diff --git a/Makefile b/Makefile index bbc1a58..1d43626 100644 --- a/Makefile +++ b/Makefile @@ -19,7 +19,7 @@ stop: down: stop $(sail) down --remove-orphans --rmi local destroy: stop - $(sail) down server -v --remove-orphans --rmi all + $(sail) down laravel.testing -v --remove-orphans --rmi all # # DB diff --git a/app/Providers/TelescopeServiceProvider.php b/app/Providers/TelescopeServiceProvider.php index b05193a..3155c8b 100644 --- a/app/Providers/TelescopeServiceProvider.php +++ b/app/Providers/TelescopeServiceProvider.php @@ -13,11 +13,13 @@ class TelescopeServiceProvider extends TelescopeApplicationServiceProvider public function register(): void { $this->hideSensitiveRequestDetails(); + $this->configureDarkMode(); $isLocal = $this->app->environment('local'); Telescope::filter(function (IncomingEntry $entry) use ($isLocal) { return $isLocal + || $entry->isEvent() || $entry->hasMonitoredTag() || $entry->isFailedRequest() || $entry->isLog() @@ -41,6 +43,32 @@ protected function hideSensitiveRequestDetails(): void ]); } + /** + * Match Telescope's own dashboard theme to the app-wide `appearance` + * preference (see HandleAppearance middleware and + * resources/js/composables/useAppearance.ts). Telescope::css() inlines + * one of two fully-compiled Bootstrap stylesheets (dist/styles.css or + * dist/styles-dark.css) chosen by the static Telescope::$useDarkTheme + * flag, so — unlike the main app's `.dark` class toggle, which lets one + * stylesheet react to both states — the choice has to be made here, + * request-time, before that view renders, and can't be flipped + * client-side afterwards without a full reload. + * + * The `appearance` cookie is read directly off the request rather than + * via the shared view value, since provider registration runs before + * the `web` middleware group (incl. HandleAppearance) executes; reading + * it here is safe because bootstrap/app.php excludes `appearance` from + * cookie encryption. A `system` preference (or no cookie yet) falls + * back to the light theme, since the server has no way to know the + * browser's OS-level preference on this request. + */ + protected function configureDarkMode(): void + { + if (request()->cookie('appearance') === 'dark') { + Telescope::night(); + } + } + protected function gate(): void { Gate::define('viewTelescope', function (User $user) { diff --git a/compose.yml b/compose.yml index 4d8485a..9c9bf5a 100644 --- a/compose.yml +++ b/compose.yml @@ -1,8 +1,8 @@ name: ByteLyon services: - server: + laravel.testing: build: - context: './docker/server/8.5' + context: './docker/8.5' dockerfile: Dockerfile args: WWWGROUP: '${WWWGROUP:-1000}' @@ -35,7 +35,7 @@ services: POSTGRES_PASSWORD: '${DB_PASSWORD:-secret}' volumes: - 'sail-pgsql:/var/lib/postgresql' - - './docker/server/pgsql/create-testing-database.sql:/docker-entrypoint-initdb.d/10-create-testing-database.sql' + - './docker/pgsql/create-testing-database.sql:/docker-entrypoint-initdb.d/10-create-testing-database.sql' networks: - sail healthcheck: @@ -63,22 +63,6 @@ services: test: ['CMD', 'redis-cli', 'ping'] retries: 3 timeout: 5s - worker: - build: - context: './docker/worker' - dockerfile: Dockerfile - image: 'worker-1.0/app' - environment: - AWS_ACCESS_KEY_ID: '${AWS_ACCESS_KEY_ID}' - AWS_DEFAULT_REGION: '${AWS_DEFAULT_REGION:-us-east-1}' - AWS_SECRET_ACCESS_KEY: '${AWS_SECRET_ACCESS_KEY}' - LARAVEL_BASE_URL: 'http://server' - LARAVEL_WORKER_TOKEN: '${API_TOKEN}' - SCRAPE_JOBS_QUEUE_URL: '${SQS_PREFIX}/${SQS_QUEUE}' - networks: - - sail - deploy: - replicas: ${WORKER_REPLICAS:-10} networks: sail: driver: bridge diff --git a/docker/server/8.5/Dockerfile b/docker/8.5/Dockerfile similarity index 100% rename from docker/server/8.5/Dockerfile rename to docker/8.5/Dockerfile diff --git a/docker/server/8.5/php.ini b/docker/8.5/php.ini similarity index 100% rename from docker/server/8.5/php.ini rename to docker/8.5/php.ini diff --git a/docker/server/8.5/start-container b/docker/8.5/start-container similarity index 100% rename from docker/server/8.5/start-container rename to docker/8.5/start-container diff --git a/docker/server/8.5/supervisord.conf b/docker/8.5/supervisord.conf similarity index 100% rename from docker/server/8.5/supervisord.conf rename to docker/8.5/supervisord.conf diff --git a/docker/images/seleniumbase-base/Dockerfile b/docker/images/seleniumbase-base/Dockerfile deleted file mode 100644 index 975de07..0000000 --- a/docker/images/seleniumbase-base/Dockerfile +++ /dev/null @@ -1,131 +0,0 @@ -# Shared AWS Lambda base image for every bytelyon-handlers SeleniumBase -# handler — unlike ../../lambda/base-image (which layers Lambda glue on top -# of the official `cloakhq/cloakbrowser` Docker Hub image), there's no -# official pre-built SeleniumBase image to start from, so this is a from -# scratch ("OS-only") Lambda base image: a plain Debian-based Python image, -# system Chromium, and the same Lambda Runtime Interface Client / Emulator / -# entrypoint glue as the CloakBrowser base image. -# -# SeleniumBase (https://seleniumbase.io/) is used here as the *stealth -# browser provider for Playwright* — see seleniumbase_playwright.py, baked -# into this image. SeleniumBase's "Stealthy Playwright Mode" launches its -# hardened/patched Chromium via CDP Mode (`seleniumbase.sb_cdp.Chrome()`), -# then hands a normal `playwright.sync_api` `Browser` back, connected to -# that already-running, already-stealthy browser over the Chrome DevTools -# Protocol (`playwright.chromium.connect_over_cdp(...)`). Every downstream -# handler built FROM this image still writes ordinary Playwright code -# (`page.goto()`, `page.content()`, `page.screenshot()`, ...) — only the -# *launch* call differs from the CloakBrowser-based handlers: -# from seleniumbase_playwright import launch # instead of: from cloakbrowser import launch -# -# ─── Build & push this image FIRST, before building any *-seleniumbase -# handler ────────────────────────────────────────────────────────────── -# docker buildx build --platform linux/arm64 --provenance=false --sbom=false \ -# -t .dkr.ecr..amazonaws.com/bytelyon-seleniumbase-base:latest \ -# --push . -# -# (Use --load instead of --push, and tag it locally, e.g. -# bytelyon-seleniumbase-base:arm64, to test handler builds against a -# not-yet-published base image via each handler's `BASE_IMAGE` build arg — -# same pattern as ../../lambda/base-image/INSTRUCTIONS.md.) -# -# --provenance=false --sbom=false is required here AND in every handler image -# built FROM this one: without it, buildx emits a manifest-list/attestation -# image that Lambda's CreateFunction/UpdateFunctionCode rejects with -# "image manifest, config or layer media type ... is not supported". -# -# Push new revisions under a new tag (e.g. :v2) rather than overwriting -# :latest in place, then bump the `BASE_IMAGE` default in every handler -# Dockerfile that should pick it up — same versioning rationale as -# ../../lambda/base-image/INSTRUCTIONS.md. - -ARG PYTHON_VERSION=3.12 -FROM python:${PYTHON_VERSION}-slim-bookworm - -WORKDIR /app - -# ─── System Chromium (the actual browser SeleniumBase's CDP Mode drives) ────── -# SeleniumBase's CDP Mode launches the browser directly as a subprocess and -# speaks the Chrome DevTools Protocol to it over its own -# --remote-debugging-port — no chromedriver/webdriver binary is involved at -# all (that's only needed for SeleniumBase's classic Selenium/WebDriver -# mode, which this image does not set up). Debian's `chromium` package is -# used instead of Google Chrome because Google only publishes amd64 Linux -# builds, and this image (like every other image in this repo) targets -# arm64 as well. `find_chrome_executable()` -# (seleniumbase/undetected/cdp_driver/config.py, vendored inside the -# `seleniumbase` pip package) auto-discovers `chromium`/`chromium-browser` -# on PATH, so no `browser_executable_path=` override is needed at call time. -# `xvfb` supports the dual-mode entrypoint's headed invocation surfaces, -# same rationale as ../../lambda/base-image/Dockerfile, even though no -# current handler needs headed mode (all launch with headless=True). -RUN apt-get update && apt-get install -y --no-install-recommends \ - chromium \ - xvfb \ - ca-certificates \ - fonts-liberation \ - && rm -rf /var/lib/apt/lists/* - -# ─── Lambda Runtime Interface Client + S3 client + browser automation ───────── -# awslambdaric / boto3: identical rationale to the CloakBrowser base image -# (../../lambda/base-image/Dockerfile) — this is not the AWS-managed Lambda -# Python base image, so neither is bundled by default. -# seleniumbase: provides CDP Mode (`seleniumbase.sb_cdp`), which launches -# and drives the stealthy Chromium instance installed above. -# playwright: the *client* library only (`playwright.sync_api`) — used -# purely to `connect_over_cdp()` to the browser SeleniumBase already -# launched. `playwright install` (which downloads Playwright's own managed -# browser binaries) is deliberately never run: this image always drives the -# system Chromium installed above, never a Playwright-managed one. -RUN pip install --no-cache-dir \ - awslambdaric \ - boto3 \ - seleniumbase \ - playwright - -# ─── Lambda Runtime Interface Emulator (local `docker run` testing) ─────────── -# Bundled into the image so users can hit the standard local-invoke endpoint -# without mounting the RIE separately. TARGETARCH is provided by buildx. -ARG TARGETARCH -ADD https://github.com/aws/aws-lambda-runtime-interface-emulator/releases/latest/download/aws-lambda-rie-${TARGETARCH} \ - /usr/local/bin/aws-lambda-rie -RUN chmod +x /usr/local/bin/aws-lambda-rie - -# ─── Lambda glue ────────────────────────────────────────────────────────────── -# Dual-mode entrypoint: same Xvfb startup + `module.func` CMD routing through -# awslambdaric as ../../lambda/base-image/lambda-entrypoint.sh (verbatim -# copy — the logic is generic, not CloakBrowser-specific). -COPY lambda-entrypoint.sh /entrypoint.sh -RUN chmod +x /entrypoint.sh - -# ─── bytelyon Playwright-over-SeleniumBase-CDP glue module ──────────────────── -# Every handler built FROM this image imports `seleniumbase_playwright.launch()` -# instead of `cloakbrowser.launch()`. Baked in here (rather than duplicated -# per-handler) for the same reason the CloakBrowser base image centralizes -# its Lambda glue: one place to patch, smaller/faster per-handler layers, no -# risk of sibling handlers' copies drifting apart. Installed under /opt/python -# (added to PYTHONPATH below) rather than next to /app/lambda_handler.py so -# it isn't shadowed/duplicated by each handler's own thin COPY layer. -COPY seleniumbase_playwright.py /opt/python/seleniumbase_playwright.py -ENV PYTHONPATH=/opt/python - -# ─── Lambda runtime env ─────────────────────────────────────────────────────── -# HOME=/tmp gives Chromium (and SeleniumBase's own per-launch temp profile -# dirs) a writable scratch dir — Lambda only allows writes under /tmp. -# Unlike the CloakBrowser base image, no non-root readability fix is needed -# here: the Chromium binary comes from an apt package under /usr/lib and -# /usr/bin (world-readable/executable by normal Debian packaging), not from -# a build-time download into root's HOME. -ENV HOME=/tmp - -ENTRYPOINT ["/entrypoint.sh"] - -# Every handler ships a module literally named lambda_handler.py with a -# `handler(event, context)` function (see ../../lambda/grab-seleniumbase's -# own Dockerfile, which COPYs its handler to /app/lambda_handler.py), so -# this default CMD is already correct for all of them. -CMD ["lambda_handler.handler"] - -# ─── Local invocation / debugging surfaces ───────────────────────────────────── -# docker run --rm -it :arm64 python # REPL -# docker run --rm -it :arm64 bash # shell diff --git a/docker/images/seleniumbase-base/INSTRUCTIONS.md b/docker/images/seleniumbase-base/INSTRUCTIONS.md deleted file mode 100644 index 3aa2c75..0000000 --- a/docker/images/seleniumbase-base/INSTRUCTIONS.md +++ /dev/null @@ -1,130 +0,0 @@ -# Shared SeleniumBase Lambda base image - -The [SeleniumBase](https://seleniumbase.io/) counterpart of -[`../../lambda/base-image`](../../lambda/base-image): a shared AWS Lambda -base image that every `*-seleniumbase` handler builds `FROM`, published to -its own ECR repository. Where the CloakBrowser base image layers Lambda glue -on top of the official `cloakhq/cloakbrowser` Docker Hub image, this one is -built from scratch (a plain Python image + system Chromium), since there's -no official pre-built SeleniumBase image to start from. - -## Why SeleniumBase instead of CloakBrowser here - -SeleniumBase provides its own stealth browser automation stack (CDP Mode — -see the [SeleniumBase README](https://seleniumbase.io/)), and its **Stealthy -Playwright Mode** lets a normal `playwright.sync_api` session drive that -already-stealthy browser: - -1. `seleniumbase.sb_cdp.Chrome(...)` launches SeleniumBase's patched, - hardened Chromium directly (CDP Mode — no chromedriver/webdriver binary - involved at all). -2. `sb.get_endpoint_url()` returns the browser's CDP endpoint. -3. `playwright.sync_api.sync_playwright().chromium.connect_over_cdp(...)` - attaches an ordinary Playwright `Browser` to that already-running - session. - -[`seleniumbase_playwright.py`](seleniumbase_playwright.py) (baked into this -image) wraps that three-step dance behind a `launch()` function with the -same call shape as `cloakbrowser.launch()`, so a handler built `FROM` this -image is a near line-for-line port of its CloakBrowser-based counterpart: -only the `launch` import changes, every `page.goto()` / `page.content()` / -`page.screenshot()` call is identical, because both providers hand back a -standard Playwright `Page`. See the module docstring in -`seleniumbase_playwright.py` for the handful of interface differences -(`humanize`/`human_preset`/`geoip` are accepted but no-ops here). - -## Why a base image at all - -Same reasoning as [`../../lambda/base-image`](../../lambda/base-image): - -- **Smaller deployment packages** — a handler rebuild/push only needs to - transfer its own thin `lambda_handler.py` COPY layer; the shared layers - (system Chromium, awslambdaric, the RIE, `seleniumbase` + `playwright`, - the entrypoint, this glue module) already live in ECR and are reused via - Docker's layer cache. -- **Reusability** — one Dockerfile to patch (a CVE fix, an entrypoint - tweak, a SeleniumBase/Playwright version bump) instead of one per - handler. -- **Faster deployments** — once the base image is in ECR, a handler - build/push touches ~1 new layer instead of ~10. - -See [`../../lambda/base-image/INSTRUCTIONS.md`](../../lambda/base-image/INSTRUCTIONS.md)'s -"A note on AWS Lambda Layers" section — it applies identically here (every -handler in this repo is `PackageType: Image`, so the native Lambda `Layers` -API field doesn't apply; a shared base image is the container-image -equivalent). - -## Files in this directory - -| File | Purpose | -| -------------------------- | ---------------------------------------------------------------------------------------------- | -| `Dockerfile` | Plain Python image + system Chromium + all shared Lambda glue. No handler code. | -| `lambda-entrypoint.sh` | Dual-mode entrypoint. Verbatim copy of the CloakBrowser base image's — the logic is generic. | -| `seleniumbase_playwright.py`| The `launch()` shim described above. Installed to `/opt/python` (on `PYTHONPATH`) in the image. | -| `INSTRUCTIONS.md` | This file. | - -## System Chromium, not Google Chrome - -This image installs Debian's `chromium` apt package rather than Google -Chrome. Google only publishes **amd64** Linux builds of Chrome, and this -image — like every other image in this repo — targets **arm64** as well. -Debian's `chromium` package is available for both architectures and is -auto-discovered on `PATH` by SeleniumBase's `find_chrome_executable()`, so -no `browser_executable_path=` override is needed anywhere in -`seleniumbase_playwright.py` or handler code. - -No `chromium-driver` package is installed: CDP Mode never touches -chromedriver, so it isn't needed. This means classic -Selenium/WebDriver-mode SeleniumBase usage (`Driver()`, `BaseCase`, `SB()` -without CDP Mode) is **not** supported by this image — only CDP Mode / -Stealthy Playwright Mode, which is all any handler built from it uses. - -## Build & push - -Do this before (re)building any `*-seleniumbase` handler image — handler -Dockerfiles default to pulling this image by tag from ECR. - -```bash -aws ecr get-login-password --region us-east-1 \ - | docker login --username AWS --password-stdin .dkr.ecr.us-east-1.amazonaws.com - -docker buildx build --platform linux/arm64 --provenance=false --sbom=false \ - -t .dkr.ecr.us-east-1.amazonaws.com/bytelyon-seleniumbase-base:latest \ - --push . -``` - -`--provenance=false --sbom=false` is required — without it, buildx produces -a manifest-list/attestation image that Lambda's `CreateFunction`/ -`UpdateFunctionCode` rejects with `InvalidParameterValueException: ... image -manifest, config or layer media type ... is not supported`. This applies -transitively: every handler image built `FROM` this one must also be built -with these flags, every time. - -For x86_64, switch `--platform linux/amd64` (and rebuild every handler for -the same architecture — mixing architectures between the base and a handler -will fail at Lambda invocation time, not at build time). - -### Local-only testing (skip the ECR push) - -```bash -docker buildx build --platform linux/arm64 --provenance=false --sbom=false \ - -t bytelyon-seleniumbase-base:arm64 --load . -``` - -Then point a handler at it without touching ECR: - -```bash -docker buildx build --platform linux/arm64 --provenance=false --sbom=false \ - --build-arg BASE_IMAGE=bytelyon-seleniumbase-base:arm64 \ - -t bytelyon-grab-seleniumbase:arm64 --load ../../lambda/grab-seleniumbase -``` - -## Versioning - -Same policy as [`../../lambda/base-image`](../../lambda/base-image): prefer -pushing a new tag (e.g. `:v2`) over overwriting `:latest` in place, then -bump the `BASE_IMAGE` default `ARG` in every handler `Dockerfile` you want -to pick up the change. Already-deployed Lambda functions are pinned to the -image **digest** Lambda resolved at the most recent `update-function-code` -call — moving `:latest` in ECR doesn't affect them until each is explicitly -redeployed. diff --git a/docker/images/seleniumbase-base/lambda-entrypoint.sh b/docker/images/seleniumbase-base/lambda-entrypoint.sh deleted file mode 100644 index 915ef96..0000000 --- a/docker/images/seleniumbase-base/lambda-entrypoint.sh +++ /dev/null @@ -1,54 +0,0 @@ -#!/bin/sh -# Dual-mode entrypoint for the SeleniumBase Lambda image. Verbatim copy of -# ../../lambda/base-image/lambda-entrypoint.sh's logic — generic Lambda/X11 -# glue, nothing CloakBrowser-specific, so it's reused as-is here. -# -# 1. Always start Xvfb on :99 so headed Chromium would work no matter how -# the container is invoked (no current handler uses headed mode, but -# this keeps the same invocation surfaces as every other image here). -# 2. Detect whether the CMD looks like a Lambda handler (a single -# `module.func`-shaped argument). If yes, route through the Lambda -# runtime client (using the bundled aws-lambda-rie locally, or talking -# to the real Lambda Runtime API when AWS_LAMBDA_RUNTIME_API is set in -# production). -# 3. Otherwise exec the CMD directly (`python`, `bash`, etc.). -set -e - -mkdir -p /tmp/.X11-unix -chmod 1777 /tmp/.X11-unix 2>/dev/null || true - -# Clean any stale Xvfb state. If a previous Xvfb died and left its lock file -# behind (we observed this in cold-start storms), a new Xvfb refuses to start -# with "Server is already active for display 99". Removing both files makes -# Xvfb start cleanly every time. -rm -f /tmp/.X99-lock /tmp/.X11-unix/X99 - -Xvfb :99 -screen 0 1920x1080x24 -nolisten tcp >/tmp/Xvfb.log 2>&1 & - -# Wait for the X11 socket to appear AND for Xvfb to be ready to serve. The -# socket file appears at bind(), but listen() and the first accept() come -# slightly later — under cold-start CPU contention this gap matters. -i=0 -while [ ! -e /tmp/.X11-unix/X99 ] && [ "$i" -lt 200 ]; do - i=$((i + 1)) - sleep 0.05 -done -# Small buffer after the socket appears so Xvfb has a moment to call listen() -# and start accepting clients. Cheap insurance against the bind/listen gap. -sleep 0.2 - -# Lambda handler shape: exactly one arg, dotted identifier (no spaces, no slashes, -# no leading dot). `python`, `bash`, etc. all fail this test and pass through -# to plain exec. -if [ $# -eq 1 ] && \ - echo "$1" | grep -qE '^[a-zA-Z_][a-zA-Z0-9_]*(\.[a-zA-Z_][a-zA-Z0-9_]*)+$'; then - if [ -z "${AWS_LAMBDA_RUNTIME_API}" ]; then - # Local invocation via bundled RIE. - exec /usr/local/bin/aws-lambda-rie /usr/local/bin/python -m awslambdaric "$@" - else - # Real Lambda — runtime API endpoint already provided by the platform. - exec /usr/local/bin/python -m awslambdaric "$@" - fi -fi - -exec "$@" diff --git a/docker/images/seleniumbase-base/seleniumbase_playwright.py b/docker/images/seleniumbase-base/seleniumbase_playwright.py deleted file mode 100644 index 44239c5..0000000 --- a/docker/images/seleniumbase-base/seleniumbase_playwright.py +++ /dev/null @@ -1,345 +0,0 @@ -"""bytelyon glue: launch a stealthy SeleniumBase CDP Mode Chromium instance -and hand back a normal Playwright `Browser`, connected to it over the Chrome -DevTools Protocol. - -This exists so every *-seleniumbase handler's lambda_handler.py (see -../../lambda/grab-seleniumbase) can be a near line-for-line port of its -CloakBrowser-based counterpart (../../lambda/grab, ../../lambda/page-scraper, -...): the launch call site becomes - - from seleniumbase_playwright import launch # instead of: from cloakbrowser import launch - -and every other line — every `page.goto()` / `page.content()` / -`page.screenshot()` / etc. call — is unchanged, because both return a -standard `playwright.sync_api.Page`. - -How it works (see https://seleniumbase.io/ — "Stealthy Playwright Mode"): - 1. `seleniumbase.sb_cdp.Chrome(...)` launches SeleniumBase's patched, - stealthy Chromium directly as a subprocess (CDP Mode talks to it over - its own --remote-debugging-port — no chromedriver/webdriver binary is - ever involved). - 2. `sb.get_endpoint_url()` returns that CDP endpoint's HTTP address. - 3. `playwright.sync_api.sync_playwright().chromium.connect_over_cdp(...)` - attaches a *normal* Playwright `Browser` to that already-running, - already-stealthy browser session. -From that point on, every Playwright API (contexts, pages, network -interception, etc.) behaves exactly as it would against any other -Playwright-launched browser — SeleniumBase is only responsible for how the -browser process itself was started and hardened against bot detection. - -Interface differences from cloakbrowser.launch() (kept as close as possible -for handler-code compatibility, but these are NOT drop-in equivalent): - - `humanize` / `human_preset`: cloakbrowser-specific synthetic - mouse/keyboard/scroll timing. SeleniumBase's stealth comes from its CDP - Mode launch flags and patches rather than replayed human-like input - timing, so these two are accepted (purely so handler code doesn't need - an `if` for which provider it's using) but have no effect here. - - `geoip`: cloakbrowser auto-derives timezone/locale from the proxy's - exit IP. SeleniumBase has no built-in equivalent lookup. Accepted but - unused; pass `timezone=`/`locale=` explicitly via **kwargs instead if - you already know the values you want (forwarded straight to - `seleniumbase.sb_cdp.Chrome()` — see its docstring / cdp_util.start() - for the full list of supported kwargs). -""" - -from __future__ import annotations - -import asyncio -import logging -import os -import time -from contextlib import suppress -from typing import Any - -from playwright.sync_api import Browser, Playwright, sync_playwright -from seleniumbase import sb_cdp - -__all__ = ["launch"] - -logger = logging.getLogger("seleniumbase_playwright") - -# Empirically, a fresh Lambda execution environment's *first* Chromium -# launch attempt occasionally fails outright (observed failure modes range -# from an early, silent exit -- see _last_chromium_output -- to the -# generic "Failed to connect to the browser" once SeleniumBase's own single -# internal retry, in Browser.create(), is also exhausted), while a second -# attempt in the very same (now-warm) container succeeds reliably and -# quickly. This looks like resource/scheduling contention specific to a -# microVM's very first process spawn rather than anything wrong with the -# launch arguments -- retrying here, a couple of times, costs nothing on -# the (common) case where the first attempt just works. -_LAUNCH_ATTEMPTS = 3 -_LAUNCH_RETRY_DELAY_S = 1.0 - -# Rolling buffer of the most recently launched Chromium subprocess's own -# stdout+stderr, captured by _patch_subprocess_for_diagnostics() below. -# SeleniumBase's Browser.start() (seleniumbase/undetected/cdp_driver/browser.py) -# always redirects the Chromium subprocess's stdout/stderr to DEVNULL, so a -# launch failure normally only ever surfaces as a generic, contentless -# "Failed to connect to the browser" exception -- exactly the kind of -# failure that's otherwise near-impossible to debug from CloudWatch Logs -# alone (this surfaced for real: the exact same image that works fine under -# a plain `docker run` fails to connect when actually deployed to Lambda, -# and DEVNULL hid the reason why). -_last_chromium_output: list[str] = [] -_MAX_CAPTURED_LINES = 200 - - -def _patch_subprocess_for_diagnostics() -> None: - """Monkeypatch `asyncio.create_subprocess_exec` (module-level, process-wide, - applied once) so every subprocess it launches -- in practice, just the - Chromium process started by seleniumbase/undetected/cdp_driver/browser.py's - `Browser.start()` -- has its stdout+stderr piped and drained into - `_last_chromium_output` instead of silently discarded via DEVNULL. This - only ever *adds* an observer; it doesn't change how the caller's own - stdin/stdout/stderr kwargs behave otherwise (they're overridden here - specifically because DEVNULL is what browser.py always passes). - """ - if getattr(asyncio, "_bytelyon_subprocess_diagnostics_patched", False): - return - orig_create_subprocess_exec = asyncio.create_subprocess_exec - - async def _patched_create_subprocess_exec(*args, **kwargs): - kwargs["stdout"] = asyncio.subprocess.PIPE - kwargs["stderr"] = asyncio.subprocess.STDOUT - process = await orig_create_subprocess_exec(*args, **kwargs) - _last_chromium_output.clear() - - async def _drain() -> None: - assert process.stdout is not None - while True: - line = await process.stdout.readline() - if not line: - break - _last_chromium_output.append( - line.decode("utf-8", "replace").rstrip("\n") - ) - del _last_chromium_output[:-_MAX_CAPTURED_LINES] - - asyncio.get_event_loop().create_task(_drain()) - return process - - asyncio.create_subprocess_exec = _patched_create_subprocess_exec - asyncio._bytelyon_subprocess_diagnostics_patched = True - - -# SeleniumBase assumes it can create relative-path directories/lock files in -# the current working directory -- e.g. "./downloaded_files/" (its default -# download folder, per https://seleniumbase.io/) and a pip/driver "find -# lock" file under it, both created unconditionally on every CDP Mode launch -# (seleniumbase/undetected/cdp_driver/cdp_util.py, __activate_virtual_display_as_needed() -# -- runs even when headless=True). That's fine for a normal dev machine, -# but AWS Lambda's runtime filesystem is read-only everywhere except /tmp, -# so a launch() call from the handler's own CWD (/app) fails with -# "[Errno 30] Read-only file system: b'downloaded_files'" the moment -# SeleniumBase tries to create it. Switching the process's CWD to /tmp -# (Lambda's one writable path; also always writable in plain `docker run` -# testing) once, here, sidesteps every one of these relative-path -# assumptions at the source instead of chasing them down individually. -def _ensure_writable_cwd() -> None: - with suppress(Exception): - if os.getcwd() != "/tmp": - os.chdir("/tmp") - - -def _normalize_proxy(proxy: Any) -> str | None: - """Convert a cloakbrowser/Playwright-style proxy value (a - "http://[user:pass@]host:port" string, a bare "[user:pass@]host:port" - string, or a Playwright `ProxySettings` dict) into the plain - "[user:pass@]host:port" string that `seleniumbase.sb_cdp.Chrome(proxy=...)` - expects (see `seleniumbase/undetected/cdp_driver/cdp_util.py`'s - `start()` docstring: `proxy: "host:port" or "user:pass@host:port"`). - - Unlike Chromium's raw --proxy-server flag, SeleniumBase's CDP Mode - handles the embedded credentials itself — Browser.get() installs a CDP - Fetch.authRequired handler using the username/password parsed back out - of this same string — so passing "user:pass@host:port" straight through - is sufficient. No separate proxy-auth extension is needed (contrast - with ../../lambda/grab/lambda_handler.py's `_normalize_proxy()`, which - has to split credentials into Playwright's own ProxySettings shape for - cloakbrowser/plain Playwright's launch(proxy=...) to authenticate). - """ - if not proxy: - return None - if isinstance(proxy, str): - # Strip a "http://"/"https://"/"socks5://" scheme prefix, if any. - if "://" in proxy: - proxy = proxy.split("://", 1)[1] - return proxy - if isinstance(proxy, dict): - server = proxy.get("server", "") - if "://" in server: - server = server.split("://", 1)[1] - username = proxy.get("username") - password = proxy.get("password") - if username and password: - return f"{username}:{password}@{server}" - return server - raise ValueError(f"'proxy' must be a string or dict, got {type(proxy).__name__}") - - -class _ManagedBrowser: - """Thin wrapper so callers can treat this exactly like a Playwright - `Browser` (`.new_page()` / `.close()`, plus anything else via - delegation), while a single `.close()` also tears down the underlying - SeleniumBase Chromium process and the Playwright client connection — - mirroring cloakbrowser's `Browser.close()` contract, where one call - releases everything. - """ - - def __init__(self, playwright: Playwright, browser: Browser, sb: Any): - self._playwright = playwright - self._browser = browser - self._sb = sb - - def __getattr__(self, item): - # Delegate everything else (new_context, contexts, version, ...) - # straight through to the real Playwright Browser object. - return getattr(self._browser, item) - - def new_page(self, **kwargs) -> Any: - # SeleniumBase's own CDP Mode launch already opened one tab - # (navigated to "about:blank"). Reuse it by default -- same as the - # seleniumbase.io "Stealthy Playwright Mode" example - # (`page = browser.contexts[0].pages[0]`) -- instead of opening a - # second, redundant tab. Any kwargs (e.g. a custom viewport) mean - # the caller wants a genuinely new page/context, so fall through to - # Playwright's normal `new_page()` in that case. - if not kwargs and self._browser.contexts and self._browser.contexts[0].pages: - return self._browser.contexts[0].pages[0] - return self._browser.new_page(**kwargs) - - def close(self) -> None: - with suppress(Exception): - self._browser.close() - with suppress(Exception): - self._playwright.stop() - with suppress(Exception): - self._sb.quit() - - -def launch( - *, - headless: bool = True, - humanize: bool = False, # accepted for interface parity; see module docstring - human_preset: str = "careful", # accepted for interface parity; unused - proxy: Any = None, - geoip: bool = False, # accepted for interface parity; see module docstring - args: list[str] | None = None, - **kwargs: Any, -) -> _ManagedBrowser: - """Launch a stealthy SeleniumBase CDP Mode Chromium instance and return - a Playwright `Browser` connected to it over CDP. See the module - docstring for the full cloakbrowser.launch() compatibility notes. - - :param headless: Same meaning as cloakbrowser.launch(headless=...). - :param proxy: "http://[user:pass@]host:port" string, a bare - "[user:pass@]host:port" string, or a Playwright `ProxySettings` dict. - :param args: Extra Chromium command-line flags (e.g. for Lambda -- see - ../../lambda/grab-seleniumbase/lambda_handler.py). "--no-zygote" is - always added regardless (see below); no need to pass it yourself. - :param kwargs: Forwarded as-is to `seleniumbase.sb_cdp.Chrome()`, e.g. - `locale=`, `timezone=`, `user_agent=`, `incognito=`, `guest=` -- - see https://seleniumbase.io/ and `cdp_util.start()`'s docstring - (in the `seleniumbase` package) for the full list. - - Retries the actual Chromium launch up to `_LAUNCH_ATTEMPTS` times before - giving up -- see that constant's comment for why (in short: a fresh - Lambda execution environment's very first launch attempt is observed to - occasionally fail even with otherwise-correct arguments, while retrying - in the same, now-warm container succeeds reliably). - """ - _ensure_writable_cwd() - _patch_subprocess_for_diagnostics() - - browser_args = list(args or []) - # Lambda's restricted process/container model needs "--no-zygote" - # regardless of caller-supplied args -- same rationale as every - # CloakBrowser-based handler (see ../../lambda/base-image/Dockerfile and - # ../../lambda/grab/lambda_handler.py): Lambda can't fork from - # Chromium's zygote process. "--disable-gpu" avoids spawning a separate - # GPU process no Lambda container can use anyway. - # ("--disable-dev-shm-usage" is already one of SeleniumBase's own - # default CDP Mode browser args -- see - # seleniumbase/undetected/cdp_driver/config.py -- so it isn't repeated - # here.) - # - # Deliberately NOT added: "--single-process". It looks like an obvious - # extra hardening flag for Lambda's restricted process model (and is - # what various third-party "Chromium on Lambda" projects use), but - # confirmed by testing against a real deployed Lambda function, it's - # actively harmful here: with it, Chromium starts (no crash, no fatal - # error -- see the captured-output diagnostics below) but never opens - # its CDP debug port, so every launch times out with a contentless - # "Failed to connect to the browser" - # (seleniumbase/undetected/cdp_driver/browser.py, start()). Removing it - # fixed launches immediately. This reproduced only in real Lambda, not - # in a plain `docker run` of the same image -- Lambda's extra syscall - # sandboxing on top of the container apparently doesn't get along with - # whatever `--single-process` needs internally. If you're tempted to - # re-add it (e.g. chasing a different problem), test against a real - # deployed Lambda function first, not just `docker run`/RIE locally. - for flag in ("--no-zygote", "--disable-gpu"): - if flag not in browser_args: - browser_args.append(flag) - - sb = None - last_exc: Exception | None = None - for attempt in range(1, _LAUNCH_ATTEMPTS + 1): - try: - sb = sb_cdp.Chrome( - headless=headless, - proxy=_normalize_proxy(proxy), - browser_args=browser_args, - # Chromium's own internal sandbox needs namespace privileges - # that Lambda's container runtime doesn't grant -- same - # reason headless Chrome/Chromium in most container/CI - # environments needs --no-sandbox. seleniumbase's Config - # auto-disables this when running as uid 0, but Lambda's - # runtime user isn't guaranteed to be root, so it's forced - # off explicitly here instead of relying on that - # auto-detection. - sandbox=False, - **kwargs, - ) - break - except Exception as e: - last_exc = e - if attempt < _LAUNCH_ATTEMPTS: - logger.warning( - "seleniumbase.sb_cdp.Chrome() launch attempt %d/%d " - "failed (%s); retrying in %.1fs. Captured Chromium " - "stdout/stderr so far:\n%s", - attempt, - _LAUNCH_ATTEMPTS, - e, - _LAUNCH_RETRY_DELAY_S, - "\n".join(_last_chromium_output) or "(none captured)", - ) - time.sleep(_LAUNCH_RETRY_DELAY_S) - if sb is None: - # SeleniumBase's own exception here is a generic, contentless - # "Failed to connect to the browser" -- re-raise with whatever the - # Chromium subprocess itself printed before dying/hanging on the - # final attempt, captured by _patch_subprocess_for_diagnostics() - # above, since that's almost always the actual actionable - # information. - if _last_chromium_output: - raise RuntimeError( - f"seleniumbase.sb_cdp.Chrome() failed to launch Chromium " - f"after {_LAUNCH_ATTEMPTS} attempt(s). Captured Chromium " - "stdout/stderr from the last attempt:\n" - + "\n".join(_last_chromium_output) - ) from last_exc - raise last_exc - endpoint_url = sb.get_endpoint_url() - playwright = sync_playwright().start() - try: - browser = playwright.chromium.connect_over_cdp(endpoint_url) - except Exception: - with suppress(Exception): - playwright.stop() - with suppress(Exception): - sb.quit() - raise - return _ManagedBrowser(playwright, browser, sb) diff --git a/docker/lambda/base-image/Dockerfile b/docker/lambda/base-image/Dockerfile deleted file mode 100644 index 92698c5..0000000 --- a/docker/lambda/base-image/Dockerfile +++ /dev/null @@ -1,184 +0,0 @@ -# Shared AWS Lambda base image for every bytelyon-handlers CloakBrowser -# handler — derived from the official CloakHQ image. -# -# `FROM cloakhq/cloakbrowser:` is an official distribution channel under -# the CloakBrowser Binary License — pulling it isn't redistribution. This -# image layers the Lambda glue that used to be copy-pasted verbatim into -# every sibling handler's own Dockerfile (../news-scraper, -# ../page-scraper, ../scraper, ../serp-scraper, ../site-crawler): the Lambda -# Runtime Interface Client (awslambdaric), the Lambda Runtime Interface -# Emulator (for local `docker run` testing), boto3 (some handlers upload -# screenshots to S3), the dual-mode entrypoint, the non-root readability fix, -# and the Lambda runtime ENV. -# -# Building this once and publishing it to ECR means every handler's own -# Dockerfile shrinks to just: -# ARG BASE_IMAGE= -# FROM ${BASE_IMAGE} -# COPY lambda_handler.py /app/lambda_handler.py -# -# That's the point of this directory: smaller/faster handler builds and -# pushes (Docker only needs to build+push the thin per-handler layer — the -# shared layers are already in ECR and get reused via layer caching), a -# single place to patch Lambda glue/CVEs instead of five, and no more -# risk of the sibling Dockerfiles quietly drifting apart. -# -# ─── Build & push this image FIRST, before building any handler ────────────── -# docker buildx build --platform linux/arm64 --provenance=false --sbom=false \ -# -t .dkr.ecr..amazonaws.com/bytelyon-cloakbrowser-base:latest \ -# --push . -# -# (Use --load instead of --push, and tag it locally, e.g. -# bytelyon-cloakbrowser-base:arm64, to test handler builds against a -# not-yet-published base image via each handler's `BASE_IMAGE` build arg.) -# -# --provenance=false --sbom=false is required here AND in every handler image -# built FROM this one: without it, buildx emits a manifest-list/attestation -# image that Lambda's CreateFunction/UpdateFunctionCode rejects with -# "image manifest, config or layer media type ... is not supported". -# -# Pin a specific tag (e.g. cloakhq/cloakbrowser:0.3.25), via -# `--build-arg CLOAKBROWSER_TAG=0.3.25`, for reproducible builds; `latest` -# floats with CloakHQ's release cadence. When you bump it, push this image -# under a new tag (e.g. :v2) rather than overwriting :latest in place, then -# bump the `BASE_IMAGE` default in every handler Dockerfile to match — that -# keeps already-deployed Lambda functions pinned to the digest they were last -# updated with (Lambda resolves and freezes the digest at -# create/update-function-code time) until each is explicitly redeployed. - -ARG CLOAKBROWSER_TAG=latest -FROM cloakhq/cloakbrowser:${CLOAKBROWSER_TAG} - -# ─── Lambda Runtime Interface Client + S3 client ────────────────────────────── -# Neither is bundled in the canonical image (it's not the AWS-managed Lambda -# Python base image). boto3 is only imported by the handlers that upload -# screenshots to S3 (page-scraper, site-crawler) but is baked in for every -# handler here anyway — it's a small, pure-Python dependency, and shipping one -# base image that works for all handlers is the entire point of this file. -RUN pip install --no-cache-dir awslambdaric boto3 - -# ─── Lambda Runtime Interface Emulator (local `docker run` testing) ─────────── -# Bundled into the image so users can hit the standard local-invoke endpoint -# without mounting the RIE separately. TARGETARCH is provided by buildx. -ARG TARGETARCH -ADD https://github.com/aws/aws-lambda-runtime-interface-emulator/releases/latest/download/aws-lambda-rie-${TARGETARCH} \ - /usr/local/bin/aws-lambda-rie -RUN chmod +x /usr/local/bin/aws-lambda-rie - -# ─── Lambda glue ────────────────────────────────────────────────────────────── -# Dual-mode entrypoint replaces the canonical bin/docker-entrypoint.sh: same -# Xvfb startup, plus routing for `module.func` CMDs through awslambdaric. Xvfb -# isn't strictly required by any current handler (they all launch -# headless=True by default), but it's kept so the other invocation surfaces -# below (python, cloakserve, etc.) still support headed mode exactly like the -# canonical image. -COPY lambda-entrypoint.sh /entrypoint.sh -RUN chmod +x /entrypoint.sh - -# ─── Baked-in GeoIP database (for cloakbrowser's optional geoip=True) ───────── -# cloakbrowser's geoip.py downloads GeoLite2-City.mmdb (~65 MB) on first use -# into get_cache_dir()/geoip/, which — because CLOAKBROWSER_CACHE_DIR (set -# below) points at /root/.cloakbrowser, the same read-only-at-runtime path as -# the baked Chromium binary — fails silently in real Lambda (mkdir on a -# read-only FS), surfacing as "GeoIP resolution failed: GeoIP database is -# unavailable" from any handler that passes geoip=True. Baking the DB in at -# build time avoids any runtime write entirely: _ensure_geoip_db() just finds -# the file already present and returns immediately. Same P3TERX MaxMind -# GeoLite2-City mirror cloakbrowser itself downloads from (no license key -# needed); re-bake periodically by rebuilding this image to pick up updates -# (cloakbrowser's own 30-day background refresh can't write here either, but -# fails silently in a background thread without affecting resolution). -ADD https://github.com/P3TERX/GeoLite.mmdb/raw/download/GeoLite2-City.mmdb \ - /root/.cloakbrowser/geoip/GeoLite2-City.mmdb - -# ─── Real Microsoft fonts (for Windows-fingerprint spoofing) ───────────────── -# CloakBrowser's own guidance (https://github.com/CloakHQ/cloakbrowser#font-setup-on-linux): -# a bare Linux image's font list is itself a detection signal when spoofing a -# Windows fingerprint — every launch otherwise logs "Incomplete Windows font -# set". `ttf-mscorefonts-installer` is Debian's packaging of Microsoft's own -# "Core Fonts for the Web" (Arial, Times New Roman, Courier New, Georgia, -# Verdana, ...) — genuine Microsoft font files fetched from Sourceforge at -# build time under Microsoft's redistribution EULA (auto-accepted below via -# debconf-set-selections, the standard hands-off way to script this package; -# lives in the "contrib" component, which isn't enabled by default). This is -# real Microsoft font data, not a metric-compatible substitute like -# fonts-liberation (already present upstream, and left in place — more -# font coverage only helps). -# -# CloakBrowser's own docs call this "not enough" by itself — a real Windows -# install has 200+ fonts, Office adds ~50 more, versus ~13 here. See -# ./fonts/README.md for how to add the rest if you have legal access to them; -# that directory is empty by default (just its own README, which fc-cache -# silently ignores) so this COPY is always valid even with nothing added. -RUN sed -i 's/Components: main$/Components: main contrib/' /etc/apt/sources.list.d/debian.sources \ - && echo "ttf-mscorefonts-installer msttcorefonts/accepted-mscorefonts-eula select true" | debconf-set-selections \ - && apt-get update \ - && DEBIAN_FRONTEND=noninteractive apt-get install -y --no-install-recommends ttf-mscorefonts-installer fontconfig \ - && rm -rf /var/lib/apt/lists/* -COPY fonts/ /usr/share/fonts/windows/ -RUN fc-cache -f - -# ─── Lambda non-root readability fix ────────────────────────────────────────── -# The canonical image bakes the Chromium binary at /root/.cloakbrowser/ (root's -# HOME at build time). Lambda runs the container as a non-root user that can't -# read /root by default (mode 750). Make the whole binary tree world-readable -# and traversable. -RUN chmod -R o+rX /root /root/.cloakbrowser - -# ─── Silence the CloakBrowser free-tier welcome banner ──────────────────────── -# Every keyless launch, cloakbrowser.download._show_welcome() prints a -# multi-line "CloakBrowser - stealth Chromium for automation ... Star us if -# CloakBrowser helps your project!" banner to stderr (CloudWatch, for a Lambda -# handler), gated by a cache-dir marker file (.welcome_shown): shown once if -# missing, and re-shown on the free tier once the marker is older than 24h -# (WELCOME_FREE_INTERVAL). Baking in a fresh marker at build time (as this -# image used to) only defers the banner by a day — a long-lived ECR image -# would start reprinting it on cold starts once the marker goes stale, and -# Lambda's read-only runtime FS means nothing can ever refresh it. Patch the -# one line that gates the banner so _show_welcome() always returns -# immediately, regardless of tier or marker state. Fails the build loudly -# (assert) if a future upstream release reshapes this function, rather than -# silently leaving the banner in place. -RUN python3 -c 'import pathlib, cloakbrowser.download as m; p = pathlib.Path(m.__file__); src = p.read_text(); anchor = " marker = get_cache_dir() / \".welcome_shown\"\n if not _welcome_due(marker, pro=(tier == \"pro\")):\n return\n"; assert anchor in src, "cloakbrowser _show_welcome anchor not found - upstream source changed"; p.write_text(src.replace(anchor, " return # bytelyon-cloakbrowser-base: banner disabled (see base-image/Dockerfile)\n" + anchor, 1))' - -# ─── Lambda runtime env ─────────────────────────────────────────────────────── -# HOME=/tmp gives Chromium a writable scratch dir (Lambda only allows writes -# under /tmp). CLOAKBROWSER_CACHE_DIR points at the baked binary location since -# HOME=/tmp would otherwise make get_cache_dir() resolve to /tmp/.cloakbrowser -# (empty). Auto-update is disabled because the runtime FS is read-only. -# -# CLOAKBROWSER_SUPPRESS_FONT_WARNING: same read-only-FS problem as the welcome -# banner above, applied to a check we can only ever partially satisfy. The -# "Incomplete Windows font set" warning (browser.py's _maybe_warn_windows_fonts) -# requires ALL of 8 specific fonts to be present: Segoe UI, Segoe UI Light, -# Calibri, Marlett, MS UI Gothic, Franklin Gothic, Consolas, Courier New. The -# real-Microsoft-fonts step above (ttf-mscorefonts-installer) only covers -# Courier New — the other 7 are Windows-OS/Office-exclusive system fonts with -# no legal, freely-redistributable source; they aren't part of any apt package, -# core-fonts-for-the-web included (see ./fonts/README.md if you have licensed -# access to add them yourself). Since the check can never fully pass here, and -# its "shown once" marker lives under this same read-only CLOAKBROWSER_CACHE_DIR -# (the write silently fails every time, so without this it would otherwise -# reprint on every single cold start, forever — confirmed in practice), -# suppress it outright rather than let an unactionable warning spam CloudWatch. -ENV HOME=/tmp \ - CLOAKBROWSER_CACHE_DIR=/root/.cloakbrowser \ - CLOAKBROWSER_AUTO_UPDATE=false \ - CLOAKBROWSER_SUPPRESS_FONT_WARNING=1 - -ENTRYPOINT ["/entrypoint.sh"] - -# Every handler ships a module literally named lambda_handler.py with a -# `handler(event, context)` function (see each sibling's own Dockerfile, -# which COPYs its handler to /app/lambda_handler.py), so this default CMD is -# already correct for all of them and none needs to repeat it. Override it in -# a handler's own Dockerfile only if that convention ever changes. -CMD ["lambda_handler.handler"] - -# ─── Same as the canonical CloakHQ image (CMD overridden per-handler) ───────── -# docker run --rm -it :arm64 python # REPL -# docker run --rm :arm64 python examples/basic.py # examples -# docker run --rm -p 9222:9222 :arm64 cloakserve --port=9222 # CDP server -# docker run --rm :arm64 cloaktest # stealth tests -# docker run --rm -it :arm64 node # JS wrapper -# docker run --rm -it :arm64 bash # shell diff --git a/docker/lambda/base-image/INSTRUCTIONS.md b/docker/lambda/base-image/INSTRUCTIONS.md deleted file mode 100644 index 5632515..0000000 --- a/docker/lambda/base-image/INSTRUCTIONS.md +++ /dev/null @@ -1,99 +0,0 @@ -# Shared CloakBrowser Lambda base image - -The common scaffolding previously duplicated verbatim across every handler's -own `Dockerfile` (`../news-scraper`, `../page-scraper`, `../scraper`, -`../serp-scraper`, `../site-crawler`) — the Lambda Runtime Interface Client, -the Lambda Runtime Interface Emulator, boto3, the dual-mode entrypoint, the -non-root readability fix, and the runtime `ENV` — now lives here, in a single -image, published to its own ECR repository. Every handler's `Dockerfile` -builds `FROM` it and adds only its own `lambda_handler.py`. - -## Why - -- **Smaller deployment packages** — a handler rebuild/push only needs to - transfer the thin per-handler layer (a single `lambda_handler.py` COPY); - the shared layers already live in ECR and are reused via Docker's layer - cache, instead of every handler independently re-installing - `awslambdaric`/`boto3`, re-downloading the RIE binary, etc. -- **Reusability** — one Dockerfile to patch when the Lambda glue needs to - change (a CVE fix in `awslambdaric`, an entrypoint tweak, a new baked-in - dependency), instead of five that can silently drift apart. -- **Faster deployments** — `docker build`/`push` for a handler after the base - image is already in ECR touches ~1 new layer instead of ~10. - -## Files in this directory - -| File | Purpose | -| ---------------------- | ------------------------------------------------------------------------------------------------------ | -| `Dockerfile` | `FROM cloakhq/cloakbrowser` plus all shared Lambda glue. No handler code, no `lambda_handler.py` COPY. | -| `lambda-entrypoint.sh` | Dual-mode entrypoint, identical to what every handler used to carry individually. | -| `fonts/` | Empty by default (just its own README) — optional real Windows/Office fonts, see below. | -| `INSTRUCTIONS.md` | This file. | - -## Windows font spoofing - -CloakBrowser warns (`[cloakbrowser] Incomplete Windows font set ...`) when spoofing a Windows fingerprint without a full Windows font set installed — a bare Linux font list is itself a detection signal. The `Dockerfile` installs `ttf-mscorefonts-installer` (Debian's packaging of Microsoft's real "Core Fonts for the Web": Arial, Times New Roman, Courier New, Georgia, Verdana, etc. — genuine MS font files under Microsoft's own redistribution EULA, not metric-compatible substitutes), which is a legitimate, fully-automatable improvement. - -It cannot fully satisfy CloakBrowser's check, though: that requires 8 specific fonts (`Segoe UI`, `Segoe UI Light`, `Calibri`, `Marlett`, `MS UI Gothic`, `Franklin Gothic`, `Consolas`, `Courier New`), and 7 of those 8 are Windows-OS/Office-exclusive system fonts with no legal, freely-redistributable source — they're not in `ttf-mscorefonts-installer` or any other apt package. If you have legal access to them (e.g. a licensed Windows/Office install), drop the `.ttf`/`.otf` files into `fonts/` before building — see `fonts/README.md`. The warning is suppressed by default (`CLOAKBROWSER_SUPPRESS_FONT_WARNING=1` in the `Dockerfile`'s `ENV`) since it can otherwise never fully resolve here, and its "shown once" marker lives on the same read-only-at-runtime path as the welcome banner (see the `Dockerfile` comment), so it would otherwise reprint on every single cold start. - -## A note on AWS Lambda Layers (the actual `Layers` API field) - -The native `Layers` field on `CreateFunction`/`UpdateFunctionConfiguration` -only applies to **zip-package** Lambda functions. Every function in this repo -is `PackageType: Image` (needed because the CloakBrowser Chromium binary and -its dependencies are far too large — and too native/OS-specific — for a zip -layer), and container-image Lambda functions **cannot** attach a `Layers` -resource at all; everything must be baked into the image itself. A shared -**base image** published to ECR is the standard equivalent for container -image functions, and is what this directory provides. - -## Build & push - -Do this before (re)building any handler image — handler Dockerfiles default -to pulling this image by tag from ECR. - -```bash -aws ecr get-login-password --region us-east-1 \ - | docker login --username AWS --password-stdin .dkr.ecr.us-east-1.amazonaws.com - -docker buildx build --platform linux/arm64 --provenance=false --sbom=false \ - -t .dkr.ecr.us-east-1.amazonaws.com/bytelyon-cloakbrowser-base:latest \ - --push . -``` - -`--provenance=false --sbom=false` is required — without it, buildx produces a -manifest-list/attestation image that Lambda's `CreateFunction`/ -`UpdateFunctionCode` rejects with `InvalidParameterValueException: ... image -manifest, config or layer media type ... is not supported`. This applies -transitively: every handler image built `FROM` this one must also be built -with these flags, every time. - -For x86_64, switch `--platform linux/amd64` (and rebuild every handler for -the same architecture — mixing architectures between the base and a handler -will fail at Lambda invocation time, not at build time). - -### Local-only testing (skip the ECR push) - -```bash -docker buildx build --platform linux/arm64 --provenance=false --sbom=false \ - -t bytelyon-cloakbrowser-base:arm64 --load . -``` - -Then point a handler at it without touching ECR: - -```bash -docker buildx build --platform linux/arm64 --provenance=false --sbom=false \ - --build-arg BASE_IMAGE=bytelyon-cloakbrowser-base:arm64 \ - -t bytelyon-page-scraper:arm64 --load ../page-scraper -``` - -## Versioning - -Prefer pushing a new tag (e.g. `:v2`) over overwriting `:latest` in place, -then bump the `BASE_IMAGE` default `ARG` in every handler `Dockerfile` you -want to pick up the change. Already-deployed Lambda functions are pinned to -the image **digest** Lambda resolved at the most recent -`update-function-code` call — moving `:latest` in ECR doesn't affect them -until each is explicitly redeployed, but two handlers built against `:latest` -at different times can silently end up on different digests. Tagging -deliberately avoids that ambiguity. diff --git a/docker/lambda/base-image/fonts/.gitignore b/docker/lambda/base-image/fonts/.gitignore deleted file mode 100644 index a0af254..0000000 --- a/docker/lambda/base-image/fonts/.gitignore +++ /dev/null @@ -1,5 +0,0 @@ -# Real Windows/Office fonts are proprietary — never commit them. Only this -# directory's own README.md (and this .gitignore) are tracked. -* -!.gitignore -!README.md diff --git a/docker/lambda/base-image/fonts/README.md b/docker/lambda/base-image/fonts/README.md deleted file mode 100644 index d734122..0000000 --- a/docker/lambda/base-image/fonts/README.md +++ /dev/null @@ -1,34 +0,0 @@ -# Optional: real Windows/Office fonts - -This directory is baked into the image at `/usr/share/fonts/windows/` (see -`../Dockerfile`) and is empty by default — just this README, which -`fc-cache` silently ignores (not a font file). - -The `Dockerfile` already installs `ttf-mscorefonts-installer` (Debian's -packaging of Microsoft's real "Core Fonts for the Web" — Arial, Times New -Roman, Courier New, Georgia, Verdana, etc. — fetched from Sourceforge at -build time under Microsoft's own redistribution EULA). That's a legitimate, -automatable source of *genuine* Microsoft font files, not a metric-compatible -substitute like `fonts-liberation`. - -CloakBrowser's own guidance is that this still isn't enough on its own — a -real Windows install has 200+ font files, and Office adds ~50 more, versus -~13 in the core-fonts package. If you have legal access to those (e.g. a -`C:\Windows\Fonts\` folder and Office fonts from a machine/license you -control), drop the `.ttf`/`.otf` files directly into this directory before -building the base image: - -```bash -cp /path/to/windows/Fonts/*.ttf docker/lambda/base-image/fonts/ -cp /path/to/office/fonts/*.ttf docker/lambda/base-image/fonts/ -``` - -Then rebuild `../Dockerfile` as usual (see `../INSTRUCTIONS.md`) — the -`COPY fonts/ /usr/share/fonts/windows/` step picks up whatever's here, and -`fc-cache -f` registers it. - -**Do not** commit real Microsoft font files to this repo — they're -proprietary and not freely redistributable. This directory is `.gitignore`d -except for this README specifically so each engineer can drop their own -locally, without them landing in version control or the shared team image -unless you specifically choose to publish a base image built with them. diff --git a/docker/lambda/base-image/lambda-entrypoint.sh b/docker/lambda/base-image/lambda-entrypoint.sh deleted file mode 100644 index 0118084..0000000 --- a/docker/lambda/base-image/lambda-entrypoint.sh +++ /dev/null @@ -1,52 +0,0 @@ -#!/bin/sh -# Dual-mode entrypoint for the CloakBrowser Lambda image. -# -# 1. Always start Xvfb on :99 (same as the canonical bin/docker-entrypoint.sh) -# so headed Chromium works no matter how the container is invoked. -# 2. Detect whether the CMD looks like a Lambda handler (a single -# `module.func`-shaped argument). If yes, route through the Lambda runtime -# client (using the bundled aws-lambda-rie locally, or talking to the real -# Lambda Runtime API when AWS_LAMBDA_RUNTIME_API is set in production). -# 3. Otherwise exec the CMD directly — preserving the canonical Dockerfile's -# interaction surface (`python`, `cloakserve`, `cloaktest`, `node`, `bash`, -# `python examples/basic.py`, etc.). -set -e - -mkdir -p /tmp/.X11-unix -chmod 1777 /tmp/.X11-unix 2>/dev/null || true - -# Clean any stale Xvfb state. If a previous Xvfb died and left its lock file -# behind (we observed this in cold-start storms), a new Xvfb refuses to start -# with "Server is already active for display 99". Removing both files makes -# Xvfb start cleanly every time. -rm -f /tmp/.X99-lock /tmp/.X11-unix/X99 - -Xvfb :99 -screen 0 1920x1080x24 -nolisten tcp >/tmp/Xvfb.log 2>&1 & - -# Wait for the X11 socket to appear AND for Xvfb to be ready to serve. The -# socket file appears at bind(), but listen() and the first accept() come -# slightly later — under cold-start CPU contention this gap matters. -i=0 -while [ ! -e /tmp/.X11-unix/X99 ] && [ "$i" -lt 200 ]; do - i=$((i + 1)) - sleep 0.05 -done -# Small buffer after the socket appears so Xvfb has a moment to call listen() -# and start accepting clients. Cheap insurance against the bind/listen gap. -sleep 0.2 - -# Lambda handler shape: exactly one arg, dotted identifier (no spaces, no slashes, -# no leading dot). `python`, `cloakserve`, `cloaktest`, `bash`, `node` all fail -# this test and pass through to plain exec. -if [ $# -eq 1 ] && \ - echo "$1" | grep -qE '^[a-zA-Z_][a-zA-Z0-9_]*(\.[a-zA-Z_][a-zA-Z0-9_]*)+$'; then - if [ -z "${AWS_LAMBDA_RUNTIME_API}" ]; then - # Local invocation via bundled RIE. - exec /usr/local/bin/aws-lambda-rie /usr/local/bin/python -m awslambdaric "$@" - else - # Real Lambda — runtime API endpoint already provided by the platform. - exec /usr/local/bin/python -m awslambdaric "$@" - fi -fi - -exec "$@" diff --git a/docker/lambda/cloak-example/Dockerfile b/docker/lambda/cloak-example/Dockerfile deleted file mode 100644 index 6de2592..0000000 --- a/docker/lambda/cloak-example/Dockerfile +++ /dev/null @@ -1,86 +0,0 @@ -# CloakBrowser on AWS Lambda — derived from the official CloakHQ image. -# -# `FROM cloakhq/cloakbrowser:` is an official distribution channel under -# the CloakBrowser Binary License — pulling it isn't redistribution. We just -# layer Lambda glue on top: the Lambda Runtime Interface Client (awslambdaric), -# the Lambda Runtime Interface Emulator (for local `docker run` testing), the -# dual-mode entrypoint, and the handler module. -# -# This directory is self-contained — copy/clone it anywhere and build from -# inside it. No files outside this directory are referenced. -# -# Verbatim copy of CloakHQ/CloakBrowser's official -# examples/integrations/aws_lambda/ (fetched 2026-09-15) — a clean vendor -# baseline, deliberately kept unmodified by any of bytelyon's own -# customizations (compare against ../base-image + ../serp-scraper, which -# fork from the same starting point but add GeoIP DB baking, real MS fonts, -# a Google-SERP-specific handler, proxy retry/sticky-session logic, etc.). -# -# ─── Lambda invocation (default CMD) ────────────────────────────────────────── -# # From inside this directory: -# docker buildx build --platform linux/arm64 -t cloakbrowser-lambda:arm64 --load . -# -# # Or from a parent dir, pointing at this directory as the build context: -# docker buildx build --platform linux/arm64 \ -# -f path/to/aws_lambda/Dockerfile -t cloakbrowser-lambda:arm64 --load \ -# path/to/aws_lambda -# -# docker run --rm -p 9000:8080 cloakbrowser-lambda:arm64 -# curl -XPOST http://localhost:9000/2015-03-31/functions/function/invocations \ -# -d '{"url":"https://example.com"}' -# -# ─── Same as the canonical CloakHQ image (CMD overridden) ───────────────────── -# docker run --rm -it cloakbrowser-lambda:arm64 python # REPL -# docker run --rm cloakbrowser-lambda:arm64 python examples/basic.py # examples -# docker run --rm -p 9222:9222 cloakbrowser-lambda:arm64 cloakserve --port=9222 # CDP server -# docker run --rm cloakbrowser-lambda:arm64 cloaktest # stealth tests -# docker run --rm -it cloakbrowser-lambda:arm64 node # JS wrapper -# docker run --rm -it cloakbrowser-lambda:arm64 bash # shell -# -# Pin a specific tag (e.g. cloakhq/cloakbrowser:0.3.25) for reproducible builds; -# `latest` floats with CloakHQ's release cadence. - -FROM cloakhq/cloakbrowser:latest - -# ─── Lambda Runtime Interface Client ────────────────────────────────────────── -RUN pip install --no-cache-dir awslambdaric - -# ─── Lambda Runtime Interface Emulator (local `docker run` testing) ─────────── -# Bundled into the image so users can hit the standard local-invoke endpoint -# without mounting the RIE separately. TARGETARCH is provided by buildx. -ARG TARGETARCH -ADD https://github.com/aws/aws-lambda-runtime-interface-emulator/releases/latest/download/aws-lambda-rie-${TARGETARCH} \ - /usr/local/bin/aws-lambda-rie -RUN chmod +x /usr/local/bin/aws-lambda-rie - -# ─── Lambda glue ────────────────────────────────────────────────────────────── -# Dual-mode entrypoint replaces the canonical bin/docker-entrypoint.sh: same -# Xvfb startup, plus routing for `module.func` CMDs through awslambdaric. -COPY lambda-entrypoint.sh /entrypoint.sh -RUN chmod +x /entrypoint.sh - -# Handler sits at /app (already on Python's import path in the canonical image, -# WORKDIR=/app), imports cloakbrowser as a normal library. -COPY lambda_handler.py /app/lambda_handler.py - -# ─── Lambda non-root readability fix ────────────────────────────────────────── -# The canonical image bakes the Chromium binary at /root/.cloakbrowser/ (root's -# HOME at build time). Lambda runs the container as a non-root user that can't -# read /root by default (mode 750). Make the whole binary tree world-readable -# and traversable. Also restore the .welcome_shown marker the canonical image -# rm's (Lambda's read-only runtime FS can't recreate it, so the welcome would -# print to CloudWatch on every cold start otherwise). -RUN touch /root/.cloakbrowser/.welcome_shown \ - && chmod -R o+rX /root /root/.cloakbrowser - -# ─── Lambda runtime env ─────────────────────────────────────────────────────── -# HOME=/tmp gives Chromium a writable scratch dir (Lambda only allows writes -# under /tmp). CLOAKBROWSER_CACHE_DIR points at the baked binary location since -# HOME=/tmp would otherwise make get_cache_dir() resolve to /tmp/.cloakbrowser -# (empty). Auto-update is disabled because the runtime FS is read-only. -ENV HOME=/tmp \ - CLOAKBROWSER_CACHE_DIR=/root/.cloakbrowser \ - CLOAKBROWSER_AUTO_UPDATE=false - -ENTRYPOINT ["/entrypoint.sh"] -CMD ["lambda_handler.handler"] diff --git a/docker/lambda/cloak-example/INSTRUCTIONS.md b/docker/lambda/cloak-example/INSTRUCTIONS.md deleted file mode 100644 index 5571009..0000000 --- a/docker/lambda/cloak-example/INSTRUCTIONS.md +++ /dev/null @@ -1,206 +0,0 @@ -# CloakBrowser on AWS Lambda (vendor baseline — unmodified) - -> **bytelyon note**: This directory is a verbatim copy of CloakHQ's official -> `examples/integrations/aws_lambda/` (fetched 2026-09-15 from -> https://github.com/CloakHQ/CloakBrowser). It is kept **deliberately -> unmodified** as a clean reference baseline to compare against -> `../base-image` + `../serp-scraper`, which fork from the same starting -> point but add GeoIP DB baking, real MS fonts, a Google-SERP-specific -> handler with block/CAPTCHA detection, and proxy retry/sticky-session -> logic. Deploy this as its own Lambda function (e.g. `bytelyon-cloak-example`) -> — do not overwrite `bytelyon-serp-scraper`. - -Run stealth Chromium one-shot scrapes inside an AWS Lambda function (container image package type). The image derives directly from the official CloakHQ Docker Hub image (`cloakhq/cloakbrowser`) and adds Lambda runtime support on top — Lambda is an additional invocation surface, not a replacement. Every other surface from the canonical image (`python`, `cloakserve`, `cloaktest`, `node`, `bash`, examples) keeps working. - -This document covers what the image is, how to build and locally test it, and the event/response contract. **It does not prescribe a deployment method** — push the resulting image to ECR and create the Lambda function however you prefer (AWS CLI, CDK, Terraform, SAM, console, etc.). Configuration tips for whichever tool you use are at the bottom. - -## Files in this directory - -| File | Purpose | -|---|---| -| `Dockerfile` | `FROM cloakhq/cloakbrowser` plus a thin Lambda layer. Self-contained — no files outside this directory are referenced. | -| `lambda-entrypoint.sh` | Dual-mode entrypoint. Starts Xvfb, then routes `module.func` CMDs through `awslambdaric` (via the bundled `aws-lambda-rie` locally, or the AWS Runtime API in production), and execs everything else (`python`, `cloakserve`, `cloaktest`, `node`, `bash`) directly. | -| `lambda_handler.py` | Default handler. Takes `{url, ...}`, returns `{title, url, html, screenshot_b64?}`. Always headed via Xvfb. | -| `INSTRUCTIONS.md` | This file. | - -The Lambda layer is ~30 lines on top of the official image — no apt list, no Node install, no JS-wrapper build, no Chromium download. The canonical CloakHQ image owns those. - -This directory is **standalone**: copy or clone it anywhere (its own repo, a subdirectory of an existing project, a CI artifact bundle) and the build still works. It depends only on the upstream `cloakhq/cloakbrowser` image on Docker Hub and the `aws-lambda-rie` binary on GitHub Releases — both fetched at build time. - -## Build - -From inside this directory: - -```bash -docker buildx build --platform linux/arm64 -t cloakbrowser-lambda:arm64 --load . -``` - -Or from anywhere, pointing at this directory as the build context: - -```bash -docker buildx build --platform linux/arm64 \ - -f path/to/aws_lambda/Dockerfile \ - -t cloakbrowser-lambda:arm64 --load \ - path/to/aws_lambda -``` - -The build pulls `cloakhq/cloakbrowser:latest` from Docker Hub and adds the Lambda layer on top. Pin a specific tag (e.g. `cloakhq/cloakbrowser:0.3.25`) in the `FROM` line for reproducible builds; `latest` floats with the upstream release cadence. - -For x86_64, switch `--platform linux/amd64` (slower on Apple Silicon under emulation). - -> **bytelyon note**: our other Lambda images build with `--provenance=false --sbom=false` because AWS Lambda's image deploy rejects OCI manifests with those attestations. Add the same flags here when building for actual deployment (see the header comment in `Dockerfile`). - -## Local smoke test (no AWS account needed) - -> **What's the RIE?** Lambda container images can't be run with a plain `docker run` — they expect to talk to AWS's Runtime API (the HTTP service Lambda exposes inside its sandbox to deliver events and collect responses). AWS publishes a small binary called the **Runtime Interface Emulator** that stands up a fake Runtime API on localhost so you can test the container exactly the way Lambda will invoke it, without deploying. We bake the RIE into the image, and the dual-mode entrypoint uses it automatically when `AWS_LAMBDA_RUNTIME_API` isn't set (i.e. you're not running in real Lambda). - -The image bakes in `aws-lambda-rie`, so the standard Lambda local-invoke endpoint works without mounting anything: - -```bash -docker run --rm -p 9000:8080 cloakbrowser-lambda:arm64 - -# In another shell: -curl -sS -XPOST "http://localhost:9000/2015-03-31/functions/function/invocations" \ - -d '{"url":"https://example.com"}' -``` - -Other invocation surfaces stay intact (these match the canonical CloakHQ image): - -```bash -docker run --rm -it cloakbrowser-lambda:arm64 python # REPL -docker run --rm cloakbrowser-lambda:arm64 python examples/basic.py # examples -docker run --rm -p 9222:9222 cloakbrowser-lambda:arm64 cloakserve --port=9222 # CDP server -docker run --rm cloakbrowser-lambda:arm64 cloaktest # stealth tests -docker run --rm -it cloakbrowser-lambda:arm64 node # JS wrapper -``` - -## Event schema - -Only `url` is required. Everything else is optional. - -### Launch options (forwarded to `cloakbrowser.launch_context_async`) - -| Field | Type | Default | -|---|---|---| -| `url` | str | required — `http://` and `https://` only | -| `proxy` | str / dict | none — `http://user:pass@host:port` or a Playwright proxy dict | -| `humanize` | bool | `false` — enable human-like mouse / keyboard / scroll | -| `human_preset` | str | `"default"` or `"careful"` | -| `geoip` | bool | `false` — auto timezone+locale from proxy IP | -| `timezone` | str | none — IANA tz, e.g. `"America/New_York"` | -| `locale` | str | none — BCP-47, e.g. `"en-US"` | -| `viewport` | `{width,height}` | `1920x947` (cloakbrowser default) | -| `user_agent` | str | none | - -### Navigation - -| Field | Type | Default | -|---|---|---| -| `wait_until` | str | `"domcontentloaded"` — `load` / `domcontentloaded` / `networkidle` / `commit` | -| `goto_timeout_ms` | int | `30000` | - -### Post-navigation waits - -`smart_wait` is the default when no other wait is specified. It polls `document.documentElement.outerHTML.length` and returns when the size hasn't changed for `dom_stable_ms`. Robust for at-scale scraping because it ignores network activity (analytics beacons, long-poll, websockets) that doesn't mutate the DOM — `wait_until: "networkidle"` is unreliable on modern SPAs for exactly this reason. - -| Field | Type | Default | -|---|---|---| -| `smart_wait` | bool | `true` if no other wait is set | -| `dom_stable_ms` | int | `1500` | -| `max_settle_ms` | int | `15000` | -| `wait_for_load_state` | str | none — `load` / `domcontentloaded` / `networkidle` | -| `wait_for_load_state_timeout_ms` | int | `30000` | -| `wait_for_selector` | str | none — CSS or XPath | -| `wait_for_selector_state` | str | `"visible"` — also `attached` / `detached` / `hidden` | -| `wait_for_selector_timeout_ms` | int | `30000` | -| `wait_ms` | int | none — fixed pause | - -### Capture - -| Field | Type | Default | -|---|---|---| -| `screenshot` | bool | `true` | -| `full_page_screenshot` | bool | `false` | - -### Retry orchestration - -The handler retries transient navigation failures inline within the same Lambda invocation. Two layers, both built-in: - -- **Launch retries** — 3 attempts with 0.3 s + 0.6 s backoff. Recovers Xvfb / Chromium spawn races at cold start. Fast and cheap; not configurable. -- **Strategy retries** — default 1 attempt, configurable via the `retries` event field. Recovers specific post-launch error classes by relaunching with adjusted internal Chromium args / page-load budgets. - -| Field | Type | Default | -|---|---|---| -| `retries` | int | `1` — number of strategy-retry attempts after the first failure. Set to `0` to disable retry entirely. | - -Strategies (priority order — first match wins): - -| Error pattern | Strategy applied | -|---|---| -| `ERR_CERT_*` (any cert error) | `extra_args: ["--ignore-certificate-errors"]`, `goto_timeout_ms: 60000` | -| `Timeout … exceeded` | `goto_timeout_ms: 90000`, `max_settle_ms: 25000` | -| `ERR_CONNECTION_TIMED_OUT` | same as `Timeout … exceeded` | - -Errors that are **not retried** (no anonymous scraper can recover): `ERR_NAME_NOT_RESOLVED`, `ERR_SSL_PROTOCOL_ERROR`, `ERR_CONNECTION_REFUSED`, `ERR_HTTP_RESPONSE_CODE_FAILURE`. These bail immediately. - -On final failure, the raised `RuntimeError`'s message includes a `retry_history` block listing every attempt (strategy applied + error seen). Successful invocations return the standard response shape unchanged — no surprise fields when retries didn't fire. - -### Response - -```json -{ - "title": "...", - "url": "https://example.com/", - "html": "...", - "screenshot_b64": "" -} -``` - -## Lambda-specific Chromium hardening (baked in, do not remove) - -Two flags are forced on every launch by `lambda_handler.py`: - -- `--disable-dev-shm-usage` — Lambda's `/dev/shm` is ~64 MB; Chromium's renderer crashes mid-paint without this. -- `--no-zygote` — Lambda's restricted process model can't fork from Chromium's zygote process; without this the browser launches but child renderers fail to spawn and the first `page.new_page()` raises `TargetClosedError`. - -## Function configuration recommendations - -Whatever tool you use to create the Lambda function (CLI, CDK, Terraform, SAM, console), apply these settings: - -| Setting | Value | Why | -|---|---|---| -| Package type | Image | Required — this is a container image, not a zip. | -| Architecture | `arm64` | Roughly 20% cheaper than x86_64. Native build on Apple Silicon. Match the architecture you built for. | -| Memory | 3008 MB | Memory in Lambda is tied to vCPU. Below ~1769 MB Chromium starts noticeably slower. | -| Timeout | 120–180 s | Single-attempt scrapes complete in 3–15 s warm; under retry, a `Timeout`-class first failure (30 s default) plus a longer-budget retry (90 s) plus cleanup can total ~120-130 s. 180 s leaves headroom; below 120 s the function will time out before the retry completes. Cold-start init adds 5-10 s on top. | -| Ephemeral storage (`/tmp`) | 1024 MB | Chromium profile dirs and screenshots can fill the 512 MB default. | -| Networking | Default (no VPC) | Binary is baked in, no network needed at cold start. Add VPC + NAT only if your proxy egress requires it. | -| Execution role | `AWSLambdaBasicExecutionRole` | Just CloudWatch Logs. Add more permissions only if your handler needs them. | - -## Cold start - -First invocation in a new container takes ~80–90 s (image extraction, Chromium binary mmap, JS engine warmup, no DNS/TLS caches). Subsequent warm invocations on the same container are 3–15 s. - -For latency-sensitive use cases: provision concurrency, schedule a CloudWatch/EventBridge warmer ping, or accept the cold tail. - -If you see empty/missing dynamic content on cold-start invocations, raise `max_settle_ms` in the event payload (e.g. `25000`) — the default `15000` is tuned for warm runs. - -## Security - -The handler validates all incoming URLs before navigation: - -- **Scheme restriction** — only `http://` and `https://` are accepted. `file://`, `data:`, `javascript:`, and other schemes are rejected. -- **SSRF protection** — hostnames are resolved before navigation and checked against private, loopback, link-local, reserved, and multicast IP ranges. This blocks access to cloud metadata endpoints (e.g. `169.254.169.254`), localhost services, and internal networks. -- **Post-navigation re-validation** — the final URL is re-checked after page load and after post-navigation waits to catch server-side redirects to blocked destinations. -- **No caller-controlled Chromium flags** — the handler does not accept arbitrary CLI flags from the event. Internal retry strategies add flags as needed (e.g. `--ignore-certificate-errors` for cert errors). -- **No arbitrary JS execution** — `wait_for_function` is not exposed. Use `wait_for_selector` or `smart_wait` instead. - -**Limitations**: -- Post-navigation re-validation prevents response *exfiltration*, but does not prevent the browser from *making* the request. If an internal endpoint has side effects on GET, the request will still reach it before validation rejects the response. Use network-level controls (security groups, VPC) to protect side-effect-bearing internal endpoints. -- DNS rebinding attacks can bypass pre-navigation IP checks in theory, though the post-navigation re-validation provides a second layer of defense. - -**Trust boundary**: if this handler is exposed to untrusted callers (Lambda Function URL, API Gateway without auth, public ALB), add an authentication layer (API Gateway authorizer, IAM auth, etc.). The URL validation above is defense-in-depth, not a substitute for access control. - -## License - -The patched Chromium binary inside the upstream `cloakhq/cloakbrowser` image is governed by the **CloakBrowser Binary License** (published at https://github.com/CloakHQ/CloakBrowser/blob/main/BINARY-LICENSE.md). Internal organizational use (private ECR, your own scraping pipelines, your own business) is free. Exposing this Lambda as a paid API to third-party customers — i.e. browser-as-a-service — requires an OEM/SaaS license from CloakHQ (`cloakhq@pm.me`). Do not push the resulting image to a public registry; that would be redistribution and is prohibited. diff --git a/docker/lambda/cloak-example/lambda-entrypoint.sh b/docker/lambda/cloak-example/lambda-entrypoint.sh deleted file mode 100644 index 0118084..0000000 --- a/docker/lambda/cloak-example/lambda-entrypoint.sh +++ /dev/null @@ -1,52 +0,0 @@ -#!/bin/sh -# Dual-mode entrypoint for the CloakBrowser Lambda image. -# -# 1. Always start Xvfb on :99 (same as the canonical bin/docker-entrypoint.sh) -# so headed Chromium works no matter how the container is invoked. -# 2. Detect whether the CMD looks like a Lambda handler (a single -# `module.func`-shaped argument). If yes, route through the Lambda runtime -# client (using the bundled aws-lambda-rie locally, or talking to the real -# Lambda Runtime API when AWS_LAMBDA_RUNTIME_API is set in production). -# 3. Otherwise exec the CMD directly — preserving the canonical Dockerfile's -# interaction surface (`python`, `cloakserve`, `cloaktest`, `node`, `bash`, -# `python examples/basic.py`, etc.). -set -e - -mkdir -p /tmp/.X11-unix -chmod 1777 /tmp/.X11-unix 2>/dev/null || true - -# Clean any stale Xvfb state. If a previous Xvfb died and left its lock file -# behind (we observed this in cold-start storms), a new Xvfb refuses to start -# with "Server is already active for display 99". Removing both files makes -# Xvfb start cleanly every time. -rm -f /tmp/.X99-lock /tmp/.X11-unix/X99 - -Xvfb :99 -screen 0 1920x1080x24 -nolisten tcp >/tmp/Xvfb.log 2>&1 & - -# Wait for the X11 socket to appear AND for Xvfb to be ready to serve. The -# socket file appears at bind(), but listen() and the first accept() come -# slightly later — under cold-start CPU contention this gap matters. -i=0 -while [ ! -e /tmp/.X11-unix/X99 ] && [ "$i" -lt 200 ]; do - i=$((i + 1)) - sleep 0.05 -done -# Small buffer after the socket appears so Xvfb has a moment to call listen() -# and start accepting clients. Cheap insurance against the bind/listen gap. -sleep 0.2 - -# Lambda handler shape: exactly one arg, dotted identifier (no spaces, no slashes, -# no leading dot). `python`, `cloakserve`, `cloaktest`, `bash`, `node` all fail -# this test and pass through to plain exec. -if [ $# -eq 1 ] && \ - echo "$1" | grep -qE '^[a-zA-Z_][a-zA-Z0-9_]*(\.[a-zA-Z_][a-zA-Z0-9_]*)+$'; then - if [ -z "${AWS_LAMBDA_RUNTIME_API}" ]; then - # Local invocation via bundled RIE. - exec /usr/local/bin/aws-lambda-rie /usr/local/bin/python -m awslambdaric "$@" - else - # Real Lambda — runtime API endpoint already provided by the platform. - exec /usr/local/bin/python -m awslambdaric "$@" - fi -fi - -exec "$@" diff --git a/docker/lambda/cloak-example/lambda_handler.py b/docker/lambda/cloak-example/lambda_handler.py deleted file mode 100644 index 0f0136e..0000000 --- a/docker/lambda/cloak-example/lambda_handler.py +++ /dev/null @@ -1,382 +0,0 @@ -"""AWS Lambda handler for one-off stealth-browser invocations. - -Always runs **headed** via the Xvfb display started by `lambda-entrypoint.sh`. - -Event schema (all fields except `url` are optional): - - Launch options (passed to cloakbrowser.launch_context_async): - url str required, the page to scrape (http/https only) - proxy str|dict http://user:pass@host:port or Playwright proxy dict - humanize bool False — enable human-like mouse/keyboard/scroll - human_preset str "default" | "careful" - geoip bool False — auto timezone+locale from proxy IP - timezone str IANA tz, e.g. "America/New_York" - locale str BCP-47, e.g. "en-US" - viewport {width,height} defaults to 1920x947 (cloakbrowser DEFAULT_VIEWPORT) - user_agent str custom UA (rare — cloakbrowser sets one already) - - Navigation options (passed to page.goto): - wait_until str "load"|"domcontentloaded"|"networkidle"|"commit" - default "domcontentloaded" - goto_timeout_ms int 30000 - - Post-navigation waits (run in this order if specified): - smart_wait bool ON by default if no other wait is set. - Polls document.outerHTML.length and bails when it - hasn't changed for `dom_stable_ms`. Handles lazy - hydration, async chunks, and lazy images, and is - immune to analytics beacons / long-poll that keep - the network busy without mutating the DOM. - dom_stable_ms int 1500 — how long DOM must be quiet - max_settle_ms int 15000 — hard cap on smart_wait - wait_for_load_state str "load"|"domcontentloaded"|"networkidle" - wait_for_load_state_timeout_ms int 30000 - wait_for_selector str CSS or XPath selector - wait_for_selector_state str "attached"|"detached"|"visible"|"hidden", default "visible" - wait_for_selector_timeout_ms int 30000 - wait_ms int fixed pause in ms (page.wait_for_timeout) - - Capture options: - screenshot bool True - full_page_screenshot bool False — capture entire scrollable page - - Retry orchestration: - retries int default 1. Number of retry attempts after the first - failure. Set to 0 to disable retries entirely (the - handler will fail fast on the first error). - Retried errors: - ERR_CERT_* -> retry with --ignore-certificate-errors - Timeout exceeded -> retry with goto_timeout_ms=90000, max_settle_ms=25000 - ERR_CONNECTION_TIMED_OUT -> same as Timeout - Not retried (unrecoverable): ERR_NAME_NOT_RESOLVED, - ERR_SSL_PROTOCOL_ERROR, generic ERR_CONNECTION_REFUSED. - On final failure, the error message includes a - retry_history block with strategy + error per attempt. - -Returns: - {"title": ..., "url": ..., "html": ..., "screenshot_b64"?: ...} -""" - -from __future__ import annotations - -import asyncio -import base64 -import ipaddress -import json -import logging -import socket -import subprocess -from pathlib import Path -from typing import Any -from urllib.parse import urlparse - -from cloakbrowser import launch_context_async - -logger = logging.getLogger("cloakbrowser.lambda") -logger.setLevel(logging.INFO) - - -def _validate_url(url: str) -> None: - """Reject non-HTTP schemes and URLs that resolve to private/internal IPs.""" - parsed = urlparse(url) - if parsed.scheme.lower() not in ("http", "https"): - raise ValueError( - f"Only http:// and https:// URLs are supported, got: {parsed.scheme!r}" - ) - hostname = parsed.hostname - if not hostname: - raise ValueError("URL has no hostname") - try: - infos = socket.getaddrinfo(hostname, None, socket.AF_UNSPEC, socket.SOCK_STREAM) - except socket.gaierror: - raise ValueError(f"Cannot resolve hostname: {hostname}") - for info in infos: - addr = ipaddress.ip_address(info[4][0]) - if not addr.is_global: - raise ValueError("URLs targeting private/internal networks are blocked") - - -def _diag_snapshot() -> str: - """Capture Xvfb status, Xvfb log, X11 socket state, and env for error reports.""" - import os - - parts = [] - try: - r = subprocess.run(["pgrep", "-fa", "Xvfb"], capture_output=True, text=True) - parts.append(f"pgrep Xvfb: rc={r.returncode} stdout={r.stdout.strip()!r}") - except Exception as e: - parts.append(f"pgrep failed: {e}") - try: - r = subprocess.run( - ["ls", "-la", "/tmp/.X11-unix"], capture_output=True, text=True - ) - parts.append(f"ls /tmp/.X11-unix:\n{r.stdout}{r.stderr}") - except Exception as e: - parts.append(f"ls /tmp/.X11-unix failed: {e}") - try: - log = Path("/tmp/Xvfb.log").read_text() - parts.append(f"/tmp/Xvfb.log:\n{log}") - except Exception as e: - parts.append(f"Xvfb log unreadable: {e}") - parts.append( - f"env: DISPLAY={os.environ.get('DISPLAY')!r} HOME={os.environ.get('HOME')!r}" - ) - return "\n".join(parts) - - -def handler(event: dict, context: Any) -> dict: - return asyncio.run(_run(event)) - - -def _build_launch_kwargs(event: dict) -> dict: - """Translate the event dict into kwargs for launch_context_async. - - Only includes keys explicitly set in the event so cloakbrowser's defaults - (DEFAULT_VIEWPORT etc.) kick in when fields are absent — passing - viewport=None would *disable* viewport emulation, which we don't want. - """ - kwargs: dict = { - "headless": False, # always headed via Xvfb - "args": [ - # Lambda /dev/shm is ~64 MB — Chromium crashes mid-render without this. - "--disable-dev-shm-usage", - # Lambda's restricted process model can't fork from Chromium's zygote - # — without this, child renderer processes fail to spawn. - "--no-zygote", - *event.get("_strategy_args", []), - ], - } - for key in ( - "proxy", - "humanize", - "human_preset", - "geoip", - "timezone", - "locale", - "viewport", - "user_agent", - ): - if key in event: - kwargs[key] = event[key] - return kwargs - - -async def _smart_wait( - page, dom_stable_ms: int = 1500, max_settle_ms: int = 15000 -) -> None: - """Wait until the document HTML hasn't changed for `dom_stable_ms`. - - Generic stopping condition for at-scale scraping when you can't tune - selectors per site. More robust than `networkidle` because it ignores - network activity that doesn't mutate the DOM (analytics beacons, - long-poll, websockets, web vitals streams). - """ - js = f""" - (() => {{ - if (!window.__cb_settle) {{ - window.__cb_settle = {{ len: -1, since: Date.now() }}; - }} - const cur = document.documentElement.outerHTML.length; - const s = window.__cb_settle; - if (cur !== s.len) {{ - s.len = cur; - s.since = Date.now(); - return false; - }} - return (Date.now() - s.since) >= {int(dom_stable_ms)}; - }})() - """ - try: - await page.wait_for_function(js, timeout=max_settle_ms, polling=200) - except Exception: - # Hit max_settle_ms cap — return what we have rather than fail the whole invoke - logger.warning("smart_wait hit max_settle_ms=%d cap", max_settle_ms) - - -_EXPLICIT_WAIT_KEYS = ( - "wait_for_load_state", - "wait_for_selector", - "wait_ms", -) - - -async def _post_nav_waits(page, event: dict) -> None: - """Run waits in priority order. smart_wait is the default unless the - caller asked for a more specific stopping condition.""" - explicit = any(k in event for k in _EXPLICIT_WAIT_KEYS) - if event.get("smart_wait", not explicit): - await _smart_wait( - page, - dom_stable_ms=event.get("dom_stable_ms", 1500), - max_settle_ms=event.get("max_settle_ms", 15000), - ) - if "wait_for_load_state" in event: - await page.wait_for_load_state( - event["wait_for_load_state"], - timeout=event.get("wait_for_load_state_timeout_ms", 30000), - ) - if "wait_for_selector" in event: - await page.wait_for_selector( - event["wait_for_selector"], - state=event.get("wait_for_selector_state", "visible"), - timeout=event.get("wait_for_selector_timeout_ms", 30000), - ) - if "wait_ms" in event: - await page.wait_for_timeout(event["wait_ms"]) - - -async def _launch_with_retry(event: dict, attempts: int = 3, backoff_s: float = 0.3): - """Retry launch_context_async up to `attempts` times with linear backoff. - - Lambda cold-start storms occasionally race Xvfb readiness or hit transient - Chromium spawn failures — both surface as "Target page, context or browser - has been closed" at launch. The failure is fast (~0.5s) so retries are - cheap, and a retry on a now-warm container almost always succeeds. - - Pairs with the lock-cleanup + socket-poll in lambda-entrypoint.sh: the - entrypoint catches the common case at container init; this catches the - residual race when the first invocation hits before Xvfb is fully ready. - """ - last_err: Exception | None = None - for i in range(attempts): - try: - return await launch_context_async(**_build_launch_kwargs(event)) - except Exception as e: - last_err = e - logger.warning( - "launch attempt %d/%d failed: %s", i + 1, attempts, str(e)[:200] - ) - if i + 1 < attempts: - await asyncio.sleep(backoff_s * (i + 1)) # 0.3s, 0.6s - raise last_err # type: ignore[misc] - - -def _classify_error(err: Exception) -> dict | None: - """Map a Playwright error to a retry-strategy override dict, or None - if the error is unrecoverable. - - Match on str(e) because Playwright errors carry their codes inside the - message (Error.__str__ includes ERR_CERT_AUTHORITY_INVALID etc.); there - is no stable structured `.error_code` attribute to rely on. - - Strategies (priority order — first match wins): - ERR_CERT_* -> --ignore-certificate-errors + 60s goto budget - Timeout exceeded -> 90s goto budget + 25s smart_wait cap - ERR_CONNECTION_TIMED_OUT -> same as Timeout - Returns None for unrecoverable site issues (DNS, SSL, refused, HTTP 4xx/5xx). - """ - msg = str(err) - if "ERR_CERT" in msg: - return { - "_strategy_args": ["--ignore-certificate-errors"], - "goto_timeout_ms": 60000, - } - if ("Timeout" in msg and "exceeded" in msg) or "ERR_CONNECTION_TIMED_OUT" in msg: - return { - "goto_timeout_ms": 90000, - "max_settle_ms": 25000, - } - return None - - -async def _attempt_scrape(url: str, event: dict) -> dict: - """One self-contained scrape attempt: launch, navigate, wait, capture, close. - - Extracted from `_run` so the retry loop can call it repeatedly with an - overridden event dict. Each attempt relaunches the browser — uniform - behavior across strategies (the cert-bypass strategy *requires* a relaunch - because `--ignore-certificate-errors` is a Chromium CLI arg, not a per- - context switch), and the ~3-5s relaunch cost is fine on the slow path. - """ - ctx = await _launch_with_retry(event) - try: - page = await ctx.new_page() - await page.goto( - url, - wait_until=event.get("wait_until", "domcontentloaded"), - timeout=event.get("goto_timeout_ms", 30000), - ) - _validate_url(page.url) - - await _post_nav_waits(page, event) - _validate_url(page.url) - - result: dict = { - "title": await page.title(), - "url": page.url, - "html": await page.content(), - } - - if event.get("screenshot", True): - png = await page.screenshot( - full_page=event.get("full_page_screenshot", False), - ) - result["screenshot_b64"] = base64.b64encode(png).decode() - - return result - finally: - try: - await ctx.close() - except Exception: - pass - - -def _raise_with_history(err: Exception, history: list[dict]) -> None: - """Surface a final failure with a retry_history block embedded in the - error message, so callers see what was tried before bailing.""" - diag = _diag_snapshot() - if history: - diag = "retry_history: " + json.dumps(history, default=str) + "\n\n" + diag - logger.error( - "scrape failed (after %d retries): %s\nDIAG:\n%s", len(history), err, diag - ) - raise RuntimeError(f"scrape failed: {err}\n--- DIAG ---\n{diag}") from err - - -async def _run(event: dict) -> dict: - """Top-level scrape with strategy-based retry orchestration. - - First attempt uses the event verbatim. If it fails with a classifiable - error (cert / timeout), retry with that strategy's overrides merged into - the event. `retries` bounds the number of strategy retries (default 1; - set to 0 to disable retry entirely). - """ - url = event["url"] - _validate_url(url) - event = { - k: v for k, v in event.items() if k not in ("extra_args", "_strategy_args") - } - retries_left = max(0, int(event.get("retries", 1))) - history: list[dict] = [] - current_event = event - - while True: - try: - return await _attempt_scrape(url, current_event) - except Exception as e: - if retries_left <= 0: - _raise_with_history(e, history) - strategy = _classify_error(e) - if strategy is None: - _raise_with_history(e, history) - history.append( - { - "attempt": len(history) + 1, - "error": str(e)[:300], - "strategy": strategy, - } - ) - logger.warning( - "attempt %d failed (%s); retrying with strategy=%s", - len(history), - str(e)[:120], - strategy, - ) - merged_args = list(current_event.get("_strategy_args", [])) + list( - strategy.get("_strategy_args", []) - ) - current_event = {**current_event, **strategy, "_strategy_args": merged_args} - retries_left -= 1 - # No backoff: strategy overrides change goto budget directly; - # the prior failure was either fast (cert reject) or already - # waited its full timeout. Container is warm. diff --git a/docker/lambda/grab-seleniumbase/Dockerfile b/docker/lambda/grab-seleniumbase/Dockerfile deleted file mode 100644 index 5c4adfd..0000000 --- a/docker/lambda/grab-seleniumbase/Dockerfile +++ /dev/null @@ -1,46 +0,0 @@ -# SeleniumBase (Stealthy Playwright Mode) fast single-page capture (HTML + -# full-page screenshot) on AWS Lambda. -# -# Built FROM the shared bytelyon-seleniumbase-base image (see -# ../../images/seleniumbase-base/Dockerfile and -# ../../images/seleniumbase-base/INSTRUCTIONS.md), which carries every bit of -# common Lambda glue (awslambdaric, the Lambda Runtime Interface Emulator, -# boto3, the dual-mode entrypoint) plus system Chromium, `seleniumbase`, and -# `playwright`, and the `seleniumbase_playwright.launch()` shim that connects -# a normal Playwright `Browser` to SeleniumBase's stealthy CDP Mode Chromium. -# This Dockerfile only adds this handler's own module — see -# lambda_handler.py. -# -# This is the SeleniumBase counterpart of ../grab: same event schema, same -# S3 upload behavior, same response shape. The only difference is the -# stealth-browser provider — SeleniumBase's CDP Mode -# (https://seleniumbase.io/) instead of CloakBrowser — see -# ../../images/seleniumbase-base/seleniumbase_playwright.py for how that -# swap is bridged behind an (almost) identical `launch()` call. -# -# ─── Build (the base image must already exist, locally or in ECR) ──────────── -# docker buildx build --platform linux/arm64 --provenance=false --sbom=false \ -# -t bytelyon-grab-seleniumbase:arm64 --load . -# -# docker run --rm -p 9000:8080 \ -# -e AWS_ACCESS_KEY_ID -e AWS_SECRET_ACCESS_KEY -e AWS_SESSION_TOKEN -e AWS_REGION=us-east-1 \ -# bytelyon-grab-seleniumbase:arm64 -# curl -XPOST http://localhost:9000/2015-03-31/functions/function/invocations \ -# -d '{"url":"https://example.com"}' -# -# --provenance=false --sbom=false is required: without it, buildx emits a -# manifest-list/attestation image that Lambda's CreateFunction/UpdateFunctionCode -# rejects with "image manifest, config or layer media type ... is not supported". -# -# To build against a locally-loaded (not-yet-pushed) base image instead of -# pulling from ECR, see ../../images/seleniumbase-base/INSTRUCTIONS.md: -# --build-arg BASE_IMAGE=bytelyon-seleniumbase-base:arm64 - -ARG BASE_IMAGE=138305277395.dkr.ecr.us-east-1.amazonaws.com/bytelyon-seleniumbase-base:latest -FROM ${BASE_IMAGE} - -# Handler sits at /app (already on Python's import path, WORKDIR=/app), -# imports playwright.sync_api types plus this repo's seleniumbase_playwright -# shim (on PYTHONPATH via /opt/python). ENTRYPOINT/CMD are inherited from the -# base image (both already correct for this handler's module name). -COPY lambda_handler.py /app/lambda_handler.py diff --git a/docker/lambda/grab-seleniumbase/INSTRUCTIONS.md b/docker/lambda/grab-seleniumbase/INSTRUCTIONS.md deleted file mode 100644 index 12334cc..0000000 --- a/docker/lambda/grab-seleniumbase/INSTRUCTIONS.md +++ /dev/null @@ -1,209 +0,0 @@ -# SeleniumBase fast single-page capture on AWS Lambda - -The [SeleniumBase](https://seleniumbase.io/) counterpart of -[`../grab`](../grab): given a single URL (with an optional proxy), uploads -the rendered HTML and a full-page screenshot to S3, and returns just the two -S3 keys. Same event schema, same S3 upload behavior, same response shape as -`../grab` -- the only difference is the stealth-browser provider. - -This handler builds `FROM` [`../../images/seleniumbase-base`](../../images/seleniumbase-base) -instead of `../base-image`. That base image bakes in system Chromium, -`seleniumbase`, `playwright` (client only), and a `seleniumbase_playwright.launch()` -shim that launches SeleniumBase's stealthy CDP Mode Chromium and hands back -a normal `playwright.sync_api.Browser`, connected to it over the Chrome -DevTools Protocol ("Stealthy Playwright Mode" -- see the -[SeleniumBase README](https://seleniumbase.io/)). Every line of this -handler past the `launch()` call is otherwise identical to `../grab`, -because both providers hand back a standard Playwright `Page`. - -## Files in this directory - -| File | Purpose | -| ------------------- | ------------------------------------------------------------------------------------------------------- | -| `Dockerfile` | `FROM` the shared `../../images/seleniumbase-base`, plus a single `COPY lambda_handler.py`. | -| `lambda_handler.py` | Handler. Takes `{url, proxy?, ...}`, returns `{url, content_key, screenshot_key}`. Headless by default. | -| `INSTRUCTIONS.md` | This file. | - -This directory depends on the shared base image -([`../../images/seleniumbase-base`](../../images/seleniumbase-base)) having -been built and either published to ECR or loaded locally first -- see that -directory's `INSTRUCTIONS.md`. - -## Build - -Build and publish `../../images/seleniumbase-base` first (see its -`INSTRUCTIONS.md`), then: - -```bash -docker buildx build --platform linux/arm64 --provenance=false --sbom=false \ - -t bytelyon-grab-seleniumbase:arm64 --load . -``` - -`--provenance=false --sbom=false` is required -- without it, buildx produces -a manifest-list/attestation image that Lambda's `CreateFunction` rejects -with `InvalidParameterValueException: ... image manifest, config or layer -media type ... is not supported`. - -To build against a locally-loaded (not-yet-pushed) base image instead of -pulling from ECR: - -```bash -docker buildx build --platform linux/arm64 --provenance=false --sbom=false \ - --build-arg BASE_IMAGE=bytelyon-seleniumbase-base:arm64 \ - -t bytelyon-grab-seleniumbase:arm64 --load . -``` - -For x86_64, switch `--platform linux/amd64` (and build the base image for -the same architecture). Unlike CloakBrowser's official image, the -SeleniumBase base image's system Chromium supports both architectures -equally, since it comes from Debian's `chromium` package rather than Google -Chrome (amd64-only). - -## Local smoke test (no Lambda deploy needed, but real AWS credentials required) - -Both artifacts are genuinely uploaded to S3 even in local RIE testing, so -pass through real AWS credentials with write access to the target bucket: - -```bash -docker run --rm -p 9000:8080 \ - -e AWS_ACCESS_KEY_ID -e AWS_SECRET_ACCESS_KEY -e AWS_SESSION_TOKEN -e AWS_REGION=us-east-1 \ - bytelyon-grab-seleniumbase:arm64 - -# In another shell: -curl -sS -XPOST "http://localhost:9000/2015-03-31/functions/function/invocations" \ - -d '{"url":"https://example.com"}' - -# With a proxy: -curl -sS -XPOST "http://localhost:9000/2015-03-31/functions/function/invocations" \ - -d '{"url":"https://example.com", "proxy":"http://user:pass@host:port"}' -``` - -## Event schema - -| Field | Type | Default | -| ---------------------- | --------- | ----------------------------------------------------------------------------------------------------------- | -| `url` | str | required -- the page to capture (http/https only) | -| `proxy` | str\|dict | none -- `"http://user:pass@host:port"` (credentials split out automatically) or a `ProxySettings` dict | -| `geoip` | bool | `false` -- accepted for parity with `../grab`; **no-op** with this provider (see below) | -| `bucket` | str | `"bytelyon-private"` | -| `prefix` | str | `"grabs/{aws_request_id}/"` -- S3 key prefix for both uploads | -| `headless` | bool | `true` | -| `humanize` | bool | `false` -- accepted for parity with `../grab`; **no-op** with this provider (see below) | -| `human_preset` | str | `"careful"` -- accepted for parity; **no-op** | -| `wait_until` | str | `"domcontentloaded"` -- passed to `page.goto` (`"load"`\|`"domcontentloaded"`\|`"networkidle"`\|`"commit"`) | -| `goto_timeout_ms` | int | `30000` | -| `full_page_screenshot` | bool | `true` | - -### Response - -```json -{ - "url": "https://example.com/", - "content_key": "grabs//example_com-.html", - "screenshot_key": "grabs//example_com-.png" -} -``` - -## Differences from `../grab` (CloakBrowser) - -- **Stealth mechanism**: SeleniumBase's CDP Mode launch flags/patches - instead of CloakBrowser's own stealth Chromium build. Neither this - handler nor `../grab` makes any claim about which is more effective - against any particular anti-bot vendor -- pick based on what works for - your target sites. -- **`humanize` / `human_preset` are no-ops** -- these are CloakBrowser - concepts (synthetic mouse/keyboard/scroll timing). Accepted here purely - so callers can point either handler at the same event payload. -- **`geoip` is a no-op** -- CloakBrowser auto-derives timezone/locale from - the proxy's exit IP; SeleniumBase has no built-in equivalent. If you need - specific timezone/locale values, this handler would need extending to - accept and forward explicit `timezone`/`locale` fields (both are already - accepted by `seleniumbase_playwright.launch(**kwargs)` -- see - `../../images/seleniumbase-base/seleniumbase_playwright.py`). -- **No chromedriver** -- SeleniumBase's CDP Mode talks directly to Chromium - over the DevTools Protocol, so the base image doesn't install one. - -## Lambda-specific additions - -- **Chromium hardening (do not remove)**: `--no-zygote` and `--disable-gpu` - are always applied by `seleniumbase_playwright.launch()` - (`--disable-dev-shm-usage` is already one of SeleniumBase's own default - CDP Mode args, and `--no-sandbox` is always forced on for the same - reason -- see that module's docstring). Notably, `--single-process` is - deliberately **not** applied, despite being a common recommendation for - "Chromium on Lambda" setups: confirmed by testing against a real deployed - Lambda function, it actively breaks launches here (Chromium starts but - its CDP debug port never opens) -- see the long comment above the flag - list in `seleniumbase_playwright.py` for the full story. This only - reproduced in real Lambda, never in a plain `docker run` of the same - image, so if you're debugging a _different_ launch problem, don't trust - a local `docker run`/RIE pass alone -- test against a real deployed - function too. -- **Launch retries + captured Chromium output**: `seleniumbase_playwright.launch()` - retries the actual Chromium launch up to 3 times (a fresh Lambda - execution environment's very first launch attempt is occasionally flaky - even with correct arguments -- observed empirically, not just - theorized), and on final failure raises with whatever the Chromium - subprocess itself printed to stdout/stderr (normally discarded to - DEVNULL by SeleniumBase), turning a generic "Failed to connect to the - browser" into an actionable error. -- **URL validation**: scheme + SSRF checks on the input `url` before - navigation, and again on `page.url` after navigation/redirects. -- **Proxy TCP preflight**: if `proxy` is set, a raw TCP connect to the - proxy's host:port is attempted first (~8s timeout) so an - unreachable/blocking proxy fails fast with a clear error instead of - hanging until the function timeout. Same as `../grab` and - `../serp-scraper`. - -## IAM permissions - -In addition to `AWSLambdaBasicExecutionRole` (CloudWatch Logs), the -execution role needs write access to the target bucket: - -```json -{ - "Version": "2012-10-17", - "Statement": [ - { - "Effect": "Allow", - "Action": ["s3:PutObject"], - "Resource": ["arn:aws:s3:::bytelyon-private/*"] - } - ] -} -``` - -## Function configuration recommendations - -| Setting | Value | Why | -| -------------------------- | ----------------------------------------------------------------- | ------------------------------------------------------------------------------------------------------------- | -| Package type | Image | Required -- this is a container image, not a zip. | -| Architecture | `arm64` | Roughly 20% cheaper than x86_64. Match the architecture you built for. | -| Memory | 3008 MB | Memory in Lambda is tied to vCPU. Below ~1769 MB Chromium starts noticeably slower. | -| Timeout | 30-45 s | Single page load + full-page screenshot + concurrent uploads is fast once warm; budget for cold start on top. | -| Ephemeral storage (`/tmp`) | 512 MB (default) | Both artifacts are held in memory and streamed to S3, not accumulated on disk. | -| Networking | Default (no VPC) | Only outbound HTTPS to the target site (or proxy) and to S3 is needed. | -| Execution role | Basic execution + S3 `PutObject` on the target bucket (see above) | | - -## Cold start - -Measured against a real deployed Lambda function (`arm64`, 3008 MB): first -invocation in a new execution environment typically completes in -**~4-10s** end-to-end (Lambda's own `Init Duration` for this image is -~2.3s; the rest is the Chromium launch + page load + S3 upload). Warm -invocations (same execution environment, fresh Chromium process each time --- this handler doesn't persist a browser across invocations) run in -**~3-4s**. This is notably faster than `../grab`'s documented ~80-90s -cold start -- unclear whether that's inherent to CloakBrowser's own bundled -Chromium vs. this image's Debian `chromium` package, image size/layer -count differences, or something else; take both numbers as rough guidance -rather than a rigorous head-to-head benchmark. - -Occasionally (empirically, roughly 1 in 4-5 cold starts in initial -testing) a fresh execution environment's _very first_ Chromium launch -attempt fails outright before the debug port ever opens, while a retry in -the same, now-warm container succeeds immediately -- `seleniumbase_playwright.launch()` -retries internally for exactly this reason (see "Lambda-specific -additions" above), so this shouldn't surface as a handler-level failure, -but it does mean an occasional cold start takes a few seconds longer than -the happy path above while a retry plays out. diff --git a/docker/lambda/grab-seleniumbase/lambda_handler.py b/docker/lambda/grab-seleniumbase/lambda_handler.py deleted file mode 100644 index b57f1cb..0000000 --- a/docker/lambda/grab-seleniumbase/lambda_handler.py +++ /dev/null @@ -1,311 +0,0 @@ -"""AWS Lambda handler: fast single-page capture (HTML + full-page -screenshot) to S3 -- the SeleniumBase (Stealthy Playwright Mode) counterpart -of ../grab/lambda_handler.py. - -Given a single URL, this handler: - 1. Navigates to the page (optionally through a proxy). - 2. Captures the rendered HTML and a full-page screenshot. - 3. Uploads both to S3, concurrently. - -Deliberately minimal -- no link/meta extraction, no consent-dialog handling, -no retry orchestration, no headed/Xvfb mode. Every step that isn't required -for "visit a URL, save its content and a screenshot" is left out on purpose, -same as ../grab. - -Uses the same **sync** Playwright API as ../grab (`page.goto`, `page.content`, -`page.screenshot`, ...) -- the only difference from ../grab is *how* the -stealthy browser is launched: `seleniumbase_playwright.launch()` (baked into -../../images/seleniumbase-base) instead of `cloakbrowser.launch()`. Both -return an object exposing `.new_page()` / `.close()` and hand back a normal -`playwright.sync_api.Page`, so every line below this import is unchanged -from ../grab. See ../../images/seleniumbase-base/seleniumbase_playwright.py -for how that provider swap is bridged, and for the handful of interface -differences (`humanize`/`human_preset`/`geoip` are accepted below for -call-site parity with ../grab, but are no-ops with this provider -- see that -module's docstring for why). - -Event schema: - url str required -- the page to capture (http/https only) - proxy str|dict none -- "http://user:pass@host:port" or a - Playwright ProxySettings dict - (credentials are split out automatically -- - see _normalize_proxy) - geoip bool false -- accepted for parity with ../grab; - NO-OP with this provider (SeleniumBase has no - built-in geoip-based timezone/locale - derivation -- pass `timezone=`/`locale=` - directly if you need specific values) - bucket str "bytelyon-private" - prefix str "grabs/{aws_request_id}/" -- S3 key prefix - headless bool true - humanize bool false -- accepted for parity with ../grab; - NO-OP with this provider (see module - docstring in seleniumbase_playwright.py) - human_preset str "careful" -- accepted for parity; NO-OP - wait_until str "domcontentloaded" -- passed to page.goto - ("load"|"domcontentloaded"|"networkidle"|"commit") - goto_timeout_ms int 30000 - full_page_screenshot bool true - -Returns: - { - "url": "https://example.com/", - "content_key": "grabs//example_com-.html", - "screenshot_key": "grabs//example_com-.png" - } - -Lambda-specific additions (see ../../images/seleniumbase-base/INSTRUCTIONS.md -for the same rationale in the base image, and ../grab/lambda_handler.py for -the CloakBrowser-based version of the same additions): - - `--no-zygote` Chromium flag (`--disable-dev-shm-usage` is already one of - SeleniumBase's own default CDP Mode browser args, and `--no-sandbox` is - always applied by seleniumbase_playwright.launch() -- neither needs to - be passed here). - - URL scheme + SSRF validation before *and* after navigation (defense in - depth against private/internal network access). - - A TCP preflight on the proxy (if any) so an unreachable/blocking proxy - fails in ~8s instead of hanging for the full function timeout (same as - ../grab and ../serp-scraper). - -Speed choices (identical rationale to ../grab): - - `humanize`/`human_preset` are no-ops here (see above), but harmless to - pass -- no need to special-case handler code per stealth provider. - - `wait_until` defaults to `"domcontentloaded"`, which fires as soon as - the DOM is parsed rather than waiting for every subresource (images, - fonts, analytics beacons) to finish loading. - - No link/meta extraction, no consent-dialog handling, no retry - orchestration. - - HTML and screenshot are uploaded to S3 concurrently (two independent - network calls), not sequentially. -""" - -from __future__ import annotations - -import hashlib -import ipaddress -import logging -import re -import socket -import time -from concurrent.futures import ThreadPoolExecutor -from typing import Any -from urllib.parse import unquote, urlparse - -import boto3 -from playwright.sync_api import Page, ProxySettings -from playwright.sync_api import TimeoutError as PlaywrightTimeoutError -from seleniumbase_playwright import launch - -logger = logging.getLogger("seleniumbase.lambda") -logger.setLevel(logging.INFO) - -# awslambdaric wires the root logger with a formatter like -# "[INFO]\t2024-01-15T10:30:45.123Z\t\tmessage" -- CloudWatch already -# timestamps every event server-side, so repeating timestamp + request id on -# every line just burns terminal columns. Trim to level + message. -for _h in logging.getLogger().handlers: - _h.setFormatter(logging.Formatter("%(levelname)s %(message)s")) - -DEFAULT_BUCKET = "bytelyon-private" - -# How long to wait for a raw TCP handshake with the proxy before giving up. -# `seleniumbase_playwright.launch(proxy=...)` has no timeout of its own -- if -# the proxy silently drops the connection (common for residential-proxy -# providers that block/throttle datacenter-origin traffic, e.g. AWS Lambda's -# egress IPs) the whole invocation would otherwise hang until Lambda's -# function timeout, burning the full billed duration with no diagnostic -# signal. This preflight fails fast with a clear, actionable error instead. -# Same as ../grab and ../serp-scraper. -_PROXY_PREFLIGHT_TIMEOUT_S = 8.0 - - -def _validate_url(url: str) -> None: - """Reject non-HTTP schemes and URLs that resolve to private/internal IPs.""" - parsed = urlparse(url) - if parsed.scheme.lower() not in ("http", "https"): - raise ValueError( - f"Only http:// and https:// URLs are supported, got: {parsed.scheme!r}" - ) - hostname = parsed.hostname - if not hostname: - raise ValueError("URL has no hostname") - try: - infos = socket.getaddrinfo(hostname, None, socket.AF_UNSPEC, socket.SOCK_STREAM) - except socket.gaierror: - raise ValueError(f"Cannot resolve hostname: {hostname}") - for info in infos: - addr = ipaddress.ip_address(info[4][0]) - if not addr.is_global: - raise ValueError("URLs targeting private/internal networks are blocked") - - -def _normalize_proxy(proxy: Any) -> ProxySettings | None: - """Normalize a `proxy` event value into Playwright's `ProxySettings` - shape: {"server": "://:", "bypass"?, "username"?, - "password"?}. - - seleniumbase_playwright.launch() accepts this same shape (or a plain - string) and converts it internally into the "[user:pass@]host:port" - string `seleniumbase.sb_cdp.Chrome(proxy=...)` expects -- see that - module's `_normalize_proxy()` docstring for why SeleniumBase's CDP Mode - doesn't need Playwright's own proxy-auth handling. A dict already in - this shape is passed through unchanged. Same logic as ../grab and - ../serp-scraper. - """ - if not proxy: - return None - if isinstance(proxy, dict): - return proxy - if not isinstance(proxy, str): - raise ValueError( - f"'proxy' must be a string or dict, got {type(proxy).__name__}" - ) - - parsed = urlparse(proxy) - if not parsed.hostname: - raise ValueError(f"'proxy' is not a valid proxy URL: {proxy!r}") - - server = f"{parsed.scheme or 'http'}://{parsed.hostname}" - if parsed.port: - server += f":{parsed.port}" - - normalized: ProxySettings = ProxySettings(server=server) - if parsed.username: - normalized["username"] = unquote(parsed.username) - if parsed.password: - normalized["password"] = unquote(parsed.password) - return normalized - - -def _proxy_host_port(proxy: ProxySettings | None) -> tuple[str, int] | None: - """Best-effort (host, port) extraction from an already-normalized proxy - dict, for the TCP preflight below. Returns None (preflight skipped, not - fatal) if it can't be determined. - """ - if not proxy: - return None - server = proxy.get("server") - if not isinstance(server, str) or not server: - return None - parsed = urlparse(server if "://" in server else f"http://{server}") - if not parsed.hostname or not parsed.port: - return None - return parsed.hostname, parsed.port - - -def _preflight_proxy( - proxy: ProxySettings | None, timeout: float = _PROXY_PREFLIGHT_TIMEOUT_S -) -> None: - """Raw TCP reachability check only -- deliberately does not attempt proxy - auth or a CONNECT tunnel, just confirms *something* is listening on - host:port so an unreachable/blocking proxy fails in ~8s instead of - hanging for the full function timeout. A pass here does not guarantee - the proxy will actually authenticate or tunnel traffic. - """ - host_port = _proxy_host_port(proxy) - if host_port is None: - return - host, port = host_port - started = time.monotonic() - try: - with socket.create_connection((host, port), timeout=timeout): - pass - except OSError as e: - raise RuntimeError( - f"Proxy preflight failed: could not open a TCP connection to " - f"{host}:{port} within {timeout}s ({e}). This is a network-level " - f"failure, not a credentials issue -- it usually means the proxy " - f"provider is blocking or silently dropping connections from " - f"this network (e.g. AWS Lambda's egress IP ranges). Verify with " - f"the proxy provider whether cloud/datacenter source IPs are " - f"allowed on this gateway." - ) from e - logger.info( - "Proxy preflight OK: TCP connect to %s:%s succeeded in %.2fs", - host, - port, - time.monotonic() - started, - ) - - -def _s3_key(prefix: str, url: str, ext: str) -> str: - parsed = urlparse(url) - path = parsed.path.strip("/") or "index" - safe = re.sub(r"[^a-zA-Z0-9/_-]", "_", f"{parsed.netloc}/{path}") - digest = hashlib.sha256(url.encode()).hexdigest()[:10] - return f"{prefix.rstrip('/')}/{safe}-{digest}.{ext}" - - -def handler(event: dict, context: Any) -> dict: - url = event.get("url") - if not isinstance(url, str) or not url.strip(): - raise ValueError("event must include a non-empty 'url' string") - _validate_url(url) - - bucket = event.get("bucket", DEFAULT_BUCKET) - prefix = event.get( - "prefix", f"grabs/{getattr(context, 'aws_request_id', 'local')}/" - ) - - s3 = boto3.client("s3") - - proxy: ProxySettings | None = _normalize_proxy(event.get("proxy")) - _preflight_proxy(proxy) - - browser = launch( - headless=event.get("headless", True), - humanize=event.get("humanize", False), - human_preset=event.get("human_preset", "careful"), - proxy=proxy, - # Accepted for call-site parity with ../grab -- currently a no-op - # with this provider. See seleniumbase_playwright.py's docstring. - geoip=event.get("geoip", False), - ) - page: Page = browser.new_page() - try: - try: - page.goto( - url, - wait_until=event.get("wait_until", "domcontentloaded"), - timeout=event.get("goto_timeout_ms", 30_000), - ) - except PlaywrightTimeoutError: - # Best-effort: capture whatever loaded so far rather than fail - # the whole invocation. - pass - - _validate_url(page.url) # re-check after redirects - - html = page.content() - png = page.screenshot(full_page=event.get("full_page_screenshot", True)) - - content_key = _s3_key(prefix, page.url, "html") - screenshot_key = _s3_key(prefix, page.url, "png") - - # Two independent uploads -- run them concurrently instead of - # back-to-back to shave the S3 round-trip time roughly in half. - with ThreadPoolExecutor(max_workers=2) as pool: - content_future = pool.submit( - s3.put_object, - Bucket=bucket, - Key=content_key, - Body=html.encode("utf-8"), - ContentType="text/html; charset=utf-8", - ) - screenshot_future = pool.submit( - s3.put_object, - Bucket=bucket, - Key=screenshot_key, - Body=png, - ContentType="image/png", - ) - content_future.result() - screenshot_future.result() - - return { - "url": page.url, - "content_key": content_key, - "screenshot_key": screenshot_key, - } - finally: - browser.close() diff --git a/docker/lambda/grab/Dockerfile b/docker/lambda/grab/Dockerfile deleted file mode 100644 index 23bb145..0000000 --- a/docker/lambda/grab/Dockerfile +++ /dev/null @@ -1,42 +0,0 @@ -# CloakBrowser fast single-page capture (HTML + full-page screenshot) on -# AWS Lambda. -# -# Built FROM the shared bytelyon-cloakbrowser-base image (see -# ../base-image/Dockerfile and ../base-image/INSTRUCTIONS.md), which carries -# every bit of common CloakBrowser + Lambda glue: awslambdaric, the Lambda -# Runtime Interface Emulator, boto3, the dual-mode entrypoint, the non-root -# readability fix, and the runtime ENV. This Dockerfile only adds this -# handler's own module — see lambda_handler.py. -# -# Sibling of ../news-scraper, ../page-scraper, ../scraper, ../serp-scraper, -# and ../site-crawler (same base image, different lambda_handler.py). This -# handler takes a single URL (with an optional proxy), uploads the rendered -# HTML and a full-page screenshot to S3, and returns just the two S3 keys — -# deliberately minimal, optimized for speed over ../page-scraper's link/meta -# extraction. -# -# ─── Build (the base image must already exist, locally or in ECR) ──────────── -# docker buildx build --platform linux/arm64 --provenance=false --sbom=false \ -# -t bytelyon-grab:arm64 --load . -# -# docker run --rm -p 9000:8080 \ -# -e AWS_ACCESS_KEY_ID -e AWS_SECRET_ACCESS_KEY -e AWS_SESSION_TOKEN -e AWS_REGION=us-east-1 \ -# bytelyon-grab:arm64 -# curl -XPOST http://localhost:9000/2015-03-31/functions/function/invocations \ -# -d '{"url":"https://example.com"}' -# -# --provenance=false --sbom=false is required: without it, buildx emits a -# manifest-list/attestation image that Lambda's CreateFunction/UpdateFunctionCode -# rejects with "image manifest, config or layer media type ... is not supported". -# -# To build against a locally-loaded (not-yet-pushed) base image instead of -# pulling from ECR, see ../base-image/INSTRUCTIONS.md: -# --build-arg BASE_IMAGE=bytelyon-cloakbrowser-base:arm64 - -ARG BASE_IMAGE=138305277395.dkr.ecr.us-east-1.amazonaws.com/bytelyon-cloakbrowser-base:latest -FROM ${BASE_IMAGE} - -# Handler sits at /app (already on Python's import path, WORKDIR=/app), -# imports cloakbrowser as a normal library. ENTRYPOINT/CMD are inherited from -# the base image (both already correct for this handler's module name). -COPY lambda_handler.py /app/lambda_handler.py diff --git a/docker/lambda/grab/INSTRUCTIONS.md b/docker/lambda/grab/INSTRUCTIONS.md deleted file mode 100644 index b79276e..0000000 --- a/docker/lambda/grab/INSTRUCTIONS.md +++ /dev/null @@ -1,141 +0,0 @@ -# CloakBrowser fast single-page capture on AWS Lambda - -Given a single URL (with an optional proxy): uploads the rendered HTML and a full-page screenshot to S3, and returns just the two S3 keys — using CloakBrowser's stealth Chromium inside an AWS Lambda function (container image package type). - -This handler is deliberately minimal and optimized for **speed**: no link/meta extraction, no consent-dialog handling, no retry orchestration, no headed/Xvfb mode. It's a sibling of [`../news-scraper`](../news-scraper), [`../page-scraper`](../page-scraper), [`../scraper`](../scraper), [`../serp-scraper`](../serp-scraper), and [`../site-crawler`](../site-crawler) — all six build `FROM` the shared [`../base-image`](../base-image), which carries the Dockerfile/entrypoint scaffolding derived from the official `cloakhq/cloakbrowser` Docker Hub image (plus `boto3` for the S3 uploads) that used to be duplicated in every handler's own `Dockerfile`. Every other invocation surface from the canonical image (`python`, `cloakserve`, `cloaktest`, `node`, `bash`, examples) keeps working. - -If you need link/meta extraction as well, see [`../page-scraper`](../page-scraper) instead — this handler trades that off for lower latency. - -## Files in this directory - -| File | Purpose | -| ------------------- | ----------------------------------------------------------------------------------------------------------------------| -| `Dockerfile` | `FROM` the shared `../base-image`, plus a single `COPY lambda_handler.py`. See `../base-image/INSTRUCTIONS.md`. | -| `lambda_handler.py` | Handler. Takes `{url, proxy?, ...}`, returns `{url, content_key, screenshot_key}`. Headless by default. | -| `INSTRUCTIONS.md` | This file. | - -This directory is no longer fully standalone: it depends on the shared base image (`../base-image`) having been built and either published to ECR or loaded locally first — see `../base-image/INSTRUCTIONS.md`. Everything specific to this handler (its Dockerfile and `lambda_handler.py`) still lives entirely in this directory. - -## Build - -Build and publish `../base-image` first (see `../base-image/INSTRUCTIONS.md`), then: - -```bash -docker buildx build --platform linux/arm64 --provenance=false --sbom=false \ - -t bytelyon-grab:arm64 --load . -``` - -`--provenance=false --sbom=false` is required — without it, buildx produces a manifest-list/attestation image that Lambda's `CreateFunction` rejects with `InvalidParameterValueException: ... image manifest, config or layer media type ... is not supported`. - -To build against a locally-loaded (not-yet-pushed) base image instead of pulling from ECR: - -```bash -docker buildx build --platform linux/arm64 --provenance=false --sbom=false \ - --build-arg BASE_IMAGE=bytelyon-cloakbrowser-base:arm64 \ - -t bytelyon-grab:arm64 --load . -``` - -For x86_64, switch `--platform linux/amd64` (and build the base image for the same architecture). - -## Local smoke test (no Lambda deploy needed, but real AWS credentials required) - -Both artifacts are genuinely uploaded to S3 even in local RIE testing, so pass through real AWS credentials with write access to the target bucket: - -```bash -docker run --rm -p 9000:8080 \ - -e AWS_ACCESS_KEY_ID -e AWS_SECRET_ACCESS_KEY -e AWS_SESSION_TOKEN -e AWS_REGION=us-east-1 \ - bytelyon-grab:arm64 - -# In another shell: -curl -sS -XPOST "http://localhost:9000/2015-03-31/functions/function/invocations" \ - -d '{"url":"https://example.com"}' - -# With a proxy: -curl -sS -XPOST "http://localhost:9000/2015-03-31/functions/function/invocations" \ - -d '{"url":"https://example.com", "proxy":"http://user:pass@host:port", "geoip":true}' -``` - -Other invocation surfaces stay intact (these match the canonical CloakHQ image): - -```bash -docker run --rm -it bytelyon-grab:arm64 python # REPL -docker run --rm bytelyon-grab:arm64 python examples/basic.py # examples -docker run --rm -p 9222:9222 bytelyon-grab:arm64 cloakserve --port=9222 # CDP server -docker run --rm bytelyon-grab:arm64 cloaktest # stealth tests -docker run --rm -it bytelyon-grab:arm64 node # JS wrapper -``` - -## Event schema - -| Field | Type | Default | -| ---------------------- | ---------- | ----------------------------------------------------------------------------------------------------------------------------- | -| `url` | str | required — the page to capture (http/https only) | -| `proxy` | str\|dict | none — `"http://user:pass@host:port"` (credentials split out automatically) or a `ProxySettings` dict | -| `geoip` | bool | `false` — auto-derive timezone/locale from the proxy's exit IP (only meaningful with `proxy` set) | -| `bucket` | str | `"bytelyon-private"` | -| `prefix` | str | `"grabs/{aws_request_id}/"` — S3 key prefix for both uploads | -| `headless` | bool | `true` | -| `humanize` | bool | `false` — off by default for speed; set `true` for sites that fingerprint mouse/keyboard/scroll behavior | -| `human_preset` | str | `"careful"` | -| `wait_until` | str | `"domcontentloaded"` — passed to `page.goto` (`"load"`\|`"domcontentloaded"`\|`"networkidle"`\|`"commit"`) | -| `goto_timeout_ms` | int | `30000` | -| `full_page_screenshot` | bool | `true` | - -### Response - -```json -{ - "url": "https://example.com/", - "content_key": "grabs//example_com-.html", - "screenshot_key": "grabs//example_com-.png" -} -``` - -- **`content_key`** / **`screenshot_key`** — S3 keys (under `bucket`) for the rendered HTML (`page.content()`) and the screenshot, uploaded **concurrently** to shave the round-trip time roughly in half. -- **`url`** — the final (post-redirect) URL. - -## Speed choices (this handler's reason to exist) - -- **`humanize` defaults to `false`** — no synthetic mouse/keyboard/scroll delay. The sibling handlers default this to `true` for stealth; here it's opt-in, since the ask was raw speed. Set it to `true` per-invocation for sites that fingerprint interaction behavior. -- **`wait_until` defaults to `"domcontentloaded"`** — fires as soon as the DOM is parsed, without waiting for every subresource (images, fonts, analytics beacons) like `"load"` or `"networkidle"` would. -- **No link/meta extraction, no consent-dialog handling, no retry orchestration** — every extra step the sibling handlers take is left out. Best-effort on navigation timeout: whatever loaded so far still gets captured and uploaded rather than failing the whole invocation. -- **Concurrent S3 uploads** — HTML and screenshot are two independent network calls, submitted to a 2-worker thread pool instead of run back-to-back. - -## Lambda-specific additions - -- **Chromium hardening (do not remove)**: `--disable-dev-shm-usage` and `--no-zygote`, same as the sibling handlers. -- **URL validation**: scheme + SSRF checks on the input `url` before navigation, and again on `page.url` after navigation/redirects. -- **Proxy TCP preflight**: if `proxy` is set, a raw TCP connect to the proxy's host:port is attempted first (~8s timeout) so an unreachable/blocking proxy fails fast with a clear error instead of hanging until the function timeout. Same as `../serp-scraper`. - -## IAM permissions - -In addition to `AWSLambdaBasicExecutionRole` (CloudWatch Logs), the execution role needs write access to the target bucket: - -```json -{ - "Version": "2012-10-17", - "Statement": [ - { - "Effect": "Allow", - "Action": ["s3:PutObject"], - "Resource": ["arn:aws:s3:::bytelyon-private/*"] - } - ] -} -``` - -## Function configuration recommendations - -| Setting | Value | Why | -| --------------------------- | ------------------------------------------------------------------ | -------------------------------------------------------------------------------------------------------------| -| Package type | Image | Required — this is a container image, not a zip. | -| Architecture | `arm64` | Roughly 20% cheaper than x86_64. Match the architecture you built for. | -| Memory | 3008 MB | Memory in Lambda is tied to vCPU. Below ~1769 MB Chromium starts noticeably slower. | -| Timeout | 30-45 s | Single page load + full-page screenshot + concurrent uploads is fast once warm; budget for cold start (~80-90 s) on top. | -| Ephemeral storage (`/tmp`) | 512 MB (default) | Both artifacts are held in memory and streamed to S3, not accumulated on disk. | -| Networking | Default (no VPC) | Binary is baked in; only outbound HTTPS to the target site (or proxy) and to S3 is needed. | -| Execution role | Basic execution + S3 `PutObject` on the target bucket (see above) | | - -## Cold start - -Same profile as the sibling examples: first invocation in a new container takes ~80–90 s. Subsequent warm invocations are much faster — this handler is intentionally the leanest of the family, so warm-invocation latency should be at or below the other handlers'. diff --git a/docker/lambda/grab/lambda_handler.py b/docker/lambda/grab/lambda_handler.py deleted file mode 100644 index 33ddf8f..0000000 --- a/docker/lambda/grab/lambda_handler.py +++ /dev/null @@ -1,308 +0,0 @@ -"""AWS Lambda handler: fast single-page capture (HTML + full-page -screenshot) to S3. - -Given a single URL, this handler: - 1. Navigates to the page (optionally through a proxy). - 2. Captures the rendered HTML and a full-page screenshot. - 3. Uploads both to S3, concurrently. - -Deliberately minimal — no link/meta extraction, no consent-dialog handling, -no retry orchestration, no headed/Xvfb mode. Every step that isn't required -for "visit a URL, save its content and a screenshot" is left out on purpose: -see ../page-scraper (adds link/meta extraction), ../serp-scraper (adds -Google-specific extraction + block detection), and ../scraper (adds retry -orchestration + headed Xvfb mode) for handlers that trade some of this -speed for richer output. - -Uses the **sync** cloakbrowser/Playwright API (`cloakbrowser.launch`), same -as ../page-scraper, ../news-scraper, ../serp-scraper, and ../site-crawler — -this avoids the async event-loop plumbing and headed/Xvfb startup cost that -../scraper pays for its extra features. - -Event schema: - url str required — the page to capture (http/https only) - proxy str|dict none — "http://user:pass@host:port" or a - cloakbrowser/Playwright ProxySettings dict - (credentials are split out automatically — - see _normalize_proxy) - geoip bool false — auto-derive timezone/locale from the - proxy's exit IP (only meaningful with `proxy` set) - bucket str "bytelyon-private" - prefix str "grabs/{aws_request_id}/" — S3 key prefix - headless bool true - humanize bool false — off by default for speed; set true for - sites that fingerprint mouse/keyboard/scroll behavior - human_preset str "careful" - wait_until str "domcontentloaded" — passed to page.goto - ("load"|"domcontentloaded"|"networkidle"|"commit") - goto_timeout_ms int 30000 - full_page_screenshot bool true - -Returns: - { - "url": "https://example.com/", - "content_key": "grabs//example_com-.html", - "screenshot_key": "grabs//example_com-.png" - } - -Lambda-specific additions (see ../base-image/INSTRUCTIONS.md for the same -rationale in the sibling handlers): - - `--disable-dev-shm-usage` / `--no-zygote` Chromium flags. - - URL scheme + SSRF validation before *and* after navigation (defense in - depth against private/internal network access). - - A TCP preflight on the proxy (if any) so an unreachable/blocking proxy - fails in ~8s instead of hanging for the full function timeout (same as - ../serp-scraper). - -Speed choices: - - `humanize` defaults to `false` (no synthetic mouse/keyboard/scroll - delay) — turn it on only for sites that need it. - - `wait_until` defaults to `"domcontentloaded"`, which fires as soon as - the DOM is parsed rather than waiting for every subresource (images, - fonts, analytics beacons) to finish loading. - - No link/meta extraction, no consent-dialog handling, no retry - orchestration. - - HTML and screenshot are uploaded to S3 concurrently (two independent - network calls), not sequentially. -""" - -from __future__ import annotations - -import hashlib -import ipaddress -import logging -import re -import socket -import time -from concurrent.futures import ThreadPoolExecutor -from typing import Any -from urllib.parse import unquote, urlparse - -import boto3 -from cloakbrowser import ProxySettings, launch -from playwright.sync_api import Page -from playwright.sync_api import TimeoutError as PlaywrightTimeoutError - -logger = logging.getLogger("cloakbrowser.lambda") -logger.setLevel(logging.INFO) - -# awslambdaric wires the root logger with a formatter like -# "[INFO]\t2024-01-15T10:30:45.123Z\t\tmessage" — CloudWatch already -# timestamps every event server-side, so repeating timestamp + request id on -# every line just burns terminal columns. Trim to level + message. -for _h in logging.getLogger().handlers: - _h.setFormatter(logging.Formatter("%(levelname)s %(message)s")) - -DEFAULT_BUCKET = "bytelyon-private" - -# How long to wait for a raw TCP handshake with the proxy before giving up. -# `cloakbrowser.launch(proxy=...)` has no timeout of its own — if the proxy -# silently drops the connection (common for residential-proxy providers that -# block/throttle datacenter-origin traffic, e.g. AWS Lambda's egress IPs) the -# whole invocation would otherwise hang until Lambda's function timeout, -# burning the full billed duration with no diagnostic signal. This preflight -# fails fast with a clear, actionable error instead. Same as ../serp-scraper. -_PROXY_PREFLIGHT_TIMEOUT_S = 8.0 - - -def _validate_url(url: str) -> None: - """Reject non-HTTP schemes and URLs that resolve to private/internal IPs.""" - parsed = urlparse(url) - if parsed.scheme.lower() not in ("http", "https"): - raise ValueError( - f"Only http:// and https:// URLs are supported, got: {parsed.scheme!r}" - ) - hostname = parsed.hostname - if not hostname: - raise ValueError("URL has no hostname") - try: - infos = socket.getaddrinfo(hostname, None, socket.AF_UNSPEC, socket.SOCK_STREAM) - except socket.gaierror: - raise ValueError(f"Cannot resolve hostname: {hostname}") - for info in infos: - addr = ipaddress.ip_address(info[4][0]) - if not addr.is_global: - raise ValueError("URLs targeting private/internal networks are blocked") - - -def _normalize_proxy(proxy: Any) -> ProxySettings | None: - """Normalize a `proxy` event value into cloakbrowser/Playwright's - `ProxySettings` shape: {"server": "://:", "bypass"?, - "username"?, "password"?}. - - This matters more than it looks: Chromium's underlying `--proxy-server` - flag has no concept of credentials embedded in a URL. A caller-supplied - string like "http://user:pass@host:port" connects to the proxy just fine - but never authenticates — Chromium silently drops the embedded - credentials, the proxy responds 407 Proxy Authentication Required, and - the navigation hangs until Playwright/Lambda's own timeout. Splitting - `username`/`password` into their own fields is what makes Playwright - wire up its automatic proxy-auth handling. A dict already in that shape - is passed through unchanged. Same logic as ../serp-scraper. - """ - if not proxy: - return None - if isinstance(proxy, dict): - return proxy - if not isinstance(proxy, str): - raise ValueError( - f"'proxy' must be a string or dict, got {type(proxy).__name__}" - ) - - parsed = urlparse(proxy) - if not parsed.hostname: - raise ValueError(f"'proxy' is not a valid proxy URL: {proxy!r}") - - server = f"{parsed.scheme or 'http'}://{parsed.hostname}" - if parsed.port: - server += f":{parsed.port}" - - normalized: ProxySettings = ProxySettings(server=server) - if parsed.username: - normalized["username"] = unquote(parsed.username) - if parsed.password: - normalized["password"] = unquote(parsed.password) - return normalized - - -def _proxy_host_port(proxy: ProxySettings | None) -> tuple[str, int] | None: - """Best-effort (host, port) extraction from an already-normalized proxy - dict, for the TCP preflight below. Returns None (preflight skipped, not - fatal) if it can't be determined. - """ - if not proxy: - return None - server = proxy.get("server") - if not isinstance(server, str) or not server: - return None - parsed = urlparse(server if "://" in server else f"http://{server}") - if not parsed.hostname or not parsed.port: - return None - return parsed.hostname, parsed.port - - -def _preflight_proxy( - proxy: ProxySettings | None, timeout: float = _PROXY_PREFLIGHT_TIMEOUT_S -) -> None: - """Raw TCP reachability check only — deliberately does not attempt proxy - auth or a CONNECT tunnel, just confirms *something* is listening on - host:port so an unreachable/blocking proxy fails in ~8s instead of - hanging for the full function timeout. A pass here does not guarantee - the proxy will actually authenticate or tunnel traffic. - """ - host_port = _proxy_host_port(proxy) - if host_port is None: - return - host, port = host_port - started = time.monotonic() - try: - with socket.create_connection((host, port), timeout=timeout): - pass - except OSError as e: - raise RuntimeError( - f"Proxy preflight failed: could not open a TCP connection to " - f"{host}:{port} within {timeout}s ({e}). This is a network-level " - f"failure, not a credentials issue — it usually means the proxy " - f"provider is blocking or silently dropping connections from " - f"this network (e.g. AWS Lambda's egress IP ranges). Verify with " - f"the proxy provider whether cloud/datacenter source IPs are " - f"allowed on this gateway." - ) from e - logger.info( - "Proxy preflight OK: TCP connect to %s:%s succeeded in %.2fs", - host, - port, - time.monotonic() - started, - ) - - -def _s3_key(prefix: str, url: str, ext: str) -> str: - parsed = urlparse(url) - path = parsed.path.strip("/") or "index" - safe = re.sub(r"[^a-zA-Z0-9/_-]", "_", f"{parsed.netloc}/{path}") - digest = hashlib.sha256(url.encode()).hexdigest()[:10] - return f"{prefix.rstrip('/')}/{safe}-{digest}.{ext}" - - -def handler(event: dict, context: Any) -> dict: - url = event.get("url") - if not isinstance(url, str) or not url.strip(): - raise ValueError("event must include a non-empty 'url' string") - _validate_url(url) - - bucket = event.get("bucket", DEFAULT_BUCKET) - prefix = event.get( - "prefix", f"grabs/{getattr(context, 'aws_request_id', 'local')}/" - ) - - s3 = boto3.client("s3") - - proxy: ProxySettings | None = _normalize_proxy(event.get("proxy")) - _preflight_proxy(proxy) - - browser = launch( - headless=event.get("headless", True), - humanize=event.get("humanize", False), - human_preset=event.get("human_preset", "careful"), - proxy=proxy, - # Auto-derive timezone/locale from the proxy's exit IP so the browser - # fingerprint matches where the traffic is actually coming from — a - # mismatch against a residential proxy's geolocation is itself a - # detection signal. Only meaningful when `proxy` is set. - geoip=event.get("geoip", False), - args=[ - # Lambda /dev/shm is ~64 MB — Chromium crashes mid-render without this. - "--disable-dev-shm-usage", - # Lambda's restricted process model can't fork from Chromium's - # zygote — without this, child renderer processes fail to spawn. - "--no-zygote", - ], - ) - page: Page = browser.new_page() - try: - try: - page.goto( - url, - wait_until=event.get("wait_until", "domcontentloaded"), - timeout=event.get("goto_timeout_ms", 30_000), - ) - except PlaywrightTimeoutError: - # Best-effort: capture whatever loaded so far rather than fail - # the whole invocation. - pass - - _validate_url(page.url) # re-check after redirects - - html = page.content() - png = page.screenshot(full_page=event.get("full_page_screenshot", True)) - - content_key = _s3_key(prefix, page.url, "html") - screenshot_key = _s3_key(prefix, page.url, "png") - - # Two independent uploads — run them concurrently instead of - # back-to-back to shave the S3 round-trip time roughly in half. - with ThreadPoolExecutor(max_workers=2) as pool: - content_future = pool.submit( - s3.put_object, - Bucket=bucket, - Key=content_key, - Body=html.encode("utf-8"), - ContentType="text/html; charset=utf-8", - ) - screenshot_future = pool.submit( - s3.put_object, - Bucket=bucket, - Key=screenshot_key, - Body=png, - ContentType="image/png", - ) - content_future.result() - screenshot_future.result() - - return { - "url": page.url, - "content_key": content_key, - "screenshot_key": screenshot_key, - } - finally: - browser.close() diff --git a/docker/lambda/http-test/Dockerfile b/docker/lambda/http-test/Dockerfile deleted file mode 100644 index 5bccc14..0000000 --- a/docker/lambda/http-test/Dockerfile +++ /dev/null @@ -1,25 +0,0 @@ -# Minimal, throwaway diagnostic Lambda: plain `requests` HTTP client, no -# Chromium/Playwright/cloakbrowser at all. Built FROM the AWS-managed Lambda -# Python base image (not ../base-image) since it needs none of the CloakBrowser -# Lambda glue \u2014 just requests (+ PySocks for socks5:// proxy support). -# -# Purpose: determine whether authenticated proxy traffic hangs specifically -# when it originates from AWS Lambda, independent of whether the client is a -# full browser or a plain HTTP client. See lambda_handler.py's docstring. -# -# Build: -# docker buildx build --platform linux/arm64 --provenance=false --sbom=false \ -# -t bytelyon-http-test:arm64 --load . -# -# Local smoke test: -# docker run --rm -p 9000:8080 bytelyon-http-test:arm64 -# curl -XPOST http://localhost:9000/2015-03-31/functions/function/invocations \ -# -d '{"proxy":"http://user:pass@host:port"}' - -FROM public.ecr.aws/lambda/python:3.12 - -RUN pip install --no-cache-dir requests "requests[socks]" - -COPY lambda_handler.py ${LAMBDA_TASK_ROOT}/lambda_handler.py - -CMD ["lambda_handler.handler"] diff --git a/docker/lambda/http-test/lambda_handler.py b/docker/lambda/http-test/lambda_handler.py deleted file mode 100644 index ecde740..0000000 --- a/docker/lambda/http-test/lambda_handler.py +++ /dev/null @@ -1,80 +0,0 @@ -"""Minimal, no-browser diagnostic Lambda: does a plain HTTP GET (via -`requests`, no Playwright/Chromium at all) directly or through a proxy, and -reports back timing + outcome for each URL tried. - -This exists purely to answer one question: does authenticated proxy traffic -hang/fail specifically when it originates from AWS Lambda, independent of -whether the client is a full browser (cloakbrowser/Chromium) or a plain -Python HTTP client? See ../serp-scraper/INSTRUCTIONS.md and this repo's -conversation history for the browser-based (Chromium) and httpx-based -failures this is meant to cross-check against. - -Event schema: - urls list[str] default: a small fixed set of IP-echo - services + a Google search URL - proxy str | None "http://user:pass@host:port" (or - "socks5://user:pass@host:port"), applied - to both http:// and https:// requests. - None/omitted = direct, no proxy. - timeout_s float default 25 \u2014 per-request timeout - -Returns: - {"proxy": "", "results": [ - {"url": ..., "ok": bool, "status_code": int|null, "body": str|null, - "error": str|null, "elapsed_s": float} - ]} -""" - -from __future__ import annotations - -import time -from typing import Any -from urllib.parse import urlparse - -import requests # pyright: ignore[reportMissingImports] - -DEFAULT_URLS = [ - "https://api.ipify.org", - "https://checkip.amazonaws.com", - "https://www.google.com/search?q=sailing+blocks", -] - - -def _redact_proxy(proxy: str | None) -> str | None: - if not proxy: - return None - parsed = urlparse(proxy) - if not parsed.hostname: - return proxy - netloc = parsed.hostname - if parsed.port: - netloc += f":{parsed.port}" - return f"{parsed.scheme}://{netloc}" - - -def handler(event: dict, context: Any) -> dict: - urls = event.get("urls") or DEFAULT_URLS - proxy = event.get("proxy") - timeout_s = float(event.get("timeout_s", 25)) - - proxies = {"http": proxy, "https": proxy} if proxy else None - - results = [] - for url in urls: - started = time.monotonic() - entry: dict[str, Any] = {"url": url} - try: - resp = requests.get(url, proxies=proxies, timeout=timeout_s) - entry["ok"] = True - entry["status_code"] = resp.status_code - entry["body"] = resp.text[:500] - entry["error"] = None - except Exception as e: # noqa: BLE001 - report every failure mode - entry["ok"] = False - entry["status_code"] = None - entry["body"] = None - entry["error"] = f"{type(e).__name__}: {e}" - entry["elapsed_s"] = round(time.monotonic() - started, 2) - results.append(entry) - - return {"proxy": _redact_proxy(proxy), "results": results} diff --git a/docker/lambda/news-scraper/Dockerfile b/docker/lambda/news-scraper/Dockerfile deleted file mode 100644 index f2d5481..0000000 --- a/docker/lambda/news-scraper/Dockerfile +++ /dev/null @@ -1,37 +0,0 @@ -# CloakBrowser news/article metadata scraper on AWS Lambda. -# -# Built FROM the shared bytelyon-cloakbrowser-base image (see -# ../base-image/Dockerfile and ../base-image/INSTRUCTIONS.md), which carries -# every bit of common CloakBrowser + Lambda glue: awslambdaric, the Lambda -# Runtime Interface Emulator, boto3, the dual-mode entrypoint, the non-root -# readability fix, and the runtime ENV. This Dockerfile only adds this -# handler's own module — see lambda_handler.py. -# -# Sibling of ../page-scraper, ../scraper, ../serp-scraper, and -# ../site-crawler (same base image, different lambda_handler.py). This -# handler batches over one or more URLs per invocation and extracts article -# body text + OpenGraph/Twitter meta (image, description, keywords). -# -# ─── Build (the base image must already exist, locally or in ECR) ──────────── -# docker buildx build --platform linux/arm64 --provenance=false --sbom=false \ -# -t bytelyon-news-scraper:arm64 --load . -# -# docker run --rm -p 9000:8080 bytelyon-news-scraper:arm64 -# curl -XPOST http://localhost:9000/2015-03-31/functions/function/invocations \ -# -d '{"urls":["https://example.com"]}' -# -# --provenance=false --sbom=false is required: without it, buildx emits a -# manifest-list/attestation image that Lambda's CreateFunction/UpdateFunctionCode -# rejects with "image manifest, config or layer media type ... is not supported". -# -# To build against a locally-loaded (not-yet-pushed) base image instead of -# pulling from ECR, see ../base-image/INSTRUCTIONS.md: -# --build-arg BASE_IMAGE=bytelyon-cloakbrowser-base:arm64 - -ARG BASE_IMAGE=138305277395.dkr.ecr.us-east-1.amazonaws.com/bytelyon-cloakbrowser-base:latest -FROM ${BASE_IMAGE} - -# Handler sits at /app (already on Python's import path, WORKDIR=/app), -# imports cloakbrowser as a normal library. ENTRYPOINT/CMD are inherited from -# the base image (both already correct for this handler's module name). -COPY lambda_handler.py /app/lambda_handler.py diff --git a/docker/lambda/news-scraper/INSTRUCTIONS.md b/docker/lambda/news-scraper/INSTRUCTIONS.md deleted file mode 100644 index 81199f9..0000000 --- a/docker/lambda/news-scraper/INSTRUCTIONS.md +++ /dev/null @@ -1,144 +0,0 @@ -# CloakBrowser news/article metadata scraper on AWS Lambda - -Batch-extract article body text and OpenGraph/Twitter/news metadata (image, description, keywords) from one or more URLs per invocation, using CloakBrowser's stealth Chromium inside an AWS Lambda function (container image package type). - -This is a sibling of [`../page-scraper`](../page-scraper), [`../scraper`](../scraper), [`../serp-scraper`](../serp-scraper), and [`../site-crawler`](../site-crawler) — all five build `FROM` the shared [`../base-image`](../base-image), which carries the Dockerfile/entrypoint scaffolding derived from the official `cloakhq/cloakbrowser` Docker Hub image that used to be duplicated in every handler's own `Dockerfile`. This handler is ported from a standalone script that scraped a list of URLs given on the command line. Every other invocation surface from the canonical image (`python`, `cloakserve`, `cloaktest`, `node`, `bash`, examples) keeps working, same as the sibling handlers. - -This document does not prescribe a deployment method — push the resulting image to ECR and create the Lambda function however you prefer (AWS CLI, CDK, Terraform, SAM, console, etc.). - -## Files in this directory - -| File | Purpose | -| ------------------- | --------------------------------------------------------------------------------------------------------------------------------------------------------------------------- | -| `Dockerfile` | `FROM` the shared `../base-image`, plus a single `COPY lambda_handler.py`. See `../base-image/INSTRUCTIONS.md` for what the base image provides. | -| `lambda_handler.py` | Batch handler. Takes `{url or urls, ...}`, returns a list of `{url, body, img_src, img_alt, description, keywords}`. Headless by default (no Xvfb needed for this handler). | -| `INSTRUCTIONS.md` | This file. | - -This directory is no longer fully standalone: it depends on the shared base image (`../base-image`) having been built and either published to ECR or loaded locally first — see `../base-image/INSTRUCTIONS.md`. Everything specific to this handler (its Dockerfile and `lambda_handler.py`) still lives entirely in this directory. - -## Build - -Build and publish `../base-image` first (see `../base-image/INSTRUCTIONS.md`), then, from inside this directory: - -```bash -docker buildx build --platform linux/arm64 --provenance=false --sbom=false \ - -t bytelyon-news-scraper:arm64 --load . -``` - -`--provenance=false --sbom=false` is required — without it, buildx produces a manifest-list/attestation image that Lambda's `CreateFunction` rejects with `InvalidParameterValueException: ... image manifest, config or layer media type ... is not supported`. - -Or from anywhere, pointing at this directory as the build context: - -```bash -docker buildx build --platform linux/arm64 --provenance=false --sbom=false \ - -f path/to/news-scraper/Dockerfile \ - -t bytelyon-news-scraper:arm64 --load \ - path/to/news-scraper -``` - -To build against a locally-loaded (not-yet-pushed) base image instead of pulling from ECR, add `--build-arg BASE_IMAGE=bytelyon-cloakbrowser-base:arm64`. For x86_64, switch `--platform linux/amd64` (and build the base image for the same architecture). - -## Local smoke test (no AWS account needed) - -The image bakes in `aws-lambda-rie`, so the standard Lambda local-invoke endpoint works without mounting anything: - -```bash -docker run --rm -p 9000:8080 bytelyon-news-scraper:arm64 - -# In another shell: -curl -sS -XPOST "http://localhost:9000/2015-03-31/functions/function/invocations" \ - -d '{"urls":["https://example.com"]}' -``` - -Other invocation surfaces stay intact (these match the canonical CloakHQ image): - -```bash -docker run --rm -it bytelyon-news-scraper:arm64 python # REPL -docker run --rm bytelyon-news-scraper:arm64 python examples/basic.py # examples -docker run --rm -p 9222:9222 bytelyon-news-scraper:arm64 cloakserve --port=9222 # CDP server -docker run --rm bytelyon-news-scraper:arm64 cloaktest # stealth tests -docker run --rm -it bytelyon-news-scraper:arm64 node # JS wrapper -``` - -## Event schema - -Exactly one of `url` / `urls` is required. - -| Field | Type | Default | -| -------------------------------- | --------- | ----------------------------------------------------------------- | -| `url` | str | a single URL to process | -| `urls` | list[str] | one or more URLs to process, checked before `url` if both present | -| `headless` | bool | `true` — matches the source script | -| `humanize` | bool | `true` — matches the source script | -| `human_preset` | str | `"careful"` — matches the source script | -| `goto_timeout_ms` | int | `30000` — Playwright default; the source script left this unset | -| `wait_for_load_state_timeout_ms` | int | `5000` — matches the source script's hardcoded `5_000` | - -One browser + one page is launched per invocation and reused across every URL in `urls`, in order — same as the source script's loop over `sys.argv[1:]`. - -### Response - -A JSON array, one object per URL, in the same order as the request: - -```json -[ - { - "url": "https://example.com", - "body": "...", - "img_src": "...", - "img_alt": "...", - "description": "...", - "keywords": ["..."] - } -] -``` - -If navigation/extraction for a given URL raises anything other than a Playwright timeout (the source script already tolerates timeouts and extracts whatever loaded), that URL's entry becomes `{"url": ..., "error": "..."}` instead of failing the whole batch — this is the one behavioral addition beyond a literal port, so one bad URL in a multi-URL request doesn't discard results already gathered for the others. - -## Ported from the source script — behavior notes - -The handler is a close port of a CLI script that did: - -```python -browser = launch(headless=True, humanize=True, human_preset="careful") -page = browser.new_page() -for url in sys.argv[1:]: - page.goto(url) - page.wait_for_load_state('domcontentloaded', timeout=5_000) - data.append({...}) -print(json.dumps(data, indent=2)) -``` - -Two quirks from the source script are **preserved as-is** (not fixed, since the ask was to reproduce its result): - -- `body()`'s selector fallback loop (`for selector in ['article', 'main', 'body', 'html']: locator = page.locator(selector); if locator is not None: break`) always breaks on the first iteration — `page.locator(...)` returns a lazy handle and is never `None`, even when nothing matches. In practice `body()` always resolves against the `article` selector and never falls back to `main`/`body`/`html`. If a page has no `
` element, `body` comes back empty. -- The source script's `if txt is not "":` was changed to `if txt != "":` here — the former is an identity comparison against a string literal, which raises a `SyntaxWarning` on modern Python and isn't guaranteed to behave consistently across interpreters. Behavior is unchanged in practice (CPython interns short string literals). - -## Lambda-specific additions (not in the source script) - -The source script assumes a persistent local process with a trusted, operator-supplied URL list. Running the same logic as an internet-reachable-ish Lambda function warrants a couple of defensive additions, mirroring `../aws_lambda`: - -- **Lambda Chromium hardening (do not remove)**: `--disable-dev-shm-usage` (Lambda's `/dev/shm` is ~64 MB) and `--no-zygote` (Lambda's restricted process model can't fork from Chromium's zygote process) are always passed to `launch()`. -- **URL validation** — scheme restricted to `http://`/`https://`, and hostnames are resolved and checked against private/loopback/link-local/reserved/multicast IP ranges before navigation (blocks SSRF to things like `169.254.169.254`) and re-checked against `page.url` after navigation to catch server-side redirects. See the **Security** section of `../aws_lambda/INSTRUCTIONS.md` for the same caveats (this doesn't stop a GET with side effects from reaching an internal endpoint before the response is discarded; DNS rebinding can theoretically bypass the pre-navigation check). - -No retry orchestration is implemented (unlike `../aws_lambda`) — this keeps the handler a faithful, minimal adaptation of the source script rather than a redesign. Add it yourself if flaky navigations are a problem for your target sites. - -## Function configuration recommendations - -Whatever tool you use to create the Lambda function (CLI, CDK, Terraform, SAM, console), apply these settings: - -| Setting | Value | Why | -| -------------------------- | ----------------------------- | ------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------- | -| Package type | Image | Required — this is a container image, not a zip. | -| Architecture | `arm64` | Roughly 20% cheaper than x86_64. Native build on Apple Silicon. Match the architecture you built for. | -| Memory | 3008 MB | Memory in Lambda is tied to vCPU. Below ~1769 MB Chromium starts noticeably slower, even headless. | -| Timeout | scale with URL count | Each URL can take up to `goto_timeout_ms` (default 30 s) + `wait_for_load_state_timeout_ms` (default 5 s) worst case, sequentially. E.g. for up to 5 URLs per invocation, budget ~200 s; for a single URL, 60-90 s is comfortable including cold start. | -| Ephemeral storage (`/tmp`) | 512 MB (default) | No screenshots are captured here, so the 512 MB default is normally enough headroom for Chromium's profile dir. | -| Networking | Default (no VPC) | Binary is baked in, no network needed at cold start. | -| Execution role | `AWSLambdaBasicExecutionRole` | Just CloudWatch Logs. Add more permissions only if your handler needs them. | - -## Cold start - -Same profile as `../aws_lambda`: first invocation in a new container takes ~80–90 s (image extraction, Chromium binary mmap, JS engine warmup). Subsequent warm invocations on the same container are much faster (page-load-bound, not init-bound). - - diff --git a/docker/lambda/news-scraper/lambda_handler.py b/docker/lambda/news-scraper/lambda_handler.py deleted file mode 100644 index a5ef8a8..0000000 --- a/docker/lambda/news-scraper/lambda_handler.py +++ /dev/null @@ -1,248 +0,0 @@ -"""AWS Lambda handler: batch news/article metadata scraping via cloakbrowser. - -Adapted from a standalone CLI script that did: - - browser = launch(headless=True, humanize=True, human_preset="careful") - page = browser.new_page() - for url in sys.argv[1:]: - page.goto(url) - page.wait_for_load_state('domcontentloaded', timeout=5_000) - data.append({...}) - print(json.dumps(data, indent=2)) - -The Lambda version takes `urls` (or `url`) from the event instead of -sys.argv, reuses a single browser/page across every URL in one invocation -(same as the source script), and returns the list of extracted records -directly instead of printing it. - -Uses the **sync** cloakbrowser/Playwright API (`cloakbrowser.launch`), same -as the source script — this is fine under awslambdaric, which invokes -`handler(event, context)` synchronously with no asyncio event loop already -running. No Xvfb/headed display is required: the browser launches -headless=True by default, exactly like the source script. - -Event schema: - url str a single URL to process - urls list[str] one or more URLs to process (checked before `url`) - Exactly one of `url` / `urls` is required. - headless bool True (matches source script) - humanize bool True (matches source script) - human_preset str "careful" (matches source script) - goto_timeout_ms int 30000 — Playwright default; source script left this unset - wait_for_load_state_timeout_ms int 5000 (matches source script's hardcoded 5_000) - -Returns: - A JSON array, one object per URL (source script's `data` list): - [{"url", "body", "img_src", "img_alt", "description", "keywords"}, ...] - - If navigation/extraction for a given URL raises an error other than a - Playwright timeout (which the source script already tolerates), that - URL's entry becomes {"url": ..., "error": "..."} instead of aborting the - whole batch — added so one bad URL in a multi-URL request doesn't throw - away results already gathered for the others. - -Known quirks carried over from the source script (preserved for behavioral -fidelity, not fixed): - - `body()`'s `for selector in [...]: locator = page.locator(selector); if - locator is not None: break` always breaks on the first iteration — - `page.locator(...)` returns a lazy handle and is never None, even when - no element matches. In practice this means `body()` always resolves - against the 'article' selector, never falling back to main/body/html. - - The source script's `if txt is not "":` (identity comparison against a - string literal) was changed to `if txt != "":` here since the former - raises a SyntaxWarning on modern Python and is not reliable across - interpreters; behavior is unchanged in practice. - -Lambda-specific additions (not present in the source script, required for -correct operation in Lambda's environment — see ../aws_lambda/INSTRUCTIONS.md): - - `--disable-dev-shm-usage` / `--no-zygote` Chromium flags. - - URL scheme + SSRF validation before *and* after navigation (defense in - depth against private/internal network access), matching the sibling - ../aws_lambda handler. -""" - -from __future__ import annotations - -import ipaddress -import logging -import socket -from typing import Any -from urllib.parse import urlparse - -from cloakbrowser import launch # pyright: ignore[reportMissingImports] -from playwright.sync_api import Page # pyright: ignore[reportMissingImports] -from playwright.sync_api import ( # pyright: ignore[reportMissingImports] - TimeoutError as PlaywrightTimeoutError, -) - -logger = logging.getLogger("cloakbrowser.lambda") -logger.setLevel(logging.INFO) - -# awslambdaric wires the root logger with a formatter like -# "[INFO]\t2024-01-15T10:30:45.123Z\t\tmessage" — CloudWatch already -# timestamps every event server-side, so repeating timestamp + request id on -# every line just burns terminal columns. Trim to level + message. -for _h in logging.getLogger().handlers: - _h.setFormatter(logging.Formatter("%(levelname)s %(message)s")) - -KEYWORD_ATTRS = ["property='article:tag'", "name='news_keywords'", "name='keywords'"] -DESCRIPTION_ATTRS = [ - "name='description'", - "property='og:description'", - "name='twitter:description'", - "name='abstract'", -] -IMG_ALT_ATTRS = ["property='og:image:alt'", "name='twitter:image:alt'"] -IMG_SRC_ATTRS = [ - "name='image'", - "property='og:image'", - "property='og:image:secure_url'", - "name='twitter:image'", - "name='twitter:image:src'", -] - - -def _validate_url(url: str) -> None: - """Reject non-HTTP schemes and URLs that resolve to private/internal IPs.""" - parsed = urlparse(url) - if parsed.scheme.lower() not in ("http", "https"): - raise ValueError( - f"Only http:// and https:// URLs are supported, got: {parsed.scheme!r}" - ) - hostname = parsed.hostname - if not hostname: - raise ValueError("URL has no hostname") - try: - infos = socket.getaddrinfo(hostname, None, socket.AF_UNSPEC, socket.SOCK_STREAM) - except socket.gaierror: - raise ValueError(f"Cannot resolve hostname: {hostname}") - for info in infos: - addr = ipaddress.ip_address(info[4][0]) - if not addr.is_global: - raise ValueError("URLs targeting private/internal networks are blocked") - - -def meta_content(page: Page, attrs: list[str]) -> str: - for attr in attrs: - try: - for txt in page.locator(f"meta[{attr}]").all(): - text = txt.get_attribute("content", timeout=1_500) - if text is not None: - return text - except PlaywrightTimeoutError: - pass - return "" - - -def body(page: Page) -> str: - text = [] - for selector in ["article", "main", "body", "html"]: - if page.locator(selector).count() == 0: - continue - for p in page.locator(selector).locator("p").all(): - txt = p.text_content() - if txt is None: - continue - text.append(txt.strip()) - out = " ".join(text) - out = out.replace( - "This is a modal window. Beginning of dialog window. Escape will cancel and close the window. End of dialog window.", - "", - ) - return out - - -def keywords(page: Page) -> list[str]: - words = meta_content(page, KEYWORD_ATTRS).split(",") - for idx, word in enumerate(words): - txt = word.strip() - if txt != "": - words[idx] = txt - words.sort() - return words - - -def _extract(page: Page, url: str) -> dict: - return { - "url": url, - "body": body(page), - "img_src": meta_content(page, IMG_SRC_ATTRS), - "img_alt": meta_content(page, IMG_ALT_ATTRS), - "description": meta_content(page, DESCRIPTION_ATTRS), - "keywords": keywords(page), - } - - -def _urls_from_event(event: dict) -> list[str]: - urls = event.get("urls") - if urls is not None: - if ( - not isinstance(urls, list) - or not urls - or not all(isinstance(u, str) for u in urls) - ): - raise ValueError("'urls' must be a non-empty list of strings") - return urls - url = event.get("url") - if isinstance(url, str) and url: - return [url] - raise ValueError("event must include 'url' (str) or 'urls' (list[str])") - - -def _process_url(page: Page, url: str, event: dict) -> dict: - # Validated per-URL (not upfront for the whole batch) so one bad/blocked - # URL doesn't discard results already gathered for the others. - _validate_url(url) - - try: - page.goto(url, timeout=event.get("goto_timeout_ms", 30_000)) - page.wait_for_load_state( - "domcontentloaded", - timeout=event.get("wait_for_load_state_timeout_ms", 5_000), - ) - except PlaywrightTimeoutError: - # Matches the source script: a slow/never-settling page still gets a - # best-effort extraction from whatever loaded so far. - pass - - # Re-check after navigation/redirects — defense in depth against - # server-side redirects to blocked destinations (not in the source - # script; see module docstring). - _validate_url(page.url) - - return _extract(page, url) - - -def handler(event: dict, context: Any) -> list[dict]: - urls = _urls_from_event(event) - - logger.info("🏎️\turls=[%d]", len(urls)) - - browser = launch( - headless=event.get("headless", True), - humanize=event.get("humanize", True), - human_preset=event.get("human_preset", "careful"), - args=[ - # Lambda /dev/shm is ~64 MB — Chromium crashes mid-render without this. - "--disable-dev-shm-usage", - # Lambda's restricted process model can't fork from Chromium's - # zygote — without this, child renderer processes fail to spawn. - "--no-zygote", - ], - ) - page: Page = browser.new_page() - data: list[dict] = [] - try: - for url in urls: - try: - data.append(_process_url(page, url, event)) - logger.info("🏆\t%s", url) - except Exception as e: - logger.warning("processing failed for %s: %s", url, e) - data.append({"url": url, "error": str(e)}) - finally: - browser.close() - - logger.info("🏁\turls=[%d/%d]", len(data), len(urls)) - - return data diff --git a/docker/lambda/page-scraper/Dockerfile b/docker/lambda/page-scraper/Dockerfile deleted file mode 100644 index e8f24f6..0000000 --- a/docker/lambda/page-scraper/Dockerfile +++ /dev/null @@ -1,39 +0,0 @@ -# CloakBrowser single-page scraper on AWS Lambda. -# -# Built FROM the shared bytelyon-cloakbrowser-base image (see -# ../base-image/Dockerfile and ../base-image/INSTRUCTIONS.md), which carries -# every bit of common CloakBrowser + Lambda glue: awslambdaric, the Lambda -# Runtime Interface Emulator, boto3, the dual-mode entrypoint, the non-root -# readability fix, and the runtime ENV. This Dockerfile only adds this -# handler's own module — see lambda_handler.py. -# -# Sibling of ../news-scraper, ../scraper, ../serp-scraper, and -# ../site-crawler (same base image, different lambda_handler.py). This -# handler takes a single URL, uploads a full-page screenshot to S3, and -# returns the page's links, meta tags, final URL, and title as JSON. -# -# ─── Build (the base image must already exist, locally or in ECR) ──────────── -# docker buildx build --platform linux/arm64 --provenance=false --sbom=false \ -# -t bytelyon-page-scraper:arm64 --load . -# -# docker run --rm -p 9000:8080 \ -# -e AWS_ACCESS_KEY_ID -e AWS_SECRET_ACCESS_KEY -e AWS_SESSION_TOKEN -e AWS_REGION=us-east-1 \ -# bytelyon-page-scraper:arm64 -# curl -XPOST http://localhost:9000/2015-03-31/functions/function/invocations \ -# -d '{"url":"https://example.com"}' -# -# --provenance=false --sbom=false is required: without it, buildx emits a -# manifest-list/attestation image that Lambda's CreateFunction/UpdateFunctionCode -# rejects with "image manifest, config or layer media type ... is not supported". -# -# To build against a locally-loaded (not-yet-pushed) base image instead of -# pulling from ECR, see ../base-image/INSTRUCTIONS.md: -# --build-arg BASE_IMAGE=bytelyon-cloakbrowser-base:arm64 - -ARG BASE_IMAGE=138305277395.dkr.ecr.us-east-1.amazonaws.com/bytelyon-cloakbrowser-base:latest -FROM ${BASE_IMAGE} - -# Handler sits at /app (already on Python's import path, WORKDIR=/app), -# imports cloakbrowser as a normal library. ENTRYPOINT/CMD are inherited from -# the base image (both already correct for this handler's module name). -COPY lambda_handler.py /app/lambda_handler.py diff --git a/docker/lambda/page-scraper/INSTRUCTIONS.md b/docker/lambda/page-scraper/INSTRUCTIONS.md deleted file mode 100644 index 4a2e95f..0000000 --- a/docker/lambda/page-scraper/INSTRUCTIONS.md +++ /dev/null @@ -1,134 +0,0 @@ -# CloakBrowser single-page scraper on AWS Lambda - -Given a single URL: uploads a full-page screenshot to S3, and returns the page's links, meta tags, final URL, and title as JSON — using CloakBrowser's stealth Chromium inside an AWS Lambda function (container image package type). - -This is a sibling of [`../news-scraper`](../news-scraper), [`../scraper`](../scraper), [`../serp-scraper`](../serp-scraper), and [`../site-crawler`](../site-crawler) — all five build `FROM` the shared [`../base-image`](../base-image), which carries the Dockerfile/entrypoint scaffolding derived from the official `cloakhq/cloakbrowser` Docker Hub image (plus `boto3` for the S3 upload) that used to be duplicated in every handler's own `Dockerfile`. Every other invocation surface from the canonical image (`python`, `cloakserve`, `cloaktest`, `node`, `bash`, examples) keeps working. - -## Files in this directory - -| File | Purpose | -| ------------------- | ------------------------------------------------------------------------------------------------------------------------------------------------ | -| `Dockerfile` | `FROM` the shared `../base-image`, plus a single `COPY lambda_handler.py`. See `../base-image/INSTRUCTIONS.md` for what the base image provides. | -| `lambda_handler.py` | Handler. Takes `{url, ...}`, returns `{url, title, links, meta, screenshot_s3_uri}`. Headless by default. | -| `INSTRUCTIONS.md` | This file. | - -This directory is no longer fully standalone: it depends on the shared base image (`../base-image`) having been built and either published to ECR or loaded locally first — see `../base-image/INSTRUCTIONS.md`. Everything specific to this handler (its Dockerfile and `lambda_handler.py`) still lives entirely in this directory. - -## Build - -Build and publish `../base-image` first (see `../base-image/INSTRUCTIONS.md`), then: - -```bash -docker buildx build --platform linux/arm64 --provenance=false --sbom=false \ - -t bytelyon-page-scraper:arm64 --load . -``` - -`--provenance=false --sbom=false` is required — without it, buildx produces a manifest-list/attestation image that Lambda's `CreateFunction` rejects with `InvalidParameterValueException: ... image manifest, config or layer media type ... is not supported`. - -To build against a locally-loaded (not-yet-pushed) base image instead of pulling from ECR: - -```bash -docker buildx build --platform linux/arm64 --provenance=false --sbom=false \ - --build-arg BASE_IMAGE=bytelyon-cloakbrowser-base:arm64 \ - -t bytelyon-page-scraper:arm64 --load . -``` - -For x86_64, switch `--platform linux/amd64` (and build the base image for the same architecture). - -## Local smoke test (no Lambda deploy needed, but real AWS credentials required) - -The screenshot is genuinely uploaded to S3 even in local RIE testing, so pass through real AWS credentials with write access to the target bucket: - -```bash -docker run --rm -p 9000:8080 \ - -e AWS_ACCESS_KEY_ID -e AWS_SECRET_ACCESS_KEY -e AWS_SESSION_TOKEN -e AWS_REGION=us-east-1 \ - bytelyon-page-scraper:arm64 - -# In another shell: -curl -sS -XPOST "http://localhost:9000/2015-03-31/functions/function/invocations" \ - -d '{"url":"https://example.com"}' -``` - -Other invocation surfaces stay intact (these match the canonical CloakHQ image): - -```bash -docker run --rm -it bytelyon-page-scraper:arm64 python # REPL -docker run --rm bytelyon-page-scraper:arm64 python examples/basic.py # examples -docker run --rm -p 9222:9222 bytelyon-page-scraper:arm64 cloakserve --port=9222 # CDP server -docker run --rm bytelyon-page-scraper:arm64 cloaktest # stealth tests -docker run --rm -it bytelyon-page-scraper:arm64 node # JS wrapper -``` - -## Event schema - -| Field | Type | Default | -| -------------------------------- | ---- | ------------------------------------------------------------------------------------------------------------------------------ | -| `url` | str | required — the page to scrape (http/https only) | -| `bucket` | str | `"bytelyon-private"` | -| `prefix` | str | `"page-scrapes/{aws_request_id}/"` — S3 key prefix for the screenshot | -| `headless` | bool | `true` | -| `humanize` | bool | `true` | -| `human_preset` | str | `"careful"` | -| `goto_timeout_ms` | int | `30000` | -| `wait_for_load_state_timeout_ms` | int | `5000` | -| `include_links` | bool | `true` — when `false`, link extraction is skipped entirely (no DOM walk over anchors) and `links` is omitted from the response | - -### Response - -```json -{ - "url": "https://example.com/", - "title": "Example Domain", - "links": ["https://example.com/about", "https://example.com/contact"], - "meta": [ - { "charset": "utf-8" }, - { "name": "description", "content": "..." }, - { "property": "og:title", "content": "..." } - ], - "screenshot_s3_uri": "s3://bytelyon-private/page-scrapes//example_com-.png" -} -``` - -- **`links`** — every `` on the page, resolved to absolute URLs, in DOM order, filtered to same-domain links only (the linked host must match the page's own host, ignoring a `www.` prefix on either side) with any link containing a URL fragment (`#...`) omitted entirely. Among what remains, `www.` and non-`www.` variants of the same URL (same scheme, path, and query, differing only by a `www.` host prefix) are treated as duplicates; the first-encountered form is kept. Pass `"include_links": false` to skip this extraction step altogether and omit `links` from the response. -- **`meta`** — every `` tag on the page, not a curated subset. Each entry includes whichever of `name` / `property` / `http-equiv` / `charset` is present on that tag, plus `content` if present. (For a curated OpenGraph/Twitter/news-style extraction instead, see `../news-scraper`.) -- **`url`** / **`title`** — the final (post-redirect) URL and `document.title`. -- **`screenshot_s3_uri`** — not part of the original ask, but included since otherwise there'd be no way to locate the uploaded screenshot from the response. Remove it from `lambda_handler.py`'s return dict if you'd rather keep the response to exactly the four requested fields. - -## Lambda-specific additions - -- **Chromium hardening (do not remove)**: `--disable-dev-shm-usage` and `--no-zygote`, same as the sibling examples. -- **URL validation**: scheme + SSRF checks on the input `url` before navigation, and again on `page.url` after navigation/redirects — same defense-in-depth as the other handlers that accept caller-supplied URLs. - -## IAM permissions - -In addition to `AWSLambdaBasicExecutionRole` (CloudWatch Logs), the execution role needs write access to the target bucket: - -```json -{ - "Version": "2012-10-17", - "Statement": [ - { - "Effect": "Allow", - "Action": ["s3:PutObject"], - "Resource": ["arn:aws:s3:::bytelyon-private/*"] - } - ] -} -``` - -## Function configuration recommendations - -| Setting | Value | Why | -| -------------------------- | ----------------------------------------------------------------- | ------------------------------------------------------------------------------------------------------------ | -| Package type | Image | Required — this is a container image, not a zip. | -| Architecture | `arm64` | Roughly 20% cheaper than x86_64. Match the architecture you built for. | -| Memory | 3008 MB | Memory in Lambda is tied to vCPU. Below ~1769 MB Chromium starts noticeably slower. | -| Timeout | 60-90 s | Single page load + full-page screenshot + upload is fast once warm; budget for cold start (~80-90 s) on top. | -| Ephemeral storage (`/tmp`) | 512 MB (default) | The screenshot is streamed to S3, not accumulated on disk. | -| Networking | Default (no VPC) | Binary is baked in; only outbound HTTPS to the target site and to S3 is needed. | -| Execution role | Basic execution + S3 `PutObject` on the target bucket (see above) | | - -## Cold start - -Same profile as the sibling examples: first invocation in a new container takes ~80–90 s. Subsequent warm invocations are much faster. - diff --git a/docker/lambda/page-scraper/lambda_handler.py b/docker/lambda/page-scraper/lambda_handler.py deleted file mode 100644 index 957c764..0000000 --- a/docker/lambda/page-scraper/lambda_handler.py +++ /dev/null @@ -1,237 +0,0 @@ -"""AWS Lambda handler: single-page scrape with full-page screenshot to S3. - -Given a single URL, this handler: - 1. Navigates to the page. - 2. Takes a full-page screenshot and uploads it to S3. - 3. Returns the page's links, meta tags, final URL, and title as JSON. - -Uses the **sync** cloakbrowser/Playwright API (`cloakbrowser.launch`), same -as ../news-scraper, ../aws_lambda_site_crawler, and -../aws_lambda_serp_scraper. - -Event schema: - url str required — the page to scrape (http/https only) - bucket str "bytelyon-private" - prefix str "page-scrapes/{aws_request_id}/" — S3 key prefix - headless bool true - humanize bool true - human_preset str "careful" - goto_timeout_ms int 30000 - wait_for_load_state_timeout_ms int 5000 - include_links bool true — when false, link extraction is - skipped entirely and "links" is omitted - from the response - -Returns: - { - "url": "https://example.com/", - "title": "Example Domain", - "links": ["https://example.com/about", ...], - "meta": [{"name": "description", "content": "..."}, ...], - "screenshot_s3_uri": "s3://bytelyon-private/page-scrapes/.../example_com-.png" - } - -`links` only includes same-domain links (the linked host must match the -page's own host, ignoring a `www.` prefix on either side) and excludes any -link with a URL fragment (`#...`). Among the remaining links, `www.` vs. -non-`www.` variants of the same URL are deduplicated (e.g. -"https://www.example.com/x" and "https://example.com/x" are treated as the -same link) — the first-encountered form (in DOM order) is kept. Pass -`"include_links": false` in the event to skip link extraction altogether -(no DOM walk over anchors) and omit `links` from the response. `meta` -includes every `` tag on the page (whichever of `name`/`property`/ -`http-equiv`/`charset` is present, plus `content`), not a curated subset — -see ../news-scraper for a handler that extracts a curated -set of OpenGraph/Twitter/news fields instead. - -`screenshot_s3_uri` is not part of the literal ask but is included since -otherwise there'd be no way to locate the uploaded screenshot from the -response alone. - -Lambda-specific additions (see ../aws_lambda/INSTRUCTIONS.md for the same -rationale in the sibling handlers): - - `--disable-dev-shm-usage` / `--no-zygote` Chromium flags. - - URL scheme + SSRF validation before *and* after navigation (defense in - depth against private/internal network access). -""" - -from __future__ import annotations - -import hashlib -import ipaddress -import logging -import re -import socket -from typing import Any -from urllib.parse import urlparse - -import boto3 -from cloakbrowser import launch -from playwright.sync_api import Page -from playwright.sync_api import TimeoutError as PlaywrightTimeoutError - -logger = logging.getLogger("cloakbrowser.lambda") -logger.setLevel(logging.INFO) - -# awslambdaric wires the root logger with a formatter like -# "[INFO]\t2024-01-15T10:30:45.123Z\t\tmessage" — CloudWatch already -# timestamps every event server-side, so repeating timestamp + request id on -# every line just burns terminal columns. Trim to level + message. -for _h in logging.getLogger().handlers: - _h.setFormatter(logging.Formatter("%(levelname)s %(message)s")) - -DEFAULT_BUCKET = "bytelyon-private" - -# Structural extraction: same-domain, fragment-free page links (www./non-www. -# deduplicated) + every tag on the page. Link extraction is skipped -# entirely (no DOM walk over anchors) when `includeLinks` is falsy. -_PAGE_EXTRACT_JS = r""" -(opts) => { - const includeLinks = !!(opts && opts.includeLinks); - const links = []; - - if (includeLinks) { - const seen = new Set(); - const pageHost = location.host.replace(/^www\./i, ""); - - document.querySelectorAll("a[href]").forEach((a) => { - const href = a.href; - if (!href) return; - - let u; - try { - u = new URL(href); - } catch (e) { - return; // a.href is normally already absolute; skip if unparseable. - } - - // Check the raw href for a literal '#' rather than u.hash: an href of - // "...#" (empty fragment) parses to u.hash === "" (falsy) but still - // contains a fragment delimiter, which should be omitted too. a.href is - // the browser's fully-resolved URL, so any literal '#' in a path segment - // would already be percent-encoded as %23 -- a raw '#' here always means - // an actual fragment delimiter. - if (href.includes("#")) return; // omit links with fragments - - const host = u.host.replace(/^www\./i, ""); - if (host !== pageHost) return; // omit cross-domain links - - const key = u.protocol + "//" + host + u.pathname + u.search; - if (!seen.has(key)) { - seen.add(key); - links.push(href); - } - }); - } - -const meta = {}; -Array.from(document.querySelectorAll('meta')) - .forEach((m) => { - const v = m.getAttribute('content')?.trim(); - const n = m.getAttribute('name')?.trim(); - const p = m.getAttribute('property')?.trim(); - if (v && (n || p)) { - meta[n ? n : p] = v - } - }); -console.log(meta); - - return { links, meta }; -} -""" - - -def _validate_url(url: str) -> None: - """Reject non-HTTP schemes and URLs that resolve to private/internal IPs.""" - parsed = urlparse(url) - if parsed.scheme.lower() not in ("http", "https"): - raise ValueError( - f"Only http:// and https:// URLs are supported, got: {parsed.scheme!r}" - ) - hostname = parsed.hostname - if not hostname: - raise ValueError("URL has no hostname") - try: - infos = socket.getaddrinfo(hostname, None, socket.AF_UNSPEC, socket.SOCK_STREAM) - except socket.gaierror: - raise ValueError(f"Cannot resolve hostname: {hostname}") - for info in infos: - addr = ipaddress.ip_address(info[4][0]) - if not addr.is_global: - raise ValueError("URLs targeting private/internal networks are blocked") - - -def _domain(url: str) -> str: - """Return the registrable-ish domain for display: hostname minus a leading - 'www.' (e.g. "https://www.publix.com/takeout?query=foo" -> "publix.com"). - """ - hostname = urlparse(url).hostname or "" - return re.sub(r"^www\.", "", hostname, flags=re.IGNORECASE) - - -def _s3_key(prefix: str, url: str) -> str: - parsed = urlparse(url) - path = parsed.path.strip("/") or "index" - safe = re.sub(r"[^a-zA-Z0-9/_-]", "_", f"{parsed.netloc}/{path}") - digest = hashlib.sha256(url.encode()).hexdigest()[:10] - return f"{prefix.rstrip('/')}/{safe}-{digest}.png" - - -def handler(event: dict, context: Any) -> dict: - url = event.get("url") - if not isinstance(url, str) or not url.strip(): - raise ValueError("event must include a non-empty 'url' string") - _validate_url(url) - - bucket = event.get("bucket", DEFAULT_BUCKET) - prefix = event.get( - "prefix", f"page-scrapes/{getattr(context, 'aws_request_id', 'local')}/" - ) - - s3 = boto3.client("s3") - - browser = launch( - headless=event.get("headless", True), - humanize=event.get("humanize", True), - human_preset=event.get("human_preset", "careful"), - args=[ - # Lambda /dev/shm is ~64 MB — Chromium crashes mid-render without this. - "--disable-dev-shm-usage", - # Lambda's restricted process model can't fork from Chromium's - # zygote — without this, child renderer processes fail to spawn. - "--no-zygote", - ], - ) - page: Page = browser.new_page() - try: - try: - page.goto(url, timeout=event.get("goto_timeout_ms", 30_000)) - page.wait_for_load_state( - "domcontentloaded", - timeout=event.get("wait_for_load_state_timeout_ms", 5_000), - ) - except PlaywrightTimeoutError: - # Best-effort: screenshot/extract from whatever loaded so far. - pass - - _validate_url(page.url) # re-check after redirects - - png = page.screenshot(full_page=True) - key = _s3_key(prefix, page.url) - s3.put_object(Bucket=bucket, Key=key, Body=png, ContentType="image/png") - - include_links = event.get("include_links", True) - extracted = page.evaluate(_PAGE_EXTRACT_JS, {"includeLinks": include_links}) - - result = { - "url": page.url, - "domain": _domain(page.url), - "title": page.title(), - "meta": extracted["meta"], - "screenshot_key": key, - } - if include_links: - result["links"] = extracted["links"] - return result - finally: - browser.close() diff --git a/docker/lambda/scraper/Dockerfile b/docker/lambda/scraper/Dockerfile deleted file mode 100644 index 13d87ad..0000000 --- a/docker/lambda/scraper/Dockerfile +++ /dev/null @@ -1,37 +0,0 @@ -# CloakBrowser on AWS Lambda — general-purpose one-shot scraper. -# -# Built FROM the shared bytelyon-cloakbrowser-base image (see -# ../base-image/Dockerfile and ../base-image/INSTRUCTIONS.md), which carries -# every bit of common CloakBrowser + Lambda glue: awslambdaric, the Lambda -# Runtime Interface Emulator, boto3, the dual-mode entrypoint, the non-root -# readability fix, and the runtime ENV. This Dockerfile only adds this -# handler's own module — see lambda_handler.py. -# -# Sibling of ../news-scraper, ../page-scraper, ../serp-scraper, and -# ../site-crawler (same base image, different lambda_handler.py). This -# handler is always headed via Xvfb and returns -# `{title, url, html, screenshot_b64?}`. -# -# ─── Build (the base image must already exist, locally or in ECR) ──────────── -# docker buildx build --platform linux/arm64 --provenance=false --sbom=false \ -# -t bytelyon-scraper:arm64 --load . -# -# docker run --rm -p 9000:8080 bytelyon-scraper:arm64 -# curl -XPOST http://localhost:9000/2015-03-31/functions/function/invocations \ -# -d '{"url":"https://example.com"}' -# -# --provenance=false --sbom=false is required: without it, buildx emits a -# manifest-list/attestation image that Lambda's CreateFunction/UpdateFunctionCode -# rejects with "image manifest, config or layer media type ... is not supported". -# -# To build against a locally-loaded (not-yet-pushed) base image instead of -# pulling from ECR, see ../base-image/INSTRUCTIONS.md: -# --build-arg BASE_IMAGE=bytelyon-cloakbrowser-base:arm64 - -ARG BASE_IMAGE=138305277395.dkr.ecr.us-east-1.amazonaws.com/bytelyon-cloakbrowser-base:latest -FROM ${BASE_IMAGE} - -# Handler sits at /app (already on Python's import path, WORKDIR=/app), -# imports cloakbrowser as a normal library. ENTRYPOINT/CMD are inherited from -# the base image (both already correct for this handler's module name). -COPY lambda_handler.py /app/lambda_handler.py diff --git a/docker/lambda/scraper/INSTRUCTIONS.md b/docker/lambda/scraper/INSTRUCTIONS.md deleted file mode 100644 index 5bc2f4c..0000000 --- a/docker/lambda/scraper/INSTRUCTIONS.md +++ /dev/null @@ -1,192 +0,0 @@ -# CloakBrowser on AWS Lambda - -Run stealth Chromium one-shot scrapes inside an AWS Lambda function (container image package type). The image derives directly from the official CloakHQ Docker Hub image (`cloakhq/cloakbrowser`) and adds Lambda runtime support on top — Lambda is an additional invocation surface, not a replacement. Every other surface from the canonical image (`python`, `cloakserve`, `cloaktest`, `node`, `bash`, examples) keeps working. - -This document covers what the image is, how to build and locally test it, and the event/response contract. **It does not prescribe a deployment method** — push the resulting image to ECR and create the Lambda function however you prefer (AWS CLI, CDK, Terraform, SAM, console, etc.). Configuration tips for whichever tool you use are at the bottom. - -This is a sibling of [`../news-scraper`](../news-scraper), [`../page-scraper`](../page-scraper), [`../serp-scraper`](../serp-scraper), and [`../site-crawler`](../site-crawler) — all five build `FROM` the shared [`../base-image`](../base-image), which carries the Dockerfile/entrypoint scaffolding derived from the official `cloakhq/cloakbrowser` Docker Hub image that used to be duplicated in every handler's own `Dockerfile`. - -## Files in this directory - -| File | Purpose | -| ------------------- | ------------------------------------------------------------------------------------------------------------------------------------------------ | -| `Dockerfile` | `FROM` the shared `../base-image`, plus a single `COPY lambda_handler.py`. See `../base-image/INSTRUCTIONS.md` for what the base image provides. | -| `lambda_handler.py` | Default handler. Takes `{url, ...}`, returns `{title, url, html, screenshot_b64?}`. Always headed via Xvfb. | -| `INSTRUCTIONS.md` | This file. | - -This directory is no longer fully standalone: it depends on the shared base image (`../base-image`) having been built and either published to ECR or loaded locally first — see `../base-image/INSTRUCTIONS.md`. Everything specific to this handler (its Dockerfile and `lambda_handler.py`) still lives entirely in this directory. - -## Build - -Build and publish `../base-image` first (see `../base-image/INSTRUCTIONS.md`), then, from inside this directory: - -```bash -docker buildx build --platform linux/arm64 --provenance=false --sbom=false \ - -t bytelyon-scraper:arm64 --load . -``` - -Or from anywhere, pointing at this directory as the build context: - -```bash -docker buildx build --platform linux/arm64 --provenance=false --sbom=false \ - -f path/to/scraper/Dockerfile \ - -t bytelyon-scraper:arm64 --load \ - path/to/scraper -``` - -`--provenance=false --sbom=false` is required — without it, buildx produces a manifest-list/attestation image that Lambda's `CreateFunction`/`UpdateFunctionCode` rejects with `InvalidParameterValueException: ... image manifest, config or layer media type ... is not supported`. - -To build against a locally-loaded (not-yet-pushed) base image instead of pulling from ECR, add `--build-arg BASE_IMAGE=bytelyon-cloakbrowser-base:arm64`. For x86_64, switch `--platform linux/amd64` (and build the base image for the same architecture; slower on Apple Silicon under emulation). - -## Local smoke test (no AWS account needed) - -> **What's the RIE?** Lambda container images can't be run with a plain `docker run` — they expect to talk to AWS's Runtime API (the HTTP service Lambda exposes inside its sandbox to deliver events and collect responses). AWS publishes a small binary called the **Runtime Interface Emulator** that stands up a fake Runtime API on localhost so you can test the container exactly the way Lambda will invoke it, without deploying. We bake the RIE into the image, and the dual-mode entrypoint uses it automatically when `AWS_LAMBDA_RUNTIME_API` isn't set (i.e. you're not running in real Lambda). - -The image bakes in `aws-lambda-rie`, so the standard Lambda local-invoke endpoint works without mounting anything: - -```bash -docker run --rm -p 9000:8080 bytelyon-scraper:arm64 - -# In another shell: -curl -sS -XPOST "http://localhost:9000/2015-03-31/functions/function/invocations" \ - -d '{"url":"https://example.com"}' -``` - -Other invocation surfaces stay intact (these match the canonical CloakHQ image): - -```bash -docker run --rm -it bytelyon-scraper:arm64 python # REPL -docker run --rm bytelyon-scraper:arm64 python examples/basic.py # examples -docker run --rm -p 9222:9222 bytelyon-scraper:arm64 cloakserve --port=9222 # CDP server -docker run --rm bytelyon-scraper:arm64 cloaktest # stealth tests -docker run --rm -it bytelyon-scraper:arm64 node # JS wrapper -``` - -## Event schema - -Only `url` is required. Everything else is optional. - -### Launch options (forwarded to `cloakbrowser.launch_context_async`) - -| Field | Type | Default | -| -------------- | ---------------- | -------------------------------------------------------------- | -| `url` | str | required — `http://` and `https://` only | -| `proxy` | str / dict | none — `http://user:pass@host:port` or a Playwright proxy dict | -| `humanize` | bool | `false` — enable human-like mouse / keyboard / scroll | -| `human_preset` | str | `"default"` or `"careful"` | -| `geoip` | bool | `false` — auto timezone+locale from proxy IP | -| `timezone` | str | none — IANA tz, e.g. `"America/New_York"` | -| `locale` | str | none — BCP-47, e.g. `"en-US"` | -| `viewport` | `{width,height}` | `1920x947` (cloakbrowser default) | -| `user_agent` | str | none | - -### Navigation - -| Field | Type | Default | -| ----------------- | ---- | ----------------------------------------------------------------------------- | -| `wait_until` | str | `"domcontentloaded"` — `load` / `domcontentloaded` / `networkidle` / `commit` | -| `goto_timeout_ms` | int | `30000` | - -### Post-navigation waits - -`smart_wait` is the default when no other wait is specified. It polls `document.documentElement.outerHTML.length` and returns when the size hasn't changed for `dom_stable_ms`. Robust for at-scale scraping because it ignores network activity (analytics beacons, long-poll, websockets) that doesn't mutate the DOM — `wait_until: "networkidle"` is unreliable on modern SPAs for exactly this reason. - -| Field | Type | Default | -| -------------------------------- | ---- | ----------------------------------------------------- | -| `smart_wait` | bool | `true` if no other wait is set | -| `dom_stable_ms` | int | `1500` | -| `max_settle_ms` | int | `15000` | -| `wait_for_load_state` | str | none — `load` / `domcontentloaded` / `networkidle` | -| `wait_for_load_state_timeout_ms` | int | `30000` | -| `wait_for_selector` | str | none — CSS or XPath | -| `wait_for_selector_state` | str | `"visible"` — also `attached` / `detached` / `hidden` | -| `wait_for_selector_timeout_ms` | int | `30000` | -| `wait_ms` | int | none — fixed pause | - -### Capture - -| Field | Type | Default | -| ---------------------- | ---- | ------- | -| `screenshot` | bool | `true` | -| `full_page_screenshot` | bool | `false` | - -### Retry orchestration - -The handler retries transient navigation failures inline within the same Lambda invocation. Two layers, both built-in: - -- **Launch retries** — 3 attempts with 0.3 s + 0.6 s backoff. Recovers Xvfb / Chromium spawn races at cold start. Fast and cheap; not configurable. -- **Strategy retries** — default 1 attempt, configurable via the `retries` event field. Recovers specific post-launch error classes by relaunching with adjusted internal Chromium args / page-load budgets. - -| Field | Type | Default | -| --------- | ---- | ------------------------------------------------------------------------------------------------------ | -| `retries` | int | `1` — number of strategy-retry attempts after the first failure. Set to `0` to disable retry entirely. | - -Strategies (priority order — first match wins): - -| Error pattern | Strategy applied | -| ----------------------------- | ----------------------------------------------------------------------- | -| `ERR_CERT_*` (any cert error) | `extra_args: ["--ignore-certificate-errors"]`, `goto_timeout_ms: 60000` | -| `Timeout … exceeded` | `goto_timeout_ms: 90000`, `max_settle_ms: 25000` | -| `ERR_CONNECTION_TIMED_OUT` | same as `Timeout … exceeded` | - -Errors that are **not retried** (no anonymous scraper can recover): `ERR_NAME_NOT_RESOLVED`, `ERR_SSL_PROTOCOL_ERROR`, `ERR_CONNECTION_REFUSED`, `ERR_HTTP_RESPONSE_CODE_FAILURE`. These bail immediately. - -On final failure, the raised `RuntimeError`'s message includes a `retry_history` block listing every attempt (strategy applied + error seen). Successful invocations return the standard response shape unchanged — no surprise fields when retries didn't fire. - -### Response - -```json -{ - "title": "...", - "url": "https://example.com/", - "html": "...", - "screenshot_b64": "" -} -``` - -## Lambda-specific Chromium hardening (baked in, do not remove) - -Two flags are forced on every launch by `lambda_handler.py`: - -- `--disable-dev-shm-usage` — Lambda's `/dev/shm` is ~64 MB; Chromium's renderer crashes mid-paint without this. -- `--no-zygote` — Lambda's restricted process model can't fork from Chromium's zygote process; without this the browser launches but child renderers fail to spawn and the first `page.new_page()` raises `TargetClosedError`. - -## Function configuration recommendations - -Whatever tool you use to create the Lambda function (CLI, CDK, Terraform, SAM, console), apply these settings: - -| Setting | Value | Why | -| -------------------------- | ----------------------------- | -------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------- | -| Package type | Image | Required — this is a container image, not a zip. | -| Architecture | `arm64` | Roughly 20% cheaper than x86_64. Native build on Apple Silicon. Match the architecture you built for. | -| Memory | 3008 MB | Memory in Lambda is tied to vCPU. Below ~1769 MB Chromium starts noticeably slower. | -| Timeout | 120–180 s | Single-attempt scrapes complete in 3–15 s warm; under retry, a `Timeout`-class first failure (30 s default) plus a longer-budget retry (90 s) plus cleanup can total ~120-130 s. 180 s leaves headroom; below 120 s the function will time out before the retry completes. Cold-start init adds 5-10 s on top. | -| Ephemeral storage (`/tmp`) | 1024 MB | Chromium profile dirs and screenshots can fill the 512 MB default. | -| Networking | Default (no VPC) | Binary is baked in, no network needed at cold start. Add VPC + NAT only if your proxy egress requires it. | -| Execution role | `AWSLambdaBasicExecutionRole` | Just CloudWatch Logs. Add more permissions only if your handler needs them. | - -## Cold start - -First invocation in a new container takes ~80–90 s (image extraction, Chromium binary mmap, JS engine warmup, no DNS/TLS caches). Subsequent warm invocations on the same container are 3–15 s. - -For latency-sensitive use cases: provision concurrency, schedule a CloudWatch/EventBridge warmer ping, or accept the cold tail. - -If you see empty/missing dynamic content on cold-start invocations, raise `max_settle_ms` in the event payload (e.g. `25000`) — the default `15000` is tuned for warm runs. - -## Security - -The handler validates all incoming URLs before navigation: - -- **Scheme restriction** — only `http://` and `https://` are accepted. `file://`, `data:`, `javascript:`, and other schemes are rejected. -- **SSRF protection** — hostnames are resolved before navigation and checked against private, loopback, link-local, reserved, and multicast IP ranges. This blocks access to cloud metadata endpoints (e.g. `169.254.169.254`), localhost services, and internal networks. -- **Post-navigation re-validation** — the final URL is re-checked after page load and after post-navigation waits to catch server-side redirects to blocked destinations. -- **No caller-controlled Chromium flags** — the handler does not accept arbitrary CLI flags from the event. Internal retry strategies add flags as needed (e.g. `--ignore-certificate-errors` for cert errors). -- **No arbitrary JS execution** — `wait_for_function` is not exposed. Use `wait_for_selector` or `smart_wait` instead. - -**Limitations**: - -- Post-navigation re-validation prevents response _exfiltration_, but does not prevent the browser from _making_ the request. If an internal endpoint has side effects on GET, the request will still reach it before validation rejects the response. Use network-level controls (security groups, VPC) to protect side-effect-bearing internal endpoints. -- DNS rebinding attacks can bypass pre-navigation IP checks in theory, though the post-navigation re-validation provides a second layer of defense. - -**Trust boundary**: if this handler is exposed to untrusted callers (Lambda Function URL, API Gateway without auth, public ALB), add an authentication layer (API Gateway authorizer, IAM auth, etc.). The URL validation above is defense-in-depth, not a substitute for access control. - diff --git a/docker/lambda/scraper/lambda_handler.py b/docker/lambda/scraper/lambda_handler.py deleted file mode 100644 index 32698cd..0000000 --- a/docker/lambda/scraper/lambda_handler.py +++ /dev/null @@ -1,389 +0,0 @@ -"""AWS Lambda handler for one-off stealth-browser invocations. - -Always runs **headed** via the Xvfb display started by `lambda-entrypoint.sh`. - -Event schema (all fields except `url` are optional): - - Launch options (passed to cloakbrowser.launch_context_async): - url str required, the page to scrape (http/https only) - proxy str|dict http://user:pass@host:port or Playwright proxy dict - humanize bool False — enable human-like mouse/keyboard/scroll - human_preset str "default" | "careful" - geoip bool False — auto timezone+locale from proxy IP - timezone str IANA tz, e.g. "America/New_York" - locale str BCP-47, e.g. "en-US" - viewport {width,height} defaults to 1920x947 (cloakbrowser DEFAULT_VIEWPORT) - user_agent str custom UA (rare — cloakbrowser sets one already) - - Navigation options (passed to page.goto): - wait_until str "load"|"domcontentloaded"|"networkidle"|"commit" - default "domcontentloaded" - goto_timeout_ms int 30000 - - Post-navigation waits (run in this order if specified): - smart_wait bool ON by default if no other wait is set. - Polls document.outerHTML.length and bails when it - hasn't changed for `dom_stable_ms`. Handles lazy - hydration, async chunks, and lazy images, and is - immune to analytics beacons / long-poll that keep - the network busy without mutating the DOM. - dom_stable_ms int 1500 — how long DOM must be quiet - max_settle_ms int 15000 — hard cap on smart_wait - wait_for_load_state str "load"|"domcontentloaded"|"networkidle" - wait_for_load_state_timeout_ms int 30000 - wait_for_selector str CSS or XPath selector - wait_for_selector_state str "attached"|"detached"|"visible"|"hidden", default "visible" - wait_for_selector_timeout_ms int 30000 - wait_ms int fixed pause in ms (page.wait_for_timeout) - - Capture options: - screenshot bool True - full_page_screenshot bool False — capture entire scrollable page - - Retry orchestration: - retries int default 1. Number of retry attempts after the first - failure. Set to 0 to disable retries entirely (the - handler will fail fast on the first error). - Retried errors: - ERR_CERT_* -> retry with --ignore-certificate-errors - Timeout exceeded -> retry with goto_timeout_ms=90000, max_settle_ms=25000 - ERR_CONNECTION_TIMED_OUT -> same as Timeout - Not retried (unrecoverable): ERR_NAME_NOT_RESOLVED, - ERR_SSL_PROTOCOL_ERROR, generic ERR_CONNECTION_REFUSED. - On final failure, the error message includes a - retry_history block with strategy + error per attempt. - -Returns: - {"title": ..., "url": ..., "html": ..., "screenshot_b64"?: ...} -""" - -from __future__ import annotations - -import asyncio -import base64 -import ipaddress -import json -import logging -import socket -import subprocess -from pathlib import Path -from typing import Any -from urllib.parse import urlparse - -from cloakbrowser import launch_context_async - -logger = logging.getLogger("cloakbrowser.lambda") -logger.setLevel(logging.INFO) - -# awslambdaric wires the root logger with a formatter like -# "[INFO]\t2024-01-15T10:30:45.123Z\t\tmessage" — CloudWatch already -# timestamps every event server-side, so repeating timestamp + request id on -# every line just burns terminal columns. Trim to level + message. -for _h in logging.getLogger().handlers: - _h.setFormatter(logging.Formatter("%(levelname)s %(message)s")) - - -def _validate_url(url: str) -> None: - """Reject non-HTTP schemes and URLs that resolve to private/internal IPs.""" - parsed = urlparse(url) - if parsed.scheme.lower() not in ("http", "https"): - raise ValueError( - f"Only http:// and https:// URLs are supported, got: {parsed.scheme!r}" - ) - hostname = parsed.hostname - if not hostname: - raise ValueError("URL has no hostname") - try: - infos = socket.getaddrinfo(hostname, None, socket.AF_UNSPEC, socket.SOCK_STREAM) - except socket.gaierror: - raise ValueError(f"Cannot resolve hostname: {hostname}") - for info in infos: - addr = ipaddress.ip_address(info[4][0]) - if not addr.is_global: - raise ValueError("URLs targeting private/internal networks are blocked") - - -def _diag_snapshot() -> str: - """Capture Xvfb status, Xvfb log, X11 socket state, and env for error reports.""" - import os - - parts = [] - try: - r = subprocess.run(["pgrep", "-fa", "Xvfb"], capture_output=True, text=True) - parts.append(f"pgrep Xvfb: rc={r.returncode} stdout={r.stdout.strip()!r}") - except Exception as e: - parts.append(f"pgrep failed: {e}") - try: - r = subprocess.run( - ["ls", "-la", "/tmp/.X11-unix"], capture_output=True, text=True - ) - parts.append(f"ls /tmp/.X11-unix:\n{r.stdout}{r.stderr}") - except Exception as e: - parts.append(f"ls /tmp/.X11-unix failed: {e}") - try: - log = Path("/tmp/Xvfb.log").read_text() - parts.append(f"/tmp/Xvfb.log:\n{log}") - except Exception as e: - parts.append(f"Xvfb log unreadable: {e}") - parts.append( - f"env: DISPLAY={os.environ.get('DISPLAY')!r} HOME={os.environ.get('HOME')!r}" - ) - return "\n".join(parts) - - -def handler(event: dict, context: Any) -> dict: - return asyncio.run(_run(event)) - - -def _build_launch_kwargs(event: dict) -> dict: - """Translate the event dict into kwargs for launch_context_async. - - Only includes keys explicitly set in the event so cloakbrowser's defaults - (DEFAULT_VIEWPORT etc.) kick in when fields are absent — passing - viewport=None would *disable* viewport emulation, which we don't want. - """ - kwargs: dict = { - "headless": False, # always headed via Xvfb - "args": [ - # Lambda /dev/shm is ~64 MB — Chromium crashes mid-render without this. - "--disable-dev-shm-usage", - # Lambda's restricted process model can't fork from Chromium's zygote - # — without this, child renderer processes fail to spawn. - "--no-zygote", - *event.get("_strategy_args", []), - ], - } - for key in ( - "proxy", - "humanize", - "human_preset", - "geoip", - "timezone", - "locale", - "viewport", - "user_agent", - ): - if key in event: - kwargs[key] = event[key] - return kwargs - - -async def _smart_wait( - page, dom_stable_ms: int = 1500, max_settle_ms: int = 15000 -) -> None: - """Wait until the document HTML hasn't changed for `dom_stable_ms`. - - Generic stopping condition for at-scale scraping when you can't tune - selectors per site. More robust than `networkidle` because it ignores - network activity that doesn't mutate the DOM (analytics beacons, - long-poll, websockets, web vitals streams). - """ - js = f""" - (() => {{ - if (!window.__cb_settle) {{ - window.__cb_settle = {{ len: -1, since: Date.now() }}; - }} - const cur = document.documentElement.outerHTML.length; - const s = window.__cb_settle; - if (cur !== s.len) {{ - s.len = cur; - s.since = Date.now(); - return false; - }} - return (Date.now() - s.since) >= {int(dom_stable_ms)}; - }})() - """ - try: - await page.wait_for_function(js, timeout=max_settle_ms, polling=200) - except Exception: - # Hit max_settle_ms cap — return what we have rather than fail the whole invoke - logger.warning("smart_wait hit max_settle_ms=%d cap", max_settle_ms) - - -_EXPLICIT_WAIT_KEYS = ( - "wait_for_load_state", - "wait_for_selector", - "wait_ms", -) - - -async def _post_nav_waits(page, event: dict) -> None: - """Run waits in priority order. smart_wait is the default unless the - caller asked for a more specific stopping condition.""" - explicit = any(k in event for k in _EXPLICIT_WAIT_KEYS) - if event.get("smart_wait", not explicit): - await _smart_wait( - page, - dom_stable_ms=event.get("dom_stable_ms", 1500), - max_settle_ms=event.get("max_settle_ms", 15000), - ) - if "wait_for_load_state" in event: - await page.wait_for_load_state( - event["wait_for_load_state"], - timeout=event.get("wait_for_load_state_timeout_ms", 30000), - ) - if "wait_for_selector" in event: - await page.wait_for_selector( - event["wait_for_selector"], - state=event.get("wait_for_selector_state", "visible"), - timeout=event.get("wait_for_selector_timeout_ms", 30000), - ) - if "wait_ms" in event: - await page.wait_for_timeout(event["wait_ms"]) - - -async def _launch_with_retry(event: dict, attempts: int = 3, backoff_s: float = 0.3): - """Retry launch_context_async up to `attempts` times with linear backoff. - - Lambda cold-start storms occasionally race Xvfb readiness or hit transient - Chromium spawn failures — both surface as "Target page, context or browser - has been closed" at launch. The failure is fast (~0.5s) so retries are - cheap, and a retry on a now-warm container almost always succeeds. - - Pairs with the lock-cleanup + socket-poll in lambda-entrypoint.sh: the - entrypoint catches the common case at container init; this catches the - residual race when the first invocation hits before Xvfb is fully ready. - """ - last_err: Exception | None = None - for i in range(attempts): - try: - return await launch_context_async(**_build_launch_kwargs(event)) - except Exception as e: - last_err = e - logger.warning( - "launch attempt %d/%d failed: %s", i + 1, attempts, str(e)[:200] - ) - if i + 1 < attempts: - await asyncio.sleep(backoff_s * (i + 1)) # 0.3s, 0.6s - raise last_err # type: ignore[misc] - - -def _classify_error(err: Exception) -> dict | None: - """Map a Playwright error to a retry-strategy override dict, or None - if the error is unrecoverable. - - Match on str(e) because Playwright errors carry their codes inside the - message (Error.__str__ includes ERR_CERT_AUTHORITY_INVALID etc.); there - is no stable structured `.error_code` attribute to rely on. - - Strategies (priority order — first match wins): - ERR_CERT_* -> --ignore-certificate-errors + 60s goto budget - Timeout exceeded -> 90s goto budget + 25s smart_wait cap - ERR_CONNECTION_TIMED_OUT -> same as Timeout - Returns None for unrecoverable site issues (DNS, SSL, refused, HTTP 4xx/5xx). - """ - msg = str(err) - if "ERR_CERT" in msg: - return { - "_strategy_args": ["--ignore-certificate-errors"], - "goto_timeout_ms": 60000, - } - if ("Timeout" in msg and "exceeded" in msg) or "ERR_CONNECTION_TIMED_OUT" in msg: - return { - "goto_timeout_ms": 90000, - "max_settle_ms": 25000, - } - return None - - -async def _attempt_scrape(url: str, event: dict) -> dict: - """One self-contained scrape attempt: launch, navigate, wait, capture, close. - - Extracted from `_run` so the retry loop can call it repeatedly with an - overridden event dict. Each attempt relaunches the browser — uniform - behavior across strategies (the cert-bypass strategy *requires* a relaunch - because `--ignore-certificate-errors` is a Chromium CLI arg, not a per- - context switch), and the ~3-5s relaunch cost is fine on the slow path. - """ - ctx = await _launch_with_retry(event) - try: - page = await ctx.new_page() - await page.goto( - url, - wait_until=event.get("wait_until", "domcontentloaded"), - timeout=event.get("goto_timeout_ms", 30000), - ) - _validate_url(page.url) - - await _post_nav_waits(page, event) - _validate_url(page.url) - - result: dict = { - "title": await page.title(), - "url": page.url, - "html": await page.content(), - } - - if event.get("screenshot", True): - png = await page.screenshot( - full_page=event.get("full_page_screenshot", False), - ) - result["screenshot_b64"] = base64.b64encode(png).decode() - - return result - finally: - try: - await ctx.close() - except Exception: - pass - - -def _raise_with_history(err: Exception, history: list[dict]) -> None: - """Surface a final failure with a retry_history block embedded in the - error message, so callers see what was tried before bailing.""" - diag = _diag_snapshot() - if history: - diag = "retry_history: " + json.dumps(history, default=str) + "\n\n" + diag - logger.error( - "scrape failed (after %d retries): %s\nDIAG:\n%s", len(history), err, diag - ) - raise RuntimeError(f"scrape failed: {err}\n--- DIAG ---\n{diag}") from err - - -async def _run(event: dict) -> dict: - """Top-level scrape with strategy-based retry orchestration. - - First attempt uses the event verbatim. If it fails with a classifiable - error (cert / timeout), retry with that strategy's overrides merged into - the event. `retries` bounds the number of strategy retries (default 1; - set to 0 to disable retry entirely). - """ - url = event["url"] - _validate_url(url) - event = { - k: v for k, v in event.items() if k not in ("extra_args", "_strategy_args") - } - retries_left = max(0, int(event.get("retries", 1))) - history: list[dict] = [] - current_event = event - - while True: - try: - return await _attempt_scrape(url, current_event) - except Exception as e: - if retries_left <= 0: - _raise_with_history(e, history) - strategy = _classify_error(e) - if strategy is None: - _raise_with_history(e, history) - history.append( - { - "attempt": len(history) + 1, - "error": str(e)[:300], - "strategy": strategy, - } - ) - logger.warning( - "attempt %d failed (%s); retrying with strategy=%s", - len(history), - str(e)[:120], - strategy, - ) - merged_args = list(current_event.get("_strategy_args", [])) + list( - strategy.get("_strategy_args", []) - ) - current_event = {**current_event, **strategy, "_strategy_args": merged_args} - retries_left -= 1 - # No backoff: strategy overrides change goto budget directly; - # the prior failure was either fast (cert reject) or already - # waited its full timeout. Container is warm. diff --git a/docker/lambda/serp-di/Dockerfile b/docker/lambda/serp-di/Dockerfile deleted file mode 100644 index ece70d5..0000000 --- a/docker/lambda/serp-di/Dockerfile +++ /dev/null @@ -1,56 +0,0 @@ -# CloakBrowser Google SERP scraper on AWS Lambda — DataImpulse SOCKS5 proxy, -# wired up exactly the way it ran successfully overnight as a standalone -# `uv run` script (see lambda_handler.py's module docstring for the original). -# -# Built FROM the shared bytelyon-cloakbrowser-base image (see -# ../base-image/Dockerfile), same as ../serp-scraper — gets the baked-in -# GeoIP database, real MS fonts, Xvfb entrypoint, and non-root readability fix -# for free. A sibling of ../serp-scraper: same retry-on-block/relaunch -# philosophy and the same structured `data` SERP-parsing output contract -# (ported verbatim, see lambda_handler.py), but reaches Google through -# DataImpulse's SOCKS5 proxy instead of Oxylabs, and its own -# launch()/goto()/type() navigation sequence is the one already proven -# reliable overnight as a plain script. -# -# This same image (unmodified) is also the base for ../serp-di/Dockerfile.worker -# — a local-machine SQS worker that imports lambda_handler.py's `main()` -# directly and runs the identical browser logic outside Lambda entirely, since -# Google trusts residential/home-network egress far more than any datacenter -# IP (Lambda's own included). See worker.py's module docstring. -# -# ─── Build (the base image must already exist, locally or in ECR) ──────────── -# docker buildx build --platform linux/arm64 --provenance=false --sbom=false \ -# -t bytelyon-serp-di:arm64 --load . -# -# docker run --rm -p 9000:8080 bytelyon-serp-di:arm64 -# curl -XPOST http://localhost:9000/2015-03-31/functions/function/invocations \ -# -d '{"query":"sailing blocks"}' -# -# --provenance=false --sbom=false is required: without it, buildx emits a -# manifest-list/attestation image that Lambda's CreateFunction/UpdateFunctionCode -# rejects with "image manifest, config or layer media type ... is not supported". -# -# To build against a locally-loaded (not-yet-pushed) base image instead of -# pulling from ECR: -# --build-arg BASE_IMAGE=bytelyon-cloakbrowser-base:arm64 - -ARG BASE_IMAGE=138305277395.dkr.ecr.us-east-1.amazonaws.com/bytelyon-cloakbrowser-base:latest -FROM ${BASE_IMAGE} - -# `requests` + PySocks (`requests[socks]`), used by lambda_handler.py's -# `_resolve_geo()` instead of cloakbrowser's own geoip=True. Confirmed via the -# bytelyon-http-test diagnostic Lambda: this same SOCKS5 proxy responds to -# `requests` from AWS Lambda's network in under a second, every time, while -# cloakbrowser's httpx-based geoip resolution reliably hangs to its full -# timeout when run inside Lambda specifically (not reproducible locally in -# Docker, and not reproducible with a different HTTP client from the same -# Lambda function) — an httpx/socksio incompatibility with Lambda's sandboxed -# network namespace, not a proxy problem. geoip2 (for reading the DB baked -# into the base image) and cloakbrowser's own country->locale map are already -# available via the `cloakbrowser[geoip]` extra baked into this base image. -RUN pip install --no-cache-dir requests "requests[socks]" - -# Handler sits at /app (already on Python's import path, WORKDIR=/app), -# imports cloakbrowser as a normal library. ENTRYPOINT/CMD are inherited from -# the base image (both already correct for this handler's module name). -COPY lambda_handler.py /app/lambda_handler.py diff --git a/docker/lambda/serp-di/INSTRUCTIONS.md b/docker/lambda/serp-di/INSTRUCTIONS.md deleted file mode 100644 index 2f19ba7..0000000 --- a/docker/lambda/serp-di/INSTRUCTIONS.md +++ /dev/null @@ -1,31 +0,0 @@ -# serp-di: standalone Lambda (reference/one-off testing) - -This directory is now **Lambda-mode only**. The production path — one or -more local-machine workers pulling jobs from a shared SQS queue — moved to -`../../worker/` (a single consolidated location for the serp/news/sitemap -worker fleet). See `../../worker/INSTRUCTIONS.md` for that. - -`lambda_handler.py` here is kept as a standalone, full-featured Lambda -function (DataImpulse proxy, GeoIP, SERP parsing, S3 upload, all in one -Python module) — useful for one-off testing or comparing Lambda-vs-local- -worker block rates, but no longer what `SearchBotJob` calls in production. - -## Build & deploy (unchanged) - -```bash -docker buildx build --platform linux/arm64 --provenance=false --sbom=false \ - --build-arg BASE_IMAGE=138305277395.dkr.ecr.us-east-1.amazonaws.com/bytelyon-cloakbrowser-base:latest \ - -t bytelyon-serp-di:arm64 --load . - -docker run --rm -p 9000:8080 bytelyon-serp-di:arm64 -curl -XPOST http://localhost:9000/2015-03-31/functions/function/invocations \ - -d '{"query":"sailing blocks"}' -``` - -Deployed as the `bytelyon-serp-di` Lambda function (arm64, 3008MB, 300s -timeout) in `us-east-1`. - -## Event schema / response - -See `lambda_handler.py`'s own module docstring — same `data`/ -`screenshot_key`/`content_key` contract as `../serp-scraper`. diff --git a/docker/lambda/serp-di/lambda_handler.py b/docker/lambda/serp-di/lambda_handler.py deleted file mode 100644 index a8a26a9..0000000 --- a/docker/lambda/serp-di/lambda_handler.py +++ /dev/null @@ -1,605 +0,0 @@ -"""AWS Lambda handler / SQS worker core wrapping — as closely as possible — -the CloakBrowser script that ran reliably against Google all night as a -standalone `uv run` script: - - #!/usr/bin/env -S uv run --script - # - # /// script - # requires-python = ">=3.14" - # dependencies = [ - # "cloakbrowser", - # "cloakbrowser[geoip]", - # ] - # /// - import argparse - from pathlib import Path - from uuid import uuid5, NAMESPACE_URL - - from cloakbrowser import launch, ProxySettings - from playwright.sync_api import Browser, Page - - GOOGLE_URL = "https://www.google.com" - - def main(query: str, path: str, headless: bool) -> None: - Path(path).mkdir(parents=True, exist_ok=True) - browser: Browser = launch( - proxy=ProxySettings( - username='2b9bc9b578274c5e9c8f__cr.us', - password='a1d0893a3f201adf', - bypass='.google.com', - server='socks5://gw.dataimpulse.com:824', - ), - headless=headless, - human_preset="careful", - geoip=True - ) - page: Page = browser.new_page() - page.goto(GOOGLE_URL) - page.locator('textarea[name="q"]').first.click() - page.keyboard.type(query) - page.keyboard.press('Enter') - page.wait_for_selector('#sfooter') - name = uuid5(NAMESPACE_URL, f"{GOOGLE_URL}?q={query.replace(' ', '+')}") - page.screenshot(path=f"{path}/{name}.png", full_page=True) - with open(f"{path}/{name}.html", "w", encoding="utf-8") as file: - file.write(page.content()) - browser.close() - -The launch()/goto()/type() sequence, the DataImpulse SOCKS5 proxy -credentials, and the `human_preset="careful"` launch option are copied -verbatim — do not "improve" or re-tune any of that without re-validating -against a live run first, since this exact combination is the one that's -already proven reliable. - -This module is used two ways: - 1. As an AWS Lambda handler (`handler(event, context)`), same as every - sibling handler in ../serp-scraper etc. - 2. Imported directly by worker.py's SQS polling loop, which runs this same - browser logic on a local machine instead of in Lambda — see - ../serp-di/worker.py and its own docstring for why (Google trusts - residential/home-network egress far more than any datacenter IP, - Lambda's included). - -Changes relative to the original script above: - 1. `argparse` -> function args / event fields (`query`, `headless`). - 2. The capture step (screenshot + HTML) no longer sits behind a bare - `page.wait_for_selector("#sfooter")`. Confirmed in testing: Google's - CAPTCHA/interstitial page (served from google.com/sorry/...) never has - that selector, so a block was indistinguishable from a crash — the - handler raised a TimeoutError before capturing a single byte, with no - screenshot or HTML to diagnose *why*. The wait is now soft-bounded, and - capture always runs afterward regardless of that outcome. - 3. Screenshot/HTML are captured in memory (`page.screenshot()` / - `page.content()`) and uploaded straight to S3, then parsed into the same - structured `data` shape (organic results, ads, PAA/similar queries, - etc.) that ../serp-scraper's Lambda handler produces — see - `_extract_data` below, ported from there — so this handler's output is - a drop-in match for `Serp::update()`'s `data`/`screenshot_key`/ - `content_key` contract regardless of which handler (or which machine) - produced it. No local disk round-trip needed anymore. - 4. GeoIP: `geoip=True` is NOT used — see `_resolve_geo()`'s docstring for - why (it hangs specifically inside AWS Lambda's network namespace). - Timezone/locale are resolved independently instead. Harmless overhead - when run outside Lambda (worker.py), just an extra IP lookup. - 5. Lambda-only Chromium hardening (`--disable-dev-shm-usage --no-zygote`) - is always applied. It's a no-op outside Lambda (worker.py) — Chromium - ignores flags it doesn't need — so there's no reason to special-case it - away for the local-worker use case. - -Function args (`main`) / event schema (`handler`): - query str required, e.g. "sailing blocks" - headless bool optional, default True - bucket str optional, default "bytelyon-private" — S3 bucket for - page content + screenshot uploads - prefix str optional, default "serp-scrapes/di//" — S3 key - prefix for both uploads - -Returns: - { - "query": "...", - "url": "https://www.google.com/search?q=...", - "blocked": bool, - "screenshot_key": "serp-scrapes/di//sailing_blocks-.png", - "content_key": "serp-scrapes/di//sailing_blocks-.html", - "data": { - "sponsored_results": [...], "sponsored_products": [...], - "organic_results": [...], "organic_products": [...], - "similar_queries": [...] - } - } - -Same `data` shape (and same degrade-independently-to-`[]`-on-failure -behavior) as ../serp-scraper/lambda_handler.py — see that module's own -docstring for the full per-field shape of each section. -""" - -from __future__ import annotations - -import hashlib -import logging -import re -import time -import uuid -from typing import Any -from urllib.parse import urlparse - -import boto3 -from cloakbrowser import ProxySettings, launch -from cloakbrowser.geoip import COUNTRY_LOCALE_MAP -from playwright.sync_api import Browser, Page -from playwright.sync_api import TimeoutError as PlaywrightTimeoutError - -logger = logging.getLogger("cloakbrowser.serp_di") -logger.setLevel(logging.INFO) - -GOOGLE_URL = "https://www.google.com" -PROXY_USERNAME = "2b9bc9b578274c5e9c8f__cr.us" -PROXY_PASSWORD = "a1d0893a3f201adf" -PROXY_HOST = "gw.dataimpulse.com" -PROXY_PORT = 824 -PROXY_URL = f"socks5://{PROXY_USERNAME}:{PROXY_PASSWORD}@{PROXY_HOST}:{PROXY_PORT}" -GEOIP_DB_PATH = "/root/.cloakbrowser/geoip/GeoLite2-City.mmdb" -DEFAULT_BUCKET = "bytelyon-private" - -# DataImpulse's SOCKS5 endpoint has a known non-trivial raw connection-failure -# rate (confirmed separately via plain HTTP sampling against this same proxy). -# It's shown up here at every stage: GeoIP resolution timing out (the exit-IP -# lookup itself can't get through), launch-time "Target crashed", and -# page.goto() itself timing out mid-navigation. All are proxy-connection -# flakiness, not a code bug, so the retry wraps the *entire* attempt (fresh -# browser, fresh proxy connection, fresh everything) rather than just launch() -# — a fresh SOCKS5 connection on the next attempt routinely succeeds where the -# previous one didn't, but only if it's actually a fresh connection. -SCRAPE_ATTEMPTS = 2 - -_BLOCK_PHRASES = ( - "unusual traffic", - "our systems have detected", -) - -# Google's consent interstitial (shown for some EU-ish exit IPs regardless of -# hl/gl) — #L2AGLb ("I agree") has been a long-lived id for this button, but -# fall back to text matching since it's not guaranteed to still exist. -_CONSENT_SELECTORS = [ - "#L2AGLb", - "button:has-text('Accept all')", - "button:has-text('I agree')", -] - - -def _resolve_geo() -> tuple[str | None, str | None]: - """Resolve (timezone, locale) for the proxy's exit IP ourselves, instead - of using cloakbrowser's built-in geoip=True. - - Confirmed by direct diagnostic (bytelyon-http-test Lambda, plain - `requests` + PySocks against this same SOCKS5 proxy): the proxy itself - responds through AWS Lambda's network in under a second, every time. - But cloakbrowser's own geoip=True path (httpx's SOCKS5 client) reliably - hangs to its full timeout when run inside Lambda specifically — not - reproducible locally in Docker, and not reproducible with a different - HTTP client from the very same Lambda function. That points at an - httpx/socksio incompatibility with Lambda's sandboxed network namespace, - not a proxy problem. Reusing the same baked GeoLite2 DB and - country->locale map cloakbrowser itself uses, just via `requests` - (already proven reliable here) instead of httpx. Also used unmodified by - worker.py (outside Lambda) — a plain, reliable lookup either way. - """ - import geoip2.database - import requests - - try: - resp = requests.get( - "https://api.ipify.org", - proxies={"https": PROXY_URL}, - timeout=10, - ) - resp.raise_for_status() - ip = resp.text.strip() - except Exception: - return None, None - - try: - with geoip2.database.Reader(GEOIP_DB_PATH) as reader: - city = reader.city(ip) - timezone = city.location.time_zone - country = city.country.iso_code - locale = COUNTRY_LOCALE_MAP.get(country) if country else None - return timezone, locale - except Exception: - return None, None - - -def _launch(headless: bool) -> Browser: - timezone, locale = _resolve_geo() - return launch( - proxy=ProxySettings( - username=PROXY_USERNAME, - password=PROXY_PASSWORD, - bypass=".google.com", - server=f"socks5://{PROXY_HOST}:{PROXY_PORT}", - ), - headless=headless, - human_preset="careful", - # geoip=True is deliberately NOT used here — see _resolve_geo()'s - # docstring. timezone/locale are resolved ourselves instead, using a - # client already confirmed reliable through this proxy from Lambda. - # Falls back to no timezone/locale override (Chromium's own defaults) - # if resolution fails, rather than blocking the whole launch on it. - timezone=timezone, - locale=locale, - # Lambda-only hardening, harmless outside Lambda (worker.py) — - # Chromium simply ignores flags it doesn't need there. Without these - # in Lambda, `browser.new_page()` reliably raises "Target crashed": - # Lambda's /dev/shm is ~64MB (Chromium's renderer needs more) and its - # sandboxed process model can't fork from Chromium's zygote process. - # Same fix CloakHQ's own official AWS Lambda example bakes in - # unconditionally for this exact reason. - args=["--disable-dev-shm-usage", "--no-zygote"], - ) - - -def _empty_data() -> dict: - """All-empty `data` shape, used when the page is blocked or extraction - fails outright, so callers can always rely on the same response shape. - Identical to ../serp-scraper/lambda_handler.py's own `_empty_data`. - """ - return { - "sponsored_results": [], - "sponsored_products": [], - "organic_results": [], - "organic_products": [], - "similar_queries": [], - } - - -# Ported verbatim from ../serp-scraper/lambda_handler.py — see that module's -# own comments for the full rationale (structural, best-effort parsing that -# degrades gracefully as Google's unversioned/obfuscated markup shifts). -_DATA_EXTRACT_JS = r""" -() => { - const textOf = (el) => (el && el.textContent ? el.textContent.trim() : ""); - - const sponsoredResults = Array.from(document.querySelectorAll("[data-pcu]")).map((e, i) => { - const spans = e.querySelectorAll("span"); - const rawBrand = spans[1] ? textOf(spans[1]) : ""; - return { - index: i, - title: spans[0] ? textOf(spans[0]) : "", - brand: rawBrand.split("https://")[0].trim(), - href: e.getAttribute("href") || "", - }; - }); - - const organicProducts = Array.from(document.querySelectorAll("product-viewer-entrypoint")).map((e, i) => { - const div = e.querySelector("div"); - const img = e.querySelector("img"); - let title = div ? (div.getAttribute("aria-label") || "") : ""; - if (!title && img) title = img.getAttribute("alt") || ""; - return { - index: i, - title, - image: img ? (img.getAttribute("src") || "") : "", - }; - }); - - const sponsoredProducts = Array.from(document.querySelectorAll("[data-dtld]")).map((e, i) => { - const div = e.querySelector("div.pla-unit-container"); - const a = div ? div.querySelector("a.pla-unit-img-container-link") : null; - const titleEl = div ? div.querySelector("div[data-call_grow_wiz_event='true']") : null; - const priceEl = div ? div.querySelector("div[aria-label]") : null; - const brandEl = div ? div.querySelector("span[role='text']") : null; - const img = a ? a.querySelector("img") : null; - return { - index: i, - domain: e.getAttribute("data-dtld") || "", - href: a ? (a.getAttribute("href") || "") : "", - image: img ? (img.getAttribute("src") || "") : "", - title: textOf(titleEl), - price: priceEl ? (priceEl.getAttribute("aria-label") || "") : "", - brand: textOf(brandEl), - }; - }); - - const organicResults = Array.from(document.querySelectorAll("h3[id]")).map((e, i) => { - const parent = e.parentElement; - return { - index: i, - title: textOf(e), - href: parent ? (parent.getAttribute("href") || "") : "", - }; - }); - - const similarQueries = []; - document.querySelectorAll("div[data-notify-expansion]").forEach((e) => { - const q = e.getAttribute("data-q") || ""; - if (q.length > 4) similarQueries.push(q); - }); - const botstuff = document.querySelector("div#botstuff"); - if (botstuff) { - botstuff.querySelectorAll("a").forEach((a) => { - const t = textOf(a); - if (t.length > 4) similarQueries.push(t); - }); - } - - return { sponsoredResults, organicProducts, sponsoredProducts, organicResults, similarQueries }; -} -""" - -# In-memory cache of resolved destination URLs, shared across invocations in -# the same warm process (Lambda container or long-running worker.py loop). -_URL_CACHE: dict[str, tuple[float, str]] = {} -_URL_CACHE_TTL_S = 1800 # 30 minutes - - -def _to_domain(url: str) -> str: - host = urlparse(url).hostname or "" - return host.removeprefix("www.") - - -def _final_url(page: Page, link: str, timeout_ms: int) -> str: - """Resolve a (possibly relative/redirect) Google result link to its final - destination URL by actually requesting it, through the same browser - context (and therefore the same proxy/session) as the search itself. - Falls back to the normalized-but-unresolved link on any failure. - """ - if not link: - return link - if not link.startswith("https://www.google.com"): - if not link.startswith("/"): - link = f"/{link}" - link = f"https://www.google.com{link}" - - cache_key = hashlib.sha1(link.encode()).hexdigest() - now = time.monotonic() - cached = _URL_CACHE.get(cache_key) - if cached and cached[0] > now: - return cached[1] - - try: - final_url = page.request.get(link, timeout=timeout_ms).url - except Exception: - return link - - expires_at = now + _URL_CACHE_TTL_S - _URL_CACHE[cache_key] = (expires_at, final_url) - _URL_CACHE[hashlib.sha1(final_url.encode()).hexdigest()] = (expires_at, final_url) - return final_url - - -def _extract_data(page: Page, url_resolve_timeout_ms: int) -> dict: - """Best-effort structural SERP parser — ported verbatim from - ../serp-scraper/lambda_handler.py. See that module for full rationale. - """ - try: - raw = page.evaluate(_DATA_EXTRACT_JS) - except Exception: - logger.exception("SERP data extraction failed") - return _empty_data() - - def resolve(link: str) -> str: - return _final_url(page, link, url_resolve_timeout_ms) - - sponsored_results: list[dict] = [] - try: - for item in raw.get("sponsoredResults", []): - url = resolve(item.get("href", "")) - sponsored_results.append( - { - "kind": "sponsored_result", - "index": item["index"], - "title": item.get("title", ""), - "brand": item.get("brand", ""), - "url": url, - "domain": _to_domain(url), - } - ) - except Exception: - logger.exception("sponsored_results extraction failed") - sponsored_results = [] - - organic_products: list[dict] = [] - try: - organic_products = [ - { - "kind": "organic_product", - "index": item["index"], - "title": item.get("title", ""), - "image": item.get("image", ""), - } - for item in raw.get("organicProducts", []) - ] - except Exception: - logger.exception("organic_products extraction failed") - organic_products = [] - - sponsored_products: list[dict] = [] - try: - for item in raw.get("sponsoredProducts", []): - url = resolve(item.get("href", "")) - sponsored_products.append( - { - "kind": "sponsored_product", - "index": item["index"], - "domain": item.get("domain", ""), - "url": url, - "image": item.get("image", ""), - "title": item.get("title", ""), - "price": item.get("price", ""), - "brand": item.get("brand", ""), - } - ) - except Exception: - logger.exception("sponsored_products extraction failed") - sponsored_products = [] - - organic_results: list[dict] = [] - try: - for item in raw.get("organicResults", []): - url = resolve(item.get("href", "")) - organic_results.append( - { - "kind": "organic_result", - "index": item["index"], - "title": item.get("title", ""), - "url": url, - "domain": _to_domain(url), - } - ) - except Exception: - logger.exception("organic_results extraction failed") - organic_results = [] - - similar_queries: list[dict] = [] - try: - similar_queries = [ - {"kind": "similar_query", "index": i, "value": value} - for i, value in enumerate(raw.get("similarQueries", [])) - ] - except Exception: - logger.exception("similar_queries extraction failed") - similar_queries = [] - - return { - "sponsored_results": sponsored_results, - "sponsored_products": sponsored_products, - "organic_results": organic_results, - "organic_products": organic_products, - "similar_queries": similar_queries, - } - - -def _dismiss_consent(page: Page) -> None: - """Best-effort click-through for Google's cookie-consent interstitial.""" - for selector in _CONSENT_SELECTORS: - try: - locator = page.locator(selector).first - if locator.is_visible(timeout=1_000): - locator.click(timeout=2_000) - page.wait_for_load_state("domcontentloaded", timeout=5_000) - return - except Exception: - continue - - -def _is_blocked(page: Page, html: str, sfooter_found: bool) -> bool: - """Checks the already-captured page source rather than issuing a second, - separate live DOM query — see ../serp-scraper/lambda_handler.py's - `_is_blocked` for why that matters (a real CAPTCHA page can be missed by - a live query that races the page settling). `sfooter_found` folds in - this handler's own original wait_for_selector('#sfooter') signal: that - selector only ever appears on a real, unblocked results page. - """ - if "/sorry/" in page.url: - return True - if not sfooter_found: - return True - return any(phrase in html.lower() for phrase in _BLOCK_PHRASES) - - -def _s3_key(prefix: str, query: str, url: str, ext: str) -> str: - safe_query = ( - re.sub(r"[^a-zA-Z0-9_-]+", "_", query.strip()).strip("_")[:80] or "query" - ) - digest = hashlib.sha256(url.encode()).hexdigest()[:10] - return f"{prefix.rstrip('/')}/{safe_query}-{digest}.{ext}" - - -def _attempt(query: str, headless: bool, bucket: str, prefix: str) -> dict: - browser: Browser = _launch(headless) - try: - page: Page = browser.new_page() - page.goto(GOOGLE_URL, timeout=45_000) - page.locator('textarea[name="q"]').first.click() - page.keyboard.type(query) - page.keyboard.press("Enter") - - # #sfooter only ever appears on a real, unblocked results page. - # Google's CAPTCHA/interstitial page (served from google.com/sorry/...) - # never has it, so a naked `wait_for_selector` throws before a single - # byte of the page is ever captured — the original script's biggest - # blind spot: a block looked identical to a crash, with no - # screenshot/HTML to diagnose it. Downgrade the wait to a soft check - # (bounded timeout, exception caught) so the capture step below - # always runs and always returns something to look at, whether - # that's a clean SERP or a CAPTCHA page. - try: - page.wait_for_selector("#sfooter", timeout=30_000) - sfooter_found = True - except PlaywrightTimeoutError: - sfooter_found = False - - if sfooter_found: - _dismiss_consent(page) - - final_url = page.url - html = page.content() - png = page.screenshot(full_page=True) - blocked = _is_blocked(page, html, sfooter_found) - - content_key = _s3_key(prefix, query, final_url, "html") - screenshot_key = _s3_key(prefix, query, final_url, "png") - - s3 = boto3.client("s3") - s3.put_object( - Bucket=bucket, - Key=content_key, - Body=html.encode("utf-8"), - ContentType="text/html; charset=utf-8", - ) - s3.put_object( - Bucket=bucket, Key=screenshot_key, Body=png, ContentType="image/png" - ) - - if blocked: - logger.warning("Google served a block/CAPTCHA page for query=%r", query) - data = _empty_data() - else: - data = _extract_data(page, url_resolve_timeout_ms=10_000) - - return { - "query": query, - "url": final_url, - "blocked": blocked, - "screenshot_key": screenshot_key, - "content_key": content_key, - "data": data, - } - finally: - browser.close() - - -def main( - query: str, - headless: bool = False, - bucket: str = DEFAULT_BUCKET, - prefix: str | None = None, -) -> dict: - if prefix is None: - prefix = f"serp-scrapes/di/{uuid.uuid4()}/" - last_err: Exception | None = None - for attempt in range(1, SCRAPE_ATTEMPTS + 1): - try: - return _attempt(query, headless, bucket, prefix) - except Exception as e: - last_err = e - logger.warning( - "scrape attempt %d/%d failed: %s", attempt, SCRAPE_ATTEMPTS, e - ) - raise last_err # type: ignore[misc] - - -def handler(event: dict, context: Any) -> dict: - query = event.get("query") - if not isinstance(query, str) or not query.strip(): - raise ValueError("event must include a non-empty 'query' string") - return main( - query=query, - # False (headed, via the base image's Xvfb entrypoint) matches both - # ../serp-scraper's own default and CloakBrowser's own guidance that - # headed mode meaningfully reduces blocks — "most blocks come from - # missing residential proxy + geoip + headed mode." - headless=event.get("headless", False), - bucket=event.get("bucket", DEFAULT_BUCKET), - prefix=event.get("prefix"), - ) diff --git a/docker/lambda/serp-scraper/Dockerfile b/docker/lambda/serp-scraper/Dockerfile deleted file mode 100644 index 70bf51a..0000000 --- a/docker/lambda/serp-scraper/Dockerfile +++ /dev/null @@ -1,39 +0,0 @@ -# CloakBrowser Google SERP scraper on AWS Lambda. -# -# Built FROM the shared bytelyon-cloakbrowser-base image (see -# ../base-image/Dockerfile and ../base-image/INSTRUCTIONS.md), which carries -# every bit of common CloakBrowser + Lambda glue: awslambdaric, the Lambda -# Runtime Interface Emulator, boto3, the dual-mode entrypoint, the non-root -# readability fix, and the runtime ENV. This Dockerfile only adds this -# handler's own module — see lambda_handler.py. -# -# Sibling of ../news-scraper, ../page-scraper, ../scraper, and -# ../site-crawler (same base image, different lambda_handler.py). This -# handler runs a Google search and parses the results page into a JSON -# object keyed by SERP section (organic, featured snippet, People Also Ask, -# related searches, ads, knowledge panel) — read INSTRUCTIONS.md before -# relying on this against Google at any volume. -# -# ─── Build (the base image must already exist, locally or in ECR) ──────────── -# docker buildx build --platform linux/arm64 --provenance=false --sbom=false \ -# -t bytelyon-serp-scraper:arm64 --load . -# -# docker run --rm -p 9000:8080 bytelyon-serp-scraper:arm64 -# curl -XPOST http://localhost:9000/2015-03-31/functions/function/invocations \ -# -d '{"query":"openai"}' -# -# --provenance=false --sbom=false is required: without it, buildx emits a -# manifest-list/attestation image that Lambda's CreateFunction/UpdateFunctionCode -# rejects with "image manifest, config or layer media type ... is not supported". -# -# To build against a locally-loaded (not-yet-pushed) base image instead of -# pulling from ECR, see ../base-image/INSTRUCTIONS.md: -# --build-arg BASE_IMAGE=bytelyon-cloakbrowser-base:arm64 - -ARG BASE_IMAGE=138305277395.dkr.ecr.us-east-1.amazonaws.com/bytelyon-cloakbrowser-base:latest -FROM ${BASE_IMAGE} - -# Handler sits at /app (already on Python's import path, WORKDIR=/app), -# imports cloakbrowser as a normal library. ENTRYPOINT/CMD are inherited from -# the base image (both already correct for this handler's module name). -COPY lambda_handler.py /app/lambda_handler.py diff --git a/docker/lambda/serp-scraper/INSTRUCTIONS.md b/docker/lambda/serp-scraper/INSTRUCTIONS.md deleted file mode 100644 index b02d5e1..0000000 --- a/docker/lambda/serp-scraper/INSTRUCTIONS.md +++ /dev/null @@ -1,215 +0,0 @@ -# CloakBrowser Google SERP scraper on AWS Lambda - -Runs a Google search and parses the results page into a JSON object grouped by SERP section (organic results, featured snippet, People Also Ask, related searches, ads, knowledge panel), using CloakBrowser's stealth Chromium inside an AWS Lambda function (container image package type). - -This is a sibling of [`../news-scraper`](../news-scraper), [`../page-scraper`](../page-scraper), [`../scraper`](../scraper), and [`../site-crawler`](../site-crawler) — all five build `FROM` the shared [`../base-image`](../base-image), which carries the Dockerfile/entrypoint scaffolding derived from the official `cloakhq/cloakbrowser` Docker Hub image that used to be duplicated in every handler's own `Dockerfile`. Every other invocation surface from the canonical image (`python`, `cloakserve`, `cloaktest`, `node`, `bash`, examples) keeps working. - -## Read this before using it against Google at any volume - -1. **Google IP-reputation-blocks automated traffic, independent of browser fingerprinting.** Requests from datacenter IPs — including AWS Lambda's own IP ranges — are commonly served a "Our systems have detected unusual traffic from your computer network" interstitial instead of a results page. This was confirmed while building this example: identical requests from a shared/cloud egress IP were blocked outright, with no captcha to solve — a hard IP-based block. CloakBrowser's stealth Chromium (fingerprint patches, humanize) does not get you past this on its own; it addresses bot _detection_, not IP _reputation_. Use the `proxy` event field with a quality residential/mobile proxy for reliable results. -2. Because of (1), this handler was built and tested **structurally** rather than against a live, verified-working SERP capture — see "How the parser works" below. Treat the section-extraction logic as best-effort, and expect to revisit it as you observe real results (or as Google's markup shifts). -3. Google's SERP markup is unversioned, changes without notice, and leans on obfuscated/rotating class names. There is no long-term-stable public API for this — that's exactly why the parser favors structural patterns (e.g. "an `

` near a link" = an organic result) over hardcoded class names, but it's inherently fragile compared to a real API. -4. **Scraping Google's SERPs may be subject to Google's Terms of Service.** That's a legal/policy call for your use case — this example doesn't make it for you. - -## A note on proxy credentials - -Chromium's underlying `--proxy-server` flag has no concept of credentials embedded in a URL. If you pass `"proxy": "http://user:pass@host:port"`, the handler automatically splits the credentials into separate `username`/`password` fields before launching (see `_normalize_proxy` in `lambda_handler.py`, which returns a `cloakbrowser.ProxySettings` dict) — this is what actually makes proxy authentication work. Without that split, Chromium connects to the proxy, silently drops the embedded credentials, gets a `407 Proxy Authentication Required` response, and the navigation hangs with no clear error until the Lambda function times out. If you already have a `ProxySettings`-shaped dict (`{"server", "username", "password", "bypass"}`), pass it as-is — it's used unchanged. Dict form is also the only way to set `bypass` or use a non-`http` proxy scheme (e.g. `socks5://`). - -**Don't set `bypass` to `.google.com` (or anything matching it).** `bypass` tells Chromium which hosts should skip the proxy and connect _directly_ instead — the opposite of what you want here, since the whole point of `proxy` in this handler is to route the Google search request through it. Bypassing google.com sends that request straight from Lambda's own datacenter IP, which is exactly the IP-reputation block described above. Only use `bypass` for other hosts you specifically want excluded from the proxy. - -A fast TCP preflight (`_preflight_proxy`, ~8s timeout) runs before every browser launch and raises a clear error if nothing is listening on the proxy's host:port, instead of silently hanging for the full function timeout. It only checks that a TCP connection can be opened — it does not attempt authentication or a CONNECT tunnel, so a pass here doesn't guarantee the proxy will actually route traffic. - -## Retries on a bad proxy connection - -Rotating residential/mobile proxies typically hand out a fresh exit IP per TCP connection, with no session stickiness. At any given moment some fraction of exit peers in a shared pool are already rate-limited or blocked by Google — this was confirmed empirically: a small back-to-back sample of plain HTTP requests through a real residential proxy came back 3x `200`, 1x `429`, 1x `403` straight from Google, with the proxy connection itself succeeding every single time in under 1.5s (no proxy-side blocking or slowness at all). A full browser session opens far more concurrent connections than a single request, so it's proportionally more likely to land at least one bad peer — usually surfacing as a hard `page.goto` failure (`net::ERR_CONNECTION_CLOSED`, `net::ERR_TIMED_OUT`, etc.) rather than a slow load. - -On a hard navigation failure, the handler closes the browser and relaunches a fresh one — a new browser means a new proxy connection, which means a new (hopefully clean) exit IP — up to `goto_retry_attempts` (default `3`) total attempts before giving up and raising. A `PlaywrightTimeoutError` (page loaded, just slowly) does **not** count as a failure and is not retried; only a hard connection-level error does. A CAPTCHA/block page is also retried (not just hard connection failures) for the same reason — see `_is_blocked` usage in `lambda_handler.py`. - -## Oxylabs: skip country targeting - -`App\Services\LambdaService::proxyUsername()` normalizes the `customer-` username prefix Oxylabs' targeting suffixes require, and `lambda_handler.py`'s `_oxylabs_sticky_proxy()` appends a fresh `-sessid-` per retry attempt (see its docstring for why that has to happen per-attempt, not once per invocation). Deliberately **not** included: a `-cc-` suffix. It's tempting — tighter `geoip` timezone/locale matching — but confirmed empirically to make things measurably worse: several consecutive Google blocks in a row with `-cc-US` set, across different exit IPs and different queries (ruling out a single bad IP or query-specific rate-limiting), immediately followed by a clean success the moment `-cc-US` was removed and the plain username used instead. Whatever sub-pool Oxylabs routes `-cc-US` traffic through, it was smaller/more commonly flagged than the default pool at the time this was tested. Re-verify carefully (small sample sizes here, and proxy pool quality can drift over time) before reintroducing country targeting. - -## Files in this directory - -| File | Purpose | -| ------------------- | ------------------------------------------------------------------------------------------------------------------------------------------------ | -| `Dockerfile` | `FROM` the shared `../base-image`, plus a single `COPY lambda_handler.py`. See `../base-image/INSTRUCTIONS.md` for what the base image provides. | -| `lambda_handler.py` | Search + parse handler. Takes `{query, ...}`, returns a JSON object keyed by SERP section. Headed by default (see below). | -| `INSTRUCTIONS.md` | This file. | - -This directory is no longer fully standalone: it depends on the shared base image (`../base-image`) having been built and either published to ECR or loaded locally first — see `../base-image/INSTRUCTIONS.md`. Everything specific to this handler (its Dockerfile and `lambda_handler.py`) still lives entirely in this directory. - -## Build - -Build and publish `../base-image` first (see `../base-image/INSTRUCTIONS.md`), then: - -```bash -docker buildx build --platform linux/arm64 --provenance=false --sbom=false \ - -t bytelyon-serp-scraper:arm64 --load . -``` - -`--provenance=false --sbom=false` is required — without it, buildx produces a manifest-list/attestation image that Lambda's `CreateFunction` rejects with `InvalidParameterValueException: ... image manifest, config or layer media type ... is not supported`. - -To build against a locally-loaded (not-yet-pushed) base image instead of pulling from ECR, add `--build-arg BASE_IMAGE=bytelyon-cloakbrowser-base:arm64`. For x86_64, switch `--platform linux/amd64` (and build the base image for the same architecture). - -## Local smoke test (no AWS account needed) - -```bash -docker run --rm -p 9000:8080 bytelyon-serp-scraper:arm64 - -# In another shell: -curl -sS -XPOST "http://localhost:9000/2015-03-31/functions/function/invocations" \ - -d '{"query":"openai"}' - -# With a proxy (recommended — see above), string form: -curl -sS -XPOST "http://localhost:9000/2015-03-31/functions/function/invocations" \ - -d '{"query":"openai","proxy":"http://user:pass@proxy-host:port"}' - -# ...or dict form (required for non-http proxy schemes, e.g. socks5, and to -# set `bypass`; also what the string form above is normalized into internally). -# Do NOT set `bypass` to ".google.com" here — see "A note on proxy credentials" above: -curl -sS -XPOST "http://localhost:9000/2015-03-31/functions/function/invocations" \ - -d '{"query":"openai","proxy":{"server":"socks5://proxy-host:port","username":"user","password":"pass"}}' -``` - -Other invocation surfaces stay intact (these match the canonical CloakHQ image): - -```bash -docker run --rm -it bytelyon-serp-scraper:arm64 python # REPL -docker run --rm bytelyon-serp-scraper:arm64 python examples/basic.py # examples -docker run --rm -p 9222:9222 bytelyon-serp-scraper:arm64 cloakserve --port=9222 # CDP server -docker run --rm bytelyon-serp-scraper:arm64 cloaktest # stealth tests -docker run --rm -it bytelyon-serp-scraper:arm64 node # JS wrapper -``` - -## Event schema - -| Field | Type | Default | -| -------------------------------- | --------------------- | ------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------ | -| `query` | str | required — the search query | -| `hl` | str | `"en"` — Google UI language | -| `gl` | str | `"us"` — Google country | -| `num` | int | `10` — requested result count (Google may ignore or cap this) | -| `proxy` | str / `ProxySettings` | none — **strongly recommended**, see above. Either `"http://user:pass@host:port"` (credentials are split out into `username`/`password` automatically — see "A note on proxy credentials" below) or a `cloakbrowser.ProxySettings` dict: `{"server": "socks5://host:port", "username": ..., "password": ...}`. Do **not** set `bypass` to `.google.com` — see "A note on proxy credentials" above. | -| `geoip` | bool | `false` — auto-derive timezone/locale from the proxy's exit IP; only meaningful when `proxy` is set. A timezone/locale that doesn't match the proxy's geolocation is itself a detection signal. | -| `headless` | bool | `false` — headed via Xvfb, baked into the base image (`:99`, `DISPLAY=:99`). CloakBrowser's own guidance: "most blocks come from missing" a residential proxy, `geoip`, or headed mode — "some sites detect headless even with our C++ patches." | -| `humanize` | bool | `true` | -| `human_preset` | str | `"careful"` | -| `goto_timeout_ms` | int | `30000` | -| `goto_retry_attempts` | int | `3` — rotating residential/mobile proxies hand out a fresh exit IP per connection with no stickiness, and any individual peer may already be rate-limited/blocked by Google independent of this proxy/account (confirmed empirically — see below). On a hard navigation failure the browser is closed and relaunched — a fresh proxy connection means a fresh exit IP — up to this many attempts before giving up. | -| `wait_for_load_state_timeout_ms` | int | `5000` | -| `url_resolve_timeout_ms` | int | `10000` — per-link timeout when resolving a result/product link to its final destination URL | -| `bucket` | str | `"bytelyon-private"` — S3 bucket for the saved page content + screenshot | -| `prefix` | str | `"serp-scrapes/{aws_request_id}/"` — S3 key prefix for both uploads | - -### Response - -```json -{ - "query": "openai", - "url": "https://www.google.com/search?q=openai&hl=en&gl=us", - "screenshot_key": "serp-scrapes//openai-.png", - "content_key": "serp-scrapes//openai-.html", - "data": { - "sponsored_results": [ - { - "kind": "sponsored_result", - "index": 0, - "title": "...", - "brand": "...", - "url": "https://...", - "domain": "example.com" - } - ], - "sponsored_products": [ - { - "kind": "sponsored_product", - "index": 0, - "domain": "example.com", - "url": "https://...", - "image": "https://...", - "title": "...", - "price": "$19.99", - "brand": "..." - } - ], - "organic_results": [ - { - "kind": "organic_result", - "index": 0, - "title": "...", - "url": "https://...", - "domain": "example.com" - } - ], - "organic_products": [ - { - "kind": "organic_product", - "index": 0, - "title": "...", - "image": "https://..." - } - ], - "similar_queries": [ - { "kind": "similar_query", "index": 0, "value": "..." } - ] - } -} -``` - -`data` groups the parsed SERP sections: `sponsored_results`, `sponsored_products`, `organic_results`, `organic_products`, and `similar_queries`. Each item's `url`/`domain` (where present) reflect the **final destination** the link resolves to, not Google's redirect/tracking link — see [How the parser works](#how-the-parser-works). Any section that fails to parse (e.g. an unexpected markup change) degrades independently to `[]` rather than failing the whole request. - -### Saved artifacts (S3) - -Every invocation uploads two objects to `bucket`/`prefix` under the same key stem (sanitized query + a short hash of the final URL): a full-page `.png` screenshot (`screenshot_key`) and the raw `.html` page content (`content_key`). This happens **unconditionally** — for real results and for blocked/CAPTCHA pages alike — since the block page itself is the most useful artifact for diagnosing _why_ a given proxy/session got flagged. - -### Blocked responses - -If Google serves its "unusual traffic" interstitial instead of a results page, the handler does **not** raise and does **not** change the response shape — it logs a warning to CloudWatch (`Google served a block/CAPTCHA page for query=...`) and still returns `content_key`/`screenshot_key` pointing at the saved interstitial page, plus an all-empty `data` (parsing isn't attempted against a block page), so you can inspect _why_ a given proxy/session got blocked without having to parse a Lambda error payload. - -## How the parser works - -Rather than hardcoding Google's current (obfuscated, rotating) class names, `_DATA_EXTRACT_JS` in `lambda_handler.py` looks for long-lived structural hooks: - -- **Sponsored results**: elements matching `[data-pcu]`; the title is the first ``'s text, the brand is the second ``'s text truncated before the first `https://` substring (Google often renders "Brand https://displayed-url.com" as one string), and the link comes from the element's own `href`. -- **Organic results**: `h3[id]` elements; the title is the heading text and the link comes from the heading's parent `href` (Google wraps organic headings directly in the result's anchor). -- **Organic products**: `product-viewer-entrypoint` custom elements; the title comes from the inner `
`'s `aria-label`, falling back to the ``'s `alt` text when that's empty, and the image is the ``'s `src`. -- **Sponsored products**: elements matching `[data-dtld]` (the attribute itself is the pre-resolved domain); within `div.pla-unit-container`, the link/image come from `a.pla-unit-img-container-link`, and title/price/brand come from `div[data-call_grow_wiz_event='true']`, `div[aria-label]`, and `span[role='text']` respectively. -- **Similar (related) queries**: `data-q` attributes on `div[data-notify-expansion]` elements, plus link text inside `div#botstuff`; entries under 5 characters are filtered out (matches common noise like stray punctuation). - -Each section is parsed independently and falls back to `[]` on failure rather than failing the whole extraction. Links (`href`s) captured by the JS above are frequently `/url?q=...`-style Google redirects rather than the destination itself, so `_final_url()` resolves each one to its final destination by issuing a real HTTP request through the same browser context/proxy as the search, with a small in-memory cache (`_URL_CACHE`, ~30 minute TTL) to avoid redundant lookups for links/destinations that repeat within or across nearby invocations. - -A cookie-consent interstitial (shown for some EU-ish exit IPs regardless of `hl`/`gl`) is best-effort dismissed before parsing, by clicking `#L2AGLb` ("I agree") or an "Accept all"/"I agree" button if present. - -## IAM permissions - -In addition to `AWSLambdaBasicExecutionRole` (CloudWatch Logs), the execution role needs write access to the target bucket: - -```json -{ - "Version": "2012-10-17", - "Statement": [ - { - "Effect": "Allow", - "Action": ["s3:PutObject"], - "Resource": ["arn:aws:s3:::bytelyon-private/*"] - } - ] -} -``` - -## Function configuration recommendations - -| Setting | Value | Why | -| -------------------------- | ----------------------------------------------------------------- | ---------------------------------------------------------------------------------------------------------------------------- | -| Package type | Image | Required — this is a container image, not a zip. | -| Architecture | `arm64` | Roughly 20% cheaper than x86_64. Match the architecture you built for. | -| Memory | 3008 MB | Memory in Lambda is tied to vCPU. Below ~1769 MB Chromium starts noticeably slower. | -| Timeout | 60-90 s | Single search + parse is fast once loaded (a few seconds warm); budget for cold start (~80-90 s) plus proxy latency if used. | -| Ephemeral storage (`/tmp`) | 512 MB (default) | The screenshot and page content are streamed directly to S3, not accumulated on disk. | -| Networking | Default (no VPC) | Binary is baked in; only outbound HTTPS to Google, your proxy (if any), and S3 is needed. | -| Execution role | Basic execution + S3 `PutObject` on the target bucket (see above) | | - -## Cold start - -Same profile as the sibling examples: first invocation in a new container takes ~80–90 s. Subsequent warm invocations are much faster. diff --git a/docker/lambda/serp-scraper/lambda_handler.py b/docker/lambda/serp-scraper/lambda_handler.py deleted file mode 100644 index 1c27f09..0000000 --- a/docker/lambda/serp-scraper/lambda_handler.py +++ /dev/null @@ -1,850 +0,0 @@ -"""AWS Lambda handler: Google search + SERP parsing by section. - -Runs a single Google search and returns a JSON object with the results -grouped by section under `data`: sponsored results, sponsored products, -organic results, organic products, and similar (related) queries. The raw -page HTML and a full-page screenshot are always saved to S3 too. - -Uses the **sync** cloakbrowser/Playwright API (`cloakbrowser.launch`), same -as ../news-scraper and ../aws_lambda_site_crawler. - -IMPORTANT — read before using this against Google at any volume: - - 1. Google aggressively IP-reputation-blocks automated traffic, independent - of browser fingerprinting. Requests from datacenter IPs (including AWS - Lambda's own IP ranges) are commonly served a "Our systems have - detected unusual traffic from your computer network" interstitial - instead of a results page — CloakBrowser's stealth Chromium does not - get you past this on its own. Use the `proxy` event field with a - quality residential/mobile proxy for reliable results. This handler - detects the block page and logs a warning, but does not fail the - invocation — the saved HTML/screenshot are the best way to confirm - whether a given call was blocked, and `data` is returned all-empty - rather than attempting to parse a block/CAPTCHA page. - 2. Google's SERP markup is unversioned, changes without notice, and uses - largely obfuscated/rotating class names. The extraction in - `_DATA_EXTRACT_JS` is deliberately **structural** (looks for long-lived - attributes/tag names like `[data-pcu]` or `h3[id]` rather than hardcoded - class names) so it degrades gracefully as markup shifts, but it is - inherently best-effort. Expect to revisit it over time. - 3. Result/product links are resolved to their final destination URL with a - real HTTP request per link (see `_final_url`), through the same browser - context/proxy as the search itself. This adds latency proportional to - the number of results on the page; a small in-memory cache - (`_URL_CACHE`) avoids redundant lookups for repeated links. - 4. Scraping Google's SERPs may be subject to Google's Terms of Service. - That's a legal/policy call for you to make for your use case — this - example doesn't make it for you. - -Event schema: - query str required — the search query - hl str "en" — Google UI language - gl str "us" — Google country - num int 10 — requested result count (Google may ignore/cap this) - proxy str|ProxySettings none — strongly recommended, see above. - Either "http://user:pass@host:port" (credentials - are split out automatically — see _normalize_proxy) - or a cloakbrowser/Playwright `ProxySettings` dict: - {"server": "socks5://host:port", "username": ..., - "password": ...}. Do NOT set `bypass` to - ".google.com" (or anything matching it) here — this - handler's entire point is to route google.com traffic - *through* the proxy; `bypass` makes matching hosts skip - the proxy and connect directly instead, which sends the - actual search request straight from Lambda's own - datacenter IP — precisely the IP-reputation block in (1) - above. `bypass` is only useful for *other* hosts you want - to exclude from the proxy. - geoip bool false — auto-derive timezone/locale from the - proxy's exit IP (only meaningful with `proxy` set) - bucket str "bytelyon-private" — S3 bucket for page - content + screenshot - prefix str "serp-scrapes/{aws_request_id}/" — S3 key - prefix for both uploads - headless bool false — headed via Xvfb (:99, baked into - the base image); CloakBrowser's own guidance is - that some sites detect headless even with the - stealth patches - humanize bool true - human_preset str "careful" - goto_timeout_ms int 30000 - goto_retry_attempts int 3 — rotating residential/mobile - proxies hand out a fresh exit IP per - connection with no stickiness, and any - individual peer may already be rate- - limited/blocked by Google independent of - this proxy/account. On a hard navigation - failure (e.g. net::ERR_CONNECTION_CLOSED, - net::ERR_TIMED_OUT) the browser is closed - and relaunched — a fresh proxy connection - means a fresh exit IP — up to this many - attempts total before giving up. - wait_for_load_state_timeout_ms int 5000 - url_resolve_timeout_ms int 10000 — per-link timeout when resolving - result/product links to their final destination - -Returns: - { - "query": "...", - "url": "https://www.google.com/search?q=...", - "screenshot_key": "serp-scrapes//openai-.png", - "content_key": "serp-scrapes//openai-.html", - "data": { - "sponsored_results": [ - {"kind": "sponsored_result", "index": 0, "title": "...", "brand": "...", "url": "...", "domain": "..."} - ], - "sponsored_products": [ - {"kind": "sponsored_product", "index": 0, "domain": "...", "url": "...", "image": "...", "title": "...", "price": "...", "brand": "..."} - ], - "organic_results": [ - {"kind": "organic_result", "index": 0, "title": "...", "url": "...", "domain": "..."} - ], - "organic_products": [ - {"kind": "organic_product", "index": 0, "title": "...", "image": "..."} - ], - "similar_queries": [ - {"kind": "similar_query", "index": 0, "value": "..."} - ] - } - } - -Every section in `data` degrades independently to `[]` on a parsing failure -(e.g. an unexpected markup change), and the whole `data` object is all-empty -when the page was blocked or extraction failed outright — see `_extract_data` -and `_empty_data`. - -The full-page screenshot and the raw page HTML are always uploaded to S3 -(same key stem, `.png` / `.html` extensions) regardless of whether Google -served real results or a block/CAPTCHA page — the latter is deliberately -still saved since it's the most useful artifact for diagnosing *why* a given -proxy/session got blocked. Both `screenshot_key` and `content_key` are -returned in the response. - -Lambda-specific additions (see ../aws_lambda/INSTRUCTIONS.md for the same -rationale in the sibling handlers): - - `--disable-dev-shm-usage` / `--no-zygote` Chromium flags. - - `query` is URL-encoded and appended to a fixed `https://www.google.com/search` - base — there's no SSRF surface here since the destination host isn't - caller-controlled (unlike the other handlers, which navigate to - caller-supplied URLs). -""" - -from __future__ import annotations - -import hashlib -import logging -import re -import secrets -import socket -import time -from typing import Any -from urllib.parse import unquote, urlencode, urlparse - -import boto3 # pyright: ignore[reportMissingImports] -from cloakbrowser import ProxySettings, launch # pyright: ignore[reportMissingImports] -from playwright.sync_api import Page # pyright: ignore[reportMissingImports] -from playwright.sync_api import ( # pyright: ignore[reportMissingImports] - TimeoutError as PlaywrightTimeoutError, -) - -logger = logging.getLogger("cloakbrowser.lambda") -logger.setLevel(logging.INFO) - -# awslambdaric wires the root logger with a formatter like -# "[INFO]\t2024-01-15T10:30:45.123Z\t\tmessage" — CloudWatch already -# timestamps every event server-side, so repeating timestamp + request id on -# every line just burns terminal columns. Trim to level + message. -for _h in logging.getLogger().handlers: - _h.setFormatter(logging.Formatter("%(levelname)s %(message)s")) - -DEFAULT_BUCKET = "bytelyon-private" - -# How long to wait for a raw TCP handshake with the proxy before giving up. -# `cloakbrowser.launch(proxy=...)` has no timeout of its own — if the proxy -# silently drops the connection (common for residential-proxy providers that -# block/throttle datacenter-origin traffic, e.g. AWS Lambda's egress IPs) the -# whole invocation would otherwise hang until Lambda's function timeout, -# burning the full billed duration with no diagnostic signal. This preflight -# fails fast with a clear, actionable error instead. -_PROXY_PREFLIGHT_TIMEOUT_S = 8.0 - - -def _normalize_proxy(proxy: Any) -> ProxySettings | None: - """Normalize a `proxy` event value into cloakbrowser/Playwright's - `ProxySettings` shape: {"server": "://:", "bypass"?, - "username"?, "password"?}. - - This matters more than it looks: Chromium's underlying `--proxy-server` - flag has no concept of credentials embedded in a URL. A caller-supplied - string like "http://user:pass@host:port" connects to the proxy just fine - but never authenticates — Chromium silently drops the embedded - credentials, the proxy responds 407 Proxy Authentication Required, and - nothing in the stack retries with credentials or surfaces an error, so - the navigation hangs until Playwright/Lambda's own timeout. Splitting - `username`/`password` into their own fields is what makes Playwright - wire up its automatic proxy-auth handling. A dict already in that shape - (e.g. `{"server": "socks5://host:port", "username": ..., "password": ...}`) - is passed through unchanged. Don't set `bypass` to anything matching - google.com here — see the module docstring's `proxy` entry for why. - """ - if not proxy: - return None - if isinstance(proxy, dict): - return proxy - if not isinstance(proxy, str): - raise ValueError( - f"'proxy' must be a string or dict, got {type(proxy).__name__}" - ) - - parsed = urlparse(proxy) - if not parsed.hostname: - raise ValueError(f"'proxy' is not a valid proxy URL: {proxy!r}") - - server = f"{parsed.scheme}://{parsed.hostname}" - if parsed.port: - server += f":{parsed.port}" - - normalized: ProxySettings = ProxySettings(server=server) - if parsed.username: - normalized["username"] = unquote(parsed.username) - if parsed.password: - normalized["password"] = unquote(parsed.password) - return normalized - - -def _oxylabs_sticky_proxy(proxy: ProxySettings | None) -> ProxySettings | None: - """Return a copy of `proxy` with a fresh Oxylabs sticky-session segment - appended to the username — call this once per browser launch (i.e. once - per retry attempt in `handler`, not once for the whole invocation). - - Oxylabs rotates to a new residential exit peer per TCP connection with no - stickiness by default (confirmed empirically), and a full browser session - opens far more connections per page load (main doc, JS, CSS, images, - consent dialog, ...) than a single request — proportionally more likely - to land at least one bad/flagged peer mid-page-load. `-sessid-` pins - every connection carrying that exact username to the same exit peer for - the session's lifetime (~10 min by default), so generating a fresh, - random id *per launch* keeps one attempt's connections consistent while - still letting each retry roll a new exit IP — the entire point of - retrying with a fresh browser/proxy connection in the first place. A - session id fixed for the whole invocation (e.g. assigned once PHP-side) - would instead pin every retry to the same IP and defeat that. - - Requires the `customer-` username prefix to take effect at all (confirmed - empirically — Oxylabs otherwise responds 407); auto-added if missing. - No-op for anything that isn't an Oxylabs proxy (host doesn't contain - "oxylabs") or has no username — other providers (e.g. DataImpulse's - `__cr.us` convention) use incompatible suffix syntax. - """ - if not proxy or not isinstance(proxy, dict): - return proxy - server = proxy.get("server") or "" - username = proxy.get("username") - if not username or "oxylabs" not in server.lower(): - return proxy - - base = username if username.startswith("customer-") else f"customer-{username}" - session_id = secrets.token_hex(4) - return {**proxy, "username": f"{base}-sessid-{session_id}"} - - -def _proxy_host_port(proxy: ProxySettings | None) -> tuple[str, int] | None: - """Best-effort (host, port) extraction from an already-normalized proxy - dict, for the TCP preflight below. Returns None (preflight skipped, not - fatal) if it can't be determined. - """ - if not proxy: - return None - server = proxy.get("server") - if not isinstance(server, str) or not server: - return None - parsed = urlparse(server if "://" in server else f"http://{server}") - if not parsed.hostname or not parsed.port: - return None - return parsed.hostname, parsed.port - - -def _preflight_proxy( - proxy: ProxySettings | None, timeout: float = _PROXY_PREFLIGHT_TIMEOUT_S -) -> None: - """Raw TCP reachability check only — deliberately does not attempt proxy - auth or a CONNECT tunnel, just confirms *something* is listening on - host:port so an unreachable/blocking proxy fails in ~8s instead of - hanging for the full function timeout. A pass here does not guarantee - the proxy will actually authenticate or tunnel traffic. - """ - host_port = _proxy_host_port(proxy) - if host_port is None: - return - host, port = host_port - started = time.monotonic() - try: - with socket.create_connection((host, port), timeout=timeout): - pass - except OSError as e: - raise RuntimeError( - f"Proxy preflight failed: could not open a TCP connection to " - f"{host}:{port} within {timeout}s ({e}). This is a network-level " - f"failure, not a credentials issue — it usually means the proxy " - f"provider is blocking or silently dropping connections from " - f"this network (e.g. AWS Lambda's egress IP ranges). Verify with " - f"the proxy provider whether cloud/datacenter source IPs are " - f"allowed on this gateway." - ) from e - logger.info( - "Proxy preflight OK: TCP connect to %s:%s succeeded in %.2fs", - host, - port, - time.monotonic() - started, - ) - - -_BLOCK_PHRASES = ( - "unusual traffic", - "our systems have detected", -) - -# Google's consent interstitial (shown for some EU-ish exit IPs regardless of -# hl/gl) — #L2AGLb ("I agree") has been a long-lived id for this button, but -# fall back to text matching since it's not guaranteed to still exist. -_CONSENT_SELECTORS = [ - "#L2AGLb", - "button:has-text('Accept all')", - "button:has-text('I agree')", -] - - -def _empty_data() -> dict: - """All-empty `data` shape, used when the page is blocked or extraction - fails outright, so callers can always rely on the same response shape. - """ - return { - "sponsored_results": [], - "sponsored_products": [], - "organic_results": [], - "organic_products": [], - "similar_queries": [], - } - - -# Structural, best-effort SERP parser: looks for long-lived structural hooks -# (attributes/tag names) rather than Google's obfuscated/rotating class -# names, so it degrades gracefully as markup shifts. Runs in the page via -# `page.evaluate` and returns raw values only (hrefs, text) — resolving a -# link to its final destination requires a real HTTP request, which happens -# afterwards in `_final_url` since that can't happen synchronously from -# within `page.evaluate`. -_DATA_EXTRACT_JS = r""" -() => { - const textOf = (el) => (el && el.textContent ? el.textContent.trim() : ""); - - const sponsoredResults = Array.from(document.querySelectorAll("[data-pcu]")).map((e, i) => { - const spans = e.querySelectorAll("span"); - const rawBrand = spans[1] ? textOf(spans[1]) : ""; - return { - index: i, - title: spans[0] ? textOf(spans[0]) : "", - brand: rawBrand.split("https://")[0].trim(), - href: e.getAttribute("href") || "", - }; - }); - - const organicProducts = Array.from(document.querySelectorAll("product-viewer-entrypoint")).map((e, i) => { - const div = e.querySelector("div"); - const img = e.querySelector("img"); - let title = div ? (div.getAttribute("aria-label") || "") : ""; - if (!title && img) title = img.getAttribute("alt") || ""; - return { - index: i, - title, - image: img ? (img.getAttribute("src") || "") : "", - }; - }); - - const sponsoredProducts = Array.from(document.querySelectorAll("[data-dtld]")).map((e, i) => { - const div = e.querySelector("div.pla-unit-container"); - const a = div ? div.querySelector("a.pla-unit-img-container-link") : null; - const titleEl = div ? div.querySelector("div[data-call_grow_wiz_event='true']") : null; - const priceEl = div ? div.querySelector("div[aria-label]") : null; - const brandEl = div ? div.querySelector("span[role='text']") : null; - const img = a ? a.querySelector("img") : null; - return { - index: i, - domain: e.getAttribute("data-dtld") || "", - href: a ? (a.getAttribute("href") || "") : "", - image: img ? (img.getAttribute("src") || "") : "", - title: textOf(titleEl), - price: priceEl ? (priceEl.getAttribute("aria-label") || "") : "", - brand: textOf(brandEl), - }; - }); - - const organicResults = Array.from(document.querySelectorAll("h3[id]")).map((e, i) => { - const parent = e.parentElement; - return { - index: i, - title: textOf(e), - href: parent ? (parent.getAttribute("href") || "") : "", - }; - }); - - const similarQueries = []; - document.querySelectorAll("div[data-notify-expansion]").forEach((e) => { - const q = e.getAttribute("data-q") || ""; - if (q.length > 4) similarQueries.push(q); - }); - const botstuff = document.querySelector("div#botstuff"); - if (botstuff) { - botstuff.querySelectorAll("a").forEach((a) => { - const t = textOf(a); - if (t.length > 4) similarQueries.push(t); - }); - } - - return { sponsoredResults, organicProducts, sponsoredProducts, organicResults, similarQueries }; -} -""" - -# In-memory cache of resolved destination URLs, shared across invocations in -# the same warm Lambda container. Google's organic/sponsored result links -# are frequently `/url?q=...`-style redirects rather than the destination -# itself; resolving one costs a real HTTP round-trip, and the same -# destination often repeats across a single SERP (or across nearby -# invocations against the same warm container), so caching meaningfully -# cuts down on redundant requests. -_URL_CACHE: dict[str, tuple[float, str]] = {} -_URL_CACHE_TTL_S = 1800 # 30 minutes - - -def _to_domain(url: str) -> str: - host = urlparse(url).hostname or "" - return host.removeprefix("www.") - - -def _final_url(page: Page, link: str, timeout_ms: int) -> str: - """Resolve a (possibly relative/redirect) Google result link to its final - destination URL by actually requesting it, through the same browser - context (and therefore the same proxy/session) as the search itself. - Falls back to the normalized-but-unresolved link on any failure. - """ - if not link: - return link - if not link.startswith("https://www.google.com"): - if not link.startswith("/"): - link = f"/{link}" - link = f"https://www.google.com{link}" - - cache_key = hashlib.sha1(link.encode()).hexdigest() - now = time.monotonic() - cached = _URL_CACHE.get(cache_key) - if cached and cached[0] > now: - return cached[1] - - try: - final_url = page.request.get(link, timeout=timeout_ms).url - except Exception: - return link - - expires_at = now + _URL_CACHE_TTL_S - _URL_CACHE[cache_key] = (expires_at, final_url) - _URL_CACHE[hashlib.sha1(final_url.encode()).hexdigest()] = (expires_at, final_url) - return final_url - - -def _extract_data(page: Page, url_resolve_timeout_ms: int) -> dict: - """Best-effort structural SERP parser. - - Runs `_DATA_EXTRACT_JS` in the page to pull raw text/attributes, then - resolves each result's link to its final destination URL. Each section - is independently guarded — a failure in one (e.g. an unexpected markup - change) falls back to an empty list for that section rather than - failing the whole extraction. - """ - try: - raw = page.evaluate(_DATA_EXTRACT_JS) - except Exception: - logger.exception("SERP data extraction failed") - return _empty_data() - - def resolve(link: str) -> str: - return _final_url(page, link, url_resolve_timeout_ms) - - sponsored_results: list[dict] = [] - try: - for item in raw.get("sponsoredResults", []): - url = resolve(item.get("href", "")) - sponsored_results.append( - { - "kind": "sponsored_result", - "index": item["index"], - "title": item.get("title", ""), - "brand": item.get("brand", ""), - "url": url, - "domain": _to_domain(url), - } - ) - except Exception: - logger.exception("sponsored_results extraction failed") - sponsored_results = [] - - organic_products: list[dict] = [] - try: - organic_products = [ - { - "kind": "organic_product", - "index": item["index"], - "title": item.get("title", ""), - "image": item.get("image", ""), - } - for item in raw.get("organicProducts", []) - ] - except Exception: - logger.exception("organic_products extraction failed") - organic_products = [] - - sponsored_products: list[dict] = [] - try: - for item in raw.get("sponsoredProducts", []): - url = resolve(item.get("href", "")) - sponsored_products.append( - { - "kind": "sponsored_product", - "index": item["index"], - "domain": item.get("domain", ""), - "url": url, - "image": item.get("image", ""), - "title": item.get("title", ""), - "price": item.get("price", ""), - "brand": item.get("brand", ""), - } - ) - except Exception: - logger.exception("sponsored_products extraction failed") - sponsored_products = [] - - organic_results: list[dict] = [] - try: - for item in raw.get("organicResults", []): - url = resolve(item.get("href", "")) - organic_results.append( - { - "kind": "organic_result", - "index": item["index"], - "title": item.get("title", ""), - "url": url, - "domain": _to_domain(url), - } - ) - except Exception: - logger.exception("organic_results extraction failed") - organic_results = [] - - similar_queries: list[dict] = [] - try: - similar_queries = [ - {"kind": "similar_query", "index": i, "value": value} - for i, value in enumerate(raw.get("similarQueries", [])) - ] - except Exception: - logger.exception("similar_queries extraction failed") - similar_queries = [] - - return { - "sponsored_results": sponsored_results, - "sponsored_products": sponsored_products, - "organic_results": organic_results, - "organic_products": organic_products, - "similar_queries": similar_queries, - } - - -def _dismiss_consent(page: Page) -> None: - """Best-effort click-through for Google's cookie-consent interstitial.""" - for selector in _CONSENT_SELECTORS: - try: - locator = page.locator(selector).first - if locator.is_visible(timeout=1_000): - locator.click(timeout=2_000) - page.wait_for_load_state("domcontentloaded", timeout=5_000) - return - except Exception: - continue - - -def _is_blocked(page: Page, html: str) -> bool: - """Checks the already-captured page source rather than issuing a second, - separate live DOM query (`page.locator(...).inner_text()`). That query - used to run independently of — and noticeably later than — the html/png - capture, and confirmed empirically to sometimes race: a real CAPTCHA page - (verified from the saved screenshot, with the block text plainly visible) - was still missed here because the live query happened to fail or return - stale/partial text, silently swallowed by the old `except: return False`. - Checking the exact html string already being saved/returned removes that - second query — and the race — entirely. - """ - if "/sorry/" in page.url: - return True - return any(phrase in html.lower() for phrase in _BLOCK_PHRASES) - - -def _s3_key(prefix: str, query: str, url: str, ext: str) -> str: - safe_query = ( - re.sub(r"[^a-zA-Z0-9_-]+", "_", query.strip()).strip("_")[:80] or "query" - ) - digest = hashlib.sha256(url.encode()).hexdigest()[:10] - return f"{prefix.rstrip('/')}/{safe_query}-{digest}.{ext}" - - -def handler(event: dict, context: Any) -> dict: - query = event.get("query") - if not isinstance(query, str) or not query.strip(): - raise ValueError("event must include a non-empty 'query' string") - - params = {"q": query, "hl": event.get("hl", "en"), "gl": event.get("gl", "us")} - if "num" in event: - params["num"] = event["num"] - search_url = "https://www.google.com/search?" + urlencode(params) - - bucket = event.get("bucket", DEFAULT_BUCKET) - prefix = event.get( - "prefix", f"serp-scrapes/{getattr(context, 'aws_request_id', 'local')}/" - ) - s3 = boto3.client("s3") - - proxy: ProxySettings | None = _normalize_proxy(event.get("proxy")) - _preflight_proxy(proxy) - - geoip = event.get("geoip", False) - launch_kwargs: dict[str, Any] = { - # CloakBrowser's own "recommended config for anti-bot sites" is - # explicit: "most blocks come from missing one of these three things" - # — residential proxy, geoip, and *headed* mode — "not from browser - # fingerprint detection": "some sites detect headless even with our - # C++ patches." We already had the first two; defaulting to headed - # here closes the gap. Works in Lambda because the base image's - # entrypoint always starts Xvfb on :99 (with DISPLAY=:99 baked into - # the canonical image's own ENV) regardless of this setting. - "headless": event.get("headless", False), - "humanize": event.get("humanize", True), - "human_preset": event.get("human_preset", "careful"), - "args": [ - # Lambda /dev/shm is ~64 MB — Chromium crashes mid-render without this. - "--disable-dev-shm-usage", - # Lambda's restricted process model can't fork from Chromium's - # zygote — without this, child renderer processes fail to spawn. - "--no-zygote", - # Aligns font metrics with the spoofed Windows platform — without - # it, font metrics leak the real Linux host underneath, a signal - # FingerprintJS-style "browser tampering" checks pick up on. - # Chromium 148+ binary only (we're on Pro 151); needs Windows - # fonts installed to have any effect — see ../base-image/Dockerfile - # (real MS core fonts) and ../base-image/fonts/README.md (the rest, - # if you have licensed access) — harmless no-op without them. - "--fingerprint-windows-font-metrics", - ], - } - # `proxy` deliberately isn't in `launch_kwargs` above — it's computed fresh - # per launch (see `_launch_browser`) so Oxylabs sticky-session ids don't - # get reused across retries. - # Tried `--fingerprint-transparent-proxy` here (CloakBrowser Pro's own - # suggested fix for "works direct, blocks through a proxy") against - # DataImpulse specifically — empirically made things worse: all 3 retry - # attempts failed identically with "CloakBrowser Pro: couldn't verify your - # license (license server unreachable)" instead of the usual - # net::ERR_CONNECTION_CLOSED, suggesting it routes *all* Chromium traffic - # through the proxy, including CloakBrowser's own license heartbeat — - # compounding exposure to a flaky proxy rather than reducing it. Reverted. - # Revisit only alongside `--license-through-proxy` and a more reliable - # proxy pool, not as a fix for DataImpulse's own connection reliability. - - def _launch_browser(): - # Fresh Oxylabs sticky-session id every launch (no-op for other - # providers) — see `_oxylabs_sticky_proxy` for why this must happen - # per attempt rather than once for the whole invocation. - attempt_proxy = _oxylabs_sticky_proxy(proxy) - kwargs = {**launch_kwargs, "proxy": attempt_proxy} - try: - # Auto-derive timezone/locale from the proxy's exit IP so the browser - # fingerprint matches where the traffic is actually coming from — - # a timezone/locale mismatch against a residential proxy's geolocation - # is itself a detection signal. Only meaningful when `proxy` is set. - return launch(geoip=geoip, **kwargs) - except RuntimeError as e: - # cloakbrowser's own geoip lookup (routed through the proxy, to a - # third-party IP-geolocation service) has no retry/fallback of its - # own — a slow or blocking exit node turns into a hard failure of - # the *entire* invocation, even though geoip is purely cosmetic - # (timezone/locale matching) and the search itself doesn't need it. - # Degrade to no-geoip instead of failing outright. - if not geoip or "geoip" not in str(e).lower(): - raise - logger.warning( - "geoip resolution failed (%s) — retrying without geoip; " - "timezone/locale will not be auto-matched to the proxy's exit IP", - e, - ) - return launch(geoip=False, **kwargs) - - # Rotating residential/mobile proxies (the kind this handler needs — see - # "Read this before using it against Google at any volume" above) typically - # hand out a fresh exit IP per TCP connection, with no session stickiness. - # Any individual exit peer may already be rate-limited/blocked by Google — - # confirmed empirically: a small back-to-back sample through a real - # residential proxy came back 3x 200, 1x 429, 1x 403 straight from Google, - # with the proxy connection itself succeeding every time in under 1.5s. A - # full browser session opens far more concurrent connections than a single - # request, so it's proportionally more likely to land at least one bad - # peer — usually surfacing as a hard `page.goto` failure (e.g. - # net::ERR_CONNECTION_CLOSED, net::ERR_TIMED_OUT) rather than a slow load. - # Retrying with a *fresh* browser (not just a fresh page/goto on the same - # browser) is what actually gets a new proxy connection on the next try. - max_attempts = max(1, int(event.get("goto_retry_attempts", 3) or 1)) - goto_timeout_ms = event.get("goto_timeout_ms", 30_000) - - browser = None - page: Page | None = None - last_error: BaseException | None = None - html: str | None = None - png: bytes | None = None - blocked = False - for attempt in range(1, max_attempts + 1): - browser = _launch_browser() - page = browser.new_page() - nav_failed = False - try: - page.goto(search_url, timeout=goto_timeout_ms) - page.wait_for_load_state( - "domcontentloaded", - timeout=event.get("wait_for_load_state_timeout_ms", 5_000), - ) - except PlaywrightTimeoutError: - # Got *some* response, just slow to reach the requested load - # state — not a connection failure, so don't burn a retry on it. - pass - except Exception as e: - nav_failed = True - last_error = e - - # Consent dismissal clicks a real button (Google's "Accept all"), - # which itself triggers a follow-up navigation/request — on a fresh - # proxy connection, just as exposed to a bad exit peer as the initial - # `page.goto` above. Run it here, *inside* the retry loop, so a - # failure at this step also gets a fresh browser/proxy retry instead - # of silently clobbering an otherwise-good page after the loop exits. - if not nav_failed: - _dismiss_consent(page) - - # Chromium resolves *some* hard network failures (net::ERR_TIMED_OUT - # in particular, confirmed empirically — unlike net::ERR_CONNECTION_CLOSED, - # which raises) into a "successful" navigation to its own internal - # network-error interstitial rather than raising, so the except blocks - # above never fire and `page.goto` returns normally. Confirmed via a - # real capture that `page.url` still reports the *original* destination - # (https://www.google.com/...), not a chrome-error:// URL, so URL alone - # can't detect this — the interstitial's own DOM (a `main-frame-error` - # element, always present in Chromium's built-in error page template) - # is the reliable signal. Without this check, this page would sail - # through as if it were a real (if resultless) Google response. Checked - # again here (not just right after the initial goto) since the - # consent-dismissal navigation above can *introduce* this failure too. - if not nav_failed: - try: - nav_failed = 'id="main-frame-error"' in page.content() - except Exception: - pass - if nav_failed: - last_error = RuntimeError( - f"navigation resolved to a Chromium network-error page (url={page.url})" - ) - - # A CAPTCHA/block page is a "successful" navigation (no exception, no - # network-error interstitial) but useless for extraction. Each retry - # gets a fresh browser/proxy connection — on a rotating residential - # proxy that usually means a different exit IP — so it's often just as - # fixable as a hard connection failure. Confirmed empirically against - # Oxylabs: a clean connection on the very first attempt, just a - # CAPTCHA from that specific exit IP. Unlike a hard failure, still - # being blocked after the last attempt isn't fatal — the loop falls - # through and the last captured html/png/blocked state is used as-is. - # - # html/png are captured *here* (once per attempt, reused below — no - # second capture later) rather than checking `_is_blocked` on its own - # right after goto: confirmed empirically that a CAPTCHA page can pass - # an immediately-after-goto check as "not blocked" and only reveal - # itself a few hundred ms later, once something (e.g. a full-page - # screenshot forcing layout/render) gives it time to settle. Checking - # against the *exact* content being saved/returned avoids that gap. - blocked = False - if not nav_failed: - try: - html = page.content() - png = page.screenshot(full_page=True) - blocked = _is_blocked(page, html) - except Exception as e: - nav_failed = True - last_error = e - - if not nav_failed and not blocked: - break - if blocked and attempt == max_attempts: - break - - browser.close() - browser = None - page = None - if nav_failed and attempt == max_attempts: - raise last_error - logger.warning( - "goto attempt %d/%d %s — retrying with a fresh browser/proxy connection", - attempt, - max_attempts, - f"failed ({last_error})" - if nav_failed - else "hit a Google block/CAPTCHA page", - ) - - if browser is None or page is None or html is None or png is None: - # Unreachable in practice: the loop above always either `break`s with - # all four set, or `raise`s on the final attempt. - raise RuntimeError("failed to establish a browser session") - - try: - # Saved regardless of blocked/success outcome — useful for debugging - # block pages too (the last several rounds of proxy debugging on this - # handler would have been far faster with these on hand). Reuses the - # exact html/png/blocked captured inside the retry loop above rather - # than re-capturing/re-checking here — see the loop's own comment for - # why a second, later check previously disagreed with the first. - content_key = _s3_key(prefix, query, page.url, "html") - screenshot_key = _s3_key(prefix, query, page.url, "png") - s3.put_object( - Bucket=bucket, - Key=content_key, - Body=html.encode("utf-8"), - ContentType="text/html; charset=utf-8", - ) - s3.put_object( - Bucket=bucket, Key=screenshot_key, Body=png, ContentType="image/png" - ) - - if blocked: - logger.warning( - "Google served a block/CAPTCHA page for query=%r — this is an " - "IP-reputation block, not a fingerprinting issue; pass a " - "residential/mobile proxy via the 'proxy' event field.", - query, - ) - data = _empty_data() - else: - data = _extract_data( - page, url_resolve_timeout_ms=event.get("url_resolve_timeout_ms", 10_000) - ) - - return { - "query": query, - "screenshot_key": screenshot_key, - "content_key": content_key, - "data": data, - } - finally: - browser.close() diff --git a/docker/lambda/site-crawler/Dockerfile b/docker/lambda/site-crawler/Dockerfile deleted file mode 100644 index 7cc5375..0000000 --- a/docker/lambda/site-crawler/Dockerfile +++ /dev/null @@ -1,40 +0,0 @@ -# CloakBrowser site crawler on AWS Lambda. -# -# Built FROM the shared bytelyon-cloakbrowser-base image (see -# ../base-image/Dockerfile and ../base-image/INSTRUCTIONS.md), which carries -# every bit of common CloakBrowser + Lambda glue: awslambdaric, the Lambda -# Runtime Interface Emulator, boto3, the dual-mode entrypoint, the non-root -# readability fix, and the runtime ENV. This Dockerfile only adds this -# handler's own module — see lambda_handler.py. -# -# Sibling of ../news-scraper, ../page-scraper, ../scraper, and -# ../serp-scraper (same base image, different lambda_handler.py). This -# handler crawls a site breadth-first from one or more seed URLs, -# screenshots every page it visits, uploads each screenshot to S3, and -# returns the list of URLs it crawled. -# -# ─── Build (the base image must already exist, locally or in ECR) ──────────── -# docker buildx build --platform linux/arm64 --provenance=false --sbom=false \ -# -t bytelyon-site-crawler:arm64 --load . -# -# docker run --rm -p 9000:8080 \ -# -e AWS_ACCESS_KEY_ID -e AWS_SECRET_ACCESS_KEY -e AWS_SESSION_TOKEN -e AWS_REGION=us-east-1 \ -# bytelyon-site-crawler:arm64 -# curl -XPOST http://localhost:9000/2015-03-31/functions/function/invocations \ -# -d '{"url":"https://example.com","max_pages":5,"max_depth":1}' -# -# --provenance=false --sbom=false is required: without it, buildx emits a -# manifest-list/attestation image that Lambda's CreateFunction/UpdateFunctionCode -# rejects with "image manifest, config or layer media type ... is not supported". -# -# To build against a locally-loaded (not-yet-pushed) base image instead of -# pulling from ECR, see ../base-image/INSTRUCTIONS.md: -# --build-arg BASE_IMAGE=bytelyon-cloakbrowser-base:arm64 - -ARG BASE_IMAGE=138305277395.dkr.ecr.us-east-1.amazonaws.com/bytelyon-cloakbrowser-base:latest -FROM ${BASE_IMAGE} - -# Handler sits at /app (already on Python's import path, WORKDIR=/app), -# imports cloakbrowser as a normal library. ENTRYPOINT/CMD are inherited from -# the base image (both already correct for this handler's module name). -COPY lambda_handler.py /app/lambda_handler.py diff --git a/docker/lambda/site-crawler/INSTRUCTIONS.md b/docker/lambda/site-crawler/INSTRUCTIONS.md deleted file mode 100644 index 656e329..0000000 --- a/docker/lambda/site-crawler/INSTRUCTIONS.md +++ /dev/null @@ -1,143 +0,0 @@ -# CloakBrowser site crawler on AWS Lambda - -Breadth-first crawl of a site from one or more seed URLs, using CloakBrowser's stealth Chromium inside an AWS Lambda function (container image package type). Every page visited is screenshotted and the screenshot is uploaded to S3; the function returns the list of URLs it crawled. - -This is a sibling of [`../news-scraper`](../news-scraper), [`../page-scraper`](../page-scraper), [`../scraper`](../scraper), and [`../serp-scraper`](../serp-scraper) — all five build `FROM` the shared [`../base-image`](../base-image), which carries the Dockerfile/entrypoint scaffolding derived from the official `cloakhq/cloakbrowser` Docker Hub image (plus `boto3` for the S3 upload) that used to be duplicated in every handler's own `Dockerfile`. Every other invocation surface from the canonical image (`python`, `cloakserve`, `cloaktest`, `node`, `bash`, examples) keeps working. - -This document does not prescribe a deployment method — push the resulting image to ECR and create the Lambda function however you prefer (AWS CLI, CDK, Terraform, SAM, console, etc.). - -## Files in this directory - -| File | Purpose | -| ------------------- | ------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------- | -| `Dockerfile` | `FROM` the shared `../base-image`, plus a single `COPY lambda_handler.py`. See `../base-image/INSTRUCTIONS.md` for what the base image provides (including `boto3`, used here for the S3 upload). | -| `lambda_handler.py` | Crawl handler. Takes `{url or urls, bucket, ...}`, returns a list of crawled URLs. Headless by default. | -| `INSTRUCTIONS.md` | This file. | - -This directory is no longer fully standalone: it depends on the shared base image (`../base-image`) having been built and either published to ECR or loaded locally first — see `../base-image/INSTRUCTIONS.md`. Everything specific to this handler (its Dockerfile and `lambda_handler.py`) still lives entirely in this directory. - -## Build - -Build and publish `../base-image` first (see `../base-image/INSTRUCTIONS.md`), then: - -```bash -docker buildx build --platform linux/arm64 --provenance=false --sbom=false \ - -t bytelyon-site-crawler:arm64 --load . -``` - -`--provenance=false --sbom=false` is required — without it, buildx produces a manifest-list/attestation image that Lambda's `CreateFunction` rejects with `InvalidParameterValueException: ... image manifest, config or layer media type ... is not supported`. - -To build against a locally-loaded (not-yet-pushed) base image instead of pulling from ECR, add `--build-arg BASE_IMAGE=bytelyon-cloakbrowser-base:arm64`. For x86_64, switch `--platform linux/amd64` (and build the base image for the same architecture). - -## Local smoke test (no Lambda deploy needed, but real AWS credentials required) - -Screenshots are genuinely uploaded to S3 even in local RIE testing, so pass through real AWS credentials with S3 write access to the target bucket: - -```bash -docker run --rm -p 9000:8080 \ - -e AWS_ACCESS_KEY_ID -e AWS_SECRET_ACCESS_KEY -e AWS_SESSION_TOKEN -e AWS_REGION=us-east-1 \ - bytelyon-site-crawler:arm64 - -# In another shell: -curl -sS -XPOST "http://localhost:9000/2015-03-31/functions/function/invocations" \ - -d '{"url":"https://example.com","max_pages":5,"max_depth":1}' -``` - -Other invocation surfaces stay intact (these match the canonical CloakHQ image): - -```bash -docker run --rm -it bytelyon-site-crawler:arm64 python # REPL -docker run --rm bytelyon-site-crawler:arm64 python examples/basic.py # examples -docker run --rm -p 9222:9222 bytelyon-site-crawler:arm64 cloakserve --port=9222 # CDP server -docker run --rm bytelyon-site-crawler:arm64 cloaktest # stealth tests -docker run --rm -it bytelyon-site-crawler:arm64 node # JS wrapper -``` - -## Event schema - -Exactly one of `url` / `urls` is required. - -| Field | Type | Default | -| -------------------------------- | --------- | ------------------------------------------------------------------------------- | -| `url` | str | a single seed URL | -| `urls` | list[str] | one or more seed URLs, checked before `url` if both present | -| `bucket` | str | `"bytelyon-private"` | -| `prefix` | str | `"crawls/{aws_request_id}/"` — S3 key prefix for this invocation's screenshots | -| `max_pages` | int | `20` — hard cap on total pages visited across all seeds | -| `max_depth` | int | `2` — link-following depth from each seed (`0` = seeds only, no link-following) | -| `same_domain_only` | bool | `true` — only follow links whose hostname exactly matches the seed's hostname | -| `full_page_screenshot` | bool | `true` — capture the entire scrollable page, not just the viewport | -| `headless` | bool | `true` | -| `humanize` | bool | `true` | -| `human_preset` | str | `"careful"` | -| `goto_timeout_ms` | int | `30000` | -| `wait_for_load_state_timeout_ms` | int | `5000` | - -One browser + one page is launched per invocation and reused for every page in the crawl (sequential, not concurrent — simplest and cheapest for a single Lambda invocation). - -### Crawl algorithm - -Breadth-first from the seed(s): - -1. Dequeue the next URL (fragment-stripped for dedup); skip if already visited. -2. Validate its scheme + resolve its hostname, rejecting private/internal targets (see Security below); navigate, wait, then re-validate the (possibly redirected) final URL. -3. Screenshot the page and upload it to `s3://{bucket}/{prefix}{sanitized-host-and-path}-{sha256[:10]}.png`. -4. Record the final URL in the result list. -5. If under `max_depth`, extract every `` on the page, keep the ones that are same-scheme, unvisited, and (if `same_domain_only`) share the seed's exact hostname, and enqueue them at `depth + 1`. -6. Repeat until the queue is empty, `max_pages` is reached, or the invocation is running low on time (see below). - -### Response - -```json -[ - "https://example.com/", - "https://example.com/about", - "https://example.com/contact" -] -``` - -Screenshots are a side effect written to S3 — they are **not** included in the response payload (unlike `../aws_lambda`'s one-shot handler, which returns `screenshot_b64` inline). Fetch them from S3 using the same key scheme, or list `s3://{bucket}/{prefix}` after the invocation completes. - -## Lambda-specific additions - -- **Chromium hardening (do not remove)**: `--disable-dev-shm-usage` and `--no-zygote`, same as the sibling examples. -- **URL validation, applied continuously, not just at the seed**: every discovered link is scheme- and SSRF-checked before navigation, and the final (possibly redirected) URL is re-checked after navigation — the same defense-in-depth as `../aws_lambda`, but more important here because a crawl follows links found _on the pages it visits_, which is attacker-influenced input if the crawled site is untrusted or compromised. -- **Time-budget-aware crawling**: before starting each new page, the handler checks `context.get_remaining_time_in_millis()` and stops enqueuing further pages once fewer than 10 seconds remain, returning whatever was crawled so far instead of getting killed mid-request by the Lambda platform. Set the function timeout generously relative to `max_pages` (see below) so this is a rare safety net, not the normal stopping condition. -- **`same_domain_only`**: matches hostname exactly (not a registrable-domain / public-suffix match), so `www.example.com` and `example.com` are treated as different hosts. Widen this yourself if you need subdomain-inclusive crawling. - -## IAM permissions - -In addition to `AWSLambdaBasicExecutionRole` (CloudWatch Logs), the execution role needs write access to the target bucket: - -```json -{ - "Version": "2012-10-17", - "Statement": [ - { - "Effect": "Allow", - "Action": ["s3:PutObject"], - "Resource": ["arn:aws:s3:::bytelyon-private/*"] - } - ] -} -``` - -## Function configuration recommendations - -| Setting | Value | Why | -| -------------------------- | ----------------------------------------------------------------- | -------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------- | -| Package type | Image | Required — this is a container image, not a zip. | -| Architecture | `arm64` | Roughly 20% cheaper than x86_64. Match the architecture you built for. | -| Memory | 3008 MB | Memory in Lambda is tied to vCPU. Below ~1769 MB Chromium starts noticeably slower. | -| Timeout | scale with `max_pages` | Budget ~10-15 s/page (nav + settle + screenshot + upload) plus ~10 s cold start. E.g. `max_pages=20` → 300-330 s; keep the default `max_pages=20` under a 300 s timeout, or lower `max_pages` for a shorter timeout. | -| Ephemeral storage (`/tmp`) | 512 MB (default) | Screenshots are streamed to S3, not accumulated on disk. | -| Networking | Default (no VPC) | Binary is baked in; only outbound HTTPS to the crawled sites and to S3 is needed. | -| Execution role | Basic execution + S3 `PutObject` on the target bucket (see above) | | - -## Cold start - -Same profile as the sibling examples: first invocation in a new container takes ~80–90 s. Subsequent warm invocations are much faster. - -## License - -The patched Chromium binary inside the upstream `cloakhq/cloakbrowser` image is governed by the **CloakBrowser Binary License** (published at https://github.com/CloakHQ/CloakBrowser/blob/main/BINARY-LICENSE.md). Internal organizational use (private ECR, your own scraping pipelines, your own business) is free. Exposing this Lambda as a paid API to third-party customers — i.e. browser-as-a-service — requires an OEM/SaaS license from CloakHQ (`cloakhq@pm.me`). Do not push the resulting image to a public registry; that would be redistribution and is prohibited. diff --git a/docker/lambda/site-crawler/lambda_handler.py b/docker/lambda/site-crawler/lambda_handler.py deleted file mode 100644 index 98caaf9..0000000 --- a/docker/lambda/site-crawler/lambda_handler.py +++ /dev/null @@ -1,261 +0,0 @@ -"""AWS Lambda handler: breadth-first site crawl with per-page screenshots -uploaded to S3. - -Starting from one or more seed URLs, this handler: - 1. Visits a page with cloakbrowser's stealth Chromium. - 2. Screenshots it and uploads the screenshot to S3. - 3. Extracts same-site links from the page and enqueues unvisited ones. - 4. Repeats breadth-first until `max_pages`/`max_depth` is hit, the crawl - runs out of same-site links, or the Lambda invocation is running low on - time. - -Returns the list of URLs actually crawled (in visit order) — screenshots are -a side effect written to S3, not part of the return value. - -Uses the **sync** cloakbrowser/Playwright API (`cloakbrowser.launch`), same -as ../news-scraper — this is fine under awslambdaric, which -invokes `handler(event, context)` synchronously with no asyncio event loop -already running. - -Event schema: - url str a single seed URL - urls list[str] one or more seed URLs (checked before `url`) - Exactly one of `url` / `urls` is required. - bucket str "bytelyon-private" - prefix str "crawls/{aws_request_id}/" — S3 key prefix - max_pages int 20 — hard cap on pages visited across all seeds - max_depth int 2 — link-following depth from each seed (0 = seeds only) - same_domain_only bool true — only follow links whose hostname matches the seed's - full_page_screenshot bool true — capture the entire scrollable page - headless bool true — matches ../news-scraper - humanize bool true - human_preset str "careful" - goto_timeout_ms int 30000 - wait_for_load_state_timeout_ms int 5000 - -Returns: - ["https://example.com/", "https://example.com/about", ...] - -S3 layout: - s3://{bucket}/{prefix}{sanitized-host-and-path}-{sha256[:10]}.png - -Lambda-specific additions (see ../aws_lambda/INSTRUCTIONS.md for the same -rationale in the sibling handlers): - - `--disable-dev-shm-usage` / `--no-zygote` Chromium flags. - - URL scheme + SSRF validation before *and* after navigating to every URL — - seeds and, crucially, every link discovered on a crawled page, since - those are effectively attacker-influenced input if the crawled site is - untrusted. - - A remaining-time check (`context.get_remaining_time_in_millis()`) stops - enqueuing new pages once the invocation is close to its timeout, so the - function returns whatever it crawled so far instead of being killed - mid-request. -""" - -from __future__ import annotations - -import hashlib -import ipaddress -import logging -import re -import socket -from collections import deque -from typing import Any -from urllib.parse import urldefrag, urlparse - -import boto3 -from cloakbrowser import launch -from playwright.sync_api import Page -from playwright.sync_api import TimeoutError as PlaywrightTimeoutError - -logger = logging.getLogger("cloakbrowser.lambda") -logger.setLevel(logging.INFO) - -# awslambdaric wires the root logger with a formatter like -# "[INFO]\t2024-01-15T10:30:45.123Z\t\tmessage" — CloudWatch already -# timestamps every event server-side, so repeating timestamp + request id on -# every line just burns terminal columns. Trim to level + message. -for _h in logging.getLogger().handlers: - _h.setFormatter(logging.Formatter("%(levelname)s %(message)s")) - -DEFAULT_BUCKET = "bytelyon-private" -_TIME_BUDGET_MS = 10_000 # stop starting new pages this close to the Lambda deadline - - -def _validate_url(url: str) -> None: - """Reject non-HTTP schemes and URLs that resolve to private/internal IPs.""" - parsed = urlparse(url) - if parsed.scheme.lower() not in ("http", "https"): - raise ValueError( - f"Only http:// and https:// URLs are supported, got: {parsed.scheme!r}" - ) - hostname = parsed.hostname - if not hostname: - raise ValueError("URL has no hostname") - try: - infos = socket.getaddrinfo(hostname, None, socket.AF_UNSPEC, socket.SOCK_STREAM) - except socket.gaierror: - raise ValueError(f"Cannot resolve hostname: {hostname}") - for info in infos: - addr = ipaddress.ip_address(info[4][0]) - if not addr.is_global: - raise ValueError("URLs targeting private/internal networks are blocked") - - -def _normalize(url: str) -> str: - """Strip the fragment so '#section' anchors don't count as distinct pages.""" - return urldefrag(url)[0] - - -def _same_site(seed_host: str, candidate_host: str) -> bool: - return seed_host.lower() == candidate_host.lower() - - -def _s3_key(prefix: str, url: str) -> str: - parsed = urlparse(url) - path = parsed.path.strip("/") or "index" - safe = re.sub(r"[^a-zA-Z0-9/_-]", "_", f"{parsed.netloc}/{path}") - digest = hashlib.sha256(url.encode()).hexdigest()[:10] - return f"{prefix.rstrip('/')}/{safe}-{digest}.png" - - -def _extract_links(page: Page) -> list[str]: - try: - return page.eval_on_selector_all("a[href]", "els => els.map(e => e.href)") - except Exception as e: - logger.warning("link extraction failed on %s: %s", page.url, e) - return [] - - -def _seeds_from_event(event: dict) -> list[str]: - urls = event.get("urls") - if urls is not None: - if ( - not isinstance(urls, list) - or not urls - or not all(isinstance(u, str) for u in urls) - ): - raise ValueError("'urls' must be a non-empty list of strings") - return urls - url = event.get("url") - if isinstance(url, str) and url: - return [url] - raise ValueError("event must include 'url' (str) or 'urls' (list[str])") - - -def _remaining_ms(context: Any) -> int: - try: - return context.get_remaining_time_in_millis() - except Exception: - return 10**9 # local RIE testing has no deadline — treat as unlimited - - -def handler(event: dict, context: Any) -> list[str]: - seeds = _seeds_from_event(event) - for seed in seeds: - _validate_url(seed) - - bucket = event.get("bucket", DEFAULT_BUCKET) - prefix = event.get( - "prefix", f"crawls/{getattr(context, 'aws_request_id', 'local')}/" - ) - max_pages = int(event.get("max_pages", 20)) - max_depth = int(event.get("max_depth", 2)) - same_domain_only = event.get("same_domain_only", True) - full_page_screenshot = event.get("full_page_screenshot", True) - - s3 = boto3.client("s3") - - browser = launch( - headless=event.get("headless", True), - humanize=event.get("humanize", True), - human_preset=event.get("human_preset", "careful"), - args=[ - # Lambda /dev/shm is ~64 MB — Chromium crashes mid-render without this. - "--disable-dev-shm-usage", - # Lambda's restricted process model can't fork from Chromium's - # zygote — without this, child renderer processes fail to spawn. - "--no-zygote", - ], - ) - page: Page = browser.new_page() - - visited: set[str] = set() - crawled: list[str] = [] - queue: deque[tuple[str, int, str]] = deque() # (url, depth, seed_host) - for seed in seeds: - seed_host = urlparse(seed).hostname or "" - queue.append((_normalize(seed), 0, seed_host)) - - try: - while queue and len(crawled) < max_pages: - if _remaining_ms(context) < _TIME_BUDGET_MS: - logger.warning( - "stopping crawl early: %d ms remaining, %d/%d pages done", - _remaining_ms(context), - len(crawled), - max_pages, - ) - break - - url, depth, seed_host = queue.popleft() - if url in visited: - continue - visited.add(url) - - try: - _validate_url(url) - page.goto(url, timeout=event.get("goto_timeout_ms", 30_000)) - page.wait_for_load_state( - "domcontentloaded", - timeout=event.get("wait_for_load_state_timeout_ms", 5_000), - ) - except PlaywrightTimeoutError: - # Best-effort: screenshot/links from whatever loaded so far. - pass - except Exception as e: - logger.warning("skipping %s: %s", url, e) - continue - - try: - _validate_url(page.url) # re-check after redirects - except Exception as e: - logger.warning("skipping %s after redirect: %s", url, e) - continue - - final_url = _normalize(page.url) - if final_url in visited: - # Reached a URL already crawled under a different alias (e.g. - # the seed lacked a trailing slash but resolved to one, or a - # redirect landed on an already-visited page). - continue - visited.add(final_url) - - try: - png = page.screenshot(full_page=full_page_screenshot) - s3.put_object( - Bucket=bucket, - Key=_s3_key(prefix, final_url), - Body=png, - ContentType="image/png", - ) - except Exception as e: - logger.warning("screenshot/upload failed for %s: %s", final_url, e) - - crawled.append(final_url) - - if depth < max_depth: - for link in _extract_links(page): - link = _normalize(link) - if link in visited: - continue - link_host = urlparse(link).hostname or "" - if same_domain_only and not _same_site(seed_host, link_host): - continue - if urlparse(link).scheme.lower() not in ("http", "https"): - continue - queue.append((link, depth + 1, seed_host)) - finally: - browser.close() - - return crawled diff --git a/docker/lambda/warmer/Dockerfile b/docker/lambda/warmer/Dockerfile deleted file mode 100644 index 7dc3c02..0000000 --- a/docker/lambda/warmer/Dockerfile +++ /dev/null @@ -1,43 +0,0 @@ -# CloakBrowser base-image warmer on AWS Lambda. -# -# Built FROM the shared bytelyon-cloakbrowser-base image (see -# ../base-image/Dockerfile and ../base-image/INSTRUCTIONS.md), which carries -# every bit of common CloakBrowser + Lambda glue: awslambdaric, the Lambda -# Runtime Interface Emulator, boto3, the dual-mode entrypoint, the non-root -# readability fix, and the runtime ENV. This Dockerfile only adds this -# handler's own module — see lambda_handler.py. -# -# Unlike every other handler built from this base image, this one does no -# scraping at all: its sole job is to launch (and immediately close) a -# CloakBrowser browser instance, exercising the same expensive cold-start -# path (baked Chromium binary extraction/verification, renderer process -# spawn, stealth patch injection) every other handler pays for on its own -# first invocation — without any navigation, scraping, or S3/network I/O. -# -# Sibling of ../news-scraper, ../page-scraper, ../scraper, -# ../serp-scraper, and ../site-crawler (same base image, different -# lambda_handler.py) — see INSTRUCTIONS.md for why invoking this function -# does NOT warm any of those. -# -# ─── Build (the base image must already exist, locally or in ECR) ──────────── -# docker buildx build --platform linux/arm64 --provenance=false --sbom=false \ -# -t bytelyon-warmer:arm64 --load . -# -# docker run --rm -p 9000:8080 bytelyon-warmer:arm64 -# curl -XPOST http://localhost:9000/2015-03-31/functions/function/invocations -d '{}' -# -# --provenance=false --sbom=false is required: without it, buildx emits a -# manifest-list/attestation image that Lambda's CreateFunction/UpdateFunctionCode -# rejects with "image manifest, config or layer media type ... is not supported". -# -# To build against a locally-loaded (not-yet-pushed) base image instead of -# pulling from ECR, see ../base-image/INSTRUCTIONS.md: -# --build-arg BASE_IMAGE=bytelyon-cloakbrowser-base:arm64 - -ARG BASE_IMAGE=138305277395.dkr.ecr.us-east-1.amazonaws.com/bytelyon-cloakbrowser-base:latest -FROM ${BASE_IMAGE} - -# Handler sits at /app (already on Python's import path, WORKDIR=/app), -# imports cloakbrowser as a normal library. ENTRYPOINT/CMD are inherited from -# the base image (both already correct for this handler's module name). -COPY lambda_handler.py /app/lambda_handler.py diff --git a/docker/lambda/warmer/INSTRUCTIONS.md b/docker/lambda/warmer/INSTRUCTIONS.md deleted file mode 100644 index 4731f68..0000000 --- a/docker/lambda/warmer/INSTRUCTIONS.md +++ /dev/null @@ -1,120 +0,0 @@ -# CloakBrowser base-image warmer on AWS Lambda - -A single-purpose Lambda function that launches (and immediately closes) a CloakBrowser browser instance and returns some timing/version diagnostics — nothing else. It does not scrape, navigate anywhere meaningful, write to S3, or accept a proxy. Its only job is to exercise the expensive part of a cold start (baked Chromium binary extraction/verification, renderer process spawn, CloakBrowser's stealth patch injection) so you can either: - -1. Point a periodic trigger (an EventBridge scheduled rule) at it to keep its own execution environment warm, or -2. Invoke it once after building/pushing a new base image as a fast, cheap, side-effect-free smoke test that it still boots. - -This is a sibling of [`../news-scraper`](../news-scraper), [`../page-scraper`](../page-scraper), [`../scraper`](../scraper), [`../serp-scraper`](../serp-scraper), and [`../site-crawler`](../site-crawler) — same shared [`../base-image`](../base-image), different (much smaller) `lambda_handler.py`. - -## Read this before wiring up a scheduled trigger - -**Invoking this function does NOT warm any _other_ Lambda function.** Even functions built from the byte-identical container image get their own independent pool of execution environments — AWS Lambda does not share warm containers across function ARNs, regardless of shared image layers. Pinging this function only keeps _this function's own_ containers warm. If your actual goal is to stop `../serp-scraper` (or any other handler) from going cold, your options are: - -- **AWS Lambda Provisioned Concurrency** on that handler's own function — the correct, AWS-native fix, though it has an hourly cost regardless of invocations. -- **A scheduled ping of that handler's own function** with a cheap-but-real event (e.g. `{"query": "test"}` for `serp-scraper`) — simple, but runs that handler's actual logic (and its side effects, like S3 writes) on every ping, and consumes its normal IAM permissions/proxy config. -- **This function**, if what you actually want is: a function with no side effects and no extra IAM permissions that proves the shared base image still boots, and/or you're fine with only _this_ function itself staying warm (e.g. you're about to fan out a burst of concurrent invocations across the other handlers and want to prime the account's Lambda container-image pull/cache path first — image-layer pull caching within a region/account is an internal Lambda implementation detail, not a documented guarantee, so treat this as a best-effort nudge, not a fix). - -## Files in this directory - -| File | Purpose | -| ------------------- | --------------------------------------------------------------------------------------------------------------- | -| `Dockerfile` | `FROM` the shared `../base-image`, plus a single `COPY lambda_handler.py`. See `../base-image/INSTRUCTIONS.md`. | -| `lambda_handler.py` | Launch + close. No scraping, no S3, no proxy support. Headless by default. | -| `INSTRUCTIONS.md` | This file. | - -This directory is no longer fully standalone: it depends on the shared base image (`../base-image`) having been built and either published to ECR or loaded locally first — see `../base-image/INSTRUCTIONS.md`. - -## Build - -Build and publish `../base-image` first (see `../base-image/INSTRUCTIONS.md`), then: - -```bash -docker buildx build --platform linux/arm64 --provenance=false --sbom=false \ - -t bytelyon-warmer:arm64 --load . -``` - -`--provenance=false --sbom=false` is required — without it, buildx produces a manifest-list/attestation image that Lambda's `CreateFunction` rejects with `InvalidParameterValueException: ... image manifest, config or layer media type ... is not supported`. - -To build against a locally-loaded (not-yet-pushed) base image instead of pulling from ECR, add `--build-arg BASE_IMAGE=bytelyon-cloakbrowser-base:arm64`. For x86_64, switch `--platform linux/amd64` (and build the base image for the same architecture). - -## Local smoke test (no AWS account needed) - -```bash -docker run --rm -p 9000:8080 bytelyon-warmer:arm64 - -# In another shell — no payload needed at all: -curl -sS -XPOST "http://localhost:9000/2015-03-31/functions/function/invocations" -d '{}' - -# Invoke again on the same container to see cold_start flip to false: -curl -sS -XPOST "http://localhost:9000/2015-03-31/functions/function/invocations" -d '{}' - -# Also exercise navigation (not just process startup): -curl -sS -XPOST "http://localhost:9000/2015-03-31/functions/function/invocations" \ - -d '{"navigate": true}' -``` - -No AWS credentials, S3 bucket, or proxy are needed for this handler at all — that's deliberate, so it can't fail for reasons unrelated to "does the base image still boot". - -## Event schema - -| Field | Type | Default | Description | -| ---------- | ---- | ------- | ------------------------------------------------------------------------------------------------------------------------------ | -| `headless` | bool | `true` | | -| `navigate` | bool | `false` | Also open a page and navigate it to `about:blank` (no network call) to exercise the render pipeline, not just process startup. | - -### Response - -```json -{ - "warm": true, - "cold_start": true, - "cloakbrowser_version": "0.5.10", - "chromium_version": "146.0.7680.177.5", - "launch_ms": 812.4, - "navigate_ms": null, - "total_ms": 815.9 -} -``` - -`cold_start` is `true` only on a given container's first invocation (tracked with a module-level flag, so it resets whenever Lambda spins up a fresh execution environment) — on later invocations of an already-warm container it's `false`. If you're pinging this function on a schedule specifically to keep it warm, seeing `cold_start: true` on a ping means the container was recycled since the last one (Lambda reclaims idle containers, typically well within an hour of inactivity) — check your trigger's interval if that's happening more than expected. `navigate_ms` is `null` unless `"navigate": true` was passed. - -## IAM permissions - -Just `AWSLambdaBasicExecutionRole` (CloudWatch Logs) — nothing else. This handler never touches S3, a proxy, or any caller-supplied URL, so it needs no additional grants. - -## Scheduling a periodic warm-up (optional) - -An EventBridge scheduled rule targeting this function's ARN, e.g. every 5 minutes: - -```bash -aws events put-rule --name warm-cloakbrowser --schedule-expression "rate(5 minutes)" -aws lambda add-permission --function-name bytelyon-warmer \ - --statement-id AllowEventBridge --action lambda:InvokeFunction \ - --principal events.amazonaws.com --source-arn -aws events put-targets --rule warm-cloakbrowser \ - --targets "Id"="1","Arn"="","Input"="{}" -``` - -Remember: this only keeps _this_ function's own containers warm (see the warning above) — it is not a substitute for Provisioned Concurrency on a handler whose cold start latency actually matters to you. - -## Function configuration recommendations - -| Setting | Value | Why | -| -------------------------------- | ------------------------------------------- | --------------------------------------------------------------------------------------------------------------------------------------------------------------- | -| Package type | Image | Required — this is a container image, not a zip. | -| Architecture | `arm64` | Roughly 20% cheaper than x86_64. Match the architecture you built for. | -| Memory | 3008 MB | Same as every other handler here — Chromium still has to actually launch. Below ~1769 MB it starts noticeably slower, which would defeat the point of a warmer. | -| Timeout | 30 s | A launch+close with no navigation completes in well under a second warm, a few seconds on a genuine cold start. 30 s leaves generous headroom. | -| Ephemeral storage (`/tmp`) | 512 MB (default) | Nothing is written to disk. | -| Networking | Default (no VPC) | No outbound network call at all unless `"navigate": true` (and even then, `about:blank` needs none). | -| Execution role | `AWSLambdaBasicExecutionRole` | Just CloudWatch Logs — see IAM permissions above. | -| Reserved/provisioned concurrency | none needed for this function's own purpose | Only relevant if you're using _this_ function as a literal warm pool rather than just a smoke test. | - -## Cold start - -Same profile as the sibling examples: first invocation in a new container takes ~80–90 s (baked Chromium binary extraction/verification is the dominant cost). Subsequent warm invocations complete in well under a second, since there's no navigation or I/O by default. - -## License - -Same as every sibling handler: the patched Chromium binary inside the upstream `cloakhq/cloakbrowser` image is governed by the **CloakBrowser Binary License** (published at https://github.com/CloakHQ/CloakBrowser/blob/main/BINARY-LICENSE.md). Internal organizational use (private ECR, your own scraping pipelines, your own business) is free. Do not push the resulting image to a public registry; that would be redistribution and is prohibited. diff --git a/docker/lambda/warmer/lambda_handler.py b/docker/lambda/warmer/lambda_handler.py deleted file mode 100644 index 47ec820..0000000 --- a/docker/lambda/warmer/lambda_handler.py +++ /dev/null @@ -1,107 +0,0 @@ -"""AWS Lambda handler: warm up the shared CloakBrowser base image. - -Sole purpose: launch (and immediately close) a CloakBrowser browser instance -on every invocation. That launch is the expensive part of a cold start every -other handler in this repo also pays for on its own first invocation — -extracting/verifying the baked Chromium binary, spawning the -renderer/GPU-process tree, injecting CloakBrowser's stealth patches — but -this handler does no navigation, no scraping, and no S3/network I/O of its -own. Point a periodic trigger (e.g. an EventBridge scheduled rule) at this -function to keep its execution environment warm, or invoke it once after a -deploy as a cheap smoke test that the base image still boots correctly. - -IMPORTANT — this does NOT warm any *other* Lambda function's execution -environment. Even functions built from the exact same container image get -their own independent pool of warm containers; invoking this function has no -guaranteed effect on ../news-scraper, ../page-scraper, ../scraper, -../serp-scraper, or ../site-crawler. Deploy this as its own function when you -specifically want a side-effect-free, zero-extra-IAM-permission way to keep -*a* CloakBrowser container warm and/or verify the base image boots — if your -goal is instead to stop a *specific* handler from going cold, the more -direct fix is either AWS Lambda Provisioned Concurrency on that handler's own -function, or a scheduled ping of that handler's own function with a cheap -real event. - -Same sibling of ../news-scraper, ../page-scraper, ../scraper, -../serp-scraper, and ../site-crawler — built FROM the shared ../base-image. - -Event schema: - headless bool true - navigate bool false — additionally open a page and navigate it to - "about:blank" (no network call) to also exercise - Chromium's page/render pipeline, not just process - startup - -Returns: - { - "warm": true, - "cold_start": true, -- whether this was this container's - first invocation since it started; - expect "false" on later pings if - the warm-up trigger is working - "cloakbrowser_version": "0.5.10", - "chromium_version": "146.0.7680.177.5", - "launch_ms": 812.4, - "navigate_ms": null, -- null unless "navigate": true - "total_ms": 815.9 - } -""" - -from __future__ import annotations - -import time -from typing import Any - -import cloakbrowser # pyright: ignore[reportMissingImports] -from cloakbrowser import launch # pyright: ignore[reportMissingImports] -from playwright.sync_api import Page # pyright: ignore[reportMissingImports] - -# Module-level state persists for the life of the container (i.e. across -# every invocation it serves after the first), same as any other global in a -# Lambda handler module. Flips to True on this container's very first -# invocation, so every later invocation on the same (warm) container reports -# cold_start=False — a simple, honest signal that the warm-up trigger is -# actually keeping this function's containers alive. -_warmed = False - - -def handler(event: dict, context: Any) -> dict: - global _warmed - cold_start = not _warmed - _warmed = True - - started = time.monotonic() - - browser = launch( - headless=event.get("headless", True), - args=[ - # Lambda /dev/shm is ~64 MB — Chromium crashes mid-render without this. - "--disable-dev-shm-usage", - # Lambda's restricted process model can't fork from Chromium's - # zygote — without this, child renderer processes fail to spawn. - "--no-zygote", - ], - ) - launch_done = time.monotonic() - - navigate_ms: float | None = None - try: - page: Page = browser.new_page() - if event.get("navigate", False): - nav_started = time.monotonic() - page.goto("about:blank") - navigate_ms = (time.monotonic() - nav_started) * 1000 - finally: - browser.close() - - total_ms = (time.monotonic() - started) * 1000 - - return { - "warm": True, - "cold_start": cold_start, - "cloakbrowser_version": cloakbrowser.__version__, - "chromium_version": cloakbrowser.CHROMIUM_VERSION, - "launch_ms": round((launch_done - started) * 1000, 1), - "navigate_ms": round(navigate_ms, 1) if navigate_ms is not None else None, - "total_ms": round(total_ms, 1), - } diff --git a/docker/server/pgsql/create-testing-database.sql b/docker/pgsql/create-testing-database.sql similarity index 100% rename from docker/server/pgsql/create-testing-database.sql rename to docker/pgsql/create-testing-database.sql diff --git a/docker/worker/Dockerfile b/docker/worker/Dockerfile deleted file mode 100644 index a1d09fd..0000000 --- a/docker/worker/Dockerfile +++ /dev/null @@ -1,88 +0,0 @@ -# Consolidated local-machine SQS worker for every bytelyon scrape job type -# (serp, news, sitemap, ...) — one image, one queue, one location. See -# worker.py's module docstring for the full architecture and env-var -# contract, and handlers/*.py for the per-job-type navigation logic. -# -# Built FROM the shared bytelyon-cloakbrowser-base image (see -# ../lambda/base-image/Dockerfile) — same GeoIP database, real MS fonts, -# Xvfb entrypoint, and non-root readability fix every Lambda handler in -# ../lambda/ already gets. boto3 and requests are already present there; -# nothing extra needed for handlers/serp.py. -# -# ALSO layers in SeleniumBase's Stealthy Playwright Mode (Debian's system -# `chromium` + the `seleniumbase` pip package + the -# ../images/seleniumbase-base/seleniumbase_playwright.py launch shim, -# copied straight out of that already-published image rather than -# duplicated here — see the COPY --from= below) as a *second* browser -# provider for handlers/generic.py (news/sitemap). Why bother running two -# browser stacks in one image: CloakBrowser Pro's license caps concurrent -# sessions at 5 seats. handlers/serp.py actually needs CloakBrowser's -# Google-specific stealth patches and reserves a seat for that; news and -# sitemap targets are far less aggressive about bot detection and don't — -# SeleniumBase's CDP Mode is plenty for them, and defaulting news/sitemap to -# it (see handlers/generic.py's GENERIC_BROWSER_PROVIDER) means those jobs -# never compete with serp jobs for one of the 5 coveted cloak seats, no -# matter how many run concurrently. -# -# This never gets deployed to AWS Lambda — it's meant to run as a long-lived -# container on a home/office network machine, so there's no Lambda-specific -# build flag concern here (no --provenance=false --sbom=false needed, but -# harmless to keep for consistency with every other image in this repo). -# -# ─── Build ───────────────────────────────────────────────────────────────── -# docker buildx build --platform linux/arm64 --provenance=false --sbom=false \ -# -t bytelyon-worker:arm64 --load . -# -# (Swap --platform for your machine's actual architecture — build natively -# for whatever's running the container, there's no reason to cross-build -# something that never leaves this machine.) -# -# ─── Run (see worker.py's docstring for what each env var does) ──────────── -# docker run -d --name bytelyon-worker --restart unless-stopped \ -# -e AWS_ACCESS_KEY_ID=... \ -# -e AWS_SECRET_ACCESS_KEY=... \ -# -e AWS_DEFAULT_REGION=us-east-1 \ -# -e SCRAPE_JOBS_QUEUE_URL=https://sqs.us-east-1.amazonaws.com//bytelyon-scrape-jobs \ -# -e LARAVEL_BASE_URL=http://host.docker.internal \ -# -e LARAVEL_WORKER_TOKEN=... \ -# bytelyon-worker:arm64 -# -# docker logs -f bytelyon-worker - -ARG SELENIUMBASE_BASE_IMAGE=138305277395.dkr.ecr.us-east-1.amazonaws.com/bytelyon-seleniumbase-base:latest -ARG BASE_IMAGE=138305277395.dkr.ecr.us-east-1.amazonaws.com/bytelyon-cloakbrowser-base:latest - -FROM ${SELENIUMBASE_BASE_IMAGE} AS seleniumbase_source - -FROM ${BASE_IMAGE} - -# Debian's `chromium` package — the actual browser SeleniumBase's CDP Mode -# drives (see ../images/seleniumbase-base/Dockerfile for the full rationale, -# identical here). `xvfb`/`ca-certificates`/`fonts-liberation` are already -# present via the CloakBrowser base image, so only `chromium` itself is new. -RUN apt-get update && apt-get install -y --no-install-recommends \ - chromium \ - && rm -rf /var/lib/apt/lists/* - -# seleniumbase + playwright's *client* library. playwright is already -# present (CloakBrowser depends on it too), but pip will no-op rather than -# error if versions already satisfy seleniumbase's requirement. -RUN pip install --no-cache-dir seleniumbase - -# The launch() shim itself — single source of truth stays -# ../images/seleniumbase-base/seleniumbase_playwright.py; copied out of that -# already-published image instead of duplicating the file here so the two -# never drift apart. Same /opt/python + PYTHONPATH placement as that image, -# so it isn't shadowed by this Dockerfile's own /app COPY layer below. -COPY --from=seleniumbase_source /opt/python/seleniumbase_playwright.py /opt/python/seleniumbase_playwright.py -ENV PYTHONPATH=/opt/python - -COPY common.py worker.py /app/ -COPY handlers/ /app/handlers/ - -# Override the base image's Lambda-shaped CMD — this container is never -# invoked by the Lambda Runtime API, it just runs worker.py's poll loop -# directly under Xvfb (started by the inherited /entrypoint.sh, same as -# every headed launch in this image family). -ENTRYPOINT ["/entrypoint.sh"] -CMD ["python3", "worker.py"] diff --git a/docker/worker/INSTRUCTIONS.md b/docker/worker/INSTRUCTIONS.md deleted file mode 100644 index 7219610..0000000 --- a/docker/worker/INSTRUCTIONS.md +++ /dev/null @@ -1,342 +0,0 @@ -# bytelyon-worker: local-machine SQS worker (serp, news, sitemap) - -One consolidated worker fleet for every bot type that needs a real browser: -Search (Google SERP), News (article capture), and Sitemap (site crawl). One -image, one queue, one location — replacing what used to be per-job-type -Lambda functions plus, most recently, a serp-only worker. - -## Why local workers instead of Lambda - -Google (and to a lesser extent other sites) trusts residential/home-network -egress far more than any datacenter IP, Lambda's own included. Running the -actual browser on a home network, driven by jobs pulled from a shared SQS -queue, sidesteps that entirely — and any number of machines can run this -same worker against the same queue for free, horizontal, pull-based scaling -(SQS's visibility timeout guarantees only one worker processes a given -message at a time). - -## Architecture - -``` -BotJob (Laravel) - -> picks one of the bot's configured proxies at random, if it has any - (App\Models\Bot::randomProxy(), configured per-bot from one-to-many - proxies created in Settings > Proxies) - -> Sqs::enqueueScrape($type, $id, $fields) -- $fields includes a `proxy` - field (`{scheme, host, port, username, pass, bypass}`) whenever a - proxy was picked - -> SQS queue: bytelyon-scrape-jobs[-dev] (DLQ: ...-dlq, maxReceive=3) - one queue pair per environment -- see "One-time setup" below - -> worker.py (any number of machines, long-polling) - -> handlers/serp.py (type=serp) launch+navigate, using the - job's `proxy` if present, else falling back to a hardcoded - DataImpulse SOCKS5 proxy (Google needs *some* trusted-geo - egress; see the module docstring) - -> handlers/generic.py (type=news|sitemap) plain page.goto(url), - using the job's `proxy` if present, else no proxy at all - -> both return {url, screenshot_key, content_key} — nothing else. - No parsing happens in Python at all. `proxy` itself is consumed - by the handler and never forwarded past this point (see - common.proxy_settings() and worker.py's passthrough filter). - -> uploads html/screenshot to s3://bytelyon-private/worker-scrapes//... - -> POST {LARAVEL_BASE_URL}/api/scrape-jobs/{type}/{id}/complete - Authorization: Bearer - -> ScrapeJobController (app/Http/Controllers/Api/ScrapeJobController.php) - -> serp: App\Support\Serp parses SERP structure -> Serp::data - -> news: App\Data\Html\Page parses article content -> Article - -> sitemap: App\Support\Html parses links -> Page rows + re-enqueues - unseen links (bounded by `depth`, carried in the job) -``` - -**Every worker's response payload is identical** regardless of job type: -`{url, screenshot_key, content_key}`. All parsing/business logic (SERP -structure, article body extraction, sitemap link discovery) lives in the -main Laravel app, in PHP, not scattered across Python handlers — one place, -one language, easy to test and iterate on without rebuilding/redeploying a -worker image. - -### What actually gets POSTed back to Laravel - -```mermaid -sequenceDiagram - participant Bot as SearchBotJob / NewsBotJob / SitemapBotJob - participant SQS as SQS queue - participant Worker as worker.py - participant Handler as handler (serp.py / generic.py) - participant S3 as S3 (bytelyon-private) - participant Api as ScrapeJobController - - Bot->>SQS: enqueueScrape(type, id, fields) - SQS-->>Worker: receive_message (long-poll) - Worker->>Handler: run(job) - Handler->>S3: put_object(html) - Handler->>S3: put_object(screenshot png) - Handler-->>Worker: {url, screenshot_key, content_key} - Worker->>Api: POST /api/scrape-jobs/{type}/{id}/complete - Note right of Worker: body = {...passthrough, url, screenshot_key, content_key} - Api-->>Worker: 204 No Content - Worker->>SQS: delete_message -``` - -The POST body is `{**passthrough, **result}` (see `_handle_message` / -`_report_result` in `worker.py`) — every SQS message field _except_ -`type`/`id`/`headless`, merged with the handler's `{url, screenshot_key, -content_key}`. Note `result` is spread last, so its `url` (the page's -final, post-navigation/redirect URL) always wins over any `url` the -original SQS message carried. Concretely, per job type: - -```jsonc -// POST /api/scrape-jobs/search/42/complete -// SQS message was {"type": "search", "id": 42, "query": "sailing blocks"} -{ - "query": "sailing blocks", // passthrough — untouched by this worker - "url": "https://www.google.com/search?q=sailing+blocks", - "screenshot_key": "worker-scrapes/serp/3f9e.../www.google.com/search-a1b2c3d4e5.png", - "content_key": "worker-scrapes/serp/3f9e.../www.google.com/search-a1b2c3d4e5.html", -} -``` - -(The `search` job type's own S3 key prefix and route parameter name are -still `serp`/`{serp}` — `handlers/serp.py` hardcodes `job_type="serp"` for -its S3 folder naming, and `routes/api.php` binds the `Serp` model via -`{serp}` — only the URI segment and SQS `type` field needed to agree on -`search` to fix the callback 404 this section used to describe -incorrectly.) - -```jsonc -// POST /api/scrape-jobs/news/17/complete -// SQS message was {"type": "news", "id": 17, "url": "https://www.bbc.com/news/some-article"} -{ - "url": "https://www.bbc.com/news/some-article", // handler's result.url overrides passthrough's - "screenshot_key": "worker-scrapes/news/7c1d.../www.bbc.com/news/some-article-5f6a7b8c9d.png", - "content_key": "worker-scrapes/news/7c1d.../www.bbc.com/news/some-article-5f6a7b8c9d.html", -} -``` - -```jsonc -// POST /api/scrape-jobs/sitemap/5/complete -// SQS message was {"type": "sitemap", "id": 5, "url": "https://bytelyon.com/about", "depth": 4} -{ - "depth": 4, // passthrough — read by ScrapeJobController::sitemap() to gate re-enqueueing discovered links - "url": "https://bytelyon.com/about", - "screenshot_key": "worker-scrapes/sitemap/9a2b.../bytelyon.com/about-1a2b3c4d5e.png", - "content_key": "worker-scrapes/sitemap/9a2b.../bytelyon.com/about-1a2b3c4d5e.html", -} -``` - -All three routes validate against the same rules -(`app/Concerns/ScrapeValidationRules.php`, used by -`app/Http/Requests/Api/PageSaveRequest.php`): `url` (required, valid URL), -`screenshot_key`/`content_key` (required strings), `depth` (nullable -integer, sitemap-only in practice but not type-restricted at the -validation layer). - -### Two browser providers, one image - -CloakBrowser Pro's license caps concurrent sessions at **5 seats**. -`handlers/serp.py` actually needs CloakBrowser's Google-specific stealth -patches and always uses it. `handlers/generic.py` (news/sitemap) doesn't — -ordinary websites are far less aggressive about bot detection — so it -defaults to **SeleniumBase's Stealthy Playwright Mode** instead -(`../images/seleniumbase-base/seleniumbase_playwright.py`, copied into this -image at build time), which has no such seat limit. That means news/sitemap -jobs never compete with serp jobs for one of the 5 coveted cloak seats, no -matter how many run concurrently. - -Both providers are baked into the one image (see the Dockerfile's -multi-stage `COPY --from=` of the SeleniumBase shim). Set the _starting_ -provider per job with a `provider` field in the SQS message (`"cloakbrowser"` -or `"seleniumbase"`), or fleet-wide with the `GENERIC_BROWSER_PROVIDER` env -var. - -**Automatic fallback is wired up**: if the starting attempt raises (launch -or navigation failure) or comes back with a block-shaped HTTP status -(403/429/503), `handlers/generic.py` retries once with `cloakbrowser` -- -never the other direction, and never past `cloakbrowser` (nothing left to -escalate to). A plain `page.goto` timeout is still treated as best-effort -(capture whatever loaded) rather than a trigger -- plenty of ordinary slow -pages hit that without being blocked at all. - -At-least-once delivery: if a worker crashes or the callback POST fails, the -message is left alone (not deleted) and SQS redelivers it automatically -after the queue's visibility timeout (300s) — up to 3 attempts before it -lands in the DLQ. No retry logic needed in the worker itself. - -## ⚠️ Before testing: sync code to the actual running app - -The Sail dev stack (`bytelyon-server-1`) mounts `~/PhpstormProjects/bytelyon`, -**not** this checkout (`~/tmp/bytelyon`, branch `bugfix/flaky-serp`). All of -the following need to exist there too before any callback endpoint works: - -- `routes/api.php` (the three `scrape-jobs/{type}/{id}/complete` routes) -- `app/Http/Controllers/Api/ScrapeJobController.php` -- `../../app/Http/Requests/Api/PageSaveRequest.php` -- `app/Support/Serp.php` -- `app/Services/SqsService.php` -- `app/Facades/Sqs.php` -- `app/Jobs/SearchBotJob.php`, `app/Jobs/NewsBotJob.php`, `app/Jobs/SitemapBotJob.php` -- `config/services.php` (`sqs` block) -- `.env` — add `SQS_SCRAPE_JOBS_QUEUE_URL=...` pointing at whichever queue - matches that checkout's own environment (`bytelyon-scrape-jobs-dev` for a - local/dev app, `bytelyon-scrape-jobs` for production — see "One-time - setup" below for why these must stay paired with a matching worker) -- `tests/Feature/Jobs/{Search,News,Sitemap}BotJobTest.php` (updated to mock `Sqs` instead of asserting synchronous side effects) - -Easiest path is probably pushing this branch and pulling/merging it into the -other checkout, rather than copying files by hand. - -## One-time setup - -**1. Issue a worker API token** (once the code above is live): log in as -whichever user should own the "system scraper" identity, go to -Settings → API Tokens, create one (any name) — it's automatically issued -the `worker` ability (`ApiTokenController::store`). Copy the plaintext -token shown once. - -**2. AWS resources** (already created, nothing to do unless rebuilding from -scratch) — **one queue pair per environment**, not one shared queue: - -| | Production | Dev/local | -| ------------------------------------ | -------------------------- | ------------------------------ | -| Queue | `bytelyon-scrape-jobs` | `bytelyon-scrape-jobs-dev` | -| DLQ | `bytelyon-scrape-jobs-dlq` | `bytelyon-scrape-jobs-dev-dlq` | -| `.env`'s `SQS_SCRAPE_JOBS_QUEUE_URL` | points at the prod queue | points at the dev queue | - -Both pairs use the same settings (300s visibility timeout, 20s long-poll, -4-day retention, redrive after 3 receives) and the same IAM user, -`bytelyon-worker` — its policy already covers `sqs:ReceiveMessage` / -`sqs:DeleteMessage` / `sqs:GetQueueAttributes` on _both_ queues, plus -`s3:PutObject` on `bytelyon-private/worker-scrapes/*`. Rotate its access key -via `aws iam create-access-key --user-name bytelyon-worker` / -`aws iam delete-access-key ...` if needed. - -**Why two queues instead of one `--laravel-base-url` switch on a shared -queue**: a job's `id` only means anything against the database it was -enqueued from. A dev-enqueued job picked up by a prod-pointed worker (or -vice versa) would either 404 or silently complete a same-numbered but -totally unrelated production/dev record. Keeping the queues themselves -separate makes that class of mistake structurally impossible instead of -relying on remembering to set the right env var every time. - -## Running a worker - -Every setting has both a CLI flag and an env var — the flag wins if both are -given. Env vars suit `docker run -e`; flags suit one-off overrides or -running the script directly. `python3 worker.py --help` lists all of them. - -```bash -# From this directory: -docker buildx build --platform linux/arm64 --provenance=false --sbom=false \ - -t bytelyon-worker:arm64 --load . -``` - -After any change to this worker fleet's Python code (`common.py`, -`worker.py`, `handlers/*.py`), run `./update.sh` instead of the above -- -it refreshes the ECR auth token (which expires periodically and otherwise -fails the build partway through with a bare 403), then rebuilds _both_ -tags this repo actually uses in one shot: the standalone -`bytelyon-worker:arm64` above, and `worker-1.0/app:latest` (the tag -`../../compose.yml`'s `worker` service runs, via `docker compose build -worker`) -- easy to update one and forget the other otherwise. Neither -command restarts an already-running container; recreate whichever one you -actually run afterward to pick up the change (`update.sh`'s own output -prints the exact command either way). - -**Local dev** (points at `bytelyon-scrape-jobs-dev` + this machine's own Sail app): - -```bash -docker run -d --name bytelyon-worker-dev --restart unless-stopped \ - -e AWS_ACCESS_KEY_ID= \ - -e AWS_SECRET_ACCESS_KEY= \ - bytelyon-worker:arm64 python3 worker.py \ - --queue-url https://sqs.us-east-1.amazonaws.com/138305277395/bytelyon-scrape-jobs-dev \ - --laravel-base-url http://host.docker.internal \ - --worker-token -``` - -**Production** (points at `bytelyon-scrape-jobs` + the public site): - -```bash -docker run -d --name bytelyon-worker --restart unless-stopped \ - -e AWS_ACCESS_KEY_ID= \ - -e AWS_SECRET_ACCESS_KEY= \ - bytelyon-worker:arm64 python3 worker.py \ - --queue-url https://sqs.us-east-1.amazonaws.com/138305277395/bytelyon-scrape-jobs \ - --laravel-base-url https://bytelyon.com \ - --worker-token -``` - -`docker logs -f bytelyon-worker` (or `-dev`) either way. - -`http://host.docker.internal` reaches the Sail app's port 80 on the Docker -host — works out of the box on Docker Desktop for Mac. If the worker runs -on a genuinely different machine than the Laravel app it's reporting to (a -separate box on the same LAN, or a machine reporting to production over the -internet), it needs a real, reachable URL instead — that's exactly what -`--laravel-base-url`/`LARAVEL_BASE_URL` is for. - -`GENERIC_BROWSER_PROVIDER` (env-only, no CLI flag currently) is optional — -defaults to `seleniumbase` already (see "Two browser providers, one image" -above); only set it to `cloakbrowser` if you want news/sitemap jobs to use -CloakBrowser fleet-wide instead. - -Run as many workers as you want _per environment_ — they all pull from -their one queue and handle whatever job type comes up (serp/news/sitemap), -and SQS's visibility timeout guarantees no two workers process the same job -concurrently. Don't point a dev and a prod worker at the same queue (see -above for why). - -**Always stop workers before rebuilding** -(`docker stop bytelyon-worker && docker rm bytelyon-worker`) — a running -worker holds no external lock, but you'll otherwise have two containers -racing to poll the same queue with old vs new code. - -## Manual testing without the full Laravel loop - -Enqueue a job directly (bypassing the Bot jobs) to test a handler + S3 side -in isolation — the callback POST will 404/fail until a real record + live -route exist, which is expected; the worker just leaves the message for -redrive in that case. Use the **dev** queue for this so stray test jobs -never land in front of a production worker: - -```bash -DEV_QUEUE=https://sqs.us-east-1.amazonaws.com/138305277395/bytelyon-scrape-jobs-dev - -# serp -aws sqs send-message --queue-url $DEV_QUEUE --region us-east-1 \ - --message-body '{"type":"serp","id":1,"query":"sailing blocks"}' - -# news -aws sqs send-message --queue-url $DEV_QUEUE --region us-east-1 \ - --message-body '{"type":"news","id":1,"url":"https://www.bbc.com/news"}' - -# sitemap (root of a crawl) -aws sqs send-message --queue-url $DEV_QUEUE --region us-east-1 \ - --message-body '{"type":"sitemap","id":1,"url":"https://bytelyon.com","depth":5}' -``` - -## Known trade-offs (from moving these off a single synchronous execution) - -- **Sitemap freshness**: only the root URL is unconditionally re-crawled on - every bot run. A previously-discovered page only refreshes if it's - re-reachable via some newly-discovered path this run — it's no longer - proactively re-crawled just because it was seen before. The persisted - `pages` table is now the crawl's "seen" set (replacing what used to be an - in-memory set scoped to one job execution). -- **No aggregate "N found" notification** for sitemap crawls (there's no - single point in time that "the crawl finished" in a distributed, - re-enqueueing design without extra completion-tracking infra). News still - fires `BotResultsPersisted` at _enqueue_ time ("N articles queued"), - reworded from the old "N articles found" since the actual scraping now - happens later, asynchronously. -- **SERP link resolution**: `App\Support\Serp::resolveUrl()` only decodes - Google's simple `/url?q=` redirect format directly from the - query string — no live HTTP request. Confirmed against real captured SERPs - that Google also uses an opaque `/goto?url=` format for some - results, which can't be decoded without a live request through Google's own - redirect service; those links currently pass through unresolved - (`domain` ends up as `google.com` rather than the true destination). The - Python version used to resolve these via a real HTTP request through the - same browser/proxy session; that capability didn't move to PHP. Worth - revisiting if destination-URL accuracy matters more than the current - title/kind/index-only fidelity for those specific results. diff --git a/docker/worker/common.py b/docker/worker/common.py deleted file mode 100644 index 31e7413..0000000 --- a/docker/worker/common.py +++ /dev/null @@ -1,99 +0,0 @@ -"""Shared helpers for every job-type handler under docker/worker/handlers/. - -Deliberately minimal: this worker fleet's whole job is "go get this page -(optionally via a proxy/search box), save the HTML and a screenshot to S3, -report back {url, screenshot_key, content_key}." No parsing, no business -logic, no per-domain knowledge lives here or in any handler — that all moved -to the main Laravel app (see app/Support/Html.php, app/Data/Html/Page.php, -app/Support/Serp.php), so every worker's response payload is identical -regardless of job type, and the parsing logic lives in one place, in one -language, versioned with the rest of the app instead of scattered across -Python handlers that are harder to test/iterate on than the PHP app itself. -""" - -from __future__ import annotations - -import hashlib -import re -import uuid -from urllib.parse import urlparse - -import boto3 - -DEFAULT_BUCKET = "bytelyon-private" - - -def proxy_settings(job: dict) -> dict | None: - """Convert the bot's selected-proxy payload -- added to the SQS message - by `App\\Jobs\\BotJob::proxyPayload()` on the Laravel side as - `{scheme, host, port, username, pass, bypass}` -- into the - Playwright-compatible `ProxySettings` shape (`{server, username?, - password?, bypass?}`) that both `cloakbrowser.launch()` and - `seleniumbase_playwright.launch()` accept as their `proxy` kwarg. - - `BotJob` picks one proxy at random (server-side, in PHP) whenever a bot - has more than one configured -- this worker only ever sees the single - already-chosen proxy for a given job, if any. Returns None when the job - carries no `proxy` field at all (the bot has none configured), so - callers can pass the result straight through as `proxy=...` without a - None-check of their own. - """ - proxy = job.get("proxy") - if not proxy or not proxy.get("host"): - return None - - scheme = proxy.get("scheme") or "http" - port = proxy.get("port") - server = f"{scheme}://{proxy['host']}" + (f":{port}" if port else "") - - settings: dict = {"server": server} - if proxy.get("username"): - settings["username"] = proxy["username"] - if proxy.get("pass"): - settings["password"] = proxy["pass"] - if proxy.get("bypass"): - settings["bypass"] = proxy["bypass"] - return settings - - -def s3_key(prefix: str, url: str, ext: str) -> str: - parsed = urlparse(url) - path = parsed.path.strip("/") or "index" - safe = re.sub(r"[^a-zA-Z0-9/_-]", "_", f"{parsed.netloc}/{path}") - digest = hashlib.sha256(url.encode()).hexdigest()[:10] - return f"{prefix.rstrip('/')}/{safe}-{digest}.{ext}" - - -def default_prefix(job_type: str) -> str: - return f"worker-scrapes/{job_type}/{uuid.uuid4()}/" - - -def capture_and_upload(page, job_type: str, bucket: str = DEFAULT_BUCKET) -> dict: - """Capture the current page's HTML + full-page screenshot and upload - both to S3. Returns the generic `{url, screenshot_key, content_key}` - contract every handler returns — the *only* thing worker.py's callback - POST needs, regardless of job type. - """ - url = page.url - html = page.content() - png = page.screenshot(full_page=True) - - prefix = default_prefix(job_type) - content_key = s3_key(prefix, url, "html") - screenshot_key = s3_key(prefix, url, "png") - - s3 = boto3.client("s3") - s3.put_object( - Bucket=bucket, - Key=content_key, - Body=html.encode("utf-8"), - ContentType="text/html; charset=utf-8", - ) - s3.put_object( - Bucket=bucket, - Key=screenshot_key, - Body=png, - ContentType="image/png", - ) - - return {"url": url, "screenshot_key": screenshot_key, "content_key": content_key} diff --git a/docker/worker/handlers/__init__.py b/docker/worker/handlers/__init__.py deleted file mode 100644 index e69de29..0000000 diff --git a/docker/worker/handlers/generic.py b/docker/worker/handlers/generic.py deleted file mode 100644 index dd351fc..0000000 --- a/docker/worker/handlers/generic.py +++ /dev/null @@ -1,121 +0,0 @@ -"""Generic single-URL page capture — used for `news` and `sitemap` jobs (and -any future job type that's just "go to this URL and save it"). Local-worker -equivalent of ../../lambda/grab/lambda_handler.py's core logic, trimmed to -this worker fleet's shared contract. - -Proxy optional: news/sitemap targets are ordinary websites, not Google — -nowhere near as aggressive about datacenter-vs-residential IP reputation, -and this worker's home-network egress is already more trustworthy than -Lambda's own IPs were anyway. So unlike handlers/serp.py, there's no -default proxy here — jobs run proxy-less unless the bot that enqueued them -had one or more proxies configured in Settings, in which case Laravel's -`BotJob` picks one at random and rides it along as the job's `proxy` field -(see common.proxy_settings()). - -Two interchangeable browser providers, both exposing the same -`launch(...) -> Browser`-with-`.new_page()`/`.close()` interface: - - "seleniumbase" (default) — SeleniumBase's Stealthy Playwright Mode (see - ../../images/seleniumbase-base/seleniumbase_playwright.py). Not as - battle-tested against Google specifically as CloakBrowser, but plenty - stealthy for ordinary news/sitemap targets, and — the actual point of - defaulting to it here — doesn't consume one of CloakBrowser Pro's 5 - concurrent-session seats. Those are reserved for handlers/serp.py, - which actually needs CloakBrowser's Google-specific patches; news and - sitemap jobs don't, and can run as many of these concurrently as the - machine can handle without ever touching the CloakBrowser seat pool. - - "cloakbrowser" — same provider handlers/serp.py uses. -Override the *starting* provider per-job with a `provider` field in the SQS -message, or fleet-wide via the GENERIC_BROWSER_PROVIDER env var. - -Automatic fallback: if the starting attempt raises (launch/navigation -failure) or comes back with a block-shaped HTTP status (403/429/503), this -retries once with "cloakbrowser" — never the other direction, since -cloakbrowser is already the stronger/more expensive option and there's -nothing left to escalate to if *it* fails. This only escalates on a real -signal (an exception, or a status code that specifically means "you got -blocked/rate-limited"), not merely "the page looked different than -expected" — this handler has no per-site knowledge to judge that with, so -it doesn't try to. A `page.goto` timeout is still treated as best-effort -(capture whatever loaded rather than fail/escalate) — same as before — -since plenty of ordinary slow pages hit that without being blocked at all. -""" - -from __future__ import annotations - -import logging -import os - -from cloakbrowser import launch as _cloakbrowser_launch -from common import capture_and_upload, proxy_settings -from playwright.sync_api import Page -from playwright.sync_api import TimeoutError as PlaywrightTimeoutError -from seleniumbase_playwright import launch as _seleniumbase_launch - -logger = logging.getLogger("worker.generic") - -_LAUNCHERS = { - "cloakbrowser": _cloakbrowser_launch, - "seleniumbase": _seleniumbase_launch, -} -DEFAULT_PROVIDER = ( - os.environ.get("GENERIC_BROWSER_PROVIDER", "seleniumbase").strip().lower() -) - -# HTTP statuses that specifically mean "you got blocked/rate-limited", not -# just "this page doesn't exist" (404) or "the site is broken" (500) — -# escalating the provider wouldn't help with those, only with statuses a -# stealthier browser might actually get past. -_BLOCK_STATUS_CODES = {403, 429, 503} - - -def _attempt( - url: str, job_type: str, headless: bool, provider: str, proxy: dict | None -) -> dict: - launch = _LAUNCHERS[provider] - browser = launch( - headless=headless, - proxy=proxy, - args=["--disable-dev-shm-usage", "--no-zygote"], - ) - try: - page: Page = browser.new_page() - response = None - try: - response = page.goto(url, wait_until="domcontentloaded", timeout=30_000) - except PlaywrightTimeoutError: - # Best-effort: capture whatever loaded so far rather than fail - # the whole job. Plenty of ordinary slow pages hit this without - # being blocked at all, so this alone doesn't trigger fallback. - pass - - if response is not None and response.status in _BLOCK_STATUS_CODES: - raise RuntimeError(f"got HTTP {response.status} from {url} via {provider}") - - return capture_and_upload(page, job_type=job_type) - finally: - browser.close() - - -def run(job: dict) -> dict: - url = job["url"] - job_type = job["type"] - headless = job.get("headless", True) - provider = str(job.get("provider", DEFAULT_PROVIDER)).strip().lower() - if provider not in _LAUNCHERS: - provider = "seleniumbase" - proxy = proxy_settings(job) - - try: - return _attempt(url, job_type, headless, provider, proxy) - except Exception as e: - if provider == "cloakbrowser": - # Already the strongest provider available — nothing left to - # escalate to, let the caller's own retry/redrive handle it. - raise - logger.warning( - "generic scrape via %s failed (%s); falling back to cloakbrowser for %s", - provider, - e, - url, - ) - return _attempt(url, job_type, headless, "cloakbrowser", proxy) diff --git a/docker/worker/handlers/serp.py b/docker/worker/handlers/serp.py deleted file mode 100644 index 13749ae..0000000 --- a/docker/worker/handlers/serp.py +++ /dev/null @@ -1,145 +0,0 @@ -"""Google-SERP navigation handler — the local-machine equivalent of -../../lambda/serp-di/lambda_handler.py, trimmed to the generic worker -contract: no SERP parsing here anymore (see app/Support/Serp.php in the main -app), just launch/navigate/capture/upload, same as every other handler. - -The launch()/goto()/type() sequence, the DataImpulse SOCKS5 proxy -credentials, and the `human_preset="careful"` launch option are copied -verbatim from the script that ran reliably against Google overnight — do -not "improve" or re-tune any of that without re-validating against a live -run first. - -The DataImpulse proxy is only the *fallback*: if the bot that enqueued this -job has one or more proxies configured in Settings, Laravel's `BotJob` -picks one at random and rides it along as the job's `proxy` field (see -common.proxy_settings()), which takes priority over DataImpulse. Bots with -no proxy of their own keep using DataImpulse, since Google needs *some* -residential/trusted-geo egress to reliably avoid a block/CAPTCHA page. - -Unlike the Lambda version this was derived from, this runs in a normal -Docker container (not AWS Lambda), so: - - `geoip=True` is used directly. cloakbrowser's own httpx-based geoip - resolution only hangs inside *Lambda's* sandboxed network namespace - (confirmed empirically — see ../../lambda/serp-di/lambda_handler.py's - `_resolve_geo` docstring for the full story); it works fine in an - ordinary container. Still wrapped in a fallback to `geoip=False` in - case a given proxy/exit-IP combination misbehaves, same degrade- - gracefully pattern ../../lambda/serp-scraper uses. - - The `--disable-dev-shm-usage --no-zygote` Chromium args are Lambda- - specific hardening and technically unnecessary here, but harmless - (Chromium ignores flags it doesn't need) — kept for parity with the - Lambda version rather than maintaining two divergent arg lists. -""" - -from __future__ import annotations - -import logging - -from cloakbrowser import ProxySettings, launch -from common import capture_and_upload, proxy_settings -from playwright.sync_api import Browser, Page -from playwright.sync_api import TimeoutError as PlaywrightTimeoutError - -logger = logging.getLogger("worker.serp") - -GOOGLE_URL = "https://www.google.com" -PROXY_USERNAME = "2b9bc9b578274c5e9c8f__cr.us" -PROXY_PASSWORD = "a1d0893a3f201adf" -PROXY_HOST = "gw.dataimpulse.com" -PROXY_PORT = 824 - -_CONSENT_SELECTORS = [ - "#L2AGLb", - "button:has-text('Accept all')", - "button:has-text('I agree')", -] - -# DataImpulse's SOCKS5 endpoint has a known non-trivial raw connection- -# failure rate — the retry wraps the *entire* attempt (fresh browser, fresh -# proxy connection) rather than just launch(), since a fresh SOCKS5 -# connection on the next attempt routinely succeeds where the previous one -# didn't, and that failure has shown up at every stage (launch, geoip, -# navigation) in testing. -SCRAPE_ATTEMPTS = 2 - - -def _default_proxy() -> ProxySettings: - return ProxySettings( - username=PROXY_USERNAME, - password=PROXY_PASSWORD, - bypass=".google.com", - server=f"socks5://{PROXY_HOST}:{PROXY_PORT}", - ) - - -def _launch(headless: bool, proxy: ProxySettings | None) -> Browser: - # A bot-configured proxy (picked at random on the Laravel side when a - # bot has more than one -- see common.proxy_settings()) always wins; - # this DataImpulse SOCKS5 endpoint is only the fallback for bots with no - # proxy of their own, since Google needs *some* residential/trusted-geo - # egress to reliably avoid a block/CAPTCHA page. - common_kwargs = dict( - proxy=proxy or _default_proxy(), - headless=headless, - human_preset="careful", - args=["--disable-dev-shm-usage", "--no-zygote"], - ) - try: - return launch(geoip=True, **common_kwargs) - except RuntimeError as e: - if "geoip" not in str(e).lower(): - raise - logger.warning("geoip resolution failed (%s); retrying without geoip", e) - return launch(geoip=False, **common_kwargs) - - -def _dismiss_consent(page: Page) -> None: - for selector in _CONSENT_SELECTORS: - try: - locator = page.locator(selector).first - if locator.is_visible(timeout=1_000): - locator.click(timeout=2_000) - page.wait_for_load_state("domcontentloaded", timeout=5_000) - return - except Exception: - continue - - -def _attempt(query: str, headless: bool, proxy: ProxySettings | None) -> dict: - browser = _launch(headless, proxy) - try: - page: Page = browser.new_page() - page.goto(GOOGLE_URL, timeout=45_000) - page.locator('textarea[name="q"]').first.click() - page.keyboard.type(query) - page.keyboard.press("Enter") - - # A soft wait, not a hard requirement: Google's CAPTCHA/interstitial - # page (google.com/sorry/...) never has #sfooter, so a bare - # wait_for_selector would throw before a single byte gets captured. - # Whether it showed up or not, capture unconditionally below — the - # main app decides "blocked or not" once it has the HTML in hand - # (see app/Support/Serp.php::blocked()), not this worker. - try: - page.wait_for_selector("#sfooter", timeout=30_000) - _dismiss_consent(page) - except PlaywrightTimeoutError: - pass - - return capture_and_upload(page, job_type="serp") - finally: - browser.close() - - -def run(job: dict) -> dict: - query = job["query"] - headless = job.get("headless", False) - proxy = proxy_settings(job) - last_err: Exception | None = None - for attempt in range(1, SCRAPE_ATTEMPTS + 1): - try: - return _attempt(query, headless, proxy) - except Exception as e: - last_err = e - logger.warning("serp attempt %d/%d failed: %s", attempt, SCRAPE_ATTEMPTS, e) - raise last_err # type: ignore[misc] diff --git a/docker/worker/update.sh b/docker/worker/update.sh deleted file mode 100755 index 1eec3db..0000000 --- a/docker/worker/update.sh +++ /dev/null @@ -1,43 +0,0 @@ -#!/bin/sh -# Rebuilds every image this worker fleet's Python code (common.py, worker.py, -# handlers/*.py) ends up in, in one shot, whenever any of that code changes. -# There are two separate tags this repo actually uses day to day, and it's -# easy to update one and forget the other: -# - bytelyon-worker:arm64 standalone tag from build.sh's direct buildx -# build, for a plain `docker run` per -# INSTRUCTIONS.md's "Running a worker" section. -# - worker-1.0/app:latest the tag ../../compose.yml's `worker` service -# actually runs, built via `docker compose build` -# so Sail/`docker compose up worker` picks up the -# same change. -# -# Both build FROM the same private ECR base images (bytelyon-cloakbrowser-base, -# bytelyon-seleniumbase-base), so this also refreshes the ECR auth token -# first -- that expires periodically, and buildx otherwise fails with a bare -# "403 Forbidden" / "authorization token has expired" partway through. -# -# Usage: ./update.sh (run from anywhere -- paths below are script-relative) -set -e - -script_dir=$(CDPATH= cd -- "$(dirname -- "$0")" && pwd) -repo_root=$(CDPATH= cd -- "$script_dir/../.." && pwd) - -aws_region="${AWS_DEFAULT_REGION:-us-east-1}" -aws_account="${AWS_ACCOUNT_ID:-138305277395}" -ecr_host="$aws_account.dkr.ecr.$aws_region.amazonaws.com" - -echo "==> Refreshing ECR auth for $ecr_host" -aws ecr get-login-password --region "$aws_region" \ - | docker login --username AWS --password-stdin "$ecr_host" - -echo "==> Building bytelyon-worker:arm64 (docker/worker/build.sh)" -(cd "$script_dir" && docker buildx build --platform linux/arm64 --provenance=false --sbom=false -t bytelyon-worker:arm64 --load .) - -echo "==> Building worker-1.0/app:latest (docker compose build worker)" -(cd "$repo_root" && docker compose build worker) - -echo "==> Done. Both image tags now include the latest Python code." -echo " Neither is picked up by an already-running container -- recreate" -echo " whichever one you actually run to pick up the change:" -echo " docker compose up -d --force-recreate worker # Sail/compose" -echo " docker stop bytelyon-worker && docker rm bytelyon-worker && docker run ... # standalone, see INSTRUCTIONS.md" diff --git a/docker/worker/worker.py b/docker/worker/worker.py deleted file mode 100644 index 3edc493..0000000 --- a/docker/worker/worker.py +++ /dev/null @@ -1,311 +0,0 @@ -"""Generic SQS polling worker for all bytelyon scrape job types (search, news, -sitemap, ...) — the single consolidated location this whole worker fleet -lives in, replacing what used to be a per-job-type Lambda function. - -Why local workers instead of Lambda at all: Google (and to a lesser extent -other sites) trusts residential/home-network egress far more than any -datacenter IP, Lambda's own included. Running the actual browser on a home -network, driven by jobs pulled from a shared SQS queue, sidesteps that -entirely — and any number of machines can run this same worker against the -same queue for free, horizontal, pull-based scaling (SQS's visibility -timeout guarantees only one worker processes a given message at a time). - -Every handler under handlers/ returns the exact same shape: - {"url": ..., "screenshot_key": ..., "content_key": ...} -No parsing, no per-job business logic lives in Python at all — see -handlers/serp.py and handlers/generic.py's own docstrings. All of that -(SERP structure, article body extraction, sitemap link discovery) lives in -the main Laravel app (app/Support/Html.php, app/Data/Html/Page.php, -app/Support/Serp.php), reached via one callback endpoint per job. - -Flow, per message: - 1. Long-poll receive from SQS (`--queue-url` / `SCRAPE_JOBS_QUEUE_URL`). - 2. Parse `{"type": "search"|"news"|"sitemap", "id": int, ...}` from the - message body. `type` picks the handler; `id` identifies which - record to update on the Laravel side (its meaning is type-specific - — see routes/api.php and ScrapeJobController for what `id` means - per type). Most other fields (e.g. sitemap's `depth`) are opaque to - this worker — it doesn't need to understand them, just carry them - through to the callback unchanged. The exception is `proxy` - (`{scheme, host, port, username, pass, bypass}`, present whenever - the enqueuing bot has one or more proxies configured -- Laravel's - `BotJob` picks one at random when there's more than one): handlers - consume it directly (see common.proxy_settings()) and it is *not* - forwarded to the completion callback. - 3. Run the matching handler's `run(job)` -> {url, screenshot_key, - content_key}. - 4. POST {**passthrough fields, url, screenshot_key, content_key} to - POST {laravel_base_url}/api/scrape-jobs/{type}/{id}/complete, - Authorization: Bearer {worker_token} (a Sanctum personal access - token with the `worker` ability, issued from Settings > API Tokens). - 5. On success, delete the SQS message. On any failure (scrape or - callback), leave it alone — SQS's visibility timeout expires and - redelivers it (up to the queue's maxReceiveCount) before it lands in - the DLQ, so a transient failure gets retried automatically without - any retry logic needed here. - -Configuration — every setting has both a CLI flag and an env var; the flag -wins if both are given. Env vars are the natural fit for `docker run -e`; -CLI flags are there for running this as a plain script, or for one-off -overrides without touching a container's env: - - --queue-url SCRAPE_JOBS_QUEUE_URL required, full SQS queue URL - --laravel-base-url LARAVEL_BASE_URL default "http://host.docker.internal" - --worker-token LARAVEL_WORKER_TOKEN required, Sanctum PAT with - the `worker` ability - --aws-region AWS_DEFAULT_REGION default "us-east-1" - - AWS_ACCESS_KEY_ID / AWS_SECRET_ACCESS_KEY (env-only, standard boto3 - credential resolution — no CLI flag; don't put secrets on a command - line) scoped IAM credentials (sqs:ReceiveMessage/DeleteMessage on the - queue above, s3:PutObject on bytelyon-private/worker-scrapes/*). - -Local dev vs production — use a *separate queue per environment*, not just -a different --laravel-base-url on an otherwise-shared queue. A job's `id` -only means anything against the database it was enqueued from; a worker -pointed at the wrong environment's Laravel app would either 404 or, worse, -silently complete a same-numbered but unrelated record. Point each worker -at the one queue *and* Laravel URL pair that actually go together: - - # local dev (Sail on this machine) - python3 worker.py \\ - --queue-url https://sqs.us-east-1.amazonaws.com//bytelyon-scrape-jobs-dev \\ - --laravel-base-url http://host.docker.internal \\ - --worker-token - - # production - python3 worker.py \\ - --queue-url https://sqs.us-east-1.amazonaws.com//bytelyon-scrape-jobs \\ - --laravel-base-url https://bytelyon.com \\ - --worker-token - -See INSTRUCTIONS.md for how to provision the dev queue (same shape as the -production one, just a second queue + DLQ pair). -""" - -from __future__ import annotations - -import argparse -import json -import logging -import os -import sys -import time -from dataclasses import dataclass - -import boto3 -import requests -from handlers import generic, serp - - -class _ColorFormatter(logging.Formatter): - """Colors just the level name, so INFO/WARNING/ERROR/etc. are easy to - pick out at a glance in a terminal, without touching the message itself - (so it stays plain, greppable text). Falls back to no color at all when - stdout isn't a TTY (e.g. piped to a file or a log collector), since ANSI - codes would just show up as garbage escape sequences there. - """ - - _COLORS = { - logging.DEBUG: "\033[36m", # cyan - logging.INFO: "\033[32m", # green - logging.WARNING: "\033[33m", # yellow - logging.ERROR: "\033[31m", # red - logging.CRITICAL: "\033[1;37;41m", # bold white on red bg - } - _ABBREVIATIONS = { - logging.DEBUG: "DBG", - logging.INFO: "INF", - logging.WARNING: "WAR", - logging.ERROR: "ERR", - logging.CRITICAL: "FAT", - } - _RESET = "\033[0m" - - def __init__(self, fmt: str, *, colorize: bool) -> None: - super().__init__(fmt, datefmt="%H:%M") - self._colorize = colorize - - def format(self, record: logging.LogRecord) -> str: - levelname = self._ABBREVIATIONS.get(record.levelno, record.levelname) - if self._colorize: - color = self._COLORS.get(record.levelno, "") - if color: - levelname = f"{color}{levelname}{self._RESET}" - if levelname != record.levelname: - record = logging.makeLogRecord(record.__dict__) - record.levelname = levelname - return super().format(record) - - -# Not gated on sys.stdout.isatty(): this worker normally runs inside a -# plain (non-tty) Docker container, piping stdout straight to the log -# driver — isatty() would be False there even though whoever's actually -# reading the logs (`docker compose logs -f`, a terminal) is a real, -# color-capable TTY on the *other* end of that pipe. Default to color on; -# respect the standard NO_COLOR convention (https://no-color.org/) for -# anyone piping logs somewhere that can't render ANSI codes. -_handler = logging.StreamHandler(stream=sys.stdout) -_handler.setFormatter( - _ColorFormatter( - "%(asctime)s %(levelname)s %(message)s", - colorize=not os.environ.get("NO_COLOR"), - ) -) -logging.basicConfig(level=logging.INFO, handlers=[_handler]) -logger = logging.getLogger("worker") - -_HANDLERS = { - "search": serp.run, - "news": generic.run, - "sitemap": generic.run, -} - - -@dataclass(frozen=True) -class Config: - queue_url: str - laravel_base_url: str - worker_token: str - aws_region: str - - -def _parse_config(argv: list[str] | None = None) -> Config: - parser = argparse.ArgumentParser( - description="bytelyon scrape-jobs SQS worker (search/news/sitemap)" - ) - parser.add_argument( - "--queue-url", - default=None, - help="SQS queue URL to poll. Env: SCRAPE_JOBS_QUEUE_URL. Required " - "(one or the other).", - ) - parser.add_argument( - "--laravel-base-url", - default=None, - help="Base URL of the Laravel app to POST callbacks to — e.g. " - "https://bytelyon.com in production, http://host.docker.internal " - "for local dev (reaches the Sail app's port 80 on the Docker host " - "from inside this container, on Docker Desktop for Mac). Env: " - "LARAVEL_BASE_URL. Default: http://host.docker.internal.", - ) - parser.add_argument( - "--worker-token", - default=None, - help="Sanctum personal access token with the `worker` ability " - "(Settings > API Tokens). Env: LARAVEL_WORKER_TOKEN. Required " - "(one or the other).", - ) - parser.add_argument( - "--aws-region", - default=None, - help="AWS region the SQS queue lives in. Env: AWS_DEFAULT_REGION. " - "Default: us-east-1.", - ) - args = parser.parse_args(argv) - - queue_url = args.queue_url or os.environ.get("SCRAPE_JOBS_QUEUE_URL") - laravel_base_url = args.laravel_base_url or os.environ.get( - "LARAVEL_BASE_URL", "http://host.docker.internal" - ) - worker_token = args.worker_token or os.environ.get("LARAVEL_WORKER_TOKEN") - aws_region = args.aws_region or os.environ.get("AWS_DEFAULT_REGION", "us-east-1") - - if not queue_url: - parser.error("--queue-url is required (or set SCRAPE_JOBS_QUEUE_URL)") - if not worker_token: - parser.error("--worker-token is required (or set LARAVEL_WORKER_TOKEN)") - - return Config( - queue_url=queue_url, - laravel_base_url=laravel_base_url.rstrip("/"), - worker_token=worker_token, - aws_region=aws_region, - ) - - -def _report_result( - config: Config, job_type: str, job_id: int, passthrough: dict, result: dict -) -> None: - url = f"{config.laravel_base_url}/api/scrape-jobs/{job_type}/{job_id}/complete" - resp = requests.post( - url, - json={**passthrough, **result}, - headers={"Authorization": f"Bearer {config.worker_token}"}, - timeout=15, - ) - resp.raise_for_status() - - -def _handle_message(config: Config, sqs, message: dict) -> None: - receipt_handle = message["ReceiptHandle"] - try: - job = json.loads(message["Body"]) - job_type = str(job["type"]) - job_id = int(job["id"]) - handler = _HANDLERS[job_type] - except Exception: - logger.exception( - "malformed/unknown job message, leaving for redrive: %s", - message.get("Body"), - ) - return - - # Everything except type/id/headless/proxy rides through to the - # callback unchanged (e.g. sitemap's `depth`) — this worker doesn't need - # to understand it, only the handler (for its own inputs, e.g. `query` - # or `url`) and the Laravel callback (for continuing the workflow) do. - # `proxy` is deliberately excluded: it's only meant for the handler (see - # handlers/serp.py, handlers/generic.py, common.proxy_settings()) — - # there's no reason to send proxy credentials back over the wire in the - # completion callback, and the callback endpoints don't expect it. - passthrough = { - k: v for k, v in job.items() if k not in ("type", "id", "headless", "proxy") - } - - logger.info("job received: type=%s id=%s", job_type, job_id) - try: - result = handler(job) - logger.info( - "job scraped: type=%s id=%s url=%s", job_type, job_id, result["url"] - ) - _report_result(config, job_type, job_id, passthrough, result) - except Exception: - logger.exception( - "job failed (type=%s id=%s) — leaving message for SQS redrive/DLQ", - job_type, - job_id, - ) - return - - sqs.delete_message(QueueUrl=config.queue_url, ReceiptHandle=receipt_handle) - logger.info("job complete: type=%s id=%s", job_type, job_id) - - -def main(argv: list[str] | None = None) -> None: - config = _parse_config(argv) - sqs = boto3.client("sqs", region_name=config.aws_region) - - logger.info( - "worker starting, polling %s, reporting to %s", - config.queue_url, - config.laravel_base_url, - ) - while True: - try: - resp = sqs.receive_message( - QueueUrl=config.queue_url, - MaxNumberOfMessages=1, - WaitTimeSeconds=20, - ) - except Exception: - logger.exception("SQS receive_message failed, backing off 5s") - time.sleep(5) - continue - - for message in resp.get("Messages", []): - _handle_message(config, sqs, message) - - -if __name__ == "__main__": - main() diff --git a/public/css/telescope-theme.css b/public/css/telescope-theme.css index d0d4081..938129e 100644 --- a/public/css/telescope-theme.css +++ b/public/css/telescope-theme.css @@ -52,6 +52,55 @@ --tele-radius-lg: 10px; } +/* + * Dark mode — active when resources/views/vendor/telescope/layout.blade.php + * puts a `dark` class on (driven by the `appearance` cookie via + * TelescopeServiceProvider::configureDarkMode(), which is what picks + * Telescope's own dark dist for this same request). Every rule below this + * point that uses one of these --tele-* variables re-themes for free; the + * values aren't a literal inversion of the light scale above, they're + * hand-picked per actual usage (see the numbered usage list in each + * variable's comment) so the *role* each step plays (page bg, surface, + * border, muted text, strong text…) stays correct once the ramp flips. + * Copied by hand from resources/css/app.css's own `.dark` block, same as + * the light values above — re-copy if the app's dark palette changes. + */ +.dark { + --tele-gray-0: #22252f; /* header / .card surface (was white) */ + --tele-gray-25: #12131a; /* page background (was near-white) */ + --tele-gray-50: #2a2e3a; /* subtle hover/secondary bg (was near-white) */ + --tele-gray-100: #363b4d; /* borders, badge-secondary bg (was near-white) */ + --tele-gray-200: #4b5268; /* header/form-control border (was light gray) */ + --tele-gray-300: #666e85; /* control-action icon fill (was light gray) */ + --tele-gray-400: #8c93a8; /* default sidebar icon fill (unchanged — already mid-tone) */ + --tele-gray-500: #b7bccb; /* nav-pills default text (was mid gray) */ + --tele-gray-600: #d7dae3; /* sidebar/btn-muted/badge default text (was mid gray) */ + --tele-gray-800: #eaecf2; /* nav-pills hover text (was near-black) */ + --tele-gray-900: #f5f6f9; /* body text, logo, btn-muted hover text (was near-black) */ + + /* Pale tints unreadable on a dark surface — same fix app.css's .dark + block applies to the semantic tints, just for our one pink tint + (sidebar active state, new-entry flash, ::selection, focus ring). */ + --tele-pink-100: rgb(228 5 111 / 0.28); + + --tele-success-tint: #0b3a24; + --tele-success-strong: #6ee7b0; + --tele-warning-tint: #3d2a06; + --tele-warning-strong: #fbcf7a; + --tele-danger-tint: #45140f; + --tele-danger-strong: #f8a49c; + --tele-info-tint: #0d2a4d; + --tele-info-strong: #93c5fd; +} + +/* Bootstrap's own $card-border-color isn't parameterised per-theme by + Telescope (resources/sass/styles(-dark).scss), so it stays a translucent + black in both dists — invisible against a dark card. Give dark cards a + visible edge using our own (already re-themed) border step instead. */ +.dark .card { + border-color: var(--tele-gray-100); +} + /* ── Type ─────────────────────────────────────────────────────────────── */ body { font-family: Manrope, ui-sans-serif, system-ui, sans-serif; diff --git a/resources/views/vendor/telescope/layout.blade.php b/resources/views/vendor/telescope/layout.blade.php index 3bc355d..3af9be1 100644 --- a/resources/views/vendor/telescope/layout.blade.php +++ b/resources/views/vendor/telescope/layout.blade.php @@ -1,5 +1,5 @@ - + ($appearance ?? 'system') === 'dark'])> @@ -50,6 +50,14 @@ Telescope's own base.scss only sizes a `.logo svg`, not an `img`), and "Laravel" dropped from the title so the header reads as this app's own brand, not Laravel's. + 5. `` gets a `dark` class when the `appearance` cookie is + `dark`, matching the exact condition TelescopeServiceProvider's + configureDarkMode() uses to pick Telescope's own dark dist (see + that method for why `system` can't be resolved server-side). + This class doesn't drive any Bootstrap styling itself -- it + only gates the dark-specific variable overrides in + telescope-theme.css, which must stay in lockstep with whichever + dist got loaded or the two layers of styling will fight. Re-diff against the vendor file after any `composer update laravel/telescope` in case the upstream markup changes. --}}