diff --git a/.env.example b/.env.example index 83fdcf4..d6e545d 100644 --- a/.env.example +++ b/.env.example @@ -31,5 +31,7 @@ PERPLEXITY_PROXY= PERPLEXITY_CDP_ENDPOINT= # Optional convenience default for --url, if you always scrape the same -# single article. There is no equivalent for --urls-file. -PERPLEXITY_URL= +# single article. Ignored whenever --urls-file or --discover is typed, so it +# never turns a feed or file run into a one-article run. Left commented out +# here so a copied example changes nothing; uncomment to use it. +# PERPLEXITY_URL=https://www.perplexity.ai/page/... diff --git a/.github/ci_checks.py b/.github/ci_checks.py index 38e84d8..f444b85 100644 --- a/.github/ci_checks.py +++ b/.github/ci_checks.py @@ -7,9 +7,9 @@ Unlike tipranks-scraper (no credential model at all), this repo has a real one: a 2Captcha API key, proxy credentials, a Scraping Browser CDP endpoint, a Fingerprint API key. `env_config.py` is the only place any of -these are read from (`.env`, never argv — CLAUDE.md Sec.3), and +these are read from (`.env`, never argv), and `proxy_pool.redact_credentials` is the only place a credentialed URL gets -logged or re-raised (CLAUDE.md Sec.8). This scanner is the third, +logged or re-raised. This scanner is the third, independent check: it never trusts that either of those held everywhere they should have -- it just re-reads every committed/staged file and looks for the shapes a real credential takes, the same way a reviewer @@ -26,8 +26,7 @@ ROOT = Path(__file__).resolve().parents[1] FIXTURE_ALLOWLIST = { ".github/ci_checks.py", # contains the detector patterns themselves - ".env.example", # documents placeholder credentials verbatim, on purpose (CLAUDE.md Sec.17) - "smoke_test.py", # deliberate credential-shaped masking/redaction test fixtures + ".env.example", # documents placeholder credentials verbatim, on purpose } TOKEN_RULES = { @@ -38,7 +37,7 @@ } URL_CREDENTIALS = re.compile(r"\b(?:https?|wss?)://([^\s/@:]+):([^\s/@]+)@", re.I) # The name may itself be quoted, and JSON-escaped inside a string -# (`"api_key": "…"`, `\"api_key\": \"…\"`) — CLAUDE.md §24: the old form +# (`"api_key": "…"`, `\"api_key\": \"…\"`) — the old form # required `=`/`:` right after the bare name and missed both. SECRET_ASSIGNMENT = re.compile( r"\b(?:api[_-]?key|client[_-]?key|twocaptcha[_-]?key|TWOCAPTCHA_KEY|PERPLEXITY_PROXY|" @@ -81,7 +80,7 @@ def _words(token: str) -> set[str]: def _in_virtualenv(relative: str) -> bool: """True when any parent directory holds a `pyvenv.cfg` — a virtualenv under ANY name (`myenv/`, `.venv-selenium/`), not just the names the - regex above knows (CLAUDE.md §23).""" + regex above knows.""" parent = (ROOT / relative).parent while parent != ROOT and ROOT in parent.parents: if (parent / "pyvenv.cfg").exists(): @@ -145,7 +144,7 @@ def _looks_like_type_hint(value: str) -> bool: def scan_text(relative: str, text: str) -> list[tuple[str, int, str]]: """The rules, on one file's text — separate so a check can PLANT a credential and require it to be found (a scanner that cannot fail is - not a scanner, CLAUDE.md §25).""" + not a scanner).""" findings: list[tuple[str, int, str]] = [] for line_no, line in enumerate(text.splitlines(), 1): if True: @@ -168,8 +167,13 @@ def scan_text(relative: str, text: str) -> list[tuple[str, int, str]]: return findings -def scan() -> list[tuple[str, int, str]]: +# A scan that read nothing would print "passed" exactly like a clean one. +MIN_FILES_SCANNED = 10 + + +def scan(stats: dict | None = None) -> list[tuple[str, int, str]]: findings: list[tuple[str, int, str]] = [] + scanned = 0 for relative in repository_files(): if relative in FIXTURE_ALLOWLIST: continue @@ -177,14 +181,16 @@ def scan() -> list[tuple[str, int, str]]: text = (ROOT / relative).read_text(encoding="utf-8") except (UnicodeDecodeError, OSError): continue + scanned += 1 findings.extend(scan_text(relative, text)) + if stats is not None: + stats["scanned"] = scanned return findings def sample_check() -> int: """sample_output.{json,csv} carry exactly the Product columns. Lives - here so tests.yml calls it instead of importing a local module inline - (CLAUDE.md §26).""" + here so tests.yml calls it instead of importing a local module inline.""" import csv import json sys.path.insert(0, str(ROOT)) @@ -200,13 +206,18 @@ def sample_check() -> int: def main() -> int: if "--sample-check" in sys.argv[1:]: return sample_check() - findings = scan() + stats: dict = {} + findings = scan(stats) if findings: for path, line, rule in findings: print(f"{path}:{line}: possible committed credential ({rule})") print("credential scan failed", file=sys.stderr) return 1 - print("credential scan passed") + if stats["scanned"] < MIN_FILES_SCANNED: + print(f"credential scan read only {stats['scanned']} file(s) — refusing to call that clean", + file=sys.stderr) + return 1 + print(f"credential scan passed ({stats['scanned']} files)") return 0 diff --git a/.github/workflows/canary.yml b/.github/workflows/canary.yml index f492847..bcad7db 100644 --- a/.github/workflows/canary.yml +++ b/.github/workflows/canary.yml @@ -14,8 +14,10 @@ jobs: # Two inputs, so one deleted article cannot turn the job red on its own: # 1. `--discover top --max-results 5` — the live feed, always fresh; # 2. one known classic Page (`vars.PERPLEXITY_CANARY_URL` overrides it). - # Blocked, empty, partial and remote-error outcomes are red. Without the - # PERPLEXITY_CDP_ENDPOINT secret the job SKIPS with a notice — not red. + # Blocked, empty, partial and remote-error outcomes are red. So is a run + # WITHOUT the PERPLEXITY_CDP_ENDPOINT secret: a canary that skips green + # tests nothing while its badge says the site works (13 of 13 scheduled + # runs from 2026-10-02 to 2026-10-14 skipped exactly that way). canary-cdp: runs-on: ubuntu-latest env: @@ -26,12 +28,13 @@ jobs: steps: - uses: actions/checkout@v4 - - name: Skip — no PERPLEXITY_CDP_ENDPOINT secret configured + - name: Fail — no PERPLEXITY_CDP_ENDPOINT secret configured if: env.HAS_CDP != 'true' run: | - echo "::notice::Skipping: no PERPLEXITY_CDP_ENDPOINT secret. Set it (a Scraping \ - Browser API ws:// URL) as a repo secret to turn the canary on. This is a SKIP, \ - not a pass or a fail." + echo "::error::Missing repo secret PERPLEXITY_CDP_ENDPOINT (a 2Captcha Scraping \ + Browser API ws:// endpoint). Cloudflare refuses every local browser on this site, \ + so without it the canary cannot reach perplexity.ai and has tested nothing." + exit 1 - uses: actions/setup-python@v5 if: env.HAS_CDP == 'true' diff --git a/.github/workflows/claude-code-review.yml b/.github/workflows/claude-code-review.yml index 6539b57..e0f0ab3 100644 --- a/.github/workflows/claude-code-review.yml +++ b/.github/workflows/claude-code-review.yml @@ -1,7 +1,7 @@ name: Claude Code Review # Automated first-pass review on every PR. Extra to the family core -# (CLAUDE.md §11) — same token-guard and stable-channel reasoning as +# — same token-guard and stable-channel reasoning as # claude.yml, see the comments there. on: @@ -38,10 +38,21 @@ jobs: claude_code_oauth_token: ${{ secrets.CLAUDE_CODE_OAUTH_TOKEN }} path_to_claude_code_executable: ${{ env.CLAUDE_CODE_PATH }} prompt: | - Review this PR against CLAUDE.md's invariants (the ones in - ~/2scraper/CLAUDE.md if this checkout carries a copy, section 8 - in particular) — do not re-litigate them, explain instead of - complying if a change seems to conflict with one. Flag banned - wording (section 12), output-contract drift (section 9), and - anything that touches pagination or captcha handling without - the corresponding smoke_test.py coverage. + Review this PR for correctness against this repository's own + invariants. Do not re-litigate them; if a change seems to + conflict with one, explain the conflict instead of complying: + - A run that finds nothing writes nothing (no overwrite of good + output with an empty file unless --allow-empty). + - Exit codes: 0 ok, 1 crash, 2 bad usage, 3 blocked, 4 genuinely + empty, 5 nothing obtained (fetch/parse/remote failure), + 6 partial (rows, then a failure). Blocked, empty and partial + must stay distinguishable, identically in all three engines. + - Output files and the .meta.json sidecar are written atomically; + CSV cells starting with = + - @ tab CR LF are escaped. + - Never present a guess as a fact: unknown values are null. + - Credentials never reach argv, logs or exception messages. + - Paid captcha solves go through the run's one solve budget. + Flag any product wording the smoke test bans (the remote browser + product is the "Scraping Browser API"), output-contract drift in the Product + row or sidecar, and anything that touches fetching or captcha + handling without matching smoke_test.py coverage. diff --git a/.github/workflows/claude.yml b/.github/workflows/claude.yml index db717dd..9f1429d 100644 --- a/.github/workflows/claude.yml +++ b/.github/workflows/claude.yml @@ -1,7 +1,7 @@ name: Claude # Mention @claude in an issue, PR review, or PR review comment to have -# Claude Code act on it. Extra to the family core (CLAUDE.md §11) — the +# Claude Code act on it. Extra to the family core — the # offline/canary split in tests.yml/canary.yml is what actually gates # merges; this is a convenience on top of it. @@ -37,8 +37,8 @@ jobs: # checked here through step `env` instead — a repo copied without # CLAUDE_CODE_OAUTH_TOKEN gets a clean skip on the steps below # rather than the action's own environment validation FAILING the - # job and leaving a permanently red check on every PR (CLAUDE.md - # §11 — a check that's always red teaches everyone to ignore checks). + # job and leaving a permanently red check on every PR (a check + # that's always red teaches everyone to ignore checks). - name: Check token is configured id: token_check env: diff --git a/.gitignore b/.gitignore index 0abee20..a554271 100644 --- a/.gitignore +++ b/.gitignore @@ -34,7 +34,7 @@ node_modules/ .idea/ .vscode/ -# secrets — `.env*` so a `.env.bak`/`.env.local` copy is never one `git add -A` away (CLAUDE.md §22) +# secrets — `.env*` so a `.env.bak`/`.env.local` copy is never one `git add -A` away *.log .env* !.env.example diff --git a/CHANGELOG.md b/CHANGELOG.md index e8e8617..386f415 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -9,6 +9,52 @@ rather than being a silent violation of that. ## [Unreleased] +> **Behaviour changes.** `--retries`, `--retry-delay`, +> `--delay-between-pages`, `--max-solves` and `--proxy-block-retries` now +> reject a negative value with a usage error (exit 2) in the Selenium and +> pyppeteer engines too, as Playwright already did; `--retries -1` used to +> navigate zero times and end in exit 5. `PERPLEXITY_URL` from the +> environment or `.env` no longer replaces a typed `--discover` or +> `--urls-file`. CSV cells that start with `=`, `+`, `-`, `@`, a tab, CR or +> LF now carry a leading apostrophe (CSV only). `diff_runs.py +> --fail-on-change` can now fire, on a content change. The canary job now +> fails, instead of skipping green, when its `PERPLEXITY_CDP_ENDPOINT` +> secret is missing. + +### Fixed +- Output files and the `.meta.json` sidecar are written to a temporary file + beside the target and renamed over it, so a crash or a full disk during a + write leaves the previous good output intact instead of a truncated file. + A new file gets the usual umask-derived mode; an existing file keeps its + own. +- CSV formula injection: user-written titles, summaries and author names + can no longer run as spreadsheet formulas. Numbers are never escaped, + the JSON output is unchanged, and the sidecar's `csv_cells_escaped` + says how many cells were prefixed. +- `PERPLEXITY_URL` left in `.env` (as the shipped `.env.example` invited) + silently turned `--discover top` or `--urls-file` into a one-article run + with exit 0. It now fills `--url` only when neither is typed, and + `.env.example` ships it commented out. +- `diff_runs.py` compared only the price and currency columns, which are + always empty for an article, so no change was ever reported. It now + reports content changes (`changed`) and counter moves + (`counters_changed`) separately. +- Selenium and pyppeteer accepted negative counts and delays (see above). +- The credential scan now reports how many files it read, fails if that is + implausibly few, and scans `smoke_test.py` too (its credential-shaped test + values are assembled from pieces). +- `python3 env_config.py` now prints only whether a secret is set and its + length, not its first and last characters. +- The Scraping Browser 401 message no longer claims endpoints expire after + "about a day": lifetimes vary, and one lasted more than ten days. + +### Changed +- Code comments, check labels, the CI review prompt and this changelog no + longer point at a maintainer document the public cannot open; each rule + is stated in this repository's own words, and a smoke check keeps it that + way. Comments that described other sites instead of this one were + rewritten or removed. + ## [0.1.0] — 2026-10-01 First public release. Everything below was developed before it; the @@ -158,7 +204,7 @@ entries are kept in the order they were written. `get_event_loop()`. Its "Target closed" / "No session with given id" futures no longer log as ERROR on every page close. - `Product`: removed `follow_up_question_count` (nothing the site serves - carries it — CLAUDE.md §9); added `like_count`, `fork_count`, `summary`, + carries it, and a column that is null on every row should not exist); added `like_count`, `fork_count`, `summary`, `read_time_minutes`, `updated_at`. `category` is the Discover topic. - Real API captures committed as fixtures (`tests/fixtures/*_live_20260930.json`); `sample_output.*` are real rows. @@ -212,7 +258,7 @@ entries are kept in the order they were written. `skyscanner-scraper`, `lidl-scraper`), targeting [Perplexity Pages](https://www.perplexity.ai/) — public, wiki-style articles at `https://www.perplexity.ai/page/{slug}-{id}`. -- **Genuine CLI divergence from every sibling repo (CLAUDE.md §1), +- **Genuine CLI divergence from every sibling repo, documented rather than silent**: this repo takes `--url`/`--urls-file` instead of `--query`/`--category`. Perplexity Pages have no site-search mechanism to point a query at — `robots.txt` explicitly disallows @@ -221,7 +267,7 @@ entries are kept in the order they were written. via its own specific URL. See `page_parser.py`'s module docstring for the full research this is based on. - Ported the family-shared, no-site-knowledge modules near-verbatim from - `lidl-scraper` (CLAUDE.md §7): `output_writer.py` (exit codes, + `lidl-scraper`: `output_writer.py` (exit codes, `STATUS_BY_EXIT`, `finish_run()` precedence — unchanged), `proxy_pool.py`, `captcha_solver.py`, `fingerprint_client.py`, `scraper_api_client.py`, `diff_runs.py`, `env_config.py` (renamed to `PERPLEXITY_*` env keys). @@ -250,9 +296,9 @@ entries are kept in the order they were written. attempted and reported as a failure. - Standard doc set, CI workflows (`tests.yml`'s offline/docker/ engine-smoke jobs, `canary.yml`, `claude.yml`, `claude-code-review.yml`), - `Dockerfile` (Playwright-only, per CLAUDE.md §14), and a fictional + `Dockerfile` (Playwright-only), and a fictional `sample_output.json`/`.csv` (clearly marked as such — no real capture - exists yet, per CLAUDE.md §15). + exists yet). ### Known gap at the initial build (closed 2026-09-30 — see above) - **No live browser capture of perplexity.ai exists yet.** Direct HTTP diff --git a/README.md b/README.md index 7736b0f..8a9c4d0 100644 --- a/README.md +++ b/README.md @@ -184,7 +184,11 @@ previous good file. ## Comparing runs `diff_runs.py old.json new.json` compares two complete runs by `sku`: -articles added and removed. It refuses comparisons that would mislead: +articles added and removed, articles whose content changed (`changed`: +title, author, summary, dates, section/word/source counts) and articles +whose view/like/fork counters moved (`counters_changed`). +`--fail-on-change` exits 1 on a content change only; counters move on +almost every run and never fail it. It refuses comparisons that would mislead: two runs of different selections (including the full URL list and limit), or a `.meta.json` that does not match its file. When a run was capped by `--max-results`, an article missing @@ -199,7 +203,7 @@ the command line. | Option | Default | | |---|---|---| -| `--url` / `--urls-file` / `--discover TOPIC` | | what to scrape, in that order of precedence; only `top` has Discover items for an anonymous visitor | +| `--url` / `--urls-file` / `--discover TOPIC` | | what to scrape, in that order of precedence; only `top` has Discover items for an anonymous visitor. `PERPLEXITY_URL` in `.env` fills `--url` only when neither `--urls-file` nor `--discover` is given | | `--max-results` | 30 | articles to fetch | | `--delay-between-pages` | 2s | pause between articles and between feed pages | | `--format` / `--out` | json / `perplexity_results.` | output format and path | diff --git a/TESTING.md b/TESTING.md index cacc90a..1d7edb8 100644 --- a/TESTING.md +++ b/TESTING.md @@ -189,7 +189,7 @@ git push -u origin main Then, in the GitHub repo's Settings → **Secrets and variables → Actions**: -- `PERPLEXITY_CDP_ENDPOINT` — the canary SKIPS without it (there is no +- `PERPLEXITY_CDP_ENDPOINT` — the canary FAILS without it (there is no local-browser canary: Cloudflare blocks local browsers). - `TWOCAPTCHA_KEY` — optional for the canary. - `CLAUDE_CODE_OAUTH_TOKEN` — for `claude.yml` / `claude-code-review.yml`; diff --git a/captcha_solver.py b/captcha_solver.py index 8eb1784..9058ca5 100644 --- a/captcha_solver.py +++ b/captcha_solver.py @@ -1,7 +1,7 @@ #!/usr/bin/env python3 """captcha_solver.py — detection + solving policy. Comments may name a site for context; no site-specific selector or URL lives here. Per-site markers -are passed in by the caller (see BOT_CHALLENGE_MARKERS in product_parser.py) +are passed in by the caller (see BOT_CHALLENGE_MARKERS in page_parser.py) rather than hardcoded, so this module stays reusable across the family. Policy, matching the family's hard-won rule that "detected != blocking": @@ -22,24 +22,15 @@ for detection/solving: this module owns policy and JS-source construction, never the act of running JS against a real page. -**Until 2026-09-21, no engine in this family actually injected a solved -token back into a locally-launched (non `--cdp-endpoint`) browser at -all** — every engine's own `_maybe_solve_captcha` only logged "solved", a -real gap Roman asked about directly after GeeTest support was added -without it. `build_injection_script()` closes that gap using each widget -type's OWN standard, publicly-documented client-integration convention -(a hidden `g-recaptcha-response` textarea for reCAPTCHA v2, a -`cf-turnstile-response` field for Turnstile, and so on) — **not** anything -confirmed against a real shein.com (or any family site's) actual widget -markup, which has never been captured for ANY type. It is the same -"standard convention, explicitly unconfirmed against this site" honesty -posture `shein_parser.py`'s own DOM fallback selectors use. reCAPTCHA v3 -has no such convention — it's invisible, and the token is typically -consumed the instant the SITE'S OWN JavaScript resolves -`grecaptcha.execute()`'s promise (often straight into an XHR, never read -back off a DOM element) — so `build_injection_script()` returns `None` for -it rather than guess at site-specific code, which this shared module's own -charter says does not belong here. +`build_injection_script()` returns a token-injection script for a locally +launched browser using each widget type's OWN standard, publicly documented +client-integration convention (a hidden `g-recaptcha-response` textarea for +reCAPTCHA v2, a `cf-turnstile-response` field for Turnstile, and so on). It +is not confirmed against this site's markup. reCAPTCHA v3 has no such +convention (the site's own JavaScript consumes the token directly), so it +returns `None` rather than guess at site-specific code. In this repo the +engines do not deliver tokens locally at all: local solving is disabled +and no paid task is created (see each engine's `_maybe_solve_captcha`). Over `--cdp-endpoint`, none of this applies: 2Captcha's own `Captcha.setAutoSolve` CDP domain (see each engine's own @@ -96,48 +87,21 @@ class CaptchaType(str, Enum): _RECAPTCHA_V3_LOADER_RE = re.compile(r"recaptcha/api\.js\?render=([\w-]+)") _RECAPTCHA_EXPLICIT_LOADER_RE = re.compile(r"recaptcha/api\.js\?render=explicit") -# reCAPTCHA v2's sitekey doesn't always ship as static `
` markup, the only shape -# `_SITEKEY_PATTERNS[RECAPTCHA_V2]` above can see — CONFIRMED live on -# shein.com, 2026-09-21 (see README "Known limitations" and Roman's own -# question about whether other captcha vendors are in play): the site -# loads `https://www.google.com/recaptcha/api.js` and has a live -# `window.grecaptcha` v2 object (`render`/`execute`/`getResponse`/ -# `reset`/`ready` — not `.enterprise`) on every normal page, but the -# sitekey itself lives in a JS config object under a site-chosen key -# name (`window.gbCommonInfo.GOOGLE_VERIFY_SITEKEY`), presumably handed -# to `grecaptcha.render()` programmatically later (e.g. after a -# suspicious login attempt) rather than rendered into the initial HTML -# at all. The static-markup regex above would silently MISS this real, -# live sitekey — a real detection gap, not a hypothetical one. reCAPTCHA -# sitekeys have a fixed, distinctive shape regardless of what variable -# or markup carries them (`6L` + 38 more URL-safe-base64 characters, 40 -# total, confirmed against Google's own documented format and against -# shein.com's actual `6LcoBR4UAAAAAIi5xU3U_q37C3nFaSckeMaT-P5j`) — so -# this searches for that shape ANYWHERE in the page, gated on the v2 -# loader actually being present so it can't misfire on an unrelated -# 40-character token elsewhere that happens to start with "6L". +# reCAPTCHA v2's sitekey doesn't always ship as static `data-sitekey` +# markup: a site can keep it in a JS config object and hand it to +# `grecaptcha.render()` later, which the static regex above misses. A +# reCAPTCHA sitekey has a fixed shape (`6L` + 38 URL-safe-base64 +# characters, 40 total), so this searches for that shape anywhere in the +# page, gated on the v2 loader being present so an unrelated token cannot +# misfire. _RECAPTCHA_SITEKEY_ANYWHERE_RE = re.compile(r"\b(6L[\w-]{38})\b") _RECAPTCHA_V2_LOADER_RE = re.compile(r"recaptcha/api\.js") -# GeeTest — added 2026-09-21 after shein.com's own risk gateway -# (captcha_type=909, see shein_parser.BOT_CHALLENGE_MARKERS) correlated -# with GeeTest via three circumstantial signals documented in -# shein_parser.py's module docstring (the site's global stylesheet defines -# .geetest_wind/.geetest_panel on EVERY page, robots.txt disallows -# /geetest/, and the numeric captcha_type code is consistent with GeeTest's -# own convention) — NOT a confirmed vendor string the way every other -# CaptchaType below was found (an actual `cf-turnstile`/`g-recaptcha`/ -# `h-captcha` class or loader URL, seen live). No family site, shein.com -# included, has ever had its actual GeeTest widget markup captured — only -# a waiting/redirect shell page, never what renders when a real challenge -# is actually shown. These two patterns are therefore UNCONFIRMED -# best-effort, built from GeeTest's own public client-integration -# documentation (a `data-captcha-id`/`captchaId` attribute for v4; `gt` -# and `challenge` fields together for v3 — both are 32-character -# hex-like ids in GeeTest's own docs), not a shein.com-specific fact. -# `# TODO: verify live` applies here exactly like shein_parser.py's own -# DOM fallback selectors — update these the moment a real capture exists. +# GeeTest: UNCONFIRMED best-effort patterns built from GeeTest's own public +# client-integration documentation (a `data-captcha-id`/`captchaId` +# attribute for v4; `gt` and `challenge` together for v3, each a 32-char +# hex id). No real GeeTest widget has been captured on a site this code +# runs against; update these when one is. _GEETEST_V4_ID_PATTERNS = ( re.compile(r'data-captcha-id=["\']([a-f0-9]{32})["\']'), re.compile(r'captchaId["\']?\s*:\s*["\']([a-f0-9]{32})["\']'), @@ -168,7 +132,7 @@ class CaptchaSignal: # `page.content()` over --cdp-endpoint captures THOSE injected