diff --git a/.github/ISSUE_TEMPLATE/bug_report.yml b/.github/ISSUE_TEMPLATE/bug_report.yml index 9b07f80..4b58f5a 100644 --- a/.github/ISSUE_TEMPLATE/bug_report.yml +++ b/.github/ISSUE_TEMPLATE/bug_report.yml @@ -56,8 +56,6 @@ body: options: - category - product - - transfers - - player - not applicable validations: required: true diff --git a/.github/ISSUE_TEMPLATE/site_changed.yml b/.github/ISSUE_TEMPLATE/site_changed.yml index 5d28ebf..50acea5 100644 --- a/.github/ISSUE_TEMPLATE/site_changed.yml +++ b/.github/ISSUE_TEMPLATE/site_changed.yml @@ -25,8 +25,6 @@ body: options: - category - product - - transfers - - player - not sure / more than one validations: required: true @@ -37,7 +35,7 @@ body: label: What stopped matching options: - Rows (0 rows on a page that visibly has data) - - A specific field is null or wrong (price, club, position, fee, ...) + - A specific field is null or wrong (price, currency, shade, availability, ...) - Price coverage dropped below the floor logged in the run output - Pagination stops early or repeats a page - The bot-challenge detector finds nothing on a page that has one @@ -62,13 +60,13 @@ body: description: | Whichever of these you can get. A capture beats a description. - - The relevant table row's HTML (right-click the row on the real site - -> Copy -> Copy outerHTML), or the output of + - The relevant product tile's HTML (right-click the tile on the real + site -> Copy -> Copy outerHTML), or the output of `--dump-html capture.html` against this URL. - For a field problem: the value you got, and the value on the page. - - For a player-profile problem: whether the page has a - `itemtype="https://schema.org/Person"` block at all (view source - and search for it). + - For a product-page problem: whether the page has a + `", re.IGNORECASE | re.DOTALL) # The site's own asset signature. Present 276-512 times on every one of the # 14 served captures (measured 2026-09-18) and structurally absent from an -# interstitial, which is CLAUDE.md §8's "a served page is built out of the -# site's own assets" applied here. +# interstitial: a served page is built out of the site's own assets. _SITE_ASSET_MARKER = "Sites-givenchy-beauty" # Positive content hooks, each measured to be unambiguous: @@ -556,6 +554,18 @@ def is_self_clearing_challenge(html: str) -> bool: _LISTING_HOOK = "js-ProductItem" _PRODUCT_HOOK = "js-pdp" +# The site's own "this address does not exist" page. Measured 2026-10-14 with +# plain curl: `/fr/fr/makeup/lips/` and a made-up `/us/makeup//` +# both answer HTTP 410 with a full storefront page (275-330 asset +# references) whose Demandware page context reads +# `{"title":"General Error","type":"error","ns":"error"}`, and neither +# carries a listing or product hook. A served-but-empty category +# (`/es/maquillaje/sets-de-regalo/`, HTTP 200) reads `"ns":"category"` +# instead, so this marker does not turn a genuinely empty listing into a +# missing one. It is what lets Selenium and pyppeteer, which have no HTTP +# status to read, reach the same answer as the engines that do. +_NOT_FOUND_HOOK = '"type":"error","ns":"error"' + def detect_bot_challenge(html: str, url: Optional[str] = None) -> Optional[str]: """The challenge vendor found in `html`, or None.""" @@ -583,9 +593,10 @@ def _header(headers: Optional[Dict[str, str]], name: str) -> Optional[str]: def detect_page_state(html: str, status: Optional[int] = None, url: Optional[str] = None, headers: Optional[Dict[str, str]] = None) -> str: - """content | blocked | captcha | empty — see page_flow.STATE_POLICY. + """content | blocked | captcha | empty | not_found — see + page_flow.STATE_POLICY. - Ordered by how much each check PROVES, per CLAUDE.md §17, not by cost: + Ordered by how much each check PROVES, not by cost: 1. `x-amzn-waf-action` is the front naming its own action, so nothing overrides it. @@ -594,7 +605,7 @@ def detect_page_state(html: str, status: Optional[int] = None, assets, and an engine enumerating one must not read that as a block. 4. The site's OWN hooks are an unambiguous positive, and run BEFORE the vendor scan because the scan can match a marker that is not the - site's at all (the auto-solve extension, §19). + site's at all (the auto-solve extension). 5. The vendor scan then only refines the reason for a page that already failed to look like content. 6. A non-2xx status comes last among the positive tests: it proves @@ -615,6 +626,17 @@ def detect_page_state(html: str, status: Optional[int] = None, if not html: return "blocked" + # A wrong or retired address. Terminal and NOT a block: retrying it + # cannot help, and calling it blocked (exit 3) sends the reader to buy a + # proxy when the thing to fix is the URL. Read from the status where an + # engine has one, and from the site's own error page where it does not. + if status in (404, 410): + return "not_found" + if (status is None and _SITE_ASSET_MARKER in html + and _NOT_FOUND_HOOK in html + and _LISTING_HOOK not in html and _PRODUCT_HOOK not in html): + return "not_found" + status_ok = status is None or 200 <= status < 300 if status_ok and (" Optional[float]: """Discount computed from the two figures, never read from the badge. - The site prints its own `-21%`, but §4's Farfetch lesson is that a + The site prints its own `-21%`, but on a sibling site a printed percentage can be the first of two compounding discounts. The computed value is the one that matches what the customer pays. Returns None — not 0 and not a negative — when the two figures are not what they @@ -912,7 +933,7 @@ def _tile_availability(article, gtm: dict) -> Optional[str]: def _jsonld_blocks(soup) -> List[dict]: """Every JSON-LD object on the page, flattened. - Handles the shapes CLAUDE.md §4 lists as legal and naive-parser-fatal: a + Handles the shapes that are legal schema.org and fatal to a naive parser: a block may be a list, and products may hide inside `@graph` rather than at the top level — which loses every product SILENTLY, reporting an empty page for an unread format. A malformed block costs itself, not the run. @@ -950,7 +971,7 @@ def _first_offer(node: dict) -> dict: """`offers` as a dict, whatever legal shape it arrived in. `"offers": null` is an explicit null, so a `.get("offers", {})` default - does NOT apply — that is the `AttributeError` §4 names first. A list may + does NOT apply — that is an `AttributeError`. A list may also hold non-dicts. """ offers = node.get("offers") @@ -1004,7 +1025,7 @@ def parse_product(html: str, base_url: str) -> Optional[Product]: JSON-LD is primary where it exists — on a priced locale it carries the price, the currency, the sku and the canonical offer URL, and `offers.priceCurrency` is a FACT the DOM is not allowed to overwrite - (CLAUDE.md §4). On `int/en` and `ru` there is no JSON-LD at all and no + On `int/en` and `ru` there is no JSON-LD at all and no price to read, so the DOM supplies name, size and breadcrumb and the row comes out with `price=None` rather than not at all. """ @@ -1057,8 +1078,7 @@ def parse_product(html: str, base_url: str) -> Optional[Product]: original = list_price url = (_clean(offer.get("url")) or canonical_url or base_url) - # `sku` may be absent from an otherwise-valid Product block (CLAUDE.md - # §4's last shape), so `mpn` and then the URL id are the fallbacks. + # `sku` may be absent from an otherwise-valid Product block, so `mpn` and then the URL id are the fallbacks. sku = None if node: sku = _clean(node.get("sku")) or _clean(node.get("mpn")) diff --git a/puppeteer_scraper.py b/puppeteer_scraper.py index 0ebc9cf..517fd70 100644 --- a/puppeteer_scraper.py +++ b/puppeteer_scraper.py @@ -46,7 +46,7 @@ from urllib.parse import urlparse from typing import List, Optional -# At module level, deliberately — see CLAUDE.md §10 and §16. The offline +# At module level, deliberately. The offline # suite guards `import puppeteer_scraper` behind try/except ImportError and # REPORTS the skip; CI's engine-smoke job fails on any reported skip. That # mechanism only catches a broken import if importing this module actually @@ -339,7 +339,7 @@ def _next_page_href(session) -> Optional[str]: def handle_captcha_if_present(session, args, ready_selector: str, - proxy=None) -> bool: + proxy=None, budget=None) -> bool: bridge, page = session.bridge, session.page html = _driver(session)["content"]() if html is None: @@ -365,7 +365,7 @@ def handle_captcha_if_present(session, args, ready_selector: str, logger.info("%s detected via %s, but content is already on the " "page — not solving it.", challenge.kind, challenge.source) return False - # Section 19: "unsolvable" is a property of a PAGE -- it means the page + # "Unsolvable" is a property of a PAGE -- it means the page # carries no widget. An AWS WAF CHALLENGE-action page is exactly that: # challenge.js only, no puzzle rendered, nothing for a solver to work # on. Sending it anyway buys a token for a widget that was never there, @@ -390,6 +390,11 @@ def handle_captcha_if_present(session, args, ready_selector: str, "being solved. 2captcha does solve this type — set " "TWOCAPTCHA_KEY (or pass --twocaptcha-key) to use it.") return False + if budget is not None and not budget.spend(): + logger.warning("This page's paid-solve allowance is used up (%d of " + "%d) — not buying another token.", budget.spent, + budget.limit) + return False try: token = solve_recaptcha(challenge, args.twocaptcha_key, api_version=args.captcha_api, @@ -423,15 +428,24 @@ def _fetch_one_page(session, args, pool, page_num: int, url: str) -> PageOutcome ready_count = page_flow.ready_count(args.mode) block_retries = args.proxy_block_retries if (pool and len(pool) > 1) else 0 + # One allowance for this page, kept across retries and exit rotations. + solve_budget = page_flow.SolveBudget() html, state, load_failed = None, "ok", False for block_attempt in range(block_retries + 1): logger.info("Fetching page %d: %s", page_num, url) load_failed = False + resp_status = None for attempt in range(1, max(1, args.retries) + 1): try: - bridge.run(page.goto(url, {"waitUntil": "domcontentloaded", - "timeout": 60000})) + # The response is kept for its HTTP status: a 404/410 is a + # wrong address, not a block, and the classifier can only + # tell that from the status or from the site's error page. + response = bridge.run(page.goto(url, {"waitUntil": "domcontentloaded", + "timeout": 60000})) + resp_status = getattr(response, "status", None) + if not isinstance(resp_status, int): + resp_status = None load_failed = False break except Exception as e: # noqa: BLE001 — pyppeteer raises many types @@ -457,18 +471,25 @@ def _fetch_one_page(session, args, pool, page_num: int, url: str) -> PageOutcome if load_failed: break - if handle_captcha_if_present(session, args, ready_selector, - proxy=pool.current if pool else None): + if solve_budget.remaining() and handle_captcha_if_present( + session, args, ready_selector, + proxy=pool.current if pool else None, budget=solve_budget): time.sleep(1) + # The document now in the page replaced the challenge, so the + # status of the original navigation no longer describes it. + resp_status = None html = d["content"]() or "" if is_self_clearing_challenge(html): # Shared with the other two engines -- see page_flow for why the # wait condition is "the site's content hooks appeared", not # "the challenge markers went away". - html, _cleared = page_flow.wait_out_self_clearing_challenge( + html, cleared = page_flow.wait_out_self_clearing_challenge( d["content"], d["sleep"], url=page.url, log=logger.info) - state = page_flow.classify(html, url=page.url) + if cleared: + # The original status described the interstitial. + resp_status = None + state = page_flow.classify(html, status=resp_status, url=page.url) if not page_flow.should_retry(state): break @@ -489,6 +510,15 @@ def _fetch_one_page(session, args, pool, page_num: int, url: str) -> PageOutcome outcome.state = state + if state == "not_found": + logger.error("%s does not exist on this site (HTTP 404/410, or the " + "site's own error page) — nothing to parse and nothing " + "a retry or a proxy would change. Category paths differ " + "between locales: take them from the locale's own " + "sitemap_1-category.xml.", url) + outcome.final_url = page.url + return outcome + if state == "blocked": debug_html = f"{args.out}_page{page_num}_debug.html" with open(debug_html, "w", encoding="utf-8") as f: @@ -580,9 +610,9 @@ def scrape(args) -> int: "the remote browser has its own exit.") pool = None if args.concurrency > 1: - logger.warning("--concurrency is ignored in this engine: parallel " - "page fetching (--mode transfers) is implemented in " - "playwright_scraper.py. Running sequentially.") + logger.warning("--concurrency is ignored: no mode on this site has a " + "second page to fetch in parallel (a listing's own " + "paging is robots-disallowed). Running sequentially.") bridge = _AsyncBridge() session = _Session(bridge, args, pool).open() @@ -594,6 +624,8 @@ def scrape(args) -> int: stop_reason = ("page_load_timeout" if first.load_failed else f"blocked_{first.blocked_by}") blocked = first.blocked_by is not None + elif first.state == "not_found": + stop_reason = "not_found" elif args.mode not in PAGINATED_MODES: pass else: @@ -671,6 +703,18 @@ def scrape(args) -> int: start_url=args.url, final_url=final_url) +def _non_negative_float(value): + """argparse type for a delay in seconds: zero is fine, negative is not + (`time.sleep` raises ValueError on a negative value, mid-run).""" + try: + n = float(value) + except ValueError: + raise argparse.ArgumentTypeError(f"must be a number, got {value!r}") + if n < 0 or n != n: + raise argparse.ArgumentTypeError(f"must be 0 or greater, got {value}") + return n + + def _non_negative(value): """argparse type for a count that may be zero but never negative.""" n = int(value) @@ -703,23 +747,23 @@ def parse_args(): help="The exact page to read. Required for --mode product. " "Falls back to $GIVENCHY_URL, then to what " "--category builds.") - p.add_argument("--category", default="makeup/lips", + p.add_argument("--category", default=None, help="Category path WITHOUT the locale prefix, e.g. " "makeup/lips. Take real paths from the locale's own " "sitemap_1-category.xml: they are NOT shared between " "locales.") - p.add_argument("--site-locale", default="us", choices=list(LOCALES), + p.add_argument("--site-locale", default=None, choices=list(LOCALES), help="The locale PATH PREFIX on www.givenchybeauty.com " "(default us). Not --locale, which is the browser's " "own. int/en and ru publish no prices.") p.add_argument("--pages", type=_positive_int, default=1, help="Accepted for family compatibility; a listing is one " "page on this site (robots-disallowed paging).") - p.add_argument("--delay", type=float, default=2.0) + p.add_argument("--delay", type=_non_negative_float, default=2.0) p.add_argument("--concurrency", type=_positive_int, default=1, help="Ignored in this engine — see playwright_scraper.py.") p.add_argument("--retries", type=_positive_int, default=3) - p.add_argument("--retry-delay", type=float, default=2.0) + p.add_argument("--retry-delay", type=_non_negative_float, default=2.0) p.add_argument("--format", choices=["json", "csv", "both"], default="both") p.add_argument("--out", default="givenchy_products") p.add_argument("--proxy", default=None) @@ -730,10 +774,10 @@ def parse_args(): "--cdp-endpoint session (measurement mode: it " "lets a challenge be met and left unsolved, " "which is what a control needs).") - p.add_argument("--proxy-sessions", type=int, default=None, + p.add_argument("--proxy-sessions", type=_non_negative, default=None, help="With --proxy pointing at a 2Captcha proxy gateway, mint this many session-pinned exits from that one credential instead of keeping a file of them. Each session is a different exit address (measured 2026-09-17: ten sessions, ten distinct addresses). Has no effect with --proxy-file, and is refused for a non-2Captcha host.") p.add_argument("--proxy-shuffle", action="store_true") - p.add_argument("--proxy-block-retries", type=int, default=2) + p.add_argument("--proxy-block-retries", type=_non_negative, default=2) p.add_argument("--twocaptcha-key", default=None) p.add_argument("--fingerprint", action="store_true") p.add_argument("--fp-tags", default="Windows") @@ -763,6 +807,21 @@ def parse_args(): "Google Chrome'. Not a credential, so the command " "line is the right place for it.") args = p.parse_args() + # What the user TYPED, captured before env_config fills anything in: + # afterwards a $GIVENCHY_URL from .env is indistinguishable from a typed + # --url. Typed builder flags (--category/--site-locale) build the URL + # themselves, so an environment URL can no longer silently override them. + typed_url = args.url + builder_typed = args.category is not None or args.site_locale is not None + if args.category is None: + args.category = "makeup/lips" + if args.site_locale is None: + args.site_locale = "us" + if typed_url and builder_typed: + logger.warning("--url was given, so --category/--site-locale are " + "ignored: the URL is read exactly as typed.") + elif builder_typed and args.mode != "product": + args.url = f"{BASE}/{args.site_locale}/{args.category.strip('/')}/" env_config.apply(args) if not args.url: diff --git a/scraper_api_client.py b/scraper_api_client.py index 3920571..6ceb758 100644 --- a/scraper_api_client.py +++ b/scraper_api_client.py @@ -85,7 +85,7 @@ PAGINATED_MODES = () -# Global on purpose, not a single find/partition -- see CLAUDE.md §8: "A +# Global on purpose, not a single find/partition: "A # masker that handles the first occurrence prints the password the other # four times and looks like it is working." Same pattern the three browser # engines use in their own _mask_credentials. @@ -98,20 +98,62 @@ def _mask_credentials(text: str) -> str: return _CREDENTIALS_IN_URL_RE.sub(r"\1***:***@", text or "") -def _build_wait_for(args) -> Optional[str]: - """`waitFor` must be a JSON STRING (double-encoded), per the API docs. - Passing a nested object is silently wrong. +def _build_wait_for(args) -> Optional[dict]: + """`waitFor` is sent as a JSON OBJECT. + + This client used to send it as a double-encoded JSON string, which is + what an older reading of the API docs said. The API now refuses the + string form with HTTP 422 ("params.waitFor must be an object") and still + bills the call, while the object form answers 200 -- so every --wait-* + run was a paid exit 5. Default (no flag): wait for the DOM. Not needed for a page that arrives as complete HTML with no hydration (this site, per the module docstring) -- --wait-text/--wait-element exist for the day that stops being true, or for a --cdp-url run through a challenge page.""" if args.wait_text: - return json.dumps({"text": args.wait_text}) + return {"text": args.wait_text} if args.wait_element: - return json.dumps({"element": args.wait_element, "checkVisible": True}) + return {"element": args.wait_element, "checkVisible": True} if args.wait_state: - return json.dumps({"state": args.wait_state}) + return {"state": args.wait_state} + return None + + +# Credentials embedded ANYWHERE in a blob of text, and every occurrence. +_KEY_IN_TEXT_RE = re.compile( + r"((?:client)?key|token|api[_-]?key)=([^&\s'\"]{6,})", re.IGNORECASE) + + +def _redact_debug_header(value: str) -> str: + """The x-debug header, safe to log. + + The API echoes back the task it ran in this header, so a run driven + through a credentialled --cdp-url put that endpoint's username and + password into the log verbatim, and a key passed as a query parameter + would go the same way. Redaction rather than an allowlist of fields: the + header's shape is the API's, and an allowlist would silently drop the + cost and timing figures this is logged for the first time a field is + added. + """ + return _KEY_IN_TEXT_RE.sub(r"\1=***", _mask_credentials(value)) + + +def _target_status(body: dict) -> Optional[int]: + """The TARGET page's HTTP status from a `format=json` response. + + The target's status is `http_code`. `status` is the API's own verdict + string ("success", "warn", ...), which can read `warn` on a 200 and on a + 403 alike; handing it to the page classifier crashed the run with + `TypeError: '<=' not supported between 'int' and 'str'`. `status` is used + only when it is itself an integer. + """ + code = body.get("http_code") + if isinstance(code, int) and not isinstance(code, bool): + return code + legacy = body.get("status") + if isinstance(legacy, int) and not isinstance(legacy, bool): + return legacy return None @@ -120,14 +162,14 @@ def fetch_html(args, url: str, timeout: int): "task_type": "scrape", "url": url, "data_format": "raw", # we want HTML; product_parser does the rest - "format": "json", # so we get {"status", "headers", "body"} + "format": "json", # {"status": API verdict, "http_code": target status, "headers", "body"} "timeout": min(timeout, MAX_API_TIMEOUT), } wait_for = _build_wait_for(args) if wait_for: payload["waitFor"] = wait_for - logger.info("waitFor: %s", wait_for) + logger.info("waitFor: %s", json.dumps(wait_for, ensure_ascii=False)) if args.cdp_url: payload["cdpurl"] = args.cdp_url @@ -150,7 +192,7 @@ def fetch_html(args, url: str, timeout: int): # cost of the call shows up. debug = resp.headers.get("x-debug") if debug: - logger.info("x-debug: %s", debug) + logger.info("x-debug: %s", _redact_debug_header(debug)) if resp.status_code != 200: # 422 = task ran but errored (this is what a bad/unreachable @@ -165,9 +207,10 @@ def fetch_html(args, url: str, timeout: int): body = resp.json() html = body.get("body") or "" - upstream_status = body.get("status") + upstream_status = _target_status(body) upstream_headers = body.get("headers") or {} - logger.info("Upstream page status %s, %d bytes of HTML.", upstream_status, len(html)) + logger.info("Upstream page HTTP status %s (API verdict %r), %d bytes of " + "HTML.", upstream_status, body.get("status"), len(html)) # The STATUS and the HEADERS both travel WITH the HTML, neither thrown # away. The Scraper API hands back `{"status", "headers", "body"}` and # until v0.4.1 this function kept the first two thirds of that and @@ -220,6 +263,13 @@ def _fetch_one_page(args, page_num: int, url: str): state = detect_page_state(html, status=upstream_status, url=url, headers=upstream_headers) + if state == "not_found": + # Terminal, not a block: a retry cannot make the address exist. + logger.error("%s does not exist on this site (upstream HTTP %s) " + "— nothing to parse. Category paths differ between " + "locales: take them from the locale's own " + "sitemap_1-category.xml.", url, upstream_status) + return [], None, upstream_status, html if state == "blocked": # Retried like a vendor challenge, NOT returned immediately: a # transient upstream error (a non-2xx status, or an empty body) @@ -267,12 +317,13 @@ def _fetch_one_page(args, page_num: int, url: str): coverage = priced / len(rows) logger.info("Price coverage on page %d: %d/%d (%.0f%%).", page_num, priced, len(rows), 100.0 * coverage) - # See page_flow.PRICE_COVERAGE_FLOOR for why transfers is excluded. + # See page_flow.PRICE_COVERAGE_FLOOR for why showcase locales are + # excluded. if coverage < page_flow.PRICE_COVERAGE_FLOOR: logger.warning("Price coverage on page %d (%.0f%%) is below the " "%.0f%% floor for --mode %s — check for a " "parsing regression rather than assuming these " - "players are all free agents.", + "products are genuinely unpriced.", page_num, 100.0 * coverage, 100.0 * page_flow.PRICE_COVERAGE_FLOOR, args.mode) @@ -299,6 +350,9 @@ def scrape(args) -> int: stop_reason = f"blocked_{blocked_by}" blocked = True pages_failed.append(1) + elif not rows and detect_page_state(page1_html, status=_status, + url=args.url) == "not_found": + stop_reason = "not_found" else: pages_completed = 1 fresh = dedupe_by_key(rows, seen_keys, key="sku") @@ -308,11 +362,10 @@ def scrape(args) -> int: if args.pages > 1 and args.mode in PAGINATED_MODES: # Same "verify first" check the browser engines make off the # live DOM (page_flow.pagination_is_addressable) -- made here - # off the fetched HTML instead. See page_flow.py's "gallery - # This site has no addressable pagination at all (see the - # module docstring), so this always answers False. - # transfers), and only rarely is not -- this checks which is - # true FOR THIS RUN rather than assuming either way. + # off the fetched HTML instead. This site has no addressable + # pagination at all (see the module docstring), so this always + # answers False today; it is kept so the answer is checked per + # run rather than assumed. soup = BeautifulSoup(page1_html, "html.parser") next_el = soup.select_one("link[rel='next'], a[rel='next']") site_next = next_el.get("href") if next_el else None @@ -414,6 +467,18 @@ def main() -> int: return EXIT_REMOTE_API_ERROR +def _non_negative_float(value): + """argparse type for a delay in seconds: zero is fine, negative is not + (`time.sleep` raises ValueError on a negative value, mid-run).""" + try: + n = float(value) + except ValueError: + raise argparse.ArgumentTypeError(f"must be a number, got {value!r}") + if n < 0 or n != n: + raise argparse.ArgumentTypeError(f"must be 0 or greater, got {value}") + return n + + def _non_negative(value): """argparse type for a count that may be zero but never negative.""" n = int(value) @@ -457,20 +522,20 @@ def parse_args(): help="Override the URL this mode would otherwise build. " "Falls back to $GIVENCHY_URL, then to the " "mode's own canonical URL.") - p.add_argument("--category", default="makeup/lips", + p.add_argument("--category", default=None, help="Category path without the locale prefix, e.g. " "makeup/lips.") - p.add_argument("--site-locale", default="us", choices=list(LOCALES), + p.add_argument("--site-locale", default=None, choices=list(LOCALES), help="Locale path prefix on www.givenchybeauty.com.") p.add_argument("--pages", type=_positive_int, default=1, help="Listing pages to crawl. Checked per run against " "page 1's own next-link (page_flow.pagination_is_" "addressable). A listing is one page on this site: " "its own paging is robots-disallowed.") - p.add_argument("--delay", type=float, default=1.0, help="Delay between pages, seconds") + p.add_argument("--delay", type=_non_negative_float, default=1.0, help="Delay between pages, seconds") p.add_argument("--format", choices=["json", "csv", "both"], default="both") p.add_argument("--out", default="givenchy_products_scraperapi", help="Output file prefix") - p.add_argument("--timeout", type=int, default=60, + p.add_argument("--timeout", type=_positive_int, default=60, help=f"API-side task timeout in seconds (1-{MAX_API_TIMEOUT}, default 60)") p.add_argument("--cdp-url", default=None, help="Route the fetch through an existing browser session over CDP " @@ -490,7 +555,7 @@ def parse_args(): p.add_argument("--retries", type=_positive_int, default=1, help="Extra attempts if a bot-challenge page comes back. Each attempt " "is a separate billable task, so this defaults to 1.") - p.add_argument("--retry-delay", type=int, default=10, help="Seconds between retries") + p.add_argument("--retry-delay", type=_non_negative_float, default=10, help="Seconds between retries") p.add_argument("--dump-html", default=None, help="Save the exact HTML the parser is given (always, even on success)") args = p.parse_args() @@ -498,6 +563,21 @@ def parse_args(): # This client uses --key rather than --twocaptcha-key, so the env # mapping is spelled out instead of defaulted -- same pattern this # family's mediamarkt-scraper uses for its own scraper_api_client.py. + # What the user TYPED, captured before env_config fills anything in: + # afterwards a $GIVENCHY_URL from .env is indistinguishable from a typed + # --url. Typed builder flags (--category/--site-locale) build the URL + # themselves, so an environment URL can no longer silently override them. + typed_url = args.url + builder_typed = args.category is not None or args.site_locale is not None + if args.category is None: + args.category = "makeup/lips" + if args.site_locale is None: + args.site_locale = "us" + if typed_url and builder_typed: + logger.warning("--url was given, so --category/--site-locale are " + "ignored: the URL is read exactly as typed.") + elif builder_typed and args.mode != "product": + args.url = f"{BASE}/{args.site_locale}/{args.category.strip('/')}/" env_config.apply(args, keys={ "TWOCAPTCHA_KEY": "key", "GIVENCHY_CDP_ENDPOINT": "cdp_url", @@ -505,7 +585,7 @@ def parse_args(): }) if key_on_argv: - # CLAUDE.md §8: credentials never reach argv. Warn rather than refuse, + # Credentials never reach argv. Warn rather than refuse, # because refusing would break a caller mid-pipeline -- but say it # plainly, because `ps` makes this readable to every other user on the # box and it lands in shell history. diff --git a/selenium_scraper.py b/selenium_scraper.py index c67bf80..da04917 100644 --- a/selenium_scraper.py +++ b/selenium_scraper.py @@ -276,7 +276,7 @@ def _next_page_href(session) -> Optional[str]: def handle_captcha_if_present(session, args, ready_selector: str, - proxy=None) -> bool: + proxy=None, budget=None) -> bool: driver = session.driver d = _driver(session) html = d["content"]() @@ -304,7 +304,7 @@ def handle_captcha_if_present(session, args, ready_selector: str, logger.info("%s detected via %s, but content is already on the " "page — not solving it.", challenge.kind, challenge.source) return False - # Section 19: "unsolvable" is a property of a PAGE -- it means the page + # "Unsolvable" is a property of a PAGE -- it means the page # carries no widget. An AWS WAF CHALLENGE-action page is exactly that: # challenge.js only, no puzzle rendered, nothing for a solver to work # on. Sending it anyway buys a token for a widget that was never there, @@ -329,6 +329,11 @@ def handle_captcha_if_present(session, args, ready_selector: str, "being solved. 2captcha does solve this type — set " "TWOCAPTCHA_KEY (or pass --twocaptcha-key) to use it.") return False + if budget is not None and not budget.spend(): + logger.warning("This page's paid-solve allowance is used up (%d of " + "%d) — not buying another token.", budget.spent, + budget.limit) + return False try: token = solve_recaptcha(challenge, args.twocaptcha_key, api_version=args.captcha_api, @@ -371,6 +376,8 @@ def _fetch_one_page(session, args, pool, page_num: int, url: str) -> PageOutcome html, state, load_failed = None, "ok", False block_retries = args.proxy_block_retries if (pool and len(pool) > 1) else 0 + # One allowance for this page, kept across retries and exit rotations. + solve_budget = page_flow.SolveBudget() for block_attempt in range(block_retries + 1): logger.info("Fetching page %d: %s", page_num, url) @@ -405,8 +412,9 @@ def _fetch_one_page(session, args, pool, page_num: int, url: str) -> PageOutcome if load_failed: break - if handle_captcha_if_present(session, args, ready_selector, - proxy=pool.current if pool else None): + if solve_budget.remaining() and handle_captcha_if_present( + session, args, ready_selector, + proxy=pool.current if pool else None, budget=solve_budget): time.sleep(1) html = d["content"]() or "" @@ -436,6 +444,15 @@ def _fetch_one_page(session, args, pool, page_num: int, url: str) -> PageOutcome outcome.state = state + if state == "not_found": + logger.error("%s does not exist on this site (HTTP 404/410, or the " + "site's own error page) — nothing to parse and nothing " + "a retry or a proxy would change. Category paths differ " + "between locales: take them from the locale's own " + "sitemap_1-category.xml.", url) + outcome.final_url = d["current_url"]() + return outcome + if state == "blocked": debug_html = f"{args.out}_page{page_num}_debug.html" with open(debug_html, "w", encoding="utf-8") as f: @@ -526,9 +543,9 @@ def scrape(args) -> int: "the remote browser has its own exit.") pool = None if args.concurrency > 1: - logger.warning("--concurrency is ignored in this engine: parallel " - "page fetching (--mode transfers) is implemented in " - "playwright_scraper.py. Running sequentially.") + logger.warning("--concurrency is ignored: no mode on this site has a " + "second page to fetch in parallel (a listing's own " + "paging is robots-disallowed). Running sequentially.") session = _Session(args, pool).open() try: @@ -539,6 +556,8 @@ def scrape(args) -> int: stop_reason = ("page_load_timeout" if first.load_failed else f"blocked_{first.blocked_by}") blocked = first.blocked_by is not None + elif first.state == "not_found": + stop_reason = "not_found" elif args.mode not in PAGINATED_MODES: pass else: @@ -615,6 +634,18 @@ def scrape(args) -> int: start_url=args.url, final_url=final_url) +def _non_negative_float(value): + """argparse type for a delay in seconds: zero is fine, negative is not + (`time.sleep` raises ValueError on a negative value, mid-run).""" + try: + n = float(value) + except ValueError: + raise argparse.ArgumentTypeError(f"must be a number, got {value!r}") + if n < 0 or n != n: + raise argparse.ArgumentTypeError(f"must be 0 or greater, got {value}") + return n + + def _non_negative(value): """argparse type for a count that may be zero but never negative.""" n = int(value) @@ -647,32 +678,32 @@ def parse_args(): help="The exact page to read. Required for --mode product. " "Falls back to $GIVENCHY_URL, then to what " "--category builds.") - p.add_argument("--category", default="makeup/lips", + p.add_argument("--category", default=None, help="Category path WITHOUT the locale prefix, e.g. " "makeup/lips. Take real paths from the locale's own " "sitemap_1-category.xml: they are NOT shared between " "locales.") - p.add_argument("--site-locale", default="us", choices=list(LOCALES), + p.add_argument("--site-locale", default=None, choices=list(LOCALES), help="The locale PATH PREFIX on www.givenchybeauty.com " "(default us). Not --locale, which is the browser's " "own. int/en and ru publish no prices.") p.add_argument("--pages", type=_positive_int, default=1, help="Accepted for family compatibility; a listing is one " "page on this site (robots-disallowed paging).") - p.add_argument("--delay", type=float, default=2.0) + p.add_argument("--delay", type=_non_negative_float, default=2.0) p.add_argument("--concurrency", type=_positive_int, default=1, help="Ignored in this engine — see playwright_scraper.py.") p.add_argument("--retries", type=_positive_int, default=3) - p.add_argument("--retry-delay", type=float, default=2.0) + p.add_argument("--retry-delay", type=_non_negative_float, default=2.0) p.add_argument("--format", choices=["json", "csv", "both"], default="both") p.add_argument("--out", default="givenchy_products") p.add_argument("--proxy", default=None) p.add_argument("--proxy-file", default=None) p.add_argument("--proxy-rotate", choices=list(ROTATE_MODES), default="per-run") - p.add_argument("--proxy-sessions", type=int, default=None, + p.add_argument("--proxy-sessions", type=_non_negative, default=None, help="With --proxy pointing at a 2Captcha proxy gateway, mint this many session-pinned exits from that one credential instead of keeping a file of them. Each session is a different exit address (measured 2026-09-17: ten sessions, ten distinct addresses). Has no effect with --proxy-file, and is refused for a non-2Captcha host.") p.add_argument("--proxy-shuffle", action="store_true") - p.add_argument("--proxy-block-retries", type=int, default=2) + p.add_argument("--proxy-block-retries", type=_non_negative, default=2) p.add_argument("--twocaptcha-key", default=None) p.add_argument("--allow-empty", action="store_true") p.add_argument("--fingerprint", action="store_true") @@ -690,6 +721,21 @@ def parse_args(): p.add_argument("--headless", action="store_true", default=True) p.add_argument("--headful", dest="headless", action="store_false") args = p.parse_args() + # What the user TYPED, captured before env_config fills anything in: + # afterwards a $GIVENCHY_URL from .env is indistinguishable from a typed + # --url. Typed builder flags (--category/--site-locale) build the URL + # themselves, so an environment URL can no longer silently override them. + typed_url = args.url + builder_typed = args.category is not None or args.site_locale is not None + if args.category is None: + args.category = "makeup/lips" + if args.site_locale is None: + args.site_locale = "us" + if typed_url and builder_typed: + logger.warning("--url was given, so --category/--site-locale are " + "ignored: the URL is read exactly as typed.") + elif builder_typed and args.mode != "product": + args.url = f"{BASE}/{args.site_locale}/{args.category.strip('/')}/" env_config.apply(args) if not args.url: diff --git a/smoke_test.py b/smoke_test.py index 0a861bf..2d7da46 100644 --- a/smoke_test.py +++ b/smoke_test.py @@ -46,7 +46,9 @@ import csv import inspect import json +import logging import os +import pathlib import re import shutil import subprocess @@ -126,7 +128,7 @@ def _raises(fn): # the `js-pdp` hook). Full pages run 600-960KB and this suite needs the # STRUCTURE, not 300 tiles. # -# Scrubbed per CLAUDE.md §10, with PATTERNS rather than literals so the next +# Scrubbed of session material, with PATTERNS rather than literals so the next # capture is checked too (see test_no_capture_leaks): the Bazaarvoice # `api_key` and merchant ids in each tile's `pr-category-snippet`, the # listing's own `searchID`, and any `dwsid`/`session_id`. No customer data @@ -784,7 +786,7 @@ def test_money_parsing(): group("parse_money: every grouping convention this site can print") ok = True # The first four are shapes measured on this site; the rest are the - # conventions CLAUDE.md §4 requires every repo in this family to handle, + # conventions every repo in this family handles, # kept because a currency this site does not print today is one deploy # away from being printed tomorrow, and because getting them wrong is # silent. @@ -835,7 +837,7 @@ def test_tile_parsing(): group("parse_category: values pinned against six real tiles") ok = True # Values, not coverage. A column can be 100% populated and entirely - # wrong (CLAUDE.md §10). + # wrong. expected = { "TILE_US": dict(fixture=TILE_US, url=US_LIPS, sku="P000476", master_id="F20100269", price=50.0, currency="USD", @@ -979,7 +981,7 @@ def parsed(payload): return product_parser.parse_product(HEAD % payload, URL) # "offers": null is an EXPLICIT null -- a .get(..., {}) default does not - # apply to it, which is the AttributeError CLAUDE.md §4 names first. + # apply to it, which is an AttributeError in a naive parser. r = parsed('{"@type":"Product","name":"N","sku":"P1","offers":null}') ok &= check('"offers": null does not raise', r is not None) ok &= check('"offers": null leaves price None', r.price is None) @@ -1172,6 +1174,20 @@ def test_showcase_and_states(): product_parser.detect_page_state(_listing(TILE_US), 403) != "content") ok &= check("`empty` is not retried -- it is a correct answer", page_flow.should_retry("empty") is False) + # REGRESSION, audit 2026-10-14: a 404/410 classified as `blocked` when + # the engine had a status (retried, exit 3, "buy a proxy") and as + # `empty` when it had none (exit 4). It is neither. + for status in (None, 404, 410): + ok &= check("the site's 410 error page reads as not_found (status=%s)" + % status, product_parser.detect_page_state( + FIX_NOT_FOUND_410, status) == "not_found") + ok &= check("a served listing with no tiles is still `empty`, not " + "not_found -- the empty category says ns=category", + product_parser.detect_page_state(_listing(), None) == "empty") + ok &= check("`not_found` is neither retried nor counted as blocked", + page_flow.should_retry("not_found") is False + and page_flow.counts_as_blocked("not_found") is False + and page_flow.should_solve("not_found") is False) return ok @@ -1191,6 +1207,29 @@ def _page(body_html, ready_marker=True): return "%s%s" % (marker, body_html) +# The site's own answer for an address that does not exist. Trimmed from a +# real capture taken 2026-10-14 with plain curl (HTTP 410, 646 KB) of a +# made-up category path under /us/: the title, one asset reference, the +# Demandware page context and the page wrapper are verbatim lines from it; +# everything else is cut. `/fr/fr/makeup/lips/` answered the same 410 with +# the same page context. Verified to classify identically to the untrimmed +# capture (not_found with no status, not_found with 410). +FIX_NOT_FOUND_410 = """ + + + Sites-givenchy-beauty-us-Site + + + + +
+
+ +""" + + # The challenge this site actually serves. Transcribed from a real capture # taken 2026-09-16 (HTTP 405, `x-amzn-waf-action: captcha`, `server: # CloudFront`, 2331 bytes) -- the gokuProps values are truncated here and @@ -1218,7 +1257,7 @@ def test_aws_waf(): ok = True import captcha_solver as cs - # --- §18's control: every marker must be ABSENT from real pages ------- + # --- the control: every marker must be ABSENT from real pages ------- # A marker that fires on a good page is worse than no marker at all. good_pages = { "us listing": _listing(TILE_US), @@ -1258,7 +1297,7 @@ def test_aws_waf(): ok &= check("and the policy for that state actually solves", page_flow.should_solve("captcha") and not page_flow.should_solve("blocked")) - # --- §17's classification-order trap --------------------------------- + # --- classification order: proof before heuristics------------------------------ served_with_hunter = _page( '') @@ -1691,13 +1730,42 @@ def test_finish_run(): "-- a bot-check is a different failure than an empty page", rc == EXIT_BLOCKED) - # 0 rows, not blocked (a genuinely empty page) -> EXIT_NO_PRODUCTS. + # 0 rows, not blocked, and the page WAS served (a genuinely empty + # category ends its run with the ordinary complete reason) -> + # EXIT_NO_PRODUCTS. This check used `stop_reason="empty"`, a reason + # no engine sets; under the "not complete -> exit 5" rule below it + # would say the content was never obtained, which is not what it + # meant to model. rc = finish_run([], prefix, "json", allow_empty=False, blocked=False, - stop_reason="empty", pages_requested=1, pages_completed=1, - start_url="https://x/1", final_url="https://x/1", mode="category") - ok &= check("0 rows, NOT blocked, returns EXIT_NO_PRODUCTS", + stop_reason="single_page_mode", pages_requested=1, + pages_completed=1, start_url="https://x/1", + final_url="https://x/1", mode="category") + ok &= check("0 rows, NOT blocked, page served returns EXIT_NO_PRODUCTS", rc == EXIT_NO_PRODUCTS) + # 0 rows because the page never arrived -> exit 5, not 4. Every + # engine records a load failure as `page_load_timeout`; before this + # it exited 4, so a dead proxy read as an empty catalogue. + for reason in ("page_load_timeout", "proxy_failed", + "a_reason_nobody_has_named_yet"): + rc = finish_run([], prefix, "json", allow_empty=False, + blocked=False, stop_reason=reason, + pages_requested=1, pages_completed=0, + start_url="https://x/1", final_url="https://x/1", + mode="category") + ok &= check("0 rows, not blocked, stop_reason=%s returns " + "EXIT_REMOTE_API_ERROR (5), not EXIT_NO_PRODUCTS" + % reason, rc == EXIT_REMOTE_API_ERROR) + + # A complete reason with a failed page is not complete. + rc = finish_run(rows, prefix, "json", allow_empty=False, blocked=False, + stop_reason="completed", pages_requested=2, + pages_completed=1, pages_failed=[2], + start_url="https://x/1", final_url="https://x/1", + mode="category") + ok &= check("rows + a complete stop_reason + pages_failed is PARTIAL", + rc == EXIT_PARTIAL) + ok &= check("COMPLETE_STOP_REASONS names the reasons that count as a full run", {"completed", "pagination_exhausted", "single_page_mode"} <= set(COMPLETE_STOP_REASONS)) return ok @@ -1709,7 +1777,7 @@ def test_attempt_record(): # REGRESSION, external audit 2026-09-18: after a blocked run the sidecar # still said `status=complete, products=1` from the PREVIOUS run, so a # consumer reading files rather than exit codes read yesterday's success - # as today's. §9's rule (a failed run writes no `meta.json`, so a good + # as today's. The rule (a failed run writes no `meta.json`, so a good # dataset is never contradicted) is kept; the attempt record answers the # other question. with tempfile.TemporaryDirectory() as d: @@ -2449,7 +2517,7 @@ def test_env_example_matches_env_keys(): ok = True path = os.path.join(REPO_ROOT, ".env.example") if not os.path.exists(path): - return check(".env.example exists (CLAUDE.md §3: it is WRITTEN, never " + return check(".env.example exists (it is WRITTEN, never " "copied from a sibling repo)", False) text = open(path, encoding="utf-8").read() documented = {line.split("=", 1)[0].strip() @@ -2463,7 +2531,7 @@ def test_env_example_matches_env_keys(): "than a missing one" % sorted(documented - declared), documented <= declared) - # §17: a braced placeholder in a copied example reads as CONFIGURED and + # A braced placeholder in a copied example reads as CONFIGURED and # produces a 401 a long way from its cause. env_config treats both # shapes as unset; this asserts the example only ever uses those. for name in sorted(documented): @@ -2586,7 +2654,7 @@ def test_aws_waf_two_actions(): got is not None) ok &= check("...but is reported as the challenge action", got and got.aws_waf_action == "challenge") - ok &= check("...and is recognised as carrying NO widget -- section 19's " + ok &= check("...and is recognised as carrying NO widget -- " "'unsolvable is a property of a page', measured rather than " "assumed", got and not got.has_captcha_widget) ok &= check("both actions still yield the fields a task would need, so " @@ -2920,6 +2988,62 @@ def test_scraper_api_client(): "page_flow.PRICE_COVERAGE_FLOOR" in src and "page_flow.is_thin_page(" in src) + # The Scraper API contract, driven through fetch_html with + # `requests.post` stubbed -- no network, no key, no bill. + import types as _types + ns = _types.SimpleNamespace + wf = sac._build_wait_for(ns(wait_text=None, wait_element="div.x", + wait_state=None)) + ok &= check("waitFor is built as an OBJECT -- the string form is refused " + "with a billed HTTP 422", wf == {"element": "div.x", + "checkVisible": True}) + SECRET = "SECRET" + "PW" + debug_cdp = "ws://" + "user-zone-x:" + SECRET + "@cb.2captcha.com:9222" + + class _Resp: + status_code = 200 + headers = {"x-debug": json.dumps({"cdpurl": debug_cdp, + "price": 0.0005})} + text = "" + + def json(self): + return {"status": "warn", "http_code": 403, "headers": {}, + "body": "refused"} + + sent = {} + + def _fake_post(url, headers=None, json=None, timeout=None): + sent["payload"] = json + return _Resp() + + real_post = sac.requests.post + records = [] + + class _Grab(logging.Handler): + def emit(self, record): + records.append(record.getMessage()) + + grab = _Grab() + sac.logger.addHandler(grab) + sac.requests.post = _fake_post + try: + _html, status, _h = sac.fetch_html( + ns(wait_text="Lips", wait_element=None, wait_state=None, + cdp_url=None, key="k"), "https://www.givenchybeauty.com/us/", 30) + finally: + sac.requests.post = real_post + sac.logger.removeHandler(grab) + ok &= check("fetch_html sends waitFor as an object in the real payload", + isinstance(sent["payload"].get("waitFor"), dict)) + ok &= check("the target status is read from http_code, not the API's " + "`status` verdict string", status == 403) + ok &= check("a string verdict alone never reaches the classifier as a " + "status", sac._target_status({"status": "success"}) is None) + log_text = "\n".join(records) + ok &= check("the x-debug header is logged with its credentials masked", + "x-debug" in log_text and SECRET not in log_text + and "cb.2captcha.com:9222" in log_text) + # Functional: drive scrape() against this suite's own real-capture # fixtures, the same ones test_market_values/test_club_squad/ # test_transfers/test_player_detail already verify parse_* against -- @@ -2974,6 +3098,24 @@ class _FakeArgs: else: ok &= check("--mode %s wrote an output file" % mode, False) + # A retired category: HTTP 410 and the site's own error page. Not a + # block (exit 3) and not an empty catalogue (exit 4). + gone_calls = [] + + def _gone(args, url, timeout): + gone_calls.append(url) + return FIX_NOT_FOUND_410, 410, {} + sac.fetch_html = _gone + a = _FakeArgs(mode="category", retries=3, + out=os.path.join(tmpdir, "out_gone")) + rc = sac.scrape(a) + attempt = json.load(open(a.out + ".attempt.json", encoding="utf-8")) + ok &= check("a 410 address: one fetch, no retries, stop_reason " + "not_found, not blocked, exit 5", + len(gone_calls) == 1 and rc == EXIT_REMOTE_API_ERROR + and attempt["stop_reason"] == "not_found" + and attempt["blocked"] is False) + # This site has no addressable pagination at all, so the check # below is that the engine fetches ONCE rather than re-requesting # page 1 under a second page number. @@ -3064,8 +3206,22 @@ def test_no_capture_leaks(): if os.path.exists(gi_path): gitignore = open(gi_path, encoding="utf-8").read() ok &= check("the .gitignore excludes .env, so one cannot be committed", - any(line.strip() in (".env", "*.env", "/.env") + any(line.strip() in (".env", "*.env", "/.env", ".env*") for line in gitignore.splitlines())) + # Asked of git, not matched against text: a renamed working copy + # (.env.bak, .env.local) carries the same keys, and was NOT ignored + # before 2026-10-14. The template itself must stay committable. + in_git = subprocess.run(["git", "rev-parse", "--is-inside-work-tree"], + cwd=REPO_ROOT, capture_output=True, + text=True).stdout.strip() == "true" + if in_git: + for name in (".env", ".env.bak", ".env.local", ".env.save"): + ign = subprocess.run(["git", "check-ignore", "-q", name], + cwd=REPO_ROOT).returncode == 0 + ok &= check("git refuses to commit %s" % name, ign) + ok &= check("...while .env.example stays committable", + subprocess.run(["git", "check-ignore", "-q", ".env.example"], + cwd=REPO_ROOT).returncode == 1) tracked = subprocess.run(["git", "ls-files", "--error-unmatch", ".env"], cwd=REPO_ROOT, capture_output=True) if tracked.returncode == 0: @@ -3077,7 +3233,7 @@ def test_no_capture_leaks(): return ok -# Wording the family enforces. See CLAUDE.md's own note on why: two of these +# Wording the family enforces. Two of these # names were used for a placeholder endpoint that never existed, and one # names a real 2Captcha product under the wrong term. BANNED_PHRASES = ( @@ -3130,7 +3286,9 @@ def test_no_sibling_site_vocabulary(): # cries wolf gets allowlisted into uselessness. BANNED = ("spieler", "verein", "hauptlink", "posrela", "table.items", "market-values", "club-squad", "transfer_id", "marktwert", - "schema.org/Person") + "schema.org/person", "--mode transfers", "neuestetransfers", + "free agent", "--player-id", "--club-id", "data-header__label", + "without club") # This file names them in order to ban them, exactly as the family's # banned-phrase check exempts itself. SELF = os.path.basename(__file__) @@ -3139,7 +3297,8 @@ def test_no_sibling_site_vocabulary(): dirs[:] = [d for d in dirs if d not in ("captures", "__pycache__", ".git", ".venv")] for name in files: - if name == SELF or not name.endswith((".py", ".md", ".yml", ".yaml")) \ + if name == SELF or not name.endswith((".py", ".md", ".yml", ".yaml", + ".sh", ".toml", ".example")) \ and name != "Dockerfile": continue path = os.path.join(root, name) @@ -3346,7 +3505,7 @@ def test_fingerprint_client_reads_env(): # This file used to be the one CLI in the repo that skipped env_config # entirely and read only its own --key flag -- so a TWOCAPTCHA_KEY set in # .env or exported the way every other script here reads it was silently - # ignored, and the first thing CLAUDE.md tells someone to run when a key + # ignored, and the first thing to run when a key # "isn't working" (`python3 fingerprint_client.py`) could not see it. saved = os.environ.pop("TWOCAPTCHA_KEY", None) old_argv = sys.argv @@ -3757,6 +3916,427 @@ def test_sample_output(): # --------------------------------------------------------------------------- +def test_output_modes_and_csv_guard(): + group("output files: mode follows the umask, CSV formula cells escaped") + ok = True + import stat as _stat + with tempfile.TemporaryDirectory() as d: + # REGRESSION, audit 2026-10-14: the temp-file-then-rename writer + # produced 0600 files whatever the umask, and narrowed an existing + # 0664 file to 0600. 002 is in the matrix on purpose: under 022 a + # flat 0644 and the right rule agree, so 022 alone proves nothing. + for umask in (0o022, 0o002, 0o077): + path = os.path.join(d, "u%o.json" % umask) + old = os.umask(umask) + try: + write_json([Product(sku="A")], path) + finally: + os.umask(old) + mode = _stat.S_IMODE(os.stat(path).st_mode) + ok &= check("a new output under umask %03o is %03o, not 0600" + % (umask, 0o666 & ~umask), mode == 0o666 & ~umask) + path = os.path.join(d, "kept.csv") + open(path, "w").close() + os.chmod(path, 0o664) + old = os.umask(0o022) + try: + write_csv([Product(sku="A")], path) + finally: + os.umask(old) + ok &= check("an existing 0664 output keeps 0664 when replaced", + _stat.S_IMODE(os.stat(path).st_mode) == 0o664) + + # Formula-shaped strings: escaped in CSV only, numbers untouched, + # lists escaped after joining, count in the sidecar. + row = Product(sku="F1", title='=HYPERLINK("x")', category="+cmd", + badge="-2+3", shades=["@SUM(A1)", "NUDE-1"], + subtitle="\tTab", price=-5.0) + prefix = os.path.join(d, "f") + rc = finish_run([row], prefix, "both", allow_empty=False, + blocked=False, stop_reason="single_page_mode", + pages_requested=1, pages_completed=1, + start_url="u", final_url="u", mode="category") + with open(prefix + ".csv", encoding="utf-8", newline="") as f: + got = list(csv.DictReader(f))[0] + ok &= check("CSV: a title starting with = is prefixed with an apostrophe", + got["title"] == """'=HYPERLINK("x")""") + ok &= check("CSV: + and - and TAB leads are escaped too", + got["category"] == "'+cmd" and got["badge"] == "'-2+3" + and got["subtitle"] == "'\tTab") + ok &= check("CSV: a joined list whose first element is formula-shaped " + "is escaped", got["shades"].startswith("'@SUM(A1)")) + ok &= check("CSV: a negative NUMBER is left alone", got["price"] == "-5.0") + data = json.load(open(prefix + ".json", encoding="utf-8"))[0] + ok &= check("JSON keeps the site's text unescaped", + data["title"] == '=HYPERLINK("x")' and data["category"] == "+cmd") + meta = json.load(open(prefix + ".meta.json", encoding="utf-8")) + ok &= check("the sidecar records csv_cells_escaped", + rc == 0 and meta.get("csv_cells_escaped") == 5) + finish_run([row], prefix, "both", allow_empty=False, blocked=False, + stop_reason="single_page_mode", pages_requested=1, + pages_completed=1, start_url="u", final_url="u", + mode="category", extra={"csv_cells_escaped": "caller"}) + meta = json.load(open(prefix + ".meta.json", encoding="utf-8")) + ok &= check("a caller-provided extra key wins over the housekeeping one", + meta.get("csv_cells_escaped") == "caller") + return ok + + +def _load_engine_from_disk(filename): + """Load an engine module from its source file with its browser driver + replaced by inert stand-in modules, so the engine's own fetch loop can be + driven offline in a venv that has no driver installed at all. A check + gated behind importing the real engine would skip for every absent + driver -- i.e. for all three in the offline CI job.""" + import importlib.util + import types + + class _Err(Exception): + pass + + def mod(name, **attrs): + m = types.ModuleType(name) + for k, v in attrs.items(): + setattr(m, k, v) + return m + + fakes = { + "playwright": mod("playwright"), + "playwright.sync_api": mod("playwright.sync_api", + sync_playwright=lambda: None, + Error=_Err, TimeoutError=_Err), + "selenium": mod("selenium", webdriver=mod("selenium.webdriver")), + "selenium.webdriver": mod("selenium.webdriver"), + "selenium.common": mod("selenium.common"), + "selenium.common.exceptions": mod( + "selenium.common.exceptions", TimeoutException=_Err, + WebDriverException=_Err, JavascriptException=_Err), + "selenium.webdriver.chrome": mod("selenium.webdriver.chrome"), + "selenium.webdriver.chrome.options": mod( + "selenium.webdriver.chrome.options", Options=object), + "pyppeteer": mod("pyppeteer", launch=None, connect=None), + } + saved = {k: sys.modules.get(k) for k in fakes} + sys.modules.update(fakes) + try: + name = "_ondisk_" + filename.replace(".py", "") + spec = importlib.util.spec_from_file_location( + name, os.path.join(REPO_ROOT, filename)) + module = importlib.util.module_from_spec(spec) + spec.loader.exec_module(module) + return module + finally: + for k, v in saved.items(): + if v is None: + sys.modules.pop(k, None) + else: + sys.modules[k] = v + + +class _StubPage: + url = "https://www.givenchybeauty.com/us/makeup/lips/" + + def goto(self, *a, **k): + return types_ns(status=200, headers={}) + + def wait_for_timeout(self, ms): + pass + + def screenshot(self, **k): + pass + + def content(self): + return "" + + +def types_ns(**kw): + import types + return types.SimpleNamespace(**kw) + + +class _StubSession: + """Answers the primitives all three engines' fetch loops reach for.""" + + def __init__(self): + self.page = _StubPage() + self.driver = types_ns(get=lambda url: None, + current_url=_StubPage.url, page_source="", + save_screenshot=lambda path: None) + self.bridge = types_ns(run=lambda x: x) + self.relaunches = 0 + + def relaunch(self): + self.relaunches += 1 + return self + + +def test_captcha_budget_per_page(): + group("paid captcha solves: one allowance per page, kept through rotation") + ok = True + budget = page_flow.SolveBudget() + ok &= check("SolveBudget allows SOLVES_PER_PAGE purchases and refuses the next", + [budget.spend() for _ in range(page_flow.SOLVES_PER_PAGE + 1)] + == [True] * page_flow.SOLVES_PER_PAGE + [False]) + from proxy_pool import ProxyPool + for filename in ("playwright_scraper.py", "selenium_scraper.py", + "puppeteer_scraper.py"): + eng = _load_engine_from_disk(filename) + src = open(os.path.join(REPO_ROOT, filename), encoding="utf-8").read() + # Inside the captcha handler the allowance is checked before the + # solver is asked for a token, so a call site that forgot the guard + # still cannot buy a second one. + fn = src[src.index("def handle_captcha_if_present("):] + fn = fn[:fn.index("\ndef ")] + ok &= check("%s: budget.spend() is checked BEFORE solve_recaptcha()" + % filename, "budget.spend()" in fn + and fn.index("budget.spend()") < fn.index("solve_recaptcha(")) + + calls = {"handler": 0, "paid": 0} + + def stub_handler(session, args, ready_selector, proxy=None, budget=None): + calls["handler"] += 1 + if budget is None or budget.spend(): + calls["paid"] += 1 + return False + + empty = {"content": lambda: "", "count": lambda sel: 0, + "sleep": lambda ms: None, + "current_url": lambda: _StubPage.url} + eng.handle_captcha_if_present = stub_handler + eng._driver = lambda *a, **k: empty + if hasattr(eng, "_content_when_settled"): + eng._content_when_settled = lambda *a, **k: "" + with tempfile.TemporaryDirectory() as d: + args = types_ns(mode="category", proxy_block_retries=2, retries=1, + retry_delay=0, out=os.path.join(d, "o"), + dump_html=None, solve_captcha="when-blocked", + twocaptcha_key="x", headless=True) + pool = ProxyPool(["http://a.example:1", "http://b.example:2", + "http://c.example:3"]) + session = _StubSession() + try: + outcome = eng._fetch_one_page(session, args, pool, 1, + _StubPage.url) + raised = None + except Exception as e: # noqa: BLE001 + outcome, raised = None, e + ok &= check("%s: a page refused on 3 exits ran without raising (%s)" + % (filename, raised), raised is None) + ok &= check("%s: ...and rotated through the pool (%d relaunches)" + % (filename, session.relaunches), session.relaunches == 2) + ok &= check("%s: ...and bought at most SOLVES_PER_PAGE solves for " + "that ONE page (paid %d)" % (filename, calls["paid"]), + calls["paid"] <= page_flow.SOLVES_PER_PAGE + and calls["handler"] >= 1) + # The same refused page with NO proxy at all: every pool access in + # the retry path must be guarded, or the mirrors crash with + # AttributeError on None instead of reporting a block. + with tempfile.TemporaryDirectory() as d: + args.out = os.path.join(d, "o") + try: + outcome = eng._fetch_one_page(_StubSession(), args, None, 1, + _StubPage.url) + raised = None + except Exception as e: # noqa: BLE001 + outcome, raised = None, e + ok &= check("%s: with no proxy pool a refused page is reported, not " + "an exception (%r)" % (filename, raised), + raised is None and outcome is not None + and outcome.blocked_by is not None) + # The site's own error page, served to a driver that has no status + # to give (or a 410 from one that has): terminal, not a block. + gone = {"content": lambda: FIX_NOT_FOUND_410, "count": lambda sel: 0, + "sleep": lambda ms: None, + "current_url": lambda: _StubPage.url} + eng._driver = lambda *a, **k: gone + if hasattr(eng, "_content_when_settled"): + eng._content_when_settled = lambda *a, **k: FIX_NOT_FOUND_410 + gone_session = _StubSession() + gone_session.page.goto = lambda *a, **k: types_ns(status=410, headers={}) + gone_session.driver.page_source = FIX_NOT_FOUND_410 + parsed = [] + eng._parse_for_mode = lambda *a, **k: parsed.append(a) or [] + with tempfile.TemporaryDirectory() as d: + args.out = os.path.join(d, "o") + try: + outcome = eng._fetch_one_page(gone_session, args, None, 1, + _StubPage.url) + except Exception as e: # noqa: BLE001 + outcome = types_ns(state="raised: %r" % e, blocked_by=None, + rows=[]) + dumped = os.listdir(d) + ok &= check("%s: a missing address comes back as state=not_found, " + "not blocked, no rows, not retried" % filename, + outcome.state == "not_found" and outcome.blocked_by is None + and not outcome.rows and gone_session.relaunches == 0) + ok &= check("%s: ...and stops there: the error page is not handed to " + "the parser or dumped as a '0 rows' debug file" % filename, + not parsed and not dumped) + src_scrape = src[src.index("def scrape("):] + ok &= check("%s: scrape() turns a not_found page 1 into " + "stop_reason='not_found'" % filename, + 'first.state == "not_found"' in src_scrape + and 'stop_reason = "not_found"' in src_scrape) + return ok + + +def test_typed_flags_beat_env_url(): + group("typed --category/--site-locale are not overridden by $GIVENCHY_URL") + ok = True + # REGRESSION, audit 2026-10-14: env_config.apply fills only UNSET values, + # so a GIVENCHY_URL in the environment filled --url and the typed builder + # flags were silently ignored. + env_url = "https://www.givenchybeauty.com/us/makeup/lips/" + for filename in ("playwright_scraper.py", "selenium_scraper.py", + "puppeteer_scraper.py", "scraper_api_client.py"): + eng = _load_engine_from_disk(filename) + saved_argv, saved_env = sys.argv, dict(os.environ) + os.environ["GIVENCHY_URL"] = env_url + os.environ["TWOCAPTCHA_KEY"] = "0" * 8 + try: + sys.argv = [filename, "--category", "skincare", + "--site-locale", "gb"] + built = eng.parse_args().url + sys.argv = [filename] + from_env = eng.parse_args().url + sys.argv = [filename, "--url", + "https://www.givenchybeauty.com/gb/makeup/lips/"] + typed = eng.parse_args().url + finally: + sys.argv = saved_argv + os.environ.clear() + os.environ.update(saved_env) + ok &= check("%s: typed --category/--site-locale build the URL even " + "with $GIVENCHY_URL set" % filename, + built == "https://www.givenchybeauty.com/gb/skincare/") + ok &= check("%s: with no flag typed, $GIVENCHY_URL is still the " + "fallback" % filename, from_env == env_url) + ok &= check("%s: a typed --url wins over both" % filename, + typed == "https://www.givenchybeauty.com/gb/makeup/lips/") + return ok + + +def test_negative_counts_rejected(): + group("negative delays and counts are usage errors naming the flag") + ok = True + import contextlib + import io + cases = { + "playwright_scraper.py": ["--delay", "--retry-delay", + "--proxy-block-retries", "--proxy-sessions", + "--pages", "--retries", "--concurrency"], + "selenium_scraper.py": ["--delay", "--retry-delay", + "--proxy-block-retries", "--proxy-sessions", + "--pages", "--retries", "--concurrency"], + "puppeteer_scraper.py": ["--delay", "--retry-delay", + "--proxy-block-retries", "--proxy-sessions", + "--pages", "--retries", "--concurrency"], + "scraper_api_client.py": ["--delay", "--retry-delay", "--timeout", + "--pages", "--retries"], + } + for filename, flags in cases.items(): + eng = _load_engine_from_disk(filename) + saved_argv, saved_env = sys.argv, dict(os.environ) + os.environ["TWOCAPTCHA_KEY"] = "0" * 8 + try: + for flag in flags: + sys.argv = [filename, flag, "-1"] + err = io.StringIO() + code = None + with contextlib.redirect_stderr(err): + try: + eng.parse_args() + except SystemExit as e: + code = e.code + ok &= check("%s %s -1 exits 2 with an error naming the flag" + % (filename, flag), + code == 2 and ("argument %s" % flag) in err.getvalue()) + finally: + sys.argv = saved_argv + os.environ.clear() + os.environ.update(saved_env) + return ok + + +def test_secret_scan_scope(): + group("credential scan: judged on repo-relative paths, venvs by pyvenv.cfg") + ok = True + import importlib.util + spec = importlib.util.spec_from_file_location( + "ci_checks_scope", os.path.join(REPO_ROOT, ".github", "ci_checks.py")) + ci = importlib.util.module_from_spec(spec) + spec.loader.exec_module(ci) + # Assembled from pieces so this file never carries the shapes it plants. + hex_key = "ab" * 16 + field = "\\" + '"api' + 'Key\\": \\"' + "Q7w" * 10 + '\\"' + saved = ci.REPO + try: + with tempfile.TemporaryDirectory() as d: + # A clone that lives under a directory called `venv`. + repo = pathlib.Path(d) / "venv" / "clone" + repo.mkdir(parents=True) + (repo / "a.py").write_text("k = '%s'\n" % hex_key) + ci.REPO = repo + failed = ci.secret_check() + ok &= check("a clone under an ancestor named venv is still " + "scanned (planted key found)", + any("a.py" in f for f in failed)) + (repo / "a.py").write_text("x = 1\n") + env = repo / "myenv" / "lib" + env.mkdir(parents=True) + (repo / "myenv" / "pyvenv.cfg").write_text("home = /usr\n") + (env / "vendored.py").write_text("# %s\n" % hex_key) + ok &= check("a virtualenv named anything is skipped, recognised by " + "pyvenv.cfg", ci.secret_check() == []) + (repo / "fixture.py").write_text("F = '%s'\n" % field) + ok &= check("a key-shaped field inside a JSON-escaped fixture is " + "reported", any("fixture.py" in f + for f in ci.secret_check())) + with tempfile.TemporaryDirectory() as d: + ci.REPO = pathlib.Path(d) + ok &= check("a scan that scanned 0 files FAILS instead of " + "reporting clean", any("0 files scanned" in f + for f in ci.secret_check())) + finally: + ci.REPO = saved + return ok + + +def test_no_private_citations(): + group("shipped files cite no private notes or bare section numbers") + ok = True + # The family's internal notes are not public, so a pointer into them + # ("see , rule eight") reads as a reference nobody outside can + # follow. State the rule in this repo's own words instead. Tokens are + # assembled from pieces so this file does not match itself. + banned = [re.escape("CLAUDE" + ".md"), re.escape("claude_" + "2scraper"), + re.escape("2scraper_" + "repo_rules"), "play" + "book", + "\u00a7" + r"\s?\d", r"\b[Ss]ection " + r"\d+\b"] + pattern = re.compile("|".join(banned)) + try: + listed = subprocess.run(["git", "ls-files"], cwd=REPO_ROOT, + capture_output=True, text=True, check=True + ).stdout.split() + except (OSError, subprocess.CalledProcessError): + listed = [os.path.relpath(os.path.join(r, f), REPO_ROOT) + for r, dirs, fs in os.walk(REPO_ROOT) + if ".git" not in r.split(os.sep) for f in fs] + hits, scanned = [], 0 + for rel in listed: + path = os.path.join(REPO_ROOT, rel) + if not os.path.isfile(path) or rel.endswith((".png", ".jpg", ".ico")): + continue + scanned += 1 + text = open(path, encoding="utf-8", errors="replace").read() + for lineno, line in enumerate(text.splitlines(), 1): + if pattern.search(line): + hits.append("%s:%d" % (rel, lineno)) + ok &= check("scanned the shipped files (%d)" % scanned, scanned > 10) + ok &= check("no private-note citation or bare section number (%s)" + % (", ".join(hits[:8]) or "clean"), not hits) + return ok + + def main() -> int: ok = True # Checks that could not run because an optional engine library is absent. @@ -3778,12 +4358,16 @@ def main() -> int: ok &= test_output_contract() ok &= test_writers() ok &= test_finish_run() + ok &= test_output_modes_and_csv_guard() ok &= test_attempt_record() ok &= test_retry_floor() + ok &= test_negative_counts_rejected() ok &= test_diff() ok &= test_captcha() + ok &= test_captcha_budget_per_page() ok &= test_remote_api_error() ok &= test_env_config() + ok &= test_typed_flags_beat_env_url() ok &= test_proxy_pool() ok &= test_engines(skips) ok &= test_engine_parity(skips) @@ -3797,12 +4381,14 @@ def main() -> int: ok &= test_no_capture_leaks() ok &= test_no_sibling_site_vocabulary() ok &= test_wording() + ok &= test_no_private_citations() ok &= test_fingerprint_application() ok &= test_fingerprint_client_reads_env() ok &= test_credentials_never_reach_a_log() ok &= test_concurrent_dispatch(skips) ok &= test_no_undefined_names() ok &= test_ci_checks_is_actually_wired_up() + ok &= test_secret_scan_scope() ok &= test_dockerfile_copies_what_it_runs() ok &= test_sample_output() diff --git a/tools/browser_profile_client.py b/tools/browser_profile_client.py index 4c16c5a..d54b3dd 100644 --- a/tools/browser_profile_client.py +++ b/tools/browser_profile_client.py @@ -36,7 +36,7 @@ navigation. `connection` is the POST path, for when a custom proxy has to be attached to the request itself. -Credentials come from `.env` via env_config (CLAUDE.md §3) -- never argv. +Credentials come from `.env` via env_config -- never argv. Nothing secret is printed: logins, passwords and the assembled connection URI are shown as scheme://***:***@host:port, and `--write-env` puts the real URI straight into `.env` without it passing through the terminal. diff --git a/tools/verify_browser_api.sh b/tools/verify_browser_api.sh deleted file mode 100755 index a20405c..0000000 --- a/tools/verify_browser_api.sh +++ /dev/null @@ -1,159 +0,0 @@ -#!/usr/bin/env bash -# verify_browser_api.sh — does the Scraping Browser API actually clear this -# site's AWS WAF challenge, or does it merely never meet one? -# -# Those two outcomes look identical from a single successful run, and telling -# them apart is the whole point of this script. A measurement with no negative -# case is an anecdote: if the exit you are testing from is not challenged in -# the first place, a green run through the Browser API proves nothing about -# auto-solve. -# -# So it runs three legs and compares them: -# -# 1. CONTROL — a plain HTTP probe from THIS machine, to establish whether -# this network position gets challenged at all. -# 2. BASELINE — a local Chromium run, no Browser API. -# 3. TEST — the same run through --cdp-endpoint. -# -# Read the verdict at the end; it says which of the four possible worlds you -# are in, including the two where the answer is "inconclusive, and here is -# why". -# -# Needs: .env with TWOCAPTCHA_KEY and GIVENCHY_CDP_ENDPOINT, and -# playwright installed. Prints no secrets. -# -# bash tools/verify_browser_api.sh - -set -u -cd "$(dirname "$0")/.." || exit 2 -LOGS=$(mktemp -d) -trap 'echo; echo "logs kept in $LOGS"' EXIT - -# `timeout` is GNU coreutils and is NOT on a stock macOS, where this script is -# most likely to be run (the machine with a browser on it). Homebrew installs -# it as `gtimeout`. Fall back to running the command unbounded rather than -# failing with "command not found" on the very line that was supposed to stop -# a run hanging. -if command -v timeout >/dev/null 2>&1; then RUN_TIMEOUT="timeout" -elif command -v gtimeout >/dev/null 2>&1; then RUN_TIMEOUT="gtimeout" -else RUN_TIMEOUT="" - echo "note: no timeout(1) on this system — the two browser runs below are" - echo " unbounded. Ctrl-C if one hangs. (brew install coreutils adds it.)" - echo -fi -bounded() { # bounded - local secs="$1"; shift - if [ -n "$RUN_TIMEOUT" ]; then "$RUN_TIMEOUT" "$secs" "$@"; else "$@"; fi -} - -echo "==============================================================" -echo " Scraping Browser API — does it clear the challenge, or dodge it?" -echo "==============================================================" -echo - -# ---------------------------------------------------------------- config -if [ ! -f .env ]; then - echo "!! no .env next to the scripts. Copy .env.example to .env and put" - echo " TWOCAPTCHA_KEY and GIVENCHY_CDP_ENDPOINT in it." - exit 2 -fi -echo "--- what the loader picks up (no secrets printed) ---" -python3 env_config.py || exit 2 -have_cdp=$(python3 - <<'PY' -import env_config -class A: pass -a = A() -for d in set(env_config.ENV_KEYS.values()): - setattr(a, d, None) -setattr(a, "out", "x") -env_config.apply(a) -print("yes" if getattr(a, "cdp_endpoint", None) else "no") -PY -) -echo - -# ---------------------------------------------------------------- 1. control -echo "=== 1. CONTROL — is this exit challenged at all? ===" -bash tools/waf_probe.sh 10 market-values 2>&1 | tee "$LOGS/control.txt" | tail -4 -challenged=$(grep -c "waf=captcha" "$LOGS/control.txt") -echo - -# ---------------------------------------------------------------- 2. baseline -echo "=== 2. BASELINE — local Chromium, no Browser API ===" -bounded 300 python3 playwright_scraper.py --mode market-values --pages 1 \ - --out "$LOGS/baseline" --format json > "$LOGS/baseline.log" 2>&1 -base_rc=$? -base_rows=$(python3 -c " -import json,sys -try: print(len(json.load(open('$LOGS/baseline.json')))) -except Exception: print(0)") -echo " exit=$base_rc rows=$base_rows" -grep -iE "captcha|waf|blocked" "$LOGS/baseline.log" | head -3 | sed 's/^/ /' -echo - -# ---------------------------------------------------------------- 3. test -if [ "$have_cdp" != "yes" ]; then - echo "=== 3. TEST — SKIPPED: no GIVENCHY_CDP_ENDPOINT in .env ===" - echo - echo "VERDICT: cannot answer the question without the Browser API leg." - exit 0 -fi - -echo "=== 3. TEST — through the Scraping Browser API ===" -echo " (a profile allows ONE live connection; if this 500s, another run" -echo " still holds that pid. A 401 here is a malformed or superseded" -echo " login, not an expired one: the vendor documents profiles as stored" -echo " 90 days from creation or until deleted, and the credentials" -echo " themselves do not expire on a timer. Fetch a fresh connectionUri" -echo " with tools/browser_profile_client.py rather than editing the" -echo " string.)" -bounded 300 python3 playwright_scraper.py --mode market-values --pages 1 \ - --out "$LOGS/viacdp" --format json > "$LOGS/viacdp.log" 2>&1 -cdp_rc=$? -cdp_rows=$(python3 -c " -import json,sys -try: print(len(json.load(open('$LOGS/viacdp.json')))) -except Exception: print(0)") -echo " exit=$cdp_rc rows=$cdp_rows" -autosolve_on=$(grep -c "setAutoSolve enabled" "$LOGS/viacdp.log") -solved=$(grep -c "CAPTCHA solved automatically" "$LOGS/viacdp.log") -detected=$(grep -c "CAPTCHA detected on page" "$LOGS/viacdp.log") -grep -iE "Scraping Browser|setAutoSolve" "$LOGS/viacdp.log" | head -4 | sed 's/^/ /' -echo - -# ---------------------------------------------------------------- verdict -echo "==============================================================" -echo " VERDICT" -echo "==============================================================" -printf " control: %s of 10 plain requests challenged\n" "$challenged" -printf " baseline (local browser): exit=%s rows=%s\n" "$base_rc" "$base_rows" -printf " test (Browser API): exit=%s rows=%s\n" "$cdp_rc" "$cdp_rows" -printf " auto-solve enabled: %s | challenge detected: %s | solved: %s\n" \ - "$([ "$autosolve_on" -gt 0 ] && echo yes || echo NO)" \ - "$([ "$detected" -gt 0 ] && echo yes || echo no)" \ - "$([ "$solved" -gt 0 ] && echo yes || echo no)" -echo - -if [ "$autosolve_on" -eq 0 ]; then - echo " INCONCLUSIVE — auto-solve never switched on. Either this endpoint is" - echo " not a Scraping Browser, or it does not carry the Captcha CDP domain." - echo " Check the log line 'Captcha.setAutoSolve not available'." -elif [ "$challenged" -eq 0 ]; then - echo " INCONCLUSIVE, and this is the trap this script exists for: your exit" - echo " was NOT challenged even once without the Browser API, so a green run" - echo " through it shows only that it fetches the page — not that it cleared" - echo " anything. Re-run from an address that IS challenged (a cloud VM or a" - echo " CI runner), or the answer stays unknown." -elif [ "$solved" -gt 0 ]; then - echo " ANSWERED: the exit is challenged ($challenged/10), the Browser API met" - echo " a challenge and reported solving it, and the run returned rows. That" - echo " is auto-solve doing the work, with a negative case behind it." -elif [ "$cdp_rc" -eq 0 ] && [ "$base_rc" -ne 0 ]; then - echo " ANSWERED (weaker form): the exit is challenged ($challenged/10), the" - echo " local browser failed and the Browser API succeeded. It cleared the" - echo " challenge, though no explicit solve event was logged — its own exit" - echo " may simply not be challenged. Useful either way." -else - echo " MIXED — read the logs. Both legs behaved the same way, so the Browser" - echo " API made no measurable difference from this address." -fi diff --git a/tools/waf_probe.sh b/tools/waf_probe.sh deleted file mode 100755 index 361b918..0000000 --- a/tools/waf_probe.sh +++ /dev/null @@ -1,136 +0,0 @@ -#!/usr/bin/env bash -# waf_probe.sh — measure how often transfermarkt.com answers with an AWS WAF -# captcha instead of the page, FROM WHEREVER YOU RUN IT. -# -# Why this ships in the repo rather than living in a notebook: the README and -# .env.example both quote a figure (8 of 10 requests challenged from a bare -# datacentre address, 2026-09-16), and a number describing a living thing has -# to arrive with the command that reproduces it or not be written down at -# all. That figure is a property of an address on a day, not of the site — -# from a proxied exit the same day, the real page came back every time. -# -# ./tools/waf_probe.sh # 10 requests to market-values -# ./tools/waf_probe.sh 20 # 20 requests -# ./tools/waf_probe.sh 10 transfers # a different mode -# ./tools/waf_probe.sh 10 market-values --direct # ignore any proxy -# -# Modes: market-values | club-squad | transfers | player -# -# THE EXIT IT MEASURES -# -------------------- -# By default this goes through GIVENCHY_PROXY when .env sets one, -# because "is this address challenged" is a question about an EXIT, and -# measuring the machine you happen to be sitting at answers it for one exit -# only. With no proxy configured, or with --direct, it measures this machine. -# -# The proxy is resolved by env_config.py, the same loader every engine uses, -# so precedence is the documented one (a real environment variable beats -# .env) and only one place decides it. It reaches curl through -# `https_proxy`/`http_proxy` in the ENVIRONMENT -- never a command-line flag, -# because `ps` reads argv. Only the masked host and port are ever printed. -# -# Reading the output: a page the site actually served answers from -# `server: nginx`, its own origin. A refusal answers from `server: CloudFront` -# with `x-amzn-waf-action: captcha` — CloudFront generated it at the edge and -# the request never reached Transfermarkt at all. - -set -u - -N="${1:-10}" -MODE="${2:-market-values}" -DIRECT="${3:-}" - -case "$MODE" in - market-values) URL="https://www.transfermarkt.com/spieler-statistik/wertvollstespieler/marktwertetop" ;; - club-squad) URL="https://www.transfermarkt.com/x/startseite/verein/281" ;; - transfers) URL="https://www.transfermarkt.com/statistik/neuestetransfers" ;; - player) URL="https://www.transfermarkt.com/x/profil/spieler/418560" ;; - *) echo "unknown mode: $MODE (expected market-values|club-squad|transfers|player)"; exit 2 ;; -esac - -UA="Mozilla/5.0 (Macintosh; Intel Mac OS X 10_15_7) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/140.0.0.0 Safari/537.36" - -# --- which exit -------------------------------------------------------------- -# The python below prints two shell assignments and nothing else: a MASKED -# label, safe to show, and the export lines carrying the real URL. `eval` -# consumes the second, so the credential never reaches argv, a log or the -# terminal. proxy_pool.mask() is the repo's own masker and is documented -# never to raise -- it is the last thing between a password and a log. -PROXY_LABEL="direct (this machine)" -if [ "$DIRECT" != "--direct" ]; then - eval "$(cd "$(dirname "$0")/.." && python3 - <<'ENVPY' -import shlex -import env_config -import proxy_pool - -env_config.load_env() -url = env_config.env_value("GIVENCHY_PROXY") -if url: - print("PROXY_LABEL=%s" % shlex.quote(proxy_pool.mask(url))) - for name in ("https_proxy", "http_proxy", "HTTPS_PROXY", "HTTP_PROXY"): - print("export %s=%s" % (name, shlex.quote(url))) -else: - print("PROXY_LABEL=%s" % shlex.quote("direct (no GIVENCHY_PROXY in .env)")) -ENVPY -)" -fi -HDR="$(mktemp)"; BODY="$(mktemp)" -trap 'rm -f "$HDR" "$BODY"' EXIT - -echo "URL: $URL" -echo "exit: $PROXY_LABEL" -echo "when: $(date -u '+%Y-%m-%d %H:%M:%SZ') requests: $N" -echo - -challenged=0 -unreachable=0 -served=0 -for i in $(seq 1 "$N"); do - # -L matters: the player and club URLs answer 301 to their canonical slug, - # and without it every redirect would be counted as a clean response. - code=$(curl -sSL -o "$BODY" -D "$HDR" -w "%{http_code}" "$URL" \ - -H "User-Agent: $UA" --max-time 40 2>/dev/null) - srv=$(grep -i '^server:' "$HDR" | tr -d '\r' | tail -1 | awk '{print $2}') - waf=$(grep -i '^x-amzn-waf-action:' "$HDR" | tr -d '\r' | tail -1 | awk '{print $2}') - title=$(grep -o -i '[^<]*' "$BODY" | head -1 \ - | sed 's/<[^>]*>//g' | cut -c1-46) - [ -n "${waf:-}" ] && challenged=$((challenged + 1)) - # curl reports 000 when it never got an HTTP response at all -- a dead or - # unauthenticated proxy, a DNS failure, a timeout. Counted SEPARATELY: - # without this, ten failed connections and ten clean pages both print - # "challenged: 0 of 10", which is the one thing this probe must never do. - if [ "$code" = "000" ]; then - unreachable=$((unreachable + 1)) - else - served=$((served + 1)) - fi - printf "%3d http=%-4s server=%-11s waf=%-8s %s\n" \ - "$i" "$code" "${srv:-?}" "${waf:-none}" "$title" - sleep 2 -done - -echo -echo "exit: $PROXY_LABEL" -echo "requests: $N" -echo "got an HTTP response: $served" -echo "never connected: $unreachable" -echo "challenged: $challenged of $served answered" -echo -if [ "$unreachable" -eq "$N" ]; then - echo "INCONCLUSIVE: not one request reached the site, so this says nothing" - echo "about the WAF. Every line above is a transport failure -- check the" - echo "exit itself before reading anything into a challenge count of zero." -elif [ "$unreachable" -gt 0 ]; then - echo "PARTIAL: $unreachable of $N never connected. The challenge rate below" - echo "is over the $served that did answer; the rest measured nothing." -fi -if [ "$challenged" -gt 0 ]; then - echo "This exit meets the WAF. A proxy or the Scraping Browser API is what" - echo "gets the page from here; TWOCAPTCHA_KEY is what solves the challenge" - echo "when one still appears (task type AmazonTaskProxyless)." -elif [ "$served" -gt 0 ]; then - echo "This exit was not challenged in $served answered request(s). That is a" - echo "fact about this address today, not about the site: on 2026-09-16 the" - echo "same URL from one datacentre address returned 200 and 405 interleaved." - echo "Run it again, and run at least 10 before concluding anything." -fi