From 17f432d579d9b73caff74d41d2152426bab91e42 Mon Sep 17 00:00:00 2001 From: Petr Date: Thu, 22 Oct 2026 22:18:58 +0000 Subject: [PATCH] Scraper API on the shared output contract; --url keeps its market; docs and packaging MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit From a manual test of v0.2.0 (#14, #15, #16), plus one defect found while verifying #14. A market in --url was overwritten. `--gl`/`--hl` defaulted to US/en and were written over the URL, so `--url ".../search?q=vpn&hl=ja&gl=JP"` fetched the US store — USD, English titles, rows labelled US — silently. Found by comparing the Scraper API with Playwright on that URL: 23 of 30 apps shared and 58 field differences, with the Scraper API the one showing JPY. The URL's own gl/hl now outrank the defaults; a typed flag replaces them and says so. After the fix the two engines returned the same 30 apps with no field different. Scraper API (#14): it wrote rows only — no `.meta.json`, so `diff_runs.py` could not use it; `page`/`position` empty; `--format both` where the engines default to json; and "not run against this site" text. It now reads the page state through the engines' `_handle_state` and writes through `finish_run`, fills page and position, defaults to json, normalises `--out` the same way, and pins hl/gl in the request — without them Play chose the market from the API's exit while rows were labelled with a default. Run live 2026-10-22: a category (96 rows), a JP search (30), an app (1, position 1), a developer (10), all exit 0 with a sidecar; a `gl` Play does not serve exits 2; $0.0006 for the five requests. An app row from `parse_products` also gets position 1, as the engines' own app run gives it. diff_runs (#15): a refusal now ends with advice for its actual reason (the same --gl, the same --mode, re-run the incomplete side, compare reviews on review_id) instead of "re-run the incomplete side" for every reason. Docs (#15, #16 and the pyppeteer note): README explains why `original_price` and `discount_pct` are null (0 of 411 capture rows, 0 of 877 live rows); the Scraper API is in the engine table; pyppeteer on Apple Silicon is documented — its Chromium download has no arm64 entry. `.dockerignore` leaves out `.venv-*/`, `venv-*/` and `captures/`: a build with a planted 20 MB venv sent 288 kB of context, the image built, ran `--help` and holds no `.env` and no captures. Tests: the Scraper API path through its own `main()` with the network replaced by captures; market precedence for `--url` with and without typed flags, and a fake-driver run of a JP URL; diff_runs' advice line for two complete markets and for an incomplete run. Six controls, each caught by its named check — one of them showed the diff test matching "same --gl" in the problem text rather than the advice, and the test now reads the advice line. Co-Authored-By: Claude Opus 5.5 Claude-Session: https://claude.ai/code/session_01VrpkXSoXyZMGV5nqF9JV6P --- .dockerignore | 6 ++ CHANGELOG.md | 26 ++++++++ README.md | 12 +++- diff_runs.py | 29 +++++++-- engine_core.py | 43 +++++++++++-- product_parser.py | 6 +- scraper_api_client.py | 107 +++++++++++++++++++++++++------ smoke_test.py | 143 ++++++++++++++++++++++++++++++++++++++++++ 8 files changed, 341 insertions(+), 31 deletions(-) diff --git a/.dockerignore b/.dockerignore index cc2f9d1..104496d 100644 --- a/.dockerignore +++ b/.dockerignore @@ -10,6 +10,12 @@ __pycache__/ .fingerprint_cache/ .venv/ venv/ +# One virtualenv per engine is what the README recommends, under any name; +# and captures/ is test material the image never copies. Neither belongs in +# the build context: ~240 MB of it was sent to the daemon on every build. +.venv-*/ +venv-*/ +captures/ env/ googleplay_apps.* *_debug.html diff --git a/CHANGELOG.md b/CHANGELOG.md index c897a6d..d801c29 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -98,9 +98,35 @@ discovered from a bill or a diff. while a previous session on the same profile closes. Other refusals — 401, a closed port, a timeout — fail at once. +- **A market in `--url` is kept.** `--gl` and `--hl` defaulted to `US` and + `en` and were then written over the URL, so `--url "…/search?q=vpn&hl=ja&gl=JP"` + fetched the US store — USD prices, English titles, rows labelled US — with + nothing said. The URL's own `gl`/`hl` now win over the defaults; a typed + flag still replaces them, and says so. Found while comparing the Scraper + API with Playwright: after the fix the two agreed on all 30 apps of that + search with no field different. +- **The Scraper API path follows the same output contract.** It writes a + `.meta.json` (so `diff_runs.py` can use its output), fills `page` and + `position`, defaults to `--format json` like the browser engines, and + normalises `--out` the same way. It also pins `hl`/`gl` in the request: + without them Play chose the market from the API's own exit while the rows + were labelled with a default. The "not run against this site yet" notes + are replaced by what was measured. +- `diff_runs.py` ends a refusal with advice for the actual reason — the same + `--gl`, the same `--mode`, re-run the incomplete side — instead of "re-run + the incomplete side" whatever the reason. +- `.dockerignore` leaves out `.venv-*/`, `venv-*/` and `captures/`: with one + virtualenv per engine, as the README recommends, ~240 MB was sent to the + build. The image itself copies files by name and is unchanged. + ### Changed - CI only: the canary now runs weekly (Mondays 06:17 UTC) instead of daily, and on a manual dispatch. +- README: why `original_price` and `discount_pct` are null on every row (0 + of 411 rows from the captures, 0 of 877 from live runs). +- README: the Scraper API is in the engine table. +- README: pyppeteer on Apple Silicon — its Chromium download has no arm64 + entry; use it through `--cdp-endpoint` there, or use Playwright. ## [0.2.0] — 2026-10-15 diff --git a/README.md b/README.md index 8099eec..090abc8 100644 --- a/README.md +++ b/README.md @@ -206,7 +206,8 @@ package ids were identical. |---|---|---| | **Playwright** (primary) | `requirements-playwright.txt` | the one to use | | Selenium | `requirements-selenium.txt` | no authenticated remote CDP endpoint (`--cdp-endpoint` is refused, exit 2); `--proxy-server` cannot authenticate at all, so credentials are stripped and a warning printed; exposes no HTTP status, so classification leans on the structural signals (Play's own 404 template is recognised from its body); `--fingerprint` is refused | -| pyppeteer | `requirements-puppeteer.txt` | effectively unmaintained upstream; here for parity; `--fingerprint` is refused | +| pyppeteer | `requirements-puppeteer.txt` | effectively unmaintained upstream; here for parity; `--fingerprint` is refused. On Apple Silicon Macs, use it through `--cdp-endpoint` or use Playwright: the Chromium pyppeteer downloads comes from its `Mac` (x86_64) snapshot channel — it has no Apple Silicon entry — and was reported to crash on macOS arm64 | +| Scraper API | `scraper_api_client.py --url …` | no local browser: the 2Captcha Scraper API fetches one page per request (measured 2026-10-22 at $0.0003 or less each). Same rows, `.meta.json` and exit codes as the browser engines; a grid's further shelves and later review pages are the browser engines' job. Which apps fill a category's shelves depends on the requesting address, so a grid can differ from a local run (82 of 96 apps shared in one comparison) while a search came back identical | | Scraping Browser API | `--cdp-endpoint` | a managed browser; never stack `--proxy` or `--fingerprint` on it. Playwright and pyppeteer open their own page on it and leave the browser running when the run ends; a handshake refused with HTTP 500/502/503 is retried twice (3 s, 6 s), anything else fails at once with exit 5 | **Install exactly one per virtualenv.** Playwright and pyppeteer declare @@ -229,6 +230,12 @@ every repo in this family; Play's own follow. See Exit codes: `0` ok · `1` crash · `2` bad usage · `3` blocked · `4` zero apps · `5` **the content was never obtained** · `6` partial. +`original_price` and `discount_pct` are part of the family prefix and are +null on every row: 0 of 411 rows parsed from the captures and 0 of 877 rows +from live runs on 2026-10-22. Play does show a struck-through price on some +discounted paid apps; this repository has no capture of one and does not read +a was-price, so the columns stay empty rather than guess. + `5` and `4` are the distinction worth knowing: a run that never got its page must not report "the catalogue is empty", or a dead proxy and an empty category look the same to whatever reads the file. A run that gathered @@ -265,6 +272,9 @@ many cells differ. 2026-10-22) gets Play's own HTTP 400 page, which this scraper reports as a usage error naming the flag — with no retries and no proxy advice. `ZZ`, by contrast, is answered with a full page. +* **A market in `--url` is kept.** `--url "…/search?q=vpn&hl=ja&gl=JP"` + fetches Japan; `--gl`/`--hl` replace it only when typed, and say so. Up to + 0.2.0 the defaults (`US`, `en`) silently replaced it. * **`--out` is a prefix.** `--out data/run.json` writes `data/run.json`, `data/run.csv` and `data/run.meta.json`, creating `data/` if needed. * **`genre_id` is null on every listing row, and that is the store.** A grid diff --git a/diff_runs.py b/diff_runs.py index b41e8c1..5e6c6dd 100644 --- a/diff_runs.py +++ b/diff_runs.py @@ -223,9 +223,9 @@ def _print_summary(result: dict) -> None: def _run_status(path: str) -> Tuple[Optional[str], Optional[dict]]: """Read the `.meta.json` sidecar beside a run's JSON output. - Returns (status, meta), or (None, None) when there is no sidecar — which - is the normal case for output written before run metadata existed, or by - `scraper_api_client.py` (single fetch, no pagination to cut short). + Returns (status, meta), or (None, None) when there is no sidecar — the + case for output written before run metadata existed. Every engine + writes one now, `scraper_api_client.py` included. """ meta_path = re.sub(r"\.json$", "", path) + ".meta.json" try: @@ -247,6 +247,16 @@ def _check_comparable(args) -> bool: this is a refusal with a --force escape hatch rather than a hard error. """ problems = [] + # What would make the two runs comparable, one line per KIND of problem, + # in the order found. The closing advice used to be "re-run the + # incomplete side" whatever the reason, which is wrong for two complete + # runs of different markets and for a reviews run. + advice = [] + + def advise(line): + if line not in advice: + advice.append(line) + modes = {} for label, path in (("--old", args.old), ("--new", args.new)): status, meta = _run_status(path) @@ -265,16 +275,20 @@ def _check_comparable(args) -> bool: f"{label} ({path}) is a {mode!r} run, which is not one row " f"per sku. This tool diffs one row per sku on price, so there " f"is nothing here it can compare.") + advise("compare reviews runs on review_id yourself; this tool " + "diffs app rows") if status != "complete": problems.append( f"{label} ({path}) was a {status!r} run — stopped after " f"{meta.get('pages_completed')} of {meta.get('pages_requested')} " f"page(s), reason {meta.get('stop_reason')!r}") + advise("re-run the incomplete side") if len(set(modes.values())) > 1: problems.append( f"the two runs are different modes ({modes}). A listing row and a " f"detail row carry different fields, so `added`/`removed` would " f"describe the mode change rather than the catalogue.") + advise("diff two runs of the same --mode") # A CURRENCY MISMATCH. Google Play prices per market: the README's # table shows one app quoted in USD, EUR, JPY, INR and BRL from one @@ -296,6 +310,7 @@ def _check_comparable(args) -> bool: f"{label} ({path}) holds more than one currency ({sorted(seen)}) " f"— that run was redirected mid-way and its own prices are not " f"comparable with each other, let alone with another run's.") + advise("re-run that side") if len(set(currencies.values())) > 1: problems.append( f"the two runs quote different currencies ({currencies}). " @@ -303,6 +318,7 @@ def _check_comparable(args) -> bool: f"of different --gl values, and their prices and ratings are not " f"comparable. Diff two runs of the same --gl. `source` cannot " f"catch it: it is 'play.google.com' on both sides.") + advise("diff two runs of the same --gl") # A MARKET mismatch, read from the rows' own `gl`. The currency above # cannot tell two markets that share one apart — DE and FR are both EUR — @@ -323,6 +339,7 @@ def _check_comparable(args) -> bool: f"the two runs describe different markets ({markets}). Ratings and " f"prices are per --gl, so every difference would be the market " f"rather than the store moving. Diff two runs of the same --gl.") + advise("diff two runs of the same --gl") if not problems: return True @@ -334,9 +351,9 @@ def _check_comparable(args) -> bool: print("[!] Refusing to diff these two runs:") for line in problems: print(f" {line}") - print(" Re-run the incomplete side, or pass --force to compare anyway " - "(added/removed will include products that were simply never " - "fetched).") + print(" To compare them: " + "; ".join(advice) + ".") + print(" --force compares anyway, and then the differences describe the " + "mismatch above rather than the store.") return False diff --git a/engine_core.py b/engine_core.py index fe8f5ec..34b9b7d 100644 --- a/engine_core.py +++ b/engine_core.py @@ -48,6 +48,7 @@ def close() -> None import re import sys import time +import urllib.parse from typing import Any, Callable, Dict, List, Optional, Sequence, Tuple import captcha_solver @@ -127,11 +128,14 @@ def build_parser() -> argparse.ArgumentParser: ap.add_argument("--query", help="a search query (--mode search)") ap.add_argument("--app-id", help="an Android package id, e.g. com.whatsapp") ap.add_argument("--developer", help="a developer id or name") - ap.add_argument("--gl", default="US", type=country_code, + ap.add_argument("--gl", default=None, type=country_code, help="the MARKET, as a two-letter country code. Changes " "the rating, the currency and which apps are listed. " - "A code Play does not serve is exit 2. Default US.") - ap.add_argument("--hl", default="en", help="the display language. Default en.") + "A code Play does not serve is exit 2. Default: " + "the gl in --url, else US.") + ap.add_argument("--hl", default=None, + help="the display language. Default: the hl in --url, " + "else en.") ap.add_argument("--pages", type=positive_int, default=1, help="how many fetches to make. On a grid this follows " "the page's own shelves; on reviews it follows the " @@ -205,12 +209,42 @@ def build_parser() -> argparse.ArgumentParser: return ap +DEFAULT_HL, DEFAULT_GL = "en", "US" + + +def _settle_market(args: argparse.Namespace) -> None: + """The market of a `--url` run: the URL's own, unless a flag was typed. + + `--gl` and `--hl` used to DEFAULT to US and en and were then written over + the URL, so `--url ".../search?q=vpn&hl=ja&gl=JP"` was fetched as + `hl=en&gl=US` — measured 2026-10-22: USD prices and English titles, rows + labelled US — with nothing said. A market in the URL is something the + caller typed and outranks a default. A flag the caller typed outranks the + URL, and says so. + """ + query = dict(urllib.parse.parse_qsl(urllib.parse.urlsplit(args.url).query)) + for flag, attr, default in (("--gl", "gl", DEFAULT_GL), ("--hl", "hl", DEFAULT_HL)): + in_url = query.get(attr) + if attr == "gl" and in_url: + in_url = in_url.upper() + typed = getattr(args, attr) + if typed and in_url and typed != in_url: + print(f"[info] {flag} {typed} replaces {attr}={in_url} in --url", + file=sys.stderr) + setattr(args, attr, typed or in_url or default) + + def resolve_target(args: argparse.Namespace) -> Tuple[Optional[str], Optional[str], Optional[str]]: - """`(mode, url, error)` — what this invocation is actually asking for.""" + """`(mode, url, error)` — what this invocation is actually asking for. + + Also settles the MARKET, and writes it back to `args.gl` / `args.hl` so the + request, the rows and every message agree on it. + """ if args.url: reason = parser.refusal_reason(args.url) if reason: return None, None, reason + _settle_market(args) mode = parser.mode_for_url(args.url) # An app page is the address of BOTH app-level modes: `--mode app` # reads the page, `--mode reviews` pages that app's reviews. Refusing @@ -226,6 +260,7 @@ def resolve_target(args: argparse.Namespace) -> Tuple[Optional[str], Optional[st f"{mode} URL") return mode, parser.with_market(args.url, args.hl, args.gl), None + args.hl, args.gl = args.hl or DEFAULT_HL, args.gl or DEFAULT_GL if args.mode == "listing": if not args.category: return None, None, "--mode listing needs --category (e.g. GAME) or --url" diff --git a/product_parser.py b/product_parser.py index b59542c..3174a76 100644 --- a/product_parser.py +++ b/product_parser.py @@ -1513,7 +1513,11 @@ def parse_products(markup: str, url: str = "", category: Optional[str] = None, """ mode = mode_for_url(url) if url else None if mode == "app": - row = parse_detail(markup, package_from_url(url), hl=hl, gl=gl, page=page) + # Position 1, as the browser engines' app run gives it: one page is + # one app. Left unset here, an app row from the Scraper API came back + # with `position` null beside the same row's `page` 1. + row = parse_detail(markup, package_from_url(url), hl=hl, gl=gl, page=page, + position=1 if page is not None else None) rows = [row] if row is not None else [] else: rows = parse_grid(markup, hl=hl, gl=gl, page=page) diff --git a/scraper_api_client.py b/scraper_api_client.py index e352b7f..2f0ba21 100644 --- a/scraper_api_client.py +++ b/scraper_api_client.py @@ -6,13 +6,20 @@ the HTML; this client parses it with the same `product_parser` the browser engines use, so the rows and columns are identical. -NOT RUN ON THIS SITE YET ------------------------- -This path has not been run against Google Play; no measurement of it is -recorded in this repo. An unrun path says "not run", never "does not work" — -a sibling repo once shipped a README telling readers a 2Captcha key would not -help them, when the truth was that the client had not implemented the task -type. +WHAT WAS MEASURED HERE +--------------------- +Run live 2026-10-22, one request each: a category grid (96 rows), a search +with `hl=ja&gl=JP` (30 rows, JPY, Japanese titles), an app page (1 row, the +detail columns filled) and a developer grid (10 rows), all exit 0 with a +`.meta.json`, at $0.0003 or less per request as the API's own `x-debug` +header reported. A `gl` Play does not serve came back as Play's HTTP 400 and +exits 2. + +The same search through Playwright gave the same 30 apps with no field +different. A category grid did differ — 82 of 96 apps shared — while the +same grid fetched twice from one address was identical: which apps fill a +grid's shelves depends on the address that asks, and this path asks from the +API's. WHAT TO KNOW BEFORE USING IT HERE --------------------------------- @@ -22,6 +29,10 @@ * One request is one page. A grid's further shelves and a reviews run's continuation token are followed by the browser engines, not by this client. +* The market is pinned in the request: a URL without `hl`/`gl` gets the + browser engines' defaults (en, US) before it is sent, so the rows' market + columns describe what was fetched rather than whatever the API's exit + country would have chosen. python3 scraper_api_client.py \\ --url "https://play.google.com/store/apps/category/GAME" @@ -42,9 +53,13 @@ import requests +import urllib.parse + from product_parser import (parse_products, detect_bot_challenge, - detect_page_state, BOT_CHALLENGE_MARKERS) -from output_writer import save + detect_page_state, BOT_CHALLENGE_MARKERS, + mode_for_url, with_market) +from output_writer import finish_run +import engine_core import env_config logging.basicConfig(level=logging.INFO, format="%(asctime)s [%(levelname)s] %(message)s") @@ -243,6 +258,7 @@ def _run_once(args, attempt: int = 1, attempts: int = 1) -> int: with open(args.dump_html, "w", encoding="utf-8") as f: f.write(html) logger.info("Raw HTML written to %s", args.dump_html) + hl, gl = market_of(args.url) # Same policy as the browser engines. detect_page_state returns # `(state, reason)`; comparing the tuple itself to "blocked" never @@ -277,8 +293,24 @@ def _run_once(args, attempt: int = 1, attempts: int = 1) -> int: "playwright_scraper.py / puppeteer_scraper.py directly.") return 3 - products = parse_products(html, args.url, category=args.category) - logger.info("Parsed %d products.", len(products)) + # From here on, the same contract as the browser engines: the page state + # is read by the same `_handle_state`, rows carry `page` and `position`, + # and the run is written by the same `finish_run` — so a `.meta.json` + # sidecar exists and `diff_runs.py` can compare a Scraper API run with + # a browser run. Up to 0.2.0 this path wrote rows only, with `page` and + # `position` empty, and reported "0 products" as exit 4 whatever the + # page had been. + out = engine_core.RunOutcome() + products = [] + if engine_core._handle_state(out, state, reason, 1): + products = parse_products(html, args.url, category=args.category, + hl=hl, gl=gl, page=1) + if products: + out.pages_completed = 1 + else: + out.pages_failed.append(1) + out.stop_reason = "parser_found_nothing" + logger.info("Parsed %d products (page state %s).", len(products), state) if not products: dump = f"{args.out}_scraperapi_debug.html" @@ -286,17 +318,43 @@ def _run_once(args, attempt: int = 1, attempts: int = 1) -> int: f.write(html) logger.warning("0 products parsed — saved the raw response to %s so you can see " "what actually came back.", dump) - return 4 - return save(products, args.out, args.format, allow_empty=args.allow_empty) + return finish_run( + products, args.out, args.format, args.allow_empty, + blocked=out.blocked, stop_reason=out.stop_reason, + pages_requested=1, pages_completed=out.pages_completed, + start_url=args.url, final_url=args.url, + pages_failed=out.pages_failed or None, + mode=mode_for_url(args.url) or "listing", + extra={"engine": "scraper_api", "gl": gl, "hl": hl}) + + +def market_of(url: str): + """`(hl, gl)` as the URL states them.""" + query = dict(urllib.parse.parse_qsl(urllib.parse.urlsplit(url).query)) + return query.get("hl") or "en", (query.get("gl") or "US").upper() + + +def pin_market(url: str) -> str: + """Put `hl`/`gl` on a URL that lacks them, with the engines' defaults. + + The browser engines always send both. Without them Play picks the market + from the address the request comes from — here the Scraper API's own exit, + whose country this client does not choose — while every row would still + be labelled with a default market. A row's market is a column on this + site because the rating and the currency follow it, so it is pinned in + the request rather than guessed afterwards. + """ + hl, gl = market_of(url) + return with_market(url, hl, gl) def parse_args(): p = argparse.ArgumentParser( - description="Google Play scraper — 2captcha Scraper API edition (no " - "local browser). One request is one page; not run " - "against this site yet — see this file's docstring, and " - "prefer playwright_scraper.py.") + description="Google Play scraper — 2Captcha Scraper API edition (no " + "local browser). One request is one page: a grid's " + "further shelves and a reviews run's later pages are the " + "browser engines' job.") # NOT required: prefer the TWOCAPTCHA_KEY env var. A key passed on the # command line is visible to anyone who can run `ps`, and it lands in # shell history and in any log that echoes the command line. @@ -308,8 +366,11 @@ def parse_args(): "app page or a developer page). Required, unless " "GOOGLEPLAY_URL is set in the environment or in .env.") p.add_argument("--category", default=None, help="Label to tag output rows with. Defaults to the category segment of the URL, so the column is never empty just because the flag was omitted.") - p.add_argument("--format", choices=["json", "csv", "both"], default="both") - p.add_argument("--out", default="googleplay_apps_scraperapi", help="Output file prefix") + p.add_argument("--format", choices=["json", "csv", "both"], default="json") + p.add_argument("--out", default="googleplay_apps_scraperapi", + help="output prefix: writes PREFIX.json / .csv / .meta.json. " + "A trailing .json or .csv is dropped, and missing " + "directories are created.") p.add_argument("--timeout", type=int, default=60, help=f"API-side task timeout in seconds (1-{MAX_API_TIMEOUT}, default 60)") p.add_argument("--cdp-url", default=None, @@ -346,6 +407,14 @@ def parse_args(): if not args.url: p.error("no --url given, and GOOGLEPLAY_URL is not set in the environment " "or in .env.") + args.url = pin_market(args.url) + args.out = os.path.expanduser(engine_core.output_prefix(args.out)) + if args.dump_html: + args.dump_html = os.path.expanduser(args.dump_html) + for flag, path in (("--out", args.out), ("--dump-html", args.dump_html)): + problem = engine_core.ensure_parent_dir(path) + if problem: + p.error(f"{flag} {path}: {problem}") return args diff --git a/smoke_test.py b/smoke_test.py index faa4497..5d731cd 100644 --- a/smoke_test.py +++ b/smoke_test.py @@ -1984,6 +1984,118 @@ async def flaky(endpoint, timeout_s): return ok +def test_scraper_api_contract(): + """The Scraper API path writes what the browser engines write. + + Driven through its own `main()` with the network call replaced by a real + capture. Up to 0.2.0 it wrote rows only — `page` and `position` empty, no + `.meta.json`, so `diff_runs.py` could not use its output — and labelled + every row with the default market whatever the URL asked for. + """ + import scraper_api_client as api + ok = True + grid, missing = capture("search_vpn_jp.html"), capture("notfound.html") + if None in (grid, missing): + skip("scraper-api", "captures needed") + return True + + def run(url, page, extra=()): + saved_fetch, saved_argv = api.fetch_html, sys.argv + sent = {} + + def fake_fetch(args): + sent["url"] = args.url + return page + + api.fetch_html = fake_fetch + with tempfile.TemporaryDirectory() as tmp: + prefix = os.path.join(tmp, "api.json") + sys.argv = ["scraper_api_client.py", "--key", "test-key", "--url", url, + "--out", prefix, "--retries", "0", *extra] + try: + with redirect_stdout(io.StringIO()), redirect_stderr(io.StringIO()): + rc = api.main() + except SystemExit as exc: + rc = exc.code + finally: + api.fetch_html, sys.argv = saved_fetch, saved_argv + got = {} + for ext in (".json", ".meta.json", ".csv"): + path = os.path.join(tmp, "api" + ext) + if os.path.exists(path): + with open(path, encoding="utf-8") as fh: + got[ext] = fh.read() + return rc, got, sent + + rc, files, sent = run("https://play.google.com/store/search?q=vpn&c=apps&hl=ja&gl=JP", + (grid, 200)) + rows = json.loads(files.get(".json", "[]")) + meta = json.loads(files.get(".meta.json", "{}")) + ok &= check("[api] a served grid is exit 0", rc == 0, f"rc={rc}") + ok &= check("[api] --out api.json writes api.json, not api.json.json", + ".json" in files) + ok &= check("[api] json is the default format, like the engines", + ".csv" not in files) + ok &= check("[api] a .meta.json sidecar is written", meta.get("status") == "complete", + str(meta)[:120]) + ok &= check("[api] rows carry page and position", + rows and all(r["page"] == 1 for r in rows) + and [r["position"] for r in rows] == list(range(1, len(rows) + 1))) + ok &= check("[api] rows carry the URL's market, not a default", + rows and all((r["gl"], r["hl"]) == ("JP", "ja") for r in rows), + str(rows[0]["gl"] if rows else None)) + + rc, _files, sent = run("https://play.google.com/store/apps/category/GAME", (grid, 200)) + ok &= check("[api] a URL without a market is sent WITH the engines' default", + "gl=US" in sent.get("url", "") and "hl=en" in sent.get("url", ""), + sent.get("url")) + + rc, files, _ = run("https://play.google.com/store/apps/details?id=com.no.such.app", + (missing, 404)) + ok &= check("[api] a package id that does not exist is exit 4", rc == 4, f"rc={rc}") + + app = capture("app_free_us.html") + if app is not None: + rc, files, _ = run("https://play.google.com/store/apps/details?id=com.whatsapp", + (app, 200)) + rows = json.loads(files.get(".json", "[]")) + ok &= check("[api] an app row is page 1, position 1, like the engines'", + rows and (rows[0]["page"], rows[0]["position"]) == (1, 1), + str(rows and (rows[0]["page"], rows[0]["position"]))) + return ok + + +def test_url_market_precedence(): + """A market typed in --url outranks the default; a typed flag outranks the URL.""" + import engine_core + ok = True + jp = "https://play.google.com/store/search?q=vpn&c=apps&hl=ja&gl=JP" + cases = ( + ([jp], "JP", "ja"), + ([jp, "--gl", "DE"], "DE", "ja"), + (["https://play.google.com/store/search?q=vpn&c=apps"], "US", "en"), + ) + for argv, gl, hl in cases: + args = engine_core.build_parser().parse_args(["--url", *argv]) + with redirect_stderr(io.StringIO()): + _mode, url, _err = engine_core.resolve_target(args) + ok &= check(f"[market] {' '.join(argv[1:]) or 'no flags'} on {argv[0][-22:]}" + f" -> gl={gl} hl={hl}", + (args.gl, args.hl) == (gl, hl) and f"gl={gl}" in url + and f"hl={hl}" in url, f"{args.gl} {args.hl} {url}") + + page = capture("search_vpn_jp.html") + if page is not None: + rc, world, files = _run_main(["--url", jp], [(page, 200)]) + rows = json.loads(files.get(".json", "[]")) + ok &= check("[market] --url with gl=JP is fetched as JP", + world["fetches"] and "gl=JP" in world["fetches"][0][0], + str(world["fetches"][:1])) + ok &= check("[market] and its rows say JP", + rows and all((r["gl"], r["hl"]) == ("JP", "ja") for r in rows)) + return ok + + def test_run_contract_captcha_budget(): """At most one paid solve per page, whatever retries and rotations do.""" ok = True @@ -2256,9 +2368,38 @@ def test_diff_refuses_two_markets_in_one_currency(): ok &= check("[diff] two markets sharing a currency are refused", verdict is False and "different markets" in buf.getvalue(), buf.getvalue()[-300:]) + # The advice must follow the reason. Both runs here are COMPLETE, so + # "re-run the incomplete side" — what every refusal used to end with — + # would send the reader to re-run something that did not fail. + # Read the ADVICE line itself: the problem text above it also says + # "same --gl", and a check on the whole output passed with the advice + # broken. + advice = _advice_line(buf.getvalue()) + ok &= check("[diff] the advice for two markets is the same --gl", + "same --gl" in advice, repr(advice)) + ok &= check("[diff] and not to re-run a side that was complete", + "incomplete" not in advice.lower(), repr(advice)) + + # And for an incomplete run, that IS the advice. + meta = os.path.join(tmp, "DE.meta.json") + with open(meta, "w", encoding="utf-8") as fh: + json.dump({"status": "partial", "pages_completed": 1, + "pages_requested": 3, "stop_reason": "x"}, fh) + args = _ap.Namespace(old=paths["DE"], new=paths["DE"], force=False) + buf = io.StringIO() + with redirect_stdout(buf), redirect_stderr(buf): + diff_runs._check_comparable(args) + advice = _advice_line(buf.getvalue()) + ok &= check("[diff] an incomplete run is told to re-run that side", + "re-run the incomplete side" in advice, repr(advice)) return ok +def _advice_line(output): + lines = [l.strip() for l in output.splitlines() if "To compare them:" in l] + return lines[0] if lines else "" + + def test_workflows_import_no_local_module_inline(): """A workflow step calls ci_checks.py; it does not re-implement it. @@ -2325,6 +2466,8 @@ def main(): test_cdp_connect_policy, test_cdp_teardown_playwright, test_cdp_teardown_pyppeteer, + test_scraper_api_contract, + test_url_market_precedence, test_run_contract_captcha_budget, test_cli_rejects_counts_that_cannot_run, test_env_url_does_not_override_a_typed_builder_flag,