From 9f98ba0ce857efcd2d031d1120d36a860eae8e5c Mon Sep 17 00:00:00 2001 From: juliantellez <4896851+juliantellez@users.noreply.github.com> Date: Mon, 31 Aug 2026 11:15:35 +0000 Subject: [PATCH 1/7] feat: name what the platform answered at the boundary instead of calling it a missing video --- src/contract.ts | 18 ++++- src/handler.test.ts | 2 +- src/observe.test.ts | 38 +++++++++++ src/observe.ts | 25 +++++++ src/page/render.test.ts | 3 +- src/store/lookup.test.ts | 8 ++- src/store/lookup.ts | 8 ++- src/transcript/diagnosis.test.ts | 112 +++++++++++++++++++++++++++++++ src/transcript/diagnosis.ts | 94 ++++++++++++++++++++++++++ src/transcript/youtube.test.ts | 55 ++++++++++++--- src/transcript/youtube.ts | 107 +++++++++++++++++++++++------ 11 files changed, 435 insertions(+), 35 deletions(-) create mode 100644 src/observe.test.ts create mode 100644 src/observe.ts create mode 100644 src/transcript/diagnosis.test.ts create mode 100644 src/transcript/diagnosis.ts diff --git a/src/contract.ts b/src/contract.ts index a4db155..c1c3735 100644 --- a/src/contract.ts +++ b/src/contract.ts @@ -20,11 +20,27 @@ export interface TranscriptItem { source: string; } +// Why a read did not produce a transcript. The reader is told a different thing for each one, and +// the log names the same value, so a page and a log line can be lined up. +// +// The default is never "the video is missing". A page this code does not recognise is a refusal, +// because reporting a refusal as a missing video is how a working video came to read as deleted. +export type Cause = + | "video_missing" + | "bot_check" + | "consent_wall" + | "rate_limited" + | "platform_error" + | "unrecognised_page" + | "captions_refused" + | "captions_not_json" + | "captions_empty"; + export type LookupResult = | { kind: "ok"; item: TranscriptItem; cached: boolean } | { kind: "bad_id"; given: string } | { kind: "not_found"; video_id: string } - | { kind: "upstream_failed"; video_id: string; reason: string }; + | { kind: "upstream_failed"; video_id: string; cause: Cause; reason: string }; export const VIDEO_ID_PATTERN = /^[A-Za-z0-9_-]{11}$/; diff --git a/src/handler.test.ts b/src/handler.test.ts index 548a1fe..0575f1d 100644 --- a/src/handler.test.ts +++ b/src/handler.test.ts @@ -83,7 +83,7 @@ describe("what the edge is told to hold", () => { }); it("never holds a failure, so the next visit reaches the platform", () => { - expect(cacheControlFor({ kind: "upstream_failed", video_id: "x", reason: "r" })).toBe("no-store"); + expect(cacheControlFor({ kind: "upstream_failed", video_id: "x", cause: "bot_check", reason: "r" })).toBe("no-store"); }); }); diff --git a/src/observe.test.ts b/src/observe.test.ts new file mode 100644 index 0000000..5b299ad --- /dev/null +++ b/src/observe.test.ts @@ -0,0 +1,38 @@ +import { describe, expect, it } from "vitest"; +import { line, withoutSecrets } from "./observe.js"; + +describe("writing a line", () => { + it("names the event and carries the fields", () => { + expect(JSON.parse(line("watch_page_read", { video_id: "gyN9lV9QgyA", status: 200 }))).toEqual({ + event: "watch_page_read", + video_id: "gyN9lV9QgyA", + status: 200, + }); + }); + + it("writes one line, so one read is one record", () => { + expect(line("a", { b: "c\nd" })).not.toContain("\n"); + }); +}); + +describe("keeping credentials out of the log", () => { + // The caption address carries the signature the platform issued, the token minted for this video + // and whatever else travels in the query. None of it belongs in a log that is kept for a fortnight. + const signed = + "https://www.youtube.com/api/timedtext?v=gyN9lV9QgyA&signature=s3cr3t&expire=1756000000&pot=t0ken&fmt=json3"; + + it("keeps the host and the path and drops the query", () => { + expect(withoutSecrets(signed)).toBe("https://www.youtube.com/api/timedtext"); + }); + + it("drops the signature, the token and the expiry", () => { + const written = withoutSecrets(signed); + expect(written).not.toContain("s3cr3t"); + expect(written).not.toContain("t0ken"); + expect(written).not.toContain("expire"); + }); + + it("says so rather than throwing when the address does not parse", () => { + expect(withoutSecrets("not an address")).toBe("an address that does not parse"); + }); +}); diff --git a/src/observe.ts b/src/observe.ts new file mode 100644 index 0000000..1fa1157 --- /dev/null +++ b/src/observe.ts @@ -0,0 +1,25 @@ +// One line of JSON per thing worth knowing. CloudWatch reads a JSON line as fields, so a query can +// count causes without a regular expression over prose. +// +// Nothing here ever carries a credential. The cookies the watch page sets, the proof of origin +// token and the signature on a caption address are all secrets, so an address is reduced to its +// host and path before it is written, and no header is written at all. + +export type Fields = Record; + +export function withoutSecrets(url: string): string { + try { + const parsed = new URL(url); + return `${parsed.origin}${parsed.pathname}`; + } catch { + return "an address that does not parse"; + } +} + +export function line(event: string, fields: Fields): string { + return JSON.stringify({ event, ...fields }); +} + +export function observe(event: string, fields: Fields): void { + console.log(line(event, fields)); +} diff --git a/src/page/render.test.ts b/src/page/render.test.ts index b4ce41d..4a7a851 100644 --- a/src/page/render.test.ts +++ b/src/page/render.test.ts @@ -145,6 +145,7 @@ describe("the pages for what went wrong", () => { const page = renderPage({ kind: "upstream_failed", video_id: "gyN9lV9QgyA", + cause: "captions_refused", reason: "the caption endpoint returned an empty body", }); expect(page.status).toBe(502); @@ -156,7 +157,7 @@ describe("the pages for what went wrong", () => { for (const result of [ { kind: "bad_id", given: "x" }, { kind: "not_found", video_id: "aaaaaaaaaaa" }, - { kind: "upstream_failed", video_id: "aaaaaaaaaaa", reason: "r" }, + { kind: "upstream_failed", video_id: "aaaaaaaaaaa", cause: "bot_check", reason: "r" }, ] satisfies LookupResult[]) { expect(renderPage(result).html).toContain('
{ it("says the platform refused, carrying the reason", async () => { const lookup = makeLookup( new MemoryStore(), - () => Promise.reject(new PlatformRefused("the caption endpoint returned an empty body")), + () => Promise.reject(new PlatformRefused("captions_refused", "an empty body")), ); expect(await lookup("gyN9lV9QgyA")).toEqual({ kind: "upstream_failed", video_id: "gyN9lV9QgyA", - reason: "the caption endpoint returned an empty body", + cause: "captions_refused", + reason: "an empty body", }); }); @@ -91,6 +92,7 @@ describe("looking a video up", () => { expect(await lookup("gyN9lV9QgyA")).toEqual({ kind: "upstream_failed", video_id: "gyN9lV9QgyA", + cause: "platform_error", reason: "socket closed", }); }); @@ -99,7 +101,7 @@ describe("looking a video up", () => { const store = new MemoryStore(); const fetcher = vi .fn() - .mockRejectedValueOnce(new PlatformRefused("refused")) + .mockRejectedValueOnce(new PlatformRefused("bot_check", "refused")) .mockResolvedValueOnce(item()); const lookup = makeLookup(store, fetcher); diff --git a/src/store/lookup.ts b/src/store/lookup.ts index 62131c1..21cb065 100644 --- a/src/store/lookup.ts +++ b/src/store/lookup.ts @@ -1,4 +1,5 @@ import { isVideoId, type LookupResult, type TranscriptItem } from "../contract.js"; +import { observe } from "../observe.js"; import { PlatformRefused, VideoNotFound } from "../transcript/youtube.js"; import type { Store } from "./store.js"; @@ -23,10 +24,13 @@ export function makeLookup( } catch (cause) { if (cause instanceof VideoNotFound) return { kind: "not_found", video_id: given }; if (cause instanceof PlatformRefused) { - return { kind: "upstream_failed", video_id: given, reason: cause.message }; + return { kind: "upstream_failed", video_id: given, cause: cause.why, reason: cause.message }; } + // Anything else is the attestation or the network giving way, which is the platform not + // answering rather than the video being absent. const reason = cause instanceof Error ? cause.message : "the fetch failed"; - return { kind: "upstream_failed", video_id: given, reason }; + observe("fetch_threw", { video_id: given, reason }); + return { kind: "upstream_failed", video_id: given, cause: "platform_error", reason }; } await store.put(item); diff --git a/src/transcript/diagnosis.test.ts b/src/transcript/diagnosis.test.ts new file mode 100644 index 0000000..1bbf33f --- /dev/null +++ b/src/transcript/diagnosis.test.ts @@ -0,0 +1,112 @@ +import { describe, expect, it } from "vitest"; +import { WHAT_HAPPENED, causeOf, readShape } from "./diagnosis.js"; +import type { Cause } from "../contract.js"; + +const page = (inner: string, title = "YouTube") => + `${title}`; + +const player = (status: string, reason: string) => + `"playabilityStatus":{"status":"${status}","reason":"${reason}"}`; + +const causeOfPage = (html: string, status = 200, contentType = "text/html"): Cause => + causeOf(readShape(status, contentType, html)); + +describe("reading the shape of what came back", () => { + it("records the status, the size and the kind of body", () => { + const shape = readShape(200, "text/html; charset=utf-8", page(player("ERROR", "Video unavailable"))); + expect(shape.status).toBe(200); + expect(shape.content_type).toBe("text/html; charset=utf-8"); + expect(shape.bytes).toBeGreaterThan(0); + }); + + it("records what the platform said about playing the video", () => { + const shape = readShape(200, "text/html", page(player("LOGIN_REQUIRED", "Sign in to confirm"))); + expect(shape.playability).toBe("LOGIN_REQUIRED"); + expect(shape.playability_reason).toBe("Sign in to confirm"); + expect(shape.has_player_response).toBe(true); + }); + + it("records the title of the page, which is what tells a wall apart from a watch page", () => { + expect(readShape(200, "text/html", page("", "Before you continue")).page_title).toBe("Before you continue"); + }); + + it("records whether the video details and the caption tracks are there at all", () => { + const shape = readShape(200, "text/html", page(`"videoDetails":{},"captionTracks":[]`)); + expect(shape.has_video_details).toBe(true); + expect(shape.has_caption_tracks).toBe(true); + }); + + it("keeps the page itself out of the record, because a watch page is over a megabyte", () => { + const shape = readShape(200, "text/html", page(player("ERROR", "Video unavailable"))); + expect(JSON.stringify(shape).length).toBeLessThan(600); + }); +}); + +describe("naming the cause, one per way a read can fail", () => { + it("calls a video the platform will not play a missing video", () => { + expect(causeOfPage(page(player("ERROR", "Video unavailable")))).toBe("video_missing"); + expect(causeOfPage(page(player("UNPLAYABLE", "This video is private")))).toBe("video_missing"); + }); + + it("calls a demand for proof of a person a bot check", () => { + expect(causeOfPage(page("", "Before you continue") + "Sign in to confirm you are not a bot")).toBe("bot_check"); + expect(causeOfPage("Our systems have detected unusual traffic from your computer network")).toBe("bot_check"); + expect(causeOfPage(``)).toBe("bot_check"); + expect(causeOfPage(page(player("LOGIN_REQUIRED", "Sign in")))).toBe("bot_check"); + }); + + it("calls a 403 a bot check, because that is what a blocked caller is told", () => { + expect(causeOfPage("", 403)).toBe("bot_check"); + }); + + it("calls a consent page a consent wall when the platform sent no player response", () => { + expect(causeOfPage(`Before you continue to YouTube`)).toBe("consent_wall"); + }); + + it("calls too many requests a rate limit", () => { + expect(causeOfPage("", 429)).toBe("rate_limited"); + }); + + it("calls a status that is not a page a platform error", () => { + expect(causeOfPage("", 503)).toBe("platform_error"); + expect(causeOfPage("", 301)).toBe("platform_error"); + }); + + // This is the defect that made a working video read as deleted. A page with nothing recognisable + // in it is a refusal, because that is what the platform sends when it does not want to answer. + it("calls a page it does not recognise a refusal, never a missing video", () => { + expect(causeOfPage("something else entirely")).toBe("unrecognised_page"); + expect(causeOfPage("")).toBe("unrecognised_page"); + }); + + // A real watch page carries the string consent.youtube.com in its own scripts, so the page for a + // video that genuinely does not exist matches the consent marker too. The stated verdict wins. + it("prefers the verdict the platform stated over a word found in the body", () => { + const both = page(player("ERROR", "Video unavailable")) + "https://consent.youtube.com/"; + expect(causeOfPage(both)).toBe("video_missing"); + }); + + it("still calls a caller who is told to prove it is a person a bot check, whatever else is on the page", () => { + const both = page(player("ERROR", "Video unavailable")) + "Sign in to confirm you are not a bot"; + expect(causeOfPage(both)).toBe("bot_check"); + }); +}); + +describe("what a cause is called in words", () => { + it("gives every cause a sentence, so no cause reaches a reader unnamed", () => { + const causes: Cause[] = [ + "video_missing", + "bot_check", + "consent_wall", + "rate_limited", + "platform_error", + "unrecognised_page", + "captions_refused", + "captions_not_json", + "captions_empty", + ]; + for (const cause of causes) { + expect(WHAT_HAPPENED[cause]).toBeTruthy(); + } + }); +}); diff --git a/src/transcript/diagnosis.ts b/src/transcript/diagnosis.ts new file mode 100644 index 0000000..213fbe8 --- /dev/null +++ b/src/transcript/diagnosis.ts @@ -0,0 +1,94 @@ +import type { Cause } from "../contract.js"; + +// The watch page arrives as one of several things and they all used to read the same way: as a +// missing video. This reads the shape of what arrived and names it. +// +// Only the shape is read, never the whole body. A watch page is over a megabyte, and writing a +// megabyte per request costs money and buries the line that matters. + +export interface WatchPageShape { + status: number; + content_type: string; + bytes: number; + has_video_details: boolean; + has_caption_tracks: boolean; + has_player_response: boolean; + playability: string; + playability_reason: string; + page_title: string; + markers: string[]; +} + +// A full watch page happens to contain the words "consent" and "video unavailable" in its own +// scripts, so a marker means nothing on a page that carries a title. Every caller here reads the +// shape only after the title comes back missing. +const MARKERS: { name: string; pattern: RegExp }[] = [ + { name: "not a bot", pattern: /sign in to confirm[^<"]{0,40}not a bot/i }, + { name: "unusual traffic", pattern: /unusual traffic from your computer network/i }, + { name: "captcha", pattern: /g-recaptcha|recaptcha\/api\.js|\/sorry\/index/i }, + { name: "consent", pattern: /consent\.youtube\.com|before you continue to youtube/i }, + { name: "sign in", pattern: /"status"\s*:\s*"LOGIN_REQUIRED"/ }, + { name: "unplayable", pattern: /"status"\s*:\s*"(ERROR|UNPLAYABLE)"/ }, +]; + +function first(html: string, pattern: RegExp): string { + return pattern.exec(html)?.[1] ?? ""; +} + +export function readShape(status: number, contentType: string, html: string): WatchPageShape { + return { + status, + content_type: contentType, + bytes: html.length, + has_video_details: html.includes('"videoDetails"'), + has_caption_tracks: html.includes('"captionTracks"'), + has_player_response: html.includes('"playabilityStatus"'), + playability: first(html, /"playabilityStatus":\s*\{\s*"status"\s*:\s*"([A-Z_]+)"/), + playability_reason: first(html, /"playabilityStatus":[^}]*?"reason"\s*:\s*"([^"]{0,120})"/).slice(0, 120), + page_title: first(html, /]*>([^<]{0,120}) m.pattern.test(html)).map((m) => m.name), + }; +} + +// The order matters, and the rule is that a verdict the platform states beats a word found in the +// body. A real watch page mentions consent.youtube.com in its own scripts, so the page for a video +// that genuinely does not exist carries the consent marker as well as its ERROR status. Reading the +// marker first would report a deleted video as a consent wall. +// +// A missing video is proved, never assumed: the platform has to say the video is unplayable. Every +// answer left over is a refusal, because a refusal is what an unexplained page usually is. +export function causeOf(shape: WatchPageShape): Cause { + if (shape.status === 429) return "rate_limited"; + if (shape.status >= 500) return "platform_error"; + if (shape.status === 403) return "bot_check"; + if (shape.status !== 200) return "platform_error"; + + const marked = (name: string): boolean => shape.markers.includes(name); + + // These three name the caller rather than the video, so nothing outranks them. + if (marked("not a bot") || marked("unusual traffic") || marked("captcha")) return "bot_check"; + + if (shape.has_player_response) { + if (shape.playability === "ERROR" || shape.playability === "UNPLAYABLE") return "video_missing"; + if (shape.playability === "LOGIN_REQUIRED" || shape.playability === "AGE_VERIFICATION_REQUIRED") { + return "bot_check"; + } + } + + // Only now, with no verdict from the platform to read, is a word in the body worth anything. + if (marked("consent")) return "consent_wall"; + return "unrecognised_page"; +} + +// What the log says about a cause, in the words a person reading it needs. +export const WHAT_HAPPENED: Record = { + video_missing: "the platform holds no video under that id", + bot_check: "the platform asked the caller to prove it is not a machine", + consent_wall: "the platform answered with a consent page instead of the video", + rate_limited: "the platform is refusing this caller for asking too often", + platform_error: "the platform answered with a status that is not a page", + unrecognised_page: "the platform answered with a page that carries no video and names no reason", + captions_refused: "the caption endpoint answered with an empty body, which is how it refuses", + captions_not_json: "the caption endpoint answered with something that is not json", + captions_empty: "the caption endpoint answered with json that carries no usable line", +}; diff --git a/src/transcript/youtube.test.ts b/src/transcript/youtube.test.ts index 5e5dea3..10da44a 100644 --- a/src/transcript/youtube.test.ts +++ b/src/transcript/youtube.test.ts @@ -18,16 +18,29 @@ const watchPage = (title: string, withTracks = true) => (withTracks ? `,${TRACK_JSON}` : "") + `};`; +// What the platform really sends when there is no video: a watch page carrying a player response +// that says so. A bare page with the words on it is not what a missing video looks like. +const missingVideoPage = + `YouTube`; + +const botCheckPage = + `YouTube`; + const minter = { mint: async () => "a-token", expiresAt: Date.now() + 60_000 }; const getMinter = async () => minter; -const responseOf = (body: string, status = 200, cookies: string[] = []) => - ({ +// Real headers rather than an object with the one method the code happens to call. A double that +// answers less than the real thing turns a missing call into a green test. +const responseOf = (body: string, status = 200, cookies: string[] = [], contentType = "text/html") => { + const headers = new Headers({ "content-type": contentType }); + for (const cookie of cookies) headers.append("set-cookie", cookie); + return { ok: status >= 200 && status < 300, status, text: async () => body, - headers: { getSetCookie: () => cookies }, - }) as unknown as Response; + headers, + } as unknown as Response; +}; const captionBody = JSON.stringify({ events: [ @@ -174,13 +187,39 @@ describe("fetching a transcript", () => { expect(f).toHaveBeenCalledTimes(1); }); - it("says the video does not exist when the watch page carries no video", async () => { - const f = vi.fn().mockResolvedValueOnce(responseOf("Video unavailable")); + it("says the video does not exist when the platform says the video is unplayable", async () => { + const f = vi.fn().mockResolvedValueOnce(responseOf(missingVideoPage)); await expect( fetchTranscript("aaaaaaaaaaa", { fetch: f as unknown as typeof fetch, minter: getMinter }), ).rejects.toBeInstanceOf(VideoNotFound); }); + // This is the defect the page was built on. A page the code does not recognise carries no video + // and names no reason, which is exactly what a refusal looks like, so calling it a missing video + // told the reader a working video had been deleted. + it("does not call a page it cannot recognise a missing video", async () => { + const f = vi.fn().mockResolvedValueOnce(responseOf("something else entirely")); + const failure = await fetchTranscript("gyN9lV9QgyA", { + fetch: f as unknown as typeof fetch, + minter: getMinter, + }).catch((thrown: unknown) => thrown); + + expect(failure).toBeInstanceOf(PlatformRefused); + expect(failure).not.toBeInstanceOf(VideoNotFound); + expect((failure as PlatformRefused).why).toBe("unrecognised_page"); + }); + + it("says the platform refused, not that the video is missing, when it asks for proof of a person", async () => { + const f = vi.fn().mockResolvedValueOnce(responseOf(botCheckPage)); + const failure = await fetchTranscript("gyN9lV9QgyA", { + fetch: f as unknown as typeof fetch, + minter: getMinter, + }).catch((thrown: unknown) => thrown); + + expect(failure).toBeInstanceOf(PlatformRefused); + expect((failure as PlatformRefused).why).toBe("bot_check"); + }); + it("says the platform refused when the caption body comes back empty", async () => { const f = vi .fn() @@ -205,7 +244,7 @@ describe("fetching a transcript", () => { .mockResolvedValueOnce(responseOf("no")); await expect( fetchTranscript("gyN9lV9QgyA", { fetch: f as unknown as typeof fetch, minter: getMinter }), - ).rejects.toThrow(/not json/); + ).rejects.toThrow(/is not json/); }); it("says the platform refused when every event is empty", async () => { @@ -215,6 +254,6 @@ describe("fetching a transcript", () => { .mockResolvedValueOnce(responseOf(JSON.stringify({ events: [{ tStartMs: 0, segs: [{ utf8: " " }] }] }))); await expect( fetchTranscript("gyN9lV9QgyA", { fetch: f as unknown as typeof fetch, minter: getMinter }), - ).rejects.toThrow(/no usable segments/); + ).rejects.toThrow(/no usable line/); }); }); diff --git a/src/transcript/youtube.ts b/src/transcript/youtube.ts index 8b5dc8f..f626e6c 100644 --- a/src/transcript/youtube.ts +++ b/src/transcript/youtube.ts @@ -1,5 +1,7 @@ -import { SCHEMA_VERSION, type Segment, type TranscriptItem } from "../contract.js"; +import { SCHEMA_VERSION, type Cause, type Segment, type TranscriptItem } from "../contract.js"; +import { observe, withoutSecrets } from "../observe.js"; import { heldMinter, type Minter } from "./attestation.js"; +import { WHAT_HAPPENED, causeOf, readShape } from "./diagnosis.js"; const USER_AGENT = "Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/131.0.0.0 Safari/537.36"; @@ -7,7 +9,17 @@ const USER_AGENT = export const SOURCE = "youtube.com timedtext, json3, with a proof of origin token bound to the video id"; export class VideoNotFound extends Error {} -export class PlatformRefused extends Error {} + +// The cause travels with the refusal, so the page the reader gets and the line in the log name the +// same thing. It is not called cause, because Error already has a field by that name. +export class PlatformRefused extends Error { + constructor( + readonly why: Cause, + message: string, + ) { + super(message); + } +} interface CaptionTrack { baseUrl: string; @@ -17,6 +29,8 @@ interface CaptionTrack { } interface WatchPage { + status: number; + contentType: string; html: string; cookies: string; } @@ -33,18 +47,24 @@ interface TimedTextEvent { segs?: { utf8?: string }[]; } +// A status that is not 200 is not thrown here. The body of a refusal is where the reason is +// written, so it is read and classified like any other answer. async function readWatchPage(videoId: string, f: typeof fetch): Promise { const response = await f(`https://www.youtube.com/watch?v=${videoId}`, { headers: { "user-agent": USER_AGENT, "accept-language": "en-US,en;q=0.9" }, }); - if (!response.ok) throw new PlatformRefused(`the watch page answered ${response.status}`); - const html = await response.text(); + const html = await response.text().catch(() => ""); const cookies = response.headers .getSetCookie() .map((c) => c.split(";")[0] ?? "") .filter(Boolean) .join("; "); - return { html, cookies }; + return { + status: response.status, + contentType: response.headers.get("content-type") ?? "", + html, + cookies, + }; } export function readTitle(html: string): string | null { @@ -102,6 +122,13 @@ export function eventsToSegments(events: TimedTextEvent[]): Segment[] { return segments; } +function captionStatusCause(status: number): Cause { + if (status === 429) return "rate_limited"; + if (status === 403) return "bot_check"; + if (status >= 500) return "platform_error"; + return "captions_refused"; +} + export async function fetchTranscript( videoId: string, deps: { fetch?: typeof fetch; minter?: () => Promise } = {}, @@ -109,12 +136,35 @@ export async function fetchTranscript( const f = deps.fetch ?? fetch; const getMinter = deps.minter ?? (() => heldMinter()); - const { html, cookies } = await readWatchPage(videoId, f); - const title = readTitle(html); - if (title === null) throw new VideoNotFound(`no video with the id ${videoId}`); + const page = await readWatchPage(videoId, f); + const title = page.status === 200 ? readTitle(page.html) : null; + + // The title is missing for several different reasons and they used to be indistinguishable. The + // shape of what came back is written here, once, whatever the reason turns out to be. + if (title === null) { + const shape = readShape(page.status, page.contentType, page.html); + const why = causeOf(shape); + observe("watch_page_unreadable", { + video_id: videoId, + cause: why, + what_happened: WHAT_HAPPENED[why], + ...shape, + }); + if (why === "video_missing") throw new VideoNotFound(WHAT_HAPPENED[why]); + throw new PlatformRefused(why, WHAT_HAPPENED[why]); + } const now = new Date().toISOString(); - const track = chooseTrack(readCaptionTracks(html)); + const track = chooseTrack(readCaptionTracks(page.html)); + + observe("watch_page_read", { + video_id: videoId, + status: page.status, + bytes: page.html.length, + has_caption_tracks: track !== null, + track_kind: track?.kind ?? "", + language_code: track?.languageCode ?? "", + }); if (!track) { return { @@ -140,30 +190,49 @@ export async function fetchTranscript( headers: { "user-agent": USER_AGENT, "accept-language": "en-US,en;q=0.9", - cookie: cookies, + cookie: page.cookies, referer: `https://www.youtube.com/watch?v=${videoId}`, origin: "https://www.youtube.com", }, }); - if (!response.ok) throw new PlatformRefused(`the caption endpoint answered ${response.status}`); + const body = response.ok ? await response.text().catch(() => "") : ""; + + // The address is signed and the request carried cookies and a token, so only the host and the + // path are written. The rest is a credential. + const refuse = (why: Cause): PlatformRefused => { + observe("captions_unreadable", { + video_id: videoId, + cause: why, + what_happened: WHAT_HAPPENED[why], + status: response.status, + content_type: response.headers.get("content-type") ?? "", + bytes: body.length, + address: withoutSecrets(url), + }); + return new PlatformRefused(why, WHAT_HAPPENED[why]); + }; - const body = await response.text(); + if (!response.ok) throw refuse(captionStatusCause(response.status)); // An empty body with a 200 is how the platform refuses a caption request it does not trust. - if (body.length === 0) { - throw new PlatformRefused("the caption endpoint returned an empty body"); - } + if (body.length === 0) throw refuse("captions_refused"); let parsed: { events?: TimedTextEvent[] }; try { parsed = JSON.parse(body) as { events?: TimedTextEvent[] }; } catch { - throw new PlatformRefused("the caption endpoint returned something that is not json"); + throw refuse("captions_not_json"); } const segments = eventsToSegments(parsed.events ?? []); - if (segments.length === 0) { - throw new PlatformRefused("the caption endpoint returned no usable segments"); - } + if (segments.length === 0) throw refuse("captions_empty"); + + observe("captions_read", { + video_id: videoId, + status: response.status, + bytes: body.length, + segments: segments.length, + characters: segments.reduce((n, segment) => n + segment.text.length, 0), + }); return { video_id: videoId, From c933d0f1aa783862cf97216efb711bd50893e016 Mon Sep 17 00:00:00 2001 From: juliantellez <4896851+juliantellez@users.noreply.github.com> Date: Mon, 31 Aug 2026 11:29:32 +0000 Subject: [PATCH 2/7] fix: tell the reader which refusal happened instead of reporting a missing video --- ...-refusal-is-not-a-missing-video.feature.ts | 116 +++++++++++ features/scenario.ts | 46 +++++ src/boundary.test.ts | 191 ++++++++++++++++++ src/handler.ts | 13 +- src/page/render.test.ts | 26 ++- src/page/render.ts | 127 ++++++++++-- tsconfig.json | 2 +- vitest.config.ts | 4 +- 8 files changed, 502 insertions(+), 23 deletions(-) create mode 100644 features/a-refusal-is-not-a-missing-video.feature.ts create mode 100644 features/scenario.ts create mode 100644 src/boundary.test.ts diff --git a/features/a-refusal-is-not-a-missing-video.feature.ts b/features/a-refusal-is-not-a-missing-video.feature.ts new file mode 100644 index 0000000..50d7e3f --- /dev/null +++ b/features/a-refusal-is-not-a-missing-video.feature.ts @@ -0,0 +1,116 @@ +import { describe, expect, vi } from "vitest"; +import { route, type FunctionUrlEvent } from "../src/handler.js"; +import { makeLookup } from "../src/store/lookup.js"; +import { MemoryStore } from "../src/store/store.js"; +import { fetchTranscript } from "../src/transcript/youtube.js"; +import { scenario, theAnswer, type World } from "./scenario.js"; + +// The deployed page reported a working video as deleted. The platform was refusing the address the +// function reads from, and a refusal, a rate limit and a genuinely missing video all reached the +// reader as one 404 that said "No video with that id". +// +// The page the platform really sent is copied here from the function's own log: HTTP 200, over a +// megabyte, no caption track, and a player response asking the caller to prove it is a person. + +const REFUSED_BY_THE_PLATFORM = + ` - YouTube`; + +const DELETED_BY_ITS_OWNER = + ` - YouTube`; + +const answering = (body: string, status = 200) => { + const headers = new Headers({ "content-type": "text/html; charset=utf-8" }); + return { ok: status >= 200 && status < 300, status, text: async () => body, headers } as unknown as Response; +}; + +function readerAsksFor(videoId: string, platformAnswers: Response): (world: World) => Promise { + return async (world) => { + const spy = vi.spyOn(console, "log").mockImplementation(() => undefined); + try { + const lookup = makeLookup(new MemoryStore(), (id) => + fetchTranscript(id, { + fetch: (async () => platformAnswers) as unknown as typeof fetch, + minter: async () => ({ mint: async () => "a-token", expiresAt: Date.now() + 60_000 }), + }), + ); + const request: FunctionUrlEvent = { + rawPath: "/videos", + queryStringParameters: { id: videoId }, + requestContext: { http: { method: "GET", path: "/videos" } }, + }; + world.answer = await route(request, lookup); + world.log = spy.mock.calls.map((call) => JSON.parse(String(call[0])) as Record); + } finally { + spy.mockRestore(); + } + }; +} + +describe("a refusal is not a missing video", () => { + scenario("the platform refuses the caller, and the reader is told the truth", ({ given, when, then, and }) => { + given("the platform asks this page to prove it is a person", () => { + expect(REFUSED_BY_THE_PLATFORM).toContain("LOGIN_REQUIRED"); + }); + + when("a reader opens the page for a video that really exists", readerAsksFor("gyN9lV9QgyA", answering(REFUSED_BY_THE_PLATFORM))); + + then("the page does not say the video is missing", (world) => { + expect(theAnswer(world).body).not.toContain("No video with that id"); + expect(theAnswer(world).statusCode).not.toBe(404); + }); + + and("the status says the fault is upstream, not with the request", (world) => { + expect(theAnswer(world).statusCode).toBe(502); + }); + + and("the page says the platform refused this page rather than the video", (world) => { + expect(theAnswer(world).body).toContain("The platform refused this page, not your video"); + expect(theAnswer(world).body).toContain("Your id is right and the video is fine"); + }); + + and("the page offers the video where it can still be watched", (world) => { + expect(theAnswer(world).body).toContain("https://www.youtube.com/watch?v=gyN9lV9QgyA"); + }); + + and("the page does not promise that trying again will work", (world) => { + expect(theAnswer(world).body).toContain("Trying again will not help"); + }); + + and("nothing is held at the edge, so a later visit reaches the platform again", (world) => { + expect(theAnswer(world).headers["cache-control"]).toBe("no-store"); + }); + + and("the log names the cause, so an operator can tell this from a deleted video", (world) => { + const boundary = world.log.find((entry) => entry.event === "watch_page_unreadable"); + expect(boundary?.cause).toBe("bot_check"); + expect(boundary?.playability).toBe("LOGIN_REQUIRED"); + }); + + and("the log carries no credential", (world) => { + expect(JSON.stringify(world.log)).not.toContain("a-token"); + }); + }); + + scenario("a video that really is gone is still reported as gone", ({ given, when, then, and }) => { + given("the platform says the video is unavailable", () => { + expect(DELETED_BY_ITS_OWNER).toContain("ERROR"); + }); + + when("a reader opens the page for it", readerAsksFor("aaaaaaaaaaa", answering(DELETED_BY_ITS_OWNER))); + + then("the page answers 404", (world) => { + expect(theAnswer(world).statusCode).toBe(404); + }); + + and("the page says there is no video with that id", (world) => { + expect(theAnswer(world).body).toContain("No video with that id"); + }); + + and("the log calls it a missing video, not a refusal", (world) => { + expect(world.log.find((entry) => entry.event === "watch_page_unreadable")?.cause).toBe("video_missing"); + }); + }); +}); diff --git a/features/scenario.ts b/features/scenario.ts new file mode 100644 index 0000000..bfe217e --- /dev/null +++ b/features/scenario.ts @@ -0,0 +1,46 @@ +import { expect, it } from "vitest"; + +// A scenario reads as one sentence per step. When a step fails, the report names the step and the +// whole scenario, so a failure says what the reader was supposed to get rather than a line number. + +export interface World { + answer: { statusCode: number; headers: Record; body: string } | null; + log: Record[]; +} + +type Step = (world: World) => Promise | void; + +export interface Steps { + given(text: string, step: Step): void; + when(text: string, step: Step): void; + then(text: string, step: Step): void; + and(text: string, step: Step): void; +} + +export function scenario(title: string, build: (steps: Steps) => void): void { + const steps: { word: string; text: string; step: Step }[] = []; + const add = (word: string) => (text: string, step: Step) => { + steps.push({ word, text, step }); + }; + + build({ given: add("Given"), when: add("When"), then: add("Then"), and: add("And") }); + + if (steps.length === 0) throw new Error(`the scenario "${title}" declares no step`); + + it(title, async () => { + const world: World = { answer: null, log: [] }; + for (const { word, text, step } of steps) { + try { + await step(world); + } catch (failure) { + const why = failure instanceof Error ? failure.message : String(failure); + expect.fail(`${word} ${text}\n\n${why}`); + } + } + }); +} + +export function theAnswer(world: World): { statusCode: number; headers: Record; body: string } { + if (world.answer === null) throw new Error("no page was asked for yet"); + return world.answer; +} diff --git a/src/boundary.test.ts b/src/boundary.test.ts new file mode 100644 index 0000000..6cb2e93 --- /dev/null +++ b/src/boundary.test.ts @@ -0,0 +1,191 @@ +import { afterEach, describe, expect, it, vi } from "vitest"; +import type { Cause } from "./contract.js"; +import { route, type FunctionUrlEvent } from "./handler.js"; +import { makeLookup } from "./store/lookup.js"; +import { MemoryStore } from "./store/store.js"; +import { fetchTranscript } from "./transcript/youtube.js"; + +// From the boundary to the page, with only the platform faked. Every one of these used to reach the +// reader as the same 404 saying the video does not exist. + +const get = (id: string): FunctionUrlEvent => ({ + rawPath: "/videos", + queryStringParameters: { id }, + requestContext: { http: { method: "GET", path: "/videos" } }, +}); + +const responseOf = (body: string, status = 200, cookies: string[] = []) => { + const headers = new Headers({ "content-type": "text/html; charset=utf-8" }); + for (const cookie of cookies) headers.append("set-cookie", cookie); + return { ok: status >= 200 && status < 300, status, text: async () => body, headers } as unknown as Response; +}; + +// The shape the deployed function really received, copied from its own log: a full watch page, +// HTTP 200, carrying videoDetails but no title, no caption track, and a player response that asks +// the caller to prove it is a person. +const BOT_CHECK_PAGE = + ` - YouTube`; + +const MISSING_VIDEO_PAGE = + ` - YouTube`; + +const GOOD_WATCH_PAGE = + `A title - YouTube`; + +const minter = async () => ({ mint: async () => "a-token", expiresAt: Date.now() + 60_000 }); + +function pageFor(responses: Response[]) { + const f = vi.fn(); + for (const response of responses) f.mockResolvedValueOnce(response); + const lookup = makeLookup(new MemoryStore(), (id) => + fetchTranscript(id, { fetch: f as unknown as typeof fetch, minter }), + ); + return { lookup, f }; +} + +afterEach(() => { + vi.restoreAllMocks(); +}); + +describe("what the reader is told, one answer per cause", () => { + it("a bot check is a refusal, and it never reads as a missing video", async () => { + const { lookup } = pageFor([responseOf(BOT_CHECK_PAGE)]); + const answer = await route(get("gyN9lV9QgyA"), lookup); + + expect(answer.statusCode).toBe(502); + expect(answer.body).toContain("The platform refused this page, not your video"); + expect(answer.body).toContain("Your id is right and the video is fine"); + expect(answer.body).not.toContain("No video with that id"); + expect(answer.body).not.toContain("may be deleted"); + }); + + it("a bot check offers the video on the platform, and does not promise that trying again works", async () => { + const { lookup } = pageFor([responseOf(BOT_CHECK_PAGE)]); + const answer = await route(get("gyN9lV9QgyA"), lookup); + + expect(answer.body).toContain("https://www.youtube.com/watch?v=gyN9lV9QgyA"); + expect(answer.body).toContain("Trying again will not help"); + }); + + it("a video that really is missing still answers 404", async () => { + const { lookup } = pageFor([responseOf(MISSING_VIDEO_PAGE)]); + const answer = await route(get("aaaaaaaaaaa"), lookup); + + expect(answer.statusCode).toBe(404); + expect(answer.body).toContain("No video with that id"); + }); + + it("a rate limit answers 429 and says when to come back", async () => { + const { lookup } = pageFor([responseOf("", 429)]); + const answer = await route(get("gyN9lV9QgyA"), lookup); + + expect(answer.statusCode).toBe(429); + expect(answer.headers["retry-after"]).toBe("60"); + expect(answer.body).toContain("asking too often"); + }); + + it("a platform error answers 502, not 404", async () => { + const { lookup } = pageFor([responseOf("", 503)]); + const answer = await route(get("gyN9lV9QgyA"), lookup); + + expect(answer.statusCode).toBe(502); + expect(answer.body).toContain("The platform did not answer"); + }); + + it("captions refused after the video was found answers 502 and says the video is fine", async () => { + const { lookup } = pageFor([responseOf(GOOD_WATCH_PAGE, 200, ["VISITOR=1; Path=/"]), responseOf("")]); + const answer = await route(get("gyN9lV9QgyA"), lookup); + + expect(answer.statusCode).toBe(502); + expect(answer.body).toContain("The platform refused the caption text"); + expect(answer.body).toContain("Your id is right and the video is fine"); + }); + + it("a page carrying no video and no reason is a refusal, not a deletion", async () => { + const { lookup } = pageFor([responseOf("something else entirely")]); + const answer = await route(get("gyN9lV9QgyA"), lookup); + + expect(answer.statusCode).toBe(502); + expect(answer.body).toContain("The platform sent something this page cannot read"); + }); + + it("a working video still serves its text, so none of this broke the good path", async () => { + const captions = JSON.stringify({ + events: [{ tStartMs: 0, dDurationMs: 1500, segs: [{ utf8: "the words" }] }], + }); + const { lookup } = pageFor([ + responseOf(GOOD_WATCH_PAGE, 200, ["VISITOR=1; Path=/"]), + responseOf(captions), + ]); + const answer = await route(get("gyN9lV9QgyA"), lookup); + + expect(answer.statusCode).toBe(200); + expect(answer.body).toContain("the words"); + }); + + it("never holds a refusal at the edge, so the next visit reaches the platform", async () => { + const { lookup } = pageFor([responseOf(BOT_CHECK_PAGE)]); + const answer = await route(get("gyN9lV9QgyA"), lookup); + expect(answer.headers["cache-control"]).toBe("no-store"); + }); +}); + +describe("what the log says while the reader is being told", () => { + it("names the cause, the status and the shape of the body", async () => { + const spy = vi.spyOn(console, "log").mockImplementation(() => undefined); + const { lookup } = pageFor([responseOf(BOT_CHECK_PAGE)]); + await route(get("gyN9lV9QgyA"), lookup); + + const lines = spy.mock.calls.map((call) => JSON.parse(String(call[0])) as Record); + const boundary = lines.find((l) => l.event === "watch_page_unreadable"); + + expect(boundary).toBeDefined(); + expect(boundary?.cause satisfies unknown as Cause).toBe("bot_check"); + expect(boundary?.status).toBe(200); + expect(boundary?.playability).toBe("LOGIN_REQUIRED"); + expect(boundary?.has_caption_tracks).toBe(false); + expect(String(boundary?.playability_reason)).toContain("not a bot"); + }); + + it("writes the cause the page shows, so a log line and a page line up", async () => { + const spy = vi.spyOn(console, "log").mockImplementation(() => undefined); + const { lookup } = pageFor([responseOf(MISSING_VIDEO_PAGE)]); + await route(get("aaaaaaaaaaa"), lookup); + + const lines = spy.mock.calls.map((call) => JSON.parse(String(call[0])) as Record); + expect(lines.find((l) => l.event === "watch_page_unreadable")?.cause).toBe("video_missing"); + }); + + // The caption address carries a signature and the request carries cookies and a minted token. + it("never writes a credential, whatever the platform answered", async () => { + const spy = vi.spyOn(console, "log").mockImplementation(() => undefined); + const { lookup } = pageFor([responseOf(GOOD_WATCH_PAGE, 200, ["VISITOR=s3ss10n; Path=/"]), responseOf("")]); + await route(get("gyN9lV9QgyA"), lookup); + + const everything = spy.mock.calls.map((call) => String(call[0])).join("\n"); + expect(everything).toContain("captions_unreadable"); + expect(everything).not.toContain("s3cr3t"); + expect(everything).not.toContain("s3ss10n"); + expect(everything).not.toContain("a-token"); + expect(everything).not.toContain("signature"); + }); + + it("is silent about nothing: a read that worked is written too", async () => { + const spy = vi.spyOn(console, "log").mockImplementation(() => undefined); + const captions = JSON.stringify({ + events: [{ tStartMs: 0, dDurationMs: 1500, segs: [{ utf8: "the words" }] }], + }); + const { lookup } = pageFor([responseOf(GOOD_WATCH_PAGE, 200, []), responseOf(captions)]); + await route(get("gyN9lV9QgyA"), lookup); + + const events = spy.mock.calls.map((call) => (JSON.parse(String(call[0])) as { event: string }).event); + expect(events).toContain("watch_page_read"); + expect(events).toContain("captions_read"); + }); +}); diff --git a/src/handler.ts b/src/handler.ts index ca12185..996c91a 100644 --- a/src/handler.ts +++ b/src/handler.ts @@ -32,10 +32,19 @@ export function cacheControlFor(result: LookupResult | null): string { } } -function answer(page: Rendered, cacheControl: string): HttpAnswer { +// A rate limit is the one refusal that says when to come back, so the answer says it too. +export function retryAfterFor(result: LookupResult): Record { + if (result.kind === "upstream_failed" && result.cause === "rate_limited") { + return { "retry-after": "60" }; + } + return {}; +} + +function answer(page: Rendered, cacheControl: string, extra: Record = {}): HttpAnswer { return { statusCode: page.status, headers: { + ...extra, "content-type": "text/html; charset=utf-8", "cache-control": cacheControl, "x-content-type-options": "nosniff", @@ -61,7 +70,7 @@ export async function route(event: FunctionUrlEvent, lookup: Lookup): Promise = {}): TranscriptItem => ({ video_id: "gyN9lV9QgyA", @@ -141,16 +141,30 @@ describe("the pages for what went wrong", () => { expect(page.html).toContain("aaaaaaaaaaa"); }); - it("answers 502 when the platform refused, and carries the reason", () => { + it("answers 502 when the platform refused, and says which refusal it was", () => { const page = renderPage({ kind: "upstream_failed", video_id: "gyN9lV9QgyA", cause: "captions_refused", - reason: "the caption endpoint returned an empty body", + reason: "an empty body", }); expect(page.status).toBe(502); - expect(page.html).toContain("The platform did not answer"); - expect(page.html).toContain("the caption endpoint returned an empty body"); + expect(page.html).toContain("The platform refused the caption text"); + }); + + // A refusal that answered 404 is the defect this page was built on. Nothing in this map may. + it("answers 404 for a missing video and for nothing else", () => { + for (const [cause, refusal] of Object.entries(REFUSALS)) { + expect(refusal.status === 404).toBe(cause === "video_missing"); + } + }); + + it("gives every cause its own page, and never leaves one unrendered", () => { + for (const cause of Object.keys(REFUSALS) as Cause[]) { + const page = renderPage({ kind: "upstream_failed", video_id: "gyN9lV9QgyA", cause, reason: "r" }); + expect(page.html).toContain(REFUSALS[cause].title); + expect(page.status).toBeGreaterThanOrEqual(400); + } }); it("offers a way onward from every problem page", () => { diff --git a/src/page/render.ts b/src/page/render.ts index a3055eb..88a8ab3 100644 --- a/src/page/render.ts +++ b/src/page/render.ts @@ -1,4 +1,4 @@ -import type { LookupResult, TranscriptItem } from "../contract.js"; +import type { Cause, LookupResult, TranscriptItem } from "../contract.js"; import { escapeHtml } from "./escape.js"; import { STYLE } from "./style.js"; @@ -124,6 +124,111 @@ ${body} }; } + +// One page per cause. A refusal is not a missing video, so none of these answers 404, and none of +// them tells the reader to try again when trying again cannot work. +// +// Every refusal offers the video on the platform itself, because that is the way onward that does +// work while this page cannot read it. +interface Refusal { + status: number; + title: string; + lines: (videoId: string) => string[]; +} + +function watchLink(videoId: string): string { + const id = escapeHtml(videoId); + return `You can watch it on the platform: + youtube.com/watch?v=${id}.`; +} + +export const REFUSALS: Record = { + // The reader did nothing wrong and cannot fix this, so the page says who has to. + bot_check: { + status: 502, + title: "The platform refused this page, not your video", + lines: (videoId) => [ + `The platform asked this page to prove that it is a person. This page is a server, so it + cannot.`, + `Your id is right and the video is fine. ${watchLink(videoId)}`, + `Trying again will not help. The platform refuses the address this page reads from, and a + person has to change how this page reaches the platform.`, + ], + }, + consent_wall: { + status: 502, + title: "The platform asked for consent instead of sending the video", + lines: (videoId) => [ + `The platform answered with a consent page. It did not send the video.`, + `Your id is right. ${watchLink(videoId)}`, + `Trying again is unlikely to help until a person changes how this page asks.`, + ], + }, + rate_limited: { + status: 429, + title: "The platform is refusing this page for asking too often", + lines: (videoId) => [ + `The platform limits how often this page can ask it for a video. That limit is reached.`, + `Nothing was stored. Wait a minute, then try again. ${watchLink(videoId)}`, + ], + }, + platform_error: { + status: 502, + title: "The platform did not answer", + lines: (videoId) => [ + `The platform answered with an error. The fault is at the platform, not in the id you gave.`, + `Nothing was stored. Wait a minute, then try again. ${watchLink(videoId)}`, + ], + }, + unrecognised_page: { + status: 502, + title: "The platform sent something this page cannot read", + lines: (videoId) => [ + `The platform answered with a page that carries no video, and it gave no reason.`, + `The video may be fine. ${watchLink(videoId)}`, + `Nothing was stored. Trying again is worth doing.`, + ], + }, + captions_refused: { + status: 502, + title: "The platform refused the caption text", + lines: (videoId) => [ + `This page found the video and found its caption track. The platform then refused to send the + text of those captions.`, + `Your id is right and the video is fine. ${watchLink(videoId)}`, + `Nothing was stored. Trying again is worth doing.`, + ], + }, + captions_not_json: { + status: 502, + title: "The platform refused the caption text", + lines: (videoId) => [ + `This page found the video and asked for its captions. The platform answered with something + that is not caption text.`, + `Your id is right and the video is fine. ${watchLink(videoId)}`, + `Nothing was stored. Trying again is worth doing.`, + ], + }, + captions_empty: { + status: 502, + title: "The caption track carries no text", + lines: (videoId) => [ + `This page found a caption track for this video, but the track holds no readable line.`, + `${watchLink(videoId)}`, + `Nothing was stored. Trying again is worth doing.`, + ], + }, + // A missing video never reaches this map. It is a 404 and it is rendered as one. + video_missing: { + status: 404, + title: "No video with that id", + lines: (videoId) => [ + `The platform holds no video under ${escapeHtml(videoId)}.`, + `The video may be private, it may be deleted, or the id may have one character wrong.`, + ], + }, +}; + export function renderPage(result: LookupResult): Rendered { switch (result.kind) { case "ok": @@ -135,18 +240,14 @@ export function renderPage(result: LookupResult): Rendered { `Take the id from the address of the video. In youtube.com/watch?v=gyN9lV9QgyA the id is the part after v=.`, ]); - case "not_found": - return problemPage(404, "No video with that id", [ - `The id ${escapeHtml(result.video_id)} has the right shape, but the platform - holds no video under it.`, - `The video may be private, it may be deleted, or the id may have one character wrong.`, - ]); - case "upstream_failed": - return problemPage(502, "The platform did not answer", [ - `The text of ${escapeHtml(result.video_id)} could not be read just now.`, - `Nothing was stored, so trying again in a minute is worth doing.`, - `What went wrong: ${escapeHtml(result.reason)}.`, - ]); + case "not_found": { + const missing = REFUSALS.video_missing; + return problemPage(missing.status, missing.title, missing.lines(result.video_id)); + } + case "upstream_failed": { + const refusal = REFUSALS[result.cause]; + return problemPage(refusal.status, refusal.title, refusal.lines(result.video_id)); + } } } diff --git a/tsconfig.json b/tsconfig.json index 9a6d004..74789dc 100644 --- a/tsconfig.json +++ b/tsconfig.json @@ -13,5 +13,5 @@ "forceConsistentCasingInFileNames": true, "verbatimModuleSyntax": true }, - "include": ["src", "scripts", "vitest.config.ts", "vitest.live.config.ts"] + "include": ["src", "features", "scripts", "vitest.config.ts", "vitest.live.config.ts"] } diff --git a/vitest.config.ts b/vitest.config.ts index da0a648..e9aedd1 100644 --- a/vitest.config.ts +++ b/vitest.config.ts @@ -2,7 +2,9 @@ import { defineConfig } from "vitest/config"; export default defineConfig({ test: { - include: ["src/**/*.test.ts"], + // Two tiers in one run: the unit and integration tests beside the code, and the scenarios in + // features, which drive a reader request all the way to the page that comes back. + include: ["src/**/*.test.ts", "features/**/*.feature.ts"], environment: "node", passWithNoTests: false, }, From d592e0a90c699cc6f6aeb3918febc0f84b11d4cd Mon Sep 17 00:00:00 2001 From: juliantellez <4896851+juliantellez@users.noreply.github.com> Date: Mon, 31 Aug 2026 11:29:33 +0000 Subject: [PATCH 3/7] ci: name the cause when the deployed page cannot read a transcript --- .github/workflows/deploy.yml | 35 +++++++++++++++++++++++++++++------ 1 file changed, 29 insertions(+), 6 deletions(-) diff --git a/.github/workflows/deploy.yml b/.github/workflows/deploy.yml index 5ca5b34..c124948 100644 --- a/.github/workflows/deploy.yml +++ b/.github/workflows/deploy.yml @@ -121,22 +121,45 @@ jobs: test "$code" = "200" working-directory: infra - # Green means the steps ran. It says nothing about whether the page works, so the page is - # opened and read before this run is called a success. - - name: read the page that is now serving + # The deploy serving at all is a different question from the product working, so they are + # asked separately. This one only proves the distribution reaches the function. + - name: read the front page + run: | + set -euo pipefail + address="$(terraform output -raw address)" + code=$(curl -s -o /tmp/front.html -w '%{http_code}' --max-time 90 "$address/" || true) + echo "the front page answered $code with $(wc -c < /tmp/front.html) bytes" + test "$code" = "200" + grep -q "Read a video as text" /tmp/front.html + working-directory: infra + + # Green means the steps ran. It says nothing about whether a reader gets a transcript, so one + # is asked for here. A failure names what the platform did rather than the status alone, + # because a bare 404 told nobody anything for a day. The page states the cause in its title. + - name: read a transcript from the page that is now serving run: | set -euo pipefail address="$(terraform output -raw address)" - cd "$GITHUB_WORKSPACE" for attempt in 1 2 3 4 5 6 7 8; do code=$(curl -s -o /tmp/page.html -w '%{http_code}' --max-time 90 "$address/videos?id=gyN9lV9QgyA" || true) if [ "$code" = "200" ]; then break; fi echo "attempt $attempt answered $code, the distribution may still be spreading" sleep 30 done - test "$code" = "200" bytes=$(wc -c < /tmp/page.html) - echo "the page answered $code with $bytes bytes" + said=$(sed -n "s/.*\\(.*\\)<\\/title>.*/\\1/p" /tmp/page.html | head -1) + echo "the page answered $code with $bytes bytes, and said: $said" + if [ "$code" != "200" ]; then + { + echo "## The page is serving, and it cannot read a transcript" + echo "" + echo "It answered $code and said: $said" + echo "" + echo "The cause is in the function log, on the watch_page_unreadable line." + } >> "$GITHUB_STEP_SUMMARY" + echo "::error::the page answered $code and said: $said" + exit 1 + fi test "$bytes" -gt 5000 grep -q "Transcript" /tmp/page.html echo "the page carries text" From 84d9fe37c3414e82dc8937a7046a626cc590d8c0 Mon Sep 17 00:00:00 2001 From: juliantellez <4896851+juliantellez@users.noreply.github.com> Date: Mon, 31 Aug 2026 11:29:33 +0000 Subject: [PATCH 4/7] docs: record what the deployed function gets from the platform --- README.md | 41 +++++++++++++++++++++++++++++++++++++++++ 1 file changed, 41 insertions(+) diff --git a/README.md b/README.md index e597a2f..29fd6bb 100644 --- a/README.md +++ b/README.md @@ -41,6 +41,47 @@ it is correct for the player, and for captions it returns zero bytes. The integrity token behind the mint lasts twelve hours, so a warm function attests once and mints per video after that. +## What the deployed function actually gets + +Captured from the deployed function on 31 August 2026, in eu-west-1, on the `watch_page_unreadable` +line of its own log: + + {"event":"watch_page_unreadable","video_id":"gyN9lV9QgyA","cause":"bot_check", + "status":200,"bytes":1412860,"has_video_details":true,"has_caption_tracks":false, + "has_player_response":true,"playability":"LOGIN_REQUIRED", + "playability_reason":"Sign in to confirm you are not a bot"} + +The platform serves the function a full watch page, HTTP 200, over a megabyte, and that page carries +no caption track and a player response that asks the caller to prove it is a person. The same +request from a laptop returns the video and 560 caption segments. The block is on the address the +function calls from, and it lands on the watch page, before the proof of origin work above is +reached. + +Reading it needed the boundary to say so. Before this, the title came back null and every cause threw +the same error, so a refusal, a rate limit and a deleted video all reached the reader as one 404 +saying "No video with that id". + +## What a read can fail with + +Each cause gets its own page, its own status and its own line in the log, and only one of them is a +404: + + video_missing 404 the platform says the video is unplayable + bot_check 502 the platform wants proof the caller is a person + consent_wall 502 a consent page came back instead of the video + rate_limited 429 too many requests, and the answer carries retry-after + platform_error 502 a status that is not a page + unrecognised_page 502 a page with no video and no reason given + captions_refused 502 the video was found, the caption text was refused + captions_not_json 502 the caption endpoint answered with something else + captions_empty 502 the track holds no readable line + +A missing video is proved, never assumed. Anything the code cannot recognise is a refusal, because +calling an unrecognised page a missing video is what told readers a working video was deleted. + +Nothing written to the log is a credential. The caption address is signed and the request carries +cookies and a minted token, so an address is cut to its host and path before it is written. + ## The gates npm ci From b69b4c1b44e7af8be98084526a4f55562c6e4fa2 Mon Sep 17 00:00:00 2001 From: juliantellez <4896851+juliantellez@users.noreply.github.com> Date: Tue, 1 Sep 2026 08:51:07 +0000 Subject: [PATCH 5/7] fix: read the text through the player endpoint when the watch page is refused --- src/live.livetest.ts | 29 ++++++ src/transcript/innertube.test.ts | 149 +++++++++++++++++++++++++++++ src/transcript/innertube.ts | 157 +++++++++++++++++++++++++++++++ src/transcript/timedtext.ts | 52 ++++++++++ src/transcript/youtube.test.ts | 76 +++++++++++++++ src/transcript/youtube.ts | 79 ++++++---------- 6 files changed, 494 insertions(+), 48 deletions(-) create mode 100644 src/transcript/innertube.test.ts create mode 100644 src/transcript/innertube.ts create mode 100644 src/transcript/timedtext.ts diff --git a/src/live.livetest.ts b/src/live.livetest.ts index 3438e74..a884213 100644 --- a/src/live.livetest.ts +++ b/src/live.livetest.ts @@ -41,3 +41,32 @@ describe("against the real platform", () => { expect(first?.segments.length).toBeGreaterThan(100); }); }); + +// What the deployed function is served at a datacentre address: a watch page carrying the bot +// check. Nothing else is faked, so the player endpoint and the caption address are the real ones. +// This is the check that the deploy gate was failing on. +describe("against the real platform, with the watch page refused", () => { + const BOT_CHECK = + `<html><title>YouTube`; + + it("still reaches the text through the player endpoint", { timeout: 120_000 }, async () => { + const real = globalThis.fetch; + const refused = (async (url: string | URL | Request, options?: RequestInit) => { + if (String(url).includes("/watch?v=")) { + return new Response(BOT_CHECK, { status: 200, headers: { "content-type": "text/html; charset=utf-8" } }); + } + return real(url, options); + }) as unknown as typeof fetch; + + const item = await fetchTranscript("gyN9lV9QgyA", { fetch: refused }); + + console.log( + `refused watch page captions=${item.has_captions} segments=${item.segments.length}` + + ` chars=${item.text.length} source=${item.source}`, + ); + expect(item.has_captions).toBe(true); + expect(item.segments.length).toBeGreaterThan(100); + expect(item.title.length).toBeGreaterThan(0); + }); +}); diff --git a/src/transcript/innertube.test.ts b/src/transcript/innertube.test.ts new file mode 100644 index 0000000..9f870b6 --- /dev/null +++ b/src/transcript/innertube.test.ts @@ -0,0 +1,149 @@ +import { describe, expect, it, vi } from "vitest"; +import { readThroughPlayer } from "./innertube.js"; + +// Real headers rather than an object with the one method the code happens to call. A double that +// answers less than the real thing turns a missing call into a green test. +const jsonOf = (body: unknown, status = 200) => + ({ + ok: status >= 200 && status < 300, + status, + json: async () => body, + text: async () => JSON.stringify(body), + headers: new Headers({ "content-type": "application/json" }), + }) as unknown as Response; + +const textOf = (body: string, status = 200) => + ({ + ok: status >= 200 && status < 300, + status, + text: async () => body, + headers: new Headers({ "content-type": "application/json" }), + }) as unknown as Response; + +const playable = (withTrack = true) => ({ + playabilityStatus: { status: "OK" }, + videoDetails: { title: "A title" }, + captions: withTrack + ? { + playerCaptionsTracklistRenderer: { + captionTracks: [ + { + baseUrl: "https://www.youtube.com/api/timedtext?v=x&signature=secret", + name: { simpleText: "English (auto-generated)" }, + languageCode: "en", + kind: "asr", + }, + ], + }, + } + : {}, +}); + +const captionBody = JSON.stringify({ + events: [ + { tStartMs: 0, dDurationMs: 1500, segs: [{ utf8: "first" }, { utf8: " line" }] }, + { tStartMs: 1500, dDurationMs: 2000, segs: [{ utf8: "second line" }] }, + ], +}); + +describe("reading through the player endpoint", () => { + it("returns the title, the track and the lines when the video plays", async () => { + const f = vi.fn().mockResolvedValueOnce(jsonOf(playable())).mockResolvedValueOnce(textOf(captionBody)); + + const read = await readThroughPlayer("gyN9lV9QgyA", f as unknown as typeof fetch); + + expect(read?.title).toBe("A title"); + expect(read?.track?.languageCode).toBe("en"); + expect(read?.track?.kind).toBe("asr"); + expect(read?.segments).toHaveLength(2); + expect(read?.segments.map((s) => s.text).join(" ")).toBe("first line second line"); + }); + + // The whole reason this path exists: it must not carry the proof of origin token the watch page + // path needs, because minting one is what the refused address cannot do. + it("asks as the phone client, and asks for the captions with no proof of origin token", async () => { + const f = vi.fn().mockResolvedValueOnce(jsonOf(playable())).mockResolvedValueOnce(textOf(captionBody)); + + await readThroughPlayer("gyN9lV9QgyA", f as unknown as typeof fetch); + + const [playerUrl, playerOptions] = f.mock.calls[0] as [string, RequestInit]; + expect(playerUrl).toContain("/youtubei/v1/player"); + expect(playerOptions.method).toBe("POST"); + const sent = JSON.parse(playerOptions.body as string) as { + videoId: string; + context: { client: { clientName: string } }; + }; + expect(sent.videoId).toBe("gyN9lV9QgyA"); + expect(sent.context.client.clientName).toBe("IOS"); + + const [captionUrl] = f.mock.calls[1] as [string]; + expect(captionUrl).toContain("fmt=json3"); + expect(captionUrl).not.toContain("pot="); + }); + + it("returns a read with no captions when the video plays and carries no track", async () => { + const f = vi.fn().mockResolvedValueOnce(jsonOf(playable(false))); + + const read = await readThroughPlayer("gyN9lV9QgyA", f as unknown as typeof fetch); + + expect(read).not.toBeNull(); + expect(read?.track).toBeNull(); + expect(read?.segments).toEqual([]); + expect(read?.title).toBe("A title"); + expect(f).toHaveBeenCalledTimes(1); + }); + + // Everything below returns null rather than a reason. This path is a second opinion, so when it + // has nothing to add the caller reports what the watch page said. + it("returns null when the video does not play", async () => { + const f = vi + .fn() + .mockResolvedValueOnce(jsonOf({ playabilityStatus: { status: "ERROR", reason: "This video is unavailable" } })); + + expect(await readThroughPlayer("aaaaaaaaaaa", f as unknown as typeof fetch)).toBeNull(); + }); + + it("returns null when the player endpoint answers with a status", async () => { + const f = vi.fn().mockResolvedValueOnce(jsonOf({}, 429)); + expect(await readThroughPlayer("gyN9lV9QgyA", f as unknown as typeof fetch)).toBeNull(); + }); + + it("returns null when the player endpoint cannot be reached", async () => { + const f = vi.fn().mockRejectedValueOnce(new Error("no route to host")); + expect(await readThroughPlayer("gyN9lV9QgyA", f as unknown as typeof fetch)).toBeNull(); + }); + + it("returns null when the caption address answers with a status", async () => { + const f = vi.fn().mockResolvedValueOnce(jsonOf(playable())).mockResolvedValueOnce(textOf("", 403)); + expect(await readThroughPlayer("gyN9lV9QgyA", f as unknown as typeof fetch)).toBeNull(); + }); + + it("returns null when the caption address answers with something that is not json", async () => { + const f = vi + .fn() + .mockResolvedValueOnce(jsonOf(playable())) + .mockResolvedValueOnce(textOf("")); + expect(await readThroughPlayer("gyN9lV9QgyA", f as unknown as typeof fetch)).toBeNull(); + }); + + it("returns null when every line of the track is empty", async () => { + const f = vi + .fn() + .mockResolvedValueOnce(jsonOf(playable())) + .mockResolvedValueOnce(textOf(JSON.stringify({ events: [{ tStartMs: 0, segs: [{ utf8: " " }] }] }))); + expect(await readThroughPlayer("gyN9lV9QgyA", f as unknown as typeof fetch)).toBeNull(); + }); + + // The caption address is signed. Writing it whole puts the signature in the log. + it("never writes the signature on the caption address to the log", async () => { + const written: string[] = []; + const log = vi.spyOn(console, "log").mockImplementation((line: string) => void written.push(line)); + const f = vi.fn().mockResolvedValueOnce(jsonOf(playable())).mockResolvedValueOnce(textOf("", 403)); + + await readThroughPlayer("gyN9lV9QgyA", f as unknown as typeof fetch); + log.mockRestore(); + + expect(written.join("\n")).not.toContain("secret"); + expect(written.join("\n")).toContain("https://www.youtube.com/api/timedtext"); + }); +}); diff --git a/src/transcript/innertube.ts b/src/transcript/innertube.ts new file mode 100644 index 0000000..db6dc3d --- /dev/null +++ b/src/transcript/innertube.ts @@ -0,0 +1,157 @@ +import { observe, withoutSecrets } from "../observe.js"; +import { type CaptionTrack, type TimedTextEvent, chooseTrack, eventsToSegments } from "./timedtext.js"; +import type { Segment } from "../contract.js"; + +// The watch page is not readable from every address. From a datacentre the platform serves a full +// page that carries no caption track and a player response asking the caller to prove it is a +// person, which is where the deployed function stopped. +// +// The player endpoint answers the same question for a phone client. That client is not asked for +// the proof, and the caption address it hands back needs no proof of origin token either, so this +// path reaches the text where the watch page cannot. + +const CLIENT = { + clientName: "IOS", + clientVersion: "20.10.4", + deviceMake: "Apple", + deviceModel: "iPhone16,2", + osName: "iOS", + osVersion: "18.0.0.22A3354", + hl: "en", + gl: "US", +}; + +const CLIENT_NAME_ID = "5"; + +const USER_AGENT = "com.google.ios.youtube/20.10.4 (iPhone16,2; U; CPU iOS 18_0 like Mac OS X)"; + +const PLAYER = "https://www.youtube.com/youtubei/v1/player?prettyPrint=false"; + +export const PLAYER_SOURCE = + "youtube.com youtubei v1 player, ios client, timedtext json3, no proof of origin token"; + +// What the player endpoint gave back. A null track is a video that plays and carries no captions, +// which is an answer rather than a failure. +export interface PlayerRead { + title: string; + track: CaptionTrack | null; + segments: Segment[]; +} + +interface PlayerResponse { + playabilityStatus?: { status?: string; reason?: string }; + videoDetails?: { title?: string }; + captions?: { playerCaptionsTracklistRenderer?: { captionTracks?: CaptionTrack[] } }; +} + +// This is a second opinion, asked only after the first one failed. Anything unexpected returns null +// so the caller reports what the watch page said, rather than this path inventing a new reason. +export async function readThroughPlayer(videoId: string, f: typeof fetch): Promise { + let answer: PlayerResponse; + let status: number; + try { + const response = await f(PLAYER, { + method: "POST", + headers: { + "content-type": "application/json", + "user-agent": USER_AGENT, + "x-youtube-client-name": CLIENT_NAME_ID, + "x-youtube-client-version": CLIENT.clientVersion, + }, + body: JSON.stringify({ + videoId, + context: { client: CLIENT }, + contentCheckOk: true, + racyCheckOk: true, + }), + }); + status = response.status; + if (!response.ok) { + observe("player_unusable", { video_id: videoId, reason: "the player endpoint answered a status", status }); + return null; + } + answer = (await response.json()) as PlayerResponse; + } catch { + observe("player_unusable", { video_id: videoId, reason: "the player endpoint could not be read" }); + return null; + } + + const playability = answer.playabilityStatus?.status ?? ""; + const title = answer.videoDetails?.title; + if (playability !== "OK" || title === undefined || title === "") { + observe("player_unusable", { + video_id: videoId, + reason: "the player endpoint did not return a playable video", + status, + playability, + playability_reason: answer.playabilityStatus?.reason ?? "", + }); + return null; + } + + const track = chooseTrack(answer.captions?.playerCaptionsTracklistRenderer?.captionTracks ?? []); + observe("player_read", { + video_id: videoId, + status, + has_caption_tracks: track !== null, + track_kind: track?.kind ?? "", + language_code: track?.languageCode ?? "", + }); + if (!track) return { title, track: null, segments: [] }; + + // The address is signed, so only its host and path are ever written. + const address = `${track.baseUrl}&fmt=json3`; + let body: string; + try { + const captions = await f(address, { headers: { "user-agent": USER_AGENT } }); + if (!captions.ok) { + observe("player_captions_unusable", { + video_id: videoId, + reason: "the caption address answered a status", + status: captions.status, + address: withoutSecrets(address), + }); + return null; + } + body = await captions.text(); + } catch { + observe("player_captions_unusable", { + video_id: videoId, + reason: "the caption address could not be read", + address: withoutSecrets(address), + }); + return null; + } + + let parsed: { events?: TimedTextEvent[] }; + try { + parsed = JSON.parse(body) as { events?: TimedTextEvent[] }; + } catch { + observe("player_captions_unusable", { + video_id: videoId, + reason: "the caption address did not answer with json", + bytes: body.length, + address: withoutSecrets(address), + }); + return null; + } + + const segments = eventsToSegments(parsed.events ?? []); + if (segments.length === 0) { + observe("player_captions_unusable", { + video_id: videoId, + reason: "the track holds no readable line", + bytes: body.length, + address: withoutSecrets(address), + }); + return null; + } + + observe("player_captions_read", { + video_id: videoId, + bytes: body.length, + segments: segments.length, + characters: segments.reduce((n, segment) => n + segment.text.length, 0), + }); + return { title, track, segments }; +} diff --git a/src/transcript/timedtext.ts b/src/transcript/timedtext.ts new file mode 100644 index 0000000..5a2632c --- /dev/null +++ b/src/transcript/timedtext.ts @@ -0,0 +1,52 @@ +import type { Segment } from "../contract.js"; + +// The pieces both entry points share. The watch page and the player endpoint describe a caption +// track with the same shape and hand back the same json3 body, so the choosing and the parsing live +// here rather than once per entry point. + +export interface CaptionTrack { + baseUrl: string; + languageCode?: string; + kind?: string; + name?: { simpleText?: string; runs?: { text: string }[] }; +} + +export interface TimedTextEvent { + tStartMs?: number; + dDurationMs?: number; + segs?: { utf8?: string }[]; +} + +// A track a person wrote is preferred over one a machine generated, and English over anything else. +export function chooseTrack(tracks: CaptionTrack[]): CaptionTrack | null { + if (tracks.length === 0) return null; + const score = (t: CaptionTrack): number => { + let n = 0; + if (t.kind !== "asr") n += 2; + if ((t.languageCode ?? "").startsWith("en")) n += 1; + return n; + }; + return [...tracks].sort((a, b) => score(b) - score(a))[0] ?? null; +} + +export function trackName(track: CaptionTrack): string { + return track.name?.simpleText ?? track.name?.runs?.[0]?.text ?? ""; +} + +export function eventsToSegments(events: TimedTextEvent[]): Segment[] { + const segments: Segment[] = []; + for (const event of events) { + const text = (event.segs ?? []) + .map((s) => s.utf8 ?? "") + .join("") + .replace(/\s+/g, " ") + .trim(); + if (!text) continue; + segments.push({ + start_seconds: Math.round((event.tStartMs ?? 0) / 10) / 100, + duration_seconds: Math.round((event.dDurationMs ?? 0) / 10) / 100, + text, + }); + } + return segments; +} diff --git a/src/transcript/youtube.test.ts b/src/transcript/youtube.test.ts index 10da44a..caeccd4 100644 --- a/src/transcript/youtube.test.ts +++ b/src/transcript/youtube.test.ts @@ -257,3 +257,79 @@ describe("fetching a transcript", () => { ).rejects.toThrow(/no usable line/); }); }); + +// The watch page is refused at a datacentre address and readable elsewhere, which is what stopped +// the deployed function. A refusal is now a question asked again, not an answer to the reader. +describe("when the watch page is refused, the player endpoint is asked", () => { + const playerAnswer = { + "playabilityStatus": { + "status": "OK" + }, + "videoDetails": { + "title": "A title the player knows" + }, + "captions": { + "playerCaptionsTracklistRenderer": { + "captionTracks": [ + { + "baseUrl": "https://www.youtube.com/api/timedtext?v=x&signature=1", + "name": { + "simpleText": "English (auto-generated)" + }, + "languageCode": "en", + "kind": "asr" + } + ] + } + } + }; + + const jsonOf = (body: unknown, status = 200) => + ({ + ok: status >= 200 && status < 300, + status, + json: async () => body, + text: async () => JSON.stringify(body), + headers: new Headers({ "content-type": "application/json" }), + }) as unknown as Response; + + it("returns the transcript the player found instead of the refusal", async () => { + const f = vi + .fn() + .mockResolvedValueOnce(responseOf(botCheckPage)) + .mockResolvedValueOnce(jsonOf(playerAnswer)) + .mockResolvedValueOnce(responseOf(captionBody, 200, [], "application/json")); + + const item = await fetchTranscript("gyN9lV9QgyA", { fetch: f as unknown as typeof fetch, minter: getMinter }); + + expect(item.has_captions).toBe(true); + expect(item.title).toBe("A title the player knows"); + expect(item.text).toBe("first line second line"); + expect(item.source).toContain("ios client"); + }); + + it("still refuses when the player endpoint has nothing to add", async () => { + const f = vi + .fn() + .mockResolvedValueOnce(responseOf(botCheckPage)) + .mockResolvedValueOnce(jsonOf({ playabilityStatus: { status: "LOGIN_REQUIRED" } })); + + const failure = await fetchTranscript("gyN9lV9QgyA", { + fetch: f as unknown as typeof fetch, + minter: getMinter, + }).catch((thrown: unknown) => thrown); + + expect(failure).toBeInstanceOf(PlatformRefused); + expect((failure as PlatformRefused).why).toBe("bot_check"); + }); + + // A missing video is proved from the watch page, so there is nothing to ask again about. + it("does not ask the player when the platform proved the video is missing", async () => { + const f = vi.fn().mockResolvedValueOnce(responseOf(missingVideoPage)); + + await expect( + fetchTranscript("aaaaaaaaaaa", { fetch: f as unknown as typeof fetch, minter: getMinter }), + ).rejects.toBeInstanceOf(VideoNotFound); + expect(f).toHaveBeenCalledTimes(1); + }); +}); diff --git a/src/transcript/youtube.ts b/src/transcript/youtube.ts index f626e6c..9709fc2 100644 --- a/src/transcript/youtube.ts +++ b/src/transcript/youtube.ts @@ -1,7 +1,12 @@ -import { SCHEMA_VERSION, type Cause, type Segment, type TranscriptItem } from "../contract.js"; +import { SCHEMA_VERSION, type Cause, type TranscriptItem } from "../contract.js"; import { observe, withoutSecrets } from "../observe.js"; import { heldMinter, type Minter } from "./attestation.js"; import { WHAT_HAPPENED, causeOf, readShape } from "./diagnosis.js"; +import { PLAYER_SOURCE, type PlayerRead, readThroughPlayer } from "./innertube.js"; +import { type CaptionTrack, type TimedTextEvent, chooseTrack, eventsToSegments, trackName } from "./timedtext.js"; + +export { chooseTrack, eventsToSegments, trackName }; +export type { CaptionTrack, TimedTextEvent }; const USER_AGENT = "Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/131.0.0.0 Safari/537.36"; @@ -21,13 +26,6 @@ export class PlatformRefused extends Error { } } -interface CaptionTrack { - baseUrl: string; - languageCode?: string; - kind?: string; - name?: { simpleText?: string; runs?: { text: string }[] }; -} - interface WatchPage { status: number; contentType: string; @@ -41,12 +39,6 @@ export interface Fetched { events: TimedTextEvent[]; } -interface TimedTextEvent { - tStartMs?: number; - dDurationMs?: number; - segs?: { utf8?: string }[]; -} - // A status that is not 200 is not thrown here. The body of a refusal is where the reason is // written, so it is read and classified like any other answer. async function readWatchPage(videoId: string, f: typeof fetch): Promise { @@ -88,40 +80,6 @@ export function readCaptionTracks(html: string): CaptionTrack[] { } } -// A track a person wrote is preferred over one a machine generated, and English over anything else. -export function chooseTrack(tracks: CaptionTrack[]): CaptionTrack | null { - if (tracks.length === 0) return null; - const score = (t: CaptionTrack): number => { - let n = 0; - if (t.kind !== "asr") n += 2; - if ((t.languageCode ?? "").startsWith("en")) n += 1; - return n; - }; - return [...tracks].sort((a, b) => score(b) - score(a))[0] ?? null; -} - -export function trackName(track: CaptionTrack): string { - return track.name?.simpleText ?? track.name?.runs?.[0]?.text ?? ""; -} - -export function eventsToSegments(events: TimedTextEvent[]): Segment[] { - const segments: Segment[] = []; - for (const event of events) { - const text = (event.segs ?? []) - .map((s) => s.utf8 ?? "") - .join("") - .replace(/\s+/g, " ") - .trim(); - if (!text) continue; - segments.push({ - start_seconds: Math.round((event.tStartMs ?? 0) / 10) / 100, - duration_seconds: Math.round((event.dDurationMs ?? 0) / 10) / 100, - text, - }); - } - return segments; -} - function captionStatusCause(status: number): Cause { if (status === 429) return "rate_limited"; if (status === 403) return "bot_check"; @@ -129,6 +87,23 @@ function captionStatusCause(status: number): Cause { return "captions_refused"; } +function itemFromPlayer(videoId: string, read: PlayerRead): TranscriptItem { + const track = read.track; + return { + video_id: videoId, + schema_version: SCHEMA_VERSION, + fetched_at: new Date().toISOString(), + title: read.title, + has_captions: track !== null, + language_code: track?.languageCode ?? "", + track_kind: track === null ? "" : track.kind === "asr" ? "asr" : "standard", + track_name: track === null ? "" : trackName(track), + segments: read.segments, + text: read.segments.map((s) => s.text).join(" "), + source: PLAYER_SOURCE, + }; +} + export async function fetchTranscript( videoId: string, deps: { fetch?: typeof fetch; minter?: () => Promise } = {}, @@ -151,6 +126,14 @@ export async function fetchTranscript( ...shape, }); if (why === "video_missing") throw new VideoNotFound(WHAT_HAPPENED[why]); + + // The watch page is readable from some addresses and refused at others, so a refusal here is + // not the end of the question. The player endpoint is asked before the reader is told no, and + // it answers where the watch page will not. A missing video is already proved above, so this + // asks only about a refusal. + const read = await readThroughPlayer(videoId, f); + if (read) return itemFromPlayer(videoId, read); + throw new PlatformRefused(why, WHAT_HAPPENED[why]); } From 293fa50592ec5351e1dd0e8ce1696d52d00740ba Mon Sep 17 00:00:00 2001 From: juliantellez <4896851+juliantellez@users.noreply.github.com> Date: Tue, 1 Sep 2026 08:51:12 +0000 Subject: [PATCH 6/7] docs: record the request that gets the text from a refused address --- README.md | 32 ++++++++++++++++++++++++++++++++ 1 file changed, 32 insertions(+) diff --git a/README.md b/README.md index 29fd6bb..cff4239 100644 --- a/README.md +++ b/README.md @@ -61,6 +61,38 @@ Reading it needed the boundary to say so. Before this, the title came back null the same error, so a refusal, a rate limit and a deleted video all reached the reader as one 404 saying "No video with that id". +## What gets the text when the watch page is refused + +The refusal above lands on the watch page, so no caption address is ever read and the proof of origin +work is never reached. Asking the same question a different way answers it. This request works, and +it carries no token at all: + + POST https://www.youtube.com/youtubei/v1/player?prettyPrint=false + content-type: application/json + user-agent: com.google.ios.youtube/20.10.4 (iPhone16,2; U; CPU iOS 18_0 like Mac OS X) + x-youtube-client-name: 5 + x-youtube-client-version: 20.10.4 + + {"videoId":"gyN9lV9QgyA","context":{"client":{"clientName":"IOS","clientVersion":"20.10.4", + "deviceMake":"Apple","deviceModel":"iPhone16,2","hl":"en","gl":"US"}}, + "contentCheckOk":true,"racyCheckOk":true} + +It answers with the title and a caption track. That track's address with `&fmt=json3` added returns +406,491 bytes of json3, which is 560 segments and 19,420 characters on `gyN9lV9QgyA`. That is the +same count the watch page path produces. + +Measured on 1 September 2026. Three readings behind it: + +- The caption address from the watch page returns HTTP 200 and zero bytes without a minted token, + from an address the platform does not refuse at all. The token is what that path needs. +- The caption address from the player endpoint needs no token. Plain, it returns 406,491 bytes. +- The `ANDROID` client answers too, but ignores `fmt=json3` and returns XML, so `IOS` is the one used. + +The watch page is still read first, because it is the path that tells a video with no captions apart +from a video that is missing. The player endpoint is asked only after a refusal, and a proved missing +video never reaches it. When the player endpoint has nothing to add, the refusal the watch page named +is what the reader gets, unchanged. + ## What a read can fail with Each cause gets its own page, its own status and its own line in the log, and only one of them is a From 9e819482aad60f86b7c085402545b1c49f7cfb53 Mon Sep 17 00:00:00 2001 From: juliantellez <4896851+juliantellez@users.noreply.github.com> Date: Tue, 1 Sep 2026 08:59:47 +0000 Subject: [PATCH 7/7] docs: the player endpoint is refused from the function address too --- README.md | 35 ++++++++++++++++++++++++++++++----- 1 file changed, 30 insertions(+), 5 deletions(-) diff --git a/README.md b/README.md index cff4239..dbcfc69 100644 --- a/README.md +++ b/README.md @@ -64,8 +64,8 @@ saying "No video with that id". ## What gets the text when the watch page is refused The refusal above lands on the watch page, so no caption address is ever read and the proof of origin -work is never reached. Asking the same question a different way answers it. This request works, and -it carries no token at all: +work is never reached. Asking the same question a different way answers it, from an address the +platform does not refuse. This request carries no token at all: POST https://www.youtube.com/youtubei/v1/player?prettyPrint=false content-type: application/json @@ -81,10 +81,10 @@ It answers with the title and a caption track. That track's address with `&fmt=j 406,491 bytes of json3, which is 560 segments and 19,420 characters on `gyN9lV9QgyA`. That is the same count the watch page path produces. -Measured on 1 September 2026. Three readings behind it: +Measured on 1 September 2026, from an address the platform does not refuse. Three readings: -- The caption address from the watch page returns HTTP 200 and zero bytes without a minted token, - from an address the platform does not refuse at all. The token is what that path needs. +- The caption address from the watch page returns HTTP 200 and zero bytes without a minted token. + The token is what that path needs. - The caption address from the player endpoint needs no token. Plain, it returns 406,491 bytes. - The `ANDROID` client answers too, but ignores `fmt=json3` and returns XML, so `IOS` is the one used. @@ -93,6 +93,31 @@ from a video that is missing. The player endpoint is asked only after a refusal, video never reaches it. When the player endpoint has nothing to add, the refusal the watch page named is what the reader gets, unchanged. +## The player endpoint is refused from the function too + +The paragraph above is true from an address the platform accepts. It is not true from the deployed +function. Captured on 1 September 2026 from the function log in eu-west-1, on the run that deployed +this code: + + {"event":"watch_page_unreadable","video_id":"gyN9lV9QgyA","cause":"bot_check","status":200, + "bytes":1339782,"has_video_details":true,"has_caption_tracks":false, + "playability":"LOGIN_REQUIRED","playability_reason":"Sign in to confirm you're not a bot"} + {"event":"player_unusable","video_id":"gyN9lV9QgyA", + "reason":"the player endpoint did not return a playable video","status":200, + "playability":"LOGIN_REQUIRED","playability_reason":"Sign in to confirm you're not a bot"} + +Both entry points are refused, with the same reason, on the same request. So the block is on the +address the function calls from, and it is not a property of the client the request claims to be. +Changing the client does not move it. That is the measurement, and it rules out the whole family of +fixes that consist of asking as something else. + +What is left is the address. Reaching the platform from somewhere it does not refuse needs an egress +this project refuses on cost: a network address translation gateway bills while nothing happens, and +a proxy service is a paid dependency. Neither is a choice to make here, so the deploy gate that asks +the page for a transcript stays red until a person decides which egress to buy. + +The gate is correct and it is not weakened. It asks for a transcript and it does not get one. + ## What a read can fail with Each cause gets its own page, its own status and its own line in the log, and only one of them is a