fix: report what the platform actually answered instead of a missing video

.github/workflows/deploy.yml

@@ -121,22 +121,45 @@ jobs:
           test "$code" = "200"
         working-directory: infra
 
-      # Green means the steps ran. It says nothing about whether the page works, so the page is
-      # opened and read before this run is called a success.
-      - name: read the page that is now serving
+      # The deploy serving at all is a different question from the product working, so they are
+      # asked separately. This one only proves the distribution reaches the function.
+      - name: read the front page
+        run: |
+          set -euo pipefail
+          address="$(terraform output -raw address)"
+          code=$(curl -s -o /tmp/front.html -w '%{http_code}' --max-time 90 "$address/" || true)
+          echo "the front page answered $code with $(wc -c < /tmp/front.html) bytes"
+          test "$code" = "200"
+          grep -q "Read a video as text" /tmp/front.html
+        working-directory: infra
+
+      # Green means the steps ran. It says nothing about whether a reader gets a transcript, so one
+      # is asked for here. A failure names what the platform did rather than the status alone,
+      # because a bare 404 told nobody anything for a day. The page states the cause in its title.
+      - name: read a transcript from the page that is now serving
         run: |
           set -euo pipefail
           address="$(terraform output -raw address)"
-          cd "$GITHUB_WORKSPACE"
           for attempt in 1 2 3 4 5 6 7 8; do
             code=$(curl -s -o /tmp/page.html -w '%{http_code}' --max-time 90 "$address/videos?id=gyN9lV9QgyA" || true)
             if [ "$code" = "200" ]; then break; fi
             echo "attempt $attempt answered $code, the distribution may still be spreading"
             sleep 30
           done
-          test "$code" = "200"
           bytes=$(wc -c < /tmp/page.html)
-          echo "the page answered $code with $bytes bytes"
+          said=$(sed -n "s/.*<title>\\(.*\\)<\\/title>.*/\\1/p" /tmp/page.html | head -1)
+          echo "the page answered $code with $bytes bytes, and said: $said"
+          if [ "$code" != "200" ]; then
+            {
+              echo "## The page is serving, and it cannot read a transcript"
+              echo ""
+              echo "It answered $code and said: $said"
+              echo ""
+              echo "The cause is in the function log, on the watch_page_unreadable line."
+            } >> "$GITHUB_STEP_SUMMARY"
+            echo "::error::the page answered $code and said: $said"
+            exit 1
+          fi
           test "$bytes" -gt 5000
           grep -q "Transcript" /tmp/page.html
           echo "the page carries text"

README.md

@@ -41,6 +41,104 @@ it is correct for the player, and for captions it returns zero bytes.
 The integrity token behind the mint lasts twelve hours, so a warm function attests once and mints per
 video after that.
 
+## What the deployed function actually gets
+
+Captured from the deployed function on 31 August 2026, in eu-west-1, on the `watch_page_unreadable`
+line of its own log:
+
+    {"event":"watch_page_unreadable","video_id":"gyN9lV9QgyA","cause":"bot_check",
+     "status":200,"bytes":1412860,"has_video_details":true,"has_caption_tracks":false,
+     "has_player_response":true,"playability":"LOGIN_REQUIRED",
+     "playability_reason":"Sign in to confirm you are not a bot"}
+
+The platform serves the function a full watch page, HTTP 200, over a megabyte, and that page carries
+no caption track and a player response that asks the caller to prove it is a person. The same
+request from a laptop returns the video and 560 caption segments. The block is on the address the
+function calls from, and it lands on the watch page, before the proof of origin work above is
+reached.
+
+Reading it needed the boundary to say so. Before this, the title came back null and every cause threw
+the same error, so a refusal, a rate limit and a deleted video all reached the reader as one 404
+saying "No video with that id".
+
+## What gets the text when the watch page is refused
+
+The refusal above lands on the watch page, so no caption address is ever read and the proof of origin
+work is never reached. Asking the same question a different way answers it, from an address the
+platform does not refuse. This request carries no token at all:
+
+    POST https://www.youtube.com/youtubei/v1/player?prettyPrint=false
+    content-type: application/json
+    user-agent: com.google.ios.youtube/20.10.4 (iPhone16,2; U; CPU iOS 18_0 like Mac OS X)
+    x-youtube-client-name: 5
+    x-youtube-client-version: 20.10.4
+
+    {"videoId":"gyN9lV9QgyA","context":{"client":{"clientName":"IOS","clientVersion":"20.10.4",
+     "deviceMake":"Apple","deviceModel":"iPhone16,2","hl":"en","gl":"US"}},
+     "contentCheckOk":true,"racyCheckOk":true}
+
+It answers with the title and a caption track. That track's address with `&fmt=json3` added returns
+406,491 bytes of json3, which is 560 segments and 19,420 characters on `gyN9lV9QgyA`. That is the
+same count the watch page path produces.
+
+Measured on 1 September 2026, from an address the platform does not refuse. Three readings:
+
+- The caption address from the watch page returns HTTP 200 and zero bytes without a minted token.
+  The token is what that path needs.
+- The caption address from the player endpoint needs no token. Plain, it returns 406,491 bytes.
+- The `ANDROID` client answers too, but ignores `fmt=json3` and returns XML, so `IOS` is the one used.
+
+The watch page is still read first, because it is the path that tells a video with no captions apart
+from a video that is missing. The player endpoint is asked only after a refusal, and a proved missing
+video never reaches it. When the player endpoint has nothing to add, the refusal the watch page named
+is what the reader gets, unchanged.
+
+## The player endpoint is refused from the function too
+
+The paragraph above is true from an address the platform accepts. It is not true from the deployed
+function. Captured on 1 September 2026 from the function log in eu-west-1, on the run that deployed
+this code:
+
+    {"event":"watch_page_unreadable","video_id":"gyN9lV9QgyA","cause":"bot_check","status":200,
+     "bytes":1339782,"has_video_details":true,"has_caption_tracks":false,
+     "playability":"LOGIN_REQUIRED","playability_reason":"Sign in to confirm you're not a bot"}
+    {"event":"player_unusable","video_id":"gyN9lV9QgyA",
+     "reason":"the player endpoint did not return a playable video","status":200,
+     "playability":"LOGIN_REQUIRED","playability_reason":"Sign in to confirm you're not a bot"}
+
+Both entry points are refused, with the same reason, on the same request. So the block is on the
+address the function calls from, and it is not a property of the client the request claims to be.
+Changing the client does not move it. That is the measurement, and it rules out the whole family of
+fixes that consist of asking as something else.
+
+What is left is the address. Reaching the platform from somewhere it does not refuse needs an egress
+this project refuses on cost: a network address translation gateway bills while nothing happens, and
+a proxy service is a paid dependency. Neither is a choice to make here, so the deploy gate that asks
+the page for a transcript stays red until a person decides which egress to buy.
+
+The gate is correct and it is not weakened. It asks for a transcript and it does not get one.
+
+## What a read can fail with
+
+Each cause gets its own page, its own status and its own line in the log, and only one of them is a
+404:
+
+    video_missing      404  the platform says the video is unplayable
+    bot_check          502  the platform wants proof the caller is a person
+    consent_wall       502  a consent page came back instead of the video
+    rate_limited       429  too many requests, and the answer carries retry-after
+    platform_error     502  a status that is not a page
+    unrecognised_page  502  a page with no video and no reason given
+    captions_refused   502  the video was found, the caption text was refused
+    captions_not_json  502  the caption endpoint answered with something else
+    captions_empty     502  the track holds no readable line
+
+A missing video is proved, never assumed. Anything the code cannot recognise is a refusal, because
+calling an unrecognised page a missing video is what told readers a working video was deleted.
+
+Nothing written to the log is a credential. The caption address is signed and the request carries
+cookies and a minted token, so an address is cut to its host and path before it is written.
+
 ## The gates
 
     npm ci

features/a-refusal-is-not-a-missing-video.feature.ts

@@ -0,0 +1,116 @@
+import { describe, expect, vi } from "vitest";
+import { route, type FunctionUrlEvent } from "../src/handler.js";
+import { makeLookup } from "../src/store/lookup.js";
+import { MemoryStore } from "../src/store/store.js";
+import { fetchTranscript } from "../src/transcript/youtube.js";
+import { scenario, theAnswer, type World } from "./scenario.js";
+
+// The deployed page reported a working video as deleted. The platform was refusing the address the
+// function reads from, and a refusal, a rate limit and a genuinely missing video all reached the
+// reader as one 404 that said "No video with that id".
+//
+// The page the platform really sent is copied here from the function's own log: HTTP 200, over a
+// megabyte, no caption track, and a player response asking the caller to prove it is a person.
+
+const REFUSED_BY_THE_PLATFORM =
+  `<html><head><title> - YouTube</title></head><body><script>var ytInitialPlayerResponse = ` +
+  `{"playabilityStatus":{"status":"LOGIN_REQUIRED","reason":"Sign in to confirm you’re not a bot"},` +
+  `"videoDetails":{"videoId":"gyN9lV9QgyA"}};</script></body></html>`;
+
+const DELETED_BY_ITS_OWNER =
+  `<html><head><title> - YouTube</title></head><body><script>var x = ` +
+  `{"playabilityStatus":{"status":"ERROR","reason":"Video unavailable"}};</script></body></html>`;
+
+const answering = (body: string, status = 200) => {
+  const headers = new Headers({ "content-type": "text/html; charset=utf-8" });
+  return { ok: status >= 200 && status < 300, status, text: async () => body, headers } as unknown as Response;
+};
+
+function readerAsksFor(videoId: string, platformAnswers: Response): (world: World) => Promise<void> {
+  return async (world) => {
+    const spy = vi.spyOn(console, "log").mockImplementation(() => undefined);
+    try {
+      const lookup = makeLookup(new MemoryStore(), (id) =>
+        fetchTranscript(id, {
+          fetch: (async () => platformAnswers) as unknown as typeof fetch,
+          minter: async () => ({ mint: async () => "a-token", expiresAt: Date.now() + 60_000 }),
+        }),
+      );
+      const request: FunctionUrlEvent = {
+        rawPath: "/videos",
+        queryStringParameters: { id: videoId },
+        requestContext: { http: { method: "GET", path: "/videos" } },
+      };
+      world.answer = await route(request, lookup);
+      world.log = spy.mock.calls.map((call) => JSON.parse(String(call[0])) as Record<string, unknown>);
+    } finally {
+      spy.mockRestore();
+    }
+  };
+}
+
+describe("a refusal is not a missing video", () => {
+  scenario("the platform refuses the caller, and the reader is told the truth", ({ given, when, then, and }) => {
+    given("the platform asks this page to prove it is a person", () => {
+      expect(REFUSED_BY_THE_PLATFORM).toContain("LOGIN_REQUIRED");
+    });
+
+    when("a reader opens the page for a video that really exists", readerAsksFor("gyN9lV9QgyA", answering(REFUSED_BY_THE_PLATFORM)));
+
+    then("the page does not say the video is missing", (world) => {
+      expect(theAnswer(world).body).not.toContain("No video with that id");
+      expect(theAnswer(world).statusCode).not.toBe(404);
+    });
+
+    and("the status says the fault is upstream, not with the request", (world) => {
+      expect(theAnswer(world).statusCode).toBe(502);
+    });
+
+    and("the page says the platform refused this page rather than the video", (world) => {
+      expect(theAnswer(world).body).toContain("The platform refused this page, not your video");
+      expect(theAnswer(world).body).toContain("Your id is right and the video is fine");
+    });
+
+    and("the page offers the video where it can still be watched", (world) => {
+      expect(theAnswer(world).body).toContain("https://www.youtube.com/watch?v=gyN9lV9QgyA");
+    });
+
+    and("the page does not promise that trying again will work", (world) => {
+      expect(theAnswer(world).body).toContain("Trying again will not help");
+    });
+
+    and("nothing is held at the edge, so a later visit reaches the platform again", (world) => {
+      expect(theAnswer(world).headers["cache-control"]).toBe("no-store");
+    });
+
+    and("the log names the cause, so an operator can tell this from a deleted video", (world) => {
+      const boundary = world.log.find((entry) => entry.event === "watch_page_unreadable");
+      expect(boundary?.cause).toBe("bot_check");
+      expect(boundary?.playability).toBe("LOGIN_REQUIRED");
+    });
+
+    and("the log carries no credential", (world) => {
+      expect(JSON.stringify(world.log)).not.toContain("a-token");
+    });
+  });
+
+  scenario("a video that really is gone is still reported as gone", ({ given, when, then, and }) => {
+    given("the platform says the video is unavailable", () => {
+      expect(DELETED_BY_ITS_OWNER).toContain("ERROR");
+    });
+
+    when("a reader opens the page for it", readerAsksFor("aaaaaaaaaaa", answering(DELETED_BY_ITS_OWNER)));
+
+    then("the page answers 404", (world) => {
+      expect(theAnswer(world).statusCode).toBe(404);
+    });
+
+    and("the page says there is no video with that id", (world) => {
+      expect(theAnswer(world).body).toContain("No video with that id");
+    });
+
+    and("the log calls it a missing video, not a refusal", (world) => {
+      expect(world.log.find((entry) => entry.event === "watch_page_unreadable")?.cause).toBe("video_missing");
+    });
+  });
+});

features/scenario.ts

@@ -0,0 +1,46 @@
+import { expect, it } from "vitest";
+
+// A scenario reads as one sentence per step. When a step fails, the report names the step and the
+// whole scenario, so a failure says what the reader was supposed to get rather than a line number.
+
+export interface World {
+  answer: { statusCode: number; headers: Record<string, string>; body: string } | null;
+  log: Record<string, unknown>[];
+}
+
+type Step = (world: World) => Promise<void> | void;
+
+export interface Steps {
+  given(text: string, step: Step): void;
+  when(text: string, step: Step): void;
+  then(text: string, step: Step): void;
+  and(text: string, step: Step): void;
+}
+
+export function scenario(title: string, build: (steps: Steps) => void): void {
+  const steps: { word: string; text: string; step: Step }[] = [];
+  const add = (word: string) => (text: string, step: Step) => {
+    steps.push({ word, text, step });
+  };
+
+  build({ given: add("Given"), when: add("When"), then: add("Then"), and: add("And") });
+
+  if (steps.length === 0) throw new Error(`the scenario "${title}" declares no step`);
+
+  it(title, async () => {
+    const world: World = { answer: null, log: [] };
+    for (const { word, text, step } of steps) {
+      try {
+        await step(world);
+      } catch (failure) {
+        const why = failure instanceof Error ? failure.message : String(failure);
+        expect.fail(`${word} ${text}\n\n${why}`);
+      }
+    }
+  });
+}
+
+export function theAnswer(world: World): { statusCode: number; headers: Record<string, string>; body: string } {
+  if (world.answer === null) throw new Error("no page was asked for yet");
+  return world.answer;
+}