From 1753502fc023741ada243978f3d699089265efd8 Mon Sep 17 00:00:00 2001 From: Jose Montes de Oca Date: Thu, 17 Sep 2026 21:22:42 -0400 Subject: [PATCH 1/3] fix: stop encoded probes from poisoning the robots.txt cache A request for /robots%2Etxt missed the static robots route, fell through to the docs catch-all, and had its slug decoded before the page was cached. The resulting 404 render was written under the /robots.txt key, so every later robots.txt request failed with "app-route received invalid cache entry APP_PAGE" and returned 500 to Google. Two guards close this. The catch-all now sets dynamicParams = false, so unknown slugs 404 without rendering or writing anything to disk, which also ends the unbounded growth of 404 pages in the server cache. The proxy answers any percent-encoded path with a 404 before routing, because on its own the segment config still left encoded aliases of route handlers returning 500. A new smoke:standalone script boots the standalone server, replays the probe cold and warm, and checks robots.txt, sitemap, docs pages, Markdown negotiation and the cache directory over HTTP and on disk. --- __tests__/proxy.test.ts | 60 ++++- app/[[...slug]]/page.tsx | 8 + package.json | 3 +- proxy.ts | 19 ++ scripts/smoke-standalone.ts | 432 ++++++++++++++++++++++++++++++++++++ 5 files changed, 520 insertions(+), 2 deletions(-) create mode 100644 scripts/smoke-standalone.ts diff --git a/__tests__/proxy.test.ts b/__tests__/proxy.test.ts index 501c21c..4122185 100644 --- a/__tests__/proxy.test.ts +++ b/__tests__/proxy.test.ts @@ -1,5 +1,9 @@ +// @vitest-environment node +// The proxy runs on the server. happy-dom's Request drops the Host header, +// which the legacy-host redirect depends on. import { describe, expect, it } from 'vitest'; -import { resolveDocsHostRedirect } from '../proxy'; +import { NextRequest } from 'next/server'; +import proxy, { hasEncodedPathname, resolveDocsHostRedirect } from '../proxy'; describe('resolveDocsHostRedirect', () => { it('redirects the legacy docs host to docs.prose.md', () => { @@ -18,3 +22,57 @@ describe('resolveDocsHostRedirect', () => { ).toBeNull(); }); }); + +describe('hasEncodedPathname', () => { + it('accepts plain paths', () => { + expect(hasEncodedPathname('/robots.txt')).toBe(false); + expect(hasEncodedPathname('/sitemap.xml')).toBe(false); + expect(hasEncodedPathname('/setup')).toBe(false); + expect(hasEncodedPathname('/')).toBe(false); + }); + + it('flags percent-encoded aliases of route handlers', () => { + expect(hasEncodedPathname('/robots%2Etxt')).toBe(true); + expect(hasEncodedPathname('/sitemap%2Exml')).toBe(true); + }); + + it('flags malformed escape sequences', () => { + expect(hasEncodedPathname('/%E0%A4%A')).toBe(true); + }); +}); + +describe('proxy', () => { + it('returns 404 for a percent-encoded robots path', () => { + const res = proxy(new NextRequest('https://docs.prose.md/robots%2Etxt')); + expect(res.status).toBe(404); + expect(res.headers.get('x-middleware-next')).toBeNull(); + }); + + it('returns 404 for a percent-encoded sitemap path', () => { + const res = proxy(new NextRequest('https://docs.prose.md/sitemap%2Exml')); + expect(res.status).toBe(404); + }); + + it('rejects encoded paths before the legacy host redirect', () => { + const headers = { host: 'docs.openprose.ai' }; + const plain = proxy( + new NextRequest('https://docs.openprose.ai/robots.txt', { headers }), + ); + expect(plain.status).toBe(301); + + const encoded = proxy( + new NextRequest('https://docs.openprose.ai/robots%2Etxt', { headers }), + ); + expect(encoded.status).toBe(404); + }); + + it('passes /robots.txt through untouched', () => { + const res = proxy(new NextRequest('https://docs.prose.md/robots.txt')); + expect(res.headers.get('x-middleware-next')).toBe('1'); + }); + + it('passes /sitemap.xml through untouched', () => { + const res = proxy(new NextRequest('https://docs.prose.md/sitemap.xml')); + expect(res.headers.get('x-middleware-next')).toBe('1'); + }); +}); diff --git a/app/[[...slug]]/page.tsx b/app/[[...slug]]/page.tsx index 8493ad7..67c110b 100644 --- a/app/[[...slug]]/page.tsx +++ b/app/[[...slug]]/page.tsx @@ -70,6 +70,14 @@ function BrandedTitle({ title }: { title: string }) { ); } +// Every docs page is known at build time, so an unknown slug 404s without +// rendering or writing a cache entry. Otherwise every path a scanner tries is +// rendered and persisted to disk, and because Next decodes the slug before +// keying the cache, an encoded probe such as /robots%2Etxt can overwrite the +// /robots.txt route's entry ("app-route received invalid cache entry +// APP_PAGE"). proxy.ts also turns encoded paths away before routing. +export const dynamicParams = false; + export async function generateStaticParams() { return source.generateParams(); } diff --git a/package.json b/package.json index cbf5891..dc29cec 100644 --- a/package.json +++ b/package.json @@ -23,7 +23,8 @@ "pretest": "fumadocs-mdx", "test": "vitest run", "check:emdash": "scripts/check-em-dash-in-content.sh", - "check:links": "tsx scripts/check-links.ts" + "check:links": "tsx scripts/check-links.ts", + "smoke:standalone": "tsx scripts/smoke-standalone.ts" }, "dependencies": { "@shikijs/transformers": "^4.0.2", diff --git a/proxy.ts b/proxy.ts index 72c986d..d0ab112 100644 --- a/proxy.ts +++ b/proxy.ts @@ -22,7 +22,26 @@ export function resolveDocsHostRedirect( return `https://docs.prose.md${pathWithSearch}`; } +// No docs route has a percent-encoded path: slugs are plain ASCII and search +// terms travel in the query string. Next decodes a dynamic route's params +// before keying its cache, so an encoded alias such as /robots%2Etxt misses +// the static /robots.txt route, reaches the catch-all page, and is looked up +// under the robots route's cache key. Rejecting encoded paths here keeps them +// away from routing and the cache entirely. +export function hasEncodedPathname(pathname: string): boolean { + try { + return decodeURIComponent(pathname) !== pathname; + } catch { + // A malformed escape sequence cannot be a docs path either. + return true; + } +} + export default function proxy(request: NextRequest) { + if (hasEncodedPathname(request.nextUrl.pathname)) { + return new NextResponse(null, { status: 404 }); + } + const hostRedirect = resolveDocsHostRedirect( request.headers.get('host') ?? '', `${request.nextUrl.pathname}${request.nextUrl.search}`, diff --git a/scripts/smoke-standalone.ts b/scripts/smoke-standalone.ts new file mode 100644 index 0000000..3589cea --- /dev/null +++ b/scripts/smoke-standalone.ts @@ -0,0 +1,432 @@ +#!/usr/bin/env tsx +/** + * Boots the production standalone server exactly the way the Dockerfile's run + * stage does and checks it over real HTTP. This is the only check in the repo + * that exercises routing, the proxy, and Next's incremental cache together, + * which is where the /robots.txt 500 lived: a percent-encoded probe such as + * GET /robots%2Etxt fell through to the [[...slug]] catch-all, which decoded + * the slug and wrote its 404 render into the robots route's cache slot. Every + * later GET /robots.txt then threw "app-route received invalid cache entry". + * + * Two guards are in place, and the checks map onto them: + * - proxy.ts answers any percent-encoded path with 404 before routing. That + * is what makes both probes of /robots%2Etxt return 404 and keeps them out + * of the robots route's cache slot (the robots.txt.* and .meta checks). + * - `dynamicParams = false` on the catch-all makes unknown slugs 404 without + * rendering, so nothing is written to disk for them (the unknown-page + * check). It also stops the write for an encoded path that reaches the + * catch-all, but on its own such a probe still answers 500, because Next + * either retries the request until it gives up or reads the robots entry + * back from the cache. + * + * Run after `pnpm exec next build`: + * pnpm smoke:standalone --mode public (DOCS_PREVIEW_MODE=false build) + * pnpm smoke:standalone --mode preview (DOCS_PREVIEW_MODE=true build) + */ +import { spawn, type ChildProcess } from "node:child_process"; +import { cpSync, existsSync, readFileSync, readdirSync } from "node:fs"; +import { resolve } from "node:path"; +import { parseArgs } from "node:util"; + +type Mode = "public" | "preview"; + +const REPO_ROOT = process.cwd(); +const STANDALONE_DIR = resolve(REPO_ROOT, ".next/standalone"); +const SERVER_ENTRY = resolve(STANDALONE_DIR, "server.js"); +const APP_CACHE_DIR = resolve(STANDALONE_DIR, ".next/server/app"); +const CANONICAL_SITEMAP = "https://docs.prose.md/sitemap.xml"; +const BOOT_TIMEOUT_MS = 30_000; +const REQUEST_TIMEOUT_MS = 15_000; +const SHUTDOWN_TIMEOUT_MS = 5_000; + +// A path the catch-all does not know. Rendering it used to persist +// .html/.rsc/.meta/.segments under the app cache directory. +const UNKNOWN_PAGE = "smoke-nonexistent-page"; + +// The percent-encoded probe. Without the proxy guard it misses the static +// /robots.txt route, matches the catch-all, and decodes to the robots +// route's cache key. +const ROBOTS_PROBE = "/robots%2Etxt"; + +// What the robots route logs when it reads a page entry from its cache slot. +const ROBOTS_INVARIANT = "app-route received invalid cache entry"; + +function parseCli(): { mode: Mode; port: number } { + const { values } = parseArgs({ + options: { + mode: { type: "string" }, + port: { type: "string", default: "3100" }, + }, + strict: true, + }); + if (values.mode !== "public" && values.mode !== "preview") { + console.error("Usage: smoke-standalone --mode public|preview [--port 3100]"); + process.exit(2); + } + const port = Number.parseInt(values.port ?? "3100", 10); + if (!Number.isInteger(port) || port <= 0 || port > 65535) { + console.error(`Invalid --port: ${values.port}`); + process.exit(2); + } + return { mode: values.mode, port }; +} + +// --------------------------------------------------------------------------- +// Assertions: plain functions feeding one collected failure list. +// --------------------------------------------------------------------------- + +interface HttpResult { + status: number; + contentType: string; + body: string; +} + +const failures: string[] = []; +let passed = 0; + +function pass(name: string): void { + passed += 1; + console.log(`PASS ${name}`); +} + +function fail(name: string, detail: string): void { + failures.push(`${name}: ${detail}`); + console.log(`FAIL ${name}\n ${detail.split("\n").join("\n ")}`); +} + +function describeResponse(res: HttpResult): string { + const head = res.body.slice(0, 200).replace(/\s+/g, " ").trim(); + return `status ${res.status}, content-type "${res.contentType}", body: ${JSON.stringify(head)}`; +} + +function check(name: string, ok: boolean, detail: string): void { + if (ok) pass(name); + else fail(name, detail); +} + +async function get( + baseUrl: string, + path: string, + headers: Record = {}, +): Promise { + const res = await fetch(`${baseUrl}${path}`, { + headers, + redirect: "manual", + signal: AbortSignal.timeout(REQUEST_TIMEOUT_MS), + }); + return { + status: res.status, + contentType: res.headers.get("content-type") ?? "", + body: await res.text(), + }; +} + +/** Fetches a path and asserts its status (and optional content-type prefix). */ +async function expectStatus( + baseUrl: string, + path: string, + status: number, + options: { contentType?: string; headers?: Record } = {}, +): Promise { + const label = options.headers?.Accept + ? `GET ${path} (Accept: ${options.headers.Accept})` + : `GET ${path}`; + const expectation = options.contentType + ? `${status} ${options.contentType}` + : `${status}`; + let res: HttpResult; + try { + res = await get(baseUrl, path, options.headers); + } catch (error) { + fail(`${label} -> ${expectation}`, `request failed: ${String(error)}`); + return null; + } + const ok = + res.status === status && + (!options.contentType || res.contentType.startsWith(options.contentType)); + check(`${label} -> ${expectation}`, ok, describeResponse(res)); + return res; +} + +function checkRobotsBody(label: string, mode: Mode, res: HttpResult): void { + if (mode === "public") { + check( + `${label} allows Googlebot`, + /^User-Agent: Googlebot\r?\nAllow: \/\r?$/m.test(res.body), + `expected a "User-Agent: Googlebot" line followed by "Allow: /"; ${describeResponse(res)}`, + ); + check( + `${label} advertises ${CANONICAL_SITEMAP}`, + res.body.includes(`Sitemap: ${CANONICAL_SITEMAP}`), + `expected "Sitemap: ${CANONICAL_SITEMAP}"; ${describeResponse(res)}`, + ); + } else { + check( + `${label} disallows every crawler`, + /^User-Agent: \*\r?\nDisallow: \/\r?$/m.test(res.body), + `expected "User-Agent: *" followed by "Disallow: /"; ${describeResponse(res)}`, + ); + } +} + +/** Cache files written for a key: `.body`, `.html`, `.segments`... */ +function listCacheFiles(key: string): string[] { + return readdirSync(APP_CACHE_DIR) + .filter((name) => name.startsWith(`${key}.`)) + .sort(); +} + +// --------------------------------------------------------------------------- +// Standalone assembly and server lifecycle. +// --------------------------------------------------------------------------- + +/** + * `output: "standalone"` leaves out static assets and public/. The Dockerfile + * copies them in; do the same so the smoke run serves what production serves. + */ +function assembleStandalone(): void { + if (!existsSync(SERVER_ENTRY)) { + console.error( + `Missing ${SERVER_ENTRY}. Run \`pnpm exec next build\` before the smoke check.`, + ); + process.exit(2); + } + cpSync( + resolve(REPO_ROOT, ".next/static"), + resolve(STANDALONE_DIR, ".next/static"), + { recursive: true }, + ); + cpSync(resolve(REPO_ROOT, "public"), resolve(STANDALONE_DIR, "public"), { + recursive: true, + }); +} + +async function isPortAnswering(baseUrl: string): Promise { + try { + await fetch(baseUrl, { signal: AbortSignal.timeout(1_000) }); + return true; + } catch { + return false; + } +} + +interface Server { + child: ChildProcess; + output: () => string; + exited: Promise; +} + +function startServer(port: number): Server { + // Mirror the run stage's environment. DOCS_PREVIEW_MODE is dropped on + // purpose: the image never carries it, and the robots body must come from + // the build, not from whatever the shell running this script has set. + const env: NodeJS.ProcessEnv = { + ...process.env, + NODE_ENV: "production", + NEXT_TELEMETRY_DISABLED: "1", + PORT: String(port), + HOSTNAME: "127.0.0.1", + }; + delete env.DOCS_PREVIEW_MODE; + + const child = spawn(process.execPath, [SERVER_ENTRY], { + cwd: STANDALONE_DIR, + env, + stdio: ["ignore", "pipe", "pipe"], + }); + let output = ""; + child.stdout?.on("data", (chunk: Buffer) => (output += chunk.toString())); + child.stderr?.on("data", (chunk: Buffer) => (output += chunk.toString())); + const exited = new Promise((resolveExit) => { + child.once("exit", (code) => resolveExit(code)); + }); + return { child, output: () => output, exited }; +} + +async function stopServer(server: Server): Promise { + if (server.child.exitCode !== null || server.child.signalCode !== null) { + return; + } + server.child.kill("SIGTERM"); + const timedOut = await Promise.race([ + server.exited.then(() => false), + new Promise((r) => setTimeout(() => r(true), SHUTDOWN_TIMEOUT_MS)), + ]); + if (timedOut) { + server.child.kill("SIGKILL"); + await server.exited; + } +} + +async function waitForReady(baseUrl: string, server: Server): Promise { + const deadline = Date.now() + BOOT_TIMEOUT_MS; + let lastError = "no response yet"; + while (Date.now() < deadline) { + if (server.child.exitCode !== null) { + throw new Error( + `server exited with code ${server.child.exitCode} before answering`, + ); + } + try { + const res = await fetch(`${baseUrl}/`, { + signal: AbortSignal.timeout(2_000), + }); + await res.arrayBuffer(); + if (res.status === 200) return; + lastError = `GET / returned ${res.status}`; + } catch (error) { + lastError = String(error); + } + await new Promise((r) => setTimeout(r, 250)); + } + throw new Error( + `server did not answer GET / with 200 within ${BOOT_TIMEOUT_MS}ms (${lastError})`, + ); +} + +// --------------------------------------------------------------------------- +// The smoke sequence. +// --------------------------------------------------------------------------- + +interface BuildArtifacts { + robotsBody: string; + robotsMeta: string; +} + +/** Nothing may have been written into the robots route's cache slot. */ +function checkRobotsSlot(when: string, build: BuildArtifacts): void { + const robotsFiles = listCacheFiles("robots.txt"); + check( + `cache dir ${when}: robots.txt.* is exactly {body, meta}`, + JSON.stringify(robotsFiles) === + JSON.stringify(["robots.txt.body", "robots.txt.meta"]), + `found ${JSON.stringify(robotsFiles)} in ${APP_CACHE_DIR}`, + ); + const robotsMeta = readFileSync(resolve(APP_CACHE_DIR, "robots.txt.meta"), "utf-8"); + check( + `cache dir ${when}: robots.txt.meta is unchanged since the build`, + robotsMeta === build.robotsMeta, + `build: ${build.robotsMeta}\nnow: ${robotsMeta}`, + ); +} + +/** /robots.txt answers 200 text/plain with the body the build produced. */ +async function checkRobots( + baseUrl: string, + when: string, + mode: Mode, + build: BuildArtifacts, +): Promise { + const res = await expectStatus(baseUrl, "/robots.txt", 200, { + contentType: "text/plain", + }); + if (!res) return; + checkRobotsBody(`/robots.txt ${when}`, mode, res); + check( + `/robots.txt ${when} serves the build's robots.txt.body`, + res.body === build.robotsBody, + `build: ${JSON.stringify(build.robotsBody.slice(0, 200))}\nnow: ${JSON.stringify(res.body.slice(0, 200))}`, + ); +} + +async function runChecks( + baseUrl: string, + mode: Mode, + server: Server, + build: BuildArtifacts, +): Promise { + // 1. Cold probe, before anything has read /robots.txt. The robots entry is + // not in the in-memory cache yet, so a catch-all lookup under the + // decoded key would miss. This is the ordering that poisoned + // production: the catch-all rendered its 404 and stored it as + // /robots.txt. The proxy guard must answer it with 404 first. + await expectStatus(baseUrl, ROBOTS_PROBE, 404); + checkRobotsSlot("after the cold probe", build); + await checkRobots(baseUrl, "after the cold probe", mode, build); + + // 2. The rest of the public surface still works. + await expectStatus(baseUrl, "/sitemap.xml", 200); + await expectStatus(baseUrl, "/setup", 200, { contentType: "text/html" }); + await expectStatus(baseUrl, "/llms.mdx/setup/content.md", 200); + // Markdown negotiation must still rewrite the docs root. + await expectStatus(baseUrl, "/", 200, { + contentType: "text/markdown", + headers: { Accept: "text/markdown" }, + }); + + // 3. Unknown paths 404 without writing anything to disk. This one reaches + // the catch-all, so it is `dynamicParams = false` that keeps it off disk. + await expectStatus(baseUrl, `/${UNKNOWN_PAGE}`, 404); + const unknownFiles = listCacheFiles(UNKNOWN_PAGE); + check( + `cache dir: no ${UNKNOWN_PAGE}.* entry was written`, + unknownFiles.length === 0, + `found ${JSON.stringify(unknownFiles)} in ${APP_CACHE_DIR}`, + ); + + // 4. Warm replay: the same probe now that /robots.txt sits in the + // in-memory cache, then the robots route again. Past the proxy, this + // ordering would read the robots entry back as a page and fail. + await expectStatus(baseUrl, ROBOTS_PROBE, 404); + checkRobotsSlot("after the warm probe", build); + await checkRobots(baseUrl, "after the warm probe", mode, build); + + const invariantLines = server + .output() + .split("\n") + .filter((line) => line.includes(ROBOTS_INVARIANT)); + check( + `server output never reports "${ROBOTS_INVARIANT}"`, + invariantLines.length === 0, + invariantLines.join("\n"), + ); +} + +async function main(): Promise { + const { mode, port } = parseCli(); + const baseUrl = `http://127.0.0.1:${port}`; + + assembleStandalone(); + const build: BuildArtifacts = { + robotsBody: readFileSync(resolve(APP_CACHE_DIR, "robots.txt.body"), "utf-8"), + robotsMeta: readFileSync(resolve(APP_CACHE_DIR, "robots.txt.meta"), "utf-8"), + }; + + if (await isPortAnswering(baseUrl)) { + console.error(`Something is already listening on ${baseUrl}; pass --port.`); + process.exit(2); + } + + console.log(`Smoke-testing the ${mode} standalone build at ${baseUrl}\n`); + const server = startServer(port); + + const onSignal = (signal: NodeJS.Signals) => { + void stopServer(server).finally(() => { + process.exit(signal === "SIGINT" ? 130 : 143); + }); + }; + process.once("SIGINT", onSignal); + process.once("SIGTERM", onSignal); + + try { + await waitForReady(baseUrl, server); + await runChecks(baseUrl, mode, server, build); + } catch (error) { + fail("smoke run", String(error)); + } finally { + await stopServer(server); + } + + if (failures.length > 0) { + console.log(`\n--- server output ---\n${server.output().trimEnd()}\n---------------------`); + console.error(`\n${failures.length} check(s) failed, ${passed} passed.`); + process.exitCode = 1; + return; + } + console.log(`\nAll ${passed} checks passed for the ${mode} build.`); +} + +main().catch((error: unknown) => { + console.error(error); + process.exitCode = 1; +}); From 08287e719766fd47d26d1bd8285b90128907be68 Mon Sep 17 00:00:00 2001 From: Jose Montes de Oca Date: Thu, 17 Sep 2026 21:31:33 -0400 Subject: [PATCH 2/3] ci: smoke both docs build modes over HTTP on every PR Verify built preview mode once and never made a request, so a broken robots route in the public build that production ships could pass CI behind a healthy build step. The job now builds public mode and runs the standalone smoke check against it, then does the same for preview mode. That checks both what crawlers may do and what they may not on every pull request. The timeout grows to 20 minutes to cover the second build. --- .github/workflows/verify.yml | 16 ++++++++++++++-- 1 file changed, 14 insertions(+), 2 deletions(-) diff --git a/.github/workflows/verify.yml b/.github/workflows/verify.yml index 8e4c955..b196925 100644 --- a/.github/workflows/verify.yml +++ b/.github/workflows/verify.yml @@ -11,7 +11,7 @@ permissions: jobs: verify: runs-on: ubuntu-latest - timeout-minutes: 15 + timeout-minutes: 20 steps: - uses: actions/checkout@v4 - uses: pnpm/action-setup@b906affcce14559ad1aafd4ab0e942779e9f58b1 # v4.3.0 @@ -31,6 +31,18 @@ jobs: - run: pnpm check:emdash - run: pnpm check:links - run: pnpm test - - run: pnpm exec next build + # Build the standalone server in both modes and check each one over + # HTTP. Public mode is what production ships, so it goes first. `pnpm + # exec` skips the prebuild hook, matching the Dockerfile. + - name: Build public mode + run: pnpm exec next build + env: + DOCS_PREVIEW_MODE: "false" + - name: Smoke public build over HTTP + run: pnpm smoke:standalone --mode public + - name: Build preview mode + run: pnpm exec next build env: DOCS_PREVIEW_MODE: "true" + - name: Smoke preview build over HTTP + run: pnpm smoke:standalone --mode preview From 1603e8ac8fb08bfd5c962875fa433d8de8c022ae Mon Sep 17 00:00:00 2001 From: Jose Montes de Oca Date: Thu, 17 Sep 2026 21:42:22 -0400 Subject: [PATCH 3/3] ci: check the live robots.txt after each docs deploy The Fly health check only probes /, so the deploy stayed green while robots.txt returned 500 and Google stopped crawling the site. A new check-live-robots.sh fetches /robots.txt as Googlebot, retrying while a stopped machine starts, and fails unless it gets a 200 text/plain response that allows Googlebot and advertises the canonical sitemap. The deploy workflow runs it against docs.prose.md after flyctl deploy, so a broken robots route turns the deploy run red instead of going unnoticed. Run against production before this fix, it fails with 500 on every attempt. --- .github/workflows/deploy-docs.yml | 6 ++ scripts/check-live-robots.sh | 121 ++++++++++++++++++++++++++++++ 2 files changed, 127 insertions(+) create mode 100755 scripts/check-live-robots.sh diff --git a/.github/workflows/deploy-docs.yml b/.github/workflows/deploy-docs.yml index a7b700c..dea5e7d 100644 --- a/.github/workflows/deploy-docs.yml +++ b/.github/workflows/deploy-docs.yml @@ -29,3 +29,9 @@ jobs: --build-arg NEXT_PUBLIC_POSTHOG_KEY=${{ secrets.NEXT_PUBLIC_POSTHOG_KEY }} env: FLY_API_TOKEN: ${{ secrets.FLY_API_TOKEN }} + + # The Fly health check only probes /, so it stays green when a route + # handler breaks. Check the route crawlers need on the live site. A + # failure does not roll back; it marks this run failed. + - name: Verify live robots.txt + run: scripts/check-live-robots.sh https://docs.prose.md diff --git a/scripts/check-live-robots.sh b/scripts/check-live-robots.sh new file mode 100755 index 0000000..1c5d1f4 --- /dev/null +++ b/scripts/check-live-robots.sh @@ -0,0 +1,121 @@ +#!/usr/bin/env bash +set -euo pipefail + +# Checks a running docs site's /robots.txt the way a crawler fetches it: +# HTTP 200, text/plain, a Googlebot allow rule, and the canonical sitemap. +# A 5xx on this route makes Google stop crawling the whole site, and the Fly +# health check only probes /, so a healthy homepage can hide it. The body +# checks mirror the public-mode checks in scripts/smoke-standalone.ts. +# +# Usage: scripts/check-live-robots.sh [BASE_URL] +# BASE_URL defaults to https://docs.prose.md. Point it at a local +# standalone server (http://127.0.0.1:3100) to try it against a build. + +if [[ $# -gt 1 ]]; then + echo "Usage: $0 [BASE_URL]" >&2 + exit 2 +fi + +BASE_URL="${1:-https://docs.prose.md}" +URL="${BASE_URL%/}/robots.txt" +# app/robots.ts always advertises the canonical host, whatever host serves it. +SITEMAP_LINE="Sitemap: https://docs.prose.md/sitemap.xml" +# The machine may be stopped and take a few seconds to start, so retry until +# the route answers 200: at most 10 requests, 6 seconds apart. +ATTEMPTS=10 +RETRY_DELAY_SECONDS=6 + +tmp_dir=$(mktemp -d) +trap 'rm -rf "$tmp_dir"' EXIT +body="$tmp_dir/body" +headers="$tmp_dir/headers" + +print_body_head() { + if [[ -s "$body" ]]; then + echo " body (first 200 bytes):" + head -c 200 "$body" | awk '{ print " " $0 }' + fi +} + +attempt=0 +while true; do + attempt=$((attempt + 1)) + rm -f "$body" "$headers" + curl_exit=0 + # With no HTTP response at all, curl prints 000 and exits non-zero. + status=$(curl -sS -A "Googlebot" --connect-timeout 10 --max-time 20 \ + -o "$body" -D "$headers" -w '%{http_code}' "$URL") || curl_exit=$? + if [[ "$curl_exit" -eq 0 && "$status" == "200" ]]; then + break + fi + + result="HTTP ${status:-000}" + if [[ "$curl_exit" -ne 0 ]]; then + result="$result (curl exit $curl_exit)" + fi + if [[ "$attempt" -ge "$ATTEMPTS" ]]; then + echo "FAIL GET $URL returned $result after $attempt attempt(s)" + print_body_head + exit 1 + fi + echo "Attempt $attempt of $ATTEMPTS: $result, retrying in ${RETRY_DELAY_SECONDS}s" + sleep "$RETRY_DELAY_SECONDS" +done +echo "PASS GET $URL -> 200 (attempt $attempt of $ATTEMPTS)" + +# Last Content-Type header, without the name or a trailing CR. +content_type=$(awk ' + tolower($0) ~ /^content-type:/ { + sub(/\r$/, ""); sub(/^[^:]*:[ \t]*/, ""); value = $0 + } + END { print value } +' "$headers") + +is_text_plain() { + local lowered + lowered=$(printf '%s' "$content_type" | tr '[:upper:]' '[:lower:]') + [[ "$lowered" == text/plain* ]] +} + +# True when the body has a line equal to $1. +has_line() { + awk -v want="$1" ' + { sub(/\r$/, "") } + $0 == want { found = 1 } + END { exit found ? 0 : 1 } + ' "$body" +} + +# True when the body has a line equal to $1 directly followed by one equal to $2. +has_line_pair() { + awk -v first="$1" -v second="$2" ' + { sub(/\r$/, "") } + previous == first && $0 == second { found = 1 } + { previous = $0 } + END { exit found ? 0 : 1 } + ' "$body" +} + +failures=0 +check() { + local name=$1 + shift + if "$@"; then + echo "PASS $name" + else + echo "FAIL $name" + failures=$((failures + 1)) + fi +} + +check "content-type is text/plain (got \"$content_type\")" is_text_plain +check "\"User-Agent: Googlebot\" is followed by \"Allow: /\"" \ + has_line_pair "User-Agent: Googlebot" "Allow: /" +check "advertises \"$SITEMAP_LINE\"" has_line "$SITEMAP_LINE" + +if [[ "$failures" -gt 0 ]]; then + echo "$failures check(s) failed for $URL" + print_body_head + exit 1 +fi +echo "robots.txt OK after $attempt attempt(s)"