fix: stop encoded probes from poisoning the robots.txt cache

.github/workflows/deploy-docs.yml

@@ -29,3 +29,9 @@ jobs:
             --build-arg NEXT_PUBLIC_POSTHOG_KEY=${{ secrets.NEXT_PUBLIC_POSTHOG_KEY }}
         env:
           FLY_API_TOKEN: ${{ secrets.FLY_API_TOKEN }}
+
+      # The Fly health check only probes /, so it stays green when a route
+      # handler breaks. Check the route crawlers need on the live site. A
+      # failure does not roll back; it marks this run failed.
+      - name: Verify live robots.txt
+        run: scripts/check-live-robots.sh https://docs.prose.md

.github/workflows/verify.yml

@@ -11,7 +11,7 @@ permissions:
 jobs:
   verify:
     runs-on: ubuntu-latest
-    timeout-minutes: 15
+    timeout-minutes: 20
     steps:
       - uses: actions/checkout@v4
       - uses: pnpm/action-setup@b906affcce14559ad1aafd4ab0e942779e9f58b1 # v4.3.0
@@ -31,6 +31,18 @@ jobs:
       - run: pnpm check:emdash
       - run: pnpm check:links
       - run: pnpm test
-      - run: pnpm exec next build
+      # Build the standalone server in both modes and check each one over
+      # HTTP. Public mode is what production ships, so it goes first. `pnpm
+      # exec` skips the prebuild hook, matching the Dockerfile.
+      - name: Build public mode
+        run: pnpm exec next build
+        env:
+          DOCS_PREVIEW_MODE: "false"
+      - name: Smoke public build over HTTP
+        run: pnpm smoke:standalone --mode public
+      - name: Build preview mode
+        run: pnpm exec next build
         env:
           DOCS_PREVIEW_MODE: "true"
+      - name: Smoke preview build over HTTP
+        run: pnpm smoke:standalone --mode preview

__tests__/proxy.test.ts

@@ -1,5 +1,9 @@
+// @vitest-environment node
+// The proxy runs on the server. happy-dom's Request drops the Host header,
+// which the legacy-host redirect depends on.
 import { describe, expect, it } from 'vitest';
-import { resolveDocsHostRedirect } from '../proxy';
+import { NextRequest } from 'next/server';
+import proxy, { hasEncodedPathname, resolveDocsHostRedirect } from '../proxy';
 
 describe('resolveDocsHostRedirect', () => {
   it('redirects the legacy docs host to docs.prose.md', () => {
@@ -18,3 +22,57 @@ describe('resolveDocsHostRedirect', () => {
     ).toBeNull();
   });
 });
+
+describe('hasEncodedPathname', () => {
+  it('accepts plain paths', () => {
+    expect(hasEncodedPathname('/robots.txt')).toBe(false);
+    expect(hasEncodedPathname('/sitemap.xml')).toBe(false);
+    expect(hasEncodedPathname('/setup')).toBe(false);
+    expect(hasEncodedPathname('/')).toBe(false);
+  });
+
+  it('flags percent-encoded aliases of route handlers', () => {
+    expect(hasEncodedPathname('/robots%2Etxt')).toBe(true);
+    expect(hasEncodedPathname('/sitemap%2Exml')).toBe(true);
+  });
+
+  it('flags malformed escape sequences', () => {
+    expect(hasEncodedPathname('/%E0%A4%A')).toBe(true);
+  });
+});
+
+describe('proxy', () => {
+  it('returns 404 for a percent-encoded robots path', () => {
+    const res = proxy(new NextRequest('https://docs.prose.md/robots%2Etxt'));
+    expect(res.status).toBe(404);
+    expect(res.headers.get('x-middleware-next')).toBeNull();
+  });
+
+  it('returns 404 for a percent-encoded sitemap path', () => {
+    const res = proxy(new NextRequest('https://docs.prose.md/sitemap%2Exml'));
+    expect(res.status).toBe(404);
+  });
+
+  it('rejects encoded paths before the legacy host redirect', () => {
+    const headers = { host: 'docs.openprose.ai' };
+    const plain = proxy(
+      new NextRequest('https://docs.openprose.ai/robots.txt', { headers }),
+    );
+    expect(plain.status).toBe(301);
+
+    const encoded = proxy(
+      new NextRequest('https://docs.openprose.ai/robots%2Etxt', { headers }),
+    );
+    expect(encoded.status).toBe(404);
+  });
+
+  it('passes /robots.txt through untouched', () => {
+    const res = proxy(new NextRequest('https://docs.prose.md/robots.txt'));
+    expect(res.headers.get('x-middleware-next')).toBe('1');
+  });
+
+  it('passes /sitemap.xml through untouched', () => {
+    const res = proxy(new NextRequest('https://docs.prose.md/sitemap.xml'));
+    expect(res.headers.get('x-middleware-next')).toBe('1');
+  });
+});

app/[[...slug]]/page.tsx

@@ -70,6 +70,14 @@ function BrandedTitle({ title }: { title: string }) {
   );
 }
 
+// Every docs page is known at build time, so an unknown slug 404s without
+// rendering or writing a cache entry. Otherwise every path a scanner tries is
+// rendered and persisted to disk, and because Next decodes the slug before
+// keying the cache, an encoded probe such as /robots%2Etxt can overwrite the
+// /robots.txt route's entry ("app-route received invalid cache entry
+// APP_PAGE"). proxy.ts also turns encoded paths away before routing.
+export const dynamicParams = false;
+
 export async function generateStaticParams() {
   return source.generateParams();
 }

package.json

@@ -23,7 +23,8 @@
     "pretest": "fumadocs-mdx",
     "test": "vitest run",
     "check:emdash": "scripts/check-em-dash-in-content.sh",
-    "check:links": "tsx scripts/check-links.ts"
+    "check:links": "tsx scripts/check-links.ts",
+    "smoke:standalone": "tsx scripts/smoke-standalone.ts"
   },
   "dependencies": {
     "@shikijs/transformers": "^4.0.2",

proxy.ts

@@ -22,7 +22,26 @@ export function resolveDocsHostRedirect(
   return `https://docs.prose.md${pathWithSearch}`;
 }
 
+// No docs route has a percent-encoded path: slugs are plain ASCII and search
+// terms travel in the query string. Next decodes a dynamic route's params
+// before keying its cache, so an encoded alias such as /robots%2Etxt misses
+// the static /robots.txt route, reaches the catch-all page, and is looked up
+// under the robots route's cache key. Rejecting encoded paths here keeps them
+// away from routing and the cache entirely.
+export function hasEncodedPathname(pathname: string): boolean {
+  try {
+    return decodeURIComponent(pathname) !== pathname;
+  } catch {
+    // A malformed escape sequence cannot be a docs path either.
+    return true;
+  }
+}
+
 export default function proxy(request: NextRequest) {
+  if (hasEncodedPathname(request.nextUrl.pathname)) {
+    return new NextResponse(null, { status: 404 });
+  }
+
   const hostRedirect = resolveDocsHostRedirect(
     request.headers.get('host') ?? '',
     `${request.nextUrl.pathname}${request.nextUrl.search}`,

scripts/check-live-robots.sh

@@ -0,0 +1,121 @@
+#!/usr/bin/env bash
+set -euo pipefail
+
+# Checks a running docs site's /robots.txt the way a crawler fetches it:
+# HTTP 200, text/plain, a Googlebot allow rule, and the canonical sitemap.
+# A 5xx on this route makes Google stop crawling the whole site, and the Fly
+# health check only probes /, so a healthy homepage can hide it. The body
+# checks mirror the public-mode checks in scripts/smoke-standalone.ts.
+#
+# Usage: scripts/check-live-robots.sh [BASE_URL]
+#   BASE_URL defaults to https://docs.prose.md. Point it at a local
+#   standalone server (http://127.0.0.1:3100) to try it against a build.
+
+if [[ $# -gt 1 ]]; then
+  echo "Usage: $0 [BASE_URL]" >&2
+  exit 2
+fi
+
+BASE_URL="${1:-https://docs.prose.md}"
+URL="${BASE_URL%/}/robots.txt"
+# app/robots.ts always advertises the canonical host, whatever host serves it.
+SITEMAP_LINE="Sitemap: https://docs.prose.md/sitemap.xml"
+# The machine may be stopped and take a few seconds to start, so retry until
+# the route answers 200: at most 10 requests, 6 seconds apart.
+ATTEMPTS=10
+RETRY_DELAY_SECONDS=6
+
+tmp_dir=$(mktemp -d)
+trap 'rm -rf "$tmp_dir"' EXIT
+body="$tmp_dir/body"
+headers="$tmp_dir/headers"
+
+print_body_head() {
+  if [[ -s "$body" ]]; then
+    echo "      body (first 200 bytes):"
+    head -c 200 "$body" | awk '{ print "        " $0 }'
+  fi
+}
+
+attempt=0
+while true; do
+  attempt=$((attempt + 1))
+  rm -f "$body" "$headers"
+  curl_exit=0
+  # With no HTTP response at all, curl prints 000 and exits non-zero.
+  status=$(curl -sS -A "Googlebot" --connect-timeout 10 --max-time 20 \
+    -o "$body" -D "$headers" -w '%{http_code}' "$URL") || curl_exit=$?
+  if [[ "$curl_exit" -eq 0 && "$status" == "200" ]]; then
+    break
+  fi
+
+  result="HTTP ${status:-000}"
+  if [[ "$curl_exit" -ne 0 ]]; then
+    result="$result (curl exit $curl_exit)"
+  fi
+  if [[ "$attempt" -ge "$ATTEMPTS" ]]; then
+    echo "FAIL  GET $URL returned $result after $attempt attempt(s)"
+    print_body_head
+    exit 1
+  fi
+  echo "Attempt $attempt of $ATTEMPTS: $result, retrying in ${RETRY_DELAY_SECONDS}s"
+  sleep "$RETRY_DELAY_SECONDS"
+done
+echo "PASS  GET $URL -> 200 (attempt $attempt of $ATTEMPTS)"
+
+# Last Content-Type header, without the name or a trailing CR.
+content_type=$(awk '
+  tolower($0) ~ /^content-type:/ {
+    sub(/\r$/, ""); sub(/^[^:]*:[ \t]*/, ""); value = $0
+  }
+  END { print value }
+' "$headers")
+
+is_text_plain() {
+  local lowered
+  lowered=$(printf '%s' "$content_type" | tr '[:upper:]' '[:lower:]')
+  [[ "$lowered" == text/plain* ]]
+}
+
+# True when the body has a line equal to $1.
+has_line() {
+  awk -v want="$1" '
+    { sub(/\r$/, "") }
+    $0 == want { found = 1 }
+    END { exit found ? 0 : 1 }
+  ' "$body"
+}
+
+# True when the body has a line equal to $1 directly followed by one equal to $2.
+has_line_pair() {
+  awk -v first="$1" -v second="$2" '
+    { sub(/\r$/, "") }
+    previous == first && $0 == second { found = 1 }
+    { previous = $0 }
+    END { exit found ? 0 : 1 }
+  ' "$body"
+}
+
+failures=0
+check() {
+  local name=$1
+  shift
+  if "$@"; then
+    echo "PASS  $name"
+  else
+    echo "FAIL  $name"
+    failures=$((failures + 1))
+  fi
+}
+
+check "content-type is text/plain (got \"$content_type\")" is_text_plain
+check "\"User-Agent: Googlebot\" is followed by \"Allow: /\"" \
+  has_line_pair "User-Agent: Googlebot" "Allow: /"
+check "advertises \"$SITEMAP_LINE\"" has_line "$SITEMAP_LINE"
+
+if [[ "$failures" -gt 0 ]]; then
+  echo "$failures check(s) failed for $URL"
+  print_body_head
+  exit 1
+fi
+echo "robots.txt OK after $attempt attempt(s)"