From 26fe62b6837c8718d0ae63aed14e5b57c085f3d6 Mon Sep 17 00:00:00 2001 From: Joe Date: Mon, 3 Aug 2026 14:56:48 -0400 Subject: [PATCH] feat(plugin): sitemap fetch can skip the staging pin (sitemap.useStagingIp); v0.25.0 MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit The sitemap refresh followed staging.ip unconditionally — the right edge differs per deployment, so make it a sitemap-scoped opt-out. Off, sitemap fetches go direct to the production origin; the security token is sent either way. The render/proxy staging passthrough is unaffected. Co-Authored-By: Claude Fable 5 --- package-lock.json | 2 +- packages/plugin/README.md | 6 ++++++ packages/plugin/package.json | 2 +- packages/plugin/src/config.js | 17 ++++++++++++----- packages/plugin/src/resources/Sitemap.js | 14 +++++++------- packages/plugin/src/util/upstream.js | 9 +++++++++ packages/plugin/test/upstream.test.js | 19 +++++++++++++++++++ 7 files changed, 55 insertions(+), 14 deletions(-) diff --git a/package-lock.json b/package-lock.json index 421f4b7..762071f 100644 --- a/package-lock.json +++ b/package-lock.json @@ -9007,7 +9007,7 @@ }, "packages/plugin": { "name": "@harperfast/prerender", - "version": "0.24.0", + "version": "0.25.0", "license": "Apache-2.0", "dependencies": { "fast-xml-parser": "^5.0.9", diff --git a/packages/plugin/README.md b/packages/plugin/README.md index cff5f9d..76480f0 100644 --- a/packages/plugin/README.md +++ b/packages/plugin/README.md @@ -107,6 +107,7 @@ rest: true # required for the @export-ed table REST endpoints filteredWarnPercent: 50 # filtered share of one sitemap that is reported as an ERROR node: '' # pin the scheduled refresh to this node ('' disables it) workerIndex: 0 # ...and this worker + useStagingIp: true # sitemap fetches follow staging.ip when set; false = direct to prod background: true # POST returns a handle immediately; the walk runs in the background staleRunMs: 600000 # 10m — un-updated progress after which a run is treated as dead removedSampleCap: 20 # sample size of unlinked keys in the result (the COUNT is exact) @@ -370,6 +371,11 @@ certificate (the server-side equivalent of a `host-resolver-rules` / `/etc/hosts is unaffected unless a staging IP is explicitly configured. - With the `debugHeader` also present, a staging-served response is tagged with the `x-harper-origin: staging` response header so you can confirm it. +- **Sitemap fetches follow `staging.ip` by default.** They have no incoming request to carry the + toggle header, so `sitemap.useStagingIp` is their equivalent: `true` (the default) pins every + sitemap fetch to the staging edge; `false` sends them direct to the production origin. The + security token is sent either way, so a direct fetch requires the token to be accepted on the + production property — otherwise every sitemap fetch is bounced with a 403. ### Database topology diff --git a/packages/plugin/package.json b/packages/plugin/package.json index 527cde2..554ff6c 100644 --- a/packages/plugin/package.json +++ b/packages/plugin/package.json @@ -1,6 +1,6 @@ { "name": "@harperfast/prerender", - "version": "0.24.0", + "version": "0.25.0", "type": "module", "description": "Configurable Harper plugin for prerendering pages for bots and crawlers", "license": "Apache-2.0", diff --git a/packages/plugin/src/config.js b/packages/plugin/src/config.js index 040d6b0..97e63b5 100644 --- a/packages/plugin/src/config.js +++ b/packages/plugin/src/config.js @@ -168,10 +168,9 @@ const defaultConfig = () => ({ // cached page regardless of it. Empty `ip` disables the feature — production is // unaffected unless a staging IP is explicitly configured. // - // The sitemap refresh reuses this `ip` too, but unconditionally (no toggle header — it has - // no incoming request): whenever `ip` is set, every sitemap fetch is pinned to it, so all - // Harper→origin traffic hits the same edge. The security token often only authenticates - // against the staging edge, so a direct prod sitemap fetch is bounced with a 403. + // The sitemap refresh reuses this `ip` too, gated by `sitemap.useStagingIp` instead of the + // toggle header (it has no incoming request to carry one): when that flag is on and `ip` is + // set, every sitemap fetch is pinned to it, so all Harper→origin traffic hits the same edge. staging: { ip: '', header: 'x-harper-staging', @@ -356,6 +355,14 @@ const defaultConfig = () => ({ node: '', workerIndex: 0, + // Whether sitemap fetches follow `staging.ip` when it is set. The sitemap fetch has no + // incoming request to carry the staging toggle header, so this flag is its equivalent. + // On (the default), sitemap XML is fetched via the staging edge like every other + // Harper→origin fetch. Off, sitemap fetches go direct to the production origin — the + // security token still rides along, so the edge's bot-mitigation bypass must accept it + // on the production property, not just staging, or every fetch 403s. + useStagingIp: true, + // Run `POST /Sitemap/` as a background walk and answer immediately with a handle, // instead of holding the request open for the whole traversal. A sitemap index is not an // HTTP-request-sized unit of work — a real one fans out to tens of children and over a @@ -659,7 +666,7 @@ export const collectConfigWarnings = () => { add( 'info', 'staging.ip', - `staging passthrough ENABLED — cache-miss requests carrying "${config.staging.header}" are proxied to ${config.staging.ip} (Host/SNI preserved). Toggling this on/off contaminates the URL-keyed page cache; wipe it when switching.` + `staging passthrough ENABLED — cache-miss requests carrying "${config.staging.header}" are proxied to ${config.staging.ip} (Host/SNI preserved); sitemap fetches go ${config.sitemap.useStagingIp ? `to ${config.staging.ip} too (sitemap.useStagingIp)` : 'direct to the production origin (sitemap.useStagingIp: false)'}. Toggling this on/off contaminates the URL-keyed page cache; wipe it when switching.` ); } else { add( diff --git a/packages/plugin/src/resources/Sitemap.js b/packages/plugin/src/resources/Sitemap.js index 1c21b50..78700f0 100644 --- a/packages/plugin/src/resources/Sitemap.js +++ b/packages/plugin/src/resources/Sitemap.js @@ -5,7 +5,7 @@ import { classifyUrl, PASSTHROUGH, PRERENDER, UNCLASSIFIED } from '../util/route import { currentMinuteMs, epochMsOf, getNextSitemapRefreshTime } from '../util/time.js'; import { parseSitemap, partitionSitemapEntries } from '../util/sitemap.js'; import { actionForExisting, canSkipLookup, createRefreshRun, TargetAction } from '../util/sitemapRun.js'; -import { configuredStagingIp, dispatcherFor } from '../util/upstream.js'; +import { dispatcherFor, sitemapStagingIp } from '../util/upstream.js'; import { setImmediate } from 'node:timers/promises'; import { applyInBatches, collectFromScan } from '../util/scan.js'; @@ -627,12 +627,12 @@ function getTtlFromChangeFreq(changefreq, { minTtl, defaultTtl }) { } async function fetchLatestSitemap(url) { - // Route every Harper→origin sitemap fetch through the same edge as the render/origin-fetch - // path: whenever a staging IP is configured, pin the TCP connection to it (Host/SNI stay the - // real origin, exactly like upstream.js). The security token typically only authenticates - // against the staging edge, so a direct prod fetch is bounced with a 403 "Access Denied". - // Empty staging.ip → normal direct fetch (production, once the token is valid at the origin). - const stagingIp = configuredStagingIp(); + // Pin the TCP connection to the staging edge when `sitemap.useStagingIp` + `staging.ip` + // say so (Host/SNI stay the real origin, exactly like upstream.js). The flag exists because + // the right edge differs per deployment: a token that only authenticates against the + // staging edge needs the pin, while a token accepted on the production property can fetch + // the real sitemap direct — `useStagingIp: false`. Either way the token is sent. + const stagingIp = sitemapStagingIp(); const via = stagingIp ? ` (via staging ${stagingIp})` : ''; const res = await fetch(url, { diff --git a/packages/plugin/src/util/upstream.js b/packages/plugin/src/util/upstream.js index eefe6bb..5df8645 100644 --- a/packages/plugin/src/util/upstream.js +++ b/packages/plugin/src/util/upstream.js @@ -28,6 +28,15 @@ export const configuredStagingIp = () => { return ip && isIP(ip) ? ip : undefined; }; +/** + * The staging IP a sitemap fetch should connect to, or undefined for a direct production + * fetch. The sitemap refresh has no incoming request to carry the staging toggle header, so + * `sitemap.useStagingIp` is its equivalent: on, sitemap fetches follow `staging.ip` like + * every other Harper→origin fetch; off, they go direct to the production origin (the + * security token is sent either way). + */ +export const sitemapStagingIp = () => (config.sitemap.useStagingIp ? configuredStagingIp() : undefined); + // Dispatchers that pin DNS resolution to a fixed IP (staging passthrough), one per IP. // Only the connect address is overridden — the origin (so Host header + TLS SNI + cert // validation) stays the real origin host, the server-side equivalent of Chrome's diff --git a/packages/plugin/test/upstream.test.js b/packages/plugin/test/upstream.test.js index 91ce5f1..f8808e4 100644 --- a/packages/plugin/test/upstream.test.js +++ b/packages/plugin/test/upstream.test.js @@ -5,6 +5,7 @@ import { configuredStagingIp, resolveUpstreamHeaders, sanitizeOriginResponseHeaders, + sitemapStagingIp, stagingTargetIp, } from '../src/util/upstream.js'; @@ -74,6 +75,24 @@ test('configuredStagingIp ignores an invalid configured ip', () => { assert.equal(configuredStagingIp(), undefined); }); +test('sitemapStagingIp follows staging.ip by default', () => { + applyOptions({ staging: { ip: '192.0.2.27' } }); + assert.equal(config.sitemap.useStagingIp, true); + assert.equal(sitemapStagingIp(), '192.0.2.27'); +}); + +test('sitemapStagingIp is undefined when sitemap.useStagingIp is off (direct prod fetch)', () => { + applyOptions({ staging: { ip: '192.0.2.27' }, sitemap: { useStagingIp: false } }); + // The render/proxy path still pins to staging; only the sitemap fetch goes direct. + assert.equal(configuredStagingIp(), '192.0.2.27'); + assert.equal(sitemapStagingIp(), undefined); +}); + +test('sitemapStagingIp is undefined when no staging ip is configured, regardless of the flag', () => { + applyOptions({ sitemap: { useStagingIp: true } }); + assert.equal(sitemapStagingIp(), undefined); +}); + test('ignoredHeaders defaults to an empty list', () => { applyOptions({}); assert.deepEqual(config.ignoredHeaders, []);