feat(plugin): sitemap fetch can skip the staging pin (sitemap.useStagingIp); v0.25.0

packages/plugin/README.md

@@ -107,6 +107,7 @@ rest: true # required for the @export-ed table REST endpoints
     filteredWarnPercent: 50 # filtered share of one sitemap that is reported as an ERROR
     node: '' # pin the scheduled refresh to this node ('' disables it)
     workerIndex: 0 # ...and this worker
+    useStagingIp: true # sitemap fetches follow staging.ip when set; false = direct to prod
     background: true # POST returns a handle immediately; the walk runs in the background
     staleRunMs: 600000 # 10m — un-updated progress after which a run is treated as dead
     removedSampleCap: 20 # sample size of unlinked keys in the result (the COUNT is exact)
@@ -370,6 +371,11 @@ certificate (the server-side equivalent of a `host-resolver-rules` / `/etc/hosts
   is unaffected unless a staging IP is explicitly configured.
 - With the `debugHeader` also present, a staging-served response is tagged with the
   `x-harper-origin: staging` response header so you can confirm it.
+- **Sitemap fetches follow `staging.ip` by default.** They have no incoming request to carry the
+  toggle header, so `sitemap.useStagingIp` is their equivalent: `true` (the default) pins every
+  sitemap fetch to the staging edge; `false` sends them direct to the production origin. The
+  security token is sent either way, so a direct fetch requires the token to be accepted on the
+  production property — otherwise every sitemap fetch is bounced with a 403.
 
 ### Database topology
 

packages/plugin/package.json

@@ -1,6 +1,6 @@
 {
 	"name": "@harperfast/prerender",
-	"version": "0.24.0",
+	"version": "0.25.0",
 	"type": "module",
 	"description": "Configurable Harper plugin for prerendering pages for bots and crawlers",
 	"license": "Apache-2.0",

packages/plugin/src/config.js

@@ -168,10 +168,9 @@ const defaultConfig = () => ({
 	// cached page regardless of it. Empty `ip` disables the feature — production is
 	// unaffected unless a staging IP is explicitly configured.
 	//
-	// The sitemap refresh reuses this `ip` too, but unconditionally (no toggle header — it has
-	// no incoming request): whenever `ip` is set, every sitemap fetch is pinned to it, so all
-	// Harper→origin traffic hits the same edge. The security token often only authenticates
-	// against the staging edge, so a direct prod sitemap fetch is bounced with a 403.
+	// The sitemap refresh reuses this `ip` too, gated by `sitemap.useStagingIp` instead of the
+	// toggle header (it has no incoming request to carry one): when that flag is on and `ip` is
+	// set, every sitemap fetch is pinned to it, so all Harper→origin traffic hits the same edge.
 	staging: {
 		ip: '',
 		header: 'x-harper-staging',
@@ -356,6 +355,14 @@ const defaultConfig = () => ({
 		node: '',
 		workerIndex: 0,
 
+		// Whether sitemap fetches follow `staging.ip` when it is set. The sitemap fetch has no
+		// incoming request to carry the staging toggle header, so this flag is its equivalent.
+		// On (the default), sitemap XML is fetched via the staging edge like every other
+		// Harper→origin fetch. Off, sitemap fetches go direct to the production origin — the
+		// security token still rides along, so the edge's bot-mitigation bypass must accept it
+		// on the production property, not just staging, or every fetch 403s.
+		useStagingIp: true,
+
 		// Run `POST /Sitemap/<url>` as a background walk and answer immediately with a handle,
 		// instead of holding the request open for the whole traversal. A sitemap index is not an
 		// HTTP-request-sized unit of work — a real one fans out to tens of children and over a
@@ -659,7 +666,7 @@ export const collectConfigWarnings = () => {
 			add(
 				'info',
 				'staging.ip',
-				`staging passthrough ENABLED — cache-miss requests carrying "${config.staging.header}" are proxied to ${config.staging.ip} (Host/SNI preserved). Toggling this on/off contaminates the URL-keyed page cache; wipe it when switching.`
+				`staging passthrough ENABLED — cache-miss requests carrying "${config.staging.header}" are proxied to ${config.staging.ip} (Host/SNI preserved); sitemap fetches go ${config.sitemap.useStagingIp ? `to ${config.staging.ip} too (sitemap.useStagingIp)` : 'direct to the production origin (sitemap.useStagingIp: false)'}. Toggling this on/off contaminates the URL-keyed page cache; wipe it when switching.`
 			);
 		} else {
 			add(

packages/plugin/src/resources/Sitemap.js

@@ -5,7 +5,7 @@ import { classifyUrl, PASSTHROUGH, PRERENDER, UNCLASSIFIED } from '../util/route
 import { currentMinuteMs, epochMsOf, getNextSitemapRefreshTime } from '../util/time.js';
 import { parseSitemap, partitionSitemapEntries } from '../util/sitemap.js';
 import { actionForExisting, canSkipLookup, createRefreshRun, TargetAction } from '../util/sitemapRun.js';
-import { configuredStagingIp, dispatcherFor } from '../util/upstream.js';
+import { dispatcherFor, sitemapStagingIp } from '../util/upstream.js';
 import { setImmediate } from 'node:timers/promises';
 import { applyInBatches, collectFromScan } from '../util/scan.js';
 
@@ -627,12 +627,12 @@ function getTtlFromChangeFreq(changefreq, { minTtl, defaultTtl }) {
 }
 
 async function fetchLatestSitemap(url) {
-	// Route every Harper→origin sitemap fetch through the same edge as the render/origin-fetch
-	// path: whenever a staging IP is configured, pin the TCP connection to it (Host/SNI stay the
-	// real origin, exactly like upstream.js). The security token typically only authenticates
-	// against the staging edge, so a direct prod fetch is bounced with a 403 "Access Denied".
-	// Empty staging.ip → normal direct fetch (production, once the token is valid at the origin).
-	const stagingIp = configuredStagingIp();
+	// Pin the TCP connection to the staging edge when `sitemap.useStagingIp` + `staging.ip`
+	// say so (Host/SNI stay the real origin, exactly like upstream.js). The flag exists because
+	// the right edge differs per deployment: a token that only authenticates against the
+	// staging edge needs the pin, while a token accepted on the production property can fetch
+	// the real sitemap direct — `useStagingIp: false`. Either way the token is sent.
+	const stagingIp = sitemapStagingIp();
 	const via = stagingIp ? ` (via staging ${stagingIp})` : '';
 
 	const res = await fetch(url, {

packages/plugin/src/util/upstream.js

@@ -28,6 +28,15 @@ export const configuredStagingIp = () => {
 	return ip && isIP(ip) ? ip : undefined;
 };
 
+/**
+ * The staging IP a sitemap fetch should connect to, or undefined for a direct production
+ * fetch. The sitemap refresh has no incoming request to carry the staging toggle header, so
+ * `sitemap.useStagingIp` is its equivalent: on, sitemap fetches follow `staging.ip` like
+ * every other Harper→origin fetch; off, they go direct to the production origin (the
+ * security token is sent either way).
+ */
+export const sitemapStagingIp = () => (config.sitemap.useStagingIp ? configuredStagingIp() : undefined);
+
 // Dispatchers that pin DNS resolution to a fixed IP (staging passthrough), one per IP.
 // Only the connect address is overridden — the origin (so Host header + TLS SNI + cert
 // validation) stays the real origin host, the server-side equivalent of Chrome's

packages/plugin/test/upstream.test.js

@@ -5,6 +5,7 @@ import {
 	configuredStagingIp,
 	resolveUpstreamHeaders,
 	sanitizeOriginResponseHeaders,
+	sitemapStagingIp,
 	stagingTargetIp,
 } from '../src/util/upstream.js';
 
@@ -74,6 +75,24 @@ test('configuredStagingIp ignores an invalid configured ip', () => {
 	assert.equal(configuredStagingIp(), undefined);
 });
 
+test('sitemapStagingIp follows staging.ip by default', () => {
+	applyOptions({ staging: { ip: '192.0.2.27' } });
+	assert.equal(config.sitemap.useStagingIp, true);
+	assert.equal(sitemapStagingIp(), '192.0.2.27');
+});
+
+test('sitemapStagingIp is undefined when sitemap.useStagingIp is off (direct prod fetch)', () => {
+	applyOptions({ staging: { ip: '192.0.2.27' }, sitemap: { useStagingIp: false } });
+	// The render/proxy path still pins to staging; only the sitemap fetch goes direct.
+	assert.equal(configuredStagingIp(), '192.0.2.27');
+	assert.equal(sitemapStagingIp(), undefined);
+});
+
+test('sitemapStagingIp is undefined when no staging ip is configured, regardless of the flag', () => {
+	applyOptions({ sitemap: { useStagingIp: true } });
+	assert.equal(sitemapStagingIp(), undefined);
+});
+
 test('ignoredHeaders defaults to an empty list', () => {
 	applyOptions({});
 	assert.deepEqual(config.ignoredHeaders, []);