From e83b8c8c3fdb71269c17da25e3b73ac774e5c544 Mon Sep 17 00:00:00 2001 From: Justin Rowles Date: Tue, 20 Oct 2026 11:50:15 +0100 Subject: [PATCH] Add EC2 vs ECS integration comparison tests --- .../fixtures/manual/validationArticles.ts | 15 + .../integration-test/article-html/.gitignore | 1 + .../integration-test/article-html/README.md | 122 +++++ .../article-html/article-html.integration.ts | 24 + .../article-html/ignore-rules.ts | 30 ++ .../article-html/support.test.ts | 456 ++++++++++++++++++ .../integration-test/article-html/support.ts | 249 ++++++++++ dotcom-rendering/package.json | 1 + dotcom-rendering/src/model/validate.test.ts | 34 +- 9 files changed, 899 insertions(+), 33 deletions(-) create mode 100644 dotcom-rendering/fixtures/manual/validationArticles.ts create mode 100644 dotcom-rendering/integration-test/article-html/.gitignore create mode 100644 dotcom-rendering/integration-test/article-html/README.md create mode 100644 dotcom-rendering/integration-test/article-html/article-html.integration.ts create mode 100644 dotcom-rendering/integration-test/article-html/ignore-rules.ts create mode 100644 dotcom-rendering/integration-test/article-html/support.test.ts create mode 100644 dotcom-rendering/integration-test/article-html/support.ts diff --git a/dotcom-rendering/fixtures/manual/validationArticles.ts b/dotcom-rendering/fixtures/manual/validationArticles.ts new file mode 100644 index 00000000000..f05ef7caa6d --- /dev/null +++ b/dotcom-rendering/fixtures/manual/validationArticles.ts @@ -0,0 +1,15 @@ +import { Comment } from '../generated/fe-articles/Comment'; +import { Feature } from '../generated/fe-articles/Feature'; +import { Live } from '../generated/fe-articles/Live'; +import { MatchReport } from '../generated/fe-articles/MatchReport'; +import { Review } from '../generated/fe-articles/Review'; +import { Standard } from '../generated/fe-articles/Standard'; + +export const articles = [ + { name: 'Standard', data: Standard }, + { name: 'Feature', data: Feature }, + { name: 'Comment', data: Comment }, + { name: 'Match Report', data: MatchReport }, + { name: 'Review', data: Review }, + { name: 'Liveblog', data: Live }, +] as const; diff --git a/dotcom-rendering/integration-test/article-html/.gitignore b/dotcom-rendering/integration-test/article-html/.gitignore new file mode 100644 index 00000000000..b05c2dfa700 --- /dev/null +++ b/dotcom-rendering/integration-test/article-html/.gitignore @@ -0,0 +1 @@ +__snapshots__ diff --git a/dotcom-rendering/integration-test/article-html/README.md b/dotcom-rendering/integration-test/article-html/README.md new file mode 100644 index 00000000000..73602f80d5b --- /dev/null +++ b/dotcom-rendering/integration-test/article-html/README.md @@ -0,0 +1,122 @@ +# Article HTML snapshots + +This opt-in integration suite POSTs the six articles shared with +`src/model/validate.test.ts` to an already-running renderer and snapshots its +HTML response and response headers. It does not start a server or execute the returned scripts. +Hosted-content fixtures are not included. + +To test a renderer behind an internal load balancer, first set up an SSH tunnel +through a host that can reach that load balancer. Forward a local port to the +load balancer's HTTPS port, then point `ARTICLE_TEST_URL` at the tunnel on +`localhost`, including the renderer endpoint path. For example, with local port +443 forwarded: + +```sh +export ARTICLE_TEST_URL='https://localhost/Interactive' +``` + +Certificate validation is expressly disabled for this suite to support testing +through these tunnels: the load balancer's certificate will not match +`localhost`. Use only tunnels and endpoints you trust. + +Run from `dotcom-rendering/`: + +```sh +ARTICLE_TEST_URL='http://localhost:9000/Article' npm run test:article-html +``` + +The URL is the full POST endpoint; no path is appended. Optional headers are a +JSON object of strings: + +```sh +ARTICLE_TEST_URL='https://your-renderer.example/Article' \ +ARTICLE_TEST_HEADERS='{"X-Gu-Target-Group":"ecs"}' \ +npm run test:article-html +``` + +The request content type is always `application/json`. Keep credentials out of +committed files. Requests run sequentially with a 15-second timeout, require +HTTP 200 and `Content-Type: text/html`, and reject redirects. + +HTTPS requests deliberately ignore certificate verification errors, including +self-signed, expired, and hostname-mismatched certificates. This applies only to +this suite's request agent, not global TLS settings. Use trusted test endpoints: +the server's identity is not verified. + +## Creating and updating baselines + +Create the initial baseline against your chosen reference deployment: + +```sh +ARTICLE_TEST_URL='http://localhost:9000/Article' \ +npm run test:article-html -- --updateSnapshot +``` + +Jest writes separate HTML and response-header snapshots for each article into +`__snapshots__/article-html.integration.ts.snap` beside the suite. Review and +commit this file. Use the same explicit update command for intentional changes. +Ordinary runs fail for missing or changed snapshots, including outside CI. + +The `*.integration.ts` filename is deliberately outside the usual Jest +`*.test.*` pattern. Neither `npm test` nor `test:ci` runs this HTTP suite. +Only the dedicated configuration selects it. + +## Ignoring volatile fields + +Response headers are compared with lowercase, sorted names, preserving repeated +values and their order. `ignoredResponseHeaders` in `ignore-rules.ts` masks +`date` (the response clock) and `etag` (a hash of HTML containing runtime clocks). +Masked headers remain in the snapshot, so their removal is still detected. +`x-gu-backend-app-target-group` is excluded entirely from snapshots. When the +request includes `X-Gu-Target-Group`, the response must contain +`x-gu-backend-app-target-group` with exactly that value; a missing or mismatched +value fails the test. Without the request header, no target-group assertion is +made. Header names are case-insensitive; values are compared exactly. +All other response headers are compared. +Add header names to this list only when their values are intentionally variable. +Review snapshots before committing, particularly if an endpoint returns cookies +or other sensitive headers. + +`ignore-rules.ts` masks the hash in the `DCR commit hash ...` HTML comment, +retaining the comment and checking that it is present. Other comments and hashes +in page content or scripts remain compared. +It also masks runtime clock values in island JSON `props`: +`serverTime` on `OnwardsUpper` and `KeyEventsCarousel`, and `now` on `RelativeTime`. +Article publication dates and all other properties remain compared. +The liveblog-only selectors explicitly allow no matches on other article types. +Without rules the response is compared verbatim, including significant whitespace, +CSS, and scripts. Additional narrowly targeted rules can be added, for example: + +```ts +export const ignoreRules: readonly IgnoreRule[] = [ + { kind: 'text', selector: '#rendered-at' }, + { + kind: 'attribute', + selector: 'meta[name="rendered-at"]', + attribute: 'content', + }, + { + kind: 'json', + selector: 'script#page-data', + paths: [ + ['config', 'renderedAt'], + ['events', '0', 'timestamp'], + ], + }, +]; +``` + +These replace values with `[IGNORED]`, retaining the element, attribute, or JSON +property. JSON rules apply to valid JSON text, not JavaScript assignments. +Set `attribute: 'props'` on a JSON rule to read and mask JSON stored in an HTML +attribute rather than element text. +Text rules require leaf elements so that they cannot hide child markup. +Selectors must match in every article unless a JSON rule explicitly sets +`allowNoMatches: true`; missing attributes, JSON paths, invalid +selectors, and invalid JSON fail rather than silently skipping a rule. +Array indexes in JSON paths are strings. + +With rules enabled, JSDOM parses and serializes the HTML without running scripts +or loading resources. This canonicalizes HTML syntax, but does not pretty-print, +collapse text whitespace, or broadly remove dates. Embedded JSON selected by a +rule is reserialized. Review updated snapshots whenever the ignore list changes. diff --git a/dotcom-rendering/integration-test/article-html/article-html.integration.ts b/dotcom-rendering/integration-test/article-html/article-html.integration.ts new file mode 100644 index 00000000000..27357e7595d --- /dev/null +++ b/dotcom-rendering/integration-test/article-html/article-html.integration.ts @@ -0,0 +1,24 @@ +import { articles } from '../../fixtures/manual/validationArticles'; +import { ignoredResponseHeaders, ignoreRules } from './ignore-rules'; +import { + normalizeHeaders, + normalizeHtml, + postArticle, + readRequestConfig, +} from './support'; + +describe('Article HTML', () => { + let config: ReturnType; + + beforeAll(() => { + config = readRequestConfig(process.env); + }); + + it.each(articles)('$name', async ({ data }) => { + const { html, headers } = await postArticle(data, config); + expect(normalizeHtml(html, ignoreRules)).toMatchSnapshot(); + expect( + normalizeHeaders(headers, ignoredResponseHeaders), + ).toMatchSnapshot('response headers'); + }); +}); diff --git a/dotcom-rendering/integration-test/article-html/ignore-rules.ts b/dotcom-rendering/integration-test/article-html/ignore-rules.ts new file mode 100644 index 00000000000..4d75c62eb63 --- /dev/null +++ b/dotcom-rendering/integration-test/article-html/ignore-rules.ts @@ -0,0 +1,30 @@ +import type { IgnoreRule } from './support'; + +export const ignoredResponseHeaders: readonly string[] = ['date', 'etag']; + +export const ignoreRules: readonly IgnoreRule[] = [ + { + kind: 'comment', + pattern: /(?<=^\s*DCR commit hash )[a-f0-9]{40}(?=\s*$)/, + }, + { + kind: 'json', + selector: 'gu-island[name="OnwardsUpper"]', + attribute: 'props', + paths: [['serverTime']], + }, + { + kind: 'json', + selector: 'gu-island[name="KeyEventsCarousel"]', + attribute: 'props', + paths: [['serverTime']], + allowNoMatches: true, + }, + { + kind: 'json', + selector: 'gu-island[name="RelativeTime"]', + attribute: 'props', + paths: [['now']], + allowNoMatches: true, + }, +]; diff --git a/dotcom-rendering/integration-test/article-html/support.test.ts b/dotcom-rendering/integration-test/article-html/support.test.ts new file mode 100644 index 00000000000..7aa4376c284 --- /dev/null +++ b/dotcom-rendering/integration-test/article-html/support.test.ts @@ -0,0 +1,456 @@ +/** + * @jest-environment node + */ + +import { Agent } from 'node:https'; +import type * as NodeFetch from 'node-fetch'; +import fetch, { Response } from 'node-fetch'; +import { Standard } from '../../fixtures/generated/fe-articles/Standard'; +import { ignoreRules } from './ignore-rules'; +import { + normalizeHeaders, + normalizeHtml, + postArticle, + readRequestConfig, +} from './support'; + +jest.mock('node-fetch', () => ({ + ...jest.requireActual('node-fetch'), + __esModule: true, + default: jest.fn(), +})); + +describe('article HTML request configuration', () => { + it('requires an explicit HTTP(S) URL', () => { + expect(() => readRequestConfig({})).toThrow('ARTICLE_TEST_URL'); + expect(() => + readRequestConfig({ ARTICLE_TEST_URL: 'file:///tmp/article.html' }), + ).toThrow('HTTP or HTTPS'); + expect(() => + readRequestConfig({ ARTICLE_TEST_URL: 'not a URL' }), + ).toThrow(); + }); + + it.each(['null', '[]', '"headers"', '{"Authorization":123}'])( + 'rejects non-string header objects: %s', + (headers) => { + expect(() => + readRequestConfig({ + ARTICLE_TEST_URL: 'http://localhost:9000/Article', + ARTICLE_TEST_HEADERS: headers, + }), + ).toThrow('ARTICLE_TEST_HEADERS'); + }, + ); + + it('rejects malformed header JSON', () => { + expect(() => + readRequestConfig({ + ARTICLE_TEST_URL: 'http://localhost:9000/Article', + ARTICLE_TEST_HEADERS: '{', + }), + ).toThrow(SyntaxError); + }); + + it('accepts routing headers while enforcing JSON content type', () => { + const config = readRequestConfig({ + ARTICLE_TEST_URL: 'https://example.com/Article?edition=UK', + ARTICLE_TEST_HEADERS: + '{"X-Gu-Target-Group":"ecs","content-type":"text/plain"}', + }); + expect(config.url.href).toBe('https://example.com/Article?edition=UK'); + expect(config.headers.get('X-Gu-Target-Group')).toBe('ecs'); + expect(config.headers.get('Content-Type')).toBe('application/json'); + }); +}); + +describe('posting articles', () => { + afterEach(() => { + jest.restoreAllMocks(); + jest.mocked(fetch).mockReset(); + }); + + const config = readRequestConfig({ + ARTICLE_TEST_URL: 'http://localhost:9000/Article', + ARTICLE_TEST_HEADERS: '{"X-Gu-Target-Group":"ecs"}', + }); + + it('posts the complete article and returns the HTML unchanged', async () => { + const html = '

Article

'; + const fetchMock = jest.mocked(fetch).mockResolvedValue( + new Response(html, { + headers: { + 'content-type': 'text/html; charset=utf-8', + 'x-gu-backend-app-target-group': 'ecs', + }, + }), + ); + await expect(postArticle(Standard, config)).resolves.toEqual({ + html, + headers: { + 'content-type': ['text/html; charset=utf-8'], + 'x-gu-backend-app-target-group': ['ecs'], + }, + }); + expect(fetchMock).toHaveBeenCalledWith(config.url, { + method: 'POST', + headers: Object.fromEntries(config.headers), + body: JSON.stringify(Standard), + redirect: 'error', + signal: expect.any(AbortSignal), + agent: undefined, + }); + }); + + it.each([undefined, 'ec2', 'ecs, ec2'])( + 'rejects a missing or mismatched response target group: %s', + async (targetGroup) => { + jest.mocked(fetch).mockResolvedValue( + new Response('

Article

', { + headers: { + 'content-type': 'text/html', + ...(targetGroup === undefined + ? {} + : { 'X-Gu-Backend-App-Target-Group': targetGroup }), + }, + }), + ); + await expect(postArticle(Standard, config)).rejects.toThrow( + 'Expected x-gu-backend-app-target-group to match X-Gu-Target-Group "ecs"', + ); + }, + ); + + it('matches target group header names case-insensitively', async () => { + jest.mocked(fetch).mockResolvedValue( + new Response('

Article

', { + headers: { + 'Content-Type': 'text/html', + 'X-Gu-Backend-App-Target-Group': 'ec2', + }, + }), + ); + await expect( + postArticle( + Standard, + readRequestConfig({ + ARTICLE_TEST_URL: 'http://localhost/Article', + ARTICLE_TEST_HEADERS: '{"x-gu-target-group":"ec2"}', + }), + ), + ).resolves.toHaveProperty('html', '

Article

'); + }); + + it('disables certificate verification only on its HTTPS agent and cleans it up', async () => { + const fetchMock = jest.mocked(fetch).mockResolvedValue( + new Response('

Article

', { + headers: { 'content-type': 'text/html' }, + }), + ); + const destroy = jest.spyOn(Agent.prototype, 'destroy'); + await postArticle( + Standard, + readRequestConfig({ + ARTICLE_TEST_URL: 'https://localhost/Article', + }), + ); + expect(fetchMock.mock.calls[0]?.[1]?.agent).toEqual( + expect.objectContaining({ + options: expect.objectContaining({ rejectUnauthorized: false }), + }), + ); + expect(destroy).toHaveBeenCalledTimes(1); + }); + + it('rejects HTTP errors with a bounded response excerpt', async () => { + jest.mocked(fetch).mockResolvedValue( + new Response('Renderer failed', { status: 500 }), + ); + await expect(postArticle(Standard, config)).rejects.toThrow( + 'Expected HTTP 200, received 500: Renderer failed', + ); + }); + + it('rejects non-HTML responses', async () => { + jest.mocked(fetch).mockResolvedValue( + new Response('{}', { + headers: { 'content-type': 'application/json' }, + }), + ); + await expect(postArticle(Standard, config)).rejects.toThrow( + 'Content-Type text/html', + ); + }); + + it('rejects empty HTML', async () => { + jest.mocked(fetch).mockResolvedValue( + new Response(' ', { headers: { 'content-type': 'text/html' } }), + ); + await expect(postArticle(Standard, config)).rejects.toThrow( + 'empty HTML', + ); + }); + + it('propagates network and timeout failures', async () => { + const error = new Error('Request timed out'); + jest.mocked(fetch).mockRejectedValue(error); + await expect(postArticle(Standard, config)).rejects.toBe(error); + }); +}); + +describe('normalizing response headers', () => { + it('canonicalizes header names and order without changing values or duplicates', () => { + expect( + normalizeHeaders( + { + 'X-Gu-Backend-App': ['interactive-rendering'], + 'Set-Cookie': ['first=1', 'second=2'], + 'Content-Type': ['text/html'], + }, + [], + ), + ).toEqual({ + 'content-type': ['text/html'], + 'set-cookie': ['first=1', 'second=2'], + 'x-gu-backend-app': ['interactive-rendering'], + }); + expect( + Object.keys(normalizeHeaders({ Zebra: ['z'], Alpha: ['a'] }, [])), + ).toEqual(['alpha', 'zebra']); + }); + + it('masks only explicitly ignored values, retaining header presence', () => { + const headers = { + date: ['Tue, 20 Oct 2026 10:23:18 GMT'], + etag: ['W/"first"'], + 'x-gu-backend-app': ['interactive-rendering'], + }; + const normalized = normalizeHeaders(headers, ['Date', 'ETag']); + expect(normalized).toEqual({ + date: ['[IGNORED]'], + etag: ['[IGNORED]'], + 'x-gu-backend-app': ['interactive-rendering'], + }); + expect( + normalizeHeaders( + { + ...headers, + date: ['later'], + etag: ['W/"second"'], + }, + ['date', 'etag'], + ), + ).toEqual(normalized); + expect( + normalizeHeaders( + { + ...headers, + 'x-gu-backend-app': ['article-rendering'], + }, + ['date', 'etag'], + ), + ).not.toEqual(normalized); + expect( + normalizeHeaders( + { + etag: headers.etag, + 'x-gu-backend-app': ['interactive-rendering'], + }, + ['date', 'etag'], + ), + ).not.toEqual(normalized); + expect(headers.etag).toEqual(['W/"first"']); + }); + + it('excludes the target group header entirely regardless of case or value', () => { + const otherHeaders = { 'content-type': ['text/html'] }; + for (const name of [ + 'x-gu-backend-app-target-group', + 'X-Gu-Backend-App-Target-Group', + ]) { + expect( + normalizeHeaders({ ...otherHeaders, [name]: ['ecs'] }, []), + ).toEqual(normalizeHeaders(otherHeaders, [])); + } + }); +}); + +describe('normalizing article HTML', () => { + const html = + '' + + '' + + '

Published 2024-01-01 unchanged

' + + ''; + + it('preserves the exact response with no ignore rules', () => { + expect(normalizeHtml(html, [])).toBe(html); + }); + + it('masks only the DCR hash inside comments, preserving other comments and content', () => { + const rule = ignoreRules.filter((item) => item.kind === 'comment'); + const makeHtml = (hash: string) => + `` + + `

DCR commit hash ${'a'.repeat(40)}

` + + ``; + const normalized = normalizeHtml(makeHtml('a'.repeat(40)), rule); + expect(normalized).toBe(normalizeHtml(makeHtml('b'.repeat(40)), rule)); + expect(normalized).toContain(''); + expect(normalized).toContain(''); + expect(normalized).toContain( + `

DCR commit hash ${'a'.repeat(40)}

`, + ); + expect(normalized).toContain(`const hash = "${'a'.repeat(40)}";`); + expect(() => normalizeHtml('

No hash comment

', rule)).toThrow( + 'matched no comments', + ); + }); + + it('ignores runtime island clocks while preserving article dates and other props', () => { + const makeHtml = (clock: number) => + `` + + `` + + `` + + ``; + const normalized = normalizeHtml(makeHtml(1000), ignoreRules); + expect(normalized).toBe(normalizeHtml(makeHtml(61000), ignoreRules)); + expect(normalized).toContain( + '"serverTime":"[IGNORED]"', + ); + expect(normalized).toContain('"now":"[IGNORED]"'); + expect(normalized).toContain('"then":456'); + expect(normalized).toContain('"timestamp":123'); + expect(normalized).toContain( + '', + ); + expect( + normalizeHtml( + makeHtml(1000).replace('Article', 'Changed'), + ignoreRules, + ), + ).not.toBe(normalized); + expect( + normalizeHtml( + makeHtml(1000).replace('"then":456', '"then":789'), + ignoreRules, + ), + ).not.toBe(normalized); + }); + + it('allows liveblog-only islands to be absent without making all selectors optional', () => { + const standard = ``; + expect(() => normalizeHtml(standard, ignoreRules)).not.toThrow(); + expect(() => normalizeHtml('

No islands

', ignoreRules)).toThrow( + 'matched no comments', + ); + }); + + it('rejects missing or invalid JSON attributes and missing clock properties', () => { + for (const attributes of [ + '', + 'props="invalid"', + `props='{"other":123}'`, + ]) { + expect(() => + normalizeHtml( + ``, + ignoreRules, + ), + ).toThrow(); + } + }); + + it('masks only configured text, attributes, and nested JSON fields', () => { + expect( + normalizeHtml(html, [ + { kind: 'text', selector: '#generated' }, + { + kind: 'attribute', + selector: '#generated', + attribute: 'datetime', + }, + { + kind: 'json', + selector: '#data', + paths: [ + ['config', 'timestamp'], + ['events', '0', 'at'], + ], + }, + ]), + ).toBe( + html + .replace( + 'datetime="2026-10-20">Today', + 'datetime="[IGNORED]">[IGNORED]', + ) + .replace('"timestamp":123', '"timestamp":"[IGNORED]"') + .replace('"at":456', '"at":"[IGNORED]"'), + ); + }); + + it('masks every matching element', () => { + expect( + normalizeHtml('

One

Two

', [ + { kind: 'text', selector: 'p' }, + ]), + ).toContain('

[IGNORED]

[IGNORED]

'); + }); + + it('fails for unmatched or invalid selectors', () => { + expect(() => + normalizeHtml(html, [{ kind: 'text', selector: '#missing' }]), + ).toThrow('matched no elements'); + expect(() => + normalizeHtml(html, [{ kind: 'text', selector: '[' }]), + ).toThrow(); + }); + + it('does not silently hide child markup', () => { + expect(() => + normalizeHtml(html, [{ kind: 'text', selector: 'p' }]), + ).toThrow('leaf elements'); + }); + + it('fails for missing attributes', () => { + expect(() => + normalizeHtml(html, [ + { + kind: 'attribute', + selector: '#generated', + attribute: 'missing', + }, + ]), + ).toThrow('Missing attribute'); + }); + + it.each([ + { paths: [['config', 'missing']] }, + { paths: [['config', 'timestamp', 'missing']] }, + { paths: [['toString']] }, + { paths: [[]] }, + { paths: [] }, + ])('rejects missing or empty JSON paths: $paths', ({ paths }) => { + expect(() => + normalizeHtml(html, [{ kind: 'json', selector: '#data', paths }]), + ).toThrow(); + }); + + it('fails for invalid embedded JSON', () => { + expect(() => + normalizeHtml('', [ + { kind: 'json', selector: '#data', paths: [['timestamp']] }, + ]), + ).toThrow(SyntaxError); + }); + + it('does not execute scripts or allow JSON to introduce closing script tags', () => { + const output = normalizeHtml( + '' + + '', + [{ kind: 'json', selector: '#data', paths: [['timestamp']] }], + ); + expect(output).toContain('"text":"\\u003c/script>"'); + }); +}); diff --git a/dotcom-rendering/integration-test/article-html/support.ts b/dotcom-rendering/integration-test/article-html/support.ts new file mode 100644 index 00000000000..bd4423f0b90 --- /dev/null +++ b/dotcom-rendering/integration-test/article-html/support.ts @@ -0,0 +1,249 @@ +import { Agent } from 'node:https'; +import { JSDOM } from 'jsdom'; +import fetch from 'node-fetch'; +import type { FEArticle } from '../../src/frontend/feArticle'; + +export type IgnoreRule = + | { kind: 'comment'; pattern: RegExp } + | { selector: string; kind: 'text' } + | { selector: string; kind: 'attribute'; attribute: string } + | { + selector: string; + kind: 'json'; + paths: readonly (readonly string[])[]; + attribute?: string; + allowNoMatches?: boolean; + }; + +const placeholder = '[IGNORED]'; +const requestTimeoutMs = 15000; +const targetGroupResponseHeader = 'x-gu-backend-app-target-group'; + +const isRecord = (value: unknown): value is Record => + typeof value === 'object' && value !== null; + +type RequestConfig = { + url: URL; + headers: Headers; +}; + +type ArticleResponse = { + html: string; + headers: Record; +}; + +export const normalizeHeaders = ( + headers: Record, + ignoredHeaders: readonly string[], +): Record => { + const ignored = new Set(ignoredHeaders.map((name) => name.toLowerCase())); + const normalized: Record = {}; + for (const [name, values] of Object.entries(headers)) { + const key = name.toLowerCase(); + if (key === targetGroupResponseHeader) continue; + normalized[key] = ignored.has(key) ? [placeholder] : [...values]; + } + return Object.fromEntries( + Object.entries(normalized).sort(([a], [b]) => a.localeCompare(b)), + ); +}; + +export const readRequestConfig = (env: NodeJS.ProcessEnv): RequestConfig => { + if (!env.ARTICLE_TEST_URL) { + throw new Error( + 'ARTICLE_TEST_URL must contain the full HTTP(S) POST URL', + ); + } + const url = new URL(env.ARTICLE_TEST_URL); + if (url.protocol !== 'http:' && url.protocol !== 'https:') { + throw new Error('ARTICLE_TEST_URL must use HTTP or HTTPS'); + } + const headers = new Headers({ Accept: 'text/html' }); + if (env.ARTICLE_TEST_HEADERS !== undefined) { + const values: unknown = JSON.parse(env.ARTICLE_TEST_HEADERS); + if (!isRecord(values) || Array.isArray(values)) { + throw new Error( + 'ARTICLE_TEST_HEADERS must be a JSON object of strings', + ); + } + for (const [key, value] of Object.entries(values)) { + if (typeof value !== 'string') { + throw new Error( + `ARTICLE_TEST_HEADERS value for "${key}" must be a string`, + ); + } + headers.set(key, value); + } + } + headers.set('Content-Type', 'application/json'); + return { url, headers }; +}; + +export const postArticle = async ( + article: FEArticle, + config: RequestConfig, +): Promise => { + const agent = + config.url.protocol === 'https:' + ? new Agent({ rejectUnauthorized: false }) + : undefined; + try { + const response = await fetch(config.url, { + method: 'POST', + headers: Object.fromEntries(config.headers), + body: JSON.stringify(article), + redirect: 'error', + signal: AbortSignal.timeout(requestTimeoutMs), + agent, + }); + const html = await response.text(); + if (response.status !== 200) { + throw new Error( + `Expected HTTP 200, received ${response.status}: ${html.slice(0, 500)}`, + ); + } + if ( + !/^text\/html(?:;|$)/i.test( + response.headers.get('content-type') ?? '', + ) + ) { + throw new Error( + 'Expected an HTML response with Content-Type text/html', + ); + } + if (!html.trim()) { + throw new Error('Received an empty HTML response'); + } + const requestedTargetGroup = config.headers.get('X-Gu-Target-Group'); + if (requestedTargetGroup !== null) { + const actualTargetGroup = response.headers.get( + targetGroupResponseHeader, + ); + if (actualTargetGroup !== requestedTargetGroup) { + throw new Error( + `Expected ${targetGroupResponseHeader} to match X-Gu-Target-Group "${requestedTargetGroup}", received ${JSON.stringify(actualTargetGroup)}`, + ); + } + } + return { html, headers: response.headers.raw() }; + } finally { + agent?.destroy(); + } +}; + +const maskJsonPath = (json: unknown, path: readonly string[]): void => { + if (path.length === 0) { + throw new Error('JSON ignore paths must not be empty'); + } + let parent = json; + for (const [index, key] of path.entries()) { + if (!isRecord(parent) || !Object.hasOwn(parent, key)) { + throw new Error(`JSON ignore path not found: ${path.join('.')}`); + } + if (index === path.length - 1) { + parent[key] = placeholder; + } else { + parent = parent[key]; + } + } +}; + +export const normalizeHtml = ( + html: string, + rules: readonly IgnoreRule[], +): string => { + if (rules.length === 0) return html; + // Default JSDOM options neither execute scripts nor load external resources. + const dom = new JSDOM(html); + try { + for (const rule of rules) { + if (rule.kind === 'comment') { + const comments = dom.window.document.createTreeWalker( + dom.window.document, + dom.window.NodeFilter.SHOW_COMMENT, + ); + let matched = false; + let comment = comments.nextNode(); + while (comment) { + comment.nodeValue = (comment.nodeValue ?? '').replace( + rule.pattern, + () => { + matched = true; + return placeholder; + }, + ); + comment = comments.nextNode(); + } + if (!matched) { + throw new Error( + `HTML comment ignore pattern matched no comments: ${rule.pattern}`, + ); + } + continue; + } + const elements = dom.window.document.querySelectorAll( + rule.selector, + ); + if ( + elements.length === 0 && + !(rule.kind === 'json' && rule.allowNoMatches) + ) { + throw new Error( + `HTML ignore selector matched no elements: ${rule.selector}`, + ); + } + for (const element of elements) { + switch (rule.kind) { + case 'text': + if (element.childElementCount > 0) { + throw new Error( + `Text ignore rules must target leaf elements: ${rule.selector}`, + ); + } + element.textContent = placeholder; + break; + case 'attribute': + if (!element.hasAttribute(rule.attribute)) { + throw new Error( + `Missing attribute "${rule.attribute}" on ${rule.selector}`, + ); + } + element.setAttribute(rule.attribute, placeholder); + break; + case 'json': { + if (rule.paths.length === 0) { + throw new Error( + 'JSON ignore rules must specify at least one path', + ); + } + const source = rule.attribute + ? element.getAttribute(rule.attribute) + : element.textContent; + if (source === null) { + throw new Error( + `Missing JSON attribute "${rule.attribute}" on ${rule.selector}`, + ); + } + const json: unknown = JSON.parse(source); + for (const path of rule.paths) maskJsonPath(json, path); + if (rule.attribute) { + element.setAttribute( + rule.attribute, + JSON.stringify(json), + ); + } else { + element.textContent = JSON.stringify(json).replace( + /