feat(updater): closed funds skip fetch and guard, guard rejection is failed, test groups of at most five

scripts/update-data.test.ts

@@ -7,10 +7,12 @@ import { tmpdir } from 'node:os';
 import { join } from 'node:path';
 import { pathToFileURL } from 'node:url';
 import {
+  CLOSED_FUNDS,
   CONTROL_NAMES,
   RETURNS_BASIS,
   annualizedToTotal,
   applyHistoryRange,
+  closedFund,
   configurePacing,
   deriveCatalogMetrics,
   distributionHistoryUrl,
@@ -178,7 +180,9 @@ describe('controls', () => {
     for (const b of [{ CONCURRENCY: 'abc' }, { MAX_FETCHES: 'x' }, { MAX_RETRIES: '0' }, { STORE_RAW_DOWNLOADS: 'maybe' }]) expect(() => loadConfig(b)).toThrow();
     expect(loadConfig({ STORE_RAW_DOWNLOADS: 'off' }).storeRawDownloads).toBe(false);
   });
+});
 
+describe('controls 2', () => {
   test('range parsers: bounds, K/M/B/T and presets, percent signs, rejection of colonless or inverted values', () => {
     for (const empty of ['', ':', '  :  ']) expect(parseRange(empty, 'X')).toBeUndefined();
     expect(parseRange('5:20', 'X')).toEqual({ min: 5, max: 20 });
@@ -283,7 +287,9 @@ describe('parsing', () => {
     expect(holdingsGuard(headers, allocation, 2, ['sum'])).toBeNull();
     expect(holdingsGuard(headers, rows, 5)).toContain('less than half');
   });
+});
 
+describe('parsing 2', () => {
   test('NAV and premium/discount history workbooks (blank rows dropped, registered mark stripped)', () => {
     const nav = table(buildXlsx([
       ['Fund Name:', 'SPDR Gold MiniShares'], ['Ticker Symbol:', 'GLDM®'], ['Date', 'NAV', 'Shares Outstanding', 'Total Net Assets'],
@@ -331,9 +337,10 @@ describe('parsing', () => {
 });
 
 // ---------------------------------------------------------------------------
-describe('metrics', () => {
-  const monthEnd = { ytd: 10.06, yr1: 19.4, yr3: 19.18, yr5: 12.72, yr10: 14.93, sinceInception: 10.8 };
+// month-end returns shared by both metrics groups
+const monthEnd = { ytd: 10.06, yr1: 19.4, yr3: 19.18, yr5: 12.72, yr10: 14.93, sinceInception: 10.8 };
 
+describe('metrics', () => {
   test('annualized -> total conversion, null (never 0) on bad input', () => {
     expect(annualizedToTotal(19.18, 3)).toBeCloseTo(69.28, 2);
     expect(annualizedToTotal(14.93, 10)).toBeCloseTo(302.1, 1);
@@ -385,7 +392,9 @@ describe('metrics', () => {
     expect([official.dividendYield, official.dividendYieldText, official.dividendYieldSource]).toEqual([0.99, '0.99%', 'official']);
     expect(official.indicatedDividendYield).toBe(0.99);
   });
+});
 
+describe('metrics 2', () => {
   test('dividendYieldBasis: code per yield source, null with a null yield, same key set on fresh/legacy/empty rows', () => {
     const dist = { frequency: 'Quarterly', exDate: '06/18/2026', dividend: '1.903516' };
     const none = { display: null, value: null };
@@ -575,7 +584,9 @@ describe('pipeline', () => {
     expect(row(dir, 'XLK')).toEqual(before.find((f: any) => f.ticker === 'XLK'));
     expect(readFileSync(join(dir, 'funds/XLK/meta.json'), 'utf8')).toBe(xlkMeta);
   });
+});
 
+describe('pipeline 2', () => {
   test('a second identical run writes nothing (same files, bytes and mtimes, no temp files)', async () => {
     const dir = tempRoot();
     installMock();
@@ -641,6 +652,56 @@ describe('pipeline', () => {
     expect(readFileSync(join(dir, 'funds/SPY/meta.json'), 'utf8')).toBe(meta);
   });
 
+  test('a closed fund is never fetched or reported failed: its files stay as published, its index row says closed, a rerun writes nothing', async () => {
+    const entry = { ticker: 'XLK', closed: '2026-10-01', source: 'https://www.sec.gov/Archives/edgar/data/1/notice.htm' };
+    expect(() => closedFund([{ ...entry, closed: 'soon' }], 'XLK')).toThrow('CLOSED_FUNDS');
+    expect(closedFund([entry], 'xlk')).toEqual(entry);
+    const dir = tempRoot();
+    installMock();
+    await runMain();
+    const files = () => listFiles(dir).filter((f) => f !== 'index.json').map((f) => `${f}:${readFileSync(join(dir, f), 'utf8')}`);
+    const snap = () => listFiles(dir).map((f) => `${f}:${readFileSync(join(dir, f), 'utf8')}`);
+    const first = files();
+    const published = row(dir, 'XLK');
+    const calls: string[] = [];
+    const watch = (tickers: string[]) => {
+      installMock({ tickers, nav: { SPY: 500, XLK: 999, GLD: 300 } });
+      const good = globalThis.fetch;
+      globalThis.fetch = (async (input: unknown, init?: RequestInit) => {
+        calls.push(String(input));
+        // a header-only XLK sheet: the guard would reject it
+        return String(input).includes('holdings-daily-us-en-xlk') ? new Response(new Uint8Array(buildXlsx(holdingsSheet('XLK').slice(0, 4)))) : good(input as string, init);
+      }) as unknown as typeof fetch;
+    };
+    CLOSED_FUNDS.push(entry);
+    try {
+      watch(['SPY', 'XLK', 'GLD']);
+      const lines: string[] = [];
+      await runMain({ TICKERS: 'XLK' }, lines);
+      await runMain({}, lines);
+      expect(calls.filter((url) => /xlk/i.test(url))).toEqual([]);
+      expect(process.exitCode).not.toBe(1);
+      expect(lines.join('\n')).not.toMatch(/XLK +failed/);
+      expect(files()).toEqual(first);
+      expect(readJson(dir, 'index.json').funds.map((f: any) => f.ticker)).toEqual(['GLD', 'SPY', 'XLK']);
+      expect(row(dir, 'XLK')).toEqual({ ...published, status: 'closed', closedDate: entry.closed, closedSource: entry.source });
+      expect(row(dir, 'XLK').dataFile).toBe('./funds/XLK/meta.json');
+      const closed = snap();
+      await runMain();
+      expect(snap()).toEqual(closed);
+      // the live catalog dropped the closed fund: a TICKERS run for it is still accepted, its row and files survive
+      watch(['SPY', 'GLD']);
+      await runMain({ TICKERS: 'XLK' });
+      await runMain();
+      expect(calls.filter((url) => /xlk/i.test(url))).toEqual([]);
+      expect(snap()).toEqual(closed);
+    } finally {
+      CLOSED_FUNDS.splice(CLOSED_FUNDS.indexOf(entry), 1);
+    }
+  });
+});
+
+describe('pipeline 3', () => {
   test('unknown TICKERS is an error that writes nothing; all selected funds failing is an error', async () => {
     const dir = tempRoot();
     installMock();

scripts/update-data.ts

@@ -202,6 +202,8 @@ export function setApiRoot(root: URL): void {
 const HOLDINGS_PAGE_SIZE_FALLBACK = 250;
 /** Funds whose holdings legitimately fail a holdingsGuard check (geared, futures, look-through, top-N lists): ticker, or '*' for every fund -> relaxed checks. */
 const HOLDINGS_GUARD_RELAX: Record<string, string[]> = {};
+/** Funds the issuer closed, each confirmed from an SEC filing or the issuer notice (`source`): never fetched or guarded, last published data kept, index row marked closed. */
+export const CLOSED_FUNDS: ClosedFund[] = [];
 const HISTORY_PAGE_SIZE_FALLBACK = 1000;
 const CONCURRENCY_FALLBACK = 2;
 const REQUEST_SLEEP_FALLBACK = 1;
@@ -1704,7 +1706,9 @@ export async function main(env: Record<string, string | undefined> = process.env
 
   console.log(`[ ${'catalog'.padEnd(9)}] ${catalog.length} SPDR ETFs (${catalogSource})`);
 
-  const unknownTickers = config.tickers.filter((ticker) => !catalog.some((fund) => fund.ticker === ticker));
+  // A closed fund whose row is published is a known ticker even after the live catalog dropped it (the run then writes only its closed mark).
+  const publishedTickers = new Set<string>((previousIndex?.funds || []).map((row: JsonRecord) => String(row.ticker)));
+  const unknownTickers = config.tickers.filter((ticker) => !catalog.some((fund) => fund.ticker === ticker) && !(closedFund(CLOSED_FUNDS, ticker) && publishedTickers.has(ticker)));
   if (unknownTickers.length) throw new Error(`TICKERS: unknown ticker(s) ${unknownTickers.join(', ')} (not in the SPDR catalog)`);
 
   const previousRows = new Map<string, JsonRecord>((previousIndex?.funds || []).map((row: JsonRecord) => [row.ticker, row]));
@@ -1718,12 +1722,13 @@ export async function main(env: Record<string, string | undefined> = process.env
 
   // Every filter is applied before the cursor, so a bounded run counts only funds that pass the filters.
   // Return filters exclude funds with null for a bounded range (matchesRange treats null as no match).
-  const selected = catalog.filter(
+  // Closed funds leave the queue here, before the MAX_FETCHES budget: never fetched, guarded or reported.
+  const selected = withoutClosedFunds(catalog.filter(
     (fund) =>
       catalogFiltersPass(fund, config, productData.get(fund.ticker)) &&
       returnFiltersPass(fund, config) &&
       yieldFiltersPass(deriveCatalogMetrics(fund.monthEnd, fund.navValue, distributions.get(fund.ticker), productData.get(fund.ticker)), config),
-  );
+  ), (fund) => fund.ticker, CLOSED_FUNDS);
   outputPrintFilter(selected.length, catalog.length);
 
   // Bounded runs continue after the committed cursor (deterministic ticker order), only for the same filter set.
@@ -1819,7 +1824,7 @@ export async function main(env: Record<string, string | undefined> = process.env
   };
 
   // Fund-level consistency: a refreshed fund gets a fresh row; any other fund keeps its previous row untouched
-  // (never a new return next to stale files). Funds with a meta.json that left the catalog stay listed.
+  // (never a new return next to stale files). Funds with a meta.json that left the catalog stay listed, closed funds always.
   const indexFunds: JsonRecord[] = [];
   const catalogTickers = new Set(catalog.map((fund) => fund.ticker));
   for (const fund of catalog) {
@@ -1828,9 +1833,10 @@ export async function main(env: Record<string, string | undefined> = process.env
     else indexFunds.push({ ...previous, metrics: withYieldBasis(previous.metrics), dataFile: (await fileExists(metaPath(fund.ticker))) ? `./funds/${fund.ticker}/meta.json` : null });
   }
   for (const [ticker, previous] of previousRows) {
-    if (!catalogTickers.has(ticker) && (await fileExists(metaPath(ticker)))) indexFunds.push({ ...previous, metrics: withYieldBasis(previous.metrics) });
+    if (!catalogTickers.has(ticker) && (closedFund(CLOSED_FUNDS, ticker) || (await fileExists(metaPath(ticker))))) indexFunds.push({ ...previous, metrics: withYieldBasis(previous.metrics) });
   }
   indexFunds.sort((a, b) => String(a.ticker).localeCompare(String(b.ticker)));
+  const funds = markClosedFunds(indexFunds, CLOSED_FUNDS);
 
   const indexPayload = {
     generatedAt: previousIndex?.generatedAt || '',
@@ -1842,12 +1848,12 @@ export async function main(env: Record<string, string | undefined> = process.env
       productData: PRODUCT_DATA_URL,
     },
     counts: {
-      funds: indexFunds.length,
-      holdings: indexFunds.reduce((sum, fund) => sum + (fund.holdings || 0), 0),
-      history: indexFunds.reduce((sum, fund) => sum + (fund.history || 0), 0),
-      premiumDiscountHistory: indexFunds.reduce((sum, fund) => sum + (fund.premiumDiscountHistory || 0), 0),
+      funds: funds.length,
+      holdings: funds.reduce((sum, fund) => sum + (fund.holdings || 0), 0),
+      history: funds.reduce((sum, fund) => sum + (fund.history || 0), 0),
+      premiumDiscountHistory: funds.reduce((sum, fund) => sum + (fund.premiumDiscountHistory || 0), 0),
     },
-    funds: indexFunds,
+    funds,
   };
   // The stamp moves only when some content moved, so an identical rerun leaves a zero git diff.
   if (!previousIndex || outputContentKey(indexPayload) !== outputContentKey(previousIndex) || !indexPayload.generatedAt) {
@@ -1967,7 +1973,7 @@ if (import.meta.main) {
  * Weight (%) or Market Weight) are required, >= 80% of the rows carry a numeric weight, the weights add up to
  * 50..150 and the sheet keeps >= 50% of the previously published rows. `relax` switches off the checks a fund
  * legitimately fails: 'numeric' and 'sum' (geared, inverse, futures, look-through, top-N lists) and 'count'.
- * Rows are arrays in header order or objects keyed by the header text.
+ * Rows are arrays in header order or objects keyed by the header text. A fund in the brand's CLOSED_FUNDS never reaches the guard.
  */
 export function holdingsGuard(headers: ReadonlyArray<string>, rows: ReadonlyArray<unknown>, previousRows: number | null | undefined, relax: ReadonlyArray<string> = []): string | null {
   const previous = Number(previousRows) > 0 ? Number(previousRows) : 0;
@@ -2003,4 +2009,43 @@ function holdingsGuardNumber(value: unknown): number | null {
   if (!/^[-+]?(\d+\.?\d*|\.\d+)(e[-+]?\d+)?$/i.test(text)) return null;
   return negative ? -Number(text) : Number(text);
 }
+
+/** A fund its issuer closed (liquidated, delisted or merged away), confirmed from an official source: an SEC filing or the issuer's notice. */
+export type ClosedFund = { ticker: string; closed: string; source: string };
+
+/**
+ * The brand's CLOSED_FUNDS entry for a ticker, or null. A closed fund is never fetched, never guarded and never reported failed: it keeps
+ * its last published data and its index row carries `status: 'closed'`, `closedDate` and `closedSource`. A malformed entry throws, so a
+ * typo (no ISO date, no https source) never ships.
+ */
+export function closedFund(list: ReadonlyArray<ClosedFund>, ticker: unknown): ClosedFund | null {
+  const wanted = String(ticker === null || ticker === undefined ? '' : ticker).trim().toUpperCase();
+  let found: ClosedFund | null = null;
+  for (let i = 0; i < list.length; i++) {
+    const fund = list[i];
+    if (!/^[A-Z0-9][A-Z0-9.-]{0,9}$/.test(fund.ticker) || !/^\d{4}-\d{2}-\d{2}$/.test(fund.closed) || !/^https:\/\/\S+$/.test(fund.source)) throw new Error(`CLOSED_FUNDS: ${fund.ticker} needs an upper-case ticker, an ISO closing date and an https source`);
+    if (fund.ticker === wanted) found = fund;
+  }
+  return found;
+}
+
+/** The index rows with every closed fund marked (`status: 'closed'`, `closedDate`, `closedSource`); other rows are returned as they are. */
+export function markClosedFunds<T extends object>(rows: ReadonlyArray<T>, list: ReadonlyArray<ClosedFund>): T[] {
+  return rows.map((row) => {
+    const fund = closedFund(list, (row as { ticker?: unknown }).ticker);
+    return fund ? ({ ...row, status: 'closed', closedDate: fund.closed, closedSource: fund.source } as T) : row;
+  });
+}
+
+/** A run's queue without the closed funds, which are named once in the log: they are neither fetched nor counted as failed. */
+export function withoutClosedFunds<T>(queue: ReadonlyArray<T>, tickerOf: (item: T) => string, list: ReadonlyArray<ClosedFund>): T[] {
+  const closed: string[] = [];
+  const open: T[] = [];
+  for (let i = 0; i < queue.length; i++) {
+    if (closedFund(list, tickerOf(queue[i]))) closed.push(tickerOf(queue[i]));
+    else open.push(queue[i]);
+  }
+  if (closed.length) console.log(`[ closed   ] ${closed.join(', ')}: closed by the issuer, published data kept`);
+  return open;
+}
 // </holdings-guard>