From f701ae6d9f25220cabd18f6a20be82243f14f9c5 Mon Sep 17 00:00:00 2001 From: Maksim Kostromin Date: Mon, 26 Oct 2026 01:03:38 -0400 Subject: [PATCH] feat(updater): closed funds skip fetch and guard, guard rejection is failed, test groups of at most five - open-items etfs Q1: shared CLOSED_FUNDS block (empty for SPDR), closed funds leave the queue before any request and their index row is marked closed; a published closed fund passes the unknown TICKERS check and is never pruned - open-items etfs Q2: already compliant (guard rejection is failed, a new fund gets a catalog-only row), verified with mocked runs - open-items etfs Q12: test groups split to at most five tests, shared month-end fixture moved to module scope --- scripts/update-data.test.ts | 65 +++++++++++++++++++++++++++++++++-- scripts/update-data.ts | 67 +++++++++++++++++++++++++++++++------ 2 files changed, 119 insertions(+), 13 deletions(-) diff --git a/scripts/update-data.test.ts b/scripts/update-data.test.ts index 99e19701..af2e4076 100644 --- a/scripts/update-data.test.ts +++ b/scripts/update-data.test.ts @@ -7,10 +7,12 @@ import { tmpdir } from 'node:os'; import { join } from 'node:path'; import { pathToFileURL } from 'node:url'; import { + CLOSED_FUNDS, CONTROL_NAMES, RETURNS_BASIS, annualizedToTotal, applyHistoryRange, + closedFund, configurePacing, deriveCatalogMetrics, distributionHistoryUrl, @@ -178,7 +180,9 @@ describe('controls', () => { for (const b of [{ CONCURRENCY: 'abc' }, { MAX_FETCHES: 'x' }, { MAX_RETRIES: '0' }, { STORE_RAW_DOWNLOADS: 'maybe' }]) expect(() => loadConfig(b)).toThrow(); expect(loadConfig({ STORE_RAW_DOWNLOADS: 'off' }).storeRawDownloads).toBe(false); }); +}); +describe('controls 2', () => { test('range parsers: bounds, K/M/B/T and presets, percent signs, rejection of colonless or inverted values', () => { for (const empty of ['', ':', ' : ']) expect(parseRange(empty, 'X')).toBeUndefined(); expect(parseRange('5:20', 'X')).toEqual({ min: 5, max: 20 }); @@ -283,7 +287,9 @@ describe('parsing', () => { expect(holdingsGuard(headers, allocation, 2, ['sum'])).toBeNull(); expect(holdingsGuard(headers, rows, 5)).toContain('less than half'); }); +}); +describe('parsing 2', () => { test('NAV and premium/discount history workbooks (blank rows dropped, registered mark stripped)', () => { const nav = table(buildXlsx([ ['Fund Name:', 'SPDR Gold MiniShares'], ['Ticker Symbol:', 'GLDM®'], ['Date', 'NAV', 'Shares Outstanding', 'Total Net Assets'], @@ -331,9 +337,10 @@ describe('parsing', () => { }); // --------------------------------------------------------------------------- -describe('metrics', () => { - const monthEnd = { ytd: 10.06, yr1: 19.4, yr3: 19.18, yr5: 12.72, yr10: 14.93, sinceInception: 10.8 }; +// month-end returns shared by both metrics groups +const monthEnd = { ytd: 10.06, yr1: 19.4, yr3: 19.18, yr5: 12.72, yr10: 14.93, sinceInception: 10.8 }; +describe('metrics', () => { test('annualized -> total conversion, null (never 0) on bad input', () => { expect(annualizedToTotal(19.18, 3)).toBeCloseTo(69.28, 2); expect(annualizedToTotal(14.93, 10)).toBeCloseTo(302.1, 1); @@ -385,7 +392,9 @@ describe('metrics', () => { expect([official.dividendYield, official.dividendYieldText, official.dividendYieldSource]).toEqual([0.99, '0.99%', 'official']); expect(official.indicatedDividendYield).toBe(0.99); }); +}); +describe('metrics 2', () => { test('dividendYieldBasis: code per yield source, null with a null yield, same key set on fresh/legacy/empty rows', () => { const dist = { frequency: 'Quarterly', exDate: '06/18/2026', dividend: '1.903516' }; const none = { display: null, value: null }; @@ -575,7 +584,9 @@ describe('pipeline', () => { expect(row(dir, 'XLK')).toEqual(before.find((f: any) => f.ticker === 'XLK')); expect(readFileSync(join(dir, 'funds/XLK/meta.json'), 'utf8')).toBe(xlkMeta); }); +}); +describe('pipeline 2', () => { test('a second identical run writes nothing (same files, bytes and mtimes, no temp files)', async () => { const dir = tempRoot(); installMock(); @@ -641,6 +652,56 @@ describe('pipeline', () => { expect(readFileSync(join(dir, 'funds/SPY/meta.json'), 'utf8')).toBe(meta); }); + test('a closed fund is never fetched or reported failed: its files stay as published, its index row says closed, a rerun writes nothing', async () => { + const entry = { ticker: 'XLK', closed: '2026-10-01', source: 'https://www.sec.gov/Archives/edgar/data/1/notice.htm' }; + expect(() => closedFund([{ ...entry, closed: 'soon' }], 'XLK')).toThrow('CLOSED_FUNDS'); + expect(closedFund([entry], 'xlk')).toEqual(entry); + const dir = tempRoot(); + installMock(); + await runMain(); + const files = () => listFiles(dir).filter((f) => f !== 'index.json').map((f) => `${f}:${readFileSync(join(dir, f), 'utf8')}`); + const snap = () => listFiles(dir).map((f) => `${f}:${readFileSync(join(dir, f), 'utf8')}`); + const first = files(); + const published = row(dir, 'XLK'); + const calls: string[] = []; + const watch = (tickers: string[]) => { + installMock({ tickers, nav: { SPY: 500, XLK: 999, GLD: 300 } }); + const good = globalThis.fetch; + globalThis.fetch = (async (input: unknown, init?: RequestInit) => { + calls.push(String(input)); + // a header-only XLK sheet: the guard would reject it + return String(input).includes('holdings-daily-us-en-xlk') ? new Response(new Uint8Array(buildXlsx(holdingsSheet('XLK').slice(0, 4)))) : good(input as string, init); + }) as unknown as typeof fetch; + }; + CLOSED_FUNDS.push(entry); + try { + watch(['SPY', 'XLK', 'GLD']); + const lines: string[] = []; + await runMain({ TICKERS: 'XLK' }, lines); + await runMain({}, lines); + expect(calls.filter((url) => /xlk/i.test(url))).toEqual([]); + expect(process.exitCode).not.toBe(1); + expect(lines.join('\n')).not.toMatch(/XLK +failed/); + expect(files()).toEqual(first); + expect(readJson(dir, 'index.json').funds.map((f: any) => f.ticker)).toEqual(['GLD', 'SPY', 'XLK']); + expect(row(dir, 'XLK')).toEqual({ ...published, status: 'closed', closedDate: entry.closed, closedSource: entry.source }); + expect(row(dir, 'XLK').dataFile).toBe('./funds/XLK/meta.json'); + const closed = snap(); + await runMain(); + expect(snap()).toEqual(closed); + // the live catalog dropped the closed fund: a TICKERS run for it is still accepted, its row and files survive + watch(['SPY', 'GLD']); + await runMain({ TICKERS: 'XLK' }); + await runMain(); + expect(calls.filter((url) => /xlk/i.test(url))).toEqual([]); + expect(snap()).toEqual(closed); + } finally { + CLOSED_FUNDS.splice(CLOSED_FUNDS.indexOf(entry), 1); + } + }); +}); + +describe('pipeline 3', () => { test('unknown TICKERS is an error that writes nothing; all selected funds failing is an error', async () => { const dir = tempRoot(); installMock(); diff --git a/scripts/update-data.ts b/scripts/update-data.ts index 69de84e6..b3aa0164 100755 --- a/scripts/update-data.ts +++ b/scripts/update-data.ts @@ -202,6 +202,8 @@ export function setApiRoot(root: URL): void { const HOLDINGS_PAGE_SIZE_FALLBACK = 250; /** Funds whose holdings legitimately fail a holdingsGuard check (geared, futures, look-through, top-N lists): ticker, or '*' for every fund -> relaxed checks. */ const HOLDINGS_GUARD_RELAX: Record = {}; +/** Funds the issuer closed, each confirmed from an SEC filing or the issuer notice (`source`): never fetched or guarded, last published data kept, index row marked closed. */ +export const CLOSED_FUNDS: ClosedFund[] = []; const HISTORY_PAGE_SIZE_FALLBACK = 1000; const CONCURRENCY_FALLBACK = 2; const REQUEST_SLEEP_FALLBACK = 1; @@ -1704,7 +1706,9 @@ export async function main(env: Record = process.env console.log(`[ ${'catalog'.padEnd(9)}] ${catalog.length} SPDR ETFs (${catalogSource})`); - const unknownTickers = config.tickers.filter((ticker) => !catalog.some((fund) => fund.ticker === ticker)); + // A closed fund whose row is published is a known ticker even after the live catalog dropped it (the run then writes only its closed mark). + const publishedTickers = new Set((previousIndex?.funds || []).map((row: JsonRecord) => String(row.ticker))); + const unknownTickers = config.tickers.filter((ticker) => !catalog.some((fund) => fund.ticker === ticker) && !(closedFund(CLOSED_FUNDS, ticker) && publishedTickers.has(ticker))); if (unknownTickers.length) throw new Error(`TICKERS: unknown ticker(s) ${unknownTickers.join(', ')} (not in the SPDR catalog)`); const previousRows = new Map((previousIndex?.funds || []).map((row: JsonRecord) => [row.ticker, row])); @@ -1718,12 +1722,13 @@ export async function main(env: Record = process.env // Every filter is applied before the cursor, so a bounded run counts only funds that pass the filters. // Return filters exclude funds with null for a bounded range (matchesRange treats null as no match). - const selected = catalog.filter( + // Closed funds leave the queue here, before the MAX_FETCHES budget: never fetched, guarded or reported. + const selected = withoutClosedFunds(catalog.filter( (fund) => catalogFiltersPass(fund, config, productData.get(fund.ticker)) && returnFiltersPass(fund, config) && yieldFiltersPass(deriveCatalogMetrics(fund.monthEnd, fund.navValue, distributions.get(fund.ticker), productData.get(fund.ticker)), config), - ); + ), (fund) => fund.ticker, CLOSED_FUNDS); outputPrintFilter(selected.length, catalog.length); // Bounded runs continue after the committed cursor (deterministic ticker order), only for the same filter set. @@ -1819,7 +1824,7 @@ export async function main(env: Record = process.env }; // Fund-level consistency: a refreshed fund gets a fresh row; any other fund keeps its previous row untouched - // (never a new return next to stale files). Funds with a meta.json that left the catalog stay listed. + // (never a new return next to stale files). Funds with a meta.json that left the catalog stay listed, closed funds always. const indexFunds: JsonRecord[] = []; const catalogTickers = new Set(catalog.map((fund) => fund.ticker)); for (const fund of catalog) { @@ -1828,9 +1833,10 @@ export async function main(env: Record = process.env else indexFunds.push({ ...previous, metrics: withYieldBasis(previous.metrics), dataFile: (await fileExists(metaPath(fund.ticker))) ? `./funds/${fund.ticker}/meta.json` : null }); } for (const [ticker, previous] of previousRows) { - if (!catalogTickers.has(ticker) && (await fileExists(metaPath(ticker)))) indexFunds.push({ ...previous, metrics: withYieldBasis(previous.metrics) }); + if (!catalogTickers.has(ticker) && (closedFund(CLOSED_FUNDS, ticker) || (await fileExists(metaPath(ticker))))) indexFunds.push({ ...previous, metrics: withYieldBasis(previous.metrics) }); } indexFunds.sort((a, b) => String(a.ticker).localeCompare(String(b.ticker))); + const funds = markClosedFunds(indexFunds, CLOSED_FUNDS); const indexPayload = { generatedAt: previousIndex?.generatedAt || '', @@ -1842,12 +1848,12 @@ export async function main(env: Record = process.env productData: PRODUCT_DATA_URL, }, counts: { - funds: indexFunds.length, - holdings: indexFunds.reduce((sum, fund) => sum + (fund.holdings || 0), 0), - history: indexFunds.reduce((sum, fund) => sum + (fund.history || 0), 0), - premiumDiscountHistory: indexFunds.reduce((sum, fund) => sum + (fund.premiumDiscountHistory || 0), 0), + funds: funds.length, + holdings: funds.reduce((sum, fund) => sum + (fund.holdings || 0), 0), + history: funds.reduce((sum, fund) => sum + (fund.history || 0), 0), + premiumDiscountHistory: funds.reduce((sum, fund) => sum + (fund.premiumDiscountHistory || 0), 0), }, - funds: indexFunds, + funds, }; // The stamp moves only when some content moved, so an identical rerun leaves a zero git diff. if (!previousIndex || outputContentKey(indexPayload) !== outputContentKey(previousIndex) || !indexPayload.generatedAt) { @@ -1967,7 +1973,7 @@ if (import.meta.main) { * Weight (%) or Market Weight) are required, >= 80% of the rows carry a numeric weight, the weights add up to * 50..150 and the sheet keeps >= 50% of the previously published rows. `relax` switches off the checks a fund * legitimately fails: 'numeric' and 'sum' (geared, inverse, futures, look-through, top-N lists) and 'count'. - * Rows are arrays in header order or objects keyed by the header text. + * Rows are arrays in header order or objects keyed by the header text. A fund in the brand's CLOSED_FUNDS never reaches the guard. */ export function holdingsGuard(headers: ReadonlyArray, rows: ReadonlyArray, previousRows: number | null | undefined, relax: ReadonlyArray = []): string | null { const previous = Number(previousRows) > 0 ? Number(previousRows) : 0; @@ -2003,4 +2009,43 @@ function holdingsGuardNumber(value: unknown): number | null { if (!/^[-+]?(\d+\.?\d*|\.\d+)(e[-+]?\d+)?$/i.test(text)) return null; return negative ? -Number(text) : Number(text); } + +/** A fund its issuer closed (liquidated, delisted or merged away), confirmed from an official source: an SEC filing or the issuer's notice. */ +export type ClosedFund = { ticker: string; closed: string; source: string }; + +/** + * The brand's CLOSED_FUNDS entry for a ticker, or null. A closed fund is never fetched, never guarded and never reported failed: it keeps + * its last published data and its index row carries `status: 'closed'`, `closedDate` and `closedSource`. A malformed entry throws, so a + * typo (no ISO date, no https source) never ships. + */ +export function closedFund(list: ReadonlyArray, ticker: unknown): ClosedFund | null { + const wanted = String(ticker === null || ticker === undefined ? '' : ticker).trim().toUpperCase(); + let found: ClosedFund | null = null; + for (let i = 0; i < list.length; i++) { + const fund = list[i]; + if (!/^[A-Z0-9][A-Z0-9.-]{0,9}$/.test(fund.ticker) || !/^\d{4}-\d{2}-\d{2}$/.test(fund.closed) || !/^https:\/\/\S+$/.test(fund.source)) throw new Error(`CLOSED_FUNDS: ${fund.ticker} needs an upper-case ticker, an ISO closing date and an https source`); + if (fund.ticker === wanted) found = fund; + } + return found; +} + +/** The index rows with every closed fund marked (`status: 'closed'`, `closedDate`, `closedSource`); other rows are returned as they are. */ +export function markClosedFunds(rows: ReadonlyArray, list: ReadonlyArray): T[] { + return rows.map((row) => { + const fund = closedFund(list, (row as { ticker?: unknown }).ticker); + return fund ? ({ ...row, status: 'closed', closedDate: fund.closed, closedSource: fund.source } as T) : row; + }); +} + +/** A run's queue without the closed funds, which are named once in the log: they are neither fetched nor counted as failed. */ +export function withoutClosedFunds(queue: ReadonlyArray, tickerOf: (item: T) => string, list: ReadonlyArray): T[] { + const closed: string[] = []; + const open: T[] = []; + for (let i = 0; i < queue.length; i++) { + if (closedFund(list, tickerOf(queue[i]))) closed.push(tickerOf(queue[i])); + else open.push(queue[i]); + } + if (closed.length) console.log(`[ closed ] ${closed.join(', ')}: closed by the issuer, published data kept`); + return open; +} //