Skip to content
Merged
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
15 changes: 15 additions & 0 deletions scripts/update-data.test.ts
Original file line number Diff line number Diff line change
Expand Up @@ -598,6 +598,8 @@ describe('pipeline', () => {
expect(readJson(dir, 'funds/SPY/meta.json').holdings.totalRows).toBe(1);
const sheets: Record<string, string[][]> = {
renamed: holdingsSheet('SPY').map((r) => r.map((cell) => (cell === 'Weight' ? 'Weighting' : cell))),
// the shared guard accepts Weight (%), but the UI reads only Weight
percent: holdingsSheet('SPY').map((r) => r.map((cell) => (cell === 'Weight' ? 'Weight (%)' : cell))),
headerOnly: holdingsSheet('SPY').slice(0, 4),
};
for (const [name, sheet] of Object.entries(sheets)) {
Expand All @@ -611,6 +613,19 @@ describe('pipeline', () => {
expect([name, snap()]).toEqual([name, before]);
expect(lines.join('\n')).toMatch(/SPY +failed .*reason=holdings guard: /);
}
// a new fund whose workbook is shifted (header row not found, 0 rows) is failed, not published as "empty" holdings
const fresh = tempRoot();
installMock();
const good = globalThis.fetch;
const shifted = holdingsSheet('XLK').slice(0, 4).map((r) => r.map((cell) => (cell === 'Weight' ? 'Weight (%)' : cell)));
globalThis.fetch = (async (input: unknown, init?: RequestInit) => (String(input).includes('holdings-daily-us-en-xlk')
? new Response(new Uint8Array(buildXlsx(shifted)))
: good(input as string, init))) as unknown as typeof fetch;
const lines: string[] = [];
await runMain({}, lines);
expect(lines.join('\n')).toMatch(/XLK +failed .*reason=holdings guard: no Name and Weight header row/);
expect(existsSync(join(fresh, 'funds/XLK/meta.json'))).toBe(false);
expect(row(fresh, 'XLK').dataFile).toBeNull();
});

test('a fund whose source failed keeps its previous index row and meta exactly as published', async () => {
Expand Down
13 changes: 12 additions & 1 deletion scripts/update-data.ts
Original file line number Diff line number Diff line change
Expand Up @@ -1469,6 +1469,17 @@ export function distributionsWorksheet(
};
}

/**
* The shared holdingsGuard plus what only SPDR needs: the UI reads holdings cells by the exact header text Name and
* Weight, so a workbook that answered 200 must carry that header row. A renamed column (Weight (%), which the shared
* guard accepts for other brands) or a shifted sheet whose header row was not found (a new fund's 0-row result would
* otherwise be published as status "empty") is broken. Commodity trusts without holdings answer 404 and never get here.
*/
export function spdrHoldingsProblem(table: SheetTable, previousRows: number | null | undefined, relax: ReadonlyArray<string>): string | null {
if (!table.headers.includes('Name') || !table.headers.includes('Weight')) return `no Name and Weight header row (headers: ${table.headers.join(', ') || 'none'})`;
return holdingsGuard(table.headers, table.rows, previousRows, relax);
}

async function processFund(
fund: CatalogFund,
distribution: JsonRecord | undefined,
Expand Down Expand Up @@ -1501,7 +1512,7 @@ async function processFund(
// Freshly fetched holdings that look broken (no weight column, non-numeric weights, a bad weight sum, half the rows gone)
// never replace the published ones: the fund fails here, before anything of it is written, and keeps its previous state.
const previousMeta = await readPreviousMeta();
const holdingsProblem = holdingsGuard(holdings.table.headers, holdings.table.rows, previousMeta?.holdings?.totalRows, HOLDINGS_GUARD_RELAX[ticker] ?? HOLDINGS_GUARD_RELAX['*'] ?? []);
const holdingsProblem = spdrHoldingsProblem(holdings.table, previousMeta?.holdings?.totalRows, HOLDINGS_GUARD_RELAX[ticker] ?? HOLDINGS_GUARD_RELAX['*'] ?? []);
if (holdingsProblem) throw new Error(`holdings guard: ${holdingsProblem}`);
}

Expand Down
Loading