From cb99b78ebafb19484b66485430d0e78e91e78fd2 Mon Sep 17 00:00:00 2001 From: Cory Rylan Date: Tue, 27 Oct 2026 20:51:57 -0500 Subject: [PATCH] chore(ci): enhance ci performance and timeout settings - Added a new performance audit document detailing a 12.62% reduction in median CI time. - Updated timeout settings in various test suites to 30 seconds for improved reliability preventing retries. - Enhanced the CI profiling script to include concurrency information. - Improved error handling in file operations to ensure robustness during concurrent builds. Signed-off-by: Cory Rylan --- projects/cli/src/index.test.ts | 2 +- projects/internals/ci/ci-profile.js | 25 ++++++- projects/internals/ci/ci-profile.test.js | 72 +++++++++++++++++++ .../metadata/src/tasks/wireit.utils.test.ts | 10 ++- .../metadata/src/tasks/wireit.utils.ts | 12 +++- projects/internals/tools/src/index.test.ts | 2 +- projects/pages/index.js | 49 +++++++++++-- projects/site/src/sitemap/index.test.ts | 2 +- 8 files changed, 160 insertions(+), 14 deletions(-) create mode 100644 projects/internals/ci/ci-profile.test.js diff --git a/projects/cli/src/index.test.ts b/projects/cli/src/index.test.ts index d0206656ed..fec261c55f 100644 --- a/projects/cli/src/index.test.ts +++ b/projects/cli/src/index.test.ts @@ -17,7 +17,7 @@ const NATIVE_BINARY_FILENAMES = new Map([ ['win32-x64', 'nve-windows-x64.exe'] ]); -describe('index', () => { +describe('index', { timeout: 15_000 }, () => { const output = execFileSync(process.execPath, ['dist/index.js']).toString(); it('should run the native binary for the current platform', () => { diff --git a/projects/internals/ci/ci-profile.js b/projects/internals/ci/ci-profile.js index e8f75bd5b1..60fda8769f 100644 --- a/projects/internals/ci/ci-profile.js +++ b/projects/internals/ci/ci-profile.js @@ -217,6 +217,10 @@ export function renderReport({ metadata, runs, scripts }) { const medianCiMs = median(runs.map(run => run.durationMs)); const generatedAt = new Date(metadata.generatedAt).toISOString(); const target = metadata.target ?? 'ci'; + const concurrency = + metadata.wireitParallel && metadata.wireitParallel !== 'default' + ? `\`WIREIT_PARALLEL=${metadata.wireitParallel}\`` + : "Wireit's default concurrency"; const rows = topScripts.map( (script, index) => `| ${index + 1} | \`${script.id}\` | ${seconds(script.medianMs)} s | ${seconds(script.minMs)} s | ${seconds(script.maxMs)} s | ${percent(script.medianMs, medianCiMs)} | \`${escapeTable(script.command)}\` |` @@ -234,7 +238,7 @@ export function renderReport({ metadata, runs, scripts }) { '', '## Methodology', '', - `The profiler used reset mode \`${metadata.reset ?? 'cold'}\` for ${runs.length} samples, then timed \`pnpm run ${target}\` with \`CI=true\`, \`WIREIT_CACHE=none\`, \`PAGES_BASE_URL=/elements/\`, and Wireit's default concurrency. Cold mode runs \`pnpm run ci:reset\`; none mode preserves existing outputs and incremental freshness. Reset and dependency installation time is separate from execution time. Lighthouse samples must execute every test command in the current Wireit graph. Rankings exclude dependency wait time, cache hits, commandless targets, and composite commands.`, + `The profiler used reset mode \`${metadata.reset ?? 'cold'}\` for ${runs.length} samples, then timed \`pnpm run ${target}\` with \`CI=true\`, \`WIREIT_CACHE=none\`, \`PAGES_BASE_URL=/elements/\`, and ${concurrency}. Cold mode runs \`pnpm run ci:reset\`; none mode preserves existing outputs and incremental freshness. Reset and dependency installation time is separate from execution time. Lighthouse samples must execute every test command in the current Wireit graph. Rankings exclude dependency wait time, cache hits, commandless targets, and composite commands.`, '', `Environment: ${metadata.platform} ${metadata.release}, ${metadata.cpuModel}, ${metadata.cpuCount} logical CPUs, ${metadata.memoryGiB} GiB memory, Node ${metadata.node}, pnpm ${metadata.pnpm}, Wireit ${metadata.wireit}.`, '', @@ -304,6 +308,7 @@ function getMetadata() { node: process.version, pnpm: runSync('pnpm', ['--version']), wireit: wireitPackage.version, + wireitParallel: process.env.WIREIT_PARALLEL ?? 'default', profilerSha256: createHash('sha256') .update(readFileSync(fileURLToPath(import.meta.url))) .digest('hex') @@ -507,6 +512,12 @@ async function benchmarkRun(index, temporaryDirectory, options, graph) { /** Preserves complete or partial samples, including the documented run/reset log names. */ export function copyProfileArtifacts(temporaryDirectory, destination = METRICS_DIR) { mkdirSync(destination, { recursive: true }); + for (const entry of readdirSync(destination, { withFileTypes: true })) { + if (!entry.isFile() && !entry.isSymbolicLink()) continue; + if (/^ci-profile-(?:run|reset|phases)-\d+(?:[.-]|$)/.test(entry.name)) { + rmSync(path.join(destination, entry.name), { force: true }); + } + } for (const fileName of readdirSync(temporaryDirectory)) { if (fileName === 'vale-styles') continue; if (fileName.startsWith('phases-')) { @@ -679,7 +690,17 @@ export async function main() { } } for (let index = 1; index <= options.samples; index++) { - runs.push(await benchmarkRun(index, temporaryDirectory, options, metadata.graph)); + const run = await benchmarkRun(index, temporaryDirectory, options, metadata.graph); + runs.push(run); + const runMetadata = { + ...metadata, + wireit: JSON.parse(readFileSync(path.join(ROOT_DIR, 'node_modules/wireit/package.json'), 'utf8')).version + }; + // Keep successful samples available even when a later sample fails. + writeFileSync( + path.join(temporaryDirectory, `run-${index}.json`), + JSON.stringify({ metadata: runMetadata, run }, null, 2) + ); } // Reset replaces the comparison's bootstrap dependencies with this revision's. metadata.wireit = JSON.parse(readFileSync(path.join(ROOT_DIR, 'node_modules/wireit/package.json'), 'utf8')).version; diff --git a/projects/internals/ci/ci-profile.test.js b/projects/internals/ci/ci-profile.test.js new file mode 100644 index 0000000000..cdde4a27af --- /dev/null +++ b/projects/internals/ci/ci-profile.test.js @@ -0,0 +1,72 @@ +import assert from 'node:assert/strict'; +import { mkdirSync, mkdtempSync, readdirSync, readFileSync, rmSync, writeFileSync } from 'node:fs'; +import { tmpdir } from 'node:os'; +import path from 'node:path'; +import { it } from 'node:test'; +import { copyProfileArtifacts } from './ci-profile.js'; + +it('replaces numbered sample artifacts while preserving canonical reports', t => { + const root = mkdtempSync(path.join(tmpdir(), 'ci-profile-artifacts-')); + t.after(() => rmSync(root, { recursive: true, force: true })); + const source = path.join(root, 'source'); + const destination = path.join(root, 'destination'); + mkdirSync(source); + mkdirSync(destination); + + const preserved = { + 'ci-profile.json': '{"canonical":true}', + 'ci-profile.md': '# Existing report\n', + 'ci-profile-run-summary.json': 'summary', + 'notes.txt': 'notes' + }; + for (const [name, contents] of Object.entries(preserved)) { + writeFileSync(path.join(destination, name), contents); + } + for (const name of [ + 'ci-profile-run-1.json', + 'ci-profile-run-1-ci.log.resources.json', + 'ci-profile-run-1-vale.log', + 'ci-profile-run-2-ci.log', + 'ci-profile-run-2.log', + 'ci-profile-run-2-reset.log', + 'ci-profile-reset-2.log', + 'ci-profile-run-3-phases.json', + 'ci-profile-run-10.json', + 'ci-profile-phases-1-previous.jsonl', + 'ci-profile-phases-3-previous.jsonl' + ]) { + writeFileSync(path.join(destination, name), 'stale'); + } + + writeFileSync(path.join(source, 'run-1.json'), '{"run":1}'); + writeFileSync(path.join(source, 'run-1-ci.log'), 'current CI'); + writeFileSync(path.join(source, 'run-1-reset.log'), 'current reset'); + mkdirSync(path.join(source, 'phases-1')); + writeFileSync(path.join(source, 'phases-1/current.jsonl'), 'current phase'); + mkdirSync(path.join(source, 'vale-styles')); + + copyProfileArtifacts(source, destination); + + const expected = { + ...preserved, + 'ci-profile-run-1.json': '{"run":1}', + 'ci-profile-run-1-ci.log': 'current CI', + 'ci-profile-run-1.log': 'current CI', + 'ci-profile-run-1-reset.log': 'current reset', + 'ci-profile-reset-1.log': 'current reset', + 'ci-profile-phases-1-current.jsonl': 'current phase' + }; + assert.deepEqual(readdirSync(destination).sort(), Object.keys(expected).sort()); + for (const [name, contents] of Object.entries(expected)) { + assert.equal(readFileSync(path.join(destination, name), 'utf8'), contents); + } + + rmSync(source, { recursive: true }); + mkdirSync(source); + copyProfileArtifacts(source, destination); + + assert.deepEqual(readdirSync(destination).sort(), Object.keys(preserved).sort()); + for (const [name, contents] of Object.entries(preserved)) { + assert.equal(readFileSync(path.join(destination, name), 'utf8'), contents); + } +}); diff --git a/projects/internals/metadata/src/tasks/wireit.utils.test.ts b/projects/internals/metadata/src/tasks/wireit.utils.test.ts index ebadbad460..5b0688666c 100644 --- a/projects/internals/metadata/src/tasks/wireit.utils.test.ts +++ b/projects/internals/metadata/src/tasks/wireit.utils.test.ts @@ -1,7 +1,7 @@ // SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. // SPDX-License-Identifier: Apache-2.0 -import { mkdirSync, rmSync, writeFileSync } from 'node:fs'; +import { mkdirSync, rmSync, symlinkSync, writeFileSync } from 'node:fs'; import { join } from 'node:path'; import { afterAll, describe, expect, it, vi } from 'vitest'; import { generateGraphData } from './wireit.utils.js'; @@ -13,6 +13,10 @@ const missingPackageDep = '../../../../../../../../upkeep-missing-wireit-pkg:bui function installFixtures() { rmSync(fixtureRoot, { recursive: true, force: true }); + mkdirSync(fixtureRoot, { recursive: true }); + if (process.platform !== 'win32') { + symlinkSync('missing-entry-target', join(fixtureRoot, '000-missing-entry')); + } mkdirSync(join(fixturePackageDir, 'nested', 'deeper'), { recursive: true }); writeFileSync(join(fixturePackageDir, 'nested', 'package.json'), '{ not json'); writeFileSync( @@ -411,6 +415,10 @@ describe('generateGraphData', () => { expect(result.nodes.length).toBeGreaterThan(0); }); + it.skipIf(process.platform === 'win32')('should keep scanning packages after a missing filesystem entry', () => { + expect(result.nodes.some(node => node.packageName === 'labs/upkeep-wireit-fixture')).toBe(true); + }); + it('should keep generating a graph when a wireit dependency cannot be parsed', () => { expect(errorMessages.some(message => message.startsWith('error parsing dependency'))).toBe(true); }); diff --git a/projects/internals/metadata/src/tasks/wireit.utils.ts b/projects/internals/metadata/src/tasks/wireit.utils.ts index 11caa1c4c9..451e8c33a9 100644 --- a/projects/internals/metadata/src/tasks/wireit.utils.ts +++ b/projects/internals/metadata/src/tasks/wireit.utils.ts @@ -1,7 +1,7 @@ // SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. // SPDX-License-Identifier: Apache-2.0 -import { readFileSync, readdirSync, statSync } from 'fs'; +import { readFileSync, readdirSync, statSync, type Stats } from 'fs'; import { join, relative, resolve, dirname } from 'path'; const rootDir = resolve(import.meta.dirname, '../../../../../'); @@ -9,7 +9,7 @@ const wireitScripts = new Map(); // key: "package-path:script-name", value: scri function findPackageJsonFiles(dir: string, results: string[] = []): string[] { try { - const items = readdirSync(dir); + const items = readdirSync(dir).sort(); for (const item of items) { if (item === 'node_modules' || item === '.git' || item === 'dist' || item === 'coverage' || item === '.wireit') { @@ -17,7 +17,13 @@ function findPackageJsonFiles(dir: string, results: string[] = []): string[] { } const fullPath = join(dir, item); - const stat = statSync(fullPath); + let stat: Stats; + try { + stat = statSync(fullPath); + } catch { + // Continue when a concurrent build removes an entry. + continue; + } if (stat.isDirectory()) { findPackageJsonFiles(fullPath, results); diff --git a/projects/internals/tools/src/index.test.ts b/projects/internals/tools/src/index.test.ts index 8a6cbc869a..88710be597 100644 --- a/projects/internals/tools/src/index.test.ts +++ b/projects/internals/tools/src/index.test.ts @@ -28,7 +28,7 @@ describe('tools', () => { it('should be defined', async () => { await expect(getToolNames()).resolves.toBeDefined(); - }); + }, 30_000); it('should omit playground tools without runtime playground url configuration', async () => { delete process.env.ELEMENTS_PLAYGROUND_BASE_URL; diff --git a/projects/pages/index.js b/projects/pages/index.js index 68dfd727b6..ee50220638 100644 --- a/projects/pages/index.js +++ b/projects/pages/index.js @@ -1,11 +1,14 @@ -import { cpSync, existsSync, mkdirSync } from 'node:fs'; +import { spawnSync } from 'node:child_process'; +import { cpSync, existsSync, lstatSync, mkdirSync, readdirSync, rmSync, statSync } from 'node:fs'; +import { join } from 'node:path'; -if (!existsSync('./dist')) { +const freshOutput = !existsSync('./dist'); +if (freshOutput) { mkdirSync('./dist/starters', { recursive: true }); mkdirSync('./dist/api', { recursive: true }); } -cpSync('../site/dist/', './dist/', { recursive: true }); +copySite(freshOutput); cpSync('../internals/metadata/static/api.json', './dist/metadata/api.json'); cpSync('../internals/metadata/static/examples.json', './dist/metadata/examples.json'); cpSync('../internals/metadata/static/projects.json', './dist/metadata/projects.json'); @@ -18,9 +21,9 @@ if (existsSync('../cli/dist')) { const src = `../cli/dist/${bin}`; const srcExe = `../cli/dist/${bin}.exe`; if (existsSync(src)) { - cpSync(src, `./dist/cli/${bin}`); + copyBinary(src, `./dist/cli/${bin}`); } else if (existsSync(srcExe)) { - cpSync(srcExe, `./dist/cli/${bin}.exe`); + copyBinary(srcExe, `./dist/cli/${bin}.exe`); } } if (existsSync('../cli/dist/manifest.json')) { @@ -43,3 +46,39 @@ cpSync('../starters/svelte/dist/', './dist/starters/svelte/', { recursive: true cpSync('../starters/typescript/dist/', './dist/starters/typescript/', { recursive: true }); cpSync('../starters/vue/dist/', './dist/starters/vue/', { recursive: true }); cpSync('../starters/hugo/dist/', './dist/starters/hugo/', { recursive: true }); + +function copySite(freshOutput) { + const source = '../site/dist/'; + const maskedPermissions = process.umask() | 0o7000; + if ( + freshOutput && + process.platform === 'darwin' && + readdirSync(source, { recursive: true, withFileTypes: true }).every(entry => { + if (!entry.isDirectory() && !entry.isFile()) return false; + const mode = statSync(join(entry.parentPath, entry.name)).mode; + if (entry.isDirectory() && (mode & 0o700) !== 0o700) return false; + return (mode & (entry.isFile() ? maskedPermissions : 0o7000)) === 0; + }) + ) { + // macOS supports filesystem clones through cp; Node's clone flag falls back to copying here. + const result = spawnSync('/bin/cp', ['-cR', `${source}.`, './dist/'], { stdio: 'ignore' }); + if (result.status === 0) return; + // This invocation owns the fresh directory, so discard any partial clone before retrying. + rmSync('./dist/', { recursive: true, force: true }); + mkdirSync('./dist/starters', { recursive: true }); + mkdirSync('./dist/api', { recursive: true }); + } + cpSync(source, './dist/', { recursive: true }); +} + +function copyBinary(source, destination) { + if (process.platform === 'darwin' && !lstatSync(destination, { throwIfNoEntry: false })) { + const info = lstatSync(source); + if (info.isFile() && (info.mode & (process.umask() | 0o7000)) === 0) { + const result = spawnSync('/bin/cp', ['-c', source, destination], { stdio: 'ignore' }); + if (result.status === 0) return; + rmSync(destination, { force: true }); + } + } + cpSync(source, destination); +} diff --git a/projects/site/src/sitemap/index.test.ts b/projects/site/src/sitemap/index.test.ts index 0b09a3ff0d..74ba7db88c 100644 --- a/projects/site/src/sitemap/index.test.ts +++ b/projects/site/src/sitemap/index.test.ts @@ -223,5 +223,5 @@ describe('human sitemap', () => { expect(await readFile(new URL('sitemap.xml', output), 'utf8')).toContain( 'https://nvidia.github.io/elements/sitemap/' ); - }); + }, 30_000); });