Skip to content

Commit fd5cda5

Browse files
committed
chore(ci): improve lighthouse test performance
Signed-off-by: Cory Rylan <crylan@nvidia.com>
1 parent 4f2a699 commit fd5cda5

13 files changed

Lines changed: 595 additions & 81 deletions

File tree

‎.github/workflows/ci.yml‎

Lines changed: 4 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -121,7 +121,10 @@ jobs:
121121
lfs: true
122122
- uses: ./.github/actions/setup-ci
123123
- name: Run Lighthouse tests
124-
run: WIREIT_FAILURES=kill pnpm run lighthouse && node ./projects/internals/ci/cache-validate.js lighthouse && node ./projects/internals/ci/metrics.lighthouse.js
124+
run: >-
125+
WIREIT_FAILURES=kill mise exec -- pnpm run lighthouse
126+
&& mise exec -- node ./projects/internals/ci/cache-validate.js lighthouse
127+
&& mise exec -- node ./projects/internals/ci/metrics.lighthouse.js
125128
- name: Write lighthouse job summary
126129
if: always()
127130
run: node ./projects/internals/ci/lighthouse-summary.js >> "$GITHUB_STEP_SUMMARY"

‎projects/core/package.json‎

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -4,7 +4,7 @@
44
"release": {
55
"extends": "../../release.config.js"
66
},
7-
"description": "The Design Language for AI/ML Factories Building at the Speed of Light. Provides a set of accessible and themeable UI elements.",
7+
"description": "The UI SDK and agent-ready developer toolkit for AI factories, robotics, and autonomous vehicles. Provides a set of accessible and themeable UI components.",
88
"keywords": [
99
"nvidia",
1010
"web-components",

‎projects/core/vitest.lighthouse.ts‎

Lines changed: 1 addition & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -7,6 +7,7 @@ export default mergeConfig(libraryLighthouseTestConfig, {
77
alias: { '@nvidia-elements/core': resolve(import.meta.dirname, './dist') }
88
},
99
test: {
10+
maxWorkers: 4,
1011
include: ['src/**/*.test.lighthouse.ts'],
1112
outputFile: {
1213
junit: './coverage/lighthouse/junit.xml'

‎projects/internals/ci/ci-profile.js‎

Lines changed: 411 additions & 47 deletions
Large diffs are not rendered by default.

‎projects/internals/vite/src/configs/lighthouse.js‎

Lines changed: 3 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -1,4 +1,5 @@
11
import { hideExpectedTestConsoleMessage } from './console.js';
2+
import { LIGHTHOUSE_WORKERS } from '../runners/lighthouse-workers.js';
23

34
/** @type {import('vite').UserConfig} */
45
export const libraryLighthouseTestConfig = {
@@ -10,8 +11,8 @@ export const libraryLighthouseTestConfig = {
1011
test: {
1112
retry: 2,
1213
isolate: false,
13-
maxWorkers: 1,
14-
fileParallelism: false,
14+
maxWorkers: LIGHTHOUSE_WORKERS,
15+
fileParallelism: LIGHTHOUSE_WORKERS > 1,
1516
onConsoleLog: hideExpectedTestConsoleMessage,
1617
reporters: [
1718
'default',

‎projects/internals/vite/src/playwright/index.js‎

Lines changed: 6 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -2,6 +2,8 @@ import { exec } from 'node:child_process';
22
import { existsSync, writeFileSync, mkdirSync } from 'node:fs';
33
import { resolve } from 'path';
44
import * as lockFile from 'proper-lockfile';
5+
import { performance } from 'node:perf_hooks';
6+
import { recordPhase } from '../runners/profile.js';
57

68
/**
79
* creates a lockfile to force parallel playwright runners to execute one at a time, example:
@@ -22,13 +24,16 @@ if (!existsSync(lockPath)) {
2224
writeFileSync(lockPath, '');
2325
}
2426

27+
const waitStart = performance.now();
2528
const timerId = setInterval(() => {
2629
lockFile
2730
.lock(lockPath, {
2831
stale: 300000 // 5 minutes
2932
})
3033
.then(async release => {
3134
clearInterval(timerId);
35+
recordPhase('browser-lock-wait', waitStart, { lockName });
36+
const heldStart = performance.now();
3237
await new Promise(r => {
3338
exec(cmd, (error, stdout, stderr) => {
3439
console.log(stdout);
@@ -45,6 +50,7 @@ const timerId = setInterval(() => {
4550
r();
4651
});
4752
});
53+
recordPhase('browser-lock-held', heldStart, { lockName });
4854
return release();
4955
})
5056
.catch(_ => {});
Lines changed: 7 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,7 @@
1+
import { globSync } from 'node:fs';
2+
3+
export const MAX_LIGHTHOUSE_WORKERS = 4;
4+
5+
// Keep audits bounded while Wireit builds other projects. Single-file suites
6+
// use one browser because their tests still execute sequentially.
7+
export const LIGHTHOUSE_WORKERS = Math.max(1, Math.min(3, globSync('src/**/*.test.lighthouse.ts').length));

‎projects/internals/vite/src/runners/lighthouse.js‎

Lines changed: 68 additions & 11 deletions
Original file line numberDiff line numberDiff line change
@@ -3,13 +3,15 @@ import fs from 'fs';
33
import { resolve } from 'path';
44
import { VitePlaywrightRunner, buildPage } from './playwright.js';
55
import { withSoftwareWebGPUChromiumArgs } from './utils.js';
6+
import { profilePhase } from './profile.js';
7+
import { LIGHTHOUSE_WORKERS, MAX_LIGHTHOUSE_WORKERS } from './lighthouse-workers.js';
68

79
const output = process.env.CI ? ['json'] : ['json', 'html'];
810
const RUNNER_ID = 'lighthouse';
911
const LIGHTHOUSE_FLAGS = { logLevel: 'error', output };
1012
const CHROMIUM_ARGS = withSoftwareWebGPUChromiumArgs([
1113
'--headless',
12-
'--remote-debugging-port=9222',
14+
'--remote-debugging-port=0',
1315
'--disable-dev-shm-usage',
1416
'--disable-extensions',
1517
'--disable-background-networking',
@@ -38,27 +40,78 @@ const LIGHTHOUSE_CONFIG = {
3840

3941
export class LighthouseRunner {
4042
#runner;
43+
#pool = [];
44+
#chromiumArgs;
4145

4246
constructor({ chromiumArgs = CHROMIUM_ARGS } = {}) {
43-
this.#runner = new VitePlaywrightRunner({ runnerID: RUNNER_ID, chromiumArgs });
47+
this.#chromiumArgs = chromiumArgs;
48+
this.#runner = new VitePlaywrightRunner({
49+
runnerID: RUNNER_ID,
50+
instanceID: process.env.VITEST_POOL_ID ?? '1',
51+
chromiumArgs
52+
});
4453
}
4554

46-
async open() {
47-
await this.#runner.open();
55+
async open({ workers = LIGHTHOUSE_WORKERS } = {}) {
56+
if (!Number.isInteger(workers) || workers < 1 || workers > MAX_LIGHTHOUSE_WORKERS) {
57+
throw new Error(`Lighthouse requires between 1 and ${MAX_LIGHTHOUSE_WORKERS} workers`);
58+
}
59+
// Global setup owns the browsers. Each Vitest worker reads only its own ports.
60+
// The project-wide browser lock bounds machine-wide audit concurrency.
61+
this.#pool = Array.from(
62+
{ length: workers },
63+
(_, index) =>
64+
new VitePlaywrightRunner({
65+
runnerID: RUNNER_ID,
66+
instanceID: String(index + 1),
67+
chromiumArgs: this.#chromiumArgs
68+
})
69+
);
70+
try {
71+
for (const runner of this.#pool) {
72+
await profilePhase('browser-server-setup', () => runner.open());
73+
}
74+
} catch (error) {
75+
try {
76+
await this.close();
77+
} catch (cleanupError) {
78+
throw new AggregateError([error, cleanupError], 'Lighthouse setup and cleanup failed');
79+
}
80+
throw error;
81+
}
4882
}
4983

5084
async close() {
51-
await this.#runner.close();
85+
const results = await Promise.allSettled(
86+
this.#pool.map(runner => profilePhase('report-collection-cleanup', () => runner.close()))
87+
);
88+
this.#pool = [];
89+
const failures = results.filter(result => result.status === 'rejected').map(result => result.reason);
90+
if (failures.length) throw new AggregateError(failures, 'Failed to close Lighthouse browsers');
5291
}
5392

5493
async getReport(name, content) {
5594
console.log('lighthouse-runner: building test page...');
56-
await buildPage(name, RUNNER_ID, t => t.replace('</body>', `${content}</body>`));
95+
await profilePhase(
96+
'fixture-build',
97+
() => buildPage(name, RUNNER_ID, t => t.replace('</body>', `${content}</body>`)),
98+
{ name }
99+
);
57100
console.log('lighthouse-runner: running tests...');
58-
const runnerResult = await lighthouse(
59-
`http://localhost:${this.#runner.port}/${name}/index.html`,
60-
LIGHTHOUSE_FLAGS,
61-
LIGHTHOUSE_CONFIG
101+
const runnerResult = await profilePhase(
102+
'lighthouse-execution',
103+
() =>
104+
lighthouse(
105+
`http://localhost:${this.#runner.port}/${name}/index.html`,
106+
{ ...LIGHTHOUSE_FLAGS, port: this.#runner.debuggingPort },
107+
LIGHTHOUSE_CONFIG
108+
),
109+
{
110+
name,
111+
worker: process.env.VITEST_POOL_ID ?? '1',
112+
port: this.#runner.port,
113+
debuggingPort: this.#runner.debuggingPort
114+
}
62115
);
63116

64117
const networkRequests = runnerResult.lhr.audits['network-requests']?.details?.items;
@@ -83,7 +136,11 @@ export class LighthouseRunner {
83136
}
84137

85138
console.log('lighthouse-runner: writing test report...');
86-
fs.writeFileSync(`${dist}/report.json`, JSON.stringify(report, null, 2));
139+
await profilePhase(
140+
'report-write',
141+
async () => fs.writeFileSync(`${dist}/report.json`, JSON.stringify(report, null, 2)),
142+
{ name }
143+
);
87144
console.log(`lighthouse-runner: test report written to ${dist}/report.json`);
88145
return report;
89146
}

‎projects/internals/vite/src/runners/playwright.js‎

Lines changed: 59 additions & 16 deletions
Original file line numberDiff line numberDiff line change
@@ -19,10 +19,12 @@ export class VitePlaywrightRunner {
1919
#server;
2020
/** @type {import('playwright').Browser} */
2121
#browser;
22+
#browserServer;
2223
/** @type {import('playwright').LaunchOptions} */
2324
#chromiumArgs;
2425
/** @type {string} */
2526
#runnerID;
27+
#instanceID;
2628

2729
get page() {
2830
return this.#page;
@@ -45,7 +47,13 @@ export class VitePlaywrightRunner {
4547
}
4648

4749
get #portFile() {
48-
return `.${this.#runnerID}/port`;
50+
return `.${this.#runnerID}/port${this.#instanceID ? `-${this.#instanceID}` : ''}`;
51+
}
52+
53+
get debuggingPort() {
54+
const port = Number(fs.readFileSync(`${this.#portFile}-debugging`, 'utf8').trim());
55+
if (!Number.isInteger(port) || port < 1 || port > 65535) throw new Error('Invalid Chromium debugging port');
56+
return port;
4957
}
5058

5159
get #serverPort() {
@@ -64,6 +72,7 @@ export class VitePlaywrightRunner {
6472

6573
constructor(config = {}) {
6674
this.#runnerID = config.runnerID ?? 'playwright';
75+
this.#instanceID = config.instanceID;
6776
this.#chromiumArgs = config.chromiumArgs ?? [
6877
'--headless',
6978
'--disable-dev-shm-usage',
@@ -97,13 +106,25 @@ export class VitePlaywrightRunner {
97106
}
98107

99108
console.log('playwright-runner: launching browser');
100-
this.#browser = await chromium.launch({ args: this.#chromiumArgs }).catch(error => {
101-
console.error('playwright-runner: error launching browser', error);
102-
throw error;
103-
});
109+
// Chromium reserves the debugging port itself, avoiding a free-port race.
110+
if (this.#chromiumArgs.includes('--remote-debugging-port=0')) {
111+
this.#browserServer = await chromium.launchServer({
112+
args: this.#chromiumArgs
113+
});
114+
this.#browser = await chromium.connect(this.#browserServer.wsEndpoint());
115+
const args = this.#browserServer.process().spawnargs;
116+
const userDataDir = args.find(arg => arg.startsWith('--user-data-dir='))?.slice('--user-data-dir='.length);
117+
if (!userDataDir) throw new Error('Chromium did not expose its isolated profile directory');
118+
const port = fs.readFileSync(path.join(userDataDir, 'DevToolsActivePort'), 'utf8').split('\n')[0];
119+
fs.writeFileSync(`${this.#portFile}-debugging`, port);
120+
} else {
121+
this.#browser = await chromium.launch({ args: this.#chromiumArgs });
122+
}
104123
console.log('playwright-runner: creating context');
105124
this.#page = await (
106-
await this.#browser.newContext({ viewport: { width: VIEWPORT_WIDTH, height: VIEWPORT_HEIGHT } })
125+
await this.#browser.newContext({
126+
viewport: { width: VIEWPORT_WIDTH, height: VIEWPORT_HEIGHT }
127+
})
107128
).newPage();
108129
this.#page.on('crash', data => console.error('playwright-runner: browser crashed', data));
109130
console.log('playwright-runner: creating server');
@@ -117,21 +138,43 @@ export class VitePlaywrightRunner {
117138
}
118139

119140
async close() {
120-
const report = collectReports(this.#root);
121-
122-
fs.writeFileSync(`${this.#dist}/report.json`, JSON.stringify(report, null, 2));
123-
await this.#page.close();
124-
await this.#browser.close();
125-
await this.#server.close();
141+
const failures = [];
142+
try {
143+
if (fs.existsSync(this.#dist)) {
144+
const report = collectReports(this.#root);
145+
fs.writeFileSync(`${this.#dist}/report.json`, JSON.stringify(report, null, 2));
146+
}
147+
} catch (error) {
148+
failures.push(error);
149+
}
150+
for (const close of [
151+
() => this.#page?.close(),
152+
() => this.#browser?.close(),
153+
() => this.#browserServer?.close(),
154+
() => this.#server?.close()
155+
]) {
156+
try {
157+
await close();
158+
} catch (error) {
159+
failures.push(error);
160+
}
161+
}
162+
this.#page = undefined;
163+
this.#browser = undefined;
164+
this.#browserServer = undefined;
165+
this.#server = undefined;
166+
if (failures.length) throw new AggregateError(failures, 'Failed to close Playwright runner');
126167
}
127168
}
128169

129170
/** Collects individual runner reports without re-reading the aggregate report. */
130171
export function collectReports(root) {
131-
return globSync(path.resolve(root, 'dist/*/report.json')).reduce((reports, reportPath) => {
132-
const report = JSON.parse(fs.readFileSync(reportPath));
133-
return { ...reports, [report.name]: report };
134-
}, {});
172+
return globSync(path.resolve(root, 'dist/*/report.json'))
173+
.sort()
174+
.reduce((reports, reportPath) => {
175+
const report = JSON.parse(fs.readFileSync(reportPath));
176+
return { ...reports, [report.name]: report };
177+
}, {});
135178
}
136179

137180
/**
Lines changed: 29 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,29 @@
1+
import { appendFileSync, mkdirSync } from 'node:fs';
2+
import path from 'node:path';
3+
import { performance } from 'node:perf_hooks';
4+
5+
/** Records optional runner timings outside cached test outputs. */
6+
export function recordPhase(phase, start, details = {}) {
7+
const directory = process.env.LIGHTHOUSE_PROFILE_DIR;
8+
if (!directory) return;
9+
mkdirSync(directory, { recursive: true });
10+
appendFileSync(
11+
path.join(directory, `${process.pid}.jsonl`),
12+
`${JSON.stringify({
13+
project: process.cwd(),
14+
phase,
15+
startTime: performance.timeOrigin + start,
16+
durationMs: performance.now() - start,
17+
...details
18+
})}\n`
19+
);
20+
}
21+
22+
export async function profilePhase(phase, operation, details) {
23+
const start = performance.now();
24+
try {
25+
return await operation();
26+
} finally {
27+
recordPhase(phase, start, details);
28+
}
29+
}

0 commit comments

Comments
 (0)