From a23040d6cc1762f42c44de2a7d4796bef6ea654c Mon Sep 17 00:00:00 2001 From: quality Date: Thu, 8 Oct 2026 07:58:51 -0400 Subject: [PATCH 1/3] test(e2e): give each fixture overlay directory its own coverage build The end-to-end fixture mechanism supported exactly two builds: the ordinary coverage build, and one "variant" build selected by E2E_COVERAGE_VARIANT=1. That ceiling is reached. ReferenceArchitectures' absent-revision guard needs sources.architectures.revision cleared while metrics.generatedAt stays present, and the variant build already clears generatedAt to reach the same component's ': null' syncDate arm -- an arm that lives inside the guard. One build cannot hold both shapes, so covering either arm un-covers the other. Replace the boolean with a name: every tests/e2e/fixtures/data-/ directory declares one build, compiled into build/e2e-coverage- under base URL /e2e-coverage-/ with E2E_COVERAGE_BUILD=. The build list is the directory listing rather than a registry, so adding a build is adding a directory. An unknown name is an error rather than a silent fall-back to the ordinary build, which would otherwise serve a site that covers nothing new. The overlay engine, the base fixtures and the coverage report's containment fold are unchanged -- the report already resolves scripts from the whole build/ tree, so it needed no edit. tests/e2e/fixtures/data-variants/ moves to data-variant/ so the directory, the build name, the base URL and the output directory are all derived from one name. No route changes. Then the first consumer of the third build: data-no-revision/ clears sources.architectures.revision, and a spec pairs /architectures with /e2e-coverage-no-revision/architectures, asserting the provenance line renders on the first and is absent on the second. Verified at this revision: npm run build:e2e:coverage compiles all three sites, and the rendered architectures/index.html carries the provenance line with a 3ddf917 revision in build/ and build/e2e-coverage-variant/ and carries neither in build/e2e-coverage-no-revision/, while the catalog section itself still renders there. npm run test:unit is 2083/2083 and npm run check is clean. Signed-off-by: quality --- CONTRIBUTING.md | 23 ++++ package.json | 4 +- tests/e2e-data-fixture-integrity.test.mjs | 27 ++-- tests/e2e-data-fixture-loader.test.mjs | 22 ++-- tests/e2e-data-fixtures.test.mjs | 123 ++++++++++++------ tests/e2e/case-studies-variant.spec.js | 4 +- tests/e2e/data-variants.spec.js | 4 +- .../fixtures/data-no-revision/metrics.json | 6 + .../awards.json | 0 .../case-studies.json | 0 .../community-groups.json | 0 .../community-people.json | 0 .../members.json | 0 .../metrics.json | 0 .../radar-reports.json | 0 tests/e2e/group-link-status-variant.spec.js | 4 +- ...member-directory-freshness-variant.spec.js | 4 +- .../metrics-empty-collections-variant.spec.js | 4 +- tests/e2e/radar-reports-variant.spec.js | 4 +- ...eference-architectures-no-revision.spec.js | 110 ++++++++++++++++ tests/tools/e2e-coverage-builds.mjs | 65 +++++++++ tests/tools/e2e-data-fixture-loader.cjs | 6 +- tests/tools/e2e-data-fixtures.cjs | 104 +++++++++++---- 23 files changed, 413 insertions(+), 101 deletions(-) create mode 100644 tests/e2e/fixtures/data-no-revision/metrics.json rename tests/e2e/fixtures/{data-variants => data-variant}/awards.json (100%) rename tests/e2e/fixtures/{data-variants => data-variant}/case-studies.json (100%) rename tests/e2e/fixtures/{data-variants => data-variant}/community-groups.json (100%) rename tests/e2e/fixtures/{data-variants => data-variant}/community-people.json (100%) rename tests/e2e/fixtures/{data-variants => data-variant}/members.json (100%) rename tests/e2e/fixtures/{data-variants => data-variant}/metrics.json (100%) rename tests/e2e/fixtures/{data-variants => data-variant}/radar-reports.json (100%) create mode 100644 tests/e2e/reference-architectures-no-revision.spec.js create mode 100644 tests/tools/e2e-coverage-builds.mjs diff --git a/CONTRIBUTING.md b/CONTRIBUTING.md index b24425db..bd56d50e 100644 --- a/CONTRIBUTING.md +++ b/CONTRIBUTING.md @@ -282,6 +282,29 @@ format, and `tests/e2e/data-fixtures.spec.js` for the specs that drive the branches. A spec that asserts against a data file should read it through `loadSiteData()` so it describes the build it is running against. +An overlay in `tests/e2e/fixtures/data/` only helps where the missing shape is +an _additional_ record. A branch that turns on a document-level field — an +awards file with no verification date, an architectures source with no resolved +revision — cannot be reached that way: clearing the field swaps which arm the +one page renders rather than adding a case, trading covered lines for the arm it +displaces. Those branches get a build of their own. Every directory named +`tests/e2e/fixtures/data-/` declares one: `npm run build:e2e:coverage` +compiles it into `build/e2e-coverage-` with those overlays layered on top +of `tests/e2e/fixtures/data/`, under base URL `/e2e-coverage-/`. One +`docusaurus serve` offers every site at once, so a single Playwright run visits +the real page and each fixture build, and the report unions what each reached — +the builds compile the same `src/**` sources, so their scripts fold onto the +same lines. + +Adding a build is adding a directory; nothing else has to be told about it. Pair +the new route with the real one in the spec — on its own, an assertion that a +page omits something passes just as well when the page is broken. Each build is +a full Docusaurus compile, so the coverage job's wall time grows linearly in the +number of these directories: add one when the shape it needs provably conflicts +with every existing build, not as the first reach for a branch an additive +overlay could cover instead. `tests/e2e/data-variants.spec.js` and +`tests/e2e/reference-architectures-no-revision.spec.js` are worked examples. + Data overlays reach only components that read `data/*.json`. A branch whose props arrive through generated MDX — the architecture pages under `docs/architectures/` are committed output of `npm run import:architectures`, so diff --git a/package.json b/package.json index 656fe65b..3f53c1d9 100644 --- a/package.json +++ b/package.json @@ -72,9 +72,9 @@ "test:unit:coverage": "TZ=UTC node tests/tools/coverage-report.mjs", "test:unit:coverage:check": "TZ=UTC node tests/tools/coverage-report.mjs --check 99 --check-source 100 --check-regions 95 --check-source-regions 99 --check-source-file-regions 97 --require-source-files", "test:e2e": "playwright test", - "build:e2e:coverage": "npm run build:e2e:coverage:site && npm run build:e2e:coverage:variant", + "build:e2e:coverage": "npm run build:e2e:coverage:site && npm run build:e2e:coverage:fixtures", "build:e2e:coverage:site": "DOCUSAURUS_NO_PERSISTENT_CACHE=1 E2E_COVERAGE=1 npm run build:production", - "build:e2e:coverage:variant": "DOCUSAURUS_NO_PERSISTENT_CACHE=1 E2E_COVERAGE=1 E2E_COVERAGE_VARIANT=1 BASE_URL=/e2e-coverage-variant/ npm run docus:build -- --out-dir build/e2e-coverage-variant", + "build:e2e:coverage:fixtures": "node tests/tools/e2e-coverage-builds.mjs", "test:e2e:coverage": "E2E_COVERAGE=1 playwright test --workers=2", "report:e2e:coverage": "node tests/tools/e2e-coverage-report.mjs" }, diff --git a/tests/e2e-data-fixture-integrity.test.mjs b/tests/e2e-data-fixture-integrity.test.mjs index 248a057d..4798c80f 100644 --- a/tests/e2e-data-fixture-integrity.test.mjs +++ b/tests/e2e-data-fixture-integrity.test.mjs @@ -11,8 +11,9 @@ // which looks the same from CI as a branch that was never covered. // // The cases below are therefore derived from the contents of -// tests/e2e/fixtures/data/ and tests/e2e/fixtures/data-variants/, so a newly -// committed overlay is held to them without anyone remembering to register it. +// tests/e2e/fixtures/data/ and of every tests/e2e/fixtures/data-/ build +// directory, so a newly committed overlay -- and a newly committed build -- is +// held to them without anyone remembering to register it. // // This is deliberately not an assertion about *which* branch an overlay // reaches — that belongs with the spec that drives it. It is the weaker @@ -27,8 +28,9 @@ import test from 'node:test'; import { DATA_DIR, FIXTURE_DIR, - VARIANT_FIXTURE_DIR, applyOverlay, + coverageBuildNames, + overlayDirFor, } from './tools/e2e-data-fixtures.cjs'; const REPO_ROOT = new URL('..', import.meta.url).pathname; @@ -42,23 +44,32 @@ function overlaysIn(dir) { } const FIXTURE_OVERLAYS = overlaysIn(FIXTURE_DIR); -const VARIANT_OVERLAYS = overlaysIn(VARIANT_FIXTURE_DIR); -const ALL_OVERLAYS = [...FIXTURE_OVERLAYS, ...VARIANT_OVERLAYS]; +const BUILD_DIRS = coverageBuildNames().map(overlayDirFor); +const BUILD_OVERLAYS = BUILD_DIRS.flatMap(overlaysIn); +const ALL_OVERLAYS = [...FIXTURE_OVERLAYS, ...BUILD_OVERLAYS]; const label = (overlayPath) => relative(REPO_ROOT, overlayPath); // A directory that has gone empty would make every test below vacuous: each // one iterates the list, so zero overlays means zero assertions and a green // run that proves nothing. -test('both fixture directories hold at least one committed overlay', () => { +test('every fixture directory holds at least one committed overlay', () => { assert.ok( FIXTURE_OVERLAYS.length > 0, `${label(FIXTURE_DIR)} holds no overlay; the tests below would assert nothing`, ); assert.ok( - VARIANT_OVERLAYS.length > 0, - `${label(VARIANT_FIXTURE_DIR)} holds no overlay; the tests below would assert nothing`, + BUILD_DIRS.length > 0, + 'no tests/e2e/fixtures/data-/ build directory; the tests below would assert nothing', ); + // An empty build directory is worse than no build directory: it still costs + // a full Docusaurus compile in the coverage job, and the site it produces + // is byte-for-byte the ordinary coverage build. + for (const dir of BUILD_DIRS) + assert.ok( + overlaysIn(dir).length > 0, + `${label(dir)} holds no overlay; its build would compile the ordinary coverage site again`, + ); }); // overlayPathFor maps data/ to /, so an overlay whose name diff --git a/tests/e2e-data-fixture-loader.test.mjs b/tests/e2e-data-fixture-loader.test.mjs index 3a048685..62914fa6 100644 --- a/tests/e2e-data-fixture-loader.test.mjs +++ b/tests/e2e-data-fixture-loader.test.mjs @@ -43,7 +43,7 @@ const loader = require('./tools/e2e-data-fixture-loader.cjs'); const { DATA_DIR, FIXTURE_DIR, - VARIANT_FIXTURE_DIR, + overlayDirFor, overlayPathsFor, } = require('./tools/e2e-data-fixtures.cjs'); @@ -64,21 +64,21 @@ function run(relativeDataPath, { variant = false } = {}) { const resourcePath = join(DATA_DIR, relativeDataPath); const source = readFileSync(resourcePath, 'utf8'); const context = loaderContext(resourcePath); - // The loader reads the variant flag through process.env, so setting it here - // is the only way to drive the second pass. A unit run never arrives with it - // set -- the variant build is a separate pass of `npm run + // The loader reads the build name through process.env, so setting it here + // is the only way to drive a fixture pass. A unit run never arrives with it + // set -- each fixture build is a separate pass of `npm run // build:e2e:coverage` -- so the helper asserts that and restores by // deleting, rather than carrying a restore branch no test can reach. assert.equal( - process.env.E2E_COVERAGE_VARIANT, + process.env.E2E_COVERAGE_BUILD, undefined, - 'E2E_COVERAGE_VARIANT leaked into the unit run', + 'E2E_COVERAGE_BUILD leaked into the unit run', ); - if (variant) process.env.E2E_COVERAGE_VARIANT = '1'; + if (variant) process.env.E2E_COVERAGE_BUILD = 'variant'; try { return { source, context, patched: loader.call(context, source) }; } finally { - delete process.env.E2E_COVERAGE_VARIANT; + delete process.env.E2E_COVERAGE_BUILD; } } @@ -113,10 +113,10 @@ test('the variant build layers both overlays and registers both', () => { // community-people.json is the one data file carrying an overlay in each // directory, so it is the only path on which the ordering is observable. assert.deepEqual( - overlayPathsFor(resourcePath, { E2E_COVERAGE_VARIANT: '1' }), + overlayPathsFor(resourcePath, { E2E_COVERAGE_BUILD: 'variant' }), [ join(FIXTURE_DIR, 'community-people.json'), - join(VARIANT_FIXTURE_DIR, 'community-people.json'), + join(overlayDirFor('variant'), 'community-people.json'), ], ); @@ -124,7 +124,7 @@ test('the variant build layers both overlays and registers both', () => { assert.deepEqual(context.dependencies, [ join(FIXTURE_DIR, 'community-people.json'), - join(VARIANT_FIXTURE_DIR, 'community-people.json'), + join(overlayDirFor('variant'), 'community-people.json'), ]); // The variant overlay empties fetchedAt and is applied second, so seeing it // win proves the base overlay did not overwrite it on the way past. diff --git a/tests/e2e-data-fixtures.test.mjs b/tests/e2e-data-fixtures.test.mjs index 50f7c273..78bf5041 100644 --- a/tests/e2e-data-fixtures.test.mjs +++ b/tests/e2e-data-fixtures.test.mjs @@ -12,16 +12,17 @@ // instead of taking the coverage with it. import assert from 'node:assert/strict'; -import { readFileSync } from 'node:fs'; +import { readFileSync, statSync } from 'node:fs'; import { join } from 'node:path'; import test from 'node:test'; import { DATA_DIR, FIXTURE_DIR, - VARIANT_FIXTURE_DIR, applyOverlay, + coverageBuildNames, loadSiteData, + overlayDirFor, overlayDirs, overlayPathFor, overlayPathsFor, @@ -370,39 +371,72 @@ test('overlaySource reports the overlay it applied', () => { ); }); -// The variant build is a second site compiled from the same sources and +// A named fixture build is a second site compiled from the same sources and // served beside the first, which is the only way to cover a branch that turns // on a document-level field: clearing it in the one coverage build would swap // which arm the one page renders rather than add a case. The directory is -// additive and second, so a variant overlay patches what the ordinary -// coverage build already produced. -test('the variant overlay directory is additive and applies last', () => { +// additive and second, so its overlay patches what the ordinary coverage +// build already produced. +test('a named build directory is additive and applies last', () => { assert.deepEqual(overlayDirs({}), [FIXTURE_DIR]); - assert.deepEqual(overlayDirs({ E2E_COVERAGE_VARIANT: '1' }), [ + assert.deepEqual(overlayDirs({ E2E_COVERAGE_BUILD: 'variant' }), [ FIXTURE_DIR, - VARIANT_FIXTURE_DIR, + overlayDirFor('variant'), ]); }); +// Every build name is a directory and every directory is a build: the list is +// what package.json's build pass loops over, so a directory missing from it +// would be a site nothing compiles and a spec asserting against a route that +// 404s. +test('the build names are the data-* fixture directories', () => { + const names = coverageBuildNames(); + assert.ok( + names.includes('variant'), + 'the variant build must still be declared', + ); + assert.ok( + names.includes('no-revision'), + 'the no-revision build must still be declared', + ); + assert.deepEqual(names, [...names].sort(), 'build order must be stable'); + for (const name of names) + assert.ok( + statSync(overlayDirFor(name)).isDirectory(), + `${name}: no overlay directory`, + ); +}); + +// A misspelled name must not quietly compile the ordinary coverage build +// under a second base URL: that site serves, passes a smoke test, and covers +// nothing the real build did not already cover. +test('an unknown build name is an error, not a silent ordinary build', () => { + assert.throws( + () => overlayDirs({ E2E_COVERAGE_BUILD: 'no-such-build' }), + /names no overlay directory/, + ); +}); + test('a data file patched by both directories collects both overlays', () => { const path = join(DATA_DIR, 'awards.json'); assert.deepEqual(overlayPathsFor(path, {}), []); - assert.deepEqual(overlayPathsFor(path, { E2E_COVERAGE_VARIANT: '1' }), [ - join(VARIANT_FIXTURE_DIR, 'awards.json'), + assert.deepEqual(overlayPathsFor(path, { E2E_COVERAGE_BUILD: 'variant' }), [ + join(overlayDirFor('variant'), 'awards.json'), ]); const groups = join(DATA_DIR, 'community-groups.json'); - assert.deepEqual(overlayPathsFor(groups, { E2E_COVERAGE_VARIANT: '1' }), [ + assert.deepEqual(overlayPathsFor(groups, { E2E_COVERAGE_BUILD: 'variant' }), [ join(FIXTURE_DIR, 'community-groups.json'), - join(VARIANT_FIXTURE_DIR, 'community-groups.json'), + join(overlayDirFor('variant'), 'community-groups.json'), ]); // The contrast case: a file the ordinary coverage build overlays and the // variant build has nothing to add to still collects one path, which is - // what proves the variant directory is consulted only when it has + // what proves the named directory is consulted only when it has // something to say rather than always appended. const catalog = join(DATA_DIR, 'architectures', 'catalog.json'); - assert.deepEqual(overlayPathsFor(catalog, { E2E_COVERAGE_VARIANT: '1' }), [ - join(FIXTURE_DIR, 'architectures', 'catalog.json'), - ]); + assert.deepEqual( + overlayPathsFor(catalog, { E2E_COVERAGE_BUILD: 'variant' }), + [join(FIXTURE_DIR, 'architectures', 'catalog.json')], + ); }); // Both arms have to be reachable in one Playwright run, which is the whole @@ -426,7 +460,10 @@ test('the variant build clears the fields the ordinary build keeps', () => { ); assert.equal( field( - loadSiteData(name, { E2E_COVERAGE: '1', E2E_COVERAGE_VARIANT: '1' }), + loadSiteData(name, { + E2E_COVERAGE: '1', + E2E_COVERAGE_BUILD: 'variant', + }), ), cleared, `${name}: the variant build must clear it`, @@ -439,7 +476,7 @@ test('the variant build clears the fields the ordinary build keeps', () => { !field( loadSiteData(name, { E2E_COVERAGE: '1', - E2E_COVERAGE_VARIANT: '1', + E2E_COVERAGE_BUILD: 'variant', }), ), `${name}: the cleared value must be falsy`, @@ -538,33 +575,37 @@ test('the coverage build does not share a bundler cache with the real build', () const scripts = JSON.parse( readFileSync(new URL('../package.json', import.meta.url), 'utf8'), ).scripts; - // Both passes compile different data from the same sources, so the second - // would replay the first's cache just as readily as it would the production - // build's. - for (const name of [ - 'build:e2e:coverage:site', - 'build:e2e:coverage:variant', - ]) { - assert.match( - scripts[name], - /DOCUSAURUS_NO_PERSISTENT_CACHE=1/, - `${name} must opt out of the shared bundler cache`, - ); - } + const driver = readFileSync( + new URL('./tools/e2e-coverage-builds.mjs', import.meta.url), + 'utf8', + ); + // The site pass compiles different data from the same sources, so the + // fixture passes would replay its cache just as readily as the production + // build's -- and so would it replay theirs. assert.match( - scripts['build:e2e:coverage'], - /build:e2e:coverage:site(?:.|\n)*build:e2e:coverage:variant/, - 'build:e2e:coverage must run the site build before the variant build', + scripts['build:e2e:coverage:site'], + /DOCUSAURUS_NO_PERSISTENT_CACHE=1/, + 'build:e2e:coverage:site must opt out of the shared bundler cache', ); - // The variant is written inside the ordinary build output so one - // `docusaurus serve` offers both sites, and under its own base URL so the - // real routes keep their paths. assert.match( - scripts['build:e2e:coverage:variant'], - /BASE_URL=\/e2e-coverage-variant\//, + driver, + /DOCUSAURUS_NO_PERSISTENT_CACHE: '1'/, + 'every fixture build must opt out of the shared bundler cache', + ); + assert.match( + scripts['build:e2e:coverage'], + /build:e2e:coverage:site(?:.|\n)*build:e2e:coverage:fixtures/, + 'build:e2e:coverage must run the site build before the fixture builds', ); assert.match( - scripts['build:e2e:coverage:variant'], - /--out-dir build\/e2e-coverage-variant/, + scripts['build:e2e:coverage:fixtures'], + /tests\/tools\/e2e-coverage-builds\.mjs/, ); + // Each fixture build is written inside the ordinary build output so one + // `docusaurus serve` offers every site, and under its own base URL so the + // real routes keep their paths. Both are derived from the build name, which + // is what lets a new build be a new directory and nothing else. + assert.match(driver, /BASE_URL: `\/e2e-coverage-\$\{name\}\/`/); + assert.match(driver, /`build\/e2e-coverage-\$\{name\}`/); + assert.match(driver, /E2E_COVERAGE_BUILD: name/); }); diff --git a/tests/e2e/case-studies-variant.spec.js b/tests/e2e/case-studies-variant.spec.js index 1f8bab81..7db8e3e0 100644 --- a/tests/e2e/case-studies-variant.spec.js +++ b/tests/e2e/case-studies-variant.spec.js @@ -29,7 +29,7 @@ // AwardsTimeline's provenance paragraph do — see the preamble of // tests/e2e/data-variants.spec.js for the mechanism. // `npm run build:e2e:coverage` compiles a second site under -// /e2e-coverage-variant/ with tests/e2e/fixtures/data-variants/** layered on, +// /e2e-coverage-variant/ with tests/e2e/fixtures/data-variant/** layered on, // one `docusaurus serve` offers both, and the report unions what each build // reached. // @@ -62,7 +62,7 @@ const describeCoverage = process.env.E2E_COVERAGE === '1' ? test.describe : test.describe.skip; const COVERAGE_ENV = { E2E_COVERAGE: '1' }; -const VARIANT_ENV = { E2E_COVERAGE: '1', E2E_COVERAGE_VARIANT: '1' }; +const VARIANT_ENV = { E2E_COVERAGE: '1', E2E_COVERAGE_BUILD: 'variant' }; // Read the corpus through the overlay rather than hard-coding a count, so an // edited overlay fails here instead of leaving a test that asserts nothing. diff --git a/tests/e2e/data-variants.spec.js b/tests/e2e/data-variants.spec.js index 540736f2..8b8bbb13 100644 --- a/tests/e2e/data-variants.spec.js +++ b/tests/e2e/data-variants.spec.js @@ -24,7 +24,7 @@ // // `npm run build:e2e:coverage` therefore produces two sites: the ordinary // coverage build, and a second one under /e2e-coverage-variant/ with -// tests/e2e/fixtures/data-variants/** layered on top. One `docusaurus serve` +// tests/e2e/fixtures/data-variant/** layered on top. One `docusaurus serve` // offers both, so a single Playwright run visits the real page and its variant // and the report unions what each reached -- the two builds compile the same // src/** sources, so their scripts fold onto the same lines. @@ -43,7 +43,7 @@ const VARIANT_BASE = '/e2e-coverage-variant'; const describeCoverage = process.env.E2E_COVERAGE === '1' ? test.describe : test.describe.skip; -const VARIANT_ENV = { E2E_COVERAGE: '1', E2E_COVERAGE_VARIANT: '1' }; +const VARIANT_ENV = { E2E_COVERAGE: '1', E2E_COVERAGE_BUILD: 'variant' }; const COVERAGE_ENV = { E2E_COVERAGE: '1' }; // Read through the overlay rather than hard-coded, so an edited variant diff --git a/tests/e2e/fixtures/data-no-revision/metrics.json b/tests/e2e/fixtures/data-no-revision/metrics.json new file mode 100644 index 00000000..4b66c8f7 --- /dev/null +++ b/tests/e2e/fixtures/data-no-revision/metrics.json @@ -0,0 +1,6 @@ +{ + "description": "Clears sources.architectures.revision so src/components/ReferenceArchitectures/index.js takes the absent-revision guard at line 13 -- SyncStatus returns null and the whole provenance line goes with it. The field is a scalar on a singleton object of the one file the one /architectures page reads, so clearing it in the ordinary coverage build would not add a case: it would take lines 14-34 with it, including the `on ${syncDate}` arm the real build is the only one to reach. It cannot share the variant build either, which clears metrics.generatedAt to reach the ': null' syncDate arm at line 23 -- that arm is inside the guard this overlay closes, so one build cannot hold both shapes. That standoff is why the fixture mechanism takes a directory per build rather than a fixed second one. metrics.generatedAt is deliberately left present here: with it cleared this build would cover nothing the variant does not.", + "set": { + "sources.architectures.revision": null + } +} diff --git a/tests/e2e/fixtures/data-variants/awards.json b/tests/e2e/fixtures/data-variant/awards.json similarity index 100% rename from tests/e2e/fixtures/data-variants/awards.json rename to tests/e2e/fixtures/data-variant/awards.json diff --git a/tests/e2e/fixtures/data-variants/case-studies.json b/tests/e2e/fixtures/data-variant/case-studies.json similarity index 100% rename from tests/e2e/fixtures/data-variants/case-studies.json rename to tests/e2e/fixtures/data-variant/case-studies.json diff --git a/tests/e2e/fixtures/data-variants/community-groups.json b/tests/e2e/fixtures/data-variant/community-groups.json similarity index 100% rename from tests/e2e/fixtures/data-variants/community-groups.json rename to tests/e2e/fixtures/data-variant/community-groups.json diff --git a/tests/e2e/fixtures/data-variants/community-people.json b/tests/e2e/fixtures/data-variant/community-people.json similarity index 100% rename from tests/e2e/fixtures/data-variants/community-people.json rename to tests/e2e/fixtures/data-variant/community-people.json diff --git a/tests/e2e/fixtures/data-variants/members.json b/tests/e2e/fixtures/data-variant/members.json similarity index 100% rename from tests/e2e/fixtures/data-variants/members.json rename to tests/e2e/fixtures/data-variant/members.json diff --git a/tests/e2e/fixtures/data-variants/metrics.json b/tests/e2e/fixtures/data-variant/metrics.json similarity index 100% rename from tests/e2e/fixtures/data-variants/metrics.json rename to tests/e2e/fixtures/data-variant/metrics.json diff --git a/tests/e2e/fixtures/data-variants/radar-reports.json b/tests/e2e/fixtures/data-variant/radar-reports.json similarity index 100% rename from tests/e2e/fixtures/data-variants/radar-reports.json rename to tests/e2e/fixtures/data-variant/radar-reports.json diff --git a/tests/e2e/group-link-status-variant.spec.js b/tests/e2e/group-link-status-variant.spec.js index 130203b5..7f851058 100644 --- a/tests/e2e/group-link-status-variant.spec.js +++ b/tests/e2e/group-link-status-variant.spec.js @@ -33,7 +33,7 @@ // AwardsTimeline's provenance paragraph does; see the preamble of // tests/e2e/data-variants.spec.js for the mechanism. `npm run // build:e2e:coverage` compiles a second site under /e2e-coverage-variant/ -// with tests/e2e/fixtures/data-variants/** layered on, one `docusaurus serve` +// with tests/e2e/fixtures/data-variant/** layered on, one `docusaurus serve` // offers both, and the report unions what each build reached. // // The real route is asserted alongside the variant one. On its own, an @@ -56,7 +56,7 @@ const describeCoverage = process.env.E2E_COVERAGE === '1' ? test.describe : test.describe.skip; const COVERAGE_ENV = { E2E_COVERAGE: '1' }; -const VARIANT_ENV = { E2E_COVERAGE: '1', E2E_COVERAGE_VARIANT: '1' }; +const VARIANT_ENV = { E2E_COVERAGE: '1', E2E_COVERAGE_BUILD: 'variant' }; // Read the corpus through the overlays rather than hard-coding names or a // count, so an edited overlay fails here instead of leaving a test that diff --git a/tests/e2e/member-directory-freshness-variant.spec.js b/tests/e2e/member-directory-freshness-variant.spec.js index aae968df..3c9c72eb 100644 --- a/tests/e2e/member-directory-freshness-variant.spec.js +++ b/tests/e2e/member-directory-freshness-variant.spec.js @@ -18,7 +18,7 @@ // and so does the ordinary coverage build. Clearing it there would not add a // case: it would trade the membership sentence covered by // tests/e2e/member-directory-freshness.spec.js for the arms above. The -// timestamp is made unparseable in tests/e2e/fixtures/data-variants/members.json +// timestamp is made unparseable in tests/e2e/fixtures/data-variant/members.json // instead, the mechanism tests/e2e/data-variants.spec.js documents for this // class of branch, so one Playwright run visits /community/members and // /e2e-coverage-variant/community/members and the report unions what each @@ -39,7 +39,7 @@ const describeCoverage = process.env.E2E_COVERAGE === '1' ? test.describe : test.describe.skip; const COVERAGE_ENV = { E2E_COVERAGE: '1' }; -const VARIANT_ENV = { E2E_COVERAGE: '1', E2E_COVERAGE_VARIANT: '1' }; +const VARIANT_ENV = { E2E_COVERAGE: '1', E2E_COVERAGE_BUILD: 'variant' }; // Read through the overlay rather than hard-coded, so an edited variant // overlay fails here instead of leaving a test that asserts nothing. Each diff --git a/tests/e2e/metrics-empty-collections-variant.spec.js b/tests/e2e/metrics-empty-collections-variant.spec.js index 9ef5cc33..96e2531a 100644 --- a/tests/e2e/metrics-empty-collections-variant.spec.js +++ b/tests/e2e/metrics-empty-collections-variant.spec.js @@ -28,7 +28,7 @@ // AwardsTimeline's provenance paragraph does — see the preamble of // tests/e2e/data-variants.spec.js for the mechanism. `npm run // build:e2e:coverage` compiles a second site under /e2e-coverage-variant/ with -// tests/e2e/fixtures/data-variants/** layered on, one `docusaurus serve` +// tests/e2e/fixtures/data-variant/** layered on, one `docusaurus serve` // offers both, and the report unions what each build reached. // // `referenceArchitectureLifecycle.trends` at line 76 used to be the one @@ -71,7 +71,7 @@ const describeCoverage = process.env.E2E_COVERAGE === '1' ? test.describe : test.describe.skip; const COVERAGE_ENV = { E2E_COVERAGE: '1' }; -const VARIANT_ENV = { E2E_COVERAGE: '1', E2E_COVERAGE_VARIANT: '1' }; +const VARIANT_ENV = { E2E_COVERAGE: '1', E2E_COVERAGE_BUILD: 'variant' }; // Read both documents through the overlay rather than hard-coding labels or // counts, so an edited overlay fails here instead of leaving a test that diff --git a/tests/e2e/radar-reports-variant.spec.js b/tests/e2e/radar-reports-variant.spec.js index a2a8c2fc..ad42da09 100644 --- a/tests/e2e/radar-reports-variant.spec.js +++ b/tests/e2e/radar-reports-variant.spec.js @@ -26,7 +26,7 @@ // the variant build for the same reason AwardsTimeline's provenance paragraph // does — see the preamble of tests/e2e/data-variants.spec.js for the // mechanism. `npm run build:e2e:coverage` compiles a second site under -// /e2e-coverage-variant/ with tests/e2e/fixtures/data-variants/** layered on, +// /e2e-coverage-variant/ with tests/e2e/fixtures/data-variant/** layered on, // one `docusaurus serve` offers both, and the report unions what each build // reached. // @@ -63,7 +63,7 @@ const describeCoverage = process.env.E2E_COVERAGE === '1' ? test.describe : test.describe.skip; const COVERAGE_ENV = { E2E_COVERAGE: '1' }; -const VARIANT_ENV = { E2E_COVERAGE: '1', E2E_COVERAGE_VARIANT: '1' }; +const VARIANT_ENV = { E2E_COVERAGE: '1', E2E_COVERAGE_BUILD: 'variant' }; // Read the corpus through the overlay rather than hard-coding a count, so an // edited overlay fails here instead of leaving a test that asserts nothing. diff --git a/tests/e2e/reference-architectures-no-revision.spec.js b/tests/e2e/reference-architectures-no-revision.spec.js new file mode 100644 index 00000000..2b18f398 --- /dev/null +++ b/tests/e2e/reference-architectures-no-revision.spec.js @@ -0,0 +1,110 @@ +// End-to-end coverage for the absent-revision guard of +// src/components/ReferenceArchitectures/index.js, which needs a build of its +// own — and was the first branch in the repository to prove that two builds +// are not enough. +// +// The component reads data/metrics.json at module scope and opens with: +// +// 12 const architectures = metrics?.sources?.architectures; +// 13 if (!architectures?.revision) return null; +// +// The guard is an ordinary shape rather than defensive dead code: +// data/metrics.json is regenerated by `npm run collect:metrics`, and a +// collector run that reaches the architectures source but cannot resolve a +// commit emits the document without a revision. The checked-in file always +// carries one, so no browser test reaches line 13 against the shipped data. +// +// Why this cannot share the variant build. tests/e2e/fixtures/data-variant/ +// clears metrics.generatedAt, which is what makes the same component render +// the `: null` arm of its syncDate ternary (line 23) and the `''` arm beside +// it (line 33). Both of those arms live *inside* the guard this overlay +// closes: clearing revision in that build would return at line 13 and take +// lines 14–34 with it, trading the arms the variant exists for against the one +// gained. Nor can an additive overlay reach it — `add` works for a keyed +// collection, where a fixture record renders beside the real ones, and +// sources.architectures.revision is a scalar on a singleton with nothing to +// sit beside. +// +// So it gets tests/e2e/fixtures/data-no-revision/, compiled by +// `npm run build:e2e:coverage` into build/e2e-coverage-no-revision under base +// URL /e2e-coverage-no-revision/. One `docusaurus serve` offers it beside the +// real site and the variant, one Playwright run visits all three, and the +// report unions what each reached — the builds compile the same src/** +// sources, so their scripts fold onto the same lines. See the preamble of +// tests/tools/e2e-data-fixtures.cjs for the mechanism. +// +// The real route is asserted alongside the no-revision one. On its own, an +// assertion that a page has no provenance line passes just as well when the +// page failed to build or the route is wrong; pairing it with the real route +// is what makes the pair evidence that the guard *switched*. +import { test, expect } from '../tools/e2e-coverage.cjs'; +import { loadSiteData } from '../tools/e2e-data-fixtures.cjs'; + +const ARCHITECTURES_PATH = '/architectures'; +const NO_REVISION_BASE = '/e2e-coverage-no-revision'; +const SECTION_NAME = 'Reference architecture catalog'; +const SYNC_STATUS = /Last synced from/; + +// The fixture sites are only built by the coverage run; outside it the base +// path does not exist and the ordinary page still carries a revision. +const describeCoverage = + process.env.E2E_COVERAGE === '1' ? test.describe : test.describe.skip; + +const COVERAGE_ENV = { E2E_COVERAGE: '1' }; +const NO_REVISION_ENV = { + E2E_COVERAGE: '1', + E2E_COVERAGE_BUILD: 'no-revision', +}; + +// Read through the overlay rather than hard-coding, so an edited overlay fails +// here instead of leaving a test that asserts nothing. +const metrics = loadSiteData('metrics.json', COVERAGE_ENV); +const noRevisionMetrics = loadSiteData('metrics.json', NO_REVISION_ENV); + +const sectionOf = (page) => page.getByRole('region', { name: SECTION_NAME }); + +describeCoverage('an architectures source with no resolved revision', () => { + test('the real page links the commit it last synced from', async ({ + page, + }) => { + // The premise of the whole file: against the data the site ships, the + // guard below is unreachable. + expect(metrics.sources.architectures.revision).toBeTruthy(); + + await page.goto(ARCHITECTURES_PATH); + await expect(sectionOf(page)).toBeVisible(); + const syncStatus = page.getByText(SYNC_STATUS); + await expect(syncStatus).toBeVisible(); + // The short revision is what the guard gates: its presence is the + // evidence the component got past line 13 rather than merely rendering + // some paragraph. + await expect(syncStatus).toContainText( + metrics.sources.architectures.revision.slice(0, 7), + ); + }); + + test('the no-revision page drops the whole provenance line', async ({ + page, + }) => { + expect(noRevisionMetrics.sources.architectures.revision).toBeNull(); + // generatedAt stays present, so this build differs from the variant one + // in exactly the field under test. + expect(noRevisionMetrics.generatedAt).toBeTruthy(); + + await page.goto(`${NO_REVISION_BASE}${ARCHITECTURES_PATH}`); + // The route built and served: the catalog count is prose beside + // SyncStatus in the same section, so it renders either way. + const section = sectionOf(page); + await expect(section).toBeVisible(); + await expect( + section.getByText(/architecture reports from CNCF end users/), + ).toBeVisible(); + // The guard returning null is the provenance line not existing at all — + // not an empty paragraph, and not a line that merely lost its date, which + // is what tests/e2e/data-variants.spec.js pins for the variant build. + await expect(page.getByText(SYNC_STATUS)).toHaveCount(0); + await expect( + section.getByRole('link', { name: 'cncf/architecture' }), + ).toHaveCount(0); + }); +}); diff --git a/tests/tools/e2e-coverage-builds.mjs b/tests/tools/e2e-coverage-builds.mjs new file mode 100644 index 00000000..fafae6e1 --- /dev/null +++ b/tests/tools/e2e-coverage-builds.mjs @@ -0,0 +1,65 @@ +// Compiles the additional fixture builds the end-to-end coverage run serves +// beside the ordinary coverage build. +// +// `npm run build:e2e:coverage` runs the ordinary coverage build first and then +// this script, which compiles one site per `tests/e2e/fixtures/data-/` +// directory into `build/e2e-coverage-` under base URL +// `/e2e-coverage-/`. One `docusaurus serve` of build/ then offers all of +// them, so a single Playwright run visits the real pages and every fixture +// build, and tests/tools/e2e-coverage-report.mjs unions what each reached -- +// the builds compile the same `src/**` sources, so their scripts fold onto the +// same lines. +// +// The loop lives here rather than in package.json because the build list is +// the directory listing (see tests/tools/e2e-data-fixtures.cjs): a shell +// one-liner would have to repeat the names, which is exactly the registry the +// directory convention exists to avoid. +// +// Each build is a full compile, so the job's wall time grows linearly in the +// number of fixture build directories. + +import { spawnSync } from 'node:child_process'; +import { createRequire } from 'node:module'; +import { fileURLToPath } from 'node:url'; + +const require = createRequire(import.meta.url); +const { coverageBuildNames } = require('./e2e-data-fixtures.cjs'); + +const repoRoot = fileURLToPath(new URL('../..', import.meta.url)); + +function main() { + const names = coverageBuildNames(); + if (names.length === 0) { + // Not a no-op to shrug at: every variant spec asserts against a build + // under /e2e-coverage-/, so serving none of them would fail the + // suite far from the cause. + console.error( + 'No tests/e2e/fixtures/data-/ directory: the end-to-end coverage run has no fixture build to serve.', + ); + process.exit(1); + } + for (const name of names) { + console.log(`\n==> e2e coverage build "${name}"`); + const result = spawnSync( + 'npm', + ['run', 'docus:build', '--', '--out-dir', `build/e2e-coverage-${name}`], + { + cwd: repoRoot, + stdio: 'inherit', + env: { + ...process.env, + DOCUSAURUS_NO_PERSISTENT_CACHE: '1', + E2E_COVERAGE: '1', + E2E_COVERAGE_BUILD: name, + BASE_URL: `/e2e-coverage-${name}/`, + }, + }, + ); + if (result.status !== 0) { + console.error(`e2e coverage build "${name}" failed`); + process.exit(result.status === null ? 1 : result.status); + } + } +} + +main(); diff --git a/tests/tools/e2e-data-fixture-loader.cjs b/tests/tools/e2e-data-fixture-loader.cjs index e2c16905..d4408564 100644 --- a/tests/tools/e2e-data-fixture-loader.cjs +++ b/tests/tools/e2e-data-fixture-loader.cjs @@ -4,9 +4,9 @@ // docusaurus.config.js installs this on the site's data directory only when // E2E_COVERAGE=1, so the production build and the gating end-to-end job // compile the checked-in data untouched. A data file with no committed -// overlay passes through byte-for-byte. The second pass of -// `npm run build:e2e:coverage` additionally sets E2E_COVERAGE_VARIANT=1, which -// layers tests/e2e/fixtures/data-variants/** on top for that build only. +// overlay passes through byte-for-byte. Each additional pass of +// `npm run build:e2e:coverage` sets E2E_COVERAGE_BUILD=, which layers +// tests/e2e/fixtures/data-/** on top for that build only. // // See tests/tools/e2e-data-fixtures.cjs for the overlay format and for why // the branches involved are unreachable without it. diff --git a/tests/tools/e2e-data-fixtures.cjs b/tests/tools/e2e-data-fixtures.cjs index 03922253..16374472 100644 --- a/tests/tools/e2e-data-fixtures.cjs +++ b/tests/tools/e2e-data-fixtures.cjs @@ -14,14 +14,31 @@ // clearing the field swaps which arm the one page renders rather than adding // a case, trading one covered line for the arm it displaces. // -// Those branches get a *second* build instead. With E2E_COVERAGE_VARIANT=1 the -// overlays in tests/e2e/fixtures/data-variants/** are applied on top of the -// ones above, and `npm run build:e2e:coverage` compiles that variant into a -// sub-directory of the ordinary coverage build under its own base URL. One -// `docusaurus serve` then offers both sites at once: the real page keeps -// rendering the field-present arm, the variant page renders the arm beside it, -// and because the two builds compile the same `src/**` sources the coverage -// report folds their scripts onto the same lines and unions what each reached. +// Those branches get a build of their own instead. Every directory named +// `tests/e2e/fixtures/data-/` declares one additional build: with +// E2E_COVERAGE_BUILD= its overlays are applied on top of the ones above, +// and `npm run build:e2e:coverage` compiles that build into +// `build/e2e-coverage-` under base URL `/e2e-coverage-/`. One +// `docusaurus serve` then offers every site at once: the real page keeps +// rendering the field-present arm, each named build renders the arm beside it, +// and because the builds compile the same `src/**` sources the coverage report +// folds their scripts onto the same lines and unions what each reached. +// +// The build list is the directory listing, not a registry: adding +// `tests/e2e/fixtures/data-/` adds a build, and nothing else has to be +// told about it. That matters because one additional build is not enough. +// `data-variant` clears `metrics.generatedAt`, which is what reaches +// ReferenceArchitectures' `: null` syncDate arm; the same component's +// absent-revision guard needs `sources.architectures.revision` cleared while +// `generatedAt` stays present, so the two shapes cannot share a build. With +// a fixed number of builds every such pair is a standoff where covering one +// arm un-covers another; with a directory per build it is one more directory. +// +// Each build is a full Docusaurus compile, so CI time for the end-to-end +// coverage job grows linearly in the number of `data-/` directories. +// A new build is worth adding when the shape it needs provably conflicts with +// every existing one -- not as the first reach for a branch an additive +// overlay in `tests/e2e/fixtures/data/` could cover instead. // // This module applies a small, committed overlay to a data file so the missing // shapes exist in the coverage build only. It is used from two places: @@ -43,7 +60,7 @@ // shape change fails the build instead of quietly dropping the coverage it // was written for. // -// Overlay format (tests/e2e/fixtures/data[-variants]/.json): +// Overlay format (tests/e2e/fixtures/data[-]/.json): // // { // "description": "why this overlay exists", @@ -94,14 +111,9 @@ const path = require('node:path'); const REPO_ROOT = path.resolve(__dirname, '..', '..'); const DATA_DIR = path.join(REPO_ROOT, 'data'); -const FIXTURE_DIR = path.join(REPO_ROOT, 'tests', 'e2e', 'fixtures', 'data'); -const VARIANT_FIXTURE_DIR = path.join( - REPO_ROOT, - 'tests', - 'e2e', - 'fixtures', - 'data-variants', -); +const FIXTURES_ROOT = path.join(REPO_ROOT, 'tests', 'e2e', 'fixtures'); +const FIXTURE_DIR = path.join(FIXTURES_ROOT, 'data'); +const BUILD_DIR_PREFIX = 'data-'; const OVERLAY_KEYS = new Set([ 'description', @@ -126,21 +138,63 @@ function overlayPathFor(dataPath, fixtureDir = FIXTURE_DIR) { return path.join(fixtureDir, relative); } +/** + * The overlay directory of one named build. + * + * @param {string} name build name, e.g. 'variant' + * @returns {string} absolute path to tests/e2e/fixtures/data- + */ +function overlayDirFor(name) { + return path.join(FIXTURES_ROOT, `${BUILD_DIR_PREFIX}${name}`); +} + +/** + * Every named build the repository declares, in a stable order. + * + * The list is the directory listing rather than a committed registry: a build + * exists because `tests/e2e/fixtures/data-/` exists, so adding one is + * adding a directory and nothing has to be kept in step with it. Sorted so the + * build order, the base URLs and the test expectations do not depend on the + * order a filesystem happens to return. + * + * @param {string} [fixturesRoot] + * @returns {string[]} build names, without the `data-` prefix + */ +function coverageBuildNames(fixturesRoot = FIXTURES_ROOT) { + return fs + .readdirSync(fixturesRoot, { withFileTypes: true }) + .filter( + (entry) => entry.isDirectory() && entry.name.startsWith(BUILD_DIR_PREFIX), + ) + .map((entry) => entry.name.slice(BUILD_DIR_PREFIX.length)) + .sort(); +} + /** * The overlay directories a build reads, in the order they are applied. * - * The variant directory is additive and comes second, so a variant overlay + * The named build's directory is additive and comes second, so its overlay * patches the document the ordinary coverage build was already compiled from - * rather than replacing it. Only `npm run build:e2e:coverage`'s second pass - * sets E2E_COVERAGE_VARIANT. + * rather than replacing it. Only the per-build passes of + * `npm run build:e2e:coverage` set E2E_COVERAGE_BUILD; the ordinary coverage + * build leaves it unset and reads tests/e2e/fixtures/data/ alone. + * + * An unknown name is an error rather than a silent fall-back to the ordinary + * build: a typo would otherwise compile a site that looks right, serves, and + * covers nothing the real build did not already cover. * * @param {NodeJS.ProcessEnv} [env] * @returns {string[]} absolute fixture directories */ function overlayDirs(env = process.env) { - return env.E2E_COVERAGE_VARIANT === '1' - ? [FIXTURE_DIR, VARIANT_FIXTURE_DIR] - : [FIXTURE_DIR]; + const name = env.E2E_COVERAGE_BUILD; + if (!name) return [FIXTURE_DIR]; + const known = coverageBuildNames(); + if (!known.includes(name)) + throw new Error( + `E2E_COVERAGE_BUILD="${name}" names no overlay directory; expected one of ${known.join(', ')} (tests/e2e/fixtures/${BUILD_DIR_PREFIX}/)`, + ); + return [FIXTURE_DIR, overlayDirFor(name)]; } /** @@ -336,10 +390,12 @@ function loadSiteData(relativePath, env = process.env) { module.exports = { DATA_DIR, + FIXTURES_ROOT, FIXTURE_DIR, - VARIANT_FIXTURE_DIR, applyOverlay, + coverageBuildNames, loadSiteData, + overlayDirFor, overlayDirs, overlayPathFor, overlayPathsFor, From a9d8fd2d72747d809cd1ab89f60b9b7eb68e5bb5 Mon Sep 17 00:00:00 2001 From: quality Date: Thu, 8 Oct 2026 09:52:42 -0400 Subject: [PATCH 2/3] test(e2e): cover the fixture-build driver the coverage gate requires tests/tools/e2e-coverage-builds.mjs was added as a bare top-level script, so nothing could import it and no coverage record ever named it. The Validate repository gate runs the unit coverage reporter with --require-source-files, which requires every file under tests/tools/ to be exercised, and failed the run with "1 source file(s) were never measured". Give the driver the entrypoint-guard shape the sibling tools already use (tests/tools/e2e-coverage-run.mjs, tests/tools/e2e-coverage-report.mjs): export buildCommand() and main(), run main() only when the file is the process entrypoint, and return an exit code instead of calling process.exit from inside the loop. Behaviour is unchanged -- the same npm invocation, the same derived out-dir and BASE_URL, the same stop-at-first-failure. tests/e2e-coverage-builds.test.mjs covers it: a spawn stub for the ordering, the signalled-build (status null) arm and the no-fixture-directory arm, and two CLI runs with a stub npm first on PATH so the entrypoint guard and the exit code it forwards are exercised rather than described. The file now reports 100.00% lines / 100.00% regions and the gate exits 0. Signed-off-by: quality --- tests/e2e-coverage-builds.test.mjs | 200 ++++++++++++++++++++++++++++ tests/tools/e2e-coverage-builds.mjs | 92 +++++++++---- 2 files changed, 268 insertions(+), 24 deletions(-) create mode 100644 tests/e2e-coverage-builds.test.mjs diff --git a/tests/e2e-coverage-builds.test.mjs b/tests/e2e-coverage-builds.test.mjs new file mode 100644 index 00000000..2de34bb3 --- /dev/null +++ b/tests/e2e-coverage-builds.test.mjs @@ -0,0 +1,200 @@ +// Unit coverage for tests/tools/e2e-coverage-builds.mjs, the driver +// `npm run build:e2e:coverage:fixtures` runs. +// +// The compiles themselves are Docusaurus builds measured in minutes, so the +// in-process tests inject a spawn stub and assert the command, the derived +// output directory and base URL, and the stop-at-first-failure contract. The +// CLI tests run the real script with a stub `npm` first on PATH, so the +// entrypoint guard and the exit codes it forwards are exercised rather than +// described. + +import assert from 'node:assert/strict'; +import test from 'node:test'; +import { spawnSync } from 'node:child_process'; +import { chmodSync, mkdtempSync, rmSync, writeFileSync } from 'node:fs'; +import { tmpdir } from 'node:os'; +import { delimiter, join } from 'node:path'; +import { createRequire } from 'node:module'; +import { fileURLToPath } from 'node:url'; + +import { buildCommand, main } from './tools/e2e-coverage-builds.mjs'; + +const require = createRequire(import.meta.url); +const { coverageBuildNames } = require('./tools/e2e-data-fixtures.cjs'); + +const BUILDS_TOOL = fileURLToPath( + new URL('./tools/e2e-coverage-builds.mjs', import.meta.url), +); +const REPO_ROOT = fileURLToPath(new URL('..', import.meta.url)); + +function recordingSpawn(statuses = []) { + const calls = []; + let index = 0; + return { + calls, + spawn(command, args, options) { + calls.push({ command, args, options }); + const status = index < statuses.length ? statuses[index] : 0; + index += 1; + return { status }; + }, + }; +} + +function runMain(deps) { + const logs = []; + const errors = []; + const code = main({ + log: (line) => logs.push(line), + error: (line) => errors.push(line), + ...deps, + }); + return { code, logs, errors }; +} + +// A stub `npm` first on PATH: the driver shells out to `npm run docus:build`, +// and the point of the CLI tests is the driver's own control flow, not a site +// compile. +function withStubNpm(exitCode, run) { + const binDir = mkdtempSync(join(tmpdir(), 'endusers-e2e-builds-bin-')); + const log = join(binDir, 'calls.log'); + try { + const npm = join(binDir, 'npm'); + writeFileSync( + npm, + `#!/bin/sh\nprintf '%s\\n' "$E2E_COVERAGE_BUILD|$BASE_URL|$DOCUSAURUS_NO_PERSISTENT_CACHE|$E2E_COVERAGE|$*" >> ${JSON.stringify(log)}\nexit ${exitCode}\n`, + ); + chmodSync(npm, 0o755); + return run({ + env: { + ...process.env, + PATH: `${binDir}${delimiter}${process.env.PATH}`, + }, + log, + }); + } finally { + rmSync(binDir, { recursive: true, force: true }); + } +} + +test('buildCommand derives the out-dir and base URL from the build name', () => { + const { command, args, options } = buildCommand('variant', { HOME: '/home' }); + assert.equal(command, 'npm'); + assert.deepEqual(args, [ + 'run', + 'docus:build', + '--', + '--out-dir', + 'build/e2e-coverage-variant', + ]); + assert.equal(options.cwd, REPO_ROOT); + assert.equal(options.stdio, 'inherit'); + assert.equal(options.env.BASE_URL, '/e2e-coverage-variant/'); + assert.equal(options.env.E2E_COVERAGE_BUILD, 'variant'); + assert.equal(options.env.E2E_COVERAGE, '1'); + assert.equal(options.env.DOCUSAURUS_NO_PERSISTENT_CACHE, '1'); + // The build inherits the ambient environment rather than replacing it. + assert.equal(options.env.HOME, '/home'); +}); + +test('main compiles every named build, in the order it was given', () => { + const { calls, spawn } = recordingSpawn(); + const { code, logs, errors } = runMain({ + names: ['no-revision', 'variant'], + spawn, + }); + assert.equal(code, 0); + assert.deepEqual(errors, []); + assert.deepEqual( + calls.map((call) => call.options.env.E2E_COVERAGE_BUILD), + ['no-revision', 'variant'], + ); + assert.deepEqual( + calls.map((call) => call.args.at(-1)), + ['build/e2e-coverage-no-revision', 'build/e2e-coverage-variant'], + ); + assert.deepEqual(logs, [ + '\n==> e2e coverage build "no-revision"', + '\n==> e2e coverage build "variant"', + ]); +}); + +test('main stops at the first failing build and forwards its status', () => { + const { calls, spawn } = recordingSpawn([0, 3]); + const { code, errors } = runMain({ + names: ['first', 'second', 'third'], + spawn, + }); + assert.equal(code, 3); + assert.equal(calls.length, 2, 'the third build must not be started'); + assert.deepEqual(errors, ['e2e coverage build "second" failed']); +}); + +// spawnSync reports a signalled child as status null, which would otherwise +// be forwarded as a zero exit code and pass the job on a build that never +// finished. +test('main reports a signalled build as a failure', () => { + const { spawn } = recordingSpawn([null]); + const { code, errors } = runMain({ names: ['variant'], spawn }); + assert.equal(code, 1); + assert.deepEqual(errors, ['e2e coverage build "variant" failed']); +}); + +test('main fails when no fixture build directory exists', () => { + const { calls, spawn } = recordingSpawn(); + const { code, logs, errors } = runMain({ names: [], spawn }); + assert.equal(code, 1); + assert.deepEqual(calls, []); + assert.deepEqual(logs, []); + assert.match(errors[0], /no fixture build to serve/); +}); + +// The default name list is the directory listing, so the checked-in fixture +// directories are the builds the real run compiles. +test('main defaults to the committed fixture build directories', () => { + const { calls, spawn } = recordingSpawn(); + const { code } = runMain({ spawn }); + assert.equal(code, 0); + assert.ok(calls.length > 0, 'the repository must declare a fixture build'); + assert.deepEqual( + calls.map((call) => call.options.env.E2E_COVERAGE_BUILD), + coverageBuildNames(), + ); +}); + +test('the CLI compiles each build and exits 0', () => { + const { status, calls } = withStubNpm(0, ({ env, log }) => { + const result = spawnSync(process.execPath, [BUILDS_TOOL], { + env, + encoding: 'utf8', + }); + return { + status: result.status, + calls: spawnSync('cat', [log], { encoding: 'utf8' }).stdout ?? '', + }; + }); + assert.equal(status, 0); + const lines = calls.trim().split('\n').filter(Boolean); + assert.ok(lines.length > 0, 'the CLI must run at least one build'); + for (const line of lines) { + const [name, baseUrl, noCache, coverage, argv] = line.split('|'); + assert.equal(baseUrl, `/e2e-coverage-${name}/`); + assert.equal(noCache, '1'); + assert.equal(coverage, '1'); + assert.equal( + argv, + `run docus:build -- --out-dir build/e2e-coverage-${name}`, + ); + } +}); + +test('the CLI exits non-zero when a build fails', () => { + const status = withStubNpm(3, ({ env }) => { + const result = spawnSync(process.execPath, [BUILDS_TOOL], { + env, + encoding: 'utf8', + }); + return result.status; + }); + assert.equal(status, 3); +}); diff --git a/tests/tools/e2e-coverage-builds.mjs b/tests/tools/e2e-coverage-builds.mjs index fafae6e1..787e6a93 100644 --- a/tests/tools/e2e-coverage-builds.mjs +++ b/tests/tools/e2e-coverage-builds.mjs @@ -20,46 +20,90 @@ import { spawnSync } from 'node:child_process'; import { createRequire } from 'node:module'; -import { fileURLToPath } from 'node:url'; +import { fileURLToPath, pathToFileURL } from 'node:url'; const require = createRequire(import.meta.url); const { coverageBuildNames } = require('./e2e-data-fixtures.cjs'); const repoRoot = fileURLToPath(new URL('../..', import.meta.url)); -function main() { - const names = coverageBuildNames(); +/** + * The compile one named fixture build runs. + * + * Both the output directory and the base URL are derived from the build name, + * which is what lets a new build be a new `tests/e2e/fixtures/data-/` + * directory and nothing else. The build opts out of the persistent bundler + * cache because every pass compiles the same sources from different data, so + * a shared cache would replay one build's output into the next. + * + * @param {string} name build name, without the `data-` prefix + * @param {NodeJS.ProcessEnv} [env] environment the build inherits + * @returns {{command: string, args: string[], options: object}} + */ +export function buildCommand(name, env = process.env) { + return { + command: 'npm', + args: [ + 'run', + 'docus:build', + '--', + '--out-dir', + `build/e2e-coverage-${name}`, + ], + options: { + cwd: repoRoot, + stdio: 'inherit', + env: { + ...env, + DOCUSAURUS_NO_PERSISTENT_CACHE: '1', + E2E_COVERAGE: '1', + E2E_COVERAGE_BUILD: name, + BASE_URL: `/e2e-coverage-${name}/`, + }, + }, + }; +} + +/** + * Compiles every named fixture build, stopping at the first failure. + * + * @param {object} [deps] + * @param {string[]} [deps.names] builds to compile + * @param {Function} [deps.spawn] `spawnSync`-shaped runner + * @param {Function} [deps.log] progress sink + * @param {Function} [deps.error] failure sink + * @returns {number} process exit code + */ +export function main({ + names = coverageBuildNames(), + spawn = spawnSync, + log = console.log, + error = console.error, +} = {}) { if (names.length === 0) { // Not a no-op to shrug at: every variant spec asserts against a build // under /e2e-coverage-/, so serving none of them would fail the // suite far from the cause. - console.error( + error( 'No tests/e2e/fixtures/data-/ directory: the end-to-end coverage run has no fixture build to serve.', ); - process.exit(1); + return 1; } for (const name of names) { - console.log(`\n==> e2e coverage build "${name}"`); - const result = spawnSync( - 'npm', - ['run', 'docus:build', '--', '--out-dir', `build/e2e-coverage-${name}`], - { - cwd: repoRoot, - stdio: 'inherit', - env: { - ...process.env, - DOCUSAURUS_NO_PERSISTENT_CACHE: '1', - E2E_COVERAGE: '1', - E2E_COVERAGE_BUILD: name, - BASE_URL: `/e2e-coverage-${name}/`, - }, - }, - ); + log(`\n==> e2e coverage build "${name}"`); + const { command, args, options } = buildCommand(name); + const result = spawn(command, args, options); if (result.status !== 0) { - console.error(`e2e coverage build "${name}" failed`); - process.exit(result.status === null ? 1 : result.status); + error(`e2e coverage build "${name}" failed`); + return result.status === null ? 1 : result.status; } } + return 0; } -main(); +if ( + process.argv[1] && + import.meta.url === pathToFileURL(process.argv[1]).href +) { + process.exit(main()); +} From e3c95d6da87d5fde4527af7f47c6d2181ee799fc Mon Sep 17 00:00:00 2001 From: mrbobbytables Date: Fri, 9 Oct 2026 12:21:28 -0500 Subject: [PATCH 3/3] test(e2e): pin the no-revision spec in the coverage-gated inventory Co-authored-by: Copilot <223556219+Copilot@users.noreply.github.com> Signed-off-by: mrbobbytables --- tests/e2e-coverage-describe-gate.test.mjs | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/tests/e2e-coverage-describe-gate.test.mjs b/tests/e2e-coverage-describe-gate.test.mjs index ba55db12..3abfc7d5 100644 --- a/tests/e2e-coverage-describe-gate.test.mjs +++ b/tests/e2e-coverage-describe-gate.test.mjs @@ -6,7 +6,7 @@ import { fileURLToPath } from 'node:url'; import { isCoverageEnabled } from './tools/e2e-coverage.cjs'; -// Eleven end-to-end specs hold cases that only the coverage build can satisfy +// Twelve end-to-end specs hold cases that only the coverage build can satisfy // -- they navigate a route that exists only under E2E_COVERAGE=1, or they // assert against the fixture overlay that only that build layers in. Each one // gates itself by hand: @@ -58,6 +58,7 @@ const COVERAGE_GATED = [ 'metrics-empty-collections-variant.spec.js', 'metrics-sparkline.spec.js', 'radar-reports-variant.spec.js', + 'reference-architectures-no-revision.spec.js', ]; const specNames = readdirSync(specDir)