test(e2e): give each fixture overlay directory its own coverage build

CONTRIBUTING.md

@@ -286,6 +286,29 @@ format, and `tests/e2e/data-fixtures.spec.js` for the specs that drive the
 branches. A spec that asserts against a data file should read it through
 `loadSiteData()` so it describes the build it is running against.
 
+An overlay in `tests/e2e/fixtures/data/` only helps where the missing shape is
+an _additional_ record. A branch that turns on a document-level field — an
+awards file with no verification date, an architectures source with no resolved
+revision — cannot be reached that way: clearing the field swaps which arm the
+one page renders rather than adding a case, trading covered lines for the arm it
+displaces. Those branches get a build of their own. Every directory named
+`tests/e2e/fixtures/data-<name>/` declares one: `npm run build:e2e:coverage`
+compiles it into `build/e2e-coverage-<name>` with those overlays layered on top
+of `tests/e2e/fixtures/data/`, under base URL `/e2e-coverage-<name>/`. One
+`docusaurus serve` offers every site at once, so a single Playwright run visits
+the real page and each fixture build, and the report unions what each reached —
+the builds compile the same `src/**` sources, so their scripts fold onto the
+same lines.
+
+Adding a build is adding a directory; nothing else has to be told about it. Pair
+the new route with the real one in the spec — on its own, an assertion that a
+page omits something passes just as well when the page is broken. Each build is
+a full Docusaurus compile, so the coverage job's wall time grows linearly in the
+number of these directories: add one when the shape it needs provably conflicts
+with every existing build, not as the first reach for a branch an additive
+overlay could cover instead. `tests/e2e/data-variants.spec.js` and
+`tests/e2e/reference-architectures-no-revision.spec.js` are worked examples.
+
 Data overlays reach only components that read `data/*.json`. A branch whose
 props arrive through generated MDX — the architecture pages under
 `docs/architectures/` are committed output of `npm run import:architectures`, so

package.json

@@ -72,9 +72,9 @@
     "test:unit:coverage": "TZ=UTC node tests/tools/coverage-report.mjs",
     "test:unit:coverage:check": "TZ=UTC node tests/tools/coverage-report.mjs --check 99 --check-source 100 --check-regions 95 --check-source-regions 99 --check-source-file-regions 97 --check-harness 100 --check-harness-regions 97 --check-harness-file-regions 93 --require-source-files",
     "test:e2e": "playwright test",
-    "build:e2e:coverage": "npm run build:e2e:coverage:site && npm run build:e2e:coverage:variant",
+    "build:e2e:coverage": "npm run build:e2e:coverage:site && npm run build:e2e:coverage:fixtures",
     "build:e2e:coverage:site": "DOCUSAURUS_NO_PERSISTENT_CACHE=1 E2E_COVERAGE=1 npm run build:production",
-    "build:e2e:coverage:variant": "DOCUSAURUS_NO_PERSISTENT_CACHE=1 E2E_COVERAGE=1 E2E_COVERAGE_VARIANT=1 BASE_URL=/e2e-coverage-variant/ npm run docus:build -- --out-dir build/e2e-coverage-variant",
+    "build:e2e:coverage:fixtures": "node tests/tools/e2e-coverage-builds.mjs",
     "test:e2e:coverage": "E2E_COVERAGE=1 playwright test --workers=2",
     "report:e2e:coverage": "node tests/tools/e2e-coverage-report.mjs"
   },

tests/e2e-coverage-builds.test.mjs

@@ -0,0 +1,200 @@
+// Unit coverage for tests/tools/e2e-coverage-builds.mjs, the driver
+// `npm run build:e2e:coverage:fixtures` runs.
+//
+// The compiles themselves are Docusaurus builds measured in minutes, so the
+// in-process tests inject a spawn stub and assert the command, the derived
+// output directory and base URL, and the stop-at-first-failure contract. The
+// CLI tests run the real script with a stub `npm` first on PATH, so the
+// entrypoint guard and the exit codes it forwards are exercised rather than
+// described.
+
+import assert from 'node:assert/strict';
+import test from 'node:test';
+import { spawnSync } from 'node:child_process';
+import { chmodSync, mkdtempSync, rmSync, writeFileSync } from 'node:fs';
+import { tmpdir } from 'node:os';
+import { delimiter, join } from 'node:path';
+import { createRequire } from 'node:module';
+import { fileURLToPath } from 'node:url';
+
+import { buildCommand, main } from './tools/e2e-coverage-builds.mjs';
+
+const require = createRequire(import.meta.url);
+const { coverageBuildNames } = require('./tools/e2e-data-fixtures.cjs');
+
+const BUILDS_TOOL = fileURLToPath(
+  new URL('./tools/e2e-coverage-builds.mjs', import.meta.url),
+);
+const REPO_ROOT = fileURLToPath(new URL('..', import.meta.url));
+
+function recordingSpawn(statuses = []) {
+  const calls = [];
+  let index = 0;
+  return {
+    calls,
+    spawn(command, args, options) {
+      calls.push({ command, args, options });
+      const status = index < statuses.length ? statuses[index] : 0;
+      index += 1;
+      return { status };
+    },
+  };
+}
+
+function runMain(deps) {
+  const logs = [];
+  const errors = [];
+  const code = main({
+    log: (line) => logs.push(line),
+    error: (line) => errors.push(line),
+    ...deps,
+  });
+  return { code, logs, errors };
+}
+
+// A stub `npm` first on PATH: the driver shells out to `npm run docus:build`,
+// and the point of the CLI tests is the driver's own control flow, not a site
+// compile.
+function withStubNpm(exitCode, run) {
+  const binDir = mkdtempSync(join(tmpdir(), 'endusers-e2e-builds-bin-'));
+  const log = join(binDir, 'calls.log');
+  try {
+    const npm = join(binDir, 'npm');
+    writeFileSync(
+      npm,
+      `#!/bin/sh\nprintf '%s\\n' "$E2E_COVERAGE_BUILD|$BASE_URL|$DOCUSAURUS_NO_PERSISTENT_CACHE|$E2E_COVERAGE|$*" >> ${JSON.stringify(log)}\nexit ${exitCode}\n`,
+    );
+    chmodSync(npm, 0o755);
+    return run({
+      env: {
+        ...process.env,
+        PATH: `${binDir}${delimiter}${process.env.PATH}`,
+      },
+      log,
+    });
+  } finally {
+    rmSync(binDir, { recursive: true, force: true });
+  }
+}
+
+test('buildCommand derives the out-dir and base URL from the build name', () => {
+  const { command, args, options } = buildCommand('variant', { HOME: '/home' });
+  assert.equal(command, 'npm');
+  assert.deepEqual(args, [
+    'run',
+    'docus:build',
+    '--',
+    '--out-dir',
+    'build/e2e-coverage-variant',
+  ]);
+  assert.equal(options.cwd, REPO_ROOT);
+  assert.equal(options.stdio, 'inherit');
+  assert.equal(options.env.BASE_URL, '/e2e-coverage-variant/');
+  assert.equal(options.env.E2E_COVERAGE_BUILD, 'variant');
+  assert.equal(options.env.E2E_COVERAGE, '1');
+  assert.equal(options.env.DOCUSAURUS_NO_PERSISTENT_CACHE, '1');
+  // The build inherits the ambient environment rather than replacing it.
+  assert.equal(options.env.HOME, '/home');
+});
+
+test('main compiles every named build, in the order it was given', () => {
+  const { calls, spawn } = recordingSpawn();
+  const { code, logs, errors } = runMain({
+    names: ['no-revision', 'variant'],
+    spawn,
+  });
+  assert.equal(code, 0);
+  assert.deepEqual(errors, []);
+  assert.deepEqual(
+    calls.map((call) => call.options.env.E2E_COVERAGE_BUILD),
+    ['no-revision', 'variant'],
+  );
+  assert.deepEqual(
+    calls.map((call) => call.args.at(-1)),
+    ['build/e2e-coverage-no-revision', 'build/e2e-coverage-variant'],
+  );
+  assert.deepEqual(logs, [
+    '\n==> e2e coverage build "no-revision"',
+    '\n==> e2e coverage build "variant"',
+  ]);
+});
+
+test('main stops at the first failing build and forwards its status', () => {
+  const { calls, spawn } = recordingSpawn([0, 3]);
+  const { code, errors } = runMain({
+    names: ['first', 'second', 'third'],
+    spawn,
+  });
+  assert.equal(code, 3);
+  assert.equal(calls.length, 2, 'the third build must not be started');
+  assert.deepEqual(errors, ['e2e coverage build "second" failed']);
+});
+
+// spawnSync reports a signalled child as status null, which would otherwise
+// be forwarded as a zero exit code and pass the job on a build that never
+// finished.
+test('main reports a signalled build as a failure', () => {
+  const { spawn } = recordingSpawn([null]);
+  const { code, errors } = runMain({ names: ['variant'], spawn });
+  assert.equal(code, 1);
+  assert.deepEqual(errors, ['e2e coverage build "variant" failed']);
+});
+
+test('main fails when no fixture build directory exists', () => {
+  const { calls, spawn } = recordingSpawn();
+  const { code, logs, errors } = runMain({ names: [], spawn });
+  assert.equal(code, 1);
+  assert.deepEqual(calls, []);
+  assert.deepEqual(logs, []);
+  assert.match(errors[0], /no fixture build to serve/);
+});
+
+// The default name list is the directory listing, so the checked-in fixture
+// directories are the builds the real run compiles.
+test('main defaults to the committed fixture build directories', () => {
+  const { calls, spawn } = recordingSpawn();
+  const { code } = runMain({ spawn });
+  assert.equal(code, 0);
+  assert.ok(calls.length > 0, 'the repository must declare a fixture build');
+  assert.deepEqual(
+    calls.map((call) => call.options.env.E2E_COVERAGE_BUILD),
+    coverageBuildNames(),
+  );
+});
+
+test('the CLI compiles each build and exits 0', () => {
+  const { status, calls } = withStubNpm(0, ({ env, log }) => {
+    const result = spawnSync(process.execPath, [BUILDS_TOOL], {
+      env,
+      encoding: 'utf8',
+    });
+    return {
+      status: result.status,
+      calls: spawnSync('cat', [log], { encoding: 'utf8' }).stdout ?? '',
+    };
+  });
+  assert.equal(status, 0);
+  const lines = calls.trim().split('\n').filter(Boolean);
+  assert.ok(lines.length > 0, 'the CLI must run at least one build');
+  for (const line of lines) {
+    const [name, baseUrl, noCache, coverage, argv] = line.split('|');
+    assert.equal(baseUrl, `/e2e-coverage-${name}/`);
+    assert.equal(noCache, '1');
+    assert.equal(coverage, '1');
+    assert.equal(
+      argv,
+      `run docus:build -- --out-dir build/e2e-coverage-${name}`,
+    );
+  }
+});
+
+test('the CLI exits non-zero when a build fails', () => {
+  const status = withStubNpm(3, ({ env }) => {
+    const result = spawnSync(process.execPath, [BUILDS_TOOL], {
+      env,
+      encoding: 'utf8',
+    });
+    return result.status;
+  });
+  assert.equal(status, 3);
+});

tests/e2e-coverage-describe-gate.test.mjs

@@ -6,7 +6,7 @@ import { fileURLToPath } from 'node:url';
 
 import { isCoverageEnabled } from './tools/e2e-coverage.cjs';
 
-// Eleven end-to-end specs hold cases that only the coverage build can satisfy
+// Twelve end-to-end specs hold cases that only the coverage build can satisfy
 // -- they navigate a route that exists only under E2E_COVERAGE=1, or they
 // assert against the fixture overlay that only that build layers in. Each one
 // gates itself by hand:
@@ -58,6 +58,7 @@ const COVERAGE_GATED = [
   'metrics-empty-collections-variant.spec.js',
   'metrics-sparkline.spec.js',
   'radar-reports-variant.spec.js',
+  'reference-architectures-no-revision.spec.js',
 ];
 
 const specNames = readdirSync(specDir)

tests/e2e-data-fixture-integrity.test.mjs

@@ -11,8 +11,9 @@
 // which looks the same from CI as a branch that was never covered.
 //
 // The cases below are therefore derived from the contents of
-// tests/e2e/fixtures/data/ and tests/e2e/fixtures/data-variants/, so a newly
-// committed overlay is held to them without anyone remembering to register it.
+// tests/e2e/fixtures/data/ and of every tests/e2e/fixtures/data-<name>/ build
+// directory, so a newly committed overlay -- and a newly committed build -- is
+// held to them without anyone remembering to register it.
 //
 // This is deliberately not an assertion about *which* branch an overlay
 // reaches — that belongs with the spec that drives it. It is the weaker
@@ -27,8 +28,9 @@ import test from 'node:test';
 import {
   DATA_DIR,
   FIXTURE_DIR,
-  VARIANT_FIXTURE_DIR,
   applyOverlay,
+  coverageBuildNames,
+  overlayDirFor,
 } from './tools/e2e-data-fixtures.cjs';
 
 const REPO_ROOT = new URL('..', import.meta.url).pathname;
@@ -42,23 +44,32 @@ function overlaysIn(dir) {
 }
 
 const FIXTURE_OVERLAYS = overlaysIn(FIXTURE_DIR);
-const VARIANT_OVERLAYS = overlaysIn(VARIANT_FIXTURE_DIR);
-const ALL_OVERLAYS = [...FIXTURE_OVERLAYS, ...VARIANT_OVERLAYS];
+const BUILD_DIRS = coverageBuildNames().map(overlayDirFor);
+const BUILD_OVERLAYS = BUILD_DIRS.flatMap(overlaysIn);
+const ALL_OVERLAYS = [...FIXTURE_OVERLAYS, ...BUILD_OVERLAYS];
 
 const label = (overlayPath) => relative(REPO_ROOT, overlayPath);
 
 // A directory that has gone empty would make every test below vacuous: each
 // one iterates the list, so zero overlays means zero assertions and a green
 // run that proves nothing.
-test('both fixture directories hold at least one committed overlay', () => {
+test('every fixture directory holds at least one committed overlay', () => {
   assert.ok(
     FIXTURE_OVERLAYS.length > 0,
     `${label(FIXTURE_DIR)} holds no overlay; the tests below would assert nothing`,
   );
   assert.ok(
-    VARIANT_OVERLAYS.length > 0,
-    `${label(VARIANT_FIXTURE_DIR)} holds no overlay; the tests below would assert nothing`,
+    BUILD_DIRS.length > 0,
+    'no tests/e2e/fixtures/data-<name>/ build directory; the tests below would assert nothing',
   );
+  // An empty build directory is worse than no build directory: it still costs
+  // a full Docusaurus compile in the coverage job, and the site it produces
+  // is byte-for-byte the ordinary coverage build.
+  for (const dir of BUILD_DIRS)
+    assert.ok(
+      overlaysIn(dir).length > 0,
+      `${label(dir)} holds no overlay; its build would compile the ordinary coverage site again`,
+    );
 });
 
 // overlayPathFor maps data/<name> to <dir>/<name>, so an overlay whose name

tests/e2e-data-fixture-loader.test.mjs

@@ -43,7 +43,7 @@ const loader = require('./tools/e2e-data-fixture-loader.cjs');
 const {
   DATA_DIR,
   FIXTURE_DIR,
-  VARIANT_FIXTURE_DIR,
+  overlayDirFor,
   overlayPathsFor,
 } = require('./tools/e2e-data-fixtures.cjs');
 
@@ -64,21 +64,21 @@ function run(relativeDataPath, { variant = false } = {}) {
   const resourcePath = join(DATA_DIR, relativeDataPath);
   const source = readFileSync(resourcePath, 'utf8');
   const context = loaderContext(resourcePath);
-  // The loader reads the variant flag through process.env, so setting it here
-  // is the only way to drive the second pass. A unit run never arrives with it
-  // set -- the variant build is a separate pass of `npm run
+  // The loader reads the build name through process.env, so setting it here
+  // is the only way to drive a fixture pass. A unit run never arrives with it
+  // set -- each fixture build is a separate pass of `npm run
   // build:e2e:coverage` -- so the helper asserts that and restores by
   // deleting, rather than carrying a restore branch no test can reach.
   assert.equal(
-    process.env.E2E_COVERAGE_VARIANT,
+    process.env.E2E_COVERAGE_BUILD,
     undefined,
-    'E2E_COVERAGE_VARIANT leaked into the unit run',
+    'E2E_COVERAGE_BUILD leaked into the unit run',
   );
-  if (variant) process.env.E2E_COVERAGE_VARIANT = '1';
+  if (variant) process.env.E2E_COVERAGE_BUILD = 'variant';
   try {
     return { source, context, patched: loader.call(context, source) };
   } finally {
-    delete process.env.E2E_COVERAGE_VARIANT;
+    delete process.env.E2E_COVERAGE_BUILD;
   }
 }
 
@@ -113,18 +113,18 @@ test('the variant build layers both overlays and registers both', () => {
   // community-people.json is the one data file carrying an overlay in each
   // directory, so it is the only path on which the ordering is observable.
   assert.deepEqual(
-    overlayPathsFor(resourcePath, { E2E_COVERAGE_VARIANT: '1' }),
+    overlayPathsFor(resourcePath, { E2E_COVERAGE_BUILD: 'variant' }),
     [
       join(FIXTURE_DIR, 'community-people.json'),
-      join(VARIANT_FIXTURE_DIR, 'community-people.json'),
+      join(overlayDirFor('variant'), 'community-people.json'),
     ],
   );
 
   const { patched, context } = run('community-people.json', { variant: true });
 
   assert.deepEqual(context.dependencies, [
     join(FIXTURE_DIR, 'community-people.json'),
-    join(VARIANT_FIXTURE_DIR, 'community-people.json'),
+    join(overlayDirFor('variant'), 'community-people.json'),
   ]);
   // The variant overlay empties fetchedAt and is applied second, so seeing it
   // win proves the base overlay did not overwrite it on the way past.