test(e2e): add a per-file region floor to the e2e coverage reporter

tests/e2e-coverage-report-file-region-floor.test.mjs

@@ -0,0 +1,240 @@
+// `--check-source-regions` is an aggregate, and an aggregate hides where its
+// own slack is spent. The `e2e-coverage` job asks for 95%, so roughly one
+// region in twenty may be uncovered anywhere across `src/**` -- and nothing
+// says those regions have to be spread out. A single component can hold all of
+// the slack and sit far below the floor while the job stays green, which is
+// the same hole `tests/tools/coverage-report.mjs` closed on the unit run with
+// `--check-source-file-regions`.
+//
+// The companion line gate does not catch it either: a lost *region* need not
+// be a lost *line*. An unexecuted ternary arm or `??` fallback sits on a line
+// the surrounding statement still covers, so a file can report 100% lines
+// while its region percentage falls.
+//
+// The fixture below is the proof rather than an illustration. Two source files
+// share one bundle; one is fully covered and the other is not, and the numbers
+// are chosen so the aggregate clears a floor that the worse file fails. A gate
+// that only ever asked the aggregate would pass that run.
+
+import assert from 'node:assert/strict';
+import test from 'node:test';
+import { mkdir, mkdtemp, readFile, rm, writeFile } from 'node:fs/promises';
+import { tmpdir } from 'node:os';
+import { join } from 'node:path';
+
+import { main } from './tools/e2e-coverage-report.mjs';
+import {
+  initCoverageRun,
+  sealCoverageRun,
+  writeCoverageArtifact,
+} from './tools/e2e-coverage-run.mjs';
+
+const VLQ_CHARS =
+  'ABCDEFGHIJKLMNOPQRSTUVWXYZabcdefghijklmnopqrstuvwxyz0123456789+/';
+
+function vlq(value) {
+  let encoded = value < 0 ? (-value << 1) | 1 : value << 1;
+  let output = '';
+  do {
+    let digit = encoded & 31;
+    encoded >>>= 5;
+    if (encoded > 0) digit |= 32;
+    output += VLQ_CHARS[digit];
+  } while (encoded > 0);
+  return output;
+}
+
+function mapLines(entries) {
+  let previousSource = 0;
+  let previousLine = 0;
+  return entries
+    .map(([sourceIndex, originalLine]) => {
+      const segment = [
+        vlq(0),
+        vlq(sourceIndex - previousSource),
+        vlq(originalLine - previousLine),
+        vlq(0),
+      ].join('');
+      previousSource = sourceIndex;
+      previousLine = originalLine;
+      return segment;
+    })
+    .join(';');
+}
+
+// Two bundles, one src file each, as Docusaurus emits them. `Covered/index.js`
+// has both of its lines executed; `Uneven/index.js` has one of two. The run
+// therefore clears a 66% aggregate (two of three regions) while
+// `Uneven/index.js` alone sits at 50%.
+//
+// Each file gets its own bundle and its own map deliberately: a single map
+// naming two sources is resolved by `v8-to-istanbul` into one source only, so
+// a merged fixture would silently measure one file and prove nothing.
+async function unevenRun() {
+  const root = await mkdtemp(join(tmpdir(), 'endusers-e2e-file-floor-'));
+  const runDir = join(root, 'coverage');
+  const buildDir = join(root, 'build');
+  await mkdir(join(buildDir, 'assets/js'), { recursive: true });
+  await mkdir(join(root, 'src/components/Covered'), { recursive: true });
+  await mkdir(join(root, 'src/components/Uneven'), { recursive: true });
+  await initCoverageRun(runDir, 'run-file-floor', { project: 'chromium' });
+
+  const bundles = [
+    { name: 'covered', dir: 'Covered', body: 'const a = 1;\nconst b = 2;\n' },
+    { name: 'uneven', dir: 'Uneven', body: 'const c = 3;\nconst d = 4;\n' },
+  ];
+  const result = [];
+  for (const bundle of bundles) {
+    const scriptText = `${bundle.body}\n//# sourceMappingURL=${bundle.name}.js.map\n`;
+    await writeFile(
+      join(root, `src/components/${bundle.dir}/index.js`),
+      bundle.body,
+    );
+    await writeFile(join(buildDir, `assets/js/${bundle.name}.js`), scriptText);
+    await writeFile(
+      join(buildDir, `assets/js/${bundle.name}.js.map`),
+      JSON.stringify({
+        version: 3,
+        file: `${bundle.name}.js`,
+        sources: [`webpack://endusers/./src/components/${bundle.dir}/index.js`],
+        sourcesContent: [bundle.body],
+        names: [],
+        mappings: mapLines([
+          [0, 0],
+          [0, 1],
+        ]),
+      }),
+    );
+    // The uneven bundle leaves its second statement unexecuted; the covered
+    // one is entered whole.
+    const ranges =
+      bundle.dir === 'Uneven'
+        ? [
+            {
+              startOffset: bundle.body.indexOf('const d'),
+              endOffset: scriptText.length,
+              count: 0,
+            },
+            { startOffset: 0, endOffset: scriptText.length, count: 1 },
+          ]
+        : [{ startOffset: 0, endOffset: scriptText.length, count: 1 }];
+    result.push({
+      url: `http://localhost:3000/assets/js/${bundle.name}.js`,
+      scriptId: String(result.length + 1),
+      functions: [{ functionName: '', isBlockCoverage: true, ranges }],
+    });
+  }
+  await writeCoverageArtifact(runDir, 'worker-0-page-0', {
+    schemaVersion: 1,
+    kind: 'endusers.playwright.v8-coverage',
+    runId: 'run-file-floor',
+    result,
+  });
+  await sealCoverageRun(runDir, 'passed');
+  return { root, runDir, buildDir };
+}
+
+function reportArgs(fixture, extra = []) {
+  return [
+    '--input',
+    fixture.runDir,
+    '--root',
+    fixture.root,
+    '--build',
+    fixture.buildDir,
+    '--json',
+    join(fixture.root, 'report.json'),
+    '--text',
+    join(fixture.root, 'report.txt'),
+    ...extra,
+  ];
+}
+
+// The whole argument for the flag, in one assertion: the same run that clears
+// the aggregate floor fails the per-file one.
+test('a per-file floor catches a file the aggregate floor averages away', async () => {
+  const fixture = await unevenRun();
+  try {
+    const passed = await main(
+      reportArgs(fixture, ['--check-source-regions', '66']),
+    );
+    assert.equal(passed.summary.regionPercent, 66.67);
+
+    await assert.rejects(
+      () => main(reportArgs(fixture, ['--check-source-file-regions', '66'])),
+      (error) => {
+        assert.match(
+          error.message,
+          /1 source file\(s\) fall below the required 66% region coverage per file:/,
+        );
+        assert.match(
+          error.message,
+          /src\/components\/Uneven\/index\.js 50\.00% \(1\/2 regions; uncovered at 2\)/,
+        );
+        // The file that is fine must not be named: an offender list that
+        // reports every file is a list nobody reads.
+        assert.doesNotMatch(error.message, /Covered/);
+        return true;
+      },
+    );
+  } finally {
+    await rm(fixture.root, { recursive: true, force: true });
+  }
+});
+
+// Same ordering contract the aggregate gates already hold: a threshold failure
+// must still leave behind the report that explains it.
+test('--check-source-file-regions keeps the artifacts it fails on', async () => {
+  const fixture = await unevenRun();
+  try {
+    await assert.rejects(() =>
+      main(reportArgs(fixture, ['--check-source-file-regions', '100'])),
+    );
+    const json = JSON.parse(
+      await readFile(join(fixture.root, 'report.json'), 'utf8'),
+    );
+    assert.equal(json.status, 'ok');
+    const uneven = json.sources.find((row) =>
+      row.file.endsWith('Uneven/index.js'),
+    );
+    assert.equal(uneven.regionPercent, 50);
+    assert.ok(
+      (await readFile(join(fixture.root, 'report.txt'), 'utf8')).length > 0,
+    );
+  } finally {
+    await rm(fixture.root, { recursive: true, force: true });
+  }
+});
+
+test('--check-source-file-regions passes when every file meets the floor', async () => {
+  const fixture = await unevenRun();
+  try {
+    const report = await main(
+      reportArgs(fixture, ['--check-source-file-regions', '50']),
+    );
+    assert.equal(report.status, 'ok');
+  } finally {
+    await rm(fixture.root, { recursive: true, force: true });
+  }
+});
+
+test('omitting --check-source-file-regions leaves the reporter ungated', async () => {
+  const fixture = await unevenRun();
+  try {
+    const report = await main(reportArgs(fixture));
+    assert.equal(report.summary.regionPercent, 66.67);
+  } finally {
+    await rm(fixture.root, { recursive: true, force: true });
+  }
+});
+
+// A silently-ignored bad threshold would read as a gate in the workflow while
+// enforcing nothing, which is worse than no gate at all.
+test('--check-source-file-regions rejects thresholds that are not a percentage', async () => {
+  for (const value of ['', 'ninety', '-1', '101']) {
+    await assert.rejects(
+      () => main(['--input', 'unused', '--check-source-file-regions', value]),
+      /--check-source-file-regions expects a percentage between 0 and 100/,
+    );
+  }
+});

tests/tools/e2e-coverage-report.mjs

@@ -70,6 +70,7 @@ function parseArgs(argv) {
     text: null,
     checkSource: null,
     checkSourceRegions: null,
+    checkSourceFileRegions: null,
     requireSourceFiles: false,
   };
   for (let index = 0; index < argv.length; index += 1) {
@@ -86,6 +87,11 @@ function parseArgs(argv) {
         argv[++index],
         '--check-source-regions',
       );
+    else if (arg === '--check-source-file-regions')
+      options.checkSourceFileRegions = parsePercent(
+        argv[++index],
+        '--check-source-file-regions',
+      );
     else if (arg === '--require-source-files')
       options.requireSourceFiles = true;
     else throw new Error(`unknown e2e coverage report option: ${arg}`);
@@ -902,6 +908,42 @@ export async function main(argv = process.argv.slice(2)) {
       `Source region coverage ${sourceRegionPercent.toFixed(2)}% is below the required ${options.checkSourceRegions}%.`,
     );
   }
+  // --check-source-regions is an aggregate, and an aggregate hides where its
+  // own slack is spent: the gate is cleared by the whole of src/ together, so
+  // the regions one file loses are paid for by every other file that still has
+  // them. At the 95% the CI job asks for, a single small component may sit at
+  // nothing and the job stays green. The companion line gate does not catch
+  // that either, because a lost *region* need not be a lost *line* -- an
+  // unexecuted ternary arm or `??` fallback sits on a line the surrounding
+  // statement still covers. This applies the floor to each source file on its
+  // own, so a regression concentrated in one file fails on that file's name
+  // instead of being averaged away. tests/tools/coverage-report.mjs already
+  // gates the unit run this way; this is the same guarantee for the browser
+  // run.
+  if (options.checkSourceFileRegions !== null) {
+    // No vacuous-pass guard is needed here, unlike the unit reporter's
+    // equivalent: collectE2ECoverage above already refuses a run that
+    // attributed nothing to src/**, so this list can never be empty for the
+    // reason that guard exists to catch.
+    const below = report.sources.filter(
+      (row) =>
+        row.regions > 0 &&
+        row.regionPercent + 1e-9 < options.checkSourceFileRegions,
+    );
+    if (below.length > 0) {
+      throw new Error(
+        `${below.length} source file(s) fall below the required ` +
+          `${options.checkSourceFileRegions}% region coverage per file:\n` +
+          below
+            .map(
+              (row) =>
+                `  ${row.file} ${row.regionPercent.toFixed(2)}% ` +
+                `(${row.coveredRegions}/${row.regions} regions; uncovered at ${row.uncoveredRegions.join(' ')})`,
+            )
+            .join('\n'),
+      );
+    }
+  }
   if (options.requireSourceFiles && report.missingSourceFiles.length > 0) {
     throw new Error(
       '--require-source-files was requested, but the run never measured:\n' +