From e45dee333efbe27708ef729ae6cc8ad63e61510c Mon Sep 17 00:00:00 2001 From: tpluscode Date: Thu, 16 Jul 2026 12:35:02 +0200 Subject: [PATCH] feat: sequential execution --- .changeset/gentle-parrots-sniff.md | 5 +++ .changeset/mighty-doors-thank.md | 6 +++ .github/workflows/tests.yml | 12 +++++- README.md | 23 ++++++++++- index.ts | 61 ++++++++++++++++++++---------- 5 files changed, 84 insertions(+), 23 deletions(-) create mode 100644 .changeset/gentle-parrots-sniff.md create mode 100644 .changeset/mighty-doors-thank.md diff --git a/.changeset/gentle-parrots-sniff.md b/.changeset/gentle-parrots-sniff.md new file mode 100644 index 0000000..fcbf67b --- /dev/null +++ b/.changeset/gentle-parrots-sniff.md @@ -0,0 +1,5 @@ +--- +"api-tuner": patch +--- + +Added `--sequential` flag so that tests are not executed concurrently diff --git a/.changeset/mighty-doors-thank.md b/.changeset/mighty-doors-thank.md new file mode 100644 index 0000000..cb28c9e --- /dev/null +++ b/.changeset/mighty-doors-thank.md @@ -0,0 +1,6 @@ +--- +"api-tuner": patch +--- + +Write output of tests as they are completed without wating for all to finish + \ No newline at end of file diff --git a/.github/workflows/tests.yml b/.github/workflows/tests.yml index 6b682e6..d5e5858 100644 --- a/.github/workflows/tests.yml +++ b/.github/workflows/tests.yml @@ -15,6 +15,15 @@ jobs: - run: npm run prepack tests: runs-on: ubuntu-latest + strategy: + fail-fast: false + matrix: + runMode: + - name: concurrent + flags: '' + - name: sequential + flags: --sequential + name: tests (mode=${{ matrix.runMode.name }}) steps: - uses: actions/checkout@v4 - uses: actions/setup-node@v4 @@ -24,7 +33,8 @@ jobs: - run: npm ci - run: docker compose up -d - run: npx wait-on http-get://localhost:1080/example.com - - run: npm test -- --silent + - name: Run tests + run: npm test -- --silent ${{ matrix.runMode.flags }} shell: bash - run: if [[ `git status --porcelain` ]]; then git status --porcelain && exit 1; fi lint: diff --git a/README.md b/README.md index 0127c54..d2a513a 100644 --- a/README.md +++ b/README.md @@ -6,6 +6,7 @@ - [Installation](#installation) - [Usage](#usage) - [Example](#example) +- [Run modes](#run-modes) - [Documentation](#documentation) - [Namespaces](#namespaces) - [Defining a Test Case](#defining-a-test-case) @@ -42,7 +43,8 @@ Options: --lib Specify rules to include in all tests. Can be used multiple times. Make sure to surround globs in quotes to prevent expansion. --silent Less output --debug Enable debug output - --raw Output raw results from eye + --sequential Run test suites sequentially instead of concurrently + --raw Output raw results from eyes --grep Only run tests whose name or IRI matches the pattern (case-insensitive) --base-iri Specify the base IRI for parsing the test case files --version Show version information @@ -108,6 +110,25 @@ api-tuner --grep "login" tests/**/*.n3 api-tuner --grep "Simple GET" test.n3 ``` +## Run modes + +By default, API Tun3r runs multiple test suites concurrently for faster feedback. Add `--sequential` flag to run suites one by one in the given order. + +Notes and recommendations: + +- Write tests to be independent and free of shared mutable state (e.g., unique test data, isolated resources). Independent tests enable safe parallel execution and faster CI times. +- If your suites currently depend on ordering or shared state, use `--sequential` as a temporary measure while refactoring toward independence. + +Examples: + +```sh +# Default (concurrent) +api-tuner --base-iri http://localhost:1080/ tests/**/*.n3 + +# Force sequential execution +api-tuner --base-iri http://localhost:1080/ --sequential tests/**/*.n3 +``` + ## More examples See the [Documentation](#documentation) section below for more examples and detailed documentation of the N3 rules. diff --git a/index.ts b/index.ts index 92e8d20..f01aa2f 100644 --- a/index.ts +++ b/index.ts @@ -19,6 +19,7 @@ program .option('--lib ', 'Specify rules to include in all tests. Can be used multiple times. Make sure to surround globs in quotes to prevent expansion.', (lib, arr: string[]) => [...arr, lib], []) .option('--silent', 'Less output', false) .option('--debug', 'Enable debug output', false) + .option('--sequential', 'Run test suites sequentially instead of concurrently', false) .option('--raw', 'Output raw results from eyes') .option('--grep ', 'Only run tests matching the given pattern') .requiredOption('--base-iri ', 'Specify the base IRI for parsing the test case files') @@ -61,6 +62,16 @@ const levelIcon = { 'Failed assertion': 'āŒ Failed assertion', } as const +function writeSummary(summary: string) { + if (options.raw) return + const out = summary.endsWith('\n') ? summary : summary + '\n' + process.stdout.write(out) +} + +function suiteHeader(absolutePath: string) { + return `\nšŸ”Ž SUITE \n` +} + interface EyeProcessResult { stdout: Readable stderr: Readable @@ -113,7 +124,7 @@ function tunerParams({ path }: { path: string }): Readable { return Readable.from(graph.dataset.toCanonical()) } -const testSuites = program.args.map(async (path) => { +async function runSuite(path: string) { const absolutePath = resolve(process.cwd(), path) const result = await processPath(path) @@ -126,10 +137,9 @@ const testSuites = program.args.map(async (path) => { const validationResult = await summariseResults(summaryPassThrough) if (validationResult.allTestsSkipped) { - return { - success: true, - summary: `ā³ SKIP `, - } + const summary = `ā³ SKIP ` + writeSummary(summary) + return true } if (options.raw) { @@ -139,13 +149,12 @@ const testSuites = program.args.map(async (path) => { } if (!result.success) { - return { - summary: `\nšŸ”Ž SUITE \nāŒ FAIL Test script failed`, - success: false, - } + const summary = `${suiteHeader(absolutePath)}āŒ FAIL Test script failed` + writeSummary(summary) + return false } - let summary = `\nšŸ”Ž SUITE \n` + let summary = suiteHeader(absolutePath) if (!validationResult.success || !options.silent) { const stderr = await getStream(result.stderr) summary += stderr.replace(/"([^"]*)" TRACE ("([^"]*)")?/gm, (_, level: keyof typeof levelIcon, quoted, text) => { @@ -154,18 +163,28 @@ const testSuites = program.args.map(async (path) => { } summary += validationResult.summary + '\n' + writeSummary(summary) + return validationResult.success +} - return { - summary, - success: validationResult.success, - } -}) +async function runAllSuites() { + let results: boolean[] = [] -Promise.all(testSuites).then((results) => { - const summary = results.map(result => result.summary).join('\n') - if (!options.raw) { - process.stdout.write(summary + '\n') + if (options.sequential) { + for (const path of program.args as string[]) { + // Run each suite one by one + // Raw output (if enabled) is already streamed inside runSuite + const ok = await runSuite(path) + results.push(ok) + } + } else { + results = await Promise.all((program.args as string[]).map(runSuite)) } + + // Summaries are written directly by runSuite (unless --raw), + // so nothing else to print here. // exit code equals number of failed tests - process.exit(results.filter(result => !result.success).length) -}) + process.exit(results.filter(success => !success).length) +} + +runAllSuites()