feat: sequential execution

.changeset/gentle-parrots-sniff.md

@@ -0,0 +1,5 @@
+---
+"api-tuner": patch
+---
+
+Added `--sequential` flag so that tests are not executed concurrently

.changeset/mighty-doors-thank.md

@@ -0,0 +1,6 @@
+---
+"api-tuner": patch
+---
+
+Write output of tests as they are completed without wating for all to finish 
+

.github/workflows/tests.yml

@@ -15,6 +15,15 @@ jobs:
       - run: npm run prepack
   tests:
     runs-on: ubuntu-latest
+    strategy:
+      fail-fast: false
+      matrix:
+        runMode:
+          - name: concurrent
+            flags: ''
+          - name: sequential
+            flags: --sequential
+    name: tests (mode=${{ matrix.runMode.name }})
     steps:
       - uses: actions/checkout@v4
       - uses: actions/setup-node@v4
@@ -24,7 +33,8 @@ jobs:
       - run: npm ci
       - run: docker compose up -d
       - run: npx wait-on http-get://localhost:1080/example.com
-      - run: npm test -- --silent
+      - name: Run tests
+        run: npm test -- --silent ${{ matrix.runMode.flags }}
         shell: bash
       - run: if [[ `git status --porcelain` ]]; then git status --porcelain && exit 1; fi
   lint:

README.md

@@ -6,6 +6,7 @@
 - [Installation](#installation)
 - [Usage](#usage)
 - [Example](#example)
+- [Run modes](#run-modes)
 - [Documentation](#documentation)
   - [Namespaces](#namespaces)
   - [Defining a Test Case](#defining-a-test-case)
@@ -42,7 +43,8 @@ Options:
   --lib <path>       Specify rules to include in all tests. Can be used multiple times. Make sure to surround globs in quotes to prevent expansion.
   --silent           Less output
   --debug            Enable debug output
-  --raw              Output raw results from eye
+  --sequential       Run test suites sequentially instead of concurrently
+  --raw              Output raw results from eyes
   --grep <pattern>   Only run tests whose name or IRI matches the pattern (case-insensitive)
   --base-iri <iri>   Specify the base IRI for parsing the test case files
   --version          Show version information
@@ -108,6 +110,25 @@ api-tuner --grep "login" tests/**/*.n3
 api-tuner --grep "Simple GET" test.n3
 ```
 
+## Run modes
+
+By default, API Tun3r runs multiple test suites concurrently for faster feedback. Add `--sequential` flag to run suites one by one in the given order.
+
+Notes and recommendations:
+
+- Write tests to be independent and free of shared mutable state (e.g., unique test data, isolated resources). Independent tests enable safe parallel execution and faster CI times.
+- If your suites currently depend on ordering or shared state, use `--sequential` as a temporary measure while refactoring toward independence.
+
+Examples:
+
+```sh
+# Default (concurrent)
+api-tuner --base-iri http://localhost:1080/ tests/**/*.n3
+
+# Force sequential execution
+api-tuner --base-iri http://localhost:1080/ --sequential tests/**/*.n3
+```
+
 ## More examples
 
 See the [Documentation](#documentation) section below for more examples and detailed documentation of the N3 rules.

index.ts

@@ -19,6 +19,7 @@ program
   .option('--lib <lib>', 'Specify rules to include in all tests. Can be used multiple times. Make sure to surround globs in quotes to prevent expansion.', (lib, arr: string[]) => [...arr, lib], [])
   .option('--silent', 'Less output', false)
   .option('--debug', 'Enable debug output', false)
+  .option('--sequential', 'Run test suites sequentially instead of concurrently', false)
   .option('--raw', 'Output raw results from eyes')
   .option('--grep <pattern>', 'Only run tests matching the given pattern')
   .requiredOption('--base-iri <baseIri>', 'Specify the base IRI for parsing the test case files')
@@ -61,6 +62,16 @@ const levelIcon = {
   'Failed assertion': '❌ Failed assertion',
 } as const
 
+function writeSummary(summary: string) {
+  if (options.raw) return
+  const out = summary.endsWith('\n') ? summary : summary + '\n'
+  process.stdout.write(out)
+}
+
+function suiteHeader(absolutePath: string) {
+  return `\n🔎 SUITE   <file://${absolutePath}>\n`
+}
+
 interface EyeProcessResult {
   stdout: Readable
   stderr: Readable
@@ -113,7 +124,7 @@ function tunerParams({ path }: { path: string }): Readable {
   return Readable.from(graph.dataset.toCanonical())
 }
 
-const testSuites = program.args.map(async (path) => {
+async function runSuite(path: string) {
   const absolutePath = resolve(process.cwd(), path)
 
   const result = await processPath(path)
@@ -126,10 +137,9 @@ const testSuites = program.args.map(async (path) => {
   const validationResult = await summariseResults(summaryPassThrough)
 
   if (validationResult.allTestsSkipped) {
-    return {
-      success: true,
-      summary: `⏳  SKIP   <file://${absolutePath}>`,
-    }
+    const summary = `⏳  SKIP   <file://${absolutePath}>`
+    writeSummary(summary)
+    return true
   }
 
   if (options.raw) {
@@ -139,13 +149,12 @@ const testSuites = program.args.map(async (path) => {
   }
 
   if (!result.success) {
-    return {
-      summary: `\n🔎 SUITE   <file://${absolutePath}>\n❌  FAIL   Test script failed`,
-      success: false,
-    }
+    const summary = `${suiteHeader(absolutePath)}❌  FAIL   Test script failed`
+    writeSummary(summary)
+    return false
   }
 
-  let summary = `\n🔎 SUITE   <file://${absolutePath}>\n`
+  let summary = suiteHeader(absolutePath)
   if (!validationResult.success || !options.silent) {
     const stderr = await getStream(result.stderr)
     summary += stderr.replace(/"([^"]*)" TRACE ("([^"]*)")?/gm, (_, level: keyof typeof levelIcon, quoted, text) => {
@@ -154,18 +163,28 @@ const testSuites = program.args.map(async (path) => {
   }
 
   summary += validationResult.summary + '\n'
+  writeSummary(summary)
+  return validationResult.success
+}
 
-  return {
-    summary,
-    success: validationResult.success,
-  }
-})
+async function runAllSuites() {
+  let results: boolean[] = []
 
-Promise.all(testSuites).then((results) => {
-  const summary = results.map(result => result.summary).join('\n')
-  if (!options.raw) {
-    process.stdout.write(summary + '\n')
+  if (options.sequential) {
+    for (const path of program.args as string[]) {
+      // Run each suite one by one
+      // Raw output (if enabled) is already streamed inside runSuite
+      const ok = await runSuite(path)
+      results.push(ok)
+    }
+  } else {
+    results = await Promise.all((program.args as string[]).map(runSuite))
   }
+
+  // Summaries are written directly by runSuite (unless --raw),
+  // so nothing else to print here.
   // exit code equals number of failed tests
-  process.exit(results.filter(result => !result.success).length)
-})
+  process.exit(results.filter(success => !success).length)
+}
+
+runAllSuites()