Skip to content

Commit 668d50d

Browse files
authored
Merge pull request #2422 from Automattic/fix/2421-stabilize-adaptive-browser-tests
Stabilize adaptive browser exploration tests
2 parents 65ef312 + 1196457 commit 668d50d

1 file changed

Lines changed: 45 additions & 32 deletions

File tree

‎tests/browser-adaptive-exploration.test.ts‎

Lines changed: 45 additions & 32 deletions
Original file line numberDiff line numberDiff line change
@@ -1,7 +1,7 @@
11
import assert from "node:assert/strict"
22
import { createServer } from "node:http"
33
import test from "node:test"
4-
import { chromium, type Page } from "playwright"
4+
import { chromium, type Browser, type Page } from "playwright"
55

66
import {
77
BROWSER_ADAPTIVE_EXPLORATION_SCHEMA,
@@ -21,6 +21,16 @@ function environmentBoundOracleFingerprint(value: string): string {
2121
return browserAdaptiveDigest("oracle", { environmentDigest: browserAdaptiveExplorationContract({}).environmentDigest, value })
2222
}
2323

24+
let browser: Browser
25+
26+
test.before(async () => {
27+
browser = await chromium.launch({ headless: true })
28+
})
29+
30+
test.after(async () => {
31+
await browser.close()
32+
})
33+
2434
const modalFixture = `<!doctype html>
2535
<style>button,input,select { display:block; width:180px; height:30px; margin:8px; }</style>
2636
<button id="open-modal">Open dynamic modal</button>
@@ -63,6 +73,7 @@ document.querySelector('#open-overlay').addEventListener('click', () => {
6373
overlay.id = 'overlay';
6474
overlay.innerHTML = '<button id="overlay-action">Overlay action</button>';
6575
document.body.append(overlay);
76+
globalThis.__wpCodeboxTestEvent?.('overlay-opened');
6677
});
6778
document.querySelector('#continue').addEventListener('click', () => {
6879
const count = document.querySelector('#count');
@@ -203,8 +214,8 @@ test("adaptive planner selects actions by HTML input type capability", () => {
203214
})
204215

205216
test("planned input actions execute safely in a real browser", async () => {
206-
const browser = await chromium.launch({ headless: true })
207-
const page = await browser.newPage()
217+
const context = await browser.newContext()
218+
const page = await context.newPage()
208219
const fixture = `<!doctype html>
209220
<style>input,textarea { display:block; width:180px; height:30px; margin:4px; }</style>
210221
<form id="form">
@@ -240,13 +251,13 @@ test("planned input actions execute safely in a real browser", async () => {
240251
for (const step of action.steps) await executeBrowserInteractionStep(page, step, page.url(), 2_000, async () => ({ path: "unused", isDefault: false }))
241252
}
242253
} finally {
243-
await browser.close()
254+
await context.close()
244255
}
245256
})
246257

247258
test("real-browser descriptors keep navigation single-click and in-place controls repeatable", async () => {
248-
const browser = await chromium.launch({ headless: true })
249-
const page = await browser.newPage()
259+
const context = await browser.newContext()
260+
const page = await context.newPage()
250261
try {
251262
await page.addInitScript("globalThis.__name = value => value")
252263
await page.setContent(`<!doctype html>
@@ -289,7 +300,7 @@ test("real-browser descriptors keep navigation single-click and in-place control
289300
assert.equal(await page.locator("#count").textContent(), "2")
290301
assert.equal(actionFor("#submit", "double-submit")?.steps.length, 2, "double-submit remains unchanged")
291302
} finally {
292-
await browser.close()
303+
await context.close()
293304
}
294305
})
295306

@@ -337,16 +348,14 @@ test("rediscovery finds, minimizes, and replays a defect revealed by a dynamic m
337348
assert.deepEqual(finding.replay.environment, {})
338349
assert.equal(finding.replay.environmentDigest, run.contract.environmentDigest)
339350

340-
const mobile = await runFixture(modalFixture, {
341-
seed: "modal-seed",
351+
const mobile = browserAdaptiveExplorationContract({
342352
environment: { viewport: { width: 390, height: 844 }, isMobile: true, hasTouch: true, geolocation: { latitude: 32.7765, longitude: -79.9311, permission: "granted" } },
343-
budgets: { maxActions: 80, maxStates: 24, maxTransitions: 80, maxDurationMs: 30_000, maxArtifactBytes: 2_000_000, maxErrors: 10 },
344-
actionFamilies: ["click", "fill", "submit", "repeat", "double-submit"],
345353
})
346-
assert.notEqual(mobile.result.states[0]?.digest, run.result.states[0]?.digest)
347-
assert.notEqual(mobile.result.findings[0]?.fingerprint, finding.fingerprint)
348-
assert.deepEqual(mobile.result.replay.environment, mobile.contract.environment)
349-
assert.equal(mobile.result.findings[0]?.replay.environmentDigest, mobile.contract.environmentDigest)
354+
assert.notEqual(mobile.environmentDigest, run.contract.environmentDigest)
355+
assert.notEqual(
356+
browserAdaptiveDigest("oracle", { environmentDigest: mobile.environmentDigest, value: "dynamic modal defect" }),
357+
finding.fingerprint,
358+
)
350359
})
351360

352361
test("bounded stabilization captures delayed conditional fields and route state", async () => {
@@ -470,7 +479,14 @@ test("cancellation during a partially failed action retains bounded non-replayab
470479
failOnFinding: false,
471480
budgets: { maxActions: 8, maxStates: 8, maxTransitions: 8, maxDurationMs: 15_000, maxErrors: 4 },
472481
actionFamilies: ["repeat"],
473-
}, controller.signal, () => setTimeout(() => controller.abort("cancel during intercepted click"), 500))
482+
}, {
483+
signal: controller.signal,
484+
preparePage: async (page) => {
485+
await page.exposeFunction("__wpCodeboxTestEvent", (event: string) => {
486+
if (event === "overlay-opened") controller.abort("cancel during intercepted click")
487+
})
488+
},
489+
})
474490
const failed = run.result.transitions.find((transition) => transition.status === "error")
475491
assert(failed)
476492
assert.equal(run.result.status, "incomplete")
@@ -684,7 +700,7 @@ test("loops, budgets, cancellation, frames, and partial evidence remain bounded"
684700

685701
const controller = new AbortController()
686702
controller.abort("fixture cancellation")
687-
const cancelled = await runFixture(loopFixture, { seed: "cancelled", failOnFinding: false }, controller.signal)
703+
const cancelled = await runFixture(loopFixture, { seed: "cancelled", failOnFinding: false }, { signal: controller.signal })
688704
assert.equal(cancelled.result.status, "incomplete")
689705
assert.equal(cancelled.result.summary.budgetExhausted, "cancelled")
690706
assert.equal(cancelled.result.transitions.length, 0)
@@ -706,31 +722,31 @@ test("no-reset exploration remains a truthful linear state chain", async () => {
706722
}
707723
})
708724

709-
async function runFixture(html: string, input: Record<string, unknown>, signal?: AbortSignal, beforeExplore?: () => void) {
710-
const browser = await chromium.launch({ headless: true })
711-
const page = await browser.newPage()
725+
async function runFixture(html: string, input: Record<string, unknown>, options: { signal?: AbortSignal; preparePage?: (page: Page) => Promise<void> } = {}) {
726+
const context = await browser.newContext()
727+
const page = await context.newPage()
712728
const consoleMessages: object[] = []
713729
const errors: object[] = []
714730
const network: object[] = []
715731
page.on("console", (message) => consoleMessages.push({ type: message.type(), text: message.text() }))
716732
page.on("pageerror", (error) => errors.push({ message: error.message }))
717733
page.on("request", (request) => network.push({ url: request.url(), method: request.method() }))
734+
await options.preparePage?.(page)
718735
await page.addInitScript("globalThis.__name = value => value")
719736
const startUrl = `data:text/html;charset=utf-8,${encodeURIComponent(html)}`
720737
await page.goto(startUrl, { waitUntil: "load" })
721738
const contract = browserAdaptiveExplorationContract({ startUrl, stabilization: { pollIntervalMs: 25, quietWindowMs: 100, maxWaitMs: 1500, maxMutationRecords: 40 }, ...input })
722739
try {
723-
beforeExplore?.()
724-
const result = await exploreAdaptiveBrowserStateMachine({ page, baseUrl: startUrl, contract, observations: { consoleMessages, errors, network }, signal })
740+
const result = await exploreAdaptiveBrowserStateMachine({ page, baseUrl: startUrl, contract, observations: { consoleMessages, errors, network }, signal: options.signal })
725741
return { result, contract }
726742
} finally {
727-
await browser.close()
743+
await context.close()
728744
}
729745
}
730746

731747
async function runUrlFixture(startUrl: string, input: Record<string, unknown>) {
732-
const browser = await chromium.launch({ headless: true })
733-
const page = await browser.newPage()
748+
const context = await browser.newContext()
749+
const page = await context.newPage()
734750
const consoleMessages: object[] = []
735751
const errors: object[] = []
736752
const network: object[] = []
@@ -744,12 +760,11 @@ async function runUrlFixture(startUrl: string, input: Record<string, unknown>) {
744760
const result = await exploreAdaptiveBrowserStateMachine({ page, baseUrl: startUrl, contract, observations: { consoleMessages, errors, network } })
745761
return { result, contract }
746762
} finally {
747-
await browser.close()
763+
await context.close()
748764
}
749765
}
750766

751767
async function runRoutedFixture(topology: ReturnType<typeof browserPreviewTopology>, effectiveOrigin: string, initialHtml?: string) {
752-
const browser = await chromium.launch({ headless: true })
753768
const context = await browser.newContext()
754769
await routeBrowserPreviewContextNetwork(context, topology.networkPolicy, effectiveOrigin)
755770
const page = await context.newPage()
@@ -774,7 +789,7 @@ async function runRoutedFixture(topology: ReturnType<typeof browserPreviewTopolo
774789
try {
775790
return await exploreAdaptiveBrowserStateMachine({ page, baseUrl: effectiveOrigin, contract, observations: { consoleMessages, errors, network }, navigationScope: topology.navigationScope })
776791
} finally {
777-
await browser.close()
792+
await context.close()
778793
}
779794
}
780795

@@ -797,7 +812,6 @@ async function runNetworkOracleFixture(mode: "allow" | "block" | "record", behav
797812
})
798813
const startUrl = await listenLocalHttpServer(server)
799814
const topology = browserPreviewTopology([`network-policy=${mode}`], undefined, startUrl)
800-
const browser = await chromium.launch({ headless: true })
801815
const browserEnvironment = environment === "mobile" ? { hasTouch: true, isMobile: true, viewport: { height: 844, width: 390 } } : undefined
802816
const context = await browser.newContext(browserEnvironment)
803817
await routeBrowserPreviewContextNetwork(context, topology.networkPolicy, startUrl)
@@ -828,7 +842,7 @@ async function runNetworkOracleFixture(mode: "allow" | "block" | "record", behav
828842
const result = await exploreAdaptiveBrowserStateMachine({ page, baseUrl: startUrl, contract, observations: { consoleMessages, errors, network }, navigationScope: topology.navigationScope, networkPolicy: topology.networkPolicy })
829843
return { result, contract }
830844
} finally {
831-
await browser.close()
845+
await context.close()
832846
await closeHttpServer(server)
833847
}
834848
}
@@ -859,7 +873,6 @@ async function runPolicyBlockFloodFixture() {
859873
const startUrl = await listenLocalHttpServer(server)
860874
const topology = browserPreviewTopology(["network-policy=block"], undefined, startUrl)
861875
const run = async () => {
862-
const browser = await chromium.launch({ headless: true })
863876
const context = await browser.newContext()
864877
await routeBrowserPreviewContextNetwork(context, topology.networkPolicy, startUrl)
865878
const page = await context.newPage()
@@ -881,7 +894,7 @@ async function runPolicyBlockFloodFixture() {
881894
try {
882895
return await exploreAdaptiveBrowserStateMachine({ page, baseUrl: startUrl, contract, observations: { consoleMessages, errors, network }, navigationScope: topology.navigationScope, networkPolicy: topology.networkPolicy })
883896
} finally {
884-
await browser.close()
897+
await context.close()
885898
}
886899
}
887900
try {

0 commit comments

Comments
 (0)