Skip to content

Commit 3e01210

Browse files
feat(browser-actions): add high-fidelity video capture (#2563)
* fix: cook wp-codebox * fix(browser-actions): encode high-quality video with Playwright's bundled ffmpeg The CI runner has no system ffmpeg, so video-quality=high failed with spawn ffmpeg ENOENT. Resolve FFMPEG_PATH, then Playwright's bundled ffmpeg, then PATH (checked for executability). Pipe JPEG frames as MJPEG over pipe:0 and encode VP8, the formats the minimal bundled build supports. --------- Co-authored-by: homeboy-ci[bot] <266378653+homeboy-ci[bot]@users.noreply.github.com>
1 parent f0c1d5d commit 3e01210

3 files changed

Lines changed: 150 additions & 8 deletions

File tree

‎packages/runtime-core/src/command-registry.ts‎

Lines changed: 2 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -167,6 +167,7 @@ const browserActionsValidation: CommandValidationDescriptor = {
167167
{ name: "step-timeout", kind: "duration", code: "invalid-duration", message: "wordpress.browser-actions step-timeout must look like 500ms or 2s." },
168168
{ name: "timeout", kind: "duration", code: "invalid-duration", message: "wordpress.browser-actions timeout must look like 500ms or 2s." },
169169
{ name: "capture", kind: "comma-list-enum", values: browserActionCaptureValues, code: "invalid-capture", message: "wordpress.browser-actions capture does not support" },
170+
{ name: "video-quality", kind: "enum", values: ["default", "high"], code: "invalid-video-quality", message: "wordpress.browser-actions video-quality must be default or high." },
170171
{ name: "viewport", kind: "viewport", code: "invalid-viewport", message: "wordpress.browser-actions viewport must use <width>x<height>, for example 390x844." },
171172
{ name: "device-scale-factor", kind: "number", minimum: 0.01, maximum: 100, code: "invalid-device-scale-factor", message: "wordpress.browser-actions device-scale-factor must be a finite positive number." },
172173
{ name: "is-mobile", kind: "boolean", code: "invalid-is-mobile", message: "wordpress.browser-actions is-mobile must be true or false." },
@@ -1244,6 +1245,7 @@ export const commandRegistry = [
12441245
{ name: "storage-state", description: "Optional Playwright storageState JSON, or @<path> to JSON, imported into a fresh browser context. Summaries redact cookie/localStorage values and report only schema/kind, counts, hosts, and diagnostics.", format: "JSON object or @path" },
12451246
{ name: "transport-faults-json", description: "Optional canonical wp-codebox/transport-fault-model/v1 schedule applied only to HTTP(S) requests emitted by this isolated browser context. Server-side WordPress HTTP API requests are not intercepted.", format: "JSON object or @path" },
12461247
{ name: "capture", description: "Comma-separated artifacts to capture after interactions.", format: "steps,console,errors,html,network,screenshot,dom-snapshot" },
1248+
{ name: "video-quality", description: "Video encoding mode when capture includes video. high captures device-pixel PNG frames and encodes them once at the end.", format: "default|high" },
12471249
{ name: "max-dom-snapshot-elements", description: "Maximum visible elements captured in each screenshot sidecar DOM/style snapshot; defaults to 160.", format: "positive integer" },
12481250
...browserActionEnvironmentAcceptedArgs,
12491251
],

‎packages/runtime-playground/src/browser-actions-runner.ts‎

Lines changed: 104 additions & 6 deletions
Original file line numberDiff line numberDiff line change
@@ -1,5 +1,8 @@
1-
import { copyFile, readFile, stat, writeFile } from "node:fs/promises"
2-
import { join } from "node:path"
1+
import { access, copyFile, mkdir, readdir, readFile, stat, writeFile } from "node:fs/promises"
2+
import { constants as fsConstants } from "node:fs"
3+
import { homedir } from "node:os"
4+
import { spawn } from "node:child_process"
5+
import { delimiter, join } from "node:path"
36
import { BROWSER_ACTION_CORPUS_SCHEMA, BROWSER_ADAPTIVE_EXPLORATION_SCHEMA, BROWSER_MULTI_ACTOR_SCENARIO_SCHEMA, BROWSER_PROBE_PROFILES, BROWSER_TOOL_VERIFIER_RESULT_SCHEMA, HostToolRegistry, assertRuntimeCommandAllowed, browserActionCorpusArtifact, browserActionCorpusContract, browserAdaptiveExplorationContract, browserEnvironment, browserEnvironmentDigest, browserGeolocation, browserInteractionScriptUsesEvaluate, browserToolVerifierInputSummary, createHostToolRegistry, executeHostTool, resolveCommandPath, transportFaultModel, validateBrowserInteractionScript, type BrowserActionCorpusArtifact, type BrowserActionCorpusContract, type BrowserAdaptiveExplorationArtifact, type BrowserAdaptiveExplorationContract, type BrowserEnvironment, type BrowserGeolocationPermissionState, type BrowserInteractionStep, type BrowserMultiActorScenario, type BrowserToolVerifierResult, type ExecutionSpec, type HostToolDefinition, type JsonValue, type RuntimeCreateSpec, type TransportFaultModel } from "@automattic/wp-codebox-core"
47
import { now, sha256 } from "@automattic/wp-codebox-core/internals"
58
import { browserInteractionStepsFromArgs, browserStepTimeoutMs, durationStringMs, sanitizeScreenshotName } from "./browser-actions.js"
@@ -49,6 +52,7 @@ export interface BrowserActionsRunPlan {
4952
networkSettleTimeoutMs: number
5053
requestedViewport?: { width: number; height: number }
5154
videoSize?: { width: number; height: number }
55+
videoQuality?: "default" | "high"
5256
requestedEnvironment?: BrowserEnvironment
5357
authRequest?: { userId: number }
5458
storageStateImport?: BrowserStorageStateImport
@@ -151,6 +155,11 @@ export async function runBrowserActionsCommand({
151155
let screenshotSha256: string | undefined
152156
const videoStagingDirectory = artifactSession.absolutePath("video-source")
153157
let videoRecording: import("playwright").Video | null = null
158+
let videoCaptureTask: Promise<void> | undefined
159+
let stopVideoCapture = false
160+
let videoFrameCount = 0
161+
let videoFrameError: Error | undefined
162+
const highVideo = runPlan.videoQuality === "high"
154163
let videoSaved = false
155164
let videoCaptureFailed = false
156165
let videoStartedAt: number | undefined
@@ -222,7 +231,7 @@ export async function runBrowserActionsCommand({
222231
...topology.contextOptions(),
223232
...(storageStateImport ? { storageState: storageStateImport.storageState } : {}),
224233
...(runPlan.transportFaults ? { serviceWorkers: "block" as const } : {}),
225-
...(capture.has("video") ? { recordVideo: { dir: videoStagingDirectory, ...(browserVideoSize(runPlan.videoSize, requestedEnvironment.viewport) ? { size: browserVideoSize(runPlan.videoSize, requestedEnvironment.viewport) } : {}) } } : {}),
234+
...(capture.has("video") && !highVideo ? { recordVideo: { dir: videoStagingDirectory, ...(browserVideoSize(runPlan.videoSize, requestedEnvironment.viewport) ? { size: browserVideoSize(runPlan.videoSize, requestedEnvironment.viewport) } : {}) } } : {}),
226235
},
227236
}) : undefined)
228237
const context = environmentRuntime?.context ?? null
@@ -232,9 +241,29 @@ export async function runBrowserActionsCommand({
232241
if (context && runPlan.transportFaults) installedTransportFaults = await installBrowserTransportFaults(context, runPlan.transportFaults, { policy: browserPreviewTransportFaultPolicy(networkPolicy, topology.origins.localProxyOrigin), serviceWorkersBlocked: true })
233242
const page = activePage = environmentRuntime?.page ?? await browser.newPage()
234243
if (capture.has("video")) {
235-
videoRecording = page.video()
244+
if (highVideo) {
245+
const viewport = requestedEnvironment.viewport ?? { width: 1280, height: 720 }
246+
const scale = requestedEnvironment.deviceScaleFactor ?? 1
247+
videoDimensions = { width: Math.round(viewport.width * scale), height: Math.round(viewport.height * scale) }
248+
const frameDirectory = join(videoStagingDirectory, "frames")
249+
await mkdir(frameDirectory, { recursive: true })
250+
videoCaptureTask = (async () => {
251+
while (!stopVideoCapture) {
252+
try {
253+
const frame = await page.screenshot({ type: "jpeg", quality: 95, scale: "device", animations: "allow" })
254+
await writeFile(join(frameDirectory, `frame-${String(++videoFrameCount).padStart(6, "0")}.jpg`), frame)
255+
} catch (error) {
256+
if (stopVideoCapture) return
257+
}
258+
const nextFrameAt = (videoRecordingOrigin ?? performance.now()) + videoFrameCount * 100
259+
await new Promise((resolve) => setTimeout(resolve, Math.max(0, nextFrameAt - performance.now())))
260+
}
261+
})()
262+
} else {
263+
videoRecording = page.video()
264+
videoDimensions = browserVideoSize(runPlan.videoSize, requestedEnvironment.viewport) ?? requestedEnvironment.viewport ?? { width: 1280, height: 720 }
265+
}
236266
videoStartedAt = videoRecordingOrigin
237-
videoDimensions = browserVideoSize(runPlan.videoSize, requestedEnvironment.viewport) ?? requestedEnvironment.viewport ?? { width: 1280, height: 720 }
238267
}
239268
navigationTracker = trackBrowserNavigation(page)
240269
if (onProgress) {
@@ -557,12 +586,37 @@ export async function runBrowserActionsCommand({
557586
})
558587
}
559588
if (activePage && resolvedEnvironment) environmentEvidence = await observePlaywrightBrowserEnvironment(activePage, requestedEnvironment, resolvedEnvironment).catch(() => environmentEvidence)
589+
if (videoCaptureTask) {
590+
videoFinishedAt = performance.now()
591+
stopVideoCapture = true
592+
await videoCaptureTask
593+
}
560594
const cleanupBrowser = session ? { close: async () => {} } : { close: async () => { await environmentRuntime?.close(); await browser.close() } }
561595
for (const routeError of await closeBrowserAndDrainPreviewRoutes(cleanupBrowser, routeTracker)) {
562596
errors.push(serializeBrowserError("probe-error", routeError))
563597
if (browserPreviewCleanupErrorIsFatal(routeError)) pendingError ??= routeError
564598
}
565-
if (videoRecording) {
599+
if (videoCaptureTask) {
600+
try {
601+
videoFinishedAt ??= performance.now()
602+
if (videoFrameError) throw videoFrameError
603+
if (videoFrameCount === 0) throw new Error("wordpress.browser-actions capture=video-quality=high produced no frames")
604+
const frames = join(videoStagingDirectory, "frames")
605+
const output = join(videoStagingDirectory, "high-quality.webm")
606+
const ffmpeg = await browserFfmpegPath()
607+
// Frames are piped as MJPEG and encoded as VP8 so the encoder works with
608+
// Playwright's bundled minimal ffmpeg (VP8 encoder, image2pipe + mjpeg
609+
// decoder only) as well as a full system build. Quality-first settings
610+
// replace the realtime recorder's ~1 Mbps warm-up.
611+
const frameFiles = (await readdir(frames)).filter((entry) => entry.endsWith(".jpg")).sort().map((entry) => join(frames, entry))
612+
await runFfmpeg(ffmpeg, ["-y", "-f", "image2pipe", "-c:v", "mjpeg", "-framerate", "10", "-i", "pipe:0", "-c:v", "libvpx", "-deadline", "good", "-cpu-used", "0", "-crf", "4", "-b:v", "40M", "-qmin", "0", "-qmax", "12", "-pix_fmt", "yuv420p", "-an", output], frameFiles)
613+
await artifactSession.writeGenerated("video", "video.webm", (path) => copyFile(output, path))
614+
videoSaved = true
615+
} catch (error) {
616+
errors.push(serializeBrowserError("probe-error", error))
617+
pendingError ??= error instanceof Error ? error : new Error(String(error))
618+
}
619+
} else if (videoRecording) {
566620
// Playwright finalizes a recording on context close and names it itself. The
567621
// recording is adopted from its staging directory rather than through
568622
// video.saveAs(), which requires a browser connection that cleanup has closed.
@@ -633,6 +687,7 @@ export async function runBrowserActionsCommand({
633687
path: "files/browser/video.webm",
634688
width: videoDimensions?.width ?? 0,
635689
height: videoDimensions?.height ?? 0,
690+
fps: highVideo ? 10 : 30,
636691
durationMs: Math.max(0, Math.round((videoFinishedAt ?? performance.now()) - (videoStartedAt ?? performance.now()))),
637692
markers: videoMarkers,
638693
} : undefined
@@ -866,6 +921,7 @@ async function browserActionsRunPlanFromArgs(args: string[], artifactRoot: strin
866921
networkSettleTimeoutMs: durationArg(args, "network-settle-timeout", browserCommandLivenessPolicy().networkSettleTimeoutMs),
867922
requestedViewport: viewportArg(args, "viewport"),
868923
videoSize: browserVideoSizeArg(args),
924+
videoQuality: browserVideoQualityArg(args),
869925
requestedEnvironment: adaptiveExplorationPlan.requestedEnvironment,
870926
authRequest: browserAuthRequest(args),
871927
storageStateImport: await browserStorageStateImportFromArgs(args, "wordpress.browser-actions", artifactRoot),
@@ -878,6 +934,48 @@ async function browserActionsRunPlanFromArgs(args: string[], artifactRoot: strin
878934
}
879935
}
880936

937+
function browserVideoQualityArg(args: string[]): "default" | "high" {
938+
const value = argValue(args, "video-quality")?.trim() ?? "default"
939+
if (value !== "default" && value !== "high") throw new Error("wordpress.browser-actions video-quality must be default or high")
940+
return value
941+
}
942+
943+
async function browserFfmpegPath(): Promise<string> {
944+
// Prefer an explicit override, then the ffmpeg Playwright installs next to
945+
// its browsers (present wherever video capture already works), then PATH.
946+
const candidates = [process.env.FFMPEG_PATH].filter((candidate): candidate is string => Boolean(candidate))
947+
const browsersPath = process.env.PLAYWRIGHT_BROWSERS_PATH ?? join(homedir(), ".cache", "ms-playwright")
948+
const binary = process.platform === "linux" ? "ffmpeg-linux" : process.platform === "darwin" ? "ffmpeg-mac" : "ffmpeg-win64.exe"
949+
for (const entry of (await readdir(browsersPath).catch(() => [] as string[])).sort().reverse()) {
950+
if (entry.startsWith("ffmpeg-")) candidates.push(join(browsersPath, entry, binary))
951+
}
952+
for (const directory of (process.env.PATH ?? "").split(delimiter).filter(Boolean)) {
953+
candidates.push(join(directory, process.platform === "win32" ? "ffmpeg.exe" : "ffmpeg"))
954+
}
955+
for (const candidate of candidates) {
956+
if (await access(candidate, fsConstants.X_OK).then(() => true, () => false)) return candidate
957+
}
958+
throw new Error("video-quality=high requires ffmpeg: install Playwright's ffmpeg (npx playwright install ffmpeg), put ffmpeg on PATH, or set FFMPEG_PATH")
959+
}
960+
961+
function runFfmpeg(command: string, args: string[], stdinFiles: string[] = []): Promise<void> {
962+
return new Promise((resolve, reject) => {
963+
const child = spawn(command, args, { stdio: ["pipe", "ignore", "pipe"] })
964+
let stderr = ""
965+
child.stderr?.on("data", (chunk: Buffer) => { stderr = (stderr + chunk.toString()).slice(-4000) })
966+
child.once("error", reject)
967+
child.once("exit", (code) => code === 0 ? resolve() : reject(new Error(`ffmpeg exited with status ${code}: ${stderr.trim()}`)))
968+
child.stdin?.on("error", () => { /* surfaced through the exit status */ })
969+
void (async () => {
970+
for (const file of stdinFiles) {
971+
const chunk = await readFile(file)
972+
if (!child.stdin?.write(chunk)) await new Promise((resume) => child.stdin?.once("drain", resume))
973+
}
974+
child.stdin?.end()
975+
})().catch((error) => { child.kill(); reject(error) })
976+
})
977+
}
978+
881979
async function browserAnnotationThemeFromArgs(args: string[]): Promise<BrowserAnnotationTheme | undefined> {
882980
const raw = argValue(args, "annotation-theme-json")
883981
if (!raw) return undefined

‎tests/browser-actions-video-capture.browser.test.ts‎

Lines changed: 44 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -1,7 +1,7 @@
11
import assert from "node:assert/strict"
2-
import { mkdtemp, readFile, rm, stat } from "node:fs/promises"
2+
import { mkdtemp, readdir, readFile, rm, stat } from "node:fs/promises"
33
import { createServer } from "node:http"
4-
import { tmpdir } from "node:os"
4+
import { homedir, tmpdir } from "node:os"
55
import { join } from "node:path"
66
import test from "node:test"
77
import { spawnSync } from "node:child_process"
@@ -122,6 +122,48 @@ test("browser actions records high-density viewport video without padding", asyn
122122
}
123123
})
124124

125+
test("browser actions high video quality captures and encodes device-pixel frames", async () => {
126+
const fixture = await pageFixture()
127+
const artifactRoot = await mkdtemp(join(tmpdir(), "wp-codebox-browser-video-high-"))
128+
try {
129+
await runBrowserActionsCommand({
130+
artifactRoot,
131+
runtimeSpec,
132+
server: fixture.server,
133+
spec: { command: "wordpress.browser-actions", args: [] },
134+
plan: {
135+
steps: [{ kind: "navigate", url: fixture.url, waitFor: "load" }, { kind: "waitFor", selector: "#target", marker: "ready" }],
136+
capture: new Set(["steps", "video"]),
137+
videoQuality: "high",
138+
requestedEnvironment: { viewport: { width: 540, height: 960 }, deviceScaleFactor: 2 },
139+
stepTimeoutMs: 2_000,
140+
totalTimeoutMs: 10_000,
141+
networkSettleTimeoutMs: 100,
142+
maxDomSnapshotElements: 20,
143+
},
144+
})
145+
const recording = join(artifactRoot, "files/browser/video.webm")
146+
assert((await stat(recording)).size > 0)
147+
// Decode with the same ffmpeg the encoder resolves (Playwright's bundled
148+
// build on CI), so the test needs no system ffmpeg. Its stream header
149+
// reports the true encoded frame size.
150+
const browsers = process.env.PLAYWRIGHT_BROWSERS_PATH ?? join(homedir(), ".cache", "ms-playwright")
151+
const bundled = (await readdir(browsers).catch(() => [] as string[])).filter((entry) => entry.startsWith("ffmpeg-")).sort().reverse()
152+
const ffmpeg = process.env.FFMPEG_PATH ?? (bundled[0] ? join(browsers, bundled[0], process.platform === "linux" ? "ffmpeg-linux" : process.platform === "darwin" ? "ffmpeg-mac" : "ffmpeg-win64.exe") : "ffmpeg")
153+
const decoded = spawnSync(ffmpeg, ["-hide_banner", "-i", recording, "-frames:v", "1", "-c:v", "libvpx", "-f", "webm", "-y", join(artifactRoot, "decoded-probe.webm")], { timeout: 30_000, encoding: "utf8" })
154+
assert.equal(decoded.status, 0, `encoded high-quality video should decode: ${decoded.stderr}`)
155+
assert.match(decoded.stderr, /Video: vp8[^\n]*\b1080x1920\b/, "video stream is encoded at device pixels")
156+
const summary = JSON.parse(await readFile(join(artifactRoot, "files/browser/action-summary.json"), "utf8"))
157+
assert.deepEqual([summary.video.width, summary.video.height], [1080, 1920])
158+
assert.equal(summary.video.fps, 10)
159+
assert.deepEqual(summary.video.markers.map((marker: { name: string }) => marker.name), ["ready"])
160+
assert(summary.video.markers[0].endMs >= summary.video.markers[0].startMs)
161+
} finally {
162+
await rm(artifactRoot, { recursive: true, force: true })
163+
await fixture.close()
164+
}
165+
})
166+
125167
test("browser actions rejects an unsupported capture value", async () => {
126168
const fixture = await pageFixture()
127169
const artifactRoot = await mkdtemp(join(tmpdir(), "wp-codebox-browser-video-"))

0 commit comments

Comments
 (0)