Skip to content

Commit 6d54f09

Browse files
fix(browser-actions): record video at the CSS viewport size instead of padding a scaled frame (#2552) (#2555)
Co-authored-by: homeboy-ci[bot] <266378653+homeboy-ci[bot]@users.noreply.github.com>
1 parent 1bbf423 commit 6d54f09

3 files changed

Lines changed: 31 additions & 11 deletions

File tree

‎README.md‎

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -766,7 +766,7 @@ For synthetic web performance investigations, enable `capture=performance` and u
766766

767767
Action journeys and adaptive exploration can declare their browser context with `browser-environment-json=<object>` (inline or `@<path>`) or the compatible `device`, `viewport`, `device-scale-factor`, `is-mobile`, `has-touch`, `user-agent`, `permissions`, `locale`, `timezone`, and `geolocation-*` arguments. These controls are applied when the Playwright context is created, so mobile/touch emulation and granted, denied, or prompt geolocation remain active for the full journey. Action summaries keep requested configuration, provider-resolved configuration, and browser-observed state separate, with explicit unsupported and inconclusive dimensions. `wordpress.browser-scenario` accepts the same arguments or an `environment` object inside `scenario-json`; probe collection and authored actions share one owned context/page, while multi-actor scenarios create one isolated configured context per actor.
768768

769-
For `wordpress.browser-actions capture=video`, `video-size=<width>x<height>` sets the WebM recording dimensions without changing its format or encoding. When omitted, recordings retain Playwright's default size unless `device-scale-factor` is greater than 1, in which case the viewport dimensions are multiplied by that scale. Requested dimensions are capped at 8,294,400 pixels total with aspect ratio preserved. Video step records include `videoOffsetMs: { startMs, endMs }` relative to recording start; named `screenshot` steps and steps with a non-empty free-form `marker` label appear in the `video.markers` timeline in `action-summary.json`.
769+
For `wordpress.browser-actions capture=video`, `video-size=<width>x<height>` sets the WebM recording dimensions without changing its format or encoding. When omitted, recordings use the CSS viewport size. Playwright's recorder captures at CSS pixels and pads any larger frame, so `video-size` can only scale a recording down; upscale with an encoder afterwards when a higher-resolution output is needed. Requested dimensions are capped at 8,294,400 pixels total with aspect ratio preserved. Video step records include `videoOffsetMs: { startMs, endMs }` relative to recording start; named `screenshot` steps and steps with a non-empty free-form `marker` label appear in the `video.markers` timeline in `action-summary.json`.
770770

771771
Step kinds: `navigate` (`url`, optional `waitFor=domcontentloaded|load|networkidle`), `click`/`hover` (`selector` or `text`), `fill`/`type` (`selector`, `value`), `press` (`key`, optional `selector`), `drag` (`from` selector, `to` as `{ "selector": ... }` or `{ "x": n, "y": n }`), `select` (`selector`, `value` or `values`), `scroll` (`selector`, `position=top|bottom`, or `by: {x,y}`), `waitFor` (`selector` or `waitFor=domcontentloaded|load|networkidle|duration|selector:<sel>`), `evaluate` (`expression`, optional `assert` to deep-equal the result), `expect` (`selector`, optional `state=visible|hidden|attached|detached|enabled|disabled|checked|unchecked|editable`), `annotate` (highlight, spotlight, arrow, anchored label, fixed caption, or clear), and `screenshot` (optional `name` and boolean `fullPage`; defaults to `true`, except with `capture=video` when it defaults to `false` to keep viewport-sized video frames. Set `fullPage: true` explicitly to request a full-document screenshot while recording). Every step may set its own `timeout=<n>s`; the command also accepts a global `step-timeout=<n>s` (per step) and `timeout=<n>s` (total-script budget). Both are bounded and deterministic — the run stops cleanly on the first failing step, with no silent partial success.
772772

‎packages/runtime-playground/src/browser-actions-runner.ts‎

Lines changed: 5 additions & 5 deletions
Original file line numberDiff line numberDiff line change
@@ -221,7 +221,7 @@ export async function runBrowserActionsCommand({
221221
...topology.contextOptions(),
222222
...(storageStateImport ? { storageState: storageStateImport.storageState } : {}),
223223
...(runPlan.transportFaults ? { serviceWorkers: "block" as const } : {}),
224-
...(capture.has("video") ? { recordVideo: { dir: videoStagingDirectory, ...(browserVideoSize(runPlan.videoSize, requestedEnvironment.viewport, requestedEnvironment.deviceScaleFactor) ? { size: browserVideoSize(runPlan.videoSize, requestedEnvironment.viewport, requestedEnvironment.deviceScaleFactor) } : {}) } } : {}),
224+
...(capture.has("video") ? { recordVideo: { dir: videoStagingDirectory, ...(browserVideoSize(runPlan.videoSize, requestedEnvironment.viewport) ? { size: browserVideoSize(runPlan.videoSize, requestedEnvironment.viewport) } : {}) } } : {}),
225225
},
226226
}) : undefined)
227227
const context = environmentRuntime?.context ?? null
@@ -233,7 +233,7 @@ export async function runBrowserActionsCommand({
233233
if (capture.has("video")) {
234234
videoRecording = page.video()
235235
videoStartedAt = videoRecordingOrigin
236-
videoDimensions = browserVideoSize(runPlan.videoSize, requestedEnvironment.viewport, requestedEnvironment.deviceScaleFactor) ?? requestedEnvironment.viewport ?? { width: 1280, height: 720 }
236+
videoDimensions = browserVideoSize(runPlan.videoSize, requestedEnvironment.viewport) ?? requestedEnvironment.viewport ?? { width: 1280, height: 720 }
237237
}
238238
navigationTracker = trackBrowserNavigation(page)
239239
if (onProgress) {
@@ -891,11 +891,11 @@ function browserVideoSizeArg(args: string[]): { width: number; height: number }
891891
const width = Number(match[1])
892892
const height = Number(match[2])
893893
if (!Number.isSafeInteger(width) || !Number.isSafeInteger(height) || width < 1 || height < 1) throw new Error("wordpress.browser-actions video-size dimensions must be positive integers")
894-
return browserVideoSize({ width, height }, undefined, 1)
894+
return browserVideoSize({ width, height }, undefined)
895895
}
896896

897-
function browserVideoSize(explicit: { width: number; height: number } | undefined, viewport: { width: number; height: number } | undefined, scale = 1): { width: number; height: number } | undefined {
898-
const requested = explicit ?? (viewport && scale > 1 ? { width: Math.round(viewport.width * scale), height: Math.round(viewport.height * scale) } : undefined)
897+
function browserVideoSize(explicit: { width: number; height: number } | undefined, viewport: { width: number; height: number } | undefined): { width: number; height: number } | undefined {
898+
const requested = explicit ?? viewport
899899
if (!requested) return undefined
900900
const size = requested
901901
const factor = Math.min(1, Math.sqrt(BROWSER_VIDEO_MAX_PIXELS / (size.width * size.height)))

‎tests/browser-actions-video-capture.browser.test.ts‎

Lines changed: 25 additions & 5 deletions
Original file line numberDiff line numberDiff line change
@@ -4,6 +4,8 @@ import { createServer } from "node:http"
44
import { tmpdir } from "node:os"
55
import { join } from "node:path"
66
import test from "node:test"
7+
import { spawnSync } from "node:child_process"
8+
import { existsSync } from "node:fs"
79

810
import { runBrowserActionsCommand } from "../packages/runtime-playground/src/browser-actions-runner.js"
911
import { wordpressRuntimeSpec } from "../scripts/test-kit.js"
@@ -83,8 +85,8 @@ test("screenshot steps during video capture default to viewport-sized captures",
8385
}
8486
})
8587

86-
test("browser actions scales video dimensions from a high-density viewport by default", async () => {
87-
const fixture = await pageFixture()
88+
test("browser actions records high-density viewport video without padding", async () => {
89+
const fixture = await pageFixture(false, true)
8890
const artifactRoot = await mkdtemp(join(tmpdir(), "wp-codebox-browser-video-"))
8991
try {
9092
await runBrowserActionsCommand({
@@ -104,7 +106,14 @@ test("browser actions scales video dimensions from a high-density viewport by de
104106
})
105107

106108
const summary = JSON.parse(await readFile(join(artifactRoot, "files/browser/action-summary.json"), "utf8"))
107-
assert.deepEqual({ width: summary.video.width, height: summary.video.height }, { width: 1075, height: 2330 })
109+
assert.deepEqual({ width: summary.video.width, height: summary.video.height }, { width: 430, height: 932 })
110+
111+
const pixel = bottomRightVideoPixel(join(artifactRoot, "files/browser/video.webm"))
112+
if (pixel) {
113+
assert(pixel[2]! > 200 && pixel[0]! < 50, `bottom-right pixel should be blue page content, got ${pixel}`)
114+
} else {
115+
test.diagnostic?.("ffmpeg unavailable; skipped bottom-right pixel padding check")
116+
}
108117
} finally {
109118
await rm(artifactRoot, { recursive: true, force: true })
110119
await fixture.close()
@@ -138,10 +147,10 @@ test("browser actions rejects an unsupported capture value", async () => {
138147
}
139148
})
140149

141-
async function pageFixture(tall = false) {
150+
async function pageFixture(tall = false, fullBleed = false) {
142151
const httpServer = createServer((_request, response) => {
143152
response.setHeader("content-type", "text/html")
144-
response.end(`<!doctype html><title>video fixture</title><button id="target">press</button><main style="height:${tall ? 10000 : 500}px">ready</main>`)
153+
response.end(`<!doctype html><title>video fixture</title><body style="margin:0;${fullBleed ? "background:rgb(0,0,255)" : ""}"><button id="target">press</button><main style="height:${tall ? 10000 : 500}px">ready</main></body>`)
145154
})
146155
await new Promise<void>((resolve) => httpServer.listen(0, "127.0.0.1", resolve))
147156
const address = httpServer.address()
@@ -157,3 +166,14 @@ async function pageFixture(tall = false) {
157166
close: () => new Promise<void>((resolve, reject) => httpServer.close((error) => error ? reject(error) : resolve())),
158167
}
159168
}
169+
170+
function bottomRightVideoPixel(videoPath: string): number[] | undefined {
171+
const browsers = process.env.PLAYWRIGHT_BROWSERS_PATH ?? join(process.env.HOME ?? "", ".cache/ms-playwright")
172+
const candidates = [process.env.FFMPEG_PATH, "ffmpeg", join(browsers, "ffmpeg-1011", "ffmpeg-linux")].filter((value): value is string => Boolean(value))
173+
for (const ffmpeg of candidates) {
174+
if (ffmpeg !== "ffmpeg" && !existsSync(ffmpeg)) continue
175+
const result = spawnSync(ffmpeg, ["-v", "error", "-ss", "0.4", "-i", videoPath, "-frames:v", "1", "-vf", "crop=1:1:iw-2:ih-2", "-f", "rawvideo", "-pix_fmt", "rgb24", "-"], { timeout: 30_000 })
176+
if (result.status === 0 && result.stdout.length >= 3) return [...result.stdout.subarray(0, 3)]
177+
}
178+
return undefined
179+
}

0 commit comments

Comments
 (0)