Skip to content

Commit 1a8e455

Browse files
fix(browser-actions): stream high-quality video from the CDP screencast so motion stays smooth
video-quality=high captured a full-page screenshot every ~100ms, so cursor glides and smooth scrolls rendered as ~10fps jumps. Stream compositor frames with Page.startScreencast instead (~60fps during motion, nothing while idle) and resample them by paint timestamp onto a constant 30fps timeline anchored at the marker origin. Chromium only composites screencast frames at device pixels when the scale is forced at launch, so high-quality capture launches with --force-device-scale-factor alongside the context scale.
1 parent 0810a9f commit 1a8e455

3 files changed

Lines changed: 68 additions & 21 deletions

File tree

‎packages/runtime-playground/src/browser-actions-runner.ts‎

Lines changed: 52 additions & 13 deletions
Original file line numberDiff line numberDiff line change
@@ -145,7 +145,11 @@ export async function runBrowserActionsCommand({
145145
const startedAt = now()
146146
const startedAtMs = Date.now()
147147
const progress = createBrowserProbeProgressTracker(startedAt, 0)
148-
const browser = session?.browser ?? await launchChromiumBrowser()
148+
// High-fidelity capture streams compositor frames over CDP. Chromium only
149+
// composites at device pixels for the screencast when the scale is forced
150+
// at launch; the context's deviceScaleFactor alone yields CSS-pixel frames.
151+
const highVideoScale = runPlan.videoQuality === "high" && capture.has("video") ? (requestedEnvironment.deviceScaleFactor ?? 1) : 1
152+
const browser = session?.browser ?? await launchChromiumBrowser(highVideoScale !== 1 ? { args: [`--force-device-scale-factor=${highVideoScale}`] } : {})
149153
const topology = browserPreviewTopology(args, runtimeSpec, server.serverUrl, server.previewProxyDiagnostics?.targetOrigin)
150154
const { preview, networkPolicy } = topology
151155
const routeTracker = session?.routeTracker ?? createBrowserPreviewRouteTracker()
@@ -159,6 +163,7 @@ export async function runBrowserActionsCommand({
159163
let stopVideoCapture = false
160164
let videoFrameCount = 0
161165
let videoFrameError: Error | undefined
166+
const videoFrameTimes: number[] = []
162167
const highVideo = runPlan.videoQuality === "high"
163168
let videoSaved = false
164169
let videoCaptureFailed = false
@@ -247,17 +252,25 @@ export async function runBrowserActionsCommand({
247252
videoDimensions = { width: Math.round(viewport.width * scale), height: Math.round(viewport.height * scale) }
248253
const frameDirectory = join(videoStagingDirectory, "frames")
249254
await mkdir(frameDirectory, { recursive: true })
255+
// Stream compositor frames over CDP: Chromium emits a frame on every
256+
// repaint (~60fps during motion, none while idle), so cursor glides and
257+
// smooth scrolls stay continuous. Each frame keeps its paint timestamp
258+
// and is encoded with its real duration.
259+
const cdp = await page.context().newCDPSession(page)
260+
const frameWrites: Array<Promise<void>> = []
261+
cdp.on("Page.screencastFrame", (frame: { data: string; sessionId: number; metadata: { timestamp?: number } }) => {
262+
const index = ++videoFrameCount
263+
const paintedAtMs = typeof frame.metadata.timestamp === "number" ? frame.metadata.timestamp * 1000 : Date.now()
264+
videoFrameTimes.push(paintedAtMs)
265+
frameWrites.push(writeFile(join(frameDirectory, `frame-${String(index).padStart(6, "0")}.jpg`), Buffer.from(frame.data, "base64")).catch((error) => { videoFrameError ??= error instanceof Error ? error : new Error(String(error)) }))
266+
void cdp.send("Page.screencastFrameAck", { sessionId: frame.sessionId }).catch(() => undefined)
267+
})
268+
await cdp.send("Page.startScreencast", { format: "jpeg", quality: 92, maxWidth: videoDimensions.width, maxHeight: videoDimensions.height, everyNthFrame: 1 })
250269
videoCaptureTask = (async () => {
251-
while (!stopVideoCapture) {
252-
try {
253-
const frame = await page.screenshot({ type: "jpeg", quality: 95, scale: "device", animations: "allow" })
254-
await writeFile(join(frameDirectory, `frame-${String(++videoFrameCount).padStart(6, "0")}.jpg`), frame)
255-
} catch (error) {
256-
if (stopVideoCapture) return
257-
}
258-
const nextFrameAt = (videoRecordingOrigin ?? performance.now()) + videoFrameCount * 100
259-
await new Promise((resolve) => setTimeout(resolve, Math.max(0, nextFrameAt - performance.now())))
260-
}
270+
while (!stopVideoCapture) await new Promise((resolve) => setTimeout(resolve, 50))
271+
await cdp.send("Page.stopScreencast").catch(() => undefined)
272+
await Promise.all(frameWrites)
273+
await cdp.detach().catch(() => undefined)
261274
})()
262275
} else {
263276
videoRecording = page.video()
@@ -609,7 +622,13 @@ export async function runBrowserActionsCommand({
609622
// decoder only) as well as a full system build. Quality-first settings
610623
// replace the realtime recorder's ~1 Mbps warm-up.
611624
const frameFiles = (await readdir(frames)).filter((entry) => entry.endsWith(".jpg")).sort().map((entry) => join(frames, entry))
612-
await runFfmpeg(ffmpeg, ["-y", "-f", "image2pipe", "-c:v", "mjpeg", "-framerate", "10", "-i", "pipe:0", "-c:v", "libvpx", "-deadline", "good", "-cpu-used", "0", "-crf", "4", "-b:v", "40M", "-qmin", "0", "-qmax", "12", "-pix_fmt", "yuv420p", "-an", output], frameFiles)
625+
// Screencast frames arrive only when the page repaints. Resample them
626+
// onto a constant 30fps timeline anchored at the recording origin (the
627+
// same origin step markers use): each output tick shows the latest
628+
// frame painted at or before it.
629+
const durationMs = Math.max(1, (videoFinishedAt ?? performance.now()) - (videoStartedAt ?? videoRecordingOrigin ?? performance.now()))
630+
const timeline = highVideoFrameTimeline(videoFrameTimes.slice(0, frameFiles.length), videoStartedWallAt ?? videoFrameTimes[0] ?? 0, durationMs, HIGH_VIDEO_FPS)
631+
await runFfmpeg(ffmpeg, ["-y", "-f", "image2pipe", "-c:v", "mjpeg", "-framerate", String(HIGH_VIDEO_FPS), "-i", "pipe:0", "-c:v", "libvpx", "-deadline", "good", "-cpu-used", "0", "-crf", "4", "-b:v", "40M", "-qmin", "0", "-qmax", "12", "-pix_fmt", "yuv420p", "-an", output], timeline.map((index) => frameFiles[index]!))
613632
await artifactSession.writeGenerated("video", "video.webm", (path) => copyFile(output, path))
614633
videoSaved = true
615634
} catch (error) {
@@ -687,7 +706,7 @@ export async function runBrowserActionsCommand({
687706
path: "files/browser/video.webm",
688707
width: videoDimensions?.width ?? 0,
689708
height: videoDimensions?.height ?? 0,
690-
fps: highVideo ? 10 : 30,
709+
fps: 30,
691710
durationMs: Math.max(0, Math.round((videoFinishedAt ?? performance.now()) - (videoStartedAt ?? performance.now()))),
692711
markers: videoMarkers,
693712
} : undefined
@@ -934,6 +953,26 @@ async function browserActionsRunPlanFromArgs(args: string[], artifactRoot: strin
934953
}
935954
}
936955

956+
const HIGH_VIDEO_FPS = 30
957+
958+
/**
959+
* Map a constant-rate output timeline onto variable-rate screencast frames.
960+
* Returns, for each output tick, the index of the latest frame painted at or
961+
* before that tick (the first frame before any paint).
962+
*/
963+
export function highVideoFrameTimeline(frameTimesMs: number[], originMs: number, durationMs: number, fps: number): number[] {
964+
if (frameTimesMs.length === 0) return []
965+
const ticks = Math.max(1, Math.round((durationMs / 1000) * fps))
966+
const out: number[] = []
967+
let cursor = 0
968+
for (let tick = 0; tick < ticks; tick++) {
969+
const at = originMs + (tick * 1000) / fps
970+
while (cursor + 1 < frameTimesMs.length && frameTimesMs[cursor + 1]! <= at) cursor++
971+
out.push(cursor)
972+
}
973+
return out
974+
}
975+
937976
function browserVideoQualityArg(args: string[]): "default" | "high" {
938977
const value = argValue(args, "video-quality")?.trim() ?? "default"
939978
if (value !== "default" && value !== "high") throw new Error("wordpress.browser-actions video-quality must be default or high")

‎packages/runtime-playground/src/browser-capture-session.ts‎

Lines changed: 6 additions & 6 deletions
Original file line numberDiff line numberDiff line change
@@ -5,14 +5,14 @@ import { serializeBrowserConsoleMessage, serializeBrowserError, serializeBrowser
55
import type { Browser, Page, Request } from "playwright"
66
import { assertPlaywrightBrowserReady } from "./playwright-browser-provenance.js"
77

8-
export async function launchChromiumBrowser(): Promise<Browser> {
8+
export async function launchChromiumBrowser(options: { args?: string[] } = {}): Promise<Browser> {
99
const { chromium } = await import("playwright")
1010
await assertPlaywrightBrowserReady()
11-
return chromium.launch(
12-
process.env.WP_CODEBOX_BROWSER_CHANNEL
13-
? { channel: process.env.WP_CODEBOX_BROWSER_CHANNEL }
14-
: undefined,
15-
)
11+
const launchOptions = {
12+
...(process.env.WP_CODEBOX_BROWSER_CHANNEL ? { channel: process.env.WP_CODEBOX_BROWSER_CHANNEL } : {}),
13+
...(options.args?.length ? { args: options.args } : {}),
14+
}
15+
return chromium.launch(Object.keys(launchOptions).length ? launchOptions : undefined)
1616
}
1717

1818
export function chromiumBrowserMetadata(browser: Browser): { name: "chromium"; channel: string; version: string } {

‎tests/browser-actions-video-capture.browser.test.ts‎

Lines changed: 10 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -7,7 +7,7 @@ import test from "node:test"
77
import { spawnSync } from "node:child_process"
88
import { existsSync } from "node:fs"
99

10-
import { runBrowserActionsCommand } from "../packages/runtime-playground/src/browser-actions-runner.js"
10+
import { highVideoFrameTimeline, runBrowserActionsCommand } from "../packages/runtime-playground/src/browser-actions-runner.js"
1111
import { wordpressRuntimeSpec } from "../scripts/test-kit.js"
1212

1313
const runtimeSpec = wordpressRuntimeSpec({ commands: ["wordpress.browser-actions"] })
@@ -155,7 +155,7 @@ test("browser actions high video quality captures and encodes device-pixel frame
155155
assert.match(decoded.stderr, /Video: vp8[^\n]*\b1080x1920\b/, "video stream is encoded at device pixels")
156156
const summary = JSON.parse(await readFile(join(artifactRoot, "files/browser/action-summary.json"), "utf8"))
157157
assert.deepEqual([summary.video.width, summary.video.height], [1080, 1920])
158-
assert.equal(summary.video.fps, 10)
158+
assert.equal(summary.video.fps, 30)
159159
assert.deepEqual(summary.video.markers.map((marker: { name: string }) => marker.name), ["ready"])
160160
assert(summary.video.markers[0].endMs >= summary.video.markers[0].startMs)
161161
} finally {
@@ -221,3 +221,11 @@ function bottomRightVideoPixel(videoPath: string): number[] | undefined {
221221
}
222222
return undefined
223223
}
224+
225+
test("high video quality resamples variable-rate screencast frames onto a constant timeline", () => {
226+
// Frames painted at 0ms, 10ms, 500ms; 1s at 10fps → ticks at 0..900ms.
227+
assert.deepEqual(highVideoFrameTimeline([1000, 1010, 1500], 1000, 1000, 10), [0, 1, 1, 1, 1, 2, 2, 2, 2, 2])
228+
// Ticks before the first paint show the first frame.
229+
assert.deepEqual(highVideoFrameTimeline([1200], 1000, 500, 10), [0, 0, 0, 0, 0])
230+
assert.deepEqual(highVideoFrameTimeline([], 0, 1000, 30), [])
231+
})

0 commit comments

Comments
 (0)