diff --git a/README.md b/README.md index 49c64a80..279d1f44 100644 --- a/README.md +++ b/README.md @@ -768,7 +768,21 @@ Action journeys and adaptive exploration can declare their browser context with For `wordpress.browser-actions capture=video`, `video-size=x` sets the WebM recording dimensions without changing its format or encoding. When omitted, recordings retain Playwright's default size unless `device-scale-factor` is greater than 1, in which case the viewport dimensions are multiplied by that scale. Requested dimensions are capped at 8,294,400 pixels total with aspect ratio preserved. Video step records include `videoOffsetMs: { startMs, endMs }` relative to recording start; named `screenshot` steps and steps with a non-empty free-form `marker` label appear in the `video.markers` timeline in `action-summary.json`. -Step kinds: `navigate` (`url`, optional `waitFor=domcontentloaded|load|networkidle`), `click`/`hover` (`selector` or `text`), `fill`/`type` (`selector`, `value`), `press` (`key`, optional `selector`), `drag` (`from` selector, `to` as `{ "selector": ... }` or `{ "x": n, "y": n }`), `select` (`selector`, `value` or `values`), `waitFor` (`selector` or `waitFor=domcontentloaded|load|networkidle|duration|selector:`), `evaluate` (`expression`, optional `assert` to deep-equal the result), `expect` (`selector`, optional `state=visible|hidden|attached|detached|enabled|disabled|checked|unchecked|editable`), and `screenshot` (optional `name` and boolean `fullPage`; defaults to `true`, except with `capture=video` when it defaults to `false` to keep viewport-sized video frames. Set `fullPage: true` explicitly to request a full-document screenshot while recording). Every step may set its own `timeout=s`; the command also accepts a global `step-timeout=s` (per step) and `timeout=s` (total-script budget). Both are bounded and deterministic — the run stops cleanly on the first failing step, with no silent partial success. +Step kinds: `navigate` (`url`, optional `waitFor=domcontentloaded|load|networkidle`), `click`/`hover` (`selector` or `text`), `fill`/`type` (`selector`, `value`), `press` (`key`, optional `selector`), `drag` (`from` selector, `to` as `{ "selector": ... }` or `{ "x": n, "y": n }`), `select` (`selector`, `value` or `values`), `scroll` (`selector`, `position=top|bottom`, or `by: {x,y}`), `waitFor` (`selector` or `waitFor=domcontentloaded|load|networkidle|duration|selector:`), `evaluate` (`expression`, optional `assert` to deep-equal the result), `expect` (`selector`, optional `state=visible|hidden|attached|detached|enabled|disabled|checked|unchecked|editable`), `annotate` (highlight, spotlight, arrow, anchored label, fixed caption, or clear), and `screenshot` (optional `name` and boolean `fullPage`; defaults to `true`, except with `capture=video` when it defaults to `false` to keep viewport-sized video frames. Set `fullPage: true` explicitly to request a full-document screenshot while recording). Every step may set its own `timeout=s`; the command also accepts a global `step-timeout=s` (per step) and `timeout=s` (total-script budget). Both are bounded and deterministic — the run stops cleanly on the first failing step, with no silent partial success. + +Annotations render in a pointer-transparent, isolated shadow overlay and do not require the `evaluate` policy grant. They follow element targets through scrolling. Use `id` to replace or later clear an annotation, `durationMs` for automatic removal, `animate` (`fade`, `draw`, or `none`), and run-level `annotation-theme-json=` (`accentColor`, `textColor`, `background`, `fontFamily`, `fontSize`, `strokeWidth`, `radius`) to customize appearance. Arrow `direction` identifies the side/corner from which the SVG arrow points toward its target. A full journey can sequence generic callouts and clear them when finished: + +```jsonc +[ + { "kind": "navigate", "url": "/" }, + { "kind": "annotate", "id": "focus", "shape": "highlight", "selector": "button.save", "style": { "variant": "ring", "padding": 8 } }, + { "kind": "annotate", "shape": "label", "text": "Select this control to continue", "anchor": { "selector": "button.save", "placement": "top" } }, + { "kind": "annotate", "shape": "arrow", "selector": "button.save", "direction": "bottom-left", "animate": "draw" }, + { "kind": "click", "selector": "button.save" }, + { "kind": "annotate", "shape": "caption", "text": "Changes are saved", "position": "bottom", "durationMs": 1800 }, + { "kind": "annotate", "clear": ["focus"] } +] +``` Opt in to human-paced presentation with `presentation-json=` (or `@`). It supports `pointer: { enabled, style: "arrow"|"dot"|"touch", size, color }`, `clickFeedback: "ripple"|"pulse"`, `motion: { moveDurationMs, easing }`, and `typing: { delayMs, applyToFill }`. The pointer lives in an isolated, non-hit-testable shadow overlay and is reinstalled at document startup on navigation. Presentation actions bring targets into view and move to their center; typing is character-paced, while fills remain instant unless `applyToFill` is true. Add a `scroll` step with `selector`, `position: "top"|"bottom"`, or `by: {x,y}`, plus optional `behavior`, `durationMs`, and `block`; scrolling waits for the viewport to settle. diff --git a/package.json b/package.json index 290c03f0..bc08a587 100644 --- a/package.json +++ b/package.json @@ -187,7 +187,8 @@ "check": "npm run smoke -- --group=check", "check:all": "npm run test:all && npm run cloudflare:check", "test:browser-video-capture": "tsx tests/browser-actions-video-capture.browser.test.ts", - "test:browser-presentation": "tsx tests/browser-actions-presentation.browser.test.ts" + "test:browser-presentation": "tsx tests/browser-actions-presentation.browser.test.ts", + "test:browser-annotations": "tsx --test tests/browser-annotations.browser.test.ts" }, "workspaces": [ "packages/cli", diff --git a/packages/runtime-core/src/browser-interaction.ts b/packages/runtime-core/src/browser-interaction.ts index 678b4044..5a14e70f 100644 --- a/packages/runtime-core/src/browser-interaction.ts +++ b/packages/runtime-core/src/browser-interaction.ts @@ -31,6 +31,7 @@ export const BROWSER_INTERACTION_STEP_KINDS = [ "screenshot", "capture", "callTool", + "annotate", ] as const export type BrowserInteractionStepKind = typeof BROWSER_INTERACTION_STEP_KINDS[number] @@ -99,11 +100,18 @@ export interface BrowserInteractionStep { tool?: string /** JSON input passed to the caller-provided host tool. */ input?: JsonValue - position?: "top" | "bottom" + position?: "top" | "center" | "bottom" by?: { x: number; y: number } behavior?: "smooth" | "instant" - durationMs?: number block?: "start" | "center" | "end" + id?: string + shape?: "highlight" | "spotlight" | "arrow" | "label" | "caption" + clear?: string[] | true + style?: { variant?: "ring" | "box" | "underline"; padding?: number } + anchor?: { selector: string; placement?: "top" | "bottom" | "left" | "right" } + durationMs?: number + animate?: "draw" | "fade" | "none" + direction?: "top" | "bottom" | "left" | "right" | "top-left" | "top-right" | "bottom-left" | "bottom-right" } export interface BrowserToolVerifierInputSummary { @@ -368,6 +376,21 @@ export function validateBrowserInteractionScript(input: unknown): BrowserInterac issues.push({ index, message: "callTool step input must be JSON-serializable" }) } break + case "annotate": + if (step.clear !== undefined) { + if (step.clear !== true && (!Array.isArray(step.clear) || step.clear.some((id) => typeof id !== "string" || !id))) issues.push({ index, message: "annotate clear must be true or an array of ids" }) + if (step.shape !== undefined) issues.push({ index, message: "annotate clear cannot include shape" }) + break + } + if (!( ["highlight", "spotlight", "arrow", "label", "caption"] as const).includes(step.shape as never)) issues.push({ index, message: "annotate requires shape: highlight, spotlight, arrow, label, or caption" }) + if (step.durationMs !== undefined && (!Number.isFinite(step.durationMs) || step.durationMs < 0)) issues.push({ index, message: "annotate durationMs must be a non-negative number" }) + if (step.animate !== undefined && !["draw", "fade", "none"].includes(step.animate)) issues.push({ index, message: "annotate animate must be draw, fade, or none" }) + if (["highlight", "spotlight", "arrow"].includes(String(step.shape)) && !hasSelector) issues.push({ index, message: `annotate ${step.shape} requires selector` }) + if (["label", "caption"].includes(String(step.shape)) && !hasText) issues.push({ index, message: `annotate ${step.shape} requires text` }) + if (step.shape === "label" && (!step.anchor || typeof step.anchor.selector !== "string" || !step.anchor.selector)) issues.push({ index, message: "annotate label requires anchor.selector" }) + if (step.shape === "caption" && step.position !== undefined && !["top", "center", "bottom"].includes(step.position)) issues.push({ index, message: "annotate caption position must be top, center, or bottom" }) + if (step.shape === "arrow" && step.direction !== undefined && !["top", "bottom", "left", "right", "top-left", "top-right", "bottom-left", "bottom-right"].includes(step.direction)) issues.push({ index, message: "annotate arrow direction is invalid" }) + break } steps.push(step) diff --git a/packages/runtime-playground/src/browser-actions-runner.ts b/packages/runtime-playground/src/browser-actions-runner.ts index 2a7018f0..7e433b2c 100644 --- a/packages/runtime-playground/src/browser-actions-runner.ts +++ b/packages/runtime-playground/src/browser-actions-runner.ts @@ -29,6 +29,7 @@ import { createBrowserAccessibilityCollector } from "./browser-accessibility-col import { browserEnvironmentCell, createPlaywrightBrowserEnvironmentContext, observePlaywrightBrowserEnvironment, resolvePlaywrightBrowserEnvironment, type PlaywrightBrowserEnvironmentSession } from "./browser-environment-matrix.js" import { installBrowserTransportFaults, type BrowserTransportFaultReport, type InstalledBrowserTransportFaults } from "./browser-transport-faults.js" import { browserPresentationInitScript, validateBrowserPresentation, type BrowserPresentation } from "./browser-presentation.js" +import { setBrowserAnnotationTheme, type BrowserAnnotationTheme } from "./browser-annotations.js" export { discoverBrowserActionCorpusDescriptors } from "./browser-action-discovery.js" @@ -57,6 +58,7 @@ export interface BrowserActionsRunPlan { transportFaults?: TransportFaultModel reusePage?: boolean presentation?: BrowserPresentation + annotationTheme?: BrowserAnnotationTheme } interface BrowserRunPlan { @@ -213,6 +215,7 @@ export async function runBrowserActionsCommand({ videoRecordingOrigin = performance.now() videoStartedWallAt = Date.now() } + setBrowserAnnotationTheme(runPlan.annotationTheme ?? {}) environmentRuntime = session?.runtime ?? (needsEnvironmentContext ? await createPlaywrightBrowserEnvironmentContext(browser, resolvedEnvironment, { contextOptions: { ...topology.contextOptions(), @@ -865,10 +868,19 @@ async function browserActionsRunPlanFromArgs(args: string[], artifactRoot: strin actionCorpus: browserActionCorpusFromArgs(args), adaptiveExploration: adaptiveExplorationPlan.contract, presentation: await browserPresentationFromArgs(args), + annotationTheme: await browserAnnotationThemeFromArgs(args), transportFaults: await transportFaultModelFromArg(args), } } +async function browserAnnotationThemeFromArgs(args: string[]): Promise { + const raw = argValue(args, "annotation-theme-json") + if (!raw) return undefined + const value: unknown = JSON.parse(raw.startsWith("@") ? await readFile(resolveCommandPath(raw.slice(1)), "utf8") : raw) + if (!value || typeof value !== "object" || Array.isArray(value)) throw new Error("annotation-theme-json must be an object") + return value as BrowserAnnotationTheme +} + const BROWSER_VIDEO_MAX_PIXELS = 8_294_400 function browserVideoSizeArg(args: string[]): { width: number; height: number } | undefined { diff --git a/packages/runtime-playground/src/browser-annotations.ts b/packages/runtime-playground/src/browser-annotations.ts new file mode 100644 index 00000000..a850fa95 --- /dev/null +++ b/packages/runtime-playground/src/browser-annotations.ts @@ -0,0 +1,108 @@ +import type { Page } from "playwright" +import type { BrowserInteractionStep } from "@automattic/wp-codebox-core" + +type AnnotationSpec = { + id?: string + shape?: string + selector?: string + text?: string + clear?: string[] | true + durationMs?: number + animate?: string + direction?: string + position?: string + style?: { variant?: string; padding?: number } + anchor?: { selector?: string; placement?: string } + theme?: BrowserAnnotationTheme +} + +export type BrowserAnnotationTheme = { accentColor?: string; textColor?: string; background?: string; fontFamily?: string; fontSize?: number; strokeWidth?: number; radius?: number } +let annotationTheme: BrowserAnnotationTheme = {} +export function setBrowserAnnotationTheme(theme: BrowserAnnotationTheme): void { annotationTheme = theme } + +export async function executeBrowserAnnotation(page: Page, step: BrowserInteractionStep): Promise { + if (step.clear !== undefined) { + await page.evaluate((ids) => { + const host = document.querySelector("#__wp_codebox_annotations") as (HTMLElement & { __annotations?: Map void }> }) | null + const annotations = host?.__annotations + if (!annotations) return + const selected = ids === true ? [...annotations.keys()] : ids + for (const id of selected) { annotations.get(id)?.remove(); annotations.delete(id) } + if (annotations.size === 0) host?.remove() + }, step.clear) + return + } + if (step.selector) await page.locator(step.selector).waitFor({ state: "visible" }) + if (step.anchor?.selector) await page.locator(step.anchor.selector).waitFor({ state: "visible" }) + const spec: AnnotationSpec = { + id: step.id, shape: step.shape, selector: step.selector, text: step.text, durationMs: step.durationMs, + animate: step.animate, direction: step.direction, position: step.position, + style: step.style, anchor: step.anchor, theme: annotationTheme, + } + await page.evaluate("globalThis.__name ??= (value) => value") + await page.evaluate((spec) => { + type Annotation = { remove: () => void } + type AnnotationHost = HTMLElement & { __annotations?: Map } + let host = document.querySelector("#__wp_codebox_annotations") as AnnotationHost | null + if (!host) { + host = document.createElement("div") as AnnotationHost + host.id = "__wp_codebox_annotations" + host.setAttribute("aria-hidden", "true") + Object.assign(host.style, { position: "fixed", inset: "0", zIndex: "2147483647", pointerEvents: "none" }) + document.documentElement.append(host) + const root = host.attachShadow({ mode: "open" }) + const layer = document.createElement("div") + root.append(layer) + ;(host as AnnotationHost & { __layer?: HTMLElement }).__layer = layer + host.__annotations = new Map() + } + const layer = (host as AnnotationHost & { __layer?: HTMLElement }).__layer! + Object.assign(layer.style, { position: "fixed", inset: "0", pointerEvents: "none", fontFamily: "system-ui, sans-serif", color: "#fff" }) + ;(host as AnnotationHost & { __layer?: HTMLElement }).__layer = layer + const annotations = host.__annotations ??= new Map() + const id = spec.id || `annotation-${Date.now()}-${Math.random()}` + annotations.get(id)?.remove() + const node = document.createElement("div") + const target = spec.shape === "label" ? document.querySelector(spec.anchor?.selector || "") : spec.selector ? document.querySelector(spec.selector) : null + if (["highlight", "spotlight", "arrow", "label"].includes(String(spec.shape)) && !target) throw new Error("Annotation target was not found") + const theme = spec.theme ?? {} + const color = String(theme.accentColor || "#64748b") + const textColor = String(theme.textColor || "#fff") + const background = String(theme.background || "rgba(15, 23, 42, .94)") + function rect() { return target?.getBoundingClientRect() } + function setRect(value: DOMRect) { + const padding = Number(spec.style?.padding ?? 0) + Object.assign(node.style, { left: `${value.left - padding}px`, top: `${value.top - padding}px`, width: `${value.width + padding * 2}px`, height: `${value.height + padding * 2}px` }) + } + function render() { + const value = rect() + if (!value) return + if (spec.shape === "highlight") setRect(value) + if (spec.shape === "label") Object.assign(node.style, { left: `${value.left}px`, top: `${spec.anchor?.placement === "bottom" ? value.bottom + 8 : value.top - 44}px` }) + if (spec.shape === "spotlight") { const p = Number(spec.style?.padding ?? 8); const l=value.left-p,t=value.top-p,w=value.width+p*2,h=value.height+p*2; Object.assign(node.style,{clipPath:`polygon(0 0,100% 0,100% 100%,0 100%,0 0,${l}px ${t}px,${l}px ${t+h}px,${l+w}px ${t+h}px,${l+w}px ${t}px,${l}px ${t}px)`,background:"rgba(0,0,0,.62)"}) } + if (spec.shape === "arrow") { + const direction = spec.direction || "bottom-left" + const [vertical, horizontal] = direction.split("-").length === 2 ? direction.split("-") : [direction, "center"] + Object.assign(node.style, { left: `${horizontal === "left" ? value.left - 50 : horizontal === "right" ? value.right + 10 : value.left + value.width / 2}px`, top: `${vertical === "top" ? value.top - 50 : vertical === "bottom" ? value.bottom + 10 : value.top + value.height / 2}px` }) + } + } + Object.assign(node.style, { position: "fixed", boxSizing: "border-box", pointerEvents: "none", transition: spec.animate === "fade" || spec.animate === undefined ? "opacity .25s ease" : "none", opacity: "1" }) + if (spec.shape === "highlight") Object.assign(node.style, { border: `${Number(theme.strokeWidth ?? 3)}px solid ${color}`, borderRadius: spec.style?.variant === "box" ? `${Number(theme.radius ?? 0)}px` : "999px" }) + if (spec.shape === "spotlight") Object.assign(node.style, { inset: "0" }) + if (spec.shape === "arrow") { const svg=document.createElementNS("http://www.w3.org/2000/svg","svg"); svg.setAttribute("width","88");svg.setAttribute("height","64");svg.setAttribute("viewBox","0 0 88 64");const path=document.createElementNS(svg.namespaceURI,"path");path.setAttribute("d","M4 56 Q30 52 72 12 M54 12 L72 12 L72 30");path.setAttribute("fill","none");path.setAttribute("stroke",color);path.setAttribute("stroke-width",String(theme.strokeWidth??3));path.setAttribute("stroke-linecap","round");path.setAttribute("stroke-linejoin","round");if(spec.animate==="draw"){path.setAttribute("stroke-dasharray","120");path.setAttribute("stroke-dashoffset","120");path.animate([{strokeDashoffset:"120"},{strokeDashoffset:"0"}],{duration:600,fill:"forwards"})}svg.append(path);node.append(svg);Object.assign(node.style,{transform:spec.direction?.startsWith("top")?"rotate(180deg)":spec.direction?.startsWith("left")?"rotate(90deg)":spec.direction?.startsWith("right")?"rotate(-90deg)":"none"}) } + if (spec.shape === "label" || spec.shape === "caption") { + node.textContent = spec.text || "" + Object.assign(node.style, { padding: "8px 12px", borderRadius: `${Number(theme.radius ?? 6)}px`, color: textColor, background, fontFamily: theme.fontFamily ?? "system-ui, sans-serif", fontSize: `${Number(theme.fontSize ?? 16)}px`, whiteSpace: "nowrap" }) + if (spec.shape === "caption") Object.assign(node.style, { left: "10%", right: "10%", top: spec.position === "top" ? "8%" : spec.position === "center" ? "45%" : "auto", bottom: spec.position === "bottom" || !spec.position ? "8%" : "auto", textAlign: "center" }) + } + node.dataset.annotationId = id + layer.append(node) + let frame = 0 + function update() { render(); frame = requestAnimationFrame(update) } + render() + if (target) frame = requestAnimationFrame(update) + const remove = () => { cancelAnimationFrame(frame); node.remove() } + annotations.set(id, { remove }) + if ((spec.durationMs ?? 0) > 0) setTimeout(() => { remove(); annotations.delete(id) }, spec.durationMs) + }, spec) +} diff --git a/packages/runtime-playground/src/browser-interactions.ts b/packages/runtime-playground/src/browser-interactions.ts index 6ac28cb1..ef073e08 100644 --- a/packages/runtime-playground/src/browser-interactions.ts +++ b/packages/runtime-playground/src/browser-interactions.ts @@ -5,6 +5,7 @@ import { browserActionLoadState, browserDeepEqual, browserStepTimeoutMs, duratio import type { BrowserEditorMutationSummary, BrowserProbeErrorRecord, BrowserStepAssertion, BrowserStepReadiness, BrowserStepRecord } from "./browser-artifacts.js" import { browserCommandLivenessPolicy, withBrowserCommandLiveness } from "./browser-liveness.js" import type { BrowserPresentation } from "./browser-presentation.js" +import { executeBrowserAnnotation } from "./browser-annotations.js" export interface BrowserStepOutcome { assertion?: BrowserStepAssertion @@ -214,6 +215,9 @@ export async function executeBrowserInteractionStep( } case "capture": return {} + case "annotate": + await executeBrowserAnnotation(page, step) + return {} case "callTool": throw new Error("wordpress.browser-actions callTool requires the host-tool execution bridge, which is not available in this browser step executor") } diff --git a/tests/browser-annotations.browser.test.ts b/tests/browser-annotations.browser.test.ts new file mode 100644 index 00000000..62e21972 --- /dev/null +++ b/tests/browser-annotations.browser.test.ts @@ -0,0 +1,43 @@ +import assert from "node:assert/strict" +import { mkdtemp, readFile, rm } from "node:fs/promises" +import { createServer } from "node:http" +import { tmpdir } from "node:os" +import { join } from "node:path" +import test from "node:test" +import { validateBrowserInteractionScript } from "../packages/runtime-core/src/browser-interaction.js" +import { runBrowserActionsCommand } from "../packages/runtime-playground/src/browser-actions-runner.js" +import { wordpressRuntimeSpec } from "../scripts/test-kit.js" + +test("annotation shapes and clear validate", () => { + for (const step of [ + { kind: "annotate", shape: "highlight", selector: "button" }, + { kind: "annotate", shape: "spotlight", selector: "button" }, + { kind: "annotate", shape: "arrow", selector: "button", direction: "bottom-left" }, + { kind: "annotate", shape: "label", text: "Continue", anchor: { selector: "button" } }, + { kind: "annotate", shape: "caption", text: "Next" }, + { kind: "annotate", clear: ["one"] }, + { kind: "annotate", clear: true }, + ]) assert.equal(validateBrowserInteractionScript([step]).valid, true) + assert.equal(validateBrowserInteractionScript([{ kind: "annotate", shape: "label", text: "x" }]).valid, false) + assert.equal(validateBrowserInteractionScript([{ kind: "annotate", shape: "arrow", selector: "button", direction: "nearby" }]).valid, false) +}) + +test("annotation renders in isolated overlay, follows scrolling, passes clicks, and clears by id", async () => { + const server = createServer((_request, response) => { response.setHeader("content-type", "text/html"); response.end('
') }) + await new Promise((resolve) => server.listen(0, "127.0.0.1", resolve)) + const address = server.address(); assert(address && typeof address === "object") + const url = `http://127.0.0.1:${address.port}` + const fixture = { serverUrl: url, playground: { async run() { return { text: "", exitCode: 0 } } }, async [Symbol.asyncDispose]() {} } + const artifactRoot = await mkdtemp(join(tmpdir(), "wp-codebox-annotations-")) + try { + await runBrowserActionsCommand({ artifactRoot, runtimeSpec: wordpressRuntimeSpec({ commands: ["wordpress.browser-actions"] }), server: fixture, spec: { command: "wordpress.browser-actions", args: [] }, plan: { + steps: [{ kind: "navigate", url }, { kind: "annotate", id: "focus", shape: "highlight", selector: "#target" }, { kind: "click", selector: "#target" }, { kind: "press", key: "PageDown" }, { kind: "waitFor", waitFor: "duration", duration: "100ms" }, { kind: "annotate", clear: ["focus"] }], + capture: new Set(["steps"]), stepTimeoutMs: 2000, totalTimeoutMs: 10000, networkSettleTimeoutMs: 100, maxDomSnapshotElements: 20, + } }) + const records = (await readFile(join(artifactRoot, "files/browser/steps.jsonl"), "utf8")).trim().split("\n").map((line) => JSON.parse(line)) + assert.equal(records[1].kind, "annotate") + assert.equal(records[2].status, "ok", "overlay must not intercept target click") + assert.equal(records[4].status, "ok", "target scroll should complete while annotation is active") + assert.equal(records[5].status, "ok", "annotation should clear by id") + } finally { await rm(artifactRoot, { recursive: true, force: true }); await new Promise((resolve, reject) => server.close((error) => error ? reject(error) : resolve())) } +})