feat(browser-actions): annotate step for instructional recordings

README.md

@@ -768,7 +768,21 @@ Action journeys and adaptive exploration can declare their browser context with
 
 For `wordpress.browser-actions capture=video`, `video-size=<width>x<height>` sets the WebM recording dimensions without changing its format or encoding. When omitted, recordings retain Playwright's default size unless `device-scale-factor` is greater than 1, in which case the viewport dimensions are multiplied by that scale. Requested dimensions are capped at 8,294,400 pixels total with aspect ratio preserved. Video step records include `videoOffsetMs: { startMs, endMs }` relative to recording start; named `screenshot` steps and steps with a non-empty free-form `marker` label appear in the `video.markers` timeline in `action-summary.json`.
 
-Step kinds: `navigate` (`url`, optional `waitFor=domcontentloaded|load|networkidle`), `click`/`hover` (`selector` or `text`), `fill`/`type` (`selector`, `value`), `press` (`key`, optional `selector`), `drag` (`from` selector, `to` as `{ "selector": ... }` or `{ "x": n, "y": n }`), `select` (`selector`, `value` or `values`), `waitFor` (`selector` or `waitFor=domcontentloaded|load|networkidle|duration|selector:<sel>`), `evaluate` (`expression`, optional `assert` to deep-equal the result), `expect` (`selector`, optional `state=visible|hidden|attached|detached|enabled|disabled|checked|unchecked|editable`), and `screenshot` (optional `name` and boolean `fullPage`; defaults to `true`, except with `capture=video` when it defaults to `false` to keep viewport-sized video frames. Set `fullPage: true` explicitly to request a full-document screenshot while recording). Every step may set its own `timeout=<n>s`; the command also accepts a global `step-timeout=<n>s` (per step) and `timeout=<n>s` (total-script budget). Both are bounded and deterministic — the run stops cleanly on the first failing step, with no silent partial success.
+Step kinds: `navigate` (`url`, optional `waitFor=domcontentloaded|load|networkidle`), `click`/`hover` (`selector` or `text`), `fill`/`type` (`selector`, `value`), `press` (`key`, optional `selector`), `drag` (`from` selector, `to` as `{ "selector": ... }` or `{ "x": n, "y": n }`), `select` (`selector`, `value` or `values`), `scroll` (`selector`, `position=top|bottom`, or `by: {x,y}`), `waitFor` (`selector` or `waitFor=domcontentloaded|load|networkidle|duration|selector:<sel>`), `evaluate` (`expression`, optional `assert` to deep-equal the result), `expect` (`selector`, optional `state=visible|hidden|attached|detached|enabled|disabled|checked|unchecked|editable`), `annotate` (highlight, spotlight, arrow, anchored label, fixed caption, or clear), and `screenshot` (optional `name` and boolean `fullPage`; defaults to `true`, except with `capture=video` when it defaults to `false` to keep viewport-sized video frames. Set `fullPage: true` explicitly to request a full-document screenshot while recording). Every step may set its own `timeout=<n>s`; the command also accepts a global `step-timeout=<n>s` (per step) and `timeout=<n>s` (total-script budget). Both are bounded and deterministic — the run stops cleanly on the first failing step, with no silent partial success.
+
+Annotations render in a pointer-transparent, isolated shadow overlay and do not require the `evaluate` policy grant. They follow element targets through scrolling. Use `id` to replace or later clear an annotation, `durationMs` for automatic removal, `animate` (`fade`, `draw`, or `none`), and run-level `annotation-theme-json=<object|@file>` (`accentColor`, `textColor`, `background`, `fontFamily`, `fontSize`, `strokeWidth`, `radius`) to customize appearance. Arrow `direction` identifies the side/corner from which the SVG arrow points toward its target. A full journey can sequence generic callouts and clear them when finished:
+
+```jsonc
+[
+  { "kind": "navigate", "url": "/" },
+  { "kind": "annotate", "id": "focus", "shape": "highlight", "selector": "button.save", "style": { "variant": "ring", "padding": 8 } },
+  { "kind": "annotate", "shape": "label", "text": "Select this control to continue", "anchor": { "selector": "button.save", "placement": "top" } },
+  { "kind": "annotate", "shape": "arrow", "selector": "button.save", "direction": "bottom-left", "animate": "draw" },
+  { "kind": "click", "selector": "button.save" },
+  { "kind": "annotate", "shape": "caption", "text": "Changes are saved", "position": "bottom", "durationMs": 1800 },
+  { "kind": "annotate", "clear": ["focus"] }
+]
+```
 
 Opt in to human-paced presentation with `presentation-json=<object>` (or `@<path>`). It supports `pointer: { enabled, style: "arrow"|"dot"|"touch", size, color }`, `clickFeedback: "ripple"|"pulse"`, `motion: { moveDurationMs, easing }`, and `typing: { delayMs, applyToFill }`. The pointer lives in an isolated, non-hit-testable shadow overlay and is reinstalled at document startup on navigation. Presentation actions bring targets into view and move to their center; typing is character-paced, while fills remain instant unless `applyToFill` is true. Add a `scroll` step with `selector`, `position: "top"|"bottom"`, or `by: {x,y}`, plus optional `behavior`, `durationMs`, and `block`; scrolling waits for the viewport to settle.
 

package.json

@@ -187,7 +187,8 @@
     "check": "npm run smoke -- --group=check",
     "check:all": "npm run test:all && npm run cloudflare:check",
     "test:browser-video-capture": "tsx tests/browser-actions-video-capture.browser.test.ts",
-    "test:browser-presentation": "tsx tests/browser-actions-presentation.browser.test.ts"
+    "test:browser-presentation": "tsx tests/browser-actions-presentation.browser.test.ts",
+    "test:browser-annotations": "tsx --test tests/browser-annotations.browser.test.ts"
   },
   "workspaces": [
     "packages/cli",

packages/runtime-core/src/browser-interaction.ts

@@ -31,6 +31,7 @@ export const BROWSER_INTERACTION_STEP_KINDS = [
   "screenshot",
   "capture",
   "callTool",
+  "annotate",
 ] as const
 
 export type BrowserInteractionStepKind = typeof BROWSER_INTERACTION_STEP_KINDS[number]
@@ -99,11 +100,18 @@ export interface BrowserInteractionStep {
   tool?: string
   /** JSON input passed to the caller-provided host tool. */
   input?: JsonValue
-  position?: "top" | "bottom"
+  position?: "top" | "center" | "bottom"
   by?: { x: number; y: number }
   behavior?: "smooth" | "instant"
-  durationMs?: number
   block?: "start" | "center" | "end"
+  id?: string
+  shape?: "highlight" | "spotlight" | "arrow" | "label" | "caption"
+  clear?: string[] | true
+  style?: { variant?: "ring" | "box" | "underline"; padding?: number }
+  anchor?: { selector: string; placement?: "top" | "bottom" | "left" | "right" }
+  durationMs?: number
+  animate?: "draw" | "fade" | "none"
+  direction?: "top" | "bottom" | "left" | "right" | "top-left" | "top-right" | "bottom-left" | "bottom-right"
 }
 
 export interface BrowserToolVerifierInputSummary {
@@ -368,6 +376,21 @@ export function validateBrowserInteractionScript(input: unknown): BrowserInterac
           issues.push({ index, message: "callTool step input must be JSON-serializable" })
         }
         break
+      case "annotate":
+        if (step.clear !== undefined) {
+          if (step.clear !== true && (!Array.isArray(step.clear) || step.clear.some((id) => typeof id !== "string" || !id))) issues.push({ index, message: "annotate clear must be true or an array of ids" })
+          if (step.shape !== undefined) issues.push({ index, message: "annotate clear cannot include shape" })
+          break
+        }
+        if (!( ["highlight", "spotlight", "arrow", "label", "caption"] as const).includes(step.shape as never)) issues.push({ index, message: "annotate requires shape: highlight, spotlight, arrow, label, or caption" })
+        if (step.durationMs !== undefined && (!Number.isFinite(step.durationMs) || step.durationMs < 0)) issues.push({ index, message: "annotate durationMs must be a non-negative number" })
+        if (step.animate !== undefined && !["draw", "fade", "none"].includes(step.animate)) issues.push({ index, message: "annotate animate must be draw, fade, or none" })
+        if (["highlight", "spotlight", "arrow"].includes(String(step.shape)) && !hasSelector) issues.push({ index, message: `annotate ${step.shape} requires selector` })
+        if (["label", "caption"].includes(String(step.shape)) && !hasText) issues.push({ index, message: `annotate ${step.shape} requires text` })
+        if (step.shape === "label" && (!step.anchor || typeof step.anchor.selector !== "string" || !step.anchor.selector)) issues.push({ index, message: "annotate label requires anchor.selector" })
+        if (step.shape === "caption" && step.position !== undefined && !["top", "center", "bottom"].includes(step.position)) issues.push({ index, message: "annotate caption position must be top, center, or bottom" })
+        if (step.shape === "arrow" && step.direction !== undefined && !["top", "bottom", "left", "right", "top-left", "top-right", "bottom-left", "bottom-right"].includes(step.direction)) issues.push({ index, message: "annotate arrow direction is invalid" })
+        break
     }
 
     steps.push(step)

packages/runtime-playground/src/browser-actions-runner.ts

@@ -29,6 +29,7 @@ import { createBrowserAccessibilityCollector } from "./browser-accessibility-col
 import { browserEnvironmentCell, createPlaywrightBrowserEnvironmentContext, observePlaywrightBrowserEnvironment, resolvePlaywrightBrowserEnvironment, type PlaywrightBrowserEnvironmentSession } from "./browser-environment-matrix.js"
 import { installBrowserTransportFaults, type BrowserTransportFaultReport, type InstalledBrowserTransportFaults } from "./browser-transport-faults.js"
 import { browserPresentationInitScript, validateBrowserPresentation, type BrowserPresentation } from "./browser-presentation.js"
+import { setBrowserAnnotationTheme, type BrowserAnnotationTheme } from "./browser-annotations.js"
 
 export { discoverBrowserActionCorpusDescriptors } from "./browser-action-discovery.js"
 
@@ -57,6 +58,7 @@ export interface BrowserActionsRunPlan {
   transportFaults?: TransportFaultModel
   reusePage?: boolean
   presentation?: BrowserPresentation
+  annotationTheme?: BrowserAnnotationTheme
 }
 
 interface BrowserRunPlan {
@@ -213,6 +215,7 @@ export async function runBrowserActionsCommand({
       videoRecordingOrigin = performance.now()
       videoStartedWallAt = Date.now()
     }
+    setBrowserAnnotationTheme(runPlan.annotationTheme ?? {})
     environmentRuntime = session?.runtime ?? (needsEnvironmentContext ? await createPlaywrightBrowserEnvironmentContext(browser, resolvedEnvironment, {
       contextOptions: {
         ...topology.contextOptions(),
@@ -865,10 +868,19 @@ async function browserActionsRunPlanFromArgs(args: string[], artifactRoot: strin
     actionCorpus: browserActionCorpusFromArgs(args),
     adaptiveExploration: adaptiveExplorationPlan.contract,
     presentation: await browserPresentationFromArgs(args),
+    annotationTheme: await browserAnnotationThemeFromArgs(args),
     transportFaults: await transportFaultModelFromArg(args),
   }
 }
 
+async function browserAnnotationThemeFromArgs(args: string[]): Promise<BrowserAnnotationTheme | undefined> {
+  const raw = argValue(args, "annotation-theme-json")
+  if (!raw) return undefined
+  const value: unknown = JSON.parse(raw.startsWith("@") ? await readFile(resolveCommandPath(raw.slice(1)), "utf8") : raw)
+  if (!value || typeof value !== "object" || Array.isArray(value)) throw new Error("annotation-theme-json must be an object")
+  return value as BrowserAnnotationTheme
+}
+
 const BROWSER_VIDEO_MAX_PIXELS = 8_294_400
 
 function browserVideoSizeArg(args: string[]): { width: number; height: number } | undefined {

packages/runtime-playground/src/browser-annotations.ts

@@ -0,0 +1,108 @@
+import type { Page } from "playwright"
+import type { BrowserInteractionStep } from "@automattic/wp-codebox-core"
+
+type AnnotationSpec = {
+  id?: string
+  shape?: string
+  selector?: string
+  text?: string
+  clear?: string[] | true
+  durationMs?: number
+  animate?: string
+  direction?: string
+  position?: string
+  style?: { variant?: string; padding?: number }
+  anchor?: { selector?: string; placement?: string }
+  theme?: BrowserAnnotationTheme
+}
+
+export type BrowserAnnotationTheme = { accentColor?: string; textColor?: string; background?: string; fontFamily?: string; fontSize?: number; strokeWidth?: number; radius?: number }
+let annotationTheme: BrowserAnnotationTheme = {}
+export function setBrowserAnnotationTheme(theme: BrowserAnnotationTheme): void { annotationTheme = theme }
+
+export async function executeBrowserAnnotation(page: Page, step: BrowserInteractionStep): Promise<void> {
+  if (step.clear !== undefined) {
+    await page.evaluate((ids) => {
+      const host = document.querySelector<HTMLElement>("#__wp_codebox_annotations") as (HTMLElement & { __annotations?: Map<string, { remove: () => void }> }) | null
+      const annotations = host?.__annotations
+      if (!annotations) return
+      const selected = ids === true ? [...annotations.keys()] : ids
+      for (const id of selected) { annotations.get(id)?.remove(); annotations.delete(id) }
+      if (annotations.size === 0) host?.remove()
+    }, step.clear)
+    return
+  }
+  if (step.selector) await page.locator(step.selector).waitFor({ state: "visible" })
+  if (step.anchor?.selector) await page.locator(step.anchor.selector).waitFor({ state: "visible" })
+  const spec: AnnotationSpec = {
+    id: step.id, shape: step.shape, selector: step.selector, text: step.text, durationMs: step.durationMs,
+    animate: step.animate, direction: step.direction, position: step.position,
+    style: step.style, anchor: step.anchor, theme: annotationTheme,
+  }
+  await page.evaluate("globalThis.__name ??= (value) => value")
+  await page.evaluate<void, AnnotationSpec>((spec) => {
+    type Annotation = { remove: () => void }
+    type AnnotationHost = HTMLElement & { __annotations?: Map<string, Annotation> }
+    let host = document.querySelector<HTMLElement>("#__wp_codebox_annotations") as AnnotationHost | null
+    if (!host) {
+      host = document.createElement("div") as AnnotationHost
+      host.id = "__wp_codebox_annotations"
+      host.setAttribute("aria-hidden", "true")
+      Object.assign(host.style, { position: "fixed", inset: "0", zIndex: "2147483647", pointerEvents: "none" })
+      document.documentElement.append(host)
+      const root = host.attachShadow({ mode: "open" })
+      const layer = document.createElement("div")
+      root.append(layer)
+      ;(host as AnnotationHost & { __layer?: HTMLElement }).__layer = layer
+      host.__annotations = new Map()
+    }
+    const layer = (host as AnnotationHost & { __layer?: HTMLElement }).__layer!
+    Object.assign(layer.style, { position: "fixed", inset: "0", pointerEvents: "none", fontFamily: "system-ui, sans-serif", color: "#fff" })
+    ;(host as AnnotationHost & { __layer?: HTMLElement }).__layer = layer
+    const annotations = host.__annotations ??= new Map()
+    const id = spec.id || `annotation-${Date.now()}-${Math.random()}`
+    annotations.get(id)?.remove()
+    const node = document.createElement("div")
+    const target = spec.shape === "label" ? document.querySelector(spec.anchor?.selector || "") : spec.selector ? document.querySelector(spec.selector) : null
+    if (["highlight", "spotlight", "arrow", "label"].includes(String(spec.shape)) && !target) throw new Error("Annotation target was not found")
+    const theme = spec.theme ?? {}
+    const color = String(theme.accentColor || "#64748b")
+    const textColor = String(theme.textColor || "#fff")
+    const background = String(theme.background || "rgba(15, 23, 42, .94)")
+    function rect() { return target?.getBoundingClientRect() }
+    function setRect(value: DOMRect) {
+      const padding = Number(spec.style?.padding ?? 0)
+      Object.assign(node.style, { left: `${value.left - padding}px`, top: `${value.top - padding}px`, width: `${value.width + padding * 2}px`, height: `${value.height + padding * 2}px` })
+    }
+    function render() {
+      const value = rect()
+      if (!value) return
+      if (spec.shape === "highlight") setRect(value)
+      if (spec.shape === "label") Object.assign(node.style, { left: `${value.left}px`, top: `${spec.anchor?.placement === "bottom" ? value.bottom + 8 : value.top - 44}px` })
+      if (spec.shape === "spotlight") { const p = Number(spec.style?.padding ?? 8); const l=value.left-p,t=value.top-p,w=value.width+p*2,h=value.height+p*2; Object.assign(node.style,{clipPath:`polygon(0 0,100% 0,100% 100%,0 100%,0 0,${l}px ${t}px,${l}px ${t+h}px,${l+w}px ${t+h}px,${l+w}px ${t}px,${l}px ${t}px)`,background:"rgba(0,0,0,.62)"}) }
+      if (spec.shape === "arrow") {
+        const direction = spec.direction || "bottom-left"
+        const [vertical, horizontal] = direction.split("-").length === 2 ? direction.split("-") : [direction, "center"]
+        Object.assign(node.style, { left: `${horizontal === "left" ? value.left - 50 : horizontal === "right" ? value.right + 10 : value.left + value.width / 2}px`, top: `${vertical === "top" ? value.top - 50 : vertical === "bottom" ? value.bottom + 10 : value.top + value.height / 2}px` })
+      }
+    }
+    Object.assign(node.style, { position: "fixed", boxSizing: "border-box", pointerEvents: "none", transition: spec.animate === "fade" || spec.animate === undefined ? "opacity .25s ease" : "none", opacity: "1" })
+    if (spec.shape === "highlight") Object.assign(node.style, { border: `${Number(theme.strokeWidth ?? 3)}px solid ${color}`, borderRadius: spec.style?.variant === "box" ? `${Number(theme.radius ?? 0)}px` : "999px" })
+    if (spec.shape === "spotlight") Object.assign(node.style, { inset: "0" })
+    if (spec.shape === "arrow") { const svg=document.createElementNS("http://www.w3.org/2000/svg","svg"); svg.setAttribute("width","88");svg.setAttribute("height","64");svg.setAttribute("viewBox","0 0 88 64");const path=document.createElementNS(svg.namespaceURI,"path");path.setAttribute("d","M4 56 Q30 52 72 12 M54 12 L72 12 L72 30");path.setAttribute("fill","none");path.setAttribute("stroke",color);path.setAttribute("stroke-width",String(theme.strokeWidth??3));path.setAttribute("stroke-linecap","round");path.setAttribute("stroke-linejoin","round");if(spec.animate==="draw"){path.setAttribute("stroke-dasharray","120");path.setAttribute("stroke-dashoffset","120");path.animate([{strokeDashoffset:"120"},{strokeDashoffset:"0"}],{duration:600,fill:"forwards"})}svg.append(path);node.append(svg);Object.assign(node.style,{transform:spec.direction?.startsWith("top")?"rotate(180deg)":spec.direction?.startsWith("left")?"rotate(90deg)":spec.direction?.startsWith("right")?"rotate(-90deg)":"none"}) }
+    if (spec.shape === "label" || spec.shape === "caption") {
+      node.textContent = spec.text || ""
+      Object.assign(node.style, { padding: "8px 12px", borderRadius: `${Number(theme.radius ?? 6)}px`, color: textColor, background, fontFamily: theme.fontFamily ?? "system-ui, sans-serif", fontSize: `${Number(theme.fontSize ?? 16)}px`, whiteSpace: "nowrap" })
+      if (spec.shape === "caption") Object.assign(node.style, { left: "10%", right: "10%", top: spec.position === "top" ? "8%" : spec.position === "center" ? "45%" : "auto", bottom: spec.position === "bottom" || !spec.position ? "8%" : "auto", textAlign: "center" })
+    }
+    node.dataset.annotationId = id
+    layer.append(node)
+    let frame = 0
+    function update() { render(); frame = requestAnimationFrame(update) }
+    render()
+    if (target) frame = requestAnimationFrame(update)
+    const remove = () => { cancelAnimationFrame(frame); node.remove() }
+    annotations.set(id, { remove })
+    if ((spec.durationMs ?? 0) > 0) setTimeout(() => { remove(); annotations.delete(id) }, spec.durationMs)
+  }, spec)
+}