setups.ts

/**
 * The tool setups we compare. Every setup gets the same model, the same
 * system prompt, the same task prompts and NO other way to reach the web:
 * WebFetch, WebSearch and every other built-in tool are switched off, and
 * permission mode `dontAsk` denies any tool call a setup does not allow.
 *
 * Each setup names the package it runs and the environment variable that
 * picks its version (default `latest`, see tools.ts):
 *
 *   WDIO_VERSION            @wdio/cli, for `wdio session`
 *   WDIO_MCP_VERSION        @wdio/mcp
 *   PLAYWRIGHT_MCP_VERSION  @playwright/mcp
 *   PLAYWRIGHT_CLI_VERSION  @playwright/cli
 *   STAGEHAND_REF           git ref of browserbase/stagehand; its Claude Code
 *                           MCP server is not on npm, so it is built from source
 *
 * For development against unreleased code:
 *   WDIO_LOCAL=/path/to/webdriverio   a built checkout of webdriverio/webdriverio
 *   WDIO_MCP_LOCAL=/path/to/mcp       a built checkout of webdriverio/mcp
 */
import { createHash } from 'node:crypto'
import fs from 'node:fs/promises'
import path from 'node:path'
import { execFile } from 'node:child_process'
import { promisify } from 'node:util'
import type { Options } from '@anthropic-ai/claude-agent-sdk'

import type { GitToolSpec, InstalledTool, ToolSpec } from './tools.ts'
import { isAgentBrowserCommand, isPlaywrightCliCommand, isWdioSessionCommand } from './permit.ts'

const run = promisify(execFile)

export interface Setup {
    id: string
    label: string
    /** the npm package (or repository) this setup runs */
    tool: ToolSpec | GitToolSpec
    /** a local build to use instead of the npm package, if configured */
    local?: () => InstalledTool | undefined
    /** one or two sentences appended to the shared system prompt */
    note: string
    /** instructions the tool's own integration gives its agent, appended after the note */
    instructions?: (tool: InstalledTool) => Promise<string>
    /** build the Agent SDK options for one run in `cwd` */
    options: (tool: InstalledTool, cwd: string, runId: string, run: RunOptions) => Promise<Partial<Options>>
    /**
     * Tool calls this setup may make beyond its auto-approved `allowedTools`.
     * Everything else is denied (see run.ts).
     */
    permit?: (toolName: string, input: Record<string, unknown>) => boolean
    /** what the agent is told when a shell command is denied */
    permitHint?: string
    /** clean up anything the run left behind (browsers, daemons) */
    cleanup?: (tool: InstalledTool, cwd: string, runId: string, env: NodeJS.ProcessEnv) => Promise<void>
    /**
     * the tool's own log files in the run's working directory, relative to
     * it, kept with the transcript: they explain what a transcript can't
     * (a session that went away, a slow page)
     */
    logs?: (cwd: string) => Promise<string[]>
}

export interface RunOptions {
    /** the harness screenshots the browser after every tool call (see screenshots.ts) */
    screenshots: boolean
}

// The prompts do not mention headless or headed: each tool picks its own
// default (Playwright MCP is started with --headless, Stagehand's facade only
// runs headed), and the workflow gives every job the same virtual display.

function mcpSetup (id: string, label: string, tool: ToolSpec, args: string[] = [], prepare?: (cwd: string, run: RunOptions) => Promise<string[]>): Setup {
    return {
        id,
        label,
        tool,
        note: `Use the ${label} tools to control the browser.`,
        options: async ({ binPath }, cwd, _runId, run) => ({
            tools: [],
            mcpServers: { browser: { type: 'stdio', command: process.execPath, args: [binPath, ...args, ...(await prepare?.(cwd, run) ?? [])] } },
            allowedTools: ['mcp__browser']
        })
    }
}

/**
 * Playwright starts Chrome over a pipe, without a debugging port, so the
 * harness could not take its screenshots. With screenshots on, its config
 * adds one; nothing else about the setup changes.
 */
async function playwrightDebugPort (cwd: string, run: RunOptions) {
    if (!run.screenshots) {
        return []
    }
    const config = path.join(cwd, 'playwright-mcp.json')
    await fs.writeFile(config, JSON.stringify({ browser: { launchOptions: { args: ['--remote-debugging-port=0'] } } }))
    return ['--config', config]
}

/**
 * Give the run directory a `wdio` binary (so both `wdio session` and
 * `npx wdio session` resolve locally, never from the registry) and install
 * the wdio-session skill where Claude Code looks for project skills.
 */
async function prepareWdioSession (bin: string, cwd: string) {
    const binDir = path.join(cwd, 'node_modules', '.bin')
    await fs.mkdir(binDir, { recursive: true })
    await fs.writeFile(path.join(cwd, 'package.json'), '{"private":true}')
    const shim = path.join(binDir, 'wdio')
    await fs.writeFile(shim, `#!/bin/sh\nexec "${process.execPath}" "${bin}" "$@"\n`)
    await fs.chmod(shim, 0o755)

    await run(process.execPath, [bin, 'session', 'skill', '--install', cwd], { cwd })
    await fs.mkdir(path.join(cwd, '.claude', 'skills'), { recursive: true })
    await fs.cp(path.join(cwd, '.agents', 'skills', 'wdio-session'), path.join(cwd, '.claude', 'skills', 'wdio-session'), { recursive: true })
    return binDir
}

const PLAYWRIGHT_MCP: ToolSpec = { pkg: '@playwright/mcp', env: 'PLAYWRIGHT_MCP_VERSION' }

/**
 * agent-browser keeps one daemon per session and finds it through a socket.
 * A directory per run keeps `close --all` from reaching other runs, and a
 * short path keeps the socket under the macOS limit of 104 characters.
 */
const agentBrowserSockets = (runId: string) => `/tmp/ab-${createHash('sha1').update(runId).digest('hex').slice(0, 10)}`

/**
 * Where `wdio session` keeps its sessions for one run. Without it, the
 * runtime dir follows XDG_RUNTIME_DIR, which every run on a Linux runner
 * shares, and the cleanup's `close --all` would close the sessions of runs
 * still going on in parallel.
 */
const wdioSessionDir = (runId: string) => `/tmp/wd-${createHash('sha1').update(runId).digest('hex').slice(0, 10)}`

/**
 * The run directory gets an `agent-browser` binary and the skill the way
 * `npx skills add vercel-labs/agent-browser` installs it: a stub that points
 * the agent to `agent-browser skills get core` for the version's own guide.
 */
async function prepareAgentBrowser (bin: string, cwd: string) {
    const binDir = path.join(cwd, 'node_modules', '.bin')
    await fs.mkdir(binDir, { recursive: true })
    const shim = path.join(binDir, 'agent-browser')
    await fs.writeFile(shim, `#!/bin/sh\nexec "${process.execPath}" "${bin}" "$@"\n`)
    await fs.chmod(shim, 0o755)
    const skill = path.join(path.dirname(bin), '..', 'skills', 'agent-browser')
    await fs.cp(skill, path.join(cwd, '.claude', 'skills', 'agent-browser'), { recursive: true })
    return binDir
}

/**
 * The run directory gets a `playwright-cli` binary and the skill its own
 * `playwright-cli install --skills` installs. Its browsers belong to the
 * workspace (the run directory), so `close-all` doesn't reach other runs.
 * With screenshots on, the workspace config adds a debugging port, as for
 * Playwright MCP; nothing else about the setup changes.
 */
async function preparePlaywrightCli (bin: string, cwd: string, screenshots: boolean) {
    const binDir = path.join(cwd, 'node_modules', '.bin')
    await fs.mkdir(binDir, { recursive: true })
    const shim = path.join(binDir, 'playwright-cli')
    await fs.writeFile(shim, `#!/bin/sh\nexec "${process.execPath}" "${bin}" "$@"\n`)
    await fs.chmod(shim, 0o755)
    await run(process.execPath, [bin, 'install', '--skills'], { cwd })
    if (screenshots) {
        await fs.mkdir(path.join(cwd, '.playwright'), { recursive: true })
        await fs.writeFile(path.join(cwd, '.playwright', 'cli.config.json'), JSON.stringify({ browser: { launchOptions: { args: ['--remote-debugging-port=0'] } } }))
    }
    return binDir
}

export const SETUPS: Setup[] = [
    mcpSetup('playwright-mcp', 'Playwright MCP', PLAYWRIGHT_MCP, ['--headless', '--isolated'], playwrightDebugPort),
    mcpSetup('playwright-mcp-tuned', 'Playwright MCP', PLAYWRIGHT_MCP, ['--headless', '--isolated', '--snapshot-mode', 'none', '--codegen', 'none'], playwrightDebugPort),
    {
        // Stagehand's Claude Code integration: the facade MCP server with
        // `run`, `snapshot` and `screenshot`, wired up the way
        // packages/integrations/claude-code/src/agent.ts does it
        id: 'stagehand',
        label: 'Stagehand',
        tool: {
            repo: 'browserbase/stagehand',
            env: 'STAGEHAND_REF',
            tagPrefix: '@browserbasehq/stagehand@',
            versionFile: 'packages/sdk-ts/package.json',
            build: [
                ['npx', '--yes', 'pnpm@11', '--dir', '{dir}', 'install', '--frozen-lockfile'],
                ['npx', '--yes', 'pnpm@11', '--dir', '{dir}', 'exec', 'turbo', 'run', 'build', '--filter', '@browserbasehq/stagehand-integrations']
            ],
            entry: 'packages/integrations/core/dist/facade/stdio-server.mjs'
        },
        note: 'Use the Stagehand tools to control the browser.',
        // their agent runs with FACADE_AGENT_INSTRUCTIONS as its system prompt
        instructions: async ({ root }) => {
            const facade = await import(path.join(root!, 'packages', 'integrations', 'core', 'dist', 'facade', 'index.mjs'))
            return facade.FACADE_AGENT_INSTRUCTIONS as string
        },
        options: async ({ binPath }) => {
            // like their buildAllowlistedEnv(): only STAGEHAND_* and BROWSERBASE_*
            // reach the server, plus what a local Chrome needs. No
            // STAGEHAND_MODEL_NAME and no provider keys, so the facade runs
            // no model of its own and every token goes through the agent.
            const env: Record<string, string> = { STAGEHAND_BROWSER: 'local' }
            for (const [key, value] of Object.entries(process.env)) {
                if (value && (/^(STAGEHAND_|BROWSERBASE_)/.test(key) || ['PATH', 'HOME', 'DISPLAY', 'TMPDIR'].includes(key))) {
                    env[key] = value
                }
            }
            delete env.STAGEHAND_MODEL_NAME
            delete env.STAGEHAND_MODEL_API_KEY
            return {
                tools: [],
                mcpServers: { browser: { type: 'stdio', command: process.execPath, args: [binPath], env } },
                allowedTools: ['mcp__browser']
            }
        }
    },
    {
        // dev build of webdriverio/mcp#155 (page model from @wdio/session) until it is released
        ...mcpSetup('wdio-mcp', 'WebdriverIO MCP', { pkg: '@wdio/mcp', env: 'WDIO_MCP_VERSION', bin: 'wdio-mcp', defaultVersion: '4.0.0-dev.64' }),
        local: () => process.env.WDIO_MCP_LOCAL
            ? { pkg: '@wdio/mcp', version: 'local build', binPath: path.join(process.env.WDIO_MCP_LOCAL, 'lib', 'server.js') }
            : undefined
    },
    {
        id: 'wdio-session',
        label: 'WebdriverIO session',
        // `wdio session` ships with v10, which is in alpha until its release
        tool: { pkg: '@wdio/cli', env: 'WDIO_VERSION', bin: 'wdio', defaultVersion: '10.0.0-alpha.175' },
        local: () => process.env.WDIO_LOCAL
            ? { pkg: '@wdio/cli', version: 'local build', binPath: path.join(process.env.WDIO_LOCAL, 'packages', 'wdio-cli', 'bin', 'wdio.js') }
            : undefined,
        note: 'Use the wdio-session skill: drive the browser with `npx wdio session …` shell commands.',
        options: async ({ binPath }, cwd, runId) => {
            const binDir = await prepareWdioSession(binPath, cwd)
            return {
                tools: ['Bash', 'Skill', 'Read'],
                allowedTools: ['Skill', 'Read'],
                settingSources: ['project'],
                // only its own skill: Claude Code's bundled skills (code-review,
                // deep-research, …) would otherwise be listed on every turn
                skills: ['wdio-session'],
                env: { ...process.env, PATH: `${binDir}:${process.env.PATH}`, WDIO_SESSION: runId, WDIO_SESSION_DIR: wdioSessionDir(runId) }
            }
        },
        permit: (toolName, input) => toolName === 'Bash' && typeof input.command === 'string' && isWdioSessionCommand(input.command),
        permitHint: 'Only `wdio session …` commands are available in this benchmark setup.',
        // the agent may have named its own sessions; the run's session dir
        // holds only the ones it opened
        cleanup: async ({ binPath }, cwd, runId, env) => {
            await run(process.execPath, [binPath, 'session', 'close', '--all'], { cwd, env: { ...env, WDIO_SESSION_DIR: wdioSessionDir(runId) } }).catch(() => {})
            await fs.rm(wdioSessionDir(runId), { recursive: true, force: true })
        },
        // one artifacts directory per session the agent opened
        logs: async (cwd) => {
            const sessions = await fs.readdir(path.join(cwd, '.wdio', 'session'), { withFileTypes: true }).catch(() => [])
            return sessions.filter((entry) => entry.isDirectory()).map((entry) => path.join('.wdio', 'session', entry.name, 'daemon.log'))
        }
    },
    {
        id: 'agent-browser',
        label: 'agent-browser',
        tool: { pkg: 'agent-browser', env: 'AGENT_BROWSER_VERSION', bin: 'agent-browser' },
        note: 'Use the agent-browser skill: drive the browser with `agent-browser …` shell commands.',
        options: async ({ binPath }, cwd, runId) => {
            const binDir = await prepareAgentBrowser(binPath, cwd)
            const sockets = agentBrowserSockets(runId)
            await fs.mkdir(sockets, { recursive: true })
            return {
                tools: ['Bash', 'Skill', 'Read'],
                allowedTools: ['Skill', 'Read'],
                settingSources: ['project'],
                skills: ['agent-browser'],
                env: { ...process.env, PATH: `${binDir}:${process.env.PATH}`, AGENT_BROWSER_SESSION: runId, AGENT_BROWSER_SOCKET_DIR: sockets }
            }
        },
        permit: (toolName, input) => toolName === 'Bash' && typeof input.command === 'string' && isAgentBrowserCommand(input.command),
        permitHint: 'Only `agent-browser …` commands are available in this benchmark setup.',
        cleanup: async ({ binPath }, cwd, runId, env) => {
            const sockets = agentBrowserSockets(runId)
            await run(process.execPath, [binPath, 'close', '--all'], { cwd, env: { ...env, AGENT_BROWSER_SOCKET_DIR: sockets } }).catch(() => {})
            await fs.rm(sockets, { recursive: true, force: true })
        }
    },
    {
        id: 'playwright-cli',
        label: 'Playwright CLI',
        tool: { pkg: '@playwright/cli', env: 'PLAYWRIGHT_CLI_VERSION', bin: 'playwright-cli' },
        note: 'Use the playwright-cli skill: drive the browser with `playwright-cli …` shell commands.',
        options: async ({ binPath }, cwd, _runId, run) => {
            const binDir = await preparePlaywrightCli(binPath, cwd, run.screenshots)
            return {
                // Read: the CLI saves snapshots as files the agent reads
                tools: ['Bash', 'Skill', 'Read'],
                allowedTools: ['Skill', 'Read'],
                settingSources: ['project'],
                skills: ['playwright-cli'],
                env: { ...process.env, PATH: `${binDir}:${process.env.PATH}` }
            }
        },
        permit: (toolName, input) => toolName === 'Bash' && typeof input.command === 'string' && isPlaywrightCliCommand(input.command),
        permitHint: 'Only `playwright-cli …` commands are available in this benchmark setup.',
        cleanup: async ({ binPath }, cwd, _runId, env) => {
            await run(process.execPath, [binPath, 'close-all'], { cwd, env }).catch(() => {})
        }
    }
]