diff --git a/.gitignore b/.gitignore index de135aab5..c6821a9ee 100644 --- a/.gitignore +++ b/.gitignore @@ -53,3 +53,5 @@ workspace.zip /.trellis /AGENTS.md /CLAUDE.md +/issue_board.md +/.github/automation/ diff --git a/Cargo.lock b/Cargo.lock index 1800f9794..a58c2c3fe 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -3406,6 +3406,7 @@ dependencies = [ "objc2 0.6.4", "objc2-app-kit 0.3.2", "objc2-av-foundation", + "objc2-foundation 0.3.2", "percent-encoding", "portable-pty", "prost", diff --git a/crates/agent-gateway/web/src/pages/settings/types.ts b/crates/agent-gateway/web/src/pages/settings/types.ts index d04298452..c8a27424b 100644 --- a/crates/agent-gateway/web/src/pages/settings/types.ts +++ b/crates/agent-gateway/web/src/pages/settings/types.ts @@ -15,7 +15,8 @@ export type SectionId = | "hooks" | "cron" | "devices" - | "remote"; + | "remote" + | "cua"; export type SettingsPageProps = { settings: AppSettings; diff --git a/crates/agent-gui/src-tauri/Cargo.toml b/crates/agent-gui/src-tauri/Cargo.toml index cb67349d8..9a4910854 100644 --- a/crates/agent-gui/src-tauri/Cargo.toml +++ b/crates/agent-gui/src-tauri/Cargo.toml @@ -78,8 +78,9 @@ encoding_rs = "0.8.35" chardetng = "0.1.17" [target.'cfg(target_os = "macos")'.dependencies] -objc2-app-kit = { version = "0.3", default-features = false, features = ["std", "NSButton", "NSControl", "NSView", "NSWindow"] } +objc2-app-kit = { version = "0.3", default-features = false, features = ["std", "NSApplication", "NSButton", "NSControl", "NSView", "NSWindow"] } objc2-av-foundation = { version = "0.3", default-features = false, features = ["std", "block2", "AVCaptureDevice", "AVMediaFormat"] } +objc2-foundation = { version = "0.3", default-features = false, features = ["std", "NSArray", "NSString"] } objc2 = { version = "0.6", default-features = false, features = ["std"] } block2 = "0.6" diff --git a/crates/agent-gui/src-tauri/src/commands/app/app.rs b/crates/agent-gui/src-tauri/src/commands/app/app.rs index c0185c44e..7e7234e0e 100644 --- a/crates/agent-gui/src-tauri/src/commands/app/app.rs +++ b/crates/agent-gui/src-tauri/src/commands/app/app.rs @@ -39,6 +39,7 @@ pub fn app_frontend_ready( ) -> Result<(), String> { ready_state.0.store(true, Ordering::SeqCst); if window.is_visible().unwrap_or(false) { + crate::force_activate_main_window(&window); return Ok(()); } window @@ -46,7 +47,14 @@ pub fn app_frontend_ready( .map_err(|error| format!("failed to show frontend-ready window: {error}"))?; window .set_focus() - .map_err(|error| format!("failed to focus frontend-ready window: {error}")) + .map_err(|error| format!("failed to focus frontend-ready window: {error}"))?; + // CUA-007: macOS-only NSApp.activate + makeKeyAndOrderFront ensures the + // window reaches the frontmost-ordinary window state so cua-driver's + // `bring_to_front` and foreground delivery actually land on the + // WebView. `set_focus` alone can leave the window below the + // loginwindow or another system surface in a dev sandbox. + crate::force_activate_main_window(&window); + Ok(()) } /// 前端主动切换置顶（置顶指示器点击取消）；状态变更仍经 diff --git a/crates/agent-gui/src-tauri/src/commands/app/ax.rs b/crates/agent-gui/src-tauri/src/commands/app/ax.rs new file mode 100644 index 000000000..22823f47d --- /dev/null +++ b/crates/agent-gui/src-tauri/src/commands/app/ax.rs @@ -0,0 +1,30 @@ +//! 无障碍（AX）辅助命令。 +//! +//! 这两个命令服务于「外部自动化工具驱动 LiveAgent 自身窗口」的场景 +//! ——典型是用 `cua-driver` 跑端到端测试时，需要主窗口真的在前台、且 +//! NSWindow / WKWebView 的 AX 注解已重新广播，否则拿到的 AX 树是空的。 +//! +//! 它们与「LiveAgent 能操作用户电脑」这个功能无关：那条链路已改为把 +//! `cua-driver mcp` 当作一个普通 MCP server 接入 MCP Hub，不再有专属 +//! 的 Tauri 命令层。 +//! +//! 真正的实现在 `crate::cua_window_ready` / `crate::cua_refresh_a11y` +//! （`lib.rs`，因为要触碰 `force_activate_main_window` 与平台相关的 +//! NSWindow 细节）；这里只是命令桥。 + +/// 把主窗口重新推到最前并等待 `is_focused()` 落住，让后续的 AX 查询 +/// 能找到 WebView。 +#[tauri::command(rename_all = "camelCase")] +pub async fn cua_window_ready( + window: tauri::WebviewWindow, +) -> Result { + Ok(crate::cua_window_ready(window).await) +} + +/// 在路由切换、overlay 打开 / 关闭、外部工具主动唤起等场景重新触发 +/// NSWindow / WKWebView 的 AX 注解，并回弹广播一次 +/// `UIElementCreatedNotification`。 +#[tauri::command(rename_all = "camelCase")] +pub fn cua_refresh_a11y(window: tauri::WebviewWindow) -> crate::CuaRefreshA11yResponse { + crate::cua_refresh_a11y(&window) +} diff --git a/crates/agent-gui/src-tauri/src/commands/app/mod.rs b/crates/agent-gui/src-tauri/src/commands/app/mod.rs index 825165356..c4cb9322b 100644 --- a/crates/agent-gui/src-tauri/src/commands/app/mod.rs +++ b/crates/agent-gui/src-tauri/src/commands/app/mod.rs @@ -1,4 +1,5 @@ pub mod app; +pub mod ax; pub mod system; pub mod tray; pub mod update; diff --git a/crates/agent-gui/src-tauri/src/commands/config/settings/mod.rs b/crates/agent-gui/src-tauri/src/commands/config/settings/mod.rs index 405617681..5ffb91bf7 100644 --- a/crates/agent-gui/src-tauri/src/commands/config/settings/mod.rs +++ b/crates/agent-gui/src-tauri/src/commands/config/settings/mod.rs @@ -48,6 +48,11 @@ const SYSTEM_MISSING_WORKSPACE_PROJECT_PATHS_KEY: &str = "missingWorkspaceProjec const SYSTEM_ARCHIVED_WORKSPACE_PROJECT_PATHS_KEY: &str = "archivedWorkspaceProjectPaths"; const SYSTEM_WORKSPACE_RESOURCE_SETTINGS_KEY: &str = "workspaceResourceSettings"; const SYSTEM_SYSTEM_PROXY_KEY: &str = "systemProxy"; +// CUA 自指开关。默认 false —— cua-driver 的工具默认看不到、也点不到 +// LiveAgent 自己的窗口：让模型操作宿主界面等于让它能点掉自己的审批弹窗、 +// 改自己的设置、关掉自己。置 true 才解除（用 LiveAgent 自动化测试 +// LiveAgent 这类场景需要）。 +const SYSTEM_CUA_ALLOW_SELF_TARGETING_KEY: &str = "cuaAllowSelfTargeting"; const DEFAULT_WORKSPACE_PROJECT_ID: &str = "default-project"; const DEFAULT_WORKSPACE_PROJECT_NAME: &str = "Default Project"; pub(crate) const PROVIDER_API_KEY_UPDATES_FIELD: &str = "providerApiKeyUpdates"; diff --git a/crates/agent-gui/src-tauri/src/commands/config/settings/system.rs b/crates/agent-gui/src-tauri/src/commands/config/settings/system.rs index f61d59558..b82a95ecd 100644 --- a/crates/agent-gui/src-tauri/src/commands/config/settings/system.rs +++ b/crates/agent-gui/src-tauri/src/commands/config/settings/system.rs @@ -463,6 +463,17 @@ fn system_value_with_defaults(raw: Option, default_workdir: &str) -> Valu SYSTEM_COMMAND_SAFETY_MODE_KEY.to_string(), normalize_command_safety_mode_value(system.get(SYSTEM_COMMAND_SAFETY_MODE_KEY)), ); + // 缺省 false：安全侧的开关，任何非 true 的值（缺失、null、字符串） + // 都收敛成「不允许自指」。 + system.insert( + SYSTEM_CUA_ALLOW_SELF_TARGETING_KEY.to_string(), + Value::Bool( + system + .get(SYSTEM_CUA_ALLOW_SELF_TARGETING_KEY) + .and_then(Value::as_bool) + .unwrap_or(false), + ), + ); Value::Object(system) } @@ -572,6 +583,7 @@ fn save_system_with_default_workdir( SYSTEM_ARCHIVED_WORKSPACE_PROJECT_PATHS_KEY, SYSTEM_WORKSPACE_RESOURCE_SETTINGS_KEY, SYSTEM_SYSTEM_PROXY_KEY, + SYSTEM_CUA_ALLOW_SELF_TARGETING_KEY, ] { let value = system.get(key).cloned().unwrap_or(Value::Null); tx.execute( diff --git a/crates/agent-gui/src-tauri/src/commands/config/settings/tests.rs b/crates/agent-gui/src-tauri/src/commands/config/settings/tests.rs index 006647637..748a75c08 100644 --- a/crates/agent-gui/src-tauri/src/commands/config/settings/tests.rs +++ b/crates/agent-gui/src-tauri/src/commands/config/settings/tests.rs @@ -1185,13 +1185,14 @@ mod tests { }; let loaded = load_system(&conn).expect("load system"); - assert_eq!(row_count, 12); + assert_eq!(row_count, 13); assert_eq!( keys, vec![ SYSTEM_ACTIVE_WORKSPACE_PROJECT_ID_KEY.to_string(), SYSTEM_ARCHIVED_WORKSPACE_PROJECT_PATHS_KEY.to_string(), SYSTEM_COMMAND_SAFETY_MODE_KEY.to_string(), + SYSTEM_CUA_ALLOW_SELF_TARGETING_KEY.to_string(), SYSTEM_EXECUTION_MODE_KEY.to_string(), SYSTEM_HIDDEN_WORKSPACE_PROJECT_PATHS_KEY.to_string(), SYSTEM_MISSING_WORKSPACE_PROJECT_PATHS_KEY.to_string(), @@ -1207,6 +1208,7 @@ mod tests { loaded, Some(json!({ "activeWorkspaceProjectId": DEFAULT_WORKSPACE_PROJECT_ID, + "cuaAllowSelfTargeting": false, "executionMode": "tools", "hiddenWorkspaceProjectPaths": [], "missingWorkspaceProjectPaths": [], @@ -1506,6 +1508,7 @@ mod tests { loaded, Some(json!({ "activeWorkspaceProjectId": DEFAULT_WORKSPACE_PROJECT_ID, + "cuaAllowSelfTargeting": false, "executionMode": "tools", "hiddenWorkspaceProjectPaths": [], "missingWorkspaceProjectPaths": [], @@ -1560,6 +1563,7 @@ mod tests { loaded, Some(json!({ "activeWorkspaceProjectId": DEFAULT_WORKSPACE_PROJECT_ID, + "cuaAllowSelfTargeting": false, "executionMode": "tools", "hiddenWorkspaceProjectPaths": [], "missingWorkspaceProjectPaths": [], @@ -1631,6 +1635,7 @@ mod tests { loaded, json!({ "activeWorkspaceProjectId": DEFAULT_WORKSPACE_PROJECT_ID, + "cuaAllowSelfTargeting": false, "executionMode": "tools", "hiddenWorkspaceProjectPaths": [], "missingWorkspaceProjectPaths": [], diff --git a/crates/agent-gui/src-tauri/src/commands/integration/cua_driver.rs b/crates/agent-gui/src-tauri/src/commands/integration/cua_driver.rs new file mode 100644 index 000000000..080506085 --- /dev/null +++ b/crates/agent-gui/src-tauri/src/commands/integration/cua_driver.rs @@ -0,0 +1,69 @@ +//! `cua-driver` 引导命令桥。 +//! +//! 只覆盖「装没装 / 装一下 / 授权了没」这段引导；真正的计算机操作能力 +//! 走通用 MCP 链路（`cua-driver mcp` 是一个普通 stdio MCP server，由 +//! `commands/integration/mcp.rs` 驱动，工具从 `tools/list` 发现）。 +//! +//! 放在 `integration/` 而不是自成一域，正是因为它属于 MCP 接入的一部分。 + +use tauri::AppHandle; + +use crate::services::cua_driver::{ + self, CuaDriverPermissions, CuaDriverProbe, InstallCommandPreview, SelfIdentity, +}; + +/// 探测二进制位置、版本与 MCP 调用方式。未安装返回 `installed: false`， +/// 不是错误。只读，无副作用。 +#[tauri::command(rename_all = "camelCase")] +pub async fn cua_driver_probe() -> Result { + tauri::async_runtime::spawn_blocking(cua_driver::probe) + .await + .map_err(|error| format!("cua_driver_probe join failed: {error}")) +} + +/// 返回将要执行的安装命令**全文**，不执行。 +/// +/// UI 必须先把 `display` 展示给用户并取得显式确认，才允许调 +/// `cua_driver_install`：那条命令会从网络下载一段 shell 脚本并直接 +/// 执行，用户有权先看清楚。 +#[tauri::command(rename_all = "camelCase")] +pub fn cua_driver_install_command() -> InstallCommandPreview { + cua_driver::install_command_preview() +} + +/// 执行官方安装脚本，进度经 `cua_driver_install_progress` 事件流式回传。 +/// +/// 前置条件由 UI 保证：用户已看过 `cua_driver_install_command` 的输出 +/// 并确认。这里不做二次弹窗——后端没有 UI 上下文，弹不出可信的确认。 +#[tauri::command(rename_all = "camelCase")] +pub async fn cua_driver_install(app: AppHandle) -> Result { + tauri::async_runtime::spawn_blocking(move || cua_driver::install(&app)) + .await + .map_err(|error| format!("cua_driver_install join failed: {error}"))? +} + +/// 读取 macOS 的 Accessibility / Screen Recording 授权状态。只读， +/// 不触发系统授权弹窗。非 macOS 返回 `supported: false`。 +#[tauri::command(rename_all = "camelCase")] +pub async fn cua_driver_permissions_status() -> Result { + tauri::async_runtime::spawn_blocking(cua_driver::permissions_status) + .await + .map_err(|error| format!("cua_driver_permissions_status join failed: {error}")) +} + +/// 触发上游的授权引导：拉起 CuaDriver.app 并请求两项权限。会弹系统 +/// 对话框——授权归属 CuaDriver.app（而非 LiveAgent），这是上游推荐的 +/// 唯一正确路径。 +#[tauri::command(rename_all = "camelCase")] +pub async fn cua_driver_permissions_grant() -> Result { + tauri::async_runtime::spawn_blocking(cua_driver::permissions_grant) + .await + .map_err(|error| format!("cua_driver_permissions_grant join failed: {error}"))? +} + +/// LiveAgent 自身的进程身份。前端用它把 cua-driver 的窗口 / 应用列表里 +/// 属于宿主的记录裁掉，并拦下直接以宿主 pid 为目标的调用。只读。 +#[tauri::command(rename_all = "camelCase")] +pub fn cua_driver_self_identity() -> SelfIdentity { + cua_driver::self_identity() +} diff --git a/crates/agent-gui/src-tauri/src/commands/integration/mod.rs b/crates/agent-gui/src-tauri/src/commands/integration/mod.rs index ceb9e695a..c2cb68413 100644 --- a/crates/agent-gui/src-tauri/src/commands/integration/mod.rs +++ b/crates/agent-gui/src-tauri/src/commands/integration/mod.rs @@ -1,3 +1,4 @@ +pub mod cua_driver; pub mod gateway; pub mod mcp; pub mod memory; diff --git a/crates/agent-gui/src-tauri/src/commands/mod.rs b/crates/agent-gui/src-tauri/src/commands/mod.rs index 2b145be5e..ac3f8dcc1 100644 --- a/crates/agent-gui/src-tauri/src/commands/mod.rs +++ b/crates/agent-gui/src-tauri/src/commands/mod.rs @@ -14,6 +14,7 @@ pub mod runtime_commands; pub mod workspace_commands; pub use app_commands::app; +pub use app_commands::ax; pub use app_commands::system; pub use app_commands::tray; pub use app_commands::update; @@ -27,6 +28,7 @@ pub use history_commands::chat_history; pub use history_commands::history_db; pub use history_commands::subagent_store; +pub use integration_commands::cua_driver; pub use integration_commands::gateway; pub use integration_commands::mcp; pub use integration_commands::memory; diff --git a/crates/agent-gui/src-tauri/src/lib.rs b/crates/agent-gui/src-tauri/src/lib.rs index 986bd3e7c..ef2bf4da3 100644 --- a/crates/agent-gui/src-tauri/src/lib.rs +++ b/crates/agent-gui/src-tauri/src/lib.rs @@ -309,6 +309,16 @@ macro_rules! app_invoke_handler { commands::system::system_begin_power_activity, commands::system::system_end_power_activity, commands::system::system_clipboard_read_text, + // AX 辅助命令：让外部自动化工具（cua-driver）能读到本窗口的 + // 无障碍树。与「操作用户电脑」无关——那条链路走 MCP Hub。 + commands::ax::cua_window_ready, + commands::ax::cua_refresh_a11y, + commands::cua_driver::cua_driver_probe, + commands::cua_driver::cua_driver_install_command, + commands::cua_driver::cua_driver_install, + commands::cua_driver::cua_driver_permissions_status, + commands::cua_driver::cua_driver_permissions_grant, + commands::cua_driver::cua_driver_self_identity, commands::gateway::gateway_connect, commands::gateway::gateway_disconnect, commands::gateway::gateway_status, @@ -352,11 +362,425 @@ fn show_main_window(app: &tauri::AppHandle) -> tauri::Result<()> { window.show()?; window.unminimize()?; window.set_focus()?; + // CUA-007: on macOS, `set_focus()` alone is not enough to reclaim + // focus from a previously foreground app (e.g. a stale Problem + // Reporter dialog). `NSApp.activate(ignoringOtherApps: true)` plus + // `makeKeyAndOrderFront` is the documented pattern for "always + // bring this window to the front", which is what cua-driver needs + // for `bring_to_front` to land on the dev window. + force_activate_main_window(&window); } Ok(()) } +/// macOS-only: bring the main window to the front by activating NSApp and +/// ordering the window key. No-op on other platforms. +#[cfg(target_os = "macos")] +#[allow(deprecated)] // `activateIgnoringOtherApps` is the documented hook for our case; the new `NSApp.activate` API is not yet available in our objc2-app-kit version. +pub(crate) fn force_activate_main_window(window: &tauri::WebviewWindow) { + use objc2::rc::Retained; + use objc2_app_kit::{NSApplication, NSWindow}; + use objc2::MainThreadMarker; + // `WebviewWindow::ns_window` borrows the window, so the raw pointer we + // get back has a lifetime tied to the borrow. To hand it to the main + // thread we must first convert it to an integer (raw pointers are not + // `Send`); AppKit guarantees the underlying NSWindow outlives the + // Tauri handle, so the integer is a faithful stand-in. + let ns_window_addr = window + .ns_window() + .ok() + .filter(|ptr| !ptr.is_null()) + .map(|ptr| ptr.cast::() as usize) + .unwrap_or(0); + if ns_window_addr == 0 { + return; + } + let _ = window.run_on_main_thread(move || { + let ns_window_ptr = ns_window_addr as *mut NSWindow; + // run_on_main_thread guarantees we are on the AppKit main thread; + // MainThreadMarker::new() panics on the wrong thread, which would + // be a programmer error worth surfacing loudly. + let mtm = MainThreadMarker::new().expect("run_on_main_thread must run on the AppKit main thread"); + let ns_app: Retained = NSApplication::sharedApplication(mtm); + // Ignoring other apps is exactly what we want during dev: cua-driver + // needs the LiveAgent window to be frontmost to deliver input. In + // production, users would notice this; in dev it is the right knob + // because the desktop test harness explicitly summons the window. + ns_app.activateIgnoringOtherApps(true); + let ns_window: &NSWindow = unsafe { &*ns_window_ptr }; + ns_window.makeKeyAndOrderFront(None); + // CUA-007: the WKWebView child view must be the first responder + // for HID events to actually reach the renderer. Without this, + // cua-driver's foreground delivery reports `effect: unverifiable` + // because the input is dropped at the AppKit layer — the NSWindow + // is "frontmost" but no responder consumes the events. + // CUA-011: explicitly enable accessibility on the NSWindow so + // WindowServer's AX walker can descend into the WKWebView subtree. + // Without this, `cua-driver get_window_state` returns + // `ax_window_unresolved` because the AppKit surface we present + // does not opt into accessibility until *something* queries it. + // We call this on every re-activation so a later dev reload still + // benefits even if the renderer tears down the previous responder. + // CUA-019: also explicitly mark the NSWindow itself as an + // accessibility element with role AXWindow so cua-driver's + // AXWindow walk has at least one entry. Without this the + // window's default `isAccessibilityElement = false` keeps it + // out of the AX tree entirely, and even a perfectly-configured + // WKWebView subtree is unreachable. + // + // Caveat: the tao-rs `TaoWindow` class is a runtime-built + // subclass of NSWindow. `respondsToSelector(setIsAccessibilityElement:)` + // returns false because the subclass is registered with the + // bare minimum methods (see tao's WindowClass). Sending the + // selector anyway throws `NSInvalidArgumentException` + // (unrecognized selector) which terminates the dev binary at + // first paint. The CUA-015 guard wraps the call in + // `objc2::exception::Exception::catch` so a future tao/wry + // release that DOES implement these selectors can take + // advantage without us having to ship another round trip. + // + // The `NSAccessibilitySetOverrideEnabled(true)` function that + // would force AppKit to publish the tree regardless of consumer + // presence is NOT in the public AppKit symbol table on recent + // macOS releases (it's an internal helper), so we can't link + // against it without a private framework header. The runtime + // configuration must rely on (a) the WKWebView's remote a11y + // tree being installed automatically when WKWebView is added + // to a window whose `accessibilityEnabled` is true, and (b) + // the AX walker picking it up via the `setAccessibilityChildren:` + // call below. + let _ = objc2::exception::catch(std::panic::AssertUnwindSafe(|| unsafe { + let _: () = objc2::msg_send![&*ns_window, setAccessibilityEnabled: true]; + let _: () = objc2::msg_send![&*ns_window, setIsAccessibilityElement: true]; + let ax_window_role: Retained = objc2::msg_send![ + objc2::class!(NSString), + stringWithUTF8String: b"AXWindow\0".as_ptr() + ]; + let _: () = objc2::msg_send![&*ns_window, setAccessibilityRole: &*ax_window_role]; + })); + if let Some(content_view) = ns_window.contentView() { + // CUA-015: Tauri 2 / wry's NSWindow contentView is a + // `wry::WryWebViewParent`, which does NOT implement + // `setIsAccessibilityElement:` or `setAccessibilityRole:`. + // Calling them unconditionally throws `NSInvalidArgumentException` + // (`unrecognized selector sent to instance …`) and terminates + // the dev binary at `frontend_ready` time, blocking every + // launch. Guard with `respondsToSelector:` so we silently skip + // the annotation on classes that haven't opted into + // accessibility, and wrap the whole block in + // `objc2::exception::Exception::catch` so any future selector + // drift in a wry/Tauri release cannot take the binary down. + // CUA-011's intent (WindowServer's AX walker can descend into + // the WKWebView subtree) is preserved when wry later swaps in a + // view that does respond; the `setAccessibilityEnabled:` call on + // the NSWindow above is unaffected. + use objc2::runtime::NSObjectProtocol; + let annotates_accessibility = content_view + .respondsToSelector(objc2::sel!(setIsAccessibilityElement:)) + && content_view.respondsToSelector(objc2::sel!(setAccessibilityRole:)); + if annotates_accessibility { + // Use a raw pointer + AssertUnwindSafe so the closure stays + // `UnwindSafe` regardless of `Retained`'s auto-trait impls + // (the `Retained` itself is still owned by + // `content_view` outside the catch). + let cv_ptr = + std::ptr::addr_of!(*content_view).cast::(); + let _ = objc2::exception::catch( + std::panic::AssertUnwindSafe(move || { + let role: objc2::rc::Retained = unsafe { + objc2::msg_send![ + objc2::class!(NSString), + stringWithUTF8String: b"AXWindow\0".as_ptr() + ] + }; + unsafe { + let _: () = objc2::msg_send![ + cv_ptr, + setIsAccessibilityElement: true + ]; + let _: () = + objc2::msg_send![cv_ptr, setAccessibilityRole: &*role]; + } + }), + ); + } + // CUA-017: walk the content view's subviews to find the actual + // WKWebView. `wry::WryWebViewParent` inherits NSView's default + // `acceptsFirstResponder == false`, so calling + // `makeFirstResponder(Some(&content_view))` silently fails on + // every dev build — AppKit drops the request because the + // receiver refuses to become first responder. The WKWebView + // subclass overrides `acceptsFirstResponder` to return `true` + // (WebKit consumes keyboard events), so naming it explicitly is + // what actually plumbs foreground HID into the renderer. + let wk_webview_ptr = find_wk_webview_in_subviews(&content_view); + if let Some(wk_ptr) = wk_webview_ptr { + let wk_view: &objc2_app_kit::NSView = unsafe { wk_ptr.as_ref() }; + // Promote the WKWebView to first responder through BOTH + // pathways. `makeFirstResponder` is the window-level + // delegate; `becomeFirstResponder` is the view-level + // confirmation that exercises `acceptsFirstResponder` so + // we know the rename actually stuck. + let ok = ns_window.makeFirstResponder(Some(wk_view)); + let became = wk_view.becomeFirstResponder(); + if !ok || !became { + eprintln!( + "liveagent: WKWebView first responder not granted (makeFirstResponder={ok}, become={became}); falling back to content view" + ); + ns_window.makeFirstResponder(Some(&content_view)); + } + // CUA-019: declare the WKWebView as the content view's + // accessibility child so WindowServer's AX walker has a + // deterministic entry point into the WKWebView subtree. + // Without this, `get_window_state` reports + // `ax_window_unresolved` because + // `wry::WryWebViewParent` doesn't implement + // `accessibilityChildren` itself (the selector is + // inherited as a no-op). The WKWebView already owns a + // remote AX tree; we just need to expose it. + unsafe { + let wk_array: Retained> = + objc2::msg_send![ + objc2::class!(NSArray), + arrayWithObject: wk_view + ]; + let _: () = objc2::msg_send![ + &*content_view, + setAccessibilityChildren: &*wk_array + ]; + } + // CUA-019: also explicitly mark the WKWebView itself as + // an accessibility element with role AXWebArea so the + // WKWebView's own AX bridge has at least one anchor + // entry visible to AX walkers that look inside the + // window. WebKit installs its remote a11y tree under + // this element. The whole block is wrapped in + // exception::catch so a future WebKit selector drift + // cannot crash the dev binary. + let _ = objc2::exception::catch(std::panic::AssertUnwindSafe(|| unsafe { + let _: () = objc2::msg_send![wk_view, setAccessibilityEnabled: true]; + let _: () = objc2::msg_send![wk_view, setIsAccessibilityElement: true]; + let ax_web_area: Retained = objc2::msg_send![ + objc2::class!(NSString), + stringWithUTF8String: b"AXWebArea\0".as_ptr() + ]; + let _: () = objc2::msg_send![wk_view, setAccessibilityRole: &*ax_web_area]; + })); + // CUA-019: also broadcast a UIElementCreatedNotification + // on the content view. WindowServer's AX walker treats + // this as a cue to re-evaluate the window's subtree — if + // it had previously cached `ax_window_unresolved`, the + // next `get_window_state` call sees the WKWebView's tree. + // The notification is cheap; re-broadcasting on every + // activation also covers the dev-reload path where the + // previous WKWebView was torn down. + post_accessibility_element_created(&content_view); + } else { + // Fallback path: WKWebView not in the view hierarchy yet + // (page still loading) or wry swapped implementations. + // Keep the CUA-007 behaviour so a partial first paint + // still routes input to the responder chain's deepest + // accepting view. + ns_window.makeFirstResponder(Some(&content_view)); + } + } + }); +} + +#[cfg(target_os = "macos")] +fn find_wk_webview_in_subviews( + view: &objc2_app_kit::NSView, +) -> Option> { + // `class!` would force a hard dependency on the WebKit framework class + // list, which is not part of objc2-app-kit. Resolve WKWebView at + // runtime via the Objective-C class registry — wry loads WebKit as + // part of WKWebView construction, so the class is registered by the + // time we get here in production. In tests where no WebKit is + // loaded, `get` returns `None` and we silently fall through. + let wk_class = objc2::runtime::AnyClass::get(c"WKWebView")?; + let is_wk: bool = + unsafe { objc2::msg_send![view, isKindOfClass: wk_class] }; + if is_wk { + return Some(std::ptr::NonNull::from(view)); + } + let subviews = view.subviews(); + for sub in subviews.iter() { + if let Some(found) = find_wk_webview_in_subviews(&sub) { + return Some(found); + } + } + None +} + +#[cfg(target_os = "macos")] +fn post_accessibility_element_created(view: &objc2_app_kit::NSView) { + use objc2::rc::Retained; + // `NSAccessibilityPostNotification` is a free C function exported + // from AppKit. `NSAccessibility` itself is only a category on + // `NSObject` + a protocol — it is NOT a real class, so `class!` + // would panic at runtime. Linking the symbol directly is the + // supported path (it's how every macOS app calls this entry point). + // Wrapping the call in `objc2::exception::catch` keeps a malformed + // AppKit from taking the dev binary down if the selector ever + // changes signature in a future macOS release. + #[link(name = "AppKit", kind = "framework")] + extern "C" { + fn NSAccessibilityPostNotification( + element: *const objc2::runtime::AnyObject, + notification: *const objc2::runtime::AnyObject, + ); + } + let _ = objc2::exception::catch(std::panic::AssertUnwindSafe(|| { + unsafe { + let notification_name: Retained = objc2::msg_send![ + objc2::class!(NSString), + stringWithUTF8String: b"NSAccessibilityUIElementCreatedNotification\0".as_ptr() + ]; + NSAccessibilityPostNotification( + std::ptr::addr_of!(*view).cast::(), + std::ptr::addr_of!(*notification_name).cast::(), + ); + } + })); +} + +#[cfg(not(target_os = "macos"))] +pub(crate) fn force_activate_main_window(_window: &tauri::WebviewWindow) {} + +/// Polled result of `force_activate_main_window`: cua-driver calls +/// `cua_window_ready` between `bring_to_front` and the first AX / +/// foreground click, so this is the contract the driver can rely on. +#[derive(Debug, Clone, serde::Serialize)] +#[serde(rename_all = "camelCase")] +pub struct CuaWindowReadyResponse { + /// True once `is_focused()` reports true within the poll window. + pub focused: bool, + /// How long we waited for focus to land (ms). Useful for the caller + /// to detect a slow first paint and back off. + pub elapsed_ms: u64, + /// Whether the macOS-specific NSApp.activate path ran. False on + /// non-macOS platforms where the call is a no-op best-effort. + pub macos_activated: bool, +} + +/// Best-effort "force focus" command for cua-driver. Re-runs the macOS +/// NSApp.activate + makeKeyAndOrderFront + makeFirstResponder cycle (see +/// [`force_activate_main_window`]) and then polls `is_focused()` for up +/// to 750 ms. cua-driver awaits this between `bring_to_front` and the +/// next AX / pixel action to avoid the `effect: unverifiable` / +/// `ax_window_unresolved` regressions seen when the WKWebView is still +/// settling after first paint (CUA-011). +pub async fn cua_window_ready(window: tauri::WebviewWindow) -> CuaWindowReadyResponse { + let started = std::time::Instant::now(); + force_activate_main_window(&window); + let mut focused = window.is_focused().unwrap_or(false); + if !focused { + // Poll up to 750 ms, 50 ms cadence — long enough to outlast a + // typical first-paint cycle but short enough that the driver + // stays interactive. Each iteration re-issues the activation + // hint so AppKit keeps LiveAgent on the foreground space. + let mut elapsed_ms = 0u64; + while elapsed_ms < 750 { + tokio::time::sleep(std::time::Duration::from_millis(50)).await; + elapsed_ms = started.elapsed().as_millis() as u64; + force_activate_main_window(&window); + if window.is_focused().unwrap_or(false) { + focused = true; + break; + } + } + } + CuaWindowReadyResponse { + focused, + elapsed_ms: started.elapsed().as_millis() as u64, + macos_activated: cfg!(target_os = "macos"), + } +} + +/// CUA-020/021/022: 给前端的「重新发表 AX 表面」诊断响应。 +/// `force_activate_main_window` 是幂等的（NSWindow 注解不变），但每 +/// 次都会重新跑 `makeFirstResponder + becomeFirstResponder` 并在 +/// content view 上重新广播 `UIElementCreatedNotification`，用于 +/// `Settings overlay` 打开 / 路由切换 / WKWebView hot reload 后让 +/// cua-driver 的下一帧 AX 查询拿到刷新后的表面（不再 `unresolved`）。 +#[derive(Debug, Clone, serde::Serialize)] +#[serde(rename_all = "camelCase")] +pub struct CuaRefreshA11yResponse { + /// 重新触发了 `force_activate_main_window` 的次数（带 retry 时 >1）。 + pub activations: u32, + /// 找到的 WKWebView 子视图指针是否非空（false 表示尚未挂载）。 + pub wk_webview_found: bool, + /// NSWindow 是否真的拿到了 first responder（macOS only）。 + pub responder_granted: bool, + /// 是否在 macOS 平台跑了实际注解（非 macOS 是 no-op）。 + pub macos_activated: bool, +} + +/// 把 `force_activate_main_window` 的内容再次跑一遍——主要给前端在 +/// `Settings overlay` 打开 / 路由切换 / 模态弹出后手动调用一次，让 +/// WKWebView 的 a11y 子树被 cua-driver 看见（CUA-021）。也用于 +/// `cua_window_ready` 之外的「只修 AX、不等 focus」场景。 +pub fn cua_refresh_a11y(window: &tauri::WebviewWindow) -> CuaRefreshA11yResponse { + // 在 macOS 上 force_activate_main_window 已经覆盖了：NSWindow 注解 + // + WKWebView 第一响应 + AX 子树广播。这里再加一次「WKWebView 是 + // 否真的找到 / 是否真的成 first responder」的诊断，便于前端在 + // 端到端验证时不用再额外发一次 `cua_window_ready`。 + let activations = 1u32; + // 用 Arc 把诊断标志从 `run_on_main_thread` 的闭包里透 + // 出来——闭包按 move 捕获，无法直接拿 `&mut` 外部变量。 + let wk_flag = std::sync::Arc::new(std::sync::atomic::AtomicBool::new(false)); + let resp_flag = std::sync::Arc::new(std::sync::atomic::AtomicBool::new(false)); + let mtm_present = cfg!(target_os = "macos"); + // CUA-022: 第二次跑确保 WKWebView 在 hot reload / overlay 重渲 + // 之后重新拿到 first responder——`makeFirstResponder` 不是幂等的， + // WKWebView 被 React unmount/remount 后会重置 responder chain。 + force_activate_main_window(window); + #[cfg(target_os = "macos")] + { + use objc2_app_kit::NSWindow; + if let Ok(ptr) = window.ns_window() { + if !ptr.is_null() { + let addr = ptr.cast::() as usize; + if addr != 0 { + let wk_flag_inner = std::sync::Arc::clone(&wk_flag); + let resp_flag_inner = std::sync::Arc::clone(&resp_flag); + let _ = window.run_on_main_thread(move || { + let ns_window_ptr = addr as *mut NSWindow; + let ns_window: &NSWindow = unsafe { &*ns_window_ptr }; + if let Some(content_view) = ns_window.contentView() { + if let Some(wk_ptr) = find_wk_webview_in_subviews(&content_view) { + wk_flag_inner.store(true, std::sync::atomic::Ordering::SeqCst); + let wk_view: &objc2_app_kit::NSView = unsafe { wk_ptr.as_ref() }; + let ok = ns_window.makeFirstResponder(Some(wk_view)); + let became = wk_view.becomeFirstResponder(); + resp_flag_inner.store( + ok && became, + std::sync::atomic::Ordering::SeqCst, + ); + // CUA-021: 重广播一次 UIElementCreated 通 + // 知，让 WindowServer 的 AX walker 把最近 + // 一次渲染（含 Settings overlay 重渲）写 + // 进缓存。cua-driver 下一帧 + // `get_window_state` 不会再返回 + // `ax_window_unresolved`。 + post_accessibility_element_created(&content_view); + } + } + }); + } + } + } + } + let _ = mtm_present; + CuaRefreshA11yResponse { + activations, + wk_webview_found: wk_flag.load(std::sync::atomic::Ordering::SeqCst), + responder_granted: resp_flag.load(std::sync::atomic::Ordering::SeqCst), + macos_activated: mtm_present, + } +} + fn toggle_main_window(app: &tauri::AppHandle) { if let Some(window) = app.get_webview_window(MAIN_WINDOW_LABEL) { let visible = window.is_visible().unwrap_or(false); @@ -782,18 +1206,63 @@ pub fn run() { .manage(Arc::new(commands::hook::HookScopeRegistry::default())) .manage(stt_manager) .on_page_load(|webview, payload| { - if webview.label() != MAIN_WINDOW_LABEL - || !matches!(payload.event(), tauri::webview::PageLoadEvent::Started) - { + if webview.label() != MAIN_WINDOW_LABEL { return; } let app = webview.app_handle(); - if let Some(ready_state) = app.try_state::>() { - ready_state.0.store(false, Ordering::SeqCst); - } - if let Some(window) = app.get_webview_window(MAIN_WINDOW_LABEL) { - if window.is_visible().unwrap_or(false) { - let _ = window.hide(); + match payload.event() { + tauri::webview::PageLoadEvent::Started => { + if let Some(ready_state) = + app.try_state::>() + { + ready_state.0.store(false, Ordering::SeqCst); + } + if let Some(window) = app.get_webview_window(MAIN_WINDOW_LABEL) { + if window.is_visible().unwrap_or(false) { + let _ = window.hide(); + } + } + } + tauri::webview::PageLoadEvent::Finished => { + // CUA-011: re-issue force_activate_main_window after the + // first paint so the WKWebView is the first responder and + // the NSWindow's accessibility surface is wired up before + // cua-driver walks the AX tree. App_frontend_ready fires + // from a static-shell hook (often before the JS bundle is + // parsed); waiting for `Finished` closes the race where + // cua-driver queries the AX tree while the WKWebView + // host view is still being laid out, which is what was + // producing `ax_window_unresolved`. + // CUA-017/019: always run the activation cycle, even when + // the window is still hidden. The function is idempotent + // and the WKWebView is in the view hierarchy by the time + // `Finished` fires — calling it lets the AX walker find + // the WKWebView the moment `show_main_window` unhides + // the window, instead of waiting for cua-driver to issue + // its first foreground action. + // CUA-020: WKWebView's remote AX tree is populated + // asynchronously by the WebContent process — calling + // `force_activate_main_window` exactly at `Finished` + // races the layout pass and yields `ax_window_unresolved` + // on the very first `get_window_state`. Schedule a + // 600 ms deferred re-activation so the second pass lands + // after WebKit has registered its remote a11y children. + // The defer uses tokio (not `run_on_main_thread` + + // `std::thread::sleep`, which would block the AppKit + // main thread and starve the very layout pass we are + // trying to wait out). + if let Some(window) = app.get_webview_window(MAIN_WINDOW_LABEL) { + force_activate_main_window(&window); + let window_for_retry = window.clone(); + let app_handle_for_retry = app.clone(); + tauri::async_runtime::spawn(async move { + tokio::time::sleep(std::time::Duration::from_millis(600)).await; + let _ = app_handle_for_retry + .run_on_main_thread(move || { + force_activate_main_window(&window_for_retry); + }); + }); + } } } }) diff --git a/crates/agent-gui/src-tauri/src/services/cua_driver/mod.rs b/crates/agent-gui/src-tauri/src/services/cua_driver/mod.rs new file mode 100644 index 000000000..e997f8f64 --- /dev/null +++ b/crates/agent-gui/src-tauri/src/services/cua_driver/mod.rs @@ -0,0 +1,507 @@ +//! `cua-driver` 的探测 / 安装 / 权限查询。 +//! +//! 计算机操作能力本身**不经过这里**——`cua-driver mcp` 是一个标准的 +//! stdio MCP server，由 `commands/integration/mcp.rs` 那套通用 MCP +//! client 驱动，工具由 `tools/list` 自动发现。这个模块只负责它前面那 +//! 一小段引导：用户机器上有没有这个二进制、装在哪、要不要装、macOS +//! 的 TCC 授权给了没有。 +//! +//! 设计原则是**把活都推给上游**。版本检查、下载、解压、更新、授权引导 +//! 上游 CLI 全都有（`install.sh` / `update --apply` / `permissions +//! grant` / `doctor`），这里不重新实现，只做三件事： +//! +//! 1. 找到二进制（GUI 进程的 PATH 通常不含 `~/.local/bin`，必须补候选路径）； +//! 2. 问 `cua-driver manifest` 要 MCP 调用方式，而不是硬编码 `["mcp"]`； +//! 3. 需要安装时，转调官方安装脚本并把输出流式转发给前端。 +//! +//! macOS 上刻意**不**使用 `mcp --direct`：那会让 MCP 进程沿用宿主 +//! （LiveAgent.app）的 TCC 归属，等于要求 LiveAgent 自己去拿 +//! Accessibility 与 Screen Recording 授权。默认模式经 CuaDriver.app +//! 的守护进程代理，授权归它，宿主不需要任何 TCC 权限。 + +use std::path::{Path, PathBuf}; +use std::process::{Command, Stdio}; +use std::time::Duration; + +use serde::Serialize; +use serde_json::Value; +use tauri::{AppHandle, Emitter}; + +/// 单次外部命令的等待上限。`manifest` / `permissions status` 都在 1 秒 +/// 内返回；留足余量给冷启动的守护进程握手。 +const PROBE_TIMEOUT: Duration = Duration::from_secs(15); + +/// 安装脚本的进度事件名。前端 `CuaDriverSetupCard` 监听它滚动日志。 +pub const INSTALL_PROGRESS_EVENT: &str = "cua_driver_install_progress"; + +/// 官方安装脚本来源。展示给用户看的就是这个域名——必须与实际执行的 +/// URL 一致，否则确认对话框就是在骗人。 +const INSTALL_SCRIPT_URL_UNIX: &str = "https://cua.ai/driver/install.sh"; +const INSTALL_SCRIPT_URL_WINDOWS: &str = "https://cua.ai/driver/install.ps1"; + +#[derive(Debug, Default, Serialize)] +#[serde(rename_all = "camelCase")] +pub struct CuaDriverProbe { + pub installed: bool, + /// 二进制绝对路径。写进 MCP server 配置的就是它——不用裸名字， + /// 因为 MCP 子进程继承的是 GUI 进程那份窄 PATH。 + pub path: Option, + pub version: Option, + /// `manifest.mcp_invocation` 给出的调用方式。上游若改了子命令， + /// 这里跟着变，不需要我们发版。 + pub mcp_command: Option, + pub mcp_args: Vec, + /// 本平台是否存在需要用户处理的系统授权门槛。只有 macOS 有 TCC， + /// Windows / Linux 恒 false —— 前端据此**立即**决定要不要渲染授权 + /// 那一节，不必等 `permissions_status` 那趟子进程回来。 + pub permissions_required: bool, + /// 探测失败的原因（未安装是正常状态，不算错误，此时为 None）。 + pub error: Option, +} + +#[derive(Debug, Default, Serialize)] +#[serde(rename_all = "camelCase")] +pub struct CuaDriverPermissions { + /// 只有 macOS 有 TCC 门槛；其他平台恒 false，前端据此隐藏整段。 + pub supported: bool, + pub accessibility: bool, + pub screen_recording: bool, + /// 授权归属的 bundle id（正常是 `com.trycua.driver`）。守护进程没起 + /// 来时上游会报 unknown，此时两个布尔值不可信。 + pub attributed_to: Option, + pub daemon_running: bool, + pub error: Option, +} + +/// 安装命令预览。**只描述，不执行**——UI 必须先把 `display` 原样展示 +/// 给用户确认，才允许调 `install`。 +#[derive(Debug, Serialize)] +#[serde(rename_all = "camelCase")] +pub struct InstallCommandPreview { + pub program: String, + pub args: Vec, + /// 可直接粘进终端的完整命令。用户也可以选择自己去终端跑这一条。 + pub display: String, + /// 脚本来源 URL，用于在确认文案里点明「这会从网络下载并执行脚本」。 + pub source_url: String, +} + +#[derive(Debug, Clone, Serialize)] +#[serde(rename_all = "camelCase")] +pub struct InstallProgress { + /// `stdout` | `stderr` | `done` | `failed` + pub stream: String, + pub line: String, +} + +// ───────── 探测 ───────── + +/// 在 PATH 与平台候选目录里找 `cua-driver`。 +/// +/// 必须自己 walk 而不是靠 `Command::new("cua-driver")`：macOS 上从 +/// Finder / Dock 启动的 GUI 进程拿到的是 launchd 的默认 PATH，不含 +/// `~/.local/bin`，而那正是官方安装脚本的默认落点。 +fn find_binary() -> Option { + if let Some(found) = find_in_path("cua-driver") { + return Some(found); + } + candidate_paths().into_iter().find(|p| p.is_file()) +} + +fn find_in_path(binary: &str) -> Option { + let path_var = std::env::var_os("PATH")?; + for dir in std::env::split_paths(&path_var) { + let candidate = dir.join(binary); + if candidate.is_file() { + return Some(candidate); + } + #[cfg(target_os = "windows")] + { + let with_exe = dir.join(format!("{binary}.exe")); + if with_exe.is_file() { + return Some(with_exe); + } + } + } + None +} + +fn candidate_paths() -> Vec { + let home = dirs::home_dir(); + let mut out: Vec = Vec::new(); + + #[cfg(not(target_os = "windows"))] + { + if let Some(home) = home.as_ref() { + out.push(home.join(".local/bin/cua-driver")); + out.push(home.join(".cua/bin/cua-driver")); + } + out.push(PathBuf::from("/usr/local/bin/cua-driver")); + out.push(PathBuf::from("/opt/homebrew/bin/cua-driver")); + } + + #[cfg(target_os = "macos")] + { + // 装了 CuaDriver.app 但没建 PATH 软链的情况。 + out.push(PathBuf::from( + "/Applications/CuaDriver.app/Contents/MacOS/cua-driver", + )); + } + + #[cfg(target_os = "windows")] + { + if let Some(home) = home.as_ref() { + out.push(home.join(".local\\bin\\cua-driver.exe")); + out.push(home.join("AppData\\Local\\Programs\\cua-driver\\cua-driver.exe")); + } + } + + let _ = &home; + out +} + +fn run_capture(program: &Path, args: &[&str]) -> Result { + let mut child = Command::new(program) + .args(args) + .stdin(Stdio::null()) + .stdout(Stdio::piped()) + .stderr(Stdio::piped()) + .spawn() + .map_err(|error| format!("failed to spawn {}: {error}", program.display()))?; + + let status = match child + .wait_timeout(PROBE_TIMEOUT) + .map_err(|error| format!("wait failed: {error}"))? + { + Some(status) => status, + None => { + let _ = child.kill(); + let _ = child.wait(); + return Err(format!( + "{} {} timed out after {}s", + program.display(), + args.join(" "), + PROBE_TIMEOUT.as_secs() + )); + } + }; + + let output = child + .wait_with_output() + .map_err(|error| format!("failed to collect output: {error}"))?; + let stdout = String::from_utf8_lossy(&output.stdout).into_owned(); + if !status.success() { + let stderr = String::from_utf8_lossy(&output.stderr); + // 有些子命令（如 permissions status）业务失败也走非零退出但仍 + // 打了有效 JSON；把 stdout 一并带回去，让调用方决定怎么解析。 + return Err(format!( + "exit {}: {}", + status.code().unwrap_or(-1), + if stderr.trim().is_empty() { + stdout.trim() + } else { + stderr.trim() + } + )); + } + Ok(stdout) +} + +/// 探测安装状态。未安装不是错误——返回 `installed: false, error: None`。 +pub fn probe() -> CuaDriverProbe { + let Some(path) = find_binary() else { + return CuaDriverProbe { + permissions_required: cfg!(target_os = "macos"), + ..Default::default() + }; + }; + + let mut probe = CuaDriverProbe { + installed: true, + path: Some(path.to_string_lossy().into_owned()), + permissions_required: cfg!(target_os = "macos"), + ..Default::default() + }; + + match run_capture(&path, &["manifest"]) { + Ok(raw) => match serde_json::from_str::(&raw) { + Ok(manifest) => { + probe.version = manifest + .get("binary_version") + .and_then(Value::as_str) + .map(str::to_owned); + let invocation = manifest.get("mcp_invocation"); + probe.mcp_command = invocation + .and_then(|v| v.get("command")) + .and_then(Value::as_str) + .map(str::to_owned); + probe.mcp_args = invocation + .and_then(|v| v.get("args")) + .and_then(Value::as_array) + .map(|args| { + args.iter() + .filter_map(Value::as_str) + .map(str::to_owned) + .collect() + }) + .unwrap_or_default(); + } + Err(error) => probe.error = Some(format!("failed to parse manifest: {error}")), + }, + Err(error) => probe.error = Some(error), + } + + // manifest 没给出调用方式（老版本 / 解析失败）时回落到已知形态。 + // 刻意不加 `--direct`：见模块头注释。 + if probe.mcp_command.is_none() { + probe.mcp_command = probe.path.clone(); + probe.mcp_args = vec!["mcp".to_string()]; + } + + probe +} + +// ───────── 宿主自身身份 ───────── + +#[derive(Debug, Serialize)] +#[serde(rename_all = "camelCase")] +pub struct SelfIdentity { + pub pid: u32, + pub bundle_id: Option, +} + +/// LiveAgent 自己的进程身份，供前端把 cua-driver 的视野裁掉宿主窗口。 +/// +/// 让模型操作 LiveAgent 自己的界面是危险的自指：它能点掉自己的审批弹窗、 +/// 改自己的权限策略、或者直接把自己关了。过滤在前端做（Rust 侧的 +/// `mcp_call_tool` 是所有 MCP server 共用的通道，不该塞 cua 专属逻辑）， +/// 这里只提供比对用的事实。 +pub fn self_identity() -> SelfIdentity { + SelfIdentity { + pid: std::process::id(), + bundle_id: option_env!("TAURI_BUNDLE_IDENTIFIER").map(str::to_owned), + } +} + +// ───────── 权限（macOS） ───────── + +pub fn permissions_status() -> CuaDriverPermissions { + if !cfg!(target_os = "macos") { + return CuaDriverPermissions::default(); + } + let Some(path) = find_binary() else { + return CuaDriverPermissions { + supported: true, + error: Some("cua-driver not installed".into()), + ..Default::default() + }; + }; + + // 守护进程状态单独问一次：`permissions status` 在守护进程没起来时 + // 只会报 unknown，不区分「没装」和「没跑」，对用户不可读。 + // + // 两次调用各要 spawn 一个进程、并可能等守护进程握手，串起来就是用户 + // 盯着空白等两轮。彼此无依赖，并行跑。 + let daemon_path = path.clone(); + let daemon_probe = + std::thread::spawn(move || run_capture(&daemon_path, &["status"]).is_ok_and(|out| out.contains("is running"))); + let status = run_capture(&path, &["permissions", "status", "--json"]); + let daemon_running = daemon_probe.join().unwrap_or(false); + + match status { + Ok(raw) => match serde_json::from_str::(&raw) { + Ok(payload) => CuaDriverPermissions { + supported: true, + accessibility: payload + .get("accessibility") + .and_then(Value::as_bool) + .unwrap_or(false), + screen_recording: payload + .get("screen_recording") + .and_then(Value::as_bool) + .unwrap_or(false), + attributed_to: payload + .get("source") + .and_then(|source| source.get("bundle_id")) + .and_then(Value::as_str) + .map(str::to_owned), + daemon_running, + error: None, + }, + Err(error) => CuaDriverPermissions { + supported: true, + daemon_running, + error: Some(format!("failed to parse permissions payload: {error}")), + ..Default::default() + }, + }, + Err(error) => CuaDriverPermissions { + supported: true, + daemon_running, + error: Some(error), + ..Default::default() + }, + } +} + +/// 触发上游的授权引导。会弹系统对话框并把 CuaDriver.app 拉起来， +/// 归属正确的 bundle identity——这是唯一正确的授权路径，只读的 +/// `permissions status` 永远不会触发它。 +pub fn permissions_grant() -> Result { + if !cfg!(target_os = "macos") { + return Ok(CuaDriverPermissions::default()); + } + let path = find_binary().ok_or_else(|| "cua-driver not installed".to_string())?; + run_capture(&path, &["permissions", "grant"])?; + Ok(permissions_status()) +} + +// ───────── 安装 ───────── + +/// 描述将要执行的安装命令。**不执行任何东西。** +/// +/// 存在的理由就是让 UI 能在动手之前把命令原文摆到用户面前：这条命令 +/// 会从网络拉一段 shell 脚本直接执行，用户有权在看到全文之后再决定。 +pub fn install_command_preview() -> InstallCommandPreview { + if cfg!(target_os = "windows") { + let inner = format!("irm {INSTALL_SCRIPT_URL_WINDOWS} | iex"); + InstallCommandPreview { + program: "powershell".into(), + args: vec!["-NoProfile".into(), "-Command".into(), inner.clone()], + display: format!("powershell -NoProfile -Command \"{inner}\""), + source_url: INSTALL_SCRIPT_URL_WINDOWS.into(), + } + } else { + let inner = format!("$(curl -fsSL {INSTALL_SCRIPT_URL_UNIX})"); + InstallCommandPreview { + program: "/bin/bash".into(), + args: vec!["-c".into(), inner.clone()], + display: format!("/bin/bash -c \"{inner}\""), + source_url: INSTALL_SCRIPT_URL_UNIX.into(), + } + } +} + +/// 执行官方安装脚本，把 stdout / stderr 逐行 emit 给前端。 +/// +/// 调用方（Tauri command）必须确保用户已经在看到 +/// `install_command_preview().display` 之后显式确认过。 +pub fn install(app: &AppHandle) -> Result { + let preview = install_command_preview(); + let mut child = Command::new(&preview.program) + .args(&preview.args) + .stdin(Stdio::null()) + .stdout(Stdio::piped()) + .stderr(Stdio::piped()) + .spawn() + .map_err(|error| format!("failed to launch installer: {error}"))?; + + let stdout = child.stdout.take(); + let stderr = child.stderr.take(); + let pump = |handle: Option>, stream: &'static str| { + let app = app.clone(); + handle.map(|reader| { + std::thread::spawn(move || { + use std::io::BufRead; + for line in std::io::BufReader::new(reader).lines().map_while(Result::ok) { + let _ = app.emit( + INSTALL_PROGRESS_EVENT, + InstallProgress { + stream: stream.to_string(), + line, + }, + ); + } + }) + }) + }; + let out_pump = pump( + stdout.map(|s| Box::new(s) as Box), + "stdout", + ); + let err_pump = pump( + stderr.map(|s| Box::new(s) as Box), + "stderr", + ); + + let status = child + .wait() + .map_err(|error| format!("installer wait failed: {error}"))?; + if let Some(handle) = out_pump { + let _ = handle.join(); + } + if let Some(handle) = err_pump { + let _ = handle.join(); + } + + if !status.success() { + let message = format!("installer exited with {}", status.code().unwrap_or(-1)); + let _ = app.emit( + INSTALL_PROGRESS_EVENT, + InstallProgress { + stream: "failed".into(), + line: message.clone(), + }, + ); + return Err(message); + } + + let probe = probe(); + let _ = app.emit( + INSTALL_PROGRESS_EVENT, + InstallProgress { + stream: "done".into(), + line: probe + .version + .clone() + .map(|version| format!("cua-driver {version}")) + .unwrap_or_else(|| "installed".into()), + }, + ); + Ok(probe) +} + +use wait_timeout::ChildExt; + +#[cfg(test)] +mod tests { + use super::*; + + #[test] + fn install_preview_never_executes_and_matches_its_source_url() { + let preview = install_command_preview(); + // 展示给用户的命令必须真的包含那个 URL——确认对话框的全部意义 + // 就在于「看到的即将执行的」。 + assert!(preview.display.contains(&preview.source_url)); + assert!(preview.args.iter().any(|arg| arg.contains(&preview.source_url))); + } + + #[test] + fn permissions_required_tracks_the_platform_tcc_gate() { + // 前端靠这一位决定要不要渲染授权那一节；不能等 permissions_status + // 那趟慢查询回来才知道平台，否则卡片会「先没有、后长出来」。 + assert_eq!(probe().permissions_required, cfg!(target_os = "macos")); + assert_eq!( + CuaDriverProbe::default().permissions_required, + false, + "Default 用于「探测彻底失败」的兜底，不该声称有授权门槛" + ); + } + + #[test] + fn probe_reports_not_installed_without_error() { + // 未安装是正常状态，不该被前端当成故障红条渲染。 + let probe = CuaDriverProbe::default(); + assert!(!probe.installed); + assert!(probe.error.is_none()); + } + + #[test] + fn candidate_paths_cover_the_official_install_location() { + let paths = candidate_paths(); + assert!( + paths.iter().any(|p| p.to_string_lossy().contains(".local")), + "官方安装脚本默认落在 ~/.local/bin，GUI 进程的 PATH 通常不含它" + ); + } +} diff --git a/crates/agent-gui/src-tauri/src/services/mod.rs b/crates/agent-gui/src-tauri/src/services/mod.rs index bb39f9755..c08ec1ae2 100644 --- a/crates/agent-gui/src-tauri/src/services/mod.rs +++ b/crates/agent-gui/src-tauri/src/services/mod.rs @@ -1,5 +1,6 @@ pub mod automation; pub mod chat_run_ledger; +pub mod cua_driver; pub mod gateway; pub mod gateway_bridge; pub mod memory; diff --git a/crates/agent-gui/src-tauri/tauri.conf.json b/crates/agent-gui/src-tauri/tauri.conf.json index ea86b2977..9bfb7c1d8 100644 --- a/crates/agent-gui/src-tauri/tauri.conf.json +++ b/crates/agent-gui/src-tauri/tauri.conf.json @@ -19,6 +19,7 @@ "minWidth": 1200, "minHeight": 720, "visible": false, + "focus": true, "backgroundThrottling": "disabled" } ], diff --git a/crates/agent-gui/src/App.tsx b/crates/agent-gui/src/App.tsx index 402b49b03..de9f9cf19 100644 --- a/crates/agent-gui/src/App.tsx +++ b/crates/agent-gui/src/App.tsx @@ -513,6 +513,72 @@ export default function App() { const closeSettings = closeSettingsOverlay; + // CUA-021/022: Settings overlay 打开/关闭后会切换一棵新 React 子树， + // WKWebView 的 AX walker 会先短暂失效再重建。让前端在过渡结束、覆 + // 盖层真正可见之后主动 `cua_refresh_a11y` 一次，让 Rust 把 + // NSWindow/WKWebView 的 a11y 注解重新广播一遍——cua-driver 下一帧 + // `get_window_state` 不会再 `ax_window_unresolved`。`cua_window_ready` + // 是入口前置命令，这个是在它之后的「再补一刀」。 + useEffect(() => { + if (!settingsReady) return; + // overlay 状态切到 open/closed 后再触发，避开 React 渲染抖动。 + if (overlay !== "open" && overlay !== "closed") return; + let cancelled = false; + // 给 AppKit 50 ms 让它把当前 frame 的 AX 提交到 WindowServer—— + // 立即调用经常仍然拿到前一帧的缓存。 + const timer = window.setTimeout(() => { + if (cancelled) return; + invoke<{ wk_webview_found: boolean; responder_granted: boolean }>("cua_refresh_a11y").catch( + (error: unknown) => { + console.warn("cua_refresh_a11y failed", error); + }, + ); + }, 50); + return () => { + cancelled = true; + window.clearTimeout(timer); + }; + }, [overlay, settingsReady]); + + // CUA-031: 对设置面板内元素做 `delivery_mode: foreground` 输入投递后， + // WKWebView 的 a11y surface 会暂时进入 `ax_unresolved` 状态（CUA-021 + // 回归）。监听 visibilitychange / window focus——这类事件触发意味着刚 + // 刚发生过「窗口被另一进程抢焦点又还回来」的来回，AppKit 的 AX walker + // 可能尚未把新子树提交进 WindowServer。重新触发一次 `cua_refresh_a11y` + // 让 Rust 端把 NSWindow 注解 + WKWebView 第一响应 + UIElementCreatedNotification + // 重新广播，cua-driver 下一帧 `get_window_state` 不再 `ax_unresolved`。 + // 用 50 ms 防抖合并短时间内多次可见/聚焦事件，避免在 React 抖动期内 + // 连续打几帧 RPC。 + useEffect(() => { + if (!settingsReady) return; + if (typeof window === "undefined") return; + let debounceTimer: number | null = null; + const trigger = () => { + if (debounceTimer !== null) { + window.clearTimeout(debounceTimer); + } + debounceTimer = window.setTimeout(() => { + debounceTimer = null; + invoke<{ wk_webview_found: boolean; responder_granted: boolean }>("cua_refresh_a11y").catch( + (error: unknown) => { + console.warn("cua_refresh_a11y failed", error); + }, + ); + }, 50); + }; + const onVisibility = () => { + if (document.visibilityState === "visible") trigger(); + }; + const onFocus = () => trigger(); + document.addEventListener("visibilitychange", onVisibility); + window.addEventListener("focus", onFocus); + return () => { + if (debounceTimer !== null) window.clearTimeout(debounceTimer); + document.removeEventListener("visibilitychange", onVisibility); + window.removeEventListener("focus", onFocus); + }; + }, [settingsReady]); + // 动作总线（Rust `app:action`）中 App 拥有的动作：主题/打开设置/网关开关/ // 检查更新，以及「新建对话」时先收起设置覆盖层（会话侧由 ChatPage 处理）。 const closeSettingsRef = useRef(closeSettings); diff --git a/crates/agent-gui/src/agent-ui-adapters/settingsExtension.tsx b/crates/agent-gui/src/agent-ui-adapters/settingsExtension.tsx index 043ba2d4b..5c5c306ad 100644 --- a/crates/agent-gui/src/agent-ui-adapters/settingsExtension.tsx +++ b/crates/agent-gui/src/agent-ui-adapters/settingsExtension.tsx @@ -1,5 +1,6 @@ -import { Archive, Info, Keyboard } from "@liveagent/ui/components/IconSet"; +import { Archive, Hand, Info, Keyboard } from "@liveagent/ui/components/IconSet"; import type { SettingsSectionDefinition, UiExtensionSlots } from "@liveagent/ui/contracts/registry"; +import { CuaDriverSection } from "@liveagent/ui/pages/settings/CuaDriverSection"; import { isMacOsTauri, MacOsTitleBarSpacer } from "../components/MacOsTitleBarSpacer"; import { AboutSection } from "../pages/settings/AboutSection"; import { BackupSyncSection } from "../pages/settings/BackupSyncSection"; @@ -23,6 +24,18 @@ export function createSettingsExtension(props: SettingsPageProps): { mainLeading: , }, sections: [ + // CUA 接入引导。桌面端专属：探测 / 安装 / 授权都要 Tauri + // 后端命令，WebUI 下这些 invoke 会直接抛错，所以不在共享的 + // SettingsPage 里注册，而是走桌面 extension。 + { + id: "cua", + groupKey: "settings.groupConnectivity", + groupOrder: 40, + order: 30, + labelKey: "settings.navCua", + icon: , + render: () => , + }, { id: "shortcuts", groupKey: "settings.groupOther", diff --git a/crates/agent-gui/src/components/MacOsTitleBarSpacer.tsx b/crates/agent-gui/src/components/MacOsTitleBarSpacer.tsx index 373d1a549..80e9298ef 100644 --- a/crates/agent-gui/src/components/MacOsTitleBarSpacer.tsx +++ b/crates/agent-gui/src/components/MacOsTitleBarSpacer.tsx @@ -141,6 +141,8 @@ export function MacOsTitleBarToggle({ + + +
+
+
+ {installed ? ( + + ) : ( + + )} + {installed + ? probe?.version + ? t("settings.cuaDriver.detectedWithVersion").replace("{version}", probe.version) + : t("settings.cuaDriver.detected") + : t("settings.cuaDriver.notInstalledTitle")} +
+

+ {probe?.path ?? t("settings.cuaDriver.notInstalledDesc")} +

+
+ {installed ? null : ( + + )} +
+ + {/* 安装确认：把即将执行的命令原文摆出来。这条命令会从网络下载一段 + shell 脚本并直接执行，用户有权先看清楚再决定，也可以复制到自己 + 的终端里跑。绝不自动安装。 */} + {confirmingInstall && preview ? ( +
+

+ + {t("settings.cuaDriver.confirmTitle")} +

+

+ {t("settings.cuaDriver.confirmDesc").replace("{url}", preview.sourceUrl)} +

+ 
+              {preview.display}
+
+
+ + +
+
+ ) : null} + + {log.length > 0 ? ( + 
+            {log.join("\n")}
+
+ ) : null} + + {error ? ( +

+ + {error} +

+ ) : null} + + + {/* macOS TCC。授权归 CuaDriver.app 而非 LiveAgent —— 这是刻意选择的 + 代理模式，宿主不需要任何辅助功能 / 屏幕录制权限。 + + 渲染条件只看 probe 里的平台位：Windows / Linux 没有这道门槛，整 + 节不存在；macOS 上则整节随第一帧就位，只有里面的状态在等查询， + 不会出现「卡片过一会儿才长出来」的跳动。 */} + {showPermissions ? ( +
+ +
+
+
+ {!permissionsKnown ? ( + + ) : permissionsPending ? ( + + ) : ( + + )} + {!permissionsKnown + ? permissionsLoading + ? t("settings.cuaDriver.permissionsChecking") + : t("settings.cuaDriver.permissionsUnknown") + : permissionsPending + ? t("settings.cuaDriver.permissionsPending") + : t("settings.cuaDriver.statusGranted")} +
+

+ {!permissionsKnown || permissionsPending + ? t("settings.cuaDriver.permissionsDesc") + : t("settings.cuaDriver.permissionsGranted").replace( + "{bundleId}", + permissions?.attributedTo ?? "com.trycua.driver", + )} +

+
+ {permissionsKnown && !permissionsPending ? null : ( + + )} +
+
+ ) : null} + + {/* 参数全在这里改，MCP Hub 不再列出 cua-driver 这一条——两个入口都能 + 写同一份配置只会让人搞不清哪边说了算。 */} +
+ + +
+
+
{t("settings.cuaDriver.policyTitle")}
+

+ {t("settings.cuaDriver.policyDesc")} +

+
+ +
+ + {/* 自指闸门。默认关闭：模型操作宿主界面能点掉自己的审批弹窗、改写 + 这份设置、甚至关掉应用。 */} +
+
+
+ + {t("settings.cuaDriver.allowSelfTitle")} +
+

+ {t("settings.cuaDriver.allowSelfDesc")} +

+
+ setAllowSelfTargeting(!allowSelfTargeting)} + /> +
+ +
+ + setTimeoutDraft(event.target.value)} + onBlur={commitTimeout} + placeholder="60000" + className="w-40 font-mono text-[13px]" + /> +

+ {t("settings.cuaDriver.timeoutHint")} +

+
+
+ +
+ +

+ {t("settings.cuaDriver.description")}{" "} + + trycua/cua + + +

+

+ {t("settings.cuaDriver.policyNote")} +

+
+ + ); +} diff --git a/crates/agent-ui/src/pages/settings/SettingsShell.tsx b/crates/agent-ui/src/pages/settings/SettingsShell.tsx index 18a4ba598..96cefae2b 100644 --- a/crates/agent-ui/src/pages/settings/SettingsShell.tsx +++ b/crates/agent-ui/src/pages/settings/SettingsShell.tsx @@ -13,6 +13,38 @@ type SettingsShellProps = { hiddenSections?: readonly string[]; }; +// CUA-036 / CUA-043: the inner `.settings-section-enter` / +// `.settings-section-title-enter` keyframe animations are paused by +// WebKit/Chromium while the document is hidden (background launch, minimized +// window). CUA-033/034 only cover the outer overlay container; without this +// fallback the inner section body stays at the `from` state (opacity:0, +// translateY(14px) scale(0.985)) and the entire settings page reads as +// blank. Mirrors the LEAVE_FALLBACK_MS shape: subscribe to visibilitychange +// so the override clears the moment the window becomes visible again. +// +// CUA-043: `document.hidden` and `document.visibilityState` are supposed to +// stay in sync, but Tauri/WKWebView's background-launch state can leave +// them out of sync (hidden=true, visibilityState="visible"). Treat either +// signal as hidden so we still suspend the keyframe animation and surface +// the section body via the data-anim-suspended + inline style fallback. +function isDocumentHidden() { + if (typeof document === "undefined") return false; + return document.hidden || document.visibilityState === "hidden"; +} + +function useIsDocumentHidden() { + const [hidden, setHidden] = useState(isDocumentHidden); + useEffect(() => { + const sync = () => setHidden(isDocumentHidden()); + sync(); + document.addEventListener("visibilitychange", sync); + return () => { + document.removeEventListener("visibilitychange", sync); + }; + }, []); + return hidden; +} + function getSaveIndicator(state: SettingsSaveState, t: (key: string) => string) { switch (state.status) { case "saving": @@ -49,6 +81,7 @@ export function SettingsShell(props: SettingsShellProps) { const { t } = useLocale(); const [section, setSection] = useState(initialSection); const [navQuery, setNavQuery] = useState(""); + const isDocumentHidden = useIsDocumentHidden(); const hiddenSectionSet = useMemo(() => new Set(hiddenSections), [hiddenSections]); const sections = useMemo( () => @@ -153,9 +186,20 @@ export function SettingsShell(props: SettingsShellProps) { const active = definition.id === activeSection.id; return (