feat(cua): CUA Driver 支持 — 通过 MCP tool 操作 macOS 桌面

.gitignore

@@ -53,3 +53,5 @@ workspace.zip
 /.trellis
 /AGENTS.md
 /CLAUDE.md
+/issue_board.md
+/.github/automation/

crates/agent-gateway/web/src/pages/settings/types.ts

@@ -15,7 +15,8 @@ export type SectionId =
   | "hooks"
   | "cron"
   | "devices"
-  | "remote";
+  | "remote"
+  | "cua";
 
 export type SettingsPageProps = {
   settings: AppSettings;

crates/agent-gui/src-tauri/Cargo.toml

@@ -78,8 +78,9 @@ encoding_rs = "0.8.35"
 chardetng = "0.1.17"
 
 [target.'cfg(target_os = "macos")'.dependencies]
-objc2-app-kit = { version = "0.3", default-features = false, features = ["std", "NSButton", "NSControl", "NSView", "NSWindow"] }
+objc2-app-kit = { version = "0.3", default-features = false, features = ["std", "NSApplication", "NSButton", "NSControl", "NSView", "NSWindow"] }
 objc2-av-foundation = { version = "0.3", default-features = false, features = ["std", "block2", "AVCaptureDevice", "AVMediaFormat"] }
+objc2-foundation = { version = "0.3", default-features = false, features = ["std", "NSArray", "NSString"] }
 objc2 = { version = "0.6", default-features = false, features = ["std"] }
 block2 = "0.6"
 

crates/agent-gui/src-tauri/src/commands/app/app.rs

@@ -39,14 +39,22 @@ pub fn app_frontend_ready(
 ) -> Result<(), String> {
     ready_state.0.store(true, Ordering::SeqCst);
     if window.is_visible().unwrap_or(false) {
+        crate::force_activate_main_window(&window);
         return Ok(());
     }
     window
         .show()
         .map_err(|error| format!("failed to show frontend-ready window: {error}"))?;
     window
         .set_focus()
-        .map_err(|error| format!("failed to focus frontend-ready window: {error}"))
+        .map_err(|error| format!("failed to focus frontend-ready window: {error}"))?;
+    // CUA-007: macOS-only NSApp.activate + makeKeyAndOrderFront ensures the
+    // window reaches the frontmost-ordinary window state so cua-driver's
+    // `bring_to_front` and foreground delivery actually land on the
+    // WebView. `set_focus` alone can leave the window below the
+    // loginwindow or another system surface in a dev sandbox.
+    crate::force_activate_main_window(&window);
+    Ok(())
 }
 
 /// 前端主动切换置顶（置顶指示器点击取消）；状态变更仍经

crates/agent-gui/src-tauri/src/commands/app/ax.rs

@@ -0,0 +1,30 @@
+//! 无障碍（AX）辅助命令。
+//!
+//! 这两个命令服务于「外部自动化工具驱动 LiveAgent 自身窗口」的场景
+//! ——典型是用 `cua-driver` 跑端到端测试时，需要主窗口真的在前台、且
+//! NSWindow / WKWebView 的 AX 注解已重新广播，否则拿到的 AX 树是空的。
+//!
+//! 它们与「LiveAgent 能操作用户电脑」这个功能无关：那条链路已改为把
+//! `cua-driver mcp` 当作一个普通 MCP server 接入 MCP Hub，不再有专属
+//! 的 Tauri 命令层。
+//!
+//! 真正的实现在 `crate::cua_window_ready` / `crate::cua_refresh_a11y`
+//! （`lib.rs`，因为要触碰 `force_activate_main_window` 与平台相关的
+//! NSWindow 细节）；这里只是命令桥。
+
+/// 把主窗口重新推到最前并等待 `is_focused()` 落住，让后续的 AX 查询
+/// 能找到 WebView。
+#[tauri::command(rename_all = "camelCase")]
+pub async fn cua_window_ready(
+    window: tauri::WebviewWindow,
+) -> Result<crate::CuaWindowReadyResponse, String> {
+    Ok(crate::cua_window_ready(window).await)
+}
+
+/// 在路由切换、overlay 打开 / 关闭、外部工具主动唤起等场景重新触发
+/// NSWindow / WKWebView 的 AX 注解，并回弹广播一次
+/// `UIElementCreatedNotification`。
+#[tauri::command(rename_all = "camelCase")]
+pub fn cua_refresh_a11y(window: tauri::WebviewWindow) -> crate::CuaRefreshA11yResponse {
+    crate::cua_refresh_a11y(&window)
+}

crates/agent-gui/src-tauri/src/commands/app/mod.rs

@@ -1,4 +1,5 @@
 pub mod app;
+pub mod ax;
 pub mod system;
 pub mod tray;
 pub mod update;

crates/agent-gui/src-tauri/src/commands/config/settings/mod.rs

@@ -48,6 +48,11 @@ const SYSTEM_MISSING_WORKSPACE_PROJECT_PATHS_KEY: &str = "missingWorkspaceProjec
 const SYSTEM_ARCHIVED_WORKSPACE_PROJECT_PATHS_KEY: &str = "archivedWorkspaceProjectPaths";
 const SYSTEM_WORKSPACE_RESOURCE_SETTINGS_KEY: &str = "workspaceResourceSettings";
 const SYSTEM_SYSTEM_PROXY_KEY: &str = "systemProxy";
+// CUA 自指开关。默认 false —— cua-driver 的工具默认看不到、也点不到
+// LiveAgent 自己的窗口：让模型操作宿主界面等于让它能点掉自己的审批弹窗、
+// 改自己的设置、关掉自己。置 true 才解除（用 LiveAgent 自动化测试
+// LiveAgent 这类场景需要）。
+const SYSTEM_CUA_ALLOW_SELF_TARGETING_KEY: &str = "cuaAllowSelfTargeting";
 const DEFAULT_WORKSPACE_PROJECT_ID: &str = "default-project";
 const DEFAULT_WORKSPACE_PROJECT_NAME: &str = "Default Project";
 pub(crate) const PROVIDER_API_KEY_UPDATES_FIELD: &str = "providerApiKeyUpdates";

crates/agent-gui/src-tauri/src/commands/config/settings/system.rs

@@ -463,6 +463,17 @@ fn system_value_with_defaults(raw: Option<Value>, default_workdir: &str) -> Valu
         SYSTEM_COMMAND_SAFETY_MODE_KEY.to_string(),
         normalize_command_safety_mode_value(system.get(SYSTEM_COMMAND_SAFETY_MODE_KEY)),
     );
+    // 缺省 false：安全侧的开关，任何非 true 的值（缺失、null、字符串）
+    // 都收敛成「不允许自指」。
+    system.insert(
+        SYSTEM_CUA_ALLOW_SELF_TARGETING_KEY.to_string(),
+        Value::Bool(
+            system
+                .get(SYSTEM_CUA_ALLOW_SELF_TARGETING_KEY)
+                .and_then(Value::as_bool)
+                .unwrap_or(false),
+        ),
+    );
 
     Value::Object(system)
 }
@@ -572,6 +583,7 @@ fn save_system_with_default_workdir(
         SYSTEM_ARCHIVED_WORKSPACE_PROJECT_PATHS_KEY,
         SYSTEM_WORKSPACE_RESOURCE_SETTINGS_KEY,
         SYSTEM_SYSTEM_PROXY_KEY,
+        SYSTEM_CUA_ALLOW_SELF_TARGETING_KEY,
     ] {
         let value = system.get(key).cloned().unwrap_or(Value::Null);
         tx.execute(

crates/agent-gui/src-tauri/src/commands/config/settings/tests.rs

@@ -1185,13 +1185,14 @@ mod tests {
         };
         let loaded = load_system(&conn).expect("load system");
 
-        assert_eq!(row_count, 12);
+        assert_eq!(row_count, 13);
         assert_eq!(
             keys,
             vec![
                 SYSTEM_ACTIVE_WORKSPACE_PROJECT_ID_KEY.to_string(),
                 SYSTEM_ARCHIVED_WORKSPACE_PROJECT_PATHS_KEY.to_string(),
                 SYSTEM_COMMAND_SAFETY_MODE_KEY.to_string(),
+                SYSTEM_CUA_ALLOW_SELF_TARGETING_KEY.to_string(),
                 SYSTEM_EXECUTION_MODE_KEY.to_string(),
                 SYSTEM_HIDDEN_WORKSPACE_PROJECT_PATHS_KEY.to_string(),
                 SYSTEM_MISSING_WORKSPACE_PROJECT_PATHS_KEY.to_string(),
@@ -1207,6 +1208,7 @@ mod tests {
             loaded,
             Some(json!({
                 "activeWorkspaceProjectId": DEFAULT_WORKSPACE_PROJECT_ID,
+                "cuaAllowSelfTargeting": false,
                 "executionMode": "tools",
                 "hiddenWorkspaceProjectPaths": [],
                 "missingWorkspaceProjectPaths": [],
@@ -1506,6 +1508,7 @@ mod tests {
             loaded,
             Some(json!({
                 "activeWorkspaceProjectId": DEFAULT_WORKSPACE_PROJECT_ID,
+                "cuaAllowSelfTargeting": false,
                 "executionMode": "tools",
                 "hiddenWorkspaceProjectPaths": [],
                 "missingWorkspaceProjectPaths": [],
@@ -1560,6 +1563,7 @@ mod tests {
             loaded,
             Some(json!({
                 "activeWorkspaceProjectId": DEFAULT_WORKSPACE_PROJECT_ID,
+                "cuaAllowSelfTargeting": false,
                 "executionMode": "tools",
                 "hiddenWorkspaceProjectPaths": [],
                 "missingWorkspaceProjectPaths": [],
@@ -1631,6 +1635,7 @@ mod tests {
             loaded,
             json!({
                 "activeWorkspaceProjectId": DEFAULT_WORKSPACE_PROJECT_ID,
+                "cuaAllowSelfTargeting": false,
                 "executionMode": "tools",
                 "hiddenWorkspaceProjectPaths": [],
                 "missingWorkspaceProjectPaths": [],

crates/agent-gui/src-tauri/src/commands/integration/cua_driver.rs

@@ -0,0 +1,69 @@
+//! `cua-driver` 引导命令桥。
+//!
+//! 只覆盖「装没装 / 装一下 / 授权了没」这段引导；真正的计算机操作能力
+//! 走通用 MCP 链路（`cua-driver mcp` 是一个普通 stdio MCP server，由
+//! `commands/integration/mcp.rs` 驱动，工具从 `tools/list` 发现）。
+//!
+//! 放在 `integration/` 而不是自成一域，正是因为它属于 MCP 接入的一部分。
+
+use tauri::AppHandle;
+
+use crate::services::cua_driver::{
+    self, CuaDriverPermissions, CuaDriverProbe, InstallCommandPreview, SelfIdentity,
+};
+
+/// 探测二进制位置、版本与 MCP 调用方式。未安装返回 `installed: false`，
+/// 不是错误。只读，无副作用。
+#[tauri::command(rename_all = "camelCase")]
+pub async fn cua_driver_probe() -> Result<CuaDriverProbe, String> {
+    tauri::async_runtime::spawn_blocking(cua_driver::probe)
+        .await
+        .map_err(|error| format!("cua_driver_probe join failed: {error}"))
+}
+
+/// 返回将要执行的安装命令**全文**，不执行。
+///
+/// UI 必须先把 `display` 展示给用户并取得显式确认，才允许调
+/// `cua_driver_install`：那条命令会从网络下载一段 shell 脚本并直接
+/// 执行，用户有权先看清楚。
+#[tauri::command(rename_all = "camelCase")]
+pub fn cua_driver_install_command() -> InstallCommandPreview {
+    cua_driver::install_command_preview()
+}
+
+/// 执行官方安装脚本，进度经 `cua_driver_install_progress` 事件流式回传。
+///
+/// 前置条件由 UI 保证：用户已看过 `cua_driver_install_command` 的输出
+/// 并确认。这里不做二次弹窗——后端没有 UI 上下文，弹不出可信的确认。
+#[tauri::command(rename_all = "camelCase")]
+pub async fn cua_driver_install(app: AppHandle) -> Result<CuaDriverProbe, String> {
+    tauri::async_runtime::spawn_blocking(move || cua_driver::install(&app))
+        .await
+        .map_err(|error| format!("cua_driver_install join failed: {error}"))?
+}
+
+/// 读取 macOS 的 Accessibility / Screen Recording 授权状态。只读，
+/// 不触发系统授权弹窗。非 macOS 返回 `supported: false`。
+#[tauri::command(rename_all = "camelCase")]
+pub async fn cua_driver_permissions_status() -> Result<CuaDriverPermissions, String> {
+    tauri::async_runtime::spawn_blocking(cua_driver::permissions_status)
+        .await
+        .map_err(|error| format!("cua_driver_permissions_status join failed: {error}"))
+}
+
+/// 触发上游的授权引导：拉起 CuaDriver.app 并请求两项权限。会弹系统
+/// 对话框——授权归属 CuaDriver.app（而非 LiveAgent），这是上游推荐的
+/// 唯一正确路径。
+#[tauri::command(rename_all = "camelCase")]
+pub async fn cua_driver_permissions_grant() -> Result<CuaDriverPermissions, String> {
+    tauri::async_runtime::spawn_blocking(cua_driver::permissions_grant)
+        .await
+        .map_err(|error| format!("cua_driver_permissions_grant join failed: {error}"))?
+}
+
+/// LiveAgent 自身的进程身份。前端用它把 cua-driver 的窗口 / 应用列表里
+/// 属于宿主的记录裁掉，并拦下直接以宿主 pid 为目标的调用。只读。
+#[tauri::command(rename_all = "camelCase")]
+pub fn cua_driver_self_identity() -> SelfIdentity {
+    cua_driver::self_identity()
+}

crates/agent-gui/src-tauri/src/commands/integration/mod.rs

@@ -1,3 +1,4 @@
+pub mod cua_driver;
 pub mod gateway;
 pub mod mcp;
 pub mod memory;

crates/agent-gui/src-tauri/src/commands/mod.rs

@@ -14,6 +14,7 @@ pub mod runtime_commands;
 pub mod workspace_commands;
 
 pub use app_commands::app;
+pub use app_commands::ax;
 pub use app_commands::system;
 pub use app_commands::tray;
 pub use app_commands::update;
@@ -27,6 +28,7 @@ pub use history_commands::chat_history;
 pub use history_commands::history_db;
 pub use history_commands::subagent_store;
 
+pub use integration_commands::cua_driver;
 pub use integration_commands::gateway;
 pub use integration_commands::mcp;
 pub use integration_commands::memory;