diff --git a/packages/extension/src/chat_bridge.ts b/packages/extension/src/chat_bridge.ts index 48f52283..f829e057 100644 --- a/packages/extension/src/chat_bridge.ts +++ b/packages/extension/src/chat_bridge.ts @@ -149,6 +149,30 @@ export function handleAmicodeBridgeMessage(msg: unknown, io: BridgeIo): boolean return true; } + // Clipboard image bridge: the outer webview's navigator.clipboard.read() + // requires user activation (a keystroke on that document), but the user typed + // inside the sandboxed iframe — so the async Clipboard API throws in newer + // Chromium/Electron. Route through the extension host instead: we spawn a + // platform-native tool to read the clipboard image as PNG and reply with a + // base64 data URL. Same visibility gate as text reads. + if (msg.kind === "clipboard-image-read") { + if (!io.visible()) return true; + const nonce = (msg as { nonce?: string }).nonce; + const tab = (msg as { tab?: string }).tab; + void readClipboardImageNative().then((result) => { + io.postToWebview({ + source: "amicode", + kind: "clipboard-image", + nonce, + tab, + dataUrl: result?.dataUrl ?? null, + mime: result?.mime ?? null, + filename: result?.filename ?? null, + }); + }); + return true; + } + // Save bridge (run-card PNG + session markdown export): downloads are dead // inside the framed app — the extension shows a save dialog and writes the // file. Bounded size, basename-only: the payload is untrusted. @@ -852,3 +876,235 @@ export function handleAmicodeBridgeMessage(msg: unknown, io: BridgeIo): boolean return false; } + +// --------------------------------------------------------------------------- +// Native clipboard image reader +// --------------------------------------------------------------------------- +// Reads the system clipboard image using platform-native tools. Returns a +// base64 data URL on success, null if the clipboard holds no image or the +// platform isn't supported. This sidesteps the Chromium user-activation +// requirement that blocks navigator.clipboard.read() in the webview. + +interface ClipboardImageResult { + dataUrl: string; + mime: string; + filename: string; +} + +async function readClipboardImageNative(): Promise { + const platform = process.platform; + try { + if (platform === "darwin") { + return await readClipboardImageMac(); + } else if (platform === "linux") { + return await readClipboardImageLinux(); + } else if (platform === "win32") { + return await readClipboardImageWindows(); + } + } catch { + // Any failure → no image + } + return null; +} + +function execPromise( + cmd: string, + args: string[], + options?: { encoding?: "buffer" | BufferEncoding; timeout?: number }, +): Promise { + return new Promise((resolve, reject) => { + import("node:child_process").then(({ execFile }) => { + execFile(cmd, args, { timeout: options?.timeout ?? 3000, maxBuffer: 20 * 1024 * 1024, encoding: "buffer" }, (err, stdout) => { + if (err) reject(err); + else resolve(stdout as unknown as Buffer); + }); + }); + }); +} + +async function readClipboardImageMac(): Promise { + // Priority 1: If the clipboard has a file URL pointing to an image, read the + // file directly. Finder file copies put the file icon as image data and the + // file path as public.file-url — we want the actual file contents. + try { + const fileUrlScript = [ + 'use framework "AppKit"', + "set pb to current application's NSPasteboard's generalPasteboard()", + 'set furlType to "public.file-url"', + "set urlStr to (pb's stringForType:furlType) as text", + "if urlStr is missing value or urlStr is \"\" then", + ' error "no file url"', + "end if", + "return urlStr", + ].join("\n"); + const urlOut = await execPromise("osascript", ["-l", "AppleScript", "-e", fileUrlScript]); + const fileUrl = urlOut.toString("utf8").trim(); + if (fileUrl) { + const { readFileSync, existsSync } = await import("node:fs"); + const { basename } = await import("node:path"); + const { fileURLToPath } = await import("node:url"); + // Resolve macOS .file ID URLs via osascript + let filePath: string; + if (fileUrl.startsWith("file:///.file/id=")) { + const resolveScript = `POSIX path of ("${fileUrl}" as POSIX file)`; + const resolved = await execPromise("osascript", ["-e", resolveScript]); + filePath = resolved.toString("utf8").trim(); + } else { + filePath = fileURLToPath(fileUrl); + } + const imageExts = /\.(png|jpe?g|gif|webp|avif|tiff?|bmp|heic|svg|ico)$/i; + if (filePath && imageExts.test(filePath) && existsSync(filePath)) { + const buf = readFileSync(filePath); + const ext = filePath.match(imageExts)![1].toLowerCase(); + const mime = ext === "jpg" ? "image/jpeg" : ext === "tif" ? "image/tiff" : `image/${ext}`; + return { + dataUrl: `data:${mime};base64,${buf.toString("base64")}`, + mime, + filename: basename(filePath), + }; + } + } + } catch { + // No file URL or not an image file — fall through to pasteboard image + } + + // Priority 2: Read clipboard image via osascript + NSPasteboard. Try PNG first + // (covers screenshots, browser copies), then fall back to TIFF (some apps only + // put TIFF on the pasteboard). The TIFF path writes to a temp file and uses + // `sips` to convert rather than fighting AppleScript-ObjC's syntax for + // NSBitmapImageRep. + const pngScript = [ + 'use framework "AppKit"', + "set pb to current application's NSPasteboard's generalPasteboard()", + "set pngType to current application's NSPasteboardTypePNG", + "set imgData to pb's dataForType:pngType", + "if imgData is missing value then", + ' error "no image"', + "end if", + "set rawBytes to (imgData's base64EncodedStringWithOptions:0) as text", + "return rawBytes", + ].join("\n"); + + try { + const stdout = await execPromise("osascript", ["-l", "AppleScript", "-e", pngScript]); + const base64 = stdout.toString("utf8").trim(); + if (base64) { + return { dataUrl: `data:image/png;base64,${base64}`, mime: "image/png", filename: "pasted-image.png" }; + } + } catch { + // PNG not on clipboard — try TIFF path + } + + // TIFF fallback: write raw TIFF to a temp file, convert with sips + const tiffScript = [ + 'use framework "AppKit"', + 'use framework "Foundation"', + "set pb to current application's NSPasteboard's generalPasteboard()", + "set tiffType to current application's NSPasteboardTypeTIFF", + "set imgData to pb's dataForType:tiffType", + "if imgData is missing value then", + ' error "no image"', + "end if", + "set rawBytes to (imgData's base64EncodedStringWithOptions:0) as text", + "return rawBytes", + ].join("\n"); + + try { + const { writeFileSync, readFileSync, unlinkSync } = await import("node:fs"); + const { tmpdir } = await import("node:os"); + const { join } = await import("node:path"); + + const stdout = await execPromise("osascript", ["-l", "AppleScript", "-e", tiffScript]); + const tiffBase64 = stdout.toString("utf8").trim(); + if (!tiffBase64) return null; + + // Write TIFF, convert to PNG with sips + const tiffPath = join(tmpdir(), `amicode-paste-${Date.now()}.tiff`); + const pngPath = tiffPath.replace(".tiff", ".png"); + writeFileSync(tiffPath, Buffer.from(tiffBase64, "base64")); + await execPromise("sips", ["-s", "format", "png", tiffPath, "--out", pngPath]); + const pngBuf = readFileSync(pngPath); + unlinkSync(tiffPath); + unlinkSync(pngPath); + + return { dataUrl: `data:image/png;base64,${pngBuf.toString("base64")}`, mime: "image/png", filename: "pasted-image.png" }; + } catch { + return null; + } +} + +async function readClipboardImageLinux(): Promise { + const { readFileSync, existsSync } = await import("node:fs"); + const { basename } = await import("node:path"); + + // Priority 1: file URI list (Nautilus/Dolphin file copy) + try { + const uriOut = await execPromise("xclip", ["-selection", "clipboard", "-t", "text/uri-list", "-o"]); + const uri = uriOut.toString("utf8").trim().split("\n")[0]; + if (uri && uri.startsWith("file://")) { + const filePath = decodeURIComponent(uri.replace("file://", "")); + const imageExts = /\.(png|jpe?g|gif|webp|avif|tiff?|bmp|heic|svg|ico)$/i; + if (imageExts.test(filePath) && existsSync(filePath)) { + const buf = readFileSync(filePath); + const ext = filePath.match(imageExts)![1].toLowerCase(); + const mime = ext === "jpg" ? "image/jpeg" : ext === "tif" ? "image/tiff" : `image/${ext}`; + return { dataUrl: `data:${mime};base64,${buf.toString("base64")}`, mime, filename: basename(filePath) }; + } + } + } catch { + // No URI list — fall through to image data + } + + // Priority 2: raw image data + const stdout = await execPromise("xclip", ["-selection", "clipboard", "-t", "image/png", "-o"]); + if (!stdout.length) return null; + const base64 = stdout.toString("base64"); + return { + dataUrl: `data:image/png;base64,${base64}`, + mime: "image/png", + filename: "pasted-image.png", + }; +} + +async function readClipboardImageWindows(): Promise { + const { readFileSync, existsSync } = await import("node:fs"); + const { basename } = await import("node:path"); + + // Priority 1: file drop list (Explorer file copy) + try { + const fileScript = ` +Add-Type -AssemblyName System.Windows.Forms +$files = [System.Windows.Forms.Clipboard]::GetFileDropList() +if ($files.Count -gt 0) { $files[0] } else { exit 1 } +`; + const fileOut = await execPromise("powershell", ["-NoProfile", "-Command", fileScript]); + const filePath = fileOut.toString("utf8").trim(); + const imageExts = /\.(png|jpe?g|gif|webp|avif|tiff?|bmp|heic|svg|ico)$/i; + if (filePath && imageExts.test(filePath) && existsSync(filePath)) { + const buf = readFileSync(filePath); + const ext = filePath.match(imageExts)![1].toLowerCase(); + const mime = ext === "jpg" ? "image/jpeg" : ext === "tif" ? "image/tiff" : `image/${ext}`; + return { dataUrl: `data:${mime};base64,${buf.toString("base64")}`, mime, filename: basename(filePath) }; + } + } catch { + // No file drop list — fall through to image data + } + + // Priority 2: clipboard image data + const script = ` +Add-Type -AssemblyName System.Windows.Forms +$img = [System.Windows.Forms.Clipboard]::GetImage() +if ($img -eq $null) { exit 1 } +$ms = New-Object System.IO.MemoryStream +$img.Save($ms, [System.Drawing.Imaging.ImageFormat]::Png) +[Convert]::ToBase64String($ms.ToArray()) +`; + const stdout = await execPromise("powershell", ["-NoProfile", "-Command", script]); + const base64 = stdout.toString("utf8").trim(); + if (!base64) return null; + return { + dataUrl: `data:image/png;base64,${base64}`, + mime: "image/png", + filename: "pasted-image.png", + }; +} diff --git a/packages/extension/src/chat_panel.ts b/packages/extension/src/chat_panel.ts index c4c41be1..a01512ab 100644 --- a/packages/extension/src/chat_panel.ts +++ b/packages/extension/src/chat_panel.ts @@ -370,12 +370,13 @@ export class ChatPanel { // Lane 1 — iframe → extension (commands): MUST come from the opencode // origin; the extension side additionally allowlists commands. if (e.origin === ${origin}) { - // Image paste: the outer webview CAN read clipboard images - // (navigator.clipboard.read is granted here); the sandboxed iframe - // can't, and vscode.env.clipboard is text-only. So we answer this one - // client-side instead of forwarding it to the host. + // Image paste: forward to the extension host which reads the clipboard + // image natively (osascript/xclip/powershell). The browser Clipboard API + // in the outer webview requires user activation from a keystroke on THIS + // document, but the keystroke fires inside the iframe — so the async API + // throws. The extension host has no such constraint. if (d && d.source === "amicode" && d.kind === "clipboard-image-request") { - replyClipboardImage(d.nonce); + vscode.postMessage({ source: "amicode", kind: "clipboard-image-read", nonce: d.nonce }); return; } if (d && d.source === "amicode" && (d.kind === "command" || d.kind === "clipboard-request" || d.kind === "clipboard-write" || d.kind === "open-external" || d.kind === "open-file" || d.kind === "save-file" || d.kind === "set-default-model" || d.kind === "bug-filed" || d.kind === "bug-report-closed" || d.kind === "bug-report-poke" || d.kind === "dev-tools-update" || d.kind === "dev-tools-rebuild" || d.kind === "data-storage-query" || d.kind === "data-storage-update" || d.kind === "redo-onboarding" || d.kind === "device:refresh" || d.kind === "connections-credential" || d.kind === "connections-disconnect" || d.kind === "connections-revalidate" || d.kind === "connections-auth" || d.kind === "connections-choose-project" || d.kind === "connections-add-custom" || d.kind === "connections-remove" || d.kind === "app-ready")) { @@ -392,44 +393,6 @@ export class ChatPanel { if (f && f.contentWindow) f.contentWindow.postMessage(d, ${origin}); } }); - - // Answer a clipboard-image-request from the framed app: read the first - // image/* item off the OS clipboard (client-side; clipboard-read is - // granted to this webview) and post it into the frame as a data URL. On - // any failure we still reply with dataUrl:null so the app falls back to - // text paste rather than hanging on its timeout. - function blobToDataUrl(blob) { - return new Promise(function (res) { - var r = new FileReader(); - r.onload = function () { res(typeof r.result === "string" ? r.result : null); }; - r.onerror = function () { res(null); }; - r.readAsDataURL(blob); - }); - } - async function replyClipboardImage(nonce) { - var payload = { source: "amicode", kind: "clipboard-image", nonce: nonce, dataUrl: null, mime: null, filename: null }; - try { - if (navigator.clipboard && navigator.clipboard.read) { - var items = await navigator.clipboard.read(); - for (var i = 0; i < items.length && !payload.dataUrl; i++) { - var types = items[i].types || []; - for (var j = 0; j < types.length; j++) { - if (types[j].indexOf("image/") !== 0) continue; - var blob = await items[i].getType(types[j]); - var dataUrl = await blobToDataUrl(blob); - if (dataUrl) { - payload.dataUrl = dataUrl; - payload.mime = types[j]; - payload.filename = "pasted-image." + (types[j].split("/")[1] || "png"); - } - break; - } - } - } - } catch (e) { /* reply with dataUrl:null → app falls back to text paste */ } - var f = document.querySelector("iframe"); - if (f && f.contentWindow) f.contentWindow.postMessage(payload, ${origin}); - } })(); @@ -564,7 +527,7 @@ export class ChatPanel { return; } if (d && d.source === "amicode" && d.kind === "clipboard-image-request") { - replyClipboardImage(d.nonce); + vscode.postMessage({ source: "amicode", kind: "clipboard-image-read", nonce: d.nonce }); return; } if (d && d.source === "amicode" && (d.kind === "command" || d.kind === "clipboard-request" || d.kind === "clipboard-write" || d.kind === "open-external" || d.kind === "open-file" || d.kind === "save-file" || d.kind === "set-default-model" || d.kind === "bug-filed" || d.kind === "bug-report-closed" || d.kind === "bug-report-poke" || d.kind === "dev-tools-update" || d.kind === "dev-tools-rebuild" || d.kind === "data-storage-query" || d.kind === "data-storage-update" || d.kind === "redo-onboarding" || d.kind === "device:refresh" || d.kind === "connections-credential" || d.kind === "connections-disconnect" || d.kind === "connections-revalidate" || d.kind === "connections-auth" || d.kind === "connections-choose-project" || d.kind === "connections-add-custom" || d.kind === "connections-remove" || d.kind === "app-ready")) { @@ -577,38 +540,6 @@ export class ChatPanel { if (f && f.contentWindow) f.contentWindow.postMessage(d, origin); } }); - function blobToDataUrl(blob) { - return new Promise(function (res) { - var r = new FileReader(); - r.onload = function () { res(typeof r.result === "string" ? r.result : null); }; - r.onerror = function () { res(null); }; - r.readAsDataURL(blob); - }); - } - async function replyClipboardImage(nonce) { - var payload = { source: "amicode", kind: "clipboard-image", nonce: nonce, dataUrl: null, mime: null, filename: null }; - try { - if (navigator.clipboard && navigator.clipboard.read) { - var items = await navigator.clipboard.read(); - for (var i = 0; i < items.length && !payload.dataUrl; i++) { - var types = items[i].types || []; - for (var j = 0; j < types.length; j++) { - if (types[j].indexOf("image/") !== 0) continue; - var blob = await items[i].getType(types[j]); - var dataUrl = await blobToDataUrl(blob); - if (dataUrl) { - payload.dataUrl = dataUrl; - payload.mime = types[j]; - payload.filename = "pasted-image." + (types[j].split("/")[1] || "png"); - } - break; - } - } - } - } catch (e) { /* reply with dataUrl:null */ } - var f = document.querySelector("iframe"); - if (f && f.contentWindow) f.contentWindow.postMessage(payload, origin); - } })(); diff --git a/packages/extension/src/deck/shell.ts b/packages/extension/src/deck/shell.ts index f0f79f69..83eb92ad 100644 --- a/packages/extension/src/deck/shell.ts +++ b/packages/extension/src/deck/shell.ts @@ -461,34 +461,13 @@ window.addEventListener("message", (e) => { return; } - // clipboard-image-request is answered shell-side (the shell webview has - // clipboard-read; the sandboxed iframe doesn't) — reply to the ASKING pane. + // clipboard-image-request: forward to the extension host which reads the + // clipboard image natively. The shell webview's navigator.clipboard.read() + // requires user activation from a keystroke on THIS document, but the + // keystroke fires inside the sandboxed iframe — so the async Clipboard API + // throws in newer Chromium. The extension host has no such constraint. if (d.kind === "clipboard-image-request") { - void (async () => { - const payload = { source: "amicode", kind: "clipboard-image", nonce: d.nonce, dataUrl: null as string | null, mime: null as string | null, filename: null as string | null }; - try { - const items = await navigator.clipboard.read(); - for (const item of items) { - const type = item.types.find((t) => t.startsWith("image/")); - if (!type) continue; - const blob = await item.getType(type); - payload.dataUrl = await new Promise((res) => { - const r = new FileReader(); - r.onload = () => res(typeof r.result === "string" ? r.result : null); - r.onerror = () => res(null); - r.readAsDataURL(blob); - }); - if (payload.dataUrl) { - payload.mime = type; - payload.filename = `pasted-image.${type.split("/")[1] ?? "png"}`; - } - break; - } - } catch { - /* dataUrl:null → app falls back to text paste */ - } - (e.source as Window | null)?.postMessage(payload, boot.origin); - })(); + vscode.postMessage({ source: "amicode", kind: "clipboard-image-read", nonce: d.nonce, tab: tabId }); return; } diff --git a/packages/extension/test/chat_bridge.test.ts b/packages/extension/test/chat_bridge.test.ts index 271dba0f..00672f37 100644 --- a/packages/extension/test/chat_bridge.test.ts +++ b/packages/extension/test/chat_bridge.test.ts @@ -232,3 +232,44 @@ describe("extractReportBugModel — the command's optional model payload (amicod expect(extractReportBugModel({ model: { providerID: "p" } })).toBeUndefined(); }); }); + +describe("amicode bridge — clipboard-image-read", () => { + it("reads a clipboard image via native tools and replies with a data URL", async () => { + const host = io(); + const handled = handleAmicodeBridgeMessage( + { source: "amicode", kind: "clipboard-image-read", nonce: "img-1", tab: "pane-2" }, + host, + ); + expect(handled).toBe(true); + // The handler is async (spawns a native process); wait for it to complete. + // Multiple ticks needed: dynamic import + process spawn + result processing. + for (let i = 0; i < 60 && host.posted.length === 0; i++) { + await new Promise((r) => setTimeout(r, 50)); + } + expect(host.posted).toHaveLength(1); + const reply = host.posted[0] as Record; + expect(reply.source).toBe("amicode"); + expect(reply.kind).toBe("clipboard-image"); + expect(reply.nonce).toBe("img-1"); + expect(reply.tab).toBe("pane-2"); + // dataUrl is either null (no image) or a valid data URL with image content + if (reply.dataUrl !== null) { + expect(typeof reply.dataUrl).toBe("string"); + expect((reply.dataUrl as string).startsWith("data:image/")).toBe(true); + expect(typeof reply.mime).toBe("string"); + expect((reply.mime as string).startsWith("image/")).toBe(true); + expect(typeof reply.filename).toBe("string"); + } + }); + + it("a hidden panel never answers clipboard-image-read", async () => { + const host = io(false); + const handled = handleAmicodeBridgeMessage( + { source: "amicode", kind: "clipboard-image-read", nonce: "img-2" }, + host, + ); + expect(handled).toBe(true); + await flush(); + expect(host.posted).toHaveLength(0); + }); +}); diff --git a/packages/extension/test/chat_panel.test.ts b/packages/extension/test/chat_panel.test.ts index 91c23b5a..eeb4f149 100644 --- a/packages/extension/test/chat_panel.test.ts +++ b/packages/extension/test/chat_panel.test.ts @@ -328,3 +328,27 @@ describe("ChatPanel.adopt — transforms an existing panel into the chat singlet expect(readyFired).toHaveLength(1); }); }); + +describe("ChatPanel — clipboard-image-request routes through extension host", () => { + let restore: (() => void) | undefined; + let created: CapturedPanel[] = []; + afterEach(() => { + for (const p of created) p.dispose(); + restore?.(); + restore = undefined; + created = []; + }); + + it("the relay forwards clipboard-image-request to the extension host (not navigator.clipboard.read)", () => { + const cap = capturePanel(); + restore = cap.restore; + created = cap.created; + ChatPanel.openOrReveal(fakeCtx(), new URL("http://127.0.0.1:43117/")); + const html = cap.created[0].webview.html; + // The outer webview relay should forward clipboard-image-request to the + // extension via vscode.postMessage — NOT handle it with navigator.clipboard.read + expect(html).toContain('"clipboard-image-read"'); + // The old client-side replyClipboardImage with navigator.clipboard.read should be gone + expect(html).not.toContain("navigator.clipboard.read"); + }); +});