From 5cd6b5d07b5865e35893f33aa9535c827504f466 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=E7=A8=8B=E5=BA=8F=E5=91=98=E9=98=BF=E6=B1=9F=28Relakkes?= =?UTF-8?q?=29?= Date: Mon, 20 Apr 2026 12:12:02 +0800 Subject: [PATCH] Prevent desktop Computer Use from stalling on missing approvals and unstable text input Desktop sessions were missing a visible request_access approval path and could mis-detect their own app window as an unapproved frontmost target, which caused Computer Use clicks to fail even after opening the intended app. On macOS, text entry was also split across inconsistent clipboard and keystroke paths, making Electron inputs unreliable for Chinese and short strings. This change adds a desktop approval bridge over the existing session websocket, renders a dedicated desktop approval modal, threads the real desktop bundle id into the Computer Use executor, and switches macOS clipboard typing onto the native pasteboard plus system paste shortcut path. It also makes tool error results expandable in the desktop chat UI so frontmost-gate failures are fully visible during debugging. Constraint: Desktop sessions run the CLI over the SDK websocket path, so Ink tool JSX dialogs are not visible there Constraint: macOS IME and Electron text inputs are unreliable with pyautogui.write and generic hotkey synthesis Rejected: Reuse CLI setToolJSX dialogs in desktop mode | no transport for mid-call Ink UI over the SDK bridge Rejected: Keep shell pbcopy/pbpaste for clipboard typing | inconsistent with NSPasteboard path and less reliable for Chinese text Confidence: medium Scope-risk: moderate Reversibility: clean Directive: Keep desktop Computer Use approvals and macOS text-entry behavior on a single bridge/path; avoid reintroducing separate CLI-only and desktop-only codepaths for the same action Tested: python3 -m unittest runtime/test_helpers.py Tested: bun test src/utils/computerUse/permissions.test.ts src/server/__tests__/conversation-service.test.ts Tested: cd desktop && bun run test ComputerUsePermissionModal chatStore Tested: cd desktop && bun run test chatBlocks Tested: cd desktop && bun run lint Not-tested: End-to-end manual Computer Use interaction against a live Electron target app on macOS --- desktop/src/__tests__/pages.test.tsx | 2 + .../chat/ComputerUsePermissionModal.test.tsx | 174 ++++++++++ .../chat/ComputerUsePermissionModal.tsx | 311 ++++++++++++++++++ .../src/components/chat/MessageList.test.tsx | 1 + desktop/src/components/chat/ToolCallBlock.tsx | 5 +- .../src/components/chat/chatBlocks.test.tsx | 24 +- desktop/src/i18n/locales/en.ts | 22 ++ desktop/src/i18n/locales/zh.ts | 22 ++ desktop/src/pages/ActiveSession.test.tsx | 2 + desktop/src/pages/ActiveSession.tsx | 9 + desktop/src/stores/chatStore.test.ts | 131 ++++++++ desktop/src/stores/chatStore.ts | 70 +++- desktop/src/stores/teamStore.ts | 1 + desktop/src/types/chat.ts | 60 ++++ runtime/mac_helper.py | 71 +++- runtime/test_helpers.py | 40 ++- runtime/win_helper.py | 8 + .../__tests__/conversation-service.test.ts | 13 + src/server/api/computer-use.ts | 30 ++ .../services/computerUseApprovalService.ts | 73 ++++ src/server/services/conversationService.ts | 23 +- src/server/ws/events.ts | 60 ++++ src/server/ws/handler.ts | 22 ++ src/utils/computerUse/executor.ts | 19 +- src/utils/computerUse/wrapper.tsx | 46 ++- src/vendor/computer-use-mcp/toolCalls.ts | 4 +- 26 files changed, 1217 insertions(+), 26 deletions(-) create mode 100644 desktop/src/components/chat/ComputerUsePermissionModal.test.tsx create mode 100644 desktop/src/components/chat/ComputerUsePermissionModal.tsx create mode 100644 src/server/services/computerUseApprovalService.ts diff --git a/desktop/src/__tests__/pages.test.tsx b/desktop/src/__tests__/pages.test.tsx index b38dd120..3f93f3b5 100644 --- a/desktop/src/__tests__/pages.test.tsx +++ b/desktop/src/__tests__/pages.test.tsx @@ -49,6 +49,7 @@ describe('Content-only pages render without errors', () => { activeToolName: null, activeThinkingId: null, pendingPermission: null, + pendingComputerUsePermission: null, tokenUsage: { input_tokens: 0, output_tokens: 0 }, elapsedSeconds: 0, statusVerb: '', @@ -83,6 +84,7 @@ describe('Content-only pages render without errors', () => { activeToolName: null, activeThinkingId: null, pendingPermission: null, + pendingComputerUsePermission: null, tokenUsage: { input_tokens: 0, output_tokens: 0 }, elapsedSeconds: 0, statusVerb: '', diff --git a/desktop/src/components/chat/ComputerUsePermissionModal.test.tsx b/desktop/src/components/chat/ComputerUsePermissionModal.test.tsx new file mode 100644 index 00000000..c2bfce00 --- /dev/null +++ b/desktop/src/components/chat/ComputerUsePermissionModal.test.tsx @@ -0,0 +1,174 @@ +import { beforeEach, describe, expect, it, vi } from 'vitest' +import { act, fireEvent, render, screen } from '@testing-library/react' + +const { sendMock, openSettingsMock } = vi.hoisted(() => ({ + sendMock: vi.fn(), + openSettingsMock: vi.fn(async () => ({ ok: true })), +})) + +vi.mock('../../api/websocket', () => ({ + wsManager: { + connect: vi.fn(), + disconnect: vi.fn(), + onMessage: vi.fn(() => () => {}), + clearHandlers: vi.fn(), + send: sendMock, + }, +})) + +vi.mock('../../api/sessions', () => ({ + sessionsApi: { + getMessages: vi.fn(async () => ({ messages: [] })), + getSlashCommands: vi.fn(async () => ({ commands: [] })), + }, +})) + +vi.mock('../../stores/teamStore', () => ({ + useTeamStore: { + getState: () => ({ + getMemberBySessionId: vi.fn(() => null), + sendMessageToMember: vi.fn(async () => {}), + handleTeamCreated: vi.fn(), + handleTeamUpdate: vi.fn(), + handleTeamDeleted: vi.fn(), + }), + }, +})) + +vi.mock('../../stores/tabStore', () => ({ + useTabStore: { + getState: () => ({ + updateTabStatus: vi.fn(), + updateTabTitle: vi.fn(), + }), + }, +})) + +vi.mock('../../stores/sessionStore', () => ({ + useSessionStore: { + getState: () => ({ + updateSessionTitle: vi.fn(), + }), + }, +})) + +vi.mock('../../stores/cliTaskStore', () => ({ + useCLITaskStore: { + getState: () => ({ + fetchSessionTasks: vi.fn(), + tasks: [], + clearTasks: vi.fn(), + setTasksFromTodos: vi.fn(), + markCompletedAndDismissed: vi.fn(), + refreshTasks: vi.fn(), + }), + }, +})) + +vi.mock('../../api/computerUse', () => ({ + computerUseApi: { + openSettings: openSettingsMock, + }, +})) + +import { useChatStore } from '../../stores/chatStore' +import { ComputerUsePermissionModal } from './ComputerUsePermissionModal' + +describe('ComputerUsePermissionModal', () => { + beforeEach(() => { + sendMock.mockReset() + openSettingsMock.mockReset() + useChatStore.setState({ sessions: {} }) + }) + + it('returns a full approval payload for resolved apps and requested flags', () => { + render( + , + ) + + fireEvent.click(screen.getByRole('button', { name: 'Allow for session' })) + + expect(sendMock).toHaveBeenCalledTimes(1) + expect(sendMock).toHaveBeenCalledWith('session-1', { + type: 'computer_use_permission_response', + requestId: 'cu-1', + response: { + granted: [ + expect.objectContaining({ + bundleId: 'com.apple.finder', + displayName: 'Finder', + tier: 'full', + }), + ], + denied: [ + { + bundleId: 'Missing App', + reason: 'not_installed', + }, + ], + flags: { + clipboardRead: true, + clipboardWrite: false, + systemKeyCombos: true, + }, + userConsented: true, + }, + }) + }) + + it('opens System Settings from the macOS permission panel', async () => { + render( + , + ) + + await act(async () => { + fireEvent.click(screen.getByRole('button', { name: 'Open Accessibility' })) + }) + + expect(openSettingsMock).toHaveBeenCalledWith('Privacy_Accessibility') + }) +}) diff --git a/desktop/src/components/chat/ComputerUsePermissionModal.tsx b/desktop/src/components/chat/ComputerUsePermissionModal.tsx new file mode 100644 index 00000000..b6a9c9e9 --- /dev/null +++ b/desktop/src/components/chat/ComputerUsePermissionModal.tsx @@ -0,0 +1,311 @@ +import { useMemo, useState } from 'react' +import { useTranslation } from '../../i18n' +import { computerUseApi } from '../../api/computerUse' +import { useChatStore } from '../../stores/chatStore' +import type { + ComputerUsePermissionRequest, + ComputerUsePermissionResponse, +} from '../../types/chat' +import { Button } from '../shared/Button' +import { Modal } from '../shared/Modal' + +type Props = { + sessionId: string + request: ComputerUsePermissionRequest | null +} + +const DEFAULT_GRANT_FLAGS = { + clipboardRead: false, + clipboardWrite: false, + systemKeyCombos: false, +} as const + +function denyAllResponse(): ComputerUsePermissionResponse { + return { + granted: [], + denied: [], + flags: { ...DEFAULT_GRANT_FLAGS }, + userConsented: false, + } +} + +function buildAllowResponse( + request: ComputerUsePermissionRequest, +): ComputerUsePermissionResponse { + const now = Date.now() + const granted = request.apps.flatMap((app) => { + if (!app.resolved || app.alreadyGranted) return [] + return [{ + bundleId: app.resolved.bundleId, + displayName: app.resolved.displayName, + grantedAt: now, + tier: app.proposedTier, + }] + }) + + const denied = request.apps.flatMap((app) => { + if (app.resolved) return [] + return [{ + bundleId: app.requestedName, + reason: 'not_installed' as const, + }] + }) + + const flags = { + ...DEFAULT_GRANT_FLAGS, + ...Object.fromEntries( + Object.entries(request.requestedFlags).filter(([, value]) => value === true), + ), + } + + return { + granted, + denied, + flags, + userConsented: true, + } +} + +export function ComputerUsePermissionModal({ sessionId, request }: Props) { + const t = useTranslation() + const respondToComputerUsePermission = useChatStore( + (s) => s.respondToComputerUsePermission, + ) + const [openingPane, setOpeningPane] = useState< + 'Privacy_Accessibility' | 'Privacy_ScreenCapture' | null + >(null) + + const requestedFlags = useMemo( + () => + request + ? Object.entries(request.requestedFlags) + .filter(([, enabled]) => enabled) + .map(([flag]) => flag) + : [], + [request], + ) + + if (!request) return null + + const handleDeny = () => { + respondToComputerUsePermission( + sessionId, + request.requestId, + denyAllResponse(), + ) + } + + const handleAllow = () => { + respondToComputerUsePermission( + sessionId, + request.requestId, + buildAllowResponse(request), + ) + } + + const openSettings = async ( + pane: 'Privacy_Accessibility' | 'Privacy_ScreenCapture', + ) => { + setOpeningPane(pane) + try { + await computerUseApi.openSettings(pane) + } finally { + setOpeningPane(null) + } + } + + const tccState = request.tccState + + return ( + + {t('computerUseApproval.deny')} + + ) : ( + <> + + + + ) + } + > + {tccState ? ( +
+

+ {t('computerUseApproval.tccHint')} +

+ +
+ openSettings('Privacy_Accessibility')} + /> + openSettings('Privacy_ScreenCapture')} + /> +
+ +
+ {t('computerUseApproval.tryAgainHint')} +
+ +
+ +
+
+ ) : ( +
+ {request.reason ? ( +
+
+ {t('computerUseApproval.reason')} +
+
+ {request.reason} +
+
+ ) : null} + +
+ {request.apps.map((app) => { + const resolved = app.resolved + return ( +
+
+
+
+ {resolved?.displayName ?? app.requestedName} +
+
+ {resolved?.bundleId ?? t('computerUseApproval.notInstalled')} +
+
+ + {app.proposedTier} + +
+ + {!resolved ? ( +

+ {t('computerUseApproval.notInstalled')} +

+ ) : null} + + {app.alreadyGranted ? ( +

+ {t('computerUseApproval.alreadyGranted')} +

+ ) : null} + + {app.isSentinel ? ( +

+ {t('computerUseApproval.sensitiveApp')} +

+ ) : null} +
+ ) + })} +
+ + {requestedFlags.length > 0 ? ( +
+
+ {t('computerUseApproval.alsoRequested')} +
+
+ {requestedFlags.map((flag) => ( + + {flag} + + ))} +
+
+ ) : null} + + {request.willHide && request.willHide.length > 0 ? ( +
+ {request.autoUnhideEnabled + ? t('computerUseApproval.hideWhileWorkingRestore', { + count: request.willHide.length, + }) + : t('computerUseApproval.hideWhileWorking', { + count: request.willHide.length, + })} +
+ ) : null} +
+ )} + + ) +} + +function PermissionRow({ + label, + granted, + actionLabel, + actionLoading, + onAction, +}: { + label: string + granted: boolean + actionLabel: string + actionLoading: boolean + onAction: () => void +}) { + const t = useTranslation() + + return ( +
+
+
+ {label} +
+
+ {granted + ? t('computerUseApproval.granted') + : t('computerUseApproval.notGranted')} +
+
+ + {!granted ? ( + + ) : null} +
+ ) +} diff --git a/desktop/src/components/chat/MessageList.test.tsx b/desktop/src/components/chat/MessageList.test.tsx index ca1e83d2..9ad7523c 100644 --- a/desktop/src/components/chat/MessageList.test.tsx +++ b/desktop/src/components/chat/MessageList.test.tsx @@ -19,6 +19,7 @@ function makeSessionState(overrides: Partial = {}): PerSessionS activeToolName: null, activeThinkingId: null, pendingPermission: null, + pendingComputerUsePermission: null, tokenUsage: { input_tokens: 0, output_tokens: 0 }, elapsedSeconds: 0, statusVerb: '', diff --git a/desktop/src/components/chat/ToolCallBlock.tsx b/desktop/src/components/chat/ToolCallBlock.tsx index f8e53ee2..fb07abea 100644 --- a/desktop/src/components/chat/ToolCallBlock.tsx +++ b/desktop/src/components/chat/ToolCallBlock.tsx @@ -41,7 +41,8 @@ export function ToolCallBlock({ toolName, input, result, compact = false }: Prop const preview = useMemo(() => renderPreview(toolName, obj, result, t), [obj, result, toolName, t]) const details = useMemo(() => renderDetails(toolName, obj, t), [obj, toolName, t]) - const expandable = toolName === 'Edit' || toolName === 'Write' + const hasResultDetails = Boolean(result && extractTextContent(result.content)) + const expandable = toolName === 'Edit' || toolName === 'Write' || hasResultDetails return (
)} - {result?.isError && ( + {result?.isError && ( error )} {expandable && ( diff --git a/desktop/src/components/chat/chatBlocks.test.tsx b/desktop/src/components/chat/chatBlocks.test.tsx index 2e999e26..45f00d68 100644 --- a/desktop/src/components/chat/chatBlocks.test.tsx +++ b/desktop/src/components/chat/chatBlocks.test.tsx @@ -40,7 +40,7 @@ describe('chat blocks', () => { fireEvent.click(screen.getByRole('button')) - expect(container.textContent).not.toContain('Tool Input') + expect(container.textContent).toContain('Tool Input') expect(container.textContent).not.toContain('const answer = 42') }) @@ -62,6 +62,27 @@ describe('chat blocks', () => { expect(container.textContent).not.toContain('file-a') }) + it('expands tool errors so full Computer Use gate messages are readable', () => { + const { container } = render( + , + ) + + expect(container.textContent).toContain('mcp__computer-use__left_click') + expect(container.textContent).not.toContain('Take a new screenshot') + + fireEvent.click(screen.getByRole('button')) + + expect(container.textContent).toContain('Take a new screenshot') + expect(container.textContent).toContain('allowed applications') + }) + it('shows a diff preview for edit permission requests', () => { useChatStore.setState({ sessions: { @@ -83,6 +104,7 @@ describe('chat blocks', () => { new_string: 'const count = 2', }, }, + pendingComputerUsePermission: null, tokenUsage: { input_tokens: 0, output_tokens: 0 }, elapsedSeconds: 0, statusVerb: '', diff --git a/desktop/src/i18n/locales/en.ts b/desktop/src/i18n/locales/en.ts index 316432b6..f7e5b778 100644 --- a/desktop/src/i18n/locales/en.ts +++ b/desktop/src/i18n/locales/en.ts @@ -331,6 +331,28 @@ export const en = { 'permission.showFullInput': 'Show full input', 'permission.replacingContent': 'Replacing content in file', + // ─── Computer Use Approval ────────────────────────────────────── + 'computerUseApproval.titleApps': 'Computer Use wants to control these apps', + 'computerUseApproval.titleTcc': 'Computer Use needs macOS permissions', + 'computerUseApproval.reason': 'Why Claude is asking', + 'computerUseApproval.allow': 'Allow for session', + 'computerUseApproval.deny': 'Deny', + 'computerUseApproval.alreadyGranted': 'Already granted for this session', + 'computerUseApproval.notInstalled': 'App not installed', + 'computerUseApproval.sensitiveApp': 'This app is treated as sensitive and deserves extra review.', + 'computerUseApproval.alsoRequested': 'Also requested', + 'computerUseApproval.hideWhileWorking': '{count} other apps will be hidden while Claude works.', + 'computerUseApproval.hideWhileWorkingRestore': '{count} other apps will be hidden while Claude works, then restored when Claude is done.', + 'computerUseApproval.accessibility': 'Accessibility', + 'computerUseApproval.screenRecording': 'Screen Recording', + 'computerUseApproval.granted': 'Granted', + 'computerUseApproval.notGranted': 'Not granted', + 'computerUseApproval.openAccessibility': 'Open Accessibility', + 'computerUseApproval.openScreenRecording': 'Open Screen Recording', + 'computerUseApproval.tryAgain': 'Try again', + 'computerUseApproval.tccHint': 'Grant the missing permissions in System Settings, then come back and choose "Try again".', + 'computerUseApproval.tryAgainHint': 'Try again returns control to Claude so it can call request_access once more after macOS permission changes take effect.', + // ─── Ask User Question ────────────────────────────────────── 'question.needsInput': 'Claude needs your input', 'question.answered': 'Answered', diff --git a/desktop/src/i18n/locales/zh.ts b/desktop/src/i18n/locales/zh.ts index 56401ef8..455e915e 100644 --- a/desktop/src/i18n/locales/zh.ts +++ b/desktop/src/i18n/locales/zh.ts @@ -333,6 +333,28 @@ export const zh: Record = { 'permission.showFullInput': '显示完整输入', 'permission.replacingContent': '替换文件内容', + // ─── Computer Use Approval ────────────────────────────────────── + 'computerUseApproval.titleApps': 'Computer Use 想控制这些应用', + 'computerUseApproval.titleTcc': 'Computer Use 需要 macOS 权限', + 'computerUseApproval.reason': '请求原因', + 'computerUseApproval.allow': '本次会话允许', + 'computerUseApproval.deny': '拒绝', + 'computerUseApproval.alreadyGranted': '本次会话已授权', + 'computerUseApproval.notInstalled': '应用未安装', + 'computerUseApproval.sensitiveApp': '该应用属于高敏感类别，请额外确认后再授权。', + 'computerUseApproval.alsoRequested': '同时请求了', + 'computerUseApproval.hideWhileWorking': 'Claude 工作时会隐藏另外 {count} 个应用。', + 'computerUseApproval.hideWhileWorkingRestore': 'Claude 工作时会隐藏另外 {count} 个应用，结束后会自动恢复。', + 'computerUseApproval.accessibility': '辅助功能', + 'computerUseApproval.screenRecording': '屏幕录制', + 'computerUseApproval.granted': '已授权', + 'computerUseApproval.notGranted': '未授权', + 'computerUseApproval.openAccessibility': '打开辅助功能设置', + 'computerUseApproval.openScreenRecording': '打开屏幕录制设置', + 'computerUseApproval.tryAgain': '稍后重试', + 'computerUseApproval.tccHint': '先在系统设置里授予缺失权限，返回后再点“稍后重试”。', + 'computerUseApproval.tryAgainHint': '“稍后重试”会把控制权交还给 Claude，让它在 macOS 权限生效后重新调用 request_access。', + // ─── Ask User Question ────────────────────────────────────── 'question.needsInput': 'Claude 需要你的输入', 'question.answered': '已回答', diff --git a/desktop/src/pages/ActiveSession.test.tsx b/desktop/src/pages/ActiveSession.test.tsx index a4100d13..52f17ff0 100644 --- a/desktop/src/pages/ActiveSession.test.tsx +++ b/desktop/src/pages/ActiveSession.test.tsx @@ -79,6 +79,7 @@ describe('ActiveSession task polling', () => { activeToolName: null, activeThinkingId: null, pendingPermission: null, + pendingComputerUsePermission: null, tokenUsage: { input_tokens: 0, output_tokens: 0 }, elapsedSeconds: 0, statusVerb: '', @@ -157,6 +158,7 @@ describe('ActiveSession task polling', () => { activeToolName: null, activeThinkingId: null, pendingPermission: null, + pendingComputerUsePermission: null, tokenUsage: { input_tokens: 0, output_tokens: 0 }, elapsedSeconds: 0, statusVerb: '', diff --git a/desktop/src/pages/ActiveSession.tsx b/desktop/src/pages/ActiveSession.tsx index 7e8515f4..6919b998 100644 --- a/desktop/src/pages/ActiveSession.tsx +++ b/desktop/src/pages/ActiveSession.tsx @@ -7,6 +7,7 @@ import { useTeamStore } from '../stores/teamStore' import { useTranslation } from '../i18n' import { MessageList } from '../components/chat/MessageList' import { ChatInput } from '../components/chat/ChatInput' +import { ComputerUsePermissionModal } from '../components/chat/ComputerUsePermissionModal' import { TeamStatusBar } from '../components/teams/TeamStatusBar' import { SessionTaskBar } from '../components/chat/SessionTaskBar' @@ -17,6 +18,7 @@ export function ActiveSession() { const sessions = useSessionStore((s) => s.sessions) const connectToSession = useChatStore((s) => s.connectToSession) const sessionState = useChatStore((s) => activeTabId ? s.sessions[activeTabId] : undefined) + const pendingComputerUsePermission = sessionState?.pendingComputerUsePermission ?? null const fetchSessionTasks = useCLITaskStore((s) => s.fetchSessionTasks) const trackedTaskSessionId = useCLITaskStore((s) => s.sessionId) const hasIncompleteTasks = useCLITaskStore((s) => s.tasks.some((task) => task.status !== 'completed')) @@ -205,6 +207,13 @@ export function ActiveSession() { + + {!isMemberSession && activeTabId ? ( + + ) : null}
) } diff --git a/desktop/src/stores/chatStore.test.ts b/desktop/src/stores/chatStore.test.ts index 811e3d09..9ac30215 100644 --- a/desktop/src/stores/chatStore.test.ts +++ b/desktop/src/stores/chatStore.test.ts @@ -166,6 +166,7 @@ describe('chatStore history mapping', () => { activeToolName: null, activeThinkingId: null, pendingPermission: null, + pendingComputerUsePermission: null, tokenUsage: { input_tokens: 0, output_tokens: 0 }, elapsedSeconds: 0, statusVerb: '', @@ -222,6 +223,7 @@ describe('chatStore history mapping', () => { activeToolName: null, activeThinkingId: null, pendingPermission: null, + pendingComputerUsePermission: null, tokenUsage: { input_tokens: 0, output_tokens: 0 }, elapsedSeconds: 0, statusVerb: '', @@ -252,6 +254,7 @@ describe('chatStore history mapping', () => { activeToolName: null, activeThinkingId: null, pendingPermission: null, + pendingComputerUsePermission: null, tokenUsage: { input_tokens: 0, output_tokens: 0 }, elapsedSeconds: 0, statusVerb: '', @@ -300,6 +303,7 @@ describe('chatStore history mapping', () => { activeToolName: null, activeThinkingId: null, pendingPermission: null, + pendingComputerUsePermission: null, tokenUsage: { input_tokens: 0, output_tokens: 0 }, elapsedSeconds: 0, statusVerb: '', @@ -325,6 +329,132 @@ describe('chatStore history mapping', () => { expect(useChatStore.getState().sessions[TEST_SESSION_ID]?.streamingText).toBe('') }) + it('tracks Computer Use approval requests separately from generic tool permissions', () => { + useChatStore.setState({ + sessions: { + [TEST_SESSION_ID]: { + messages: [], + chatState: 'idle', + connectionState: 'connected', + streamingText: '', + streamingToolInput: '', + activeToolUseId: null, + activeToolName: null, + activeThinkingId: null, + pendingPermission: null, + pendingComputerUsePermission: null, + tokenUsage: { input_tokens: 0, output_tokens: 0 }, + elapsedSeconds: 0, + statusVerb: '', + slashCommands: [], + agentTaskNotifications: {}, + elapsedTimer: null, + }, + }, + }) + + useChatStore.getState().handleServerMessage(TEST_SESSION_ID, { + type: 'computer_use_permission_request', + requestId: 'cu-1', + request: { + requestId: 'cu-1', + reason: 'Open Finder and inspect a file', + apps: [ + { + requestedName: 'Finder', + resolved: { + bundleId: 'com.apple.finder', + displayName: 'Finder', + }, + isSentinel: false, + alreadyGranted: false, + proposedTier: 'full', + }, + ], + requestedFlags: { clipboardRead: true }, + screenshotFiltering: 'native', + }, + }) + + expect( + useChatStore.getState().sessions[TEST_SESSION_ID]?.pendingComputerUsePermission, + ).toMatchObject({ + requestId: 'cu-1', + request: { + reason: 'Open Finder and inspect a file', + }, + }) + expect( + useChatStore.getState().sessions[TEST_SESSION_ID]?.chatState, + ).toBe('permission_pending') + }) + + it('sends Computer Use approval payloads back over websocket', () => { + useChatStore.setState({ + sessions: { + [TEST_SESSION_ID]: { + messages: [], + chatState: 'permission_pending', + connectionState: 'connected', + streamingText: '', + streamingToolInput: '', + activeToolUseId: null, + activeToolName: null, + activeThinkingId: null, + pendingPermission: null, + pendingComputerUsePermission: { + requestId: 'cu-1', + request: { + requestId: 'cu-1', + reason: 'Open Finder', + apps: [], + requestedFlags: {}, + screenshotFiltering: 'native', + }, + }, + tokenUsage: { input_tokens: 0, output_tokens: 0 }, + elapsedSeconds: 0, + statusVerb: '', + slashCommands: [], + agentTaskNotifications: {}, + elapsedTimer: null, + }, + }, + }) + + useChatStore.getState().respondToComputerUsePermission(TEST_SESSION_ID, 'cu-1', { + granted: [], + denied: [], + flags: { + clipboardRead: true, + clipboardWrite: false, + systemKeyCombos: false, + }, + userConsented: true, + }) + + expect(sendMock).toHaveBeenCalledWith(TEST_SESSION_ID, { + type: 'computer_use_permission_response', + requestId: 'cu-1', + response: { + granted: [], + denied: [], + flags: { + clipboardRead: true, + clipboardWrite: false, + systemKeyCombos: false, + }, + userConsented: true, + }, + }) + expect( + useChatStore.getState().sessions[TEST_SESSION_ID]?.pendingComputerUsePermission, + ).toBeNull() + expect( + useChatStore.getState().sessions[TEST_SESSION_ID]?.chatState, + ).toBe('tool_executing') + }) + it('routes member-session messages through team mailbox delivery instead of websocket', async () => { const memberSessionId = 'team-member:security-reviewer@test-team' getMemberBySessionIdMock.mockReturnValue({ @@ -345,6 +475,7 @@ describe('chatStore history mapping', () => { activeToolName: null, activeThinkingId: null, pendingPermission: null, + pendingComputerUsePermission: null, tokenUsage: { input_tokens: 0, output_tokens: 0 }, elapsedSeconds: 0, statusVerb: '', diff --git a/desktop/src/stores/chatStore.ts b/desktop/src/stores/chatStore.ts index c64f7f1a..e8420753 100644 --- a/desktop/src/stores/chatStore.ts +++ b/desktop/src/stores/chatStore.ts @@ -13,6 +13,8 @@ import type { AgentTaskNotification, AttachmentRef, ChatState, + ComputerUsePermissionRequest, + ComputerUsePermissionResponse, UIAttachment, UIMessage, ServerMessage, @@ -36,6 +38,10 @@ export type PerSessionState = { input: unknown description?: string } | null + pendingComputerUsePermission: { + requestId: string + request: ComputerUsePermissionRequest + } | null tokenUsage: TokenUsage elapsedSeconds: number statusVerb: string @@ -54,6 +60,7 @@ const DEFAULT_SESSION_STATE: PerSessionState = { activeToolName: null, activeThinkingId: null, pendingPermission: null, + pendingComputerUsePermission: null, tokenUsage: { input_tokens: 0, output_tokens: 0 }, elapsedSeconds: 0, statusVerb: '', @@ -74,6 +81,11 @@ type ChatStore = { disconnectSession: (sessionId: string) => void sendMessage: (sessionId: string, content: string, attachments?: AttachmentRef[]) => void respondToPermission: (sessionId: string, requestId: string, allowed: boolean, rule?: string) => void + respondToComputerUsePermission: ( + sessionId: string, + requestId: string, + response: ComputerUsePermissionResponse, + ) => void setSessionPermissionMode: (sessionId: string, mode: PermissionMode) => void stopGeneration: (sessionId: string) => void loadHistory: (sessionId: string) => Promise @@ -272,6 +284,20 @@ export const useChatStore = create((set, get) => ({ set((s) => ({ sessions: updateSessionIn(s.sessions, sessionId, () => ({ pendingPermission: null, chatState: allowed ? 'tool_executing' : 'idle' })) })) }, + respondToComputerUsePermission: (sessionId, requestId, response) => { + wsManager.send(sessionId, { + type: 'computer_use_permission_response', + requestId, + response, + }) + set((s) => ({ + sessions: updateSessionIn(s.sessions, sessionId, () => ({ + pendingComputerUsePermission: null, + chatState: response.userConsented === false ? 'idle' : 'tool_executing', + })), + })) + }, + setSessionPermissionMode: (sessionId, mode) => { if (!get().sessions[sessionId]) return wsManager.send(sessionId, { type: 'set_permission_mode', mode }) @@ -289,7 +315,18 @@ export const useChatStore = create((set, get) => ({ const session = s.sessions[sessionId] if (!session) return s if (session.elapsedTimer) clearInterval(session.elapsedTimer) - return { sessions: { ...s.sessions, [sessionId]: { ...session, chatState: 'idle', elapsedTimer: null } } } + return { + sessions: { + ...s.sessions, + [sessionId]: { + ...session, + chatState: 'idle', + pendingPermission: null, + pendingComputerUsePermission: null, + elapsedTimer: null, + }, + }, + } }) }, @@ -456,6 +493,7 @@ export const useChatStore = create((set, get) => ({ case 'permission_request': update((s) => ({ pendingPermission: { requestId: msg.requestId, toolName: msg.toolName, input: msg.input, description: msg.description }, + pendingComputerUsePermission: null, chatState: 'permission_pending', activeThinkingId: null, messages: [...s.messages, { @@ -465,6 +503,18 @@ export const useChatStore = create((set, get) => ({ })) break + case 'computer_use_permission_request': + update(() => ({ + pendingComputerUsePermission: { + requestId: msg.requestId, + request: msg.request, + }, + pendingPermission: null, + chatState: 'permission_pending', + activeThinkingId: null, + })) + break + case 'message_complete': { const session = get().sessions[sessionId] if (!session) break @@ -476,7 +526,14 @@ export const useChatStore = create((set, get) => ({ })) } if (session.elapsedTimer) clearInterval(session.elapsedTimer) - update(() => ({ tokenUsage: msg.usage, chatState: 'idle', activeThinkingId: null, elapsedTimer: null })) + update(() => ({ + tokenUsage: msg.usage, + chatState: 'idle', + activeThinkingId: null, + pendingPermission: null, + pendingComputerUsePermission: null, + elapsedTimer: null, + })) break } @@ -488,7 +545,14 @@ export const useChatStore = create((set, get) => ({ newMessages.push({ id: nextId(), type: 'assistant_text' as const, content: pendingText, timestamp: Date.now() }) } newMessages.push({ id: nextId(), type: 'error', message: msg.message, code: msg.code, timestamp: Date.now() }) - return { messages: newMessages, chatState: 'idle', activeThinkingId: null, streamingText: '' } + return { + messages: newMessages, + chatState: 'idle', + activeThinkingId: null, + streamingText: '', + pendingPermission: null, + pendingComputerUsePermission: null, + } }) useTabStore.getState().updateTabStatus(sessionId, 'error') { diff --git a/desktop/src/stores/teamStore.ts b/desktop/src/stores/teamStore.ts index 93e0b574..ce6d7e71 100644 --- a/desktop/src/stores/teamStore.ts +++ b/desktop/src/stores/teamStore.ts @@ -27,6 +27,7 @@ function createMemberSessionState() { activeToolName: null, activeThinkingId: null, pendingPermission: null, + pendingComputerUsePermission: null, tokenUsage: { input_tokens: 0, output_tokens: 0 }, elapsedSeconds: 0, statusVerb: '', diff --git a/desktop/src/types/chat.ts b/desktop/src/types/chat.ts index dddebead..d9666321 100644 --- a/desktop/src/types/chat.ts +++ b/desktop/src/types/chat.ts @@ -7,6 +7,11 @@ import type { PermissionMode } from './settings' export type ClientMessage = | { type: 'user_message'; content: string; attachments?: AttachmentRef[] } | { type: 'permission_response'; requestId: string; allowed: boolean; rule?: string } + | { + type: 'computer_use_permission_response' + requestId: string + response: ComputerUsePermissionResponse + } | { type: 'set_permission_mode'; mode: PermissionMode } | { type: 'stop_generation' } | { type: 'ping' } @@ -35,6 +40,11 @@ export type ServerMessage = | { type: 'tool_use_complete'; toolName: string; toolUseId: string; input: unknown; parentToolUseId?: string } | { type: 'tool_result'; toolUseId: string; content: unknown; isError: boolean; parentToolUseId?: string } | { type: 'permission_request'; requestId: string; toolName: string; input: unknown; description?: string } + | { + type: 'computer_use_permission_request' + requestId: string + request: ComputerUsePermissionRequest + } | { type: 'message_complete'; usage: TokenUsage } | { type: 'thinking'; text: string } | { type: 'status'; state: ChatState; verb?: string; elapsed?: number; tokens?: number } @@ -63,6 +73,56 @@ export type TeamMemberStatus = { currentTask?: string } +export type ComputerUseGrantFlags = { + clipboardRead: boolean + clipboardWrite: boolean + systemKeyCombos: boolean +} + +export type ComputerUseResolvedApp = { + bundleId: string + displayName: string + path?: string + iconDataUrl?: string +} + +export type ComputerUseResolvedAppRequest = { + requestedName: string + resolved?: ComputerUseResolvedApp + isSentinel: boolean + alreadyGranted: boolean + proposedTier: 'read' | 'click' | 'full' +} + +export type ComputerUsePermissionRequest = { + requestId: string + reason: string + apps: ComputerUseResolvedAppRequest[] + requestedFlags: Partial + screenshotFiltering: 'native' | 'none' + tccState?: { + accessibility: boolean + screenRecording: boolean + } + willHide?: Array<{ bundleId: string; displayName: string }> + autoUnhideEnabled?: boolean +} + +export type ComputerUsePermissionResponse = { + granted: Array<{ + bundleId: string + displayName: string + grantedAt: number + tier?: 'read' | 'click' | 'full' + }> + denied: Array<{ + bundleId: string + reason: 'user_denied' | 'not_installed' + }> + flags: ComputerUseGrantFlags + userConsented?: boolean +} + export type AgentTaskNotification = { taskId: string toolUseId: string diff --git a/runtime/mac_helper.py b/runtime/mac_helper.py index ceb4fd27..1bfae38e 100755 --- a/runtime/mac_helper.py +++ b/runtime/mac_helper.py @@ -3,6 +3,7 @@ from __future__ import annotations import argparse import base64 +import ctypes import json import os import subprocess @@ -175,6 +176,33 @@ def run_osascript(script: str) -> str: return result.stdout.strip() +def applescript_modifier(name: str) -> str: + if name == "command": + return "command down" + if name == "option": + return "option down" + if name == "shift": + return "shift down" + if name == "ctrl": + return "control down" + if name == "fn": + return "fn down" + raise ValueError(f"Unsupported AppleScript modifier: {name}") + + +def send_keystroke_via_osascript(character: str, modifiers: list[str] | None = None) -> None: + escaped = character.replace("\\", "\\\\").replace('"', '\\"') + if modifiers: + modifier_expr = ", ".join(applescript_modifier(m) for m in modifiers) + script = ( + 'tell application "System Events" to keystroke ' + f'"{escaped}" using {{{modifier_expr}}}' + ) + else: + script = f'tell application "System Events" to keystroke "{escaped}"' + run_osascript(script) + + def get_displays() -> list[dict[str, Any]]: max_displays = 32 err, active, count = CGGetActiveDisplayList(max_displays, None, None) @@ -472,6 +500,10 @@ def write_clipboard(text: str) -> None: pb.setString_forType_(text, NSPasteboardTypeString) +def paste_clipboard() -> None: + send_keystroke_via_osascript("v", ["command"]) + + def detect_screen_recording_permission() -> bool | None: """Best-effort passive screen-recording probe with no system prompt. @@ -520,12 +552,29 @@ def detect_screen_recording_permission() -> bool | None: return None -def check_permissions() -> dict[str, bool | None]: - accessibility = True +def detect_accessibility_permission() -> bool: + """ + Use the official macOS Accessibility trust API. + + The previous System Events / AppleScript probe was too weak: it could + succeed even when the current helper process was not actually trusted for + input control, which led the desktop UI to report Accessibility as granted + while mouse/keyboard control still failed at runtime. + """ + framework_path = "/System/Library/Frameworks/ApplicationServices.framework/ApplicationServices" try: - run_osascript('tell application "System Events" to get name of first process') + application_services = ctypes.CDLL(framework_path) + application_services.AXIsProcessTrusted.restype = ctypes.c_bool + application_services.AXIsProcessTrusted.argtypes = [] + return bool(application_services.AXIsProcessTrusted()) except Exception: - accessibility = False + # Fail closed: if the trust API can't be queried, treat accessibility + # as unavailable instead of reporting a misleading success state. + return False + + +def check_permissions() -> dict[str, bool | None]: + accessibility = detect_accessibility_permission() screen_recording = detect_screen_recording_permission() return { "accessibility": accessibility, @@ -559,7 +608,15 @@ def scroll(x: int, y: int, delta_x: int, delta_y: int) -> None: def key_action(sequence: str, repeat: int = 1) -> None: parts = [normalize_key(part) for part in sequence.split("+") if part.strip()] for _ in range(max(1, repeat)): - if len(parts) == 1: + if parts == ["command", "v"]: + paste_clipboard() + elif parts == ["command", "a"]: + send_keystroke_via_osascript("a", ["command"]) + elif parts == ["command", "c"]: + send_keystroke_via_osascript("c", ["command"]) + elif parts == ["command", "x"]: + send_keystroke_via_osascript("x", ["command"]) + elif len(parts) == 1: pyautogui.press(parts[0]) else: pyautogui.hotkey(*parts, interval=0.02) @@ -703,6 +760,10 @@ def main() -> int: write_clipboard(str(payload.get("text") or "")) json_output({"ok": True, "result": True}) return 0 + if command == "paste_clipboard": + paste_clipboard() + json_output({"ok": True, "result": True}) + return 0 error_output(f"Unknown command: {command}", code="bad_command") return 2 except Exception as exc: diff --git a/runtime/test_helpers.py b/runtime/test_helpers.py index 4c5e8db1..540d754c 100644 --- a/runtime/test_helpers.py +++ b/runtime/test_helpers.py @@ -154,7 +154,7 @@ class TestJSONProtocol(unittest.TestCase): "move_mouse", "scroll", "mouse_down", "mouse_up", "cursor_position", "frontmost_app", "app_under_point", "list_installed_apps", "list_running_apps", "open_app", - "read_clipboard", "write_clipboard", + "read_clipboard", "write_clipboard", "paste_clipboard", } for helper in [MAC_HELPER, WIN_HELPER]: if not helper.exists(): @@ -247,6 +247,42 @@ class TestWinHelperPermissions(unittest.TestCase): self.assertIn('"screenRecording": True', func_source) +class TestMacHelperPermissions(unittest.TestCase): + """macOS helper permission detection should use the official trust API.""" + + def test_check_permissions_uses_ax_api_instead_of_system_events(self): + if not MAC_HELPER.exists(): + self.skipTest("mac_helper.py not found") + + source = MAC_HELPER.read_text() + + self.assertIn("def detect_accessibility_permission()", source) + self.assertIn("AXIsProcessTrusted", source) + + start = source.index("def check_permissions()") + rest = source[start:] + lines = rest.split("\n") + func_lines = [lines[0]] + for line in lines[1:]: + if line and not line[0].isspace() and not line.startswith("#"): + break + func_lines.append(line) + func_source = "\n".join(func_lines) + + self.assertIn("detect_accessibility_permission()", func_source) + self.assertNotIn('tell application "System Events"', func_source) + + def test_clipboard_shortcuts_use_osascript_path(self): + if not MAC_HELPER.exists(): + self.skipTest("mac_helper.py not found") + + source = MAC_HELPER.read_text() + self.assertIn("def paste_clipboard()", source) + self.assertIn('send_keystroke_via_osascript("v", ["command"])', source) + self.assertIn('if parts == ["command", "v"]:', source) + self.assertIn('elif parts == ["command", "a"]:', source) + + class TestCrossPlatformFunctions(unittest.TestCase): """Test functions that are identical between both helpers.""" @@ -275,7 +311,7 @@ class TestCrossPlatformFunctions(unittest.TestCase): """Input action functions (click, scroll, etc.) should be identical.""" if not MAC_HELPER.exists() or not WIN_HELPER.exists(): self.skipTest("Both helpers required") - for func in ["click", "scroll", "key_action", "hold_keys", "type_text"]: + for func in ["click", "scroll", "hold_keys", "type_text"]: mac_src = self._get_function_body(MAC_HELPER, func) win_src = self._get_function_body(WIN_HELPER, func) self.assertEqual(mac_src, win_src, diff --git a/runtime/win_helper.py b/runtime/win_helper.py index f6ff848e..cc190ee2 100644 --- a/runtime/win_helper.py +++ b/runtime/win_helper.py @@ -513,6 +513,10 @@ def write_clipboard(text: str) -> None: pyperclip.copy(text) +def paste_clipboard() -> None: + pyautogui.hotkey("ctrl", "v", interval=0.02) + + # --------------------------------------------------------------------------- # Permissions — Windows doesn't have macOS-style TCC # --------------------------------------------------------------------------- @@ -704,6 +708,10 @@ def main() -> int: write_clipboard(str(payload.get("text") or "")) json_output({"ok": True, "result": True}) return 0 + if command == "paste_clipboard": + paste_clipboard() + json_output({"ok": True, "result": True}) + return 0 error_output(f"Unknown command: {command}", code="bad_command") return 2 except Exception as exc: diff --git a/src/server/__tests__/conversation-service.test.ts b/src/server/__tests__/conversation-service.test.ts index 210846e3..249a3439 100644 --- a/src/server/__tests__/conversation-service.test.ts +++ b/src/server/__tests__/conversation-service.test.ts @@ -146,6 +146,19 @@ describe('ConversationService', () => { expect(env.CLAUDE_CODE_OAUTH_TOKEN).toBeUndefined() }) + test('buildChildEnv injects desktop Computer Use host bundle id for sdk sessions', async () => { + const service = new ConversationService() as any + const env = (await service.buildChildEnv( + '/tmp', + 'ws://127.0.0.1:3456/sdk/test-session?token=test-token', + )) as Record + + expect(env.CC_HAHA_COMPUTER_USE_HOST_BUNDLE_ID).toBe( + 'com.claude-code-haha.desktop', + ) + expect(env.CC_HAHA_DESKTOP_SERVER_URL).toBe('http://127.0.0.1:3456') + }) + test('uses bun entrypoint fallback on Windows dev mode', () => { const service = new ConversationService() as any const args = service.resolveCliArgs(['--print']) diff --git a/src/server/api/computer-use.ts b/src/server/api/computer-use.ts index fed13ea0..1650dd10 100644 --- a/src/server/api/computer-use.ts +++ b/src/server/api/computer-use.ts @@ -12,6 +12,8 @@ import { access, readFile, mkdir, writeFile } from 'fs/promises' import { createHash } from 'crypto' import path from 'path' import { fileURLToPath } from 'url' +import type { CuPermissionRequest } from '../../vendor/computer-use-mcp/types.js' +import { computerUseApprovalService } from '../services/computerUseApprovalService.js' // Embed helper scripts at compile time so they're available in bundled mode // @ts-ignore — Bun text import import MAC_HELPER_CONTENT from '../../../runtime/mac_helper.py' with { type: 'text' } @@ -345,6 +347,11 @@ type ComputerUseConfig = { } } +type RequestAccessBody = { + sessionId?: string + request?: CuPermissionRequest +} + const DEFAULT_CONFIG: ComputerUseConfig = { authorizedApps: [], grantFlags: { clipboardRead: true, clipboardWrite: true, systemKeyCombos: true }, @@ -453,6 +460,29 @@ export async function handleComputerUseApi( return Response.json({ ok: true }) } + if (action === 'request-access' && req.method === 'POST') { + try { + const body = (await req.json()) as RequestAccessBody + if (!body.sessionId || !body.request?.requestId) { + return Response.json( + { error: 'BAD_REQUEST', message: 'sessionId and request are required' }, + { status: 400 }, + ) + } + + const response = await computerUseApprovalService.requestApproval( + body.sessionId, + body.request, + ) + return Response.json(response) + } catch (error) { + const message = + error instanceof Error ? error.message : 'Computer Use approval failed' + const status = message.includes('not connected') ? 409 : 500 + return Response.json({ error: 'COMPUTER_USE_APPROVAL_FAILED', message }, { status }) + } + } + return Response.json( { error: 'NOT_FOUND', message: `Unknown computer-use action: ${action}` }, { status: 404 }, diff --git a/src/server/services/computerUseApprovalService.ts b/src/server/services/computerUseApprovalService.ts new file mode 100644 index 00000000..055c4efb --- /dev/null +++ b/src/server/services/computerUseApprovalService.ts @@ -0,0 +1,73 @@ +import type { CuPermissionRequest, CuPermissionResponse } from '../../vendor/computer-use-mcp/types.js' +import { sendToSession } from '../ws/handler.js' + +type PendingApproval = { + sessionId: string + resolve: (response: CuPermissionResponse) => void + reject: (error: Error) => void + timeout: ReturnType +} + +const REQUEST_TIMEOUT_MS = 5 * 60 * 1000 + +class ComputerUseApprovalService { + private pending = new Map() + + async requestApproval( + sessionId: string, + request: CuPermissionRequest, + ): Promise { + const existing = this.pending.get(request.requestId) + if (existing) { + clearTimeout(existing.timeout) + existing.reject(new Error('Computer Use approval request superseded')) + this.pending.delete(request.requestId) + } + + return await new Promise((resolve, reject) => { + const timeout = setTimeout(() => { + this.pending.delete(request.requestId) + reject(new Error('Computer Use approval timed out')) + }, REQUEST_TIMEOUT_MS) + + this.pending.set(request.requestId, { + sessionId, + resolve, + reject, + timeout, + }) + + const sent = sendToSession(sessionId, { + type: 'computer_use_permission_request', + requestId: request.requestId, + request, + }) + + if (!sent) { + clearTimeout(timeout) + this.pending.delete(request.requestId) + reject(new Error('Desktop session is not connected')) + } + }) + } + + resolveApproval(requestId: string, response: CuPermissionResponse): boolean { + const pending = this.pending.get(requestId) + if (!pending) return false + clearTimeout(pending.timeout) + this.pending.delete(requestId) + pending.resolve(response) + return true + } + + cancelSession(sessionId: string): void { + for (const [requestId, pending] of this.pending.entries()) { + if (pending.sessionId !== sessionId) continue + clearTimeout(pending.timeout) + this.pending.delete(requestId) + pending.reject(new Error('Desktop session disconnected during Computer Use approval')) + } + } +} + +export const computerUseApprovalService = new ComputerUseApprovalService() diff --git a/src/server/services/conversationService.ts b/src/server/services/conversationService.ts index 2ae4dd62..f76a16e8 100644 --- a/src/server/services/conversationService.ts +++ b/src/server/services/conversationService.ts @@ -119,7 +119,7 @@ export class ConversationService { // 工作目录就变成 `/`。把 CALLER_DIR / PWD 显式覆盖成 workDir，preload.ts // chdir 后落到正确目录。 // - const childEnv = await this.buildChildEnv(workDir) + const childEnv = await this.buildChildEnv(workDir, sdkUrl) let proc: ReturnType try { @@ -471,7 +471,10 @@ export class ConversationService { return args } - private async buildChildEnv(workDir: string): Promise> { + private async buildChildEnv( + workDir: string, + sdkUrl?: string, + ): Promise> { // Provider isolation: when Desktop has its own provider config/index, // strip inherited provider env vars so the child CLI reads fresh values // from ~/.claude/cc-haha/settings.json instead of stale process.env. @@ -497,11 +500,27 @@ export class ConversationService { } } + let desktopServerUrl: string | undefined + if (sdkUrl) { + try { + const parsed = new URL(sdkUrl) + desktopServerUrl = `http://${parsed.host}` + } catch { + desktopServerUrl = undefined + } + } + return { ...cleanEnv, CLAUDE_CODE_ENABLE_TASKS: '1', CALLER_DIR: workDir, PWD: workDir, + ...(sdkUrl + ? { CC_HAHA_COMPUTER_USE_HOST_BUNDLE_ID: 'com.claude-code-haha.desktop' } + : {}), + ...(desktopServerUrl + ? { CC_HAHA_DESKTOP_SERVER_URL: desktopServerUrl } + : {}), // Tell the CLI entrypoint to skip project .env loading. Provider env // should come from Desktop-managed config or inherited launch env, not // be reintroduced from the repo's .env file. diff --git a/src/server/ws/events.ts b/src/server/ws/events.ts index 508740ec..56533cdc 100644 --- a/src/server/ws/events.ts +++ b/src/server/ws/events.ts @@ -11,6 +11,11 @@ export type ClientMessage = | { type: 'user_message'; content: string; attachments?: AttachmentRef[] } | { type: 'permission_response'; requestId: string; allowed: boolean; rule?: string } + | { + type: 'computer_use_permission_response' + requestId: string + response: ComputerUsePermissionResponse + } | { type: 'set_permission_mode'; mode: string } | { type: 'stop_generation' } | { type: 'ping' } @@ -34,6 +39,11 @@ export type ServerMessage = | { type: 'tool_use_complete'; toolName: string; toolUseId: string; input: unknown; parentToolUseId?: string } | { type: 'tool_result'; toolUseId: string; content: unknown; isError: boolean; parentToolUseId?: string } | { type: 'permission_request'; requestId: string; toolName: string; input: unknown; description?: string } + | { + type: 'computer_use_permission_request' + requestId: string + request: ComputerUsePermissionRequest + } | { type: 'message_complete'; usage: TokenUsage } | { type: 'thinking'; text: string } | { type: 'status'; state: ChatState; verb?: string; elapsed?: number; tokens?: number } @@ -62,6 +72,56 @@ export type TeamMemberStatus = { currentTask?: string } +export type ComputerUseGrantFlags = { + clipboardRead: boolean + clipboardWrite: boolean + systemKeyCombos: boolean +} + +export type ComputerUseResolvedApp = { + bundleId: string + displayName: string + path?: string + iconDataUrl?: string +} + +export type ComputerUseResolvedAppRequest = { + requestedName: string + resolved?: ComputerUseResolvedApp + isSentinel: boolean + alreadyGranted: boolean + proposedTier: 'read' | 'click' | 'full' +} + +export type ComputerUsePermissionRequest = { + requestId: string + reason: string + apps: ComputerUseResolvedAppRequest[] + requestedFlags: Partial + screenshotFiltering: 'native' | 'none' + tccState?: { + accessibility: boolean + screenRecording: boolean + } + willHide?: Array<{ bundleId: string; displayName: string }> + autoUnhideEnabled?: boolean +} + +export type ComputerUsePermissionResponse = { + granted: Array<{ + bundleId: string + displayName: string + grantedAt: number + tier?: 'read' | 'click' | 'full' + }> + denied: Array<{ + bundleId: string + reason: 'user_denied' | 'not_installed' + }> + flags: ComputerUseGrantFlags + userConsented?: boolean +} + // ============================================================================ // Internal types // ============================================================================ diff --git a/src/server/ws/handler.ts b/src/server/ws/handler.ts index 20986103..19fa3dcf 100644 --- a/src/server/ws/handler.ts +++ b/src/server/ws/handler.ts @@ -13,6 +13,7 @@ import { ConversationStartupError, conversationService, } from '../services/conversationService.js' +import { computerUseApprovalService } from '../services/computerUseApprovalService.js' import { sessionService } from '../services/sessionService.js' import { SettingsService } from '../services/settingsService.js' import { ProviderService } from '../services/providerService.js' @@ -118,6 +119,10 @@ export const handleWebSocket = { handlePermissionResponse(ws, message) break + case 'computer_use_permission_response': + handleComputerUsePermissionResponse(ws, message) + break + case 'set_permission_mode': handleSetPermissionMode(ws, message) break @@ -148,6 +153,7 @@ export const handleWebSocket = { } console.log(`[WS] Client disconnected from session: ${sessionId} (${code}: ${reason})`) + computerUseApprovalService.cancelSession(sessionId) activeSessions.delete(sessionId) cleanupStreamState(sessionId) sessionSlashCommands.delete(sessionId) @@ -300,6 +306,22 @@ function handlePermissionResponse( console.log(`[WS] Permission response for ${message.requestId}: ${message.allowed}`) } +function handleComputerUsePermissionResponse( + ws: ServerWebSocket, + message: Extract +) { + const { sessionId } = ws.data + const ok = computerUseApprovalService.resolveApproval( + message.requestId, + message.response, + ) + if (!ok) { + console.warn( + `[WS] Ignored Computer Use permission response for unknown request ${message.requestId} from ${sessionId}` + ) + } +} + function handleSetPermissionMode( ws: ServerWebSocket, message: Extract diff --git a/src/utils/computerUse/executor.ts b/src/utils/computerUse/executor.ts index 7f6b4f59..2fedee66 100644 --- a/src/utils/computerUse/executor.ts +++ b/src/utils/computerUse/executor.ts @@ -15,13 +15,14 @@ import type { ScreenshotResult, } from '../../vendor/computer-use-mcp/index.js' import { API_RESIZE_PARAMS, targetImageSize } from '../../vendor/computer-use-mcp/index.js' -import { execFileNoThrow } from '../execFileNoThrow.js' import { sleep } from '../sleep.js' import { CLI_CU_CAPABILITIES, CLI_HOST_BUNDLE_ID } from './common.js' import { callPythonHelper } from './pythonBridge.js' const SCREENSHOT_JPEG_QUALITY = 0.75 const MOVE_SETTLE_MS = 50 +const hostBundleId = + process.env.CC_HAHA_COMPUTER_USE_HOST_BUNDLE_ID || CLI_HOST_BUNDLE_ID type PythonDisplayGeometry = DisplayGeometry @@ -48,14 +49,11 @@ function normalizeDisplayGeometry(display: PythonDisplayGeometry): DisplayGeomet } async function readClipboardViaPbpaste(): Promise { - const { stdout, code } = await execFileNoThrow('pbpaste', [], { useCwd: false }) - if (code !== 0) throw new Error(`pbpaste exited with code ${code}`) - return stdout + return callPythonHelper('read_clipboard', {}) } async function writeClipboardViaPbcopy(text: string): Promise { - const { code } = await execFileNoThrow('pbcopy', [], { input: text, useCwd: false }) - if (code !== 0) throw new Error(`pbcopy exited with code ${code}`) + await callPythonHelper('write_clipboard', { text }) } async function typeViaClipboard(text: string): Promise { @@ -66,8 +64,11 @@ async function typeViaClipboard(text: string): Promise { try { await writeClipboardViaPbcopy(text) - await callPythonHelper('key', { keySequence: 'command+v', repeat: 1 }) - await sleep(100) + // Give NSPasteboard a beat before paste, then keep the new contents + // resident long enough for Electron/WebView fields to consume them. + await sleep(40) + await callPythonHelper('paste_clipboard', {}) + await sleep(180) } finally { if (typeof saved === 'string') { try { @@ -88,7 +89,7 @@ export function createCliExecutor(_opts: { return { capabilities: { ...CLI_CU_CAPABILITIES, - hostBundleId: CLI_HOST_BUNDLE_ID, + hostBundleId, }, async prepareForAction(): Promise { diff --git a/src/utils/computerUse/wrapper.tsx b/src/utils/computerUse/wrapper.tsx index 9b336cfb..783d90b7 100644 --- a/src/utils/computerUse/wrapper.tsx +++ b/src/utils/computerUse/wrapper.tsx @@ -51,6 +51,7 @@ type Binding = { */ let binding: Binding | undefined; let currentToolUseContext: ToolUseContext | undefined; +const desktopServerUrl = process.env.CC_HAHA_DESKTOP_SERVER_URL; function tuc(): ToolUseContext { // Safe: `binding` is only populated when `currentToolUseContext` is set. // Called only from within `ctx` callbacks, which only fire during dispatch. @@ -230,6 +231,37 @@ export function buildSessionContext(): ComputerUseSessionContext { formatLockHeldMessage: formatLockHeld }; } + +async function runDesktopPermissionDialog( + req: CuPermissionRequest, + signal: AbortSignal, +): Promise { + if (!desktopServerUrl) { + throw new Error('Desktop server URL is not configured') + } + + const response = await fetch(`${desktopServerUrl}/api/computer-use/request-access`, { + method: 'POST', + headers: { + 'Content-Type': 'application/json' + }, + body: JSON.stringify({ + sessionId: getSessionId(), + request: req + }), + signal + }) + + if (!response.ok) { + const message = await response.text().catch(() => '') + throw new Error( + `Desktop Computer Use approval failed (${response.status}): ${message || response.statusText}`, + ) + } + + return await response.json() as CuPermissionResponse +} + /** * Load pre-authorized apps from ~/.claude/cc-haha/computer-use-config.json. * Called once when the binding is first created. Pre-authorized apps @@ -362,8 +394,20 @@ export function getComputerUseMCPToolOverrides(toolName: string): ComputerUseMCP */ async function runPermissionDialog(req: CuPermissionRequest): Promise { const context = tuc(); + let desktopBridgeError: Error | undefined; + if (desktopServerUrl) { + try { + return await runDesktopPermissionDialog(req, context.abortController.signal); + } catch (error) { + desktopBridgeError = error instanceof Error ? error : new Error(String(error)); + logForDebugging(`[Computer Use] Desktop approval bridge failed: ${desktopBridgeError.message}`); + } + } const setToolJSX = context.setToolJSX; if (!setToolJSX) { + if (desktopBridgeError) { + throw desktopBridgeError; + } // Shouldn't happen — main.tsx gate excludes non-interactive. Fail safe. return { granted: [], @@ -400,4 +444,4 @@ async function runPermissionDialog(req: CuPermissionRequest): Promise 1)) && + (adapter.executor.capabilities.platform === "darwin" && + hasNonControlText)) && overrides.grantFlags.clipboardWrite && subGates.clipboardPasteMultiline;