diff --git a/desktop/src/api/computerUse.ts b/desktop/src/api/computerUse.ts new file mode 100644 index 00000000..bcd36501 --- /dev/null +++ b/desktop/src/api/computerUse.ts @@ -0,0 +1,76 @@ +import { api } from './client' + +export type ComputerUseStatus = { + platform: string + supported: boolean + python: { + installed: boolean + version: string | null + path: string | null + } + venv: { + created: boolean + path: string + } + dependencies: { + installed: boolean + requirementsFound: boolean + } + permissions: { + accessibility: boolean | null + screenRecording: boolean | null + } +} + +export type SetupStep = { + name: string + ok: boolean + message: string +} + +export type SetupResult = { + success: boolean + steps: SetupStep[] +} + +export type InstalledApp = { + bundleId: string + displayName: string + path: string +} + +export type AuthorizedApp = { + bundleId: string + displayName: string + authorizedAt: string +} + +export type ComputerUseConfig = { + authorizedApps: AuthorizedApp[] + grantFlags: { + clipboardRead: boolean + clipboardWrite: boolean + systemKeyCombos: boolean + } +} + +export const computerUseApi = { + getStatus() { + return api.get('/api/computer-use/status') + }, + runSetup() { + return api.post('/api/computer-use/setup', undefined, { timeout: 300_000 }) + }, + getInstalledApps() { + return api.get<{ apps: InstalledApp[] }>('/api/computer-use/apps') + }, + getAuthorizedApps() { + return api.get('/api/computer-use/authorized-apps') + }, + setAuthorizedApps(config: Partial) { + return api.put<{ ok: true }>('/api/computer-use/authorized-apps', config) + }, + openSettings(pane: 'Privacy_ScreenCapture' | 'Privacy_Accessibility') { + return api.post<{ ok: true }>('/api/computer-use/open-settings', { pane }) + }, +} diff --git a/desktop/src/pages/ComputerUseSettings.tsx b/desktop/src/pages/ComputerUseSettings.tsx new file mode 100644 index 00000000..fda18e60 --- /dev/null +++ b/desktop/src/pages/ComputerUseSettings.tsx @@ -0,0 +1,420 @@ +import { useState, useEffect, useCallback, useMemo } from 'react' +import { computerUseApi, type ComputerUseStatus, type SetupResult, type InstalledApp, type AuthorizedApp } from '../api/computerUse' +import { useTranslation } from '../i18n' + +type CheckState = 'loading' | 'ready' | 'error' + +function StatusIcon({ ok }: { ok: boolean | null }) { + if (ok === null) { + return help + } + return ok ? ( + check_circle + ) : ( + cancel + ) +} + +function StatusRow({ label, ok, detail }: { label: string; ok: boolean | null; detail: string }) { + return ( +
+ +
+ {label} + {detail} +
+
+ ) +} + +async function openSystemSettings(pane: 'Privacy_ScreenCapture' | 'Privacy_Accessibility') { + await computerUseApi.openSettings(pane) +} + +export function ComputerUseSettings() { + const t = useTranslation() + const [status, setStatus] = useState(null) + const [checkState, setCheckState] = useState('loading') + const [setupRunning, setSetupRunning] = useState(false) + const [setupResult, setSetupResult] = useState(null) + + // App authorization state + const [installedApps, setInstalledApps] = useState([]) + const [authorizedBundleIds, setAuthorizedBundleIds] = useState>(new Set()) + const [authorizedApps, setAuthorizedApps] = useState([]) + const [appsLoading, setAppsLoading] = useState(false) + const [appsSaved, setAppsSaved] = useState(false) + const [searchQuery, setSearchQuery] = useState('') + const [clipboardAccess, setClipboardAccess] = useState(true) + const [systemKeys, setSystemKeys] = useState(true) + + const fetchStatus = useCallback(async () => { + setCheckState('loading') + try { + const s = await computerUseApi.getStatus() + setStatus(s) + setCheckState('ready') + } catch { + setCheckState('error') + } + }, []) + + const fetchApps = useCallback(async () => { + setAppsLoading(true) + try { + const [appsResult, configResult] = await Promise.all([ + computerUseApi.getInstalledApps(), + computerUseApi.getAuthorizedApps(), + ]) + setInstalledApps(appsResult.apps) + setAuthorizedApps(configResult.authorizedApps) + setAuthorizedBundleIds(new Set(configResult.authorizedApps.map(a => a.bundleId))) + setClipboardAccess(configResult.grantFlags.clipboardRead) + setSystemKeys(configResult.grantFlags.systemKeyCombos) + } catch { + // API not ready + } finally { + setAppsLoading(false) + } + }, []) + + useEffect(() => { + fetchStatus() + }, [fetchStatus]) + + // Load apps when environment is ready + const envReady = status?.venv.created && status?.dependencies.installed + useEffect(() => { + if (envReady) fetchApps() + }, [envReady, fetchApps]) + + const handleSetup = async () => { + setSetupRunning(true) + setSetupResult(null) + try { + const result = await computerUseApi.runSetup() + setSetupResult(result) + await fetchStatus() + if (result.success) await fetchApps() + } catch { + setSetupResult({ success: false, steps: [{ name: 'error', ok: false, message: 'Request failed' }] }) + } finally { + setSetupRunning(false) + } + } + + const toggleApp = (app: InstalledApp) => { + const newSet = new Set(authorizedBundleIds) + let newAuthorized = [...authorizedApps] + if (newSet.has(app.bundleId)) { + newSet.delete(app.bundleId) + newAuthorized = newAuthorized.filter(a => a.bundleId !== app.bundleId) + } else { + newSet.add(app.bundleId) + newAuthorized.push({ + bundleId: app.bundleId, + displayName: app.displayName, + authorizedAt: new Date().toISOString(), + }) + } + setAuthorizedBundleIds(newSet) + setAuthorizedApps(newAuthorized) + + // Auto-save + computerUseApi.setAuthorizedApps({ + authorizedApps: newAuthorized, + grantFlags: { clipboardRead: clipboardAccess, clipboardWrite: clipboardAccess, systemKeyCombos: systemKeys }, + }).then(() => { + setAppsSaved(true) + setTimeout(() => setAppsSaved(false), 1500) + }) + } + + const toggleFlag = (flag: 'clipboard' | 'systemKeys', value: boolean) => { + if (flag === 'clipboard') setClipboardAccess(value) + else setSystemKeys(value) + + computerUseApi.setAuthorizedApps({ + authorizedApps, + grantFlags: { + clipboardRead: flag === 'clipboard' ? value : clipboardAccess, + clipboardWrite: flag === 'clipboard' ? value : clipboardAccess, + systemKeyCombos: flag === 'systemKeys' ? value : systemKeys, + }, + }) + } + + const allReady = + status?.supported && + status.python.installed && + status.venv.created && + status.dependencies.installed + + const accessibilityNeedsAttention = status?.permissions.accessibility === false + const screenRecordingNeedsAttention = status?.permissions.screenRecording === false + const screenRecordingReady = status ? status.permissions.screenRecording !== false : null + + // Filter apps by search query + const filteredApps = useMemo(() => { + if (!searchQuery) return installedApps + const q = searchQuery.toLowerCase() + return installedApps.filter( + a => a.displayName.toLowerCase().includes(q) || a.bundleId.toLowerCase().includes(q) + ) + }, [installedApps, searchQuery]) + + // Sort: authorized apps first, then alphabetical + const sortedApps = useMemo(() => { + return [...filteredApps].sort((a, b) => { + const aAuth = authorizedBundleIds.has(a.bundleId) ? 0 : 1 + const bAuth = authorizedBundleIds.has(b.bundleId) ? 0 : 1 + if (aAuth !== bAuth) return aAuth - bAuth + return a.displayName.localeCompare(b.displayName) + }) + }, [filteredApps, authorizedBundleIds]) + + return ( +
+ {/* Title */} +
+

+ {t('settings.computerUse.title')} +

+

+ {t('settings.computerUse.description')} +

+
+ + {checkState === 'loading' ? ( +
+ {t('common.loading')} +
+ ) : checkState === 'error' ? ( +
+ Failed to check status. + +
+ ) : status ? ( + <> + {!status.supported && ( +
+ {t('settings.computerUse.notSupported')} +
+ )} + + {/* Status checks */} +
+ + + +
+ + {/* macOS Permissions */} + {envReady && ( + <> + + + {(accessibilityNeedsAttention || screenRecordingNeedsAttention) && ( +
+

{t('settings.computerUse.permRestartHint')}

+
+ {accessibilityNeedsAttention && ( + + )} + {screenRecordingNeedsAttention && ( + + )} +
+
+ )} + + )} + + {allReady && status.permissions.accessibility && screenRecordingReady && ( +
+ verified + {t('settings.computerUse.allReady')} +
+ )} + + {setupResult && ( +
+
+ {setupResult.success ? t('settings.computerUse.setupSuccess') : t('settings.computerUse.setupFail')} +
+ {setupResult.steps.map((step, i) => ( +
+ + {step.message} +
+ ))} +
+ )} + + {/* Action buttons */} +
+ {!envReady && status.python.installed && ( + + )} + +
+ + {/* ─── App Authorization Section ─── */} + {envReady && ( +
+
+

+ {t('settings.computerUse.appsTitle')} + {appsSaved && ( + + check + {t('settings.computerUse.appsSaved')} + + )} +

+

+ {t('settings.computerUse.appsDescription')} +

+
+ + {/* Grant flags */} +
+ + +
+ + {/* Search */} +
+ search + setSearchQuery(e.target.value)} + placeholder={t('settings.computerUse.appsSearch')} + className="w-full pl-9 pr-4 py-2 text-sm bg-[var(--color-surface-container-low)] border border-[var(--color-border)] rounded-lg text-[var(--color-text-primary)] placeholder:text-[var(--color-text-tertiary)] focus:outline-none focus:border-[var(--color-brand)]" + /> +
+ + {/* App list */} + {appsLoading ? ( +
+ {t('settings.computerUse.appsLoading')} +
+ ) : installedApps.length === 0 ? ( +
+ {t('settings.computerUse.appsEmpty')} +
+ ) : ( +
+ {sortedApps.map(app => { + const isAuthorized = authorizedBundleIds.has(app.bundleId) + return ( + + ) + })} +
+ )} +
+ )} + + ) : null} +
+ ) +} diff --git a/runtime/mac_helper.py b/runtime/mac_helper.py index 20c7a52f..ceb4fd27 100755 --- a/runtime/mac_helper.py +++ b/runtime/mac_helper.py @@ -28,6 +28,7 @@ from Quartz import ( CGRectContainsPoint, CGRectIntersection, CGPointMake, + CGPreflightScreenCaptureAccess, kCGNullWindowID, kCGWindowBounds, kCGWindowIsOnscreen, @@ -232,7 +233,16 @@ def choose_display(display_id: int | None) -> dict[str, Any]: raise RuntimeError(f"Unknown display: {display_id}") +def ensure_screen_recording_permission() -> None: + """No-op: CGPreflightScreenCaptureAccess is unreliable for child processes + (returns False even when the parent app has TCC permission), and any actual + capture attempt triggers a macOS popup on newer versions. Let the actual + capture call handle errors instead.""" + pass + + def capture_display(display_id: int | None, resize: tuple[int, int] | None = None) -> dict[str, Any]: + ensure_screen_recording_permission() display = choose_display(display_id) monitor = { "left": display["originX"], @@ -262,6 +272,7 @@ def capture_display(display_id: int | None, resize: tuple[int, int] | None = Non def capture_region(region: dict[str, int], resize: tuple[int, int] | None = None) -> dict[str, Any]: + ensure_screen_recording_permission() with mss.mss() as sct: raw = sct.grab(region) image = Image.frombytes("RGB", raw.size, raw.rgb) @@ -461,17 +472,61 @@ def write_clipboard(text: str) -> None: pb.setString_forType_(text, NSPasteboardTypeString) -def check_permissions() -> dict[str, bool]: +def detect_screen_recording_permission() -> bool | None: + """Best-effort passive screen-recording probe with no system prompt. + + `CGPreflightScreenCaptureAccess()` is fast and explicit when it returns + True, but on child processes launched by a TCC-authorized app bundle it can + still return False. As a fallback, inspect the visible window list: Apple + only exposes other apps' window titles when Screen Recording access is + granted. If we can see at least one title, treat the permission as granted. + If we can inspect visible windows but every title is blank, treat it as not + granted. If window enumeration itself is unavailable, return None. + """ + + try: + if CGPreflightScreenCaptureAccess(): + return True + except Exception: + pass + + try: + windows = CGWindowListCopyWindowInfo( + kCGWindowListOptionOnScreenOnly | kCGWindowListExcludeDesktopElements, + kCGNullWindowID, + ) + except Exception: + return None + + eligible_windows = 0 + for window in windows or []: + if int(window.get(kCGWindowLayer, 0)) != 0: + continue + if not bool(window.get(kCGWindowIsOnscreen, True)): + continue + + bounds = window.get(kCGWindowBounds) or {} + width = int(bounds.get("Width", 0)) + height = int(bounds.get("Height", 0)) + if width <= 1 or height <= 1: + continue + + eligible_windows += 1 + if (window.get(kCGWindowName, "") or "").strip(): + return True + + if eligible_windows > 0: + return False + return None + + +def check_permissions() -> dict[str, bool | None]: accessibility = True try: run_osascript('tell application "System Events" to get name of first process') except Exception: accessibility = False - screen_recording = True - try: - capture_display(None) - except Exception: - screen_recording = False + screen_recording = detect_screen_recording_permission() return { "accessibility": accessibility, "screenRecording": screen_recording, diff --git a/src/server/api/computer-use.ts b/src/server/api/computer-use.ts new file mode 100644 index 00000000..94fe6a86 --- /dev/null +++ b/src/server/api/computer-use.ts @@ -0,0 +1,430 @@ +/** + * Computer Use API — 环境检测与依赖安装 + * + * Routes: + * GET /api/computer-use/status — 检测 Python3、venv、依赖、权限状态 + * POST /api/computer-use/setup — 创建 venv 并安装依赖 + */ + +import { homedir } from 'os' +import { join } from 'path' +import { access, readFile, mkdir, writeFile } from 'fs/promises' +import { createHash } from 'crypto' +import path from 'path' +import { fileURLToPath } from 'url' +// Embed mac_helper.py at compile time so it's available in bundled mode +// @ts-ignore — Bun text import +import MAC_HELPER_CONTENT from '../../../runtime/mac_helper.py' with { type: 'text' } + +const __dirname = path.dirname(fileURLToPath(import.meta.url)) +const projectRoot = path.resolve(__dirname, '../../..') +const devRuntimeRoot = join(projectRoot, 'runtime') +const claudeHome = process.env.CLAUDE_CONFIG_DIR ?? join(homedir(), '.claude') +const runtimeStateRoot = join(claudeHome, '.runtime') +const venvRoot = join(runtimeStateRoot, 'venv') +const installStampPath = join(runtimeStateRoot, 'requirements.sha256') + +// Embedded content of requirements.txt — kept in sync with runtime/requirements.txt. +// This ensures the bundled sidecar can create the file without disk access. +const REQUIREMENTS_CONTENT = `mss>=10.1.0 +Pillow>=11.3.0 +pyautogui>=0.9.54 +pyobjc-core>=11.1 +pyobjc-framework-Cocoa>=11.1 +pyobjc-framework-Quartz>=11.1 +` + +// 清华大学 PyPI 镜像，国内安装速度更快 +const PIP_INDEX_URL = 'https://pypi.tuna.tsinghua.edu.cn/simple/' +const PIP_TRUSTED_HOST = 'pypi.tuna.tsinghua.edu.cn' + +// Paths that resolve correctly in both dev and bundled modes +function getRequirementsPath(): string { + return join(runtimeStateRoot, 'requirements.txt') +} + +function getHelperPath(): string { + // In bundled mode mac_helper.py is extracted to runtimeStateRoot. + // In dev mode we also copy it there during setup, so both modes + // read from the same location after setup runs. + return join(runtimeStateRoot, 'mac_helper.py') +} + +async function pathExists(target: string): Promise { + try { + await access(target) + return true + } catch { + return false + } +} + +async function runCommand( + cmd: string, + args: string[], +): Promise<{ ok: boolean; stdout: string; stderr: string; code: number }> { + try { + const proc = Bun.spawn([cmd, ...args], { + stdout: 'pipe', + stderr: 'pipe', + }) + const [stdout, stderr] = await Promise.all([ + new Response(proc.stdout).text(), + new Response(proc.stderr).text(), + ]) + const code = await proc.exited + return { ok: code === 0, stdout: stdout.trim(), stderr: stderr.trim(), code } + } catch { + return { ok: false, stdout: '', stderr: `Failed to run ${cmd}`, code: -1 } + } +} + +/** + * Ensure runtime source files (requirements.txt, mac_helper.py) exist in + * ~/.claude/.runtime/. In dev mode they are copied from the project's + * runtime/ directory; in bundled mode requirements.txt is written from the + * embedded constant and mac_helper.py is copied from the project dir (if + * available) or skipped (it will already have been extracted on a prior run). + */ +async function ensureRuntimeFiles(): Promise { + await mkdir(runtimeStateRoot, { recursive: true }) + + // requirements.txt — always write from embedded constant (authoritative) + await writeFile(getRequirementsPath(), REQUIREMENTS_CONTENT, 'utf8') + + // mac_helper.py — always write from embedded content (compile-time import) + await writeFile(getHelperPath(), MAC_HELPER_CONTENT, 'utf8') +} + +type EnvStatus = { + platform: string + supported: boolean + python: { + installed: boolean + version: string | null + path: string | null + } + venv: { + created: boolean + path: string + } + dependencies: { + installed: boolean + requirementsFound: boolean + } + permissions: { + accessibility: boolean | null + screenRecording: boolean | null + } +} + +async function checkStatus(): Promise { + const platform = process.platform + const supported = platform === 'darwin' + + // Check Python 3 + const pythonResult = await runCommand('python3', ['--version']) + const pythonInstalled = pythonResult.ok + const pythonVersion = pythonInstalled + ? pythonResult.stdout.replace('Python ', '') + : null + + let pythonPath: string | null = null + if (pythonInstalled) { + const whichResult = await runCommand('which', ['python3']) + pythonPath = whichResult.ok ? whichResult.stdout : null + } + + // Check venv + const venvCreated = await pathExists(join(venvRoot, 'bin', 'python3')) + + // Check dependencies — use the state dir copy + const reqPath = getRequirementsPath() + const requirementsFound = await pathExists(reqPath) + let depsInstalled = false + if (requirementsFound && venvCreated) { + try { + const requirements = await readFile(reqPath, 'utf8') + const digest = createHash('sha256').update(requirements).digest('hex') + const stamp = (await readFile(installStampPath, 'utf8')).trim() + depsInstalled = stamp === digest + } catch { + depsInstalled = false + } + } + + // Check macOS permissions without triggering a system prompt. The helper + // uses preflight + visible-window metadata as a passive fallback because + // plain preflight can misreport child processes launched by the desktop app. + let accessibility: boolean | null = null + let screenRecording: boolean | null = null + if (supported && venvCreated && depsInstalled) { + try { await ensureRuntimeFiles() } catch {} + const helperPath = getHelperPath() + if (await pathExists(helperPath)) { + const pythonBin = join(venvRoot, 'bin', 'python3') + const permResult = await runCommand(pythonBin, [helperPath, 'check_permissions']) + if (permResult.ok) { + try { + const parsed = JSON.parse(permResult.stdout) + if (parsed.ok && parsed.result) { + accessibility = parsed.result.accessibility ?? null + screenRecording = parsed.result.screenRecording ?? null + } + } catch {} + } + } + } + + return { + platform, + supported, + python: { installed: pythonInstalled, version: pythonVersion, path: pythonPath }, + venv: { created: venvCreated, path: venvRoot }, + dependencies: { installed: depsInstalled, requirementsFound: requirementsFound || true }, + permissions: { accessibility, screenRecording }, + } +} + +type SetupResult = { + success: boolean + steps: { name: string; ok: boolean; message: string }[] +} + +async function runSetup(): Promise { + const steps: SetupResult['steps'] = [] + + // Step 1: Check python3 + const pythonCheck = await runCommand('python3', ['--version']) + if (!pythonCheck.ok) { + steps.push({ + name: 'python_check', + ok: false, + message: 'Python 3 未安装，请先安装 Python 3', + }) + return { success: false, steps } + } + steps.push({ + name: 'python_check', + ok: true, + message: `Python ${pythonCheck.stdout.replace('Python ', '')}`, + }) + + // Step 2: Extract runtime files to ~/.claude/.runtime/ + try { + await ensureRuntimeFiles() + steps.push({ name: 'runtime_files', ok: true, message: '运行时文件已就绪' }) + } catch (err) { + steps.push({ + name: 'runtime_files', + ok: false, + message: `提取运行时文件失败: ${err}`, + }) + return { success: false, steps } + } + + // Step 3: Create venv + const venvExists = await pathExists(join(venvRoot, 'bin', 'python3')) + if (!venvExists) { + const venvResult = await runCommand('python3', ['-m', 'venv', venvRoot]) + if (!venvResult.ok) { + steps.push({ + name: 'venv', + ok: false, + message: `创建虚拟环境失败: ${venvResult.stderr}`, + }) + return { success: false, steps } + } + steps.push({ name: 'venv', ok: true, message: '虚拟环境已创建' }) + } else { + steps.push({ name: 'venv', ok: true, message: '虚拟环境已存在' }) + } + + // Step 4: Ensure pip + const pipPath = join(venvRoot, 'bin', 'pip') + if (!(await pathExists(pipPath))) { + const pythonBin = join(venvRoot, 'bin', 'python3') + const pipResult = await runCommand(pythonBin, [ + '-m', + 'ensurepip', + '--upgrade', + ]) + if (!pipResult.ok) { + steps.push({ + name: 'pip', + ok: false, + message: `安装 pip 失败: ${pipResult.stderr}`, + }) + return { success: false, steps } + } + } + steps.push({ name: 'pip', ok: true, message: 'pip 已就绪' }) + + // Step 5: Install requirements + const reqPath = getRequirementsPath() + const requirements = await readFile(reqPath, 'utf8') + const digest = createHash('sha256').update(requirements).digest('hex') + + let installedDigest = '' + try { + installedDigest = (await readFile(installStampPath, 'utf8')).trim() + } catch {} + + if (installedDigest !== digest) { + const pythonBin = join(venvRoot, 'bin', 'python3') + + // Upgrade pip first (using China mirror) + await runCommand(pythonBin, [ + '-m', 'pip', 'install', '--upgrade', 'pip', + '-i', PIP_INDEX_URL, '--trusted-host', PIP_TRUSTED_HOST, + ]) + + // Install deps (using China mirror) + const installResult = await runCommand(pythonBin, [ + '-m', 'pip', 'install', + '-r', reqPath, + '-i', PIP_INDEX_URL, '--trusted-host', PIP_TRUSTED_HOST, + ]) + if (!installResult.ok) { + steps.push({ + name: 'deps', + ok: false, + message: `安装依赖失败: ${installResult.stderr.slice(0, 500)}`, + }) + return { success: false, steps } + } + await writeFile(installStampPath, `${digest}\n`, 'utf8') + steps.push({ name: 'deps', ok: true, message: '依赖已安装' }) + } else { + steps.push({ name: 'deps', ok: true, message: '依赖已是最新' }) + } + + return { success: true, steps } +} + +// ============================================================================ +// Authorized Apps configuration — stored in ~/.claude/cc-haha/computer-use-config.json +// ============================================================================ + +const configPath = join(claudeHome, 'cc-haha', 'computer-use-config.json') + +type AuthorizedApp = { + bundleId: string + displayName: string + authorizedAt: string +} + +type ComputerUseConfig = { + authorizedApps: AuthorizedApp[] + grantFlags: { + clipboardRead: boolean + clipboardWrite: boolean + systemKeyCombos: boolean + } +} + +const DEFAULT_CONFIG: ComputerUseConfig = { + authorizedApps: [], + grantFlags: { clipboardRead: true, clipboardWrite: true, systemKeyCombos: true }, +} + +async function loadConfig(): Promise { + try { + const raw = await readFile(configPath, 'utf8') + return { ...DEFAULT_CONFIG, ...JSON.parse(raw) } + } catch { + return { ...DEFAULT_CONFIG } + } +} + +async function saveConfig(config: ComputerUseConfig): Promise { + await writeFile(configPath, JSON.stringify(config, null, 2), 'utf8') +} + +async function listInstalledApps(): Promise<{ bundleId: string; displayName: string; path: string }[]> { + const helperPath = getHelperPath() + const pythonBin = join(venvRoot, 'bin', 'python3') + + if (!(await pathExists(pythonBin)) || !(await pathExists(helperPath))) { + return [] + } + + const result = await runCommand(pythonBin, [helperPath, 'list_installed_apps']) + if (!result.ok) return [] + + try { + const parsed = JSON.parse(result.stdout) + return parsed.ok ? parsed.result : [] + } catch { + return [] + } +} + +// ============================================================================ +// Route handler +// ============================================================================ + +export async function handleComputerUseApi( + req: Request, + _url: URL, + segments: string[], +): Promise { + const action = segments[2] + + if (action === 'status' && req.method === 'GET') { + const status = await checkStatus() + return Response.json(status) + } + + if (action === 'setup' && req.method === 'POST') { + const result = await runSetup() + return Response.json(result) + } + + // GET /api/computer-use/apps — list installed macOS apps + if (action === 'apps' && req.method === 'GET') { + const apps = await listInstalledApps() + return Response.json({ apps }) + } + + // GET /api/computer-use/authorized-apps — current authorized app config + if (action === 'authorized-apps' && req.method === 'GET') { + const config = await loadConfig() + return Response.json(config) + } + + // PUT /api/computer-use/authorized-apps — update authorized apps + if (action === 'authorized-apps' && req.method === 'PUT') { + try { + const body = (await req.json()) as Partial + const config = await loadConfig() + if (body.authorizedApps) config.authorizedApps = body.authorizedApps + if (body.grantFlags) config.grantFlags = { ...config.grantFlags, ...body.grantFlags } + await saveConfig(config) + return Response.json({ ok: true }) + } catch { + return Response.json({ error: 'Invalid JSON' }, { status: 400 }) + } + } + + // POST /api/computer-use/open-settings — open macOS System Settings pane + if (action === 'open-settings' && req.method === 'POST') { + if (process.platform !== 'darwin') { + return Response.json({ error: 'macOS only' }, { status: 400 }) + } + const body = (await req.json().catch(() => ({}))) as { pane?: string } + const pane = body.pane ?? 'Privacy_ScreenCapture' + const allowed = ['Privacy_ScreenCapture', 'Privacy_Accessibility'] + if (!allowed.includes(pane)) { + return Response.json({ error: 'Invalid pane' }, { status: 400 }) + } + const url = `x-apple.systempreferences:com.apple.preference.security?${pane}` + await runCommand('open', [url]) + return Response.json({ ok: true }) + } + + return Response.json( + { error: 'NOT_FOUND', message: `Unknown computer-use action: ${action}` }, + { status: 404 }, + ) +} diff --git a/src/utils/computerUse/hostAdapter.ts b/src/utils/computerUse/hostAdapter.ts index 3c24da4d..1c5d8ad0 100644 --- a/src/utils/computerUse/hostAdapter.ts +++ b/src/utils/computerUse/hostAdapter.ts @@ -7,6 +7,7 @@ import { logForDebugging } from '../debug.js' import { COMPUTER_USE_MCP_SERVER_NAME } from './common.js' import { createCliExecutor } from './executor.js' import { getChicagoEnabled, getChicagoSubGates } from './gates.js' +import { normalizeOsPermissions } from './permissions.js' import { callPythonHelper } from './pythonBridge.js' class DebugLogger implements Logger { @@ -39,8 +40,9 @@ export function getComputerUseHostAdapter(): ComputerUseHostAdapter { getHideBeforeActionEnabled: () => getChicagoSubGates().hideBeforeAction, }), ensureOsPermissions: async () => { - const perms = await callPythonHelper<{ accessibility: boolean; screenRecording: boolean }>('check_permissions', {}) - return perms.accessibility && perms.screenRecording + const rawPerms = await callPythonHelper<{ accessibility: boolean; screenRecording: boolean | null }>('check_permissions', {}) + const perms = normalizeOsPermissions(rawPerms) + return perms.granted ? { granted: true as const } : { granted: false as const, accessibility: perms.accessibility, screenRecording: perms.screenRecording } }, diff --git a/src/utils/computerUse/permissions.test.ts b/src/utils/computerUse/permissions.test.ts new file mode 100644 index 00000000..807bb6ee --- /dev/null +++ b/src/utils/computerUse/permissions.test.ts @@ -0,0 +1,44 @@ +import { describe, expect, it } from 'bun:test' +import { normalizeOsPermissions } from './permissions.js' + +describe('normalizeOsPermissions', () => { + it('treats explicit grants as granted', () => { + expect( + normalizeOsPermissions({ accessibility: true, screenRecording: true }), + ).toEqual({ + granted: true, + accessibility: true, + screenRecording: true, + }) + }) + + it('treats screen recording unknown as non-blocking when accessibility is granted', () => { + expect( + normalizeOsPermissions({ accessibility: true, screenRecording: null }), + ).toEqual({ + granted: true, + accessibility: true, + screenRecording: true, + }) + }) + + it('still blocks when accessibility is missing', () => { + expect( + normalizeOsPermissions({ accessibility: false, screenRecording: null }), + ).toEqual({ + granted: false, + accessibility: false, + screenRecording: true, + }) + }) + + it('blocks when screen recording is explicitly denied', () => { + expect( + normalizeOsPermissions({ accessibility: true, screenRecording: false }), + ).toEqual({ + granted: false, + accessibility: true, + screenRecording: false, + }) + }) +}) diff --git a/src/utils/computerUse/permissions.ts b/src/utils/computerUse/permissions.ts new file mode 100644 index 00000000..a6e18655 --- /dev/null +++ b/src/utils/computerUse/permissions.ts @@ -0,0 +1,25 @@ +export type RawOsPermissions = { + accessibility: boolean + screenRecording: boolean | null +} + +export type NormalizedOsPermissions = { + granted: boolean + accessibility: boolean + screenRecording: boolean +} + +/** + * macOS Screen Recording passive probes can come back "unknown" for helper + * child processes even when the app bundle is already authorized. Treat that + * state as non-blocking and let the actual capture path remain the final + * source of truth. + */ +export function normalizeOsPermissions(perms: RawOsPermissions): NormalizedOsPermissions { + const screenRecording = perms.screenRecording !== false + return { + granted: perms.accessibility && screenRecording, + accessibility: perms.accessibility, + screenRecording, + } +} diff --git a/src/utils/computerUse/pythonBridge.ts b/src/utils/computerUse/pythonBridge.ts index bfeb3231..02a710bc 100644 --- a/src/utils/computerUse/pythonBridge.ts +++ b/src/utils/computerUse/pythonBridge.ts @@ -4,16 +4,25 @@ import path from 'node:path' import { fileURLToPath } from 'node:url' import { execFileNoThrow } from '../execFileNoThrow.js' import { logForDebugging } from '../debug.js' +import { getClaudeConfigHomeDir } from '../envUtils.js' const __dirname = path.dirname(fileURLToPath(import.meta.url)) const projectRoot = path.resolve(__dirname, '../../..') -const runtimeRoot = path.join(projectRoot, 'runtime') -const runtimeStateRoot = path.join(projectRoot, '.runtime') + +// All runtime state lives in ~/.claude/.runtime — writable in both dev and +// bundled (Tauri app) modes. The setup API (or ensureRuntimeFiles below) +// populates requirements.txt and mac_helper.py here. +const runtimeStateRoot = path.join(getClaudeConfigHomeDir(), '.runtime') const venvRoot = path.join(runtimeStateRoot, 'venv') -const requirementsPath = path.join(runtimeRoot, 'requirements.txt') -const helperPath = path.join(runtimeRoot, 'mac_helper.py') const installStampPath = path.join(runtimeStateRoot, 'requirements.sha256') +const PIP_INDEX_URL = 'https://pypi.tuna.tsinghua.edu.cn/simple/' +const PIP_TRUSTED_HOST = 'pypi.tuna.tsinghua.edu.cn' + +// Always read from ~/.claude/.runtime/ — works in both dev and bundled mode. +const requirementsPath = path.join(runtimeStateRoot, 'requirements.txt') +const helperPath = path.join(runtimeStateRoot, 'mac_helper.py') + let bootstrapPromise: Promise | undefined function pythonBinPath(): string { @@ -37,10 +46,34 @@ async function runOrThrow(file: string, args: string[], label: string): Promise< return stdout } +/** + * Ensure runtime source files exist in ~/.claude/.runtime/. + * In dev mode, copies from the project's runtime/ directory on first run. + * In bundled mode, these must have been placed there by the settings setup API. + */ +async function ensureRuntimeFiles(): Promise { + await mkdir(runtimeStateRoot, { recursive: true }) + + const devRequirements = path.join(projectRoot, 'runtime', 'requirements.txt') + const devHelper = path.join(projectRoot, 'runtime', 'mac_helper.py') + + // Always sync from dev runtime/ so source changes are reflected immediately. + // Previously this only copied when the dest was missing, causing stale files + // to persist after source updates — breaking mouse/keyboard actions if the + // cached copy was from an older version. + if (await pathExists(devRequirements)) { + await writeFile(requirementsPath, await readFile(devRequirements, 'utf8'), 'utf8') + } + if (await pathExists(devHelper)) { + await writeFile(helperPath, await readFile(devHelper, 'utf8'), 'utf8') + } +} + export async function ensureBootstrapped(): Promise { if (bootstrapPromise) return bootstrapPromise bootstrapPromise = (async () => { - await mkdir(runtimeStateRoot, { recursive: true }) + // Extract runtime files (requirements.txt, mac_helper.py) to state dir + await ensureRuntimeFiles() if (!(await pathExists(pythonBinPath()))) { logForDebugging('creating runtime venv at %s', { level: 'debug' }) @@ -61,10 +94,14 @@ export async function ensureBootstrapped(): Promise { if (installedDigest !== digest) { logForDebugging('installing python runtime dependencies', { level: 'debug' }) - await runOrThrow(pythonBinPath(), ['-m', 'pip', 'install', '--upgrade', 'pip'], 'pip upgrade') + await runOrThrow(pythonBinPath(), [ + '-m', 'pip', 'install', '--upgrade', 'pip', + '-i', PIP_INDEX_URL, '--trusted-host', PIP_TRUSTED_HOST, + ], 'pip upgrade') await runOrThrow( pythonBinPath(), - ['-m', 'pip', 'install', '-r', requirementsPath], + ['-m', 'pip', 'install', '-r', requirementsPath, + '-i', PIP_INDEX_URL, '--trusted-host', PIP_TRUSTED_HOST], 'python dependency install', ) await writeFile(installStampPath, `${digest}\n`, 'utf8') @@ -105,6 +142,6 @@ export async function callPythonHelper(command: string, payload: Record['call']; type Binding = { ctx: ComputerUseSessionContext; @@ -227,6 +230,64 @@ export function buildSessionContext(): ComputerUseSessionContext { formatLockHeldMessage: formatLockHeld }; } +/** + * Load pre-authorized apps from ~/.claude/cc-haha/computer-use-config.json. + * Called once when the binding is first created. Pre-authorized apps + * are injected into appState so `getAllowedApps()` returns them + * immediately — no runtime permission dialog needed. + */ +async function loadPreAuthorizedApps(): Promise { + try { + const configPath = join( + process.env.CLAUDE_CONFIG_DIR ?? join(homedir(), '.claude'), + 'cc-haha', + 'computer-use-config.json', + ) + const raw = await readFile(configPath, 'utf8') + const config = JSON.parse(raw) as { + authorizedApps?: { bundleId: string; displayName: string }[] + grantFlags?: { clipboardRead?: boolean; clipboardWrite?: boolean; systemKeyCombos?: boolean } + } + + if (!config.authorizedApps?.length) return + + const apps = config.authorizedApps.map(a => ({ + bundleId: a.bundleId, + displayName: a.displayName, + grantedAt: Date.now(), + tier: 'full' as const, + })) + const flags = { + ...DEFAULT_GRANT_FLAGS, + ...(config.grantFlags ?? {}), + } + + // Inject into appState so getAllowedApps() returns them. + // Merge with existing allowedApps (from permission dialog) instead of replacing. + if (currentToolUseContext) { + currentToolUseContext.setAppState(prev => { + const existing = prev.computerUseMcpState?.allowedApps ?? [] + const existingIds = new Set(existing.map(a => a.bundleId)) + const merged = [...existing, ...apps.filter(a => !existingIds.has(a.bundleId))] + return { + ...prev, + computerUseMcpState: { + ...prev.computerUseMcpState, + allowedApps: merged, + grantFlags: flags, + }, + } + }) + } + + logForDebugging(`[Computer Use] Loaded ${apps.length} pre-authorized apps from config`) + } catch { + // Config doesn't exist or is invalid — no pre-authorized apps + } +} + +let preAuthLoaded = false + function getOrBind(): Binding { if (binding) return binding; const ctx = buildSessionContext(); @@ -251,6 +312,12 @@ export function getComputerUseMCPToolOverrides(toolName: string): ComputerUseMCP const { dispatch } = getOrBind(); + + // Load pre-authorized apps on first tool call + if (!preAuthLoaded) { + preAuthLoaded = true; + await loadPreAuthorizedApps(); + } const { telemetry, ...result