cc-haha/src/server/api/computer-use.ts
程序员阿江(Relakkes) 5cd6b5d07b Prevent desktop Computer Use from stalling on missing approvals and unstable text input
Desktop sessions were missing a visible request_access approval path and could
mis-detect their own app window as an unapproved frontmost target, which caused
Computer Use clicks to fail even after opening the intended app. On macOS, text
entry was also split across inconsistent clipboard and keystroke paths, making
Electron inputs unreliable for Chinese and short strings.

This change adds a desktop approval bridge over the existing session websocket,
renders a dedicated desktop approval modal, threads the real desktop bundle id
into the Computer Use executor, and switches macOS clipboard typing onto the
native pasteboard plus system paste shortcut path. It also makes tool error
results expandable in the desktop chat UI so frontmost-gate failures are fully
visible during debugging.

Constraint: Desktop sessions run the CLI over the SDK websocket path, so Ink tool JSX dialogs are not visible there
Constraint: macOS IME and Electron text inputs are unreliable with pyautogui.write and generic hotkey synthesis
Rejected: Reuse CLI setToolJSX dialogs in desktop mode | no transport for mid-call Ink UI over the SDK bridge
Rejected: Keep shell pbcopy/pbpaste for clipboard typing | inconsistent with NSPasteboard path and less reliable for Chinese text
Confidence: medium
Scope-risk: moderate
Reversibility: clean
Directive: Keep desktop Computer Use approvals and macOS text-entry behavior on a single bridge/path; avoid reintroducing separate CLI-only and desktop-only codepaths for the same action
Tested: python3 -m unittest runtime/test_helpers.py
Tested: bun test src/utils/computerUse/permissions.test.ts src/server/__tests__/conversation-service.test.ts
Tested: cd desktop && bun run test ComputerUsePermissionModal chatStore
Tested: cd desktop && bun run test chatBlocks
Tested: cd desktop && bun run lint
Not-tested: End-to-end manual Computer Use interaction against a live Electron target app on macOS
2026-04-20 12:12:02 +08:00

491 lines
16 KiB
TypeScript

/**
* Computer Use API — 环境检测与依赖安装
*
* Routes:
* GET /api/computer-use/status — 检测 Python3、venv、依赖、权限状态
* POST /api/computer-use/setup — 创建 venv 并安装依赖
*/
import { homedir } from 'os'
import { join } from 'path'
import { access, readFile, mkdir, writeFile } from 'fs/promises'
import { createHash } from 'crypto'
import path from 'path'
import { fileURLToPath } from 'url'
import type { CuPermissionRequest } from '../../vendor/computer-use-mcp/types.js'
import { computerUseApprovalService } from '../services/computerUseApprovalService.js'
// Embed helper scripts at compile time so they're available in bundled mode
// @ts-ignore — Bun text import
import MAC_HELPER_CONTENT from '../../../runtime/mac_helper.py' with { type: 'text' }
// @ts-ignore — Bun text import
import WIN_HELPER_CONTENT from '../../../runtime/win_helper.py' with { type: 'text' }
const __dirname = path.dirname(fileURLToPath(import.meta.url))
const projectRoot = path.resolve(__dirname, '../../..')
const devRuntimeRoot = join(projectRoot, 'runtime')
const claudeHome = process.env.CLAUDE_CONFIG_DIR ?? join(homedir(), '.claude')
const runtimeStateRoot = join(claudeHome, '.runtime')
const venvRoot = join(runtimeStateRoot, 'venv')
const installStampPath = join(runtimeStateRoot, 'requirements.sha256')
// Embedded content of requirements — platform-specific.
const REQUIREMENTS_DARWIN = `mss>=10.1.0
Pillow>=11.3.0
pyautogui>=0.9.54
pyobjc-core>=11.1
pyobjc-framework-Cocoa>=11.1
pyobjc-framework-Quartz>=11.1
`
const REQUIREMENTS_WIN32 = `mss>=10.1.0
Pillow>=11.3.0
pyautogui>=0.9.54
pywin32>=306
psutil>=5.9.0
pyperclip>=1.8.2
screeninfo>=0.8.1
`
const isWindows = process.platform === 'win32'
const REQUIREMENTS_CONTENT = isWindows ? REQUIREMENTS_WIN32 : REQUIREMENTS_DARWIN
// 清华大学 PyPI 镜像,国内安装速度更快
const PIP_INDEX_URL = 'https://pypi.tuna.tsinghua.edu.cn/simple/'
const PIP_TRUSTED_HOST = 'pypi.tuna.tsinghua.edu.cn'
// Paths that resolve correctly in both dev and bundled modes
function getRequirementsPath(): string {
return join(runtimeStateRoot, 'requirements.txt')
}
function getHelperFileName(): string {
return isWindows ? 'win_helper.py' : 'mac_helper.py'
}
function getHelperPath(): string {
return join(runtimeStateRoot, getHelperFileName())
}
async function pathExists(target: string): Promise<boolean> {
try {
await access(target)
return true
} catch {
return false
}
}
async function runCommand(
cmd: string,
args: string[],
): Promise<{ ok: boolean; stdout: string; stderr: string; code: number }> {
try {
const proc = Bun.spawn([cmd, ...args], {
stdout: 'pipe',
stderr: 'pipe',
})
const [stdout, stderr] = await Promise.all([
new Response(proc.stdout).text(),
new Response(proc.stderr).text(),
])
const code = await proc.exited
return { ok: code === 0, stdout: stdout.trim(), stderr: stderr.trim(), code }
} catch {
return { ok: false, stdout: '', stderr: `Failed to run ${cmd}`, code: -1 }
}
}
/**
* Ensure runtime source files (requirements.txt, mac_helper.py) exist in
* ~/.claude/.runtime/. In dev mode they are copied from the project's
* runtime/ directory; in bundled mode requirements.txt is written from the
* embedded constant and mac_helper.py is copied from the project dir (if
* available) or skipped (it will already have been extracted on a prior run).
*/
async function ensureRuntimeFiles(): Promise<void> {
await mkdir(runtimeStateRoot, { recursive: true })
// requirements.txt — always write from embedded constant (authoritative)
await writeFile(getRequirementsPath(), REQUIREMENTS_CONTENT, 'utf8')
// helper script — write the platform-appropriate version
const helperContent = isWindows ? WIN_HELPER_CONTENT : MAC_HELPER_CONTENT
await writeFile(getHelperPath(), helperContent, 'utf8')
}
type EnvStatus = {
platform: string
supported: boolean
python: {
installed: boolean
version: string | null
path: string | null
}
venv: {
created: boolean
path: string
}
dependencies: {
installed: boolean
requirementsFound: boolean
}
permissions: {
accessibility: boolean | null
screenRecording: boolean | null
}
}
async function checkStatus(): Promise<EnvStatus> {
const platform = process.platform
const supported = platform === 'darwin' || platform === 'win32'
// Check Python 3 — Windows may only have `python`, not `python3`
const pythonCmd = isWindows ? 'python' : 'python3'
const pythonResult = await runCommand(pythonCmd, ['--version'])
const pythonInstalled = pythonResult.ok
const pythonVersion = pythonInstalled
? pythonResult.stdout.replace('Python ', '')
: null
let pythonPath: string | null = null
if (pythonInstalled) {
const whichCmd = isWindows ? 'where' : 'which'
const whichResult = await runCommand(whichCmd, [pythonCmd])
pythonPath = whichResult.ok ? whichResult.stdout.split('\n')[0] : null
}
// Check venv — different paths on Windows vs Unix
const venvPython = isWindows
? join(venvRoot, 'Scripts', 'python.exe')
: join(venvRoot, 'bin', 'python3')
const venvCreated = await pathExists(venvPython)
// Check dependencies — use the state dir copy
const reqPath = getRequirementsPath()
const requirementsFound = await pathExists(reqPath)
let depsInstalled = false
if (requirementsFound && venvCreated) {
try {
const requirements = await readFile(reqPath, 'utf8')
const digest = createHash('sha256').update(requirements).digest('hex')
const stamp = (await readFile(installStampPath, 'utf8')).trim()
depsInstalled = stamp === digest
} catch {
depsInstalled = false
}
}
// Check macOS permissions without triggering a system prompt. The helper
// uses preflight + visible-window metadata as a passive fallback because
// plain preflight can misreport child processes launched by the desktop app.
let accessibility: boolean | null = null
let screenRecording: boolean | null = null
if (supported && venvCreated && depsInstalled) {
try { await ensureRuntimeFiles() } catch {}
const helperPath = getHelperPath()
if (await pathExists(helperPath)) {
const permResult = await runCommand(venvPython, [helperPath, 'check_permissions'])
if (permResult.ok) {
try {
const parsed = JSON.parse(permResult.stdout)
if (parsed.ok && parsed.result) {
accessibility = parsed.result.accessibility ?? null
screenRecording = parsed.result.screenRecording ?? null
}
} catch {}
}
}
}
return {
platform,
supported,
python: { installed: pythonInstalled, version: pythonVersion, path: pythonPath },
venv: { created: venvCreated, path: venvRoot },
dependencies: { installed: depsInstalled, requirementsFound: requirementsFound || true },
permissions: { accessibility, screenRecording },
}
}
type SetupResult = {
success: boolean
steps: { name: string; ok: boolean; message: string }[]
}
async function runSetup(): Promise<SetupResult> {
const steps: SetupResult['steps'] = []
// Step 1: Check python
const pythonCmd = isWindows ? 'python' : 'python3'
const pythonCheck = await runCommand(pythonCmd, ['--version'])
if (!pythonCheck.ok) {
steps.push({
name: 'python_check',
ok: false,
message: 'Python 3 未安装,请先安装 Python 3',
})
return { success: false, steps }
}
steps.push({
name: 'python_check',
ok: true,
message: `Python ${pythonCheck.stdout.replace('Python ', '')}`,
})
// Step 2: Extract runtime files to ~/.claude/.runtime/
try {
await ensureRuntimeFiles()
steps.push({ name: 'runtime_files', ok: true, message: '运行时文件已就绪' })
} catch (err) {
steps.push({
name: 'runtime_files',
ok: false,
message: `提取运行时文件失败: ${err}`,
})
return { success: false, steps }
}
// Step 3: Create venv
const venvPython = isWindows
? join(venvRoot, 'Scripts', 'python.exe')
: join(venvRoot, 'bin', 'python3')
const venvExists = await pathExists(venvPython)
if (!venvExists) {
const venvResult = await runCommand(pythonCmd, ['-m', 'venv', venvRoot])
if (!venvResult.ok) {
steps.push({
name: 'venv',
ok: false,
message: `创建虚拟环境失败: ${venvResult.stderr}`,
})
return { success: false, steps }
}
steps.push({ name: 'venv', ok: true, message: '虚拟环境已创建' })
} else {
steps.push({ name: 'venv', ok: true, message: '虚拟环境已存在' })
}
// Step 4: Ensure pip
const pipPath = isWindows
? join(venvRoot, 'Scripts', 'pip.exe')
: join(venvRoot, 'bin', 'pip')
if (!(await pathExists(pipPath))) {
const pipResult = await runCommand(venvPython, [
'-m',
'ensurepip',
'--upgrade',
])
if (!pipResult.ok) {
steps.push({
name: 'pip',
ok: false,
message: `安装 pip 失败: ${pipResult.stderr}`,
})
return { success: false, steps }
}
}
steps.push({ name: 'pip', ok: true, message: 'pip 已就绪' })
// Step 5: Install requirements
const reqPath = getRequirementsPath()
const requirements = await readFile(reqPath, 'utf8')
const digest = createHash('sha256').update(requirements).digest('hex')
let installedDigest = ''
try {
installedDigest = (await readFile(installStampPath, 'utf8')).trim()
} catch {}
if (installedDigest !== digest) {
// Upgrade pip first (using China mirror)
await runCommand(venvPython, [
'-m', 'pip', 'install', '--upgrade', 'pip',
'-i', PIP_INDEX_URL, '--trusted-host', PIP_TRUSTED_HOST,
])
// Install deps (using China mirror)
const installResult = await runCommand(venvPython, [
'-m', 'pip', 'install',
'-r', reqPath,
'-i', PIP_INDEX_URL, '--trusted-host', PIP_TRUSTED_HOST,
])
if (!installResult.ok) {
steps.push({
name: 'deps',
ok: false,
message: `安装依赖失败: ${installResult.stderr.slice(0, 500)}`,
})
return { success: false, steps }
}
await writeFile(installStampPath, `${digest}\n`, 'utf8')
steps.push({ name: 'deps', ok: true, message: '依赖已安装' })
} else {
steps.push({ name: 'deps', ok: true, message: '依赖已是最新' })
}
return { success: true, steps }
}
// ============================================================================
// Authorized Apps configuration — stored in ~/.claude/cc-haha/computer-use-config.json
// ============================================================================
const configPath = join(claudeHome, 'cc-haha', 'computer-use-config.json')
type AuthorizedApp = {
bundleId: string
displayName: string
authorizedAt: string
}
type ComputerUseConfig = {
authorizedApps: AuthorizedApp[]
grantFlags: {
clipboardRead: boolean
clipboardWrite: boolean
systemKeyCombos: boolean
}
}
type RequestAccessBody = {
sessionId?: string
request?: CuPermissionRequest
}
const DEFAULT_CONFIG: ComputerUseConfig = {
authorizedApps: [],
grantFlags: { clipboardRead: true, clipboardWrite: true, systemKeyCombos: true },
}
async function loadConfig(): Promise<ComputerUseConfig> {
try {
const raw = await readFile(configPath, 'utf8')
return { ...DEFAULT_CONFIG, ...JSON.parse(raw) }
} catch {
return { ...DEFAULT_CONFIG }
}
}
async function saveConfig(config: ComputerUseConfig): Promise<void> {
await writeFile(configPath, JSON.stringify(config, null, 2), 'utf8')
}
async function listInstalledApps(): Promise<{ bundleId: string; displayName: string; path: string }[]> {
const helperPath = getHelperPath()
const pythonBin = isWindows
? join(venvRoot, 'Scripts', 'python.exe')
: join(venvRoot, 'bin', 'python3')
if (!(await pathExists(pythonBin)) || !(await pathExists(helperPath))) {
return []
}
const result = await runCommand(pythonBin, [helperPath, 'list_installed_apps'])
if (!result.ok) return []
try {
const parsed = JSON.parse(result.stdout)
return parsed.ok ? parsed.result : []
} catch {
return []
}
}
// ============================================================================
// Route handler
// ============================================================================
export async function handleComputerUseApi(
req: Request,
_url: URL,
segments: string[],
): Promise<Response> {
const action = segments[2]
if (action === 'status' && req.method === 'GET') {
const status = await checkStatus()
return Response.json(status)
}
if (action === 'setup' && req.method === 'POST') {
const result = await runSetup()
return Response.json(result)
}
// GET /api/computer-use/apps — list installed macOS apps
if (action === 'apps' && req.method === 'GET') {
const apps = await listInstalledApps()
return Response.json({ apps })
}
// GET /api/computer-use/authorized-apps — current authorized app config
if (action === 'authorized-apps' && req.method === 'GET') {
const config = await loadConfig()
return Response.json(config)
}
// PUT /api/computer-use/authorized-apps — update authorized apps
if (action === 'authorized-apps' && req.method === 'PUT') {
try {
const body = (await req.json()) as Partial<ComputerUseConfig>
const config = await loadConfig()
if (body.authorizedApps) config.authorizedApps = body.authorizedApps
if (body.grantFlags) config.grantFlags = { ...config.grantFlags, ...body.grantFlags }
await saveConfig(config)
return Response.json({ ok: true })
} catch {
return Response.json({ error: 'Invalid JSON' }, { status: 400 })
}
}
// POST /api/computer-use/open-settings — open system settings pane
if (action === 'open-settings' && req.method === 'POST') {
const body = (await req.json().catch(() => ({}))) as { pane?: string }
const pane = body.pane ?? 'Privacy_ScreenCapture'
const allowed = ['Privacy_ScreenCapture', 'Privacy_Accessibility']
if (!allowed.includes(pane)) {
return Response.json({ error: 'Invalid pane' }, { status: 400 })
}
if (process.platform === 'darwin') {
const url = `x-apple.systempreferences:com.apple.preference.security?${pane}`
await runCommand('open', [url])
} else if (process.platform === 'win32') {
// Windows doesn't need privacy settings like macOS TCC, but we can
// open the general privacy page if requested
await runCommand('cmd', ['/c', 'start', 'ms-settings:privacy'])
} else {
return Response.json({ error: 'Unsupported platform' }, { status: 400 })
}
return Response.json({ ok: true })
}
if (action === 'request-access' && req.method === 'POST') {
try {
const body = (await req.json()) as RequestAccessBody
if (!body.sessionId || !body.request?.requestId) {
return Response.json(
{ error: 'BAD_REQUEST', message: 'sessionId and request are required' },
{ status: 400 },
)
}
const response = await computerUseApprovalService.requestApproval(
body.sessionId,
body.request,
)
return Response.json(response)
} catch (error) {
const message =
error instanceof Error ? error.message : 'Computer Use approval failed'
const status = message.includes('not connected') ? 409 : 500
return Response.json({ error: 'COMPUTER_USE_APPROVAL_FAILED', message }, { status })
}
}
return Response.json(
{ error: 'NOT_FOUND', message: `Unknown computer-use action: ${action}` },
{ status: 404 },
)
}