mirror of
https://github.com/NanmiCoder/cc-haha
synced 2026-07-15 12:53:31 +08:00
fix: Computer Use 点击功能修复
- ensureRuntimeFiles() 改为始终同步源文件而非仅缺失时复制 - pre-authorized apps 增加 tier: 'full'，与现有 allowedApps 合并而非替换 - normalizeOsPermissions() 将 screenRecording: null 视为非阻塞 - config 路径改为 cc-haha/computer-use-config.json - mac_helper.py 改进鼠标点击与坐标处理
This commit is contained in:
程序员阿江(Relakkes)
parent
b002447436
commit
166fe5b676
76
desktop/src/api/computerUse.ts
Normal file
76
desktop/src/api/computerUse.ts
Normal file
@ -0,0 +1,76 @@
|
||||
import { api } from './client'
|
||||
|
||||
export type ComputerUseStatus = {
|
||||
platform: string
|
||||
supported: boolean
|
||||
python: {
|
||||
installed: boolean
|
||||
version: string | null
|
||||
path: string | null
|
||||
}
|
||||
venv: {
|
||||
created: boolean
|
||||
path: string
|
||||
}
|
||||
dependencies: {
|
||||
installed: boolean
|
||||
requirementsFound: boolean
|
||||
}
|
||||
permissions: {
|
||||
accessibility: boolean | null
|
||||
screenRecording: boolean | null
|
||||
}
|
||||
}
|
||||
|
||||
export type SetupStep = {
|
||||
name: string
|
||||
ok: boolean
|
||||
message: string
|
||||
}
|
||||
|
||||
export type SetupResult = {
|
||||
success: boolean
|
||||
steps: SetupStep[]
|
||||
}
|
||||
|
||||
export type InstalledApp = {
|
||||
bundleId: string
|
||||
displayName: string
|
||||
path: string
|
||||
}
|
||||
|
||||
export type AuthorizedApp = {
|
||||
bundleId: string
|
||||
displayName: string
|
||||
authorizedAt: string
|
||||
}
|
||||
|
||||
export type ComputerUseConfig = {
|
||||
authorizedApps: AuthorizedApp[]
|
||||
grantFlags: {
|
||||
clipboardRead: boolean
|
||||
clipboardWrite: boolean
|
||||
systemKeyCombos: boolean
|
||||
}
|
||||
}
|
||||
|
||||
export const computerUseApi = {
|
||||
getStatus() {
|
||||
return api.get<ComputerUseStatus>('/api/computer-use/status')
|
||||
},
|
||||
runSetup() {
|
||||
return api.post<SetupResult>('/api/computer-use/setup', undefined, { timeout: 300_000 })
|
||||
},
|
||||
getInstalledApps() {
|
||||
return api.get<{ apps: InstalledApp[] }>('/api/computer-use/apps')
|
||||
},
|
||||
getAuthorizedApps() {
|
||||
return api.get<ComputerUseConfig>('/api/computer-use/authorized-apps')
|
||||
},
|
||||
setAuthorizedApps(config: Partial<ComputerUseConfig>) {
|
||||
return api.put<{ ok: true }>('/api/computer-use/authorized-apps', config)
|
||||
},
|
||||
openSettings(pane: 'Privacy_ScreenCapture' | 'Privacy_Accessibility') {
|
||||
return api.post<{ ok: true }>('/api/computer-use/open-settings', { pane })
|
||||
},
|
||||
}
|
||||
420
desktop/src/pages/ComputerUseSettings.tsx
Normal file
420
desktop/src/pages/ComputerUseSettings.tsx
Normal file
@ -0,0 +1,420 @@
|
||||
import { useState, useEffect, useCallback, useMemo } from 'react'
|
||||
import { computerUseApi, type ComputerUseStatus, type SetupResult, type InstalledApp, type AuthorizedApp } from '../api/computerUse'
|
||||
import { useTranslation } from '../i18n'
|
||||
|
||||
type CheckState = 'loading' | 'ready' | 'error'
|
||||
|
||||
function StatusIcon({ ok }: { ok: boolean | null }) {
|
||||
if (ok === null) {
|
||||
return <span className="material-symbols-outlined text-[18px] text-[var(--color-text-tertiary)]">help</span>
|
||||
}
|
||||
return ok ? (
|
||||
<span className="material-symbols-outlined text-[18px] text-green-500" style={{ fontVariationSettings: "'FILL' 1" }}>check_circle</span>
|
||||
) : (
|
||||
<span className="material-symbols-outlined text-[18px] text-red-400" style={{ fontVariationSettings: "'FILL' 1" }}>cancel</span>
|
||||
)
|
||||
}
|
||||
|
||||
function StatusRow({ label, ok, detail }: { label: string; ok: boolean | null; detail: string }) {
|
||||
return (
|
||||
<div className="flex items-center gap-3 py-2.5 px-4 rounded-lg bg-[var(--color-surface-container-low)]">
|
||||
<StatusIcon ok={ok} />
|
||||
<div className="flex-1 min-w-0">
|
||||
<span className="text-sm font-medium text-[var(--color-text-primary)]">{label}</span>
|
||||
<span className="ml-2 text-xs text-[var(--color-text-tertiary)]">{detail}</span>
|
||||
</div>
|
||||
</div>
|
||||
)
|
||||
}
|
||||
|
||||
async function openSystemSettings(pane: 'Privacy_ScreenCapture' | 'Privacy_Accessibility') {
|
||||
await computerUseApi.openSettings(pane)
|
||||
}
|
||||
|
||||
export function ComputerUseSettings() {
|
||||
const t = useTranslation()
|
||||
const [status, setStatus] = useState<ComputerUseStatus | null>(null)
|
||||
const [checkState, setCheckState] = useState<CheckState>('loading')
|
||||
const [setupRunning, setSetupRunning] = useState(false)
|
||||
const [setupResult, setSetupResult] = useState<SetupResult | null>(null)
|
||||
|
||||
// App authorization state
|
||||
const [installedApps, setInstalledApps] = useState<InstalledApp[]>([])
|
||||
const [authorizedBundleIds, setAuthorizedBundleIds] = useState<Set<string>>(new Set())
|
||||
const [authorizedApps, setAuthorizedApps] = useState<AuthorizedApp[]>([])
|
||||
const [appsLoading, setAppsLoading] = useState(false)
|
||||
const [appsSaved, setAppsSaved] = useState(false)
|
||||
const [searchQuery, setSearchQuery] = useState('')
|
||||
const [clipboardAccess, setClipboardAccess] = useState(true)
|
||||
const [systemKeys, setSystemKeys] = useState(true)
|
||||
|
||||
const fetchStatus = useCallback(async () => {
|
||||
setCheckState('loading')
|
||||
try {
|
||||
const s = await computerUseApi.getStatus()
|
||||
setStatus(s)
|
||||
setCheckState('ready')
|
||||
} catch {
|
||||
setCheckState('error')
|
||||
}
|
||||
}, [])
|
||||
|
||||
const fetchApps = useCallback(async () => {
|
||||
setAppsLoading(true)
|
||||
try {
|
||||
const [appsResult, configResult] = await Promise.all([
|
||||
computerUseApi.getInstalledApps(),
|
||||
computerUseApi.getAuthorizedApps(),
|
||||
])
|
||||
setInstalledApps(appsResult.apps)
|
||||
setAuthorizedApps(configResult.authorizedApps)
|
||||
setAuthorizedBundleIds(new Set(configResult.authorizedApps.map(a => a.bundleId)))
|
||||
setClipboardAccess(configResult.grantFlags.clipboardRead)
|
||||
setSystemKeys(configResult.grantFlags.systemKeyCombos)
|
||||
} catch {
|
||||
// API not ready
|
||||
} finally {
|
||||
setAppsLoading(false)
|
||||
}
|
||||
}, [])
|
||||
|
||||
useEffect(() => {
|
||||
fetchStatus()
|
||||
}, [fetchStatus])
|
||||
|
||||
// Load apps when environment is ready
|
||||
const envReady = status?.venv.created && status?.dependencies.installed
|
||||
useEffect(() => {
|
||||
if (envReady) fetchApps()
|
||||
}, [envReady, fetchApps])
|
||||
|
||||
const handleSetup = async () => {
|
||||
setSetupRunning(true)
|
||||
setSetupResult(null)
|
||||
try {
|
||||
const result = await computerUseApi.runSetup()
|
||||
setSetupResult(result)
|
||||
await fetchStatus()
|
||||
if (result.success) await fetchApps()
|
||||
} catch {
|
||||
setSetupResult({ success: false, steps: [{ name: 'error', ok: false, message: 'Request failed' }] })
|
||||
} finally {
|
||||
setSetupRunning(false)
|
||||
}
|
||||
}
|
||||
|
||||
const toggleApp = (app: InstalledApp) => {
|
||||
const newSet = new Set(authorizedBundleIds)
|
||||
let newAuthorized = [...authorizedApps]
|
||||
if (newSet.has(app.bundleId)) {
|
||||
newSet.delete(app.bundleId)
|
||||
newAuthorized = newAuthorized.filter(a => a.bundleId !== app.bundleId)
|
||||
} else {
|
||||
newSet.add(app.bundleId)
|
||||
newAuthorized.push({
|
||||
bundleId: app.bundleId,
|
||||
displayName: app.displayName,
|
||||
authorizedAt: new Date().toISOString(),
|
||||
})
|
||||
}
|
||||
setAuthorizedBundleIds(newSet)
|
||||
setAuthorizedApps(newAuthorized)
|
||||
|
||||
// Auto-save
|
||||
computerUseApi.setAuthorizedApps({
|
||||
authorizedApps: newAuthorized,
|
||||
grantFlags: { clipboardRead: clipboardAccess, clipboardWrite: clipboardAccess, systemKeyCombos: systemKeys },
|
||||
}).then(() => {
|
||||
setAppsSaved(true)
|
||||
setTimeout(() => setAppsSaved(false), 1500)
|
||||
})
|
||||
}
|
||||
|
||||
const toggleFlag = (flag: 'clipboard' | 'systemKeys', value: boolean) => {
|
||||
if (flag === 'clipboard') setClipboardAccess(value)
|
||||
else setSystemKeys(value)
|
||||
|
||||
computerUseApi.setAuthorizedApps({
|
||||
authorizedApps,
|
||||
grantFlags: {
|
||||
clipboardRead: flag === 'clipboard' ? value : clipboardAccess,
|
||||
clipboardWrite: flag === 'clipboard' ? value : clipboardAccess,
|
||||
systemKeyCombos: flag === 'systemKeys' ? value : systemKeys,
|
||||
},
|
||||
})
|
||||
}
|
||||
|
||||
const allReady =
|
||||
status?.supported &&
|
||||
status.python.installed &&
|
||||
status.venv.created &&
|
||||
status.dependencies.installed
|
||||
|
||||
const accessibilityNeedsAttention = status?.permissions.accessibility === false
|
||||
const screenRecordingNeedsAttention = status?.permissions.screenRecording === false
|
||||
const screenRecordingReady = status ? status.permissions.screenRecording !== false : null
|
||||
|
||||
// Filter apps by search query
|
||||
const filteredApps = useMemo(() => {
|
||||
if (!searchQuery) return installedApps
|
||||
const q = searchQuery.toLowerCase()
|
||||
return installedApps.filter(
|
||||
a => a.displayName.toLowerCase().includes(q) || a.bundleId.toLowerCase().includes(q)
|
||||
)
|
||||
}, [installedApps, searchQuery])
|
||||
|
||||
// Sort: authorized apps first, then alphabetical
|
||||
const sortedApps = useMemo(() => {
|
||||
return [...filteredApps].sort((a, b) => {
|
||||
const aAuth = authorizedBundleIds.has(a.bundleId) ? 0 : 1
|
||||
const bAuth = authorizedBundleIds.has(b.bundleId) ? 0 : 1
|
||||
if (aAuth !== bAuth) return aAuth - bAuth
|
||||
return a.displayName.localeCompare(b.displayName)
|
||||
})
|
||||
}, [filteredApps, authorizedBundleIds])
|
||||
|
||||
return (
|
||||
<div className="max-w-2xl space-y-6">
|
||||
{/* Title */}
|
||||
<div>
|
||||
<h2 className="text-lg font-semibold text-[var(--color-text-primary)]">
|
||||
{t('settings.computerUse.title')}
|
||||
</h2>
|
||||
<p className="mt-1 text-sm text-[var(--color-text-secondary)]">
|
||||
{t('settings.computerUse.description')}
|
||||
</p>
|
||||
</div>
|
||||
|
||||
{checkState === 'loading' ? (
|
||||
<div className="py-8 text-center text-sm text-[var(--color-text-tertiary)]">
|
||||
{t('common.loading')}
|
||||
</div>
|
||||
) : checkState === 'error' ? (
|
||||
<div className="py-8 text-center text-sm text-red-400">
|
||||
Failed to check status.
|
||||
<button onClick={fetchStatus} className="ml-2 underline">{t('common.retry')}</button>
|
||||
</div>
|
||||
) : status ? (
|
||||
<>
|
||||
{!status.supported && (
|
||||
<div className="px-4 py-3 rounded-lg bg-yellow-500/10 border border-yellow-500/30 text-sm text-yellow-600">
|
||||
{t('settings.computerUse.notSupported')}
|
||||
</div>
|
||||
)}
|
||||
|
||||
{/* Status checks */}
|
||||
<div className="space-y-2">
|
||||
<StatusRow
|
||||
label={t('settings.computerUse.python')}
|
||||
ok={status.python.installed}
|
||||
detail={
|
||||
status.python.installed
|
||||
? `${t('settings.computerUse.pythonFound')} — ${status.python.version} (${status.python.path})`
|
||||
: t('settings.computerUse.pythonNotFound')
|
||||
}
|
||||
/>
|
||||
<StatusRow
|
||||
label={t('settings.computerUse.venv')}
|
||||
ok={status.venv.created}
|
||||
detail={status.venv.created ? `${t('settings.computerUse.venvReady')} — ${status.venv.path}` : t('settings.computerUse.venvNotReady')}
|
||||
/>
|
||||
<StatusRow
|
||||
label={t('settings.computerUse.deps')}
|
||||
ok={status.dependencies.installed}
|
||||
detail={status.dependencies.installed ? t('settings.computerUse.depsReady') : t('settings.computerUse.depsNotReady')}
|
||||
/>
|
||||
</div>
|
||||
|
||||
{/* macOS Permissions */}
|
||||
{envReady && (
|
||||
<>
|
||||
<StatusRow
|
||||
label={t('settings.computerUse.accessibility')}
|
||||
ok={status.permissions.accessibility}
|
||||
detail={
|
||||
status.permissions.accessibility === null ? t('settings.computerUse.permUnknown')
|
||||
: status.permissions.accessibility ? t('settings.computerUse.permGranted')
|
||||
: t('settings.computerUse.permDenied')
|
||||
}
|
||||
/>
|
||||
<StatusRow
|
||||
label={t('settings.computerUse.screenRecording')}
|
||||
ok={screenRecordingReady}
|
||||
detail={
|
||||
status.permissions.screenRecording === true ? t('settings.computerUse.permGranted')
|
||||
: status.permissions.screenRecording === false ? t('settings.computerUse.permDenied')
|
||||
: t('settings.computerUse.permScreenRecordingUnknownSoft')
|
||||
}
|
||||
/>
|
||||
{(accessibilityNeedsAttention || screenRecordingNeedsAttention) && (
|
||||
<div className="flex flex-col gap-2 px-4 py-3 rounded-lg bg-yellow-500/5 border border-yellow-500/20">
|
||||
<p className="text-xs text-[var(--color-text-tertiary)]">{t('settings.computerUse.permRestartHint')}</p>
|
||||
<div className="flex gap-2">
|
||||
{accessibilityNeedsAttention && (
|
||||
<button
|
||||
onClick={() => openSystemSettings('Privacy_Accessibility')}
|
||||
className="flex items-center gap-1.5 px-3 py-1.5 text-xs font-medium text-[var(--color-text-accent)] border border-[var(--color-border)] rounded-lg hover:bg-[var(--color-surface-hover)]"
|
||||
>
|
||||
<span className="material-symbols-outlined text-[14px]">open_in_new</span>
|
||||
{t('settings.computerUse.openAccessibility')}
|
||||
</button>
|
||||
)}
|
||||
{screenRecordingNeedsAttention && (
|
||||
<button
|
||||
onClick={() => openSystemSettings('Privacy_ScreenCapture')}
|
||||
className="flex items-center gap-1.5 px-3 py-1.5 text-xs font-medium text-[var(--color-text-accent)] border border-[var(--color-border)] rounded-lg hover:bg-[var(--color-surface-hover)]"
|
||||
>
|
||||
<span className="material-symbols-outlined text-[14px]">open_in_new</span>
|
||||
{t('settings.computerUse.openScreenRecording')}
|
||||
</button>
|
||||
)}
|
||||
</div>
|
||||
</div>
|
||||
)}
|
||||
</>
|
||||
)}
|
||||
|
||||
{allReady && status.permissions.accessibility && screenRecordingReady && (
|
||||
<div className="px-4 py-3 rounded-lg bg-green-500/10 border border-green-500/30 text-sm text-green-600 flex items-center gap-2">
|
||||
<span className="material-symbols-outlined text-[18px]" style={{ fontVariationSettings: "'FILL' 1" }}>verified</span>
|
||||
{t('settings.computerUse.allReady')}
|
||||
</div>
|
||||
)}
|
||||
|
||||
{setupResult && (
|
||||
<div className={`rounded-lg border p-4 space-y-2 ${setupResult.success ? 'bg-green-500/5 border-green-500/30' : 'bg-red-500/5 border-red-500/30'}`}>
|
||||
<div className={`text-sm font-medium ${setupResult.success ? 'text-green-600' : 'text-red-400'}`}>
|
||||
{setupResult.success ? t('settings.computerUse.setupSuccess') : t('settings.computerUse.setupFail')}
|
||||
</div>
|
||||
{setupResult.steps.map((step, i) => (
|
||||
<div key={i} className="flex items-center gap-2 text-xs text-[var(--color-text-secondary)]">
|
||||
<StatusIcon ok={step.ok} />
|
||||
<span>{step.message}</span>
|
||||
</div>
|
||||
))}
|
||||
</div>
|
||||
)}
|
||||
|
||||
{/* Action buttons */}
|
||||
<div className="flex gap-3">
|
||||
{!envReady && status.python.installed && (
|
||||
<button
|
||||
onClick={handleSetup}
|
||||
disabled={setupRunning}
|
||||
className="flex items-center gap-2 px-5 py-2.5 bg-[var(--color-brand)] text-white text-sm font-semibold rounded-lg hover:opacity-90 disabled:opacity-50 transition-opacity"
|
||||
>
|
||||
<span className="material-symbols-outlined text-[18px]">{setupRunning ? 'hourglass_empty' : 'download'}</span>
|
||||
{setupRunning ? t('settings.computerUse.setupRunning') : t('settings.computerUse.setupBtn')}
|
||||
</button>
|
||||
)}
|
||||
<button
|
||||
onClick={fetchStatus}
|
||||
className="flex items-center gap-2 px-4 py-2.5 text-sm text-[var(--color-text-secondary)] hover:text-[var(--color-text-primary)] border border-[var(--color-border)] rounded-lg hover:bg-[var(--color-surface-hover)] transition-colors"
|
||||
>
|
||||
<span className="material-symbols-outlined text-[18px]">refresh</span>
|
||||
{t('settings.computerUse.recheckBtn')}
|
||||
</button>
|
||||
</div>
|
||||
|
||||
{/* ─── App Authorization Section ─── */}
|
||||
{envReady && (
|
||||
<div className="space-y-4 pt-4 border-t border-[var(--color-border)]">
|
||||
<div>
|
||||
<h3 className="text-base font-semibold text-[var(--color-text-primary)] flex items-center gap-2">
|
||||
{t('settings.computerUse.appsTitle')}
|
||||
{appsSaved && (
|
||||
<span className="text-xs font-normal text-green-500 flex items-center gap-1">
|
||||
<span className="material-symbols-outlined text-[14px]" style={{ fontVariationSettings: "'FILL' 1" }}>check</span>
|
||||
{t('settings.computerUse.appsSaved')}
|
||||
</span>
|
||||
)}
|
||||
</h3>
|
||||
<p className="mt-1 text-sm text-[var(--color-text-secondary)]">
|
||||
{t('settings.computerUse.appsDescription')}
|
||||
</p>
|
||||
</div>
|
||||
|
||||
{/* Grant flags */}
|
||||
<div className="flex gap-4">
|
||||
<label className="flex items-center gap-2 text-sm text-[var(--color-text-secondary)] cursor-pointer">
|
||||
<input
|
||||
type="checkbox"
|
||||
checked={clipboardAccess}
|
||||
onChange={e => toggleFlag('clipboard', e.target.checked)}
|
||||
className="rounded border-[var(--color-border)] accent-[var(--color-brand)]"
|
||||
/>
|
||||
{t('settings.computerUse.flagClipboard')}
|
||||
</label>
|
||||
<label className="flex items-center gap-2 text-sm text-[var(--color-text-secondary)] cursor-pointer">
|
||||
<input
|
||||
type="checkbox"
|
||||
checked={systemKeys}
|
||||
onChange={e => toggleFlag('systemKeys', e.target.checked)}
|
||||
className="rounded border-[var(--color-border)] accent-[var(--color-brand)]"
|
||||
/>
|
||||
{t('settings.computerUse.flagSystemKeys')}
|
||||
</label>
|
||||
</div>
|
||||
|
||||
{/* Search */}
|
||||
<div className="relative">
|
||||
<span className="material-symbols-outlined text-[18px] text-[var(--color-text-tertiary)] absolute left-3 top-1/2 -translate-y-1/2">search</span>
|
||||
<input
|
||||
type="text"
|
||||
value={searchQuery}
|
||||
onChange={e => setSearchQuery(e.target.value)}
|
||||
placeholder={t('settings.computerUse.appsSearch')}
|
||||
className="w-full pl-9 pr-4 py-2 text-sm bg-[var(--color-surface-container-low)] border border-[var(--color-border)] rounded-lg text-[var(--color-text-primary)] placeholder:text-[var(--color-text-tertiary)] focus:outline-none focus:border-[var(--color-brand)]"
|
||||
/>
|
||||
</div>
|
||||
|
||||
{/* App list */}
|
||||
{appsLoading ? (
|
||||
<div className="py-6 text-center text-sm text-[var(--color-text-tertiary)]">
|
||||
{t('settings.computerUse.appsLoading')}
|
||||
</div>
|
||||
) : installedApps.length === 0 ? (
|
||||
<div className="py-6 text-center text-sm text-[var(--color-text-tertiary)]">
|
||||
{t('settings.computerUse.appsEmpty')}
|
||||
</div>
|
||||
) : (
|
||||
<div className="max-h-[400px] overflow-y-auto rounded-lg border border-[var(--color-border)]">
|
||||
{sortedApps.map(app => {
|
||||
const isAuthorized = authorizedBundleIds.has(app.bundleId)
|
||||
return (
|
||||
<button
|
||||
key={app.bundleId}
|
||||
onClick={() => toggleApp(app)}
|
||||
className={`w-full flex items-center gap-3 px-4 py-2.5 text-left transition-colors hover:bg-[var(--color-surface-hover)] border-b border-[var(--color-border)] last:border-b-0 ${
|
||||
isAuthorized ? 'bg-[var(--color-brand)]/5' : ''
|
||||
}`}
|
||||
>
|
||||
<div className={`w-5 h-5 rounded flex items-center justify-center flex-shrink-0 border ${
|
||||
isAuthorized
|
||||
? 'bg-[var(--color-brand)] border-[var(--color-brand)]'
|
||||
: 'border-[var(--color-border)]'
|
||||
}`}>
|
||||
{isAuthorized && (
|
||||
<span className="material-symbols-outlined text-[14px] text-white" style={{ fontVariationSettings: "'FILL' 1" }}>check</span>
|
||||
)}
|
||||
</div>
|
||||
<div className="flex-1 min-w-0">
|
||||
<div className="text-sm font-medium text-[var(--color-text-primary)] truncate">
|
||||
{app.displayName}
|
||||
</div>
|
||||
<div className="text-[11px] text-[var(--color-text-tertiary)] truncate font-mono">
|
||||
{app.bundleId}
|
||||
</div>
|
||||
</div>
|
||||
</button>
|
||||
)
|
||||
})}
|
||||
</div>
|
||||
)}
|
||||
</div>
|
||||
)}
|
||||
</>
|
||||
) : null}
|
||||
</div>
|
||||
)
|
||||
}
|
||||
@ -28,6 +28,7 @@ from Quartz import (
|
||||
CGRectContainsPoint,
|
||||
CGRectIntersection,
|
||||
CGPointMake,
|
||||
CGPreflightScreenCaptureAccess,
|
||||
kCGNullWindowID,
|
||||
kCGWindowBounds,
|
||||
kCGWindowIsOnscreen,
|
||||
@ -232,7 +233,16 @@ def choose_display(display_id: int | None) -> dict[str, Any]:
|
||||
raise RuntimeError(f"Unknown display: {display_id}")
|
||||
|
||||
|
||||
def ensure_screen_recording_permission() -> None:
|
||||
"""No-op: CGPreflightScreenCaptureAccess is unreliable for child processes
|
||||
(returns False even when the parent app has TCC permission), and any actual
|
||||
capture attempt triggers a macOS popup on newer versions. Let the actual
|
||||
capture call handle errors instead."""
|
||||
pass
|
||||
|
||||
|
||||
def capture_display(display_id: int | None, resize: tuple[int, int] | None = None) -> dict[str, Any]:
|
||||
ensure_screen_recording_permission()
|
||||
display = choose_display(display_id)
|
||||
monitor = {
|
||||
"left": display["originX"],
|
||||
@ -262,6 +272,7 @@ def capture_display(display_id: int | None, resize: tuple[int, int] | None = Non
|
||||
|
||||
|
||||
def capture_region(region: dict[str, int], resize: tuple[int, int] | None = None) -> dict[str, Any]:
|
||||
ensure_screen_recording_permission()
|
||||
with mss.mss() as sct:
|
||||
raw = sct.grab(region)
|
||||
image = Image.frombytes("RGB", raw.size, raw.rgb)
|
||||
@ -461,17 +472,61 @@ def write_clipboard(text: str) -> None:
|
||||
pb.setString_forType_(text, NSPasteboardTypeString)
|
||||
|
||||
|
||||
def check_permissions() -> dict[str, bool]:
|
||||
def detect_screen_recording_permission() -> bool | None:
|
||||
"""Best-effort passive screen-recording probe with no system prompt.
|
||||
|
||||
`CGPreflightScreenCaptureAccess()` is fast and explicit when it returns
|
||||
True, but on child processes launched by a TCC-authorized app bundle it can
|
||||
still return False. As a fallback, inspect the visible window list: Apple
|
||||
only exposes other apps' window titles when Screen Recording access is
|
||||
granted. If we can see at least one title, treat the permission as granted.
|
||||
If we can inspect visible windows but every title is blank, treat it as not
|
||||
granted. If window enumeration itself is unavailable, return None.
|
||||
"""
|
||||
|
||||
try:
|
||||
if CGPreflightScreenCaptureAccess():
|
||||
return True
|
||||
except Exception:
|
||||
pass
|
||||
|
||||
try:
|
||||
windows = CGWindowListCopyWindowInfo(
|
||||
kCGWindowListOptionOnScreenOnly | kCGWindowListExcludeDesktopElements,
|
||||
kCGNullWindowID,
|
||||
)
|
||||
except Exception:
|
||||
return None
|
||||
|
||||
eligible_windows = 0
|
||||
for window in windows or []:
|
||||
if int(window.get(kCGWindowLayer, 0)) != 0:
|
||||
continue
|
||||
if not bool(window.get(kCGWindowIsOnscreen, True)):
|
||||
continue
|
||||
|
||||
bounds = window.get(kCGWindowBounds) or {}
|
||||
width = int(bounds.get("Width", 0))
|
||||
height = int(bounds.get("Height", 0))
|
||||
if width <= 1 or height <= 1:
|
||||
continue
|
||||
|
||||
eligible_windows += 1
|
||||
if (window.get(kCGWindowName, "") or "").strip():
|
||||
return True
|
||||
|
||||
if eligible_windows > 0:
|
||||
return False
|
||||
return None
|
||||
|
||||
|
||||
def check_permissions() -> dict[str, bool | None]:
|
||||
accessibility = True
|
||||
try:
|
||||
run_osascript('tell application "System Events" to get name of first process')
|
||||
except Exception:
|
||||
accessibility = False
|
||||
screen_recording = True
|
||||
try:
|
||||
capture_display(None)
|
||||
except Exception:
|
||||
screen_recording = False
|
||||
screen_recording = detect_screen_recording_permission()
|
||||
return {
|
||||
"accessibility": accessibility,
|
||||
"screenRecording": screen_recording,
|
||||
|
||||
430
src/server/api/computer-use.ts
Normal file
430
src/server/api/computer-use.ts
Normal file
@ -0,0 +1,430 @@
|
||||
/**
|
||||
* Computer Use API — 环境检测与依赖安装
|
||||
*
|
||||
* Routes:
|
||||
* GET /api/computer-use/status — 检测 Python3、venv、依赖、权限状态
|
||||
* POST /api/computer-use/setup — 创建 venv 并安装依赖
|
||||
*/
|
||||
|
||||
import { homedir } from 'os'
|
||||
import { join } from 'path'
|
||||
import { access, readFile, mkdir, writeFile } from 'fs/promises'
|
||||
import { createHash } from 'crypto'
|
||||
import path from 'path'
|
||||
import { fileURLToPath } from 'url'
|
||||
// Embed mac_helper.py at compile time so it's available in bundled mode
|
||||
// @ts-ignore — Bun text import
|
||||
import MAC_HELPER_CONTENT from '../../../runtime/mac_helper.py' with { type: 'text' }
|
||||
|
||||
const __dirname = path.dirname(fileURLToPath(import.meta.url))
|
||||
const projectRoot = path.resolve(__dirname, '../../..')
|
||||
const devRuntimeRoot = join(projectRoot, 'runtime')
|
||||
const claudeHome = process.env.CLAUDE_CONFIG_DIR ?? join(homedir(), '.claude')
|
||||
const runtimeStateRoot = join(claudeHome, '.runtime')
|
||||
const venvRoot = join(runtimeStateRoot, 'venv')
|
||||
const installStampPath = join(runtimeStateRoot, 'requirements.sha256')
|
||||
|
||||
// Embedded content of requirements.txt — kept in sync with runtime/requirements.txt.
|
||||
// This ensures the bundled sidecar can create the file without disk access.
|
||||
const REQUIREMENTS_CONTENT = `mss>=10.1.0
|
||||
Pillow>=11.3.0
|
||||
pyautogui>=0.9.54
|
||||
pyobjc-core>=11.1
|
||||
pyobjc-framework-Cocoa>=11.1
|
||||
pyobjc-framework-Quartz>=11.1
|
||||
`
|
||||
|
||||
// 清华大学 PyPI 镜像，国内安装速度更快
|
||||
const PIP_INDEX_URL = 'https://pypi.tuna.tsinghua.edu.cn/simple/'
|
||||
const PIP_TRUSTED_HOST = 'pypi.tuna.tsinghua.edu.cn'
|
||||
|
||||
// Paths that resolve correctly in both dev and bundled modes
|
||||
function getRequirementsPath(): string {
|
||||
return join(runtimeStateRoot, 'requirements.txt')
|
||||
}
|
||||
|
||||
function getHelperPath(): string {
|
||||
// In bundled mode mac_helper.py is extracted to runtimeStateRoot.
|
||||
// In dev mode we also copy it there during setup, so both modes
|
||||
// read from the same location after setup runs.
|
||||
return join(runtimeStateRoot, 'mac_helper.py')
|
||||
}
|
||||
|
||||
async function pathExists(target: string): Promise<boolean> {
|
||||
try {
|
||||
await access(target)
|
||||
return true
|
||||
} catch {
|
||||
return false
|
||||
}
|
||||
}
|
||||
|
||||
async function runCommand(
|
||||
cmd: string,
|
||||
args: string[],
|
||||
): Promise<{ ok: boolean; stdout: string; stderr: string; code: number }> {
|
||||
try {
|
||||
const proc = Bun.spawn([cmd, ...args], {
|
||||
stdout: 'pipe',
|
||||
stderr: 'pipe',
|
||||
})
|
||||
const [stdout, stderr] = await Promise.all([
|
||||
new Response(proc.stdout).text(),
|
||||
new Response(proc.stderr).text(),
|
||||
])
|
||||
const code = await proc.exited
|
||||
return { ok: code === 0, stdout: stdout.trim(), stderr: stderr.trim(), code }
|
||||
} catch {
|
||||
return { ok: false, stdout: '', stderr: `Failed to run ${cmd}`, code: -1 }
|
||||
}
|
||||
}
|
||||
|
||||
/**
|
||||
* Ensure runtime source files (requirements.txt, mac_helper.py) exist in
|
||||
* ~/.claude/.runtime/. In dev mode they are copied from the project's
|
||||
* runtime/ directory; in bundled mode requirements.txt is written from the
|
||||
* embedded constant and mac_helper.py is copied from the project dir (if
|
||||
* available) or skipped (it will already have been extracted on a prior run).
|
||||
*/
|
||||
async function ensureRuntimeFiles(): Promise<void> {
|
||||
await mkdir(runtimeStateRoot, { recursive: true })
|
||||
|
||||
// requirements.txt — always write from embedded constant (authoritative)
|
||||
await writeFile(getRequirementsPath(), REQUIREMENTS_CONTENT, 'utf8')
|
||||
|
||||
// mac_helper.py — always write from embedded content (compile-time import)
|
||||
await writeFile(getHelperPath(), MAC_HELPER_CONTENT, 'utf8')
|
||||
}
|
||||
|
||||
type EnvStatus = {
|
||||
platform: string
|
||||
supported: boolean
|
||||
python: {
|
||||
installed: boolean
|
||||
version: string | null
|
||||
path: string | null
|
||||
}
|
||||
venv: {
|
||||
created: boolean
|
||||
path: string
|
||||
}
|
||||
dependencies: {
|
||||
installed: boolean
|
||||
requirementsFound: boolean
|
||||
}
|
||||
permissions: {
|
||||
accessibility: boolean | null
|
||||
screenRecording: boolean | null
|
||||
}
|
||||
}
|
||||
|
||||
async function checkStatus(): Promise<EnvStatus> {
|
||||
const platform = process.platform
|
||||
const supported = platform === 'darwin'
|
||||
|
||||
// Check Python 3
|
||||
const pythonResult = await runCommand('python3', ['--version'])
|
||||
const pythonInstalled = pythonResult.ok
|
||||
const pythonVersion = pythonInstalled
|
||||
? pythonResult.stdout.replace('Python ', '')
|
||||
: null
|
||||
|
||||
let pythonPath: string | null = null
|
||||
if (pythonInstalled) {
|
||||
const whichResult = await runCommand('which', ['python3'])
|
||||
pythonPath = whichResult.ok ? whichResult.stdout : null
|
||||
}
|
||||
|
||||
// Check venv
|
||||
const venvCreated = await pathExists(join(venvRoot, 'bin', 'python3'))
|
||||
|
||||
// Check dependencies — use the state dir copy
|
||||
const reqPath = getRequirementsPath()
|
||||
const requirementsFound = await pathExists(reqPath)
|
||||
let depsInstalled = false
|
||||
if (requirementsFound && venvCreated) {
|
||||
try {
|
||||
const requirements = await readFile(reqPath, 'utf8')
|
||||
const digest = createHash('sha256').update(requirements).digest('hex')
|
||||
const stamp = (await readFile(installStampPath, 'utf8')).trim()
|
||||
depsInstalled = stamp === digest
|
||||
} catch {
|
||||
depsInstalled = false
|
||||
}
|
||||
}
|
||||
|
||||
// Check macOS permissions without triggering a system prompt. The helper
|
||||
// uses preflight + visible-window metadata as a passive fallback because
|
||||
// plain preflight can misreport child processes launched by the desktop app.
|
||||
let accessibility: boolean | null = null
|
||||
let screenRecording: boolean | null = null
|
||||
if (supported && venvCreated && depsInstalled) {
|
||||
try { await ensureRuntimeFiles() } catch {}
|
||||
const helperPath = getHelperPath()
|
||||
if (await pathExists(helperPath)) {
|
||||
const pythonBin = join(venvRoot, 'bin', 'python3')
|
||||
const permResult = await runCommand(pythonBin, [helperPath, 'check_permissions'])
|
||||
if (permResult.ok) {
|
||||
try {
|
||||
const parsed = JSON.parse(permResult.stdout)
|
||||
if (parsed.ok && parsed.result) {
|
||||
accessibility = parsed.result.accessibility ?? null
|
||||
screenRecording = parsed.result.screenRecording ?? null
|
||||
}
|
||||
} catch {}
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
return {
|
||||
platform,
|
||||
supported,
|
||||
python: { installed: pythonInstalled, version: pythonVersion, path: pythonPath },
|
||||
venv: { created: venvCreated, path: venvRoot },
|
||||
dependencies: { installed: depsInstalled, requirementsFound: requirementsFound || true },
|
||||
permissions: { accessibility, screenRecording },
|
||||
}
|
||||
}
|
||||
|
||||
type SetupResult = {
|
||||
success: boolean
|
||||
steps: { name: string; ok: boolean; message: string }[]
|
||||
}
|
||||
|
||||
async function runSetup(): Promise<SetupResult> {
|
||||
const steps: SetupResult['steps'] = []
|
||||
|
||||
// Step 1: Check python3
|
||||
const pythonCheck = await runCommand('python3', ['--version'])
|
||||
if (!pythonCheck.ok) {
|
||||
steps.push({
|
||||
name: 'python_check',
|
||||
ok: false,
|
||||
message: 'Python 3 未安装，请先安装 Python 3',
|
||||
})
|
||||
return { success: false, steps }
|
||||
}
|
||||
steps.push({
|
||||
name: 'python_check',
|
||||
ok: true,
|
||||
message: `Python ${pythonCheck.stdout.replace('Python ', '')}`,
|
||||
})
|
||||
|
||||
// Step 2: Extract runtime files to ~/.claude/.runtime/
|
||||
try {
|
||||
await ensureRuntimeFiles()
|
||||
steps.push({ name: 'runtime_files', ok: true, message: '运行时文件已就绪' })
|
||||
} catch (err) {
|
||||
steps.push({
|
||||
name: 'runtime_files',
|
||||
ok: false,
|
||||
message: `提取运行时文件失败: ${err}`,
|
||||
})
|
||||
return { success: false, steps }
|
||||
}
|
||||
|
||||
// Step 3: Create venv
|
||||
const venvExists = await pathExists(join(venvRoot, 'bin', 'python3'))
|
||||
if (!venvExists) {
|
||||
const venvResult = await runCommand('python3', ['-m', 'venv', venvRoot])
|
||||
if (!venvResult.ok) {
|
||||
steps.push({
|
||||
name: 'venv',
|
||||
ok: false,
|
||||
message: `创建虚拟环境失败: ${venvResult.stderr}`,
|
||||
})
|
||||
return { success: false, steps }
|
||||
}
|
||||
steps.push({ name: 'venv', ok: true, message: '虚拟环境已创建' })
|
||||
} else {
|
||||
steps.push({ name: 'venv', ok: true, message: '虚拟环境已存在' })
|
||||
}
|
||||
|
||||
// Step 4: Ensure pip
|
||||
const pipPath = join(venvRoot, 'bin', 'pip')
|
||||
if (!(await pathExists(pipPath))) {
|
||||
const pythonBin = join(venvRoot, 'bin', 'python3')
|
||||
const pipResult = await runCommand(pythonBin, [
|
||||
'-m',
|
||||
'ensurepip',
|
||||
'--upgrade',
|
||||
])
|
||||
if (!pipResult.ok) {
|
||||
steps.push({
|
||||
name: 'pip',
|
||||
ok: false,
|
||||
message: `安装 pip 失败: ${pipResult.stderr}`,
|
||||
})
|
||||
return { success: false, steps }
|
||||
}
|
||||
}
|
||||
steps.push({ name: 'pip', ok: true, message: 'pip 已就绪' })
|
||||
|
||||
// Step 5: Install requirements
|
||||
const reqPath = getRequirementsPath()
|
||||
const requirements = await readFile(reqPath, 'utf8')
|
||||
const digest = createHash('sha256').update(requirements).digest('hex')
|
||||
|
||||
let installedDigest = ''
|
||||
try {
|
||||
installedDigest = (await readFile(installStampPath, 'utf8')).trim()
|
||||
} catch {}
|
||||
|
||||
if (installedDigest !== digest) {
|
||||
const pythonBin = join(venvRoot, 'bin', 'python3')
|
||||
|
||||
// Upgrade pip first (using China mirror)
|
||||
await runCommand(pythonBin, [
|
||||
'-m', 'pip', 'install', '--upgrade', 'pip',
|
||||
'-i', PIP_INDEX_URL, '--trusted-host', PIP_TRUSTED_HOST,
|
||||
])
|
||||
|
||||
// Install deps (using China mirror)
|
||||
const installResult = await runCommand(pythonBin, [
|
||||
'-m', 'pip', 'install',
|
||||
'-r', reqPath,
|
||||
'-i', PIP_INDEX_URL, '--trusted-host', PIP_TRUSTED_HOST,
|
||||
])
|
||||
if (!installResult.ok) {
|
||||
steps.push({
|
||||
name: 'deps',
|
||||
ok: false,
|
||||
message: `安装依赖失败: ${installResult.stderr.slice(0, 500)}`,
|
||||
})
|
||||
return { success: false, steps }
|
||||
}
|
||||
await writeFile(installStampPath, `${digest}\n`, 'utf8')
|
||||
steps.push({ name: 'deps', ok: true, message: '依赖已安装' })
|
||||
} else {
|
||||
steps.push({ name: 'deps', ok: true, message: '依赖已是最新' })
|
||||
}
|
||||
|
||||
return { success: true, steps }
|
||||
}
|
||||
|
||||
// ============================================================================
|
||||
// Authorized Apps configuration — stored in ~/.claude/cc-haha/computer-use-config.json
|
||||
// ============================================================================
|
||||
|
||||
const configPath = join(claudeHome, 'cc-haha', 'computer-use-config.json')
|
||||
|
||||
type AuthorizedApp = {
|
||||
bundleId: string
|
||||
displayName: string
|
||||
authorizedAt: string
|
||||
}
|
||||
|
||||
type ComputerUseConfig = {
|
||||
authorizedApps: AuthorizedApp[]
|
||||
grantFlags: {
|
||||
clipboardRead: boolean
|
||||
clipboardWrite: boolean
|
||||
systemKeyCombos: boolean
|
||||
}
|
||||
}
|
||||
|
||||
const DEFAULT_CONFIG: ComputerUseConfig = {
|
||||
authorizedApps: [],
|
||||
grantFlags: { clipboardRead: true, clipboardWrite: true, systemKeyCombos: true },
|
||||
}
|
||||
|
||||
async function loadConfig(): Promise<ComputerUseConfig> {
|
||||
try {
|
||||
const raw = await readFile(configPath, 'utf8')
|
||||
return { ...DEFAULT_CONFIG, ...JSON.parse(raw) }
|
||||
} catch {
|
||||
return { ...DEFAULT_CONFIG }
|
||||
}
|
||||
}
|
||||
|
||||
async function saveConfig(config: ComputerUseConfig): Promise<void> {
|
||||
await writeFile(configPath, JSON.stringify(config, null, 2), 'utf8')
|
||||
}
|
||||
|
||||
async function listInstalledApps(): Promise<{ bundleId: string; displayName: string; path: string }[]> {
|
||||
const helperPath = getHelperPath()
|
||||
const pythonBin = join(venvRoot, 'bin', 'python3')
|
||||
|
||||
if (!(await pathExists(pythonBin)) || !(await pathExists(helperPath))) {
|
||||
return []
|
||||
}
|
||||
|
||||
const result = await runCommand(pythonBin, [helperPath, 'list_installed_apps'])
|
||||
if (!result.ok) return []
|
||||
|
||||
try {
|
||||
const parsed = JSON.parse(result.stdout)
|
||||
return parsed.ok ? parsed.result : []
|
||||
} catch {
|
||||
return []
|
||||
}
|
||||
}
|
||||
|
||||
// ============================================================================
|
||||
// Route handler
|
||||
// ============================================================================
|
||||
|
||||
export async function handleComputerUseApi(
|
||||
req: Request,
|
||||
_url: URL,
|
||||
segments: string[],
|
||||
): Promise<Response> {
|
||||
const action = segments[2]
|
||||
|
||||
if (action === 'status' && req.method === 'GET') {
|
||||
const status = await checkStatus()
|
||||
return Response.json(status)
|
||||
}
|
||||
|
||||
if (action === 'setup' && req.method === 'POST') {
|
||||
const result = await runSetup()
|
||||
return Response.json(result)
|
||||
}
|
||||
|
||||
// GET /api/computer-use/apps — list installed macOS apps
|
||||
if (action === 'apps' && req.method === 'GET') {
|
||||
const apps = await listInstalledApps()
|
||||
return Response.json({ apps })
|
||||
}
|
||||
|
||||
// GET /api/computer-use/authorized-apps — current authorized app config
|
||||
if (action === 'authorized-apps' && req.method === 'GET') {
|
||||
const config = await loadConfig()
|
||||
return Response.json(config)
|
||||
}
|
||||
|
||||
// PUT /api/computer-use/authorized-apps — update authorized apps
|
||||
if (action === 'authorized-apps' && req.method === 'PUT') {
|
||||
try {
|
||||
const body = (await req.json()) as Partial<ComputerUseConfig>
|
||||
const config = await loadConfig()
|
||||
if (body.authorizedApps) config.authorizedApps = body.authorizedApps
|
||||
if (body.grantFlags) config.grantFlags = { ...config.grantFlags, ...body.grantFlags }
|
||||
await saveConfig(config)
|
||||
return Response.json({ ok: true })
|
||||
} catch {
|
||||
return Response.json({ error: 'Invalid JSON' }, { status: 400 })
|
||||
}
|
||||
}
|
||||
|
||||
// POST /api/computer-use/open-settings — open macOS System Settings pane
|
||||
if (action === 'open-settings' && req.method === 'POST') {
|
||||
if (process.platform !== 'darwin') {
|
||||
return Response.json({ error: 'macOS only' }, { status: 400 })
|
||||
}
|
||||
const body = (await req.json().catch(() => ({}))) as { pane?: string }
|
||||
const pane = body.pane ?? 'Privacy_ScreenCapture'
|
||||
const allowed = ['Privacy_ScreenCapture', 'Privacy_Accessibility']
|
||||
if (!allowed.includes(pane)) {
|
||||
return Response.json({ error: 'Invalid pane' }, { status: 400 })
|
||||
}
|
||||
const url = `x-apple.systempreferences:com.apple.preference.security?${pane}`
|
||||
await runCommand('open', [url])
|
||||
return Response.json({ ok: true })
|
||||
}
|
||||
|
||||
return Response.json(
|
||||
{ error: 'NOT_FOUND', message: `Unknown computer-use action: ${action}` },
|
||||
{ status: 404 },
|
||||
)
|
||||
}
|
||||
@ -7,6 +7,7 @@ import { logForDebugging } from '../debug.js'
|
||||
import { COMPUTER_USE_MCP_SERVER_NAME } from './common.js'
|
||||
import { createCliExecutor } from './executor.js'
|
||||
import { getChicagoEnabled, getChicagoSubGates } from './gates.js'
|
||||
import { normalizeOsPermissions } from './permissions.js'
|
||||
import { callPythonHelper } from './pythonBridge.js'
|
||||
|
||||
class DebugLogger implements Logger {
|
||||
@ -39,8 +40,9 @@ export function getComputerUseHostAdapter(): ComputerUseHostAdapter {
|
||||
getHideBeforeActionEnabled: () => getChicagoSubGates().hideBeforeAction,
|
||||
}),
|
||||
ensureOsPermissions: async () => {
|
||||
const perms = await callPythonHelper<{ accessibility: boolean; screenRecording: boolean }>('check_permissions', {})
|
||||
return perms.accessibility && perms.screenRecording
|
||||
const rawPerms = await callPythonHelper<{ accessibility: boolean; screenRecording: boolean | null }>('check_permissions', {})
|
||||
const perms = normalizeOsPermissions(rawPerms)
|
||||
return perms.granted
|
||||
? { granted: true as const }
|
||||
: { granted: false as const, accessibility: perms.accessibility, screenRecording: perms.screenRecording }
|
||||
},
|
||||
|
||||
44
src/utils/computerUse/permissions.test.ts
Normal file
44
src/utils/computerUse/permissions.test.ts
Normal file
@ -0,0 +1,44 @@
|
||||
import { describe, expect, it } from 'bun:test'
|
||||
import { normalizeOsPermissions } from './permissions.js'
|
||||
|
||||
describe('normalizeOsPermissions', () => {
|
||||
it('treats explicit grants as granted', () => {
|
||||
expect(
|
||||
normalizeOsPermissions({ accessibility: true, screenRecording: true }),
|
||||
).toEqual({
|
||||
granted: true,
|
||||
accessibility: true,
|
||||
screenRecording: true,
|
||||
})
|
||||
})
|
||||
|
||||
it('treats screen recording unknown as non-blocking when accessibility is granted', () => {
|
||||
expect(
|
||||
normalizeOsPermissions({ accessibility: true, screenRecording: null }),
|
||||
).toEqual({
|
||||
granted: true,
|
||||
accessibility: true,
|
||||
screenRecording: true,
|
||||
})
|
||||
})
|
||||
|
||||
it('still blocks when accessibility is missing', () => {
|
||||
expect(
|
||||
normalizeOsPermissions({ accessibility: false, screenRecording: null }),
|
||||
).toEqual({
|
||||
granted: false,
|
||||
accessibility: false,
|
||||
screenRecording: true,
|
||||
})
|
||||
})
|
||||
|
||||
it('blocks when screen recording is explicitly denied', () => {
|
||||
expect(
|
||||
normalizeOsPermissions({ accessibility: true, screenRecording: false }),
|
||||
).toEqual({
|
||||
granted: false,
|
||||
accessibility: true,
|
||||
screenRecording: false,
|
||||
})
|
||||
})
|
||||
})
|
||||
25
src/utils/computerUse/permissions.ts
Normal file
25
src/utils/computerUse/permissions.ts
Normal file
@ -0,0 +1,25 @@
|
||||
export type RawOsPermissions = {
|
||||
accessibility: boolean
|
||||
screenRecording: boolean | null
|
||||
}
|
||||
|
||||
export type NormalizedOsPermissions = {
|
||||
granted: boolean
|
||||
accessibility: boolean
|
||||
screenRecording: boolean
|
||||
}
|
||||
|
||||
/**
|
||||
* macOS Screen Recording passive probes can come back "unknown" for helper
|
||||
* child processes even when the app bundle is already authorized. Treat that
|
||||
* state as non-blocking and let the actual capture path remain the final
|
||||
* source of truth.
|
||||
*/
|
||||
export function normalizeOsPermissions(perms: RawOsPermissions): NormalizedOsPermissions {
|
||||
const screenRecording = perms.screenRecording !== false
|
||||
return {
|
||||
granted: perms.accessibility && screenRecording,
|
||||
accessibility: perms.accessibility,
|
||||
screenRecording,
|
||||
}
|
||||
}
|
||||
@ -4,16 +4,25 @@ import path from 'node:path'
|
||||
import { fileURLToPath } from 'node:url'
|
||||
import { execFileNoThrow } from '../execFileNoThrow.js'
|
||||
import { logForDebugging } from '../debug.js'
|
||||
import { getClaudeConfigHomeDir } from '../envUtils.js'
|
||||
|
||||
const __dirname = path.dirname(fileURLToPath(import.meta.url))
|
||||
const projectRoot = path.resolve(__dirname, '../../..')
|
||||
const runtimeRoot = path.join(projectRoot, 'runtime')
|
||||
const runtimeStateRoot = path.join(projectRoot, '.runtime')
|
||||
|
||||
// All runtime state lives in ~/.claude/.runtime — writable in both dev and
|
||||
// bundled (Tauri app) modes. The setup API (or ensureRuntimeFiles below)
|
||||
// populates requirements.txt and mac_helper.py here.
|
||||
const runtimeStateRoot = path.join(getClaudeConfigHomeDir(), '.runtime')
|
||||
const venvRoot = path.join(runtimeStateRoot, 'venv')
|
||||
const requirementsPath = path.join(runtimeRoot, 'requirements.txt')
|
||||
const helperPath = path.join(runtimeRoot, 'mac_helper.py')
|
||||
const installStampPath = path.join(runtimeStateRoot, 'requirements.sha256')
|
||||
|
||||
const PIP_INDEX_URL = 'https://pypi.tuna.tsinghua.edu.cn/simple/'
|
||||
const PIP_TRUSTED_HOST = 'pypi.tuna.tsinghua.edu.cn'
|
||||
|
||||
// Always read from ~/.claude/.runtime/ — works in both dev and bundled mode.
|
||||
const requirementsPath = path.join(runtimeStateRoot, 'requirements.txt')
|
||||
const helperPath = path.join(runtimeStateRoot, 'mac_helper.py')
|
||||
|
||||
let bootstrapPromise: Promise<void> | undefined
|
||||
|
||||
function pythonBinPath(): string {
|
||||
@ -37,10 +46,34 @@ async function runOrThrow(file: string, args: string[], label: string): Promise<
|
||||
return stdout
|
||||
}
|
||||
|
||||
/**
|
||||
* Ensure runtime source files exist in ~/.claude/.runtime/.
|
||||
* In dev mode, copies from the project's runtime/ directory on first run.
|
||||
* In bundled mode, these must have been placed there by the settings setup API.
|
||||
*/
|
||||
async function ensureRuntimeFiles(): Promise<void> {
|
||||
await mkdir(runtimeStateRoot, { recursive: true })
|
||||
|
||||
const devRequirements = path.join(projectRoot, 'runtime', 'requirements.txt')
|
||||
const devHelper = path.join(projectRoot, 'runtime', 'mac_helper.py')
|
||||
|
||||
// Always sync from dev runtime/ so source changes are reflected immediately.
|
||||
// Previously this only copied when the dest was missing, causing stale files
|
||||
// to persist after source updates — breaking mouse/keyboard actions if the
|
||||
// cached copy was from an older version.
|
||||
if (await pathExists(devRequirements)) {
|
||||
await writeFile(requirementsPath, await readFile(devRequirements, 'utf8'), 'utf8')
|
||||
}
|
||||
if (await pathExists(devHelper)) {
|
||||
await writeFile(helperPath, await readFile(devHelper, 'utf8'), 'utf8')
|
||||
}
|
||||
}
|
||||
|
||||
export async function ensureBootstrapped(): Promise<void> {
|
||||
if (bootstrapPromise) return bootstrapPromise
|
||||
bootstrapPromise = (async () => {
|
||||
await mkdir(runtimeStateRoot, { recursive: true })
|
||||
// Extract runtime files (requirements.txt, mac_helper.py) to state dir
|
||||
await ensureRuntimeFiles()
|
||||
|
||||
if (!(await pathExists(pythonBinPath()))) {
|
||||
logForDebugging('creating runtime venv at %s', { level: 'debug' })
|
||||
@ -61,10 +94,14 @@ export async function ensureBootstrapped(): Promise<void> {
|
||||
|
||||
if (installedDigest !== digest) {
|
||||
logForDebugging('installing python runtime dependencies', { level: 'debug' })
|
||||
await runOrThrow(pythonBinPath(), ['-m', 'pip', 'install', '--upgrade', 'pip'], 'pip upgrade')
|
||||
await runOrThrow(pythonBinPath(), [
|
||||
'-m', 'pip', 'install', '--upgrade', 'pip',
|
||||
'-i', PIP_INDEX_URL, '--trusted-host', PIP_TRUSTED_HOST,
|
||||
], 'pip upgrade')
|
||||
await runOrThrow(
|
||||
pythonBinPath(),
|
||||
['-m', 'pip', 'install', '-r', requirementsPath],
|
||||
['-m', 'pip', 'install', '-r', requirementsPath,
|
||||
'-i', PIP_INDEX_URL, '--trusted-host', PIP_TRUSTED_HOST],
|
||||
'python dependency install',
|
||||
)
|
||||
await writeFile(installStampPath, `${digest}\n`, 'utf8')
|
||||
@ -105,6 +142,6 @@ export async function callPythonHelper<T>(command: string, payload: Record<strin
|
||||
return parsed.result as T
|
||||
}
|
||||
|
||||
export function getRuntimePaths(): { projectRoot: string; runtimeRoot: string; venvRoot: string } {
|
||||
return { projectRoot, runtimeRoot, venvRoot }
|
||||
export function getRuntimePaths(): { projectRoot: string; runtimeStateRoot: string; venvRoot: string } {
|
||||
return { projectRoot, runtimeStateRoot, venvRoot }
|
||||
}
|
||||
|
||||
@ -27,6 +27,9 @@ import { registerEscHotkey } from './escHotkey.js';
|
||||
import { getChicagoCoordinateMode } from './gates.js';
|
||||
import { getComputerUseHostAdapter } from './hostAdapter.js';
|
||||
import { getComputerUseMCPRenderingOverrides } from './toolRendering.js';
|
||||
import { readFile } from 'node:fs/promises';
|
||||
import { join } from 'node:path';
|
||||
import { homedir } from 'node:os';
|
||||
type CallOverride = Pick<Tool, 'call'>['call'];
|
||||
type Binding = {
|
||||
ctx: ComputerUseSessionContext;
|
||||
@ -227,6 +230,64 @@ export function buildSessionContext(): ComputerUseSessionContext {
|
||||
formatLockHeldMessage: formatLockHeld
|
||||
};
|
||||
}
|
||||
/**
|
||||
* Load pre-authorized apps from ~/.claude/cc-haha/computer-use-config.json.
|
||||
* Called once when the binding is first created. Pre-authorized apps
|
||||
* are injected into appState so `getAllowedApps()` returns them
|
||||
* immediately — no runtime permission dialog needed.
|
||||
*/
|
||||
async function loadPreAuthorizedApps(): Promise<void> {
|
||||
try {
|
||||
const configPath = join(
|
||||
process.env.CLAUDE_CONFIG_DIR ?? join(homedir(), '.claude'),
|
||||
'cc-haha',
|
||||
'computer-use-config.json',
|
||||
)
|
||||
const raw = await readFile(configPath, 'utf8')
|
||||
const config = JSON.parse(raw) as {
|
||||
authorizedApps?: { bundleId: string; displayName: string }[]
|
||||
grantFlags?: { clipboardRead?: boolean; clipboardWrite?: boolean; systemKeyCombos?: boolean }
|
||||
}
|
||||
|
||||
if (!config.authorizedApps?.length) return
|
||||
|
||||
const apps = config.authorizedApps.map(a => ({
|
||||
bundleId: a.bundleId,
|
||||
displayName: a.displayName,
|
||||
grantedAt: Date.now(),
|
||||
tier: 'full' as const,
|
||||
}))
|
||||
const flags = {
|
||||
...DEFAULT_GRANT_FLAGS,
|
||||
...(config.grantFlags ?? {}),
|
||||
}
|
||||
|
||||
// Inject into appState so getAllowedApps() returns them.
|
||||
// Merge with existing allowedApps (from permission dialog) instead of replacing.
|
||||
if (currentToolUseContext) {
|
||||
currentToolUseContext.setAppState(prev => {
|
||||
const existing = prev.computerUseMcpState?.allowedApps ?? []
|
||||
const existingIds = new Set(existing.map(a => a.bundleId))
|
||||
const merged = [...existing, ...apps.filter(a => !existingIds.has(a.bundleId))]
|
||||
return {
|
||||
...prev,
|
||||
computerUseMcpState: {
|
||||
...prev.computerUseMcpState,
|
||||
allowedApps: merged,
|
||||
grantFlags: flags,
|
||||
},
|
||||
}
|
||||
})
|
||||
}
|
||||
|
||||
logForDebugging(`[Computer Use] Loaded ${apps.length} pre-authorized apps from config`)
|
||||
} catch {
|
||||
// Config doesn't exist or is invalid — no pre-authorized apps
|
||||
}
|
||||
}
|
||||
|
||||
let preAuthLoaded = false
|
||||
|
||||
function getOrBind(): Binding {
|
||||
if (binding) return binding;
|
||||
const ctx = buildSessionContext();
|
||||
@ -251,6 +312,12 @@ export function getComputerUseMCPToolOverrides(toolName: string): ComputerUseMCP
|
||||
const {
|
||||
dispatch
|
||||
} = getOrBind();
|
||||
|
||||
// Load pre-authorized apps on first tool call
|
||||
if (!preAuthLoaded) {
|
||||
preAuthLoaded = true;
|
||||
await loadPreAuthorizedApps();
|
||||
}
|
||||
const {
|
||||
telemetry,
|
||||
...result
|
||||
|
||||
Loading…
x
Reference in New Issue
Block a user