mirror of
https://github.com/NanmiCoder/cc-haha
synced 2026-07-26 15:03:34 +08:00
fix: Computer Use 点击功能修复
- ensureRuntimeFiles() 改为始终同步源文件而非仅缺失时复制 - pre-authorized apps 增加 tier: 'full',与现有 allowedApps 合并而非替换 - normalizeOsPermissions() 将 screenRecording: null 视为非阻塞 - config 路径改为 cc-haha/computer-use-config.json - mac_helper.py 改进鼠标点击与坐标处理
This commit is contained in:
parent
b002447436
commit
166fe5b676
76
desktop/src/api/computerUse.ts
Normal file
76
desktop/src/api/computerUse.ts
Normal file
@ -0,0 +1,76 @@
|
|||||||
|
import { api } from './client'
|
||||||
|
|
||||||
|
export type ComputerUseStatus = {
|
||||||
|
platform: string
|
||||||
|
supported: boolean
|
||||||
|
python: {
|
||||||
|
installed: boolean
|
||||||
|
version: string | null
|
||||||
|
path: string | null
|
||||||
|
}
|
||||||
|
venv: {
|
||||||
|
created: boolean
|
||||||
|
path: string
|
||||||
|
}
|
||||||
|
dependencies: {
|
||||||
|
installed: boolean
|
||||||
|
requirementsFound: boolean
|
||||||
|
}
|
||||||
|
permissions: {
|
||||||
|
accessibility: boolean | null
|
||||||
|
screenRecording: boolean | null
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
export type SetupStep = {
|
||||||
|
name: string
|
||||||
|
ok: boolean
|
||||||
|
message: string
|
||||||
|
}
|
||||||
|
|
||||||
|
export type SetupResult = {
|
||||||
|
success: boolean
|
||||||
|
steps: SetupStep[]
|
||||||
|
}
|
||||||
|
|
||||||
|
export type InstalledApp = {
|
||||||
|
bundleId: string
|
||||||
|
displayName: string
|
||||||
|
path: string
|
||||||
|
}
|
||||||
|
|
||||||
|
export type AuthorizedApp = {
|
||||||
|
bundleId: string
|
||||||
|
displayName: string
|
||||||
|
authorizedAt: string
|
||||||
|
}
|
||||||
|
|
||||||
|
export type ComputerUseConfig = {
|
||||||
|
authorizedApps: AuthorizedApp[]
|
||||||
|
grantFlags: {
|
||||||
|
clipboardRead: boolean
|
||||||
|
clipboardWrite: boolean
|
||||||
|
systemKeyCombos: boolean
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
export const computerUseApi = {
|
||||||
|
getStatus() {
|
||||||
|
return api.get<ComputerUseStatus>('/api/computer-use/status')
|
||||||
|
},
|
||||||
|
runSetup() {
|
||||||
|
return api.post<SetupResult>('/api/computer-use/setup', undefined, { timeout: 300_000 })
|
||||||
|
},
|
||||||
|
getInstalledApps() {
|
||||||
|
return api.get<{ apps: InstalledApp[] }>('/api/computer-use/apps')
|
||||||
|
},
|
||||||
|
getAuthorizedApps() {
|
||||||
|
return api.get<ComputerUseConfig>('/api/computer-use/authorized-apps')
|
||||||
|
},
|
||||||
|
setAuthorizedApps(config: Partial<ComputerUseConfig>) {
|
||||||
|
return api.put<{ ok: true }>('/api/computer-use/authorized-apps', config)
|
||||||
|
},
|
||||||
|
openSettings(pane: 'Privacy_ScreenCapture' | 'Privacy_Accessibility') {
|
||||||
|
return api.post<{ ok: true }>('/api/computer-use/open-settings', { pane })
|
||||||
|
},
|
||||||
|
}
|
||||||
420
desktop/src/pages/ComputerUseSettings.tsx
Normal file
420
desktop/src/pages/ComputerUseSettings.tsx
Normal file
@ -0,0 +1,420 @@
|
|||||||
|
import { useState, useEffect, useCallback, useMemo } from 'react'
|
||||||
|
import { computerUseApi, type ComputerUseStatus, type SetupResult, type InstalledApp, type AuthorizedApp } from '../api/computerUse'
|
||||||
|
import { useTranslation } from '../i18n'
|
||||||
|
|
||||||
|
type CheckState = 'loading' | 'ready' | 'error'
|
||||||
|
|
||||||
|
function StatusIcon({ ok }: { ok: boolean | null }) {
|
||||||
|
if (ok === null) {
|
||||||
|
return <span className="material-symbols-outlined text-[18px] text-[var(--color-text-tertiary)]">help</span>
|
||||||
|
}
|
||||||
|
return ok ? (
|
||||||
|
<span className="material-symbols-outlined text-[18px] text-green-500" style={{ fontVariationSettings: "'FILL' 1" }}>check_circle</span>
|
||||||
|
) : (
|
||||||
|
<span className="material-symbols-outlined text-[18px] text-red-400" style={{ fontVariationSettings: "'FILL' 1" }}>cancel</span>
|
||||||
|
)
|
||||||
|
}
|
||||||
|
|
||||||
|
function StatusRow({ label, ok, detail }: { label: string; ok: boolean | null; detail: string }) {
|
||||||
|
return (
|
||||||
|
<div className="flex items-center gap-3 py-2.5 px-4 rounded-lg bg-[var(--color-surface-container-low)]">
|
||||||
|
<StatusIcon ok={ok} />
|
||||||
|
<div className="flex-1 min-w-0">
|
||||||
|
<span className="text-sm font-medium text-[var(--color-text-primary)]">{label}</span>
|
||||||
|
<span className="ml-2 text-xs text-[var(--color-text-tertiary)]">{detail}</span>
|
||||||
|
</div>
|
||||||
|
</div>
|
||||||
|
)
|
||||||
|
}
|
||||||
|
|
||||||
|
async function openSystemSettings(pane: 'Privacy_ScreenCapture' | 'Privacy_Accessibility') {
|
||||||
|
await computerUseApi.openSettings(pane)
|
||||||
|
}
|
||||||
|
|
||||||
|
export function ComputerUseSettings() {
|
||||||
|
const t = useTranslation()
|
||||||
|
const [status, setStatus] = useState<ComputerUseStatus | null>(null)
|
||||||
|
const [checkState, setCheckState] = useState<CheckState>('loading')
|
||||||
|
const [setupRunning, setSetupRunning] = useState(false)
|
||||||
|
const [setupResult, setSetupResult] = useState<SetupResult | null>(null)
|
||||||
|
|
||||||
|
// App authorization state
|
||||||
|
const [installedApps, setInstalledApps] = useState<InstalledApp[]>([])
|
||||||
|
const [authorizedBundleIds, setAuthorizedBundleIds] = useState<Set<string>>(new Set())
|
||||||
|
const [authorizedApps, setAuthorizedApps] = useState<AuthorizedApp[]>([])
|
||||||
|
const [appsLoading, setAppsLoading] = useState(false)
|
||||||
|
const [appsSaved, setAppsSaved] = useState(false)
|
||||||
|
const [searchQuery, setSearchQuery] = useState('')
|
||||||
|
const [clipboardAccess, setClipboardAccess] = useState(true)
|
||||||
|
const [systemKeys, setSystemKeys] = useState(true)
|
||||||
|
|
||||||
|
const fetchStatus = useCallback(async () => {
|
||||||
|
setCheckState('loading')
|
||||||
|
try {
|
||||||
|
const s = await computerUseApi.getStatus()
|
||||||
|
setStatus(s)
|
||||||
|
setCheckState('ready')
|
||||||
|
} catch {
|
||||||
|
setCheckState('error')
|
||||||
|
}
|
||||||
|
}, [])
|
||||||
|
|
||||||
|
const fetchApps = useCallback(async () => {
|
||||||
|
setAppsLoading(true)
|
||||||
|
try {
|
||||||
|
const [appsResult, configResult] = await Promise.all([
|
||||||
|
computerUseApi.getInstalledApps(),
|
||||||
|
computerUseApi.getAuthorizedApps(),
|
||||||
|
])
|
||||||
|
setInstalledApps(appsResult.apps)
|
||||||
|
setAuthorizedApps(configResult.authorizedApps)
|
||||||
|
setAuthorizedBundleIds(new Set(configResult.authorizedApps.map(a => a.bundleId)))
|
||||||
|
setClipboardAccess(configResult.grantFlags.clipboardRead)
|
||||||
|
setSystemKeys(configResult.grantFlags.systemKeyCombos)
|
||||||
|
} catch {
|
||||||
|
// API not ready
|
||||||
|
} finally {
|
||||||
|
setAppsLoading(false)
|
||||||
|
}
|
||||||
|
}, [])
|
||||||
|
|
||||||
|
useEffect(() => {
|
||||||
|
fetchStatus()
|
||||||
|
}, [fetchStatus])
|
||||||
|
|
||||||
|
// Load apps when environment is ready
|
||||||
|
const envReady = status?.venv.created && status?.dependencies.installed
|
||||||
|
useEffect(() => {
|
||||||
|
if (envReady) fetchApps()
|
||||||
|
}, [envReady, fetchApps])
|
||||||
|
|
||||||
|
const handleSetup = async () => {
|
||||||
|
setSetupRunning(true)
|
||||||
|
setSetupResult(null)
|
||||||
|
try {
|
||||||
|
const result = await computerUseApi.runSetup()
|
||||||
|
setSetupResult(result)
|
||||||
|
await fetchStatus()
|
||||||
|
if (result.success) await fetchApps()
|
||||||
|
} catch {
|
||||||
|
setSetupResult({ success: false, steps: [{ name: 'error', ok: false, message: 'Request failed' }] })
|
||||||
|
} finally {
|
||||||
|
setSetupRunning(false)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
const toggleApp = (app: InstalledApp) => {
|
||||||
|
const newSet = new Set(authorizedBundleIds)
|
||||||
|
let newAuthorized = [...authorizedApps]
|
||||||
|
if (newSet.has(app.bundleId)) {
|
||||||
|
newSet.delete(app.bundleId)
|
||||||
|
newAuthorized = newAuthorized.filter(a => a.bundleId !== app.bundleId)
|
||||||
|
} else {
|
||||||
|
newSet.add(app.bundleId)
|
||||||
|
newAuthorized.push({
|
||||||
|
bundleId: app.bundleId,
|
||||||
|
displayName: app.displayName,
|
||||||
|
authorizedAt: new Date().toISOString(),
|
||||||
|
})
|
||||||
|
}
|
||||||
|
setAuthorizedBundleIds(newSet)
|
||||||
|
setAuthorizedApps(newAuthorized)
|
||||||
|
|
||||||
|
// Auto-save
|
||||||
|
computerUseApi.setAuthorizedApps({
|
||||||
|
authorizedApps: newAuthorized,
|
||||||
|
grantFlags: { clipboardRead: clipboardAccess, clipboardWrite: clipboardAccess, systemKeyCombos: systemKeys },
|
||||||
|
}).then(() => {
|
||||||
|
setAppsSaved(true)
|
||||||
|
setTimeout(() => setAppsSaved(false), 1500)
|
||||||
|
})
|
||||||
|
}
|
||||||
|
|
||||||
|
const toggleFlag = (flag: 'clipboard' | 'systemKeys', value: boolean) => {
|
||||||
|
if (flag === 'clipboard') setClipboardAccess(value)
|
||||||
|
else setSystemKeys(value)
|
||||||
|
|
||||||
|
computerUseApi.setAuthorizedApps({
|
||||||
|
authorizedApps,
|
||||||
|
grantFlags: {
|
||||||
|
clipboardRead: flag === 'clipboard' ? value : clipboardAccess,
|
||||||
|
clipboardWrite: flag === 'clipboard' ? value : clipboardAccess,
|
||||||
|
systemKeyCombos: flag === 'systemKeys' ? value : systemKeys,
|
||||||
|
},
|
||||||
|
})
|
||||||
|
}
|
||||||
|
|
||||||
|
const allReady =
|
||||||
|
status?.supported &&
|
||||||
|
status.python.installed &&
|
||||||
|
status.venv.created &&
|
||||||
|
status.dependencies.installed
|
||||||
|
|
||||||
|
const accessibilityNeedsAttention = status?.permissions.accessibility === false
|
||||||
|
const screenRecordingNeedsAttention = status?.permissions.screenRecording === false
|
||||||
|
const screenRecordingReady = status ? status.permissions.screenRecording !== false : null
|
||||||
|
|
||||||
|
// Filter apps by search query
|
||||||
|
const filteredApps = useMemo(() => {
|
||||||
|
if (!searchQuery) return installedApps
|
||||||
|
const q = searchQuery.toLowerCase()
|
||||||
|
return installedApps.filter(
|
||||||
|
a => a.displayName.toLowerCase().includes(q) || a.bundleId.toLowerCase().includes(q)
|
||||||
|
)
|
||||||
|
}, [installedApps, searchQuery])
|
||||||
|
|
||||||
|
// Sort: authorized apps first, then alphabetical
|
||||||
|
const sortedApps = useMemo(() => {
|
||||||
|
return [...filteredApps].sort((a, b) => {
|
||||||
|
const aAuth = authorizedBundleIds.has(a.bundleId) ? 0 : 1
|
||||||
|
const bAuth = authorizedBundleIds.has(b.bundleId) ? 0 : 1
|
||||||
|
if (aAuth !== bAuth) return aAuth - bAuth
|
||||||
|
return a.displayName.localeCompare(b.displayName)
|
||||||
|
})
|
||||||
|
}, [filteredApps, authorizedBundleIds])
|
||||||
|
|
||||||
|
return (
|
||||||
|
<div className="max-w-2xl space-y-6">
|
||||||
|
{/* Title */}
|
||||||
|
<div>
|
||||||
|
<h2 className="text-lg font-semibold text-[var(--color-text-primary)]">
|
||||||
|
{t('settings.computerUse.title')}
|
||||||
|
</h2>
|
||||||
|
<p className="mt-1 text-sm text-[var(--color-text-secondary)]">
|
||||||
|
{t('settings.computerUse.description')}
|
||||||
|
</p>
|
||||||
|
</div>
|
||||||
|
|
||||||
|
{checkState === 'loading' ? (
|
||||||
|
<div className="py-8 text-center text-sm text-[var(--color-text-tertiary)]">
|
||||||
|
{t('common.loading')}
|
||||||
|
</div>
|
||||||
|
) : checkState === 'error' ? (
|
||||||
|
<div className="py-8 text-center text-sm text-red-400">
|
||||||
|
Failed to check status.
|
||||||
|
<button onClick={fetchStatus} className="ml-2 underline">{t('common.retry')}</button>
|
||||||
|
</div>
|
||||||
|
) : status ? (
|
||||||
|
<>
|
||||||
|
{!status.supported && (
|
||||||
|
<div className="px-4 py-3 rounded-lg bg-yellow-500/10 border border-yellow-500/30 text-sm text-yellow-600">
|
||||||
|
{t('settings.computerUse.notSupported')}
|
||||||
|
</div>
|
||||||
|
)}
|
||||||
|
|
||||||
|
{/* Status checks */}
|
||||||
|
<div className="space-y-2">
|
||||||
|
<StatusRow
|
||||||
|
label={t('settings.computerUse.python')}
|
||||||
|
ok={status.python.installed}
|
||||||
|
detail={
|
||||||
|
status.python.installed
|
||||||
|
? `${t('settings.computerUse.pythonFound')} — ${status.python.version} (${status.python.path})`
|
||||||
|
: t('settings.computerUse.pythonNotFound')
|
||||||
|
}
|
||||||
|
/>
|
||||||
|
<StatusRow
|
||||||
|
label={t('settings.computerUse.venv')}
|
||||||
|
ok={status.venv.created}
|
||||||
|
detail={status.venv.created ? `${t('settings.computerUse.venvReady')} — ${status.venv.path}` : t('settings.computerUse.venvNotReady')}
|
||||||
|
/>
|
||||||
|
<StatusRow
|
||||||
|
label={t('settings.computerUse.deps')}
|
||||||
|
ok={status.dependencies.installed}
|
||||||
|
detail={status.dependencies.installed ? t('settings.computerUse.depsReady') : t('settings.computerUse.depsNotReady')}
|
||||||
|
/>
|
||||||
|
</div>
|
||||||
|
|
||||||
|
{/* macOS Permissions */}
|
||||||
|
{envReady && (
|
||||||
|
<>
|
||||||
|
<StatusRow
|
||||||
|
label={t('settings.computerUse.accessibility')}
|
||||||
|
ok={status.permissions.accessibility}
|
||||||
|
detail={
|
||||||
|
status.permissions.accessibility === null ? t('settings.computerUse.permUnknown')
|
||||||
|
: status.permissions.accessibility ? t('settings.computerUse.permGranted')
|
||||||
|
: t('settings.computerUse.permDenied')
|
||||||
|
}
|
||||||
|
/>
|
||||||
|
<StatusRow
|
||||||
|
label={t('settings.computerUse.screenRecording')}
|
||||||
|
ok={screenRecordingReady}
|
||||||
|
detail={
|
||||||
|
status.permissions.screenRecording === true ? t('settings.computerUse.permGranted')
|
||||||
|
: status.permissions.screenRecording === false ? t('settings.computerUse.permDenied')
|
||||||
|
: t('settings.computerUse.permScreenRecordingUnknownSoft')
|
||||||
|
}
|
||||||
|
/>
|
||||||
|
{(accessibilityNeedsAttention || screenRecordingNeedsAttention) && (
|
||||||
|
<div className="flex flex-col gap-2 px-4 py-3 rounded-lg bg-yellow-500/5 border border-yellow-500/20">
|
||||||
|
<p className="text-xs text-[var(--color-text-tertiary)]">{t('settings.computerUse.permRestartHint')}</p>
|
||||||
|
<div className="flex gap-2">
|
||||||
|
{accessibilityNeedsAttention && (
|
||||||
|
<button
|
||||||
|
onClick={() => openSystemSettings('Privacy_Accessibility')}
|
||||||
|
className="flex items-center gap-1.5 px-3 py-1.5 text-xs font-medium text-[var(--color-text-accent)] border border-[var(--color-border)] rounded-lg hover:bg-[var(--color-surface-hover)]"
|
||||||
|
>
|
||||||
|
<span className="material-symbols-outlined text-[14px]">open_in_new</span>
|
||||||
|
{t('settings.computerUse.openAccessibility')}
|
||||||
|
</button>
|
||||||
|
)}
|
||||||
|
{screenRecordingNeedsAttention && (
|
||||||
|
<button
|
||||||
|
onClick={() => openSystemSettings('Privacy_ScreenCapture')}
|
||||||
|
className="flex items-center gap-1.5 px-3 py-1.5 text-xs font-medium text-[var(--color-text-accent)] border border-[var(--color-border)] rounded-lg hover:bg-[var(--color-surface-hover)]"
|
||||||
|
>
|
||||||
|
<span className="material-symbols-outlined text-[14px]">open_in_new</span>
|
||||||
|
{t('settings.computerUse.openScreenRecording')}
|
||||||
|
</button>
|
||||||
|
)}
|
||||||
|
</div>
|
||||||
|
</div>
|
||||||
|
)}
|
||||||
|
</>
|
||||||
|
)}
|
||||||
|
|
||||||
|
{allReady && status.permissions.accessibility && screenRecordingReady && (
|
||||||
|
<div className="px-4 py-3 rounded-lg bg-green-500/10 border border-green-500/30 text-sm text-green-600 flex items-center gap-2">
|
||||||
|
<span className="material-symbols-outlined text-[18px]" style={{ fontVariationSettings: "'FILL' 1" }}>verified</span>
|
||||||
|
{t('settings.computerUse.allReady')}
|
||||||
|
</div>
|
||||||
|
)}
|
||||||
|
|
||||||
|
{setupResult && (
|
||||||
|
<div className={`rounded-lg border p-4 space-y-2 ${setupResult.success ? 'bg-green-500/5 border-green-500/30' : 'bg-red-500/5 border-red-500/30'}`}>
|
||||||
|
<div className={`text-sm font-medium ${setupResult.success ? 'text-green-600' : 'text-red-400'}`}>
|
||||||
|
{setupResult.success ? t('settings.computerUse.setupSuccess') : t('settings.computerUse.setupFail')}
|
||||||
|
</div>
|
||||||
|
{setupResult.steps.map((step, i) => (
|
||||||
|
<div key={i} className="flex items-center gap-2 text-xs text-[var(--color-text-secondary)]">
|
||||||
|
<StatusIcon ok={step.ok} />
|
||||||
|
<span>{step.message}</span>
|
||||||
|
</div>
|
||||||
|
))}
|
||||||
|
</div>
|
||||||
|
)}
|
||||||
|
|
||||||
|
{/* Action buttons */}
|
||||||
|
<div className="flex gap-3">
|
||||||
|
{!envReady && status.python.installed && (
|
||||||
|
<button
|
||||||
|
onClick={handleSetup}
|
||||||
|
disabled={setupRunning}
|
||||||
|
className="flex items-center gap-2 px-5 py-2.5 bg-[var(--color-brand)] text-white text-sm font-semibold rounded-lg hover:opacity-90 disabled:opacity-50 transition-opacity"
|
||||||
|
>
|
||||||
|
<span className="material-symbols-outlined text-[18px]">{setupRunning ? 'hourglass_empty' : 'download'}</span>
|
||||||
|
{setupRunning ? t('settings.computerUse.setupRunning') : t('settings.computerUse.setupBtn')}
|
||||||
|
</button>
|
||||||
|
)}
|
||||||
|
<button
|
||||||
|
onClick={fetchStatus}
|
||||||
|
className="flex items-center gap-2 px-4 py-2.5 text-sm text-[var(--color-text-secondary)] hover:text-[var(--color-text-primary)] border border-[var(--color-border)] rounded-lg hover:bg-[var(--color-surface-hover)] transition-colors"
|
||||||
|
>
|
||||||
|
<span className="material-symbols-outlined text-[18px]">refresh</span>
|
||||||
|
{t('settings.computerUse.recheckBtn')}
|
||||||
|
</button>
|
||||||
|
</div>
|
||||||
|
|
||||||
|
{/* ─── App Authorization Section ─── */}
|
||||||
|
{envReady && (
|
||||||
|
<div className="space-y-4 pt-4 border-t border-[var(--color-border)]">
|
||||||
|
<div>
|
||||||
|
<h3 className="text-base font-semibold text-[var(--color-text-primary)] flex items-center gap-2">
|
||||||
|
{t('settings.computerUse.appsTitle')}
|
||||||
|
{appsSaved && (
|
||||||
|
<span className="text-xs font-normal text-green-500 flex items-center gap-1">
|
||||||
|
<span className="material-symbols-outlined text-[14px]" style={{ fontVariationSettings: "'FILL' 1" }}>check</span>
|
||||||
|
{t('settings.computerUse.appsSaved')}
|
||||||
|
</span>
|
||||||
|
)}
|
||||||
|
</h3>
|
||||||
|
<p className="mt-1 text-sm text-[var(--color-text-secondary)]">
|
||||||
|
{t('settings.computerUse.appsDescription')}
|
||||||
|
</p>
|
||||||
|
</div>
|
||||||
|
|
||||||
|
{/* Grant flags */}
|
||||||
|
<div className="flex gap-4">
|
||||||
|
<label className="flex items-center gap-2 text-sm text-[var(--color-text-secondary)] cursor-pointer">
|
||||||
|
<input
|
||||||
|
type="checkbox"
|
||||||
|
checked={clipboardAccess}
|
||||||
|
onChange={e => toggleFlag('clipboard', e.target.checked)}
|
||||||
|
className="rounded border-[var(--color-border)] accent-[var(--color-brand)]"
|
||||||
|
/>
|
||||||
|
{t('settings.computerUse.flagClipboard')}
|
||||||
|
</label>
|
||||||
|
<label className="flex items-center gap-2 text-sm text-[var(--color-text-secondary)] cursor-pointer">
|
||||||
|
<input
|
||||||
|
type="checkbox"
|
||||||
|
checked={systemKeys}
|
||||||
|
onChange={e => toggleFlag('systemKeys', e.target.checked)}
|
||||||
|
className="rounded border-[var(--color-border)] accent-[var(--color-brand)]"
|
||||||
|
/>
|
||||||
|
{t('settings.computerUse.flagSystemKeys')}
|
||||||
|
</label>
|
||||||
|
</div>
|
||||||
|
|
||||||
|
{/* Search */}
|
||||||
|
<div className="relative">
|
||||||
|
<span className="material-symbols-outlined text-[18px] text-[var(--color-text-tertiary)] absolute left-3 top-1/2 -translate-y-1/2">search</span>
|
||||||
|
<input
|
||||||
|
type="text"
|
||||||
|
value={searchQuery}
|
||||||
|
onChange={e => setSearchQuery(e.target.value)}
|
||||||
|
placeholder={t('settings.computerUse.appsSearch')}
|
||||||
|
className="w-full pl-9 pr-4 py-2 text-sm bg-[var(--color-surface-container-low)] border border-[var(--color-border)] rounded-lg text-[var(--color-text-primary)] placeholder:text-[var(--color-text-tertiary)] focus:outline-none focus:border-[var(--color-brand)]"
|
||||||
|
/>
|
||||||
|
</div>
|
||||||
|
|
||||||
|
{/* App list */}
|
||||||
|
{appsLoading ? (
|
||||||
|
<div className="py-6 text-center text-sm text-[var(--color-text-tertiary)]">
|
||||||
|
{t('settings.computerUse.appsLoading')}
|
||||||
|
</div>
|
||||||
|
) : installedApps.length === 0 ? (
|
||||||
|
<div className="py-6 text-center text-sm text-[var(--color-text-tertiary)]">
|
||||||
|
{t('settings.computerUse.appsEmpty')}
|
||||||
|
</div>
|
||||||
|
) : (
|
||||||
|
<div className="max-h-[400px] overflow-y-auto rounded-lg border border-[var(--color-border)]">
|
||||||
|
{sortedApps.map(app => {
|
||||||
|
const isAuthorized = authorizedBundleIds.has(app.bundleId)
|
||||||
|
return (
|
||||||
|
<button
|
||||||
|
key={app.bundleId}
|
||||||
|
onClick={() => toggleApp(app)}
|
||||||
|
className={`w-full flex items-center gap-3 px-4 py-2.5 text-left transition-colors hover:bg-[var(--color-surface-hover)] border-b border-[var(--color-border)] last:border-b-0 ${
|
||||||
|
isAuthorized ? 'bg-[var(--color-brand)]/5' : ''
|
||||||
|
}`}
|
||||||
|
>
|
||||||
|
<div className={`w-5 h-5 rounded flex items-center justify-center flex-shrink-0 border ${
|
||||||
|
isAuthorized
|
||||||
|
? 'bg-[var(--color-brand)] border-[var(--color-brand)]'
|
||||||
|
: 'border-[var(--color-border)]'
|
||||||
|
}`}>
|
||||||
|
{isAuthorized && (
|
||||||
|
<span className="material-symbols-outlined text-[14px] text-white" style={{ fontVariationSettings: "'FILL' 1" }}>check</span>
|
||||||
|
)}
|
||||||
|
</div>
|
||||||
|
<div className="flex-1 min-w-0">
|
||||||
|
<div className="text-sm font-medium text-[var(--color-text-primary)] truncate">
|
||||||
|
{app.displayName}
|
||||||
|
</div>
|
||||||
|
<div className="text-[11px] text-[var(--color-text-tertiary)] truncate font-mono">
|
||||||
|
{app.bundleId}
|
||||||
|
</div>
|
||||||
|
</div>
|
||||||
|
</button>
|
||||||
|
)
|
||||||
|
})}
|
||||||
|
</div>
|
||||||
|
)}
|
||||||
|
</div>
|
||||||
|
)}
|
||||||
|
</>
|
||||||
|
) : null}
|
||||||
|
</div>
|
||||||
|
)
|
||||||
|
}
|
||||||
@ -28,6 +28,7 @@ from Quartz import (
|
|||||||
CGRectContainsPoint,
|
CGRectContainsPoint,
|
||||||
CGRectIntersection,
|
CGRectIntersection,
|
||||||
CGPointMake,
|
CGPointMake,
|
||||||
|
CGPreflightScreenCaptureAccess,
|
||||||
kCGNullWindowID,
|
kCGNullWindowID,
|
||||||
kCGWindowBounds,
|
kCGWindowBounds,
|
||||||
kCGWindowIsOnscreen,
|
kCGWindowIsOnscreen,
|
||||||
@ -232,7 +233,16 @@ def choose_display(display_id: int | None) -> dict[str, Any]:
|
|||||||
raise RuntimeError(f"Unknown display: {display_id}")
|
raise RuntimeError(f"Unknown display: {display_id}")
|
||||||
|
|
||||||
|
|
||||||
|
def ensure_screen_recording_permission() -> None:
|
||||||
|
"""No-op: CGPreflightScreenCaptureAccess is unreliable for child processes
|
||||||
|
(returns False even when the parent app has TCC permission), and any actual
|
||||||
|
capture attempt triggers a macOS popup on newer versions. Let the actual
|
||||||
|
capture call handle errors instead."""
|
||||||
|
pass
|
||||||
|
|
||||||
|
|
||||||
def capture_display(display_id: int | None, resize: tuple[int, int] | None = None) -> dict[str, Any]:
|
def capture_display(display_id: int | None, resize: tuple[int, int] | None = None) -> dict[str, Any]:
|
||||||
|
ensure_screen_recording_permission()
|
||||||
display = choose_display(display_id)
|
display = choose_display(display_id)
|
||||||
monitor = {
|
monitor = {
|
||||||
"left": display["originX"],
|
"left": display["originX"],
|
||||||
@ -262,6 +272,7 @@ def capture_display(display_id: int | None, resize: tuple[int, int] | None = Non
|
|||||||
|
|
||||||
|
|
||||||
def capture_region(region: dict[str, int], resize: tuple[int, int] | None = None) -> dict[str, Any]:
|
def capture_region(region: dict[str, int], resize: tuple[int, int] | None = None) -> dict[str, Any]:
|
||||||
|
ensure_screen_recording_permission()
|
||||||
with mss.mss() as sct:
|
with mss.mss() as sct:
|
||||||
raw = sct.grab(region)
|
raw = sct.grab(region)
|
||||||
image = Image.frombytes("RGB", raw.size, raw.rgb)
|
image = Image.frombytes("RGB", raw.size, raw.rgb)
|
||||||
@ -461,17 +472,61 @@ def write_clipboard(text: str) -> None:
|
|||||||
pb.setString_forType_(text, NSPasteboardTypeString)
|
pb.setString_forType_(text, NSPasteboardTypeString)
|
||||||
|
|
||||||
|
|
||||||
def check_permissions() -> dict[str, bool]:
|
def detect_screen_recording_permission() -> bool | None:
|
||||||
|
"""Best-effort passive screen-recording probe with no system prompt.
|
||||||
|
|
||||||
|
`CGPreflightScreenCaptureAccess()` is fast and explicit when it returns
|
||||||
|
True, but on child processes launched by a TCC-authorized app bundle it can
|
||||||
|
still return False. As a fallback, inspect the visible window list: Apple
|
||||||
|
only exposes other apps' window titles when Screen Recording access is
|
||||||
|
granted. If we can see at least one title, treat the permission as granted.
|
||||||
|
If we can inspect visible windows but every title is blank, treat it as not
|
||||||
|
granted. If window enumeration itself is unavailable, return None.
|
||||||
|
"""
|
||||||
|
|
||||||
|
try:
|
||||||
|
if CGPreflightScreenCaptureAccess():
|
||||||
|
return True
|
||||||
|
except Exception:
|
||||||
|
pass
|
||||||
|
|
||||||
|
try:
|
||||||
|
windows = CGWindowListCopyWindowInfo(
|
||||||
|
kCGWindowListOptionOnScreenOnly | kCGWindowListExcludeDesktopElements,
|
||||||
|
kCGNullWindowID,
|
||||||
|
)
|
||||||
|
except Exception:
|
||||||
|
return None
|
||||||
|
|
||||||
|
eligible_windows = 0
|
||||||
|
for window in windows or []:
|
||||||
|
if int(window.get(kCGWindowLayer, 0)) != 0:
|
||||||
|
continue
|
||||||
|
if not bool(window.get(kCGWindowIsOnscreen, True)):
|
||||||
|
continue
|
||||||
|
|
||||||
|
bounds = window.get(kCGWindowBounds) or {}
|
||||||
|
width = int(bounds.get("Width", 0))
|
||||||
|
height = int(bounds.get("Height", 0))
|
||||||
|
if width <= 1 or height <= 1:
|
||||||
|
continue
|
||||||
|
|
||||||
|
eligible_windows += 1
|
||||||
|
if (window.get(kCGWindowName, "") or "").strip():
|
||||||
|
return True
|
||||||
|
|
||||||
|
if eligible_windows > 0:
|
||||||
|
return False
|
||||||
|
return None
|
||||||
|
|
||||||
|
|
||||||
|
def check_permissions() -> dict[str, bool | None]:
|
||||||
accessibility = True
|
accessibility = True
|
||||||
try:
|
try:
|
||||||
run_osascript('tell application "System Events" to get name of first process')
|
run_osascript('tell application "System Events" to get name of first process')
|
||||||
except Exception:
|
except Exception:
|
||||||
accessibility = False
|
accessibility = False
|
||||||
screen_recording = True
|
screen_recording = detect_screen_recording_permission()
|
||||||
try:
|
|
||||||
capture_display(None)
|
|
||||||
except Exception:
|
|
||||||
screen_recording = False
|
|
||||||
return {
|
return {
|
||||||
"accessibility": accessibility,
|
"accessibility": accessibility,
|
||||||
"screenRecording": screen_recording,
|
"screenRecording": screen_recording,
|
||||||
|
|||||||
430
src/server/api/computer-use.ts
Normal file
430
src/server/api/computer-use.ts
Normal file
@ -0,0 +1,430 @@
|
|||||||
|
/**
|
||||||
|
* Computer Use API — 环境检测与依赖安装
|
||||||
|
*
|
||||||
|
* Routes:
|
||||||
|
* GET /api/computer-use/status — 检测 Python3、venv、依赖、权限状态
|
||||||
|
* POST /api/computer-use/setup — 创建 venv 并安装依赖
|
||||||
|
*/
|
||||||
|
|
||||||
|
import { homedir } from 'os'
|
||||||
|
import { join } from 'path'
|
||||||
|
import { access, readFile, mkdir, writeFile } from 'fs/promises'
|
||||||
|
import { createHash } from 'crypto'
|
||||||
|
import path from 'path'
|
||||||
|
import { fileURLToPath } from 'url'
|
||||||
|
// Embed mac_helper.py at compile time so it's available in bundled mode
|
||||||
|
// @ts-ignore — Bun text import
|
||||||
|
import MAC_HELPER_CONTENT from '../../../runtime/mac_helper.py' with { type: 'text' }
|
||||||
|
|
||||||
|
const __dirname = path.dirname(fileURLToPath(import.meta.url))
|
||||||
|
const projectRoot = path.resolve(__dirname, '../../..')
|
||||||
|
const devRuntimeRoot = join(projectRoot, 'runtime')
|
||||||
|
const claudeHome = process.env.CLAUDE_CONFIG_DIR ?? join(homedir(), '.claude')
|
||||||
|
const runtimeStateRoot = join(claudeHome, '.runtime')
|
||||||
|
const venvRoot = join(runtimeStateRoot, 'venv')
|
||||||
|
const installStampPath = join(runtimeStateRoot, 'requirements.sha256')
|
||||||
|
|
||||||
|
// Embedded content of requirements.txt — kept in sync with runtime/requirements.txt.
|
||||||
|
// This ensures the bundled sidecar can create the file without disk access.
|
||||||
|
const REQUIREMENTS_CONTENT = `mss>=10.1.0
|
||||||
|
Pillow>=11.3.0
|
||||||
|
pyautogui>=0.9.54
|
||||||
|
pyobjc-core>=11.1
|
||||||
|
pyobjc-framework-Cocoa>=11.1
|
||||||
|
pyobjc-framework-Quartz>=11.1
|
||||||
|
`
|
||||||
|
|
||||||
|
// 清华大学 PyPI 镜像,国内安装速度更快
|
||||||
|
const PIP_INDEX_URL = 'https://pypi.tuna.tsinghua.edu.cn/simple/'
|
||||||
|
const PIP_TRUSTED_HOST = 'pypi.tuna.tsinghua.edu.cn'
|
||||||
|
|
||||||
|
// Paths that resolve correctly in both dev and bundled modes
|
||||||
|
function getRequirementsPath(): string {
|
||||||
|
return join(runtimeStateRoot, 'requirements.txt')
|
||||||
|
}
|
||||||
|
|
||||||
|
function getHelperPath(): string {
|
||||||
|
// In bundled mode mac_helper.py is extracted to runtimeStateRoot.
|
||||||
|
// In dev mode we also copy it there during setup, so both modes
|
||||||
|
// read from the same location after setup runs.
|
||||||
|
return join(runtimeStateRoot, 'mac_helper.py')
|
||||||
|
}
|
||||||
|
|
||||||
|
async function pathExists(target: string): Promise<boolean> {
|
||||||
|
try {
|
||||||
|
await access(target)
|
||||||
|
return true
|
||||||
|
} catch {
|
||||||
|
return false
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
async function runCommand(
|
||||||
|
cmd: string,
|
||||||
|
args: string[],
|
||||||
|
): Promise<{ ok: boolean; stdout: string; stderr: string; code: number }> {
|
||||||
|
try {
|
||||||
|
const proc = Bun.spawn([cmd, ...args], {
|
||||||
|
stdout: 'pipe',
|
||||||
|
stderr: 'pipe',
|
||||||
|
})
|
||||||
|
const [stdout, stderr] = await Promise.all([
|
||||||
|
new Response(proc.stdout).text(),
|
||||||
|
new Response(proc.stderr).text(),
|
||||||
|
])
|
||||||
|
const code = await proc.exited
|
||||||
|
return { ok: code === 0, stdout: stdout.trim(), stderr: stderr.trim(), code }
|
||||||
|
} catch {
|
||||||
|
return { ok: false, stdout: '', stderr: `Failed to run ${cmd}`, code: -1 }
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
/**
|
||||||
|
* Ensure runtime source files (requirements.txt, mac_helper.py) exist in
|
||||||
|
* ~/.claude/.runtime/. In dev mode they are copied from the project's
|
||||||
|
* runtime/ directory; in bundled mode requirements.txt is written from the
|
||||||
|
* embedded constant and mac_helper.py is copied from the project dir (if
|
||||||
|
* available) or skipped (it will already have been extracted on a prior run).
|
||||||
|
*/
|
||||||
|
async function ensureRuntimeFiles(): Promise<void> {
|
||||||
|
await mkdir(runtimeStateRoot, { recursive: true })
|
||||||
|
|
||||||
|
// requirements.txt — always write from embedded constant (authoritative)
|
||||||
|
await writeFile(getRequirementsPath(), REQUIREMENTS_CONTENT, 'utf8')
|
||||||
|
|
||||||
|
// mac_helper.py — always write from embedded content (compile-time import)
|
||||||
|
await writeFile(getHelperPath(), MAC_HELPER_CONTENT, 'utf8')
|
||||||
|
}
|
||||||
|
|
||||||
|
type EnvStatus = {
|
||||||
|
platform: string
|
||||||
|
supported: boolean
|
||||||
|
python: {
|
||||||
|
installed: boolean
|
||||||
|
version: string | null
|
||||||
|
path: string | null
|
||||||
|
}
|
||||||
|
venv: {
|
||||||
|
created: boolean
|
||||||
|
path: string
|
||||||
|
}
|
||||||
|
dependencies: {
|
||||||
|
installed: boolean
|
||||||
|
requirementsFound: boolean
|
||||||
|
}
|
||||||
|
permissions: {
|
||||||
|
accessibility: boolean | null
|
||||||
|
screenRecording: boolean | null
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
async function checkStatus(): Promise<EnvStatus> {
|
||||||
|
const platform = process.platform
|
||||||
|
const supported = platform === 'darwin'
|
||||||
|
|
||||||
|
// Check Python 3
|
||||||
|
const pythonResult = await runCommand('python3', ['--version'])
|
||||||
|
const pythonInstalled = pythonResult.ok
|
||||||
|
const pythonVersion = pythonInstalled
|
||||||
|
? pythonResult.stdout.replace('Python ', '')
|
||||||
|
: null
|
||||||
|
|
||||||
|
let pythonPath: string | null = null
|
||||||
|
if (pythonInstalled) {
|
||||||
|
const whichResult = await runCommand('which', ['python3'])
|
||||||
|
pythonPath = whichResult.ok ? whichResult.stdout : null
|
||||||
|
}
|
||||||
|
|
||||||
|
// Check venv
|
||||||
|
const venvCreated = await pathExists(join(venvRoot, 'bin', 'python3'))
|
||||||
|
|
||||||
|
// Check dependencies — use the state dir copy
|
||||||
|
const reqPath = getRequirementsPath()
|
||||||
|
const requirementsFound = await pathExists(reqPath)
|
||||||
|
let depsInstalled = false
|
||||||
|
if (requirementsFound && venvCreated) {
|
||||||
|
try {
|
||||||
|
const requirements = await readFile(reqPath, 'utf8')
|
||||||
|
const digest = createHash('sha256').update(requirements).digest('hex')
|
||||||
|
const stamp = (await readFile(installStampPath, 'utf8')).trim()
|
||||||
|
depsInstalled = stamp === digest
|
||||||
|
} catch {
|
||||||
|
depsInstalled = false
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
// Check macOS permissions without triggering a system prompt. The helper
|
||||||
|
// uses preflight + visible-window metadata as a passive fallback because
|
||||||
|
// plain preflight can misreport child processes launched by the desktop app.
|
||||||
|
let accessibility: boolean | null = null
|
||||||
|
let screenRecording: boolean | null = null
|
||||||
|
if (supported && venvCreated && depsInstalled) {
|
||||||
|
try { await ensureRuntimeFiles() } catch {}
|
||||||
|
const helperPath = getHelperPath()
|
||||||
|
if (await pathExists(helperPath)) {
|
||||||
|
const pythonBin = join(venvRoot, 'bin', 'python3')
|
||||||
|
const permResult = await runCommand(pythonBin, [helperPath, 'check_permissions'])
|
||||||
|
if (permResult.ok) {
|
||||||
|
try {
|
||||||
|
const parsed = JSON.parse(permResult.stdout)
|
||||||
|
if (parsed.ok && parsed.result) {
|
||||||
|
accessibility = parsed.result.accessibility ?? null
|
||||||
|
screenRecording = parsed.result.screenRecording ?? null
|
||||||
|
}
|
||||||
|
} catch {}
|
||||||
|
}
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
return {
|
||||||
|
platform,
|
||||||
|
supported,
|
||||||
|
python: { installed: pythonInstalled, version: pythonVersion, path: pythonPath },
|
||||||
|
venv: { created: venvCreated, path: venvRoot },
|
||||||
|
dependencies: { installed: depsInstalled, requirementsFound: requirementsFound || true },
|
||||||
|
permissions: { accessibility, screenRecording },
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
type SetupResult = {
|
||||||
|
success: boolean
|
||||||
|
steps: { name: string; ok: boolean; message: string }[]
|
||||||
|
}
|
||||||
|
|
||||||
|
async function runSetup(): Promise<SetupResult> {
|
||||||
|
const steps: SetupResult['steps'] = []
|
||||||
|
|
||||||
|
// Step 1: Check python3
|
||||||
|
const pythonCheck = await runCommand('python3', ['--version'])
|
||||||
|
if (!pythonCheck.ok) {
|
||||||
|
steps.push({
|
||||||
|
name: 'python_check',
|
||||||
|
ok: false,
|
||||||
|
message: 'Python 3 未安装,请先安装 Python 3',
|
||||||
|
})
|
||||||
|
return { success: false, steps }
|
||||||
|
}
|
||||||
|
steps.push({
|
||||||
|
name: 'python_check',
|
||||||
|
ok: true,
|
||||||
|
message: `Python ${pythonCheck.stdout.replace('Python ', '')}`,
|
||||||
|
})
|
||||||
|
|
||||||
|
// Step 2: Extract runtime files to ~/.claude/.runtime/
|
||||||
|
try {
|
||||||
|
await ensureRuntimeFiles()
|
||||||
|
steps.push({ name: 'runtime_files', ok: true, message: '运行时文件已就绪' })
|
||||||
|
} catch (err) {
|
||||||
|
steps.push({
|
||||||
|
name: 'runtime_files',
|
||||||
|
ok: false,
|
||||||
|
message: `提取运行时文件失败: ${err}`,
|
||||||
|
})
|
||||||
|
return { success: false, steps }
|
||||||
|
}
|
||||||
|
|
||||||
|
// Step 3: Create venv
|
||||||
|
const venvExists = await pathExists(join(venvRoot, 'bin', 'python3'))
|
||||||
|
if (!venvExists) {
|
||||||
|
const venvResult = await runCommand('python3', ['-m', 'venv', venvRoot])
|
||||||
|
if (!venvResult.ok) {
|
||||||
|
steps.push({
|
||||||
|
name: 'venv',
|
||||||
|
ok: false,
|
||||||
|
message: `创建虚拟环境失败: ${venvResult.stderr}`,
|
||||||
|
})
|
||||||
|
return { success: false, steps }
|
||||||
|
}
|
||||||
|
steps.push({ name: 'venv', ok: true, message: '虚拟环境已创建' })
|
||||||
|
} else {
|
||||||
|
steps.push({ name: 'venv', ok: true, message: '虚拟环境已存在' })
|
||||||
|
}
|
||||||
|
|
||||||
|
// Step 4: Ensure pip
|
||||||
|
const pipPath = join(venvRoot, 'bin', 'pip')
|
||||||
|
if (!(await pathExists(pipPath))) {
|
||||||
|
const pythonBin = join(venvRoot, 'bin', 'python3')
|
||||||
|
const pipResult = await runCommand(pythonBin, [
|
||||||
|
'-m',
|
||||||
|
'ensurepip',
|
||||||
|
'--upgrade',
|
||||||
|
])
|
||||||
|
if (!pipResult.ok) {
|
||||||
|
steps.push({
|
||||||
|
name: 'pip',
|
||||||
|
ok: false,
|
||||||
|
message: `安装 pip 失败: ${pipResult.stderr}`,
|
||||||
|
})
|
||||||
|
return { success: false, steps }
|
||||||
|
}
|
||||||
|
}
|
||||||
|
steps.push({ name: 'pip', ok: true, message: 'pip 已就绪' })
|
||||||
|
|
||||||
|
// Step 5: Install requirements
|
||||||
|
const reqPath = getRequirementsPath()
|
||||||
|
const requirements = await readFile(reqPath, 'utf8')
|
||||||
|
const digest = createHash('sha256').update(requirements).digest('hex')
|
||||||
|
|
||||||
|
let installedDigest = ''
|
||||||
|
try {
|
||||||
|
installedDigest = (await readFile(installStampPath, 'utf8')).trim()
|
||||||
|
} catch {}
|
||||||
|
|
||||||
|
if (installedDigest !== digest) {
|
||||||
|
const pythonBin = join(venvRoot, 'bin', 'python3')
|
||||||
|
|
||||||
|
// Upgrade pip first (using China mirror)
|
||||||
|
await runCommand(pythonBin, [
|
||||||
|
'-m', 'pip', 'install', '--upgrade', 'pip',
|
||||||
|
'-i', PIP_INDEX_URL, '--trusted-host', PIP_TRUSTED_HOST,
|
||||||
|
])
|
||||||
|
|
||||||
|
// Install deps (using China mirror)
|
||||||
|
const installResult = await runCommand(pythonBin, [
|
||||||
|
'-m', 'pip', 'install',
|
||||||
|
'-r', reqPath,
|
||||||
|
'-i', PIP_INDEX_URL, '--trusted-host', PIP_TRUSTED_HOST,
|
||||||
|
])
|
||||||
|
if (!installResult.ok) {
|
||||||
|
steps.push({
|
||||||
|
name: 'deps',
|
||||||
|
ok: false,
|
||||||
|
message: `安装依赖失败: ${installResult.stderr.slice(0, 500)}`,
|
||||||
|
})
|
||||||
|
return { success: false, steps }
|
||||||
|
}
|
||||||
|
await writeFile(installStampPath, `${digest}\n`, 'utf8')
|
||||||
|
steps.push({ name: 'deps', ok: true, message: '依赖已安装' })
|
||||||
|
} else {
|
||||||
|
steps.push({ name: 'deps', ok: true, message: '依赖已是最新' })
|
||||||
|
}
|
||||||
|
|
||||||
|
return { success: true, steps }
|
||||||
|
}
|
||||||
|
|
||||||
|
// ============================================================================
|
||||||
|
// Authorized Apps configuration — stored in ~/.claude/cc-haha/computer-use-config.json
|
||||||
|
// ============================================================================
|
||||||
|
|
||||||
|
const configPath = join(claudeHome, 'cc-haha', 'computer-use-config.json')
|
||||||
|
|
||||||
|
type AuthorizedApp = {
|
||||||
|
bundleId: string
|
||||||
|
displayName: string
|
||||||
|
authorizedAt: string
|
||||||
|
}
|
||||||
|
|
||||||
|
type ComputerUseConfig = {
|
||||||
|
authorizedApps: AuthorizedApp[]
|
||||||
|
grantFlags: {
|
||||||
|
clipboardRead: boolean
|
||||||
|
clipboardWrite: boolean
|
||||||
|
systemKeyCombos: boolean
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
const DEFAULT_CONFIG: ComputerUseConfig = {
|
||||||
|
authorizedApps: [],
|
||||||
|
grantFlags: { clipboardRead: true, clipboardWrite: true, systemKeyCombos: true },
|
||||||
|
}
|
||||||
|
|
||||||
|
async function loadConfig(): Promise<ComputerUseConfig> {
|
||||||
|
try {
|
||||||
|
const raw = await readFile(configPath, 'utf8')
|
||||||
|
return { ...DEFAULT_CONFIG, ...JSON.parse(raw) }
|
||||||
|
} catch {
|
||||||
|
return { ...DEFAULT_CONFIG }
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
async function saveConfig(config: ComputerUseConfig): Promise<void> {
|
||||||
|
await writeFile(configPath, JSON.stringify(config, null, 2), 'utf8')
|
||||||
|
}
|
||||||
|
|
||||||
|
async function listInstalledApps(): Promise<{ bundleId: string; displayName: string; path: string }[]> {
|
||||||
|
const helperPath = getHelperPath()
|
||||||
|
const pythonBin = join(venvRoot, 'bin', 'python3')
|
||||||
|
|
||||||
|
if (!(await pathExists(pythonBin)) || !(await pathExists(helperPath))) {
|
||||||
|
return []
|
||||||
|
}
|
||||||
|
|
||||||
|
const result = await runCommand(pythonBin, [helperPath, 'list_installed_apps'])
|
||||||
|
if (!result.ok) return []
|
||||||
|
|
||||||
|
try {
|
||||||
|
const parsed = JSON.parse(result.stdout)
|
||||||
|
return parsed.ok ? parsed.result : []
|
||||||
|
} catch {
|
||||||
|
return []
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
// ============================================================================
|
||||||
|
// Route handler
|
||||||
|
// ============================================================================
|
||||||
|
|
||||||
|
export async function handleComputerUseApi(
|
||||||
|
req: Request,
|
||||||
|
_url: URL,
|
||||||
|
segments: string[],
|
||||||
|
): Promise<Response> {
|
||||||
|
const action = segments[2]
|
||||||
|
|
||||||
|
if (action === 'status' && req.method === 'GET') {
|
||||||
|
const status = await checkStatus()
|
||||||
|
return Response.json(status)
|
||||||
|
}
|
||||||
|
|
||||||
|
if (action === 'setup' && req.method === 'POST') {
|
||||||
|
const result = await runSetup()
|
||||||
|
return Response.json(result)
|
||||||
|
}
|
||||||
|
|
||||||
|
// GET /api/computer-use/apps — list installed macOS apps
|
||||||
|
if (action === 'apps' && req.method === 'GET') {
|
||||||
|
const apps = await listInstalledApps()
|
||||||
|
return Response.json({ apps })
|
||||||
|
}
|
||||||
|
|
||||||
|
// GET /api/computer-use/authorized-apps — current authorized app config
|
||||||
|
if (action === 'authorized-apps' && req.method === 'GET') {
|
||||||
|
const config = await loadConfig()
|
||||||
|
return Response.json(config)
|
||||||
|
}
|
||||||
|
|
||||||
|
// PUT /api/computer-use/authorized-apps — update authorized apps
|
||||||
|
if (action === 'authorized-apps' && req.method === 'PUT') {
|
||||||
|
try {
|
||||||
|
const body = (await req.json()) as Partial<ComputerUseConfig>
|
||||||
|
const config = await loadConfig()
|
||||||
|
if (body.authorizedApps) config.authorizedApps = body.authorizedApps
|
||||||
|
if (body.grantFlags) config.grantFlags = { ...config.grantFlags, ...body.grantFlags }
|
||||||
|
await saveConfig(config)
|
||||||
|
return Response.json({ ok: true })
|
||||||
|
} catch {
|
||||||
|
return Response.json({ error: 'Invalid JSON' }, { status: 400 })
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
// POST /api/computer-use/open-settings — open macOS System Settings pane
|
||||||
|
if (action === 'open-settings' && req.method === 'POST') {
|
||||||
|
if (process.platform !== 'darwin') {
|
||||||
|
return Response.json({ error: 'macOS only' }, { status: 400 })
|
||||||
|
}
|
||||||
|
const body = (await req.json().catch(() => ({}))) as { pane?: string }
|
||||||
|
const pane = body.pane ?? 'Privacy_ScreenCapture'
|
||||||
|
const allowed = ['Privacy_ScreenCapture', 'Privacy_Accessibility']
|
||||||
|
if (!allowed.includes(pane)) {
|
||||||
|
return Response.json({ error: 'Invalid pane' }, { status: 400 })
|
||||||
|
}
|
||||||
|
const url = `x-apple.systempreferences:com.apple.preference.security?${pane}`
|
||||||
|
await runCommand('open', [url])
|
||||||
|
return Response.json({ ok: true })
|
||||||
|
}
|
||||||
|
|
||||||
|
return Response.json(
|
||||||
|
{ error: 'NOT_FOUND', message: `Unknown computer-use action: ${action}` },
|
||||||
|
{ status: 404 },
|
||||||
|
)
|
||||||
|
}
|
||||||
@ -7,6 +7,7 @@ import { logForDebugging } from '../debug.js'
|
|||||||
import { COMPUTER_USE_MCP_SERVER_NAME } from './common.js'
|
import { COMPUTER_USE_MCP_SERVER_NAME } from './common.js'
|
||||||
import { createCliExecutor } from './executor.js'
|
import { createCliExecutor } from './executor.js'
|
||||||
import { getChicagoEnabled, getChicagoSubGates } from './gates.js'
|
import { getChicagoEnabled, getChicagoSubGates } from './gates.js'
|
||||||
|
import { normalizeOsPermissions } from './permissions.js'
|
||||||
import { callPythonHelper } from './pythonBridge.js'
|
import { callPythonHelper } from './pythonBridge.js'
|
||||||
|
|
||||||
class DebugLogger implements Logger {
|
class DebugLogger implements Logger {
|
||||||
@ -39,8 +40,9 @@ export function getComputerUseHostAdapter(): ComputerUseHostAdapter {
|
|||||||
getHideBeforeActionEnabled: () => getChicagoSubGates().hideBeforeAction,
|
getHideBeforeActionEnabled: () => getChicagoSubGates().hideBeforeAction,
|
||||||
}),
|
}),
|
||||||
ensureOsPermissions: async () => {
|
ensureOsPermissions: async () => {
|
||||||
const perms = await callPythonHelper<{ accessibility: boolean; screenRecording: boolean }>('check_permissions', {})
|
const rawPerms = await callPythonHelper<{ accessibility: boolean; screenRecording: boolean | null }>('check_permissions', {})
|
||||||
return perms.accessibility && perms.screenRecording
|
const perms = normalizeOsPermissions(rawPerms)
|
||||||
|
return perms.granted
|
||||||
? { granted: true as const }
|
? { granted: true as const }
|
||||||
: { granted: false as const, accessibility: perms.accessibility, screenRecording: perms.screenRecording }
|
: { granted: false as const, accessibility: perms.accessibility, screenRecording: perms.screenRecording }
|
||||||
},
|
},
|
||||||
|
|||||||
44
src/utils/computerUse/permissions.test.ts
Normal file
44
src/utils/computerUse/permissions.test.ts
Normal file
@ -0,0 +1,44 @@
|
|||||||
|
import { describe, expect, it } from 'bun:test'
|
||||||
|
import { normalizeOsPermissions } from './permissions.js'
|
||||||
|
|
||||||
|
describe('normalizeOsPermissions', () => {
|
||||||
|
it('treats explicit grants as granted', () => {
|
||||||
|
expect(
|
||||||
|
normalizeOsPermissions({ accessibility: true, screenRecording: true }),
|
||||||
|
).toEqual({
|
||||||
|
granted: true,
|
||||||
|
accessibility: true,
|
||||||
|
screenRecording: true,
|
||||||
|
})
|
||||||
|
})
|
||||||
|
|
||||||
|
it('treats screen recording unknown as non-blocking when accessibility is granted', () => {
|
||||||
|
expect(
|
||||||
|
normalizeOsPermissions({ accessibility: true, screenRecording: null }),
|
||||||
|
).toEqual({
|
||||||
|
granted: true,
|
||||||
|
accessibility: true,
|
||||||
|
screenRecording: true,
|
||||||
|
})
|
||||||
|
})
|
||||||
|
|
||||||
|
it('still blocks when accessibility is missing', () => {
|
||||||
|
expect(
|
||||||
|
normalizeOsPermissions({ accessibility: false, screenRecording: null }),
|
||||||
|
).toEqual({
|
||||||
|
granted: false,
|
||||||
|
accessibility: false,
|
||||||
|
screenRecording: true,
|
||||||
|
})
|
||||||
|
})
|
||||||
|
|
||||||
|
it('blocks when screen recording is explicitly denied', () => {
|
||||||
|
expect(
|
||||||
|
normalizeOsPermissions({ accessibility: true, screenRecording: false }),
|
||||||
|
).toEqual({
|
||||||
|
granted: false,
|
||||||
|
accessibility: true,
|
||||||
|
screenRecording: false,
|
||||||
|
})
|
||||||
|
})
|
||||||
|
})
|
||||||
25
src/utils/computerUse/permissions.ts
Normal file
25
src/utils/computerUse/permissions.ts
Normal file
@ -0,0 +1,25 @@
|
|||||||
|
export type RawOsPermissions = {
|
||||||
|
accessibility: boolean
|
||||||
|
screenRecording: boolean | null
|
||||||
|
}
|
||||||
|
|
||||||
|
export type NormalizedOsPermissions = {
|
||||||
|
granted: boolean
|
||||||
|
accessibility: boolean
|
||||||
|
screenRecording: boolean
|
||||||
|
}
|
||||||
|
|
||||||
|
/**
|
||||||
|
* macOS Screen Recording passive probes can come back "unknown" for helper
|
||||||
|
* child processes even when the app bundle is already authorized. Treat that
|
||||||
|
* state as non-blocking and let the actual capture path remain the final
|
||||||
|
* source of truth.
|
||||||
|
*/
|
||||||
|
export function normalizeOsPermissions(perms: RawOsPermissions): NormalizedOsPermissions {
|
||||||
|
const screenRecording = perms.screenRecording !== false
|
||||||
|
return {
|
||||||
|
granted: perms.accessibility && screenRecording,
|
||||||
|
accessibility: perms.accessibility,
|
||||||
|
screenRecording,
|
||||||
|
}
|
||||||
|
}
|
||||||
@ -4,16 +4,25 @@ import path from 'node:path'
|
|||||||
import { fileURLToPath } from 'node:url'
|
import { fileURLToPath } from 'node:url'
|
||||||
import { execFileNoThrow } from '../execFileNoThrow.js'
|
import { execFileNoThrow } from '../execFileNoThrow.js'
|
||||||
import { logForDebugging } from '../debug.js'
|
import { logForDebugging } from '../debug.js'
|
||||||
|
import { getClaudeConfigHomeDir } from '../envUtils.js'
|
||||||
|
|
||||||
const __dirname = path.dirname(fileURLToPath(import.meta.url))
|
const __dirname = path.dirname(fileURLToPath(import.meta.url))
|
||||||
const projectRoot = path.resolve(__dirname, '../../..')
|
const projectRoot = path.resolve(__dirname, '../../..')
|
||||||
const runtimeRoot = path.join(projectRoot, 'runtime')
|
|
||||||
const runtimeStateRoot = path.join(projectRoot, '.runtime')
|
// All runtime state lives in ~/.claude/.runtime — writable in both dev and
|
||||||
|
// bundled (Tauri app) modes. The setup API (or ensureRuntimeFiles below)
|
||||||
|
// populates requirements.txt and mac_helper.py here.
|
||||||
|
const runtimeStateRoot = path.join(getClaudeConfigHomeDir(), '.runtime')
|
||||||
const venvRoot = path.join(runtimeStateRoot, 'venv')
|
const venvRoot = path.join(runtimeStateRoot, 'venv')
|
||||||
const requirementsPath = path.join(runtimeRoot, 'requirements.txt')
|
|
||||||
const helperPath = path.join(runtimeRoot, 'mac_helper.py')
|
|
||||||
const installStampPath = path.join(runtimeStateRoot, 'requirements.sha256')
|
const installStampPath = path.join(runtimeStateRoot, 'requirements.sha256')
|
||||||
|
|
||||||
|
const PIP_INDEX_URL = 'https://pypi.tuna.tsinghua.edu.cn/simple/'
|
||||||
|
const PIP_TRUSTED_HOST = 'pypi.tuna.tsinghua.edu.cn'
|
||||||
|
|
||||||
|
// Always read from ~/.claude/.runtime/ — works in both dev and bundled mode.
|
||||||
|
const requirementsPath = path.join(runtimeStateRoot, 'requirements.txt')
|
||||||
|
const helperPath = path.join(runtimeStateRoot, 'mac_helper.py')
|
||||||
|
|
||||||
let bootstrapPromise: Promise<void> | undefined
|
let bootstrapPromise: Promise<void> | undefined
|
||||||
|
|
||||||
function pythonBinPath(): string {
|
function pythonBinPath(): string {
|
||||||
@ -37,10 +46,34 @@ async function runOrThrow(file: string, args: string[], label: string): Promise<
|
|||||||
return stdout
|
return stdout
|
||||||
}
|
}
|
||||||
|
|
||||||
|
/**
|
||||||
|
* Ensure runtime source files exist in ~/.claude/.runtime/.
|
||||||
|
* In dev mode, copies from the project's runtime/ directory on first run.
|
||||||
|
* In bundled mode, these must have been placed there by the settings setup API.
|
||||||
|
*/
|
||||||
|
async function ensureRuntimeFiles(): Promise<void> {
|
||||||
|
await mkdir(runtimeStateRoot, { recursive: true })
|
||||||
|
|
||||||
|
const devRequirements = path.join(projectRoot, 'runtime', 'requirements.txt')
|
||||||
|
const devHelper = path.join(projectRoot, 'runtime', 'mac_helper.py')
|
||||||
|
|
||||||
|
// Always sync from dev runtime/ so source changes are reflected immediately.
|
||||||
|
// Previously this only copied when the dest was missing, causing stale files
|
||||||
|
// to persist after source updates — breaking mouse/keyboard actions if the
|
||||||
|
// cached copy was from an older version.
|
||||||
|
if (await pathExists(devRequirements)) {
|
||||||
|
await writeFile(requirementsPath, await readFile(devRequirements, 'utf8'), 'utf8')
|
||||||
|
}
|
||||||
|
if (await pathExists(devHelper)) {
|
||||||
|
await writeFile(helperPath, await readFile(devHelper, 'utf8'), 'utf8')
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
export async function ensureBootstrapped(): Promise<void> {
|
export async function ensureBootstrapped(): Promise<void> {
|
||||||
if (bootstrapPromise) return bootstrapPromise
|
if (bootstrapPromise) return bootstrapPromise
|
||||||
bootstrapPromise = (async () => {
|
bootstrapPromise = (async () => {
|
||||||
await mkdir(runtimeStateRoot, { recursive: true })
|
// Extract runtime files (requirements.txt, mac_helper.py) to state dir
|
||||||
|
await ensureRuntimeFiles()
|
||||||
|
|
||||||
if (!(await pathExists(pythonBinPath()))) {
|
if (!(await pathExists(pythonBinPath()))) {
|
||||||
logForDebugging('creating runtime venv at %s', { level: 'debug' })
|
logForDebugging('creating runtime venv at %s', { level: 'debug' })
|
||||||
@ -61,10 +94,14 @@ export async function ensureBootstrapped(): Promise<void> {
|
|||||||
|
|
||||||
if (installedDigest !== digest) {
|
if (installedDigest !== digest) {
|
||||||
logForDebugging('installing python runtime dependencies', { level: 'debug' })
|
logForDebugging('installing python runtime dependencies', { level: 'debug' })
|
||||||
await runOrThrow(pythonBinPath(), ['-m', 'pip', 'install', '--upgrade', 'pip'], 'pip upgrade')
|
await runOrThrow(pythonBinPath(), [
|
||||||
|
'-m', 'pip', 'install', '--upgrade', 'pip',
|
||||||
|
'-i', PIP_INDEX_URL, '--trusted-host', PIP_TRUSTED_HOST,
|
||||||
|
], 'pip upgrade')
|
||||||
await runOrThrow(
|
await runOrThrow(
|
||||||
pythonBinPath(),
|
pythonBinPath(),
|
||||||
['-m', 'pip', 'install', '-r', requirementsPath],
|
['-m', 'pip', 'install', '-r', requirementsPath,
|
||||||
|
'-i', PIP_INDEX_URL, '--trusted-host', PIP_TRUSTED_HOST],
|
||||||
'python dependency install',
|
'python dependency install',
|
||||||
)
|
)
|
||||||
await writeFile(installStampPath, `${digest}\n`, 'utf8')
|
await writeFile(installStampPath, `${digest}\n`, 'utf8')
|
||||||
@ -105,6 +142,6 @@ export async function callPythonHelper<T>(command: string, payload: Record<strin
|
|||||||
return parsed.result as T
|
return parsed.result as T
|
||||||
}
|
}
|
||||||
|
|
||||||
export function getRuntimePaths(): { projectRoot: string; runtimeRoot: string; venvRoot: string } {
|
export function getRuntimePaths(): { projectRoot: string; runtimeStateRoot: string; venvRoot: string } {
|
||||||
return { projectRoot, runtimeRoot, venvRoot }
|
return { projectRoot, runtimeStateRoot, venvRoot }
|
||||||
}
|
}
|
||||||
|
|||||||
@ -27,6 +27,9 @@ import { registerEscHotkey } from './escHotkey.js';
|
|||||||
import { getChicagoCoordinateMode } from './gates.js';
|
import { getChicagoCoordinateMode } from './gates.js';
|
||||||
import { getComputerUseHostAdapter } from './hostAdapter.js';
|
import { getComputerUseHostAdapter } from './hostAdapter.js';
|
||||||
import { getComputerUseMCPRenderingOverrides } from './toolRendering.js';
|
import { getComputerUseMCPRenderingOverrides } from './toolRendering.js';
|
||||||
|
import { readFile } from 'node:fs/promises';
|
||||||
|
import { join } from 'node:path';
|
||||||
|
import { homedir } from 'node:os';
|
||||||
type CallOverride = Pick<Tool, 'call'>['call'];
|
type CallOverride = Pick<Tool, 'call'>['call'];
|
||||||
type Binding = {
|
type Binding = {
|
||||||
ctx: ComputerUseSessionContext;
|
ctx: ComputerUseSessionContext;
|
||||||
@ -227,6 +230,64 @@ export function buildSessionContext(): ComputerUseSessionContext {
|
|||||||
formatLockHeldMessage: formatLockHeld
|
formatLockHeldMessage: formatLockHeld
|
||||||
};
|
};
|
||||||
}
|
}
|
||||||
|
/**
|
||||||
|
* Load pre-authorized apps from ~/.claude/cc-haha/computer-use-config.json.
|
||||||
|
* Called once when the binding is first created. Pre-authorized apps
|
||||||
|
* are injected into appState so `getAllowedApps()` returns them
|
||||||
|
* immediately — no runtime permission dialog needed.
|
||||||
|
*/
|
||||||
|
async function loadPreAuthorizedApps(): Promise<void> {
|
||||||
|
try {
|
||||||
|
const configPath = join(
|
||||||
|
process.env.CLAUDE_CONFIG_DIR ?? join(homedir(), '.claude'),
|
||||||
|
'cc-haha',
|
||||||
|
'computer-use-config.json',
|
||||||
|
)
|
||||||
|
const raw = await readFile(configPath, 'utf8')
|
||||||
|
const config = JSON.parse(raw) as {
|
||||||
|
authorizedApps?: { bundleId: string; displayName: string }[]
|
||||||
|
grantFlags?: { clipboardRead?: boolean; clipboardWrite?: boolean; systemKeyCombos?: boolean }
|
||||||
|
}
|
||||||
|
|
||||||
|
if (!config.authorizedApps?.length) return
|
||||||
|
|
||||||
|
const apps = config.authorizedApps.map(a => ({
|
||||||
|
bundleId: a.bundleId,
|
||||||
|
displayName: a.displayName,
|
||||||
|
grantedAt: Date.now(),
|
||||||
|
tier: 'full' as const,
|
||||||
|
}))
|
||||||
|
const flags = {
|
||||||
|
...DEFAULT_GRANT_FLAGS,
|
||||||
|
...(config.grantFlags ?? {}),
|
||||||
|
}
|
||||||
|
|
||||||
|
// Inject into appState so getAllowedApps() returns them.
|
||||||
|
// Merge with existing allowedApps (from permission dialog) instead of replacing.
|
||||||
|
if (currentToolUseContext) {
|
||||||
|
currentToolUseContext.setAppState(prev => {
|
||||||
|
const existing = prev.computerUseMcpState?.allowedApps ?? []
|
||||||
|
const existingIds = new Set(existing.map(a => a.bundleId))
|
||||||
|
const merged = [...existing, ...apps.filter(a => !existingIds.has(a.bundleId))]
|
||||||
|
return {
|
||||||
|
...prev,
|
||||||
|
computerUseMcpState: {
|
||||||
|
...prev.computerUseMcpState,
|
||||||
|
allowedApps: merged,
|
||||||
|
grantFlags: flags,
|
||||||
|
},
|
||||||
|
}
|
||||||
|
})
|
||||||
|
}
|
||||||
|
|
||||||
|
logForDebugging(`[Computer Use] Loaded ${apps.length} pre-authorized apps from config`)
|
||||||
|
} catch {
|
||||||
|
// Config doesn't exist or is invalid — no pre-authorized apps
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
let preAuthLoaded = false
|
||||||
|
|
||||||
function getOrBind(): Binding {
|
function getOrBind(): Binding {
|
||||||
if (binding) return binding;
|
if (binding) return binding;
|
||||||
const ctx = buildSessionContext();
|
const ctx = buildSessionContext();
|
||||||
@ -251,6 +312,12 @@ export function getComputerUseMCPToolOverrides(toolName: string): ComputerUseMCP
|
|||||||
const {
|
const {
|
||||||
dispatch
|
dispatch
|
||||||
} = getOrBind();
|
} = getOrBind();
|
||||||
|
|
||||||
|
// Load pre-authorized apps on first tool call
|
||||||
|
if (!preAuthLoaded) {
|
||||||
|
preAuthLoaded = true;
|
||||||
|
await loadPreAuthorizedApps();
|
||||||
|
}
|
||||||
const {
|
const {
|
||||||
telemetry,
|
telemetry,
|
||||||
...result
|
...result
|
||||||
|
|||||||
Loading…
x
Reference in New Issue
Block a user