Exercise desktop chat through a real browser baseline

The live baseline covered the server WebSocket path, but it still did not prove the desktop app can open a session, apply a selected provider/model, send a chat turn, and surface model/tool progress through the UI. This adds an agent-browser driven smoke lane that starts the local server and Vite desktop app, restores an isolated session tab with the requested runtime selection, submits a small coding task through the composer, and accepts the run only when the fixture diff and tests pass.

Constraint: Desktop smoke must stay behind --allow-live because it launches browsers and real models.

Constraint: The smoke temporarily enables bypassPermissions for the isolated run and restores the previous mode afterward.

Rejected: Wait for a final marker phrase | model reasoning and echoed prompts can contain the same marker before the work is actually done.

Rejected: Use only DOM text as success proof | the browser can show progress while the project files are still unchanged.

Confidence: high

Scope-risk: moderate

Directive: Future desktop smoke cases should verify project state and artifacts, not only UI copy.

Tested: bun test scripts/quality-gate/*.test.ts scripts/quality-gate/baseline/*.test.ts

Tested: bun run quality:gate --mode baseline --dry-run --provider-model minimax-m2.7

Tested: bun run quality:gate --mode baseline --allow-live --provider-model minimax-m2.7 (9 passed, 0 failed)

Tested: bun run check:server

Not-tested: Kimi desktop smoke completion because the provider returned AccountQuotaExceeded / API Error 429 until its reset window.

scripts/quality-gate/baseline/execute.ts

@@ -71,7 +71,7 @@ function listFiles(root: string, current = root): string[] {
   return files.sort()
 }
 
-function changedFiles(beforeDir: string, afterDir: string) {
+export function changedFiles(beforeDir: string, afterDir: string) {
   const before = new Set(listFiles(beforeDir))
   const after = new Set(listFiles(afterDir))
   const changed = new Set<string>()
@@ -95,7 +95,7 @@ function changedFiles(beforeDir: string, afterDir: string) {
   return [...changed].sort()
 }
 
-async function writeDiffPatch(beforeDir: string, afterDir: string, patchPath: string) {
+export async function writeDiffPatch(beforeDir: string, afterDir: string, patchPath: string) {
   const result = await runCommand(['git', 'diff', '--no-index', '--', beforeDir, afterDir], process.cwd())
   writeFileSync(patchPath, `${result.stdout}${result.stderr}`)
 }

scripts/quality-gate/desktop-smoke/execute.ts

@@ -0,0 +1,376 @@
+import { appendFileSync, cpSync, mkdirSync, readFileSync, rmSync, writeFileSync } from 'node:fs'
+import { mkdtemp } from 'node:fs/promises'
+import { createServer } from 'node:net'
+import { tmpdir } from 'node:os'
+import { join } from 'node:path'
+import { changedFiles, writeDiffPatch } from '../baseline/execute'
+import type { BaselineTarget, LaneResult } from '../types'
+
+const FIXTURE = 'scripts/quality-gate/desktop-smoke/fixtures/chat-edit'
+const PROMPT = [
+  'Run the tests in this project, fix the failing greeting implementation, and rerun the tests.',
+  'Only edit src/greeting.ts. Do not edit package.json or tests.',
+  'When the tests pass, briefly say done.',
+].join(' ')
+
+async function getPort(): Promise<number> {
+  return await new Promise((resolve, reject) => {
+    const server = createServer()
+    server.on('error', reject)
+    server.listen(0, '127.0.0.1', () => {
+      const address = server.address()
+      if (!address || typeof address === 'string') {
+        server.close(() => reject(new Error('Failed to allocate a local port')))
+        return
+      }
+      const port = address.port
+      server.close(() => resolve(port))
+    })
+  })
+}
+
+async function pipeToFile(stream: ReadableStream<Uint8Array> | null, path: string) {
+  if (!stream) return
+  const reader = stream.getReader()
+  while (true) {
+    const { done, value } = await reader.read()
+    if (done) break
+    appendFileSync(path, Buffer.from(value))
+  }
+}
+
+async function waitForHttp(url: string, timeoutMs: number) {
+  const deadline = Date.now() + timeoutMs
+  let lastError = ''
+  while (Date.now() < deadline) {
+    try {
+      const response = await fetch(url)
+      if (response.ok) return
+      lastError = `HTTP ${response.status}`
+    } catch (error) {
+      lastError = error instanceof Error ? error.message : String(error)
+    }
+    await Bun.sleep(500)
+  }
+  throw new Error(`Timed out waiting for ${url}${lastError ? ` (${lastError})` : ''}`)
+}
+
+async function runLoggedCommand(
+  command: string[],
+  options: {
+    cwd: string
+    logPath: string
+    env?: Record<string, string>
+    timeoutMs?: number
+    allowFailure?: boolean
+    maxLogChars?: number
+  },
+) {
+  appendFileSync(options.logPath, `\n$ ${command.join(' ')}\n`)
+  const proc = Bun.spawn(command, {
+    cwd: options.cwd,
+    env: options.env ? { ...process.env, ...options.env } : process.env,
+    stdout: 'pipe',
+    stderr: 'pipe',
+  })
+
+  const outputPromise = Promise.all([
+    new Response(proc.stdout).text(),
+    new Response(proc.stderr).text(),
+    proc.exited,
+  ])
+  const timeout = options.timeoutMs
+    ? Bun.sleep(options.timeoutMs).then(() => {
+      proc.kill()
+      throw new Error(`Command timed out after ${options.timeoutMs}ms: ${command.join(' ')}`)
+    })
+    : null
+
+  const [stdout, stderr, exitCode] = timeout
+    ? await Promise.race([outputPromise, timeout])
+    : await outputPromise
+  const output = `${stdout}${stderr}`
+  appendFileSync(
+    options.logPath,
+    options.maxLogChars && output.length > options.maxLogChars
+      ? `${output.slice(0, options.maxLogChars)}\n[quality-gate] output truncated at ${options.maxLogChars} chars\n`
+      : output,
+  )
+
+  if (exitCode !== 0 && !options.allowFailure) {
+    throw new Error(`Command failed (${exitCode}): ${command.join(' ')}`)
+  }
+
+  return { stdout, stderr, exitCode }
+}
+
+async function fetchJson<T>(url: string, init?: RequestInit): Promise<T> {
+  const response = await fetch(url, init)
+  if (!response.ok) {
+    throw new Error(`${init?.method ?? 'GET'} ${url} failed with HTTP ${response.status}: ${await response.text()}`)
+  }
+  return response.json() as Promise<T>
+}
+
+async function setPermissionMode(baseUrl: string, mode: string) {
+  await fetchJson<{ ok: true; mode: string }>(`${baseUrl}/api/permissions/mode`, {
+    method: 'PUT',
+    headers: { 'Content-Type': 'application/json' },
+    body: JSON.stringify({ mode }),
+  })
+}
+
+async function waitForVerifiedProject(
+  browserEnv: Record<string, string>,
+  browserLogPath: string,
+  rootDir: string,
+  originalDir: string,
+  projectDir: string,
+  artifactDir: string,
+  timeoutMs: number,
+) {
+  const deadline = Date.now() + timeoutMs
+  let lastVerificationError = 'project verification has not run yet'
+  while (Date.now() < deadline) {
+    const body = await runLoggedCommand(['agent-browser', 'get', 'text', '#content-area'], {
+      cwd: rootDir,
+      env: browserEnv,
+      logPath: browserLogPath,
+      timeoutMs: 15_000,
+      allowFailure: true,
+      maxLogChars: 4_000,
+    })
+    const browserText = `${body.stdout}\n${body.stderr}`
+
+    if (browserText.includes('CLI_START_FAILED') || browserText.includes('CLI_RESTART_FAILED')) {
+      throw new Error('Desktop session reported a CLI startup failure')
+    }
+    if (
+      browserText.includes('API Error: 429') ||
+      browserText.includes('AccountQuotaExceeded') ||
+      browserText.includes('TooManyRequests')
+    ) {
+      throw new Error('Desktop session reported provider quota/rate-limit failure')
+    }
+    if (browserText.includes('处理过程中发生错误') || browserText.includes('API Error:')) {
+      throw new Error('Desktop session reported an API error')
+    }
+
+    try {
+      await verifyProject(originalDir, projectDir, artifactDir)
+      return
+    } catch (error) {
+      lastVerificationError = error instanceof Error ? error.message : String(error)
+    }
+    await Bun.sleep(5_000)
+  }
+
+  throw new Error(`Timed out waiting for desktop project verification: ${lastVerificationError}`)
+}
+
+async function verifyProject(originalDir: string, projectDir: string, artifactDir: string) {
+  await writeDiffPatch(originalDir, projectDir, join(artifactDir, 'diff.patch'))
+  const changed = changedFiles(originalDir, projectDir)
+  const unexpected = changed.filter((file) => file !== 'src/greeting.ts')
+  if (unexpected.length > 0) {
+    throw new Error(`desktop smoke changed unexpected files: ${unexpected.join(', ')}`)
+  }
+  if (!changed.includes('src/greeting.ts')) {
+    throw new Error('desktop smoke did not change src/greeting.ts')
+  }
+
+  const implementation = readFileSync(join(projectDir, 'src/greeting.ts'), 'utf8')
+  if (!implementation.includes('from desktop smoke!')) {
+    throw new Error('desktop smoke implementation is missing the expected marker text')
+  }
+
+  const proc = Bun.spawn(['bun', 'test'], {
+    cwd: projectDir,
+    stdout: 'pipe',
+    stderr: 'pipe',
+  })
+  const stdout = await new Response(proc.stdout).text()
+  const stderr = await new Response(proc.stderr).text()
+  const exitCode = await proc.exited
+  writeFileSync(join(artifactDir, 'verification.log'), `${stdout}${stderr}`)
+  if (exitCode !== 0) {
+    throw new Error(`desktop smoke verification failed with exit code ${exitCode}`)
+  }
+}
+
+export async function executeDesktopSmoke(
+  rootDir: string,
+  artifactDir: string,
+  resultId: string,
+  resultTitle: string,
+  target: BaselineTarget | undefined,
+): Promise<LaneResult> {
+  const started = Date.now()
+  mkdirSync(artifactDir, { recursive: true })
+
+  const serverLogPath = join(artifactDir, 'server.log')
+  const viteLogPath = join(artifactDir, 'vite.log')
+  const browserLogPath = join(artifactDir, 'browser.log')
+  const workRoot = await mkdtemp(join(tmpdir(), 'quality-gate-desktop-smoke-'))
+  const originalDir = join(workRoot, 'original')
+  const projectDir = join(workRoot, 'project')
+  const browserProfileDir = join(workRoot, 'browser-profile')
+  cpSync(join(rootDir, FIXTURE), originalDir, { recursive: true })
+  cpSync(join(rootDir, FIXTURE), projectDir, { recursive: true })
+
+  const serverPort = await getPort()
+  const vitePort = await getPort()
+  const baseUrl = `http://127.0.0.1:${serverPort}`
+  const appUrl = `http://127.0.0.1:${vitePort}`
+  const sessionName = `quality-gate-${Date.now()}-${Math.random().toString(36).slice(2, 8)}`
+  const browserEnv = {
+    AGENT_BROWSER_SESSION: sessionName,
+    AGENT_BROWSER_PROFILE: browserProfileDir,
+  }
+
+  const server = Bun.spawn(['bun', 'run', 'src/server/index.ts', '--host', '127.0.0.1', '--port', String(serverPort)], {
+    cwd: rootDir,
+    stdout: 'pipe',
+    stderr: 'pipe',
+  })
+  void pipeToFile(server.stdout, serverLogPath)
+  void pipeToFile(server.stderr, serverLogPath)
+
+  const vite = Bun.spawn(['bun', 'run', 'dev', '--', '--host', '127.0.0.1', '--port', String(vitePort), '--strictPort'], {
+    cwd: join(rootDir, 'desktop'),
+    env: {
+      ...process.env,
+      VITE_DESKTOP_SERVER_URL: baseUrl,
+    },
+    stdout: 'pipe',
+    stderr: 'pipe',
+  })
+  void pipeToFile(vite.stdout, viteLogPath)
+  void pipeToFile(vite.stderr, viteLogPath)
+
+  let previousPermissionMode: string | null = null
+  try {
+    await waitForHttp(`${baseUrl}/health`, 20_000)
+    await waitForHttp(appUrl, 30_000)
+
+    const permission = await fetchJson<{ mode: string }>(`${baseUrl}/api/permissions/mode`)
+    previousPermissionMode = permission.mode
+    await setPermissionMode(baseUrl, 'bypassPermissions')
+
+    const session = await fetchJson<{ sessionId: string }>(`${baseUrl}/api/sessions`, {
+      method: 'POST',
+      headers: { 'Content-Type': 'application/json' },
+      body: JSON.stringify({ workDir: projectDir }),
+    })
+    const runtimeSelection = {
+      providerId: target?.providerId ?? null,
+      modelId: target?.modelId ?? 'current',
+    }
+
+    await runLoggedCommand(['agent-browser', 'open', appUrl], {
+      cwd: rootDir,
+      env: browserEnv,
+      logPath: browserLogPath,
+      timeoutMs: 30_000,
+    })
+    await runLoggedCommand([
+      'agent-browser',
+      'eval',
+      [
+        `localStorage.setItem('cc-haha-open-tabs', ${JSON.stringify(JSON.stringify({
+          openTabs: [{ sessionId: session.sessionId, title: 'Desktop Smoke', type: 'session' }],
+          activeTabId: session.sessionId,
+        }))})`,
+        `localStorage.setItem('cc-haha-session-runtime', ${JSON.stringify(JSON.stringify({
+          [session.sessionId]: runtimeSelection,
+        }))})`,
+      ].join(';'),
+    ], {
+      cwd: rootDir,
+      env: browserEnv,
+      logPath: browserLogPath,
+      timeoutMs: 15_000,
+    })
+    await runLoggedCommand(['agent-browser', 'reload'], {
+      cwd: rootDir,
+      env: browserEnv,
+      logPath: browserLogPath,
+      timeoutMs: 30_000,
+    })
+    await runLoggedCommand(['agent-browser', 'wait', 'textarea'], {
+      cwd: rootDir,
+      env: browserEnv,
+      logPath: browserLogPath,
+      timeoutMs: 30_000,
+    })
+    await runLoggedCommand(['agent-browser', 'screenshot', join(artifactDir, 'initial.png')], {
+      cwd: rootDir,
+      env: browserEnv,
+      logPath: browserLogPath,
+      timeoutMs: 20_000,
+      allowFailure: true,
+    })
+    await runLoggedCommand(['agent-browser', 'fill', 'textarea', PROMPT], {
+      cwd: rootDir,
+      env: browserEnv,
+      logPath: browserLogPath,
+      timeoutMs: 20_000,
+    })
+    await runLoggedCommand(['agent-browser', 'press', 'Enter'], {
+      cwd: rootDir,
+      env: browserEnv,
+      logPath: browserLogPath,
+      timeoutMs: 15_000,
+    })
+
+    await waitForVerifiedProject(
+      browserEnv,
+      browserLogPath,
+      rootDir,
+      originalDir,
+      projectDir,
+      artifactDir,
+      360_000,
+    )
+    await runLoggedCommand(['agent-browser', 'screenshot', join(artifactDir, 'final.png')], {
+      cwd: rootDir,
+      env: browserEnv,
+      logPath: browserLogPath,
+      timeoutMs: 20_000,
+      allowFailure: true,
+    })
+
+    return {
+      id: resultId,
+      title: resultTitle,
+      status: 'passed',
+      durationMs: Date.now() - started,
+      artifactDir,
+    }
+  } catch (error) {
+    return {
+      id: resultId,
+      title: resultTitle,
+      status: 'failed',
+      durationMs: Date.now() - started,
+      error: error instanceof Error ? error.message : String(error),
+      artifactDir,
+    }
+  } finally {
+    if (previousPermissionMode) {
+      await setPermissionMode(baseUrl, previousPermissionMode).catch((error) => {
+        appendFileSync(serverLogPath, `\n[quality-gate] Failed to restore permission mode: ${error instanceof Error ? error.message : String(error)}\n`)
+      })
+    }
+    await runLoggedCommand(['agent-browser', 'close'], {
+      cwd: rootDir,
+      env: browserEnv,
+      logPath: browserLogPath,
+      timeoutMs: 10_000,
+      allowFailure: true,
+    }).catch(() => {})
+    server.kill()
+    vite.kill()
+    rmSync(workRoot, { recursive: true, force: true })
+  }
+}

scripts/quality-gate/desktop-smoke/fixtures/chat-edit/package.json

@@ -0,0 +1,8 @@
+{
+  "name": "desktop-smoke-chat-edit",
+  "private": true,
+  "type": "module",
+  "scripts": {
+    "test": "bun test"
+  }
+}

scripts/quality-gate/desktop-smoke/fixtures/chat-edit/src/greeting.test.ts

@@ -0,0 +1,8 @@
+import { describe, expect, test } from 'bun:test'
+import { greetingFor } from './greeting'
+
+describe('greetingFor', () => {
+  test('includes the desktop smoke marker', () => {
+    expect(greetingFor('Ada Lovelace')).toBe('Hello, Ada Lovelace from desktop smoke!')
+  })
+})

scripts/quality-gate/desktop-smoke/fixtures/chat-edit/src/greeting.ts

@@ -0,0 +1,3 @@
+export function greetingFor(name: string) {
+  return `Hello, ${name}`
+}

scripts/quality-gate/modes.ts

@@ -57,5 +57,18 @@ export function lanesForMode(mode: QualityGateMode, baselineTargets: BaselineTar
     }
   }
 
+  for (const target of targets) {
+    const targetSlug = target.label.replace(/[^a-zA-Z0-9._-]+/g, '-')
+    lanes.push({
+      id: `desktop-smoke:agent-browser-chat:${targetSlug}`,
+      title: `Desktop agent-browser chat smoke (${target.label})`,
+      description: 'Open the desktop web app with agent-browser, send a real chat task, and verify the model edits a fixture project through the UI.',
+      kind: 'desktop-smoke',
+      baselineTarget: target,
+      requiredForModes: ['baseline', 'release'],
+      live: true,
+    })
+  }
+
   return lanes.filter((lane) => lane.requiredForModes.includes(mode))
 }

scripts/quality-gate/runner.test.ts

@@ -20,6 +20,7 @@ describe('quality gate modes', () => {
     expect(lanes).toContain('baseline-catalog')
     expect(lanes).toContain('baseline:failing-unit:current-runtime')
     expect(lanes).toContain('baseline:multi-file-api:current-runtime')
+    expect(lanes).toContain('desktop-smoke:agent-browser-chat:current-runtime')
     expect(lanes).not.toContain('native-checks')
   })
 
@@ -27,6 +28,7 @@ describe('quality gate modes', () => {
     const lanes = lanesForMode('release').map((lane) => lane.id)
     expect(lanes).toContain('pr-checks')
     expect(lanes).toContain('baseline:failing-unit:current-runtime')
+    expect(lanes).toContain('desktop-smoke:agent-browser-chat:current-runtime')
     expect(lanes).toContain('native-checks')
   })
 
@@ -40,6 +42,8 @@ describe('quality gate modes', () => {
     expect(lanes).toContain('baseline:failing-unit:provider-b-model-b')
     expect(lanes).toContain('baseline:multi-file-api:provider-a-model-a')
     expect(lanes).toContain('baseline:multi-file-api:provider-b-model-b')
+    expect(lanes).toContain('desktop-smoke:agent-browser-chat:provider-a-model-a')
+    expect(lanes).toContain('desktop-smoke:agent-browser-chat:provider-b-model-b')
   })
 })
 

scripts/quality-gate/runner.ts

@@ -2,6 +2,7 @@ import { mkdirSync } from 'node:fs'
 import { join } from 'node:path'
 import { baselineCases } from './baseline/cases'
 import { executeBaselineCase } from './baseline/execute'
+import { executeDesktopSmoke } from './desktop-smoke/execute'
 import { lanesForMode } from './modes'
 import { writeReport } from './reporter'
 import type { LaneDefinition, LaneResult, QualityGateOptions, QualityGateReport } from './types'
@@ -104,6 +105,28 @@ async function runLane(lane: LaneDefinition, options: QualityGateOptions): Promi
   if (lane.kind === 'baseline-case') {
     return runBaselineCaseLane(lane, options)
   }
+  if (lane.kind === 'desktop-smoke') {
+    const started = Date.now()
+
+    if (!options.allowLive) {
+      return {
+        id: lane.id,
+        title: lane.title,
+        status: 'skipped',
+        durationMs: Date.now() - started,
+        skipReason: 'desktop agent-browser smoke requires --allow-live',
+      }
+    }
+
+    const artifactRoot = options.runOutputDir ?? join(options.rootDir, 'artifacts', 'quality-runs', options.runId ?? 'current')
+    return executeDesktopSmoke(
+      options.rootDir,
+      join(artifactRoot, 'cases', lane.id.replace(/[^a-zA-Z0-9._-]+/g, '-')),
+      lane.id,
+      lane.title,
+      lane.baselineTarget,
+    )
+  }
 
   return runCommandLane(lane, options)
 }

scripts/quality-gate/types.ts

@@ -1,6 +1,6 @@
 export type QualityGateMode = 'pr' | 'baseline' | 'release'
 
-export type LaneKind = 'command' | 'baseline-case'
+export type LaneKind = 'command' | 'baseline-case' | 'desktop-smoke'
 
 export type LaneDefinition = {
   id: string