Protect release confidence with live agent baselines

The desktop product needs a repeatable local gate that can prove the core Coding Agent loop still works after changes, not only that unit tests pass. This adds a quality-gate runner with PR, baseline, and release modes, structured reports, explicit quarantine metadata, and fixture-based live baseline cases that can run across provider/model targets.

Constraint: Existing check:pr and CI policy behavior must remain usable while the stronger baseline grows around it
Constraint: Default PR gates must not require real model credentials or provider quota
Rejected: Build a standalone QA platform first | too heavy before the baseline task shape is proven
Rejected: Keep unstable server exclusions hardcoded in run-server-tests | hides quarantine policy from reports and future review
Confidence: medium
Scope-risk: moderate
Directive: Expand baseline cases by adding focused fixtures and verifiers; do not make normal PR checks depend on live providers
Tested: bun test scripts/quality-gate/*.test.ts scripts/quality-gate/baseline/*.test.ts
Tested: bun run check:server
Tested: bun run quality:gate --mode baseline --allow-live --provider-model 2944f963-ce75-45b7-bac1-6e4f57df0970:kimi-k2.6:volc-kimi-k2.6 --provider-model 9c78d3df-7fb5-44c7-8436-3a41c3a59231:MiniMax-M2.7-highspeed:minimax-m2.7
Not-tested: desktop UI browser smoke and native release mode in this commit

.gitignore

@@ -22,6 +22,9 @@ extracted-natives/
 # E2E screenshots
 e2e-*.png
 
+# Quality gate reports
+artifacts/quality-runs/
+
 # Desktop build output
 desktop/dist/
 desktop/build-artifacts/

package.json

@@ -17,6 +17,10 @@
     "check:adapters": "cd adapters && bun test",
     "check:native": "cd desktop && bun run build:sidecars && cd src-tauri && cargo check",
     "check:docs": "npm ci && npm run docs:build",
+    "quality:gate": "bun run scripts/quality-gate/index.ts",
+    "quality:pr": "bun run quality:gate --mode pr",
+    "quality:baseline": "bun run quality:gate --mode baseline",
+    "quality:release": "bun run quality:gate --mode release",
     "docs:dev": "vitepress dev docs",
     "docs:build": "vitepress build docs",
     "docs:preview": "vitepress preview docs"

scripts/pr/run-server-tests.ts

@@ -2,18 +2,11 @@
 
 import { readdirSync, statSync } from 'node:fs'
 import { join, relative, sep } from 'node:path'
+import { quarantinedPathSet } from '../quality-gate/quarantine'
 
 const root = process.cwd()
 const roots = ['src/server', 'src/tools', 'src/utils']
-const excludedFiles = new Set([
-  // These suites are not stable enough for the default PR gate yet. Keep them
-  // out of CI until they are fixed or moved to a maintainer-only workflow.
-  'src/server/__tests__/cron-scheduler.test.ts',
-  'src/server/__tests__/providers-real.test.ts',
-  'src/server/__tests__/tasks.test.ts',
-  'src/server/__tests__/e2e/business-flow.test.ts',
-  'src/server/__tests__/e2e/full-flow.test.ts',
-])
+const excludedFiles = quarantinedPathSet()
 
 function normalize(path: string) {
   return relative(root, path).split(sep).join('/')

scripts/quality-gate/baseline/cases.test.ts

@@ -0,0 +1,28 @@
+import { describe, expect, test } from 'bun:test'
+import { existsSync } from 'node:fs'
+import { join } from 'node:path'
+import { baselineCases, validateBaselineCases } from './cases'
+
+describe('baselineCases', () => {
+  test('have valid metadata', () => {
+    expect(() => validateBaselineCases()).not.toThrow()
+  })
+
+  test('use unique ids', () => {
+    const ids = baselineCases.map((testCase) => testCase.id)
+    expect(new Set(ids).size).toBe(ids.length)
+  })
+
+  test('point at existing fixtures with package manifests', () => {
+    for (const testCase of baselineCases) {
+      expect(existsSync(testCase.fixture)).toBe(true)
+      expect(existsSync(join(testCase.fixture, 'package.json'))).toBe(true)
+    }
+  })
+
+  test('require real model capability', () => {
+    for (const testCase of baselineCases) {
+      expect(testCase.requiredCapabilities).toContain('model')
+    }
+  })
+})

scripts/quality-gate/baseline/cases.ts

@@ -0,0 +1,57 @@
+import type { BaselineCase } from '../types'
+
+export const baselineCases: BaselineCase[] = [
+  {
+    id: 'failing-unit',
+    title: 'Fix a failing unit test',
+    description: 'A tiny TypeScript project has a broken arithmetic function. The Agent must inspect the failing test, patch the implementation, and rerun the test.',
+    fixture: 'scripts/quality-gate/baseline/fixtures/failing-unit',
+    prompt: 'Run the tests, inspect the failing assertion, fix the implementation bug, and rerun the tests until they pass. Only modify the fixture source files needed for the fix.',
+    mode: 'websocket',
+    requiredCapabilities: ['model', 'file-edit', 'shell'],
+    timeoutMs: 180_000,
+    verify: {
+      commands: [['bun', 'test']],
+      expectedFiles: ['src/math.ts'],
+      forbiddenFiles: ['package.json'],
+      transcriptAssertions: ['bun test', 'src/math.ts'],
+    },
+  },
+  {
+    id: 'multi-file-api',
+    title: 'Update a multi-file API contract',
+    description: 'A small app exposes a user display API. The Agent must change the contract and update callers plus tests coherently.',
+    fixture: 'scripts/quality-gate/baseline/fixtures/multi-file-api',
+    prompt: 'Change the user display contract so it returns "Ada Lovelace <ada@example.com>" instead of only the name. Update the implementation, caller, and tests, then run the tests.',
+    mode: 'websocket',
+    requiredCapabilities: ['model', 'file-edit', 'shell'],
+    timeoutMs: 240_000,
+    verify: {
+      commands: [['bun', 'test']],
+      expectedFiles: ['src/api.ts', 'src/app.ts', 'src/app.test.ts'],
+      forbiddenFiles: ['package.json'],
+      transcriptAssertions: ['bun test', 'src/api.ts', 'src/app.ts'],
+    },
+  },
+]
+
+export function validateBaselineCases(cases = baselineCases) {
+  const ids = new Set<string>()
+
+  for (const testCase of cases) {
+    if (ids.has(testCase.id)) {
+      throw new Error(`Duplicate baseline case id: ${testCase.id}`)
+    }
+    ids.add(testCase.id)
+
+    if (!testCase.fixture) {
+      throw new Error(`Baseline case ${testCase.id} is missing a fixture`)
+    }
+    if (testCase.verify.commands.length === 0) {
+      throw new Error(`Baseline case ${testCase.id} is missing verification commands`)
+    }
+    if (testCase.timeoutMs < 30_000) {
+      throw new Error(`Baseline case ${testCase.id} timeout is too low`)
+    }
+  }
+}

scripts/quality-gate/baseline/execute.ts

@@ -0,0 +1,235 @@
+import { appendFileSync, cpSync, mkdirSync, rmSync, writeFileSync } from 'node:fs'
+import { mkdtemp } from 'node:fs/promises'
+import { tmpdir } from 'node:os'
+import { join } from 'node:path'
+import { createServer } from 'node:net'
+import type { BaselineCase, BaselineTarget, LaneResult } from '../types'
+
+type ServerMessage = {
+  type: string
+  requestId?: string
+  message?: string
+  code?: string
+  [key: string]: unknown
+}
+
+function getPort() {
+  return new Promise<number>((resolve, reject) => {
+    const server = createServer()
+    server.once('error', reject)
+    server.listen(0, '127.0.0.1', () => {
+      const address = server.address()
+      const port = typeof address === 'object' && address ? address.port : 0
+      server.close(() => resolve(port))
+    })
+  })
+}
+
+async function waitForHttp(url: string, timeoutMs: number) {
+  const deadline = Date.now() + timeoutMs
+  while (Date.now() < deadline) {
+    try {
+      const res = await fetch(url)
+      if (res.ok) return
+    } catch {
+      // Keep polling until timeout.
+    }
+    await Bun.sleep(500)
+  }
+  throw new Error(`Timed out waiting for ${url}`)
+}
+
+async function runCommand(command: string[], cwd: string) {
+  const proc = Bun.spawn(command, {
+    cwd,
+    stdout: 'pipe',
+    stderr: 'pipe',
+  })
+  const stdout = await new Response(proc.stdout).text()
+  const stderr = await new Response(proc.stderr).text()
+  const exitCode = await proc.exited
+  return { exitCode, stdout, stderr }
+}
+
+async function pipeToFile(stream: ReadableStream<Uint8Array> | null, path: string) {
+  if (!stream) return
+  const reader = stream.getReader()
+  const decoder = new TextDecoder()
+  while (true) {
+    const { value, done } = await reader.read()
+    if (done) break
+    appendFileSync(path, decoder.decode(value, { stream: true }))
+  }
+}
+
+function waitForWebSocketOpen(ws: WebSocket) {
+  return new Promise<void>((resolve, reject) => {
+    ws.onopen = () => resolve()
+    ws.onerror = () => reject(new Error('WebSocket failed to open'))
+  })
+}
+
+async function runPromptOverWebSocket(
+  baseUrl: string,
+  sessionId: string,
+  prompt: string,
+  timeoutMs: number,
+  target?: BaselineTarget,
+) {
+  const wsUrl = baseUrl.replace(/^http/, 'ws')
+  const ws = new WebSocket(`${wsUrl}/ws/${sessionId}`)
+  const messages: ServerMessage[] = []
+
+  try {
+    await waitForWebSocketOpen(ws)
+
+    await new Promise<void>((resolve, reject) => {
+      const timer = setTimeout(() => {
+        reject(new Error(`Timed out waiting for baseline case completion after ${timeoutMs}ms`))
+      }, timeoutMs)
+
+      ws.onmessage = (event) => {
+        const message = JSON.parse(String(event.data)) as ServerMessage
+        messages.push(message)
+
+        if (message.type === 'connected') {
+          if (target && target.modelId !== 'current') {
+            ws.send(JSON.stringify({
+              type: 'set_runtime_config',
+              providerId: target.providerId,
+              modelId: target.modelId,
+            }))
+          }
+          ws.send(JSON.stringify({ type: 'user_message', content: prompt }))
+          return
+        }
+
+        if (message.type === 'permission_request' && typeof message.requestId === 'string') {
+          ws.send(JSON.stringify({
+            type: 'permission_response',
+            requestId: message.requestId,
+            allowed: true,
+            rule: 'baseline-run',
+          }))
+          return
+        }
+
+        if (message.type === 'message_complete') {
+          clearTimeout(timer)
+          resolve()
+          return
+        }
+
+        if (message.type === 'error') {
+          clearTimeout(timer)
+          reject(new Error(`${message.code ?? 'WS_ERROR'}: ${message.message ?? 'unknown error'}`))
+        }
+      }
+
+      ws.onerror = () => {
+        clearTimeout(timer)
+        reject(new Error('WebSocket error during baseline case'))
+      }
+    })
+  } finally {
+    ws.close()
+  }
+
+  return messages
+}
+
+export async function executeBaselineCase(
+  testCase: BaselineCase,
+  rootDir: string,
+  artifactDir: string,
+  target?: BaselineTarget,
+): Promise<LaneResult> {
+  const started = Date.now()
+  const resultId = target
+    ? `baseline:${testCase.id}:${target.label.replace(/[^a-zA-Z0-9._-]+/g, '-')}`
+    : `baseline:${testCase.id}`
+  const resultTitle = target ? `${testCase.title} (${target.label})` : testCase.title
+  mkdirSync(artifactDir, { recursive: true })
+
+  const port = await getPort()
+  const baseUrl = `http://127.0.0.1:${port}`
+  const workRoot = await mkdtemp(join(tmpdir(), `quality-gate-${testCase.id}-`))
+  const projectDir = join(workRoot, 'project')
+  cpSync(join(rootDir, testCase.fixture), projectDir, { recursive: true })
+
+  const serverLogPath = join(artifactDir, 'server.log')
+  const transcriptPath = join(artifactDir, 'transcript.jsonl')
+  const verificationPath = join(artifactDir, 'verification.log')
+  const server = Bun.spawn(['bun', 'run', 'src/server/index.ts', '--host', '127.0.0.1', '--port', String(port)], {
+    cwd: rootDir,
+    stdout: 'pipe',
+    stderr: 'pipe',
+    env: {
+      ...process.env,
+      SERVER_PORT: String(port),
+    },
+  })
+  const stdoutPump = pipeToFile(server.stdout, serverLogPath)
+  const stderrPump = pipeToFile(server.stderr, serverLogPath)
+
+  try {
+    await waitForHttp(`${baseUrl}/health`, 60_000)
+
+    const createResponse = await fetch(`${baseUrl}/api/sessions`, {
+      method: 'POST',
+      headers: { 'Content-Type': 'application/json' },
+      body: JSON.stringify({ workDir: projectDir }),
+    })
+    if (!createResponse.ok) {
+      throw new Error(`Failed to create session: ${createResponse.status}`)
+    }
+    const session = await createResponse.json() as { sessionId?: string }
+    if (!session.sessionId) {
+      throw new Error('Session response did not include sessionId')
+    }
+
+    const messages = await runPromptOverWebSocket(baseUrl, session.sessionId, testCase.prompt, testCase.timeoutMs, target)
+    writeFileSync(transcriptPath, messages.map((message) => JSON.stringify(message)).join('\n') + '\n')
+
+    let verificationLog = ''
+    for (const command of testCase.verify.commands) {
+      const result = await runCommand(command, projectDir)
+      verificationLog += `$ ${command.join(' ')}\n${result.stdout}${result.stderr}\n`
+      if (result.exitCode !== 0) {
+        writeFileSync(verificationPath, verificationLog)
+        return {
+          id: resultId,
+          title: resultTitle,
+          status: 'failed',
+          durationMs: Date.now() - started,
+          exitCode: result.exitCode,
+          error: `verification command failed: ${command.join(' ')}`,
+          artifactDir,
+        }
+      }
+    }
+    writeFileSync(verificationPath, verificationLog)
+
+    return {
+      id: resultId,
+      title: resultTitle,
+      status: 'passed',
+      durationMs: Date.now() - started,
+      artifactDir,
+    }
+  } catch (error) {
+    return {
+      id: resultId,
+      title: resultTitle,
+      status: 'failed',
+      durationMs: Date.now() - started,
+      error: error instanceof Error ? error.message : String(error),
+      artifactDir,
+    }
+  } finally {
+    server.kill()
+    await server.exited.catch(() => undefined)
+    await Promise.all([stdoutPump, stderrPump]).catch(() => undefined)
+    rmSync(workRoot, { recursive: true, force: true })
+  }
+}

scripts/quality-gate/baseline/fixtures/failing-unit/package.json

@@ -0,0 +1,8 @@
+{
+  "name": "quality-gate-failing-unit-fixture",
+  "private": true,
+  "type": "module",
+  "scripts": {
+    "test": "bun test"
+  }
+}

scripts/quality-gate/baseline/fixtures/failing-unit/src/math.test.ts

@@ -0,0 +1,8 @@
+import { describe, expect, test } from 'bun:test'
+import { add } from './math'
+
+describe('add', () => {
+  test('adds positive numbers', () => {
+    expect(add(2, 3)).toBe(5)
+  })
+})

scripts/quality-gate/baseline/fixtures/failing-unit/src/math.ts

@@ -0,0 +1,3 @@
+export function add(a: number, b: number) {
+  return a - b
+}

scripts/quality-gate/baseline/fixtures/multi-file-api/package.json

@@ -0,0 +1,8 @@
+{
+  "name": "quality-gate-multi-file-api-fixture",
+  "private": true,
+  "type": "module",
+  "scripts": {
+    "test": "bun test"
+  }
+}

scripts/quality-gate/baseline/fixtures/multi-file-api/src/api.ts

@@ -0,0 +1,8 @@
+export type User = {
+  name: string
+  email: string
+}
+
+export function formatUser(user: User) {
+  return user.name
+}

scripts/quality-gate/baseline/fixtures/multi-file-api/src/app.test.ts

@@ -0,0 +1,8 @@
+import { describe, expect, test } from 'bun:test'
+import { renderUser } from './app'
+
+describe('renderUser', () => {
+  test('renders name and email', () => {
+    expect(renderUser({ name: 'Ada Lovelace', email: 'ada@example.com' })).toBe('User: Ada Lovelace <ada@example.com>')
+  })
+})

scripts/quality-gate/baseline/fixtures/multi-file-api/src/app.ts

@@ -0,0 +1,5 @@
+import { formatUser, type User } from './api'
+
+export function renderUser(user: User) {
+  return `User: ${formatUser(user)}`
+}

scripts/quality-gate/index.ts

@@ -0,0 +1,85 @@
+#!/usr/bin/env bun
+
+import { runQualityGate } from './runner'
+import type { BaselineTarget, QualityGateMode } from './types'
+
+function parseArgs(argv: string[]) {
+  const args = new Map<string, Array<string | boolean>>()
+
+  for (let index = 0; index < argv.length; index += 1) {
+    const arg = argv[index]
+    if (!arg.startsWith('--')) {
+      continue
+    }
+
+    const next = argv[index + 1]
+    if (next && !next.startsWith('--')) {
+      args.set(arg, [...(args.get(arg) ?? []), next])
+      index += 1
+    } else {
+      args.set(arg, [...(args.get(arg) ?? []), true])
+    }
+  }
+
+  return args
+}
+
+function firstArg(args: Map<string, Array<string | boolean>>, name: string) {
+  return args.get(name)?.[0]
+}
+
+function hasFlag(args: Map<string, Array<string | boolean>>, name: string) {
+  return args.has(name)
+}
+
+function readMode(value: string | boolean | undefined): QualityGateMode {
+  if (value === 'pr' || value === 'baseline' || value === 'release') {
+    return value
+  }
+  throw new Error('Usage: bun run quality:gate --mode <pr|baseline|release> [--dry-run] [--allow-live]')
+}
+
+function readBaselineTargets(args: Map<string, Array<string | boolean>>): BaselineTarget[] {
+  const values = (args.get('--provider-model') ?? [])
+    .filter((value): value is string => typeof value === 'string')
+    .flatMap((value) => value.split(','))
+    .map((value) => value.trim())
+    .filter(Boolean)
+
+  return values.map((value) => {
+    const [providerId, modelId, rawLabel] = value.split(':')
+    if (!providerId || !modelId) {
+      throw new Error(`Invalid --provider-model value "${value}". Expected providerId:modelId[:label].`)
+    }
+    return {
+      providerId,
+      modelId,
+      label: rawLabel || `${providerId.slice(0, 8)}-${modelId}`,
+    }
+  })
+}
+
+const args = parseArgs(process.argv.slice(2))
+const mode = readMode(firstArg(args, '--mode'))
+const dryRun = hasFlag(args, '--dry-run')
+const allowLive = hasFlag(args, '--allow-live')
+const artifactsDir = typeof firstArg(args, '--artifacts-dir') === 'string'
+  ? String(firstArg(args, '--artifacts-dir'))
+  : undefined
+const baselineTargets = readBaselineTargets(args)
+
+const { report, outputDir } = await runQualityGate({
+  mode,
+  dryRun,
+  allowLive,
+  baselineTargets,
+  rootDir: process.cwd(),
+  artifactsDir,
+})
+
+console.log(`Quality gate report: ${outputDir}/report.md`)
+console.log(`Summary: passed=${report.summary.passed} failed=${report.summary.failed} skipped=${report.summary.skipped}`)
+
+if (report.summary.failed > 0) {
+  process.exit(1)
+}

scripts/quality-gate/modes.ts

@@ -0,0 +1,61 @@
+import { baselineCases } from './baseline/cases'
+import type { BaselineTarget, LaneDefinition, QualityGateMode } from './types'
+
+export function lanesForMode(mode: QualityGateMode, baselineTargets: BaselineTarget[] = []): LaneDefinition[] {
+  const lanes: LaneDefinition[] = [
+    {
+      id: 'impact-report',
+      title: 'Impact report',
+      description: 'Summarize changed areas, required local checks, and risk notes.',
+      kind: 'command',
+      command: ['bun', 'run', 'check:impact'],
+      requiredForModes: ['pr', 'baseline', 'release'],
+    },
+    {
+      id: 'pr-checks',
+      title: 'Path-aware PR checks',
+      description: 'Run the existing local PR gate with stable path-aware checks.',
+      kind: 'command',
+      command: ['bun', 'run', 'check:pr'],
+      requiredForModes: ['pr', 'release'],
+    },
+    {
+      id: 'baseline-catalog',
+      title: 'Baseline case catalog validation',
+      description: 'Validate real Coding Agent baseline case definitions and fixture metadata.',
+      kind: 'command',
+      command: ['bun', 'test', 'scripts/quality-gate/baseline/cases.test.ts'],
+      requiredForModes: ['baseline', 'release'],
+    },
+    {
+      id: 'native-checks',
+      title: 'Native desktop checks',
+      description: 'Build sidecars and run the Tauri native compile check.',
+      kind: 'command',
+      command: ['bun', 'run', 'check:native'],
+      requiredForModes: ['release'],
+    },
+  ]
+
+  const targets = baselineTargets.length > 0
+    ? baselineTargets
+    : [{ providerId: null, modelId: 'current', label: 'current-runtime' }]
+
+  for (const testCase of baselineCases) {
+    for (const target of targets) {
+      const targetSlug = target.label.replace(/[^a-zA-Z0-9._-]+/g, '-')
+      lanes.push({
+        id: `baseline:${testCase.id}:${targetSlug}`,
+        title: `${testCase.title} (${target.label})`,
+        description: testCase.description,
+        kind: 'baseline-case',
+        baselineCaseId: testCase.id,
+        baselineTarget: target,
+        requiredForModes: ['baseline', 'release'],
+        live: true,
+      })
+    }
+  }
+
+  return lanes.filter((lane) => lane.requiredForModes.includes(mode))
+}

scripts/quality-gate/quarantine.json

@@ -0,0 +1,39 @@
+{
+  "quarantined": [
+    {
+      "id": "server:cron-scheduler",
+      "path": "src/server/__tests__/cron-scheduler.test.ts",
+      "reason": "Known unstable scheduler timing assertions; keep out of default server gate until fixed.",
+      "owner": "maintainers",
+      "reviewAfter": "2026-06-01"
+    },
+    {
+      "id": "server:providers-real",
+      "path": "src/server/__tests__/providers-real.test.ts",
+      "reason": "Live provider coverage belongs in maintainer baseline or release mode, not the default PR gate.",
+      "owner": "maintainers",
+      "reviewAfter": "2026-06-01"
+    },
+    {
+      "id": "server:tasks",
+      "path": "src/server/__tests__/tasks.test.ts",
+      "reason": "Known default-gate instability; keep quarantined until task timing is hardened.",
+      "owner": "maintainers",
+      "reviewAfter": "2026-06-01"
+    },
+    {
+      "id": "server:e2e:business-flow",
+      "path": "src/server/__tests__/e2e/business-flow.test.ts",
+      "reason": "Broad e2e flow is not deterministic enough for default PR checks.",
+      "owner": "maintainers",
+      "reviewAfter": "2026-06-01"
+    },
+    {
+      "id": "server:e2e:full-flow",
+      "path": "src/server/__tests__/e2e/full-flow.test.ts",
+      "reason": "Broad e2e flow is not deterministic enough for default PR checks.",
+      "owner": "maintainers",
+      "reviewAfter": "2026-06-01"
+    }
+  ]
+}

scripts/quality-gate/quarantine.test.ts

@@ -0,0 +1,35 @@
+import { describe, expect, test } from 'bun:test'
+import { loadQuarantineManifest, quarantinedPathSet, validateQuarantineManifest } from './quarantine'
+
+describe('quarantine manifest', () => {
+  test('loads the default manifest', () => {
+    const manifest = loadQuarantineManifest()
+    expect(manifest.quarantined.length).toBeGreaterThan(0)
+  })
+
+  test('exposes quarantined paths as a set', () => {
+    const paths = quarantinedPathSet()
+    expect(paths.has('src/server/__tests__/providers-real.test.ts')).toBe(true)
+  })
+
+  test('rejects duplicate ids', () => {
+    expect(() => validateQuarantineManifest({
+      quarantined: [
+        {
+          id: 'duplicate',
+          path: 'a.test.ts',
+          reason: 'test',
+          owner: 'maintainers',
+          reviewAfter: '2026-06-01',
+        },
+        {
+          id: 'duplicate',
+          path: 'b.test.ts',
+          reason: 'test',
+          owner: 'maintainers',
+          reviewAfter: '2026-06-01',
+        },
+      ],
+    })).toThrow('duplicate quarantine id')
+  })
+})

scripts/quality-gate/quarantine.ts

@@ -0,0 +1,50 @@
+import { readFileSync } from 'node:fs'
+import { dirname, join } from 'node:path'
+import { fileURLToPath } from 'node:url'
+
+export type QuarantineEntry = {
+  id: string
+  path: string
+  reason: string
+  owner: string
+  reviewAfter: string
+}
+
+export type QuarantineManifest = {
+  quarantined: QuarantineEntry[]
+}
+
+const defaultManifestPath = join(dirname(fileURLToPath(import.meta.url)), 'quarantine.json')
+
+export function loadQuarantineManifest(path = defaultManifestPath): QuarantineManifest {
+  const manifest = JSON.parse(readFileSync(path, 'utf8')) as QuarantineManifest
+  validateQuarantineManifest(manifest)
+  return manifest
+}
+
+export function validateQuarantineManifest(manifest: QuarantineManifest) {
+  if (!Array.isArray(manifest.quarantined)) {
+    throw new Error('quarantine manifest must contain a quarantined array')
+  }
+
+  const ids = new Set<string>()
+  const paths = new Set<string>()
+
+  for (const entry of manifest.quarantined) {
+    if (!entry.id || !entry.path || !entry.reason || !entry.owner || !entry.reviewAfter) {
+      throw new Error(`invalid quarantine entry: ${JSON.stringify(entry)}`)
+    }
+    if (ids.has(entry.id)) {
+      throw new Error(`duplicate quarantine id: ${entry.id}`)
+    }
+    if (paths.has(entry.path)) {
+      throw new Error(`duplicate quarantine path: ${entry.path}`)
+    }
+    ids.add(entry.id)
+    paths.add(entry.path)
+  }
+}
+
+export function quarantinedPathSet(manifest = loadQuarantineManifest()) {
+  return new Set(manifest.quarantined.map((entry) => entry.path))
+}

scripts/quality-gate/reporter.ts

@@ -0,0 +1,59 @@
+import { mkdirSync, writeFileSync } from 'node:fs'
+import { join } from 'node:path'
+import type { QualityGateReport } from './types'
+
+export function writeReport(report: QualityGateReport, outputDir: string) {
+  mkdirSync(outputDir, { recursive: true })
+  writeFileSync(join(outputDir, 'report.json'), JSON.stringify(report, null, 2) + '\n')
+  writeFileSync(join(outputDir, 'report.md'), renderMarkdownReport(report))
+}
+
+export function renderMarkdownReport(report: QualityGateReport) {
+  const lines = [
+    `# Quality Gate Report`,
+    '',
+    `- Run: ${report.runId}`,
+    `- Mode: ${report.mode}`,
+    `- Dry run: ${report.dryRun ? 'yes' : 'no'}`,
+    `- Live checks allowed: ${report.allowLive ? 'yes' : 'no'}`,
+    `- Git SHA: ${report.git.sha ?? 'unknown'}`,
+    `- Dirty worktree: ${report.git.dirty ? 'yes' : 'no'}`,
+    `- Started: ${report.startedAt}`,
+    `- Finished: ${report.finishedAt}`,
+    '',
+    `## Summary`,
+    '',
+    `- Passed: ${report.summary.passed}`,
+    `- Failed: ${report.summary.failed}`,
+    `- Skipped: ${report.summary.skipped}`,
+    '',
+    `## Lanes`,
+    '',
+  ]
+
+  for (const result of report.results) {
+    lines.push(`### ${result.title}`)
+    lines.push('')
+    lines.push(`- ID: ${result.id}`)
+    lines.push(`- Status: ${result.status}`)
+    lines.push(`- Duration: ${result.durationMs}ms`)
+    if (result.command) {
+      lines.push(`- Command: \`${result.command.join(' ')}\``)
+    }
+    if (result.exitCode !== undefined) {
+      lines.push(`- Exit code: ${result.exitCode}`)
+    }
+    if (result.skipReason) {
+      lines.push(`- Skip reason: ${result.skipReason}`)
+    }
+    if (result.error) {
+      lines.push(`- Error: ${result.error}`)
+    }
+    if (result.artifactDir) {
+      lines.push(`- Artifacts: ${result.artifactDir}`)
+    }
+    lines.push('')
+  }
+
+  return lines.join('\n')
+}

scripts/quality-gate/runner.test.ts

@@ -0,0 +1,108 @@
+import { describe, expect, test } from 'bun:test'
+import { mkdtempSync, readFileSync, rmSync } from 'node:fs'
+import { tmpdir } from 'node:os'
+import { join } from 'node:path'
+import { lanesForMode } from './modes'
+import { renderMarkdownReport } from './reporter'
+import { runQualityGate } from './runner'
+import type { QualityGateReport } from './types'
+
+describe('quality gate modes', () => {
+  test('pr mode includes existing path-aware PR checks', () => {
+    const lanes = lanesForMode('pr').map((lane) => lane.id)
+    expect(lanes).toContain('impact-report')
+    expect(lanes).toContain('pr-checks')
+    expect(lanes.some((lane) => lane.startsWith('baseline:'))).toBe(false)
+  })
+
+  test('baseline mode includes live baseline cases but not native checks', () => {
+    const lanes = lanesForMode('baseline').map((lane) => lane.id)
+    expect(lanes).toContain('baseline-catalog')
+    expect(lanes).toContain('baseline:failing-unit:current-runtime')
+    expect(lanes).toContain('baseline:multi-file-api:current-runtime')
+    expect(lanes).not.toContain('native-checks')
+  })
+
+  test('release mode composes PR, baseline, and native lanes', () => {
+    const lanes = lanesForMode('release').map((lane) => lane.id)
+    expect(lanes).toContain('pr-checks')
+    expect(lanes).toContain('baseline:failing-unit:current-runtime')
+    expect(lanes).toContain('native-checks')
+  })
+
+  test('baseline mode expands cases across explicit provider/model targets', () => {
+    const lanes = lanesForMode('baseline', [
+      { providerId: 'provider-a', modelId: 'model-a', label: 'provider-a-model-a' },
+      { providerId: 'provider-b', modelId: 'model-b', label: 'provider-b-model-b' },
+    ]).map((lane) => lane.id)
+
+    expect(lanes).toContain('baseline:failing-unit:provider-a-model-a')
+    expect(lanes).toContain('baseline:failing-unit:provider-b-model-b')
+    expect(lanes).toContain('baseline:multi-file-api:provider-a-model-a')
+    expect(lanes).toContain('baseline:multi-file-api:provider-b-model-b')
+  })
+})
+
+describe('runQualityGate', () => {
+  test('writes dry-run reports without executing expensive commands', async () => {
+    const artifactsDir = mkdtempSync(join(tmpdir(), 'quality-gate-test-'))
+    try {
+      const { report, outputDir } = await runQualityGate({
+        mode: 'baseline',
+        dryRun: true,
+        allowLive: false,
+        baselineTargets: [],
+        rootDir: process.cwd(),
+        artifactsDir,
+        runId: 'dry-run-test',
+      })
+
+      expect(report.mode).toBe('baseline')
+      expect(report.summary.failed).toBe(0)
+      expect(report.summary.skipped).toBeGreaterThan(0)
+      expect(readFileSync(join(outputDir, 'report.json'), 'utf8')).toContain('"mode": "baseline"')
+      expect(readFileSync(join(outputDir, 'report.md'), 'utf8')).toContain('# Quality Gate Report')
+    } finally {
+      rmSync(artifactsDir, { recursive: true, force: true })
+    }
+  })
+})
+
+describe('renderMarkdownReport', () => {
+  test('renders command, skip reason, and summary', () => {
+    const report: QualityGateReport = {
+      schemaVersion: 1,
+      runId: 'example',
+      mode: 'pr',
+      dryRun: true,
+      allowLive: false,
+      startedAt: '2026-05-02T00:00:00.000Z',
+      finishedAt: '2026-05-02T00:00:01.000Z',
+      rootDir: process.cwd(),
+      git: {
+        sha: 'abc123',
+        dirty: true,
+      },
+      results: [
+        {
+          id: 'impact-report',
+          title: 'Impact report',
+          status: 'skipped',
+          command: ['bun', 'run', 'check:impact'],
+          durationMs: 1,
+          skipReason: 'dry run',
+        },
+      ],
+      summary: {
+        passed: 0,
+        failed: 0,
+        skipped: 1,
+      },
+    }
+
+    const markdown = renderMarkdownReport(report)
+    expect(markdown).toContain('Skipped: 1')
+    expect(markdown).toContain('`bun run check:impact`')
+    expect(markdown).toContain('dry run')
+  })
+})

scripts/quality-gate/runner.ts

@@ -0,0 +1,152 @@
+import { mkdirSync } from 'node:fs'
+import { join } from 'node:path'
+import { baselineCases } from './baseline/cases'
+import { executeBaselineCase } from './baseline/execute'
+import { lanesForMode } from './modes'
+import { writeReport } from './reporter'
+import type { LaneDefinition, LaneResult, QualityGateOptions, QualityGateReport } from './types'
+
+function nowId() {
+  return new Date().toISOString().replace(/[:.]/g, '-')
+}
+
+async function output(cmd: string[], cwd: string) {
+  const proc = Bun.spawn(cmd, {
+    cwd,
+    stdout: 'pipe',
+    stderr: 'pipe',
+  })
+  const stdout = await new Response(proc.stdout).text()
+  const stderr = await new Response(proc.stderr).text()
+  const code = await proc.exited
+  if (code !== 0) {
+    return null
+  }
+  return (stdout || stderr).trim()
+}
+
+async function gitInfo(rootDir: string) {
+  const sha = await output(['git', 'rev-parse', '--short', 'HEAD'], rootDir)
+  const status = await output(['git', 'status', '--short'], rootDir)
+  return {
+    sha,
+    dirty: Boolean(status),
+  }
+}
+
+async function runCommandLane(lane: LaneDefinition, options: QualityGateOptions): Promise<LaneResult> {
+  const started = Date.now()
+  const command = lane.command ?? []
+
+  if (options.dryRun) {
+    return {
+      id: lane.id,
+      title: lane.title,
+      status: 'skipped',
+      command,
+      durationMs: Date.now() - started,
+      skipReason: 'dry run',
+    }
+  }
+
+  const proc = Bun.spawn(command, {
+    cwd: options.rootDir,
+    stdout: 'inherit',
+    stderr: 'inherit',
+  })
+  const exitCode = await proc.exited
+
+  return {
+    id: lane.id,
+    title: lane.title,
+    status: exitCode === 0 ? 'passed' : 'failed',
+    command,
+    durationMs: Date.now() - started,
+    exitCode,
+  }
+}
+
+async function runBaselineCaseLane(lane: LaneDefinition, options: QualityGateOptions): Promise<LaneResult> {
+  const started = Date.now()
+
+  if (!options.allowLive) {
+    return {
+      id: lane.id,
+      title: lane.title,
+      status: 'skipped',
+      durationMs: Date.now() - started,
+      skipReason: 'live baseline cases require --allow-live',
+    }
+  }
+
+  const caseId = lane.baselineCaseId ?? lane.id.replace(/^baseline:/, '').split(':')[0]
+  const testCase = baselineCases.find((candidate) => candidate.id === caseId)
+  if (!testCase) {
+    return {
+      id: lane.id,
+      title: lane.title,
+      status: 'failed',
+      durationMs: Date.now() - started,
+      error: `Unknown baseline case: ${caseId}`,
+    }
+  }
+
+  const artifactRoot = options.runOutputDir ?? join(options.rootDir, 'artifacts', 'quality-runs', options.runId ?? 'current')
+  return executeBaselineCase(
+    testCase,
+    options.rootDir,
+    join(artifactRoot, 'cases', lane.id.replace(/[^a-zA-Z0-9._-]+/g, '-')),
+    lane.baselineTarget,
+  )
+}
+
+async function runLane(lane: LaneDefinition, options: QualityGateOptions): Promise<LaneResult> {
+  if (lane.kind === 'baseline-case') {
+    return runBaselineCaseLane(lane, options)
+  }
+
+  return runCommandLane(lane, options)
+}
+
+function summarize(results: LaneResult[]) {
+  return {
+    passed: results.filter((result) => result.status === 'passed').length,
+    failed: results.filter((result) => result.status === 'failed').length,
+    skipped: results.filter((result) => result.status === 'skipped').length,
+  }
+}
+
+export async function runQualityGate(options: QualityGateOptions) {
+  const runId = options.runId ?? nowId()
+  const startedAt = new Date().toISOString()
+  const artifactsRoot = options.artifactsDir ?? join(options.rootDir, 'artifacts', 'quality-runs')
+  const outputDir = join(artifactsRoot, runId)
+  mkdirSync(outputDir, { recursive: true })
+
+  const runOptions = { ...options, runId, runOutputDir: outputDir }
+  const results: LaneResult[] = []
+  for (const lane of lanesForMode(options.mode, options.baselineTargets)) {
+    const result = await runLane(lane, runOptions)
+    results.push(result)
+    if (result.status === 'failed') {
+      break
+    }
+  }
+
+  const report: QualityGateReport = {
+    schemaVersion: 1,
+    runId,
+    mode: options.mode,
+    dryRun: options.dryRun,
+    allowLive: options.allowLive,
+    startedAt,
+    finishedAt: new Date().toISOString(),
+    rootDir: options.rootDir,
+    git: await gitInfo(options.rootDir),
+    results,
+    summary: summarize(results),
+  }
+
+  writeReport(report, outputDir)
+  return { report, outputDir }
+}

scripts/quality-gate/types.ts

@@ -0,0 +1,84 @@
+export type QualityGateMode = 'pr' | 'baseline' | 'release'
+
+export type LaneKind = 'command' | 'baseline-case'
+
+export type LaneDefinition = {
+  id: string
+  title: string
+  description: string
+  kind: LaneKind
+  command?: string[]
+  baselineCaseId?: string
+  baselineTarget?: BaselineTarget
+  requiredForModes: QualityGateMode[]
+  live?: boolean
+}
+
+export type BaselineCase = {
+  id: string
+  title: string
+  description: string
+  fixture: string
+  prompt: string
+  mode: 'ui' | 'websocket'
+  requiredCapabilities: Array<'model' | 'file-edit' | 'shell' | 'permission' | 'browser'>
+  timeoutMs: number
+  verify: {
+    commands: string[][]
+    expectedFiles?: string[]
+    forbiddenFiles?: string[]
+    transcriptAssertions?: string[]
+  }
+}
+
+export type BaselineTarget = {
+  providerId: string | null
+  modelId: string
+  label: string
+}
+
+export type LaneStatus = 'passed' | 'failed' | 'skipped'
+
+export type LaneResult = {
+  id: string
+  title: string
+  status: LaneStatus
+  command?: string[]
+  durationMs: number
+  exitCode?: number
+  skipReason?: string
+  error?: string
+  artifactDir?: string
+}
+
+export type QualityGateOptions = {
+  mode: QualityGateMode
+  dryRun: boolean
+  allowLive: boolean
+  baselineTargets: BaselineTarget[]
+  rootDir: string
+  artifactsDir?: string
+  runOutputDir?: string
+  runId?: string
+}
+
+export type QualityGateReport = {
+  schemaVersion: 1
+  runId: string
+  mode: QualityGateMode
+  dryRun: boolean
+  allowLive: boolean
+  startedAt: string
+  finishedAt: string
+  rootDir: string
+  git: {
+    sha: string | null
+    dirty: boolean
+  }
+  results: LaneResult[]
+  summary: {
+    passed: number
+    failed: number
+    skipped: number
+  }
+}

src/server/__tests__/conversations.test.ts

@@ -237,9 +237,11 @@ describe('ConversationService', () => {
 
   it('should reconstruct usage and metadata from a persisted transcript', async () => {
     const previousConfigDir = process.env.CLAUDE_CONFIG_DIR
+    const previousAnthropicApiKey = process.env.ANTHROPIC_API_KEY
     const tmpConfigDir = await fs.mkdtemp(path.join(os.tmpdir(), 'claude-transcript-'))
     const workDir = await fs.mkdtemp(path.join(os.tmpdir(), 'claude-workdir-'))
     process.env.CLAUDE_CONFIG_DIR = tmpConfigDir
+    process.env.ANTHROPIC_API_KEY = 'test-key'
 
     try {
       const svc = new SessionService()
@@ -290,6 +292,11 @@ describe('ConversationService', () => {
       } else {
         process.env.CLAUDE_CONFIG_DIR = previousConfigDir
       }
+      if (previousAnthropicApiKey === undefined) {
+        delete process.env.ANTHROPIC_API_KEY
+      } else {
+        process.env.ANTHROPIC_API_KEY = previousAnthropicApiKey
+      }
       await fs.rm(tmpConfigDir, { recursive: true, force: true })
       await fs.rm(workDir, { recursive: true, force: true })
     }