Broaden live agent baseline beyond toy edits

The first quality gate proved the execution path, but two tiny fixtures were not enough to reveal regressions in real agent behavior. This expands the maintained baseline with deeper failure recovery, workspace search, artifact creation, and cross-module refactor cases, and records per-case diffs so reviewers can see exactly what each model changed.

Constraint: Baseline runs must remain explicit maintainer-controlled live checks, not default PR checks.

Rejected: Require models to edit tests in every contract-change case | correct fixes can satisfy prewritten acceptance tests without modifying tests.

Rejected: Trust only command exit codes | passing tests alone does not show whether the agent edited forbidden or unrelated files.

Confidence: high

Scope-risk: moderate

Directive: Keep live baseline cases product-shaped and inspect both verification output and diff.patch before accepting future fixture changes.

Tested: bun test scripts/quality-gate/*.test.ts scripts/quality-gate/baseline/*.test.ts

Tested: bun run quality:gate --mode baseline --dry-run --provider-model volc-kimi-k2.6 --provider-model minimax-m2.7

Tested: bun run quality:gate --mode baseline --allow-live --provider-model volc-kimi-k2.6 --provider-model minimax-m2.7 (14 passed, 0 failed)

Not-tested: Desktop UI browser smoke and native Tauri release packaging remain outside this baseline expansion.

scripts/quality-gate/baseline/cases.test.ts

@@ -25,4 +25,25 @@ describe('baselineCases', () => {
       expect(testCase.requiredCapabilities).toContain('model')
     }
   })
+
+  test('define enough first-wave product baseline cases', () => {
+    expect(baselineCases.map((testCase) => testCase.id)).toEqual([
+      'failing-unit',
+      'multi-file-api',
+      'failure-recovery',
+      'workspace-search-edit',
+      'permission-artifact',
+      'cross-module-refactor',
+    ])
+  })
+
+  test('pin changed-file expectations for every case', () => {
+    for (const testCase of baselineCases) {
+      expect(testCase.verify.requiredFiles?.length ?? 0).toBeGreaterThan(0)
+      expect(testCase.verify.expectedFiles?.length ?? 0).toBeGreaterThan(0)
+      for (const file of testCase.verify.requiredFiles ?? []) {
+        expect(testCase.verify.expectedFiles).toContain(file)
+      }
+    }
+  })
 })

scripts/quality-gate/baseline/cases.ts

@@ -12,25 +12,95 @@ export const baselineCases: BaselineCase[] = [
     timeoutMs: 180_000,
     verify: {
       commands: [['bun', 'test']],
+      requiredFiles: ['src/math.ts'],
       expectedFiles: ['src/math.ts'],
       forbiddenFiles: ['package.json'],
-      transcriptAssertions: ['bun test', 'src/math.ts'],
+      transcriptAssertions: ['"type":"message_complete"'],
     },
   },
   {
     id: 'multi-file-api',
     title: 'Update a multi-file API contract',
-    description: 'A small app exposes a user display API. The Agent must change the contract and update callers plus tests coherently.',
+    description: 'A small app exposes a user display API. The Agent must change the contract and update callers coherently without loosening tests.',
     fixture: 'scripts/quality-gate/baseline/fixtures/multi-file-api',
-    prompt: 'Change the user display contract so it returns "Ada Lovelace <ada@example.com>" instead of only the name. Update the implementation, caller, and tests, then run the tests.',
+    prompt: 'The user display API contract changed to return an object with a label field, where label is "Ada Lovelace <ada@example.com>". Update the implementation and caller so the existing tests pass. Do not edit package.json.',
     mode: 'websocket',
     requiredCapabilities: ['model', 'file-edit', 'shell'],
     timeoutMs: 240_000,
     verify: {
       commands: [['bun', 'test']],
+      requiredFiles: ['src/api.ts', 'src/app.ts'],
       expectedFiles: ['src/api.ts', 'src/app.ts', 'src/app.test.ts'],
       forbiddenFiles: ['package.json'],
-      transcriptAssertions: ['bun test', 'src/api.ts', 'src/app.ts'],
+      transcriptAssertions: ['"type":"message_complete"'],
+    },
+  },
+  {
+    id: 'failure-recovery',
+    title: 'Recover from a deeper failing test suite',
+    description: 'A slug formatter has several edge-case failures. The Agent must read the failures, patch the implementation, and keep tests unchanged.',
+    fixture: 'scripts/quality-gate/baseline/fixtures/failure-recovery',
+    prompt: 'Run the tests and fix the slug formatter implementation until the whole suite passes. Do not edit the tests. Keep the implementation small and deterministic.',
+    mode: 'websocket',
+    requiredCapabilities: ['model', 'file-edit', 'shell'],
+    timeoutMs: 240_000,
+    verify: {
+      commands: [['bun', 'test']],
+      requiredFiles: ['src/slug.ts'],
+      expectedFiles: ['src/slug.ts'],
+      forbiddenFiles: ['src/slug.test.ts', 'package.json'],
+      transcriptAssertions: ['"type":"message_complete"'],
+    },
+  },
+  {
+    id: 'workspace-search-edit',
+    title: 'Find and fix a bug across a small workspace',
+    description: 'A checkout total is wrong in a multi-file fixture. The Agent must inspect the workspace and patch the correct implementation file.',
+    fixture: 'scripts/quality-gate/baseline/fixtures/workspace-search-edit',
+    prompt: 'The checkout tests are failing. Search the project, identify the real bug, fix the implementation, and rerun the tests. Do not rewrite unrelated modules.',
+    mode: 'websocket',
+    requiredCapabilities: ['model', 'file-edit', 'shell'],
+    timeoutMs: 240_000,
+    verify: {
+      commands: [['bun', 'test']],
+      requiredFiles: ['src/checkout.ts'],
+      expectedFiles: ['src/checkout.ts'],
+      forbiddenFiles: ['src/checkout.test.ts', 'src/catalog.ts', 'src/user.ts', 'package.json'],
+      transcriptAssertions: ['"type":"message_complete"'],
+    },
+  },
+  {
+    id: 'permission-artifact',
+    title: 'Create a required project artifact',
+    description: 'A test expects a generated summary file. The Agent must create a new file and validate it without changing the test.',
+    fixture: 'scripts/quality-gate/baseline/fixtures/permission-artifact',
+    prompt: 'Run the tests, create the missing project artifact they require, and rerun the tests. Do not modify the tests or package.json.',
+    mode: 'websocket',
+    requiredCapabilities: ['model', 'file-edit', 'shell', 'permission'],
+    timeoutMs: 180_000,
+    verify: {
+      commands: [['bun', 'test']],
+      requiredFiles: ['notes/summary.md'],
+      expectedFiles: ['notes/summary.md'],
+      forbiddenFiles: ['src/artifact.test.ts', 'package.json'],
+      transcriptAssertions: ['"type":"message_complete"'],
+    },
+  },
+  {
+    id: 'cross-module-refactor',
+    title: 'Complete a cross-module parser refactor',
+    description: 'A config parser contract changed but the implementation and caller are stale. The Agent must update multiple modules coherently.',
+    fixture: 'scripts/quality-gate/baseline/fixtures/cross-module-refactor',
+    prompt: 'Update the config parsing flow so parseConfig returns a structured object with enabled and retries fields. Update the implementation and caller so the existing tests pass, then run the suite.',
+    mode: 'websocket',
+    requiredCapabilities: ['model', 'file-edit', 'shell'],
+    timeoutMs: 300_000,
+    verify: {
+      commands: [['bun', 'test']],
+      requiredFiles: ['src/config.ts', 'src/runner.ts'],
+      expectedFiles: ['src/config.ts', 'src/runner.ts', 'src/runner.test.ts'],
+      forbiddenFiles: ['package.json'],
+      transcriptAssertions: ['"type":"message_complete"'],
     },
   },
 ]

scripts/quality-gate/baseline/execute.ts

@@ -1,4 +1,4 @@
-import { appendFileSync, cpSync, mkdirSync, rmSync, writeFileSync } from 'node:fs'
+import { appendFileSync, cpSync, mkdirSync, readdirSync, readFileSync, rmSync, statSync, writeFileSync } from 'node:fs'
 import { mkdtemp } from 'node:fs/promises'
 import { tmpdir } from 'node:os'
 import { join } from 'node:path'
@@ -51,6 +51,90 @@ async function runCommand(command: string[], cwd: string) {
   return { exitCode, stdout, stderr }
 }
 
+function listFiles(root: string, current = root): string[] {
+  const files: string[] = []
+  for (const entry of readdirSync(current)) {
+    if (entry === 'node_modules' || entry === '.git') {
+      continue
+    }
+
+    const fullPath = join(current, entry)
+    const stat = statSync(fullPath)
+    if (stat.isDirectory()) {
+      files.push(...listFiles(root, fullPath))
+      continue
+    }
+    if (stat.isFile()) {
+      files.push(fullPath.slice(root.length + 1))
+    }
+  }
+  return files.sort()
+}
+
+function changedFiles(beforeDir: string, afterDir: string) {
+  const before = new Set(listFiles(beforeDir))
+  const after = new Set(listFiles(afterDir))
+  const changed = new Set<string>()
+
+  for (const file of before) {
+    if (!after.has(file)) {
+      changed.add(file)
+      continue
+    }
+    if (!readFileSync(join(beforeDir, file)).equals(readFileSync(join(afterDir, file)))) {
+      changed.add(file)
+    }
+  }
+
+  for (const file of after) {
+    if (!before.has(file)) {
+      changed.add(file)
+    }
+  }
+
+  return [...changed].sort()
+}
+
+async function writeDiffPatch(beforeDir: string, afterDir: string, patchPath: string) {
+  const result = await runCommand(['git', 'diff', '--no-index', '--', beforeDir, afterDir], process.cwd())
+  writeFileSync(patchPath, `${result.stdout}${result.stderr}`)
+}
+
+function verifyChangedFiles(testCase: BaselineCase, changed: string[]) {
+  const expected = testCase.verify.expectedFiles
+  if (expected) {
+    const unexpected = changed.filter((file) => !expected.includes(file))
+    if (unexpected.length > 0) {
+      throw new Error(`unexpected changed files: ${unexpected.join(', ')}`)
+    }
+  }
+
+  const required = testCase.verify.requiredFiles ?? []
+  const missing = required.filter((file) => !changed.includes(file))
+  if (missing.length > 0) {
+    throw new Error(`required files were not changed: ${missing.join(', ')}`)
+  }
+
+  const forbidden = testCase.verify.forbiddenFiles ?? []
+  const forbiddenChanged = forbidden.filter((file) => changed.includes(file))
+  if (forbiddenChanged.length > 0) {
+    throw new Error(`forbidden files changed: ${forbiddenChanged.join(', ')}`)
+  }
+}
+
+function verifyTranscript(testCase: BaselineCase, transcriptPath: string) {
+  const assertions = testCase.verify.transcriptAssertions ?? []
+  if (assertions.length === 0) {
+    return
+  }
+
+  const transcript = readFileSync(transcriptPath, 'utf8')
+  const missing = assertions.filter((assertion) => !transcript.includes(assertion))
+  if (missing.length > 0) {
+    throw new Error(`transcript assertions missing: ${missing.join(', ')}`)
+  }
+}
+
 async function pipeToFile(stream: ReadableStream<Uint8Array> | null, path: string) {
   if (!stream) return
   const reader = stream.getReader()
@@ -154,12 +238,15 @@ export async function executeBaselineCase(
   const port = await getPort()
   const baseUrl = `http://127.0.0.1:${port}`
   const workRoot = await mkdtemp(join(tmpdir(), `quality-gate-${testCase.id}-`))
+  const originalDir = join(workRoot, 'original')
   const projectDir = join(workRoot, 'project')
+  cpSync(join(rootDir, testCase.fixture), originalDir, { recursive: true })
   cpSync(join(rootDir, testCase.fixture), projectDir, { recursive: true })
 
   const serverLogPath = join(artifactDir, 'server.log')
   const transcriptPath = join(artifactDir, 'transcript.jsonl')
   const verificationPath = join(artifactDir, 'verification.log')
+  const diffPath = join(artifactDir, 'diff.patch')
   const server = Bun.spawn(['bun', 'run', 'src/server/index.ts', '--host', '127.0.0.1', '--port', String(port)], {
     cwd: rootDir,
     stdout: 'pipe',
@@ -190,6 +277,11 @@ export async function executeBaselineCase(
 
     const messages = await runPromptOverWebSocket(baseUrl, session.sessionId, testCase.prompt, testCase.timeoutMs, target)
     writeFileSync(transcriptPath, messages.map((message) => JSON.stringify(message)).join('\n') + '\n')
+    verifyTranscript(testCase, transcriptPath)
+
+    await writeDiffPatch(originalDir, projectDir, diffPath)
+    const changed = changedFiles(originalDir, projectDir)
+    verifyChangedFiles(testCase, changed)
 
     let verificationLog = ''
     for (const command of testCase.verify.commands) {

scripts/quality-gate/baseline/fixtures/cross-module-refactor/package.json

@@ -0,0 +1,8 @@
+{
+  "name": "quality-gate-cross-module-refactor-fixture",
+  "private": true,
+  "type": "module",
+  "scripts": {
+    "test": "bun test"
+  }
+}

scripts/quality-gate/baseline/fixtures/cross-module-refactor/src/config.ts

@@ -0,0 +1,3 @@
+export function parseConfig(raw: string) {
+  return raw === 'enabled'
+}

scripts/quality-gate/baseline/fixtures/cross-module-refactor/src/runner.test.ts

@@ -0,0 +1,17 @@
+import { describe, expect, test } from 'bun:test'
+import { parseConfig } from './config'
+import { describeRun } from './runner'
+
+describe('config runner', () => {
+  test('parses structured config', () => {
+    expect(parseConfig('enabled retries=3')).toEqual({ enabled: true, retries: 3 })
+  })
+
+  test('describes enabled retry runs', () => {
+    expect(describeRun('enabled retries=3')).toBe('enabled with 3 retries')
+  })
+
+  test('describes disabled runs', () => {
+    expect(describeRun('disabled retries=0')).toBe('disabled')
+  })
+})

scripts/quality-gate/baseline/fixtures/cross-module-refactor/src/runner.ts

@@ -0,0 +1,6 @@
+import { parseConfig } from './config'
+
+export function describeRun(rawConfig: string) {
+  const enabled = parseConfig(rawConfig)
+  return enabled ? 'enabled' : 'disabled'
+}

scripts/quality-gate/baseline/fixtures/failure-recovery/package.json

@@ -0,0 +1,8 @@
+{
+  "name": "quality-gate-failure-recovery-fixture",
+  "private": true,
+  "type": "module",
+  "scripts": {
+    "test": "bun test"
+  }
+}

scripts/quality-gate/baseline/fixtures/failure-recovery/src/slug.test.ts

@@ -0,0 +1,16 @@
+import { describe, expect, test } from 'bun:test'
+import { slugify } from './slug'
+
+describe('slugify', () => {
+  test('normalizes whitespace and punctuation', () => {
+    expect(slugify('  Hello, Coding Agent!  ')).toBe('hello-coding-agent')
+  })
+
+  test('collapses repeated separators', () => {
+    expect(slugify('Ship---Stable    Releases')).toBe('ship-stable-releases')
+  })
+
+  test('trims separators from the edges', () => {
+    expect(slugify('---Quality Gate---')).toBe('quality-gate')
+  })
+})

scripts/quality-gate/baseline/fixtures/failure-recovery/src/slug.ts

@@ -0,0 +1,3 @@
+export function slugify(input: string) {
+  return input.toLowerCase().replaceAll(' ', '-')
+}

scripts/quality-gate/baseline/fixtures/multi-file-api/src/api.ts

@@ -3,6 +3,12 @@ export type User = {
   email: string
 }
 
-export function formatUser(user: User) {
-  return user.name
+export type UserDisplay = {
+  displayName: string
+}
+
+export function formatUser(user: User) {
+  return {
+    displayName: user.name,
+  } satisfies UserDisplay
 }

scripts/quality-gate/baseline/fixtures/multi-file-api/src/app.test.ts

@@ -1,8 +1,15 @@
 import { describe, expect, test } from 'bun:test'
+import { formatUser } from './api'
 import { renderUser } from './app'
 
 describe('renderUser', () => {
   test('renders name and email', () => {
     expect(renderUser({ name: 'Ada Lovelace', email: 'ada@example.com' })).toBe('User: Ada Lovelace <ada@example.com>')
   })
+
+  test('uses the structured display API contract', () => {
+    expect(formatUser({ name: 'Ada Lovelace', email: 'ada@example.com' })).toEqual({
+      label: 'Ada Lovelace <ada@example.com>',
+    })
+  })
 })

scripts/quality-gate/baseline/fixtures/multi-file-api/src/app.ts

@@ -1,5 +1,5 @@
 import { formatUser, type User } from './api'
 
 export function renderUser(user: User) {
-  return `User: ${formatUser(user)}`
+  return `User: ${formatUser(user).displayName}`
 }

scripts/quality-gate/baseline/fixtures/permission-artifact/package.json

@@ -0,0 +1,8 @@
+{
+  "name": "quality-gate-permission-artifact-fixture",
+  "private": true,
+  "type": "module",
+  "scripts": {
+    "test": "bun test"
+  }
+}

scripts/quality-gate/baseline/fixtures/permission-artifact/src/artifact.test.ts

@@ -0,0 +1,9 @@
+import { describe, expect, test } from 'bun:test'
+import { existsSync, readFileSync } from 'node:fs'
+
+describe('release artifact', () => {
+  test('has the required summary marker', () => {
+    expect(existsSync('notes/summary.md')).toBe(true)
+    expect(readFileSync('notes/summary.md', 'utf8')).toContain('baseline-ready')
+  })
+})

scripts/quality-gate/baseline/fixtures/workspace-search-edit/package.json

@@ -0,0 +1,8 @@
+{
+  "name": "quality-gate-workspace-search-edit-fixture",
+  "private": true,
+  "type": "module",
+  "scripts": {
+    "test": "bun test"
+  }
+}

scripts/quality-gate/baseline/fixtures/workspace-search-edit/src/catalog.ts

@@ -0,0 +1,9 @@
+export type CatalogItem = {
+  sku: string
+  priceCents: number
+}
+
+export const catalog: CatalogItem[] = [
+  { sku: 'agent-seat', priceCents: 1200 },
+  { sku: 'baseline-pack', priceCents: 800 },
+]

scripts/quality-gate/baseline/fixtures/workspace-search-edit/src/checkout.test.ts

@@ -0,0 +1,10 @@
+import { describe, expect, test } from 'bun:test'
+import { catalog } from './catalog'
+import { calculateTotalCents } from './checkout'
+import { testUser } from './user'
+
+describe('calculateTotalCents', () => {
+  test('applies percentage discounts to the subtotal', () => {
+    expect(calculateTotalCents(catalog, testUser)).toBe(1500)
+  })
+})

scripts/quality-gate/baseline/fixtures/workspace-search-edit/src/checkout.ts

@@ -0,0 +1,7 @@
+import type { CatalogItem } from './catalog'
+import type { User } from './user'
+
+export function calculateTotalCents(items: CatalogItem[], user: User) {
+  const subtotal = items.reduce((sum, item) => sum + item.priceCents, 0)
+  return subtotal - user.discountPercent
+}

scripts/quality-gate/baseline/fixtures/workspace-search-edit/src/user.ts

@@ -0,0 +1,9 @@
+export type User = {
+  id: string
+  discountPercent: number
+}
+
+export const testUser: User = {
+  id: 'user-1',
+  discountPercent: 25,
+}

scripts/quality-gate/types.ts

@@ -25,6 +25,7 @@ export type BaselineCase = {
   timeoutMs: number
   verify: {
     commands: string[][]
+    requiredFiles?: string[]
     expectedFiles?: string[]
     forbiddenFiles?: string[]
     transcriptAssertions?: string[]