diff --git a/scripts/quality-gate/baseline/cases.test.ts b/scripts/quality-gate/baseline/cases.test.ts index c7b624a5..c895e099 100644 --- a/scripts/quality-gate/baseline/cases.test.ts +++ b/scripts/quality-gate/baseline/cases.test.ts @@ -25,4 +25,25 @@ describe('baselineCases', () => { expect(testCase.requiredCapabilities).toContain('model') } }) + + test('define enough first-wave product baseline cases', () => { + expect(baselineCases.map((testCase) => testCase.id)).toEqual([ + 'failing-unit', + 'multi-file-api', + 'failure-recovery', + 'workspace-search-edit', + 'permission-artifact', + 'cross-module-refactor', + ]) + }) + + test('pin changed-file expectations for every case', () => { + for (const testCase of baselineCases) { + expect(testCase.verify.requiredFiles?.length ?? 0).toBeGreaterThan(0) + expect(testCase.verify.expectedFiles?.length ?? 0).toBeGreaterThan(0) + for (const file of testCase.verify.requiredFiles ?? []) { + expect(testCase.verify.expectedFiles).toContain(file) + } + } + }) }) diff --git a/scripts/quality-gate/baseline/cases.ts b/scripts/quality-gate/baseline/cases.ts index 4e308e2d..34ab7849 100644 --- a/scripts/quality-gate/baseline/cases.ts +++ b/scripts/quality-gate/baseline/cases.ts @@ -12,25 +12,95 @@ export const baselineCases: BaselineCase[] = [ timeoutMs: 180_000, verify: { commands: [['bun', 'test']], + requiredFiles: ['src/math.ts'], expectedFiles: ['src/math.ts'], forbiddenFiles: ['package.json'], - transcriptAssertions: ['bun test', 'src/math.ts'], + transcriptAssertions: ['"type":"message_complete"'], }, }, { id: 'multi-file-api', title: 'Update a multi-file API contract', - description: 'A small app exposes a user display API. The Agent must change the contract and update callers plus tests coherently.', + description: 'A small app exposes a user display API. The Agent must change the contract and update callers coherently without loosening tests.', fixture: 'scripts/quality-gate/baseline/fixtures/multi-file-api', - prompt: 'Change the user display contract so it returns "Ada Lovelace " instead of only the name. Update the implementation, caller, and tests, then run the tests.', + prompt: 'The user display API contract changed to return an object with a label field, where label is "Ada Lovelace ". Update the implementation and caller so the existing tests pass. Do not edit package.json.', mode: 'websocket', requiredCapabilities: ['model', 'file-edit', 'shell'], timeoutMs: 240_000, verify: { commands: [['bun', 'test']], + requiredFiles: ['src/api.ts', 'src/app.ts'], expectedFiles: ['src/api.ts', 'src/app.ts', 'src/app.test.ts'], forbiddenFiles: ['package.json'], - transcriptAssertions: ['bun test', 'src/api.ts', 'src/app.ts'], + transcriptAssertions: ['"type":"message_complete"'], + }, + }, + { + id: 'failure-recovery', + title: 'Recover from a deeper failing test suite', + description: 'A slug formatter has several edge-case failures. The Agent must read the failures, patch the implementation, and keep tests unchanged.', + fixture: 'scripts/quality-gate/baseline/fixtures/failure-recovery', + prompt: 'Run the tests and fix the slug formatter implementation until the whole suite passes. Do not edit the tests. Keep the implementation small and deterministic.', + mode: 'websocket', + requiredCapabilities: ['model', 'file-edit', 'shell'], + timeoutMs: 240_000, + verify: { + commands: [['bun', 'test']], + requiredFiles: ['src/slug.ts'], + expectedFiles: ['src/slug.ts'], + forbiddenFiles: ['src/slug.test.ts', 'package.json'], + transcriptAssertions: ['"type":"message_complete"'], + }, + }, + { + id: 'workspace-search-edit', + title: 'Find and fix a bug across a small workspace', + description: 'A checkout total is wrong in a multi-file fixture. The Agent must inspect the workspace and patch the correct implementation file.', + fixture: 'scripts/quality-gate/baseline/fixtures/workspace-search-edit', + prompt: 'The checkout tests are failing. Search the project, identify the real bug, fix the implementation, and rerun the tests. Do not rewrite unrelated modules.', + mode: 'websocket', + requiredCapabilities: ['model', 'file-edit', 'shell'], + timeoutMs: 240_000, + verify: { + commands: [['bun', 'test']], + requiredFiles: ['src/checkout.ts'], + expectedFiles: ['src/checkout.ts'], + forbiddenFiles: ['src/checkout.test.ts', 'src/catalog.ts', 'src/user.ts', 'package.json'], + transcriptAssertions: ['"type":"message_complete"'], + }, + }, + { + id: 'permission-artifact', + title: 'Create a required project artifact', + description: 'A test expects a generated summary file. The Agent must create a new file and validate it without changing the test.', + fixture: 'scripts/quality-gate/baseline/fixtures/permission-artifact', + prompt: 'Run the tests, create the missing project artifact they require, and rerun the tests. Do not modify the tests or package.json.', + mode: 'websocket', + requiredCapabilities: ['model', 'file-edit', 'shell', 'permission'], + timeoutMs: 180_000, + verify: { + commands: [['bun', 'test']], + requiredFiles: ['notes/summary.md'], + expectedFiles: ['notes/summary.md'], + forbiddenFiles: ['src/artifact.test.ts', 'package.json'], + transcriptAssertions: ['"type":"message_complete"'], + }, + }, + { + id: 'cross-module-refactor', + title: 'Complete a cross-module parser refactor', + description: 'A config parser contract changed but the implementation and caller are stale. The Agent must update multiple modules coherently.', + fixture: 'scripts/quality-gate/baseline/fixtures/cross-module-refactor', + prompt: 'Update the config parsing flow so parseConfig returns a structured object with enabled and retries fields. Update the implementation and caller so the existing tests pass, then run the suite.', + mode: 'websocket', + requiredCapabilities: ['model', 'file-edit', 'shell'], + timeoutMs: 300_000, + verify: { + commands: [['bun', 'test']], + requiredFiles: ['src/config.ts', 'src/runner.ts'], + expectedFiles: ['src/config.ts', 'src/runner.ts', 'src/runner.test.ts'], + forbiddenFiles: ['package.json'], + transcriptAssertions: ['"type":"message_complete"'], }, }, ] diff --git a/scripts/quality-gate/baseline/execute.ts b/scripts/quality-gate/baseline/execute.ts index 020630e7..e8e18bf1 100644 --- a/scripts/quality-gate/baseline/execute.ts +++ b/scripts/quality-gate/baseline/execute.ts @@ -1,4 +1,4 @@ -import { appendFileSync, cpSync, mkdirSync, rmSync, writeFileSync } from 'node:fs' +import { appendFileSync, cpSync, mkdirSync, readdirSync, readFileSync, rmSync, statSync, writeFileSync } from 'node:fs' import { mkdtemp } from 'node:fs/promises' import { tmpdir } from 'node:os' import { join } from 'node:path' @@ -51,6 +51,90 @@ async function runCommand(command: string[], cwd: string) { return { exitCode, stdout, stderr } } +function listFiles(root: string, current = root): string[] { + const files: string[] = [] + for (const entry of readdirSync(current)) { + if (entry === 'node_modules' || entry === '.git') { + continue + } + + const fullPath = join(current, entry) + const stat = statSync(fullPath) + if (stat.isDirectory()) { + files.push(...listFiles(root, fullPath)) + continue + } + if (stat.isFile()) { + files.push(fullPath.slice(root.length + 1)) + } + } + return files.sort() +} + +function changedFiles(beforeDir: string, afterDir: string) { + const before = new Set(listFiles(beforeDir)) + const after = new Set(listFiles(afterDir)) + const changed = new Set() + + for (const file of before) { + if (!after.has(file)) { + changed.add(file) + continue + } + if (!readFileSync(join(beforeDir, file)).equals(readFileSync(join(afterDir, file)))) { + changed.add(file) + } + } + + for (const file of after) { + if (!before.has(file)) { + changed.add(file) + } + } + + return [...changed].sort() +} + +async function writeDiffPatch(beforeDir: string, afterDir: string, patchPath: string) { + const result = await runCommand(['git', 'diff', '--no-index', '--', beforeDir, afterDir], process.cwd()) + writeFileSync(patchPath, `${result.stdout}${result.stderr}`) +} + +function verifyChangedFiles(testCase: BaselineCase, changed: string[]) { + const expected = testCase.verify.expectedFiles + if (expected) { + const unexpected = changed.filter((file) => !expected.includes(file)) + if (unexpected.length > 0) { + throw new Error(`unexpected changed files: ${unexpected.join(', ')}`) + } + } + + const required = testCase.verify.requiredFiles ?? [] + const missing = required.filter((file) => !changed.includes(file)) + if (missing.length > 0) { + throw new Error(`required files were not changed: ${missing.join(', ')}`) + } + + const forbidden = testCase.verify.forbiddenFiles ?? [] + const forbiddenChanged = forbidden.filter((file) => changed.includes(file)) + if (forbiddenChanged.length > 0) { + throw new Error(`forbidden files changed: ${forbiddenChanged.join(', ')}`) + } +} + +function verifyTranscript(testCase: BaselineCase, transcriptPath: string) { + const assertions = testCase.verify.transcriptAssertions ?? [] + if (assertions.length === 0) { + return + } + + const transcript = readFileSync(transcriptPath, 'utf8') + const missing = assertions.filter((assertion) => !transcript.includes(assertion)) + if (missing.length > 0) { + throw new Error(`transcript assertions missing: ${missing.join(', ')}`) + } +} + async function pipeToFile(stream: ReadableStream | null, path: string) { if (!stream) return const reader = stream.getReader() @@ -154,12 +238,15 @@ export async function executeBaselineCase( const port = await getPort() const baseUrl = `http://127.0.0.1:${port}` const workRoot = await mkdtemp(join(tmpdir(), `quality-gate-${testCase.id}-`)) + const originalDir = join(workRoot, 'original') const projectDir = join(workRoot, 'project') + cpSync(join(rootDir, testCase.fixture), originalDir, { recursive: true }) cpSync(join(rootDir, testCase.fixture), projectDir, { recursive: true }) const serverLogPath = join(artifactDir, 'server.log') const transcriptPath = join(artifactDir, 'transcript.jsonl') const verificationPath = join(artifactDir, 'verification.log') + const diffPath = join(artifactDir, 'diff.patch') const server = Bun.spawn(['bun', 'run', 'src/server/index.ts', '--host', '127.0.0.1', '--port', String(port)], { cwd: rootDir, stdout: 'pipe', @@ -190,6 +277,11 @@ export async function executeBaselineCase( const messages = await runPromptOverWebSocket(baseUrl, session.sessionId, testCase.prompt, testCase.timeoutMs, target) writeFileSync(transcriptPath, messages.map((message) => JSON.stringify(message)).join('\n') + '\n') + verifyTranscript(testCase, transcriptPath) + + await writeDiffPatch(originalDir, projectDir, diffPath) + const changed = changedFiles(originalDir, projectDir) + verifyChangedFiles(testCase, changed) let verificationLog = '' for (const command of testCase.verify.commands) { diff --git a/scripts/quality-gate/baseline/fixtures/cross-module-refactor/package.json b/scripts/quality-gate/baseline/fixtures/cross-module-refactor/package.json new file mode 100644 index 00000000..7de3edee --- /dev/null +++ b/scripts/quality-gate/baseline/fixtures/cross-module-refactor/package.json @@ -0,0 +1,8 @@ +{ + "name": "quality-gate-cross-module-refactor-fixture", + "private": true, + "type": "module", + "scripts": { + "test": "bun test" + } +} diff --git a/scripts/quality-gate/baseline/fixtures/cross-module-refactor/src/config.ts b/scripts/quality-gate/baseline/fixtures/cross-module-refactor/src/config.ts new file mode 100644 index 00000000..b7c271ab --- /dev/null +++ b/scripts/quality-gate/baseline/fixtures/cross-module-refactor/src/config.ts @@ -0,0 +1,3 @@ +export function parseConfig(raw: string) { + return raw === 'enabled' +} diff --git a/scripts/quality-gate/baseline/fixtures/cross-module-refactor/src/runner.test.ts b/scripts/quality-gate/baseline/fixtures/cross-module-refactor/src/runner.test.ts new file mode 100644 index 00000000..dbf12a1e --- /dev/null +++ b/scripts/quality-gate/baseline/fixtures/cross-module-refactor/src/runner.test.ts @@ -0,0 +1,17 @@ +import { describe, expect, test } from 'bun:test' +import { parseConfig } from './config' +import { describeRun } from './runner' + +describe('config runner', () => { + test('parses structured config', () => { + expect(parseConfig('enabled retries=3')).toEqual({ enabled: true, retries: 3 }) + }) + + test('describes enabled retry runs', () => { + expect(describeRun('enabled retries=3')).toBe('enabled with 3 retries') + }) + + test('describes disabled runs', () => { + expect(describeRun('disabled retries=0')).toBe('disabled') + }) +}) diff --git a/scripts/quality-gate/baseline/fixtures/cross-module-refactor/src/runner.ts b/scripts/quality-gate/baseline/fixtures/cross-module-refactor/src/runner.ts new file mode 100644 index 00000000..9304d604 --- /dev/null +++ b/scripts/quality-gate/baseline/fixtures/cross-module-refactor/src/runner.ts @@ -0,0 +1,6 @@ +import { parseConfig } from './config' + +export function describeRun(rawConfig: string) { + const enabled = parseConfig(rawConfig) + return enabled ? 'enabled' : 'disabled' +} diff --git a/scripts/quality-gate/baseline/fixtures/failure-recovery/package.json b/scripts/quality-gate/baseline/fixtures/failure-recovery/package.json new file mode 100644 index 00000000..70977b30 --- /dev/null +++ b/scripts/quality-gate/baseline/fixtures/failure-recovery/package.json @@ -0,0 +1,8 @@ +{ + "name": "quality-gate-failure-recovery-fixture", + "private": true, + "type": "module", + "scripts": { + "test": "bun test" + } +} diff --git a/scripts/quality-gate/baseline/fixtures/failure-recovery/src/slug.test.ts b/scripts/quality-gate/baseline/fixtures/failure-recovery/src/slug.test.ts new file mode 100644 index 00000000..1360e711 --- /dev/null +++ b/scripts/quality-gate/baseline/fixtures/failure-recovery/src/slug.test.ts @@ -0,0 +1,16 @@ +import { describe, expect, test } from 'bun:test' +import { slugify } from './slug' + +describe('slugify', () => { + test('normalizes whitespace and punctuation', () => { + expect(slugify(' Hello, Coding Agent! ')).toBe('hello-coding-agent') + }) + + test('collapses repeated separators', () => { + expect(slugify('Ship---Stable Releases')).toBe('ship-stable-releases') + }) + + test('trims separators from the edges', () => { + expect(slugify('---Quality Gate---')).toBe('quality-gate') + }) +}) diff --git a/scripts/quality-gate/baseline/fixtures/failure-recovery/src/slug.ts b/scripts/quality-gate/baseline/fixtures/failure-recovery/src/slug.ts new file mode 100644 index 00000000..90ec1085 --- /dev/null +++ b/scripts/quality-gate/baseline/fixtures/failure-recovery/src/slug.ts @@ -0,0 +1,3 @@ +export function slugify(input: string) { + return input.toLowerCase().replaceAll(' ', '-') +} diff --git a/scripts/quality-gate/baseline/fixtures/multi-file-api/src/api.ts b/scripts/quality-gate/baseline/fixtures/multi-file-api/src/api.ts index 76df2226..2db3f4c5 100644 --- a/scripts/quality-gate/baseline/fixtures/multi-file-api/src/api.ts +++ b/scripts/quality-gate/baseline/fixtures/multi-file-api/src/api.ts @@ -3,6 +3,12 @@ export type User = { email: string } -export function formatUser(user: User) { - return user.name +export type UserDisplay = { + displayName: string +} + +export function formatUser(user: User) { + return { + displayName: user.name, + } satisfies UserDisplay } diff --git a/scripts/quality-gate/baseline/fixtures/multi-file-api/src/app.test.ts b/scripts/quality-gate/baseline/fixtures/multi-file-api/src/app.test.ts index df622a5c..14c247ca 100644 --- a/scripts/quality-gate/baseline/fixtures/multi-file-api/src/app.test.ts +++ b/scripts/quality-gate/baseline/fixtures/multi-file-api/src/app.test.ts @@ -1,8 +1,15 @@ import { describe, expect, test } from 'bun:test' +import { formatUser } from './api' import { renderUser } from './app' describe('renderUser', () => { test('renders name and email', () => { expect(renderUser({ name: 'Ada Lovelace', email: 'ada@example.com' })).toBe('User: Ada Lovelace ') }) + + test('uses the structured display API contract', () => { + expect(formatUser({ name: 'Ada Lovelace', email: 'ada@example.com' })).toEqual({ + label: 'Ada Lovelace ', + }) + }) }) diff --git a/scripts/quality-gate/baseline/fixtures/multi-file-api/src/app.ts b/scripts/quality-gate/baseline/fixtures/multi-file-api/src/app.ts index e045b09c..2c6780c0 100644 --- a/scripts/quality-gate/baseline/fixtures/multi-file-api/src/app.ts +++ b/scripts/quality-gate/baseline/fixtures/multi-file-api/src/app.ts @@ -1,5 +1,5 @@ import { formatUser, type User } from './api' export function renderUser(user: User) { - return `User: ${formatUser(user)}` + return `User: ${formatUser(user).displayName}` } diff --git a/scripts/quality-gate/baseline/fixtures/permission-artifact/package.json b/scripts/quality-gate/baseline/fixtures/permission-artifact/package.json new file mode 100644 index 00000000..40efa949 --- /dev/null +++ b/scripts/quality-gate/baseline/fixtures/permission-artifact/package.json @@ -0,0 +1,8 @@ +{ + "name": "quality-gate-permission-artifact-fixture", + "private": true, + "type": "module", + "scripts": { + "test": "bun test" + } +} diff --git a/scripts/quality-gate/baseline/fixtures/permission-artifact/src/artifact.test.ts b/scripts/quality-gate/baseline/fixtures/permission-artifact/src/artifact.test.ts new file mode 100644 index 00000000..a76f8608 --- /dev/null +++ b/scripts/quality-gate/baseline/fixtures/permission-artifact/src/artifact.test.ts @@ -0,0 +1,9 @@ +import { describe, expect, test } from 'bun:test' +import { existsSync, readFileSync } from 'node:fs' + +describe('release artifact', () => { + test('has the required summary marker', () => { + expect(existsSync('notes/summary.md')).toBe(true) + expect(readFileSync('notes/summary.md', 'utf8')).toContain('baseline-ready') + }) +}) diff --git a/scripts/quality-gate/baseline/fixtures/workspace-search-edit/package.json b/scripts/quality-gate/baseline/fixtures/workspace-search-edit/package.json new file mode 100644 index 00000000..a1c962e8 --- /dev/null +++ b/scripts/quality-gate/baseline/fixtures/workspace-search-edit/package.json @@ -0,0 +1,8 @@ +{ + "name": "quality-gate-workspace-search-edit-fixture", + "private": true, + "type": "module", + "scripts": { + "test": "bun test" + } +} diff --git a/scripts/quality-gate/baseline/fixtures/workspace-search-edit/src/catalog.ts b/scripts/quality-gate/baseline/fixtures/workspace-search-edit/src/catalog.ts new file mode 100644 index 00000000..03c43817 --- /dev/null +++ b/scripts/quality-gate/baseline/fixtures/workspace-search-edit/src/catalog.ts @@ -0,0 +1,9 @@ +export type CatalogItem = { + sku: string + priceCents: number +} + +export const catalog: CatalogItem[] = [ + { sku: 'agent-seat', priceCents: 1200 }, + { sku: 'baseline-pack', priceCents: 800 }, +] diff --git a/scripts/quality-gate/baseline/fixtures/workspace-search-edit/src/checkout.test.ts b/scripts/quality-gate/baseline/fixtures/workspace-search-edit/src/checkout.test.ts new file mode 100644 index 00000000..b329149a --- /dev/null +++ b/scripts/quality-gate/baseline/fixtures/workspace-search-edit/src/checkout.test.ts @@ -0,0 +1,10 @@ +import { describe, expect, test } from 'bun:test' +import { catalog } from './catalog' +import { calculateTotalCents } from './checkout' +import { testUser } from './user' + +describe('calculateTotalCents', () => { + test('applies percentage discounts to the subtotal', () => { + expect(calculateTotalCents(catalog, testUser)).toBe(1500) + }) +}) diff --git a/scripts/quality-gate/baseline/fixtures/workspace-search-edit/src/checkout.ts b/scripts/quality-gate/baseline/fixtures/workspace-search-edit/src/checkout.ts new file mode 100644 index 00000000..57c02a49 --- /dev/null +++ b/scripts/quality-gate/baseline/fixtures/workspace-search-edit/src/checkout.ts @@ -0,0 +1,7 @@ +import type { CatalogItem } from './catalog' +import type { User } from './user' + +export function calculateTotalCents(items: CatalogItem[], user: User) { + const subtotal = items.reduce((sum, item) => sum + item.priceCents, 0) + return subtotal - user.discountPercent +} diff --git a/scripts/quality-gate/baseline/fixtures/workspace-search-edit/src/user.ts b/scripts/quality-gate/baseline/fixtures/workspace-search-edit/src/user.ts new file mode 100644 index 00000000..ca7b062f --- /dev/null +++ b/scripts/quality-gate/baseline/fixtures/workspace-search-edit/src/user.ts @@ -0,0 +1,9 @@ +export type User = { + id: string + discountPercent: number +} + +export const testUser: User = { + id: 'user-1', + discountPercent: 25, +} diff --git a/scripts/quality-gate/types.ts b/scripts/quality-gate/types.ts index b1a09a38..e86368f4 100644 --- a/scripts/quality-gate/types.ts +++ b/scripts/quality-gate/types.ts @@ -25,6 +25,7 @@ export type BaselineCase = { timeoutMs: number verify: { commands: string[][] + requiredFiles?: string[] expectedFiles?: string[] forbiddenFiles?: string[] transcriptAssertions?: string[]