mirror of
https://github.com/NanmiCoder/cc-haha
synced 2026-07-17 13:13:35 +08:00
f6511ab278
The desktop product needs a repeatable local gate that can prove the core Coding Agent loop still works after changes, not only that unit tests pass. This adds a quality-gate runner with PR, baseline, and release modes, structured reports, explicit quarantine metadata, and fixture-based live baseline cases that can run across provider/model targets. Constraint: Existing check:pr and CI policy behavior must remain usable while the stronger baseline grows around it Constraint: Default PR gates must not require real model credentials or provider quota Rejected: Build a standalone QA platform first | too heavy before the baseline task shape is proven Rejected: Keep unstable server exclusions hardcoded in run-server-tests | hides quarantine policy from reports and future review Confidence: medium Scope-risk: moderate Directive: Expand baseline cases by adding focused fixtures and verifiers; do not make normal PR checks depend on live providers Tested: bun test scripts/quality-gate/*.test.ts scripts/quality-gate/baseline/*.test.ts Tested: bun run check:server Tested: bun run quality:gate --mode baseline --allow-live --provider-model 2944f963-ce75-45b7-bac1-6e4f57df0970:kimi-k2.6:volc-kimi-k2.6 --provider-model 9c78d3df-7fb5-44c7-8436-3a41c3a59231:MiniMax-M2.7-highspeed:minimax-m2.7 Not-tested: desktop UI browser smoke and native release mode in this commit
58 lines
2.3 KiB
TypeScript
58 lines
2.3 KiB
TypeScript
import type { BaselineCase } from '../types'
|
|
|
|
export const baselineCases: BaselineCase[] = [
|
|
{
|
|
id: 'failing-unit',
|
|
title: 'Fix a failing unit test',
|
|
description: 'A tiny TypeScript project has a broken arithmetic function. The Agent must inspect the failing test, patch the implementation, and rerun the test.',
|
|
fixture: 'scripts/quality-gate/baseline/fixtures/failing-unit',
|
|
prompt: 'Run the tests, inspect the failing assertion, fix the implementation bug, and rerun the tests until they pass. Only modify the fixture source files needed for the fix.',
|
|
mode: 'websocket',
|
|
requiredCapabilities: ['model', 'file-edit', 'shell'],
|
|
timeoutMs: 180_000,
|
|
verify: {
|
|
commands: [['bun', 'test']],
|
|
expectedFiles: ['src/math.ts'],
|
|
forbiddenFiles: ['package.json'],
|
|
transcriptAssertions: ['bun test', 'src/math.ts'],
|
|
},
|
|
},
|
|
{
|
|
id: 'multi-file-api',
|
|
title: 'Update a multi-file API contract',
|
|
description: 'A small app exposes a user display API. The Agent must change the contract and update callers plus tests coherently.',
|
|
fixture: 'scripts/quality-gate/baseline/fixtures/multi-file-api',
|
|
prompt: 'Change the user display contract so it returns "Ada Lovelace <ada@example.com>" instead of only the name. Update the implementation, caller, and tests, then run the tests.',
|
|
mode: 'websocket',
|
|
requiredCapabilities: ['model', 'file-edit', 'shell'],
|
|
timeoutMs: 240_000,
|
|
verify: {
|
|
commands: [['bun', 'test']],
|
|
expectedFiles: ['src/api.ts', 'src/app.ts', 'src/app.test.ts'],
|
|
forbiddenFiles: ['package.json'],
|
|
transcriptAssertions: ['bun test', 'src/api.ts', 'src/app.ts'],
|
|
},
|
|
},
|
|
]
|
|
|
|
export function validateBaselineCases(cases = baselineCases) {
|
|
const ids = new Set<string>()
|
|
|
|
for (const testCase of cases) {
|
|
if (ids.has(testCase.id)) {
|
|
throw new Error(`Duplicate baseline case id: ${testCase.id}`)
|
|
}
|
|
ids.add(testCase.id)
|
|
|
|
if (!testCase.fixture) {
|
|
throw new Error(`Baseline case ${testCase.id} is missing a fixture`)
|
|
}
|
|
if (testCase.verify.commands.length === 0) {
|
|
throw new Error(`Baseline case ${testCase.id} is missing verification commands`)
|
|
}
|
|
if (testCase.timeoutMs < 30_000) {
|
|
throw new Error(`Baseline case ${testCase.id} timeout is too low`)
|
|
}
|
|
}
|
|
}
|