mirror of
https://github.com/NanmiCoder/cc-haha
synced 2026-07-16 13:03:31 +08:00
d1219b9682
The first quality gate proved the execution path, but two tiny fixtures were not enough to reveal regressions in real agent behavior. This expands the maintained baseline with deeper failure recovery, workspace search, artifact creation, and cross-module refactor cases, and records per-case diffs so reviewers can see exactly what each model changed. Constraint: Baseline runs must remain explicit maintainer-controlled live checks, not default PR checks. Rejected: Require models to edit tests in every contract-change case | correct fixes can satisfy prewritten acceptance tests without modifying tests. Rejected: Trust only command exit codes | passing tests alone does not show whether the agent edited forbidden or unrelated files. Confidence: high Scope-risk: moderate Directive: Keep live baseline cases product-shaped and inspect both verification output and diff.patch before accepting future fixture changes. Tested: bun test scripts/quality-gate/*.test.ts scripts/quality-gate/baseline/*.test.ts Tested: bun run quality:gate --mode baseline --dry-run --provider-model volc-kimi-k2.6 --provider-model minimax-m2.7 Tested: bun run quality:gate --mode baseline --allow-live --provider-model volc-kimi-k2.6 --provider-model minimax-m2.7 (14 passed, 0 failed) Not-tested: Desktop UI browser smoke and native Tauri release packaging remain outside this baseline expansion.
86 lines
1.7 KiB
TypeScript
86 lines
1.7 KiB
TypeScript
export type QualityGateMode = 'pr' | 'baseline' | 'release'
|
|
|
|
export type LaneKind = 'command' | 'baseline-case'
|
|
|
|
export type LaneDefinition = {
|
|
id: string
|
|
title: string
|
|
description: string
|
|
kind: LaneKind
|
|
command?: string[]
|
|
baselineCaseId?: string
|
|
baselineTarget?: BaselineTarget
|
|
requiredForModes: QualityGateMode[]
|
|
live?: boolean
|
|
}
|
|
|
|
export type BaselineCase = {
|
|
id: string
|
|
title: string
|
|
description: string
|
|
fixture: string
|
|
prompt: string
|
|
mode: 'ui' | 'websocket'
|
|
requiredCapabilities: Array<'model' | 'file-edit' | 'shell' | 'permission' | 'browser'>
|
|
timeoutMs: number
|
|
verify: {
|
|
commands: string[][]
|
|
requiredFiles?: string[]
|
|
expectedFiles?: string[]
|
|
forbiddenFiles?: string[]
|
|
transcriptAssertions?: string[]
|
|
}
|
|
}
|
|
|
|
export type BaselineTarget = {
|
|
providerId: string | null
|
|
modelId: string
|
|
label: string
|
|
}
|
|
|
|
export type LaneStatus = 'passed' | 'failed' | 'skipped'
|
|
|
|
export type LaneResult = {
|
|
id: string
|
|
title: string
|
|
status: LaneStatus
|
|
command?: string[]
|
|
durationMs: number
|
|
exitCode?: number
|
|
skipReason?: string
|
|
error?: string
|
|
artifactDir?: string
|
|
}
|
|
|
|
export type QualityGateOptions = {
|
|
mode: QualityGateMode
|
|
dryRun: boolean
|
|
allowLive: boolean
|
|
baselineTargets: BaselineTarget[]
|
|
rootDir: string
|
|
artifactsDir?: string
|
|
runOutputDir?: string
|
|
runId?: string
|
|
}
|
|
|
|
export type QualityGateReport = {
|
|
schemaVersion: 1
|
|
runId: string
|
|
mode: QualityGateMode
|
|
dryRun: boolean
|
|
allowLive: boolean
|
|
startedAt: string
|
|
finishedAt: string
|
|
rootDir: string
|
|
git: {
|
|
sha: string | null
|
|
dirty: boolean
|
|
}
|
|
results: LaneResult[]
|
|
summary: {
|
|
passed: number
|
|
failed: number
|
|
skipped: number
|
|
}
|
|
}
|