mirror of
https://github.com/NanmiCoder/cc-haha
synced 2026-07-18 13:23:33 +08:00
f6511ab278
The desktop product needs a repeatable local gate that can prove the core Coding Agent loop still works after changes, not only that unit tests pass. This adds a quality-gate runner with PR, baseline, and release modes, structured reports, explicit quarantine metadata, and fixture-based live baseline cases that can run across provider/model targets. Constraint: Existing check:pr and CI policy behavior must remain usable while the stronger baseline grows around it Constraint: Default PR gates must not require real model credentials or provider quota Rejected: Build a standalone QA platform first | too heavy before the baseline task shape is proven Rejected: Keep unstable server exclusions hardcoded in run-server-tests | hides quarantine policy from reports and future review Confidence: medium Scope-risk: moderate Directive: Expand baseline cases by adding focused fixtures and verifiers; do not make normal PR checks depend on live providers Tested: bun test scripts/quality-gate/*.test.ts scripts/quality-gate/baseline/*.test.ts Tested: bun run check:server Tested: bun run quality:gate --mode baseline --allow-live --provider-model 2944f963-ce75-45b7-bac1-6e4f57df0970:kimi-k2.6:volc-kimi-k2.6 --provider-model 9c78d3df-7fb5-44c7-8436-3a41c3a59231:MiniMax-M2.7-highspeed:minimax-m2.7 Not-tested: desktop UI browser smoke and native release mode in this commit
85 lines
1.6 KiB
TypeScript
85 lines
1.6 KiB
TypeScript
export type QualityGateMode = 'pr' | 'baseline' | 'release'
|
|
|
|
export type LaneKind = 'command' | 'baseline-case'
|
|
|
|
export type LaneDefinition = {
|
|
id: string
|
|
title: string
|
|
description: string
|
|
kind: LaneKind
|
|
command?: string[]
|
|
baselineCaseId?: string
|
|
baselineTarget?: BaselineTarget
|
|
requiredForModes: QualityGateMode[]
|
|
live?: boolean
|
|
}
|
|
|
|
export type BaselineCase = {
|
|
id: string
|
|
title: string
|
|
description: string
|
|
fixture: string
|
|
prompt: string
|
|
mode: 'ui' | 'websocket'
|
|
requiredCapabilities: Array<'model' | 'file-edit' | 'shell' | 'permission' | 'browser'>
|
|
timeoutMs: number
|
|
verify: {
|
|
commands: string[][]
|
|
expectedFiles?: string[]
|
|
forbiddenFiles?: string[]
|
|
transcriptAssertions?: string[]
|
|
}
|
|
}
|
|
|
|
export type BaselineTarget = {
|
|
providerId: string | null
|
|
modelId: string
|
|
label: string
|
|
}
|
|
|
|
export type LaneStatus = 'passed' | 'failed' | 'skipped'
|
|
|
|
export type LaneResult = {
|
|
id: string
|
|
title: string
|
|
status: LaneStatus
|
|
command?: string[]
|
|
durationMs: number
|
|
exitCode?: number
|
|
skipReason?: string
|
|
error?: string
|
|
artifactDir?: string
|
|
}
|
|
|
|
export type QualityGateOptions = {
|
|
mode: QualityGateMode
|
|
dryRun: boolean
|
|
allowLive: boolean
|
|
baselineTargets: BaselineTarget[]
|
|
rootDir: string
|
|
artifactsDir?: string
|
|
runOutputDir?: string
|
|
runId?: string
|
|
}
|
|
|
|
export type QualityGateReport = {
|
|
schemaVersion: 1
|
|
runId: string
|
|
mode: QualityGateMode
|
|
dryRun: boolean
|
|
allowLive: boolean
|
|
startedAt: string
|
|
finishedAt: string
|
|
rootDir: string
|
|
git: {
|
|
sha: string | null
|
|
dirty: boolean
|
|
}
|
|
results: LaneResult[]
|
|
summary: {
|
|
passed: number
|
|
failed: number
|
|
skipped: number
|
|
}
|
|
}
|