cc-haha/scripts/quality-gate/runner.test.ts
程序员阿江(Relakkes) f6511ab278 Protect release confidence with live agent baselines
The desktop product needs a repeatable local gate that can prove the core Coding Agent loop still works after changes, not only that unit tests pass. This adds a quality-gate runner with PR, baseline, and release modes, structured reports, explicit quarantine metadata, and fixture-based live baseline cases that can run across provider/model targets.

Constraint: Existing check:pr and CI policy behavior must remain usable while the stronger baseline grows around it
Constraint: Default PR gates must not require real model credentials or provider quota
Rejected: Build a standalone QA platform first | too heavy before the baseline task shape is proven
Rejected: Keep unstable server exclusions hardcoded in run-server-tests | hides quarantine policy from reports and future review
Confidence: medium
Scope-risk: moderate
Directive: Expand baseline cases by adding focused fixtures and verifiers; do not make normal PR checks depend on live providers
Tested: bun test scripts/quality-gate/*.test.ts scripts/quality-gate/baseline/*.test.ts
Tested: bun run check:server
Tested: bun run quality:gate --mode baseline --allow-live --provider-model 2944f963-ce75-45b7-bac1-6e4f57df0970:kimi-k2.6:volc-kimi-k2.6 --provider-model 9c78d3df-7fb5-44c7-8436-3a41c3a59231:MiniMax-M2.7-highspeed:minimax-m2.7
Not-tested: desktop UI browser smoke and native release mode in this commit
2026-05-02 13:00:55 +08:00

109 lines
3.8 KiB
TypeScript

import { describe, expect, test } from 'bun:test'
import { mkdtempSync, readFileSync, rmSync } from 'node:fs'
import { tmpdir } from 'node:os'
import { join } from 'node:path'
import { lanesForMode } from './modes'
import { renderMarkdownReport } from './reporter'
import { runQualityGate } from './runner'
import type { QualityGateReport } from './types'
describe('quality gate modes', () => {
test('pr mode includes existing path-aware PR checks', () => {
const lanes = lanesForMode('pr').map((lane) => lane.id)
expect(lanes).toContain('impact-report')
expect(lanes).toContain('pr-checks')
expect(lanes.some((lane) => lane.startsWith('baseline:'))).toBe(false)
})
test('baseline mode includes live baseline cases but not native checks', () => {
const lanes = lanesForMode('baseline').map((lane) => lane.id)
expect(lanes).toContain('baseline-catalog')
expect(lanes).toContain('baseline:failing-unit:current-runtime')
expect(lanes).toContain('baseline:multi-file-api:current-runtime')
expect(lanes).not.toContain('native-checks')
})
test('release mode composes PR, baseline, and native lanes', () => {
const lanes = lanesForMode('release').map((lane) => lane.id)
expect(lanes).toContain('pr-checks')
expect(lanes).toContain('baseline:failing-unit:current-runtime')
expect(lanes).toContain('native-checks')
})
test('baseline mode expands cases across explicit provider/model targets', () => {
const lanes = lanesForMode('baseline', [
{ providerId: 'provider-a', modelId: 'model-a', label: 'provider-a-model-a' },
{ providerId: 'provider-b', modelId: 'model-b', label: 'provider-b-model-b' },
]).map((lane) => lane.id)
expect(lanes).toContain('baseline:failing-unit:provider-a-model-a')
expect(lanes).toContain('baseline:failing-unit:provider-b-model-b')
expect(lanes).toContain('baseline:multi-file-api:provider-a-model-a')
expect(lanes).toContain('baseline:multi-file-api:provider-b-model-b')
})
})
describe('runQualityGate', () => {
test('writes dry-run reports without executing expensive commands', async () => {
const artifactsDir = mkdtempSync(join(tmpdir(), 'quality-gate-test-'))
try {
const { report, outputDir } = await runQualityGate({
mode: 'baseline',
dryRun: true,
allowLive: false,
baselineTargets: [],
rootDir: process.cwd(),
artifactsDir,
runId: 'dry-run-test',
})
expect(report.mode).toBe('baseline')
expect(report.summary.failed).toBe(0)
expect(report.summary.skipped).toBeGreaterThan(0)
expect(readFileSync(join(outputDir, 'report.json'), 'utf8')).toContain('"mode": "baseline"')
expect(readFileSync(join(outputDir, 'report.md'), 'utf8')).toContain('# Quality Gate Report')
} finally {
rmSync(artifactsDir, { recursive: true, force: true })
}
})
})
describe('renderMarkdownReport', () => {
test('renders command, skip reason, and summary', () => {
const report: QualityGateReport = {
schemaVersion: 1,
runId: 'example',
mode: 'pr',
dryRun: true,
allowLive: false,
startedAt: '2026-05-02T00:00:00.000Z',
finishedAt: '2026-05-02T00:00:01.000Z',
rootDir: process.cwd(),
git: {
sha: 'abc123',
dirty: true,
},
results: [
{
id: 'impact-report',
title: 'Impact report',
status: 'skipped',
command: ['bun', 'run', 'check:impact'],
durationMs: 1,
skipReason: 'dry run',
},
],
summary: {
passed: 0,
failed: 0,
skipped: 1,
},
}
const markdown = renderMarkdownReport(report)
expect(markdown).toContain('Skipped: 1')
expect(markdown).toContain('`bun run check:impact`')
expect(markdown).toContain('dry run')
})
})