mirror of
https://github.com/NanmiCoder/cc-haha
synced 2026-07-16 13:03:31 +08:00
The live baseline covered the server WebSocket path, but it still did not prove the desktop app can open a session, apply a selected provider/model, send a chat turn, and surface model/tool progress through the UI. This adds an agent-browser driven smoke lane that starts the local server and Vite desktop app, restores an isolated session tab with the requested runtime selection, submits a small coding task through the composer, and accepts the run only when the fixture diff and tests pass. Constraint: Desktop smoke must stay behind --allow-live because it launches browsers and real models. Constraint: The smoke temporarily enables bypassPermissions for the isolated run and restores the previous mode afterward. Rejected: Wait for a final marker phrase | model reasoning and echoed prompts can contain the same marker before the work is actually done. Rejected: Use only DOM text as success proof | the browser can show progress while the project files are still unchanged. Confidence: high Scope-risk: moderate Directive: Future desktop smoke cases should verify project state and artifacts, not only UI copy. Tested: bun test scripts/quality-gate/*.test.ts scripts/quality-gate/baseline/*.test.ts Tested: bun run quality:gate --mode baseline --dry-run --provider-model minimax-m2.7 Tested: bun run quality:gate --mode baseline --allow-live --provider-model minimax-m2.7 (9 passed, 0 failed) Tested: bun run check:server Not-tested: Kimi desktop smoke completion because the provider returned AccountQuotaExceeded / API Error 429 until its reset window.
176 lines
4.9 KiB
TypeScript
176 lines
4.9 KiB
TypeScript
import { mkdirSync } from 'node:fs'
|
|
import { join } from 'node:path'
|
|
import { baselineCases } from './baseline/cases'
|
|
import { executeBaselineCase } from './baseline/execute'
|
|
import { executeDesktopSmoke } from './desktop-smoke/execute'
|
|
import { lanesForMode } from './modes'
|
|
import { writeReport } from './reporter'
|
|
import type { LaneDefinition, LaneResult, QualityGateOptions, QualityGateReport } from './types'
|
|
|
|
function nowId() {
|
|
return new Date().toISOString().replace(/[:.]/g, '-')
|
|
}
|
|
|
|
async function output(cmd: string[], cwd: string) {
|
|
const proc = Bun.spawn(cmd, {
|
|
cwd,
|
|
stdout: 'pipe',
|
|
stderr: 'pipe',
|
|
})
|
|
const stdout = await new Response(proc.stdout).text()
|
|
const stderr = await new Response(proc.stderr).text()
|
|
const code = await proc.exited
|
|
if (code !== 0) {
|
|
return null
|
|
}
|
|
return (stdout || stderr).trim()
|
|
}
|
|
|
|
async function gitInfo(rootDir: string) {
|
|
const sha = await output(['git', 'rev-parse', '--short', 'HEAD'], rootDir)
|
|
const status = await output(['git', 'status', '--short'], rootDir)
|
|
return {
|
|
sha,
|
|
dirty: Boolean(status),
|
|
}
|
|
}
|
|
|
|
async function runCommandLane(lane: LaneDefinition, options: QualityGateOptions): Promise<LaneResult> {
|
|
const started = Date.now()
|
|
const command = lane.command ?? []
|
|
|
|
if (options.dryRun) {
|
|
return {
|
|
id: lane.id,
|
|
title: lane.title,
|
|
status: 'skipped',
|
|
command,
|
|
durationMs: Date.now() - started,
|
|
skipReason: 'dry run',
|
|
}
|
|
}
|
|
|
|
const proc = Bun.spawn(command, {
|
|
cwd: options.rootDir,
|
|
stdout: 'inherit',
|
|
stderr: 'inherit',
|
|
})
|
|
const exitCode = await proc.exited
|
|
|
|
return {
|
|
id: lane.id,
|
|
title: lane.title,
|
|
status: exitCode === 0 ? 'passed' : 'failed',
|
|
command,
|
|
durationMs: Date.now() - started,
|
|
exitCode,
|
|
}
|
|
}
|
|
|
|
async function runBaselineCaseLane(lane: LaneDefinition, options: QualityGateOptions): Promise<LaneResult> {
|
|
const started = Date.now()
|
|
|
|
if (!options.allowLive) {
|
|
return {
|
|
id: lane.id,
|
|
title: lane.title,
|
|
status: 'skipped',
|
|
durationMs: Date.now() - started,
|
|
skipReason: 'live baseline cases require --allow-live',
|
|
}
|
|
}
|
|
|
|
const caseId = lane.baselineCaseId ?? lane.id.replace(/^baseline:/, '').split(':')[0]
|
|
const testCase = baselineCases.find((candidate) => candidate.id === caseId)
|
|
if (!testCase) {
|
|
return {
|
|
id: lane.id,
|
|
title: lane.title,
|
|
status: 'failed',
|
|
durationMs: Date.now() - started,
|
|
error: `Unknown baseline case: ${caseId}`,
|
|
}
|
|
}
|
|
|
|
const artifactRoot = options.runOutputDir ?? join(options.rootDir, 'artifacts', 'quality-runs', options.runId ?? 'current')
|
|
return executeBaselineCase(
|
|
testCase,
|
|
options.rootDir,
|
|
join(artifactRoot, 'cases', lane.id.replace(/[^a-zA-Z0-9._-]+/g, '-')),
|
|
lane.baselineTarget,
|
|
)
|
|
}
|
|
|
|
async function runLane(lane: LaneDefinition, options: QualityGateOptions): Promise<LaneResult> {
|
|
if (lane.kind === 'baseline-case') {
|
|
return runBaselineCaseLane(lane, options)
|
|
}
|
|
if (lane.kind === 'desktop-smoke') {
|
|
const started = Date.now()
|
|
|
|
if (!options.allowLive) {
|
|
return {
|
|
id: lane.id,
|
|
title: lane.title,
|
|
status: 'skipped',
|
|
durationMs: Date.now() - started,
|
|
skipReason: 'desktop agent-browser smoke requires --allow-live',
|
|
}
|
|
}
|
|
|
|
const artifactRoot = options.runOutputDir ?? join(options.rootDir, 'artifacts', 'quality-runs', options.runId ?? 'current')
|
|
return executeDesktopSmoke(
|
|
options.rootDir,
|
|
join(artifactRoot, 'cases', lane.id.replace(/[^a-zA-Z0-9._-]+/g, '-')),
|
|
lane.id,
|
|
lane.title,
|
|
lane.baselineTarget,
|
|
)
|
|
}
|
|
|
|
return runCommandLane(lane, options)
|
|
}
|
|
|
|
function summarize(results: LaneResult[]) {
|
|
return {
|
|
passed: results.filter((result) => result.status === 'passed').length,
|
|
failed: results.filter((result) => result.status === 'failed').length,
|
|
skipped: results.filter((result) => result.status === 'skipped').length,
|
|
}
|
|
}
|
|
|
|
export async function runQualityGate(options: QualityGateOptions) {
|
|
const runId = options.runId ?? nowId()
|
|
const startedAt = new Date().toISOString()
|
|
const artifactsRoot = options.artifactsDir ?? join(options.rootDir, 'artifacts', 'quality-runs')
|
|
const outputDir = join(artifactsRoot, runId)
|
|
mkdirSync(outputDir, { recursive: true })
|
|
|
|
const runOptions = { ...options, runId, runOutputDir: outputDir }
|
|
const results: LaneResult[] = []
|
|
for (const lane of lanesForMode(options.mode, options.baselineTargets)) {
|
|
const result = await runLane(lane, runOptions)
|
|
results.push(result)
|
|
if (result.status === 'failed') {
|
|
break
|
|
}
|
|
}
|
|
|
|
const report: QualityGateReport = {
|
|
schemaVersion: 1,
|
|
runId,
|
|
mode: options.mode,
|
|
dryRun: options.dryRun,
|
|
allowLive: options.allowLive,
|
|
startedAt,
|
|
finishedAt: new Date().toISOString(),
|
|
rootDir: options.rootDir,
|
|
git: await gitInfo(options.rootDir),
|
|
results,
|
|
summary: summarize(results),
|
|
}
|
|
|
|
writeReport(report, outputDir)
|
|
return { report, outputDir }
|
|
}
|