mirror of
https://github.com/NanmiCoder/cc-haha
synced 2026-07-16 13:03:31 +08:00
5ed9903d2e
The live baseline covered the server WebSocket path, but it still did not prove the desktop app can open a session, apply a selected provider/model, send a chat turn, and surface model/tool progress through the UI. This adds an agent-browser driven smoke lane that starts the local server and Vite desktop app, restores an isolated session tab with the requested runtime selection, submits a small coding task through the composer, and accepts the run only when the fixture diff and tests pass. Constraint: Desktop smoke must stay behind --allow-live because it launches browsers and real models. Constraint: The smoke temporarily enables bypassPermissions for the isolated run and restores the previous mode afterward. Rejected: Wait for a final marker phrase | model reasoning and echoed prompts can contain the same marker before the work is actually done. Rejected: Use only DOM text as success proof | the browser can show progress while the project files are still unchanged. Confidence: high Scope-risk: moderate Directive: Future desktop smoke cases should verify project state and artifacts, not only UI copy. Tested: bun test scripts/quality-gate/*.test.ts scripts/quality-gate/baseline/*.test.ts Tested: bun run quality:gate --mode baseline --dry-run --provider-model minimax-m2.7 Tested: bun run quality:gate --mode baseline --allow-live --provider-model minimax-m2.7 (9 passed, 0 failed) Tested: bun run check:server Not-tested: Kimi desktop smoke completion because the provider returned AccountQuotaExceeded / API Error 429 until its reset window.
113 lines
4.1 KiB
TypeScript
113 lines
4.1 KiB
TypeScript
import { describe, expect, test } from 'bun:test'
|
|
import { mkdtempSync, readFileSync, rmSync } from 'node:fs'
|
|
import { tmpdir } from 'node:os'
|
|
import { join } from 'node:path'
|
|
import { lanesForMode } from './modes'
|
|
import { renderMarkdownReport } from './reporter'
|
|
import { runQualityGate } from './runner'
|
|
import type { QualityGateReport } from './types'
|
|
|
|
describe('quality gate modes', () => {
|
|
test('pr mode includes existing path-aware PR checks', () => {
|
|
const lanes = lanesForMode('pr').map((lane) => lane.id)
|
|
expect(lanes).toContain('impact-report')
|
|
expect(lanes).toContain('pr-checks')
|
|
expect(lanes.some((lane) => lane.startsWith('baseline:'))).toBe(false)
|
|
})
|
|
|
|
test('baseline mode includes live baseline cases but not native checks', () => {
|
|
const lanes = lanesForMode('baseline').map((lane) => lane.id)
|
|
expect(lanes).toContain('baseline-catalog')
|
|
expect(lanes).toContain('baseline:failing-unit:current-runtime')
|
|
expect(lanes).toContain('baseline:multi-file-api:current-runtime')
|
|
expect(lanes).toContain('desktop-smoke:agent-browser-chat:current-runtime')
|
|
expect(lanes).not.toContain('native-checks')
|
|
})
|
|
|
|
test('release mode composes PR, baseline, and native lanes', () => {
|
|
const lanes = lanesForMode('release').map((lane) => lane.id)
|
|
expect(lanes).toContain('pr-checks')
|
|
expect(lanes).toContain('baseline:failing-unit:current-runtime')
|
|
expect(lanes).toContain('desktop-smoke:agent-browser-chat:current-runtime')
|
|
expect(lanes).toContain('native-checks')
|
|
})
|
|
|
|
test('baseline mode expands cases across explicit provider/model targets', () => {
|
|
const lanes = lanesForMode('baseline', [
|
|
{ providerId: 'provider-a', modelId: 'model-a', label: 'provider-a-model-a' },
|
|
{ providerId: 'provider-b', modelId: 'model-b', label: 'provider-b-model-b' },
|
|
]).map((lane) => lane.id)
|
|
|
|
expect(lanes).toContain('baseline:failing-unit:provider-a-model-a')
|
|
expect(lanes).toContain('baseline:failing-unit:provider-b-model-b')
|
|
expect(lanes).toContain('baseline:multi-file-api:provider-a-model-a')
|
|
expect(lanes).toContain('baseline:multi-file-api:provider-b-model-b')
|
|
expect(lanes).toContain('desktop-smoke:agent-browser-chat:provider-a-model-a')
|
|
expect(lanes).toContain('desktop-smoke:agent-browser-chat:provider-b-model-b')
|
|
})
|
|
})
|
|
|
|
describe('runQualityGate', () => {
|
|
test('writes dry-run reports without executing expensive commands', async () => {
|
|
const artifactsDir = mkdtempSync(join(tmpdir(), 'quality-gate-test-'))
|
|
try {
|
|
const { report, outputDir } = await runQualityGate({
|
|
mode: 'baseline',
|
|
dryRun: true,
|
|
allowLive: false,
|
|
baselineTargets: [],
|
|
rootDir: process.cwd(),
|
|
artifactsDir,
|
|
runId: 'dry-run-test',
|
|
})
|
|
|
|
expect(report.mode).toBe('baseline')
|
|
expect(report.summary.failed).toBe(0)
|
|
expect(report.summary.skipped).toBeGreaterThan(0)
|
|
expect(readFileSync(join(outputDir, 'report.json'), 'utf8')).toContain('"mode": "baseline"')
|
|
expect(readFileSync(join(outputDir, 'report.md'), 'utf8')).toContain('# Quality Gate Report')
|
|
} finally {
|
|
rmSync(artifactsDir, { recursive: true, force: true })
|
|
}
|
|
})
|
|
})
|
|
|
|
describe('renderMarkdownReport', () => {
|
|
test('renders command, skip reason, and summary', () => {
|
|
const report: QualityGateReport = {
|
|
schemaVersion: 1,
|
|
runId: 'example',
|
|
mode: 'pr',
|
|
dryRun: true,
|
|
allowLive: false,
|
|
startedAt: '2026-05-02T00:00:00.000Z',
|
|
finishedAt: '2026-05-02T00:00:01.000Z',
|
|
rootDir: process.cwd(),
|
|
git: {
|
|
sha: 'abc123',
|
|
dirty: true,
|
|
},
|
|
results: [
|
|
{
|
|
id: 'impact-report',
|
|
title: 'Impact report',
|
|
status: 'skipped',
|
|
command: ['bun', 'run', 'check:impact'],
|
|
durationMs: 1,
|
|
skipReason: 'dry run',
|
|
},
|
|
],
|
|
summary: {
|
|
passed: 0,
|
|
failed: 0,
|
|
skipped: 1,
|
|
},
|
|
}
|
|
|
|
const markdown = renderMarkdownReport(report)
|
|
expect(markdown).toContain('Skipped: 1')
|
|
expect(markdown).toContain('`bun run check:impact`')
|
|
expect(markdown).toContain('dry run')
|
|
})
|
|
})
|