From f6511ab2788b04442c9a84346d512b7c84b0e288 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=E7=A8=8B=E5=BA=8F=E5=91=98=E9=98=BF=E6=B1=9F=28Relakkes?= =?UTF-8?q?=29?= Date: Sat, 2 May 2026 13:00:55 +0800 Subject: [PATCH 1/8] Protect release confidence with live agent baselines The desktop product needs a repeatable local gate that can prove the core Coding Agent loop still works after changes, not only that unit tests pass. This adds a quality-gate runner with PR, baseline, and release modes, structured reports, explicit quarantine metadata, and fixture-based live baseline cases that can run across provider/model targets. Constraint: Existing check:pr and CI policy behavior must remain usable while the stronger baseline grows around it Constraint: Default PR gates must not require real model credentials or provider quota Rejected: Build a standalone QA platform first | too heavy before the baseline task shape is proven Rejected: Keep unstable server exclusions hardcoded in run-server-tests | hides quarantine policy from reports and future review Confidence: medium Scope-risk: moderate Directive: Expand baseline cases by adding focused fixtures and verifiers; do not make normal PR checks depend on live providers Tested: bun test scripts/quality-gate/*.test.ts scripts/quality-gate/baseline/*.test.ts Tested: bun run check:server Tested: bun run quality:gate --mode baseline --allow-live --provider-model 2944f963-ce75-45b7-bac1-6e4f57df0970:kimi-k2.6:volc-kimi-k2.6 --provider-model 9c78d3df-7fb5-44c7-8436-3a41c3a59231:MiniMax-M2.7-highspeed:minimax-m2.7 Not-tested: desktop UI browser smoke and native release mode in this commit --- .gitignore | 3 + package.json | 4 + scripts/pr/run-server-tests.ts | 11 +- scripts/quality-gate/baseline/cases.test.ts | 28 +++ scripts/quality-gate/baseline/cases.ts | 57 +++++ scripts/quality-gate/baseline/execute.ts | 235 ++++++++++++++++++ .../fixtures/failing-unit/package.json | 8 + .../fixtures/failing-unit/src/math.test.ts | 8 + .../fixtures/failing-unit/src/math.ts | 3 + .../fixtures/multi-file-api/package.json | 8 + .../fixtures/multi-file-api/src/api.ts | 8 + .../fixtures/multi-file-api/src/app.test.ts | 8 + .../fixtures/multi-file-api/src/app.ts | 5 + scripts/quality-gate/index.ts | 85 +++++++ scripts/quality-gate/modes.ts | 61 +++++ scripts/quality-gate/quarantine.json | 39 +++ scripts/quality-gate/quarantine.test.ts | 35 +++ scripts/quality-gate/quarantine.ts | 50 ++++ scripts/quality-gate/reporter.ts | 59 +++++ scripts/quality-gate/runner.test.ts | 108 ++++++++ scripts/quality-gate/runner.ts | 152 +++++++++++ scripts/quality-gate/types.ts | 84 +++++++ src/server/__tests__/conversations.test.ts | 7 + 23 files changed, 1057 insertions(+), 9 deletions(-) create mode 100644 scripts/quality-gate/baseline/cases.test.ts create mode 100644 scripts/quality-gate/baseline/cases.ts create mode 100644 scripts/quality-gate/baseline/execute.ts create mode 100644 scripts/quality-gate/baseline/fixtures/failing-unit/package.json create mode 100644 scripts/quality-gate/baseline/fixtures/failing-unit/src/math.test.ts create mode 100644 scripts/quality-gate/baseline/fixtures/failing-unit/src/math.ts create mode 100644 scripts/quality-gate/baseline/fixtures/multi-file-api/package.json create mode 100644 scripts/quality-gate/baseline/fixtures/multi-file-api/src/api.ts create mode 100644 scripts/quality-gate/baseline/fixtures/multi-file-api/src/app.test.ts create mode 100644 scripts/quality-gate/baseline/fixtures/multi-file-api/src/app.ts create mode 100644 scripts/quality-gate/index.ts create mode 100644 scripts/quality-gate/modes.ts create mode 100644 scripts/quality-gate/quarantine.json create mode 100644 scripts/quality-gate/quarantine.test.ts create mode 100644 scripts/quality-gate/quarantine.ts create mode 100644 scripts/quality-gate/reporter.ts create mode 100644 scripts/quality-gate/runner.test.ts create mode 100644 scripts/quality-gate/runner.ts create mode 100644 scripts/quality-gate/types.ts diff --git a/.gitignore b/.gitignore index b9cccb16..e14c277e 100644 --- a/.gitignore +++ b/.gitignore @@ -22,6 +22,9 @@ extracted-natives/ # E2E screenshots e2e-*.png +# Quality gate reports +artifacts/quality-runs/ + # Desktop build output desktop/dist/ desktop/build-artifacts/ diff --git a/package.json b/package.json index 4a2b324d..cf8ce53d 100644 --- a/package.json +++ b/package.json @@ -17,6 +17,10 @@ "check:adapters": "cd adapters && bun test", "check:native": "cd desktop && bun run build:sidecars && cd src-tauri && cargo check", "check:docs": "npm ci && npm run docs:build", + "quality:gate": "bun run scripts/quality-gate/index.ts", + "quality:pr": "bun run quality:gate --mode pr", + "quality:baseline": "bun run quality:gate --mode baseline", + "quality:release": "bun run quality:gate --mode release", "docs:dev": "vitepress dev docs", "docs:build": "vitepress build docs", "docs:preview": "vitepress preview docs" diff --git a/scripts/pr/run-server-tests.ts b/scripts/pr/run-server-tests.ts index baadf80a..937b4baa 100644 --- a/scripts/pr/run-server-tests.ts +++ b/scripts/pr/run-server-tests.ts @@ -2,18 +2,11 @@ import { readdirSync, statSync } from 'node:fs' import { join, relative, sep } from 'node:path' +import { quarantinedPathSet } from '../quality-gate/quarantine' const root = process.cwd() const roots = ['src/server', 'src/tools', 'src/utils'] -const excludedFiles = new Set([ - // These suites are not stable enough for the default PR gate yet. Keep them - // out of CI until they are fixed or moved to a maintainer-only workflow. - 'src/server/__tests__/cron-scheduler.test.ts', - 'src/server/__tests__/providers-real.test.ts', - 'src/server/__tests__/tasks.test.ts', - 'src/server/__tests__/e2e/business-flow.test.ts', - 'src/server/__tests__/e2e/full-flow.test.ts', -]) +const excludedFiles = quarantinedPathSet() function normalize(path: string) { return relative(root, path).split(sep).join('/') diff --git a/scripts/quality-gate/baseline/cases.test.ts b/scripts/quality-gate/baseline/cases.test.ts new file mode 100644 index 00000000..c7b624a5 --- /dev/null +++ b/scripts/quality-gate/baseline/cases.test.ts @@ -0,0 +1,28 @@ +import { describe, expect, test } from 'bun:test' +import { existsSync } from 'node:fs' +import { join } from 'node:path' +import { baselineCases, validateBaselineCases } from './cases' + +describe('baselineCases', () => { + test('have valid metadata', () => { + expect(() => validateBaselineCases()).not.toThrow() + }) + + test('use unique ids', () => { + const ids = baselineCases.map((testCase) => testCase.id) + expect(new Set(ids).size).toBe(ids.length) + }) + + test('point at existing fixtures with package manifests', () => { + for (const testCase of baselineCases) { + expect(existsSync(testCase.fixture)).toBe(true) + expect(existsSync(join(testCase.fixture, 'package.json'))).toBe(true) + } + }) + + test('require real model capability', () => { + for (const testCase of baselineCases) { + expect(testCase.requiredCapabilities).toContain('model') + } + }) +}) diff --git a/scripts/quality-gate/baseline/cases.ts b/scripts/quality-gate/baseline/cases.ts new file mode 100644 index 00000000..4e308e2d --- /dev/null +++ b/scripts/quality-gate/baseline/cases.ts @@ -0,0 +1,57 @@ +import type { BaselineCase } from '../types' + +export const baselineCases: BaselineCase[] = [ + { + id: 'failing-unit', + title: 'Fix a failing unit test', + description: 'A tiny TypeScript project has a broken arithmetic function. The Agent must inspect the failing test, patch the implementation, and rerun the test.', + fixture: 'scripts/quality-gate/baseline/fixtures/failing-unit', + prompt: 'Run the tests, inspect the failing assertion, fix the implementation bug, and rerun the tests until they pass. Only modify the fixture source files needed for the fix.', + mode: 'websocket', + requiredCapabilities: ['model', 'file-edit', 'shell'], + timeoutMs: 180_000, + verify: { + commands: [['bun', 'test']], + expectedFiles: ['src/math.ts'], + forbiddenFiles: ['package.json'], + transcriptAssertions: ['bun test', 'src/math.ts'], + }, + }, + { + id: 'multi-file-api', + title: 'Update a multi-file API contract', + description: 'A small app exposes a user display API. The Agent must change the contract and update callers plus tests coherently.', + fixture: 'scripts/quality-gate/baseline/fixtures/multi-file-api', + prompt: 'Change the user display contract so it returns "Ada Lovelace " instead of only the name. Update the implementation, caller, and tests, then run the tests.', + mode: 'websocket', + requiredCapabilities: ['model', 'file-edit', 'shell'], + timeoutMs: 240_000, + verify: { + commands: [['bun', 'test']], + expectedFiles: ['src/api.ts', 'src/app.ts', 'src/app.test.ts'], + forbiddenFiles: ['package.json'], + transcriptAssertions: ['bun test', 'src/api.ts', 'src/app.ts'], + }, + }, +] + +export function validateBaselineCases(cases = baselineCases) { + const ids = new Set() + + for (const testCase of cases) { + if (ids.has(testCase.id)) { + throw new Error(`Duplicate baseline case id: ${testCase.id}`) + } + ids.add(testCase.id) + + if (!testCase.fixture) { + throw new Error(`Baseline case ${testCase.id} is missing a fixture`) + } + if (testCase.verify.commands.length === 0) { + throw new Error(`Baseline case ${testCase.id} is missing verification commands`) + } + if (testCase.timeoutMs < 30_000) { + throw new Error(`Baseline case ${testCase.id} timeout is too low`) + } + } +} diff --git a/scripts/quality-gate/baseline/execute.ts b/scripts/quality-gate/baseline/execute.ts new file mode 100644 index 00000000..020630e7 --- /dev/null +++ b/scripts/quality-gate/baseline/execute.ts @@ -0,0 +1,235 @@ +import { appendFileSync, cpSync, mkdirSync, rmSync, writeFileSync } from 'node:fs' +import { mkdtemp } from 'node:fs/promises' +import { tmpdir } from 'node:os' +import { join } from 'node:path' +import { createServer } from 'node:net' +import type { BaselineCase, BaselineTarget, LaneResult } from '../types' + +type ServerMessage = { + type: string + requestId?: string + message?: string + code?: string + [key: string]: unknown +} + +function getPort() { + return new Promise((resolve, reject) => { + const server = createServer() + server.once('error', reject) + server.listen(0, '127.0.0.1', () => { + const address = server.address() + const port = typeof address === 'object' && address ? address.port : 0 + server.close(() => resolve(port)) + }) + }) +} + +async function waitForHttp(url: string, timeoutMs: number) { + const deadline = Date.now() + timeoutMs + while (Date.now() < deadline) { + try { + const res = await fetch(url) + if (res.ok) return + } catch { + // Keep polling until timeout. + } + await Bun.sleep(500) + } + throw new Error(`Timed out waiting for ${url}`) +} + +async function runCommand(command: string[], cwd: string) { + const proc = Bun.spawn(command, { + cwd, + stdout: 'pipe', + stderr: 'pipe', + }) + const stdout = await new Response(proc.stdout).text() + const stderr = await new Response(proc.stderr).text() + const exitCode = await proc.exited + return { exitCode, stdout, stderr } +} + +async function pipeToFile(stream: ReadableStream | null, path: string) { + if (!stream) return + const reader = stream.getReader() + const decoder = new TextDecoder() + while (true) { + const { value, done } = await reader.read() + if (done) break + appendFileSync(path, decoder.decode(value, { stream: true })) + } +} + +function waitForWebSocketOpen(ws: WebSocket) { + return new Promise((resolve, reject) => { + ws.onopen = () => resolve() + ws.onerror = () => reject(new Error('WebSocket failed to open')) + }) +} + +async function runPromptOverWebSocket( + baseUrl: string, + sessionId: string, + prompt: string, + timeoutMs: number, + target?: BaselineTarget, +) { + const wsUrl = baseUrl.replace(/^http/, 'ws') + const ws = new WebSocket(`${wsUrl}/ws/${sessionId}`) + const messages: ServerMessage[] = [] + + try { + await waitForWebSocketOpen(ws) + + await new Promise((resolve, reject) => { + const timer = setTimeout(() => { + reject(new Error(`Timed out waiting for baseline case completion after ${timeoutMs}ms`)) + }, timeoutMs) + + ws.onmessage = (event) => { + const message = JSON.parse(String(event.data)) as ServerMessage + messages.push(message) + + if (message.type === 'connected') { + if (target && target.modelId !== 'current') { + ws.send(JSON.stringify({ + type: 'set_runtime_config', + providerId: target.providerId, + modelId: target.modelId, + })) + } + ws.send(JSON.stringify({ type: 'user_message', content: prompt })) + return + } + + if (message.type === 'permission_request' && typeof message.requestId === 'string') { + ws.send(JSON.stringify({ + type: 'permission_response', + requestId: message.requestId, + allowed: true, + rule: 'baseline-run', + })) + return + } + + if (message.type === 'message_complete') { + clearTimeout(timer) + resolve() + return + } + + if (message.type === 'error') { + clearTimeout(timer) + reject(new Error(`${message.code ?? 'WS_ERROR'}: ${message.message ?? 'unknown error'}`)) + } + } + + ws.onerror = () => { + clearTimeout(timer) + reject(new Error('WebSocket error during baseline case')) + } + }) + } finally { + ws.close() + } + + return messages +} + +export async function executeBaselineCase( + testCase: BaselineCase, + rootDir: string, + artifactDir: string, + target?: BaselineTarget, +): Promise { + const started = Date.now() + const resultId = target + ? `baseline:${testCase.id}:${target.label.replace(/[^a-zA-Z0-9._-]+/g, '-')}` + : `baseline:${testCase.id}` + const resultTitle = target ? `${testCase.title} (${target.label})` : testCase.title + mkdirSync(artifactDir, { recursive: true }) + + const port = await getPort() + const baseUrl = `http://127.0.0.1:${port}` + const workRoot = await mkdtemp(join(tmpdir(), `quality-gate-${testCase.id}-`)) + const projectDir = join(workRoot, 'project') + cpSync(join(rootDir, testCase.fixture), projectDir, { recursive: true }) + + const serverLogPath = join(artifactDir, 'server.log') + const transcriptPath = join(artifactDir, 'transcript.jsonl') + const verificationPath = join(artifactDir, 'verification.log') + const server = Bun.spawn(['bun', 'run', 'src/server/index.ts', '--host', '127.0.0.1', '--port', String(port)], { + cwd: rootDir, + stdout: 'pipe', + stderr: 'pipe', + env: { + ...process.env, + SERVER_PORT: String(port), + }, + }) + const stdoutPump = pipeToFile(server.stdout, serverLogPath) + const stderrPump = pipeToFile(server.stderr, serverLogPath) + + try { + await waitForHttp(`${baseUrl}/health`, 60_000) + + const createResponse = await fetch(`${baseUrl}/api/sessions`, { + method: 'POST', + headers: { 'Content-Type': 'application/json' }, + body: JSON.stringify({ workDir: projectDir }), + }) + if (!createResponse.ok) { + throw new Error(`Failed to create session: ${createResponse.status}`) + } + const session = await createResponse.json() as { sessionId?: string } + if (!session.sessionId) { + throw new Error('Session response did not include sessionId') + } + + const messages = await runPromptOverWebSocket(baseUrl, session.sessionId, testCase.prompt, testCase.timeoutMs, target) + writeFileSync(transcriptPath, messages.map((message) => JSON.stringify(message)).join('\n') + '\n') + + let verificationLog = '' + for (const command of testCase.verify.commands) { + const result = await runCommand(command, projectDir) + verificationLog += `$ ${command.join(' ')}\n${result.stdout}${result.stderr}\n` + if (result.exitCode !== 0) { + writeFileSync(verificationPath, verificationLog) + return { + id: resultId, + title: resultTitle, + status: 'failed', + durationMs: Date.now() - started, + exitCode: result.exitCode, + error: `verification command failed: ${command.join(' ')}`, + artifactDir, + } + } + } + writeFileSync(verificationPath, verificationLog) + + return { + id: resultId, + title: resultTitle, + status: 'passed', + durationMs: Date.now() - started, + artifactDir, + } + } catch (error) { + return { + id: resultId, + title: resultTitle, + status: 'failed', + durationMs: Date.now() - started, + error: error instanceof Error ? error.message : String(error), + artifactDir, + } + } finally { + server.kill() + await server.exited.catch(() => undefined) + await Promise.all([stdoutPump, stderrPump]).catch(() => undefined) + rmSync(workRoot, { recursive: true, force: true }) + } +} diff --git a/scripts/quality-gate/baseline/fixtures/failing-unit/package.json b/scripts/quality-gate/baseline/fixtures/failing-unit/package.json new file mode 100644 index 00000000..391c0814 --- /dev/null +++ b/scripts/quality-gate/baseline/fixtures/failing-unit/package.json @@ -0,0 +1,8 @@ +{ + "name": "quality-gate-failing-unit-fixture", + "private": true, + "type": "module", + "scripts": { + "test": "bun test" + } +} diff --git a/scripts/quality-gate/baseline/fixtures/failing-unit/src/math.test.ts b/scripts/quality-gate/baseline/fixtures/failing-unit/src/math.test.ts new file mode 100644 index 00000000..b6101f8f --- /dev/null +++ b/scripts/quality-gate/baseline/fixtures/failing-unit/src/math.test.ts @@ -0,0 +1,8 @@ +import { describe, expect, test } from 'bun:test' +import { add } from './math' + +describe('add', () => { + test('adds positive numbers', () => { + expect(add(2, 3)).toBe(5) + }) +}) diff --git a/scripts/quality-gate/baseline/fixtures/failing-unit/src/math.ts b/scripts/quality-gate/baseline/fixtures/failing-unit/src/math.ts new file mode 100644 index 00000000..7759c00b --- /dev/null +++ b/scripts/quality-gate/baseline/fixtures/failing-unit/src/math.ts @@ -0,0 +1,3 @@ +export function add(a: number, b: number) { + return a - b +} diff --git a/scripts/quality-gate/baseline/fixtures/multi-file-api/package.json b/scripts/quality-gate/baseline/fixtures/multi-file-api/package.json new file mode 100644 index 00000000..66d63e7d --- /dev/null +++ b/scripts/quality-gate/baseline/fixtures/multi-file-api/package.json @@ -0,0 +1,8 @@ +{ + "name": "quality-gate-multi-file-api-fixture", + "private": true, + "type": "module", + "scripts": { + "test": "bun test" + } +} diff --git a/scripts/quality-gate/baseline/fixtures/multi-file-api/src/api.ts b/scripts/quality-gate/baseline/fixtures/multi-file-api/src/api.ts new file mode 100644 index 00000000..76df2226 --- /dev/null +++ b/scripts/quality-gate/baseline/fixtures/multi-file-api/src/api.ts @@ -0,0 +1,8 @@ +export type User = { + name: string + email: string +} + +export function formatUser(user: User) { + return user.name +} diff --git a/scripts/quality-gate/baseline/fixtures/multi-file-api/src/app.test.ts b/scripts/quality-gate/baseline/fixtures/multi-file-api/src/app.test.ts new file mode 100644 index 00000000..df622a5c --- /dev/null +++ b/scripts/quality-gate/baseline/fixtures/multi-file-api/src/app.test.ts @@ -0,0 +1,8 @@ +import { describe, expect, test } from 'bun:test' +import { renderUser } from './app' + +describe('renderUser', () => { + test('renders name and email', () => { + expect(renderUser({ name: 'Ada Lovelace', email: 'ada@example.com' })).toBe('User: Ada Lovelace ') + }) +}) diff --git a/scripts/quality-gate/baseline/fixtures/multi-file-api/src/app.ts b/scripts/quality-gate/baseline/fixtures/multi-file-api/src/app.ts new file mode 100644 index 00000000..e045b09c --- /dev/null +++ b/scripts/quality-gate/baseline/fixtures/multi-file-api/src/app.ts @@ -0,0 +1,5 @@ +import { formatUser, type User } from './api' + +export function renderUser(user: User) { + return `User: ${formatUser(user)}` +} diff --git a/scripts/quality-gate/index.ts b/scripts/quality-gate/index.ts new file mode 100644 index 00000000..fceff9c0 --- /dev/null +++ b/scripts/quality-gate/index.ts @@ -0,0 +1,85 @@ +#!/usr/bin/env bun + +import { runQualityGate } from './runner' +import type { BaselineTarget, QualityGateMode } from './types' + +function parseArgs(argv: string[]) { + const args = new Map>() + + for (let index = 0; index < argv.length; index += 1) { + const arg = argv[index] + if (!arg.startsWith('--')) { + continue + } + + const next = argv[index + 1] + if (next && !next.startsWith('--')) { + args.set(arg, [...(args.get(arg) ?? []), next]) + index += 1 + } else { + args.set(arg, [...(args.get(arg) ?? []), true]) + } + } + + return args +} + +function firstArg(args: Map>, name: string) { + return args.get(name)?.[0] +} + +function hasFlag(args: Map>, name: string) { + return args.has(name) +} + +function readMode(value: string | boolean | undefined): QualityGateMode { + if (value === 'pr' || value === 'baseline' || value === 'release') { + return value + } + throw new Error('Usage: bun run quality:gate --mode [--dry-run] [--allow-live]') +} + +function readBaselineTargets(args: Map>): BaselineTarget[] { + const values = (args.get('--provider-model') ?? []) + .filter((value): value is string => typeof value === 'string') + .flatMap((value) => value.split(',')) + .map((value) => value.trim()) + .filter(Boolean) + + return values.map((value) => { + const [providerId, modelId, rawLabel] = value.split(':') + if (!providerId || !modelId) { + throw new Error(`Invalid --provider-model value "${value}". Expected providerId:modelId[:label].`) + } + return { + providerId, + modelId, + label: rawLabel || `${providerId.slice(0, 8)}-${modelId}`, + } + }) +} + +const args = parseArgs(process.argv.slice(2)) +const mode = readMode(firstArg(args, '--mode')) +const dryRun = hasFlag(args, '--dry-run') +const allowLive = hasFlag(args, '--allow-live') +const artifactsDir = typeof firstArg(args, '--artifacts-dir') === 'string' + ? String(firstArg(args, '--artifacts-dir')) + : undefined +const baselineTargets = readBaselineTargets(args) + +const { report, outputDir } = await runQualityGate({ + mode, + dryRun, + allowLive, + baselineTargets, + rootDir: process.cwd(), + artifactsDir, +}) + +console.log(`Quality gate report: ${outputDir}/report.md`) +console.log(`Summary: passed=${report.summary.passed} failed=${report.summary.failed} skipped=${report.summary.skipped}`) + +if (report.summary.failed > 0) { + process.exit(1) +} diff --git a/scripts/quality-gate/modes.ts b/scripts/quality-gate/modes.ts new file mode 100644 index 00000000..5d635550 --- /dev/null +++ b/scripts/quality-gate/modes.ts @@ -0,0 +1,61 @@ +import { baselineCases } from './baseline/cases' +import type { BaselineTarget, LaneDefinition, QualityGateMode } from './types' + +export function lanesForMode(mode: QualityGateMode, baselineTargets: BaselineTarget[] = []): LaneDefinition[] { + const lanes: LaneDefinition[] = [ + { + id: 'impact-report', + title: 'Impact report', + description: 'Summarize changed areas, required local checks, and risk notes.', + kind: 'command', + command: ['bun', 'run', 'check:impact'], + requiredForModes: ['pr', 'baseline', 'release'], + }, + { + id: 'pr-checks', + title: 'Path-aware PR checks', + description: 'Run the existing local PR gate with stable path-aware checks.', + kind: 'command', + command: ['bun', 'run', 'check:pr'], + requiredForModes: ['pr', 'release'], + }, + { + id: 'baseline-catalog', + title: 'Baseline case catalog validation', + description: 'Validate real Coding Agent baseline case definitions and fixture metadata.', + kind: 'command', + command: ['bun', 'test', 'scripts/quality-gate/baseline/cases.test.ts'], + requiredForModes: ['baseline', 'release'], + }, + { + id: 'native-checks', + title: 'Native desktop checks', + description: 'Build sidecars and run the Tauri native compile check.', + kind: 'command', + command: ['bun', 'run', 'check:native'], + requiredForModes: ['release'], + }, + ] + + const targets = baselineTargets.length > 0 + ? baselineTargets + : [{ providerId: null, modelId: 'current', label: 'current-runtime' }] + + for (const testCase of baselineCases) { + for (const target of targets) { + const targetSlug = target.label.replace(/[^a-zA-Z0-9._-]+/g, '-') + lanes.push({ + id: `baseline:${testCase.id}:${targetSlug}`, + title: `${testCase.title} (${target.label})`, + description: testCase.description, + kind: 'baseline-case', + baselineCaseId: testCase.id, + baselineTarget: target, + requiredForModes: ['baseline', 'release'], + live: true, + }) + } + } + + return lanes.filter((lane) => lane.requiredForModes.includes(mode)) +} diff --git a/scripts/quality-gate/quarantine.json b/scripts/quality-gate/quarantine.json new file mode 100644 index 00000000..efcbc693 --- /dev/null +++ b/scripts/quality-gate/quarantine.json @@ -0,0 +1,39 @@ +{ + "quarantined": [ + { + "id": "server:cron-scheduler", + "path": "src/server/__tests__/cron-scheduler.test.ts", + "reason": "Known unstable scheduler timing assertions; keep out of default server gate until fixed.", + "owner": "maintainers", + "reviewAfter": "2026-06-01" + }, + { + "id": "server:providers-real", + "path": "src/server/__tests__/providers-real.test.ts", + "reason": "Live provider coverage belongs in maintainer baseline or release mode, not the default PR gate.", + "owner": "maintainers", + "reviewAfter": "2026-06-01" + }, + { + "id": "server:tasks", + "path": "src/server/__tests__/tasks.test.ts", + "reason": "Known default-gate instability; keep quarantined until task timing is hardened.", + "owner": "maintainers", + "reviewAfter": "2026-06-01" + }, + { + "id": "server:e2e:business-flow", + "path": "src/server/__tests__/e2e/business-flow.test.ts", + "reason": "Broad e2e flow is not deterministic enough for default PR checks.", + "owner": "maintainers", + "reviewAfter": "2026-06-01" + }, + { + "id": "server:e2e:full-flow", + "path": "src/server/__tests__/e2e/full-flow.test.ts", + "reason": "Broad e2e flow is not deterministic enough for default PR checks.", + "owner": "maintainers", + "reviewAfter": "2026-06-01" + } + ] +} diff --git a/scripts/quality-gate/quarantine.test.ts b/scripts/quality-gate/quarantine.test.ts new file mode 100644 index 00000000..f0b60d38 --- /dev/null +++ b/scripts/quality-gate/quarantine.test.ts @@ -0,0 +1,35 @@ +import { describe, expect, test } from 'bun:test' +import { loadQuarantineManifest, quarantinedPathSet, validateQuarantineManifest } from './quarantine' + +describe('quarantine manifest', () => { + test('loads the default manifest', () => { + const manifest = loadQuarantineManifest() + expect(manifest.quarantined.length).toBeGreaterThan(0) + }) + + test('exposes quarantined paths as a set', () => { + const paths = quarantinedPathSet() + expect(paths.has('src/server/__tests__/providers-real.test.ts')).toBe(true) + }) + + test('rejects duplicate ids', () => { + expect(() => validateQuarantineManifest({ + quarantined: [ + { + id: 'duplicate', + path: 'a.test.ts', + reason: 'test', + owner: 'maintainers', + reviewAfter: '2026-06-01', + }, + { + id: 'duplicate', + path: 'b.test.ts', + reason: 'test', + owner: 'maintainers', + reviewAfter: '2026-06-01', + }, + ], + })).toThrow('duplicate quarantine id') + }) +}) diff --git a/scripts/quality-gate/quarantine.ts b/scripts/quality-gate/quarantine.ts new file mode 100644 index 00000000..ec771442 --- /dev/null +++ b/scripts/quality-gate/quarantine.ts @@ -0,0 +1,50 @@ +import { readFileSync } from 'node:fs' +import { dirname, join } from 'node:path' +import { fileURLToPath } from 'node:url' + +export type QuarantineEntry = { + id: string + path: string + reason: string + owner: string + reviewAfter: string +} + +export type QuarantineManifest = { + quarantined: QuarantineEntry[] +} + +const defaultManifestPath = join(dirname(fileURLToPath(import.meta.url)), 'quarantine.json') + +export function loadQuarantineManifest(path = defaultManifestPath): QuarantineManifest { + const manifest = JSON.parse(readFileSync(path, 'utf8')) as QuarantineManifest + validateQuarantineManifest(manifest) + return manifest +} + +export function validateQuarantineManifest(manifest: QuarantineManifest) { + if (!Array.isArray(manifest.quarantined)) { + throw new Error('quarantine manifest must contain a quarantined array') + } + + const ids = new Set() + const paths = new Set() + + for (const entry of manifest.quarantined) { + if (!entry.id || !entry.path || !entry.reason || !entry.owner || !entry.reviewAfter) { + throw new Error(`invalid quarantine entry: ${JSON.stringify(entry)}`) + } + if (ids.has(entry.id)) { + throw new Error(`duplicate quarantine id: ${entry.id}`) + } + if (paths.has(entry.path)) { + throw new Error(`duplicate quarantine path: ${entry.path}`) + } + ids.add(entry.id) + paths.add(entry.path) + } +} + +export function quarantinedPathSet(manifest = loadQuarantineManifest()) { + return new Set(manifest.quarantined.map((entry) => entry.path)) +} diff --git a/scripts/quality-gate/reporter.ts b/scripts/quality-gate/reporter.ts new file mode 100644 index 00000000..dde39b57 --- /dev/null +++ b/scripts/quality-gate/reporter.ts @@ -0,0 +1,59 @@ +import { mkdirSync, writeFileSync } from 'node:fs' +import { join } from 'node:path' +import type { QualityGateReport } from './types' + +export function writeReport(report: QualityGateReport, outputDir: string) { + mkdirSync(outputDir, { recursive: true }) + writeFileSync(join(outputDir, 'report.json'), JSON.stringify(report, null, 2) + '\n') + writeFileSync(join(outputDir, 'report.md'), renderMarkdownReport(report)) +} + +export function renderMarkdownReport(report: QualityGateReport) { + const lines = [ + `# Quality Gate Report`, + '', + `- Run: ${report.runId}`, + `- Mode: ${report.mode}`, + `- Dry run: ${report.dryRun ? 'yes' : 'no'}`, + `- Live checks allowed: ${report.allowLive ? 'yes' : 'no'}`, + `- Git SHA: ${report.git.sha ?? 'unknown'}`, + `- Dirty worktree: ${report.git.dirty ? 'yes' : 'no'}`, + `- Started: ${report.startedAt}`, + `- Finished: ${report.finishedAt}`, + '', + `## Summary`, + '', + `- Passed: ${report.summary.passed}`, + `- Failed: ${report.summary.failed}`, + `- Skipped: ${report.summary.skipped}`, + '', + `## Lanes`, + '', + ] + + for (const result of report.results) { + lines.push(`### ${result.title}`) + lines.push('') + lines.push(`- ID: ${result.id}`) + lines.push(`- Status: ${result.status}`) + lines.push(`- Duration: ${result.durationMs}ms`) + if (result.command) { + lines.push(`- Command: \`${result.command.join(' ')}\``) + } + if (result.exitCode !== undefined) { + lines.push(`- Exit code: ${result.exitCode}`) + } + if (result.skipReason) { + lines.push(`- Skip reason: ${result.skipReason}`) + } + if (result.error) { + lines.push(`- Error: ${result.error}`) + } + if (result.artifactDir) { + lines.push(`- Artifacts: ${result.artifactDir}`) + } + lines.push('') + } + + return lines.join('\n') +} diff --git a/scripts/quality-gate/runner.test.ts b/scripts/quality-gate/runner.test.ts new file mode 100644 index 00000000..8012b534 --- /dev/null +++ b/scripts/quality-gate/runner.test.ts @@ -0,0 +1,108 @@ +import { describe, expect, test } from 'bun:test' +import { mkdtempSync, readFileSync, rmSync } from 'node:fs' +import { tmpdir } from 'node:os' +import { join } from 'node:path' +import { lanesForMode } from './modes' +import { renderMarkdownReport } from './reporter' +import { runQualityGate } from './runner' +import type { QualityGateReport } from './types' + +describe('quality gate modes', () => { + test('pr mode includes existing path-aware PR checks', () => { + const lanes = lanesForMode('pr').map((lane) => lane.id) + expect(lanes).toContain('impact-report') + expect(lanes).toContain('pr-checks') + expect(lanes.some((lane) => lane.startsWith('baseline:'))).toBe(false) + }) + + test('baseline mode includes live baseline cases but not native checks', () => { + const lanes = lanesForMode('baseline').map((lane) => lane.id) + expect(lanes).toContain('baseline-catalog') + expect(lanes).toContain('baseline:failing-unit:current-runtime') + expect(lanes).toContain('baseline:multi-file-api:current-runtime') + expect(lanes).not.toContain('native-checks') + }) + + test('release mode composes PR, baseline, and native lanes', () => { + const lanes = lanesForMode('release').map((lane) => lane.id) + expect(lanes).toContain('pr-checks') + expect(lanes).toContain('baseline:failing-unit:current-runtime') + expect(lanes).toContain('native-checks') + }) + + test('baseline mode expands cases across explicit provider/model targets', () => { + const lanes = lanesForMode('baseline', [ + { providerId: 'provider-a', modelId: 'model-a', label: 'provider-a-model-a' }, + { providerId: 'provider-b', modelId: 'model-b', label: 'provider-b-model-b' }, + ]).map((lane) => lane.id) + + expect(lanes).toContain('baseline:failing-unit:provider-a-model-a') + expect(lanes).toContain('baseline:failing-unit:provider-b-model-b') + expect(lanes).toContain('baseline:multi-file-api:provider-a-model-a') + expect(lanes).toContain('baseline:multi-file-api:provider-b-model-b') + }) +}) + +describe('runQualityGate', () => { + test('writes dry-run reports without executing expensive commands', async () => { + const artifactsDir = mkdtempSync(join(tmpdir(), 'quality-gate-test-')) + try { + const { report, outputDir } = await runQualityGate({ + mode: 'baseline', + dryRun: true, + allowLive: false, + baselineTargets: [], + rootDir: process.cwd(), + artifactsDir, + runId: 'dry-run-test', + }) + + expect(report.mode).toBe('baseline') + expect(report.summary.failed).toBe(0) + expect(report.summary.skipped).toBeGreaterThan(0) + expect(readFileSync(join(outputDir, 'report.json'), 'utf8')).toContain('"mode": "baseline"') + expect(readFileSync(join(outputDir, 'report.md'), 'utf8')).toContain('# Quality Gate Report') + } finally { + rmSync(artifactsDir, { recursive: true, force: true }) + } + }) +}) + +describe('renderMarkdownReport', () => { + test('renders command, skip reason, and summary', () => { + const report: QualityGateReport = { + schemaVersion: 1, + runId: 'example', + mode: 'pr', + dryRun: true, + allowLive: false, + startedAt: '2026-05-02T00:00:00.000Z', + finishedAt: '2026-05-02T00:00:01.000Z', + rootDir: process.cwd(), + git: { + sha: 'abc123', + dirty: true, + }, + results: [ + { + id: 'impact-report', + title: 'Impact report', + status: 'skipped', + command: ['bun', 'run', 'check:impact'], + durationMs: 1, + skipReason: 'dry run', + }, + ], + summary: { + passed: 0, + failed: 0, + skipped: 1, + }, + } + + const markdown = renderMarkdownReport(report) + expect(markdown).toContain('Skipped: 1') + expect(markdown).toContain('`bun run check:impact`') + expect(markdown).toContain('dry run') + }) +}) diff --git a/scripts/quality-gate/runner.ts b/scripts/quality-gate/runner.ts new file mode 100644 index 00000000..131738a8 --- /dev/null +++ b/scripts/quality-gate/runner.ts @@ -0,0 +1,152 @@ +import { mkdirSync } from 'node:fs' +import { join } from 'node:path' +import { baselineCases } from './baseline/cases' +import { executeBaselineCase } from './baseline/execute' +import { lanesForMode } from './modes' +import { writeReport } from './reporter' +import type { LaneDefinition, LaneResult, QualityGateOptions, QualityGateReport } from './types' + +function nowId() { + return new Date().toISOString().replace(/[:.]/g, '-') +} + +async function output(cmd: string[], cwd: string) { + const proc = Bun.spawn(cmd, { + cwd, + stdout: 'pipe', + stderr: 'pipe', + }) + const stdout = await new Response(proc.stdout).text() + const stderr = await new Response(proc.stderr).text() + const code = await proc.exited + if (code !== 0) { + return null + } + return (stdout || stderr).trim() +} + +async function gitInfo(rootDir: string) { + const sha = await output(['git', 'rev-parse', '--short', 'HEAD'], rootDir) + const status = await output(['git', 'status', '--short'], rootDir) + return { + sha, + dirty: Boolean(status), + } +} + +async function runCommandLane(lane: LaneDefinition, options: QualityGateOptions): Promise { + const started = Date.now() + const command = lane.command ?? [] + + if (options.dryRun) { + return { + id: lane.id, + title: lane.title, + status: 'skipped', + command, + durationMs: Date.now() - started, + skipReason: 'dry run', + } + } + + const proc = Bun.spawn(command, { + cwd: options.rootDir, + stdout: 'inherit', + stderr: 'inherit', + }) + const exitCode = await proc.exited + + return { + id: lane.id, + title: lane.title, + status: exitCode === 0 ? 'passed' : 'failed', + command, + durationMs: Date.now() - started, + exitCode, + } +} + +async function runBaselineCaseLane(lane: LaneDefinition, options: QualityGateOptions): Promise { + const started = Date.now() + + if (!options.allowLive) { + return { + id: lane.id, + title: lane.title, + status: 'skipped', + durationMs: Date.now() - started, + skipReason: 'live baseline cases require --allow-live', + } + } + + const caseId = lane.baselineCaseId ?? lane.id.replace(/^baseline:/, '').split(':')[0] + const testCase = baselineCases.find((candidate) => candidate.id === caseId) + if (!testCase) { + return { + id: lane.id, + title: lane.title, + status: 'failed', + durationMs: Date.now() - started, + error: `Unknown baseline case: ${caseId}`, + } + } + + const artifactRoot = options.runOutputDir ?? join(options.rootDir, 'artifacts', 'quality-runs', options.runId ?? 'current') + return executeBaselineCase( + testCase, + options.rootDir, + join(artifactRoot, 'cases', lane.id.replace(/[^a-zA-Z0-9._-]+/g, '-')), + lane.baselineTarget, + ) +} + +async function runLane(lane: LaneDefinition, options: QualityGateOptions): Promise { + if (lane.kind === 'baseline-case') { + return runBaselineCaseLane(lane, options) + } + + return runCommandLane(lane, options) +} + +function summarize(results: LaneResult[]) { + return { + passed: results.filter((result) => result.status === 'passed').length, + failed: results.filter((result) => result.status === 'failed').length, + skipped: results.filter((result) => result.status === 'skipped').length, + } +} + +export async function runQualityGate(options: QualityGateOptions) { + const runId = options.runId ?? nowId() + const startedAt = new Date().toISOString() + const artifactsRoot = options.artifactsDir ?? join(options.rootDir, 'artifacts', 'quality-runs') + const outputDir = join(artifactsRoot, runId) + mkdirSync(outputDir, { recursive: true }) + + const runOptions = { ...options, runId, runOutputDir: outputDir } + const results: LaneResult[] = [] + for (const lane of lanesForMode(options.mode, options.baselineTargets)) { + const result = await runLane(lane, runOptions) + results.push(result) + if (result.status === 'failed') { + break + } + } + + const report: QualityGateReport = { + schemaVersion: 1, + runId, + mode: options.mode, + dryRun: options.dryRun, + allowLive: options.allowLive, + startedAt, + finishedAt: new Date().toISOString(), + rootDir: options.rootDir, + git: await gitInfo(options.rootDir), + results, + summary: summarize(results), + } + + writeReport(report, outputDir) + return { report, outputDir } +} diff --git a/scripts/quality-gate/types.ts b/scripts/quality-gate/types.ts new file mode 100644 index 00000000..b1a09a38 --- /dev/null +++ b/scripts/quality-gate/types.ts @@ -0,0 +1,84 @@ +export type QualityGateMode = 'pr' | 'baseline' | 'release' + +export type LaneKind = 'command' | 'baseline-case' + +export type LaneDefinition = { + id: string + title: string + description: string + kind: LaneKind + command?: string[] + baselineCaseId?: string + baselineTarget?: BaselineTarget + requiredForModes: QualityGateMode[] + live?: boolean +} + +export type BaselineCase = { + id: string + title: string + description: string + fixture: string + prompt: string + mode: 'ui' | 'websocket' + requiredCapabilities: Array<'model' | 'file-edit' | 'shell' | 'permission' | 'browser'> + timeoutMs: number + verify: { + commands: string[][] + expectedFiles?: string[] + forbiddenFiles?: string[] + transcriptAssertions?: string[] + } +} + +export type BaselineTarget = { + providerId: string | null + modelId: string + label: string +} + +export type LaneStatus = 'passed' | 'failed' | 'skipped' + +export type LaneResult = { + id: string + title: string + status: LaneStatus + command?: string[] + durationMs: number + exitCode?: number + skipReason?: string + error?: string + artifactDir?: string +} + +export type QualityGateOptions = { + mode: QualityGateMode + dryRun: boolean + allowLive: boolean + baselineTargets: BaselineTarget[] + rootDir: string + artifactsDir?: string + runOutputDir?: string + runId?: string +} + +export type QualityGateReport = { + schemaVersion: 1 + runId: string + mode: QualityGateMode + dryRun: boolean + allowLive: boolean + startedAt: string + finishedAt: string + rootDir: string + git: { + sha: string | null + dirty: boolean + } + results: LaneResult[] + summary: { + passed: number + failed: number + skipped: number + } +} diff --git a/src/server/__tests__/conversations.test.ts b/src/server/__tests__/conversations.test.ts index 9632588d..c13e27c0 100644 --- a/src/server/__tests__/conversations.test.ts +++ b/src/server/__tests__/conversations.test.ts @@ -237,9 +237,11 @@ describe('ConversationService', () => { it('should reconstruct usage and metadata from a persisted transcript', async () => { const previousConfigDir = process.env.CLAUDE_CONFIG_DIR + const previousAnthropicApiKey = process.env.ANTHROPIC_API_KEY const tmpConfigDir = await fs.mkdtemp(path.join(os.tmpdir(), 'claude-transcript-')) const workDir = await fs.mkdtemp(path.join(os.tmpdir(), 'claude-workdir-')) process.env.CLAUDE_CONFIG_DIR = tmpConfigDir + process.env.ANTHROPIC_API_KEY = 'test-key' try { const svc = new SessionService() @@ -290,6 +292,11 @@ describe('ConversationService', () => { } else { process.env.CLAUDE_CONFIG_DIR = previousConfigDir } + if (previousAnthropicApiKey === undefined) { + delete process.env.ANTHROPIC_API_KEY + } else { + process.env.ANTHROPIC_API_KEY = previousAnthropicApiKey + } await fs.rm(tmpConfigDir, { recursive: true, force: true }) await fs.rm(workDir, { recursive: true, force: true }) } From d1219b9682de22f141023e116aef38e24db45b6d Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=E7=A8=8B=E5=BA=8F=E5=91=98=E9=98=BF=E6=B1=9F=28Relakkes?= =?UTF-8?q?=29?= Date: Sat, 2 May 2026 13:22:55 +0800 Subject: [PATCH 2/8] Broaden live agent baseline beyond toy edits The first quality gate proved the execution path, but two tiny fixtures were not enough to reveal regressions in real agent behavior. This expands the maintained baseline with deeper failure recovery, workspace search, artifact creation, and cross-module refactor cases, and records per-case diffs so reviewers can see exactly what each model changed. Constraint: Baseline runs must remain explicit maintainer-controlled live checks, not default PR checks. Rejected: Require models to edit tests in every contract-change case | correct fixes can satisfy prewritten acceptance tests without modifying tests. Rejected: Trust only command exit codes | passing tests alone does not show whether the agent edited forbidden or unrelated files. Confidence: high Scope-risk: moderate Directive: Keep live baseline cases product-shaped and inspect both verification output and diff.patch before accepting future fixture changes. Tested: bun test scripts/quality-gate/*.test.ts scripts/quality-gate/baseline/*.test.ts Tested: bun run quality:gate --mode baseline --dry-run --provider-model volc-kimi-k2.6 --provider-model minimax-m2.7 Tested: bun run quality:gate --mode baseline --allow-live --provider-model volc-kimi-k2.6 --provider-model minimax-m2.7 (14 passed, 0 failed) Not-tested: Desktop UI browser smoke and native Tauri release packaging remain outside this baseline expansion. --- scripts/quality-gate/baseline/cases.test.ts | 21 +++++ scripts/quality-gate/baseline/cases.ts | 78 ++++++++++++++- scripts/quality-gate/baseline/execute.ts | 94 ++++++++++++++++++- .../cross-module-refactor/package.json | 8 ++ .../cross-module-refactor/src/config.ts | 3 + .../cross-module-refactor/src/runner.test.ts | 17 ++++ .../cross-module-refactor/src/runner.ts | 6 ++ .../fixtures/failure-recovery/package.json | 8 ++ .../failure-recovery/src/slug.test.ts | 16 ++++ .../fixtures/failure-recovery/src/slug.ts | 3 + .../fixtures/multi-file-api/src/api.ts | 10 +- .../fixtures/multi-file-api/src/app.test.ts | 7 ++ .../fixtures/multi-file-api/src/app.ts | 2 +- .../fixtures/permission-artifact/package.json | 8 ++ .../permission-artifact/src/artifact.test.ts | 9 ++ .../workspace-search-edit/package.json | 8 ++ .../workspace-search-edit/src/catalog.ts | 9 ++ .../src/checkout.test.ts | 10 ++ .../workspace-search-edit/src/checkout.ts | 7 ++ .../workspace-search-edit/src/user.ts | 9 ++ scripts/quality-gate/types.ts | 1 + 21 files changed, 326 insertions(+), 8 deletions(-) create mode 100644 scripts/quality-gate/baseline/fixtures/cross-module-refactor/package.json create mode 100644 scripts/quality-gate/baseline/fixtures/cross-module-refactor/src/config.ts create mode 100644 scripts/quality-gate/baseline/fixtures/cross-module-refactor/src/runner.test.ts create mode 100644 scripts/quality-gate/baseline/fixtures/cross-module-refactor/src/runner.ts create mode 100644 scripts/quality-gate/baseline/fixtures/failure-recovery/package.json create mode 100644 scripts/quality-gate/baseline/fixtures/failure-recovery/src/slug.test.ts create mode 100644 scripts/quality-gate/baseline/fixtures/failure-recovery/src/slug.ts create mode 100644 scripts/quality-gate/baseline/fixtures/permission-artifact/package.json create mode 100644 scripts/quality-gate/baseline/fixtures/permission-artifact/src/artifact.test.ts create mode 100644 scripts/quality-gate/baseline/fixtures/workspace-search-edit/package.json create mode 100644 scripts/quality-gate/baseline/fixtures/workspace-search-edit/src/catalog.ts create mode 100644 scripts/quality-gate/baseline/fixtures/workspace-search-edit/src/checkout.test.ts create mode 100644 scripts/quality-gate/baseline/fixtures/workspace-search-edit/src/checkout.ts create mode 100644 scripts/quality-gate/baseline/fixtures/workspace-search-edit/src/user.ts diff --git a/scripts/quality-gate/baseline/cases.test.ts b/scripts/quality-gate/baseline/cases.test.ts index c7b624a5..c895e099 100644 --- a/scripts/quality-gate/baseline/cases.test.ts +++ b/scripts/quality-gate/baseline/cases.test.ts @@ -25,4 +25,25 @@ describe('baselineCases', () => { expect(testCase.requiredCapabilities).toContain('model') } }) + + test('define enough first-wave product baseline cases', () => { + expect(baselineCases.map((testCase) => testCase.id)).toEqual([ + 'failing-unit', + 'multi-file-api', + 'failure-recovery', + 'workspace-search-edit', + 'permission-artifact', + 'cross-module-refactor', + ]) + }) + + test('pin changed-file expectations for every case', () => { + for (const testCase of baselineCases) { + expect(testCase.verify.requiredFiles?.length ?? 0).toBeGreaterThan(0) + expect(testCase.verify.expectedFiles?.length ?? 0).toBeGreaterThan(0) + for (const file of testCase.verify.requiredFiles ?? []) { + expect(testCase.verify.expectedFiles).toContain(file) + } + } + }) }) diff --git a/scripts/quality-gate/baseline/cases.ts b/scripts/quality-gate/baseline/cases.ts index 4e308e2d..34ab7849 100644 --- a/scripts/quality-gate/baseline/cases.ts +++ b/scripts/quality-gate/baseline/cases.ts @@ -12,25 +12,95 @@ export const baselineCases: BaselineCase[] = [ timeoutMs: 180_000, verify: { commands: [['bun', 'test']], + requiredFiles: ['src/math.ts'], expectedFiles: ['src/math.ts'], forbiddenFiles: ['package.json'], - transcriptAssertions: ['bun test', 'src/math.ts'], + transcriptAssertions: ['"type":"message_complete"'], }, }, { id: 'multi-file-api', title: 'Update a multi-file API contract', - description: 'A small app exposes a user display API. The Agent must change the contract and update callers plus tests coherently.', + description: 'A small app exposes a user display API. The Agent must change the contract and update callers coherently without loosening tests.', fixture: 'scripts/quality-gate/baseline/fixtures/multi-file-api', - prompt: 'Change the user display contract so it returns "Ada Lovelace " instead of only the name. Update the implementation, caller, and tests, then run the tests.', + prompt: 'The user display API contract changed to return an object with a label field, where label is "Ada Lovelace ". Update the implementation and caller so the existing tests pass. Do not edit package.json.', mode: 'websocket', requiredCapabilities: ['model', 'file-edit', 'shell'], timeoutMs: 240_000, verify: { commands: [['bun', 'test']], + requiredFiles: ['src/api.ts', 'src/app.ts'], expectedFiles: ['src/api.ts', 'src/app.ts', 'src/app.test.ts'], forbiddenFiles: ['package.json'], - transcriptAssertions: ['bun test', 'src/api.ts', 'src/app.ts'], + transcriptAssertions: ['"type":"message_complete"'], + }, + }, + { + id: 'failure-recovery', + title: 'Recover from a deeper failing test suite', + description: 'A slug formatter has several edge-case failures. The Agent must read the failures, patch the implementation, and keep tests unchanged.', + fixture: 'scripts/quality-gate/baseline/fixtures/failure-recovery', + prompt: 'Run the tests and fix the slug formatter implementation until the whole suite passes. Do not edit the tests. Keep the implementation small and deterministic.', + mode: 'websocket', + requiredCapabilities: ['model', 'file-edit', 'shell'], + timeoutMs: 240_000, + verify: { + commands: [['bun', 'test']], + requiredFiles: ['src/slug.ts'], + expectedFiles: ['src/slug.ts'], + forbiddenFiles: ['src/slug.test.ts', 'package.json'], + transcriptAssertions: ['"type":"message_complete"'], + }, + }, + { + id: 'workspace-search-edit', + title: 'Find and fix a bug across a small workspace', + description: 'A checkout total is wrong in a multi-file fixture. The Agent must inspect the workspace and patch the correct implementation file.', + fixture: 'scripts/quality-gate/baseline/fixtures/workspace-search-edit', + prompt: 'The checkout tests are failing. Search the project, identify the real bug, fix the implementation, and rerun the tests. Do not rewrite unrelated modules.', + mode: 'websocket', + requiredCapabilities: ['model', 'file-edit', 'shell'], + timeoutMs: 240_000, + verify: { + commands: [['bun', 'test']], + requiredFiles: ['src/checkout.ts'], + expectedFiles: ['src/checkout.ts'], + forbiddenFiles: ['src/checkout.test.ts', 'src/catalog.ts', 'src/user.ts', 'package.json'], + transcriptAssertions: ['"type":"message_complete"'], + }, + }, + { + id: 'permission-artifact', + title: 'Create a required project artifact', + description: 'A test expects a generated summary file. The Agent must create a new file and validate it without changing the test.', + fixture: 'scripts/quality-gate/baseline/fixtures/permission-artifact', + prompt: 'Run the tests, create the missing project artifact they require, and rerun the tests. Do not modify the tests or package.json.', + mode: 'websocket', + requiredCapabilities: ['model', 'file-edit', 'shell', 'permission'], + timeoutMs: 180_000, + verify: { + commands: [['bun', 'test']], + requiredFiles: ['notes/summary.md'], + expectedFiles: ['notes/summary.md'], + forbiddenFiles: ['src/artifact.test.ts', 'package.json'], + transcriptAssertions: ['"type":"message_complete"'], + }, + }, + { + id: 'cross-module-refactor', + title: 'Complete a cross-module parser refactor', + description: 'A config parser contract changed but the implementation and caller are stale. The Agent must update multiple modules coherently.', + fixture: 'scripts/quality-gate/baseline/fixtures/cross-module-refactor', + prompt: 'Update the config parsing flow so parseConfig returns a structured object with enabled and retries fields. Update the implementation and caller so the existing tests pass, then run the suite.', + mode: 'websocket', + requiredCapabilities: ['model', 'file-edit', 'shell'], + timeoutMs: 300_000, + verify: { + commands: [['bun', 'test']], + requiredFiles: ['src/config.ts', 'src/runner.ts'], + expectedFiles: ['src/config.ts', 'src/runner.ts', 'src/runner.test.ts'], + forbiddenFiles: ['package.json'], + transcriptAssertions: ['"type":"message_complete"'], }, }, ] diff --git a/scripts/quality-gate/baseline/execute.ts b/scripts/quality-gate/baseline/execute.ts index 020630e7..e8e18bf1 100644 --- a/scripts/quality-gate/baseline/execute.ts +++ b/scripts/quality-gate/baseline/execute.ts @@ -1,4 +1,4 @@ -import { appendFileSync, cpSync, mkdirSync, rmSync, writeFileSync } from 'node:fs' +import { appendFileSync, cpSync, mkdirSync, readdirSync, readFileSync, rmSync, statSync, writeFileSync } from 'node:fs' import { mkdtemp } from 'node:fs/promises' import { tmpdir } from 'node:os' import { join } from 'node:path' @@ -51,6 +51,90 @@ async function runCommand(command: string[], cwd: string) { return { exitCode, stdout, stderr } } +function listFiles(root: string, current = root): string[] { + const files: string[] = [] + for (const entry of readdirSync(current)) { + if (entry === 'node_modules' || entry === '.git') { + continue + } + + const fullPath = join(current, entry) + const stat = statSync(fullPath) + if (stat.isDirectory()) { + files.push(...listFiles(root, fullPath)) + continue + } + if (stat.isFile()) { + files.push(fullPath.slice(root.length + 1)) + } + } + return files.sort() +} + +function changedFiles(beforeDir: string, afterDir: string) { + const before = new Set(listFiles(beforeDir)) + const after = new Set(listFiles(afterDir)) + const changed = new Set() + + for (const file of before) { + if (!after.has(file)) { + changed.add(file) + continue + } + if (!readFileSync(join(beforeDir, file)).equals(readFileSync(join(afterDir, file)))) { + changed.add(file) + } + } + + for (const file of after) { + if (!before.has(file)) { + changed.add(file) + } + } + + return [...changed].sort() +} + +async function writeDiffPatch(beforeDir: string, afterDir: string, patchPath: string) { + const result = await runCommand(['git', 'diff', '--no-index', '--', beforeDir, afterDir], process.cwd()) + writeFileSync(patchPath, `${result.stdout}${result.stderr}`) +} + +function verifyChangedFiles(testCase: BaselineCase, changed: string[]) { + const expected = testCase.verify.expectedFiles + if (expected) { + const unexpected = changed.filter((file) => !expected.includes(file)) + if (unexpected.length > 0) { + throw new Error(`unexpected changed files: ${unexpected.join(', ')}`) + } + } + + const required = testCase.verify.requiredFiles ?? [] + const missing = required.filter((file) => !changed.includes(file)) + if (missing.length > 0) { + throw new Error(`required files were not changed: ${missing.join(', ')}`) + } + + const forbidden = testCase.verify.forbiddenFiles ?? [] + const forbiddenChanged = forbidden.filter((file) => changed.includes(file)) + if (forbiddenChanged.length > 0) { + throw new Error(`forbidden files changed: ${forbiddenChanged.join(', ')}`) + } +} + +function verifyTranscript(testCase: BaselineCase, transcriptPath: string) { + const assertions = testCase.verify.transcriptAssertions ?? [] + if (assertions.length === 0) { + return + } + + const transcript = readFileSync(transcriptPath, 'utf8') + const missing = assertions.filter((assertion) => !transcript.includes(assertion)) + if (missing.length > 0) { + throw new Error(`transcript assertions missing: ${missing.join(', ')}`) + } +} + async function pipeToFile(stream: ReadableStream | null, path: string) { if (!stream) return const reader = stream.getReader() @@ -154,12 +238,15 @@ export async function executeBaselineCase( const port = await getPort() const baseUrl = `http://127.0.0.1:${port}` const workRoot = await mkdtemp(join(tmpdir(), `quality-gate-${testCase.id}-`)) + const originalDir = join(workRoot, 'original') const projectDir = join(workRoot, 'project') + cpSync(join(rootDir, testCase.fixture), originalDir, { recursive: true }) cpSync(join(rootDir, testCase.fixture), projectDir, { recursive: true }) const serverLogPath = join(artifactDir, 'server.log') const transcriptPath = join(artifactDir, 'transcript.jsonl') const verificationPath = join(artifactDir, 'verification.log') + const diffPath = join(artifactDir, 'diff.patch') const server = Bun.spawn(['bun', 'run', 'src/server/index.ts', '--host', '127.0.0.1', '--port', String(port)], { cwd: rootDir, stdout: 'pipe', @@ -190,6 +277,11 @@ export async function executeBaselineCase( const messages = await runPromptOverWebSocket(baseUrl, session.sessionId, testCase.prompt, testCase.timeoutMs, target) writeFileSync(transcriptPath, messages.map((message) => JSON.stringify(message)).join('\n') + '\n') + verifyTranscript(testCase, transcriptPath) + + await writeDiffPatch(originalDir, projectDir, diffPath) + const changed = changedFiles(originalDir, projectDir) + verifyChangedFiles(testCase, changed) let verificationLog = '' for (const command of testCase.verify.commands) { diff --git a/scripts/quality-gate/baseline/fixtures/cross-module-refactor/package.json b/scripts/quality-gate/baseline/fixtures/cross-module-refactor/package.json new file mode 100644 index 00000000..7de3edee --- /dev/null +++ b/scripts/quality-gate/baseline/fixtures/cross-module-refactor/package.json @@ -0,0 +1,8 @@ +{ + "name": "quality-gate-cross-module-refactor-fixture", + "private": true, + "type": "module", + "scripts": { + "test": "bun test" + } +} diff --git a/scripts/quality-gate/baseline/fixtures/cross-module-refactor/src/config.ts b/scripts/quality-gate/baseline/fixtures/cross-module-refactor/src/config.ts new file mode 100644 index 00000000..b7c271ab --- /dev/null +++ b/scripts/quality-gate/baseline/fixtures/cross-module-refactor/src/config.ts @@ -0,0 +1,3 @@ +export function parseConfig(raw: string) { + return raw === 'enabled' +} diff --git a/scripts/quality-gate/baseline/fixtures/cross-module-refactor/src/runner.test.ts b/scripts/quality-gate/baseline/fixtures/cross-module-refactor/src/runner.test.ts new file mode 100644 index 00000000..dbf12a1e --- /dev/null +++ b/scripts/quality-gate/baseline/fixtures/cross-module-refactor/src/runner.test.ts @@ -0,0 +1,17 @@ +import { describe, expect, test } from 'bun:test' +import { parseConfig } from './config' +import { describeRun } from './runner' + +describe('config runner', () => { + test('parses structured config', () => { + expect(parseConfig('enabled retries=3')).toEqual({ enabled: true, retries: 3 }) + }) + + test('describes enabled retry runs', () => { + expect(describeRun('enabled retries=3')).toBe('enabled with 3 retries') + }) + + test('describes disabled runs', () => { + expect(describeRun('disabled retries=0')).toBe('disabled') + }) +}) diff --git a/scripts/quality-gate/baseline/fixtures/cross-module-refactor/src/runner.ts b/scripts/quality-gate/baseline/fixtures/cross-module-refactor/src/runner.ts new file mode 100644 index 00000000..9304d604 --- /dev/null +++ b/scripts/quality-gate/baseline/fixtures/cross-module-refactor/src/runner.ts @@ -0,0 +1,6 @@ +import { parseConfig } from './config' + +export function describeRun(rawConfig: string) { + const enabled = parseConfig(rawConfig) + return enabled ? 'enabled' : 'disabled' +} diff --git a/scripts/quality-gate/baseline/fixtures/failure-recovery/package.json b/scripts/quality-gate/baseline/fixtures/failure-recovery/package.json new file mode 100644 index 00000000..70977b30 --- /dev/null +++ b/scripts/quality-gate/baseline/fixtures/failure-recovery/package.json @@ -0,0 +1,8 @@ +{ + "name": "quality-gate-failure-recovery-fixture", + "private": true, + "type": "module", + "scripts": { + "test": "bun test" + } +} diff --git a/scripts/quality-gate/baseline/fixtures/failure-recovery/src/slug.test.ts b/scripts/quality-gate/baseline/fixtures/failure-recovery/src/slug.test.ts new file mode 100644 index 00000000..1360e711 --- /dev/null +++ b/scripts/quality-gate/baseline/fixtures/failure-recovery/src/slug.test.ts @@ -0,0 +1,16 @@ +import { describe, expect, test } from 'bun:test' +import { slugify } from './slug' + +describe('slugify', () => { + test('normalizes whitespace and punctuation', () => { + expect(slugify(' Hello, Coding Agent! ')).toBe('hello-coding-agent') + }) + + test('collapses repeated separators', () => { + expect(slugify('Ship---Stable Releases')).toBe('ship-stable-releases') + }) + + test('trims separators from the edges', () => { + expect(slugify('---Quality Gate---')).toBe('quality-gate') + }) +}) diff --git a/scripts/quality-gate/baseline/fixtures/failure-recovery/src/slug.ts b/scripts/quality-gate/baseline/fixtures/failure-recovery/src/slug.ts new file mode 100644 index 00000000..90ec1085 --- /dev/null +++ b/scripts/quality-gate/baseline/fixtures/failure-recovery/src/slug.ts @@ -0,0 +1,3 @@ +export function slugify(input: string) { + return input.toLowerCase().replaceAll(' ', '-') +} diff --git a/scripts/quality-gate/baseline/fixtures/multi-file-api/src/api.ts b/scripts/quality-gate/baseline/fixtures/multi-file-api/src/api.ts index 76df2226..2db3f4c5 100644 --- a/scripts/quality-gate/baseline/fixtures/multi-file-api/src/api.ts +++ b/scripts/quality-gate/baseline/fixtures/multi-file-api/src/api.ts @@ -3,6 +3,12 @@ export type User = { email: string } -export function formatUser(user: User) { - return user.name +export type UserDisplay = { + displayName: string +} + +export function formatUser(user: User) { + return { + displayName: user.name, + } satisfies UserDisplay } diff --git a/scripts/quality-gate/baseline/fixtures/multi-file-api/src/app.test.ts b/scripts/quality-gate/baseline/fixtures/multi-file-api/src/app.test.ts index df622a5c..14c247ca 100644 --- a/scripts/quality-gate/baseline/fixtures/multi-file-api/src/app.test.ts +++ b/scripts/quality-gate/baseline/fixtures/multi-file-api/src/app.test.ts @@ -1,8 +1,15 @@ import { describe, expect, test } from 'bun:test' +import { formatUser } from './api' import { renderUser } from './app' describe('renderUser', () => { test('renders name and email', () => { expect(renderUser({ name: 'Ada Lovelace', email: 'ada@example.com' })).toBe('User: Ada Lovelace ') }) + + test('uses the structured display API contract', () => { + expect(formatUser({ name: 'Ada Lovelace', email: 'ada@example.com' })).toEqual({ + label: 'Ada Lovelace ', + }) + }) }) diff --git a/scripts/quality-gate/baseline/fixtures/multi-file-api/src/app.ts b/scripts/quality-gate/baseline/fixtures/multi-file-api/src/app.ts index e045b09c..2c6780c0 100644 --- a/scripts/quality-gate/baseline/fixtures/multi-file-api/src/app.ts +++ b/scripts/quality-gate/baseline/fixtures/multi-file-api/src/app.ts @@ -1,5 +1,5 @@ import { formatUser, type User } from './api' export function renderUser(user: User) { - return `User: ${formatUser(user)}` + return `User: ${formatUser(user).displayName}` } diff --git a/scripts/quality-gate/baseline/fixtures/permission-artifact/package.json b/scripts/quality-gate/baseline/fixtures/permission-artifact/package.json new file mode 100644 index 00000000..40efa949 --- /dev/null +++ b/scripts/quality-gate/baseline/fixtures/permission-artifact/package.json @@ -0,0 +1,8 @@ +{ + "name": "quality-gate-permission-artifact-fixture", + "private": true, + "type": "module", + "scripts": { + "test": "bun test" + } +} diff --git a/scripts/quality-gate/baseline/fixtures/permission-artifact/src/artifact.test.ts b/scripts/quality-gate/baseline/fixtures/permission-artifact/src/artifact.test.ts new file mode 100644 index 00000000..a76f8608 --- /dev/null +++ b/scripts/quality-gate/baseline/fixtures/permission-artifact/src/artifact.test.ts @@ -0,0 +1,9 @@ +import { describe, expect, test } from 'bun:test' +import { existsSync, readFileSync } from 'node:fs' + +describe('release artifact', () => { + test('has the required summary marker', () => { + expect(existsSync('notes/summary.md')).toBe(true) + expect(readFileSync('notes/summary.md', 'utf8')).toContain('baseline-ready') + }) +}) diff --git a/scripts/quality-gate/baseline/fixtures/workspace-search-edit/package.json b/scripts/quality-gate/baseline/fixtures/workspace-search-edit/package.json new file mode 100644 index 00000000..a1c962e8 --- /dev/null +++ b/scripts/quality-gate/baseline/fixtures/workspace-search-edit/package.json @@ -0,0 +1,8 @@ +{ + "name": "quality-gate-workspace-search-edit-fixture", + "private": true, + "type": "module", + "scripts": { + "test": "bun test" + } +} diff --git a/scripts/quality-gate/baseline/fixtures/workspace-search-edit/src/catalog.ts b/scripts/quality-gate/baseline/fixtures/workspace-search-edit/src/catalog.ts new file mode 100644 index 00000000..03c43817 --- /dev/null +++ b/scripts/quality-gate/baseline/fixtures/workspace-search-edit/src/catalog.ts @@ -0,0 +1,9 @@ +export type CatalogItem = { + sku: string + priceCents: number +} + +export const catalog: CatalogItem[] = [ + { sku: 'agent-seat', priceCents: 1200 }, + { sku: 'baseline-pack', priceCents: 800 }, +] diff --git a/scripts/quality-gate/baseline/fixtures/workspace-search-edit/src/checkout.test.ts b/scripts/quality-gate/baseline/fixtures/workspace-search-edit/src/checkout.test.ts new file mode 100644 index 00000000..b329149a --- /dev/null +++ b/scripts/quality-gate/baseline/fixtures/workspace-search-edit/src/checkout.test.ts @@ -0,0 +1,10 @@ +import { describe, expect, test } from 'bun:test' +import { catalog } from './catalog' +import { calculateTotalCents } from './checkout' +import { testUser } from './user' + +describe('calculateTotalCents', () => { + test('applies percentage discounts to the subtotal', () => { + expect(calculateTotalCents(catalog, testUser)).toBe(1500) + }) +}) diff --git a/scripts/quality-gate/baseline/fixtures/workspace-search-edit/src/checkout.ts b/scripts/quality-gate/baseline/fixtures/workspace-search-edit/src/checkout.ts new file mode 100644 index 00000000..57c02a49 --- /dev/null +++ b/scripts/quality-gate/baseline/fixtures/workspace-search-edit/src/checkout.ts @@ -0,0 +1,7 @@ +import type { CatalogItem } from './catalog' +import type { User } from './user' + +export function calculateTotalCents(items: CatalogItem[], user: User) { + const subtotal = items.reduce((sum, item) => sum + item.priceCents, 0) + return subtotal - user.discountPercent +} diff --git a/scripts/quality-gate/baseline/fixtures/workspace-search-edit/src/user.ts b/scripts/quality-gate/baseline/fixtures/workspace-search-edit/src/user.ts new file mode 100644 index 00000000..ca7b062f --- /dev/null +++ b/scripts/quality-gate/baseline/fixtures/workspace-search-edit/src/user.ts @@ -0,0 +1,9 @@ +export type User = { + id: string + discountPercent: number +} + +export const testUser: User = { + id: 'user-1', + discountPercent: 25, +} diff --git a/scripts/quality-gate/types.ts b/scripts/quality-gate/types.ts index b1a09a38..e86368f4 100644 --- a/scripts/quality-gate/types.ts +++ b/scripts/quality-gate/types.ts @@ -25,6 +25,7 @@ export type BaselineCase = { timeoutMs: number verify: { commands: string[][] + requiredFiles?: string[] expectedFiles?: string[] forbiddenFiles?: string[] transcriptAssertions?: string[] From 5ed9903d2e4a4b43a1ea5f0091e58d2b9457ebe3 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=E7=A8=8B=E5=BA=8F=E5=91=98=E9=98=BF=E6=B1=9F=28Relakkes?= =?UTF-8?q?=29?= Date: Sat, 2 May 2026 14:07:06 +0800 Subject: [PATCH 3/8] Exercise desktop chat through a real browser baseline The live baseline covered the server WebSocket path, but it still did not prove the desktop app can open a session, apply a selected provider/model, send a chat turn, and surface model/tool progress through the UI. This adds an agent-browser driven smoke lane that starts the local server and Vite desktop app, restores an isolated session tab with the requested runtime selection, submits a small coding task through the composer, and accepts the run only when the fixture diff and tests pass. Constraint: Desktop smoke must stay behind --allow-live because it launches browsers and real models. Constraint: The smoke temporarily enables bypassPermissions for the isolated run and restores the previous mode afterward. Rejected: Wait for a final marker phrase | model reasoning and echoed prompts can contain the same marker before the work is actually done. Rejected: Use only DOM text as success proof | the browser can show progress while the project files are still unchanged. Confidence: high Scope-risk: moderate Directive: Future desktop smoke cases should verify project state and artifacts, not only UI copy. Tested: bun test scripts/quality-gate/*.test.ts scripts/quality-gate/baseline/*.test.ts Tested: bun run quality:gate --mode baseline --dry-run --provider-model minimax-m2.7 Tested: bun run quality:gate --mode baseline --allow-live --provider-model minimax-m2.7 (9 passed, 0 failed) Tested: bun run check:server Not-tested: Kimi desktop smoke completion because the provider returned AccountQuotaExceeded / API Error 429 until its reset window. --- scripts/quality-gate/baseline/execute.ts | 4 +- scripts/quality-gate/desktop-smoke/execute.ts | 376 ++++++++++++++++++ .../fixtures/chat-edit/package.json | 8 + .../fixtures/chat-edit/src/greeting.test.ts | 8 + .../fixtures/chat-edit/src/greeting.ts | 3 + scripts/quality-gate/modes.ts | 13 + scripts/quality-gate/runner.test.ts | 4 + scripts/quality-gate/runner.ts | 23 ++ scripts/quality-gate/types.ts | 2 +- 9 files changed, 438 insertions(+), 3 deletions(-) create mode 100644 scripts/quality-gate/desktop-smoke/execute.ts create mode 100644 scripts/quality-gate/desktop-smoke/fixtures/chat-edit/package.json create mode 100644 scripts/quality-gate/desktop-smoke/fixtures/chat-edit/src/greeting.test.ts create mode 100644 scripts/quality-gate/desktop-smoke/fixtures/chat-edit/src/greeting.ts diff --git a/scripts/quality-gate/baseline/execute.ts b/scripts/quality-gate/baseline/execute.ts index e8e18bf1..058d83ae 100644 --- a/scripts/quality-gate/baseline/execute.ts +++ b/scripts/quality-gate/baseline/execute.ts @@ -71,7 +71,7 @@ function listFiles(root: string, current = root): string[] { return files.sort() } -function changedFiles(beforeDir: string, afterDir: string) { +export function changedFiles(beforeDir: string, afterDir: string) { const before = new Set(listFiles(beforeDir)) const after = new Set(listFiles(afterDir)) const changed = new Set() @@ -95,7 +95,7 @@ function changedFiles(beforeDir: string, afterDir: string) { return [...changed].sort() } -async function writeDiffPatch(beforeDir: string, afterDir: string, patchPath: string) { +export async function writeDiffPatch(beforeDir: string, afterDir: string, patchPath: string) { const result = await runCommand(['git', 'diff', '--no-index', '--', beforeDir, afterDir], process.cwd()) writeFileSync(patchPath, `${result.stdout}${result.stderr}`) } diff --git a/scripts/quality-gate/desktop-smoke/execute.ts b/scripts/quality-gate/desktop-smoke/execute.ts new file mode 100644 index 00000000..62676868 --- /dev/null +++ b/scripts/quality-gate/desktop-smoke/execute.ts @@ -0,0 +1,376 @@ +import { appendFileSync, cpSync, mkdirSync, readFileSync, rmSync, writeFileSync } from 'node:fs' +import { mkdtemp } from 'node:fs/promises' +import { createServer } from 'node:net' +import { tmpdir } from 'node:os' +import { join } from 'node:path' +import { changedFiles, writeDiffPatch } from '../baseline/execute' +import type { BaselineTarget, LaneResult } from '../types' + +const FIXTURE = 'scripts/quality-gate/desktop-smoke/fixtures/chat-edit' +const PROMPT = [ + 'Run the tests in this project, fix the failing greeting implementation, and rerun the tests.', + 'Only edit src/greeting.ts. Do not edit package.json or tests.', + 'When the tests pass, briefly say done.', +].join(' ') + +async function getPort(): Promise { + return await new Promise((resolve, reject) => { + const server = createServer() + server.on('error', reject) + server.listen(0, '127.0.0.1', () => { + const address = server.address() + if (!address || typeof address === 'string') { + server.close(() => reject(new Error('Failed to allocate a local port'))) + return + } + const port = address.port + server.close(() => resolve(port)) + }) + }) +} + +async function pipeToFile(stream: ReadableStream | null, path: string) { + if (!stream) return + const reader = stream.getReader() + while (true) { + const { done, value } = await reader.read() + if (done) break + appendFileSync(path, Buffer.from(value)) + } +} + +async function waitForHttp(url: string, timeoutMs: number) { + const deadline = Date.now() + timeoutMs + let lastError = '' + while (Date.now() < deadline) { + try { + const response = await fetch(url) + if (response.ok) return + lastError = `HTTP ${response.status}` + } catch (error) { + lastError = error instanceof Error ? error.message : String(error) + } + await Bun.sleep(500) + } + throw new Error(`Timed out waiting for ${url}${lastError ? ` (${lastError})` : ''}`) +} + +async function runLoggedCommand( + command: string[], + options: { + cwd: string + logPath: string + env?: Record + timeoutMs?: number + allowFailure?: boolean + maxLogChars?: number + }, +) { + appendFileSync(options.logPath, `\n$ ${command.join(' ')}\n`) + const proc = Bun.spawn(command, { + cwd: options.cwd, + env: options.env ? { ...process.env, ...options.env } : process.env, + stdout: 'pipe', + stderr: 'pipe', + }) + + const outputPromise = Promise.all([ + new Response(proc.stdout).text(), + new Response(proc.stderr).text(), + proc.exited, + ]) + const timeout = options.timeoutMs + ? Bun.sleep(options.timeoutMs).then(() => { + proc.kill() + throw new Error(`Command timed out after ${options.timeoutMs}ms: ${command.join(' ')}`) + }) + : null + + const [stdout, stderr, exitCode] = timeout + ? await Promise.race([outputPromise, timeout]) + : await outputPromise + const output = `${stdout}${stderr}` + appendFileSync( + options.logPath, + options.maxLogChars && output.length > options.maxLogChars + ? `${output.slice(0, options.maxLogChars)}\n[quality-gate] output truncated at ${options.maxLogChars} chars\n` + : output, + ) + + if (exitCode !== 0 && !options.allowFailure) { + throw new Error(`Command failed (${exitCode}): ${command.join(' ')}`) + } + + return { stdout, stderr, exitCode } +} + +async function fetchJson(url: string, init?: RequestInit): Promise { + const response = await fetch(url, init) + if (!response.ok) { + throw new Error(`${init?.method ?? 'GET'} ${url} failed with HTTP ${response.status}: ${await response.text()}`) + } + return response.json() as Promise +} + +async function setPermissionMode(baseUrl: string, mode: string) { + await fetchJson<{ ok: true; mode: string }>(`${baseUrl}/api/permissions/mode`, { + method: 'PUT', + headers: { 'Content-Type': 'application/json' }, + body: JSON.stringify({ mode }), + }) +} + +async function waitForVerifiedProject( + browserEnv: Record, + browserLogPath: string, + rootDir: string, + originalDir: string, + projectDir: string, + artifactDir: string, + timeoutMs: number, +) { + const deadline = Date.now() + timeoutMs + let lastVerificationError = 'project verification has not run yet' + while (Date.now() < deadline) { + const body = await runLoggedCommand(['agent-browser', 'get', 'text', '#content-area'], { + cwd: rootDir, + env: browserEnv, + logPath: browserLogPath, + timeoutMs: 15_000, + allowFailure: true, + maxLogChars: 4_000, + }) + const browserText = `${body.stdout}\n${body.stderr}` + + if (browserText.includes('CLI_START_FAILED') || browserText.includes('CLI_RESTART_FAILED')) { + throw new Error('Desktop session reported a CLI startup failure') + } + if ( + browserText.includes('API Error: 429') || + browserText.includes('AccountQuotaExceeded') || + browserText.includes('TooManyRequests') + ) { + throw new Error('Desktop session reported provider quota/rate-limit failure') + } + if (browserText.includes('处理过程中发生错误') || browserText.includes('API Error:')) { + throw new Error('Desktop session reported an API error') + } + + try { + await verifyProject(originalDir, projectDir, artifactDir) + return + } catch (error) { + lastVerificationError = error instanceof Error ? error.message : String(error) + } + await Bun.sleep(5_000) + } + + throw new Error(`Timed out waiting for desktop project verification: ${lastVerificationError}`) +} + +async function verifyProject(originalDir: string, projectDir: string, artifactDir: string) { + await writeDiffPatch(originalDir, projectDir, join(artifactDir, 'diff.patch')) + const changed = changedFiles(originalDir, projectDir) + const unexpected = changed.filter((file) => file !== 'src/greeting.ts') + if (unexpected.length > 0) { + throw new Error(`desktop smoke changed unexpected files: ${unexpected.join(', ')}`) + } + if (!changed.includes('src/greeting.ts')) { + throw new Error('desktop smoke did not change src/greeting.ts') + } + + const implementation = readFileSync(join(projectDir, 'src/greeting.ts'), 'utf8') + if (!implementation.includes('from desktop smoke!')) { + throw new Error('desktop smoke implementation is missing the expected marker text') + } + + const proc = Bun.spawn(['bun', 'test'], { + cwd: projectDir, + stdout: 'pipe', + stderr: 'pipe', + }) + const stdout = await new Response(proc.stdout).text() + const stderr = await new Response(proc.stderr).text() + const exitCode = await proc.exited + writeFileSync(join(artifactDir, 'verification.log'), `${stdout}${stderr}`) + if (exitCode !== 0) { + throw new Error(`desktop smoke verification failed with exit code ${exitCode}`) + } +} + +export async function executeDesktopSmoke( + rootDir: string, + artifactDir: string, + resultId: string, + resultTitle: string, + target: BaselineTarget | undefined, +): Promise { + const started = Date.now() + mkdirSync(artifactDir, { recursive: true }) + + const serverLogPath = join(artifactDir, 'server.log') + const viteLogPath = join(artifactDir, 'vite.log') + const browserLogPath = join(artifactDir, 'browser.log') + const workRoot = await mkdtemp(join(tmpdir(), 'quality-gate-desktop-smoke-')) + const originalDir = join(workRoot, 'original') + const projectDir = join(workRoot, 'project') + const browserProfileDir = join(workRoot, 'browser-profile') + cpSync(join(rootDir, FIXTURE), originalDir, { recursive: true }) + cpSync(join(rootDir, FIXTURE), projectDir, { recursive: true }) + + const serverPort = await getPort() + const vitePort = await getPort() + const baseUrl = `http://127.0.0.1:${serverPort}` + const appUrl = `http://127.0.0.1:${vitePort}` + const sessionName = `quality-gate-${Date.now()}-${Math.random().toString(36).slice(2, 8)}` + const browserEnv = { + AGENT_BROWSER_SESSION: sessionName, + AGENT_BROWSER_PROFILE: browserProfileDir, + } + + const server = Bun.spawn(['bun', 'run', 'src/server/index.ts', '--host', '127.0.0.1', '--port', String(serverPort)], { + cwd: rootDir, + stdout: 'pipe', + stderr: 'pipe', + }) + void pipeToFile(server.stdout, serverLogPath) + void pipeToFile(server.stderr, serverLogPath) + + const vite = Bun.spawn(['bun', 'run', 'dev', '--', '--host', '127.0.0.1', '--port', String(vitePort), '--strictPort'], { + cwd: join(rootDir, 'desktop'), + env: { + ...process.env, + VITE_DESKTOP_SERVER_URL: baseUrl, + }, + stdout: 'pipe', + stderr: 'pipe', + }) + void pipeToFile(vite.stdout, viteLogPath) + void pipeToFile(vite.stderr, viteLogPath) + + let previousPermissionMode: string | null = null + try { + await waitForHttp(`${baseUrl}/health`, 20_000) + await waitForHttp(appUrl, 30_000) + + const permission = await fetchJson<{ mode: string }>(`${baseUrl}/api/permissions/mode`) + previousPermissionMode = permission.mode + await setPermissionMode(baseUrl, 'bypassPermissions') + + const session = await fetchJson<{ sessionId: string }>(`${baseUrl}/api/sessions`, { + method: 'POST', + headers: { 'Content-Type': 'application/json' }, + body: JSON.stringify({ workDir: projectDir }), + }) + const runtimeSelection = { + providerId: target?.providerId ?? null, + modelId: target?.modelId ?? 'current', + } + + await runLoggedCommand(['agent-browser', 'open', appUrl], { + cwd: rootDir, + env: browserEnv, + logPath: browserLogPath, + timeoutMs: 30_000, + }) + await runLoggedCommand([ + 'agent-browser', + 'eval', + [ + `localStorage.setItem('cc-haha-open-tabs', ${JSON.stringify(JSON.stringify({ + openTabs: [{ sessionId: session.sessionId, title: 'Desktop Smoke', type: 'session' }], + activeTabId: session.sessionId, + }))})`, + `localStorage.setItem('cc-haha-session-runtime', ${JSON.stringify(JSON.stringify({ + [session.sessionId]: runtimeSelection, + }))})`, + ].join(';'), + ], { + cwd: rootDir, + env: browserEnv, + logPath: browserLogPath, + timeoutMs: 15_000, + }) + await runLoggedCommand(['agent-browser', 'reload'], { + cwd: rootDir, + env: browserEnv, + logPath: browserLogPath, + timeoutMs: 30_000, + }) + await runLoggedCommand(['agent-browser', 'wait', 'textarea'], { + cwd: rootDir, + env: browserEnv, + logPath: browserLogPath, + timeoutMs: 30_000, + }) + await runLoggedCommand(['agent-browser', 'screenshot', join(artifactDir, 'initial.png')], { + cwd: rootDir, + env: browserEnv, + logPath: browserLogPath, + timeoutMs: 20_000, + allowFailure: true, + }) + await runLoggedCommand(['agent-browser', 'fill', 'textarea', PROMPT], { + cwd: rootDir, + env: browserEnv, + logPath: browserLogPath, + timeoutMs: 20_000, + }) + await runLoggedCommand(['agent-browser', 'press', 'Enter'], { + cwd: rootDir, + env: browserEnv, + logPath: browserLogPath, + timeoutMs: 15_000, + }) + + await waitForVerifiedProject( + browserEnv, + browserLogPath, + rootDir, + originalDir, + projectDir, + artifactDir, + 360_000, + ) + await runLoggedCommand(['agent-browser', 'screenshot', join(artifactDir, 'final.png')], { + cwd: rootDir, + env: browserEnv, + logPath: browserLogPath, + timeoutMs: 20_000, + allowFailure: true, + }) + + return { + id: resultId, + title: resultTitle, + status: 'passed', + durationMs: Date.now() - started, + artifactDir, + } + } catch (error) { + return { + id: resultId, + title: resultTitle, + status: 'failed', + durationMs: Date.now() - started, + error: error instanceof Error ? error.message : String(error), + artifactDir, + } + } finally { + if (previousPermissionMode) { + await setPermissionMode(baseUrl, previousPermissionMode).catch((error) => { + appendFileSync(serverLogPath, `\n[quality-gate] Failed to restore permission mode: ${error instanceof Error ? error.message : String(error)}\n`) + }) + } + await runLoggedCommand(['agent-browser', 'close'], { + cwd: rootDir, + env: browserEnv, + logPath: browserLogPath, + timeoutMs: 10_000, + allowFailure: true, + }).catch(() => {}) + server.kill() + vite.kill() + rmSync(workRoot, { recursive: true, force: true }) + } +} diff --git a/scripts/quality-gate/desktop-smoke/fixtures/chat-edit/package.json b/scripts/quality-gate/desktop-smoke/fixtures/chat-edit/package.json new file mode 100644 index 00000000..1865effa --- /dev/null +++ b/scripts/quality-gate/desktop-smoke/fixtures/chat-edit/package.json @@ -0,0 +1,8 @@ +{ + "name": "desktop-smoke-chat-edit", + "private": true, + "type": "module", + "scripts": { + "test": "bun test" + } +} diff --git a/scripts/quality-gate/desktop-smoke/fixtures/chat-edit/src/greeting.test.ts b/scripts/quality-gate/desktop-smoke/fixtures/chat-edit/src/greeting.test.ts new file mode 100644 index 00000000..b22fc7bf --- /dev/null +++ b/scripts/quality-gate/desktop-smoke/fixtures/chat-edit/src/greeting.test.ts @@ -0,0 +1,8 @@ +import { describe, expect, test } from 'bun:test' +import { greetingFor } from './greeting' + +describe('greetingFor', () => { + test('includes the desktop smoke marker', () => { + expect(greetingFor('Ada Lovelace')).toBe('Hello, Ada Lovelace from desktop smoke!') + }) +}) diff --git a/scripts/quality-gate/desktop-smoke/fixtures/chat-edit/src/greeting.ts b/scripts/quality-gate/desktop-smoke/fixtures/chat-edit/src/greeting.ts new file mode 100644 index 00000000..de325881 --- /dev/null +++ b/scripts/quality-gate/desktop-smoke/fixtures/chat-edit/src/greeting.ts @@ -0,0 +1,3 @@ +export function greetingFor(name: string) { + return `Hello, ${name}` +} diff --git a/scripts/quality-gate/modes.ts b/scripts/quality-gate/modes.ts index 5d635550..998c5ba2 100644 --- a/scripts/quality-gate/modes.ts +++ b/scripts/quality-gate/modes.ts @@ -57,5 +57,18 @@ export function lanesForMode(mode: QualityGateMode, baselineTargets: BaselineTar } } + for (const target of targets) { + const targetSlug = target.label.replace(/[^a-zA-Z0-9._-]+/g, '-') + lanes.push({ + id: `desktop-smoke:agent-browser-chat:${targetSlug}`, + title: `Desktop agent-browser chat smoke (${target.label})`, + description: 'Open the desktop web app with agent-browser, send a real chat task, and verify the model edits a fixture project through the UI.', + kind: 'desktop-smoke', + baselineTarget: target, + requiredForModes: ['baseline', 'release'], + live: true, + }) + } + return lanes.filter((lane) => lane.requiredForModes.includes(mode)) } diff --git a/scripts/quality-gate/runner.test.ts b/scripts/quality-gate/runner.test.ts index 8012b534..f4db2237 100644 --- a/scripts/quality-gate/runner.test.ts +++ b/scripts/quality-gate/runner.test.ts @@ -20,6 +20,7 @@ describe('quality gate modes', () => { expect(lanes).toContain('baseline-catalog') expect(lanes).toContain('baseline:failing-unit:current-runtime') expect(lanes).toContain('baseline:multi-file-api:current-runtime') + expect(lanes).toContain('desktop-smoke:agent-browser-chat:current-runtime') expect(lanes).not.toContain('native-checks') }) @@ -27,6 +28,7 @@ describe('quality gate modes', () => { const lanes = lanesForMode('release').map((lane) => lane.id) expect(lanes).toContain('pr-checks') expect(lanes).toContain('baseline:failing-unit:current-runtime') + expect(lanes).toContain('desktop-smoke:agent-browser-chat:current-runtime') expect(lanes).toContain('native-checks') }) @@ -40,6 +42,8 @@ describe('quality gate modes', () => { expect(lanes).toContain('baseline:failing-unit:provider-b-model-b') expect(lanes).toContain('baseline:multi-file-api:provider-a-model-a') expect(lanes).toContain('baseline:multi-file-api:provider-b-model-b') + expect(lanes).toContain('desktop-smoke:agent-browser-chat:provider-a-model-a') + expect(lanes).toContain('desktop-smoke:agent-browser-chat:provider-b-model-b') }) }) diff --git a/scripts/quality-gate/runner.ts b/scripts/quality-gate/runner.ts index 131738a8..f5e6b58c 100644 --- a/scripts/quality-gate/runner.ts +++ b/scripts/quality-gate/runner.ts @@ -2,6 +2,7 @@ import { mkdirSync } from 'node:fs' import { join } from 'node:path' import { baselineCases } from './baseline/cases' import { executeBaselineCase } from './baseline/execute' +import { executeDesktopSmoke } from './desktop-smoke/execute' import { lanesForMode } from './modes' import { writeReport } from './reporter' import type { LaneDefinition, LaneResult, QualityGateOptions, QualityGateReport } from './types' @@ -104,6 +105,28 @@ async function runLane(lane: LaneDefinition, options: QualityGateOptions): Promi if (lane.kind === 'baseline-case') { return runBaselineCaseLane(lane, options) } + if (lane.kind === 'desktop-smoke') { + const started = Date.now() + + if (!options.allowLive) { + return { + id: lane.id, + title: lane.title, + status: 'skipped', + durationMs: Date.now() - started, + skipReason: 'desktop agent-browser smoke requires --allow-live', + } + } + + const artifactRoot = options.runOutputDir ?? join(options.rootDir, 'artifacts', 'quality-runs', options.runId ?? 'current') + return executeDesktopSmoke( + options.rootDir, + join(artifactRoot, 'cases', lane.id.replace(/[^a-zA-Z0-9._-]+/g, '-')), + lane.id, + lane.title, + lane.baselineTarget, + ) + } return runCommandLane(lane, options) } diff --git a/scripts/quality-gate/types.ts b/scripts/quality-gate/types.ts index e86368f4..c75fc6a6 100644 --- a/scripts/quality-gate/types.ts +++ b/scripts/quality-gate/types.ts @@ -1,6 +1,6 @@ export type QualityGateMode = 'pr' | 'baseline' | 'release' -export type LaneKind = 'command' | 'baseline-case' +export type LaneKind = 'command' | 'baseline-case' | 'desktop-smoke' export type LaneDefinition = { id: string From 5f59c693c498460628c595799c029fa4682fa038 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=E7=A8=8B=E5=BA=8F=E5=91=98=E9=98=BF=E6=B1=9F=28Relakkes?= =?UTF-8?q?=29?= Date: Sat, 2 May 2026 15:35:59 +0800 Subject: [PATCH 4/8] Let contributors choose quality-gate providers The live baseline previously accepted provider UUIDs, which made the gate hard to run on another contributor's machine. Add a local provider listing command and resolve quality-gate targets from stable provider-name selectors while keeping UUIDs and current runtime support. Constraint: Provider configuration is local machine state under CLAUDE_CONFIG_DIR and must not expose API keys. Rejected: Require contributors to inspect providers.json manually | too error-prone and leaks implementation detail into the workflow Confidence: high Scope-risk: narrow Directive: Keep live-provider baseline selection copyable from quality:providers before adding more live test lanes. Tested: bun test scripts/quality-gate/providerTargets.test.ts Tested: bun test scripts/quality-gate/*.test.ts scripts/quality-gate/baseline/*.test.ts Tested: bun run quality:providers Tested: bun run quality:gate --mode baseline --dry-run --provider-model codingplan:main --provider-model minimax:main Tested: bun run quality:gate --mode baseline --dry-run --provider-model custom:haiku Tested: bun run quality:gate --mode pr --dry-run Tested: bun run check:server --- package.json | 1 + scripts/quality-gate/index.ts | 24 +- scripts/quality-gate/providerTargets.test.ts | 171 ++++++++++++ scripts/quality-gate/providerTargets.ts | 257 +++++++++++++++++++ scripts/quality-gate/providers.ts | 5 + 5 files changed, 443 insertions(+), 15 deletions(-) create mode 100644 scripts/quality-gate/providerTargets.test.ts create mode 100644 scripts/quality-gate/providerTargets.ts create mode 100644 scripts/quality-gate/providers.ts diff --git a/package.json b/package.json index cf8ce53d..34ea1a7a 100644 --- a/package.json +++ b/package.json @@ -18,6 +18,7 @@ "check:native": "cd desktop && bun run build:sidecars && cd src-tauri && cargo check", "check:docs": "npm ci && npm run docs:build", "quality:gate": "bun run scripts/quality-gate/index.ts", + "quality:providers": "bun run scripts/quality-gate/providers.ts", "quality:pr": "bun run quality:gate --mode pr", "quality:baseline": "bun run quality:gate --mode baseline", "quality:release": "bun run quality:gate --mode release", diff --git a/scripts/quality-gate/index.ts b/scripts/quality-gate/index.ts index fceff9c0..68650504 100644 --- a/scripts/quality-gate/index.ts +++ b/scripts/quality-gate/index.ts @@ -1,6 +1,7 @@ #!/usr/bin/env bun import { runQualityGate } from './runner' +import { formatProviderTargets, loadProviderIndex, parseBaselineTargetValues } from './providerTargets' import type { BaselineTarget, QualityGateMode } from './types' function parseArgs(argv: string[]) { @@ -36,30 +37,23 @@ function readMode(value: string | boolean | undefined): QualityGateMode { if (value === 'pr' || value === 'baseline' || value === 'release') { return value } - throw new Error('Usage: bun run quality:gate --mode [--dry-run] [--allow-live]') + throw new Error('Usage: bun run quality:gate --mode [--dry-run] [--allow-live] [--provider-model provider:model[:label]]') } function readBaselineTargets(args: Map>): BaselineTarget[] { const values = (args.get('--provider-model') ?? []) .filter((value): value is string => typeof value === 'string') - .flatMap((value) => value.split(',')) - .map((value) => value.trim()) - .filter(Boolean) - return values.map((value) => { - const [providerId, modelId, rawLabel] = value.split(':') - if (!providerId || !modelId) { - throw new Error(`Invalid --provider-model value "${value}". Expected providerId:modelId[:label].`) - } - return { - providerId, - modelId, - label: rawLabel || `${providerId.slice(0, 8)}-${modelId}`, - } - }) + return parseBaselineTargetValues(values) } const args = parseArgs(process.argv.slice(2)) + +if (hasFlag(args, '--list-providers')) { + console.log(formatProviderTargets(loadProviderIndex())) + process.exit(0) +} + const mode = readMode(firstArg(args, '--mode')) const dryRun = hasFlag(args, '--dry-run') const allowLive = hasFlag(args, '--allow-live') diff --git a/scripts/quality-gate/providerTargets.test.ts b/scripts/quality-gate/providerTargets.test.ts new file mode 100644 index 00000000..e896ba92 --- /dev/null +++ b/scripts/quality-gate/providerTargets.test.ts @@ -0,0 +1,171 @@ +import { describe, expect, test } from 'bun:test' +import { mkdirSync, rmSync, writeFileSync } from 'node:fs' +import { tmpdir } from 'node:os' +import { join } from 'node:path' +import { + formatProviderTargets, + loadProviderIndex, + parseBaselineTargetValues, +} from './providerTargets' + +function withProviderIndex(index: unknown, fn: (configDir: string) => void) { + const configDir = join(tmpdir(), `quality-provider-targets-${crypto.randomUUID()}`) + mkdirSync(join(configDir, 'cc-haha'), { recursive: true }) + writeFileSync(join(configDir, 'cc-haha', 'providers.json'), JSON.stringify(index, null, 2)) + + try { + fn(configDir) + } finally { + rmSync(configDir, { recursive: true, force: true }) + } +} + +describe('quality gate provider targets', () => { + test('lists copyable provider-model selectors without exposing keys', () => { + withProviderIndex({ + activeId: 'provider-kimi', + providers: [ + { + id: 'provider-kimi', + presetId: 'volcengine', + name: 'Volcengine Codingplan', + apiKey: 'secret-key', + baseUrl: 'https://example.invalid', + apiFormat: 'openai_chat', + models: { + main: 'kimi-k2.6', + haiku: 'kimi-k2.6', + sonnet: 'kimi-k2.6-thinking', + opus: 'kimi-k2.6-thinking', + }, + }, + ], + }, (configDir) => { + const output = formatProviderTargets(loadProviderIndex(configDir), configDir) + + expect(output).toContain('Volcengine Codingplan') + expect(output).toContain('--provider-model volcengine-codingplan:main:volcengine-codingplan-main') + expect(output).toContain('--provider-model volcengine-codingplan:sonnet:volcengine-codingplan-sonnet') + expect(output).not.toContain('secret-key') + }) + }) + + test('parses provider-model by provider slug instead of requiring UUID', () => { + withProviderIndex({ + activeId: null, + providers: [ + { + id: 'provider-minimax', + presetId: 'minimax', + name: 'MiniMax', + apiKey: 'secret-key', + baseUrl: 'https://example.invalid', + apiFormat: 'openai_chat', + models: { + main: 'MiniMax-M2.7-highspeed', + haiku: 'MiniMax-M2.7-highspeed', + sonnet: 'MiniMax-M2.7-highspeed', + opus: 'MiniMax-M2.7-highspeed', + }, + }, + ], + }, (configDir) => { + const targets = parseBaselineTargetValues( + ['minimax:MiniMax-M2.7-highspeed'], + loadProviderIndex(configDir), + ) + + expect(targets).toEqual([ + { + providerId: 'provider-minimax', + modelId: 'MiniMax-M2.7-highspeed', + label: 'minimax-MiniMax-M2.7-highspeed', + }, + ]) + }) + }) + + test('parses provider-model by role so model IDs may contain colons', () => { + withProviderIndex({ + activeId: null, + providers: [ + { + id: 'provider-anthropic-proxy', + presetId: 'proxy', + name: 'Anthropic Proxy', + apiKey: 'secret-key', + baseUrl: 'https://example.invalid', + apiFormat: 'anthropic', + models: { + main: 'anthropic/claude-sonnet-4.6', + haiku: 'anthropic/claude-haiku-4.5:thinking', + sonnet: 'anthropic/claude-sonnet-4.6', + opus: 'anthropic/claude-opus-4.7', + }, + }, + ], + }, (configDir) => { + const targets = parseBaselineTargetValues( + ['anthropic-proxy:haiku:proxy-haiku-thinking'], + loadProviderIndex(configDir), + ) + + expect(targets).toEqual([ + { + providerId: 'provider-anthropic-proxy', + modelId: 'anthropic/claude-haiku-4.5:thinking', + label: 'proxy-haiku-thinking', + }, + ]) + }) + }) + + test('does not treat shared preset ids as provider selectors', () => { + withProviderIndex({ + activeId: null, + providers: [ + { + id: 'provider-one', + presetId: 'custom', + name: 'Provider One', + apiKey: 'secret-key', + baseUrl: 'https://example.invalid', + apiFormat: 'anthropic', + models: { main: 'model-one', haiku: 'model-one', sonnet: 'model-one', opus: 'model-one' }, + }, + { + id: 'provider-two', + presetId: 'custom', + name: 'Provider Two', + apiKey: 'secret-key', + baseUrl: 'https://example.invalid', + apiFormat: 'anthropic', + models: { main: 'model-two', haiku: 'model-two', sonnet: 'model-two', opus: 'model-two' }, + }, + ], + }, (configDir) => { + expect(() => parseBaselineTargetValues(['custom:main'], loadProviderIndex(configDir))).toThrow('Unknown provider') + expect(parseBaselineTargetValues(['provider-one:main'], loadProviderIndex(configDir))[0]).toEqual({ + providerId: 'provider-one', + modelId: 'model-one', + label: 'provider-one-model-one', + }) + }) + }) + + test('keeps current runtime and explicit UUID selectors supported', () => { + const targets = parseBaselineTargetValues([ + 'current:current', + '11111111-1111-4111-8111-111111111111:model-a:model-a-live', + ]) + + expect(targets).toEqual([ + { providerId: null, modelId: 'current', label: 'current-runtime' }, + { + providerId: '11111111-1111-4111-8111-111111111111', + modelId: 'model-a', + label: 'model-a-live', + }, + ]) + }) +}) diff --git a/scripts/quality-gate/providerTargets.ts b/scripts/quality-gate/providerTargets.ts new file mode 100644 index 00000000..ede2fc70 --- /dev/null +++ b/scripts/quality-gate/providerTargets.ts @@ -0,0 +1,257 @@ +import { existsSync, readFileSync } from 'node:fs' +import { homedir } from 'node:os' +import { join } from 'node:path' +import type { BaselineTarget } from './types' + +type ProviderModels = { + main?: string + haiku?: string + sonnet?: string + opus?: string +} + +type SavedProviderIndexEntry = { + id: string + name: string + presetId?: string + models?: ProviderModels +} + +export type ProviderIndex = { + activeId: string | null + providers: SavedProviderIndexEntry[] +} + +const DEFAULT_INDEX: ProviderIndex = { activeId: null, providers: [] } +const UUID_PATTERN = /^[0-9a-f]{8}-[0-9a-f]{4}-[1-5][0-9a-f]{3}-[89ab][0-9a-f]{3}-[0-9a-f]{12}$/i + +export function getProviderIndexPath(configDir = process.env.CLAUDE_CONFIG_DIR || join(homedir(), '.claude')) { + return join(configDir, 'cc-haha', 'providers.json') +} + +export function slugifyProviderName(value: string) { + return value + .trim() + .toLowerCase() + .replace(/[^a-z0-9]+/g, '-') + .replace(/^-+|-+$/g, '') +} + +function slugifyLabel(value: string) { + return value + .trim() + .replace(/[^a-zA-Z0-9._-]+/g, '-') + .replace(/^-+|-+$/g, '') +} + +function providerAliases(provider: SavedProviderIndexEntry) { + return new Set([ + provider.id, + provider.name, + provider.name.toLowerCase(), + slugifyProviderName(provider.name), + ].filter(Boolean)) +} + +function modelEntries(provider: SavedProviderIndexEntry) { + const roles = [ + ['main', provider.models?.main], + ['haiku', provider.models?.haiku], + ['sonnet', provider.models?.sonnet], + ['opus', provider.models?.opus], + ] as const + const seen = new Set() + + return roles + .filter((entry): entry is [typeof entry[0], string] => typeof entry[1] === 'string' && entry[1].trim().length > 0) + .filter(([, modelId]) => { + if (seen.has(modelId)) return false + seen.add(modelId) + return true + }) +} + +function splitProviderTarget(value: string) { + const separator = value.indexOf(':') + if (separator === -1) { + return null + } + + return { + providerRef: value.slice(0, separator), + modelAndLabel: value.slice(separator + 1), + } +} + +function splitModelAndLabel(value: string) { + const separator = value.lastIndexOf(':') + if (separator === -1) { + return { modelId: value, label: undefined } + } + + return { + modelId: value.slice(0, separator), + label: value.slice(separator + 1) || undefined, + } +} + +function resolveProviderModel(provider: SavedProviderIndexEntry, modelAndLabel: string) { + const roleEntries = [ + ['main', provider.models?.main], + ['haiku', provider.models?.haiku], + ['sonnet', provider.models?.sonnet], + ['opus', provider.models?.opus], + ] as const + + for (const [role, modelId] of roleEntries) { + if (!modelId) continue + if (modelAndLabel === role) { + return { modelId, label: undefined } + } + if (modelAndLabel.startsWith(`${role}:`)) { + return { modelId, label: modelAndLabel.slice(role.length + 1) || undefined } + } + } + + const knownModels = [...new Set(roleEntries.map(([, modelId]) => modelId).filter((modelId): modelId is string => Boolean(modelId)))] + .sort((left, right) => right.length - left.length) + for (const modelId of knownModels) { + if (modelAndLabel === modelId) { + return { modelId, label: undefined } + } + if (modelAndLabel.startsWith(`${modelId}:`)) { + return { modelId, label: modelAndLabel.slice(modelId.length + 1) || undefined } + } + } + + return splitModelAndLabel(modelAndLabel) +} + +export function loadProviderIndex(configDir?: string): ProviderIndex { + const filePath = getProviderIndexPath(configDir) + if (!existsSync(filePath)) { + return { ...DEFAULT_INDEX, providers: [] } + } + + const parsed = JSON.parse(readFileSync(filePath, 'utf8')) as Partial + return { + activeId: typeof parsed.activeId === 'string' ? parsed.activeId : null, + providers: Array.isArray(parsed.providers) + ? parsed.providers.filter((provider): provider is SavedProviderIndexEntry => ( + provider && + typeof provider.id === 'string' && + typeof provider.name === 'string' + )) + : [], + } +} + +function resolveProviderReference(reference: string, index: ProviderIndex) { + if (reference === 'current' || reference === 'official') { + return null + } + + const normalized = reference.toLowerCase() + const slug = slugifyProviderName(reference) + const matches = index.providers.filter((provider) => { + const aliases = providerAliases(provider) + return aliases.has(reference) || aliases.has(normalized) || aliases.has(slug) + }) + + if (matches.length === 1) { + return matches[0] + } + + if (matches.length > 1) { + throw new Error(`Ambiguous provider reference "${reference}". Use one of these IDs: ${matches.map((provider) => provider.id).join(', ')}`) + } + + if (UUID_PATTERN.test(reference)) { + return { id: reference, name: reference } + } + + throw new Error(`Unknown provider "${reference}". Run "bun run quality:providers" to list available --provider-model values.`) +} + +export function parseBaselineTargetValues(values: string[], index: ProviderIndex = loadProviderIndex()): BaselineTarget[] { + return values + .flatMap((value) => value.split(',')) + .map((value) => value.trim()) + .filter(Boolean) + .map((value) => { + const target = splitProviderTarget(value) + if (!target?.providerRef || !target.modelAndLabel) { + throw new Error(`Invalid --provider-model value "${value}". Expected provider:model[:label]. Run "bun run quality:providers" for copyable values.`) + } + + const { providerRef, modelAndLabel } = target + const provider = resolveProviderReference(providerRef, index) + if (!provider) { + const { modelId, label } = splitModelAndLabel(modelAndLabel) + return { + providerId: null, + modelId, + label: label || (providerRef === 'current' && modelId === 'current' ? 'current-runtime' : `${providerRef}-${modelId}`), + } + } + + const { modelId, label } = resolveProviderModel(provider, modelAndLabel) + const providerSlug = slugifyProviderName(provider.name) || provider.id.slice(0, 8) + return { + providerId: provider.id, + modelId, + label: label || slugifyLabel(`${providerSlug}-${modelId}`), + } + }) +} + +function providerSelector(provider: SavedProviderIndexEntry, index: ProviderIndex) { + const slug = slugifyProviderName(provider.name) + if (!slug) { + return provider.id + } + + const matchingSlugCount = index.providers.filter((candidate) => slugifyProviderName(candidate.name) === slug).length + return matchingSlugCount === 1 ? slug : provider.id +} + +export function formatProviderTargets(index: ProviderIndex = loadProviderIndex(), configDir = process.env.CLAUDE_CONFIG_DIR || join(homedir(), '.claude')) { + const lines = [ + 'Quality gate provider targets', + `Config: ${getProviderIndexPath(configDir)}`, + '', + 'Current/default runtime:', + ' --provider-model current:current:current-runtime', + '', + ] + + if (index.providers.length === 0) { + lines.push('Saved providers: none') + lines.push('') + lines.push('Add a provider in Desktop Settings > Providers, then run this command again.') + return lines.join('\n') + } + + lines.push('Saved providers:') + for (const provider of index.providers) { + const providerSlug = providerSelector(provider, index) + const active = index.activeId === provider.id ? ' (active)' : '' + lines.push(` ${provider.name}${active}`) + lines.push(` id: ${provider.id}`) + lines.push(` selector: ${providerSlug}`) + + const models = modelEntries(provider) + if (models.length === 0) { + lines.push(' models: none configured') + continue + } + + for (const [role, modelId] of models) { + const label = slugifyLabel(`${providerSlug}-${role}`) + lines.push(` ${role}: ${modelId}`) + lines.push(` --provider-model ${providerSlug}:${role}:${label}`) + } + } + + return lines.join('\n') +} diff --git a/scripts/quality-gate/providers.ts b/scripts/quality-gate/providers.ts new file mode 100644 index 00000000..bf2711c4 --- /dev/null +++ b/scripts/quality-gate/providers.ts @@ -0,0 +1,5 @@ +#!/usr/bin/env bun + +import { formatProviderTargets, loadProviderIndex } from './providerTargets' + +console.log(formatProviderTargets(loadProviderIndex())) From ef430c7618e9d6cce12c22fbb0d41a276e41d63e Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=E7=A8=8B=E5=BA=8F=E5=91=98=E9=98=BF=E6=B1=9F=28Relakkes?= =?UTF-8?q?=29?= Date: Sat, 2 May 2026 15:45:04 +0800 Subject: [PATCH 5/8] Document contributor quality gates Contributors need a visible path for local verification instead of relying on design notes or maintainer memory. Add a GitHub-facing CONTRIBUTING entrypoint, bilingual docs pages, README links, and VitePress sidebar navigation for PR gates, live model baselines, provider selection, reports, and release checks. Constraint: Live baseline providers are local machine state; docs must tell contributors how to list and choose their own providers without maintainer UUIDs. Rejected: Keep the instructions only in the quality-gate design doc | too hidden for clone-and-contribute workflows Confidence: high Scope-risk: narrow Directive: Keep quality-gate commands documented wherever contributor onboarding links are exposed. Tested: bun run check:docs Tested: bun run quality:pr Not-tested: live baseline after the docs-only wording change --- CONTRIBUTING.md | 31 +++++++ README.en.md | 1 + README.md | 1 + docs/.vitepress/config.mts | 2 + docs/en/guide/contributing.md | 164 ++++++++++++++++++++++++++++++++++ docs/guide/contributing.md | 164 ++++++++++++++++++++++++++++++++++ 6 files changed, 363 insertions(+) create mode 100644 CONTRIBUTING.md create mode 100644 docs/en/guide/contributing.md create mode 100644 docs/guide/contributing.md diff --git a/CONTRIBUTING.md b/CONTRIBUTING.md new file mode 100644 index 00000000..d8991d94 --- /dev/null +++ b/CONTRIBUTING.md @@ -0,0 +1,31 @@ +# Contributing + +Thanks for helping improve Claude Code Haha. + +For the full contributor guide, including local checks, live model baselines, quality-gate reports, and PR expectations, see: + +- Chinese: [docs/guide/contributing.md](docs/guide/contributing.md) +- English: [docs/en/guide/contributing.md](docs/en/guide/contributing.md) + +Most contributors should run this before opening a PR: + +```bash +bun install +bun run quality:pr +``` + +If you run adapter or native checks on a fresh clone, install adapter dependencies too: + +```bash +cd adapters +bun install +``` + +If your change touches the desktop chat path, provider/runtime selection, CLI bridge, permissions, tools, file editing, or release packaging, also run a live baseline with your own local model provider: + +```bash +bun run quality:providers +bun run quality:gate --mode baseline --allow-live --provider-model :main +``` + +Quality reports are written under `artifacts/quality-runs//`. diff --git a/README.en.md b/README.en.md index 21319330..48cc93ae 100644 --- a/README.en.md +++ b/README.en.md @@ -195,6 +195,7 @@ http://127.0.0.1:2024 |------|------| | [Environment Variables](docs/en/guide/env-vars.md) | Full env var reference and configuration methods | | [Third-Party Models](docs/en/guide/third-party-models.md) | Using OpenAI / DeepSeek / Ollama and other non-Anthropic models | +| [Contributing](docs/en/guide/contributing.md) | Local tests, live model baselines, PR gates, and release gates | | [Memory System](docs/memory/01-usage-guide.md) | Cross-session persistent memory usage and implementation | | [Multi-Agent System](docs/agent/01-usage-guide.md) | Agent orchestration, parallel tasks and Teams collaboration | | [Skills System](docs/skills/01-usage-guide.md) | Extensible capability plugins, custom workflows and conditional activation | diff --git a/README.md b/README.md index f136251a..a778d439 100644 --- a/README.md +++ b/README.md @@ -195,6 +195,7 @@ http://127.0.0.1:2024 |------|------| | [环境变量](docs/guide/env-vars.md) | 完整环境变量参考和配置方式 | | [第三方模型](docs/guide/third-party-models.md) | 接入 OpenAI / DeepSeek / Ollama 等非 Anthropic 模型 | +| [贡献与质量门禁](docs/guide/contributing.md) | 本地测试、真实模型 baseline、PR 和 release 门禁 | | [记忆系统](docs/memory/01-usage-guide.md) | 跨会话持久化记忆的使用与实现 | | [多 Agent 系统](docs/agent/01-usage-guide.md) | 多代理编排、并行任务执行与 Teams 协作 | | [Skills 系统](docs/skills/01-usage-guide.md) | 可扩展能力插件、自定义工作流与条件激活 | diff --git a/docs/.vitepress/config.mts b/docs/.vitepress/config.mts index ca3639ff..66547160 100644 --- a/docs/.vitepress/config.mts +++ b/docs/.vitepress/config.mts @@ -24,6 +24,7 @@ const zhSidebar = [ { text: '第三方模型', link: '/guide/third-party-models' }, { text: '全局使用', link: '/guide/global-usage' }, { text: '常见问题', link: '/guide/faq' }, + { text: '贡献与质量门禁', link: '/guide/contributing' }, ], }, { @@ -109,6 +110,7 @@ const enSidebar = [ { text: 'Third-Party Models', link: '/en/guide/third-party-models' }, { text: 'Global Usage', link: '/en/guide/global-usage' }, { text: 'FAQ', link: '/en/guide/faq' }, + { text: 'Contributing', link: '/en/guide/contributing' }, ], }, { diff --git a/docs/en/guide/contributing.md b/docs/en/guide/contributing.md new file mode 100644 index 00000000..4ef47fee --- /dev/null +++ b/docs/en/guide/contributing.md @@ -0,0 +1,164 @@ +# Contributing and Local Quality Gates + +This guide explains how to install, develop, test, and run the local quality gates before opening a PR. The goal is to help maintainers and contributors answer one question before review: did this change break the core Coding Agent workflow? + +## Setup + +Install root dependencies with Bun: + +```bash +bun install +``` + +If your change touches `desktop/`, also install desktop dependencies: + +```bash +cd desktop +bun install +``` + +If your change touches `adapters/`, or if you run `check:adapters` / `check:native`, install adapter dependencies: + +```bash +cd adapters +bun install +``` + +Do not commit local artifacts such as `artifacts/quality-runs/`, `node_modules/`, or `desktop/node_modules/`. + +## Required PR Gate + +Before opening a normal PR, run: + +```bash +bun run quality:pr +``` + +This gate does not call real models, so every contributor can run it locally. It writes reports to: + +```text +artifacts/quality-runs//report.md +artifacts/quality-runs//report.json +``` + +Include the commands you ran and the report summary in your PR description. + +## Area-Specific Checks + +Run the checks that match the files you changed: + +```bash +bun run check:server # Server API, WebSocket, providers, sessions, and related tests +bun run check:desktop # Desktop lint, Vitest, and production build +bun run check:adapters # IM adapter tests +bun run check:native # Desktop sidecars and Tauri native checks +bun run check:docs # Docs build, using npm ci + docs:build +``` + +Focused tests are fine while developing, but run `bun run quality:pr` before sending the PR. + +## Live Model Baseline + +`quality:baseline` runs real Coding Agent tasks: it starts the local server, creates isolated fixtures, asks a model through chat to fix code, runs tests, and saves transcripts, diffs, verification logs, and a report. + +The default baseline command does not call real models: + +```bash +bun run quality:baseline +``` + +To actually call models, pass `--allow-live` and choose a local provider. + +First list your local providers and copyable selectors: + +```bash +bun run quality:providers +``` + +Example output: + +```text +Saved providers: + MiniMax + selector: minimax + main: MiniMax-M2.7-highspeed + --provider-model minimax:main:minimax-main +``` + +Copy one of the listed values: + +```bash +bun run quality:gate --mode baseline --allow-live --provider-model minimax:main:minimax-main +``` + +You can run multiple models in one pass: + +```bash +bun run quality:gate --mode baseline --allow-live \ + --provider-model codingplan:main:codingplan-main \ + --provider-model minimax:main:minimax-main +``` + +Provider selectors come from the providers saved in your local Desktop Settings > Providers page. Contributors do not need the maintainer's provider UUIDs or vendor accounts. They can add their own provider locally, run `bun run quality:providers`, and choose their own model. + +## When To Run The Baseline + +Run the live baseline for changes touching: + +- Desktop chat, session resume, WebSocket, or the CLI bridge +- Provider, model, or runtime selection +- Permissions, tool calls, file edits, and task execution +- agent-browser smoke, Computer Use, Skills, or MCP +- Release preparation or broad cross-module refactors + +If you do not have model access, still run `bun run quality:pr` and state in the PR why the live baseline was not run. + +## Release Gate + +Before a release, run release mode: + +```bash +bun run quality:gate --mode release --allow-live --provider-model :main +``` + +Release mode composes PR checks, baseline catalog validation, live baseline cases, desktop smoke, and native checks. Reports are written to `artifacts/quality-runs//`. + +## PR Workflow + +1. Create a product branch such as `fix/session-reconnect` or `feat/provider-quality-gate`. +2. Install dependencies and make the change. +3. Add tests for behavior changes. +4. Run focused checks for the affected area. +5. Run `bun run quality:pr`. +6. Run the live baseline for high-risk changes. +7. In the PR description, include user impact, verification commands, report summary, and known risks. + +## FAQ + +### Can I run checks without a provider? + +Yes. Run the normal PR gate: + +```bash +bun run quality:pr +``` + +Only the live baseline needs a real model. Add your provider in Desktop Settings > Providers, then run: + +```bash +bun run quality:providers +``` + +### What if provider selectors conflict? + +If two provider names produce the same selector, `quality:providers` falls back to the provider ID. Copy the `--provider-model ...` value it prints. + +### What if a model ID contains a colon? + +Prefer role selectors: + +```bash +--provider-model custom:haiku:custom-haiku +``` + +The runner resolves `haiku` to the real model ID from your local provider configuration. diff --git a/docs/guide/contributing.md b/docs/guide/contributing.md new file mode 100644 index 00000000..cd40ad2c --- /dev/null +++ b/docs/guide/contributing.md @@ -0,0 +1,164 @@ +# 贡献指南与本地质量门禁 + +这份文档说明贡献代码前应该如何在本地安装、开发、测试和运行质量门禁。目标是让维护者和贡献者都能在提交 PR 前回答一个问题:这次改动有没有破坏核心 Coding Agent 工作流。 + +## 环境准备 + +项目根目录使用 Bun: + +```bash +bun install +``` + +如果改动涉及 `desktop/`,也安装桌面端依赖: + +```bash +cd desktop +bun install +``` + +如果改动涉及 `adapters/`,或者要运行 `check:adapters` / `check:native`,安装 adapter 依赖: + +```bash +cd adapters +bun install +``` + +不要提交本地运行产物,例如 `artifacts/quality-runs/`、`node_modules/`、`desktop/node_modules/`。 + +## 普通 PR 必跑门禁 + +普通贡献者在提交 PR 前至少运行: + +```bash +bun run quality:pr +``` + +这个门禁不调用真实大模型,适合所有人本地运行。它会生成报告: + +```text +artifacts/quality-runs//report.md +artifacts/quality-runs//report.json +``` + +PR 描述里请贴出你实际运行的命令和 summary。 + +## 按改动范围补充测试 + +根据你改动的区域补充运行: + +```bash +bun run check:server # 服务端 API、WebSocket、provider、会话等测试 +bun run check:desktop # 桌面端 lint、Vitest、生产构建 +bun run check:adapters # IM adapter 测试 +bun run check:native # 桌面 sidecar 与 Tauri native 检查 +bun run check:docs # 文档构建,使用 npm ci + docs:build +``` + +如果只改了很窄的文件,也可以先跑对应的定向测试,但 PR 前仍应跑 `bun run quality:pr`。 + +## 真实模型 Baseline + +`quality:baseline` 用来跑真实 Coding Agent 任务:启动本地服务端、创建隔离 fixture、让模型通过聊天修代码、跑测试,并保存 transcript、diff、verification log 和报告。 + +默认命令不会调用真实模型: + +```bash +bun run quality:baseline +``` + +要真正跑模型,必须显式加 `--allow-live` 并选择本机 provider。 + +先列出本机可用 provider 和可复制参数: + +```bash +bun run quality:providers +``` + +输出示例: + +```text +Saved providers: + MiniMax + selector: minimax + main: MiniMax-M2.7-highspeed + --provider-model minimax:main:minimax-main +``` + +复制输出里的参数运行 baseline: + +```bash +bun run quality:gate --mode baseline --allow-live --provider-model minimax:main:minimax-main +``` + +可以一次跑多个模型: + +```bash +bun run quality:gate --mode baseline --allow-live \ + --provider-model codingplan:main:codingplan-main \ + --provider-model minimax:main:minimax-main +``` + +`provider` selector 来自桌面端「Settings > Providers」里保存的本机配置。别人 clone 代码后不需要知道你的 provider UUID,也不需要使用你的供应商;他们可以在自己的桌面端添加 provider 后运行 `bun run quality:providers` 选择自己的模型。 + +## 什么时候必须跑 Baseline + +以下改动建议跑 live baseline: + +- 桌面聊天、会话恢复、WebSocket、CLI bridge +- provider/model/runtime 选择 +- 权限、工具调用、文件编辑、任务执行 +- agent-browser smoke、Computer Use、Skills、MCP +- release 前或风险较大的跨模块重构 + +如果没有模型额度,至少运行 `bun run quality:pr`,并在 PR 里说明未跑 live baseline 的原因。 + +## Release 门禁 + +发版前使用 release 模式: + +```bash +bun run quality:gate --mode release --allow-live --provider-model :main +``` + +release 模式会组合 PR checks、baseline catalog、live baseline、desktop smoke 和 native checks。发版报告同样写入 `artifacts/quality-runs//`。 + +## PR 提交流程 + +1. 新建普通产品分支,例如 `fix/session-reconnect` 或 `feat/provider-quality-gate`。 +2. 安装依赖并完成改动。 +3. 为行为变化补测试。 +4. 运行相关定向测试。 +5. 运行 `bun run quality:pr`。 +6. 对高风险改动运行 live baseline。 +7. 在 PR 描述里写清楚用户影响、测试命令、报告 summary、已知风险。 + +## 常见问题 + +### 没有 provider 可以跑吗? + +可以跑普通门禁: + +```bash +bun run quality:pr +``` + +只有 live baseline 需要真实模型。先在桌面端 Settings > Providers 添加自己的 provider,再运行: + +```bash +bun run quality:providers +``` + +### provider selector 冲突怎么办? + +如果两个 provider 名称生成了相同 selector,`quality:providers` 会退回输出 provider ID。直接复制它给出的 `--provider-model ...` 即可。 + +### 模型 ID 里带冒号怎么办? + +优先使用角色选择,例如: + +```bash +--provider-model custom:haiku:custom-haiku +``` + +脚本会把 `haiku` 解析成本机 provider 配置里的真实模型 ID。 From 9ed4588db3fe60cbaffbc1e1ee8ca3634b675d7c Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=E7=A8=8B=E5=BA=8F=E5=91=98=E9=98=BF=E6=B1=9F=28Relakkes?= =?UTF-8?q?=29?= Date: Sat, 2 May 2026 15:50:00 +0800 Subject: [PATCH 6/8] Teach agents to run quality gates automatically AGENTS.md is the operational contract future CodingAgents read before editing this repo. Record when to run PR, targeted, live-baseline, release, docs, and native gates so users do not need to manually enumerate the commands each time. Constraint: Live baseline depends on local provider and quota state. Rejected: Leave this only in CONTRIBUTING.md | human-facing docs do not reliably steer future agents. Confidence: high Scope-risk: narrow Directive: Do not claim PR or release readiness without running the corresponding gate or reporting the blocker. Tested: git diff --check Tested: bun run quality:gate --mode pr --dry-run Not-tested: full quality:pr because this is an AGENTS-only guidance change. --- AGENTS.md | 21 +++++++++++++++++++++ 1 file changed, 21 insertions(+) diff --git a/AGENTS.md b/AGENTS.md index a6426894..c747a369 100644 --- a/AGENTS.md +++ b/AGENTS.md @@ -13,6 +13,10 @@ Install root dependencies with `bun install`, then install desktop dependencies - `cd desktop && bun run build`: type-check and produce a production web build. - `cd desktop && bun run test`: run Vitest suites. - `cd desktop && bun run lint`: run TypeScript no-emit checks. +- `bun run quality:providers`: list configured provider/model selectors for live agent baselines. +- `bun run quality:pr`: run the local PR quality gate and write a report under `artifacts/quality-runs/`. +- `bun run quality:gate --mode baseline --allow-live --provider-model `: run live Coding Agent baseline cases, including desktop agent-browser smoke. +- `bun run quality:gate --mode release --allow-live --provider-model `: run the release gate with live baseline coverage. ## Desktop Release Workflow - Desktop releases are built remotely by GitHub Actions, not by uploading local build artifacts. @@ -32,6 +36,23 @@ Use TypeScript with 2-space indentation, ESM imports, and no semicolons to match ## Testing Guidelines Desktop tests use Vitest with Testing Library in a `jsdom` environment. Name tests `*.test.ts` or `*.test.tsx`; colocate focused tests near the file or place broader coverage in `desktop/src/__tests__/`. No coverage gate is configured, so add regression tests for any behavior you change and run the relevant suites before opening a PR. +## Quality Gate Automation +Future Coding Agents should run the right local gate themselves before claiming a change is ready. Do not ask the user to manually run the commands unless credentials, local model access, or machine resources are missing. + +- For normal code changes, run the narrow relevant check first, then `bun run quality:pr` before a PR-ready or merge-ready claim. +- Use `bun run check:server` for `src/server`, `src/tools`, provider/runtime, MCP, OAuth, WebSocket, or API behavior changes. +- Use `bun run check:desktop` for `desktop/src` UI, stores, API clients, and desktop web behavior changes. +- Use `bun run check:native` for `desktop/src-tauri`, sidecars, native packaging, release, or platform startup behavior changes. +- Use `bun run check:adapters` for `adapters/`; on a fresh checkout run `cd adapters && bun install` first if dependencies are missing. +- Use `bun run check:docs` for docs, VitePress, README, or docs workflow changes. +- For chat, agent loop, tool execution, provider routing, desktop chat UI, CLI task execution, or other core Coding Agent paths, also run a live baseline when local providers are available: first `bun run quality:providers`, then choose one or more copyable selectors and run `bun run quality:gate --mode baseline --allow-live --provider-model `. +- For release readiness, run `bun run quality:gate --mode release --allow-live --provider-model ` with at least one real provider/model selector. Prefer multiple providers when quota is available. +- If no live provider is configured, or a provider quota/key is unavailable, run the non-live gate anyway and report the live-baseline blocker explicitly instead of claiming full release confidence. +- `bun run check:docs` executes `npm ci`, which can rebuild root `node_modules`. Run docs checks sequentially, not in parallel with `quality:pr`, `check:native`, or other commands that depend on the same installed packages. +- Quality reports are written to `artifacts/quality-runs//`. Summarize the final report path and the pass/fail counts in handoffs and PR descriptions. +- Do not commit generated `artifacts/quality-runs/`, local `.omx/` state, `node_modules/`, `desktop/node_modules/`, or adapter dependency folders. +- Do not claim "complete", "ready to merge", or "ready to release" without either running the matching gate or naming the exact blocker that prevented it. + ## Commit & Pull Request Guidelines Recent history follows Conventional Commit prefixes such as `feat:`, `fix:`, and `docs:`. Keep subjects imperative and scoped to one change. PRs should explain the user-visible impact, list verification steps, link related issues, and include screenshots for desktop or docs UI changes. Keep diffs reviewable and call out any follow-up work or known gaps. Branch names should use normal product prefixes such as `fix/xxx`, `feat/xxx`, or `docs/xxx`; do not create `codex/`-prefixed branches in this repository. From 6b83ffa4f52f2522fb31b19bbf736632b37dfb8c Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=E7=A8=8B=E5=BA=8F=E5=91=98=E9=98=BF=E6=B1=9F=28Relakkes?= =?UTF-8?q?=29?= Date: Sat, 2 May 2026 16:08:28 +0800 Subject: [PATCH 7/8] Report the full quality gate impact Release quality gates are used as maintainer evidence, so a live baseline failure must not hide later provider or desktop-smoke lanes. Run every lane and summarize all pass/fail statuses so reviewers can tell whether the whole matrix was exercised. Constraint: Live model lanes can fail independently and may be slow, but release evidence needs complete coverage. Rejected: Keep fail-fast behavior | it makes reports ambiguous after the first live failure. Confidence: high Scope-risk: narrow Directive: Do not reintroduce fail-fast behavior for release gates without adding explicit unexecuted-lane reporting. Tested: bun test scripts/quality-gate/runner.test.ts scripts/quality-gate/baseline/cases.test.ts scripts/quality-gate/providerTargets.test.ts Tested: bun run quality:gate --mode release --allow-live --provider-model codingplan:main:codingplan-main --provider-model minimax:main:minimax-main Not-tested: clean-worktree release rerun before this commit; run immediately after committing. --- scripts/quality-gate/runner.test.ts | 54 +++++++++++++++++++++++++++-- scripts/quality-gate/runner.ts | 17 ++++++--- 2 files changed, 64 insertions(+), 7 deletions(-) diff --git a/scripts/quality-gate/runner.test.ts b/scripts/quality-gate/runner.test.ts index f4db2237..b120f810 100644 --- a/scripts/quality-gate/runner.test.ts +++ b/scripts/quality-gate/runner.test.ts @@ -4,8 +4,8 @@ import { tmpdir } from 'node:os' import { join } from 'node:path' import { lanesForMode } from './modes' import { renderMarkdownReport } from './reporter' -import { runQualityGate } from './runner' -import type { QualityGateReport } from './types' +import { runQualityGate, runQualityGateLanes } from './runner' +import type { LaneDefinition, QualityGateReport } from './types' describe('quality gate modes', () => { test('pr mode includes existing path-aware PR checks', () => { @@ -70,6 +70,56 @@ describe('runQualityGate', () => { rmSync(artifactsDir, { recursive: true, force: true }) } }) + + test('continues through later lanes after a failure so reports show full impact', async () => { + const artifactsDir = mkdtempSync(join(tmpdir(), 'quality-gate-test-')) + const lanes: LaneDefinition[] = [ + { + id: 'first-lane', + title: 'First lane', + description: 'Fails intentionally', + kind: 'command', + command: ['false'], + requiredForModes: ['pr'], + }, + { + id: 'second-lane', + title: 'Second lane', + description: 'Still runs', + kind: 'command', + command: ['true'], + requiredForModes: ['pr'], + }, + ] + + try { + const { report } = await runQualityGateLanes({ + mode: 'pr', + dryRun: false, + allowLive: false, + baselineTargets: [], + rootDir: process.cwd(), + artifactsDir, + runId: 'continue-after-failure-test', + }, lanes, async (lane) => ({ + id: lane.id, + title: lane.title, + status: lane.id === 'first-lane' ? 'failed' : 'passed', + command: lane.command, + durationMs: 1, + exitCode: lane.id === 'first-lane' ? 1 : 0, + })) + + expect(report.results.map((result) => result.id)).toEqual(['first-lane', 'second-lane']) + expect(report.summary).toEqual({ + passed: 1, + failed: 1, + skipped: 0, + }) + } finally { + rmSync(artifactsDir, { recursive: true, force: true }) + } + }) }) describe('renderMarkdownReport', () => { diff --git a/scripts/quality-gate/runner.ts b/scripts/quality-gate/runner.ts index f5e6b58c..f03990a4 100644 --- a/scripts/quality-gate/runner.ts +++ b/scripts/quality-gate/runner.ts @@ -7,6 +7,8 @@ import { lanesForMode } from './modes' import { writeReport } from './reporter' import type { LaneDefinition, LaneResult, QualityGateOptions, QualityGateReport } from './types' +type LaneExecutor = (lane: LaneDefinition, options: QualityGateOptions) => Promise + function nowId() { return new Date().toISOString().replace(/[:.]/g, '-') } @@ -140,6 +142,14 @@ function summarize(results: LaneResult[]) { } export async function runQualityGate(options: QualityGateOptions) { + return runQualityGateLanes(options, lanesForMode(options.mode, options.baselineTargets)) +} + +export async function runQualityGateLanes( + options: QualityGateOptions, + lanes: LaneDefinition[], + executeLane: LaneExecutor = runLane, +) { const runId = options.runId ?? nowId() const startedAt = new Date().toISOString() const artifactsRoot = options.artifactsDir ?? join(options.rootDir, 'artifacts', 'quality-runs') @@ -148,12 +158,9 @@ export async function runQualityGate(options: QualityGateOptions) { const runOptions = { ...options, runId, runOutputDir: outputDir } const results: LaneResult[] = [] - for (const lane of lanesForMode(options.mode, options.baselineTargets)) { - const result = await runLane(lane, runOptions) + for (const lane of lanes) { + const result = await executeLane(lane, runOptions) results.push(result) - if (result.status === 'failed') { - break - } } const report: QualityGateReport = { From 6f9f2c61e38bd38722e5f4244ccac58dc77a3283 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=E7=A8=8B=E5=BA=8F=E5=91=98=E9=98=BF=E6=B1=9F=28Relakkes?= =?UTF-8?q?=29?= Date: Sat, 2 May 2026 16:20:49 +0800 Subject: [PATCH 8/8] Stop tracking local Superpowers plans Superpowers plan/spec files are local execution artifacts, and this repo already ignores docs/superpowers/. Remove the previously tracked plan files from the index so future quality-gate work does not mix local planning output into product commits. Constraint: Local Superpowers files should remain available on disk but not be versioned. Rejected: Add another ignore rule | docs/superpowers/ is already ignored, and the problem was tracked files bypassing ignore rules. Confidence: high Scope-risk: narrow Directive: Keep generated quality reports and Superpowers planning artifacts out of repository commits. Tested: git diff --check --cached Tested: git ls-files -ci --exclude-standard Not-tested: runtime gates because this is repository hygiene only. --- docs/superpowers/plans/2026-04-09-skill-ui.md | 1107 ----------------- .../specs/2026-04-09-skill-ui-design.md | 271 ---- 2 files changed, 1378 deletions(-) delete mode 100644 docs/superpowers/plans/2026-04-09-skill-ui.md delete mode 100644 docs/superpowers/specs/2026-04-09-skill-ui-design.md diff --git a/docs/superpowers/plans/2026-04-09-skill-ui.md b/docs/superpowers/plans/2026-04-09-skill-ui.md deleted file mode 100644 index 9b557eb1..00000000 --- a/docs/superpowers/plans/2026-04-09-skill-ui.md +++ /dev/null @@ -1,1107 +0,0 @@ -# Skills UI Implementation Plan - -> **For agentic workers:** REQUIRED SUB-SKILL: Use superpowers:subagent-driven-development (recommended) or superpowers:executing-plans to implement this plan task-by-task. Steps use checkbox (`- [ ]`) syntax for tracking. - -**Goal:** Add a Skills browser to the Desktop Web APP Settings page, with API backend, file tree navigation, and Markdown/code content preview. - -**Architecture:** Server reads skill directories from `~/.claude/skills/`, exposes two REST endpoints (list + detail). Desktop frontend adds a "Skills" tab to Settings with list/detail views, reusing existing `MarkdownRenderer` and `CodeViewer` components. - -**Tech Stack:** Bun server (REST API), React + Zustand + Tailwind CSS (desktop), marked + react-shiki (rendering) - ---- - -## File Structure - -| Action | File | Responsibility | -|--------|------|----------------| -| Create | `src/server/api/skills.ts` | Server API handler: list skills, read skill directory tree + files | -| Modify | `src/server/router.ts` | Register `/api/skills` route | -| Create | `desktop/src/types/skill.ts` | Shared type definitions | -| Create | `desktop/src/api/skills.ts` | API client for skills endpoints | -| Create | `desktop/src/stores/skillStore.ts` | Zustand store for skill state | -| Create | `desktop/src/components/skills/SkillList.tsx` | Grouped skill list view | -| Create | `desktop/src/components/skills/SkillDetail.tsx` | File tree + content viewer | -| Modify | `desktop/src/pages/Settings.tsx` | Add Skills tab + SkillSettings wrapper | -| Modify | `desktop/src/i18n/locales/en.ts` | English translation keys | -| Modify | `desktop/src/i18n/locales/zh.ts` | Chinese translation keys | - ---- - -### Task 1: Desktop Type Definitions - -**Files:** -- Create: `desktop/src/types/skill.ts` - -- [ ] **Step 1: Create type definitions** - -```typescript -// desktop/src/types/skill.ts - -export type SkillSource = 'user' | 'project' | 'plugin' | 'mcp' | 'bundled' - -export type SkillMeta = { - name: string - displayName?: string - description: string - source: SkillSource - userInvocable: boolean - version?: string - contentLength: number - hasDirectory: boolean - pluginName?: string -} - -export type FileTreeNode = { - name: string - path: string - type: 'file' | 'directory' - children?: FileTreeNode[] -} - -export type SkillFile = { - path: string - content: string - language: string -} - -export type SkillDetail = { - meta: SkillMeta - tree: FileTreeNode[] - files: SkillFile[] - skillRoot: string -} -``` - -- [ ] **Step 2: Commit** - -```bash -git add desktop/src/types/skill.ts -git commit -m "feat(skills): add type definitions for Skills UI" -``` - ---- - -### Task 2: Server API Handler - -**Files:** -- Create: `src/server/api/skills.ts` -- Modify: `src/server/router.ts:1-79` - -- [ ] **Step 1: Create the skills API handler** - -```typescript -// src/server/api/skills.ts - -/** - * Skills REST API - * - * GET /api/skills — List all installed skills (metadata only) - * GET /api/skills/detail — Full skill data (tree + files) - * ?source=user&name=xxx - */ - -import * as os from 'os' -import * as path from 'path' -import * as fs from 'fs/promises' -import { parse as parseYaml } from 'yaml' -import { ApiError, errorResponse } from '../middleware/errorHandler.js' - -// ─── Types (server-local, mirrors desktop/src/types/skill.ts) ──────────────── - -type SkillMeta = { - name: string - displayName?: string - description: string - source: 'user' | 'project' - userInvocable: boolean - version?: string - contentLength: number - hasDirectory: boolean -} - -type FileTreeNode = { - name: string - path: string - type: 'file' | 'directory' - children?: FileTreeNode[] -} - -type SkillFile = { - path: string - content: string - language: string -} - -// ─── Constants ─────────────────────────────────────────────────────────────── - -const MAX_FILES = 50 -const MAX_FILE_SIZE = 100 * 1024 // 100 KB -const SKIP_ENTRIES = new Set(['node_modules', '.git', '__pycache__', '.DS_Store']) - -const LANG_MAP: Record = { - md: 'markdown', ts: 'typescript', tsx: 'typescript', - js: 'javascript', jsx: 'javascript', json: 'json', - yaml: 'yaml', yml: 'yaml', sh: 'bash', bash: 'bash', - py: 'python', toml: 'toml', css: 'css', html: 'html', - txt: 'text', xml: 'xml', sql: 'sql', rs: 'rust', go: 'go', -} - -// ─── Helpers ───────────────────────────────────────────────────────────────── - -function detectLanguage(filename: string): string { - const ext = filename.split('.').pop()?.toLowerCase() || '' - return LANG_MAP[ext] || 'text' -} - -function parseFrontmatter(content: string): { - frontmatter: Record - body: string -} { - const match = content.match(/^---\s*\n([\s\S]*?)\n---\s*\n?/) - if (!match) return { frontmatter: {}, body: content } - try { - const frontmatter = (parseYaml(match[1]) as Record) || {} - return { frontmatter, body: content.slice(match[0].length) } - } catch { - return { frontmatter: {}, body: content } - } -} - -function getUserSkillsDir(): string { - return path.join(os.homedir(), '.claude', 'skills') -} - -async function loadSkillMeta( - skillDir: string, - skillName: string, - source: 'user' | 'project', -): Promise { - const skillFile = path.join(skillDir, 'SKILL.md') - try { - const raw = await fs.readFile(skillFile, 'utf-8') - const { frontmatter, body } = parseFrontmatter(raw) - - const description = - (frontmatter.description as string) || - body - .split('\n') - .find((l) => l.trim().length > 0) - ?.trim() || - 'No description' - - return { - name: skillName, - displayName: (frontmatter.name as string) || undefined, - description, - source, - userInvocable: frontmatter['user-invocable'] !== false, - version: frontmatter.version != null ? String(frontmatter.version) : undefined, - contentLength: raw.length, - hasDirectory: true, - } - } catch { - return null - } -} - -async function buildFileTree( - dirPath: string, -): Promise<{ tree: FileTreeNode[]; files: SkillFile[] }> { - const tree: FileTreeNode[] = [] - const files: SkillFile[] = [] - let fileCount = 0 - - async function walk(currentPath: string, nodes: FileTreeNode[]) { - if (fileCount >= MAX_FILES) return - - let entries: import('fs').Dirent[] - try { - entries = await fs.readdir(currentPath, { withFileTypes: true }) - } catch { - return - } - - // directories first, then alphabetical - entries.sort((a, b) => { - if (a.isDirectory() !== b.isDirectory()) return a.isDirectory() ? -1 : 1 - return a.name.localeCompare(b.name) - }) - - for (const entry of entries) { - if (fileCount >= MAX_FILES) break - if (SKIP_ENTRIES.has(entry.name) || entry.name.startsWith('.')) continue - - const fullPath = path.join(currentPath, entry.name) - const relPath = path.relative(dirPath, fullPath) - - if (entry.isDirectory()) { - const node: FileTreeNode = { - name: entry.name, - path: relPath, - type: 'directory', - children: [], - } - nodes.push(node) - await walk(fullPath, node.children!) - if (node.children!.length === 0) delete node.children - } else if (entry.isFile()) { - nodes.push({ name: entry.name, path: relPath, type: 'file' }) - - try { - const stat = await fs.stat(fullPath) - if (stat.size <= MAX_FILE_SIZE) { - const content = await fs.readFile(fullPath, 'utf-8') - files.push({ - path: relPath, - content, - language: detectLanguage(entry.name), - }) - fileCount++ - } - } catch { - // skip unreadable files - } - } - } - } - - await walk(dirPath, tree) - return { tree, files } -} - -// ─── Router ────────────────────────────────────────────────────────────────── - -export async function handleSkillsApi( - req: Request, - url: URL, - segments: string[], -): Promise { - try { - if (req.method !== 'GET') { - throw new ApiError(405, `Method ${req.method} not allowed`, 'METHOD_NOT_ALLOWED') - } - - const sub = segments[2] - - switch (sub) { - case undefined: - return await listSkills() - case 'detail': - return await getSkillDetail(url) - default: - throw ApiError.notFound(`Unknown skills endpoint: ${sub}`) - } - } catch (error) { - return errorResponse(error) - } -} - -// ─── Handlers ──────────────────────────────────────────────────────────────── - -async function listSkills(): Promise { - const skillsDir = getUserSkillsDir() - const skills: SkillMeta[] = [] - - try { - const entries = await fs.readdir(skillsDir, { withFileTypes: true }) - for (const entry of entries) { - if (!entry.isDirectory() || entry.name.startsWith('.')) continue - const meta = await loadSkillMeta( - path.join(skillsDir, entry.name), - entry.name, - 'user', - ) - if (meta) skills.push(meta) - } - } catch { - // skills dir doesn't exist — return empty list - } - - skills.sort((a, b) => a.name.localeCompare(b.name)) - return Response.json({ skills }) -} - -async function getSkillDetail(url: URL): Promise { - const source = url.searchParams.get('source') - const name = url.searchParams.get('name') - - if (!source || !name) { - throw ApiError.badRequest('Missing required query parameters: source, name') - } - - // Prevent path traversal - if (name.includes('..') || name.includes('/') || name.includes('\\')) { - throw ApiError.badRequest('Invalid skill name') - } - - let skillDir: string - if (source === 'user') { - skillDir = path.join(getUserSkillsDir(), name) - } else { - throw ApiError.badRequest(`Unsupported source: ${source}`) - } - - try { - const stat = await fs.stat(skillDir) - if (!stat.isDirectory()) throw new Error() - } catch { - throw ApiError.notFound(`Skill not found: ${name}`) - } - - const meta = await loadSkillMeta(skillDir, name, source as 'user') - if (!meta) { - throw ApiError.notFound(`Skill missing SKILL.md: ${name}`) - } - - const { tree, files } = await buildFileTree(skillDir) - - return Response.json({ - detail: { meta, tree, files, skillRoot: skillDir }, - }) -} -``` - -- [ ] **Step 2: Register the route in router.ts** - -In `src/server/router.ts`, add the import and case: - -```typescript -// Add import at the top (after existing imports): -import { handleSkillsApi } from './api/skills.js' - -// Add case in the switch (before `case 'filesystem':`): - case 'skills': - return handleSkillsApi(req, url, segments) -``` - -- [ ] **Step 3: Verify server compiles** - -Run: `cd /Users/nanmi/workspace/myself_code/claude-code-haha/.claude/worktrees/feature-skill-ui && bun build src/server/index.ts --outdir /tmp/skill-check --target bun 2>&1 | tail -5` - -Expected: No type errors, successful build output. - -- [ ] **Step 4: Commit** - -```bash -git add src/server/api/skills.ts src/server/router.ts -git commit -m "feat(skills): add GET /api/skills and /api/skills/detail endpoints" -``` - ---- - -### Task 3: Desktop API Client - -**Files:** -- Create: `desktop/src/api/skills.ts` - -- [ ] **Step 1: Create the API client** - -```typescript -// desktop/src/api/skills.ts - -import { api } from './client' -import type { SkillMeta, SkillDetail } from '../types/skill' - -export const skillsApi = { - list: () => api.get<{ skills: SkillMeta[] }>('/api/skills'), - - detail: (source: string, name: string) => - api.get<{ detail: SkillDetail }>( - `/api/skills/detail?source=${encodeURIComponent(source)}&name=${encodeURIComponent(name)}`, - ), -} -``` - -- [ ] **Step 2: Commit** - -```bash -git add desktop/src/api/skills.ts -git commit -m "feat(skills): add desktop API client for skills endpoints" -``` - ---- - -### Task 4: Zustand Store - -**Files:** -- Create: `desktop/src/stores/skillStore.ts` - -- [ ] **Step 1: Create the skill store** - -```typescript -// desktop/src/stores/skillStore.ts - -import { create } from 'zustand' -import { skillsApi } from '../api/skills' -import type { SkillMeta, SkillDetail } from '../types/skill' - -type SkillStore = { - skills: SkillMeta[] - selectedSkill: SkillDetail | null - isLoading: boolean - isDetailLoading: boolean - error: string | null - - fetchSkills: () => Promise - fetchSkillDetail: (source: string, name: string) => Promise - clearSelection: () => void -} - -export const useSkillStore = create((set) => ({ - skills: [], - selectedSkill: null, - isLoading: false, - isDetailLoading: false, - error: null, - - fetchSkills: async () => { - set({ isLoading: true, error: null }) - try { - const { skills } = await skillsApi.list() - set({ skills, isLoading: false }) - } catch (err) { - set({ - error: err instanceof Error ? err.message : String(err), - isLoading: false, - }) - } - }, - - fetchSkillDetail: async (source, name) => { - set({ isDetailLoading: true, error: null }) - try { - const { detail } = await skillsApi.detail(source, name) - set({ selectedSkill: detail, isDetailLoading: false }) - } catch (err) { - set({ - error: err instanceof Error ? err.message : String(err), - isDetailLoading: false, - }) - } - }, - - clearSelection: () => set({ selectedSkill: null }), -})) -``` - -- [ ] **Step 2: Commit** - -```bash -git add desktop/src/stores/skillStore.ts -git commit -m "feat(skills): add Zustand store for skill state management" -``` - ---- - -### Task 5: i18n Translation Keys - -**Files:** -- Modify: `desktop/src/i18n/locales/en.ts:47-50` -- Modify: `desktop/src/i18n/locales/zh.ts:47-50` - -- [ ] **Step 1: Add English translations** - -In `desktop/src/i18n/locales/en.ts`, add after `'settings.tab.general': 'General',` (line 49): - -```typescript - 'settings.tab.skills': 'Skills', -``` - -Then add a new section after the `// Settings > Adapters` block (before `// Settings > General`). Insert after line 140 (`'settings.adapters.platform.feishu': 'Feishu',`): - -```typescript - - // Settings > Skills - 'settings.skills.title': 'Installed Skills', - 'settings.skills.description': 'Skills extend Claude with specialized capabilities. Manage skills in ~/.claude/skills/', - 'settings.skills.empty': 'No skills installed', - 'settings.skills.emptyHint': 'Add skills to ~/.claude/skills/ to get started', - 'settings.skills.back': 'Back to list', - 'settings.skills.files': 'files', - 'settings.skills.source.user': 'User', - 'settings.skills.source.project': 'Project', - 'settings.skills.source.plugin': 'Plugin', - 'settings.skills.source.mcp': 'MCP', - 'settings.skills.source.bundled': 'Built-in', -``` - -- [ ] **Step 2: Add Chinese translations** - -In `desktop/src/i18n/locales/zh.ts`, add after the `settings.tab.general` line (around line 50): - -```typescript - 'settings.tab.skills': '技能', -``` - -Then add matching keys after the adapters section: - -```typescript - - // Settings > Skills - 'settings.skills.title': '已安装技能', - 'settings.skills.description': '技能扩展 Claude 的能力。在 ~/.claude/skills/ 中管理技能。', - 'settings.skills.empty': '暂无已安装技能', - 'settings.skills.emptyHint': '在 ~/.claude/skills/ 中添加技能即可开始', - 'settings.skills.back': '返回列表', - 'settings.skills.files': '个文件', - 'settings.skills.source.user': '用户', - 'settings.skills.source.project': '项目', - 'settings.skills.source.plugin': '插件', - 'settings.skills.source.mcp': 'MCP', - 'settings.skills.source.bundled': '内置', -``` - -- [ ] **Step 3: Verify TypeScript is happy** - -Run: `cd /Users/nanmi/workspace/myself_code/claude-code-haha/.claude/worktrees/feature-skill-ui/desktop && npx tsc --noEmit 2>&1 | tail -5` - -Expected: No errors (the `zh.ts` Record type enforces key parity with `en.ts`). - -- [ ] **Step 4: Commit** - -```bash -git add desktop/src/i18n/locales/en.ts desktop/src/i18n/locales/zh.ts -git commit -m "feat(skills): add i18n translation keys for Skills UI" -``` - ---- - -### Task 6: SkillList Component - -**Files:** -- Create: `desktop/src/components/skills/SkillList.tsx` - -- [ ] **Step 1: Create the skill list component** - -```tsx -// desktop/src/components/skills/SkillList.tsx - -import { useEffect } from 'react' -import { useSkillStore } from '../../stores/skillStore' -import { useTranslation } from '../../i18n' -import type { SkillMeta, SkillSource } from '../../types/skill' - -const SOURCE_ORDER: SkillSource[] = ['user', 'project', 'plugin', 'mcp', 'bundled'] - -const SOURCE_ICONS: Record = { - user: 'person', - project: 'folder', - plugin: 'extension', - mcp: 'hub', - bundled: 'inventory_2', -} - -export function SkillList() { - const { skills, isLoading, error, fetchSkills, fetchSkillDetail } = - useSkillStore() - const t = useTranslation() - - useEffect(() => { - fetchSkills() - }, [fetchSkills]) - - if (isLoading) { - return ( -
-
-
- ) - } - - if (error) { - return
{error}
- } - - if (skills.length === 0) { - return ( -
- - auto_awesome - -

- {t('settings.skills.empty')} -

-

- {t('settings.skills.emptyHint')} -

-
- ) - } - - // Group by source - const grouped: Partial> = {} - for (const skill of skills) { - const src = skill.source as SkillSource - ;(grouped[src] ??= []).push(skill) - } - - return ( -
- {SOURCE_ORDER.map((source) => { - const group = grouped[source] - if (!group?.length) return null - - return ( -
- {/* Group header */} -
- - {SOURCE_ICONS[source]} - - - {t(`settings.skills.source.${source}`)} - - - ({group.length}) - -
- - {/* Skill items */} -
- {group.map((skill) => ( - - ))} -
-
- ) - })} -
- ) -} -``` - -- [ ] **Step 2: Commit** - -```bash -git add desktop/src/components/skills/SkillList.tsx -git commit -m "feat(skills): add SkillList component with grouped display" -``` - ---- - -### Task 7: SkillDetail Component - -**Files:** -- Create: `desktop/src/components/skills/SkillDetail.tsx` - -- [ ] **Step 1: Create the skill detail component with file tree and viewer** - -```tsx -// desktop/src/components/skills/SkillDetail.tsx - -import { useState } from 'react' -import { useSkillStore } from '../../stores/skillStore' -import { useTranslation } from '../../i18n' -import { MarkdownRenderer } from '../markdown/MarkdownRenderer' -import { CodeViewer } from '../chat/CodeViewer' -import type { FileTreeNode } from '../../types/skill' - -// ─── Main Component ────────────────────────────────────────────────────────── - -export function SkillDetail() { - const { selectedSkill, isDetailLoading, clearSelection } = useSkillStore() - const t = useTranslation() - const [selectedFile, setSelectedFile] = useState('SKILL.md') - - if (isDetailLoading) { - return ( -
-
-
- ) - } - - if (!selectedSkill) return null - - const { meta, tree, files } = selectedSkill - const currentFile = files.find((f) => f.path === selectedFile) || files[0] - - return ( -
- {/* Back button */} -
- -
- - {/* Skill header */} -
-
-

- {meta.displayName || meta.name} -

- - {meta.source} - -
-

- {meta.description} -

-
- ~{Math.ceil(meta.contentLength / 4)} tokens - - {files.length} {t('settings.skills.files')} - - {meta.version && v{meta.version}} -
-
- - {/* Two-panel: file tree + content viewer */} -
- {/* Left: File tree */} -
- -
- - {/* Right: File content */} -
- {currentFile && ( -
- {/* File path header */} -
- - {currentFile.path} - - - {currentFile.language} - -
- - {/* Content */} -
- {currentFile.language === 'markdown' ? ( - - ) : ( - - )} -
-
- )} -
-
-
- ) -} - -// ─── File Tree Components ──────────────────────────────────────────────────── - -function TreeView({ - nodes, - selectedPath, - onSelect, - depth, -}: { - nodes: FileTreeNode[] - selectedPath: string - onSelect: (path: string) => void - depth: number -}) { - return ( - <> - {nodes.map((node) => ( - - ))} - - ) -} - -function TreeItem({ - node, - selectedPath, - onSelect, - depth, -}: { - node: FileTreeNode - selectedPath: string - onSelect: (path: string) => void - depth: number -}) { - const [expanded, setExpanded] = useState(true) - const isSelected = node.path === selectedPath - const isDir = node.type === 'directory' - - const icon = isDir - ? expanded - ? 'folder_open' - : 'folder' - : fileIcon(node.name) - - return ( -
- - - {isDir && expanded && node.children && ( - - )} -
- ) -} - -function fileIcon(filename: string): string { - const ext = filename.split('.').pop()?.toLowerCase() - switch (ext) { - case 'md': - return 'description' - case 'ts': - case 'tsx': - case 'js': - case 'jsx': - case 'py': - case 'rs': - case 'go': - return 'code' - case 'json': - case 'yaml': - case 'yml': - case 'toml': - return 'data_object' - case 'sh': - case 'bash': - return 'terminal' - default: - return 'draft' - } -} -``` - -- [ ] **Step 2: Commit** - -```bash -git add desktop/src/components/skills/SkillDetail.tsx -git commit -m "feat(skills): add SkillDetail component with file tree and content viewer" -``` - ---- - -### Task 8: Settings Page Integration - -**Files:** -- Modify: `desktop/src/pages/Settings.tsx:1-58` - -- [ ] **Step 1: Add Skills tab to Settings page** - -In `desktop/src/pages/Settings.tsx`, apply these changes: - -**1. Add imports at the top** (after existing imports): - -```typescript -import { useSkillStore } from '../stores/skillStore' -import { SkillList } from '../components/skills/SkillList' -import { SkillDetail } from '../components/skills/SkillDetail' -``` - -**2. Extend the SettingsTab type** (line 15): - -Change: -```typescript -type SettingsTab = 'providers' | 'permissions' | 'general' | 'adapters' -``` -To: -```typescript -type SettingsTab = 'providers' | 'permissions' | 'general' | 'adapters' | 'skills' -``` - -**3. Add the TabButton** in the Settings component (after the adapters TabButton, around line 29): - -```tsx - setActiveTab('skills')} /> -``` - -**4. Add the content renderer** (after the adapters condition, around line 35): - -```tsx - {activeTab === 'skills' && } -``` - -**5. Add the SkillSettings wrapper component** at the bottom of the file (before the closing line, after `GeneralSettings`): - -```typescript -// ─── Skill Settings ────────────────────────────────────── - -function SkillSettings() { - const selectedSkill = useSkillStore((s) => s.selectedSkill) - const t = useTranslation() - - if (selectedSkill) { - return ( -
- -
- ) - } - - return ( -
-

- {t('settings.skills.title')} -

-

- {t('settings.skills.description')} -

- -
- ) -} -``` - -- [ ] **Step 2: Verify the desktop app compiles** - -Run: `cd /Users/nanmi/workspace/myself_code/claude-code-haha/.claude/worktrees/feature-skill-ui/desktop && npx tsc --noEmit 2>&1 | tail -10` - -Expected: No type errors. - -- [ ] **Step 3: Commit** - -```bash -git add desktop/src/pages/Settings.tsx -git commit -m "feat(skills): integrate Skills tab into Settings page" -``` - ---- - -### Task 9: Manual Smoke Test - -- [ ] **Step 1: Create a test skill directory** - -```bash -mkdir -p ~/.claude/skills/test-skill -cat > ~/.claude/skills/test-skill/SKILL.md << 'SKILLEOF' ---- -name: Test Skill -description: A test skill for verifying the Skills UI -version: 1.0 -user-invocable: true ---- - -# Test Skill - -This is a test skill to verify the Skills browser works correctly. - -## Usage - -Just invoke `/test-skill` from the chat. - -```typescript -console.log('Hello from test skill') -``` -SKILLEOF - -cat > ~/.claude/skills/test-skill/helper.ts << 'HELPEREOF' -export function greet(name: string): string { - return `Hello, ${name}!` -} -HELPEREOF -``` - -- [ ] **Step 2: Start the server and verify API** - -Run: `cd /Users/nanmi/workspace/myself_code/claude-code-haha/.claude/worktrees/feature-skill-ui && bun run src/server/index.ts &` - -Then test: -```bash -curl -s http://127.0.0.1:3456/api/skills | python3 -m json.tool -``` - -Expected: JSON with `skills` array containing `test-skill` entry. - -```bash -curl -s "http://127.0.0.1:3456/api/skills/detail?source=user&name=test-skill" | python3 -m json.tool -``` - -Expected: JSON with `detail` containing `meta`, `tree` (SKILL.md + helper.ts), `files` with content. - -- [ ] **Step 3: Clean up test data (optional)** - -```bash -rm -rf ~/.claude/skills/test-skill -``` - -- [ ] **Step 4: Final commit with all changes** - -```bash -git add -A -git commit -m "feat: add Skills browser to Settings page - -- Server: GET /api/skills (list) and GET /api/skills/detail (tree + files) -- Desktop: Skills tab in Settings with list/detail views -- File tree navigation with Markdown and code preview -- i18n support (EN/ZH)" -``` diff --git a/docs/superpowers/specs/2026-04-09-skill-ui-design.md b/docs/superpowers/specs/2026-04-09-skill-ui-design.md deleted file mode 100644 index d4036988..00000000 --- a/docs/superpowers/specs/2026-04-09-skill-ui-design.md +++ /dev/null @@ -1,271 +0,0 @@ -# Skills UI Feature Design - -## Overview - -Port the CLI `/skills` listing to the Desktop Web APP, adding a full Skills browser with file tree navigation and content preview in the Settings page. - -## Architecture - -``` -Desktop UI (React) Server (Bun.serve) Filesystem -┌──────────────────┐ ┌──────────────────┐ ┌──────────────────┐ -│ Settings Page │ │ GET /api/skills │ │ ~/.claude/skills/│ -│ └─ Skills Tab │──────▶│ → list meta │──────▶│ ./.claude/skills/│ -│ ├─ SkillList │ │ │ │ plugins cache/ │ -│ └─ SkillDetail │ GET /api/skills/ │ │ bundled skills/ │ -│ ├─ FileTree│──────▶│ detail?s=&n= │──────▶│ │ -│ └─ Viewer │ │ → tree + files │ │ │ -└──────────────────┘ └──────────────────┘ └──────────────────┘ -``` - -## Data Models - -### SkillMeta (list item) - -```typescript -type SkillSource = 'userSettings' | 'projectSettings' | 'policySettings' - | 'plugin' | 'mcp' | 'bundled' - -type SkillMeta = { - name: string // unique name (plugin format: "plugin:skill") - displayName?: string // custom display name from frontmatter - description: string - source: SkillSource - userInvocable: boolean - version?: string - contentLength: number // char count for token estimation - pluginName?: string // plugin name when source is 'plugin' - hasDirectory: boolean // whether the skill has a browsable directory -} -``` - -### FileTreeNode - -```typescript -type FileTreeNode = { - name: string // file/dir name - path: string // relative to skill root - type: 'file' | 'directory' - children?: FileTreeNode[] // present for directories -} -``` - -### SkillFile - -```typescript -type SkillFile = { - path: string // relative path - content: string // file content - language: string // language id (md, ts, json, yaml, sh, etc.) -} -``` - -### SkillDetail (detail view) - -```typescript -type SkillDetail = { - meta: SkillMeta - tree: FileTreeNode[] // directory tree - files: SkillFile[] // all file contents, loaded at once - skillRoot: string // absolute path (display only) -} -``` - -## Server API - -### `GET /api/skills` - -Returns metadata list of all installed skills. - -**Response:** -```json -{ - "skills": [ - { - "name": "my-skill", - "displayName": "My Skill", - "description": "Does something useful", - "source": "userSettings", - "userInvocable": true, - "contentLength": 1234, - "hasDirectory": true - } - ] -} -``` - -**Implementation:** Reuse `loadAllCommands()` from `src/commands.ts`, filter for `type === 'prompt'` commands with skill-related `loadedFrom` values. Extract metadata without loading full content. - -### `GET /api/skills/detail?source={source}&name={name}` - -Returns full skill data including file tree and all file contents. - -**Response:** -```json -{ - "detail": { - "meta": { ... }, - "tree": [ - { "name": "SKILL.md", "path": "SKILL.md", "type": "file" }, - { - "name": "examples", - "path": "examples", - "type": "directory", - "children": [ - { "name": "demo.ts", "path": "examples/demo.ts", "type": "file" } - ] - } - ], - "files": [ - { "path": "SKILL.md", "content": "---\nname: ...\n---\n# ...", "language": "md" }, - { "path": "examples/demo.ts", "content": "...", "language": "ts" } - ], - "skillRoot": "/Users/x/.claude/skills/my-skill" - } -} -``` - -**Implementation:** -1. Find matching skill by source + name from loaded commands -2. Get `skillRoot` from the command object -3. Recursively walk the directory, skip `node_modules`, `.git`, hidden dirs -4. Read all files (max 100KB per file, max 50 files total as safety limit) -5. Detect language from file extension -6. For MCP skills (no directory): return empty tree, single virtual file with the skill's prompt content - -### File: `src/server/api/skills.ts` - -New file following the pattern of `src/server/api/status.ts`. - -### Router registration: `src/server/router.ts` - -Add `case 'skills': return handleSkillsApi(req, url, segments)`. - -## Desktop UI - -### Settings Tab Addition - -Add a 5th tab "Skills" (icon: `auto_awesome`) to `desktop/src/pages/Settings.tsx`. - -``` -type SettingsTab = 'providers' | 'permissions' | 'general' | 'adapters' | 'skills' -``` - -### Skill List View (`SkillList.tsx`) - -- Grouped by source (User, Project, Plugin, MCP, Bundled) -- Each group: collapsible header with count -- Each item: name, description (truncated), source badge -- Click → navigate to detail view -- Loading state + empty state - -### Skill Detail View (`SkillDetail.tsx`) - -Two-panel layout: -- **Left panel (w-[220px]):** File tree with expand/collapse -- **Right panel (flex-1):** File content viewer -- **Header:** Back button, skill name, description, source badge, token estimate -- Default selected file: SKILL.md - -### File Tree Component (`FileTree.tsx`) - -- Recursive tree with indent -- Directory expand/collapse (default: all expanded) -- File icons based on extension -- Click file → show in right panel -- Active file highlight - -### File Viewer (`FileViewer.tsx`) - -- For `.md` files: Use existing `MarkdownRenderer` -- For code files: Use existing `CodeViewer` with language detection -- File path header with copy button -- Scrollable content area - -### Language Detection - -Map file extension to language: -```typescript -const LANG_MAP: Record = { - md: 'markdown', ts: 'typescript', tsx: 'typescript', - js: 'javascript', jsx: 'javascript', json: 'json', - yaml: 'yaml', yml: 'yaml', sh: 'bash', bash: 'bash', - py: 'python', toml: 'toml', css: 'css', html: 'html', -} -``` - -## State Management - -### `desktop/src/stores/skillStore.ts` - -```typescript -type SkillStoreState = { - skills: SkillMeta[] - selectedSkill: SkillDetail | null - isLoading: boolean - isDetailLoading: boolean - error: string | null - - fetchSkills: () => Promise - fetchSkillDetail: (source: string, name: string) => Promise - clearSelection: () => void -} -``` - -### `desktop/src/api/skills.ts` - -```typescript -export const skillsApi = { - list: () => api.get<{ skills: SkillMeta[] }>('/api/skills'), - detail: (source: string, name: string) => - api.get<{ detail: SkillDetail }>(`/api/skills/detail?source=${encodeURIComponent(source)}&name=${encodeURIComponent(name)}`), -} -``` - -### `desktop/src/types/skill.ts` - -All type definitions from Data Models section. - -## i18n - -Add to both `en.ts` and `zh.ts`: - -```typescript -'settings.tab.skills': 'Skills' / '技能', -'settings.skills.title': 'Installed Skills' / '已安装技能', -'settings.skills.description': 'Skills extend Claude...' / '技能扩展 Claude...', -'settings.skills.empty': 'No skills installed' / '暂无已安装技能', -'settings.skills.back': 'Back to list' / '返回列表', -'settings.skills.tokens': '~{count} tokens', -'settings.skills.files': '{count} files' / '{count} 个文件', -'settings.skills.source.userSettings': 'User' / '用户', -'settings.skills.source.projectSettings': 'Project' / '项目', -'settings.skills.source.plugin': 'Plugin' / '插件', -'settings.skills.source.mcp': 'MCP', -'settings.skills.source.bundled': 'Built-in' / '内置', -``` - -## Files to Create/Modify - -### New files (7): -1. `src/server/api/skills.ts` — Server API handler -2. `desktop/src/types/skill.ts` — Type definitions -3. `desktop/src/api/skills.ts` — API client -4. `desktop/src/stores/skillStore.ts` — Zustand store -5. `desktop/src/components/skills/SkillList.tsx` — Skill list component -6. `desktop/src/components/skills/SkillDetail.tsx` — Detail view (FileTree + FileViewer inline) - -### Modified files (4): -7. `src/server/router.ts` — Add skills route -8. `desktop/src/pages/Settings.tsx` — Add Skills tab -9. `desktop/src/i18n/locales/en.ts` — English translations -10. `desktop/src/i18n/locales/zh.ts` — Chinese translations - -## Design Decisions - -1. **One detail request loads all files** — Single skill directories are small (typically <50KB total), so loading all at once avoids per-file request overhead. -2. **Reuse existing components** — `MarkdownRenderer` for .md files, `CodeViewer` for code. No new rendering dependencies. -3. **source + name as identifier** — Skills can have duplicate names across sources, so both are needed. -4. **Server-side skill loading** — Reuse `loadAllCommands()` which already handles all 6 sources with deduplication and caching. -5. **Safety limits** — Max 50 files, 100KB per file to prevent issues with massive plugin directories. -6. **FileTree + FileViewer inline in SkillDetail** — No need for separate components given the simplicity; keeps code collocated.