diff --git a/.gitignore b/.gitignore index b9cccb16..e14c277e 100644 --- a/.gitignore +++ b/.gitignore @@ -22,6 +22,9 @@ extracted-natives/ # E2E screenshots e2e-*.png +# Quality gate reports +artifacts/quality-runs/ + # Desktop build output desktop/dist/ desktop/build-artifacts/ diff --git a/AGENTS.md b/AGENTS.md index a6426894..c747a369 100644 --- a/AGENTS.md +++ b/AGENTS.md @@ -13,6 +13,10 @@ Install root dependencies with `bun install`, then install desktop dependencies - `cd desktop && bun run build`: type-check and produce a production web build. - `cd desktop && bun run test`: run Vitest suites. - `cd desktop && bun run lint`: run TypeScript no-emit checks. +- `bun run quality:providers`: list configured provider/model selectors for live agent baselines. +- `bun run quality:pr`: run the local PR quality gate and write a report under `artifacts/quality-runs/`. +- `bun run quality:gate --mode baseline --allow-live --provider-model `: run live Coding Agent baseline cases, including desktop agent-browser smoke. +- `bun run quality:gate --mode release --allow-live --provider-model `: run the release gate with live baseline coverage. ## Desktop Release Workflow - Desktop releases are built remotely by GitHub Actions, not by uploading local build artifacts. @@ -32,6 +36,23 @@ Use TypeScript with 2-space indentation, ESM imports, and no semicolons to match ## Testing Guidelines Desktop tests use Vitest with Testing Library in a `jsdom` environment. Name tests `*.test.ts` or `*.test.tsx`; colocate focused tests near the file or place broader coverage in `desktop/src/__tests__/`. No coverage gate is configured, so add regression tests for any behavior you change and run the relevant suites before opening a PR. +## Quality Gate Automation +Future Coding Agents should run the right local gate themselves before claiming a change is ready. Do not ask the user to manually run the commands unless credentials, local model access, or machine resources are missing. + +- For normal code changes, run the narrow relevant check first, then `bun run quality:pr` before a PR-ready or merge-ready claim. +- Use `bun run check:server` for `src/server`, `src/tools`, provider/runtime, MCP, OAuth, WebSocket, or API behavior changes. +- Use `bun run check:desktop` for `desktop/src` UI, stores, API clients, and desktop web behavior changes. +- Use `bun run check:native` for `desktop/src-tauri`, sidecars, native packaging, release, or platform startup behavior changes. +- Use `bun run check:adapters` for `adapters/`; on a fresh checkout run `cd adapters && bun install` first if dependencies are missing. +- Use `bun run check:docs` for docs, VitePress, README, or docs workflow changes. +- For chat, agent loop, tool execution, provider routing, desktop chat UI, CLI task execution, or other core Coding Agent paths, also run a live baseline when local providers are available: first `bun run quality:providers`, then choose one or more copyable selectors and run `bun run quality:gate --mode baseline --allow-live --provider-model `. +- For release readiness, run `bun run quality:gate --mode release --allow-live --provider-model ` with at least one real provider/model selector. Prefer multiple providers when quota is available. +- If no live provider is configured, or a provider quota/key is unavailable, run the non-live gate anyway and report the live-baseline blocker explicitly instead of claiming full release confidence. +- `bun run check:docs` executes `npm ci`, which can rebuild root `node_modules`. Run docs checks sequentially, not in parallel with `quality:pr`, `check:native`, or other commands that depend on the same installed packages. +- Quality reports are written to `artifacts/quality-runs//`. Summarize the final report path and the pass/fail counts in handoffs and PR descriptions. +- Do not commit generated `artifacts/quality-runs/`, local `.omx/` state, `node_modules/`, `desktop/node_modules/`, or adapter dependency folders. +- Do not claim "complete", "ready to merge", or "ready to release" without either running the matching gate or naming the exact blocker that prevented it. + ## Commit & Pull Request Guidelines Recent history follows Conventional Commit prefixes such as `feat:`, `fix:`, and `docs:`. Keep subjects imperative and scoped to one change. PRs should explain the user-visible impact, list verification steps, link related issues, and include screenshots for desktop or docs UI changes. Keep diffs reviewable and call out any follow-up work or known gaps. Branch names should use normal product prefixes such as `fix/xxx`, `feat/xxx`, or `docs/xxx`; do not create `codex/`-prefixed branches in this repository. diff --git a/CONTRIBUTING.md b/CONTRIBUTING.md new file mode 100644 index 00000000..d8991d94 --- /dev/null +++ b/CONTRIBUTING.md @@ -0,0 +1,31 @@ +# Contributing + +Thanks for helping improve Claude Code Haha. + +For the full contributor guide, including local checks, live model baselines, quality-gate reports, and PR expectations, see: + +- Chinese: [docs/guide/contributing.md](docs/guide/contributing.md) +- English: [docs/en/guide/contributing.md](docs/en/guide/contributing.md) + +Most contributors should run this before opening a PR: + +```bash +bun install +bun run quality:pr +``` + +If you run adapter or native checks on a fresh clone, install adapter dependencies too: + +```bash +cd adapters +bun install +``` + +If your change touches the desktop chat path, provider/runtime selection, CLI bridge, permissions, tools, file editing, or release packaging, also run a live baseline with your own local model provider: + +```bash +bun run quality:providers +bun run quality:gate --mode baseline --allow-live --provider-model :main +``` + +Quality reports are written under `artifacts/quality-runs//`. diff --git a/README.en.md b/README.en.md index 21319330..48cc93ae 100644 --- a/README.en.md +++ b/README.en.md @@ -195,6 +195,7 @@ http://127.0.0.1:2024 |------|------| | [Environment Variables](docs/en/guide/env-vars.md) | Full env var reference and configuration methods | | [Third-Party Models](docs/en/guide/third-party-models.md) | Using OpenAI / DeepSeek / Ollama and other non-Anthropic models | +| [Contributing](docs/en/guide/contributing.md) | Local tests, live model baselines, PR gates, and release gates | | [Memory System](docs/memory/01-usage-guide.md) | Cross-session persistent memory usage and implementation | | [Multi-Agent System](docs/agent/01-usage-guide.md) | Agent orchestration, parallel tasks and Teams collaboration | | [Skills System](docs/skills/01-usage-guide.md) | Extensible capability plugins, custom workflows and conditional activation | diff --git a/README.md b/README.md index f136251a..a778d439 100644 --- a/README.md +++ b/README.md @@ -195,6 +195,7 @@ http://127.0.0.1:2024 |------|------| | [环境变量](docs/guide/env-vars.md) | 完整环境变量参考和配置方式 | | [第三方模型](docs/guide/third-party-models.md) | 接入 OpenAI / DeepSeek / Ollama 等非 Anthropic 模型 | +| [贡献与质量门禁](docs/guide/contributing.md) | 本地测试、真实模型 baseline、PR 和 release 门禁 | | [记忆系统](docs/memory/01-usage-guide.md) | 跨会话持久化记忆的使用与实现 | | [多 Agent 系统](docs/agent/01-usage-guide.md) | 多代理编排、并行任务执行与 Teams 协作 | | [Skills 系统](docs/skills/01-usage-guide.md) | 可扩展能力插件、自定义工作流与条件激活 | diff --git a/docs/.vitepress/config.mts b/docs/.vitepress/config.mts index ca3639ff..66547160 100644 --- a/docs/.vitepress/config.mts +++ b/docs/.vitepress/config.mts @@ -24,6 +24,7 @@ const zhSidebar = [ { text: '第三方模型', link: '/guide/third-party-models' }, { text: '全局使用', link: '/guide/global-usage' }, { text: '常见问题', link: '/guide/faq' }, + { text: '贡献与质量门禁', link: '/guide/contributing' }, ], }, { @@ -109,6 +110,7 @@ const enSidebar = [ { text: 'Third-Party Models', link: '/en/guide/third-party-models' }, { text: 'Global Usage', link: '/en/guide/global-usage' }, { text: 'FAQ', link: '/en/guide/faq' }, + { text: 'Contributing', link: '/en/guide/contributing' }, ], }, { diff --git a/docs/en/guide/contributing.md b/docs/en/guide/contributing.md new file mode 100644 index 00000000..4ef47fee --- /dev/null +++ b/docs/en/guide/contributing.md @@ -0,0 +1,164 @@ +# Contributing and Local Quality Gates + +This guide explains how to install, develop, test, and run the local quality gates before opening a PR. The goal is to help maintainers and contributors answer one question before review: did this change break the core Coding Agent workflow? + +## Setup + +Install root dependencies with Bun: + +```bash +bun install +``` + +If your change touches `desktop/`, also install desktop dependencies: + +```bash +cd desktop +bun install +``` + +If your change touches `adapters/`, or if you run `check:adapters` / `check:native`, install adapter dependencies: + +```bash +cd adapters +bun install +``` + +Do not commit local artifacts such as `artifacts/quality-runs/`, `node_modules/`, or `desktop/node_modules/`. + +## Required PR Gate + +Before opening a normal PR, run: + +```bash +bun run quality:pr +``` + +This gate does not call real models, so every contributor can run it locally. It writes reports to: + +```text +artifacts/quality-runs//report.md +artifacts/quality-runs//report.json +``` + +Include the commands you ran and the report summary in your PR description. + +## Area-Specific Checks + +Run the checks that match the files you changed: + +```bash +bun run check:server # Server API, WebSocket, providers, sessions, and related tests +bun run check:desktop # Desktop lint, Vitest, and production build +bun run check:adapters # IM adapter tests +bun run check:native # Desktop sidecars and Tauri native checks +bun run check:docs # Docs build, using npm ci + docs:build +``` + +Focused tests are fine while developing, but run `bun run quality:pr` before sending the PR. + +## Live Model Baseline + +`quality:baseline` runs real Coding Agent tasks: it starts the local server, creates isolated fixtures, asks a model through chat to fix code, runs tests, and saves transcripts, diffs, verification logs, and a report. + +The default baseline command does not call real models: + +```bash +bun run quality:baseline +``` + +To actually call models, pass `--allow-live` and choose a local provider. + +First list your local providers and copyable selectors: + +```bash +bun run quality:providers +``` + +Example output: + +```text +Saved providers: + MiniMax + selector: minimax + main: MiniMax-M2.7-highspeed + --provider-model minimax:main:minimax-main +``` + +Copy one of the listed values: + +```bash +bun run quality:gate --mode baseline --allow-live --provider-model minimax:main:minimax-main +``` + +You can run multiple models in one pass: + +```bash +bun run quality:gate --mode baseline --allow-live \ + --provider-model codingplan:main:codingplan-main \ + --provider-model minimax:main:minimax-main +``` + +Provider selectors come from the providers saved in your local Desktop Settings > Providers page. Contributors do not need the maintainer's provider UUIDs or vendor accounts. They can add their own provider locally, run `bun run quality:providers`, and choose their own model. + +## When To Run The Baseline + +Run the live baseline for changes touching: + +- Desktop chat, session resume, WebSocket, or the CLI bridge +- Provider, model, or runtime selection +- Permissions, tool calls, file edits, and task execution +- agent-browser smoke, Computer Use, Skills, or MCP +- Release preparation or broad cross-module refactors + +If you do not have model access, still run `bun run quality:pr` and state in the PR why the live baseline was not run. + +## Release Gate + +Before a release, run release mode: + +```bash +bun run quality:gate --mode release --allow-live --provider-model :main +``` + +Release mode composes PR checks, baseline catalog validation, live baseline cases, desktop smoke, and native checks. Reports are written to `artifacts/quality-runs//`. + +## PR Workflow + +1. Create a product branch such as `fix/session-reconnect` or `feat/provider-quality-gate`. +2. Install dependencies and make the change. +3. Add tests for behavior changes. +4. Run focused checks for the affected area. +5. Run `bun run quality:pr`. +6. Run the live baseline for high-risk changes. +7. In the PR description, include user impact, verification commands, report summary, and known risks. + +## FAQ + +### Can I run checks without a provider? + +Yes. Run the normal PR gate: + +```bash +bun run quality:pr +``` + +Only the live baseline needs a real model. Add your provider in Desktop Settings > Providers, then run: + +```bash +bun run quality:providers +``` + +### What if provider selectors conflict? + +If two provider names produce the same selector, `quality:providers` falls back to the provider ID. Copy the `--provider-model ...` value it prints. + +### What if a model ID contains a colon? + +Prefer role selectors: + +```bash +--provider-model custom:haiku:custom-haiku +``` + +The runner resolves `haiku` to the real model ID from your local provider configuration. diff --git a/docs/guide/contributing.md b/docs/guide/contributing.md new file mode 100644 index 00000000..cd40ad2c --- /dev/null +++ b/docs/guide/contributing.md @@ -0,0 +1,164 @@ +# 贡献指南与本地质量门禁 + +这份文档说明贡献代码前应该如何在本地安装、开发、测试和运行质量门禁。目标是让维护者和贡献者都能在提交 PR 前回答一个问题:这次改动有没有破坏核心 Coding Agent 工作流。 + +## 环境准备 + +项目根目录使用 Bun: + +```bash +bun install +``` + +如果改动涉及 `desktop/`,也安装桌面端依赖: + +```bash +cd desktop +bun install +``` + +如果改动涉及 `adapters/`,或者要运行 `check:adapters` / `check:native`,安装 adapter 依赖: + +```bash +cd adapters +bun install +``` + +不要提交本地运行产物,例如 `artifacts/quality-runs/`、`node_modules/`、`desktop/node_modules/`。 + +## 普通 PR 必跑门禁 + +普通贡献者在提交 PR 前至少运行: + +```bash +bun run quality:pr +``` + +这个门禁不调用真实大模型,适合所有人本地运行。它会生成报告: + +```text +artifacts/quality-runs//report.md +artifacts/quality-runs//report.json +``` + +PR 描述里请贴出你实际运行的命令和 summary。 + +## 按改动范围补充测试 + +根据你改动的区域补充运行: + +```bash +bun run check:server # 服务端 API、WebSocket、provider、会话等测试 +bun run check:desktop # 桌面端 lint、Vitest、生产构建 +bun run check:adapters # IM adapter 测试 +bun run check:native # 桌面 sidecar 与 Tauri native 检查 +bun run check:docs # 文档构建,使用 npm ci + docs:build +``` + +如果只改了很窄的文件,也可以先跑对应的定向测试,但 PR 前仍应跑 `bun run quality:pr`。 + +## 真实模型 Baseline + +`quality:baseline` 用来跑真实 Coding Agent 任务:启动本地服务端、创建隔离 fixture、让模型通过聊天修代码、跑测试,并保存 transcript、diff、verification log 和报告。 + +默认命令不会调用真实模型: + +```bash +bun run quality:baseline +``` + +要真正跑模型,必须显式加 `--allow-live` 并选择本机 provider。 + +先列出本机可用 provider 和可复制参数: + +```bash +bun run quality:providers +``` + +输出示例: + +```text +Saved providers: + MiniMax + selector: minimax + main: MiniMax-M2.7-highspeed + --provider-model minimax:main:minimax-main +``` + +复制输出里的参数运行 baseline: + +```bash +bun run quality:gate --mode baseline --allow-live --provider-model minimax:main:minimax-main +``` + +可以一次跑多个模型: + +```bash +bun run quality:gate --mode baseline --allow-live \ + --provider-model codingplan:main:codingplan-main \ + --provider-model minimax:main:minimax-main +``` + +`provider` selector 来自桌面端「Settings > Providers」里保存的本机配置。别人 clone 代码后不需要知道你的 provider UUID,也不需要使用你的供应商;他们可以在自己的桌面端添加 provider 后运行 `bun run quality:providers` 选择自己的模型。 + +## 什么时候必须跑 Baseline + +以下改动建议跑 live baseline: + +- 桌面聊天、会话恢复、WebSocket、CLI bridge +- provider/model/runtime 选择 +- 权限、工具调用、文件编辑、任务执行 +- agent-browser smoke、Computer Use、Skills、MCP +- release 前或风险较大的跨模块重构 + +如果没有模型额度,至少运行 `bun run quality:pr`,并在 PR 里说明未跑 live baseline 的原因。 + +## Release 门禁 + +发版前使用 release 模式: + +```bash +bun run quality:gate --mode release --allow-live --provider-model :main +``` + +release 模式会组合 PR checks、baseline catalog、live baseline、desktop smoke 和 native checks。发版报告同样写入 `artifacts/quality-runs//`。 + +## PR 提交流程 + +1. 新建普通产品分支,例如 `fix/session-reconnect` 或 `feat/provider-quality-gate`。 +2. 安装依赖并完成改动。 +3. 为行为变化补测试。 +4. 运行相关定向测试。 +5. 运行 `bun run quality:pr`。 +6. 对高风险改动运行 live baseline。 +7. 在 PR 描述里写清楚用户影响、测试命令、报告 summary、已知风险。 + +## 常见问题 + +### 没有 provider 可以跑吗? + +可以跑普通门禁: + +```bash +bun run quality:pr +``` + +只有 live baseline 需要真实模型。先在桌面端 Settings > Providers 添加自己的 provider,再运行: + +```bash +bun run quality:providers +``` + +### provider selector 冲突怎么办? + +如果两个 provider 名称生成了相同 selector,`quality:providers` 会退回输出 provider ID。直接复制它给出的 `--provider-model ...` 即可。 + +### 模型 ID 里带冒号怎么办? + +优先使用角色选择,例如: + +```bash +--provider-model custom:haiku:custom-haiku +``` + +脚本会把 `haiku` 解析成本机 provider 配置里的真实模型 ID。 diff --git a/docs/superpowers/plans/2026-04-09-skill-ui.md b/docs/superpowers/plans/2026-04-09-skill-ui.md deleted file mode 100644 index 9b557eb1..00000000 --- a/docs/superpowers/plans/2026-04-09-skill-ui.md +++ /dev/null @@ -1,1107 +0,0 @@ -# Skills UI Implementation Plan - -> **For agentic workers:** REQUIRED SUB-SKILL: Use superpowers:subagent-driven-development (recommended) or superpowers:executing-plans to implement this plan task-by-task. Steps use checkbox (`- [ ]`) syntax for tracking. - -**Goal:** Add a Skills browser to the Desktop Web APP Settings page, with API backend, file tree navigation, and Markdown/code content preview. - -**Architecture:** Server reads skill directories from `~/.claude/skills/`, exposes two REST endpoints (list + detail). Desktop frontend adds a "Skills" tab to Settings with list/detail views, reusing existing `MarkdownRenderer` and `CodeViewer` components. - -**Tech Stack:** Bun server (REST API), React + Zustand + Tailwind CSS (desktop), marked + react-shiki (rendering) - ---- - -## File Structure - -| Action | File | Responsibility | -|--------|------|----------------| -| Create | `src/server/api/skills.ts` | Server API handler: list skills, read skill directory tree + files | -| Modify | `src/server/router.ts` | Register `/api/skills` route | -| Create | `desktop/src/types/skill.ts` | Shared type definitions | -| Create | `desktop/src/api/skills.ts` | API client for skills endpoints | -| Create | `desktop/src/stores/skillStore.ts` | Zustand store for skill state | -| Create | `desktop/src/components/skills/SkillList.tsx` | Grouped skill list view | -| Create | `desktop/src/components/skills/SkillDetail.tsx` | File tree + content viewer | -| Modify | `desktop/src/pages/Settings.tsx` | Add Skills tab + SkillSettings wrapper | -| Modify | `desktop/src/i18n/locales/en.ts` | English translation keys | -| Modify | `desktop/src/i18n/locales/zh.ts` | Chinese translation keys | - ---- - -### Task 1: Desktop Type Definitions - -**Files:** -- Create: `desktop/src/types/skill.ts` - -- [ ] **Step 1: Create type definitions** - -```typescript -// desktop/src/types/skill.ts - -export type SkillSource = 'user' | 'project' | 'plugin' | 'mcp' | 'bundled' - -export type SkillMeta = { - name: string - displayName?: string - description: string - source: SkillSource - userInvocable: boolean - version?: string - contentLength: number - hasDirectory: boolean - pluginName?: string -} - -export type FileTreeNode = { - name: string - path: string - type: 'file' | 'directory' - children?: FileTreeNode[] -} - -export type SkillFile = { - path: string - content: string - language: string -} - -export type SkillDetail = { - meta: SkillMeta - tree: FileTreeNode[] - files: SkillFile[] - skillRoot: string -} -``` - -- [ ] **Step 2: Commit** - -```bash -git add desktop/src/types/skill.ts -git commit -m "feat(skills): add type definitions for Skills UI" -``` - ---- - -### Task 2: Server API Handler - -**Files:** -- Create: `src/server/api/skills.ts` -- Modify: `src/server/router.ts:1-79` - -- [ ] **Step 1: Create the skills API handler** - -```typescript -// src/server/api/skills.ts - -/** - * Skills REST API - * - * GET /api/skills — List all installed skills (metadata only) - * GET /api/skills/detail — Full skill data (tree + files) - * ?source=user&name=xxx - */ - -import * as os from 'os' -import * as path from 'path' -import * as fs from 'fs/promises' -import { parse as parseYaml } from 'yaml' -import { ApiError, errorResponse } from '../middleware/errorHandler.js' - -// ─── Types (server-local, mirrors desktop/src/types/skill.ts) ──────────────── - -type SkillMeta = { - name: string - displayName?: string - description: string - source: 'user' | 'project' - userInvocable: boolean - version?: string - contentLength: number - hasDirectory: boolean -} - -type FileTreeNode = { - name: string - path: string - type: 'file' | 'directory' - children?: FileTreeNode[] -} - -type SkillFile = { - path: string - content: string - language: string -} - -// ─── Constants ─────────────────────────────────────────────────────────────── - -const MAX_FILES = 50 -const MAX_FILE_SIZE = 100 * 1024 // 100 KB -const SKIP_ENTRIES = new Set(['node_modules', '.git', '__pycache__', '.DS_Store']) - -const LANG_MAP: Record = { - md: 'markdown', ts: 'typescript', tsx: 'typescript', - js: 'javascript', jsx: 'javascript', json: 'json', - yaml: 'yaml', yml: 'yaml', sh: 'bash', bash: 'bash', - py: 'python', toml: 'toml', css: 'css', html: 'html', - txt: 'text', xml: 'xml', sql: 'sql', rs: 'rust', go: 'go', -} - -// ─── Helpers ───────────────────────────────────────────────────────────────── - -function detectLanguage(filename: string): string { - const ext = filename.split('.').pop()?.toLowerCase() || '' - return LANG_MAP[ext] || 'text' -} - -function parseFrontmatter(content: string): { - frontmatter: Record - body: string -} { - const match = content.match(/^---\s*\n([\s\S]*?)\n---\s*\n?/) - if (!match) return { frontmatter: {}, body: content } - try { - const frontmatter = (parseYaml(match[1]) as Record) || {} - return { frontmatter, body: content.slice(match[0].length) } - } catch { - return { frontmatter: {}, body: content } - } -} - -function getUserSkillsDir(): string { - return path.join(os.homedir(), '.claude', 'skills') -} - -async function loadSkillMeta( - skillDir: string, - skillName: string, - source: 'user' | 'project', -): Promise { - const skillFile = path.join(skillDir, 'SKILL.md') - try { - const raw = await fs.readFile(skillFile, 'utf-8') - const { frontmatter, body } = parseFrontmatter(raw) - - const description = - (frontmatter.description as string) || - body - .split('\n') - .find((l) => l.trim().length > 0) - ?.trim() || - 'No description' - - return { - name: skillName, - displayName: (frontmatter.name as string) || undefined, - description, - source, - userInvocable: frontmatter['user-invocable'] !== false, - version: frontmatter.version != null ? String(frontmatter.version) : undefined, - contentLength: raw.length, - hasDirectory: true, - } - } catch { - return null - } -} - -async function buildFileTree( - dirPath: string, -): Promise<{ tree: FileTreeNode[]; files: SkillFile[] }> { - const tree: FileTreeNode[] = [] - const files: SkillFile[] = [] - let fileCount = 0 - - async function walk(currentPath: string, nodes: FileTreeNode[]) { - if (fileCount >= MAX_FILES) return - - let entries: import('fs').Dirent[] - try { - entries = await fs.readdir(currentPath, { withFileTypes: true }) - } catch { - return - } - - // directories first, then alphabetical - entries.sort((a, b) => { - if (a.isDirectory() !== b.isDirectory()) return a.isDirectory() ? -1 : 1 - return a.name.localeCompare(b.name) - }) - - for (const entry of entries) { - if (fileCount >= MAX_FILES) break - if (SKIP_ENTRIES.has(entry.name) || entry.name.startsWith('.')) continue - - const fullPath = path.join(currentPath, entry.name) - const relPath = path.relative(dirPath, fullPath) - - if (entry.isDirectory()) { - const node: FileTreeNode = { - name: entry.name, - path: relPath, - type: 'directory', - children: [], - } - nodes.push(node) - await walk(fullPath, node.children!) - if (node.children!.length === 0) delete node.children - } else if (entry.isFile()) { - nodes.push({ name: entry.name, path: relPath, type: 'file' }) - - try { - const stat = await fs.stat(fullPath) - if (stat.size <= MAX_FILE_SIZE) { - const content = await fs.readFile(fullPath, 'utf-8') - files.push({ - path: relPath, - content, - language: detectLanguage(entry.name), - }) - fileCount++ - } - } catch { - // skip unreadable files - } - } - } - } - - await walk(dirPath, tree) - return { tree, files } -} - -// ─── Router ────────────────────────────────────────────────────────────────── - -export async function handleSkillsApi( - req: Request, - url: URL, - segments: string[], -): Promise { - try { - if (req.method !== 'GET') { - throw new ApiError(405, `Method ${req.method} not allowed`, 'METHOD_NOT_ALLOWED') - } - - const sub = segments[2] - - switch (sub) { - case undefined: - return await listSkills() - case 'detail': - return await getSkillDetail(url) - default: - throw ApiError.notFound(`Unknown skills endpoint: ${sub}`) - } - } catch (error) { - return errorResponse(error) - } -} - -// ─── Handlers ──────────────────────────────────────────────────────────────── - -async function listSkills(): Promise { - const skillsDir = getUserSkillsDir() - const skills: SkillMeta[] = [] - - try { - const entries = await fs.readdir(skillsDir, { withFileTypes: true }) - for (const entry of entries) { - if (!entry.isDirectory() || entry.name.startsWith('.')) continue - const meta = await loadSkillMeta( - path.join(skillsDir, entry.name), - entry.name, - 'user', - ) - if (meta) skills.push(meta) - } - } catch { - // skills dir doesn't exist — return empty list - } - - skills.sort((a, b) => a.name.localeCompare(b.name)) - return Response.json({ skills }) -} - -async function getSkillDetail(url: URL): Promise { - const source = url.searchParams.get('source') - const name = url.searchParams.get('name') - - if (!source || !name) { - throw ApiError.badRequest('Missing required query parameters: source, name') - } - - // Prevent path traversal - if (name.includes('..') || name.includes('/') || name.includes('\\')) { - throw ApiError.badRequest('Invalid skill name') - } - - let skillDir: string - if (source === 'user') { - skillDir = path.join(getUserSkillsDir(), name) - } else { - throw ApiError.badRequest(`Unsupported source: ${source}`) - } - - try { - const stat = await fs.stat(skillDir) - if (!stat.isDirectory()) throw new Error() - } catch { - throw ApiError.notFound(`Skill not found: ${name}`) - } - - const meta = await loadSkillMeta(skillDir, name, source as 'user') - if (!meta) { - throw ApiError.notFound(`Skill missing SKILL.md: ${name}`) - } - - const { tree, files } = await buildFileTree(skillDir) - - return Response.json({ - detail: { meta, tree, files, skillRoot: skillDir }, - }) -} -``` - -- [ ] **Step 2: Register the route in router.ts** - -In `src/server/router.ts`, add the import and case: - -```typescript -// Add import at the top (after existing imports): -import { handleSkillsApi } from './api/skills.js' - -// Add case in the switch (before `case 'filesystem':`): - case 'skills': - return handleSkillsApi(req, url, segments) -``` - -- [ ] **Step 3: Verify server compiles** - -Run: `cd /Users/nanmi/workspace/myself_code/claude-code-haha/.claude/worktrees/feature-skill-ui && bun build src/server/index.ts --outdir /tmp/skill-check --target bun 2>&1 | tail -5` - -Expected: No type errors, successful build output. - -- [ ] **Step 4: Commit** - -```bash -git add src/server/api/skills.ts src/server/router.ts -git commit -m "feat(skills): add GET /api/skills and /api/skills/detail endpoints" -``` - ---- - -### Task 3: Desktop API Client - -**Files:** -- Create: `desktop/src/api/skills.ts` - -- [ ] **Step 1: Create the API client** - -```typescript -// desktop/src/api/skills.ts - -import { api } from './client' -import type { SkillMeta, SkillDetail } from '../types/skill' - -export const skillsApi = { - list: () => api.get<{ skills: SkillMeta[] }>('/api/skills'), - - detail: (source: string, name: string) => - api.get<{ detail: SkillDetail }>( - `/api/skills/detail?source=${encodeURIComponent(source)}&name=${encodeURIComponent(name)}`, - ), -} -``` - -- [ ] **Step 2: Commit** - -```bash -git add desktop/src/api/skills.ts -git commit -m "feat(skills): add desktop API client for skills endpoints" -``` - ---- - -### Task 4: Zustand Store - -**Files:** -- Create: `desktop/src/stores/skillStore.ts` - -- [ ] **Step 1: Create the skill store** - -```typescript -// desktop/src/stores/skillStore.ts - -import { create } from 'zustand' -import { skillsApi } from '../api/skills' -import type { SkillMeta, SkillDetail } from '../types/skill' - -type SkillStore = { - skills: SkillMeta[] - selectedSkill: SkillDetail | null - isLoading: boolean - isDetailLoading: boolean - error: string | null - - fetchSkills: () => Promise - fetchSkillDetail: (source: string, name: string) => Promise - clearSelection: () => void -} - -export const useSkillStore = create((set) => ({ - skills: [], - selectedSkill: null, - isLoading: false, - isDetailLoading: false, - error: null, - - fetchSkills: async () => { - set({ isLoading: true, error: null }) - try { - const { skills } = await skillsApi.list() - set({ skills, isLoading: false }) - } catch (err) { - set({ - error: err instanceof Error ? err.message : String(err), - isLoading: false, - }) - } - }, - - fetchSkillDetail: async (source, name) => { - set({ isDetailLoading: true, error: null }) - try { - const { detail } = await skillsApi.detail(source, name) - set({ selectedSkill: detail, isDetailLoading: false }) - } catch (err) { - set({ - error: err instanceof Error ? err.message : String(err), - isDetailLoading: false, - }) - } - }, - - clearSelection: () => set({ selectedSkill: null }), -})) -``` - -- [ ] **Step 2: Commit** - -```bash -git add desktop/src/stores/skillStore.ts -git commit -m "feat(skills): add Zustand store for skill state management" -``` - ---- - -### Task 5: i18n Translation Keys - -**Files:** -- Modify: `desktop/src/i18n/locales/en.ts:47-50` -- Modify: `desktop/src/i18n/locales/zh.ts:47-50` - -- [ ] **Step 1: Add English translations** - -In `desktop/src/i18n/locales/en.ts`, add after `'settings.tab.general': 'General',` (line 49): - -```typescript - 'settings.tab.skills': 'Skills', -``` - -Then add a new section after the `// Settings > Adapters` block (before `// Settings > General`). Insert after line 140 (`'settings.adapters.platform.feishu': 'Feishu',`): - -```typescript - - // Settings > Skills - 'settings.skills.title': 'Installed Skills', - 'settings.skills.description': 'Skills extend Claude with specialized capabilities. Manage skills in ~/.claude/skills/', - 'settings.skills.empty': 'No skills installed', - 'settings.skills.emptyHint': 'Add skills to ~/.claude/skills/ to get started', - 'settings.skills.back': 'Back to list', - 'settings.skills.files': 'files', - 'settings.skills.source.user': 'User', - 'settings.skills.source.project': 'Project', - 'settings.skills.source.plugin': 'Plugin', - 'settings.skills.source.mcp': 'MCP', - 'settings.skills.source.bundled': 'Built-in', -``` - -- [ ] **Step 2: Add Chinese translations** - -In `desktop/src/i18n/locales/zh.ts`, add after the `settings.tab.general` line (around line 50): - -```typescript - 'settings.tab.skills': '技能', -``` - -Then add matching keys after the adapters section: - -```typescript - - // Settings > Skills - 'settings.skills.title': '已安装技能', - 'settings.skills.description': '技能扩展 Claude 的能力。在 ~/.claude/skills/ 中管理技能。', - 'settings.skills.empty': '暂无已安装技能', - 'settings.skills.emptyHint': '在 ~/.claude/skills/ 中添加技能即可开始', - 'settings.skills.back': '返回列表', - 'settings.skills.files': '个文件', - 'settings.skills.source.user': '用户', - 'settings.skills.source.project': '项目', - 'settings.skills.source.plugin': '插件', - 'settings.skills.source.mcp': 'MCP', - 'settings.skills.source.bundled': '内置', -``` - -- [ ] **Step 3: Verify TypeScript is happy** - -Run: `cd /Users/nanmi/workspace/myself_code/claude-code-haha/.claude/worktrees/feature-skill-ui/desktop && npx tsc --noEmit 2>&1 | tail -5` - -Expected: No errors (the `zh.ts` Record type enforces key parity with `en.ts`). - -- [ ] **Step 4: Commit** - -```bash -git add desktop/src/i18n/locales/en.ts desktop/src/i18n/locales/zh.ts -git commit -m "feat(skills): add i18n translation keys for Skills UI" -``` - ---- - -### Task 6: SkillList Component - -**Files:** -- Create: `desktop/src/components/skills/SkillList.tsx` - -- [ ] **Step 1: Create the skill list component** - -```tsx -// desktop/src/components/skills/SkillList.tsx - -import { useEffect } from 'react' -import { useSkillStore } from '../../stores/skillStore' -import { useTranslation } from '../../i18n' -import type { SkillMeta, SkillSource } from '../../types/skill' - -const SOURCE_ORDER: SkillSource[] = ['user', 'project', 'plugin', 'mcp', 'bundled'] - -const SOURCE_ICONS: Record = { - user: 'person', - project: 'folder', - plugin: 'extension', - mcp: 'hub', - bundled: 'inventory_2', -} - -export function SkillList() { - const { skills, isLoading, error, fetchSkills, fetchSkillDetail } = - useSkillStore() - const t = useTranslation() - - useEffect(() => { - fetchSkills() - }, [fetchSkills]) - - if (isLoading) { - return ( -
-
-
- ) - } - - if (error) { - return
{error}
- } - - if (skills.length === 0) { - return ( -
- - auto_awesome - -

- {t('settings.skills.empty')} -

-

- {t('settings.skills.emptyHint')} -

-
- ) - } - - // Group by source - const grouped: Partial> = {} - for (const skill of skills) { - const src = skill.source as SkillSource - ;(grouped[src] ??= []).push(skill) - } - - return ( -
- {SOURCE_ORDER.map((source) => { - const group = grouped[source] - if (!group?.length) return null - - return ( -
- {/* Group header */} -
- - {SOURCE_ICONS[source]} - - - {t(`settings.skills.source.${source}`)} - - - ({group.length}) - -
- - {/* Skill items */} -
- {group.map((skill) => ( - - ))} -
-
- ) - })} -
- ) -} -``` - -- [ ] **Step 2: Commit** - -```bash -git add desktop/src/components/skills/SkillList.tsx -git commit -m "feat(skills): add SkillList component with grouped display" -``` - ---- - -### Task 7: SkillDetail Component - -**Files:** -- Create: `desktop/src/components/skills/SkillDetail.tsx` - -- [ ] **Step 1: Create the skill detail component with file tree and viewer** - -```tsx -// desktop/src/components/skills/SkillDetail.tsx - -import { useState } from 'react' -import { useSkillStore } from '../../stores/skillStore' -import { useTranslation } from '../../i18n' -import { MarkdownRenderer } from '../markdown/MarkdownRenderer' -import { CodeViewer } from '../chat/CodeViewer' -import type { FileTreeNode } from '../../types/skill' - -// ─── Main Component ────────────────────────────────────────────────────────── - -export function SkillDetail() { - const { selectedSkill, isDetailLoading, clearSelection } = useSkillStore() - const t = useTranslation() - const [selectedFile, setSelectedFile] = useState('SKILL.md') - - if (isDetailLoading) { - return ( -
-
-
- ) - } - - if (!selectedSkill) return null - - const { meta, tree, files } = selectedSkill - const currentFile = files.find((f) => f.path === selectedFile) || files[0] - - return ( -
- {/* Back button */} -
- -
- - {/* Skill header */} -
-
-

- {meta.displayName || meta.name} -

- - {meta.source} - -
-

- {meta.description} -

-
- ~{Math.ceil(meta.contentLength / 4)} tokens - - {files.length} {t('settings.skills.files')} - - {meta.version && v{meta.version}} -
-
- - {/* Two-panel: file tree + content viewer */} -
- {/* Left: File tree */} -
- -
- - {/* Right: File content */} -
- {currentFile && ( -
- {/* File path header */} -
- - {currentFile.path} - - - {currentFile.language} - -
- - {/* Content */} -
- {currentFile.language === 'markdown' ? ( - - ) : ( - - )} -
-
- )} -
-
-
- ) -} - -// ─── File Tree Components ──────────────────────────────────────────────────── - -function TreeView({ - nodes, - selectedPath, - onSelect, - depth, -}: { - nodes: FileTreeNode[] - selectedPath: string - onSelect: (path: string) => void - depth: number -}) { - return ( - <> - {nodes.map((node) => ( - - ))} - - ) -} - -function TreeItem({ - node, - selectedPath, - onSelect, - depth, -}: { - node: FileTreeNode - selectedPath: string - onSelect: (path: string) => void - depth: number -}) { - const [expanded, setExpanded] = useState(true) - const isSelected = node.path === selectedPath - const isDir = node.type === 'directory' - - const icon = isDir - ? expanded - ? 'folder_open' - : 'folder' - : fileIcon(node.name) - - return ( -
- - - {isDir && expanded && node.children && ( - - )} -
- ) -} - -function fileIcon(filename: string): string { - const ext = filename.split('.').pop()?.toLowerCase() - switch (ext) { - case 'md': - return 'description' - case 'ts': - case 'tsx': - case 'js': - case 'jsx': - case 'py': - case 'rs': - case 'go': - return 'code' - case 'json': - case 'yaml': - case 'yml': - case 'toml': - return 'data_object' - case 'sh': - case 'bash': - return 'terminal' - default: - return 'draft' - } -} -``` - -- [ ] **Step 2: Commit** - -```bash -git add desktop/src/components/skills/SkillDetail.tsx -git commit -m "feat(skills): add SkillDetail component with file tree and content viewer" -``` - ---- - -### Task 8: Settings Page Integration - -**Files:** -- Modify: `desktop/src/pages/Settings.tsx:1-58` - -- [ ] **Step 1: Add Skills tab to Settings page** - -In `desktop/src/pages/Settings.tsx`, apply these changes: - -**1. Add imports at the top** (after existing imports): - -```typescript -import { useSkillStore } from '../stores/skillStore' -import { SkillList } from '../components/skills/SkillList' -import { SkillDetail } from '../components/skills/SkillDetail' -``` - -**2. Extend the SettingsTab type** (line 15): - -Change: -```typescript -type SettingsTab = 'providers' | 'permissions' | 'general' | 'adapters' -``` -To: -```typescript -type SettingsTab = 'providers' | 'permissions' | 'general' | 'adapters' | 'skills' -``` - -**3. Add the TabButton** in the Settings component (after the adapters TabButton, around line 29): - -```tsx - setActiveTab('skills')} /> -``` - -**4. Add the content renderer** (after the adapters condition, around line 35): - -```tsx - {activeTab === 'skills' && } -``` - -**5. Add the SkillSettings wrapper component** at the bottom of the file (before the closing line, after `GeneralSettings`): - -```typescript -// ─── Skill Settings ────────────────────────────────────── - -function SkillSettings() { - const selectedSkill = useSkillStore((s) => s.selectedSkill) - const t = useTranslation() - - if (selectedSkill) { - return ( -
- -
- ) - } - - return ( -
-

- {t('settings.skills.title')} -

-

- {t('settings.skills.description')} -

- -
- ) -} -``` - -- [ ] **Step 2: Verify the desktop app compiles** - -Run: `cd /Users/nanmi/workspace/myself_code/claude-code-haha/.claude/worktrees/feature-skill-ui/desktop && npx tsc --noEmit 2>&1 | tail -10` - -Expected: No type errors. - -- [ ] **Step 3: Commit** - -```bash -git add desktop/src/pages/Settings.tsx -git commit -m "feat(skills): integrate Skills tab into Settings page" -``` - ---- - -### Task 9: Manual Smoke Test - -- [ ] **Step 1: Create a test skill directory** - -```bash -mkdir -p ~/.claude/skills/test-skill -cat > ~/.claude/skills/test-skill/SKILL.md << 'SKILLEOF' ---- -name: Test Skill -description: A test skill for verifying the Skills UI -version: 1.0 -user-invocable: true ---- - -# Test Skill - -This is a test skill to verify the Skills browser works correctly. - -## Usage - -Just invoke `/test-skill` from the chat. - -```typescript -console.log('Hello from test skill') -``` -SKILLEOF - -cat > ~/.claude/skills/test-skill/helper.ts << 'HELPEREOF' -export function greet(name: string): string { - return `Hello, ${name}!` -} -HELPEREOF -``` - -- [ ] **Step 2: Start the server and verify API** - -Run: `cd /Users/nanmi/workspace/myself_code/claude-code-haha/.claude/worktrees/feature-skill-ui && bun run src/server/index.ts &` - -Then test: -```bash -curl -s http://127.0.0.1:3456/api/skills | python3 -m json.tool -``` - -Expected: JSON with `skills` array containing `test-skill` entry. - -```bash -curl -s "http://127.0.0.1:3456/api/skills/detail?source=user&name=test-skill" | python3 -m json.tool -``` - -Expected: JSON with `detail` containing `meta`, `tree` (SKILL.md + helper.ts), `files` with content. - -- [ ] **Step 3: Clean up test data (optional)** - -```bash -rm -rf ~/.claude/skills/test-skill -``` - -- [ ] **Step 4: Final commit with all changes** - -```bash -git add -A -git commit -m "feat: add Skills browser to Settings page - -- Server: GET /api/skills (list) and GET /api/skills/detail (tree + files) -- Desktop: Skills tab in Settings with list/detail views -- File tree navigation with Markdown and code preview -- i18n support (EN/ZH)" -``` diff --git a/docs/superpowers/specs/2026-04-09-skill-ui-design.md b/docs/superpowers/specs/2026-04-09-skill-ui-design.md deleted file mode 100644 index d4036988..00000000 --- a/docs/superpowers/specs/2026-04-09-skill-ui-design.md +++ /dev/null @@ -1,271 +0,0 @@ -# Skills UI Feature Design - -## Overview - -Port the CLI `/skills` listing to the Desktop Web APP, adding a full Skills browser with file tree navigation and content preview in the Settings page. - -## Architecture - -``` -Desktop UI (React) Server (Bun.serve) Filesystem -┌──────────────────┐ ┌──────────────────┐ ┌──────────────────┐ -│ Settings Page │ │ GET /api/skills │ │ ~/.claude/skills/│ -│ └─ Skills Tab │──────▶│ → list meta │──────▶│ ./.claude/skills/│ -│ ├─ SkillList │ │ │ │ plugins cache/ │ -│ └─ SkillDetail │ GET /api/skills/ │ │ bundled skills/ │ -│ ├─ FileTree│──────▶│ detail?s=&n= │──────▶│ │ -│ └─ Viewer │ │ → tree + files │ │ │ -└──────────────────┘ └──────────────────┘ └──────────────────┘ -``` - -## Data Models - -### SkillMeta (list item) - -```typescript -type SkillSource = 'userSettings' | 'projectSettings' | 'policySettings' - | 'plugin' | 'mcp' | 'bundled' - -type SkillMeta = { - name: string // unique name (plugin format: "plugin:skill") - displayName?: string // custom display name from frontmatter - description: string - source: SkillSource - userInvocable: boolean - version?: string - contentLength: number // char count for token estimation - pluginName?: string // plugin name when source is 'plugin' - hasDirectory: boolean // whether the skill has a browsable directory -} -``` - -### FileTreeNode - -```typescript -type FileTreeNode = { - name: string // file/dir name - path: string // relative to skill root - type: 'file' | 'directory' - children?: FileTreeNode[] // present for directories -} -``` - -### SkillFile - -```typescript -type SkillFile = { - path: string // relative path - content: string // file content - language: string // language id (md, ts, json, yaml, sh, etc.) -} -``` - -### SkillDetail (detail view) - -```typescript -type SkillDetail = { - meta: SkillMeta - tree: FileTreeNode[] // directory tree - files: SkillFile[] // all file contents, loaded at once - skillRoot: string // absolute path (display only) -} -``` - -## Server API - -### `GET /api/skills` - -Returns metadata list of all installed skills. - -**Response:** -```json -{ - "skills": [ - { - "name": "my-skill", - "displayName": "My Skill", - "description": "Does something useful", - "source": "userSettings", - "userInvocable": true, - "contentLength": 1234, - "hasDirectory": true - } - ] -} -``` - -**Implementation:** Reuse `loadAllCommands()` from `src/commands.ts`, filter for `type === 'prompt'` commands with skill-related `loadedFrom` values. Extract metadata without loading full content. - -### `GET /api/skills/detail?source={source}&name={name}` - -Returns full skill data including file tree and all file contents. - -**Response:** -```json -{ - "detail": { - "meta": { ... }, - "tree": [ - { "name": "SKILL.md", "path": "SKILL.md", "type": "file" }, - { - "name": "examples", - "path": "examples", - "type": "directory", - "children": [ - { "name": "demo.ts", "path": "examples/demo.ts", "type": "file" } - ] - } - ], - "files": [ - { "path": "SKILL.md", "content": "---\nname: ...\n---\n# ...", "language": "md" }, - { "path": "examples/demo.ts", "content": "...", "language": "ts" } - ], - "skillRoot": "/Users/x/.claude/skills/my-skill" - } -} -``` - -**Implementation:** -1. Find matching skill by source + name from loaded commands -2. Get `skillRoot` from the command object -3. Recursively walk the directory, skip `node_modules`, `.git`, hidden dirs -4. Read all files (max 100KB per file, max 50 files total as safety limit) -5. Detect language from file extension -6. For MCP skills (no directory): return empty tree, single virtual file with the skill's prompt content - -### File: `src/server/api/skills.ts` - -New file following the pattern of `src/server/api/status.ts`. - -### Router registration: `src/server/router.ts` - -Add `case 'skills': return handleSkillsApi(req, url, segments)`. - -## Desktop UI - -### Settings Tab Addition - -Add a 5th tab "Skills" (icon: `auto_awesome`) to `desktop/src/pages/Settings.tsx`. - -``` -type SettingsTab = 'providers' | 'permissions' | 'general' | 'adapters' | 'skills' -``` - -### Skill List View (`SkillList.tsx`) - -- Grouped by source (User, Project, Plugin, MCP, Bundled) -- Each group: collapsible header with count -- Each item: name, description (truncated), source badge -- Click → navigate to detail view -- Loading state + empty state - -### Skill Detail View (`SkillDetail.tsx`) - -Two-panel layout: -- **Left panel (w-[220px]):** File tree with expand/collapse -- **Right panel (flex-1):** File content viewer -- **Header:** Back button, skill name, description, source badge, token estimate -- Default selected file: SKILL.md - -### File Tree Component (`FileTree.tsx`) - -- Recursive tree with indent -- Directory expand/collapse (default: all expanded) -- File icons based on extension -- Click file → show in right panel -- Active file highlight - -### File Viewer (`FileViewer.tsx`) - -- For `.md` files: Use existing `MarkdownRenderer` -- For code files: Use existing `CodeViewer` with language detection -- File path header with copy button -- Scrollable content area - -### Language Detection - -Map file extension to language: -```typescript -const LANG_MAP: Record = { - md: 'markdown', ts: 'typescript', tsx: 'typescript', - js: 'javascript', jsx: 'javascript', json: 'json', - yaml: 'yaml', yml: 'yaml', sh: 'bash', bash: 'bash', - py: 'python', toml: 'toml', css: 'css', html: 'html', -} -``` - -## State Management - -### `desktop/src/stores/skillStore.ts` - -```typescript -type SkillStoreState = { - skills: SkillMeta[] - selectedSkill: SkillDetail | null - isLoading: boolean - isDetailLoading: boolean - error: string | null - - fetchSkills: () => Promise - fetchSkillDetail: (source: string, name: string) => Promise - clearSelection: () => void -} -``` - -### `desktop/src/api/skills.ts` - -```typescript -export const skillsApi = { - list: () => api.get<{ skills: SkillMeta[] }>('/api/skills'), - detail: (source: string, name: string) => - api.get<{ detail: SkillDetail }>(`/api/skills/detail?source=${encodeURIComponent(source)}&name=${encodeURIComponent(name)}`), -} -``` - -### `desktop/src/types/skill.ts` - -All type definitions from Data Models section. - -## i18n - -Add to both `en.ts` and `zh.ts`: - -```typescript -'settings.tab.skills': 'Skills' / '技能', -'settings.skills.title': 'Installed Skills' / '已安装技能', -'settings.skills.description': 'Skills extend Claude...' / '技能扩展 Claude...', -'settings.skills.empty': 'No skills installed' / '暂无已安装技能', -'settings.skills.back': 'Back to list' / '返回列表', -'settings.skills.tokens': '~{count} tokens', -'settings.skills.files': '{count} files' / '{count} 个文件', -'settings.skills.source.userSettings': 'User' / '用户', -'settings.skills.source.projectSettings': 'Project' / '项目', -'settings.skills.source.plugin': 'Plugin' / '插件', -'settings.skills.source.mcp': 'MCP', -'settings.skills.source.bundled': 'Built-in' / '内置', -``` - -## Files to Create/Modify - -### New files (7): -1. `src/server/api/skills.ts` — Server API handler -2. `desktop/src/types/skill.ts` — Type definitions -3. `desktop/src/api/skills.ts` — API client -4. `desktop/src/stores/skillStore.ts` — Zustand store -5. `desktop/src/components/skills/SkillList.tsx` — Skill list component -6. `desktop/src/components/skills/SkillDetail.tsx` — Detail view (FileTree + FileViewer inline) - -### Modified files (4): -7. `src/server/router.ts` — Add skills route -8. `desktop/src/pages/Settings.tsx` — Add Skills tab -9. `desktop/src/i18n/locales/en.ts` — English translations -10. `desktop/src/i18n/locales/zh.ts` — Chinese translations - -## Design Decisions - -1. **One detail request loads all files** — Single skill directories are small (typically <50KB total), so loading all at once avoids per-file request overhead. -2. **Reuse existing components** — `MarkdownRenderer` for .md files, `CodeViewer` for code. No new rendering dependencies. -3. **source + name as identifier** — Skills can have duplicate names across sources, so both are needed. -4. **Server-side skill loading** — Reuse `loadAllCommands()` which already handles all 6 sources with deduplication and caching. -5. **Safety limits** — Max 50 files, 100KB per file to prevent issues with massive plugin directories. -6. **FileTree + FileViewer inline in SkillDetail** — No need for separate components given the simplicity; keeps code collocated. diff --git a/package.json b/package.json index 4a2b324d..34ea1a7a 100644 --- a/package.json +++ b/package.json @@ -17,6 +17,11 @@ "check:adapters": "cd adapters && bun test", "check:native": "cd desktop && bun run build:sidecars && cd src-tauri && cargo check", "check:docs": "npm ci && npm run docs:build", + "quality:gate": "bun run scripts/quality-gate/index.ts", + "quality:providers": "bun run scripts/quality-gate/providers.ts", + "quality:pr": "bun run quality:gate --mode pr", + "quality:baseline": "bun run quality:gate --mode baseline", + "quality:release": "bun run quality:gate --mode release", "docs:dev": "vitepress dev docs", "docs:build": "vitepress build docs", "docs:preview": "vitepress preview docs" diff --git a/scripts/pr/run-server-tests.ts b/scripts/pr/run-server-tests.ts index baadf80a..937b4baa 100644 --- a/scripts/pr/run-server-tests.ts +++ b/scripts/pr/run-server-tests.ts @@ -2,18 +2,11 @@ import { readdirSync, statSync } from 'node:fs' import { join, relative, sep } from 'node:path' +import { quarantinedPathSet } from '../quality-gate/quarantine' const root = process.cwd() const roots = ['src/server', 'src/tools', 'src/utils'] -const excludedFiles = new Set([ - // These suites are not stable enough for the default PR gate yet. Keep them - // out of CI until they are fixed or moved to a maintainer-only workflow. - 'src/server/__tests__/cron-scheduler.test.ts', - 'src/server/__tests__/providers-real.test.ts', - 'src/server/__tests__/tasks.test.ts', - 'src/server/__tests__/e2e/business-flow.test.ts', - 'src/server/__tests__/e2e/full-flow.test.ts', -]) +const excludedFiles = quarantinedPathSet() function normalize(path: string) { return relative(root, path).split(sep).join('/') diff --git a/scripts/quality-gate/baseline/cases.test.ts b/scripts/quality-gate/baseline/cases.test.ts new file mode 100644 index 00000000..c895e099 --- /dev/null +++ b/scripts/quality-gate/baseline/cases.test.ts @@ -0,0 +1,49 @@ +import { describe, expect, test } from 'bun:test' +import { existsSync } from 'node:fs' +import { join } from 'node:path' +import { baselineCases, validateBaselineCases } from './cases' + +describe('baselineCases', () => { + test('have valid metadata', () => { + expect(() => validateBaselineCases()).not.toThrow() + }) + + test('use unique ids', () => { + const ids = baselineCases.map((testCase) => testCase.id) + expect(new Set(ids).size).toBe(ids.length) + }) + + test('point at existing fixtures with package manifests', () => { + for (const testCase of baselineCases) { + expect(existsSync(testCase.fixture)).toBe(true) + expect(existsSync(join(testCase.fixture, 'package.json'))).toBe(true) + } + }) + + test('require real model capability', () => { + for (const testCase of baselineCases) { + expect(testCase.requiredCapabilities).toContain('model') + } + }) + + test('define enough first-wave product baseline cases', () => { + expect(baselineCases.map((testCase) => testCase.id)).toEqual([ + 'failing-unit', + 'multi-file-api', + 'failure-recovery', + 'workspace-search-edit', + 'permission-artifact', + 'cross-module-refactor', + ]) + }) + + test('pin changed-file expectations for every case', () => { + for (const testCase of baselineCases) { + expect(testCase.verify.requiredFiles?.length ?? 0).toBeGreaterThan(0) + expect(testCase.verify.expectedFiles?.length ?? 0).toBeGreaterThan(0) + for (const file of testCase.verify.requiredFiles ?? []) { + expect(testCase.verify.expectedFiles).toContain(file) + } + } + }) +}) diff --git a/scripts/quality-gate/baseline/cases.ts b/scripts/quality-gate/baseline/cases.ts new file mode 100644 index 00000000..34ab7849 --- /dev/null +++ b/scripts/quality-gate/baseline/cases.ts @@ -0,0 +1,127 @@ +import type { BaselineCase } from '../types' + +export const baselineCases: BaselineCase[] = [ + { + id: 'failing-unit', + title: 'Fix a failing unit test', + description: 'A tiny TypeScript project has a broken arithmetic function. The Agent must inspect the failing test, patch the implementation, and rerun the test.', + fixture: 'scripts/quality-gate/baseline/fixtures/failing-unit', + prompt: 'Run the tests, inspect the failing assertion, fix the implementation bug, and rerun the tests until they pass. Only modify the fixture source files needed for the fix.', + mode: 'websocket', + requiredCapabilities: ['model', 'file-edit', 'shell'], + timeoutMs: 180_000, + verify: { + commands: [['bun', 'test']], + requiredFiles: ['src/math.ts'], + expectedFiles: ['src/math.ts'], + forbiddenFiles: ['package.json'], + transcriptAssertions: ['"type":"message_complete"'], + }, + }, + { + id: 'multi-file-api', + title: 'Update a multi-file API contract', + description: 'A small app exposes a user display API. The Agent must change the contract and update callers coherently without loosening tests.', + fixture: 'scripts/quality-gate/baseline/fixtures/multi-file-api', + prompt: 'The user display API contract changed to return an object with a label field, where label is "Ada Lovelace ". Update the implementation and caller so the existing tests pass. Do not edit package.json.', + mode: 'websocket', + requiredCapabilities: ['model', 'file-edit', 'shell'], + timeoutMs: 240_000, + verify: { + commands: [['bun', 'test']], + requiredFiles: ['src/api.ts', 'src/app.ts'], + expectedFiles: ['src/api.ts', 'src/app.ts', 'src/app.test.ts'], + forbiddenFiles: ['package.json'], + transcriptAssertions: ['"type":"message_complete"'], + }, + }, + { + id: 'failure-recovery', + title: 'Recover from a deeper failing test suite', + description: 'A slug formatter has several edge-case failures. The Agent must read the failures, patch the implementation, and keep tests unchanged.', + fixture: 'scripts/quality-gate/baseline/fixtures/failure-recovery', + prompt: 'Run the tests and fix the slug formatter implementation until the whole suite passes. Do not edit the tests. Keep the implementation small and deterministic.', + mode: 'websocket', + requiredCapabilities: ['model', 'file-edit', 'shell'], + timeoutMs: 240_000, + verify: { + commands: [['bun', 'test']], + requiredFiles: ['src/slug.ts'], + expectedFiles: ['src/slug.ts'], + forbiddenFiles: ['src/slug.test.ts', 'package.json'], + transcriptAssertions: ['"type":"message_complete"'], + }, + }, + { + id: 'workspace-search-edit', + title: 'Find and fix a bug across a small workspace', + description: 'A checkout total is wrong in a multi-file fixture. The Agent must inspect the workspace and patch the correct implementation file.', + fixture: 'scripts/quality-gate/baseline/fixtures/workspace-search-edit', + prompt: 'The checkout tests are failing. Search the project, identify the real bug, fix the implementation, and rerun the tests. Do not rewrite unrelated modules.', + mode: 'websocket', + requiredCapabilities: ['model', 'file-edit', 'shell'], + timeoutMs: 240_000, + verify: { + commands: [['bun', 'test']], + requiredFiles: ['src/checkout.ts'], + expectedFiles: ['src/checkout.ts'], + forbiddenFiles: ['src/checkout.test.ts', 'src/catalog.ts', 'src/user.ts', 'package.json'], + transcriptAssertions: ['"type":"message_complete"'], + }, + }, + { + id: 'permission-artifact', + title: 'Create a required project artifact', + description: 'A test expects a generated summary file. The Agent must create a new file and validate it without changing the test.', + fixture: 'scripts/quality-gate/baseline/fixtures/permission-artifact', + prompt: 'Run the tests, create the missing project artifact they require, and rerun the tests. Do not modify the tests or package.json.', + mode: 'websocket', + requiredCapabilities: ['model', 'file-edit', 'shell', 'permission'], + timeoutMs: 180_000, + verify: { + commands: [['bun', 'test']], + requiredFiles: ['notes/summary.md'], + expectedFiles: ['notes/summary.md'], + forbiddenFiles: ['src/artifact.test.ts', 'package.json'], + transcriptAssertions: ['"type":"message_complete"'], + }, + }, + { + id: 'cross-module-refactor', + title: 'Complete a cross-module parser refactor', + description: 'A config parser contract changed but the implementation and caller are stale. The Agent must update multiple modules coherently.', + fixture: 'scripts/quality-gate/baseline/fixtures/cross-module-refactor', + prompt: 'Update the config parsing flow so parseConfig returns a structured object with enabled and retries fields. Update the implementation and caller so the existing tests pass, then run the suite.', + mode: 'websocket', + requiredCapabilities: ['model', 'file-edit', 'shell'], + timeoutMs: 300_000, + verify: { + commands: [['bun', 'test']], + requiredFiles: ['src/config.ts', 'src/runner.ts'], + expectedFiles: ['src/config.ts', 'src/runner.ts', 'src/runner.test.ts'], + forbiddenFiles: ['package.json'], + transcriptAssertions: ['"type":"message_complete"'], + }, + }, +] + +export function validateBaselineCases(cases = baselineCases) { + const ids = new Set() + + for (const testCase of cases) { + if (ids.has(testCase.id)) { + throw new Error(`Duplicate baseline case id: ${testCase.id}`) + } + ids.add(testCase.id) + + if (!testCase.fixture) { + throw new Error(`Baseline case ${testCase.id} is missing a fixture`) + } + if (testCase.verify.commands.length === 0) { + throw new Error(`Baseline case ${testCase.id} is missing verification commands`) + } + if (testCase.timeoutMs < 30_000) { + throw new Error(`Baseline case ${testCase.id} timeout is too low`) + } + } +} diff --git a/scripts/quality-gate/baseline/execute.ts b/scripts/quality-gate/baseline/execute.ts new file mode 100644 index 00000000..058d83ae --- /dev/null +++ b/scripts/quality-gate/baseline/execute.ts @@ -0,0 +1,327 @@ +import { appendFileSync, cpSync, mkdirSync, readdirSync, readFileSync, rmSync, statSync, writeFileSync } from 'node:fs' +import { mkdtemp } from 'node:fs/promises' +import { tmpdir } from 'node:os' +import { join } from 'node:path' +import { createServer } from 'node:net' +import type { BaselineCase, BaselineTarget, LaneResult } from '../types' + +type ServerMessage = { + type: string + requestId?: string + message?: string + code?: string + [key: string]: unknown +} + +function getPort() { + return new Promise((resolve, reject) => { + const server = createServer() + server.once('error', reject) + server.listen(0, '127.0.0.1', () => { + const address = server.address() + const port = typeof address === 'object' && address ? address.port : 0 + server.close(() => resolve(port)) + }) + }) +} + +async function waitForHttp(url: string, timeoutMs: number) { + const deadline = Date.now() + timeoutMs + while (Date.now() < deadline) { + try { + const res = await fetch(url) + if (res.ok) return + } catch { + // Keep polling until timeout. + } + await Bun.sleep(500) + } + throw new Error(`Timed out waiting for ${url}`) +} + +async function runCommand(command: string[], cwd: string) { + const proc = Bun.spawn(command, { + cwd, + stdout: 'pipe', + stderr: 'pipe', + }) + const stdout = await new Response(proc.stdout).text() + const stderr = await new Response(proc.stderr).text() + const exitCode = await proc.exited + return { exitCode, stdout, stderr } +} + +function listFiles(root: string, current = root): string[] { + const files: string[] = [] + for (const entry of readdirSync(current)) { + if (entry === 'node_modules' || entry === '.git') { + continue + } + + const fullPath = join(current, entry) + const stat = statSync(fullPath) + if (stat.isDirectory()) { + files.push(...listFiles(root, fullPath)) + continue + } + if (stat.isFile()) { + files.push(fullPath.slice(root.length + 1)) + } + } + return files.sort() +} + +export function changedFiles(beforeDir: string, afterDir: string) { + const before = new Set(listFiles(beforeDir)) + const after = new Set(listFiles(afterDir)) + const changed = new Set() + + for (const file of before) { + if (!after.has(file)) { + changed.add(file) + continue + } + if (!readFileSync(join(beforeDir, file)).equals(readFileSync(join(afterDir, file)))) { + changed.add(file) + } + } + + for (const file of after) { + if (!before.has(file)) { + changed.add(file) + } + } + + return [...changed].sort() +} + +export async function writeDiffPatch(beforeDir: string, afterDir: string, patchPath: string) { + const result = await runCommand(['git', 'diff', '--no-index', '--', beforeDir, afterDir], process.cwd()) + writeFileSync(patchPath, `${result.stdout}${result.stderr}`) +} + +function verifyChangedFiles(testCase: BaselineCase, changed: string[]) { + const expected = testCase.verify.expectedFiles + if (expected) { + const unexpected = changed.filter((file) => !expected.includes(file)) + if (unexpected.length > 0) { + throw new Error(`unexpected changed files: ${unexpected.join(', ')}`) + } + } + + const required = testCase.verify.requiredFiles ?? [] + const missing = required.filter((file) => !changed.includes(file)) + if (missing.length > 0) { + throw new Error(`required files were not changed: ${missing.join(', ')}`) + } + + const forbidden = testCase.verify.forbiddenFiles ?? [] + const forbiddenChanged = forbidden.filter((file) => changed.includes(file)) + if (forbiddenChanged.length > 0) { + throw new Error(`forbidden files changed: ${forbiddenChanged.join(', ')}`) + } +} + +function verifyTranscript(testCase: BaselineCase, transcriptPath: string) { + const assertions = testCase.verify.transcriptAssertions ?? [] + if (assertions.length === 0) { + return + } + + const transcript = readFileSync(transcriptPath, 'utf8') + const missing = assertions.filter((assertion) => !transcript.includes(assertion)) + if (missing.length > 0) { + throw new Error(`transcript assertions missing: ${missing.join(', ')}`) + } +} + +async function pipeToFile(stream: ReadableStream | null, path: string) { + if (!stream) return + const reader = stream.getReader() + const decoder = new TextDecoder() + while (true) { + const { value, done } = await reader.read() + if (done) break + appendFileSync(path, decoder.decode(value, { stream: true })) + } +} + +function waitForWebSocketOpen(ws: WebSocket) { + return new Promise((resolve, reject) => { + ws.onopen = () => resolve() + ws.onerror = () => reject(new Error('WebSocket failed to open')) + }) +} + +async function runPromptOverWebSocket( + baseUrl: string, + sessionId: string, + prompt: string, + timeoutMs: number, + target?: BaselineTarget, +) { + const wsUrl = baseUrl.replace(/^http/, 'ws') + const ws = new WebSocket(`${wsUrl}/ws/${sessionId}`) + const messages: ServerMessage[] = [] + + try { + await waitForWebSocketOpen(ws) + + await new Promise((resolve, reject) => { + const timer = setTimeout(() => { + reject(new Error(`Timed out waiting for baseline case completion after ${timeoutMs}ms`)) + }, timeoutMs) + + ws.onmessage = (event) => { + const message = JSON.parse(String(event.data)) as ServerMessage + messages.push(message) + + if (message.type === 'connected') { + if (target && target.modelId !== 'current') { + ws.send(JSON.stringify({ + type: 'set_runtime_config', + providerId: target.providerId, + modelId: target.modelId, + })) + } + ws.send(JSON.stringify({ type: 'user_message', content: prompt })) + return + } + + if (message.type === 'permission_request' && typeof message.requestId === 'string') { + ws.send(JSON.stringify({ + type: 'permission_response', + requestId: message.requestId, + allowed: true, + rule: 'baseline-run', + })) + return + } + + if (message.type === 'message_complete') { + clearTimeout(timer) + resolve() + return + } + + if (message.type === 'error') { + clearTimeout(timer) + reject(new Error(`${message.code ?? 'WS_ERROR'}: ${message.message ?? 'unknown error'}`)) + } + } + + ws.onerror = () => { + clearTimeout(timer) + reject(new Error('WebSocket error during baseline case')) + } + }) + } finally { + ws.close() + } + + return messages +} + +export async function executeBaselineCase( + testCase: BaselineCase, + rootDir: string, + artifactDir: string, + target?: BaselineTarget, +): Promise { + const started = Date.now() + const resultId = target + ? `baseline:${testCase.id}:${target.label.replace(/[^a-zA-Z0-9._-]+/g, '-')}` + : `baseline:${testCase.id}` + const resultTitle = target ? `${testCase.title} (${target.label})` : testCase.title + mkdirSync(artifactDir, { recursive: true }) + + const port = await getPort() + const baseUrl = `http://127.0.0.1:${port}` + const workRoot = await mkdtemp(join(tmpdir(), `quality-gate-${testCase.id}-`)) + const originalDir = join(workRoot, 'original') + const projectDir = join(workRoot, 'project') + cpSync(join(rootDir, testCase.fixture), originalDir, { recursive: true }) + cpSync(join(rootDir, testCase.fixture), projectDir, { recursive: true }) + + const serverLogPath = join(artifactDir, 'server.log') + const transcriptPath = join(artifactDir, 'transcript.jsonl') + const verificationPath = join(artifactDir, 'verification.log') + const diffPath = join(artifactDir, 'diff.patch') + const server = Bun.spawn(['bun', 'run', 'src/server/index.ts', '--host', '127.0.0.1', '--port', String(port)], { + cwd: rootDir, + stdout: 'pipe', + stderr: 'pipe', + env: { + ...process.env, + SERVER_PORT: String(port), + }, + }) + const stdoutPump = pipeToFile(server.stdout, serverLogPath) + const stderrPump = pipeToFile(server.stderr, serverLogPath) + + try { + await waitForHttp(`${baseUrl}/health`, 60_000) + + const createResponse = await fetch(`${baseUrl}/api/sessions`, { + method: 'POST', + headers: { 'Content-Type': 'application/json' }, + body: JSON.stringify({ workDir: projectDir }), + }) + if (!createResponse.ok) { + throw new Error(`Failed to create session: ${createResponse.status}`) + } + const session = await createResponse.json() as { sessionId?: string } + if (!session.sessionId) { + throw new Error('Session response did not include sessionId') + } + + const messages = await runPromptOverWebSocket(baseUrl, session.sessionId, testCase.prompt, testCase.timeoutMs, target) + writeFileSync(transcriptPath, messages.map((message) => JSON.stringify(message)).join('\n') + '\n') + verifyTranscript(testCase, transcriptPath) + + await writeDiffPatch(originalDir, projectDir, diffPath) + const changed = changedFiles(originalDir, projectDir) + verifyChangedFiles(testCase, changed) + + let verificationLog = '' + for (const command of testCase.verify.commands) { + const result = await runCommand(command, projectDir) + verificationLog += `$ ${command.join(' ')}\n${result.stdout}${result.stderr}\n` + if (result.exitCode !== 0) { + writeFileSync(verificationPath, verificationLog) + return { + id: resultId, + title: resultTitle, + status: 'failed', + durationMs: Date.now() - started, + exitCode: result.exitCode, + error: `verification command failed: ${command.join(' ')}`, + artifactDir, + } + } + } + writeFileSync(verificationPath, verificationLog) + + return { + id: resultId, + title: resultTitle, + status: 'passed', + durationMs: Date.now() - started, + artifactDir, + } + } catch (error) { + return { + id: resultId, + title: resultTitle, + status: 'failed', + durationMs: Date.now() - started, + error: error instanceof Error ? error.message : String(error), + artifactDir, + } + } finally { + server.kill() + await server.exited.catch(() => undefined) + await Promise.all([stdoutPump, stderrPump]).catch(() => undefined) + rmSync(workRoot, { recursive: true, force: true }) + } +} diff --git a/scripts/quality-gate/baseline/fixtures/cross-module-refactor/package.json b/scripts/quality-gate/baseline/fixtures/cross-module-refactor/package.json new file mode 100644 index 00000000..7de3edee --- /dev/null +++ b/scripts/quality-gate/baseline/fixtures/cross-module-refactor/package.json @@ -0,0 +1,8 @@ +{ + "name": "quality-gate-cross-module-refactor-fixture", + "private": true, + "type": "module", + "scripts": { + "test": "bun test" + } +} diff --git a/scripts/quality-gate/baseline/fixtures/cross-module-refactor/src/config.ts b/scripts/quality-gate/baseline/fixtures/cross-module-refactor/src/config.ts new file mode 100644 index 00000000..b7c271ab --- /dev/null +++ b/scripts/quality-gate/baseline/fixtures/cross-module-refactor/src/config.ts @@ -0,0 +1,3 @@ +export function parseConfig(raw: string) { + return raw === 'enabled' +} diff --git a/scripts/quality-gate/baseline/fixtures/cross-module-refactor/src/runner.test.ts b/scripts/quality-gate/baseline/fixtures/cross-module-refactor/src/runner.test.ts new file mode 100644 index 00000000..dbf12a1e --- /dev/null +++ b/scripts/quality-gate/baseline/fixtures/cross-module-refactor/src/runner.test.ts @@ -0,0 +1,17 @@ +import { describe, expect, test } from 'bun:test' +import { parseConfig } from './config' +import { describeRun } from './runner' + +describe('config runner', () => { + test('parses structured config', () => { + expect(parseConfig('enabled retries=3')).toEqual({ enabled: true, retries: 3 }) + }) + + test('describes enabled retry runs', () => { + expect(describeRun('enabled retries=3')).toBe('enabled with 3 retries') + }) + + test('describes disabled runs', () => { + expect(describeRun('disabled retries=0')).toBe('disabled') + }) +}) diff --git a/scripts/quality-gate/baseline/fixtures/cross-module-refactor/src/runner.ts b/scripts/quality-gate/baseline/fixtures/cross-module-refactor/src/runner.ts new file mode 100644 index 00000000..9304d604 --- /dev/null +++ b/scripts/quality-gate/baseline/fixtures/cross-module-refactor/src/runner.ts @@ -0,0 +1,6 @@ +import { parseConfig } from './config' + +export function describeRun(rawConfig: string) { + const enabled = parseConfig(rawConfig) + return enabled ? 'enabled' : 'disabled' +} diff --git a/scripts/quality-gate/baseline/fixtures/failing-unit/package.json b/scripts/quality-gate/baseline/fixtures/failing-unit/package.json new file mode 100644 index 00000000..391c0814 --- /dev/null +++ b/scripts/quality-gate/baseline/fixtures/failing-unit/package.json @@ -0,0 +1,8 @@ +{ + "name": "quality-gate-failing-unit-fixture", + "private": true, + "type": "module", + "scripts": { + "test": "bun test" + } +} diff --git a/scripts/quality-gate/baseline/fixtures/failing-unit/src/math.test.ts b/scripts/quality-gate/baseline/fixtures/failing-unit/src/math.test.ts new file mode 100644 index 00000000..b6101f8f --- /dev/null +++ b/scripts/quality-gate/baseline/fixtures/failing-unit/src/math.test.ts @@ -0,0 +1,8 @@ +import { describe, expect, test } from 'bun:test' +import { add } from './math' + +describe('add', () => { + test('adds positive numbers', () => { + expect(add(2, 3)).toBe(5) + }) +}) diff --git a/scripts/quality-gate/baseline/fixtures/failing-unit/src/math.ts b/scripts/quality-gate/baseline/fixtures/failing-unit/src/math.ts new file mode 100644 index 00000000..7759c00b --- /dev/null +++ b/scripts/quality-gate/baseline/fixtures/failing-unit/src/math.ts @@ -0,0 +1,3 @@ +export function add(a: number, b: number) { + return a - b +} diff --git a/scripts/quality-gate/baseline/fixtures/failure-recovery/package.json b/scripts/quality-gate/baseline/fixtures/failure-recovery/package.json new file mode 100644 index 00000000..70977b30 --- /dev/null +++ b/scripts/quality-gate/baseline/fixtures/failure-recovery/package.json @@ -0,0 +1,8 @@ +{ + "name": "quality-gate-failure-recovery-fixture", + "private": true, + "type": "module", + "scripts": { + "test": "bun test" + } +} diff --git a/scripts/quality-gate/baseline/fixtures/failure-recovery/src/slug.test.ts b/scripts/quality-gate/baseline/fixtures/failure-recovery/src/slug.test.ts new file mode 100644 index 00000000..1360e711 --- /dev/null +++ b/scripts/quality-gate/baseline/fixtures/failure-recovery/src/slug.test.ts @@ -0,0 +1,16 @@ +import { describe, expect, test } from 'bun:test' +import { slugify } from './slug' + +describe('slugify', () => { + test('normalizes whitespace and punctuation', () => { + expect(slugify(' Hello, Coding Agent! ')).toBe('hello-coding-agent') + }) + + test('collapses repeated separators', () => { + expect(slugify('Ship---Stable Releases')).toBe('ship-stable-releases') + }) + + test('trims separators from the edges', () => { + expect(slugify('---Quality Gate---')).toBe('quality-gate') + }) +}) diff --git a/scripts/quality-gate/baseline/fixtures/failure-recovery/src/slug.ts b/scripts/quality-gate/baseline/fixtures/failure-recovery/src/slug.ts new file mode 100644 index 00000000..90ec1085 --- /dev/null +++ b/scripts/quality-gate/baseline/fixtures/failure-recovery/src/slug.ts @@ -0,0 +1,3 @@ +export function slugify(input: string) { + return input.toLowerCase().replaceAll(' ', '-') +} diff --git a/scripts/quality-gate/baseline/fixtures/multi-file-api/package.json b/scripts/quality-gate/baseline/fixtures/multi-file-api/package.json new file mode 100644 index 00000000..66d63e7d --- /dev/null +++ b/scripts/quality-gate/baseline/fixtures/multi-file-api/package.json @@ -0,0 +1,8 @@ +{ + "name": "quality-gate-multi-file-api-fixture", + "private": true, + "type": "module", + "scripts": { + "test": "bun test" + } +} diff --git a/scripts/quality-gate/baseline/fixtures/multi-file-api/src/api.ts b/scripts/quality-gate/baseline/fixtures/multi-file-api/src/api.ts new file mode 100644 index 00000000..2db3f4c5 --- /dev/null +++ b/scripts/quality-gate/baseline/fixtures/multi-file-api/src/api.ts @@ -0,0 +1,14 @@ +export type User = { + name: string + email: string +} + +export type UserDisplay = { + displayName: string +} + +export function formatUser(user: User) { + return { + displayName: user.name, + } satisfies UserDisplay +} diff --git a/scripts/quality-gate/baseline/fixtures/multi-file-api/src/app.test.ts b/scripts/quality-gate/baseline/fixtures/multi-file-api/src/app.test.ts new file mode 100644 index 00000000..14c247ca --- /dev/null +++ b/scripts/quality-gate/baseline/fixtures/multi-file-api/src/app.test.ts @@ -0,0 +1,15 @@ +import { describe, expect, test } from 'bun:test' +import { formatUser } from './api' +import { renderUser } from './app' + +describe('renderUser', () => { + test('renders name and email', () => { + expect(renderUser({ name: 'Ada Lovelace', email: 'ada@example.com' })).toBe('User: Ada Lovelace ') + }) + + test('uses the structured display API contract', () => { + expect(formatUser({ name: 'Ada Lovelace', email: 'ada@example.com' })).toEqual({ + label: 'Ada Lovelace ', + }) + }) +}) diff --git a/scripts/quality-gate/baseline/fixtures/multi-file-api/src/app.ts b/scripts/quality-gate/baseline/fixtures/multi-file-api/src/app.ts new file mode 100644 index 00000000..2c6780c0 --- /dev/null +++ b/scripts/quality-gate/baseline/fixtures/multi-file-api/src/app.ts @@ -0,0 +1,5 @@ +import { formatUser, type User } from './api' + +export function renderUser(user: User) { + return `User: ${formatUser(user).displayName}` +} diff --git a/scripts/quality-gate/baseline/fixtures/permission-artifact/package.json b/scripts/quality-gate/baseline/fixtures/permission-artifact/package.json new file mode 100644 index 00000000..40efa949 --- /dev/null +++ b/scripts/quality-gate/baseline/fixtures/permission-artifact/package.json @@ -0,0 +1,8 @@ +{ + "name": "quality-gate-permission-artifact-fixture", + "private": true, + "type": "module", + "scripts": { + "test": "bun test" + } +} diff --git a/scripts/quality-gate/baseline/fixtures/permission-artifact/src/artifact.test.ts b/scripts/quality-gate/baseline/fixtures/permission-artifact/src/artifact.test.ts new file mode 100644 index 00000000..a76f8608 --- /dev/null +++ b/scripts/quality-gate/baseline/fixtures/permission-artifact/src/artifact.test.ts @@ -0,0 +1,9 @@ +import { describe, expect, test } from 'bun:test' +import { existsSync, readFileSync } from 'node:fs' + +describe('release artifact', () => { + test('has the required summary marker', () => { + expect(existsSync('notes/summary.md')).toBe(true) + expect(readFileSync('notes/summary.md', 'utf8')).toContain('baseline-ready') + }) +}) diff --git a/scripts/quality-gate/baseline/fixtures/workspace-search-edit/package.json b/scripts/quality-gate/baseline/fixtures/workspace-search-edit/package.json new file mode 100644 index 00000000..a1c962e8 --- /dev/null +++ b/scripts/quality-gate/baseline/fixtures/workspace-search-edit/package.json @@ -0,0 +1,8 @@ +{ + "name": "quality-gate-workspace-search-edit-fixture", + "private": true, + "type": "module", + "scripts": { + "test": "bun test" + } +} diff --git a/scripts/quality-gate/baseline/fixtures/workspace-search-edit/src/catalog.ts b/scripts/quality-gate/baseline/fixtures/workspace-search-edit/src/catalog.ts new file mode 100644 index 00000000..03c43817 --- /dev/null +++ b/scripts/quality-gate/baseline/fixtures/workspace-search-edit/src/catalog.ts @@ -0,0 +1,9 @@ +export type CatalogItem = { + sku: string + priceCents: number +} + +export const catalog: CatalogItem[] = [ + { sku: 'agent-seat', priceCents: 1200 }, + { sku: 'baseline-pack', priceCents: 800 }, +] diff --git a/scripts/quality-gate/baseline/fixtures/workspace-search-edit/src/checkout.test.ts b/scripts/quality-gate/baseline/fixtures/workspace-search-edit/src/checkout.test.ts new file mode 100644 index 00000000..b329149a --- /dev/null +++ b/scripts/quality-gate/baseline/fixtures/workspace-search-edit/src/checkout.test.ts @@ -0,0 +1,10 @@ +import { describe, expect, test } from 'bun:test' +import { catalog } from './catalog' +import { calculateTotalCents } from './checkout' +import { testUser } from './user' + +describe('calculateTotalCents', () => { + test('applies percentage discounts to the subtotal', () => { + expect(calculateTotalCents(catalog, testUser)).toBe(1500) + }) +}) diff --git a/scripts/quality-gate/baseline/fixtures/workspace-search-edit/src/checkout.ts b/scripts/quality-gate/baseline/fixtures/workspace-search-edit/src/checkout.ts new file mode 100644 index 00000000..57c02a49 --- /dev/null +++ b/scripts/quality-gate/baseline/fixtures/workspace-search-edit/src/checkout.ts @@ -0,0 +1,7 @@ +import type { CatalogItem } from './catalog' +import type { User } from './user' + +export function calculateTotalCents(items: CatalogItem[], user: User) { + const subtotal = items.reduce((sum, item) => sum + item.priceCents, 0) + return subtotal - user.discountPercent +} diff --git a/scripts/quality-gate/baseline/fixtures/workspace-search-edit/src/user.ts b/scripts/quality-gate/baseline/fixtures/workspace-search-edit/src/user.ts new file mode 100644 index 00000000..ca7b062f --- /dev/null +++ b/scripts/quality-gate/baseline/fixtures/workspace-search-edit/src/user.ts @@ -0,0 +1,9 @@ +export type User = { + id: string + discountPercent: number +} + +export const testUser: User = { + id: 'user-1', + discountPercent: 25, +} diff --git a/scripts/quality-gate/desktop-smoke/execute.ts b/scripts/quality-gate/desktop-smoke/execute.ts new file mode 100644 index 00000000..62676868 --- /dev/null +++ b/scripts/quality-gate/desktop-smoke/execute.ts @@ -0,0 +1,376 @@ +import { appendFileSync, cpSync, mkdirSync, readFileSync, rmSync, writeFileSync } from 'node:fs' +import { mkdtemp } from 'node:fs/promises' +import { createServer } from 'node:net' +import { tmpdir } from 'node:os' +import { join } from 'node:path' +import { changedFiles, writeDiffPatch } from '../baseline/execute' +import type { BaselineTarget, LaneResult } from '../types' + +const FIXTURE = 'scripts/quality-gate/desktop-smoke/fixtures/chat-edit' +const PROMPT = [ + 'Run the tests in this project, fix the failing greeting implementation, and rerun the tests.', + 'Only edit src/greeting.ts. Do not edit package.json or tests.', + 'When the tests pass, briefly say done.', +].join(' ') + +async function getPort(): Promise { + return await new Promise((resolve, reject) => { + const server = createServer() + server.on('error', reject) + server.listen(0, '127.0.0.1', () => { + const address = server.address() + if (!address || typeof address === 'string') { + server.close(() => reject(new Error('Failed to allocate a local port'))) + return + } + const port = address.port + server.close(() => resolve(port)) + }) + }) +} + +async function pipeToFile(stream: ReadableStream | null, path: string) { + if (!stream) return + const reader = stream.getReader() + while (true) { + const { done, value } = await reader.read() + if (done) break + appendFileSync(path, Buffer.from(value)) + } +} + +async function waitForHttp(url: string, timeoutMs: number) { + const deadline = Date.now() + timeoutMs + let lastError = '' + while (Date.now() < deadline) { + try { + const response = await fetch(url) + if (response.ok) return + lastError = `HTTP ${response.status}` + } catch (error) { + lastError = error instanceof Error ? error.message : String(error) + } + await Bun.sleep(500) + } + throw new Error(`Timed out waiting for ${url}${lastError ? ` (${lastError})` : ''}`) +} + +async function runLoggedCommand( + command: string[], + options: { + cwd: string + logPath: string + env?: Record + timeoutMs?: number + allowFailure?: boolean + maxLogChars?: number + }, +) { + appendFileSync(options.logPath, `\n$ ${command.join(' ')}\n`) + const proc = Bun.spawn(command, { + cwd: options.cwd, + env: options.env ? { ...process.env, ...options.env } : process.env, + stdout: 'pipe', + stderr: 'pipe', + }) + + const outputPromise = Promise.all([ + new Response(proc.stdout).text(), + new Response(proc.stderr).text(), + proc.exited, + ]) + const timeout = options.timeoutMs + ? Bun.sleep(options.timeoutMs).then(() => { + proc.kill() + throw new Error(`Command timed out after ${options.timeoutMs}ms: ${command.join(' ')}`) + }) + : null + + const [stdout, stderr, exitCode] = timeout + ? await Promise.race([outputPromise, timeout]) + : await outputPromise + const output = `${stdout}${stderr}` + appendFileSync( + options.logPath, + options.maxLogChars && output.length > options.maxLogChars + ? `${output.slice(0, options.maxLogChars)}\n[quality-gate] output truncated at ${options.maxLogChars} chars\n` + : output, + ) + + if (exitCode !== 0 && !options.allowFailure) { + throw new Error(`Command failed (${exitCode}): ${command.join(' ')}`) + } + + return { stdout, stderr, exitCode } +} + +async function fetchJson(url: string, init?: RequestInit): Promise { + const response = await fetch(url, init) + if (!response.ok) { + throw new Error(`${init?.method ?? 'GET'} ${url} failed with HTTP ${response.status}: ${await response.text()}`) + } + return response.json() as Promise +} + +async function setPermissionMode(baseUrl: string, mode: string) { + await fetchJson<{ ok: true; mode: string }>(`${baseUrl}/api/permissions/mode`, { + method: 'PUT', + headers: { 'Content-Type': 'application/json' }, + body: JSON.stringify({ mode }), + }) +} + +async function waitForVerifiedProject( + browserEnv: Record, + browserLogPath: string, + rootDir: string, + originalDir: string, + projectDir: string, + artifactDir: string, + timeoutMs: number, +) { + const deadline = Date.now() + timeoutMs + let lastVerificationError = 'project verification has not run yet' + while (Date.now() < deadline) { + const body = await runLoggedCommand(['agent-browser', 'get', 'text', '#content-area'], { + cwd: rootDir, + env: browserEnv, + logPath: browserLogPath, + timeoutMs: 15_000, + allowFailure: true, + maxLogChars: 4_000, + }) + const browserText = `${body.stdout}\n${body.stderr}` + + if (browserText.includes('CLI_START_FAILED') || browserText.includes('CLI_RESTART_FAILED')) { + throw new Error('Desktop session reported a CLI startup failure') + } + if ( + browserText.includes('API Error: 429') || + browserText.includes('AccountQuotaExceeded') || + browserText.includes('TooManyRequests') + ) { + throw new Error('Desktop session reported provider quota/rate-limit failure') + } + if (browserText.includes('处理过程中发生错误') || browserText.includes('API Error:')) { + throw new Error('Desktop session reported an API error') + } + + try { + await verifyProject(originalDir, projectDir, artifactDir) + return + } catch (error) { + lastVerificationError = error instanceof Error ? error.message : String(error) + } + await Bun.sleep(5_000) + } + + throw new Error(`Timed out waiting for desktop project verification: ${lastVerificationError}`) +} + +async function verifyProject(originalDir: string, projectDir: string, artifactDir: string) { + await writeDiffPatch(originalDir, projectDir, join(artifactDir, 'diff.patch')) + const changed = changedFiles(originalDir, projectDir) + const unexpected = changed.filter((file) => file !== 'src/greeting.ts') + if (unexpected.length > 0) { + throw new Error(`desktop smoke changed unexpected files: ${unexpected.join(', ')}`) + } + if (!changed.includes('src/greeting.ts')) { + throw new Error('desktop smoke did not change src/greeting.ts') + } + + const implementation = readFileSync(join(projectDir, 'src/greeting.ts'), 'utf8') + if (!implementation.includes('from desktop smoke!')) { + throw new Error('desktop smoke implementation is missing the expected marker text') + } + + const proc = Bun.spawn(['bun', 'test'], { + cwd: projectDir, + stdout: 'pipe', + stderr: 'pipe', + }) + const stdout = await new Response(proc.stdout).text() + const stderr = await new Response(proc.stderr).text() + const exitCode = await proc.exited + writeFileSync(join(artifactDir, 'verification.log'), `${stdout}${stderr}`) + if (exitCode !== 0) { + throw new Error(`desktop smoke verification failed with exit code ${exitCode}`) + } +} + +export async function executeDesktopSmoke( + rootDir: string, + artifactDir: string, + resultId: string, + resultTitle: string, + target: BaselineTarget | undefined, +): Promise { + const started = Date.now() + mkdirSync(artifactDir, { recursive: true }) + + const serverLogPath = join(artifactDir, 'server.log') + const viteLogPath = join(artifactDir, 'vite.log') + const browserLogPath = join(artifactDir, 'browser.log') + const workRoot = await mkdtemp(join(tmpdir(), 'quality-gate-desktop-smoke-')) + const originalDir = join(workRoot, 'original') + const projectDir = join(workRoot, 'project') + const browserProfileDir = join(workRoot, 'browser-profile') + cpSync(join(rootDir, FIXTURE), originalDir, { recursive: true }) + cpSync(join(rootDir, FIXTURE), projectDir, { recursive: true }) + + const serverPort = await getPort() + const vitePort = await getPort() + const baseUrl = `http://127.0.0.1:${serverPort}` + const appUrl = `http://127.0.0.1:${vitePort}` + const sessionName = `quality-gate-${Date.now()}-${Math.random().toString(36).slice(2, 8)}` + const browserEnv = { + AGENT_BROWSER_SESSION: sessionName, + AGENT_BROWSER_PROFILE: browserProfileDir, + } + + const server = Bun.spawn(['bun', 'run', 'src/server/index.ts', '--host', '127.0.0.1', '--port', String(serverPort)], { + cwd: rootDir, + stdout: 'pipe', + stderr: 'pipe', + }) + void pipeToFile(server.stdout, serverLogPath) + void pipeToFile(server.stderr, serverLogPath) + + const vite = Bun.spawn(['bun', 'run', 'dev', '--', '--host', '127.0.0.1', '--port', String(vitePort), '--strictPort'], { + cwd: join(rootDir, 'desktop'), + env: { + ...process.env, + VITE_DESKTOP_SERVER_URL: baseUrl, + }, + stdout: 'pipe', + stderr: 'pipe', + }) + void pipeToFile(vite.stdout, viteLogPath) + void pipeToFile(vite.stderr, viteLogPath) + + let previousPermissionMode: string | null = null + try { + await waitForHttp(`${baseUrl}/health`, 20_000) + await waitForHttp(appUrl, 30_000) + + const permission = await fetchJson<{ mode: string }>(`${baseUrl}/api/permissions/mode`) + previousPermissionMode = permission.mode + await setPermissionMode(baseUrl, 'bypassPermissions') + + const session = await fetchJson<{ sessionId: string }>(`${baseUrl}/api/sessions`, { + method: 'POST', + headers: { 'Content-Type': 'application/json' }, + body: JSON.stringify({ workDir: projectDir }), + }) + const runtimeSelection = { + providerId: target?.providerId ?? null, + modelId: target?.modelId ?? 'current', + } + + await runLoggedCommand(['agent-browser', 'open', appUrl], { + cwd: rootDir, + env: browserEnv, + logPath: browserLogPath, + timeoutMs: 30_000, + }) + await runLoggedCommand([ + 'agent-browser', + 'eval', + [ + `localStorage.setItem('cc-haha-open-tabs', ${JSON.stringify(JSON.stringify({ + openTabs: [{ sessionId: session.sessionId, title: 'Desktop Smoke', type: 'session' }], + activeTabId: session.sessionId, + }))})`, + `localStorage.setItem('cc-haha-session-runtime', ${JSON.stringify(JSON.stringify({ + [session.sessionId]: runtimeSelection, + }))})`, + ].join(';'), + ], { + cwd: rootDir, + env: browserEnv, + logPath: browserLogPath, + timeoutMs: 15_000, + }) + await runLoggedCommand(['agent-browser', 'reload'], { + cwd: rootDir, + env: browserEnv, + logPath: browserLogPath, + timeoutMs: 30_000, + }) + await runLoggedCommand(['agent-browser', 'wait', 'textarea'], { + cwd: rootDir, + env: browserEnv, + logPath: browserLogPath, + timeoutMs: 30_000, + }) + await runLoggedCommand(['agent-browser', 'screenshot', join(artifactDir, 'initial.png')], { + cwd: rootDir, + env: browserEnv, + logPath: browserLogPath, + timeoutMs: 20_000, + allowFailure: true, + }) + await runLoggedCommand(['agent-browser', 'fill', 'textarea', PROMPT], { + cwd: rootDir, + env: browserEnv, + logPath: browserLogPath, + timeoutMs: 20_000, + }) + await runLoggedCommand(['agent-browser', 'press', 'Enter'], { + cwd: rootDir, + env: browserEnv, + logPath: browserLogPath, + timeoutMs: 15_000, + }) + + await waitForVerifiedProject( + browserEnv, + browserLogPath, + rootDir, + originalDir, + projectDir, + artifactDir, + 360_000, + ) + await runLoggedCommand(['agent-browser', 'screenshot', join(artifactDir, 'final.png')], { + cwd: rootDir, + env: browserEnv, + logPath: browserLogPath, + timeoutMs: 20_000, + allowFailure: true, + }) + + return { + id: resultId, + title: resultTitle, + status: 'passed', + durationMs: Date.now() - started, + artifactDir, + } + } catch (error) { + return { + id: resultId, + title: resultTitle, + status: 'failed', + durationMs: Date.now() - started, + error: error instanceof Error ? error.message : String(error), + artifactDir, + } + } finally { + if (previousPermissionMode) { + await setPermissionMode(baseUrl, previousPermissionMode).catch((error) => { + appendFileSync(serverLogPath, `\n[quality-gate] Failed to restore permission mode: ${error instanceof Error ? error.message : String(error)}\n`) + }) + } + await runLoggedCommand(['agent-browser', 'close'], { + cwd: rootDir, + env: browserEnv, + logPath: browserLogPath, + timeoutMs: 10_000, + allowFailure: true, + }).catch(() => {}) + server.kill() + vite.kill() + rmSync(workRoot, { recursive: true, force: true }) + } +} diff --git a/scripts/quality-gate/desktop-smoke/fixtures/chat-edit/package.json b/scripts/quality-gate/desktop-smoke/fixtures/chat-edit/package.json new file mode 100644 index 00000000..1865effa --- /dev/null +++ b/scripts/quality-gate/desktop-smoke/fixtures/chat-edit/package.json @@ -0,0 +1,8 @@ +{ + "name": "desktop-smoke-chat-edit", + "private": true, + "type": "module", + "scripts": { + "test": "bun test" + } +} diff --git a/scripts/quality-gate/desktop-smoke/fixtures/chat-edit/src/greeting.test.ts b/scripts/quality-gate/desktop-smoke/fixtures/chat-edit/src/greeting.test.ts new file mode 100644 index 00000000..b22fc7bf --- /dev/null +++ b/scripts/quality-gate/desktop-smoke/fixtures/chat-edit/src/greeting.test.ts @@ -0,0 +1,8 @@ +import { describe, expect, test } from 'bun:test' +import { greetingFor } from './greeting' + +describe('greetingFor', () => { + test('includes the desktop smoke marker', () => { + expect(greetingFor('Ada Lovelace')).toBe('Hello, Ada Lovelace from desktop smoke!') + }) +}) diff --git a/scripts/quality-gate/desktop-smoke/fixtures/chat-edit/src/greeting.ts b/scripts/quality-gate/desktop-smoke/fixtures/chat-edit/src/greeting.ts new file mode 100644 index 00000000..de325881 --- /dev/null +++ b/scripts/quality-gate/desktop-smoke/fixtures/chat-edit/src/greeting.ts @@ -0,0 +1,3 @@ +export function greetingFor(name: string) { + return `Hello, ${name}` +} diff --git a/scripts/quality-gate/index.ts b/scripts/quality-gate/index.ts new file mode 100644 index 00000000..68650504 --- /dev/null +++ b/scripts/quality-gate/index.ts @@ -0,0 +1,79 @@ +#!/usr/bin/env bun + +import { runQualityGate } from './runner' +import { formatProviderTargets, loadProviderIndex, parseBaselineTargetValues } from './providerTargets' +import type { BaselineTarget, QualityGateMode } from './types' + +function parseArgs(argv: string[]) { + const args = new Map>() + + for (let index = 0; index < argv.length; index += 1) { + const arg = argv[index] + if (!arg.startsWith('--')) { + continue + } + + const next = argv[index + 1] + if (next && !next.startsWith('--')) { + args.set(arg, [...(args.get(arg) ?? []), next]) + index += 1 + } else { + args.set(arg, [...(args.get(arg) ?? []), true]) + } + } + + return args +} + +function firstArg(args: Map>, name: string) { + return args.get(name)?.[0] +} + +function hasFlag(args: Map>, name: string) { + return args.has(name) +} + +function readMode(value: string | boolean | undefined): QualityGateMode { + if (value === 'pr' || value === 'baseline' || value === 'release') { + return value + } + throw new Error('Usage: bun run quality:gate --mode [--dry-run] [--allow-live] [--provider-model provider:model[:label]]') +} + +function readBaselineTargets(args: Map>): BaselineTarget[] { + const values = (args.get('--provider-model') ?? []) + .filter((value): value is string => typeof value === 'string') + + return parseBaselineTargetValues(values) +} + +const args = parseArgs(process.argv.slice(2)) + +if (hasFlag(args, '--list-providers')) { + console.log(formatProviderTargets(loadProviderIndex())) + process.exit(0) +} + +const mode = readMode(firstArg(args, '--mode')) +const dryRun = hasFlag(args, '--dry-run') +const allowLive = hasFlag(args, '--allow-live') +const artifactsDir = typeof firstArg(args, '--artifacts-dir') === 'string' + ? String(firstArg(args, '--artifacts-dir')) + : undefined +const baselineTargets = readBaselineTargets(args) + +const { report, outputDir } = await runQualityGate({ + mode, + dryRun, + allowLive, + baselineTargets, + rootDir: process.cwd(), + artifactsDir, +}) + +console.log(`Quality gate report: ${outputDir}/report.md`) +console.log(`Summary: passed=${report.summary.passed} failed=${report.summary.failed} skipped=${report.summary.skipped}`) + +if (report.summary.failed > 0) { + process.exit(1) +} diff --git a/scripts/quality-gate/modes.ts b/scripts/quality-gate/modes.ts new file mode 100644 index 00000000..998c5ba2 --- /dev/null +++ b/scripts/quality-gate/modes.ts @@ -0,0 +1,74 @@ +import { baselineCases } from './baseline/cases' +import type { BaselineTarget, LaneDefinition, QualityGateMode } from './types' + +export function lanesForMode(mode: QualityGateMode, baselineTargets: BaselineTarget[] = []): LaneDefinition[] { + const lanes: LaneDefinition[] = [ + { + id: 'impact-report', + title: 'Impact report', + description: 'Summarize changed areas, required local checks, and risk notes.', + kind: 'command', + command: ['bun', 'run', 'check:impact'], + requiredForModes: ['pr', 'baseline', 'release'], + }, + { + id: 'pr-checks', + title: 'Path-aware PR checks', + description: 'Run the existing local PR gate with stable path-aware checks.', + kind: 'command', + command: ['bun', 'run', 'check:pr'], + requiredForModes: ['pr', 'release'], + }, + { + id: 'baseline-catalog', + title: 'Baseline case catalog validation', + description: 'Validate real Coding Agent baseline case definitions and fixture metadata.', + kind: 'command', + command: ['bun', 'test', 'scripts/quality-gate/baseline/cases.test.ts'], + requiredForModes: ['baseline', 'release'], + }, + { + id: 'native-checks', + title: 'Native desktop checks', + description: 'Build sidecars and run the Tauri native compile check.', + kind: 'command', + command: ['bun', 'run', 'check:native'], + requiredForModes: ['release'], + }, + ] + + const targets = baselineTargets.length > 0 + ? baselineTargets + : [{ providerId: null, modelId: 'current', label: 'current-runtime' }] + + for (const testCase of baselineCases) { + for (const target of targets) { + const targetSlug = target.label.replace(/[^a-zA-Z0-9._-]+/g, '-') + lanes.push({ + id: `baseline:${testCase.id}:${targetSlug}`, + title: `${testCase.title} (${target.label})`, + description: testCase.description, + kind: 'baseline-case', + baselineCaseId: testCase.id, + baselineTarget: target, + requiredForModes: ['baseline', 'release'], + live: true, + }) + } + } + + for (const target of targets) { + const targetSlug = target.label.replace(/[^a-zA-Z0-9._-]+/g, '-') + lanes.push({ + id: `desktop-smoke:agent-browser-chat:${targetSlug}`, + title: `Desktop agent-browser chat smoke (${target.label})`, + description: 'Open the desktop web app with agent-browser, send a real chat task, and verify the model edits a fixture project through the UI.', + kind: 'desktop-smoke', + baselineTarget: target, + requiredForModes: ['baseline', 'release'], + live: true, + }) + } + + return lanes.filter((lane) => lane.requiredForModes.includes(mode)) +} diff --git a/scripts/quality-gate/providerTargets.test.ts b/scripts/quality-gate/providerTargets.test.ts new file mode 100644 index 00000000..e896ba92 --- /dev/null +++ b/scripts/quality-gate/providerTargets.test.ts @@ -0,0 +1,171 @@ +import { describe, expect, test } from 'bun:test' +import { mkdirSync, rmSync, writeFileSync } from 'node:fs' +import { tmpdir } from 'node:os' +import { join } from 'node:path' +import { + formatProviderTargets, + loadProviderIndex, + parseBaselineTargetValues, +} from './providerTargets' + +function withProviderIndex(index: unknown, fn: (configDir: string) => void) { + const configDir = join(tmpdir(), `quality-provider-targets-${crypto.randomUUID()}`) + mkdirSync(join(configDir, 'cc-haha'), { recursive: true }) + writeFileSync(join(configDir, 'cc-haha', 'providers.json'), JSON.stringify(index, null, 2)) + + try { + fn(configDir) + } finally { + rmSync(configDir, { recursive: true, force: true }) + } +} + +describe('quality gate provider targets', () => { + test('lists copyable provider-model selectors without exposing keys', () => { + withProviderIndex({ + activeId: 'provider-kimi', + providers: [ + { + id: 'provider-kimi', + presetId: 'volcengine', + name: 'Volcengine Codingplan', + apiKey: 'secret-key', + baseUrl: 'https://example.invalid', + apiFormat: 'openai_chat', + models: { + main: 'kimi-k2.6', + haiku: 'kimi-k2.6', + sonnet: 'kimi-k2.6-thinking', + opus: 'kimi-k2.6-thinking', + }, + }, + ], + }, (configDir) => { + const output = formatProviderTargets(loadProviderIndex(configDir), configDir) + + expect(output).toContain('Volcengine Codingplan') + expect(output).toContain('--provider-model volcengine-codingplan:main:volcengine-codingplan-main') + expect(output).toContain('--provider-model volcengine-codingplan:sonnet:volcengine-codingplan-sonnet') + expect(output).not.toContain('secret-key') + }) + }) + + test('parses provider-model by provider slug instead of requiring UUID', () => { + withProviderIndex({ + activeId: null, + providers: [ + { + id: 'provider-minimax', + presetId: 'minimax', + name: 'MiniMax', + apiKey: 'secret-key', + baseUrl: 'https://example.invalid', + apiFormat: 'openai_chat', + models: { + main: 'MiniMax-M2.7-highspeed', + haiku: 'MiniMax-M2.7-highspeed', + sonnet: 'MiniMax-M2.7-highspeed', + opus: 'MiniMax-M2.7-highspeed', + }, + }, + ], + }, (configDir) => { + const targets = parseBaselineTargetValues( + ['minimax:MiniMax-M2.7-highspeed'], + loadProviderIndex(configDir), + ) + + expect(targets).toEqual([ + { + providerId: 'provider-minimax', + modelId: 'MiniMax-M2.7-highspeed', + label: 'minimax-MiniMax-M2.7-highspeed', + }, + ]) + }) + }) + + test('parses provider-model by role so model IDs may contain colons', () => { + withProviderIndex({ + activeId: null, + providers: [ + { + id: 'provider-anthropic-proxy', + presetId: 'proxy', + name: 'Anthropic Proxy', + apiKey: 'secret-key', + baseUrl: 'https://example.invalid', + apiFormat: 'anthropic', + models: { + main: 'anthropic/claude-sonnet-4.6', + haiku: 'anthropic/claude-haiku-4.5:thinking', + sonnet: 'anthropic/claude-sonnet-4.6', + opus: 'anthropic/claude-opus-4.7', + }, + }, + ], + }, (configDir) => { + const targets = parseBaselineTargetValues( + ['anthropic-proxy:haiku:proxy-haiku-thinking'], + loadProviderIndex(configDir), + ) + + expect(targets).toEqual([ + { + providerId: 'provider-anthropic-proxy', + modelId: 'anthropic/claude-haiku-4.5:thinking', + label: 'proxy-haiku-thinking', + }, + ]) + }) + }) + + test('does not treat shared preset ids as provider selectors', () => { + withProviderIndex({ + activeId: null, + providers: [ + { + id: 'provider-one', + presetId: 'custom', + name: 'Provider One', + apiKey: 'secret-key', + baseUrl: 'https://example.invalid', + apiFormat: 'anthropic', + models: { main: 'model-one', haiku: 'model-one', sonnet: 'model-one', opus: 'model-one' }, + }, + { + id: 'provider-two', + presetId: 'custom', + name: 'Provider Two', + apiKey: 'secret-key', + baseUrl: 'https://example.invalid', + apiFormat: 'anthropic', + models: { main: 'model-two', haiku: 'model-two', sonnet: 'model-two', opus: 'model-two' }, + }, + ], + }, (configDir) => { + expect(() => parseBaselineTargetValues(['custom:main'], loadProviderIndex(configDir))).toThrow('Unknown provider') + expect(parseBaselineTargetValues(['provider-one:main'], loadProviderIndex(configDir))[0]).toEqual({ + providerId: 'provider-one', + modelId: 'model-one', + label: 'provider-one-model-one', + }) + }) + }) + + test('keeps current runtime and explicit UUID selectors supported', () => { + const targets = parseBaselineTargetValues([ + 'current:current', + '11111111-1111-4111-8111-111111111111:model-a:model-a-live', + ]) + + expect(targets).toEqual([ + { providerId: null, modelId: 'current', label: 'current-runtime' }, + { + providerId: '11111111-1111-4111-8111-111111111111', + modelId: 'model-a', + label: 'model-a-live', + }, + ]) + }) +}) diff --git a/scripts/quality-gate/providerTargets.ts b/scripts/quality-gate/providerTargets.ts new file mode 100644 index 00000000..ede2fc70 --- /dev/null +++ b/scripts/quality-gate/providerTargets.ts @@ -0,0 +1,257 @@ +import { existsSync, readFileSync } from 'node:fs' +import { homedir } from 'node:os' +import { join } from 'node:path' +import type { BaselineTarget } from './types' + +type ProviderModels = { + main?: string + haiku?: string + sonnet?: string + opus?: string +} + +type SavedProviderIndexEntry = { + id: string + name: string + presetId?: string + models?: ProviderModels +} + +export type ProviderIndex = { + activeId: string | null + providers: SavedProviderIndexEntry[] +} + +const DEFAULT_INDEX: ProviderIndex = { activeId: null, providers: [] } +const UUID_PATTERN = /^[0-9a-f]{8}-[0-9a-f]{4}-[1-5][0-9a-f]{3}-[89ab][0-9a-f]{3}-[0-9a-f]{12}$/i + +export function getProviderIndexPath(configDir = process.env.CLAUDE_CONFIG_DIR || join(homedir(), '.claude')) { + return join(configDir, 'cc-haha', 'providers.json') +} + +export function slugifyProviderName(value: string) { + return value + .trim() + .toLowerCase() + .replace(/[^a-z0-9]+/g, '-') + .replace(/^-+|-+$/g, '') +} + +function slugifyLabel(value: string) { + return value + .trim() + .replace(/[^a-zA-Z0-9._-]+/g, '-') + .replace(/^-+|-+$/g, '') +} + +function providerAliases(provider: SavedProviderIndexEntry) { + return new Set([ + provider.id, + provider.name, + provider.name.toLowerCase(), + slugifyProviderName(provider.name), + ].filter(Boolean)) +} + +function modelEntries(provider: SavedProviderIndexEntry) { + const roles = [ + ['main', provider.models?.main], + ['haiku', provider.models?.haiku], + ['sonnet', provider.models?.sonnet], + ['opus', provider.models?.opus], + ] as const + const seen = new Set() + + return roles + .filter((entry): entry is [typeof entry[0], string] => typeof entry[1] === 'string' && entry[1].trim().length > 0) + .filter(([, modelId]) => { + if (seen.has(modelId)) return false + seen.add(modelId) + return true + }) +} + +function splitProviderTarget(value: string) { + const separator = value.indexOf(':') + if (separator === -1) { + return null + } + + return { + providerRef: value.slice(0, separator), + modelAndLabel: value.slice(separator + 1), + } +} + +function splitModelAndLabel(value: string) { + const separator = value.lastIndexOf(':') + if (separator === -1) { + return { modelId: value, label: undefined } + } + + return { + modelId: value.slice(0, separator), + label: value.slice(separator + 1) || undefined, + } +} + +function resolveProviderModel(provider: SavedProviderIndexEntry, modelAndLabel: string) { + const roleEntries = [ + ['main', provider.models?.main], + ['haiku', provider.models?.haiku], + ['sonnet', provider.models?.sonnet], + ['opus', provider.models?.opus], + ] as const + + for (const [role, modelId] of roleEntries) { + if (!modelId) continue + if (modelAndLabel === role) { + return { modelId, label: undefined } + } + if (modelAndLabel.startsWith(`${role}:`)) { + return { modelId, label: modelAndLabel.slice(role.length + 1) || undefined } + } + } + + const knownModels = [...new Set(roleEntries.map(([, modelId]) => modelId).filter((modelId): modelId is string => Boolean(modelId)))] + .sort((left, right) => right.length - left.length) + for (const modelId of knownModels) { + if (modelAndLabel === modelId) { + return { modelId, label: undefined } + } + if (modelAndLabel.startsWith(`${modelId}:`)) { + return { modelId, label: modelAndLabel.slice(modelId.length + 1) || undefined } + } + } + + return splitModelAndLabel(modelAndLabel) +} + +export function loadProviderIndex(configDir?: string): ProviderIndex { + const filePath = getProviderIndexPath(configDir) + if (!existsSync(filePath)) { + return { ...DEFAULT_INDEX, providers: [] } + } + + const parsed = JSON.parse(readFileSync(filePath, 'utf8')) as Partial + return { + activeId: typeof parsed.activeId === 'string' ? parsed.activeId : null, + providers: Array.isArray(parsed.providers) + ? parsed.providers.filter((provider): provider is SavedProviderIndexEntry => ( + provider && + typeof provider.id === 'string' && + typeof provider.name === 'string' + )) + : [], + } +} + +function resolveProviderReference(reference: string, index: ProviderIndex) { + if (reference === 'current' || reference === 'official') { + return null + } + + const normalized = reference.toLowerCase() + const slug = slugifyProviderName(reference) + const matches = index.providers.filter((provider) => { + const aliases = providerAliases(provider) + return aliases.has(reference) || aliases.has(normalized) || aliases.has(slug) + }) + + if (matches.length === 1) { + return matches[0] + } + + if (matches.length > 1) { + throw new Error(`Ambiguous provider reference "${reference}". Use one of these IDs: ${matches.map((provider) => provider.id).join(', ')}`) + } + + if (UUID_PATTERN.test(reference)) { + return { id: reference, name: reference } + } + + throw new Error(`Unknown provider "${reference}". Run "bun run quality:providers" to list available --provider-model values.`) +} + +export function parseBaselineTargetValues(values: string[], index: ProviderIndex = loadProviderIndex()): BaselineTarget[] { + return values + .flatMap((value) => value.split(',')) + .map((value) => value.trim()) + .filter(Boolean) + .map((value) => { + const target = splitProviderTarget(value) + if (!target?.providerRef || !target.modelAndLabel) { + throw new Error(`Invalid --provider-model value "${value}". Expected provider:model[:label]. Run "bun run quality:providers" for copyable values.`) + } + + const { providerRef, modelAndLabel } = target + const provider = resolveProviderReference(providerRef, index) + if (!provider) { + const { modelId, label } = splitModelAndLabel(modelAndLabel) + return { + providerId: null, + modelId, + label: label || (providerRef === 'current' && modelId === 'current' ? 'current-runtime' : `${providerRef}-${modelId}`), + } + } + + const { modelId, label } = resolveProviderModel(provider, modelAndLabel) + const providerSlug = slugifyProviderName(provider.name) || provider.id.slice(0, 8) + return { + providerId: provider.id, + modelId, + label: label || slugifyLabel(`${providerSlug}-${modelId}`), + } + }) +} + +function providerSelector(provider: SavedProviderIndexEntry, index: ProviderIndex) { + const slug = slugifyProviderName(provider.name) + if (!slug) { + return provider.id + } + + const matchingSlugCount = index.providers.filter((candidate) => slugifyProviderName(candidate.name) === slug).length + return matchingSlugCount === 1 ? slug : provider.id +} + +export function formatProviderTargets(index: ProviderIndex = loadProviderIndex(), configDir = process.env.CLAUDE_CONFIG_DIR || join(homedir(), '.claude')) { + const lines = [ + 'Quality gate provider targets', + `Config: ${getProviderIndexPath(configDir)}`, + '', + 'Current/default runtime:', + ' --provider-model current:current:current-runtime', + '', + ] + + if (index.providers.length === 0) { + lines.push('Saved providers: none') + lines.push('') + lines.push('Add a provider in Desktop Settings > Providers, then run this command again.') + return lines.join('\n') + } + + lines.push('Saved providers:') + for (const provider of index.providers) { + const providerSlug = providerSelector(provider, index) + const active = index.activeId === provider.id ? ' (active)' : '' + lines.push(` ${provider.name}${active}`) + lines.push(` id: ${provider.id}`) + lines.push(` selector: ${providerSlug}`) + + const models = modelEntries(provider) + if (models.length === 0) { + lines.push(' models: none configured') + continue + } + + for (const [role, modelId] of models) { + const label = slugifyLabel(`${providerSlug}-${role}`) + lines.push(` ${role}: ${modelId}`) + lines.push(` --provider-model ${providerSlug}:${role}:${label}`) + } + } + + return lines.join('\n') +} diff --git a/scripts/quality-gate/providers.ts b/scripts/quality-gate/providers.ts new file mode 100644 index 00000000..bf2711c4 --- /dev/null +++ b/scripts/quality-gate/providers.ts @@ -0,0 +1,5 @@ +#!/usr/bin/env bun + +import { formatProviderTargets, loadProviderIndex } from './providerTargets' + +console.log(formatProviderTargets(loadProviderIndex())) diff --git a/scripts/quality-gate/quarantine.json b/scripts/quality-gate/quarantine.json new file mode 100644 index 00000000..efcbc693 --- /dev/null +++ b/scripts/quality-gate/quarantine.json @@ -0,0 +1,39 @@ +{ + "quarantined": [ + { + "id": "server:cron-scheduler", + "path": "src/server/__tests__/cron-scheduler.test.ts", + "reason": "Known unstable scheduler timing assertions; keep out of default server gate until fixed.", + "owner": "maintainers", + "reviewAfter": "2026-06-01" + }, + { + "id": "server:providers-real", + "path": "src/server/__tests__/providers-real.test.ts", + "reason": "Live provider coverage belongs in maintainer baseline or release mode, not the default PR gate.", + "owner": "maintainers", + "reviewAfter": "2026-06-01" + }, + { + "id": "server:tasks", + "path": "src/server/__tests__/tasks.test.ts", + "reason": "Known default-gate instability; keep quarantined until task timing is hardened.", + "owner": "maintainers", + "reviewAfter": "2026-06-01" + }, + { + "id": "server:e2e:business-flow", + "path": "src/server/__tests__/e2e/business-flow.test.ts", + "reason": "Broad e2e flow is not deterministic enough for default PR checks.", + "owner": "maintainers", + "reviewAfter": "2026-06-01" + }, + { + "id": "server:e2e:full-flow", + "path": "src/server/__tests__/e2e/full-flow.test.ts", + "reason": "Broad e2e flow is not deterministic enough for default PR checks.", + "owner": "maintainers", + "reviewAfter": "2026-06-01" + } + ] +} diff --git a/scripts/quality-gate/quarantine.test.ts b/scripts/quality-gate/quarantine.test.ts new file mode 100644 index 00000000..f0b60d38 --- /dev/null +++ b/scripts/quality-gate/quarantine.test.ts @@ -0,0 +1,35 @@ +import { describe, expect, test } from 'bun:test' +import { loadQuarantineManifest, quarantinedPathSet, validateQuarantineManifest } from './quarantine' + +describe('quarantine manifest', () => { + test('loads the default manifest', () => { + const manifest = loadQuarantineManifest() + expect(manifest.quarantined.length).toBeGreaterThan(0) + }) + + test('exposes quarantined paths as a set', () => { + const paths = quarantinedPathSet() + expect(paths.has('src/server/__tests__/providers-real.test.ts')).toBe(true) + }) + + test('rejects duplicate ids', () => { + expect(() => validateQuarantineManifest({ + quarantined: [ + { + id: 'duplicate', + path: 'a.test.ts', + reason: 'test', + owner: 'maintainers', + reviewAfter: '2026-06-01', + }, + { + id: 'duplicate', + path: 'b.test.ts', + reason: 'test', + owner: 'maintainers', + reviewAfter: '2026-06-01', + }, + ], + })).toThrow('duplicate quarantine id') + }) +}) diff --git a/scripts/quality-gate/quarantine.ts b/scripts/quality-gate/quarantine.ts new file mode 100644 index 00000000..ec771442 --- /dev/null +++ b/scripts/quality-gate/quarantine.ts @@ -0,0 +1,50 @@ +import { readFileSync } from 'node:fs' +import { dirname, join } from 'node:path' +import { fileURLToPath } from 'node:url' + +export type QuarantineEntry = { + id: string + path: string + reason: string + owner: string + reviewAfter: string +} + +export type QuarantineManifest = { + quarantined: QuarantineEntry[] +} + +const defaultManifestPath = join(dirname(fileURLToPath(import.meta.url)), 'quarantine.json') + +export function loadQuarantineManifest(path = defaultManifestPath): QuarantineManifest { + const manifest = JSON.parse(readFileSync(path, 'utf8')) as QuarantineManifest + validateQuarantineManifest(manifest) + return manifest +} + +export function validateQuarantineManifest(manifest: QuarantineManifest) { + if (!Array.isArray(manifest.quarantined)) { + throw new Error('quarantine manifest must contain a quarantined array') + } + + const ids = new Set() + const paths = new Set() + + for (const entry of manifest.quarantined) { + if (!entry.id || !entry.path || !entry.reason || !entry.owner || !entry.reviewAfter) { + throw new Error(`invalid quarantine entry: ${JSON.stringify(entry)}`) + } + if (ids.has(entry.id)) { + throw new Error(`duplicate quarantine id: ${entry.id}`) + } + if (paths.has(entry.path)) { + throw new Error(`duplicate quarantine path: ${entry.path}`) + } + ids.add(entry.id) + paths.add(entry.path) + } +} + +export function quarantinedPathSet(manifest = loadQuarantineManifest()) { + return new Set(manifest.quarantined.map((entry) => entry.path)) +} diff --git a/scripts/quality-gate/reporter.ts b/scripts/quality-gate/reporter.ts new file mode 100644 index 00000000..dde39b57 --- /dev/null +++ b/scripts/quality-gate/reporter.ts @@ -0,0 +1,59 @@ +import { mkdirSync, writeFileSync } from 'node:fs' +import { join } from 'node:path' +import type { QualityGateReport } from './types' + +export function writeReport(report: QualityGateReport, outputDir: string) { + mkdirSync(outputDir, { recursive: true }) + writeFileSync(join(outputDir, 'report.json'), JSON.stringify(report, null, 2) + '\n') + writeFileSync(join(outputDir, 'report.md'), renderMarkdownReport(report)) +} + +export function renderMarkdownReport(report: QualityGateReport) { + const lines = [ + `# Quality Gate Report`, + '', + `- Run: ${report.runId}`, + `- Mode: ${report.mode}`, + `- Dry run: ${report.dryRun ? 'yes' : 'no'}`, + `- Live checks allowed: ${report.allowLive ? 'yes' : 'no'}`, + `- Git SHA: ${report.git.sha ?? 'unknown'}`, + `- Dirty worktree: ${report.git.dirty ? 'yes' : 'no'}`, + `- Started: ${report.startedAt}`, + `- Finished: ${report.finishedAt}`, + '', + `## Summary`, + '', + `- Passed: ${report.summary.passed}`, + `- Failed: ${report.summary.failed}`, + `- Skipped: ${report.summary.skipped}`, + '', + `## Lanes`, + '', + ] + + for (const result of report.results) { + lines.push(`### ${result.title}`) + lines.push('') + lines.push(`- ID: ${result.id}`) + lines.push(`- Status: ${result.status}`) + lines.push(`- Duration: ${result.durationMs}ms`) + if (result.command) { + lines.push(`- Command: \`${result.command.join(' ')}\``) + } + if (result.exitCode !== undefined) { + lines.push(`- Exit code: ${result.exitCode}`) + } + if (result.skipReason) { + lines.push(`- Skip reason: ${result.skipReason}`) + } + if (result.error) { + lines.push(`- Error: ${result.error}`) + } + if (result.artifactDir) { + lines.push(`- Artifacts: ${result.artifactDir}`) + } + lines.push('') + } + + return lines.join('\n') +} diff --git a/scripts/quality-gate/runner.test.ts b/scripts/quality-gate/runner.test.ts new file mode 100644 index 00000000..b120f810 --- /dev/null +++ b/scripts/quality-gate/runner.test.ts @@ -0,0 +1,162 @@ +import { describe, expect, test } from 'bun:test' +import { mkdtempSync, readFileSync, rmSync } from 'node:fs' +import { tmpdir } from 'node:os' +import { join } from 'node:path' +import { lanesForMode } from './modes' +import { renderMarkdownReport } from './reporter' +import { runQualityGate, runQualityGateLanes } from './runner' +import type { LaneDefinition, QualityGateReport } from './types' + +describe('quality gate modes', () => { + test('pr mode includes existing path-aware PR checks', () => { + const lanes = lanesForMode('pr').map((lane) => lane.id) + expect(lanes).toContain('impact-report') + expect(lanes).toContain('pr-checks') + expect(lanes.some((lane) => lane.startsWith('baseline:'))).toBe(false) + }) + + test('baseline mode includes live baseline cases but not native checks', () => { + const lanes = lanesForMode('baseline').map((lane) => lane.id) + expect(lanes).toContain('baseline-catalog') + expect(lanes).toContain('baseline:failing-unit:current-runtime') + expect(lanes).toContain('baseline:multi-file-api:current-runtime') + expect(lanes).toContain('desktop-smoke:agent-browser-chat:current-runtime') + expect(lanes).not.toContain('native-checks') + }) + + test('release mode composes PR, baseline, and native lanes', () => { + const lanes = lanesForMode('release').map((lane) => lane.id) + expect(lanes).toContain('pr-checks') + expect(lanes).toContain('baseline:failing-unit:current-runtime') + expect(lanes).toContain('desktop-smoke:agent-browser-chat:current-runtime') + expect(lanes).toContain('native-checks') + }) + + test('baseline mode expands cases across explicit provider/model targets', () => { + const lanes = lanesForMode('baseline', [ + { providerId: 'provider-a', modelId: 'model-a', label: 'provider-a-model-a' }, + { providerId: 'provider-b', modelId: 'model-b', label: 'provider-b-model-b' }, + ]).map((lane) => lane.id) + + expect(lanes).toContain('baseline:failing-unit:provider-a-model-a') + expect(lanes).toContain('baseline:failing-unit:provider-b-model-b') + expect(lanes).toContain('baseline:multi-file-api:provider-a-model-a') + expect(lanes).toContain('baseline:multi-file-api:provider-b-model-b') + expect(lanes).toContain('desktop-smoke:agent-browser-chat:provider-a-model-a') + expect(lanes).toContain('desktop-smoke:agent-browser-chat:provider-b-model-b') + }) +}) + +describe('runQualityGate', () => { + test('writes dry-run reports without executing expensive commands', async () => { + const artifactsDir = mkdtempSync(join(tmpdir(), 'quality-gate-test-')) + try { + const { report, outputDir } = await runQualityGate({ + mode: 'baseline', + dryRun: true, + allowLive: false, + baselineTargets: [], + rootDir: process.cwd(), + artifactsDir, + runId: 'dry-run-test', + }) + + expect(report.mode).toBe('baseline') + expect(report.summary.failed).toBe(0) + expect(report.summary.skipped).toBeGreaterThan(0) + expect(readFileSync(join(outputDir, 'report.json'), 'utf8')).toContain('"mode": "baseline"') + expect(readFileSync(join(outputDir, 'report.md'), 'utf8')).toContain('# Quality Gate Report') + } finally { + rmSync(artifactsDir, { recursive: true, force: true }) + } + }) + + test('continues through later lanes after a failure so reports show full impact', async () => { + const artifactsDir = mkdtempSync(join(tmpdir(), 'quality-gate-test-')) + const lanes: LaneDefinition[] = [ + { + id: 'first-lane', + title: 'First lane', + description: 'Fails intentionally', + kind: 'command', + command: ['false'], + requiredForModes: ['pr'], + }, + { + id: 'second-lane', + title: 'Second lane', + description: 'Still runs', + kind: 'command', + command: ['true'], + requiredForModes: ['pr'], + }, + ] + + try { + const { report } = await runQualityGateLanes({ + mode: 'pr', + dryRun: false, + allowLive: false, + baselineTargets: [], + rootDir: process.cwd(), + artifactsDir, + runId: 'continue-after-failure-test', + }, lanes, async (lane) => ({ + id: lane.id, + title: lane.title, + status: lane.id === 'first-lane' ? 'failed' : 'passed', + command: lane.command, + durationMs: 1, + exitCode: lane.id === 'first-lane' ? 1 : 0, + })) + + expect(report.results.map((result) => result.id)).toEqual(['first-lane', 'second-lane']) + expect(report.summary).toEqual({ + passed: 1, + failed: 1, + skipped: 0, + }) + } finally { + rmSync(artifactsDir, { recursive: true, force: true }) + } + }) +}) + +describe('renderMarkdownReport', () => { + test('renders command, skip reason, and summary', () => { + const report: QualityGateReport = { + schemaVersion: 1, + runId: 'example', + mode: 'pr', + dryRun: true, + allowLive: false, + startedAt: '2026-05-02T00:00:00.000Z', + finishedAt: '2026-05-02T00:00:01.000Z', + rootDir: process.cwd(), + git: { + sha: 'abc123', + dirty: true, + }, + results: [ + { + id: 'impact-report', + title: 'Impact report', + status: 'skipped', + command: ['bun', 'run', 'check:impact'], + durationMs: 1, + skipReason: 'dry run', + }, + ], + summary: { + passed: 0, + failed: 0, + skipped: 1, + }, + } + + const markdown = renderMarkdownReport(report) + expect(markdown).toContain('Skipped: 1') + expect(markdown).toContain('`bun run check:impact`') + expect(markdown).toContain('dry run') + }) +}) diff --git a/scripts/quality-gate/runner.ts b/scripts/quality-gate/runner.ts new file mode 100644 index 00000000..f03990a4 --- /dev/null +++ b/scripts/quality-gate/runner.ts @@ -0,0 +1,182 @@ +import { mkdirSync } from 'node:fs' +import { join } from 'node:path' +import { baselineCases } from './baseline/cases' +import { executeBaselineCase } from './baseline/execute' +import { executeDesktopSmoke } from './desktop-smoke/execute' +import { lanesForMode } from './modes' +import { writeReport } from './reporter' +import type { LaneDefinition, LaneResult, QualityGateOptions, QualityGateReport } from './types' + +type LaneExecutor = (lane: LaneDefinition, options: QualityGateOptions) => Promise + +function nowId() { + return new Date().toISOString().replace(/[:.]/g, '-') +} + +async function output(cmd: string[], cwd: string) { + const proc = Bun.spawn(cmd, { + cwd, + stdout: 'pipe', + stderr: 'pipe', + }) + const stdout = await new Response(proc.stdout).text() + const stderr = await new Response(proc.stderr).text() + const code = await proc.exited + if (code !== 0) { + return null + } + return (stdout || stderr).trim() +} + +async function gitInfo(rootDir: string) { + const sha = await output(['git', 'rev-parse', '--short', 'HEAD'], rootDir) + const status = await output(['git', 'status', '--short'], rootDir) + return { + sha, + dirty: Boolean(status), + } +} + +async function runCommandLane(lane: LaneDefinition, options: QualityGateOptions): Promise { + const started = Date.now() + const command = lane.command ?? [] + + if (options.dryRun) { + return { + id: lane.id, + title: lane.title, + status: 'skipped', + command, + durationMs: Date.now() - started, + skipReason: 'dry run', + } + } + + const proc = Bun.spawn(command, { + cwd: options.rootDir, + stdout: 'inherit', + stderr: 'inherit', + }) + const exitCode = await proc.exited + + return { + id: lane.id, + title: lane.title, + status: exitCode === 0 ? 'passed' : 'failed', + command, + durationMs: Date.now() - started, + exitCode, + } +} + +async function runBaselineCaseLane(lane: LaneDefinition, options: QualityGateOptions): Promise { + const started = Date.now() + + if (!options.allowLive) { + return { + id: lane.id, + title: lane.title, + status: 'skipped', + durationMs: Date.now() - started, + skipReason: 'live baseline cases require --allow-live', + } + } + + const caseId = lane.baselineCaseId ?? lane.id.replace(/^baseline:/, '').split(':')[0] + const testCase = baselineCases.find((candidate) => candidate.id === caseId) + if (!testCase) { + return { + id: lane.id, + title: lane.title, + status: 'failed', + durationMs: Date.now() - started, + error: `Unknown baseline case: ${caseId}`, + } + } + + const artifactRoot = options.runOutputDir ?? join(options.rootDir, 'artifacts', 'quality-runs', options.runId ?? 'current') + return executeBaselineCase( + testCase, + options.rootDir, + join(artifactRoot, 'cases', lane.id.replace(/[^a-zA-Z0-9._-]+/g, '-')), + lane.baselineTarget, + ) +} + +async function runLane(lane: LaneDefinition, options: QualityGateOptions): Promise { + if (lane.kind === 'baseline-case') { + return runBaselineCaseLane(lane, options) + } + if (lane.kind === 'desktop-smoke') { + const started = Date.now() + + if (!options.allowLive) { + return { + id: lane.id, + title: lane.title, + status: 'skipped', + durationMs: Date.now() - started, + skipReason: 'desktop agent-browser smoke requires --allow-live', + } + } + + const artifactRoot = options.runOutputDir ?? join(options.rootDir, 'artifacts', 'quality-runs', options.runId ?? 'current') + return executeDesktopSmoke( + options.rootDir, + join(artifactRoot, 'cases', lane.id.replace(/[^a-zA-Z0-9._-]+/g, '-')), + lane.id, + lane.title, + lane.baselineTarget, + ) + } + + return runCommandLane(lane, options) +} + +function summarize(results: LaneResult[]) { + return { + passed: results.filter((result) => result.status === 'passed').length, + failed: results.filter((result) => result.status === 'failed').length, + skipped: results.filter((result) => result.status === 'skipped').length, + } +} + +export async function runQualityGate(options: QualityGateOptions) { + return runQualityGateLanes(options, lanesForMode(options.mode, options.baselineTargets)) +} + +export async function runQualityGateLanes( + options: QualityGateOptions, + lanes: LaneDefinition[], + executeLane: LaneExecutor = runLane, +) { + const runId = options.runId ?? nowId() + const startedAt = new Date().toISOString() + const artifactsRoot = options.artifactsDir ?? join(options.rootDir, 'artifacts', 'quality-runs') + const outputDir = join(artifactsRoot, runId) + mkdirSync(outputDir, { recursive: true }) + + const runOptions = { ...options, runId, runOutputDir: outputDir } + const results: LaneResult[] = [] + for (const lane of lanes) { + const result = await executeLane(lane, runOptions) + results.push(result) + } + + const report: QualityGateReport = { + schemaVersion: 1, + runId, + mode: options.mode, + dryRun: options.dryRun, + allowLive: options.allowLive, + startedAt, + finishedAt: new Date().toISOString(), + rootDir: options.rootDir, + git: await gitInfo(options.rootDir), + results, + summary: summarize(results), + } + + writeReport(report, outputDir) + return { report, outputDir } +} diff --git a/scripts/quality-gate/types.ts b/scripts/quality-gate/types.ts new file mode 100644 index 00000000..c75fc6a6 --- /dev/null +++ b/scripts/quality-gate/types.ts @@ -0,0 +1,85 @@ +export type QualityGateMode = 'pr' | 'baseline' | 'release' + +export type LaneKind = 'command' | 'baseline-case' | 'desktop-smoke' + +export type LaneDefinition = { + id: string + title: string + description: string + kind: LaneKind + command?: string[] + baselineCaseId?: string + baselineTarget?: BaselineTarget + requiredForModes: QualityGateMode[] + live?: boolean +} + +export type BaselineCase = { + id: string + title: string + description: string + fixture: string + prompt: string + mode: 'ui' | 'websocket' + requiredCapabilities: Array<'model' | 'file-edit' | 'shell' | 'permission' | 'browser'> + timeoutMs: number + verify: { + commands: string[][] + requiredFiles?: string[] + expectedFiles?: string[] + forbiddenFiles?: string[] + transcriptAssertions?: string[] + } +} + +export type BaselineTarget = { + providerId: string | null + modelId: string + label: string +} + +export type LaneStatus = 'passed' | 'failed' | 'skipped' + +export type LaneResult = { + id: string + title: string + status: LaneStatus + command?: string[] + durationMs: number + exitCode?: number + skipReason?: string + error?: string + artifactDir?: string +} + +export type QualityGateOptions = { + mode: QualityGateMode + dryRun: boolean + allowLive: boolean + baselineTargets: BaselineTarget[] + rootDir: string + artifactsDir?: string + runOutputDir?: string + runId?: string +} + +export type QualityGateReport = { + schemaVersion: 1 + runId: string + mode: QualityGateMode + dryRun: boolean + allowLive: boolean + startedAt: string + finishedAt: string + rootDir: string + git: { + sha: string | null + dirty: boolean + } + results: LaneResult[] + summary: { + passed: number + failed: number + skipped: number + } +} diff --git a/src/server/__tests__/conversations.test.ts b/src/server/__tests__/conversations.test.ts index fe8f1975..05a0b031 100644 --- a/src/server/__tests__/conversations.test.ts +++ b/src/server/__tests__/conversations.test.ts @@ -252,9 +252,11 @@ describe('ConversationService', () => { it('should reconstruct usage and metadata from a persisted transcript', async () => { const previousConfigDir = process.env.CLAUDE_CONFIG_DIR + const previousAnthropicApiKey = process.env.ANTHROPIC_API_KEY const tmpConfigDir = await fs.mkdtemp(path.join(os.tmpdir(), 'claude-transcript-')) const workDir = await fs.mkdtemp(path.join(os.tmpdir(), 'claude-workdir-')) process.env.CLAUDE_CONFIG_DIR = tmpConfigDir + process.env.ANTHROPIC_API_KEY = 'test-key' try { const svc = new SessionService() @@ -305,6 +307,11 @@ describe('ConversationService', () => { } else { process.env.CLAUDE_CONFIG_DIR = previousConfigDir } + if (previousAnthropicApiKey === undefined) { + delete process.env.ANTHROPIC_API_KEY + } else { + process.env.ANTHROPIC_API_KEY = previousAnthropicApiKey + } await fs.rm(tmpConfigDir, { recursive: true, force: true }) await fs.rm(workDir, { recursive: true, force: true }) }