From 055854f8ec2ab4c6e1374ab88c88d60d4c15cf30 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=E7=A8=8B=E5=BA=8F=E5=91=98=E9=98=BF=E6=B1=9F=28Relakkes?= =?UTF-8?q?=29?= Date: Fri, 3 Jul 2026 18:03:23 +0800 Subject: [PATCH] fix: constrain skill market fallback caching Tested: bun test src/server/__tests__/skill-market.test.ts Tested: git diff --check Confidence: high Scope-risk: narrow --- src/server/__tests__/skill-market.test.ts | 124 ++++++++++++++++ src/server/services/skillMarket/service.ts | 159 +++++++++++++++++---- 2 files changed, 256 insertions(+), 27 deletions(-) diff --git a/src/server/__tests__/skill-market.test.ts b/src/server/__tests__/skill-market.test.ts index 9d70c171..74a013c8 100644 --- a/src/server/__tests__/skill-market.test.ts +++ b/src/server/__tests__/skill-market.test.ts @@ -478,6 +478,130 @@ describe('skill market service source selection', () => { expect(result.message).toContain('network down') }) + it('caches catalog results without caching installed state', async () => { + const fetchCalls: string[] = [] + let installedSkillNames = new Set() + const service = createSkillMarketService({ + fetchImpl: async (url) => { + fetchCalls.push(String(url)) + return Response.json(CLAWHUB_TOP_SKILLS_RESPONSE) + }, + installedSkillNames: async () => installedSkillNames, + now: () => 1_000, + }) + + const first = await service.listSkills({ source: 'clawhub', limit: 12, query: 'vetter' }) + installedSkillNames = new Set(['skill-vetter']) + const second = await service.listSkills({ source: 'clawhub', limit: 12, query: 'vetter' }) + + expect(fetchCalls).toHaveLength(1) + expect(first.items[0]).toMatchObject({ slug: 'skill-vetter', installed: false }) + expect(second.items[0]).toMatchObject({ slug: 'skill-vetter', installed: true }) + }) + + it('refreshes cached catalog results after the catalog TTL expires', async () => { + const fetchCalls: string[] = [] + let now = 10_000 + const service = createSkillMarketService({ + fetchImpl: async (url) => { + fetchCalls.push(String(url)) + return Response.json(CLAWHUB_TOP_SKILLS_RESPONSE) + }, + installedSkillNames: async () => new Set(), + now: () => now, + }) + + await service.listSkills({ source: 'clawhub', limit: 12, query: 'vetter' }) + await service.listSkills({ source: 'clawhub', limit: 12, query: 'vetter' }) + expect(fetchCalls).toHaveLength(1) + now += 5 * 60 * 1_000 + 1 + await service.listSkills({ source: 'clawhub', limit: 12, query: 'vetter' }) + + expect(fetchCalls).toHaveLength(2) + expect(fetchCalls[0]).toBe(fetchCalls[1]) + }) + + it('uses the failure cache to skip repeated ClawHub request failures in auto mode', async () => { + const fetchCalls: string[] = [] + let now = 20_000 + let clawHubRequests = 0 + const service = createSkillMarketService({ + fetchImpl: async (url) => { + const urlString = String(url) + fetchCalls.push(urlString) + if (urlString.startsWith('https://clawhub.ai/')) { + clawHubRequests += 1 + if (clawHubRequests === 1) { + return new Response('temporarily unavailable', { status: 503 }) + } + return Response.json(CLAWHUB_TOP_SKILLS_RESPONSE) + } + return Response.json(SKILLHUB_TOP_SKILLS_RESPONSE) + }, + installedSkillNames: async () => new Set(), + now: () => now, + }) + + const first = await service.listSkills({ source: 'auto', limit: 10, query: 'vetter' }) + const requestsAfterFirst = fetchCalls.length + now += 30_000 + const second = await service.listSkills({ source: 'auto', limit: 10, query: 'vetter' }) + const requestsAfterSecond = fetchCalls.length + now += 30_001 + const third = await service.listSkills({ source: 'auto', limit: 10, query: 'vetter' }) + + expect(fetchCalls.slice(0, requestsAfterFirst)).toEqual([ + expect.stringContaining('https://clawhub.ai/api/v1/skills'), + expect.stringContaining('https://api.skillhub.cn/api/skills'), + ]) + expect(fetchCalls.slice(requestsAfterFirst, requestsAfterSecond)).not.toContainEqual( + expect.stringContaining('https://clawhub.ai/'), + ) + expect(first.sourceStatus).toBe('fallback') + expect(second.source).toBe('skillhub') + expect(second.sourceStatus).toBe('fallback') + expect(second.message).toContain('ClawHub unavailable') + expect(third).toMatchObject({ source: 'clawhub', sourceStatus: 'ok' }) + expect(clawHubRequests).toBe(2) + }) + + it('does not fall back when ClawHub returns 2xx but JSON parsing fails', async () => { + const fetchCalls: string[] = [] + const service = createSkillMarketService({ + fetchImpl: async (url) => { + fetchCalls.push(String(url)) + if (String(url).startsWith('https://clawhub.ai/')) { + return new Response('{not-json', { status: 200 }) + } + return Response.json(SKILLHUB_TOP_SKILLS_RESPONSE) + }, + installedSkillNames: async () => new Set(), + }) + + await expect(service.listSkills({ source: 'auto' })).rejects.toThrow() + + expect(fetchCalls).toHaveLength(1) + expect(fetchCalls[0]).toStartWith('https://clawhub.ai/api/v1/skills') + }) + + it('does not fall back when installed skill resolution fails', async () => { + const fetchCalls: string[] = [] + const service = createSkillMarketService({ + fetchImpl: async (url) => { + fetchCalls.push(String(url)) + return Response.json(CLAWHUB_TOP_SKILLS_RESPONSE) + }, + installedSkillNames: async () => { + throw new Error('installed provider unavailable') + }, + }) + + await expect(service.listSkills({ source: 'auto' })).rejects.toThrow('installed provider unavailable') + + expect(fetchCalls).toHaveLength(1) + expect(fetchCalls[0]).toStartWith('https://clawhub.ai/api/v1/skills') + }) + it("does not fallback to SkillHub when source is 'clawhub'", async () => { const fetchCalls: string[] = [] const service = createSkillMarketService({ diff --git a/src/server/services/skillMarket/service.ts b/src/server/services/skillMarket/service.ts index bc7813d3..f823b7dc 100644 --- a/src/server/services/skillMarket/service.ts +++ b/src/server/services/skillMarket/service.ts @@ -18,6 +18,7 @@ type InstalledSkillNamesProvider = Set | (() => Set | Promise number } export type SkillMarketService = { @@ -29,10 +30,33 @@ const CLAWHUB_SKILLS_URL = 'https://clawhub.ai/api/v1/skills' const SKILLHUB_SKILLS_URL = 'https://api.skillhub.cn/api/skills' const DEFAULT_LIMIT = 24 const MAX_LIMIT = 100 +const CATALOG_CACHE_TTL_MS = 5 * 60 * 1_000 +const FAILURE_CACHE_TTL_MS = 60 * 1_000 + +type CatalogCacheEntry = { + expiresAt: number + result: SkillMarketListResult +} + +type FailureCacheEntry = { + expiresAt: number + message: string +} + +class SkillMarketRequestError extends Error { + constructor(message: string, options?: { cause?: unknown }) { + super(message) + this.name = 'SkillMarketRequestError' + this.cause = options?.cause + } +} export function createSkillMarketService(options: SkillMarketServiceOptions = {}): SkillMarketService { const fetchImpl = options.fetchImpl ?? fetch const installedSkillNames = options.installedSkillNames + const now = options.now ?? Date.now + const catalogCache = new Map() + const failureCache = new Map() async function listSkills(params: SkillMarketListParams = {}): Promise { const source = params.source ?? 'auto' @@ -49,46 +73,99 @@ export function createSkillMarketService(options: SkillMarketServiceOptions = {} throw new Error(`Unsupported skill market source: ${source}`) } + const clawHubFailure = recentFailure(clawHubCatalogCacheKey(params)) + if (clawHubFailure) { + const fallback = await listSkillHub(params) + return withInstalled({ + ...fallback, + sourceStatus: 'fallback', + message: `ClawHub unavailable: recent request failure (${clawHubFailure.message})`, + }) + } + + let clawHub: SkillMarketListResult try { - return withInstalled(await listClawHub(params)) + clawHub = await listClawHub(params) } catch (error) { - const fallback = await listSkillHub(params, 'fallback') + if (!(error instanceof SkillMarketRequestError)) { + throw error + } + const fallback = await listSkillHub(params) return withInstalled({ ...fallback, sourceStatus: 'fallback', message: `ClawHub unavailable: ${errorMessage(error)}`, }) } + return withInstalled(clawHub) } async function listClawHub(params: SkillMarketListParams): Promise { - const url = new URL(CLAWHUB_SKILLS_URL) - url.searchParams.set('sort', clawHubSort(params.sort)) - url.searchParams.set('nonSuspiciousOnly', 'true') - url.searchParams.set('limit', String(limitFor(params.limit))) - addOptionalParam(url, 'query', params.query) - addOptionalParam(url, 'cursor', params.cursor) - - const payload = await requestJson(fetchImpl, url, 'ClawHub') - return normalizeClawHubList(payload) + const url = clawHubUrlFor(params) + const cacheKey = catalogCacheKey('clawhub', url) + try { + const result = await cachedCatalog(cacheKey, async () => { + const payload = await requestJson(fetchImpl, url, 'ClawHub') + return normalizeClawHubList(payload) + }) + failureCache.delete(cacheKey) + return result + } catch (error) { + if (error instanceof SkillMarketRequestError) { + failureCache.set(cacheKey, { + expiresAt: now() + FAILURE_CACHE_TTL_MS, + message: errorMessage(error), + }) + } + throw error + } } - async function listSkillHub( - params: SkillMarketListParams, - sourceStatus: SkillMarketListResult['sourceStatus'], - ): Promise { - const url = new URL(SKILLHUB_SKILLS_URL) - url.searchParams.set('sortBy', skillHubSort(params.sort)) - url.searchParams.set('order', 'desc') - url.searchParams.set('limit', String(limitFor(params.limit))) - addOptionalParam(url, 'query', params.query) - addOptionalParam(url, 'cursor', params.cursor) + async function listSkillHub(params: SkillMarketListParams): Promise { + const url = skillHubUrlFor(params) + return cachedCatalog(catalogCacheKey('skillhub', url), async () => { + const payload = await requestJson(fetchImpl, url, 'SkillHub') + return { + ...normalizeSkillHubList(payload), + sourceStatus: 'ok', + } + }) + } - const payload = await requestJson(fetchImpl, url, 'SkillHub') - return { - ...normalizeSkillHubList(payload), - sourceStatus, + async function cachedCatalog( + cacheKey: string, + loader: () => Promise, + ): Promise { + const cached = catalogCache.get(cacheKey) + const currentTime = now() + if (cached && cached.expiresAt > currentTime) { + return { + ...cached.result, + sourceStatus: 'cached', + } } + if (cached) { + catalogCache.delete(cacheKey) + } + + const result = await loader() + catalogCache.set(cacheKey, { + expiresAt: now() + CATALOG_CACHE_TTL_MS, + result, + }) + return result + } + + function recentFailure(cacheKey: string): FailureCacheEntry | undefined { + const cached = failureCache.get(cacheKey) + if (!cached) { + return undefined + } + if (cached.expiresAt > now()) { + return cached + } + failureCache.delete(cacheKey) + return undefined } async function withInstalled(result: SkillMarketListResult): Promise { @@ -108,16 +185,44 @@ export function createSkillMarketService(options: SkillMarketServiceOptions = {} } } +function clawHubUrlFor(params: SkillMarketListParams): URL { + const url = new URL(CLAWHUB_SKILLS_URL) + url.searchParams.set('sort', clawHubSort(params.sort)) + url.searchParams.set('nonSuspiciousOnly', 'true') + url.searchParams.set('limit', String(limitFor(params.limit))) + addOptionalParam(url, 'query', params.query) + addOptionalParam(url, 'cursor', params.cursor) + return url +} + +function skillHubUrlFor(params: SkillMarketListParams): URL { + const url = new URL(SKILLHUB_SKILLS_URL) + url.searchParams.set('sortBy', skillHubSort(params.sort)) + url.searchParams.set('order', 'desc') + url.searchParams.set('limit', String(limitFor(params.limit))) + addOptionalParam(url, 'query', params.query) + addOptionalParam(url, 'cursor', params.cursor) + return url +} + +function clawHubCatalogCacheKey(params: SkillMarketListParams): string { + return catalogCacheKey('clawhub', clawHubUrlFor(params)) +} + +function catalogCacheKey(source: 'clawhub' | 'skillhub', url: URL): string { + return `${source}:${url.toString()}` +} + async function requestJson(fetchImpl: FetchImpl, url: URL, sourceName: string): Promise { let response: Response try { response = await fetchImpl(url) } catch (error) { - throw new Error(`${sourceName} request failed: ${errorMessage(error)}`) + throw new SkillMarketRequestError(`${sourceName} request failed: ${errorMessage(error)}`, { cause: error }) } if (!response.ok) { - throw new Error(`${sourceName} request failed with status ${response.status}`) + throw new SkillMarketRequestError(`${sourceName} request failed with status ${response.status}`) } return response.json()