diff --git a/src/server/__tests__/providers.test.ts b/src/server/__tests__/providers.test.ts index 68d82c7a..2354693e 100644 --- a/src/server/__tests__/providers.test.ts +++ b/src/server/__tests__/providers.test.ts @@ -1002,10 +1002,8 @@ describe('ProviderService', () => { } }) - test('injects Claude Code billing attribution with compat version and signed CCH', async () => { + test('strips leading billing attribution instead of injecting it for OpenAI-compatible upstreams', async () => { const originalFetch = globalThis.fetch - const originalEntrypoint = process.env.CLAUDE_CODE_ENTRYPOINT - delete process.env.CLAUDE_CODE_ENTRYPOINT const calls: Array<{ body: Record }> = [] globalThis.fetch = mock(async (_url: string | URL | Request, init?: RequestInit) => { calls.push({ body: JSON.parse(String(init?.body)) as Record }) @@ -1033,6 +1031,10 @@ describe('ProviderService', () => { body: JSON.stringify({ model: 'gpt-4', max_tokens: 64, + system: [ + { type: 'text', text: 'x-anthropic-billing-header: cc_version=2.1.92.693; cc_entrypoint=cli; cch=00000;' }, + { type: 'text', text: 'You are a helpful assistant.' }, + ], messages: [{ role: 'user', content: 'hello from proxy' }], }), }) @@ -1040,15 +1042,57 @@ describe('ProviderService', () => { const res = await handleProxyRequest(req, new URL(req.url)) expect(res.status).toBe(200) - const system = calls[0].body.messages as Array> - expect(system[0].role).toBe('system') - expect(system[0].content).toMatch( - /^x-anthropic-billing-header: cc_version=2\.1\.92\.693; cc_entrypoint=unknown; cch=[0-9a-f]{5};$/, - ) + // The rotating billing header would change the prompt prefix on every + // request and defeat upstream prefix caching — it must not be forwarded. + const messages = calls[0].body.messages as Array> + expect(messages[0].role).toBe('system') + expect(messages[0].content).toBe('You are a helpful assistant.') + expect(JSON.stringify(calls[0].body)).not.toContain('x-anthropic-billing-header') + } finally { + globalThis.fetch = originalFetch + } + }) + + test('forwards a stable prompt_cache_key from client session metadata for OpenAI Responses upstreams', async () => { + const originalFetch = globalThis.fetch + const calls: Array<{ body: Record }> = [] + globalThis.fetch = mock(async (_url: string | URL | Request, init?: RequestInit) => { + calls.push({ body: JSON.parse(String(init?.body)) as Record }) + return new Response(JSON.stringify({ + id: 'resp-1', + object: 'response', + created_at: 0, + model: 'gpt-5.4', + status: 'completed', + output: [{ type: 'message', role: 'assistant', content: [{ type: 'output_text', text: 'ok' }] }], + usage: { input_tokens: 1, output_tokens: 1, total_tokens: 2 }, + }), { + status: 200, + headers: { 'Content-Type': 'application/json' }, + }) + }) as typeof fetch + + try { + const svc = new ProviderService() + const provider = await svc.addProvider(sampleInput({ apiFormat: 'openai_responses' })) + await svc.activateProvider(provider.id) + + const req = new Request('http://localhost:3456/proxy/v1/messages', { + method: 'POST', + headers: { 'Content-Type': 'application/json' }, + body: JSON.stringify({ + model: 'gpt-5.4', + max_tokens: 64, + metadata: { user_id: 'user_3f7a_account_9b2c_session_sess-42aa' }, + messages: [{ role: 'user', content: 'hello from proxy' }], + }), + }) + + const res = await handleProxyRequest(req, new URL(req.url)) + expect(res.status).toBe(200) + expect(calls[0].body.prompt_cache_key).toBe('sess-42aa') } finally { globalThis.fetch = originalFetch - if (originalEntrypoint === undefined) delete process.env.CLAUDE_CODE_ENTRYPOINT - else process.env.CLAUDE_CODE_ENTRYPOINT = originalEntrypoint } }) diff --git a/src/server/__tests__/proxy-streaming.test.ts b/src/server/__tests__/proxy-streaming.test.ts index 56dd9ece..bcabadc3 100644 --- a/src/server/__tests__/proxy-streaming.test.ts +++ b/src/server/__tests__/proxy-streaming.test.ts @@ -269,6 +269,40 @@ describe('openaiChatStreamToAnthropic', () => { ) expect(firstBlockStop).toBeLessThan(toolBlockStart) }) + + test('maps cached tokens from a trailing usage-only chunk', async () => { + // stream_options.include_usage delivers usage in a final chunk with empty choices + const sseChunks = [ + 'data: {"id":"c8","object":"chat.completion.chunk","created":0,"model":"gpt-4","choices":[{"index":0,"delta":{"content":"Hi"},"finish_reason":null}]}\n\n', + 'data: {"id":"c8","object":"chat.completion.chunk","created":0,"model":"gpt-4","choices":[{"index":0,"delta":{},"finish_reason":"stop"}]}\n\n', + 'data: {"id":"c8","object":"chat.completion.chunk","created":0,"model":"gpt-4","choices":[],"usage":{"prompt_tokens":100,"completion_tokens":7,"prompt_tokens_details":{"cached_tokens":80}}}\n\n', + 'data: [DONE]\n\n', + ] + + const events = await collectSse(openaiChatStreamToAnthropic(makeStream(sseChunks), 'gpt-4')) + const msgDelta = events.find((e) => e.event === 'message_delta')! + expect(msgDelta.data.usage).toEqual({ + input_tokens: 20, + output_tokens: 7, + cache_read_input_tokens: 80, + }) + }) + + test('maps cached tokens when usage arrives with finish_reason', async () => { + const sseChunks = [ + 'data: {"id":"c9","object":"chat.completion.chunk","created":0,"model":"gpt-4","choices":[{"index":0,"delta":{"content":"Hi"},"finish_reason":null}]}\n\n', + 'data: {"id":"c9","object":"chat.completion.chunk","created":0,"model":"gpt-4","choices":[{"index":0,"delta":{},"finish_reason":"stop"}],"usage":{"prompt_tokens":50,"completion_tokens":3,"prompt_tokens_details":{"cached_tokens":40}}}\n\n', + 'data: [DONE]\n\n', + ] + + const events = await collectSse(openaiChatStreamToAnthropic(makeStream(sseChunks), 'gpt-4')) + const msgDelta = events.find((e) => e.event === 'message_delta')! + expect(msgDelta.data.usage).toEqual({ + input_tokens: 10, + output_tokens: 3, + cache_read_input_tokens: 40, + }) + }) }) // ─── OpenAI Responses SSE → Anthropic SSE ────────────────────── @@ -341,4 +375,23 @@ describe('openaiResponsesStreamToAnthropic', () => { const msgDelta = events.find((e) => e.event === 'message_delta')! expect((msgDelta.data.delta as Record).stop_reason).toBe('tool_use') }) + + test('maps cached tokens from response.completed usage', async () => { + const sseChunks = [ + 'event: response.created\ndata: {"id":"r3","model":"gpt-5.4","status":"in_progress"}\n\n', + 'event: response.output_item.added\ndata: {"output_index":0,"item":{"type":"message","role":"assistant"}}\n\n', + 'event: response.content_part.added\ndata: {"output_index":0,"content_index":0,"part":{"type":"output_text","text":""}}\n\n', + 'event: response.output_text.delta\ndata: {"output_index":0,"content_index":0,"delta":"Hi"}\n\n', + 'event: response.output_text.done\ndata: {"output_index":0,"content_index":0,"text":"Hi"}\n\n', + 'event: response.completed\ndata: {"response":{"id":"r3","model":"gpt-5.4","status":"completed","usage":{"input_tokens":1200,"output_tokens":40,"input_tokens_details":{"cached_tokens":1000}}}}\n\n', + ] + + const events = await collectSse(openaiResponsesStreamToAnthropic(makeStream(sseChunks), 'gpt-5.4')) + const msgDelta = events.find((e) => e.event === 'message_delta')! + expect(msgDelta.data.usage).toEqual({ + input_tokens: 200, + output_tokens: 40, + cache_read_input_tokens: 1000, + }) + }) }) diff --git a/src/server/__tests__/proxy-transform.test.ts b/src/server/__tests__/proxy-transform.test.ts index 657bd34e..b4ce8b52 100644 --- a/src/server/__tests__/proxy-transform.test.ts +++ b/src/server/__tests__/proxy-transform.test.ts @@ -7,8 +7,13 @@ import { anthropicToOpenaiChat } from '../proxy/transform/anthropicToOpenaiChat. import { anthropicToOpenaiResponses } from '../proxy/transform/anthropicToOpenaiResponses.js' import { openaiChatToAnthropic } from '../proxy/transform/openaiChatToAnthropic.js' import { openaiResponsesToAnthropic } from '../proxy/transform/openaiResponsesToAnthropic.js' +import { stripLeadingBillingHeader } from '../proxy/transform/billingHeader.js' +import { openaiUsageToAnthropic } from '../proxy/transform/usage.js' +import { resolvePromptCacheKey } from '../proxy/promptCacheKey.js' import type { AnthropicRequest, OpenAIChatResponse, OpenAIResponsesResponse } from '../proxy/transform/types.js' +const BILLING_HEADER = 'x-anthropic-billing-header: cc_version=2.1.92.693; cc_entrypoint=cli; cch=00000;' + // ─── anthropicToOpenaiChat ────────────────────────────────────── describe('anthropicToOpenaiChat', () => { @@ -609,3 +614,255 @@ describe('openaiResponsesToAnthropic', () => { expect(result.content).toEqual([{ type: 'text', text: '' }]) }) }) + +// ─── stripLeadingBillingHeader ────────────────────────────────── + +describe('stripLeadingBillingHeader', () => { + test('returns text unchanged when no billing header prefix', () => { + expect(stripLeadingBillingHeader('You are helpful')).toBe('You are helpful') + }) + + test('strips a single-line billing header to empty string', () => { + expect(stripLeadingBillingHeader(BILLING_HEADER)).toBe('') + }) + + test('strips leading header line and its blank separator', () => { + expect(stripLeadingBillingHeader(`${BILLING_HEADER}\n\nYou are helpful`)).toBe('You are helpful') + }) + + test('strips leading header line followed directly by text', () => { + expect(stripLeadingBillingHeader(`${BILLING_HEADER}\nYou are helpful`)).toBe('You are helpful') + }) + + test('handles CRLF line endings', () => { + expect(stripLeadingBillingHeader(`${BILLING_HEADER}\r\n\r\nYou are helpful`)).toBe('You are helpful') + }) + + test('keeps later occurrences inside user-authored text', () => { + const text = `You are helpful.\n${BILLING_HEADER}` + expect(stripLeadingBillingHeader(text)).toBe(text) + }) +}) + +// ─── resolvePromptCacheKey ────────────────────────────────────── + +describe('resolvePromptCacheKey', () => { + const baseRequest = (metadata?: AnthropicRequest['metadata']): AnthropicRequest => ({ + model: 'gpt-5.4', + max_tokens: 64, + messages: [{ role: 'user', content: 'hi' }], + ...(metadata ? { metadata } : {}), + }) + + test('extracts session suffix from metadata.user_id', () => { + const body = baseRequest({ user_id: 'user_3f7a_account_9b2c_session_sess-42aa' }) + expect(resolvePromptCacheKey(body)).toBe('sess-42aa') + }) + + test('falls back to metadata.session_id', () => { + const body = baseRequest({ session_id: 'direct-session-id' }) + expect(resolvePromptCacheKey(body)).toBe('direct-session-id') + }) + + test('falls back to the CLI session header', () => { + expect(resolvePromptCacheKey(baseRequest(), ' header-session ')).toBe('header-session') + }) + + test('prefers user_id session over session_id and header', () => { + const body = baseRequest({ user_id: 'user_x_session_from-user-id', session_id: 'from-metadata' }) + expect(resolvePromptCacheKey(body, 'from-header')).toBe('from-user-id') + }) + + test('returns undefined without any client session identity', () => { + expect(resolvePromptCacheKey(baseRequest())).toBeUndefined() + expect(resolvePromptCacheKey(baseRequest(), ' ')).toBeUndefined() + expect(resolvePromptCacheKey(baseRequest({ user_id: 'user_without_marker' }))).toBeUndefined() + }) + + test('ignores empty session suffix in user_id', () => { + expect(resolvePromptCacheKey(baseRequest({ user_id: 'user_x_session_' }))).toBeUndefined() + }) +}) + +// ─── openaiUsageToAnthropic ───────────────────────────────────── + +describe('openaiUsageToAnthropic', () => { + test('maps Responses-style cached tokens and excludes them from input', () => { + const usage = openaiUsageToAnthropic({ + input_tokens: 100, + output_tokens: 5, + input_tokens_details: { cached_tokens: 80 }, + }) + expect(usage).toEqual({ input_tokens: 20, output_tokens: 5, cache_read_input_tokens: 80 }) + }) + + test('maps Chat-style cached tokens as fallback', () => { + const usage = openaiUsageToAnthropic({ + prompt_tokens: 100, + completion_tokens: 5, + prompt_tokens_details: { cached_tokens: 30 }, + }) + expect(usage).toEqual({ input_tokens: 70, output_tokens: 5, cache_read_input_tokens: 30 }) + }) + + test('prefers direct Anthropic-style cache fields over nested details', () => { + const usage = openaiUsageToAnthropic({ + input_tokens: 100, + output_tokens: 5, + input_tokens_details: { cached_tokens: 80 }, + cache_read_input_tokens: 60, + cache_creation_input_tokens: 10, + }) + expect(usage).toEqual({ + input_tokens: 30, + output_tokens: 5, + cache_read_input_tokens: 60, + cache_creation_input_tokens: 10, + }) + }) + + test('leaves input untouched and omits cache fields without cache activity', () => { + expect(openaiUsageToAnthropic({ input_tokens: 10, output_tokens: 5 })) + .toEqual({ input_tokens: 10, output_tokens: 5 }) + }) + + test('clamps input at zero when cached exceeds reported input', () => { + const usage = openaiUsageToAnthropic({ + input_tokens: 50, + output_tokens: 5, + input_tokens_details: { cached_tokens: 80 }, + }) + expect(usage.input_tokens).toBe(0) + expect(usage.cache_read_input_tokens).toBe(80) + }) + + test('returns zeros for missing usage', () => { + expect(openaiUsageToAnthropic(undefined)).toEqual({ input_tokens: 0, output_tokens: 0 }) + }) +}) + +// ─── prompt caching semantics in request transforms ───────────── + +describe('prompt caching semantics', () => { + test('responses transform strips leading billing header from system array', () => { + const req: AnthropicRequest = { + model: 'gpt-5.4', + max_tokens: 64, + system: [ + { type: 'text', text: BILLING_HEADER }, + { type: 'text', text: 'You are helpful' }, + ], + messages: [{ role: 'user', content: 'hi' }], + } + const result = anthropicToOpenaiResponses(req) + expect(result.instructions).toBe('You are helpful') + }) + + test('responses transform strips leading billing header from system string', () => { + const req: AnthropicRequest = { + model: 'gpt-5.4', + max_tokens: 64, + system: `${BILLING_HEADER}\n\nYou are helpful`, + messages: [{ role: 'user', content: 'hi' }], + } + const result = anthropicToOpenaiResponses(req) + expect(result.instructions).toBe('You are helpful') + }) + + test('responses transform omits instructions when system is only a billing header', () => { + const req: AnthropicRequest = { + model: 'gpt-5.4', + max_tokens: 64, + system: [{ type: 'text', text: BILLING_HEADER }], + messages: [{ role: 'user', content: 'hi' }], + } + const result = anthropicToOpenaiResponses(req) + expect(result.instructions).toBeUndefined() + }) + + test('responses transform injects prompt_cache_key when provided', () => { + const req: AnthropicRequest = { + model: 'gpt-5.4', + max_tokens: 64, + messages: [{ role: 'user', content: 'hi' }], + } + expect(anthropicToOpenaiResponses(req, { cacheKey: 'sess-1' }).prompt_cache_key).toBe('sess-1') + expect(anthropicToOpenaiResponses(req).prompt_cache_key).toBeUndefined() + }) + + test('chat transform strips leading billing header from system', () => { + const req: AnthropicRequest = { + model: 'gpt-4', + max_tokens: 64, + system: [ + { type: 'text', text: BILLING_HEADER }, + { type: 'text', text: 'You are helpful' }, + ], + messages: [{ role: 'user', content: 'hi' }], + } + const result = anthropicToOpenaiChat(req) + expect(result.messages[0]).toEqual({ role: 'system', content: 'You are helpful' }) + }) + + test('chat transform omits system message when system is only a billing header', () => { + const req: AnthropicRequest = { + model: 'gpt-4', + max_tokens: 64, + system: BILLING_HEADER, + messages: [{ role: 'user', content: 'hi' }], + } + const result = anthropicToOpenaiChat(req) + expect(result.messages[0]).toEqual({ role: 'user', content: 'hi' }) + }) + + test('chat transform requests stream usage explicitly', () => { + const req: AnthropicRequest = { + model: 'gpt-4', + max_tokens: 64, + messages: [{ role: 'user', content: 'hi' }], + } + expect(anthropicToOpenaiChat(req, {}).stream_options).toBeUndefined() + expect(anthropicToOpenaiChat({ ...req, stream: true }).stream_options).toEqual({ include_usage: true }) + }) + + test('responses non-streaming maps cached tokens into Anthropic usage', () => { + const res: OpenAIResponsesResponse = { + id: 'resp_cache', + object: 'response', + created_at: 0, + model: 'gpt-5.4', + status: 'completed', + output: [{ type: 'message', role: 'assistant', content: [{ type: 'output_text', text: 'hi' }] }], + usage: { + input_tokens: 1200, + output_tokens: 40, + input_tokens_details: { cached_tokens: 1000 }, + }, + } + const result = openaiResponsesToAnthropic(res, 'gpt-5.4') + expect(result.usage).toEqual({ + input_tokens: 200, + output_tokens: 40, + cache_read_input_tokens: 1000, + }) + }) + + test('chat non-streaming subtracts cached tokens from input', () => { + const res: OpenAIChatResponse = { + id: 'x', object: 'chat.completion', created: 0, model: 'gpt-4', + choices: [{ index: 0, message: { role: 'assistant', content: 'hi' }, finish_reason: 'stop' }], + usage: { + prompt_tokens: 100, + completion_tokens: 50, + total_tokens: 150, + prompt_tokens_details: { cached_tokens: 80 }, + }, + } + const result = openaiChatToAnthropic(res, 'gpt-4') + expect(result.usage).toEqual({ + input_tokens: 20, + output_tokens: 50, + cache_read_input_tokens: 80, + }) + }) +}) diff --git a/src/server/proxy/claudeCodeAttribution.ts b/src/server/proxy/claudeCodeAttribution.ts deleted file mode 100644 index 3151f6ec..00000000 --- a/src/server/proxy/claudeCodeAttribution.ts +++ /dev/null @@ -1,57 +0,0 @@ -/** - * Proxy-side Claude Code billing attribution based on sub2api gateway behavior. - * Reference: https://github.com/Wei-Shaw/sub2api - * License: LGPL-3.0-or-later, Copyright (c) 2026 Wesley Liddick. - */ -import { - CLAUDE_CODE_BILLING_HEADER_PREFIX, - CLAUDE_CODE_COMPAT_VERSION, - formatClaudeCodeBillingHeader, -} from '../../constants/claudeCodeCompatibility.js' -import { computeFingerprint } from '../../utils/fingerprint.js' -import type { AnthropicRequest } from './transform/types.js' - -export function extractFirstUserText(body: AnthropicRequest): string { - for (const message of body.messages) { - if (message.role !== 'user') continue - - if (typeof message.content === 'string') { - return message.content - } - - const textBlock = message.content.find(block => block.type === 'text') - return textBlock?.type === 'text' ? textBlock.text : '' - } - - return '' -} - -export function ensureClaudeCodeAttribution(body: AnthropicRequest): AnthropicRequest { - if (hasBillingAttribution(body.system)) return body - - const fingerprint = computeFingerprint(extractFirstUserText(body), CLAUDE_CODE_COMPAT_VERSION) - const billingBlock = { - type: 'text' as const, - text: formatClaudeCodeBillingHeader({ - fingerprint, - entrypoint: process.env.CLAUDE_CODE_ENTRYPOINT ?? 'unknown', - }), - } - - return { - ...body, - system: typeof body.system === 'string' - ? [billingBlock, { type: 'text', text: body.system }] - : [billingBlock, ...(body.system ?? [])], - } -} - -function hasBillingAttribution(system: AnthropicRequest['system']): boolean { - if (typeof system === 'string') { - return system.startsWith(CLAUDE_CODE_BILLING_HEADER_PREFIX) - } - - return Array.isArray(system) && system.some(block => ( - block?.type === 'text' && typeof block.text === 'string' && block.text.startsWith(CLAUDE_CODE_BILLING_HEADER_PREFIX) - )) -} diff --git a/src/server/proxy/handler.ts b/src/server/proxy/handler.ts index 636409eb..b422d8d7 100644 --- a/src/server/proxy/handler.ts +++ b/src/server/proxy/handler.ts @@ -9,9 +9,8 @@ * Original work by Jason Young, MIT License */ -import { signClaudeCodeCCHInTransformedString } from '../../utils/claudeCodeCch.js' import { ProviderService } from '../services/providerService.js' -import { ensureClaudeCodeAttribution } from './claudeCodeAttribution.js' +import { resolvePromptCacheKey } from './promptCacheKey.js' import { anthropicToOpenaiChat } from './transform/anthropicToOpenaiChat.js' import { anthropicToOpenaiResponses } from './transform/anthropicToOpenaiResponses.js' import { openaiChatToAnthropic } from './transform/openaiChatToAnthropic.js' @@ -212,22 +211,23 @@ export async function handleProxyRequest(req: Request, url: URL): Promise { - const transformed = anthropicToOpenaiResponses(body) + const transformed = anthropicToOpenaiResponses(body, { cacheKey: promptCacheKey }) const url = `${baseUrl}/v1/responses` const proxyOptions = getProxyFetchOptions({ proxyUrl }) const startedAtMs = Date.now() @@ -454,7 +455,7 @@ async function handleOpenaiResponses( 'Content-Type': 'application/json', Authorization: `Bearer ${apiKey}`, }, - body: signClaudeCodeCCHInTransformedString(JSON.stringify(transformed)), + body: JSON.stringify(transformed), ...proxyOptions, }, aiRequestTimeoutMs, isStream) } catch (err) { diff --git a/src/server/proxy/promptCacheKey.ts b/src/server/proxy/promptCacheKey.ts new file mode 100644 index 00000000..bbd02ca4 --- /dev/null +++ b/src/server/proxy/promptCacheKey.ts @@ -0,0 +1,44 @@ +/** + * Stable prompt cache key resolution for OpenAI-compatible upstreams. + * Derived from cc-switch (https://github.com/farion1231/cc-switch) + * Original work by Jason Young, MIT License + */ + +import type { AnthropicRequest } from './transform/types.js' + +const SESSION_MARKER = '_session_' + +/** + * Resolve a stable `prompt_cache_key` for the OpenAI Responses API. + * + * OpenAI prompt caching routes on this key, so all requests of one + * conversation must share it. Sources in order: the `_session_` suffix of + * `metadata.user_id` (what Claude Code sends), `metadata.session_id`, then + * the CLI's `x-claude-code-session-id` header. Returns undefined when no + * client session identity exists — never a generated UUID, and never + * `previous_response_id`, which is a per-turn response cursor and would + * rotate the key every request. + */ +export function resolvePromptCacheKey( + body: AnthropicRequest, + headerSessionId?: string | null, +): string | undefined { + const userId = body.metadata?.user_id + if (typeof userId === 'string') { + const markerIndex = userId.indexOf(SESSION_MARKER) + if (markerIndex !== -1) { + const sessionId = userId.slice(markerIndex + SESSION_MARKER.length) + if (sessionId) return sessionId + } + } + + const metadataSessionId = body.metadata?.session_id + if (typeof metadataSessionId === 'string' && metadataSessionId) { + return metadataSessionId + } + + const trimmedHeader = headerSessionId?.trim() + if (trimmedHeader) return trimmedHeader + + return undefined +} diff --git a/src/server/proxy/streaming/openaiChatStreamToAnthropic.ts b/src/server/proxy/streaming/openaiChatStreamToAnthropic.ts index 5fc13fe8..c022fa1e 100644 --- a/src/server/proxy/streaming/openaiChatStreamToAnthropic.ts +++ b/src/server/proxy/streaming/openaiChatStreamToAnthropic.ts @@ -22,6 +22,7 @@ import type { OpenAIChatStreamChunk } from '../transform/types.js' import { stringifyOpenAIToolArguments } from '../transform/toolArguments.js' +import { openaiUsageToAnthropic } from '../transform/usage.js' // ─── Types ───────────────────────────────────────────────── @@ -471,7 +472,7 @@ function handleFinishReason( const stopReason = mapFinishReason(finishReason) const usage = chunk.usage - ? { output_tokens: chunk.usage.completion_tokens || 0 } + ? openaiUsageToAnthropic(chunk.usage) : { output_tokens: 0 } const messageDelta: SseEvent = { @@ -500,7 +501,7 @@ function mergeUsageIntoHeldDelta( if (!state.heldMessageDelta) return const data = state.heldMessageDelta.data as Record - data.usage = { output_tokens: usage.completion_tokens || 0 } + data.usage = openaiUsageToAnthropic(usage) state.messageDeltaSent = true state.queue.push(state.heldMessageDelta) state.heldMessageDelta = null diff --git a/src/server/proxy/streaming/openaiResponsesStreamToAnthropic.ts b/src/server/proxy/streaming/openaiResponsesStreamToAnthropic.ts index 8721c037..ef360185 100644 --- a/src/server/proxy/streaming/openaiResponsesStreamToAnthropic.ts +++ b/src/server/proxy/streaming/openaiResponsesStreamToAnthropic.ts @@ -5,6 +5,8 @@ */ import { stringifyOpenAIToolArguments } from '../transform/toolArguments.js' +import { openaiUsageToAnthropic } from '../transform/usage.js' +import type { OpenAICompatibleUsage } from '../transform/types.js' type StreamState = { nextContentIndex: number @@ -257,7 +259,7 @@ function processEvent( case 'response.completed': { const response = data.response as Record | undefined const status = (response?.status as string) || 'completed' - const usage = response?.usage as Record | undefined + const usage = response?.usage as OpenAICompatibleUsage | undefined const hasToolUse = state.toolIndexByItemId.size > 0 const stopReason = status === 'completed' @@ -267,10 +269,7 @@ function processEvent( controller.enqueue(encoder.encode(formatSse('message_delta', { type: 'message_delta', delta: { stop_reason: stopReason, stop_sequence: null }, - usage: { - input_tokens: usage?.input_tokens ?? 0, - output_tokens: usage?.output_tokens ?? 0, - }, + usage: openaiUsageToAnthropic(usage), }))) if (!state.messageStopped) { state.messageStopped = true diff --git a/src/server/proxy/transform/anthropicToOpenaiChat.ts b/src/server/proxy/transform/anthropicToOpenaiChat.ts index 01a14385..f1dcf012 100644 --- a/src/server/proxy/transform/anthropicToOpenaiChat.ts +++ b/src/server/proxy/transform/anthropicToOpenaiChat.ts @@ -14,6 +14,7 @@ import type { OpenAIToolCall, OpenAITool, } from './types.js' +import { stripLeadingBillingHeader } from './billingHeader.js' type OpenAIChatImageContentMode = 'vision' | 'text_only' @@ -34,12 +35,14 @@ export function anthropicToOpenaiChat( ): OpenAIChatRequest { const messages: OpenAIChatMessage[] = [] - // Convert system prompt + // Convert system prompt, minus the leading billing attribution: its + // rotating cch= signature would change the prefix every turn and defeat + // upstream prompt caching. if (body.system) { - if (typeof body.system === 'string') { - messages.push({ role: 'system', content: body.system }) - } else if (Array.isArray(body.system)) { - const text = body.system.map((b) => b.text).join('\n') + const text = typeof body.system === 'string' + ? stripLeadingBillingHeader(body.system) + : body.system.map((b) => stripLeadingBillingHeader(b.text)).filter(Boolean).join('\n') + if (text) { messages.push({ role: 'system', content: text }) } } @@ -56,6 +59,11 @@ export function anthropicToOpenaiChat( stream: body.stream === true, } + // Many OpenAI-compatible servers omit usage on streams unless asked. + if (result.stream) { + result.stream_options = { include_usage: true } + } + // max_tokens — omit to let upstream provider use its own default/max. // Claude Code sends very large values (e.g. 128K) that exceed many // providers' limits (DeepSeek: 8192, etc.). diff --git a/src/server/proxy/transform/anthropicToOpenaiResponses.ts b/src/server/proxy/transform/anthropicToOpenaiResponses.ts index 1a9e31d6..816f16b7 100644 --- a/src/server/proxy/transform/anthropicToOpenaiResponses.ts +++ b/src/server/proxy/transform/anthropicToOpenaiResponses.ts @@ -12,11 +12,20 @@ import type { OpenAIResponsesInputItem, OpenAIChatContentPart, } from './types.js' +import { stripLeadingBillingHeader } from './billingHeader.js' + +export type OpenAIResponsesTransformOptions = { + /** Stable cache routing key, forwarded as `prompt_cache_key`. */ + cacheKey?: string +} /** * Convert Anthropic Messages request to OpenAI Responses API request. */ -export function anthropicToOpenaiResponses(body: AnthropicRequest): OpenAIResponsesRequest { +export function anthropicToOpenaiResponses( + body: AnthropicRequest, + options: OpenAIResponsesTransformOptions = {}, +): OpenAIResponsesRequest { const input: OpenAIResponsesInputItem[] = [] // Convert messages to input items @@ -31,15 +40,22 @@ export function anthropicToOpenaiResponses(body: AnthropicRequest): OpenAIRespon store: false, } - // system → instructions + // system → instructions, minus the leading billing attribution: its + // rotating cch= signature would change the prefix every turn and defeat + // upstream prompt caching. if (body.system) { - if (typeof body.system === 'string') { - result.instructions = body.system - } else if (Array.isArray(body.system)) { - result.instructions = body.system.map((b) => b.text).join('\n') + const instructions = typeof body.system === 'string' + ? stripLeadingBillingHeader(body.system) + : body.system.map((b) => stripLeadingBillingHeader(b.text)).filter(Boolean).join('\n') + if (instructions) { + result.instructions = instructions } } + if (options.cacheKey) { + result.prompt_cache_key = options.cacheKey + } + // max_tokens — omit to let upstream provider use its own default/max. // Claude Code sends very large values that exceed many providers' limits. diff --git a/src/server/proxy/transform/billingHeader.ts b/src/server/proxy/transform/billingHeader.ts new file mode 100644 index 00000000..448c7cf3 --- /dev/null +++ b/src/server/proxy/transform/billingHeader.ts @@ -0,0 +1,33 @@ +/** + * Leading billing-attribution stripping for OpenAI-compatible transforms. + * Derived from cc-switch (https://github.com/farion1231/cc-switch) + * Original work by Jason Young, MIT License + */ + +import { CLAUDE_CODE_BILLING_HEADER_PREFIX } from '../../../constants/claudeCodeCompatibility.js' + +/** + * Strip a leading Claude Code billing attribution line from system text. + * + * The CLI can prepend dynamic `x-anthropic-billing-header:` metadata to + * `system`. Forwarding it into OpenAI Chat system messages or Responses + * `instructions` rotates the prompt prefix on every request (the `cch=` + * signature hashes the whole body), which defeats upstream prefix caching. + * Only the leading occurrence is removed so user-authored prompt text that + * happens to mention the header is kept. + */ +export function stripLeadingBillingHeader(text: string): string { + if (!text.startsWith(CLAUDE_CODE_BILLING_HEADER_PREFIX)) return text + + const lineEnd = text.search(/[\r\n]/) + if (lineEnd === -1) return '' + + let restStart = lineEnd + 1 + if (text[lineEnd] === '\r' && text[restStart] === '\n') restStart++ + + // Also consume the blank separator line that usually follows the header. + const rest = text.slice(restStart) + if (rest.startsWith('\r\n')) return rest.slice(2) + if (rest.startsWith('\n') || rest.startsWith('\r')) return rest.slice(1) + return rest +} diff --git a/src/server/proxy/transform/openaiChatToAnthropic.ts b/src/server/proxy/transform/openaiChatToAnthropic.ts index f405525b..e8df9e26 100644 --- a/src/server/proxy/transform/openaiChatToAnthropic.ts +++ b/src/server/proxy/transform/openaiChatToAnthropic.ts @@ -10,6 +10,7 @@ import type { AnthropicContentBlock, } from './types.js' import { parseOpenAIToolArguments } from './toolArguments.js' +import { openaiUsageToAnthropic } from './usage.js' /** * Convert OpenAI Chat Completions response to Anthropic Messages response. @@ -72,7 +73,7 @@ export function openaiChatToAnthropic(response: OpenAIChatResponse, model: strin model: response.model || model, stop_reason: mapFinishReason(choice.finish_reason), stop_sequence: null, - usage: mapUsage(response.usage), + usage: openaiUsageToAnthropic(response.usage), } } @@ -86,17 +87,6 @@ function mapFinishReason(reason: string | null): string { } } -function mapUsage(usage?: OpenAIChatResponse['usage']): AnthropicResponse['usage'] { - if (!usage) { - return { input_tokens: 0, output_tokens: 0 } - } - return { - input_tokens: usage.prompt_tokens || 0, - output_tokens: usage.completion_tokens || 0, - cache_read_input_tokens: usage.prompt_tokens_details?.cached_tokens || 0, - } -} - function createEmptyResponse(response: OpenAIChatResponse, model: string): AnthropicResponse { return { id: response.id || `msg_${Date.now()}`, @@ -106,6 +96,6 @@ function createEmptyResponse(response: OpenAIChatResponse, model: string): Anthr model: response.model || model, stop_reason: 'end_turn', stop_sequence: null, - usage: mapUsage(response.usage), + usage: openaiUsageToAnthropic(response.usage), } } diff --git a/src/server/proxy/transform/openaiResponsesToAnthropic.ts b/src/server/proxy/transform/openaiResponsesToAnthropic.ts index d924abd4..bb1f56ee 100644 --- a/src/server/proxy/transform/openaiResponsesToAnthropic.ts +++ b/src/server/proxy/transform/openaiResponsesToAnthropic.ts @@ -11,6 +11,7 @@ import type { AnthropicContentBlock, } from './types.js' import { parseOpenAIToolArguments } from './toolArguments.js' +import { openaiUsageToAnthropic } from './usage.js' /** * Convert OpenAI Responses API response to Anthropic Messages response. @@ -37,10 +38,7 @@ export function openaiResponsesToAnthropic(response: OpenAIResponsesResponse, mo model: response.model || model, stop_reason: mapStatus(response.status, hasToolUse), stop_sequence: null, - usage: { - input_tokens: response.usage?.input_tokens ?? response.usage?.prompt_tokens ?? 0, - output_tokens: response.usage?.output_tokens ?? response.usage?.completion_tokens ?? 0, - }, + usage: openaiUsageToAnthropic(response.usage), } } diff --git a/src/server/proxy/transform/types.ts b/src/server/proxy/transform/types.ts index fd6e6de9..d5a5d6a8 100644 --- a/src/server/proxy/transform/types.ts +++ b/src/server/proxy/transform/types.ts @@ -46,12 +46,31 @@ export type OpenAIChatRequest = { top_p?: number stop?: string | string[] stream?: boolean + stream_options?: { include_usage: boolean } tools?: OpenAITool[] tool_choice?: unknown reasoning_effort?: 'low' | 'medium' | 'high' thinking?: { type: string } } +/** + * Usage shape accepted from OpenAI-compatible upstreams. + * Responses API uses input_tokens/input_tokens_details, Chat Completions uses + * prompt_tokens/prompt_tokens_details, and some compatible servers return + * Anthropic-style cache fields directly. + */ +export type OpenAICompatibleUsage = { + input_tokens?: number + output_tokens?: number + prompt_tokens?: number + completion_tokens?: number + total_tokens?: number + input_tokens_details?: { cached_tokens?: number } + prompt_tokens_details?: { cached_tokens?: number } + cache_read_input_tokens?: number + cache_creation_input_tokens?: number +} + export type OpenAIChatResponse = { id: string object: string @@ -66,14 +85,7 @@ export type OpenAIChatResponse = { } finish_reason: string | null }> - usage?: { - prompt_tokens: number - completion_tokens: number - total_tokens: number - prompt_tokens_details?: { - cached_tokens?: number - } - } + usage?: OpenAICompatibleUsage } export type OpenAIChatStreamChunk = { @@ -125,6 +137,7 @@ export type OpenAIResponsesRequest = { }> tool_choice?: unknown reasoning?: { effort?: 'low' | 'medium' | 'high' } + prompt_cache_key?: string } export type OpenAIResponsesOutputItem = @@ -139,11 +152,7 @@ export type OpenAIResponsesResponse = { model: string status: string output: OpenAIResponsesOutputItem[] - usage?: { - input_tokens: number - output_tokens: number - total_tokens: number - } + usage?: OpenAICompatibleUsage } // ─── Anthropic Types (subset used by transforms) ─────────── @@ -164,6 +173,7 @@ export type AnthropicRequest = { model: string system?: string | Array<{ type: 'text'; text: string; cache_control?: unknown }> messages: AnthropicMessage[] + metadata?: { user_id?: string; session_id?: string } max_tokens: number temperature?: number top_p?: number diff --git a/src/server/proxy/transform/usage.ts b/src/server/proxy/transform/usage.ts new file mode 100644 index 00000000..c52e787c --- /dev/null +++ b/src/server/proxy/transform/usage.ts @@ -0,0 +1,41 @@ +/** + * Usage mapping: OpenAI-compatible → Anthropic Messages semantics. + * Derived from cc-switch (https://github.com/farion1231/cc-switch) + * Original work by Jason Young, MIT License + */ + +import type { AnthropicResponse, OpenAICompatibleUsage } from './types.js' + +/** + * Map an OpenAI-compatible usage object to Anthropic usage. + * + * Cache reads come from Anthropic-style `cache_read_input_tokens` when a + * compatible server returns it directly (authoritative), else from the + * Responses API's `input_tokens_details.cached_tokens`, else from Chat + * Completions' `prompt_tokens_details.cached_tokens`. + * + * OpenAI input counts INCLUDE cached tokens while Anthropic `input_tokens` + * excludes them, so cache reads/creations are subtracted to keep the + * Anthropic invariant: input + cache_read + cache_creation == upstream input. + */ +export function openaiUsageToAnthropic(usage: OpenAICompatibleUsage | undefined): AnthropicResponse['usage'] { + if (!usage) return { input_tokens: 0, output_tokens: 0 } + + const input = usage.input_tokens ?? usage.prompt_tokens ?? 0 + const output = usage.output_tokens ?? usage.completion_tokens ?? 0 + const cacheRead = usage.cache_read_input_tokens + ?? usage.input_tokens_details?.cached_tokens + ?? usage.prompt_tokens_details?.cached_tokens + ?? 0 + const cacheCreation = usage.cache_creation_input_tokens ?? 0 + + const result: AnthropicResponse['usage'] = { + input_tokens: cacheRead > 0 || cacheCreation > 0 + ? Math.max(0, input - cacheRead - cacheCreation) + : input, + output_tokens: output, + } + if (cacheRead > 0) result.cache_read_input_tokens = cacheRead + if (cacheCreation > 0) result.cache_creation_input_tokens = cacheCreation + return result +} diff --git a/src/utils/claudeCodeCch.test.ts b/src/utils/claudeCodeCch.test.ts index ecc48d86..39e600aa 100644 --- a/src/utils/claudeCodeCch.test.ts +++ b/src/utils/claudeCodeCch.test.ts @@ -1,5 +1,5 @@ import { describe, expect, test } from 'bun:test' -import { signClaudeCodeCCHInString, signClaudeCodeCCHInTransformedString, xxHash64Seeded } from './claudeCodeCch.js' +import { signClaudeCodeCCHInString, xxHash64Seeded } from './claudeCodeCch.js' const encoder = new TextEncoder() @@ -34,9 +34,4 @@ describe('signClaudeCodeCCHInString', () => { const body = '{"system":[{"type":"text","text":"x-anthropic-billing-header: cc_version=2.1.92.abc; cc_entrypoint=cli; cch=00000;"}],"messages":[{"role":"user","content":"literal x-anthropic-billing-header: cc_version=2.1.92.user; cc_entrypoint=cli; cch=00000;"}]}' expect(signClaudeCodeCCHInString(body)).toBe(body) }) - - test('does not partially sign transformed body when user text also contains a placeholder', () => { - const body = '{"messages":[{"role":"system","content":"x-anthropic-billing-header: cc_version=2.1.92.abc; cc_entrypoint=cli; cch=00000;"},{"role":"user","content":"literal x-anthropic-billing-header: cc_version=2.1.92.user; cc_entrypoint=cli; cch=00000;"}]}' - expect(signClaudeCodeCCHInTransformedString(body)).toBe(body) - }) }) diff --git a/src/utils/claudeCodeCch.ts b/src/utils/claudeCodeCch.ts index fbab87f6..8cdf368a 100644 --- a/src/utils/claudeCodeCch.ts +++ b/src/utils/claudeCodeCch.ts @@ -18,20 +18,7 @@ const PRIME64_5 = 2870177450012600261n const encoder = new TextEncoder() const decoder = new TextDecoder() -type BillingFieldSelector = (parsed: unknown) => string[] - export function signClaudeCodeCCHInString(body: string): string { - return signClaudeCodeCCHInStringWithSelector(body, selectAnthropicBillingFields) -} - -export function signClaudeCodeCCHInTransformedString(body: string): string { - return signClaudeCodeCCHInStringWithSelector(body, selectTransformedBillingFields) -} - -function signClaudeCodeCCHInStringWithSelector( - body: string, - selectBillingFields: BillingFieldSelector, -): string { if (!body.includes(CLAUDE_CODE_BILLING_HEADER_PREFIX) || !body.includes(CCH_PLACEHOLDER)) return body let parsed: unknown @@ -43,7 +30,7 @@ function signClaudeCodeCCHInStringWithSelector( if (countCCHPlaceholders(body) !== 1) return body - const billingFields = selectBillingFields(parsed) + const billingFields = selectAnthropicBillingFields(parsed) const placeholderCount = billingFields.reduce((count, value) => count + countCCHPlaceholders(value), 0) if (placeholderCount !== 1) return body @@ -86,26 +73,6 @@ function selectAnthropicBillingFields(parsed: unknown): string[] { )) } -function selectTransformedBillingFields(parsed: unknown): string[] { - if (!isRecord(parsed)) return [] - - const fields: string[] = [] - if (typeof parsed.instructions === 'string' && isBillingHeader(parsed.instructions)) { - fields.push(parsed.instructions) - } - - if (Array.isArray(parsed.messages)) { - for (const message of parsed.messages) { - if (!isRecord(message)) continue - if (message.role === 'system' && typeof message.content === 'string' && isBillingHeader(message.content)) { - fields.push(message.content) - } - } - } - - return fields -} - function isBillingHeader(value: string): boolean { return value.startsWith(CLAUDE_CODE_BILLING_HEADER_PREFIX) }