From 99caf8fdf57b9c1363ddb7fff3c2b91a1f06ebac Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=E7=A8=8B=E5=BA=8F=E5=91=98=E9=98=BF=E6=B1=9F=28Relakkes?= =?UTF-8?q?=29?= Date: Mon, 1 Jun 2026 00:11:33 +0800 Subject: [PATCH] fix: avoid DeepSeek OpenAI image_url rejection DeepSeek's OpenAI-compatible chat schema accepts text content rather than OpenAI vision content parts, so forwarding Anthropic image blocks as image_url makes the upstream reject the request before the model can answer. Keep normal OpenAI Chat vision behavior as the default, but switch known DeepSeek-compatible chat proxy requests to a text-only conversion that preserves prompt text and replaces image payloads with a small omission marker. Constraint: DeepSeek official Chat Completions docs list user message content as string text, while its Anthropic compatibility table marks image blocks as not supported. Rejected: Expand unsupported-image error string matching only | it still surfaces the first turn as an API error and depends on provider wording. Confidence: high Scope-risk: narrow Tested: bun test src/server/__tests__/proxy-transform.test.ts Tested: bun test src/server/__tests__/providers.test.ts --test-name-pattern "handleProxyRequest|omits image_url" Tested: bun test src/services/api/errors.test.ts tests/mediaRecoveryAndEstimation.test.ts Tested: bun run check:server --- src/server/__tests__/providers.test.ts | 61 +++++++++++++++++++ src/server/__tests__/proxy-transform.test.ts | 20 ++++++ src/server/proxy/handler.ts | 5 ++ .../proxy/transform/anthropicToOpenaiChat.ts | 47 +++++++++++--- 4 files changed, 125 insertions(+), 8 deletions(-) diff --git a/src/server/__tests__/providers.test.ts b/src/server/__tests__/providers.test.ts index 8d215bbd..6ed3ecae 100644 --- a/src/server/__tests__/providers.test.ts +++ b/src/server/__tests__/providers.test.ts @@ -993,6 +993,67 @@ describe('ProviderService', () => { else process.env.CLAUDE_CODE_ENTRYPOINT = originalEntrypoint } }) + + test('omits image_url parts for DeepSeek OpenAI Chat proxy requests', async () => { + const originalFetch = globalThis.fetch + const calls: Array<{ body: Record }> = [] + globalThis.fetch = mock(async (_url: string | URL | Request, init?: RequestInit) => { + calls.push({ body: JSON.parse(String(init?.body)) as Record }) + return new Response(JSON.stringify({ + id: 'chatcmpl-1', + object: 'chat.completion', + created: 0, + model: 'deepseek-v4-pro', + choices: [{ index: 0, message: { role: 'assistant', content: 'I cannot view images.' }, finish_reason: 'stop' }], + usage: { prompt_tokens: 1, completion_tokens: 1, total_tokens: 2 }, + }), { + status: 200, + headers: { 'Content-Type': 'application/json' }, + }) + }) as typeof fetch + + try { + const svc = new ProviderService() + const provider = await svc.addProvider(sampleInput({ + apiFormat: 'openai_chat', + baseUrl: 'https://api.deepseek.com', + models: { + main: 'deepseek-v4-pro', + haiku: 'deepseek-v4-pro', + sonnet: 'deepseek-v4-pro', + opus: 'deepseek-v4-pro', + }, + })) + await svc.activateProvider(provider.id) + + const req = new Request('http://localhost:3456/proxy/v1/messages', { + method: 'POST', + headers: { 'Content-Type': 'application/json' }, + body: JSON.stringify({ + model: 'deepseek-v4-pro', + max_tokens: 64, + messages: [{ + role: 'user', + content: [ + { type: 'text', text: 'What is in this screenshot?' }, + { type: 'image', source: { type: 'base64', media_type: 'image/png', data: 'abc123' } }, + ], + }], + }), + }) + + const res = await handleProxyRequest(req, new URL(req.url)) + expect(res.status).toBe(200) + + const serialized = JSON.stringify(calls[0].body) + expect(serialized).not.toContain('image_url') + expect(serialized).not.toContain('abc123') + expect(serialized).toContain('What is in this screenshot?') + expect(serialized).toContain('Image omitted') + } finally { + globalThis.fetch = originalFetch + } + }) }) describe('testProvider', () => { diff --git a/src/server/__tests__/proxy-transform.test.ts b/src/server/__tests__/proxy-transform.test.ts index 26433ee5..657bd34e 100644 --- a/src/server/__tests__/proxy-transform.test.ts +++ b/src/server/__tests__/proxy-transform.test.ts @@ -232,6 +232,26 @@ describe('anthropicToOpenaiChat', () => { expect(content[0].type).toBe('image_url') expect(content[0].image_url!.url).toBe('data:image/png;base64,abc123') }) + + test('text-only chat endpoints omit image payloads instead of emitting image_url parts', () => { + const req: AnthropicRequest = { + model: 'deepseek-v4-pro', + max_tokens: 100, + messages: [{ + role: 'user', + content: [ + { type: 'text', text: 'What is in this screenshot?' }, + { type: 'image', source: { type: 'base64', media_type: 'image/png', data: 'abc123' } }, + ], + }], + } + const result = anthropicToOpenaiChat(req, { imageContentMode: 'text_only' }) + expect(result.messages[0].content).toBe( + 'What is in this screenshot?\n[Image omitted: this OpenAI-compatible chat endpoint only supports text content.]', + ) + expect(JSON.stringify(result)).not.toContain('image_url') + expect(JSON.stringify(result)).not.toContain('abc123') + }) }) // ─── openaiChatToAnthropic ────────────────────────────────────── diff --git a/src/server/proxy/handler.ts b/src/server/proxy/handler.ts index cb0fc7e4..5a8112de 100644 --- a/src/server/proxy/handler.ts +++ b/src/server/proxy/handler.ts @@ -167,6 +167,7 @@ async function handleOpenaiChat( const transformed = anthropicToOpenaiChat(body, { roundTripReasoningContent: deepSeekCompatible, passThinkingToggle: deepSeekCompatible, + imageContentMode: shouldUseTextOnlyOpenAIChatContent(baseUrl) ? 'text_only' : 'vision', }) const url = `${baseUrl}/v1/chat/completions` const proxyOptions = getProxyFetchOptions({ proxyUrl }) @@ -226,6 +227,10 @@ function shouldUseDeepSeekReasoningCompat(baseUrl: string): boolean { ) } +function shouldUseTextOnlyOpenAIChatContent(baseUrl: string): boolean { + return shouldUseDeepSeekReasoningCompat(baseUrl) +} + async function handleOpenaiResponses( body: AnthropicRequest, baseUrl: string, diff --git a/src/server/proxy/transform/anthropicToOpenaiChat.ts b/src/server/proxy/transform/anthropicToOpenaiChat.ts index a637df58..123589b6 100644 --- a/src/server/proxy/transform/anthropicToOpenaiChat.ts +++ b/src/server/proxy/transform/anthropicToOpenaiChat.ts @@ -15,12 +15,22 @@ import type { OpenAITool, } from './types.js' +type OpenAIChatImageContentMode = 'vision' | 'text_only' + +type OpenAIChatTransformOptions = { + roundTripReasoningContent?: boolean + passThinkingToggle?: boolean + imageContentMode?: OpenAIChatImageContentMode +} + +const OMITTED_IMAGE_TEXT = '[Image omitted: this OpenAI-compatible chat endpoint only supports text content.]' + /** * Convert Anthropic Messages request to OpenAI Chat Completions request. */ export function anthropicToOpenaiChat( body: AnthropicRequest, - options: { roundTripReasoningContent?: boolean; passThinkingToggle?: boolean } = {}, + options: OpenAIChatTransformOptions = {}, ): OpenAIChatRequest { const messages: OpenAIChatMessage[] = [] @@ -99,7 +109,7 @@ export function anthropicToOpenaiChat( function convertMessage( msg: AnthropicMessage, output: OpenAIChatMessage[], - options: { roundTripReasoningContent?: boolean }, + options: OpenAIChatTransformOptions, ): void { const content = msg.content @@ -116,22 +126,35 @@ function convertMessage( } if (msg.role === 'user') { - convertUserMessage(content, output) + convertUserMessage(content, output, options.imageContentMode ?? 'vision') } else { convertAssistantMessage(content, output, options) } } -function convertUserMessage(blocks: AnthropicContentBlock[], output: OpenAIChatMessage[]): void { +function convertUserMessage( + blocks: AnthropicContentBlock[], + output: OpenAIChatMessage[], + imageContentMode: OpenAIChatImageContentMode, +): void { // Separate tool_result blocks from other content const contentParts: OpenAIChatContentPart[] = [] + const textOnlyParts: string[] = [] for (const block of blocks) { if (block.type === 'text') { - contentParts.push({ type: 'text', text: block.text }) + if (imageContentMode === 'text_only') { + textOnlyParts.push(block.text) + } else { + contentParts.push({ type: 'text', text: block.text }) + } } else if (block.type === 'image') { - const url = `data:${block.source.media_type};base64,${block.source.data}` - contentParts.push({ type: 'image_url', image_url: { url } }) + if (imageContentMode === 'text_only') { + textOnlyParts.push(OMITTED_IMAGE_TEXT) + } else { + const url = `data:${block.source.media_type};base64,${block.source.data}` + contentParts.push({ type: 'image_url', image_url: { url } }) + } } else if (block.type === 'tool_result') { // tool_result → separate tool message const resultContent = typeof block.content === 'string' @@ -147,7 +170,15 @@ function convertUserMessage(blocks: AnthropicContentBlock[], output: OpenAIChatM } } - if (contentParts.length > 0) { + if (imageContentMode === 'text_only') { + const content = textOnlyParts.filter(Boolean).join('\n') + if (content) { + output.push({ + role: 'user', + content, + }) + } + } else if (contentParts.length > 0) { output.push({ role: 'user', content: contentParts.length === 1 && contentParts[0].type === 'text'