Merge pull request #1053 from leehom0123/fix/openai-responses-read-image

fix(proxy): 保留 Responses 工具结果中的图片

src/server/__tests__/proxy-transform.test.ts

@@ -554,12 +554,151 @@ describe('anthropicToOpenaiResponses', () => {
       }],
     }
     const result = anthropicToOpenaiResponses(req)
-    const fco = result.input.find((i) => i.type === 'function_call_output')
-    expect(fco).toBeDefined()
-    if (fco && fco.type === 'function_call_output') {
-      expect(fco.call_id).toBe('tc_1')
-      expect(fco.output).toBe('found it')
+    expect(result.input).toEqual([{
+      type: 'function_call_output',
+      call_id: 'tc_1',
+      output: 'found it',
+    }])
+  })
+
+  test('preserves text-only tool_result arrays as strings', () => {
+    const req: AnthropicRequest = {
+      model: 'gpt-4o',
+      max_tokens: 100,
+      messages: [{
+        role: 'user',
+        content: [{
+          type: 'tool_result',
+          tool_use_id: 'tc_2',
+          content: [
+            { type: 'text', text: 'first' },
+            { type: 'text', text: 'second' },
+          ],
+        }],
+      }],
     }
+
+    const result = anthropicToOpenaiResponses(req)
+
+    expect(result.input).toEqual([{
+      type: 'function_call_output',
+      call_id: 'tc_2',
+      output: 'first\nsecond',
+    }])
+  })
+
+  test('preserves image-only tool_result content', () => {
+    const req: AnthropicRequest = {
+      model: 'gpt-4o',
+      max_tokens: 100,
+      messages: [{
+        role: 'user',
+        content: [{
+          type: 'tool_result',
+          tool_use_id: 'read_1',
+          content: [{
+            type: 'image',
+            source: { type: 'base64', media_type: 'image/png', data: 'AAECAwQ=' },
+          }],
+        }],
+      }],
+    }
+
+    const result = anthropicToOpenaiResponses(req)
+
+    expect(result.input).toEqual([{
+      type: 'function_call_output',
+      call_id: 'read_1',
+      output: [{
+        type: 'input_image',
+        image_url: 'data:image/png;base64,AAECAwQ=',
+      }],
+    }])
+  })
+
+  test('preserves mixed tool_result content in order', () => {
+    const req: AnthropicRequest = {
+      model: 'gpt-4o',
+      max_tokens: 100,
+      messages: [{
+        role: 'user',
+        content: [{
+          type: 'tool_result',
+          tool_use_id: 'read_2',
+          content: [
+            { type: 'text', text: 'before' },
+            {
+              type: 'image',
+              source: { type: 'base64', media_type: 'image/jpeg', data: '/9j/AA==' },
+            },
+            { type: 'text', text: 'after' },
+          ],
+        }],
+      }],
+    }
+
+    const result = anthropicToOpenaiResponses(req)
+
+    expect(result.input).toEqual([{
+      type: 'function_call_output',
+      call_id: 'read_2',
+      output: [
+        { type: 'input_text', text: 'before' },
+        { type: 'input_image', image_url: 'data:image/jpeg;base64,/9j/AA==' },
+        { type: 'input_text', text: 'after' },
+      ],
+    }])
+  })
+
+  test('preserves message and tool_result item order', () => {
+    const req: AnthropicRequest = {
+      model: 'gpt-4o',
+      max_tokens: 100,
+      messages: [{
+        role: 'user',
+        content: [
+          { type: 'text', text: 'before result' },
+          { type: 'tool_result', tool_use_id: 'read_3', content: 'result' },
+          { type: 'text', text: 'after result' },
+        ],
+      }],
+    }
+
+    const result = anthropicToOpenaiResponses(req)
+
+    expect(result.input).toEqual([
+      { type: 'message', role: 'user', content: 'before result' },
+      { type: 'function_call_output', call_id: 'read_3', output: 'result' },
+      { type: 'message', role: 'user', content: 'after result' },
+    ])
+  })
+
+  test('preserves mixed text and image message content', () => {
+    const req: AnthropicRequest = {
+      model: 'gpt-4o',
+      max_tokens: 100,
+      messages: [{
+        role: 'user',
+        content: [
+          { type: 'text', text: 'Describe this image' },
+          {
+            type: 'image',
+            source: { type: 'base64', media_type: 'image/png', data: 'AAECAwQ=' },
+          },
+        ],
+      }],
+    }
+
+    const result = anthropicToOpenaiResponses(req)
+
+    expect(result.input).toEqual([{
+      type: 'message',
+      role: 'user',
+      content: [
+        { type: 'input_text', text: 'Describe this image' },
+        { type: 'input_image', image_url: 'data:image/png;base64,AAECAwQ=' },
+      ],
+    }])
   })
 
   test('thinking → reasoning', () => {

src/server/proxy/transform/anthropicToOpenaiResponses.ts

@@ -10,7 +10,7 @@ import type {
   AnthropicMessage,
   OpenAIResponsesRequest,
   OpenAIResponsesInputItem,
-  OpenAIChatContentPart,
+  OpenAIResponsesInputContentPart,
 } from './types.js'
 import { stripLeadingBillingHeader } from './billingHeader.js'
 import { normalizeOpenAIReasoningEffort } from './effort.js'
@@ -116,6 +116,19 @@ export function anthropicToOpenaiResponses(
   return result
 }
 
+function convertContentBlock(
+  block: Extract<AnthropicContentBlock, { type: 'text' | 'image' }>,
+): OpenAIResponsesInputContentPart {
+  if (block.type === 'text') {
+    return { type: 'input_text', text: block.text }
+  }
+
+  return {
+    type: 'input_image',
+    image_url: `data:${block.source.media_type};base64,${block.source.data}`,
+  }
+}
+
 function convertMessageToInputItems(msg: AnthropicMessage, output: OpenAIResponsesInputItem[]): void {
   const content = msg.content
 
@@ -131,27 +144,28 @@ function convertMessageToInputItems(msg: AnthropicMessage, output: OpenAIRespons
   }
 
   // Collect text/image parts and handle tool blocks separately
-  const contentParts: (string | OpenAIChatContentPart)[] = []
+  const contentParts: OpenAIResponsesInputContentPart[] = []
+
+  const flushContentParts = (): void => {
+    if (contentParts.length === 0) return
+
+    if (contentParts.every((part) => part.type === 'input_text')) {
+      const messageContent = contentParts.map((part) => part.text).join('')
+      if (messageContent) {
+        output.push({ type: 'message', role: msg.role, content: messageContent })
+      }
+    } else {
+      output.push({ type: 'message', role: msg.role, content: [...contentParts] })
+    }
+    contentParts.length = 0
+  }
 
   for (const block of content) {
-    if (block.type === 'text') {
-      contentParts.push(block.text)
-    } else if (block.type === 'image') {
-      contentParts.push({
-        type: 'image_url',
-        image_url: { url: `data:${block.source.media_type};base64,${block.source.data}` },
-      })
+    if (block.type === 'text' || block.type === 'image') {
+      contentParts.push(convertContentBlock(block))
     } else if (block.type === 'tool_use') {
       // Flush any accumulated content first
-      if (contentParts.length > 0) {
-        const flatContent = contentParts.length === 1 && typeof contentParts[0] === 'string'
-          ? contentParts[0]
-          : contentParts.map((p) => typeof p === 'string' ? p : '').join('')
-        if (flatContent) {
-          output.push({ type: 'message', role: msg.role, content: flatContent })
-        }
-        contentParts.length = 0
-      }
+      flushContentParts()
       // Lift to function_call item
       output.push({
         type: 'function_call',
@@ -160,30 +174,30 @@ function convertMessageToInputItems(msg: AnthropicMessage, output: OpenAIRespons
         arguments: typeof block.input === 'string' ? block.input : JSON.stringify(block.input),
       })
     } else if (block.type === 'tool_result') {
+      // Flush any accumulated content first
+      flushContentParts()
       // Lift to function_call_output item
       const resultContent = typeof block.content === 'string'
         ? block.content
         : Array.isArray(block.content)
-          ? block.content.filter((b): b is Extract<AnthropicContentBlock, { type: 'text' }> => b.type === 'text').map((b) => b.text).join('\n')
+          ? block.content.filter((part): part is Extract<AnthropicContentBlock, { type: 'text' | 'image' }> => (
+              part.type === 'text' || part.type === 'image'
+            )).map(convertContentBlock)
           : ''
+      const resultOutput = Array.isArray(resultContent) && resultContent.every((part) => part.type === 'input_text')
+        ? resultContent.map((part) => part.text).join('\n')
+        : resultContent
       output.push({
         type: 'function_call_output',
         call_id: block.tool_use_id,
-        output: resultContent,
+        output: resultOutput,
       })
     }
     // Skip thinking blocks
   }
 
   // Flush remaining content
-  if (contentParts.length > 0) {
-    const flatContent = contentParts.length === 1 && typeof contentParts[0] === 'string'
-      ? contentParts[0]
-      : contentParts.map((p) => typeof p === 'string' ? p : '').join('')
-    if (flatContent) {
-      output.push({ type: 'message', role: msg.role, content: flatContent })
-    }
-  }
+  flushContentParts()
 }
 
 function convertToolChoice(choice: unknown): unknown {

src/server/proxy/transform/types.ts

@@ -117,10 +117,14 @@ export type OpenAIChatStreamChunk = {
 
 // ─── OpenAI Responses API ───────────────────────────────────
 
+export type OpenAIResponsesInputContentPart =
+  | { type: 'input_text'; text: string }
+  | { type: 'input_image'; image_url: string }
+
 export type OpenAIResponsesInputItem =
-  | { type: 'message'; role: 'user' | 'assistant' | 'system'; content: string | OpenAIChatContentPart[] }
+  | { type: 'message'; role: 'user' | 'assistant' | 'system'; content: string | OpenAIResponsesInputContentPart[] }
   | { type: 'function_call'; call_id: string; name: string; arguments: unknown }
-  | { type: 'function_call_output'; call_id: string; output: string }
+  | { type: 'function_call_output'; call_id: string; output: string | OpenAIResponsesInputContentPart[] }
 
 export type OpenAIResponsesRequest = {
   model: string