mirror of
https://github.com/NanmiCoder/cc-haha
synced 2026-07-17 13:13:35 +08:00
fix: preserve readable large image inputs
Large screenshots were being treated like oversized text payloads, which pushed readable images through an aggressive resize/compression path before the model saw them. The image path now preserves images that already fit API byte and dimension limits, prefers lossless or high-quality compression when only bytes are too large, and only downscales by vision pixel budget when the image would exceed the Read token budget. Constraint: Fixes pasted/read image failures reported in #615 and #677 while preserving screenshot readability requested in #663 Rejected: Always resize to 2048px | would unnecessarily degrade screenshots that already fit provider limits Rejected: Continue estimating image budget from base64 text length | image blocks are billed/read as vision pixels, not text payload Confidence: high Scope-risk: moderate Directive: Do not lower fallback JPEG quality or long-edge limits without revalidating real screenshot OCR/vision tasks Tested: bun test src/utils/__tests__/imageResizer.test.ts src/server/__tests__/conversation-attachments.test.ts Tested: bun run check:server Tested: bun run check:coverage (changed-line coverage 95.25%; adapters/desktop coverage lanes failed from existing non-target issues) Tested: GPT-5.5 Read smoke on 1394x4404, 4096x2304, and 7900x900 images Not-tested: Real resize/re-encode branch in this source worktree because optional sharp/native image processor is not installed locally Related: #615 Related: #677 Related: #663
This commit is contained in:
程序员阿江(Relakkes)
parent
a1f28507b2
commit
46ca911d1a
@ -44,6 +44,7 @@ import {
|
||||
compressImageBufferWithTokenLimit,
|
||||
createImageMetadataText,
|
||||
detectImageFormatFromBuffer,
|
||||
downsampleImageBufferToVisionTokenBudget,
|
||||
type ImageDimensions,
|
||||
ImageResizeError,
|
||||
maybeResizeAndDownsampleImageBuffer,
|
||||
@ -798,6 +799,18 @@ function createImageResponse(
|
||||
}
|
||||
}
|
||||
|
||||
function estimateVisionImageTokens(dimensions?: ImageDimensions): number | null {
|
||||
const width = dimensions?.displayWidth
|
||||
const height = dimensions?.displayHeight
|
||||
if (!width || !height || width <= 0 || height <= 0) {
|
||||
return null
|
||||
}
|
||||
// Claude vision charges approximately width * height / 750 image tokens.
|
||||
// Image blocks are not base64 text blocks, so using base64 length here would
|
||||
// massively over-compress ordinary screenshots before the model sees them.
|
||||
return Math.ceil((width * height) / 750)
|
||||
}
|
||||
|
||||
/**
|
||||
* Inner implementation of call, separated to allow ENOENT handling in the outer call.
|
||||
*/
|
||||
@ -1133,10 +1146,31 @@ export async function readImageWithTokenBudget(
|
||||
result = createImageResponse(imageBuffer, detectedFormat, originalSize)
|
||||
}
|
||||
|
||||
// Check if it fits in token budget
|
||||
const estimatedTokens = Math.ceil(result.file.base64.length * 0.125)
|
||||
if (estimatedTokens > maxTokens) {
|
||||
// Aggressive compression from the SAME buffer (no re-read)
|
||||
// Check if it fits in vision token budget. This is intentionally based on
|
||||
// image dimensions, not base64 payload length, because the tool result is an
|
||||
// image content block rather than text.
|
||||
const estimatedTokens = estimateVisionImageTokens(result.file.dimensions)
|
||||
if (estimatedTokens !== null && estimatedTokens > maxTokens) {
|
||||
// Downsample by vision pixel budget first. This preserves detail far better
|
||||
// than the legacy base64-text token heuristic for image blocks.
|
||||
try {
|
||||
const downsampled = await downsampleImageBufferToVisionTokenBudget(
|
||||
imageBuffer,
|
||||
originalSize,
|
||||
detectedFormat,
|
||||
maxTokens,
|
||||
)
|
||||
return createImageResponse(
|
||||
downsampled.buffer,
|
||||
downsampled.mediaType,
|
||||
originalSize,
|
||||
downsampled.dimensions,
|
||||
)
|
||||
} catch (e) {
|
||||
logError(e)
|
||||
}
|
||||
|
||||
// Compatibility fallback from the SAME buffer (no re-read)
|
||||
try {
|
||||
const compressed = await compressImageBufferWithTokenLimit(
|
||||
imageBuffer,
|
||||
|
||||
@ -14,7 +14,7 @@ export type SharpInstance = {
|
||||
palette?: boolean
|
||||
colors?: number
|
||||
}): SharpInstance
|
||||
webp(options?: { quality?: number }): SharpInstance
|
||||
webp(options?: { quality?: number; lossless?: boolean }): SharpInstance
|
||||
toBuffer(): Promise<Buffer>
|
||||
}
|
||||
|
||||
|
||||
437
src/utils/__tests__/imageResizer.test.ts
Normal file
437
src/utils/__tests__/imageResizer.test.ts
Normal file
@ -0,0 +1,437 @@
|
||||
import { beforeEach, describe, expect, mock, test } from 'bun:test'
|
||||
import { rm, writeFile } from 'node:fs/promises'
|
||||
import { tmpdir } from 'node:os'
|
||||
import { join } from 'node:path'
|
||||
|
||||
type ProcessorMode = 'metadata' | 'throw'
|
||||
|
||||
let processorMode: ProcessorMode = 'metadata'
|
||||
let resizeCalls: Array<{ width: number; height: number }> = []
|
||||
let jpegCalls: Array<{ quality?: number; resized: boolean }> = []
|
||||
let pngCalls: Array<{ compressionLevel?: number; palette?: boolean }> = []
|
||||
let webpCalls: Array<{ quality?: number; lossless?: boolean }> = []
|
||||
|
||||
let originalPngDimensions: { width?: number; height?: number; format?: string } =
|
||||
{ width: 1394, height: 4404, format: 'png' }
|
||||
let outputForOperations: (operations: Operation[]) => Buffer = () =>
|
||||
Buffer.from('resized-image')
|
||||
|
||||
type Operation =
|
||||
| { type: 'resize'; width: number; height: number }
|
||||
| { type: 'jpeg'; quality?: number }
|
||||
| { type: 'png'; compressionLevel?: number; palette?: boolean }
|
||||
| { type: 'webp'; quality?: number; lossless?: boolean }
|
||||
|
||||
mock.module('../../tools/FileReadTool/imageProcessor.js', () => ({
|
||||
getImageProcessor: async () => {
|
||||
if (processorMode === 'throw') {
|
||||
throw new Error('image processor unavailable')
|
||||
}
|
||||
|
||||
return (_input: Buffer) => {
|
||||
const operations: Operation[] = []
|
||||
const instance = {
|
||||
metadata: async () => originalPngDimensions,
|
||||
resize: (width: number, height: number) => {
|
||||
resizeCalls.push({ width, height })
|
||||
operations.push({ type: 'resize', width, height })
|
||||
return instance
|
||||
},
|
||||
jpeg: (options?: { quality?: number }) => {
|
||||
jpegCalls.push({
|
||||
quality: options?.quality,
|
||||
resized: operations.some(op => op.type === 'resize'),
|
||||
})
|
||||
operations.push({ type: 'jpeg', quality: options?.quality })
|
||||
return instance
|
||||
},
|
||||
png: (options?: { compressionLevel?: number; palette?: boolean }) => {
|
||||
pngCalls.push(options ?? {})
|
||||
operations.push({
|
||||
type: 'png',
|
||||
compressionLevel: options?.compressionLevel,
|
||||
palette: options?.palette,
|
||||
})
|
||||
return instance
|
||||
},
|
||||
webp: (options?: { quality?: number; lossless?: boolean }) => {
|
||||
webpCalls.push(options ?? {})
|
||||
operations.push({
|
||||
type: 'webp',
|
||||
quality: options?.quality,
|
||||
lossless: options?.lossless,
|
||||
})
|
||||
return instance
|
||||
},
|
||||
toBuffer: async () => outputForOperations(operations),
|
||||
}
|
||||
return instance
|
||||
}
|
||||
},
|
||||
}))
|
||||
|
||||
const {
|
||||
ImageResizeError,
|
||||
downsampleImageBufferToVisionTokenBudget,
|
||||
maybeResizeAndDownsampleImageBuffer,
|
||||
} = await import('../imageResizer.js')
|
||||
const { readImageWithTokenBudget } = await import(
|
||||
'../../tools/FileReadTool/FileReadTool.js'
|
||||
)
|
||||
|
||||
function makePngHeader(width: number, height: number): Buffer {
|
||||
const buffer = Buffer.alloc(32)
|
||||
buffer[0] = 0x89
|
||||
buffer[1] = 0x50
|
||||
buffer[2] = 0x4e
|
||||
buffer[3] = 0x47
|
||||
buffer.writeUInt32BE(width, 16)
|
||||
buffer.writeUInt32BE(height, 20)
|
||||
return buffer
|
||||
}
|
||||
|
||||
beforeEach(() => {
|
||||
processorMode = 'metadata'
|
||||
resizeCalls = []
|
||||
jpegCalls = []
|
||||
pngCalls = []
|
||||
webpCalls = []
|
||||
originalPngDimensions = { width: 1394, height: 4404, format: 'png' }
|
||||
outputForOperations = () => Buffer.from('resized-image')
|
||||
})
|
||||
|
||||
describe('maybeResizeAndDownsampleImageBuffer', () => {
|
||||
test('passes through a tall screenshot when bytes are already within API limits', async () => {
|
||||
const imageBuffer = Buffer.alloc(1024, 1)
|
||||
|
||||
const result = await maybeResizeAndDownsampleImageBuffer(
|
||||
imageBuffer,
|
||||
imageBuffer.length,
|
||||
'png',
|
||||
)
|
||||
|
||||
expect(result.buffer).toBe(imageBuffer)
|
||||
expect(result.mediaType).toBe('png')
|
||||
expect(resizeCalls).toEqual([])
|
||||
expect(result.dimensions).toEqual({
|
||||
originalWidth: 1394,
|
||||
originalHeight: 4404,
|
||||
displayWidth: 1394,
|
||||
displayHeight: 4404,
|
||||
})
|
||||
})
|
||||
|
||||
test('falls back to the original tall screenshot if local image processing is unavailable', async () => {
|
||||
processorMode = 'throw'
|
||||
const imageBuffer = makePngHeader(1394, 4404)
|
||||
|
||||
const result = await maybeResizeAndDownsampleImageBuffer(
|
||||
imageBuffer,
|
||||
imageBuffer.length,
|
||||
'png',
|
||||
)
|
||||
|
||||
expect(result.buffer).toBe(imageBuffer)
|
||||
expect(result.mediaType).toBe('png')
|
||||
})
|
||||
|
||||
test('tries lossless webp compression before lossy jpeg for oversized screenshots', async () => {
|
||||
originalPngDimensions = { width: 4096, height: 2304, format: 'png' }
|
||||
outputForOperations = operations => {
|
||||
if (operations.some(op => op.type === 'webp')) {
|
||||
return Buffer.alloc(1024, 3)
|
||||
}
|
||||
return Buffer.alloc(4 * 1024 * 1024, 2)
|
||||
}
|
||||
const imageBuffer = Buffer.alloc(4 * 1024 * 1024, 1)
|
||||
|
||||
const result = await maybeResizeAndDownsampleImageBuffer(
|
||||
imageBuffer,
|
||||
imageBuffer.length,
|
||||
'png',
|
||||
)
|
||||
|
||||
expect(result.mediaType).toBe('webp')
|
||||
expect(result.buffer.length).toBe(1024)
|
||||
expect(resizeCalls).toEqual([])
|
||||
expect(webpCalls).toEqual([{ lossless: true }])
|
||||
expect(jpegCalls).toEqual([])
|
||||
expect(result.dimensions).toEqual({
|
||||
originalWidth: 4096,
|
||||
originalHeight: 2304,
|
||||
displayWidth: 4096,
|
||||
displayHeight: 2304,
|
||||
})
|
||||
})
|
||||
|
||||
test('downsamples to a codex-sized long edge before lowering jpeg below readable quality', async () => {
|
||||
originalPngDimensions = { width: 5000, height: 3000, format: 'png' }
|
||||
outputForOperations = operations => {
|
||||
const resizeOp = operations.find(
|
||||
(op): op is Extract<Operation, { type: 'resize' }> =>
|
||||
op.type === 'resize',
|
||||
)
|
||||
const jpegOp = operations.find(
|
||||
(op): op is Extract<Operation, { type: 'jpeg' }> =>
|
||||
op.type === 'jpeg',
|
||||
)
|
||||
if (resizeOp?.width === 2048 && resizeOp.height === 1229) {
|
||||
if (jpegOp?.quality === 85) {
|
||||
return Buffer.alloc(1024, 4)
|
||||
}
|
||||
}
|
||||
return Buffer.alloc(6 * 1024 * 1024, 5)
|
||||
}
|
||||
const imageBuffer = Buffer.alloc(6 * 1024 * 1024, 1)
|
||||
|
||||
const result = await maybeResizeAndDownsampleImageBuffer(
|
||||
imageBuffer,
|
||||
imageBuffer.length,
|
||||
'png',
|
||||
)
|
||||
|
||||
expect(result.mediaType).toBe('jpeg')
|
||||
expect(result.buffer.length).toBe(1024)
|
||||
expect(resizeCalls).toContainEqual({ width: 2048, height: 1229 })
|
||||
expect(jpegCalls.some(call => (call.quality ?? 100) < 75)).toBe(false)
|
||||
expect(result.dimensions).toEqual({
|
||||
originalWidth: 5000,
|
||||
originalHeight: 3000,
|
||||
displayWidth: 2048,
|
||||
displayHeight: 1229,
|
||||
})
|
||||
})
|
||||
|
||||
test('resizes to the hard dimension cap when only height exceeds the API limit', async () => {
|
||||
originalPngDimensions = { width: 1000, height: 9000, format: 'png' }
|
||||
outputForOperations = operations => {
|
||||
const resizeOp = operations.find(
|
||||
(op): op is Extract<Operation, { type: 'resize' }> =>
|
||||
op.type === 'resize',
|
||||
)
|
||||
const jpegOp = operations.find(
|
||||
(op): op is Extract<Operation, { type: 'jpeg' }> =>
|
||||
op.type === 'jpeg',
|
||||
)
|
||||
if (
|
||||
resizeOp?.width === 889 &&
|
||||
resizeOp.height === 8000 &&
|
||||
jpegOp?.quality === 85
|
||||
) {
|
||||
return Buffer.alloc(1024, 6)
|
||||
}
|
||||
return Buffer.alloc(6 * 1024 * 1024, 5)
|
||||
}
|
||||
const imageBuffer = Buffer.alloc(200_000, 1)
|
||||
|
||||
const result = await maybeResizeAndDownsampleImageBuffer(
|
||||
imageBuffer,
|
||||
imageBuffer.length,
|
||||
'png',
|
||||
)
|
||||
|
||||
expect(result.mediaType).toBe('jpeg')
|
||||
expect(resizeCalls).toContainEqual({ width: 889, height: 8000 })
|
||||
expect(result.dimensions).toEqual({
|
||||
originalWidth: 1000,
|
||||
originalHeight: 9000,
|
||||
displayWidth: 889,
|
||||
displayHeight: 8000,
|
||||
})
|
||||
})
|
||||
|
||||
test('uses readable fallback dimensions before dropping to fallback jpeg quality', async () => {
|
||||
originalPngDimensions = { width: 6000, height: 5000, format: 'png' }
|
||||
outputForOperations = operations => {
|
||||
const resizeOp = operations.find(
|
||||
(op): op is Extract<Operation, { type: 'resize' }> =>
|
||||
op.type === 'resize',
|
||||
)
|
||||
const jpegOp = operations.find(
|
||||
(op): op is Extract<Operation, { type: 'jpeg' }> =>
|
||||
op.type === 'jpeg',
|
||||
)
|
||||
if (
|
||||
resizeOp?.width === 1568 &&
|
||||
resizeOp.height === 1307 &&
|
||||
jpegOp?.quality === 65
|
||||
) {
|
||||
return Buffer.alloc(1024, 6)
|
||||
}
|
||||
return Buffer.alloc(6 * 1024 * 1024, 5)
|
||||
}
|
||||
const imageBuffer = Buffer.alloc(6 * 1024 * 1024, 1)
|
||||
|
||||
const result = await maybeResizeAndDownsampleImageBuffer(
|
||||
imageBuffer,
|
||||
imageBuffer.length,
|
||||
'png',
|
||||
)
|
||||
|
||||
expect(result.mediaType).toBe('jpeg')
|
||||
expect(resizeCalls).toContainEqual({ width: 2048, height: 1707 })
|
||||
expect(resizeCalls).toContainEqual({ width: 1568, height: 1307 })
|
||||
expect(jpegCalls.some(call => call.quality === 65)).toBe(true)
|
||||
expect(result.dimensions).toEqual({
|
||||
originalWidth: 6000,
|
||||
originalHeight: 5000,
|
||||
displayWidth: 1568,
|
||||
displayHeight: 1307,
|
||||
})
|
||||
})
|
||||
|
||||
test('throws instead of over-compressing when readable fallback cannot fit', async () => {
|
||||
originalPngDimensions = { width: 6000, height: 5000, format: 'png' }
|
||||
outputForOperations = () => Buffer.alloc(6 * 1024 * 1024, 5)
|
||||
const imageBuffer = Buffer.alloc(6 * 1024 * 1024, 1)
|
||||
|
||||
expect(
|
||||
maybeResizeAndDownsampleImageBuffer(
|
||||
imageBuffer,
|
||||
imageBuffer.length,
|
||||
'png',
|
||||
),
|
||||
).rejects.toBeInstanceOf(ImageResizeError)
|
||||
})
|
||||
|
||||
test('tries lossless webp for oversized webp screenshots before jpeg conversion', async () => {
|
||||
originalPngDimensions = { width: 4096, height: 2304, format: 'webp' }
|
||||
outputForOperations = operations => {
|
||||
if (operations.some(op => op.type === 'webp')) {
|
||||
return Buffer.alloc(1024, 3)
|
||||
}
|
||||
return Buffer.alloc(4 * 1024 * 1024, 2)
|
||||
}
|
||||
const imageBuffer = Buffer.alloc(4 * 1024 * 1024, 1)
|
||||
|
||||
const result = await maybeResizeAndDownsampleImageBuffer(
|
||||
imageBuffer,
|
||||
imageBuffer.length,
|
||||
'webp',
|
||||
)
|
||||
|
||||
expect(result.mediaType).toBe('webp')
|
||||
expect(webpCalls).toEqual([{ lossless: true }])
|
||||
expect(jpegCalls).toEqual([])
|
||||
})
|
||||
})
|
||||
|
||||
describe('readImageWithTokenBudget', () => {
|
||||
test('does not recompress readable screenshots just because their base64 text would exceed maxTokens', async () => {
|
||||
originalPngDimensions = { width: 1920, height: 1080, format: 'png' }
|
||||
outputForOperations = () => Buffer.from('compressed-image')
|
||||
const imageBuffer = Buffer.alloc(200_000, 1)
|
||||
const imagePath = join(tmpdir(), `cc-haha-image-budget-${Date.now()}.png`)
|
||||
await writeFile(imagePath, imageBuffer)
|
||||
|
||||
try {
|
||||
const result = await readImageWithTokenBudget(imagePath, 25_000)
|
||||
|
||||
expect(result.file.base64).toBe(imageBuffer.toString('base64'))
|
||||
expect(result.file.type).toBe('image/png')
|
||||
expect(jpegCalls).toEqual([])
|
||||
expect(resizeCalls).toEqual([])
|
||||
} finally {
|
||||
await rm(imagePath, { force: true })
|
||||
}
|
||||
})
|
||||
|
||||
test('downsamples over-budget vision images by pixel budget instead of base64 text budget', async () => {
|
||||
originalPngDimensions = { width: 6000, height: 4000, format: 'png' }
|
||||
outputForOperations = operations => {
|
||||
const resizeOp = operations.find(
|
||||
(op): op is Extract<Operation, { type: 'resize' }> =>
|
||||
op.type === 'resize',
|
||||
)
|
||||
const jpegOp = operations.find(
|
||||
(op): op is Extract<Operation, { type: 'jpeg' }> =>
|
||||
op.type === 'jpeg',
|
||||
)
|
||||
if (
|
||||
resizeOp?.width === 3674 &&
|
||||
resizeOp.height === 2449 &&
|
||||
jpegOp?.quality === 85
|
||||
) {
|
||||
return Buffer.alloc(1024, 7)
|
||||
}
|
||||
return Buffer.alloc(6 * 1024 * 1024, 8)
|
||||
}
|
||||
const imageBuffer = Buffer.alloc(200_000, 1)
|
||||
const imagePath = join(
|
||||
tmpdir(),
|
||||
`cc-haha-image-vision-budget-${Date.now()}.png`,
|
||||
)
|
||||
await writeFile(imagePath, imageBuffer)
|
||||
|
||||
try {
|
||||
const result = await readImageWithTokenBudget(imagePath, 12_000)
|
||||
|
||||
expect(result.file.type).toBe('image/jpeg')
|
||||
expect(Buffer.from(result.file.base64, 'base64').length).toBe(1024)
|
||||
expect(resizeCalls).toContainEqual({ width: 3674, height: 2449 })
|
||||
expect(jpegCalls.some(call => (call.quality ?? 100) < 75)).toBe(false)
|
||||
expect(result.file.dimensions).toEqual({
|
||||
originalWidth: 6000,
|
||||
originalHeight: 4000,
|
||||
displayWidth: 3674,
|
||||
displayHeight: 2449,
|
||||
})
|
||||
} finally {
|
||||
await rm(imagePath, { force: true })
|
||||
}
|
||||
})
|
||||
|
||||
test('uses a readable fallback when the vision-budget resize cannot fit the byte cap', async () => {
|
||||
originalPngDimensions = { width: 6000, height: 4000, format: 'png' }
|
||||
outputForOperations = operations => {
|
||||
const resizeOp = operations.find(
|
||||
(op): op is Extract<Operation, { type: 'resize' }> =>
|
||||
op.type === 'resize',
|
||||
)
|
||||
const jpegOp = operations.find(
|
||||
(op): op is Extract<Operation, { type: 'jpeg' }> =>
|
||||
op.type === 'jpeg',
|
||||
)
|
||||
if (
|
||||
resizeOp?.width === 1568 &&
|
||||
resizeOp.height === 1045 &&
|
||||
jpegOp?.quality === 65
|
||||
) {
|
||||
return Buffer.alloc(1024, 7)
|
||||
}
|
||||
return Buffer.alloc(6 * 1024 * 1024, 8)
|
||||
}
|
||||
const imageBuffer = Buffer.alloc(200_000, 1)
|
||||
|
||||
const result = await downsampleImageBufferToVisionTokenBudget(
|
||||
imageBuffer,
|
||||
imageBuffer.length,
|
||||
'png',
|
||||
12_000,
|
||||
)
|
||||
|
||||
expect(result.mediaType).toBe('jpeg')
|
||||
expect(resizeCalls).toContainEqual({ width: 3674, height: 2449 })
|
||||
expect(resizeCalls).toContainEqual({ width: 1568, height: 1045 })
|
||||
expect(result.dimensions).toEqual({
|
||||
originalWidth: 6000,
|
||||
originalHeight: 4000,
|
||||
displayWidth: 1568,
|
||||
displayHeight: 1045,
|
||||
})
|
||||
})
|
||||
|
||||
test('reports an image resize error when token-budget dimensions are unavailable', async () => {
|
||||
originalPngDimensions = { format: 'png' }
|
||||
const imageBuffer = Buffer.alloc(200_000, 1)
|
||||
|
||||
expect(
|
||||
downsampleImageBufferToVisionTokenBudget(
|
||||
imageBuffer,
|
||||
imageBuffer.length,
|
||||
'png',
|
||||
12_000,
|
||||
),
|
||||
).rejects.toBeInstanceOf(ImageResizeError)
|
||||
})
|
||||
})
|
||||
@ -162,6 +162,16 @@ interface CompressedImageResult {
|
||||
originalSize: number
|
||||
}
|
||||
|
||||
type ResizeDimensions = {
|
||||
width: number
|
||||
height: number
|
||||
}
|
||||
|
||||
const IMAGE_PROMPT_SOFT_LONG_EDGE = 2048
|
||||
const IMAGE_PROMPT_READABLE_LONG_EDGE = 1568
|
||||
const READABLE_JPEG_QUALITIES = [85, 75] as const
|
||||
const FALLBACK_JPEG_QUALITIES = [65] as const
|
||||
|
||||
/**
|
||||
* Extracted from FileReadTool's readImage function
|
||||
* Resizes image buffer to meet size and dimension constraints
|
||||
@ -226,160 +236,99 @@ export async function maybeResizeAndDownsampleImageBuffer(
|
||||
}
|
||||
}
|
||||
|
||||
const hardDims = constrainDimensions(
|
||||
width,
|
||||
height,
|
||||
IMAGE_MAX_WIDTH,
|
||||
IMAGE_MAX_HEIGHT,
|
||||
)
|
||||
const needsDimensionResize =
|
||||
width > IMAGE_MAX_WIDTH || height > IMAGE_MAX_HEIGHT
|
||||
const isPng = normalizedMediaType === 'png'
|
||||
hardDims.width !== width || hardDims.height !== height
|
||||
|
||||
// If dimensions are within limits but file is too large, try compression first
|
||||
// This preserves full resolution when possible
|
||||
// Try lossless or high-quality compression at original dimensions first.
|
||||
// This keeps screenshot text sharp when the only problem is payload size.
|
||||
if (!needsDimensionResize && originalSize > IMAGE_TARGET_RAW_SIZE) {
|
||||
// For PNGs, try PNG compression first to preserve transparency
|
||||
if (isPng) {
|
||||
// Create fresh sharp instance for each compression attempt
|
||||
const pngCompressed = await sharp(imageBuffer)
|
||||
.png({ compressionLevel: 9, palette: true })
|
||||
.toBuffer()
|
||||
if (pngCompressed.length <= IMAGE_TARGET_RAW_SIZE) {
|
||||
return {
|
||||
buffer: pngCompressed,
|
||||
mediaType: 'png',
|
||||
dimensions: {
|
||||
originalWidth,
|
||||
originalHeight,
|
||||
displayWidth: width,
|
||||
displayHeight: height,
|
||||
},
|
||||
}
|
||||
}
|
||||
}
|
||||
// Try JPEG compression (lossy but much smaller)
|
||||
for (const quality of [80, 60, 40, 20]) {
|
||||
// Create fresh sharp instance for each attempt
|
||||
const compressedBuffer = await sharp(imageBuffer)
|
||||
.jpeg({ quality })
|
||||
.toBuffer()
|
||||
if (compressedBuffer.length <= IMAGE_TARGET_RAW_SIZE) {
|
||||
return {
|
||||
buffer: compressedBuffer,
|
||||
mediaType: 'jpeg',
|
||||
dimensions: {
|
||||
originalWidth,
|
||||
originalHeight,
|
||||
displayWidth: width,
|
||||
displayHeight: height,
|
||||
},
|
||||
}
|
||||
}
|
||||
}
|
||||
// Quality reduction alone wasn't enough, fall through to resize
|
||||
}
|
||||
|
||||
// Constrain dimensions if needed
|
||||
if (width > IMAGE_MAX_WIDTH) {
|
||||
height = Math.round((height * IMAGE_MAX_WIDTH) / width)
|
||||
width = IMAGE_MAX_WIDTH
|
||||
}
|
||||
|
||||
if (height > IMAGE_MAX_HEIGHT) {
|
||||
width = Math.round((width * IMAGE_MAX_HEIGHT) / height)
|
||||
height = IMAGE_MAX_HEIGHT
|
||||
}
|
||||
|
||||
// IMPORTANT: Always create fresh sharp(imageBuffer) instances for each operation.
|
||||
// The native image-processor-napi module doesn't properly apply format conversions
|
||||
// when reusing a sharp instance after calling toBuffer(). This caused a bug where
|
||||
// all compression attempts (PNG, JPEG at various qualities) returned identical sizes.
|
||||
logForDebugging(`Resizing to ${width}x${height}`)
|
||||
const resizedImageBuffer = await sharp(imageBuffer)
|
||||
.resize(width, height, {
|
||||
fit: 'inside',
|
||||
withoutEnlargement: true,
|
||||
})
|
||||
.toBuffer()
|
||||
|
||||
// If still too large after resize, try compression
|
||||
if (resizedImageBuffer.length > IMAGE_TARGET_RAW_SIZE) {
|
||||
// For PNGs, try PNG compression first to preserve transparency
|
||||
if (isPng) {
|
||||
const pngCompressed = await sharp(imageBuffer)
|
||||
.resize(width, height, {
|
||||
fit: 'inside',
|
||||
withoutEnlargement: true,
|
||||
})
|
||||
.png({ compressionLevel: 9, palette: true })
|
||||
.toBuffer()
|
||||
if (pngCompressed.length <= IMAGE_TARGET_RAW_SIZE) {
|
||||
return {
|
||||
buffer: pngCompressed,
|
||||
mediaType: 'png',
|
||||
dimensions: {
|
||||
originalWidth,
|
||||
originalHeight,
|
||||
displayWidth: width,
|
||||
displayHeight: height,
|
||||
},
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
// Try JPEG with progressively lower quality
|
||||
for (const quality of [80, 60, 40, 20]) {
|
||||
const compressedBuffer = await sharp(imageBuffer)
|
||||
.resize(width, height, {
|
||||
fit: 'inside',
|
||||
withoutEnlargement: true,
|
||||
})
|
||||
.jpeg({ quality })
|
||||
.toBuffer()
|
||||
if (compressedBuffer.length <= IMAGE_TARGET_RAW_SIZE) {
|
||||
return {
|
||||
buffer: compressedBuffer,
|
||||
mediaType: 'jpeg',
|
||||
dimensions: {
|
||||
originalWidth,
|
||||
originalHeight,
|
||||
displayWidth: width,
|
||||
displayHeight: height,
|
||||
},
|
||||
}
|
||||
}
|
||||
}
|
||||
// If still too large, resize smaller and compress aggressively
|
||||
const smallerWidth = Math.min(width, 1000)
|
||||
const smallerHeight = Math.round(
|
||||
(height * smallerWidth) / Math.max(width, 1),
|
||||
)
|
||||
logForDebugging('Still too large, compressing with JPEG')
|
||||
const compressedBuffer = await sharp(imageBuffer)
|
||||
.resize(smallerWidth, smallerHeight, {
|
||||
fit: 'inside',
|
||||
withoutEnlargement: true,
|
||||
})
|
||||
.jpeg({ quality: 20 })
|
||||
.toBuffer()
|
||||
logForDebugging(`JPEG compressed buffer size: ${compressedBuffer.length}`)
|
||||
return {
|
||||
buffer: compressedBuffer,
|
||||
mediaType: 'jpeg',
|
||||
dimensions: {
|
||||
originalWidth,
|
||||
originalHeight,
|
||||
displayWidth: smallerWidth,
|
||||
displayHeight: smallerHeight,
|
||||
},
|
||||
}
|
||||
}
|
||||
|
||||
return {
|
||||
buffer: resizedImageBuffer,
|
||||
mediaType: normalizedMediaType,
|
||||
dimensions: {
|
||||
const compressed = await tryEncodeImageCandidate({
|
||||
sharp,
|
||||
imageBuffer,
|
||||
originalWidth,
|
||||
originalHeight,
|
||||
displayWidth: width,
|
||||
displayHeight: height,
|
||||
},
|
||||
dimensions: { width, height },
|
||||
format: normalizedMediaType,
|
||||
maxBytes: IMAGE_TARGET_RAW_SIZE,
|
||||
})
|
||||
if (compressed) return compressed
|
||||
}
|
||||
|
||||
if (needsDimensionResize) {
|
||||
logForDebugging(`Resizing to ${hardDims.width}x${hardDims.height}`)
|
||||
const hardResize = await tryEncodeImageCandidate({
|
||||
sharp,
|
||||
imageBuffer,
|
||||
originalWidth,
|
||||
originalHeight,
|
||||
dimensions: hardDims,
|
||||
format: normalizedMediaType,
|
||||
maxBytes: IMAGE_TARGET_RAW_SIZE,
|
||||
})
|
||||
if (hardResize) return hardResize
|
||||
}
|
||||
|
||||
const softDims = constrainDimensions(
|
||||
hardDims.width,
|
||||
hardDims.height,
|
||||
IMAGE_PROMPT_SOFT_LONG_EDGE,
|
||||
IMAGE_PROMPT_SOFT_LONG_EDGE,
|
||||
)
|
||||
if (
|
||||
softDims.width !== hardDims.width ||
|
||||
softDims.height !== hardDims.height
|
||||
) {
|
||||
logForDebugging(
|
||||
`Downsampling image to ${softDims.width}x${softDims.height}`,
|
||||
)
|
||||
const softResize = await tryEncodeImageCandidate({
|
||||
sharp,
|
||||
imageBuffer,
|
||||
originalWidth,
|
||||
originalHeight,
|
||||
dimensions: softDims,
|
||||
format: normalizedMediaType,
|
||||
maxBytes: IMAGE_TARGET_RAW_SIZE,
|
||||
})
|
||||
if (softResize) return softResize
|
||||
}
|
||||
|
||||
const readableDims = constrainDimensions(
|
||||
hardDims.width,
|
||||
hardDims.height,
|
||||
IMAGE_PROMPT_READABLE_LONG_EDGE,
|
||||
IMAGE_PROMPT_READABLE_LONG_EDGE,
|
||||
)
|
||||
if (
|
||||
readableDims.width !== softDims.width ||
|
||||
readableDims.height !== softDims.height
|
||||
) {
|
||||
logForDebugging(
|
||||
`Downsampling image to readable fallback ${readableDims.width}x${readableDims.height}`,
|
||||
)
|
||||
const readableResize = await tryEncodeImageCandidate({
|
||||
sharp,
|
||||
imageBuffer,
|
||||
originalWidth,
|
||||
originalHeight,
|
||||
dimensions: readableDims,
|
||||
format: normalizedMediaType,
|
||||
maxBytes: IMAGE_TARGET_RAW_SIZE,
|
||||
allowFallbackQuality: true,
|
||||
})
|
||||
if (readableResize) return readableResize
|
||||
}
|
||||
|
||||
throw new ImageResizeError(
|
||||
`Unable to compress image (${formatFileSize(originalSize)}) below the 5MB API base64 limit without over-compressing it. ` +
|
||||
`Please resize the image manually or use a smaller image.`,
|
||||
)
|
||||
} catch (error) {
|
||||
// Log the error and emit analytics event
|
||||
logError(error as Error)
|
||||
@ -432,6 +381,277 @@ export async function maybeResizeAndDownsampleImageBuffer(
|
||||
}
|
||||
}
|
||||
|
||||
function constrainDimensions(
|
||||
width: number,
|
||||
height: number,
|
||||
maxWidth: number,
|
||||
maxHeight: number,
|
||||
): ResizeDimensions {
|
||||
let nextWidth = width
|
||||
let nextHeight = height
|
||||
|
||||
if (nextWidth > maxWidth) {
|
||||
nextHeight = Math.round((nextHeight * maxWidth) / nextWidth)
|
||||
nextWidth = maxWidth
|
||||
}
|
||||
|
||||
if (nextHeight > maxHeight) {
|
||||
nextWidth = Math.round((nextWidth * maxHeight) / nextHeight)
|
||||
nextHeight = maxHeight
|
||||
}
|
||||
|
||||
return {
|
||||
width: Math.max(1, nextWidth),
|
||||
height: Math.max(1, nextHeight),
|
||||
}
|
||||
}
|
||||
|
||||
function constrainDimensionsToPixelBudget(
|
||||
width: number,
|
||||
height: number,
|
||||
maxPixels: number,
|
||||
): ResizeDimensions {
|
||||
if (width * height <= maxPixels) {
|
||||
return { width, height }
|
||||
}
|
||||
|
||||
const scale = Math.sqrt(maxPixels / (width * height))
|
||||
let nextWidth = Math.max(1, Math.round(width * scale))
|
||||
let nextHeight = Math.max(1, Math.round(height * scale))
|
||||
|
||||
while (nextWidth * nextHeight > maxPixels) {
|
||||
if (nextWidth >= nextHeight) {
|
||||
nextWidth -= 1
|
||||
} else {
|
||||
nextHeight -= 1
|
||||
}
|
||||
}
|
||||
|
||||
return {
|
||||
width: nextWidth,
|
||||
height: nextHeight,
|
||||
}
|
||||
}
|
||||
|
||||
function createResizeResult(
|
||||
buffer: Buffer,
|
||||
mediaType: string,
|
||||
originalWidth: number,
|
||||
originalHeight: number,
|
||||
dimensions: ResizeDimensions,
|
||||
): ResizeResult {
|
||||
return {
|
||||
buffer,
|
||||
mediaType,
|
||||
dimensions: {
|
||||
originalWidth,
|
||||
originalHeight,
|
||||
displayWidth: dimensions.width,
|
||||
displayHeight: dimensions.height,
|
||||
},
|
||||
}
|
||||
}
|
||||
|
||||
function createImagePipeline(
|
||||
sharp: SharpFunction,
|
||||
imageBuffer: Buffer,
|
||||
originalWidth: number,
|
||||
originalHeight: number,
|
||||
dimensions: ResizeDimensions,
|
||||
): SharpInstance {
|
||||
const pipeline = sharp(imageBuffer)
|
||||
if (
|
||||
dimensions.width === originalWidth &&
|
||||
dimensions.height === originalHeight
|
||||
) {
|
||||
return pipeline
|
||||
}
|
||||
|
||||
return pipeline.resize(dimensions.width, dimensions.height, {
|
||||
fit: 'inside',
|
||||
withoutEnlargement: true,
|
||||
})
|
||||
}
|
||||
|
||||
async function tryEncodeImageCandidate({
|
||||
sharp,
|
||||
imageBuffer,
|
||||
originalWidth,
|
||||
originalHeight,
|
||||
dimensions,
|
||||
format,
|
||||
maxBytes,
|
||||
allowFallbackQuality = false,
|
||||
}: {
|
||||
sharp: SharpFunction
|
||||
imageBuffer: Buffer
|
||||
originalWidth: number
|
||||
originalHeight: number
|
||||
dimensions: ResizeDimensions
|
||||
format: string
|
||||
maxBytes: number
|
||||
allowFallbackQuality?: boolean
|
||||
}): Promise<ResizeResult | null> {
|
||||
const normalizedFormat = format === 'jpg' ? 'jpeg' : format
|
||||
|
||||
if (normalizedFormat === 'png') {
|
||||
const pngCompressed = await createImagePipeline(
|
||||
sharp,
|
||||
imageBuffer,
|
||||
originalWidth,
|
||||
originalHeight,
|
||||
dimensions,
|
||||
)
|
||||
.png({ compressionLevel: 9 })
|
||||
.toBuffer()
|
||||
if (pngCompressed.length <= maxBytes) {
|
||||
return createResizeResult(
|
||||
pngCompressed,
|
||||
'png',
|
||||
originalWidth,
|
||||
originalHeight,
|
||||
dimensions,
|
||||
)
|
||||
}
|
||||
|
||||
const webpLossless = await createImagePipeline(
|
||||
sharp,
|
||||
imageBuffer,
|
||||
originalWidth,
|
||||
originalHeight,
|
||||
dimensions,
|
||||
)
|
||||
.webp({ lossless: true })
|
||||
.toBuffer()
|
||||
if (webpLossless.length <= maxBytes) {
|
||||
return createResizeResult(
|
||||
webpLossless,
|
||||
'webp',
|
||||
originalWidth,
|
||||
originalHeight,
|
||||
dimensions,
|
||||
)
|
||||
}
|
||||
} else if (normalizedFormat === 'webp') {
|
||||
const webpLossless = await createImagePipeline(
|
||||
sharp,
|
||||
imageBuffer,
|
||||
originalWidth,
|
||||
originalHeight,
|
||||
dimensions,
|
||||
)
|
||||
.webp({ lossless: true })
|
||||
.toBuffer()
|
||||
if (webpLossless.length <= maxBytes) {
|
||||
return createResizeResult(
|
||||
webpLossless,
|
||||
'webp',
|
||||
originalWidth,
|
||||
originalHeight,
|
||||
dimensions,
|
||||
)
|
||||
}
|
||||
}
|
||||
|
||||
const jpegQualities = allowFallbackQuality
|
||||
? [...READABLE_JPEG_QUALITIES, ...FALLBACK_JPEG_QUALITIES]
|
||||
: READABLE_JPEG_QUALITIES
|
||||
for (const quality of jpegQualities) {
|
||||
const jpegBuffer = await createImagePipeline(
|
||||
sharp,
|
||||
imageBuffer,
|
||||
originalWidth,
|
||||
originalHeight,
|
||||
dimensions,
|
||||
)
|
||||
.jpeg({ quality })
|
||||
.toBuffer()
|
||||
if (jpegBuffer.length <= maxBytes) {
|
||||
return createResizeResult(
|
||||
jpegBuffer,
|
||||
'jpeg',
|
||||
originalWidth,
|
||||
originalHeight,
|
||||
dimensions,
|
||||
)
|
||||
}
|
||||
}
|
||||
|
||||
return null
|
||||
}
|
||||
|
||||
export async function downsampleImageBufferToVisionTokenBudget(
|
||||
imageBuffer: Buffer,
|
||||
originalSize: number,
|
||||
ext: string,
|
||||
maxTokens: number,
|
||||
): Promise<ResizeResult> {
|
||||
const sharp = await getImageProcessor()
|
||||
const metadata = await sharp(imageBuffer).metadata()
|
||||
const format = metadata.format ?? ext
|
||||
const normalizedMediaType = format === 'jpg' ? 'jpeg' : format
|
||||
|
||||
if (!metadata.width || !metadata.height) {
|
||||
throw new ImageResizeError(
|
||||
'Unable to downsample image for token budget because dimensions are unavailable.',
|
||||
)
|
||||
}
|
||||
|
||||
const originalWidth = metadata.width
|
||||
const originalHeight = metadata.height
|
||||
const hardDims = constrainDimensions(
|
||||
originalWidth,
|
||||
originalHeight,
|
||||
IMAGE_MAX_WIDTH,
|
||||
IMAGE_MAX_HEIGHT,
|
||||
)
|
||||
const maxPixels = Math.max(1, maxTokens * 750)
|
||||
const tokenDims = constrainDimensionsToPixelBudget(
|
||||
hardDims.width,
|
||||
hardDims.height,
|
||||
maxPixels,
|
||||
)
|
||||
|
||||
const tokenResize = await tryEncodeImageCandidate({
|
||||
sharp,
|
||||
imageBuffer,
|
||||
originalWidth,
|
||||
originalHeight,
|
||||
dimensions: tokenDims,
|
||||
format: normalizedMediaType,
|
||||
maxBytes: IMAGE_TARGET_RAW_SIZE,
|
||||
allowFallbackQuality: true,
|
||||
})
|
||||
if (tokenResize) return tokenResize
|
||||
|
||||
const readableDims = constrainDimensions(
|
||||
tokenDims.width,
|
||||
tokenDims.height,
|
||||
IMAGE_PROMPT_READABLE_LONG_EDGE,
|
||||
IMAGE_PROMPT_READABLE_LONG_EDGE,
|
||||
)
|
||||
if (
|
||||
readableDims.width !== tokenDims.width ||
|
||||
readableDims.height !== tokenDims.height
|
||||
) {
|
||||
const readableResize = await tryEncodeImageCandidate({
|
||||
sharp,
|
||||
imageBuffer,
|
||||
originalWidth,
|
||||
originalHeight,
|
||||
dimensions: readableDims,
|
||||
format: normalizedMediaType,
|
||||
maxBytes: IMAGE_TARGET_RAW_SIZE,
|
||||
allowFallbackQuality: true,
|
||||
})
|
||||
if (readableResize) return readableResize
|
||||
}
|
||||
|
||||
throw new ImageResizeError(
|
||||
`Unable to downsample image (${formatFileSize(originalSize)}) below the ${maxTokens} token budget without over-compressing it.`,
|
||||
)
|
||||
}
|
||||
|
||||
export interface ImageBlockWithDimensions {
|
||||
block: ImageBlockParam
|
||||
dimensions?: ImageDimensions
|
||||
|
||||
@ -1,87 +0,0 @@
|
||||
import { beforeEach, describe, expect, mock, test } from 'bun:test'
|
||||
|
||||
type ProcessorMode = 'metadata' | 'throw'
|
||||
|
||||
let processorMode: ProcessorMode = 'metadata'
|
||||
let resizeCalls: Array<{ width: number; height: number }> = []
|
||||
|
||||
const originalPngDimensions = { width: 1394, height: 4404, format: 'png' }
|
||||
|
||||
mock.module('../src/tools/FileReadTool/imageProcessor.js', () => ({
|
||||
getImageProcessor: async () => {
|
||||
if (processorMode === 'throw') {
|
||||
throw new Error('image processor unavailable')
|
||||
}
|
||||
|
||||
return (_input: Buffer) => {
|
||||
const instance = {
|
||||
metadata: async () => originalPngDimensions,
|
||||
resize: (width: number, height: number) => {
|
||||
resizeCalls.push({ width, height })
|
||||
return instance
|
||||
},
|
||||
jpeg: () => instance,
|
||||
png: () => instance,
|
||||
webp: () => instance,
|
||||
toBuffer: async () => Buffer.from('resized-image'),
|
||||
}
|
||||
return instance
|
||||
}
|
||||
},
|
||||
}))
|
||||
|
||||
const { maybeResizeAndDownsampleImageBuffer } = await import(
|
||||
'../src/utils/imageResizer.js'
|
||||
)
|
||||
|
||||
function makePngHeader(width: number, height: number): Buffer {
|
||||
const buffer = Buffer.alloc(32)
|
||||
buffer[0] = 0x89
|
||||
buffer[1] = 0x50
|
||||
buffer[2] = 0x4e
|
||||
buffer[3] = 0x47
|
||||
buffer.writeUInt32BE(width, 16)
|
||||
buffer.writeUInt32BE(height, 20)
|
||||
return buffer
|
||||
}
|
||||
|
||||
beforeEach(() => {
|
||||
processorMode = 'metadata'
|
||||
resizeCalls = []
|
||||
})
|
||||
|
||||
describe('maybeResizeAndDownsampleImageBuffer', () => {
|
||||
test('passes through a tall screenshot when bytes are already within API limits', async () => {
|
||||
const imageBuffer = Buffer.alloc(1024, 1)
|
||||
|
||||
const result = await maybeResizeAndDownsampleImageBuffer(
|
||||
imageBuffer,
|
||||
imageBuffer.length,
|
||||
'png',
|
||||
)
|
||||
|
||||
expect(result.buffer).toBe(imageBuffer)
|
||||
expect(result.mediaType).toBe('png')
|
||||
expect(resizeCalls).toEqual([])
|
||||
expect(result.dimensions).toEqual({
|
||||
originalWidth: 1394,
|
||||
originalHeight: 4404,
|
||||
displayWidth: 1394,
|
||||
displayHeight: 4404,
|
||||
})
|
||||
})
|
||||
|
||||
test('falls back to the original tall screenshot if local image processing is unavailable', async () => {
|
||||
processorMode = 'throw'
|
||||
const imageBuffer = makePngHeader(1394, 4404)
|
||||
|
||||
const result = await maybeResizeAndDownsampleImageBuffer(
|
||||
imageBuffer,
|
||||
imageBuffer.length,
|
||||
'png',
|
||||
)
|
||||
|
||||
expect(result.buffer).toBe(imageBuffer)
|
||||
expect(result.mediaType).toBe('png')
|
||||
})
|
||||
})
|
||||
Loading…
x
Reference in New Issue
Block a user