diff --git a/src/tools/FileReadTool/FileReadTool.ts b/src/tools/FileReadTool/FileReadTool.ts index 91f7766a..37b31f7d 100644 --- a/src/tools/FileReadTool/FileReadTool.ts +++ b/src/tools/FileReadTool/FileReadTool.ts @@ -44,6 +44,7 @@ import { compressImageBufferWithTokenLimit, createImageMetadataText, detectImageFormatFromBuffer, + downsampleImageBufferToVisionTokenBudget, type ImageDimensions, ImageResizeError, maybeResizeAndDownsampleImageBuffer, @@ -798,6 +799,18 @@ function createImageResponse( } } +function estimateVisionImageTokens(dimensions?: ImageDimensions): number | null { + const width = dimensions?.displayWidth + const height = dimensions?.displayHeight + if (!width || !height || width <= 0 || height <= 0) { + return null + } + // Claude vision charges approximately width * height / 750 image tokens. + // Image blocks are not base64 text blocks, so using base64 length here would + // massively over-compress ordinary screenshots before the model sees them. + return Math.ceil((width * height) / 750) +} + /** * Inner implementation of call, separated to allow ENOENT handling in the outer call. */ @@ -1133,10 +1146,31 @@ export async function readImageWithTokenBudget( result = createImageResponse(imageBuffer, detectedFormat, originalSize) } - // Check if it fits in token budget - const estimatedTokens = Math.ceil(result.file.base64.length * 0.125) - if (estimatedTokens > maxTokens) { - // Aggressive compression from the SAME buffer (no re-read) + // Check if it fits in vision token budget. This is intentionally based on + // image dimensions, not base64 payload length, because the tool result is an + // image content block rather than text. + const estimatedTokens = estimateVisionImageTokens(result.file.dimensions) + if (estimatedTokens !== null && estimatedTokens > maxTokens) { + // Downsample by vision pixel budget first. This preserves detail far better + // than the legacy base64-text token heuristic for image blocks. + try { + const downsampled = await downsampleImageBufferToVisionTokenBudget( + imageBuffer, + originalSize, + detectedFormat, + maxTokens, + ) + return createImageResponse( + downsampled.buffer, + downsampled.mediaType, + originalSize, + downsampled.dimensions, + ) + } catch (e) { + logError(e) + } + + // Compatibility fallback from the SAME buffer (no re-read) try { const compressed = await compressImageBufferWithTokenLimit( imageBuffer, diff --git a/src/tools/FileReadTool/imageProcessor.ts b/src/tools/FileReadTool/imageProcessor.ts index 20afa900..6d433dd9 100644 --- a/src/tools/FileReadTool/imageProcessor.ts +++ b/src/tools/FileReadTool/imageProcessor.ts @@ -14,7 +14,7 @@ export type SharpInstance = { palette?: boolean colors?: number }): SharpInstance - webp(options?: { quality?: number }): SharpInstance + webp(options?: { quality?: number; lossless?: boolean }): SharpInstance toBuffer(): Promise } diff --git a/src/utils/__tests__/imageResizer.test.ts b/src/utils/__tests__/imageResizer.test.ts new file mode 100644 index 00000000..c653718d --- /dev/null +++ b/src/utils/__tests__/imageResizer.test.ts @@ -0,0 +1,437 @@ +import { beforeEach, describe, expect, mock, test } from 'bun:test' +import { rm, writeFile } from 'node:fs/promises' +import { tmpdir } from 'node:os' +import { join } from 'node:path' + +type ProcessorMode = 'metadata' | 'throw' + +let processorMode: ProcessorMode = 'metadata' +let resizeCalls: Array<{ width: number; height: number }> = [] +let jpegCalls: Array<{ quality?: number; resized: boolean }> = [] +let pngCalls: Array<{ compressionLevel?: number; palette?: boolean }> = [] +let webpCalls: Array<{ quality?: number; lossless?: boolean }> = [] + +let originalPngDimensions: { width?: number; height?: number; format?: string } = + { width: 1394, height: 4404, format: 'png' } +let outputForOperations: (operations: Operation[]) => Buffer = () => + Buffer.from('resized-image') + +type Operation = + | { type: 'resize'; width: number; height: number } + | { type: 'jpeg'; quality?: number } + | { type: 'png'; compressionLevel?: number; palette?: boolean } + | { type: 'webp'; quality?: number; lossless?: boolean } + +mock.module('../../tools/FileReadTool/imageProcessor.js', () => ({ + getImageProcessor: async () => { + if (processorMode === 'throw') { + throw new Error('image processor unavailable') + } + + return (_input: Buffer) => { + const operations: Operation[] = [] + const instance = { + metadata: async () => originalPngDimensions, + resize: (width: number, height: number) => { + resizeCalls.push({ width, height }) + operations.push({ type: 'resize', width, height }) + return instance + }, + jpeg: (options?: { quality?: number }) => { + jpegCalls.push({ + quality: options?.quality, + resized: operations.some(op => op.type === 'resize'), + }) + operations.push({ type: 'jpeg', quality: options?.quality }) + return instance + }, + png: (options?: { compressionLevel?: number; palette?: boolean }) => { + pngCalls.push(options ?? {}) + operations.push({ + type: 'png', + compressionLevel: options?.compressionLevel, + palette: options?.palette, + }) + return instance + }, + webp: (options?: { quality?: number; lossless?: boolean }) => { + webpCalls.push(options ?? {}) + operations.push({ + type: 'webp', + quality: options?.quality, + lossless: options?.lossless, + }) + return instance + }, + toBuffer: async () => outputForOperations(operations), + } + return instance + } + }, +})) + +const { + ImageResizeError, + downsampleImageBufferToVisionTokenBudget, + maybeResizeAndDownsampleImageBuffer, +} = await import('../imageResizer.js') +const { readImageWithTokenBudget } = await import( + '../../tools/FileReadTool/FileReadTool.js' +) + +function makePngHeader(width: number, height: number): Buffer { + const buffer = Buffer.alloc(32) + buffer[0] = 0x89 + buffer[1] = 0x50 + buffer[2] = 0x4e + buffer[3] = 0x47 + buffer.writeUInt32BE(width, 16) + buffer.writeUInt32BE(height, 20) + return buffer +} + +beforeEach(() => { + processorMode = 'metadata' + resizeCalls = [] + jpegCalls = [] + pngCalls = [] + webpCalls = [] + originalPngDimensions = { width: 1394, height: 4404, format: 'png' } + outputForOperations = () => Buffer.from('resized-image') +}) + +describe('maybeResizeAndDownsampleImageBuffer', () => { + test('passes through a tall screenshot when bytes are already within API limits', async () => { + const imageBuffer = Buffer.alloc(1024, 1) + + const result = await maybeResizeAndDownsampleImageBuffer( + imageBuffer, + imageBuffer.length, + 'png', + ) + + expect(result.buffer).toBe(imageBuffer) + expect(result.mediaType).toBe('png') + expect(resizeCalls).toEqual([]) + expect(result.dimensions).toEqual({ + originalWidth: 1394, + originalHeight: 4404, + displayWidth: 1394, + displayHeight: 4404, + }) + }) + + test('falls back to the original tall screenshot if local image processing is unavailable', async () => { + processorMode = 'throw' + const imageBuffer = makePngHeader(1394, 4404) + + const result = await maybeResizeAndDownsampleImageBuffer( + imageBuffer, + imageBuffer.length, + 'png', + ) + + expect(result.buffer).toBe(imageBuffer) + expect(result.mediaType).toBe('png') + }) + + test('tries lossless webp compression before lossy jpeg for oversized screenshots', async () => { + originalPngDimensions = { width: 4096, height: 2304, format: 'png' } + outputForOperations = operations => { + if (operations.some(op => op.type === 'webp')) { + return Buffer.alloc(1024, 3) + } + return Buffer.alloc(4 * 1024 * 1024, 2) + } + const imageBuffer = Buffer.alloc(4 * 1024 * 1024, 1) + + const result = await maybeResizeAndDownsampleImageBuffer( + imageBuffer, + imageBuffer.length, + 'png', + ) + + expect(result.mediaType).toBe('webp') + expect(result.buffer.length).toBe(1024) + expect(resizeCalls).toEqual([]) + expect(webpCalls).toEqual([{ lossless: true }]) + expect(jpegCalls).toEqual([]) + expect(result.dimensions).toEqual({ + originalWidth: 4096, + originalHeight: 2304, + displayWidth: 4096, + displayHeight: 2304, + }) + }) + + test('downsamples to a codex-sized long edge before lowering jpeg below readable quality', async () => { + originalPngDimensions = { width: 5000, height: 3000, format: 'png' } + outputForOperations = operations => { + const resizeOp = operations.find( + (op): op is Extract => + op.type === 'resize', + ) + const jpegOp = operations.find( + (op): op is Extract => + op.type === 'jpeg', + ) + if (resizeOp?.width === 2048 && resizeOp.height === 1229) { + if (jpegOp?.quality === 85) { + return Buffer.alloc(1024, 4) + } + } + return Buffer.alloc(6 * 1024 * 1024, 5) + } + const imageBuffer = Buffer.alloc(6 * 1024 * 1024, 1) + + const result = await maybeResizeAndDownsampleImageBuffer( + imageBuffer, + imageBuffer.length, + 'png', + ) + + expect(result.mediaType).toBe('jpeg') + expect(result.buffer.length).toBe(1024) + expect(resizeCalls).toContainEqual({ width: 2048, height: 1229 }) + expect(jpegCalls.some(call => (call.quality ?? 100) < 75)).toBe(false) + expect(result.dimensions).toEqual({ + originalWidth: 5000, + originalHeight: 3000, + displayWidth: 2048, + displayHeight: 1229, + }) + }) + + test('resizes to the hard dimension cap when only height exceeds the API limit', async () => { + originalPngDimensions = { width: 1000, height: 9000, format: 'png' } + outputForOperations = operations => { + const resizeOp = operations.find( + (op): op is Extract => + op.type === 'resize', + ) + const jpegOp = operations.find( + (op): op is Extract => + op.type === 'jpeg', + ) + if ( + resizeOp?.width === 889 && + resizeOp.height === 8000 && + jpegOp?.quality === 85 + ) { + return Buffer.alloc(1024, 6) + } + return Buffer.alloc(6 * 1024 * 1024, 5) + } + const imageBuffer = Buffer.alloc(200_000, 1) + + const result = await maybeResizeAndDownsampleImageBuffer( + imageBuffer, + imageBuffer.length, + 'png', + ) + + expect(result.mediaType).toBe('jpeg') + expect(resizeCalls).toContainEqual({ width: 889, height: 8000 }) + expect(result.dimensions).toEqual({ + originalWidth: 1000, + originalHeight: 9000, + displayWidth: 889, + displayHeight: 8000, + }) + }) + + test('uses readable fallback dimensions before dropping to fallback jpeg quality', async () => { + originalPngDimensions = { width: 6000, height: 5000, format: 'png' } + outputForOperations = operations => { + const resizeOp = operations.find( + (op): op is Extract => + op.type === 'resize', + ) + const jpegOp = operations.find( + (op): op is Extract => + op.type === 'jpeg', + ) + if ( + resizeOp?.width === 1568 && + resizeOp.height === 1307 && + jpegOp?.quality === 65 + ) { + return Buffer.alloc(1024, 6) + } + return Buffer.alloc(6 * 1024 * 1024, 5) + } + const imageBuffer = Buffer.alloc(6 * 1024 * 1024, 1) + + const result = await maybeResizeAndDownsampleImageBuffer( + imageBuffer, + imageBuffer.length, + 'png', + ) + + expect(result.mediaType).toBe('jpeg') + expect(resizeCalls).toContainEqual({ width: 2048, height: 1707 }) + expect(resizeCalls).toContainEqual({ width: 1568, height: 1307 }) + expect(jpegCalls.some(call => call.quality === 65)).toBe(true) + expect(result.dimensions).toEqual({ + originalWidth: 6000, + originalHeight: 5000, + displayWidth: 1568, + displayHeight: 1307, + }) + }) + + test('throws instead of over-compressing when readable fallback cannot fit', async () => { + originalPngDimensions = { width: 6000, height: 5000, format: 'png' } + outputForOperations = () => Buffer.alloc(6 * 1024 * 1024, 5) + const imageBuffer = Buffer.alloc(6 * 1024 * 1024, 1) + + expect( + maybeResizeAndDownsampleImageBuffer( + imageBuffer, + imageBuffer.length, + 'png', + ), + ).rejects.toBeInstanceOf(ImageResizeError) + }) + + test('tries lossless webp for oversized webp screenshots before jpeg conversion', async () => { + originalPngDimensions = { width: 4096, height: 2304, format: 'webp' } + outputForOperations = operations => { + if (operations.some(op => op.type === 'webp')) { + return Buffer.alloc(1024, 3) + } + return Buffer.alloc(4 * 1024 * 1024, 2) + } + const imageBuffer = Buffer.alloc(4 * 1024 * 1024, 1) + + const result = await maybeResizeAndDownsampleImageBuffer( + imageBuffer, + imageBuffer.length, + 'webp', + ) + + expect(result.mediaType).toBe('webp') + expect(webpCalls).toEqual([{ lossless: true }]) + expect(jpegCalls).toEqual([]) + }) +}) + +describe('readImageWithTokenBudget', () => { + test('does not recompress readable screenshots just because their base64 text would exceed maxTokens', async () => { + originalPngDimensions = { width: 1920, height: 1080, format: 'png' } + outputForOperations = () => Buffer.from('compressed-image') + const imageBuffer = Buffer.alloc(200_000, 1) + const imagePath = join(tmpdir(), `cc-haha-image-budget-${Date.now()}.png`) + await writeFile(imagePath, imageBuffer) + + try { + const result = await readImageWithTokenBudget(imagePath, 25_000) + + expect(result.file.base64).toBe(imageBuffer.toString('base64')) + expect(result.file.type).toBe('image/png') + expect(jpegCalls).toEqual([]) + expect(resizeCalls).toEqual([]) + } finally { + await rm(imagePath, { force: true }) + } + }) + + test('downsamples over-budget vision images by pixel budget instead of base64 text budget', async () => { + originalPngDimensions = { width: 6000, height: 4000, format: 'png' } + outputForOperations = operations => { + const resizeOp = operations.find( + (op): op is Extract => + op.type === 'resize', + ) + const jpegOp = operations.find( + (op): op is Extract => + op.type === 'jpeg', + ) + if ( + resizeOp?.width === 3674 && + resizeOp.height === 2449 && + jpegOp?.quality === 85 + ) { + return Buffer.alloc(1024, 7) + } + return Buffer.alloc(6 * 1024 * 1024, 8) + } + const imageBuffer = Buffer.alloc(200_000, 1) + const imagePath = join( + tmpdir(), + `cc-haha-image-vision-budget-${Date.now()}.png`, + ) + await writeFile(imagePath, imageBuffer) + + try { + const result = await readImageWithTokenBudget(imagePath, 12_000) + + expect(result.file.type).toBe('image/jpeg') + expect(Buffer.from(result.file.base64, 'base64').length).toBe(1024) + expect(resizeCalls).toContainEqual({ width: 3674, height: 2449 }) + expect(jpegCalls.some(call => (call.quality ?? 100) < 75)).toBe(false) + expect(result.file.dimensions).toEqual({ + originalWidth: 6000, + originalHeight: 4000, + displayWidth: 3674, + displayHeight: 2449, + }) + } finally { + await rm(imagePath, { force: true }) + } + }) + + test('uses a readable fallback when the vision-budget resize cannot fit the byte cap', async () => { + originalPngDimensions = { width: 6000, height: 4000, format: 'png' } + outputForOperations = operations => { + const resizeOp = operations.find( + (op): op is Extract => + op.type === 'resize', + ) + const jpegOp = operations.find( + (op): op is Extract => + op.type === 'jpeg', + ) + if ( + resizeOp?.width === 1568 && + resizeOp.height === 1045 && + jpegOp?.quality === 65 + ) { + return Buffer.alloc(1024, 7) + } + return Buffer.alloc(6 * 1024 * 1024, 8) + } + const imageBuffer = Buffer.alloc(200_000, 1) + + const result = await downsampleImageBufferToVisionTokenBudget( + imageBuffer, + imageBuffer.length, + 'png', + 12_000, + ) + + expect(result.mediaType).toBe('jpeg') + expect(resizeCalls).toContainEqual({ width: 3674, height: 2449 }) + expect(resizeCalls).toContainEqual({ width: 1568, height: 1045 }) + expect(result.dimensions).toEqual({ + originalWidth: 6000, + originalHeight: 4000, + displayWidth: 1568, + displayHeight: 1045, + }) + }) + + test('reports an image resize error when token-budget dimensions are unavailable', async () => { + originalPngDimensions = { format: 'png' } + const imageBuffer = Buffer.alloc(200_000, 1) + + expect( + downsampleImageBufferToVisionTokenBudget( + imageBuffer, + imageBuffer.length, + 'png', + 12_000, + ), + ).rejects.toBeInstanceOf(ImageResizeError) + }) +}) diff --git a/src/utils/imageResizer.ts b/src/utils/imageResizer.ts index 149b4087..bdfd61f2 100644 --- a/src/utils/imageResizer.ts +++ b/src/utils/imageResizer.ts @@ -162,6 +162,16 @@ interface CompressedImageResult { originalSize: number } +type ResizeDimensions = { + width: number + height: number +} + +const IMAGE_PROMPT_SOFT_LONG_EDGE = 2048 +const IMAGE_PROMPT_READABLE_LONG_EDGE = 1568 +const READABLE_JPEG_QUALITIES = [85, 75] as const +const FALLBACK_JPEG_QUALITIES = [65] as const + /** * Extracted from FileReadTool's readImage function * Resizes image buffer to meet size and dimension constraints @@ -226,160 +236,99 @@ export async function maybeResizeAndDownsampleImageBuffer( } } + const hardDims = constrainDimensions( + width, + height, + IMAGE_MAX_WIDTH, + IMAGE_MAX_HEIGHT, + ) const needsDimensionResize = - width > IMAGE_MAX_WIDTH || height > IMAGE_MAX_HEIGHT - const isPng = normalizedMediaType === 'png' + hardDims.width !== width || hardDims.height !== height - // If dimensions are within limits but file is too large, try compression first - // This preserves full resolution when possible + // Try lossless or high-quality compression at original dimensions first. + // This keeps screenshot text sharp when the only problem is payload size. if (!needsDimensionResize && originalSize > IMAGE_TARGET_RAW_SIZE) { - // For PNGs, try PNG compression first to preserve transparency - if (isPng) { - // Create fresh sharp instance for each compression attempt - const pngCompressed = await sharp(imageBuffer) - .png({ compressionLevel: 9, palette: true }) - .toBuffer() - if (pngCompressed.length <= IMAGE_TARGET_RAW_SIZE) { - return { - buffer: pngCompressed, - mediaType: 'png', - dimensions: { - originalWidth, - originalHeight, - displayWidth: width, - displayHeight: height, - }, - } - } - } - // Try JPEG compression (lossy but much smaller) - for (const quality of [80, 60, 40, 20]) { - // Create fresh sharp instance for each attempt - const compressedBuffer = await sharp(imageBuffer) - .jpeg({ quality }) - .toBuffer() - if (compressedBuffer.length <= IMAGE_TARGET_RAW_SIZE) { - return { - buffer: compressedBuffer, - mediaType: 'jpeg', - dimensions: { - originalWidth, - originalHeight, - displayWidth: width, - displayHeight: height, - }, - } - } - } - // Quality reduction alone wasn't enough, fall through to resize - } - - // Constrain dimensions if needed - if (width > IMAGE_MAX_WIDTH) { - height = Math.round((height * IMAGE_MAX_WIDTH) / width) - width = IMAGE_MAX_WIDTH - } - - if (height > IMAGE_MAX_HEIGHT) { - width = Math.round((width * IMAGE_MAX_HEIGHT) / height) - height = IMAGE_MAX_HEIGHT - } - - // IMPORTANT: Always create fresh sharp(imageBuffer) instances for each operation. - // The native image-processor-napi module doesn't properly apply format conversions - // when reusing a sharp instance after calling toBuffer(). This caused a bug where - // all compression attempts (PNG, JPEG at various qualities) returned identical sizes. - logForDebugging(`Resizing to ${width}x${height}`) - const resizedImageBuffer = await sharp(imageBuffer) - .resize(width, height, { - fit: 'inside', - withoutEnlargement: true, - }) - .toBuffer() - - // If still too large after resize, try compression - if (resizedImageBuffer.length > IMAGE_TARGET_RAW_SIZE) { - // For PNGs, try PNG compression first to preserve transparency - if (isPng) { - const pngCompressed = await sharp(imageBuffer) - .resize(width, height, { - fit: 'inside', - withoutEnlargement: true, - }) - .png({ compressionLevel: 9, palette: true }) - .toBuffer() - if (pngCompressed.length <= IMAGE_TARGET_RAW_SIZE) { - return { - buffer: pngCompressed, - mediaType: 'png', - dimensions: { - originalWidth, - originalHeight, - displayWidth: width, - displayHeight: height, - }, - } - } - } - - // Try JPEG with progressively lower quality - for (const quality of [80, 60, 40, 20]) { - const compressedBuffer = await sharp(imageBuffer) - .resize(width, height, { - fit: 'inside', - withoutEnlargement: true, - }) - .jpeg({ quality }) - .toBuffer() - if (compressedBuffer.length <= IMAGE_TARGET_RAW_SIZE) { - return { - buffer: compressedBuffer, - mediaType: 'jpeg', - dimensions: { - originalWidth, - originalHeight, - displayWidth: width, - displayHeight: height, - }, - } - } - } - // If still too large, resize smaller and compress aggressively - const smallerWidth = Math.min(width, 1000) - const smallerHeight = Math.round( - (height * smallerWidth) / Math.max(width, 1), - ) - logForDebugging('Still too large, compressing with JPEG') - const compressedBuffer = await sharp(imageBuffer) - .resize(smallerWidth, smallerHeight, { - fit: 'inside', - withoutEnlargement: true, - }) - .jpeg({ quality: 20 }) - .toBuffer() - logForDebugging(`JPEG compressed buffer size: ${compressedBuffer.length}`) - return { - buffer: compressedBuffer, - mediaType: 'jpeg', - dimensions: { - originalWidth, - originalHeight, - displayWidth: smallerWidth, - displayHeight: smallerHeight, - }, - } - } - - return { - buffer: resizedImageBuffer, - mediaType: normalizedMediaType, - dimensions: { + const compressed = await tryEncodeImageCandidate({ + sharp, + imageBuffer, originalWidth, originalHeight, - displayWidth: width, - displayHeight: height, - }, + dimensions: { width, height }, + format: normalizedMediaType, + maxBytes: IMAGE_TARGET_RAW_SIZE, + }) + if (compressed) return compressed } + + if (needsDimensionResize) { + logForDebugging(`Resizing to ${hardDims.width}x${hardDims.height}`) + const hardResize = await tryEncodeImageCandidate({ + sharp, + imageBuffer, + originalWidth, + originalHeight, + dimensions: hardDims, + format: normalizedMediaType, + maxBytes: IMAGE_TARGET_RAW_SIZE, + }) + if (hardResize) return hardResize + } + + const softDims = constrainDimensions( + hardDims.width, + hardDims.height, + IMAGE_PROMPT_SOFT_LONG_EDGE, + IMAGE_PROMPT_SOFT_LONG_EDGE, + ) + if ( + softDims.width !== hardDims.width || + softDims.height !== hardDims.height + ) { + logForDebugging( + `Downsampling image to ${softDims.width}x${softDims.height}`, + ) + const softResize = await tryEncodeImageCandidate({ + sharp, + imageBuffer, + originalWidth, + originalHeight, + dimensions: softDims, + format: normalizedMediaType, + maxBytes: IMAGE_TARGET_RAW_SIZE, + }) + if (softResize) return softResize + } + + const readableDims = constrainDimensions( + hardDims.width, + hardDims.height, + IMAGE_PROMPT_READABLE_LONG_EDGE, + IMAGE_PROMPT_READABLE_LONG_EDGE, + ) + if ( + readableDims.width !== softDims.width || + readableDims.height !== softDims.height + ) { + logForDebugging( + `Downsampling image to readable fallback ${readableDims.width}x${readableDims.height}`, + ) + const readableResize = await tryEncodeImageCandidate({ + sharp, + imageBuffer, + originalWidth, + originalHeight, + dimensions: readableDims, + format: normalizedMediaType, + maxBytes: IMAGE_TARGET_RAW_SIZE, + allowFallbackQuality: true, + }) + if (readableResize) return readableResize + } + + throw new ImageResizeError( + `Unable to compress image (${formatFileSize(originalSize)}) below the 5MB API base64 limit without over-compressing it. ` + + `Please resize the image manually or use a smaller image.`, + ) } catch (error) { // Log the error and emit analytics event logError(error as Error) @@ -432,6 +381,277 @@ export async function maybeResizeAndDownsampleImageBuffer( } } +function constrainDimensions( + width: number, + height: number, + maxWidth: number, + maxHeight: number, +): ResizeDimensions { + let nextWidth = width + let nextHeight = height + + if (nextWidth > maxWidth) { + nextHeight = Math.round((nextHeight * maxWidth) / nextWidth) + nextWidth = maxWidth + } + + if (nextHeight > maxHeight) { + nextWidth = Math.round((nextWidth * maxHeight) / nextHeight) + nextHeight = maxHeight + } + + return { + width: Math.max(1, nextWidth), + height: Math.max(1, nextHeight), + } +} + +function constrainDimensionsToPixelBudget( + width: number, + height: number, + maxPixels: number, +): ResizeDimensions { + if (width * height <= maxPixels) { + return { width, height } + } + + const scale = Math.sqrt(maxPixels / (width * height)) + let nextWidth = Math.max(1, Math.round(width * scale)) + let nextHeight = Math.max(1, Math.round(height * scale)) + + while (nextWidth * nextHeight > maxPixels) { + if (nextWidth >= nextHeight) { + nextWidth -= 1 + } else { + nextHeight -= 1 + } + } + + return { + width: nextWidth, + height: nextHeight, + } +} + +function createResizeResult( + buffer: Buffer, + mediaType: string, + originalWidth: number, + originalHeight: number, + dimensions: ResizeDimensions, +): ResizeResult { + return { + buffer, + mediaType, + dimensions: { + originalWidth, + originalHeight, + displayWidth: dimensions.width, + displayHeight: dimensions.height, + }, + } +} + +function createImagePipeline( + sharp: SharpFunction, + imageBuffer: Buffer, + originalWidth: number, + originalHeight: number, + dimensions: ResizeDimensions, +): SharpInstance { + const pipeline = sharp(imageBuffer) + if ( + dimensions.width === originalWidth && + dimensions.height === originalHeight + ) { + return pipeline + } + + return pipeline.resize(dimensions.width, dimensions.height, { + fit: 'inside', + withoutEnlargement: true, + }) +} + +async function tryEncodeImageCandidate({ + sharp, + imageBuffer, + originalWidth, + originalHeight, + dimensions, + format, + maxBytes, + allowFallbackQuality = false, +}: { + sharp: SharpFunction + imageBuffer: Buffer + originalWidth: number + originalHeight: number + dimensions: ResizeDimensions + format: string + maxBytes: number + allowFallbackQuality?: boolean +}): Promise { + const normalizedFormat = format === 'jpg' ? 'jpeg' : format + + if (normalizedFormat === 'png') { + const pngCompressed = await createImagePipeline( + sharp, + imageBuffer, + originalWidth, + originalHeight, + dimensions, + ) + .png({ compressionLevel: 9 }) + .toBuffer() + if (pngCompressed.length <= maxBytes) { + return createResizeResult( + pngCompressed, + 'png', + originalWidth, + originalHeight, + dimensions, + ) + } + + const webpLossless = await createImagePipeline( + sharp, + imageBuffer, + originalWidth, + originalHeight, + dimensions, + ) + .webp({ lossless: true }) + .toBuffer() + if (webpLossless.length <= maxBytes) { + return createResizeResult( + webpLossless, + 'webp', + originalWidth, + originalHeight, + dimensions, + ) + } + } else if (normalizedFormat === 'webp') { + const webpLossless = await createImagePipeline( + sharp, + imageBuffer, + originalWidth, + originalHeight, + dimensions, + ) + .webp({ lossless: true }) + .toBuffer() + if (webpLossless.length <= maxBytes) { + return createResizeResult( + webpLossless, + 'webp', + originalWidth, + originalHeight, + dimensions, + ) + } + } + + const jpegQualities = allowFallbackQuality + ? [...READABLE_JPEG_QUALITIES, ...FALLBACK_JPEG_QUALITIES] + : READABLE_JPEG_QUALITIES + for (const quality of jpegQualities) { + const jpegBuffer = await createImagePipeline( + sharp, + imageBuffer, + originalWidth, + originalHeight, + dimensions, + ) + .jpeg({ quality }) + .toBuffer() + if (jpegBuffer.length <= maxBytes) { + return createResizeResult( + jpegBuffer, + 'jpeg', + originalWidth, + originalHeight, + dimensions, + ) + } + } + + return null +} + +export async function downsampleImageBufferToVisionTokenBudget( + imageBuffer: Buffer, + originalSize: number, + ext: string, + maxTokens: number, +): Promise { + const sharp = await getImageProcessor() + const metadata = await sharp(imageBuffer).metadata() + const format = metadata.format ?? ext + const normalizedMediaType = format === 'jpg' ? 'jpeg' : format + + if (!metadata.width || !metadata.height) { + throw new ImageResizeError( + 'Unable to downsample image for token budget because dimensions are unavailable.', + ) + } + + const originalWidth = metadata.width + const originalHeight = metadata.height + const hardDims = constrainDimensions( + originalWidth, + originalHeight, + IMAGE_MAX_WIDTH, + IMAGE_MAX_HEIGHT, + ) + const maxPixels = Math.max(1, maxTokens * 750) + const tokenDims = constrainDimensionsToPixelBudget( + hardDims.width, + hardDims.height, + maxPixels, + ) + + const tokenResize = await tryEncodeImageCandidate({ + sharp, + imageBuffer, + originalWidth, + originalHeight, + dimensions: tokenDims, + format: normalizedMediaType, + maxBytes: IMAGE_TARGET_RAW_SIZE, + allowFallbackQuality: true, + }) + if (tokenResize) return tokenResize + + const readableDims = constrainDimensions( + tokenDims.width, + tokenDims.height, + IMAGE_PROMPT_READABLE_LONG_EDGE, + IMAGE_PROMPT_READABLE_LONG_EDGE, + ) + if ( + readableDims.width !== tokenDims.width || + readableDims.height !== tokenDims.height + ) { + const readableResize = await tryEncodeImageCandidate({ + sharp, + imageBuffer, + originalWidth, + originalHeight, + dimensions: readableDims, + format: normalizedMediaType, + maxBytes: IMAGE_TARGET_RAW_SIZE, + allowFallbackQuality: true, + }) + if (readableResize) return readableResize + } + + throw new ImageResizeError( + `Unable to downsample image (${formatFileSize(originalSize)}) below the ${maxTokens} token budget without over-compressing it.`, + ) +} + export interface ImageBlockWithDimensions { block: ImageBlockParam dimensions?: ImageDimensions diff --git a/tests/imageResizer.test.ts b/tests/imageResizer.test.ts deleted file mode 100644 index 62b010c8..00000000 --- a/tests/imageResizer.test.ts +++ /dev/null @@ -1,87 +0,0 @@ -import { beforeEach, describe, expect, mock, test } from 'bun:test' - -type ProcessorMode = 'metadata' | 'throw' - -let processorMode: ProcessorMode = 'metadata' -let resizeCalls: Array<{ width: number; height: number }> = [] - -const originalPngDimensions = { width: 1394, height: 4404, format: 'png' } - -mock.module('../src/tools/FileReadTool/imageProcessor.js', () => ({ - getImageProcessor: async () => { - if (processorMode === 'throw') { - throw new Error('image processor unavailable') - } - - return (_input: Buffer) => { - const instance = { - metadata: async () => originalPngDimensions, - resize: (width: number, height: number) => { - resizeCalls.push({ width, height }) - return instance - }, - jpeg: () => instance, - png: () => instance, - webp: () => instance, - toBuffer: async () => Buffer.from('resized-image'), - } - return instance - } - }, -})) - -const { maybeResizeAndDownsampleImageBuffer } = await import( - '../src/utils/imageResizer.js' -) - -function makePngHeader(width: number, height: number): Buffer { - const buffer = Buffer.alloc(32) - buffer[0] = 0x89 - buffer[1] = 0x50 - buffer[2] = 0x4e - buffer[3] = 0x47 - buffer.writeUInt32BE(width, 16) - buffer.writeUInt32BE(height, 20) - return buffer -} - -beforeEach(() => { - processorMode = 'metadata' - resizeCalls = [] -}) - -describe('maybeResizeAndDownsampleImageBuffer', () => { - test('passes through a tall screenshot when bytes are already within API limits', async () => { - const imageBuffer = Buffer.alloc(1024, 1) - - const result = await maybeResizeAndDownsampleImageBuffer( - imageBuffer, - imageBuffer.length, - 'png', - ) - - expect(result.buffer).toBe(imageBuffer) - expect(result.mediaType).toBe('png') - expect(resizeCalls).toEqual([]) - expect(result.dimensions).toEqual({ - originalWidth: 1394, - originalHeight: 4404, - displayWidth: 1394, - displayHeight: 4404, - }) - }) - - test('falls back to the original tall screenshot if local image processing is unavailable', async () => { - processorMode = 'throw' - const imageBuffer = makePngHeader(1394, 4404) - - const result = await maybeResizeAndDownsampleImageBuffer( - imageBuffer, - imageBuffer.length, - 'png', - ) - - expect(result.buffer).toBe(imageBuffer) - expect(result.mediaType).toBe('png') - }) -})