cc-haha/src/utils/imageResizer.ts
程序员阿江(Relakkes) 46ca911d1a fix: preserve readable large image inputs
Large screenshots were being treated like oversized text payloads,
which pushed readable images through an aggressive resize/compression
path before the model saw them. The image path now preserves images
that already fit API byte and dimension limits, prefers lossless or
high-quality compression when only bytes are too large, and only
downscales by vision pixel budget when the image would exceed the
Read token budget.

Constraint: Fixes pasted/read image failures reported in #615 and #677 while preserving screenshot readability requested in #663
Rejected: Always resize to 2048px | would unnecessarily degrade screenshots that already fit provider limits
Rejected: Continue estimating image budget from base64 text length | image blocks are billed/read as vision pixels, not text payload
Confidence: high
Scope-risk: moderate
Directive: Do not lower fallback JPEG quality or long-edge limits without revalidating real screenshot OCR/vision tasks
Tested: bun test src/utils/__tests__/imageResizer.test.ts src/server/__tests__/conversation-attachments.test.ts
Tested: bun run check:server
Tested: bun run check:coverage (changed-line coverage 95.25%; adapters/desktop coverage lanes failed from existing non-target issues)
Tested: GPT-5.5 Read smoke on 1394x4404, 4096x2304, and 7900x900 images
Not-tested: Real resize/re-encode branch in this source worktree because optional sharp/native image processor is not installed locally
Related: #615
Related: #677
Related: #663
2026-06-01 20:28:05 +08:00

1101 lines
30 KiB
TypeScript

import type {
Base64ImageSource,
ImageBlockParam,
} from '@anthropic-ai/sdk/resources/messages.mjs'
import {
API_IMAGE_MAX_BASE64_SIZE,
IMAGE_MAX_HEIGHT,
IMAGE_MAX_WIDTH,
IMAGE_TARGET_RAW_SIZE,
} from '../constants/apiLimits.js'
import { logEvent } from '../services/analytics/index.js'
import {
getImageProcessor,
type SharpFunction,
type SharpInstance,
} from '../tools/FileReadTool/imageProcessor.js'
import { logForDebugging } from './debug.js'
import { errorMessage } from './errors.js'
import { formatFileSize } from './format.js'
import { logError } from './log.js'
type ImageMediaType = 'image/png' | 'image/jpeg' | 'image/gif' | 'image/webp'
// Error type constants for analytics (numeric to comply with logEvent restrictions)
const ERROR_TYPE_MODULE_LOAD = 1
const ERROR_TYPE_PROCESSING = 2
const ERROR_TYPE_UNKNOWN = 3
const ERROR_TYPE_PIXEL_LIMIT = 4
const ERROR_TYPE_MEMORY = 5
const ERROR_TYPE_TIMEOUT = 6
const ERROR_TYPE_VIPS = 7
const ERROR_TYPE_PERMISSION = 8
/**
* Error thrown when image resizing fails and the image exceeds the API limit.
*/
export class ImageResizeError extends Error {
constructor(message: string) {
super(message)
this.name = 'ImageResizeError'
}
}
/**
* Classifies image processing errors for analytics.
*
* Uses error codes when available (Node.js module errors), falls back to
* message matching for libraries like sharp that don't expose error codes.
*/
function classifyImageError(error: unknown): number {
// Check for Node.js error codes first (more reliable than string matching)
if (error instanceof Error) {
const errorWithCode = error as Error & { code?: string }
if (
errorWithCode.code === 'MODULE_NOT_FOUND' ||
errorWithCode.code === 'ERR_MODULE_NOT_FOUND' ||
errorWithCode.code === 'ERR_DLOPEN_FAILED'
) {
return ERROR_TYPE_MODULE_LOAD
}
if (errorWithCode.code === 'EACCES' || errorWithCode.code === 'EPERM') {
return ERROR_TYPE_PERMISSION
}
if (errorWithCode.code === 'ENOMEM') {
return ERROR_TYPE_MEMORY
}
}
// Fall back to message matching for errors without codes
// Note: sharp doesn't expose error codes, so we must match on messages
const message = errorMessage(error)
// Module loading errors from our native wrapper
if (message.includes('Native image processor module not available')) {
return ERROR_TYPE_MODULE_LOAD
}
// Sharp/vips processing errors (format detection, corrupt data, etc.)
if (
message.includes('unsupported image format') ||
message.includes('Input buffer') ||
message.includes('Input file is missing') ||
message.includes('Input file has corrupt header') ||
message.includes('corrupt header') ||
message.includes('corrupt image') ||
message.includes('premature end') ||
message.includes('zlib: data error') ||
message.includes('zero width') ||
message.includes('zero height')
) {
return ERROR_TYPE_PROCESSING
}
// Pixel/dimension limit errors from sharp/vips
if (
message.includes('pixel limit') ||
message.includes('too many pixels') ||
message.includes('exceeds pixel') ||
message.includes('image dimensions')
) {
return ERROR_TYPE_PIXEL_LIMIT
}
// Memory allocation failures
if (
message.includes('out of memory') ||
message.includes('Cannot allocate') ||
message.includes('memory allocation')
) {
return ERROR_TYPE_MEMORY
}
// Timeout errors
if (message.includes('timeout') || message.includes('timed out')) {
return ERROR_TYPE_TIMEOUT
}
// Vips-specific errors (VipsJpeg, VipsPng, VipsWebp, etc.)
if (message.includes('Vips')) {
return ERROR_TYPE_VIPS
}
return ERROR_TYPE_UNKNOWN
}
/**
* Computes a simple numeric hash of a string for analytics grouping.
* Uses djb2 algorithm, returning a 32-bit unsigned integer.
*/
function hashString(str: string): number {
let hash = 5381
for (let i = 0; i < str.length; i++) {
hash = ((hash << 5) + hash + str.charCodeAt(i)) | 0
}
return hash >>> 0
}
export type ImageDimensions = {
originalWidth?: number
originalHeight?: number
displayWidth?: number
displayHeight?: number
}
export interface ResizeResult {
buffer: Buffer
mediaType: string
dimensions?: ImageDimensions
}
interface ImageCompressionContext {
imageBuffer: Buffer
metadata: { width?: number; height?: number; format?: string }
format: string
maxBytes: number
originalSize: number
}
interface CompressedImageResult {
base64: string
mediaType: Base64ImageSource['media_type']
originalSize: number
}
type ResizeDimensions = {
width: number
height: number
}
const IMAGE_PROMPT_SOFT_LONG_EDGE = 2048
const IMAGE_PROMPT_READABLE_LONG_EDGE = 1568
const READABLE_JPEG_QUALITIES = [85, 75] as const
const FALLBACK_JPEG_QUALITIES = [65] as const
/**
* Extracted from FileReadTool's readImage function
* Resizes image buffer to meet size and dimension constraints
*/
export async function maybeResizeAndDownsampleImageBuffer(
imageBuffer: Buffer,
originalSize: number,
ext: string,
): Promise<ResizeResult> {
if (imageBuffer.length === 0) {
// Empty buffer would fall through the catch block below (sharp throws
// "Unable to determine image format"), and the fallback's size check
// `0 ≤ 5MB` would pass it through, yielding an empty base64 string
// that the API rejects with `image cannot be empty`.
throw new ImageResizeError('Image file is empty (0 bytes)')
}
try {
const sharp = await getImageProcessor()
const image = sharp(imageBuffer)
const metadata = await image.metadata()
const mediaType = metadata.format ?? ext
// Normalize "jpg" to "jpeg" for media type compatibility
const normalizedMediaType = mediaType === 'jpg' ? 'jpeg' : mediaType
// If dimensions aren't available from metadata
if (!metadata.width || !metadata.height) {
if (originalSize > IMAGE_TARGET_RAW_SIZE) {
// Create fresh sharp instance for compression
const compressedBuffer = await sharp(imageBuffer)
.jpeg({ quality: 80 })
.toBuffer()
return { buffer: compressedBuffer, mediaType: 'jpeg' }
}
// Return without dimensions if we can't determine them
return { buffer: imageBuffer, mediaType: normalizedMediaType }
}
// Store original dimensions (guaranteed to be defined here)
const originalWidth = metadata.width
const originalHeight = metadata.height
// Calculate dimensions while maintaining aspect ratio
let width = originalWidth
let height = originalHeight
// Check if the original file just works
if (
originalSize <= IMAGE_TARGET_RAW_SIZE &&
width <= IMAGE_MAX_WIDTH &&
height <= IMAGE_MAX_HEIGHT
) {
return {
buffer: imageBuffer,
mediaType: normalizedMediaType,
dimensions: {
originalWidth,
originalHeight,
displayWidth: width,
displayHeight: height,
},
}
}
const hardDims = constrainDimensions(
width,
height,
IMAGE_MAX_WIDTH,
IMAGE_MAX_HEIGHT,
)
const needsDimensionResize =
hardDims.width !== width || hardDims.height !== height
// Try lossless or high-quality compression at original dimensions first.
// This keeps screenshot text sharp when the only problem is payload size.
if (!needsDimensionResize && originalSize > IMAGE_TARGET_RAW_SIZE) {
const compressed = await tryEncodeImageCandidate({
sharp,
imageBuffer,
originalWidth,
originalHeight,
dimensions: { width, height },
format: normalizedMediaType,
maxBytes: IMAGE_TARGET_RAW_SIZE,
})
if (compressed) return compressed
}
if (needsDimensionResize) {
logForDebugging(`Resizing to ${hardDims.width}x${hardDims.height}`)
const hardResize = await tryEncodeImageCandidate({
sharp,
imageBuffer,
originalWidth,
originalHeight,
dimensions: hardDims,
format: normalizedMediaType,
maxBytes: IMAGE_TARGET_RAW_SIZE,
})
if (hardResize) return hardResize
}
const softDims = constrainDimensions(
hardDims.width,
hardDims.height,
IMAGE_PROMPT_SOFT_LONG_EDGE,
IMAGE_PROMPT_SOFT_LONG_EDGE,
)
if (
softDims.width !== hardDims.width ||
softDims.height !== hardDims.height
) {
logForDebugging(
`Downsampling image to ${softDims.width}x${softDims.height}`,
)
const softResize = await tryEncodeImageCandidate({
sharp,
imageBuffer,
originalWidth,
originalHeight,
dimensions: softDims,
format: normalizedMediaType,
maxBytes: IMAGE_TARGET_RAW_SIZE,
})
if (softResize) return softResize
}
const readableDims = constrainDimensions(
hardDims.width,
hardDims.height,
IMAGE_PROMPT_READABLE_LONG_EDGE,
IMAGE_PROMPT_READABLE_LONG_EDGE,
)
if (
readableDims.width !== softDims.width ||
readableDims.height !== softDims.height
) {
logForDebugging(
`Downsampling image to readable fallback ${readableDims.width}x${readableDims.height}`,
)
const readableResize = await tryEncodeImageCandidate({
sharp,
imageBuffer,
originalWidth,
originalHeight,
dimensions: readableDims,
format: normalizedMediaType,
maxBytes: IMAGE_TARGET_RAW_SIZE,
allowFallbackQuality: true,
})
if (readableResize) return readableResize
}
throw new ImageResizeError(
`Unable to compress image (${formatFileSize(originalSize)}) below the 5MB API base64 limit without over-compressing it. ` +
`Please resize the image manually or use a smaller image.`,
)
} catch (error) {
// Log the error and emit analytics event
logError(error as Error)
const errorType = classifyImageError(error)
const errorMsg = errorMessage(error)
logEvent('tengu_image_resize_failed', {
original_size_bytes: originalSize,
error_type: errorType,
error_message_hash: hashString(errorMsg),
})
// Detect actual format from magic bytes instead of trusting extension
const detected = detectImageFormatFromBuffer(imageBuffer)
const normalizedExt = detected.slice(6) // Remove 'image/' prefix
// Calculate the base64 size (API limit is on base64-encoded length)
const base64Size = Math.ceil((originalSize * 4) / 3)
// Size-under-5MB does not imply dimensions-under-cap. Don't return the
// raw buffer if the PNG header says it's oversized — fall through to
// ImageResizeError instead. PNG sig is 8 bytes, IHDR dims at 16-24.
const overDim =
imageBuffer.length >= 24 &&
imageBuffer[0] === 0x89 &&
imageBuffer[1] === 0x50 &&
imageBuffer[2] === 0x4e &&
imageBuffer[3] === 0x47 &&
(imageBuffer.readUInt32BE(16) > IMAGE_MAX_WIDTH ||
imageBuffer.readUInt32BE(20) > IMAGE_MAX_HEIGHT)
// If original image's base64 encoding is within API limit, allow it through uncompressed
if (base64Size <= API_IMAGE_MAX_BASE64_SIZE && !overDim) {
logEvent('tengu_image_resize_fallback', {
original_size_bytes: originalSize,
base64_size_bytes: base64Size,
error_type: errorType,
})
return { buffer: imageBuffer, mediaType: normalizedExt }
}
// Image is too large and we failed to compress it - fail with user-friendly error
throw new ImageResizeError(
overDim
? `Unable to resize image — dimensions exceed the ${IMAGE_MAX_WIDTH}x${IMAGE_MAX_HEIGHT}px limit and image processing failed. ` +
`Please resize the image to reduce its pixel dimensions.`
: `Unable to resize image (${formatFileSize(originalSize)} raw, ${formatFileSize(base64Size)} base64). ` +
`The image exceeds the 5MB API limit and compression failed. ` +
`Please resize the image manually or use a smaller image.`,
)
}
}
function constrainDimensions(
width: number,
height: number,
maxWidth: number,
maxHeight: number,
): ResizeDimensions {
let nextWidth = width
let nextHeight = height
if (nextWidth > maxWidth) {
nextHeight = Math.round((nextHeight * maxWidth) / nextWidth)
nextWidth = maxWidth
}
if (nextHeight > maxHeight) {
nextWidth = Math.round((nextWidth * maxHeight) / nextHeight)
nextHeight = maxHeight
}
return {
width: Math.max(1, nextWidth),
height: Math.max(1, nextHeight),
}
}
function constrainDimensionsToPixelBudget(
width: number,
height: number,
maxPixels: number,
): ResizeDimensions {
if (width * height <= maxPixels) {
return { width, height }
}
const scale = Math.sqrt(maxPixels / (width * height))
let nextWidth = Math.max(1, Math.round(width * scale))
let nextHeight = Math.max(1, Math.round(height * scale))
while (nextWidth * nextHeight > maxPixels) {
if (nextWidth >= nextHeight) {
nextWidth -= 1
} else {
nextHeight -= 1
}
}
return {
width: nextWidth,
height: nextHeight,
}
}
function createResizeResult(
buffer: Buffer,
mediaType: string,
originalWidth: number,
originalHeight: number,
dimensions: ResizeDimensions,
): ResizeResult {
return {
buffer,
mediaType,
dimensions: {
originalWidth,
originalHeight,
displayWidth: dimensions.width,
displayHeight: dimensions.height,
},
}
}
function createImagePipeline(
sharp: SharpFunction,
imageBuffer: Buffer,
originalWidth: number,
originalHeight: number,
dimensions: ResizeDimensions,
): SharpInstance {
const pipeline = sharp(imageBuffer)
if (
dimensions.width === originalWidth &&
dimensions.height === originalHeight
) {
return pipeline
}
return pipeline.resize(dimensions.width, dimensions.height, {
fit: 'inside',
withoutEnlargement: true,
})
}
async function tryEncodeImageCandidate({
sharp,
imageBuffer,
originalWidth,
originalHeight,
dimensions,
format,
maxBytes,
allowFallbackQuality = false,
}: {
sharp: SharpFunction
imageBuffer: Buffer
originalWidth: number
originalHeight: number
dimensions: ResizeDimensions
format: string
maxBytes: number
allowFallbackQuality?: boolean
}): Promise<ResizeResult | null> {
const normalizedFormat = format === 'jpg' ? 'jpeg' : format
if (normalizedFormat === 'png') {
const pngCompressed = await createImagePipeline(
sharp,
imageBuffer,
originalWidth,
originalHeight,
dimensions,
)
.png({ compressionLevel: 9 })
.toBuffer()
if (pngCompressed.length <= maxBytes) {
return createResizeResult(
pngCompressed,
'png',
originalWidth,
originalHeight,
dimensions,
)
}
const webpLossless = await createImagePipeline(
sharp,
imageBuffer,
originalWidth,
originalHeight,
dimensions,
)
.webp({ lossless: true })
.toBuffer()
if (webpLossless.length <= maxBytes) {
return createResizeResult(
webpLossless,
'webp',
originalWidth,
originalHeight,
dimensions,
)
}
} else if (normalizedFormat === 'webp') {
const webpLossless = await createImagePipeline(
sharp,
imageBuffer,
originalWidth,
originalHeight,
dimensions,
)
.webp({ lossless: true })
.toBuffer()
if (webpLossless.length <= maxBytes) {
return createResizeResult(
webpLossless,
'webp',
originalWidth,
originalHeight,
dimensions,
)
}
}
const jpegQualities = allowFallbackQuality
? [...READABLE_JPEG_QUALITIES, ...FALLBACK_JPEG_QUALITIES]
: READABLE_JPEG_QUALITIES
for (const quality of jpegQualities) {
const jpegBuffer = await createImagePipeline(
sharp,
imageBuffer,
originalWidth,
originalHeight,
dimensions,
)
.jpeg({ quality })
.toBuffer()
if (jpegBuffer.length <= maxBytes) {
return createResizeResult(
jpegBuffer,
'jpeg',
originalWidth,
originalHeight,
dimensions,
)
}
}
return null
}
export async function downsampleImageBufferToVisionTokenBudget(
imageBuffer: Buffer,
originalSize: number,
ext: string,
maxTokens: number,
): Promise<ResizeResult> {
const sharp = await getImageProcessor()
const metadata = await sharp(imageBuffer).metadata()
const format = metadata.format ?? ext
const normalizedMediaType = format === 'jpg' ? 'jpeg' : format
if (!metadata.width || !metadata.height) {
throw new ImageResizeError(
'Unable to downsample image for token budget because dimensions are unavailable.',
)
}
const originalWidth = metadata.width
const originalHeight = metadata.height
const hardDims = constrainDimensions(
originalWidth,
originalHeight,
IMAGE_MAX_WIDTH,
IMAGE_MAX_HEIGHT,
)
const maxPixels = Math.max(1, maxTokens * 750)
const tokenDims = constrainDimensionsToPixelBudget(
hardDims.width,
hardDims.height,
maxPixels,
)
const tokenResize = await tryEncodeImageCandidate({
sharp,
imageBuffer,
originalWidth,
originalHeight,
dimensions: tokenDims,
format: normalizedMediaType,
maxBytes: IMAGE_TARGET_RAW_SIZE,
allowFallbackQuality: true,
})
if (tokenResize) return tokenResize
const readableDims = constrainDimensions(
tokenDims.width,
tokenDims.height,
IMAGE_PROMPT_READABLE_LONG_EDGE,
IMAGE_PROMPT_READABLE_LONG_EDGE,
)
if (
readableDims.width !== tokenDims.width ||
readableDims.height !== tokenDims.height
) {
const readableResize = await tryEncodeImageCandidate({
sharp,
imageBuffer,
originalWidth,
originalHeight,
dimensions: readableDims,
format: normalizedMediaType,
maxBytes: IMAGE_TARGET_RAW_SIZE,
allowFallbackQuality: true,
})
if (readableResize) return readableResize
}
throw new ImageResizeError(
`Unable to downsample image (${formatFileSize(originalSize)}) below the ${maxTokens} token budget without over-compressing it.`,
)
}
export interface ImageBlockWithDimensions {
block: ImageBlockParam
dimensions?: ImageDimensions
}
/**
* Resizes an image content block if needed
* Takes an image ImageBlockParam and returns a resized version if necessary
* Also returns dimension information for coordinate mapping
*/
export async function maybeResizeAndDownsampleImageBlock(
imageBlock: ImageBlockParam,
): Promise<ImageBlockWithDimensions> {
// Only process base64 images
if (imageBlock.source.type !== 'base64') {
return { block: imageBlock }
}
// Decode base64 to buffer
const imageBuffer = Buffer.from(imageBlock.source.data, 'base64')
const originalSize = imageBuffer.length
// Extract extension from media type
const mediaType = imageBlock.source.media_type
const ext = mediaType?.split('/')[1] || 'png'
// Resize if needed
const resized = await maybeResizeAndDownsampleImageBuffer(
imageBuffer,
originalSize,
ext,
)
// Return resized image block with dimension info
return {
block: {
type: 'image',
source: {
type: 'base64',
media_type:
`image/${resized.mediaType}` as Base64ImageSource['media_type'],
data: resized.buffer.toString('base64'),
},
},
dimensions: resized.dimensions,
}
}
/**
* Compresses an image buffer to fit within a maximum byte size.
*
* Uses a multi-strategy fallback approach because simple compression often fails for
* large screenshots, high-resolution photos, or images with complex gradients. Each
* strategy is progressively more aggressive to handle edge cases where earlier
* strategies produce files still exceeding the size limit.
*
* Strategy (from FileReadTool):
* 1. Try to preserve original format (PNG, JPEG, WebP) with progressive resizing
* 2. For PNG: Use palette optimization and color reduction if needed
* 3. Last resort: Convert to JPEG with aggressive compression
*
* This ensures images fit within context windows while maintaining format when possible.
*/
export async function compressImageBuffer(
imageBuffer: Buffer,
maxBytes: number = IMAGE_TARGET_RAW_SIZE,
originalMediaType?: string,
): Promise<CompressedImageResult> {
// Extract format from originalMediaType if provided (e.g., "image/png" -> "png")
const fallbackFormat = originalMediaType?.split('/')[1] || 'jpeg'
const normalizedFallback = fallbackFormat === 'jpg' ? 'jpeg' : fallbackFormat
try {
const sharp = await getImageProcessor()
const metadata = await sharp(imageBuffer).metadata()
const format = metadata.format || normalizedFallback
const originalSize = imageBuffer.length
const context: ImageCompressionContext = {
imageBuffer,
metadata,
format,
maxBytes,
originalSize,
}
// If image is already within size limit, return as-is without processing
if (originalSize <= maxBytes) {
return createCompressedImageResult(imageBuffer, format, originalSize)
}
// Try progressive resizing with format preservation
const resizedResult = await tryProgressiveResizing(context, sharp)
if (resizedResult) {
return resizedResult
}
// For PNG, try palette optimization
if (format === 'png') {
const palettizedResult = await tryPalettePNG(context, sharp)
if (palettizedResult) {
return palettizedResult
}
}
// Try JPEG conversion with moderate compression
const jpegResult = await tryJPEGConversion(context, 50, sharp)
if (jpegResult) {
return jpegResult
}
// Last resort: ultra-compressed JPEG
return await createUltraCompressedJPEG(context, sharp)
} catch (error) {
// Log the error and emit analytics event
logError(error as Error)
const errorType = classifyImageError(error)
const errorMsg = errorMessage(error)
logEvent('tengu_image_compress_failed', {
original_size_bytes: imageBuffer.length,
max_bytes: maxBytes,
error_type: errorType,
error_message_hash: hashString(errorMsg),
})
// If original image is within the requested limit, allow it through
if (imageBuffer.length <= maxBytes) {
// Detect actual format from magic bytes instead of trusting the provided media type
const detected = detectImageFormatFromBuffer(imageBuffer)
return {
base64: imageBuffer.toString('base64'),
mediaType: detected,
originalSize: imageBuffer.length,
}
}
// Image is too large and compression failed - throw error
throw new ImageResizeError(
`Unable to compress image (${formatFileSize(imageBuffer.length)}) to fit within ${formatFileSize(maxBytes)}. ` +
`Please use a smaller image.`,
)
}
}
/**
* Compresses an image buffer to fit within a token limit.
* Converts tokens to bytes using the formula: maxBytes = (maxTokens / 0.125) * 0.75
*/
export async function compressImageBufferWithTokenLimit(
imageBuffer: Buffer,
maxTokens: number,
originalMediaType?: string,
): Promise<CompressedImageResult> {
// Convert token limit to byte limit
// base64 uses about 4/3 the original size, so we reverse this
const maxBase64Chars = Math.floor(maxTokens / 0.125)
const maxBytes = Math.floor(maxBase64Chars * 0.75)
return compressImageBuffer(imageBuffer, maxBytes, originalMediaType)
}
/**
* Compresses an image block to fit within a maximum byte size.
* Wrapper around compressImageBuffer for ImageBlockParam.
*/
export async function compressImageBlock(
imageBlock: ImageBlockParam,
maxBytes: number = IMAGE_TARGET_RAW_SIZE,
): Promise<ImageBlockParam> {
// Only process base64 images
if (imageBlock.source.type !== 'base64') {
return imageBlock
}
// Decode base64 to buffer
const imageBuffer = Buffer.from(imageBlock.source.data, 'base64')
// Check if already within size limit
if (imageBuffer.length <= maxBytes) {
return imageBlock
}
// Compress the image
const compressed = await compressImageBuffer(imageBuffer, maxBytes)
return {
type: 'image',
source: {
type: 'base64',
media_type: compressed.mediaType,
data: compressed.base64,
},
}
}
// Helper functions for compression pipeline
function createCompressedImageResult(
buffer: Buffer,
mediaType: string,
originalSize: number,
): CompressedImageResult {
const normalizedMediaType = mediaType === 'jpg' ? 'jpeg' : mediaType
return {
base64: buffer.toString('base64'),
mediaType:
`image/${normalizedMediaType}` as Base64ImageSource['media_type'],
originalSize,
}
}
async function tryProgressiveResizing(
context: ImageCompressionContext,
sharp: SharpFunction,
): Promise<CompressedImageResult | null> {
const scalingFactors = [1.0, 0.75, 0.5, 0.25]
for (const scalingFactor of scalingFactors) {
const newWidth = Math.round(
(context.metadata.width || 2000) * scalingFactor,
)
const newHeight = Math.round(
(context.metadata.height || 2000) * scalingFactor,
)
let resizedImage = sharp(context.imageBuffer).resize(newWidth, newHeight, {
fit: 'inside',
withoutEnlargement: true,
})
// Apply format-specific optimizations
resizedImage = applyFormatOptimizations(resizedImage, context.format)
const resizedBuffer = await resizedImage.toBuffer()
if (resizedBuffer.length <= context.maxBytes) {
return createCompressedImageResult(
resizedBuffer,
context.format,
context.originalSize,
)
}
}
return null
}
function applyFormatOptimizations(
image: SharpInstance,
format: string,
): SharpInstance {
switch (format) {
case 'png':
return image.png({
compressionLevel: 9,
palette: true,
})
case 'jpeg':
case 'jpg':
return image.jpeg({ quality: 80 })
case 'webp':
return image.webp({ quality: 80 })
default:
return image
}
}
async function tryPalettePNG(
context: ImageCompressionContext,
sharp: SharpFunction,
): Promise<CompressedImageResult | null> {
const palettePng = await sharp(context.imageBuffer)
.resize(800, 800, {
fit: 'inside',
withoutEnlargement: true,
})
.png({
compressionLevel: 9,
palette: true,
colors: 64, // Reduce colors to 64 for better compression
})
.toBuffer()
if (palettePng.length <= context.maxBytes) {
return createCompressedImageResult(palettePng, 'png', context.originalSize)
}
return null
}
async function tryJPEGConversion(
context: ImageCompressionContext,
quality: number,
sharp: SharpFunction,
): Promise<CompressedImageResult | null> {
const jpegBuffer = await sharp(context.imageBuffer)
.resize(600, 600, {
fit: 'inside',
withoutEnlargement: true,
})
.jpeg({ quality })
.toBuffer()
if (jpegBuffer.length <= context.maxBytes) {
return createCompressedImageResult(jpegBuffer, 'jpeg', context.originalSize)
}
return null
}
async function createUltraCompressedJPEG(
context: ImageCompressionContext,
sharp: SharpFunction,
): Promise<CompressedImageResult> {
const ultraCompressedBuffer = await sharp(context.imageBuffer)
.resize(400, 400, {
fit: 'inside',
withoutEnlargement: true,
})
.jpeg({ quality: 20 })
.toBuffer()
return createCompressedImageResult(
ultraCompressedBuffer,
'jpeg',
context.originalSize,
)
}
/**
* Detect image format from a buffer using magic bytes
* @param buffer Buffer containing image data
* @returns Media type string (e.g., 'image/png', 'image/jpeg') or 'image/png' as default
*/
export function detectImageFormatFromBuffer(buffer: Buffer): ImageMediaType {
if (buffer.length < 4) return 'image/png' // default
// Check PNG signature
if (
buffer[0] === 0x89 &&
buffer[1] === 0x50 &&
buffer[2] === 0x4e &&
buffer[3] === 0x47
) {
return 'image/png'
}
// Check JPEG signature (FFD8FF)
if (buffer[0] === 0xff && buffer[1] === 0xd8 && buffer[2] === 0xff) {
return 'image/jpeg'
}
// Check GIF signature (GIF87a or GIF89a)
if (buffer[0] === 0x47 && buffer[1] === 0x49 && buffer[2] === 0x46) {
return 'image/gif'
}
// Check WebP signature (RIFF....WEBP)
if (
buffer[0] === 0x52 &&
buffer[1] === 0x49 &&
buffer[2] === 0x46 &&
buffer[3] === 0x46
) {
if (
buffer.length >= 12 &&
buffer[8] === 0x57 &&
buffer[9] === 0x45 &&
buffer[10] === 0x42 &&
buffer[11] === 0x50
) {
return 'image/webp'
}
}
// Default to PNG if unknown
return 'image/png'
}
/**
* Detect image format from base64 data using magic bytes
* @param base64Data Base64 encoded image data
* @returns Media type string (e.g., 'image/png', 'image/jpeg') or 'image/png' as default
*/
export function detectImageFormatFromBase64(
base64Data: string,
): ImageMediaType {
try {
const buffer = Buffer.from(base64Data, 'base64')
return detectImageFormatFromBuffer(buffer)
} catch {
// Default to PNG on any error
return 'image/png'
}
}
/**
* Creates a text description of image metadata including dimensions and source path.
* Returns null if no useful metadata is available.
*/
export function createImageMetadataText(
dims: ImageDimensions,
sourcePath?: string,
): string | null {
const { originalWidth, originalHeight, displayWidth, displayHeight } = dims
// Skip if dimensions are not available or invalid
// Note: checks for undefined/null and zero to prevent division by zero
if (
!originalWidth ||
!originalHeight ||
!displayWidth ||
!displayHeight ||
displayWidth <= 0 ||
displayHeight <= 0
) {
// If we have a source path but no valid dimensions, still return source info
if (sourcePath) {
return `[Image source: ${sourcePath}]`
}
return null
}
// Check if image was resized
const wasResized =
originalWidth !== displayWidth || originalHeight !== displayHeight
// Only include metadata if there's useful info (resized or has source path)
if (!wasResized && !sourcePath) {
return null
}
// Build metadata parts
const parts: string[] = []
if (sourcePath) {
parts.push(`source: ${sourcePath}`)
}
if (wasResized) {
const scaleFactor = originalWidth / displayWidth
parts.push(
`original ${originalWidth}x${originalHeight}, displayed at ${displayWidth}x${displayHeight}. Multiply coordinates by ${scaleFactor.toFixed(2)} to map to original image.`,
)
}
return `[Image: ${parts.join(', ')}]`
}