mirror of
https://github.com/NanmiCoder/cc-haha
synced 2026-07-18 13:23:33 +08:00
fix: preserve chat flow after unsupported media
Unsupported image rejections from text-only compatible providers should not poison the session, and low-trust multimodal usage spikes should not make context indicators report a full window. Constraint: Third-party Anthropic-compatible providers may report encoded media bytes as usage tokens. Rejected: Trust all provider usage uniformly | third-party media responses can pin context to 100% incorrectly. Confidence: high Scope-risk: moderate Directive: Do not remove the media-aware fallback without checking text-only provider recovery and desktop context indicators. Tested: bun run check:server Tested: focused media/context regression suite
This commit is contained in:
程序员阿江(Relakkes)
parent
1f9af7e578
commit
eede5568d2
File diff suppressed because one or more lines are too long
File diff suppressed because one or more lines are too long
@ -529,6 +529,83 @@ describe('ConversationService', () => {
|
||||
await fs.rm(workDir, { recursive: true, force: true })
|
||||
}
|
||||
})
|
||||
|
||||
it('should not report transcript context as full for low-trust media usage spikes', async () => {
|
||||
const previousConfigDir = process.env.CLAUDE_CONFIG_DIR
|
||||
const previousNodeEnv = process.env.NODE_ENV
|
||||
const previousUseBedrock = process.env.CLAUDE_CODE_USE_BEDROCK
|
||||
const tmpConfigDir = await fs.mkdtemp(path.join(os.tmpdir(), 'claude-transcript-media-'))
|
||||
const workDir = await fs.mkdtemp(path.join(os.tmpdir(), 'claude-workdir-media-'))
|
||||
process.env.CLAUDE_CONFIG_DIR = tmpConfigDir
|
||||
process.env.NODE_ENV = 'development'
|
||||
process.env.CLAUDE_CODE_USE_BEDROCK = '1'
|
||||
|
||||
try {
|
||||
const svc = new SessionService()
|
||||
const { sessionId } = await svc.createSession(workDir)
|
||||
const found = await svc.findSessionFile(sessionId)
|
||||
expect(found).not.toBeNull()
|
||||
|
||||
await fs.appendFile(found!.filePath, JSON.stringify({
|
||||
type: 'user',
|
||||
uuid: crypto.randomUUID(),
|
||||
timestamp: '2026-04-27T12:00:00.000Z',
|
||||
cwd: workDir,
|
||||
message: {
|
||||
role: 'user',
|
||||
content: [{
|
||||
type: 'image',
|
||||
source: {
|
||||
type: 'base64',
|
||||
media_type: 'image/png',
|
||||
data: 'a'.repeat(1024),
|
||||
},
|
||||
}],
|
||||
},
|
||||
}) + '\n')
|
||||
await fs.appendFile(found!.filePath, JSON.stringify({
|
||||
type: 'assistant',
|
||||
uuid: crypto.randomUUID(),
|
||||
timestamp: '2026-04-27T12:00:01.000Z',
|
||||
cwd: workDir,
|
||||
version: '999.0.0-test',
|
||||
message: {
|
||||
role: 'assistant',
|
||||
model: 'claude-sonnet-4-6',
|
||||
content: [{ type: 'text', text: 'ok' }],
|
||||
usage: {
|
||||
input_tokens: 1_000_000,
|
||||
output_tokens: 10,
|
||||
},
|
||||
},
|
||||
}) + '\n')
|
||||
|
||||
const contextEstimate = await svc.getTranscriptContextEstimate(sessionId)
|
||||
|
||||
expect(contextEstimate?.rawMaxTokens).toBe(200_000)
|
||||
expect(contextEstimate?.totalTokens).toBeLessThan(200_000)
|
||||
expect(contextEstimate?.percentage).toBeLessThan(100)
|
||||
expect(contextEstimate?.categories[0]?.name).toBe('Estimated context')
|
||||
} finally {
|
||||
if (previousConfigDir === undefined) {
|
||||
delete process.env.CLAUDE_CONFIG_DIR
|
||||
} else {
|
||||
process.env.CLAUDE_CONFIG_DIR = previousConfigDir
|
||||
}
|
||||
if (previousNodeEnv === undefined) {
|
||||
delete process.env.NODE_ENV
|
||||
} else {
|
||||
process.env.NODE_ENV = previousNodeEnv
|
||||
}
|
||||
if (previousUseBedrock === undefined) {
|
||||
delete process.env.CLAUDE_CODE_USE_BEDROCK
|
||||
} else {
|
||||
process.env.CLAUDE_CODE_USE_BEDROCK = previousUseBedrock
|
||||
}
|
||||
await fs.rm(tmpConfigDir, { recursive: true, force: true })
|
||||
await fs.rm(workDir, { recursive: true, force: true })
|
||||
}
|
||||
})
|
||||
})
|
||||
|
||||
// ============================================================================
|
||||
|
||||
@ -195,4 +195,13 @@ describe('provider presets API', () => {
|
||||
const updatedRaw = await fs.readFile(path.join(tmpDir, 'cc-haha', 'settings.json'), 'utf-8')
|
||||
expect(JSON.parse(updatedRaw)).toEqual(updateBody)
|
||||
})
|
||||
|
||||
test('provider presets carry docs-backed context windows for current coding models', () => {
|
||||
const byId = new Map(PROVIDER_PRESETS.map((preset) => [preset.id, preset]))
|
||||
|
||||
for (const id of ['deepseek', 'zhipuglm', 'kimi', 'minimax']) {
|
||||
const preset = byId.get(id)!
|
||||
expect(preset.modelContextWindows?.[preset.defaultModels.main]).toBeGreaterThan(0)
|
||||
}
|
||||
})
|
||||
})
|
||||
|
||||
@ -16,12 +16,17 @@ import type { FileHistorySnapshot } from '../../utils/fileHistory.js'
|
||||
import { findCanonicalGitRoot } from '../../utils/git.js'
|
||||
import { calculateUSDCost, MODEL_COSTS } from '../../utils/modelCost.js'
|
||||
import {
|
||||
calculateCurrentContextTokenTotal,
|
||||
MODEL_CONTEXT_WINDOW_DEFAULT,
|
||||
getContextWindowForModel,
|
||||
getModelMaxOutputTokens,
|
||||
} from '../../utils/context.js'
|
||||
import {
|
||||
calculateContextBudget,
|
||||
getProviderUsageTrust,
|
||||
hasMediaInput,
|
||||
} from '../../utils/contextBudget.js'
|
||||
import { getCanonicalName } from '../../utils/model/model.js'
|
||||
import { isFirstPartyAnthropicBaseUrl } from '../../utils/model/providers.js'
|
||||
import {
|
||||
resolveSessionWorkspaceLaunch,
|
||||
type CreateSessionRepositoryOptions,
|
||||
@ -30,6 +35,7 @@ import {
|
||||
import { registerFilesystemAccessRoot } from './filesystemAccessRoots.js'
|
||||
import { normalizeDriveRootPathForPlatform } from './windowsDrivePath.js'
|
||||
import { cleanSessionTitleSource } from '../../utils/sessionTitleText.js'
|
||||
import { roughTokenCountEstimationForMessages } from '../../services/tokenEstimation.js'
|
||||
|
||||
// ============================================================================
|
||||
// Types
|
||||
@ -1358,18 +1364,39 @@ export class SessionService {
|
||||
|
||||
const rawMaxTokens = this.getTranscriptContextWindow(latest.model)
|
||||
const promptTokens = latest.inputTokens + latest.cacheReadInputTokens + latest.cacheCreationInputTokens
|
||||
const totalTokens = calculateCurrentContextTokenTotal(promptTokens, {
|
||||
input_tokens: latest.inputTokens,
|
||||
output_tokens: latest.outputTokens,
|
||||
cache_read_input_tokens: latest.cacheReadInputTokens,
|
||||
cache_creation_input_tokens: latest.cacheCreationInputTokens,
|
||||
}, rawMaxTokens)
|
||||
const transcriptMessages = entries.filter(entry =>
|
||||
entry.type === 'user' || entry.type === 'assistant' || entry.type === 'attachment',
|
||||
) as Parameters<typeof roughTokenCountEstimationForMessages>[0]
|
||||
const estimatedTokens =
|
||||
roughTokenCountEstimationForMessages(transcriptMessages) || promptTokens
|
||||
const contextBudget = calculateContextBudget({
|
||||
estimatedTokens,
|
||||
contextWindow: rawMaxTokens,
|
||||
currentUsage: {
|
||||
input_tokens: latest.inputTokens,
|
||||
output_tokens: latest.outputTokens,
|
||||
cache_read_input_tokens: latest.cacheReadInputTokens,
|
||||
cache_creation_input_tokens: latest.cacheCreationInputTokens,
|
||||
},
|
||||
usageTrust: getProviderUsageTrust({
|
||||
isFirstPartyAnthropic: isFirstPartyAnthropicBaseUrl(),
|
||||
}),
|
||||
hasMediaInput: hasMediaInput(transcriptMessages),
|
||||
})
|
||||
const totalTokens = contextBudget.usedTokens
|
||||
const percentage = rawMaxTokens > 0 ? Math.round((totalTokens / rawMaxTokens) * 100) : 0
|
||||
const categories: TranscriptContextEstimate['categories'] = [
|
||||
const usageCategories: TranscriptContextEstimate['categories'] = [
|
||||
{ name: 'Input tokens', tokens: latest.inputTokens, color: '#8f3217' },
|
||||
{ name: 'Cache read', tokens: latest.cacheReadInputTokens, color: '#0f5c8f' },
|
||||
{ name: 'Cache write', tokens: latest.cacheCreationInputTokens, color: '#7c3aed' },
|
||||
{ name: 'Output tokens', tokens: latest.outputTokens, color: '#2f7d32' },
|
||||
]
|
||||
const contextCategories: TranscriptContextEstimate['categories'] =
|
||||
contextBudget.ignoredUsageReason === 'low_trust_media_usage'
|
||||
? [{ name: 'Estimated context', tokens: totalTokens, color: '#8f3217' }]
|
||||
: usageCategories
|
||||
const categories: TranscriptContextEstimate['categories'] = [
|
||||
...contextCategories,
|
||||
{ name: 'Free space', tokens: Math.max(0, rawMaxTokens - totalTokens), color: '#a1a1aa', isDeferred: true },
|
||||
].filter((category) => category.tokens > 0)
|
||||
|
||||
|
||||
36
src/services/api/errors.test.ts
Normal file
36
src/services/api/errors.test.ts
Normal file
@ -0,0 +1,36 @@
|
||||
import { describe, expect, test } from 'bun:test'
|
||||
import {
|
||||
getAssistantMessageFromError,
|
||||
getImageUnsupportedErrorMessage,
|
||||
isUnsupportedImageInputErrorMessage,
|
||||
} from './errors.js'
|
||||
|
||||
describe('image unsupported API errors', () => {
|
||||
test('detects provider-specific text-only model image rejections', () => {
|
||||
expect(
|
||||
isUnsupportedImageInputErrorMessage(
|
||||
'This model does not support image blocks',
|
||||
),
|
||||
).toBe(true)
|
||||
expect(
|
||||
isUnsupportedImageInputErrorMessage(
|
||||
'unsupported modality: image input is not available',
|
||||
),
|
||||
).toBe(true)
|
||||
expect(isUnsupportedImageInputErrorMessage('image exceeds maximum')).toBe(false)
|
||||
})
|
||||
|
||||
test('maps unsupported image rejections to a recoverable synthetic error', () => {
|
||||
const msg = getAssistantMessageFromError(
|
||||
new Error('This model does not support image blocks'),
|
||||
'mimo-v2.5-pro',
|
||||
)
|
||||
|
||||
expect(msg.isApiErrorMessage).toBe(true)
|
||||
expect(msg.errorDetails).toBe('This model does not support image blocks')
|
||||
expect(msg.message.content[0]).toMatchObject({
|
||||
type: 'text',
|
||||
text: getImageUnsupportedErrorMessage(),
|
||||
})
|
||||
})
|
||||
})
|
||||
@ -188,6 +188,11 @@ export function getImageTooLargeErrorMessage(): string {
|
||||
? 'Image was too large. Try resizing the image or using a different approach.'
|
||||
: 'Image was too large. Double press esc to go back and try again with a smaller image.'
|
||||
}
|
||||
export function getImageUnsupportedErrorMessage(): string {
|
||||
return getIsNonInteractiveSession()
|
||||
? 'This model does not support images. Continue with text, or switch to a vision-capable model and send the image again.'
|
||||
: 'This model does not support images. Double press esc to go back, switch to a vision-capable model, or continue with text.'
|
||||
}
|
||||
export function getRequestTooLargeErrorMessage(): string {
|
||||
const limits = `max ${formatFileSize(PDF_TARGET_RAW_SIZE)}`
|
||||
return getIsNonInteractiveSession()
|
||||
@ -422,6 +427,20 @@ export function extractUnknownErrorFormat(value: unknown): string | undefined {
|
||||
return undefined
|
||||
}
|
||||
|
||||
export function isUnsupportedImageInputErrorMessage(message: string): boolean {
|
||||
const raw = message.toLowerCase()
|
||||
if (!raw.includes('image')) return false
|
||||
return (
|
||||
raw.includes('not support') ||
|
||||
raw.includes('not supported') ||
|
||||
raw.includes('unsupported') ||
|
||||
raw.includes('vision') ||
|
||||
raw.includes('multimodal') ||
|
||||
raw.includes('multi-modal') ||
|
||||
raw.includes('modality')
|
||||
)
|
||||
}
|
||||
|
||||
export function getAssistantMessageFromError(
|
||||
error: unknown,
|
||||
model: string,
|
||||
@ -451,6 +470,22 @@ export function getAssistantMessageFromError(
|
||||
})
|
||||
}
|
||||
|
||||
// Custom/Anthropic-compatible providers often reject image blocks with
|
||||
// provider-specific wording when the selected model is text-only. Convert it
|
||||
// to a known synthetic error so normalizeMessagesForAPI can strip the image
|
||||
// from later turns instead of poisoning the whole session.
|
||||
if (
|
||||
error instanceof Error &&
|
||||
isUnsupportedImageInputErrorMessage(error.message) &&
|
||||
(!(error instanceof APIError) || error.status === 400 || error.status === 422)
|
||||
) {
|
||||
return createAssistantAPIErrorMessage({
|
||||
content: getImageUnsupportedErrorMessage(),
|
||||
error: 'invalid_request',
|
||||
errorDetails: error.message,
|
||||
})
|
||||
}
|
||||
|
||||
// Check for emergency capacity off switch for Opus PAYG users
|
||||
if (
|
||||
error instanceof Error &&
|
||||
|
||||
@ -1,6 +1,6 @@
|
||||
import { describe, expect, test, beforeEach, afterEach } from 'bun:test'
|
||||
|
||||
import { getEffectiveContextWindowSize } from './autoCompact.js'
|
||||
import { getAutoCompactThreshold, getEffectiveContextWindowSize } from './autoCompact.js'
|
||||
import { getContextWindowForModel } from '../../utils/context.js'
|
||||
import { MODEL_CONTEXT_WINDOWS_ENV_KEY } from '../../utils/model/modelContextWindows.js'
|
||||
|
||||
@ -58,4 +58,12 @@ describe('model context window resolution', () => {
|
||||
|
||||
expect(getEffectiveContextWindowSize('unknown-future-model')).toBe(980_000)
|
||||
})
|
||||
|
||||
test('derives auto-compact thresholds from provider context windows', () => {
|
||||
expect(getAutoCompactThreshold('deepseek-v4-pro')).toBe(967_000)
|
||||
expect(getAutoCompactThreshold('glm-5.1')).toBe(167_000)
|
||||
expect(getAutoCompactThreshold('glm-4.5-air')).toBe(95_000)
|
||||
expect(getAutoCompactThreshold('kimi-k2.6')).toBe(229_144)
|
||||
expect(getAutoCompactThreshold('MiniMax-M2.7')).toBe(171_800)
|
||||
})
|
||||
})
|
||||
|
||||
@ -241,6 +241,24 @@ export function roughTokenCountEstimationForFileType(
|
||||
)
|
||||
}
|
||||
|
||||
export function roughTokenCountEstimationForAPIRequest(
|
||||
messages: readonly {
|
||||
role?: string
|
||||
content?: string | Array<Anthropic.ContentBlock> | Array<Anthropic.ContentBlockParam>
|
||||
}[],
|
||||
tools: readonly unknown[] = [],
|
||||
): number {
|
||||
let totalTokens = 0
|
||||
for (const message of messages) {
|
||||
totalTokens += roughTokenCountEstimation(message.role ?? '')
|
||||
totalTokens += roughTokenCountEstimationForContent(message.content)
|
||||
}
|
||||
if (tools.length > 0) {
|
||||
totalTokens += roughTokenCountEstimation(jsonStringify({ tools }))
|
||||
}
|
||||
return totalTokens
|
||||
}
|
||||
|
||||
/**
|
||||
* Estimates token count for a Message object by extracting and analyzing its text content.
|
||||
* This provides a more reliable estimate than getTokenUsage for messages that may have been compacted.
|
||||
@ -368,7 +386,7 @@ export function roughTokenCountEstimationForMessage(message: {
|
||||
return 0
|
||||
}
|
||||
|
||||
function roughTokenCountEstimationForContent(
|
||||
export function roughTokenCountEstimationForContent(
|
||||
content:
|
||||
| string
|
||||
| Array<Anthropic.ContentBlock>
|
||||
|
||||
@ -53,4 +53,22 @@ describe('calculateCurrentContextTokenTotal', () => {
|
||||
expect(calculateCurrentContextTokenTotal(50_000, null)).toBe(50_000)
|
||||
expect(calculateCurrentContextTokenTotal(50_000, null, 200_000)).toBe(50_000)
|
||||
})
|
||||
|
||||
test('ignores suspicious media usage that appears to count encoded bytes', () => {
|
||||
expect(calculateCurrentContextTokenTotal(30_000, {
|
||||
input_tokens: 220_000,
|
||||
cache_creation_input_tokens: 0,
|
||||
cache_read_input_tokens: 0,
|
||||
output_tokens: 1_000,
|
||||
}, 200_000, { hasMediaInput: true, usageTrust: 'low' })).toBe(30_000)
|
||||
})
|
||||
|
||||
test('keeps high provider usage for media when it is close to local estimate', () => {
|
||||
expect(calculateCurrentContextTokenTotal(180_000, {
|
||||
input_tokens: 195_000,
|
||||
cache_creation_input_tokens: 0,
|
||||
cache_read_input_tokens: 0,
|
||||
output_tokens: 5_000,
|
||||
}, 200_000, { hasMediaInput: true, usageTrust: 'high' })).toBe(200_000)
|
||||
})
|
||||
})
|
||||
|
||||
369
src/utils/__tests__/contextBudget.test.ts
Normal file
369
src/utils/__tests__/contextBudget.test.ts
Normal file
@ -0,0 +1,369 @@
|
||||
import { describe, expect, test } from 'bun:test'
|
||||
import {
|
||||
calculateContextBudget,
|
||||
calculateContextPercentagesFromTokens,
|
||||
getProviderUsageTrust,
|
||||
getUsageTokenTotal,
|
||||
hasMediaInput,
|
||||
shouldIgnoreLowTrustUsage,
|
||||
} from '../contextBudget.js'
|
||||
|
||||
function restoreEnvVar(name: string, value: string | undefined): void {
|
||||
if (value === undefined) {
|
||||
delete process.env[name]
|
||||
return
|
||||
}
|
||||
|
||||
process.env[name] = value
|
||||
}
|
||||
|
||||
describe('getProviderUsageTrust', () => {
|
||||
test('returns high for first-party Anthropic direct usage', () => {
|
||||
const originalUseBedrock = process.env.CLAUDE_CODE_USE_BEDROCK
|
||||
const originalUseVertex = process.env.CLAUDE_CODE_USE_VERTEX
|
||||
const originalUseFoundry = process.env.CLAUDE_CODE_USE_FOUNDRY
|
||||
const originalUseAzureOpenAI = process.env.CLAUDE_CODE_USE_AZURE_OPENAI
|
||||
|
||||
try {
|
||||
delete process.env.CLAUDE_CODE_USE_BEDROCK
|
||||
delete process.env.CLAUDE_CODE_USE_VERTEX
|
||||
delete process.env.CLAUDE_CODE_USE_FOUNDRY
|
||||
delete process.env.CLAUDE_CODE_USE_AZURE_OPENAI
|
||||
|
||||
expect(getProviderUsageTrust({ isFirstPartyAnthropic: true })).toBe(
|
||||
'high',
|
||||
)
|
||||
} finally {
|
||||
restoreEnvVar('CLAUDE_CODE_USE_BEDROCK', originalUseBedrock)
|
||||
restoreEnvVar('CLAUDE_CODE_USE_VERTEX', originalUseVertex)
|
||||
restoreEnvVar('CLAUDE_CODE_USE_FOUNDRY', originalUseFoundry)
|
||||
restoreEnvVar(
|
||||
'CLAUDE_CODE_USE_AZURE_OPENAI',
|
||||
originalUseAzureOpenAI,
|
||||
)
|
||||
}
|
||||
})
|
||||
|
||||
test('returns low for non-first-party providers even if the Anthropic base URL looks first-party', () => {
|
||||
const originalUseBedrock = process.env.CLAUDE_CODE_USE_BEDROCK
|
||||
|
||||
try {
|
||||
process.env.CLAUDE_CODE_USE_BEDROCK = '1'
|
||||
|
||||
expect(getProviderUsageTrust({ isFirstPartyAnthropic: true })).toBe('low')
|
||||
} finally {
|
||||
restoreEnvVar('CLAUDE_CODE_USE_BEDROCK', originalUseBedrock)
|
||||
}
|
||||
})
|
||||
|
||||
test('returns low for non-first-party Anthropic endpoints', () => {
|
||||
expect(getProviderUsageTrust({ isFirstPartyAnthropic: false })).toBe('low')
|
||||
})
|
||||
})
|
||||
|
||||
describe('hasMediaInput', () => {
|
||||
test('returns true for image file attachments', () => {
|
||||
expect(hasMediaInput([
|
||||
{
|
||||
type: 'attachment',
|
||||
attachment: {
|
||||
type: 'file',
|
||||
content: { type: 'image', source: { type: 'base64', data: 'abc' } },
|
||||
},
|
||||
},
|
||||
])).toBe(true)
|
||||
})
|
||||
|
||||
test('returns true for pdf file attachments', () => {
|
||||
expect(hasMediaInput([
|
||||
{
|
||||
type: 'attachment',
|
||||
attachment: {
|
||||
type: 'file',
|
||||
content: { type: 'pdf', pages: [] },
|
||||
},
|
||||
},
|
||||
])).toBe(true)
|
||||
})
|
||||
|
||||
test('returns true for user content with an image block', () => {
|
||||
expect(hasMediaInput([
|
||||
{
|
||||
type: 'user',
|
||||
message: {
|
||||
content: [
|
||||
{ type: 'text', text: 'look at this' },
|
||||
{ type: 'image', source: { type: 'base64', data: 'abc' } },
|
||||
],
|
||||
},
|
||||
},
|
||||
])).toBe(true)
|
||||
})
|
||||
|
||||
test('returns true for assistant content with a document block', () => {
|
||||
expect(hasMediaInput([
|
||||
{
|
||||
type: 'assistant',
|
||||
message: {
|
||||
content: [
|
||||
{ type: 'document', source: { type: 'text', data: 'report' } },
|
||||
],
|
||||
},
|
||||
},
|
||||
])).toBe(true)
|
||||
})
|
||||
|
||||
test('returns false for plain text messages', () => {
|
||||
expect(hasMediaInput([
|
||||
{ type: 'user', message: { content: 'hello' } },
|
||||
{
|
||||
type: 'assistant',
|
||||
message: { content: [{ type: 'text', text: 'hello back' }] },
|
||||
},
|
||||
])).toBe(false)
|
||||
})
|
||||
|
||||
test('returns false for text-only teammate mailbox attachments', () => {
|
||||
expect(hasMediaInput([
|
||||
{
|
||||
type: 'attachment',
|
||||
attachment: {
|
||||
type: 'teammate_mailbox',
|
||||
messages: [{ from: 'team-lead', text: 'status?', timestamp: 'now' }],
|
||||
},
|
||||
},
|
||||
])).toBe(false)
|
||||
})
|
||||
|
||||
test('returns false for text-only team context attachments', () => {
|
||||
expect(hasMediaInput([
|
||||
{
|
||||
type: 'attachment',
|
||||
attachment: {
|
||||
type: 'team_context',
|
||||
agentId: 'agent-1',
|
||||
agentName: 'worker',
|
||||
teamName: 'alpha',
|
||||
teamConfigPath: '/tmp/team.json',
|
||||
taskListPath: '/tmp/tasks.json',
|
||||
},
|
||||
},
|
||||
])).toBe(false)
|
||||
})
|
||||
|
||||
test('returns false for text-only skill discovery attachments', () => {
|
||||
expect(hasMediaInput([
|
||||
{
|
||||
type: 'attachment',
|
||||
attachment: {
|
||||
type: 'skill_discovery',
|
||||
skills: [{ name: 'foo', description: 'bar' }],
|
||||
signal: 'manual',
|
||||
source: 'native',
|
||||
},
|
||||
},
|
||||
])).toBe(false)
|
||||
})
|
||||
})
|
||||
|
||||
describe('getUsageTokenTotal', () => {
|
||||
test('includes input, cache, and output tokens by default', () => {
|
||||
expect(getUsageTokenTotal({
|
||||
input_tokens: 20_000,
|
||||
cache_creation_input_tokens: 1_000,
|
||||
cache_read_input_tokens: 2_000,
|
||||
output_tokens: 3_000,
|
||||
})).toBe(26_000)
|
||||
})
|
||||
|
||||
test('can exclude output tokens', () => {
|
||||
expect(getUsageTokenTotal({
|
||||
input_tokens: 20_000,
|
||||
cache_creation_input_tokens: 1_000,
|
||||
cache_read_input_tokens: 2_000,
|
||||
output_tokens: 3_000,
|
||||
}, { includeOutput: false })).toBe(23_000)
|
||||
})
|
||||
})
|
||||
|
||||
describe('shouldIgnoreLowTrustUsage', () => {
|
||||
test('ignores suspicious low-trust media usage', () => {
|
||||
expect(shouldIgnoreLowTrustUsage({
|
||||
usageTrust: 'low',
|
||||
hasMediaInput: true,
|
||||
usageTokens: 220_000,
|
||||
estimatedTokens: 30_000,
|
||||
contextWindow: 200_000,
|
||||
})).toBe(true)
|
||||
})
|
||||
|
||||
test('keeps high-trust usage', () => {
|
||||
expect(shouldIgnoreLowTrustUsage({
|
||||
usageTrust: 'high',
|
||||
hasMediaInput: true,
|
||||
usageTokens: 220_000,
|
||||
estimatedTokens: 30_000,
|
||||
contextWindow: 200_000,
|
||||
})).toBe(false)
|
||||
})
|
||||
|
||||
test('keeps close-to-estimate usage', () => {
|
||||
expect(shouldIgnoreLowTrustUsage({
|
||||
usageTrust: 'low',
|
||||
hasMediaInput: true,
|
||||
usageTokens: 200_000,
|
||||
estimatedTokens: 180_000,
|
||||
contextWindow: 200_000,
|
||||
})).toBe(false)
|
||||
})
|
||||
|
||||
test('keeps provider usage when there is no estimate to fall back to', () => {
|
||||
expect(shouldIgnoreLowTrustUsage({
|
||||
usageTrust: 'low',
|
||||
hasMediaInput: true,
|
||||
usageTokens: 220_000,
|
||||
estimatedTokens: 0,
|
||||
contextWindow: 200_000,
|
||||
})).toBe(false)
|
||||
})
|
||||
})
|
||||
|
||||
describe('calculateContextBudget', () => {
|
||||
test('uses larger trusted provider usage', () => {
|
||||
expect(calculateContextBudget({
|
||||
estimatedTokens: 30_000,
|
||||
contextWindow: 200_000,
|
||||
currentUsage: {
|
||||
input_tokens: 45_000,
|
||||
cache_creation_input_tokens: 1_000,
|
||||
cache_read_input_tokens: 2_000,
|
||||
output_tokens: 3_000,
|
||||
},
|
||||
usageTrust: 'high',
|
||||
hasMediaInput: false,
|
||||
})).toEqual({
|
||||
usedTokens: 51_000,
|
||||
source: 'provider_usage',
|
||||
providerUsageTokens: 51_000,
|
||||
estimatedTokens: 30_000,
|
||||
})
|
||||
})
|
||||
|
||||
test('ignores suspicious low-trust media usage', () => {
|
||||
expect(calculateContextBudget({
|
||||
estimatedTokens: 30_000,
|
||||
contextWindow: 200_000,
|
||||
currentUsage: {
|
||||
input_tokens: 220_000,
|
||||
cache_creation_input_tokens: 0,
|
||||
cache_read_input_tokens: 0,
|
||||
output_tokens: 1_000,
|
||||
},
|
||||
usageTrust: 'low',
|
||||
hasMediaInput: true,
|
||||
})).toEqual({
|
||||
usedTokens: 30_000,
|
||||
source: 'estimate',
|
||||
providerUsageTokens: 221_000,
|
||||
estimatedTokens: 30_000,
|
||||
ignoredUsageReason: 'low_trust_media_usage',
|
||||
})
|
||||
})
|
||||
|
||||
test('does not force 100% display for low-trust media usage spikes', () => {
|
||||
const budget = calculateContextBudget({
|
||||
estimatedTokens: 42_000,
|
||||
currentUsage: {
|
||||
input_tokens: 500_000,
|
||||
cache_creation_input_tokens: 0,
|
||||
cache_read_input_tokens: 0,
|
||||
output_tokens: 2_000,
|
||||
},
|
||||
contextWindow: 200_000,
|
||||
usageTrust: 'low',
|
||||
hasMediaInput: true,
|
||||
})
|
||||
|
||||
expect(calculateContextPercentagesFromTokens(
|
||||
budget.usedTokens,
|
||||
200_000,
|
||||
)).toEqual({
|
||||
used: 21,
|
||||
remaining: 79,
|
||||
})
|
||||
})
|
||||
|
||||
test('does not fall back to zero tokens for low-trust media usage spikes', () => {
|
||||
const budget = calculateContextBudget({
|
||||
estimatedTokens: 0,
|
||||
currentUsage: {
|
||||
input_tokens: 220_000,
|
||||
cache_creation_input_tokens: 0,
|
||||
cache_read_input_tokens: 0,
|
||||
output_tokens: 1_000,
|
||||
},
|
||||
contextWindow: 200_000,
|
||||
usageTrust: 'low',
|
||||
hasMediaInput: true,
|
||||
})
|
||||
|
||||
expect(budget.source).toBe('provider_usage')
|
||||
expect(budget.usedTokens).toBe(200_000)
|
||||
expect(budget.ignoredUsageReason).toBeUndefined()
|
||||
})
|
||||
|
||||
test('clamps to contextWindow', () => {
|
||||
expect(calculateContextBudget({
|
||||
estimatedTokens: 990_000,
|
||||
contextWindow: 1_000_000,
|
||||
currentUsage: {
|
||||
input_tokens: 995_000,
|
||||
cache_creation_input_tokens: 0,
|
||||
cache_read_input_tokens: 0,
|
||||
output_tokens: 8_000,
|
||||
},
|
||||
usageTrust: 'high',
|
||||
hasMediaInput: false,
|
||||
})).toEqual({
|
||||
usedTokens: 1_000_000,
|
||||
source: 'provider_usage',
|
||||
providerUsageTokens: 1_003_000,
|
||||
estimatedTokens: 990_000,
|
||||
})
|
||||
})
|
||||
})
|
||||
|
||||
describe('calculateContextPercentagesFromTokens', () => {
|
||||
test('returns nulls for null usage', () => {
|
||||
expect(calculateContextPercentagesFromTokens(null, 200_000)).toEqual({
|
||||
used: null,
|
||||
remaining: null,
|
||||
})
|
||||
})
|
||||
|
||||
test('returns nulls for invalid context windows', () => {
|
||||
expect(calculateContextPercentagesFromTokens(50_000, 0)).toEqual({
|
||||
used: null,
|
||||
remaining: null,
|
||||
})
|
||||
expect(calculateContextPercentagesFromTokens(50_000, -1)).toEqual({
|
||||
used: null,
|
||||
remaining: null,
|
||||
})
|
||||
})
|
||||
|
||||
test('rounds and clamps percentages', () => {
|
||||
expect(calculateContextPercentagesFromTokens(50_500, 200_000)).toEqual({
|
||||
used: 25,
|
||||
remaining: 75,
|
||||
})
|
||||
expect(calculateContextPercentagesFromTokens(220_000, 200_000)).toEqual({
|
||||
used: 100,
|
||||
remaining: 0,
|
||||
})
|
||||
expect(calculateContextPercentagesFromTokens(-1_000, 200_000)).toEqual({
|
||||
used: 0,
|
||||
remaining: 100,
|
||||
})
|
||||
})
|
||||
})
|
||||
@ -19,6 +19,8 @@ import {
|
||||
countMessagesTokensWithAPI,
|
||||
countTokensViaHaikuFallback,
|
||||
roughTokenCountEstimation,
|
||||
roughTokenCountEstimationForAPIRequest,
|
||||
roughTokenCountEstimationForContent,
|
||||
} from '../services/tokenEstimation.js'
|
||||
import { estimateSkillFrontmatterTokens } from '../skills/loadSkillsDir.js'
|
||||
import {
|
||||
@ -52,6 +54,7 @@ import {
|
||||
calculateCurrentContextTokenTotal,
|
||||
getContextWindowForModel,
|
||||
} from './context.js'
|
||||
import { getProviderUsageTrust, hasMediaInput } from './contextBudget.js'
|
||||
import { getCwd } from './cwd.js'
|
||||
import { logForDebugging } from './debug.js'
|
||||
import { isEnvTruthy } from './envUtils.js'
|
||||
@ -59,6 +62,7 @@ import { errorMessage, toError } from './errors.js'
|
||||
import { logError } from './log.js'
|
||||
import { normalizeMessagesForAPI } from './messages.js'
|
||||
import { getRuntimeMainLoopModel } from './model/model.js'
|
||||
import { isFirstPartyAnthropicBaseUrl } from './model/providers.js'
|
||||
import type { SettingSource } from './settings/constants.js'
|
||||
import { jsonStringify } from './slowOperations.js'
|
||||
import { buildEffectiveSystemPrompt } from './systemPrompt.js'
|
||||
@ -83,7 +87,7 @@ async function countTokensWithFallback(
|
||||
estimateOnly = false,
|
||||
): Promise<number | null> {
|
||||
if (estimateOnly) {
|
||||
return roughTokenCountEstimation(jsonStringify({ messages, tools }))
|
||||
return roughTokenCountEstimationForAPIRequest(messages, tools)
|
||||
}
|
||||
|
||||
try {
|
||||
@ -810,8 +814,7 @@ function processAssistantMessage(
|
||||
): void {
|
||||
// Process each content block individually
|
||||
for (const block of msg.message.content) {
|
||||
const blockStr = jsonStringify(block)
|
||||
const blockTokens = roughTokenCountEstimation(blockStr)
|
||||
const blockTokens = roughTokenCountEstimationForContent([block])
|
||||
|
||||
if ('type' in block && block.type === 'tool_use') {
|
||||
breakdown.toolCallTokens += blockTokens
|
||||
@ -842,8 +845,7 @@ function processUserMessage(
|
||||
|
||||
// Process each content block individually
|
||||
for (const block of msg.message.content) {
|
||||
const blockStr = jsonStringify(block)
|
||||
const blockTokens = roughTokenCountEstimation(blockStr)
|
||||
const blockTokens = roughTokenCountEstimationForContent([block])
|
||||
|
||||
if ('type' in block && block.type === 'tool_result') {
|
||||
breakdown.toolResultTokens += blockTokens
|
||||
@ -865,8 +867,12 @@ function processAttachment(
|
||||
msg: AttachmentMessage,
|
||||
breakdown: MessageBreakdown,
|
||||
): void {
|
||||
const contentStr = jsonStringify(msg.attachment)
|
||||
const tokens = roughTokenCountEstimation(contentStr)
|
||||
const userMessages = normalizeAttachmentForAPI(msg.attachment)
|
||||
const tokens = userMessages.reduce(
|
||||
(total, userMsg) =>
|
||||
total + roughTokenCountEstimationForContent(userMsg.message.content),
|
||||
0,
|
||||
)
|
||||
breakdown.attachmentTokens += tokens
|
||||
const attachType = msg.attachment.type || 'unknown'
|
||||
breakdown.attachmentsByType.set(
|
||||
@ -1208,6 +1214,12 @@ export async function analyzeContextUsage(
|
||||
totalIncludingReserved,
|
||||
apiUsage,
|
||||
contextWindow,
|
||||
{
|
||||
hasMediaInput: hasMediaInput(originalMessages ?? messages),
|
||||
usageTrust: getProviderUsageTrust({
|
||||
isFirstPartyAnthropic: isFirstPartyAnthropicBaseUrl(),
|
||||
}),
|
||||
},
|
||||
)
|
||||
|
||||
// Pre-calculate grid based on model context window and terminal width
|
||||
|
||||
@ -2,6 +2,11 @@
|
||||
import { CONTEXT_1M_BETA_HEADER } from '../constants/betas.js'
|
||||
import { getOpenAICodexContextWindowForModel } from '../services/openaiAuth/models.js'
|
||||
import { getGlobalConfig } from './config.js'
|
||||
import {
|
||||
calculateContextBudget,
|
||||
calculateContextPercentagesFromTokens,
|
||||
type ProviderUsageTrust,
|
||||
} from './contextBudget.js'
|
||||
import { isEnvTruthy } from './envUtils.js'
|
||||
import { getCanonicalName } from './model/model.js'
|
||||
import { getModelCapability } from './model/modelCapabilities.js'
|
||||
@ -145,20 +150,12 @@ export function calculateContextPercentages(
|
||||
return { used: null, remaining: null }
|
||||
}
|
||||
|
||||
const totalInputTokens =
|
||||
return calculateContextPercentagesFromTokens(
|
||||
currentUsage.input_tokens +
|
||||
currentUsage.cache_creation_input_tokens +
|
||||
currentUsage.cache_read_input_tokens
|
||||
|
||||
const usedPercentage = Math.round(
|
||||
(totalInputTokens / contextWindowSize) * 100,
|
||||
currentUsage.cache_creation_input_tokens +
|
||||
currentUsage.cache_read_input_tokens,
|
||||
contextWindowSize,
|
||||
)
|
||||
const clampedUsed = Math.min(100, Math.max(0, usedPercentage))
|
||||
|
||||
return {
|
||||
used: clampedUsed,
|
||||
remaining: 100 - clampedUsed,
|
||||
}
|
||||
}
|
||||
|
||||
/**
|
||||
@ -184,7 +181,21 @@ export function calculateCurrentContextTokenTotal(
|
||||
cache_read_input_tokens: number
|
||||
} | null,
|
||||
contextWindow?: number,
|
||||
options?: { hasMediaInput?: boolean; usageTrust?: ProviderUsageTrust },
|
||||
): number {
|
||||
const hasMediaInput = options?.hasMediaInput ?? false
|
||||
const usageTrust = options?.usageTrust ?? 'high'
|
||||
|
||||
if (contextWindow !== undefined) {
|
||||
return calculateContextBudget({
|
||||
estimatedTokens,
|
||||
contextWindow,
|
||||
currentUsage,
|
||||
usageTrust,
|
||||
hasMediaInput,
|
||||
}).usedTokens
|
||||
}
|
||||
|
||||
if (!currentUsage) return estimatedTokens
|
||||
|
||||
const totalFromAPI =
|
||||
@ -193,8 +204,7 @@ export function calculateCurrentContextTokenTotal(
|
||||
currentUsage.cache_read_input_tokens +
|
||||
(currentUsage.output_tokens ?? 0)
|
||||
|
||||
const total = Math.max(estimatedTokens, totalFromAPI)
|
||||
return contextWindow !== undefined ? Math.min(total, contextWindow) : total
|
||||
return Math.max(estimatedTokens, totalFromAPI)
|
||||
}
|
||||
|
||||
/**
|
||||
|
||||
178
src/utils/contextBudget.ts
Normal file
178
src/utils/contextBudget.ts
Normal file
@ -0,0 +1,178 @@
|
||||
import { getAPIProvider } from './model/providers.js'
|
||||
|
||||
export type ProviderUsageTrust = 'high' | 'low'
|
||||
|
||||
export type ContextUsageLike = {
|
||||
input_tokens: number
|
||||
output_tokens?: number
|
||||
cache_creation_input_tokens: number
|
||||
cache_read_input_tokens: number
|
||||
}
|
||||
|
||||
export type ContextBudgetSource = 'estimate' | 'provider_usage'
|
||||
|
||||
export type ContextBudget = {
|
||||
usedTokens: number
|
||||
source: ContextBudgetSource
|
||||
providerUsageTokens: number | null
|
||||
estimatedTokens: number
|
||||
ignoredUsageReason?: 'low_trust_media_usage'
|
||||
}
|
||||
|
||||
export function getProviderUsageTrust(args: {
|
||||
isFirstPartyAnthropic: boolean
|
||||
}): ProviderUsageTrust {
|
||||
return getAPIProvider() === 'firstParty' && args.isFirstPartyAnthropic
|
||||
? 'high'
|
||||
: 'low'
|
||||
}
|
||||
|
||||
function isMediaAttachment(attachment: unknown): boolean {
|
||||
if (typeof attachment !== 'object' || attachment === null) return false
|
||||
|
||||
const candidate = attachment as {
|
||||
type?: unknown
|
||||
content?: { type?: unknown } | null
|
||||
}
|
||||
|
||||
if (candidate.type !== 'file') return false
|
||||
|
||||
return (
|
||||
candidate.content?.type === 'image' || candidate.content?.type === 'pdf'
|
||||
)
|
||||
}
|
||||
|
||||
export function hasMediaInput(
|
||||
messages: readonly {
|
||||
type: string
|
||||
message?: { content?: unknown }
|
||||
attachment?: unknown
|
||||
}[],
|
||||
): boolean {
|
||||
return messages.some(message => {
|
||||
if (message.type === 'attachment') {
|
||||
return isMediaAttachment(message.attachment)
|
||||
}
|
||||
if (message.type !== 'user' && message.type !== 'assistant') return false
|
||||
if (!Array.isArray(message.message?.content)) return false
|
||||
|
||||
return message.message.content.some(
|
||||
block =>
|
||||
typeof block === 'object' &&
|
||||
block !== null &&
|
||||
'type' in block &&
|
||||
(block.type === 'image' || block.type === 'document'),
|
||||
)
|
||||
})
|
||||
}
|
||||
|
||||
type ShouldIgnoreLowTrustUsageArgs = {
|
||||
usageTrust: ProviderUsageTrust
|
||||
hasMediaInput: boolean
|
||||
usageTokens: number
|
||||
estimatedTokens: number
|
||||
contextWindow: number
|
||||
}
|
||||
|
||||
type CalculateContextBudgetArgs = {
|
||||
estimatedTokens: number
|
||||
contextWindow: number
|
||||
currentUsage: ContextUsageLike | null
|
||||
usageTrust: ProviderUsageTrust
|
||||
hasMediaInput: boolean
|
||||
}
|
||||
|
||||
export function getUsageTokenTotal(
|
||||
usage: ContextUsageLike,
|
||||
options?: { includeOutput?: boolean },
|
||||
): number {
|
||||
const includeOutput = options?.includeOutput ?? true
|
||||
|
||||
return (
|
||||
usage.input_tokens +
|
||||
usage.cache_creation_input_tokens +
|
||||
usage.cache_read_input_tokens +
|
||||
(includeOutput ? usage.output_tokens ?? 0 : 0)
|
||||
)
|
||||
}
|
||||
|
||||
export function shouldIgnoreLowTrustUsage(
|
||||
args: ShouldIgnoreLowTrustUsageArgs,
|
||||
): boolean {
|
||||
if (args.usageTrust === 'high') return false
|
||||
if (!args.hasMediaInput) return false
|
||||
if (args.estimatedTokens <= 0) return false
|
||||
if (args.usageTokens < args.contextWindow) return false
|
||||
if (args.estimatedTokens >= args.contextWindow) return false
|
||||
|
||||
const suspiciousFloor = Math.max(
|
||||
args.estimatedTokens * 4,
|
||||
args.estimatedTokens + 50_000,
|
||||
)
|
||||
return args.usageTokens > suspiciousFloor
|
||||
}
|
||||
|
||||
export function calculateContextBudget(
|
||||
args: CalculateContextBudgetArgs,
|
||||
): ContextBudget {
|
||||
const estimatedTokens = Math.min(args.estimatedTokens, args.contextWindow)
|
||||
|
||||
if (!args.currentUsage) {
|
||||
return {
|
||||
usedTokens: estimatedTokens,
|
||||
source: 'estimate',
|
||||
providerUsageTokens: null,
|
||||
estimatedTokens: args.estimatedTokens,
|
||||
}
|
||||
}
|
||||
|
||||
const providerUsageTokens = getUsageTokenTotal(args.currentUsage)
|
||||
|
||||
if (
|
||||
shouldIgnoreLowTrustUsage({
|
||||
usageTrust: args.usageTrust,
|
||||
hasMediaInput: args.hasMediaInput,
|
||||
usageTokens: providerUsageTokens,
|
||||
estimatedTokens: args.estimatedTokens,
|
||||
contextWindow: args.contextWindow,
|
||||
})
|
||||
) {
|
||||
return {
|
||||
usedTokens: estimatedTokens,
|
||||
source: 'estimate',
|
||||
providerUsageTokens,
|
||||
estimatedTokens: args.estimatedTokens,
|
||||
ignoredUsageReason: 'low_trust_media_usage',
|
||||
}
|
||||
}
|
||||
|
||||
const usedTokens = Math.min(
|
||||
Math.max(args.estimatedTokens, providerUsageTokens),
|
||||
args.contextWindow,
|
||||
)
|
||||
|
||||
return {
|
||||
usedTokens,
|
||||
source:
|
||||
providerUsageTokens >= args.estimatedTokens ? 'provider_usage' : 'estimate',
|
||||
providerUsageTokens,
|
||||
estimatedTokens: args.estimatedTokens,
|
||||
}
|
||||
}
|
||||
|
||||
export function calculateContextPercentagesFromTokens(
|
||||
usedTokens: number | null,
|
||||
contextWindowSize: number,
|
||||
): { used: number | null; remaining: number | null } {
|
||||
if (usedTokens === null || contextWindowSize <= 0) {
|
||||
return { used: null, remaining: null }
|
||||
}
|
||||
|
||||
const usedPercentage = Math.round((usedTokens / contextWindowSize) * 100)
|
||||
const used = Math.min(100, Math.max(0, usedPercentage))
|
||||
|
||||
return {
|
||||
used,
|
||||
remaining: 100 - used,
|
||||
}
|
||||
}
|
||||
@ -31,6 +31,7 @@ import {
|
||||
} from '../services/analytics/growthbook.js'
|
||||
import {
|
||||
getImageTooLargeErrorMessage,
|
||||
getImageUnsupportedErrorMessage,
|
||||
getPdfInvalidErrorMessage,
|
||||
getPdfPasswordProtectedErrorMessage,
|
||||
getPdfTooLargeErrorMessage,
|
||||
@ -2006,6 +2007,7 @@ export function normalizeMessagesForAPI(
|
||||
[getPdfPasswordProtectedErrorMessage()]: new Set(['document']),
|
||||
[getPdfInvalidErrorMessage()]: new Set(['document']),
|
||||
[getImageTooLargeErrorMessage()]: new Set(['image']),
|
||||
[getImageUnsupportedErrorMessage()]: new Set(['image']),
|
||||
[getRequestTooLargeErrorMessage()]: new Set(['document', 'image']),
|
||||
}
|
||||
|
||||
@ -2030,10 +2032,12 @@ export function normalizeMessagesForAPI(
|
||||
if (!blockTypesToStrip) {
|
||||
continue
|
||||
}
|
||||
// Walk backward to find the nearest preceding isMeta user message
|
||||
// Walk backward to find the nearest preceding user message. Normal pasted
|
||||
// images are ordinary user turns, while attachment-derived media can be
|
||||
// meta turns; both need to be stripped after a provider media rejection.
|
||||
for (let j = i - 1; j >= 0; j--) {
|
||||
const candidate = reorderedMessages[j]!
|
||||
if (candidate.type === 'user' && candidate.isMeta) {
|
||||
if (candidate.type === 'user') {
|
||||
const existing = stripTargets.get(candidate.uuid)
|
||||
if (existing) {
|
||||
for (const t of blockTypesToStrip) {
|
||||
@ -2044,11 +2048,11 @@ export function normalizeMessagesForAPI(
|
||||
}
|
||||
break
|
||||
}
|
||||
// Skip over other synthetic error messages or non-meta messages
|
||||
// Skip over other synthetic error messages
|
||||
if (isSyntheticApiErrorMessage(candidate)) {
|
||||
continue
|
||||
}
|
||||
// Stop if we hit an assistant message or non-meta user message
|
||||
// Stop if we hit an assistant message or any other non-user message.
|
||||
break
|
||||
}
|
||||
}
|
||||
@ -2110,11 +2114,11 @@ export function normalizeMessagesForAPI(
|
||||
)
|
||||
}
|
||||
|
||||
// Strip document/image blocks from the specific meta user message that
|
||||
// Strip document/image blocks from the specific user message that
|
||||
// preceded a PDF/image/request-too-large error, to prevent re-sending
|
||||
// the problematic content on every subsequent API call.
|
||||
const typesToStrip = stripTargets.get(normalizedMessage.uuid)
|
||||
if (typesToStrip && normalizedMessage.isMeta) {
|
||||
if (typesToStrip) {
|
||||
const content = normalizedMessage.message.content
|
||||
if (Array.isArray(content)) {
|
||||
const filtered = content.filter(
|
||||
|
||||
67
tests/mediaRecoveryAndEstimation.test.ts
Normal file
67
tests/mediaRecoveryAndEstimation.test.ts
Normal file
@ -0,0 +1,67 @@
|
||||
import { describe, expect, test } from 'bun:test'
|
||||
import { getImageUnsupportedErrorMessage } from '../src/services/api/errors.js'
|
||||
import { roughTokenCountEstimationForAPIRequest } from '../src/services/tokenEstimation.js'
|
||||
import type { UserMessage } from '../src/types/message.js'
|
||||
import {
|
||||
createAssistantAPIErrorMessage,
|
||||
createUserMessage,
|
||||
normalizeMessagesForAPI,
|
||||
} from '../src/utils/messages.js'
|
||||
|
||||
const imageBlock = (data: string) => ({
|
||||
type: 'image' as const,
|
||||
source: {
|
||||
type: 'base64' as const,
|
||||
media_type: 'image/png' as const,
|
||||
data,
|
||||
},
|
||||
})
|
||||
|
||||
describe('media error recovery', () => {
|
||||
test('strips images after an unsupported-image model error', () => {
|
||||
const imageUser = createUserMessage({
|
||||
content: [
|
||||
{ type: 'text', text: 'describe this screenshot' },
|
||||
imageBlock('base64-image-payload'),
|
||||
{ type: 'text', text: '[Image source: /tmp/screenshot.png]' },
|
||||
],
|
||||
uuid: '00000000-0000-4000-8000-000000000001',
|
||||
})
|
||||
const unsupported = createAssistantAPIErrorMessage({
|
||||
content: getImageUnsupportedErrorMessage(),
|
||||
error: 'invalid_request',
|
||||
})
|
||||
const nextUser = createUserMessage({
|
||||
content: 'continue with text only',
|
||||
uuid: '00000000-0000-4000-8000-000000000002',
|
||||
})
|
||||
|
||||
const normalized = normalizeMessagesForAPI([imageUser, unsupported, nextUser])
|
||||
const serialized = JSON.stringify(normalized)
|
||||
|
||||
expect(serialized).not.toContain('base64-image-payload')
|
||||
expect(serialized).toContain('describe this screenshot')
|
||||
expect(serialized).toContain('continue with text only')
|
||||
})
|
||||
})
|
||||
|
||||
describe('media context estimation', () => {
|
||||
test('does not count base64 image bytes as text tokens', () => {
|
||||
const rawBase64 = 'a'.repeat(1_000_000)
|
||||
const tokens = roughTokenCountEstimationForAPIRequest(
|
||||
[
|
||||
{
|
||||
role: 'user',
|
||||
content: [
|
||||
{ type: 'text', text: 'what is in this image?' },
|
||||
imageBlock(rawBase64),
|
||||
] as UserMessage['message']['content'],
|
||||
},
|
||||
],
|
||||
[],
|
||||
)
|
||||
|
||||
expect(tokens).toBeGreaterThanOrEqual(2_000)
|
||||
expect(tokens).toBeLessThan(3_000)
|
||||
})
|
||||
})
|
||||
Loading…
x
Reference in New Issue
Block a user