mirror of
https://github.com/NanmiCoder/cc-haha
synced 2026-07-19 13:33:35 +08:00
fix(provider): diagnose stalled streams
Classify stream watchdog timeouts by stream phase and include non-sensitive diagnostics for stalled provider streams. Map known watchdog failures to stable desktop WebSocket error codes, and only retry watchdog aborts before content or tool activity starts. Tested: bun test src/services/api/streamWatchdog.test.ts Tested: bun test src/server/__tests__/ws-memory-events.test.ts --test-name-pattern "maps watchdog API errors to stable desktop error codes" Tested: cd desktop && bun test src/stores/chatStore.test.ts --test-name-pattern "persists partial assistant text before an error" Tested: bun run check:server Tested: cd desktop && bun run check:desktop Tested: 5x direct CLI replay, 5x desktop-chain replay, 5x desktop-chain no-entrypoint replay with session 446f8776-538d-46e6-96d6-67080b455b07
This commit is contained in:
parent
75ddd158b4
commit
e4e7026cde
@ -4396,13 +4396,17 @@ describe('chatStore history mapping', () => {
|
|||||||
})
|
})
|
||||||
useChatStore.getState().handleServerMessage(TEST_SESSION_ID, {
|
useChatStore.getState().handleServerMessage(TEST_SESSION_ID, {
|
||||||
type: 'error',
|
type: 'error',
|
||||||
message: 'provider failed',
|
message: 'API Error: Provider stream stalled after partial response - no new chunks for 240s',
|
||||||
code: 'provider_error',
|
code: 'STREAM_IDLE_TIMEOUT',
|
||||||
})
|
})
|
||||||
|
|
||||||
expect(useChatStore.getState().sessions[TEST_SESSION_ID]?.messages).toMatchObject([
|
expect(useChatStore.getState().sessions[TEST_SESSION_ID]?.messages).toMatchObject([
|
||||||
{ type: 'assistant_text', content: 'partial answer before error' },
|
{ type: 'assistant_text', content: 'partial answer before error' },
|
||||||
{ type: 'error', message: 'provider failed' },
|
{
|
||||||
|
type: 'error',
|
||||||
|
message: 'API Error: Provider stream stalled after partial response - no new chunks for 240s',
|
||||||
|
code: 'STREAM_IDLE_TIMEOUT',
|
||||||
|
},
|
||||||
])
|
])
|
||||||
|
|
||||||
vi.runOnlyPendingTimers()
|
vi.runOnlyPendingTimers()
|
||||||
|
|||||||
@ -27,6 +27,46 @@ describe('WebSocket memory events', () => {
|
|||||||
])
|
])
|
||||||
})
|
})
|
||||||
|
|
||||||
|
it('maps watchdog API errors to stable desktop error codes', () => {
|
||||||
|
expect(translateCliMessage({
|
||||||
|
type: 'assistant',
|
||||||
|
error: 'server_error',
|
||||||
|
isApiErrorMessage: true,
|
||||||
|
message: {
|
||||||
|
role: 'assistant',
|
||||||
|
content: [{
|
||||||
|
type: 'text',
|
||||||
|
text: 'API Error: Provider stream stalled after partial response - no new chunks for 240s (last event: text_delta, content deltas: 1)',
|
||||||
|
}],
|
||||||
|
},
|
||||||
|
}, 'session-1')).toEqual([
|
||||||
|
{
|
||||||
|
type: 'error',
|
||||||
|
message: 'API Error: Provider stream stalled after partial response - no new chunks for 240s (last event: text_delta, content deltas: 1)',
|
||||||
|
code: 'STREAM_IDLE_TIMEOUT',
|
||||||
|
},
|
||||||
|
])
|
||||||
|
|
||||||
|
expect(translateCliMessage({
|
||||||
|
type: 'assistant',
|
||||||
|
error: 'server_error',
|
||||||
|
isApiErrorMessage: true,
|
||||||
|
message: {
|
||||||
|
role: 'assistant',
|
||||||
|
content: [{
|
||||||
|
type: 'text',
|
||||||
|
text: 'API Error: Stream max duration exceeded - no completion received after 600s (last event: text_delta)',
|
||||||
|
}],
|
||||||
|
},
|
||||||
|
}, 'session-1')).toEqual([
|
||||||
|
{
|
||||||
|
type: 'error',
|
||||||
|
message: 'API Error: Stream max duration exceeded - no completion received after 600s (last event: text_delta)',
|
||||||
|
code: 'STREAM_MAX_DURATION',
|
||||||
|
},
|
||||||
|
])
|
||||||
|
})
|
||||||
|
|
||||||
it('replays slash-command breadcrumbs as readable user messages', () => {
|
it('replays slash-command breadcrumbs as readable user messages', () => {
|
||||||
expect(translateCliMessage({
|
expect(translateCliMessage({
|
||||||
type: 'user',
|
type: 'user',
|
||||||
|
|||||||
@ -1357,6 +1357,19 @@ function isDuplicateOfLastApiError(
|
|||||||
)
|
)
|
||||||
}
|
}
|
||||||
|
|
||||||
|
function classifyRuntimeErrorCode(message: string, fallbackCode: string): string {
|
||||||
|
if (/Stream max duration exceeded/i.test(message)) {
|
||||||
|
return 'STREAM_MAX_DURATION'
|
||||||
|
}
|
||||||
|
if (
|
||||||
|
/Provider stream stalled after partial response/i.test(message) ||
|
||||||
|
/Stream idle timeout/i.test(message)
|
||||||
|
) {
|
||||||
|
return 'STREAM_IDLE_TIMEOUT'
|
||||||
|
}
|
||||||
|
return fallbackCode
|
||||||
|
}
|
||||||
|
|
||||||
function bindPrewarmMetadataCapture(sessionId: string) {
|
function bindPrewarmMetadataCapture(sessionId: string) {
|
||||||
for (const msg of conversationService.getRecentSdkMessages(sessionId)) {
|
for (const msg of conversationService.getRecentSdkMessages(sessionId)) {
|
||||||
cacheSessionInitMetadata(sessionId, msg)
|
cacheSessionInitMetadata(sessionId, msg)
|
||||||
@ -1444,7 +1457,8 @@ export function translateCliMessage(cliMsg: any, sessionId: string): ServerMessa
|
|||||||
return []
|
return []
|
||||||
}
|
}
|
||||||
const message = extractAssistantText(cliMsg) || cliMsg.error || 'Unknown API error'
|
const message = extractAssistantText(cliMsg) || cliMsg.error || 'Unknown API error'
|
||||||
const code = typeof cliMsg.error === 'string' ? cliMsg.error : 'API_ERROR'
|
const fallbackCode = typeof cliMsg.error === 'string' ? cliMsg.error : 'API_ERROR'
|
||||||
|
const code = classifyRuntimeErrorCode(message, fallbackCode)
|
||||||
streamState.lastApiError = { message, code }
|
streamState.lastApiError = { message, code }
|
||||||
return [{
|
return [{
|
||||||
type: 'error',
|
type: 'error',
|
||||||
@ -1744,7 +1758,7 @@ export function translateCliMessage(cliMsg: any, sessionId: string): ServerMessa
|
|||||||
{
|
{
|
||||||
type: 'error',
|
type: 'error',
|
||||||
message: resultMessage,
|
message: resultMessage,
|
||||||
code: 'CLI_ERROR',
|
code: classifyRuntimeErrorCode(resultMessage, 'CLI_ERROR'),
|
||||||
},
|
},
|
||||||
{ type: 'message_complete', usage },
|
{ type: 'message_complete', usage },
|
||||||
]
|
]
|
||||||
|
|||||||
@ -212,6 +212,10 @@ import {
|
|||||||
stopSessionActivity,
|
stopSessionActivity,
|
||||||
} from "../../utils/sessionActivity.js";
|
} from "../../utils/sessionActivity.js";
|
||||||
import { shouldTriggerNonStreamingFallbackForEmptyStream } from "./streamFallback.js";
|
import { shouldTriggerNonStreamingFallbackForEmptyStream } from "./streamFallback.js";
|
||||||
|
import {
|
||||||
|
StreamWatchdogTimeoutError,
|
||||||
|
createStreamWatchdogState,
|
||||||
|
} from "./streamWatchdog.js";
|
||||||
import { jsonStringify } from "../../utils/slowOperations.js";
|
import { jsonStringify } from "../../utils/slowOperations.js";
|
||||||
import {
|
import {
|
||||||
isBetaTracingEnabled,
|
isBetaTracingEnabled,
|
||||||
@ -2009,6 +2013,8 @@ async function* queryModel(
|
|||||||
let streamIdleAborted = false;
|
let streamIdleAborted = false;
|
||||||
// Which watchdog tripped, so the thrown error message is accurate.
|
// Which watchdog tripped, so the thrown error message is accurate.
|
||||||
let streamAbortReason: "idle" | "max_duration" | null = null;
|
let streamAbortReason: "idle" | "max_duration" | null = null;
|
||||||
|
const streamWatchdogState = createStreamWatchdogState();
|
||||||
|
let streamWatchdogTimeoutError: StreamWatchdogTimeoutError | null = null;
|
||||||
// performance.now() snapshot when watchdog fires, for measuring abort propagation delay
|
// performance.now() snapshot when watchdog fires, for measuring abort propagation delay
|
||||||
let streamWatchdogFiredAt: number | null = null;
|
let streamWatchdogFiredAt: number | null = null;
|
||||||
let streamIdleWarningTimer: ReturnType<typeof setTimeout> | null = null;
|
let streamIdleWarningTimer: ReturnType<typeof setTimeout> | null = null;
|
||||||
@ -2035,11 +2041,19 @@ async function* queryModel(
|
|||||||
const warningMs = idleMs / 2;
|
const warningMs = idleMs / 2;
|
||||||
streamIdleWarningTimer = setTimeout(
|
streamIdleWarningTimer = setTimeout(
|
||||||
(warnMs) => {
|
(warnMs) => {
|
||||||
|
const warningError = streamWatchdogState.createTimeoutError(
|
||||||
|
"idle",
|
||||||
|
warnMs,
|
||||||
|
);
|
||||||
logForDebugging(
|
logForDebugging(
|
||||||
`Streaming idle warning: no chunks received for ${warnMs / 1000}s`,
|
`Streaming idle warning: ${warningError.message}`,
|
||||||
{ level: "warn" },
|
{ level: "warn" },
|
||||||
);
|
);
|
||||||
logForDiagnosticsNoPII("warn", "cli_streaming_idle_warning");
|
logForDiagnosticsNoPII(
|
||||||
|
"warn",
|
||||||
|
"cli_streaming_idle_warning",
|
||||||
|
warningError.toDiagnosticData(),
|
||||||
|
);
|
||||||
},
|
},
|
||||||
warningMs,
|
warningMs,
|
||||||
warningMs,
|
warningMs,
|
||||||
@ -2048,17 +2062,31 @@ async function* queryModel(
|
|||||||
streamIdleAborted = true;
|
streamIdleAborted = true;
|
||||||
streamAbortReason = "idle";
|
streamAbortReason = "idle";
|
||||||
streamWatchdogFiredAt = performance.now();
|
streamWatchdogFiredAt = performance.now();
|
||||||
|
streamWatchdogTimeoutError = streamWatchdogState.createTimeoutError(
|
||||||
|
"idle",
|
||||||
|
idleMs,
|
||||||
|
);
|
||||||
logForDebugging(
|
logForDebugging(
|
||||||
`Streaming idle timeout: no chunks received for ${idleMs / 1000}s, aborting stream`,
|
`Streaming idle timeout: ${streamWatchdogTimeoutError.message}, aborting stream`,
|
||||||
{ level: "error" },
|
{ level: "error" },
|
||||||
);
|
);
|
||||||
logForDiagnosticsNoPII("error", "cli_streaming_idle_timeout");
|
logForDiagnosticsNoPII(
|
||||||
|
"error",
|
||||||
|
"cli_streaming_idle_timeout",
|
||||||
|
streamWatchdogTimeoutError.toDiagnosticData(),
|
||||||
|
);
|
||||||
logEvent("tengu_streaming_idle_timeout", {
|
logEvent("tengu_streaming_idle_timeout", {
|
||||||
model:
|
model:
|
||||||
options.model as AnalyticsMetadata_I_VERIFIED_THIS_IS_NOT_CODE_OR_FILEPATHS,
|
options.model as AnalyticsMetadata_I_VERIFIED_THIS_IS_NOT_CODE_OR_FILEPATHS,
|
||||||
request_id: (streamRequestId ??
|
request_id: (streamRequestId ??
|
||||||
"unknown") as AnalyticsMetadata_I_VERIFIED_THIS_IS_NOT_CODE_OR_FILEPATHS,
|
"unknown") as AnalyticsMetadata_I_VERIFIED_THIS_IS_NOT_CODE_OR_FILEPATHS,
|
||||||
timeout_ms: idleMs,
|
timeout_ms: idleMs,
|
||||||
|
reason:
|
||||||
|
streamWatchdogTimeoutError.reason as AnalyticsMetadata_I_VERIFIED_THIS_IS_NOT_CODE_OR_FILEPATHS,
|
||||||
|
phase:
|
||||||
|
streamWatchdogTimeoutError.phase as AnalyticsMetadata_I_VERIFIED_THIS_IS_NOT_CODE_OR_FILEPATHS,
|
||||||
|
content_delta_count:
|
||||||
|
streamWatchdogTimeoutError.streamSnapshot.contentDeltaCount,
|
||||||
});
|
});
|
||||||
releaseStreamResources();
|
releaseStreamResources();
|
||||||
}, idleMs);
|
}, idleMs);
|
||||||
@ -2072,19 +2100,29 @@ async function* queryModel(
|
|||||||
streamIdleAborted = true;
|
streamIdleAborted = true;
|
||||||
streamAbortReason = "max_duration";
|
streamAbortReason = "max_duration";
|
||||||
streamWatchdogFiredAt = performance.now();
|
streamWatchdogFiredAt = performance.now();
|
||||||
|
streamWatchdogTimeoutError = streamWatchdogState.createTimeoutError(
|
||||||
|
"max_duration",
|
||||||
|
STREAM_MAX_DURATION_MS,
|
||||||
|
);
|
||||||
logForDebugging(
|
logForDebugging(
|
||||||
`Streaming max duration exceeded: no completion after ${STREAM_MAX_DURATION_MS / 1000}s, aborting stream`,
|
`Streaming max duration exceeded: ${streamWatchdogTimeoutError.message}, aborting stream`,
|
||||||
{ level: "error" },
|
{ level: "error" },
|
||||||
);
|
);
|
||||||
logForDiagnosticsNoPII("error", "cli_streaming_max_duration_exceeded", {
|
logForDiagnosticsNoPII("error", "cli_streaming_max_duration_exceeded", {
|
||||||
timeoutMs: STREAM_MAX_DURATION_MS,
|
...streamWatchdogTimeoutError.toDiagnosticData(),
|
||||||
});
|
});
|
||||||
logEvent("tengu_streaming_idle_timeout", {
|
logEvent("tengu_streaming_max_duration_timeout", {
|
||||||
model:
|
model:
|
||||||
options.model as AnalyticsMetadata_I_VERIFIED_THIS_IS_NOT_CODE_OR_FILEPATHS,
|
options.model as AnalyticsMetadata_I_VERIFIED_THIS_IS_NOT_CODE_OR_FILEPATHS,
|
||||||
request_id: (streamRequestId ??
|
request_id: (streamRequestId ??
|
||||||
"unknown") as AnalyticsMetadata_I_VERIFIED_THIS_IS_NOT_CODE_OR_FILEPATHS,
|
"unknown") as AnalyticsMetadata_I_VERIFIED_THIS_IS_NOT_CODE_OR_FILEPATHS,
|
||||||
timeout_ms: STREAM_MAX_DURATION_MS,
|
timeout_ms: STREAM_MAX_DURATION_MS,
|
||||||
|
reason:
|
||||||
|
streamWatchdogTimeoutError.reason as AnalyticsMetadata_I_VERIFIED_THIS_IS_NOT_CODE_OR_FILEPATHS,
|
||||||
|
phase:
|
||||||
|
streamWatchdogTimeoutError.phase as AnalyticsMetadata_I_VERIFIED_THIS_IS_NOT_CODE_OR_FILEPATHS,
|
||||||
|
content_delta_count:
|
||||||
|
streamWatchdogTimeoutError.streamSnapshot.contentDeltaCount,
|
||||||
});
|
});
|
||||||
releaseStreamResources();
|
releaseStreamResources();
|
||||||
}, STREAM_MAX_DURATION_MS);
|
}, STREAM_MAX_DURATION_MS);
|
||||||
@ -2100,6 +2138,7 @@ async function* queryModel(
|
|||||||
let stallCount = 0;
|
let stallCount = 0;
|
||||||
|
|
||||||
for await (const part of stream) {
|
for await (const part of stream) {
|
||||||
|
const receivedFirstContentDelta = streamWatchdogState.recordEvent(part);
|
||||||
resetStreamIdleTimer();
|
resetStreamIdleTimer();
|
||||||
const now = Date.now();
|
const now = Date.now();
|
||||||
|
|
||||||
@ -2136,13 +2175,18 @@ async function* queryModel(
|
|||||||
}
|
}
|
||||||
endQueryProfile();
|
endQueryProfile();
|
||||||
isFirstChunk = false;
|
isFirstChunk = false;
|
||||||
// Tokens are flowing — switch the watchdog from the generous
|
}
|
||||||
// first-token (prefill) budget to the shorter mid-stream idle budget,
|
|
||||||
// so a stall *after* output started is still reclaimed quickly (#826).
|
// Content is flowing - switch the watchdog from the generous first-token
|
||||||
if (currentStreamIdleTimeoutMs !== STREAM_IDLE_TIMEOUT_MS) {
|
// (prefill) budget to the shorter mid-stream idle budget. Non-content
|
||||||
currentStreamIdleTimeoutMs = STREAM_IDLE_TIMEOUT_MS;
|
// events such as message_start must not end the first-token phase: some
|
||||||
resetStreamIdleTimer();
|
// gateways open SSE and emit bookkeeping long before useful content.
|
||||||
}
|
if (
|
||||||
|
receivedFirstContentDelta &&
|
||||||
|
currentStreamIdleTimeoutMs !== STREAM_IDLE_TIMEOUT_MS
|
||||||
|
) {
|
||||||
|
currentStreamIdleTimeoutMs = STREAM_IDLE_TIMEOUT_MS;
|
||||||
|
resetStreamIdleTimer();
|
||||||
}
|
}
|
||||||
|
|
||||||
switch (part.type) {
|
switch (part.type) {
|
||||||
@ -2504,11 +2548,13 @@ async function* queryModel(
|
|||||||
// Prevent double-emit: this throw lands in the catch block below,
|
// Prevent double-emit: this throw lands in the catch block below,
|
||||||
// whose exit_path='error' probe guards on streamWatchdogFiredAt.
|
// whose exit_path='error' probe guards on streamWatchdogFiredAt.
|
||||||
streamWatchdogFiredAt = null;
|
streamWatchdogFiredAt = null;
|
||||||
throw new Error(
|
throw streamWatchdogTimeoutError ??
|
||||||
streamAbortReason === "max_duration"
|
streamWatchdogState.createTimeoutError(
|
||||||
? "Stream max duration exceeded - no completion received"
|
streamAbortReason ?? "idle",
|
||||||
: "Stream idle timeout - no chunks received",
|
streamAbortReason === "max_duration"
|
||||||
);
|
? STREAM_MAX_DURATION_MS
|
||||||
|
: currentStreamIdleTimeoutMs,
|
||||||
|
);
|
||||||
}
|
}
|
||||||
|
|
||||||
// Detect when the stream completed without producing any assistant messages.
|
// Detect when the stream completed without producing any assistant messages.
|
||||||
@ -2660,9 +2706,26 @@ async function* queryModel(
|
|||||||
// (newMessages.length === 0): a zero-output stream means no tool_use block
|
// (newMessages.length === 0): a zero-output stream means no tool_use block
|
||||||
// ever completed, so query.ts never started a tool — no double-execution
|
// ever completed, so query.ts never started a tool — no double-execution
|
||||||
// risk (cf. #766 / inc-4258), the same precondition the zero-output
|
// risk (cf. #766 / inc-4258), the same precondition the zero-output
|
||||||
// fallback below relies on. Watchdog aborts are excluded (they have their
|
// fallback below relies on. Watchdog aborts without content/tool output
|
||||||
// own fallback/timeout handling). Thrown past the outer catch — which
|
// are handled first; remaining watchdog aborts keep their partial-output
|
||||||
|
// boundary and are not retried. Thrown past the outer catch — which
|
||||||
// re-throws it — up to withStreamRetry.
|
// re-throws it — up to withStreamRetry.
|
||||||
|
if (
|
||||||
|
streamIdleAborted &&
|
||||||
|
streamingError instanceof StreamWatchdogTimeoutError &&
|
||||||
|
streamingError.safeToRetryStream() &&
|
||||||
|
newMessages.length === 0 &&
|
||||||
|
!signal.aborted
|
||||||
|
) {
|
||||||
|
logForDebugging(
|
||||||
|
`Watchdog timeout before content/tool output, will retry stream: ${errorMessage(
|
||||||
|
streamingError,
|
||||||
|
)}`,
|
||||||
|
{ level: "warn" },
|
||||||
|
);
|
||||||
|
throw new RetriableStreamError(streamingError);
|
||||||
|
}
|
||||||
|
|
||||||
if (
|
if (
|
||||||
newMessages.length === 0 &&
|
newMessages.length === 0 &&
|
||||||
!streamIdleAborted &&
|
!streamIdleAborted &&
|
||||||
|
|||||||
@ -51,6 +51,7 @@ import {
|
|||||||
} from '../claudeAiLimits.js'
|
} from '../claudeAiLimits.js'
|
||||||
import { shouldProcessRateLimits } from '../rateLimitMocking.js' // Used for /mock-limits command
|
import { shouldProcessRateLimits } from '../rateLimitMocking.js' // Used for /mock-limits command
|
||||||
import { extractConnectionErrorDetails, formatAPIError } from './errorUtils.js'
|
import { extractConnectionErrorDetails, formatAPIError } from './errorUtils.js'
|
||||||
|
import { StreamWatchdogTimeoutError } from './streamWatchdog.js'
|
||||||
|
|
||||||
export const API_ERROR_MESSAGE_PREFIX = 'API Error'
|
export const API_ERROR_MESSAGE_PREFIX = 'API Error'
|
||||||
|
|
||||||
@ -987,6 +988,14 @@ export function getAssistantMessageFromError(
|
|||||||
})
|
})
|
||||||
}
|
}
|
||||||
|
|
||||||
|
if (error instanceof StreamWatchdogTimeoutError) {
|
||||||
|
return createAssistantAPIErrorMessage({
|
||||||
|
content: `${API_ERROR_MESSAGE_PREFIX}: ${error.message}`,
|
||||||
|
error: 'server_error',
|
||||||
|
errorDetails: JSON.stringify(error.toDiagnosticData()),
|
||||||
|
})
|
||||||
|
}
|
||||||
|
|
||||||
// Connection errors (non-timeout) — use formatAPIError for detailed messages
|
// Connection errors (non-timeout) — use formatAPIError for detailed messages
|
||||||
if (error instanceof APIConnectionError) {
|
if (error instanceof APIConnectionError) {
|
||||||
return createAssistantAPIErrorMessage({
|
return createAssistantAPIErrorMessage({
|
||||||
|
|||||||
118
src/services/api/streamWatchdog.test.ts
Normal file
118
src/services/api/streamWatchdog.test.ts
Normal file
@ -0,0 +1,118 @@
|
|||||||
|
import { describe, expect, test } from 'bun:test'
|
||||||
|
import {
|
||||||
|
StreamWatchdogTimeoutError,
|
||||||
|
createStreamWatchdogState,
|
||||||
|
} from './streamWatchdog.js'
|
||||||
|
|
||||||
|
describe('stream watchdog state', () => {
|
||||||
|
test('keeps the first-token budget until content deltas arrive', () => {
|
||||||
|
const state = createStreamWatchdogState()
|
||||||
|
|
||||||
|
expect(state.recordEvent({ type: 'message_start' })).toBe(false)
|
||||||
|
expect(state.recordEvent({
|
||||||
|
type: 'content_block_start',
|
||||||
|
index: 0,
|
||||||
|
content_block: { type: 'thinking' },
|
||||||
|
})).toBe(false)
|
||||||
|
|
||||||
|
expect(state.hasContentDelta()).toBe(false)
|
||||||
|
expect(state.snapshot().phase).toBe('before_content')
|
||||||
|
|
||||||
|
expect(state.recordEvent({
|
||||||
|
type: 'content_block_delta',
|
||||||
|
index: 0,
|
||||||
|
delta: { type: 'thinking_delta', thinking: 'working' },
|
||||||
|
})).toBe(true)
|
||||||
|
|
||||||
|
expect(state.hasContentDelta()).toBe(true)
|
||||||
|
expect(state.snapshot()).toMatchObject({
|
||||||
|
phase: 'mid_stream',
|
||||||
|
eventCount: 3,
|
||||||
|
contentDeltaCount: 1,
|
||||||
|
thinkingDeltaCount: 1,
|
||||||
|
lastEventType: 'content_block_delta',
|
||||||
|
lastDeltaType: 'thinking_delta',
|
||||||
|
lastBlockType: 'thinking',
|
||||||
|
})
|
||||||
|
})
|
||||||
|
|
||||||
|
test('describes a stream that started but never produced content', () => {
|
||||||
|
const state = createStreamWatchdogState()
|
||||||
|
state.recordEvent({ type: 'message_start' })
|
||||||
|
state.recordEvent({
|
||||||
|
type: 'content_block_start',
|
||||||
|
index: 0,
|
||||||
|
content_block: { type: 'thinking' },
|
||||||
|
})
|
||||||
|
|
||||||
|
const error = state.createTimeoutError('idle', 600_000)
|
||||||
|
|
||||||
|
expect(error).toBeInstanceOf(StreamWatchdogTimeoutError)
|
||||||
|
expect(error.code).toBe('STREAM_IDLE_TIMEOUT')
|
||||||
|
expect(error.phase).toBe('before_content')
|
||||||
|
expect(error.safeToRetryStream()).toBe(true)
|
||||||
|
expect(error.message).toContain('stream started but no content was received')
|
||||||
|
expect(error.message).toContain('last event: content_block_start')
|
||||||
|
})
|
||||||
|
|
||||||
|
test('describes a provider stream that stalls after partial output', () => {
|
||||||
|
const state = createStreamWatchdogState()
|
||||||
|
state.recordEvent({ type: 'message_start' })
|
||||||
|
state.recordEvent({
|
||||||
|
type: 'content_block_start',
|
||||||
|
index: 0,
|
||||||
|
content_block: { type: 'text' },
|
||||||
|
})
|
||||||
|
state.recordEvent({
|
||||||
|
type: 'content_block_delta',
|
||||||
|
index: 0,
|
||||||
|
delta: { type: 'text_delta', text: 'writing HTML report, partial' },
|
||||||
|
})
|
||||||
|
|
||||||
|
const error = state.createTimeoutError('idle', 240_000)
|
||||||
|
|
||||||
|
expect(error.code).toBe('STREAM_IDLE_TIMEOUT')
|
||||||
|
expect(error.phase).toBe('mid_stream')
|
||||||
|
expect(error.safeToRetryStream()).toBe(false)
|
||||||
|
expect(error.message).toContain('Provider stream stalled after partial response')
|
||||||
|
expect(error.message).toContain('last event: text_delta')
|
||||||
|
expect(error.message).not.toContain('no chunks received')
|
||||||
|
})
|
||||||
|
|
||||||
|
test('does not retry after a tool use has started', () => {
|
||||||
|
const state = createStreamWatchdogState()
|
||||||
|
state.recordEvent({ type: 'message_start' })
|
||||||
|
state.recordEvent({
|
||||||
|
type: 'content_block_start',
|
||||||
|
index: 0,
|
||||||
|
content_block: { type: 'tool_use' },
|
||||||
|
})
|
||||||
|
|
||||||
|
const error = state.createTimeoutError('idle', 240_000)
|
||||||
|
|
||||||
|
expect(error.phase).toBe('before_content')
|
||||||
|
expect(error.safeToRetryStream()).toBe(false)
|
||||||
|
})
|
||||||
|
|
||||||
|
test('classifies max-duration aborts separately from idle aborts', () => {
|
||||||
|
const state = createStreamWatchdogState()
|
||||||
|
state.recordEvent({ type: 'message_start' })
|
||||||
|
state.recordEvent({
|
||||||
|
type: 'content_block_start',
|
||||||
|
index: 0,
|
||||||
|
content_block: { type: 'text' },
|
||||||
|
})
|
||||||
|
state.recordEvent({
|
||||||
|
type: 'content_block_delta',
|
||||||
|
index: 0,
|
||||||
|
delta: { type: 'text_delta', text: 'partial' },
|
||||||
|
})
|
||||||
|
|
||||||
|
const error = state.createTimeoutError('max_duration', 600_000)
|
||||||
|
|
||||||
|
expect(error.code).toBe('STREAM_MAX_DURATION')
|
||||||
|
expect(error.phase).toBe('mid_stream')
|
||||||
|
expect(error.message).toContain('Stream max duration exceeded')
|
||||||
|
expect(error.message).toContain('last event: text_delta')
|
||||||
|
})
|
||||||
|
})
|
||||||
270
src/services/api/streamWatchdog.ts
Normal file
270
src/services/api/streamWatchdog.ts
Normal file
@ -0,0 +1,270 @@
|
|||||||
|
export type StreamWatchdogAbortReason = 'idle' | 'max_duration'
|
||||||
|
export type StreamWatchdogErrorCode = 'STREAM_IDLE_TIMEOUT' | 'STREAM_MAX_DURATION'
|
||||||
|
export type StreamWatchdogPhase =
|
||||||
|
| 'before_first_event'
|
||||||
|
| 'before_content'
|
||||||
|
| 'mid_stream'
|
||||||
|
|
||||||
|
export type StreamWatchdogSnapshot = {
|
||||||
|
phase: StreamWatchdogPhase
|
||||||
|
eventCount: number
|
||||||
|
contentDeltaCount: number
|
||||||
|
textDeltaCount: number
|
||||||
|
thinkingDeltaCount: number
|
||||||
|
toolInputDeltaCount: number
|
||||||
|
lastEventType?: string
|
||||||
|
lastDeltaType?: string
|
||||||
|
lastBlockType?: string
|
||||||
|
messageStopReceived: boolean
|
||||||
|
toolUseStarted: boolean
|
||||||
|
}
|
||||||
|
|
||||||
|
type StreamEventLike = {
|
||||||
|
type?: unknown
|
||||||
|
index?: unknown
|
||||||
|
content_block?: unknown
|
||||||
|
delta?: unknown
|
||||||
|
}
|
||||||
|
|
||||||
|
function asObject(value: unknown): Record<string, unknown> | null {
|
||||||
|
return value && typeof value === 'object'
|
||||||
|
? value as Record<string, unknown>
|
||||||
|
: null
|
||||||
|
}
|
||||||
|
|
||||||
|
function readString(value: unknown): string | undefined {
|
||||||
|
return typeof value === 'string' ? value : undefined
|
||||||
|
}
|
||||||
|
|
||||||
|
function readIndex(value: unknown): number | undefined {
|
||||||
|
return typeof value === 'number' && Number.isInteger(value) ? value : undefined
|
||||||
|
}
|
||||||
|
|
||||||
|
function hasNonEmptyStringField(
|
||||||
|
value: Record<string, unknown>,
|
||||||
|
field: string,
|
||||||
|
): boolean {
|
||||||
|
const text = value[field]
|
||||||
|
return typeof text === 'string' && text.length > 0
|
||||||
|
}
|
||||||
|
|
||||||
|
function isContentDelta(delta: Record<string, unknown>): boolean {
|
||||||
|
switch (delta.type) {
|
||||||
|
case 'text_delta':
|
||||||
|
return hasNonEmptyStringField(delta, 'text')
|
||||||
|
case 'thinking_delta':
|
||||||
|
return hasNonEmptyStringField(delta, 'thinking')
|
||||||
|
case 'input_json_delta':
|
||||||
|
return hasNonEmptyStringField(delta, 'partial_json')
|
||||||
|
case 'connector_text_delta':
|
||||||
|
return hasNonEmptyStringField(delta, 'connector_text')
|
||||||
|
default:
|
||||||
|
return false
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
function countContentDelta(
|
||||||
|
snapshot: StreamWatchdogSnapshot,
|
||||||
|
deltaType: string | undefined,
|
||||||
|
): StreamWatchdogSnapshot {
|
||||||
|
switch (deltaType) {
|
||||||
|
case 'text_delta':
|
||||||
|
case 'connector_text_delta':
|
||||||
|
return {
|
||||||
|
...snapshot,
|
||||||
|
contentDeltaCount: snapshot.contentDeltaCount + 1,
|
||||||
|
textDeltaCount: snapshot.textDeltaCount + 1,
|
||||||
|
}
|
||||||
|
case 'thinking_delta':
|
||||||
|
return {
|
||||||
|
...snapshot,
|
||||||
|
contentDeltaCount: snapshot.contentDeltaCount + 1,
|
||||||
|
thinkingDeltaCount: snapshot.thinkingDeltaCount + 1,
|
||||||
|
}
|
||||||
|
case 'input_json_delta':
|
||||||
|
return {
|
||||||
|
...snapshot,
|
||||||
|
contentDeltaCount: snapshot.contentDeltaCount + 1,
|
||||||
|
toolInputDeltaCount: snapshot.toolInputDeltaCount + 1,
|
||||||
|
}
|
||||||
|
default:
|
||||||
|
return snapshot
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
function formatSeconds(ms: number): string {
|
||||||
|
return `${Math.round(ms / 1000)}s`
|
||||||
|
}
|
||||||
|
|
||||||
|
function formatDetails(snapshot: StreamWatchdogSnapshot): string {
|
||||||
|
const lastEvent = snapshot.lastDeltaType ?? snapshot.lastEventType ?? 'none'
|
||||||
|
const parts = [
|
||||||
|
`last event: ${lastEvent}`,
|
||||||
|
`events: ${snapshot.eventCount}`,
|
||||||
|
`content deltas: ${snapshot.contentDeltaCount}`,
|
||||||
|
]
|
||||||
|
if (snapshot.lastBlockType) {
|
||||||
|
parts.push(`last block: ${snapshot.lastBlockType}`)
|
||||||
|
}
|
||||||
|
return parts.join(', ')
|
||||||
|
}
|
||||||
|
|
||||||
|
function buildMessage(
|
||||||
|
reason: StreamWatchdogAbortReason,
|
||||||
|
timeoutMs: number,
|
||||||
|
snapshot: StreamWatchdogSnapshot,
|
||||||
|
): string {
|
||||||
|
const seconds = formatSeconds(timeoutMs)
|
||||||
|
if (reason === 'max_duration') {
|
||||||
|
return `Stream max duration exceeded - no completion received after ${seconds} (${formatDetails(snapshot)})`
|
||||||
|
}
|
||||||
|
|
||||||
|
switch (snapshot.phase) {
|
||||||
|
case 'before_first_event':
|
||||||
|
return `Stream idle timeout - no stream events received for ${seconds}`
|
||||||
|
case 'before_content':
|
||||||
|
return `Stream idle timeout - stream started but no content was received for ${seconds} (${formatDetails(snapshot)})`
|
||||||
|
case 'mid_stream':
|
||||||
|
return `Provider stream stalled after partial response - no new chunks for ${seconds} (${formatDetails(snapshot)})`
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
export class StreamWatchdogTimeoutError extends Error {
|
||||||
|
readonly code: StreamWatchdogErrorCode
|
||||||
|
readonly phase: StreamWatchdogPhase
|
||||||
|
|
||||||
|
constructor(
|
||||||
|
readonly reason: StreamWatchdogAbortReason,
|
||||||
|
readonly timeoutMs: number,
|
||||||
|
readonly streamSnapshot: StreamWatchdogSnapshot,
|
||||||
|
) {
|
||||||
|
super(buildMessage(reason, timeoutMs, streamSnapshot))
|
||||||
|
this.name = 'StreamWatchdogTimeoutError'
|
||||||
|
this.code = reason === 'max_duration'
|
||||||
|
? 'STREAM_MAX_DURATION'
|
||||||
|
: 'STREAM_IDLE_TIMEOUT'
|
||||||
|
this.phase = streamSnapshot.phase
|
||||||
|
}
|
||||||
|
|
||||||
|
safeToRetryStream(): boolean {
|
||||||
|
return (
|
||||||
|
this.streamSnapshot.contentDeltaCount === 0 &&
|
||||||
|
!this.streamSnapshot.toolUseStarted
|
||||||
|
)
|
||||||
|
}
|
||||||
|
|
||||||
|
toDiagnosticData(): Record<string, unknown> {
|
||||||
|
return {
|
||||||
|
reason: this.reason,
|
||||||
|
code: this.code,
|
||||||
|
phase: this.phase,
|
||||||
|
timeoutMs: this.timeoutMs,
|
||||||
|
safeToRetry: this.safeToRetryStream(),
|
||||||
|
...this.streamSnapshot,
|
||||||
|
}
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
class StreamWatchdogState {
|
||||||
|
private snapshotValue: StreamWatchdogSnapshot = {
|
||||||
|
phase: 'before_first_event',
|
||||||
|
eventCount: 0,
|
||||||
|
contentDeltaCount: 0,
|
||||||
|
textDeltaCount: 0,
|
||||||
|
thinkingDeltaCount: 0,
|
||||||
|
toolInputDeltaCount: 0,
|
||||||
|
messageStopReceived: false,
|
||||||
|
toolUseStarted: false,
|
||||||
|
}
|
||||||
|
|
||||||
|
private readonly blockTypesByIndex = new Map<number, string>()
|
||||||
|
|
||||||
|
recordEvent(event: unknown): boolean {
|
||||||
|
const beforeContentDeltaCount = this.snapshotValue.contentDeltaCount
|
||||||
|
const rawEvent = asObject(event) as StreamEventLike | null
|
||||||
|
const type = readString(rawEvent?.type)
|
||||||
|
if (!type) {
|
||||||
|
return false
|
||||||
|
}
|
||||||
|
|
||||||
|
this.snapshotValue = {
|
||||||
|
...this.snapshotValue,
|
||||||
|
eventCount: this.snapshotValue.eventCount + 1,
|
||||||
|
lastEventType: type,
|
||||||
|
}
|
||||||
|
|
||||||
|
if (type === 'message_stop') {
|
||||||
|
this.snapshotValue = {
|
||||||
|
...this.snapshotValue,
|
||||||
|
messageStopReceived: true,
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
if (type === 'content_block_start') {
|
||||||
|
const block = asObject(rawEvent?.content_block)
|
||||||
|
const blockType = readString(block?.type)
|
||||||
|
const index = readIndex(rawEvent?.index)
|
||||||
|
if (blockType) {
|
||||||
|
this.snapshotValue = {
|
||||||
|
...this.snapshotValue,
|
||||||
|
lastBlockType: blockType,
|
||||||
|
toolUseStarted:
|
||||||
|
this.snapshotValue.toolUseStarted ||
|
||||||
|
blockType === 'tool_use' ||
|
||||||
|
blockType === 'server_tool_use',
|
||||||
|
}
|
||||||
|
if (index !== undefined) {
|
||||||
|
this.blockTypesByIndex.set(index, blockType)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
if (type === 'content_block_delta') {
|
||||||
|
const delta = asObject(rawEvent?.delta)
|
||||||
|
const deltaType = readString(delta?.type)
|
||||||
|
const index = readIndex(rawEvent?.index)
|
||||||
|
const blockType = index === undefined
|
||||||
|
? undefined
|
||||||
|
: this.blockTypesByIndex.get(index)
|
||||||
|
this.snapshotValue = {
|
||||||
|
...this.snapshotValue,
|
||||||
|
...(deltaType ? { lastDeltaType: deltaType } : {}),
|
||||||
|
...(blockType ? { lastBlockType: blockType } : {}),
|
||||||
|
}
|
||||||
|
if (delta && isContentDelta(delta)) {
|
||||||
|
this.snapshotValue = countContentDelta(this.snapshotValue, deltaType)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
this.snapshotValue = {
|
||||||
|
...this.snapshotValue,
|
||||||
|
phase: this.snapshotValue.contentDeltaCount > 0
|
||||||
|
? 'mid_stream'
|
||||||
|
: 'before_content',
|
||||||
|
}
|
||||||
|
|
||||||
|
return (
|
||||||
|
beforeContentDeltaCount === 0 &&
|
||||||
|
this.snapshotValue.contentDeltaCount > 0
|
||||||
|
)
|
||||||
|
}
|
||||||
|
|
||||||
|
hasContentDelta(): boolean {
|
||||||
|
return this.snapshotValue.contentDeltaCount > 0
|
||||||
|
}
|
||||||
|
|
||||||
|
snapshot(): StreamWatchdogSnapshot {
|
||||||
|
return { ...this.snapshotValue }
|
||||||
|
}
|
||||||
|
|
||||||
|
createTimeoutError(
|
||||||
|
reason: StreamWatchdogAbortReason,
|
||||||
|
timeoutMs: number,
|
||||||
|
): StreamWatchdogTimeoutError {
|
||||||
|
return new StreamWatchdogTimeoutError(reason, timeoutMs, this.snapshot())
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
export function createStreamWatchdogState(): StreamWatchdogState {
|
||||||
|
return new StreamWatchdogState()
|
||||||
|
}
|
||||||
Loading…
x
Reference in New Issue
Block a user