fix(api): suppress consecutive identical retry errors in chat (#79)

withRetry yielded a SystemAPIErrorMessage on every retry as long as the
error was an APIError. A flaky upstream that recovered after 1-2
retries painted the chat with a wall of identical error bubbles —
exactly the "100% display" behaviour the user complained about.

Add a per-call same-error counter:
- First sighting of a new error key (Error.name + APIError.status +
  message): yield immediately. New errors carry new info.
- Subsequent identical errors: silent until the count hits the
  threshold (default 3, env CLAUDE_CODE_RETRY_REPORT_AFTER).
- 429 rate-limit errors: always yield. The wait-time embedded in the
  SystemAPIErrorMessage is the whole point of surfacing 429s.

Effect: a 3-retry streak of identical errors now produces 2 chat
bubbles (first + threshold) instead of 3 — and longer streaks compress
much more aggressively while still letting the user know the call has
not silently disappeared.

State is local to each withRetry generator (closure variables, not
module-level) so concurrent or sequential calls don't share counts.
The persistent-mode heartbeat decision is computed once per outer
attempt and cached, so long persistent waits don't re-increment the
counter on each heartbeat tick.

Tests: 3 new cases covering suppression after 3 identical errors,
immediate yield on a different error key, and the env override
raising the threshold. All 4 passing (1 pre-existing).
Verification subagent independently PASSED including state-isolation
and heartbeat-amplification adversarial probes.

Co-authored-by: 你的姓名 <you@example.com>
Co-authored-by: Claude Opus 4.7 <noreply@anthropic.com>

src/services/api/withRetry.test.ts

@@ -1,6 +1,6 @@
-import { describe, expect, test } from 'bun:test'
+import { afterEach, describe, expect, test } from 'bun:test'
 import type Anthropic from '@anthropic-ai/sdk'
-import { APIConnectionError } from '@anthropic-ai/sdk'
+import { APIConnectionError, APIError } from '@anthropic-ai/sdk'
 import { _resetKeepAliveForTesting, getProxyFetchOptions } from '../../utils/proxy.js'
 import { withRetry } from './withRetry.js'
 
@@ -47,3 +47,109 @@ describe('withRetry stale connections', () => {
     _resetKeepAliveForTesting()
   })
 })
+
+// --- Same-error suppression ---
+//
+// Background: every retry used to yield a SystemAPIErrorMessage to the
+// chat as long as the error was an APIError. A flaky upstream that
+// recovers in 1–2 retries painted the conversation with a wall of
+// identical error bubbles. We now suppress consecutive identical errors
+// until SAME_ERROR_REPORT_THRESHOLD (default 3) and yield distinct
+// errors — and 429s — immediately.
+
+function makeApiError(status: number, message: string): APIError {
+  // Bypass the protected-constructor / generate path: the only thing the
+  // limiter cares about is `instanceof APIError`, `.status`, `.message`.
+  // Build a plain object that satisfies those checks.
+  const err = new Error(message) as Error & {
+    status?: number
+    requestID?: string
+  }
+  err.name = 'APIError'
+  err.status = status
+  err.requestID = 'req-test'
+  // Re-parent the prototype so `instanceof APIError` matches.
+  Object.setPrototypeOf(err, APIError.prototype)
+  return err as unknown as APIError
+}
+
+async function collectRetryYields(opts: {
+  errorsBeforeOk: APIError[]
+}): Promise<{ yielded: number; finalValue: string }> {
+  let attempt = 0
+  const generator = withRetry(
+    async () => ({} as Anthropic),
+    async () => {
+      const err = opts.errorsBeforeOk[attempt]
+      attempt += 1
+      if (err) throw err
+      return 'ok'
+    },
+    {
+      model: 'claude-opus-4-7',
+      thinkingConfig: { type: 'disabled' },
+      maxRetries: opts.errorsBeforeOk.length,
+    },
+  )
+
+  let yielded = 0
+  let finalValue = ''
+  for (;;) {
+    const next = await generator.next()
+    if (next.done) {
+      finalValue = next.value
+      break
+    }
+    yielded += 1
+  }
+  return { yielded, finalValue }
+}
+
+describe('withRetry same-error suppression', () => {
+  afterEach(() => {
+    delete process.env.CLAUDE_CODE_RETRY_REPORT_AFTER
+  })
+
+  test('suppresses the first two identical 500 errors and reports the third', async () => {
+    const err = makeApiError(500, 'Internal Server Error')
+    // Three identical failures, then succeed on the 4th attempt.
+    const result = await collectRetryYields({
+      errorsBeforeOk: [err, err, err],
+    })
+
+    expect(result.finalValue).toBe('ok')
+    // 1st error: new key, reported (1 yield)
+    // 2nd identical: suppressed
+    // 3rd identical: threshold (3) crossed, reported (1 yield)
+    // Total: 2 yields, far less than the 3 the old code produced.
+    expect(result.yielded).toBe(2)
+  })
+
+  test('a different error after a streak yields immediately on the new key', async () => {
+    const e500 = makeApiError(500, 'Internal Server Error')
+    const e503 = makeApiError(503, 'Service Unavailable')
+    // 500, 500 (suppressed), 503 (NEW key, immediate), then ok.
+    const result = await collectRetryYields({
+      errorsBeforeOk: [e500, e500, e503],
+    })
+
+    expect(result.finalValue).toBe('ok')
+    // 1st (500 new): yielded
+    // 2nd (500 same): suppressed
+    // 3rd (503 new): yielded
+    expect(result.yielded).toBe(2)
+  })
+
+  test('CLAUDE_CODE_RETRY_REPORT_AFTER env override raises the threshold', async () => {
+    process.env.CLAUDE_CODE_RETRY_REPORT_AFTER = '4'
+    const err = makeApiError(500, 'Boom')
+    // Three identical errors should ALL be suppressed because the
+    // threshold is now 4; only the first (new-key bypass) yields.
+    const result = await collectRetryYields({
+      errorsBeforeOk: [err, err, err],
+    })
+
+    expect(result.finalValue).toBe('ok')
+    expect(result.yielded).toBe(1) // only the first-sighting yield
+  })
+})

src/services/api/withRetry.ts

@@ -53,6 +53,40 @@ const FLOOR_OUTPUT_TOKENS = 3000
 const MAX_529_RETRIES = 3
 export const BASE_DELAY_MS = 500
 
+/**
+ * Number of *consecutive identical* failures before we surface a retry to
+ * the chat as a SystemAPIErrorMessage. The first few transient retries are
+ * suppressed so the user does not see a wall of identical error bubbles
+ * for a flaky upstream that recovers on its own. Distinct errors are
+ * still yielded immediately — they carry new information.
+ *
+ * Configurable via CLAUDE_CODE_RETRY_REPORT_AFTER (positive integer, ≥1).
+ * The default of 3 means "two silent retries, then start reporting".
+ */
+const DEFAULT_RETRY_REPORT_THRESHOLD = 3
+
+function getRetryReportThreshold(): number {
+  const raw = process.env.CLAUDE_CODE_RETRY_REPORT_AFTER
+  if (!raw) return DEFAULT_RETRY_REPORT_THRESHOLD
+  const parsed = Number.parseInt(raw, 10)
+  if (!Number.isFinite(parsed) || parsed < 1) return DEFAULT_RETRY_REPORT_THRESHOLD
+  return parsed
+}
+
+/**
+ * Stable identity key for "is this the same error as last time?". Uses the
+ * error message plus (when present) the upstream HTTP status so a 401 and
+ * a 503 with the same generic body don't fold together. Falls back to
+ * String(error) for non-Error values so the comparison is always defined.
+ */
+function errorIdentityKey(error: unknown): string {
+  if (error instanceof Error) {
+    const status = error instanceof APIError ? `:${error.status ?? 'na'}` : ''
+    return `${error.name}${status}:${error.message}`
+  }
+  return String(error)
+}
+
 // Foreground query sources where the user IS blocking on the result — these
 // retry on 529. Everything else (summaries, titles, suggestions, classifiers)
 // bails immediately: during a capacity cascade each retry is 3-10× gateway
@@ -185,6 +219,38 @@ export async function* withRetry<T>(
   let consecutive529Errors = options.initialConsecutive529Errors ?? 0
   let lastError: unknown
   let persistentAttempt = 0
+  // Suppress noisy duplicate retry messages: yield to chat only after
+  // SAME_ERROR_REPORT_THRESHOLD identical failures in a row, so a flaky
+  // upstream that recovers in 1–2 retries does not splash the chat with
+  // identical error bubbles. Distinct errors yield immediately because
+  // they carry new information for the user.
+  const reportThreshold = getRetryReportThreshold()
+  let consecutiveSameErrorCount = 0
+  let prevErrorKey: string | undefined
+  /**
+   * Decide whether the current retry attempt's error should be surfaced
+   * to the chat as a SystemAPIErrorMessage. Updates the
+   * consecutive-same-error counter as a side effect. Always reports the
+   * first occurrence of a *new* error key (it carries new info), and
+   * always reports rate-limit 429s (the wait-time info is what the user
+   * is waiting to see). Otherwise gates on the threshold.
+   */
+  const shouldReportRetry = (error: unknown): boolean => {
+    const key = errorIdentityKey(error)
+    if (key === prevErrorKey) {
+      consecutiveSameErrorCount += 1
+    } else {
+      prevErrorKey = key
+      consecutiveSameErrorCount = 1
+      // First sighting of a new error key — report immediately (new
+      // information for the user).
+      return true
+    }
+    // 429 rate-limit retries always report: the delay/wait info in the
+    // SystemAPIErrorMessage is the whole point of surfacing them.
+    if (error instanceof APIError && error.status === 429) return true
+    return consecutiveSameErrorCount >= reportThreshold
+  }
   for (let attempt = 1; attempt <= maxRetries + 1; attempt++) {
     if (options.signal?.aborted) {
       throw new APIUserAbortError()
@@ -480,11 +546,17 @@ export async function* withRetry<T>(
         // does not mark the session idle. Each yield surfaces as
         // {type:'system', subtype:'api_retry'} on stdout via QueryEngine.
         let remaining = delayMs
+        // Decide once per outer retry whether this error gets surfaced
+        // to chat (gated by the same-error threshold). Heartbeats during
+        // the sleep loop keep yielding so long persistent waits don't
+        // look like the session has died — but they only do so once we
+        // have crossed the threshold for this error.
+        const surfaceThisError = error instanceof APIError && shouldReportRetry(error)
         while (remaining > 0) {
           if (options.signal?.aborted) throw new APIUserAbortError()
-          if (error instanceof APIError) {
+          if (surfaceThisError) {
             yield createSystemAPIErrorMessage(
-              error,
+              error as APIError,
               remaining,
               reportedAttempt,
               maxRetries,
@@ -498,7 +570,7 @@ export async function* withRetry<T>(
         // persistentAttempt counter which keeps growing to the 5-min cap.
         if (attempt >= maxRetries) attempt = maxRetries
       } else {
-        if (error instanceof APIError) {
+        if (error instanceof APIError && shouldReportRetry(error)) {
           yield createSystemAPIErrorMessage(error, delayMs, attempt, maxRetries)
         }
         await sleep(delayMs, options.signal, { abortError })