mirror of
https://github.com/NanmiCoder/cc-haha
synced 2026-07-26 15:03:34 +08:00
fix(agent-limiter): add streak gate + raise verification cap to 12 (#78)
The original per-session subagent invocation limiter used a single
total-count cap that fired on the 6th call regardless of whether
prior calls succeeded or failed. With verification capped at 5, a
session running one verification per PR hit the cap on the 6th PR
and was hard-blocked even though every prior call had returned
VERDICT: PASS — exactly the "normal multi-step work" the source
comment promised the gate would not catch.
This change splits the gate into two complementary signals so the
cap actually matches the documented intent (catch pathological
loops, leave normal work alone):
1. Total cap raised: verification 5 → 12. Generous enough that
legitimate multi-PR sessions don't trip it; the new streak
gate is the real defence against loops.
2. Streak gate: noteOutcome(agentType, verdict) records each
subagent's verdict. Three consecutive FAILs (no PASS in
between) cap immediately, regardless of total count. PASS or
UNKNOWN clears the streak. This catches the verifier-loop
pattern (FAIL → fix → FAIL → fix → FAIL) within 3 attempts —
much faster and more accurately than a count-only cap.
3. Verdict parser: parseVerdict(text) extracts the standard
VERDICT: PASS/FAIL/CHANGES_NEEDED/APPROVE/UNCONFIRMED line plus
specialist-skill synonyms (SECURITY:, PLAN_REVIEW:, ROOT
CAUSE:). Conservative — UNKNOWN counts as soft-PASS so
subagents that don't emit a structured verdict (e.g. one-shot
research) are not punished.
Wired into AgentTool.tsx in the completed-result branch, with a
guard that mirrors the existing noteInvocation guard exactly so
streak counts and total counts stay in sync.
Tests: 28 cases (14 existing adapted to cap=12, 5 new streak
behaviours, 9 new parseVerdict cases) all pass. Verification
subagent independently PASSED including 8 adversarial probes.
Co-authored-by: 你的姓名 <you@example.com>
Co-authored-by: Claude Opus 4.7 <noreply@anthropic.com>
This commit is contained in:
parent
1a8d7446ac
commit
52c35424da
@ -50,7 +50,7 @@ import { agentToolResultSchema, classifyHandoffIfNeeded, emitTaskProgress, extra
|
||||
import { GENERAL_PURPOSE_AGENT } from './built-in/generalPurposeAgent.js';
|
||||
import { AGENT_TOOL_NAME, LEGACY_AGENT_TOOL_NAME, ONE_SHOT_BUILTIN_AGENT_TYPES } from './constants.js';
|
||||
import { buildForkedMessages, buildWorktreeNotice, COORDINATOR_RESEARCH_FORK_SUBAGENT_TYPE, FORK_AGENT, type ForkMode, isCoordinatorResearchForkEnabled, isForkSubagentEnabled, isInForkChild } from './forkSubagent.js';
|
||||
import { formatLimitExceededMessage, formatNearLimitWarning, isLimiterDisabled, noteInvocation } from './invocationLimiter.js';
|
||||
import { formatLimitExceededMessage, formatNearLimitWarning, isLimiterDisabled, noteInvocation, noteOutcome, parseVerdict } from './invocationLimiter.js';
|
||||
import {
|
||||
detectLazyDelegation,
|
||||
formatLazyDelegationError,
|
||||
@ -1475,6 +1475,22 @@ export const AgentTool = buildTool({
|
||||
}, ...agentResult.content];
|
||||
}
|
||||
}
|
||||
// Record the subagent's verdict so the streak gate in the
|
||||
// limiter can fast-fail on consecutive FAILs without punishing
|
||||
// legitimate multi-step work where verifications PASS. The
|
||||
// guard mirrors the one around `noteInvocation` above — both
|
||||
// sides must be symmetric or the streak count drifts away
|
||||
// from reality.
|
||||
if (
|
||||
!toolUseContext.agentId &&
|
||||
isBuiltInAgent(selectedAgent) &&
|
||||
!isLimiterDisabled()
|
||||
) {
|
||||
const combined = agentResult.content
|
||||
.map(block => (block.type === 'text' ? block.text : ''))
|
||||
.join('\n');
|
||||
noteOutcome(selectedAgent.agentType, parseVerdict(combined));
|
||||
}
|
||||
return {
|
||||
data: {
|
||||
status: 'completed' as const,
|
||||
|
||||
@ -7,6 +7,8 @@ import {
|
||||
getLimitFor,
|
||||
isLimiterDisabled,
|
||||
noteInvocation,
|
||||
noteOutcome,
|
||||
parseVerdict,
|
||||
} from './invocationLimiter.js'
|
||||
import { getSessionId } from '../../bootstrap/state.js'
|
||||
|
||||
@ -22,8 +24,8 @@ describe('invocationLimiter', () => {
|
||||
delete process.env.CLAUDE_CODE_AGENT_LIMIT_CODE_REVIEWER
|
||||
})
|
||||
|
||||
test('default cap for verification is 5', () => {
|
||||
expect(getLimitFor('verification')).toBe(5)
|
||||
test('default cap for verification is 12', () => {
|
||||
expect(getLimitFor('verification')).toBe(12)
|
||||
})
|
||||
|
||||
test('default cap for fork is 10', () => {
|
||||
@ -48,11 +50,11 @@ describe('invocationLimiter', () => {
|
||||
|
||||
test('non-numeric or zero env override falls back to default', () => {
|
||||
process.env.CLAUDE_CODE_AGENT_LIMIT_VERIFICATION = 'abc'
|
||||
expect(getLimitFor('verification')).toBe(5)
|
||||
expect(getLimitFor('verification')).toBe(12)
|
||||
process.env.CLAUDE_CODE_AGENT_LIMIT_VERIFICATION = '0'
|
||||
expect(getLimitFor('verification')).toBe(5)
|
||||
expect(getLimitFor('verification')).toBe(12)
|
||||
process.env.CLAUDE_CODE_AGENT_LIMIT_VERIFICATION = '-3'
|
||||
expect(getLimitFor('verification')).toBe(5)
|
||||
expect(getLimitFor('verification')).toBe(12)
|
||||
})
|
||||
|
||||
test('isLimiterDisabled tracks env var', () => {
|
||||
@ -62,16 +64,18 @@ describe('invocationLimiter', () => {
|
||||
})
|
||||
|
||||
test('noteInvocation increments and reports capped after threshold', () => {
|
||||
// verification cap = 5
|
||||
for (let i = 1; i <= 5; i++) {
|
||||
// verification cap = 12 (raised from 5)
|
||||
for (let i = 1; i <= 12; i++) {
|
||||
const r = noteInvocation('verification')
|
||||
expect(r.count).toBe(i)
|
||||
expect(r.limit).toBe(5)
|
||||
expect(r.limit).toBe(12)
|
||||
expect(r.capped).toBe(false)
|
||||
expect(r.cappedReason).toBe('none')
|
||||
}
|
||||
const r6 = noteInvocation('verification')
|
||||
expect(r6.count).toBe(6)
|
||||
expect(r6.capped).toBe(true)
|
||||
const r13 = noteInvocation('verification')
|
||||
expect(r13.count).toBe(13)
|
||||
expect(r13.capped).toBe(true)
|
||||
expect(r13.cappedReason).toBe('total')
|
||||
})
|
||||
|
||||
test('counters are independent per agent type', () => {
|
||||
@ -79,15 +83,20 @@ describe('invocationLimiter', () => {
|
||||
noteInvocation('verification')
|
||||
noteInvocation('code-reviewer')
|
||||
const snap = _getLimiterStateSnapshot(getSessionId())
|
||||
expect(snap?.get('verification')).toBe(2)
|
||||
expect(snap?.get('code-reviewer')).toBe(1)
|
||||
expect(snap?.total.get('verification')).toBe(2)
|
||||
expect(snap?.total.get('code-reviewer')).toBe(1)
|
||||
// No outcomes recorded yet, so streaks are unset (or 0).
|
||||
expect(snap?.failStreak.get('verification') ?? 0).toBe(0)
|
||||
})
|
||||
|
||||
test('formatLimitExceededMessage includes the counter, cap, and env hint', () => {
|
||||
test('formatLimitExceededMessage (total cap) includes the counter, cap, and env hint', () => {
|
||||
const msg = formatLimitExceededMessage('code-reviewer', {
|
||||
count: 9,
|
||||
limit: 8,
|
||||
failStreak: 0,
|
||||
failStreakLimit: 3,
|
||||
capped: true,
|
||||
cappedReason: 'total',
|
||||
nearLimit: false,
|
||||
})
|
||||
expect(msg).toContain("'code-reviewer'")
|
||||
@ -97,8 +106,26 @@ describe('invocationLimiter', () => {
|
||||
expect(msg).toContain('CLAUDE_CODE_AGENT_LIMITER_OFF=1')
|
||||
})
|
||||
|
||||
test('formatLimitExceededMessage (streak) explains the verifier-loop pattern', () => {
|
||||
const msg = formatLimitExceededMessage('verification', {
|
||||
count: 4,
|
||||
limit: 12,
|
||||
failStreak: 3,
|
||||
failStreakLimit: 3,
|
||||
capped: true,
|
||||
cappedReason: 'streak',
|
||||
nearLimit: false,
|
||||
})
|
||||
expect(msg).toContain("'verification'")
|
||||
expect(msg).toContain('FAIL 3 times in a row')
|
||||
expect(msg).toContain('streak cap is 3')
|
||||
expect(msg).toContain('PASS verdict from this agent type clears the streak')
|
||||
expect(msg).toContain('CLAUDE_CODE_AGENT_LIMIT_VERIFICATION')
|
||||
})
|
||||
|
||||
test('nearLimit fires once on the last allowed invocation', () => {
|
||||
// verification cap = 5; nearLimit should be true on the 5th call only.
|
||||
// verification cap = 12; nearLimit should be true only on the 12th call.
|
||||
process.env.CLAUDE_CODE_AGENT_LIMIT_VERIFICATION = '5'
|
||||
const flags: boolean[] = []
|
||||
for (let i = 1; i <= 5; i++) {
|
||||
const r = noteInvocation('verification')
|
||||
@ -106,7 +133,6 @@ describe('invocationLimiter', () => {
|
||||
}
|
||||
// Calls 1–4: not near; call 5: near (last allowed); call 6 would be capped.
|
||||
expect(flags).toEqual([false, false, false, false, true])
|
||||
// Verify post-cap state isn't flagged as nearLimit.
|
||||
const r6 = noteInvocation('verification')
|
||||
expect(r6.capped).toBe(true)
|
||||
expect(r6.nearLimit).toBe(false)
|
||||
@ -122,15 +148,18 @@ describe('invocationLimiter', () => {
|
||||
|
||||
test('formatNearLimitWarning is a system-reminder block with cap details', () => {
|
||||
const msg = formatNearLimitWarning('verification', {
|
||||
count: 5,
|
||||
limit: 5,
|
||||
count: 12,
|
||||
limit: 12,
|
||||
failStreak: 0,
|
||||
failStreakLimit: 3,
|
||||
capped: false,
|
||||
cappedReason: 'none',
|
||||
nearLimit: true,
|
||||
})
|
||||
expect(msg).toContain('<system-reminder>')
|
||||
expect(msg).toContain('</system-reminder>')
|
||||
expect(msg).toContain("'verification'")
|
||||
expect(msg).toContain('5 of 5')
|
||||
expect(msg).toContain('12 of 12')
|
||||
expect(msg).toContain('CLAUDE_CODE_AGENT_LIMIT_VERIFICATION')
|
||||
expect(msg).toContain('CLAUDE_CODE_AGENT_LIMITER_OFF=1')
|
||||
})
|
||||
@ -146,4 +175,145 @@ describe('invocationLimiter', () => {
|
||||
expect(r8.capped).toBe(true)
|
||||
expect(r8.count).toBe(8)
|
||||
})
|
||||
|
||||
// --- Streak gate (regression for "verifier loop" failure mode) ---
|
||||
|
||||
test('three consecutive FAILs trip the streak gate before the total cap', () => {
|
||||
// cap is 12; streak threshold is 3. Burn 3 FAILs and the next call
|
||||
// should be capped via streak even though total count is well under 12.
|
||||
noteInvocation('verification')
|
||||
noteOutcome('verification', 'FAIL')
|
||||
noteInvocation('verification')
|
||||
noteOutcome('verification', 'FAIL')
|
||||
noteInvocation('verification')
|
||||
noteOutcome('verification', 'FAIL')
|
||||
|
||||
const r4 = noteInvocation('verification')
|
||||
expect(r4.capped).toBe(true)
|
||||
expect(r4.cappedReason).toBe('streak')
|
||||
expect(r4.failStreak).toBe(3)
|
||||
expect(r4.failStreakLimit).toBe(3)
|
||||
expect(r4.count).toBe(4) // total cap not yet reached
|
||||
expect(r4.limit).toBe(12)
|
||||
})
|
||||
|
||||
test('a PASS verdict clears the FAIL streak', () => {
|
||||
noteInvocation('verification')
|
||||
noteOutcome('verification', 'FAIL')
|
||||
noteInvocation('verification')
|
||||
noteOutcome('verification', 'FAIL')
|
||||
// PASS in the middle of a streak resets it.
|
||||
noteInvocation('verification')
|
||||
noteOutcome('verification', 'PASS')
|
||||
|
||||
// Two more FAILs should NOT cap because the streak was reset.
|
||||
noteInvocation('verification')
|
||||
noteOutcome('verification', 'FAIL')
|
||||
noteInvocation('verification')
|
||||
noteOutcome('verification', 'FAIL')
|
||||
|
||||
const r6 = noteInvocation('verification')
|
||||
expect(r6.capped).toBe(false)
|
||||
expect(r6.failStreak).toBe(2) // 2 FAILs since the last PASS
|
||||
})
|
||||
|
||||
test('UNKNOWN verdict is treated as a soft-PASS (clears streak)', () => {
|
||||
// A subagent that does not emit a verdict line (e.g. one-shot
|
||||
// research / Explore) should not be punished — UNKNOWN should
|
||||
// clear the streak just like PASS does.
|
||||
noteInvocation('verification')
|
||||
noteOutcome('verification', 'FAIL')
|
||||
noteInvocation('verification')
|
||||
noteOutcome('verification', 'FAIL')
|
||||
noteInvocation('verification')
|
||||
noteOutcome('verification', 'UNKNOWN')
|
||||
|
||||
const r4 = noteInvocation('verification')
|
||||
expect(r4.capped).toBe(false)
|
||||
expect(r4.failStreak).toBe(0)
|
||||
})
|
||||
|
||||
test('streak gate is per-agent-type (PASS on one type does not clear another)', () => {
|
||||
noteInvocation('verification')
|
||||
noteOutcome('verification', 'FAIL')
|
||||
noteInvocation('code-reviewer')
|
||||
noteOutcome('code-reviewer', 'PASS')
|
||||
noteInvocation('verification')
|
||||
noteOutcome('verification', 'FAIL')
|
||||
noteInvocation('verification')
|
||||
noteOutcome('verification', 'FAIL')
|
||||
|
||||
const r4 = noteInvocation('verification')
|
||||
expect(r4.capped).toBe(true)
|
||||
expect(r4.cappedReason).toBe('streak')
|
||||
})
|
||||
|
||||
test('successful work pattern: 12 PASSes hits total cap, never streak', () => {
|
||||
// The motivating regression: 5 PRs, each with one PASSing
|
||||
// verification, then a 6th PR. With the old cap=5 + no streak
|
||||
// semantics this errored on the 6th PR. With cap=12 and PASS
|
||||
// clearing the streak, 12 successful invocations in a row are
|
||||
// allowed; the 13th hits the *total* cap (not the streak).
|
||||
for (let i = 1; i <= 12; i++) {
|
||||
const r = noteInvocation('verification')
|
||||
expect(r.capped).toBe(false)
|
||||
noteOutcome('verification', 'PASS')
|
||||
}
|
||||
const r13 = noteInvocation('verification')
|
||||
expect(r13.capped).toBe(true)
|
||||
expect(r13.cappedReason).toBe('total')
|
||||
})
|
||||
})
|
||||
|
||||
// --- parseVerdict ---
|
||||
|
||||
describe('parseVerdict', () => {
|
||||
test('returns UNKNOWN for empty / null / undefined input', () => {
|
||||
expect(parseVerdict('')).toBe('UNKNOWN')
|
||||
expect(parseVerdict(null)).toBe('UNKNOWN')
|
||||
expect(parseVerdict(undefined)).toBe('UNKNOWN')
|
||||
})
|
||||
|
||||
test('recognises the canonical "VERDICT: PASS" line', () => {
|
||||
expect(parseVerdict('Some report text\n\nVERDICT: PASS')).toBe('PASS')
|
||||
expect(parseVerdict('VERDICT: PASS\n')).toBe('PASS')
|
||||
// Case-insensitive.
|
||||
expect(parseVerdict('verdict: pass')).toBe('PASS')
|
||||
})
|
||||
|
||||
test('recognises common FAIL-shaped tokens', () => {
|
||||
expect(parseVerdict('VERDICT: FAIL')).toBe('FAIL')
|
||||
expect(parseVerdict('VERDICT: PARTIAL')).toBe('FAIL')
|
||||
expect(parseVerdict('VERDICT: CHANGES_NEEDED')).toBe('FAIL')
|
||||
expect(parseVerdict('VERDICT: UNCONFIRMED')).toBe('FAIL')
|
||||
})
|
||||
|
||||
test('recognises code-reviewer "APPROVE" as PASS', () => {
|
||||
expect(parseVerdict('Findings:\n...\nVERDICT: APPROVE')).toBe('PASS')
|
||||
})
|
||||
|
||||
test('recognises specialist verdict synonyms', () => {
|
||||
expect(parseVerdict('SECURITY: PASS')).toBe('PASS')
|
||||
expect(parseVerdict('SECURITY: CHANGES_NEEDED')).toBe('FAIL')
|
||||
expect(parseVerdict('PLAN_REVIEW: APPROVE')).toBe('PASS')
|
||||
expect(parseVerdict('PLAN_REVIEW: CHANGES_NEEDED')).toBe('FAIL')
|
||||
expect(parseVerdict('ROOT CAUSE: FOUND at file:line')).toBe('PASS')
|
||||
expect(parseVerdict('ROOT CAUSE: UNCONFIRMED')).toBe('FAIL')
|
||||
})
|
||||
|
||||
test('returns UNKNOWN when no verdict line is present', () => {
|
||||
expect(parseVerdict('Just a long report with no structured verdict.')).toBe('UNKNOWN')
|
||||
expect(parseVerdict('Tests pass and everything looks fine.')).toBe('UNKNOWN')
|
||||
})
|
||||
|
||||
test('tolerates markdown bullets and leading whitespace before VERDICT', () => {
|
||||
expect(parseVerdict(' - VERDICT: PASS')).toBe('PASS')
|
||||
expect(parseVerdict('### VERDICT: FAIL')).toBe('FAIL')
|
||||
expect(parseVerdict('> VERDICT: APPROVE')).toBe('PASS')
|
||||
})
|
||||
|
||||
test('looks at the tail when input is very long', () => {
|
||||
const huge = 'noise '.repeat(2000) + '\nVERDICT: PASS\n'
|
||||
expect(parseVerdict(huge)).toBe('PASS')
|
||||
})
|
||||
})
|
||||
|
||||
@ -7,42 +7,69 @@
|
||||
* address what the verifier is flagging. Without a cap the loop only
|
||||
* stops when the model gives up or the token budget runs out.
|
||||
*
|
||||
* This module imposes a per-session cap on each built-in agent type.
|
||||
* When the cap is exceeded the next invocation throws a tool error.
|
||||
* The model sees the error, surfaces it to the user, and the user can
|
||||
* either authorise more retries (raise the cap via env) or steer the
|
||||
* task differently.
|
||||
* This module imposes two complementary gates per agent type:
|
||||
*
|
||||
* Caps are intentionally generous so the gate only fires on pathological
|
||||
* loops, not on normal multi-step work. Verification is capped tighter
|
||||
* because verifier loops are the documented failure mode.
|
||||
* 1. A **total invocation cap** per session. Generous so legitimate
|
||||
* multi-step work (e.g. five PRs in a row each running a verifier)
|
||||
* does not trip the gate. Defends against any loop, including ones
|
||||
* where the parent rewrites prompts so each call looks different.
|
||||
*
|
||||
* 2. A **consecutive-failure streak cap**. When a subagent's result
|
||||
* reports FAIL `STREAK_FAIL_THRESHOLD` times in a row without a
|
||||
* PASS in between, we cap immediately, regardless of total count.
|
||||
* This catches pathological loops fast (the documented failure
|
||||
* mode) while leaving plenty of headroom for normal work where
|
||||
* verifiers usually return PASS.
|
||||
*
|
||||
* Caps are intentionally generous so the gate only fires on
|
||||
* pathological loops, not on normal multi-step work. When the cap is
|
||||
* exceeded the next invocation throws a tool error. The model sees the
|
||||
* error, surfaces it to the user, and the user can either authorise
|
||||
* more retries (raise the cap via env) or steer the task differently.
|
||||
*/
|
||||
|
||||
import { getSessionId } from '../../bootstrap/state.js'
|
||||
import type { SessionId } from '../../types/ids.js'
|
||||
import { isEnvTruthy } from '../../utils/envUtils.js'
|
||||
|
||||
type SessionCounters = Map<string, number>
|
||||
type SessionCounters = {
|
||||
/** Total successful + failed invocations of this agent type, capped by the total limit. */
|
||||
total: Map<string, number>
|
||||
/** Consecutive FAIL streak per agent type. Reset to 0 by any PASS. */
|
||||
failStreak: Map<string, number>
|
||||
}
|
||||
|
||||
const STATE = new Map<SessionId, SessionCounters>()
|
||||
|
||||
const FALLBACK_LIMIT = 8
|
||||
|
||||
/**
|
||||
* Per-agent default caps. Tighter for `verification` because the
|
||||
* verifier-loop pattern is the primary failure mode this gate exists
|
||||
* to catch. `fork` is also tightened (vs FALLBACK_LIMIT) because a
|
||||
* coordinator-research-fork-on session that fails to converge could
|
||||
* burn through dozens of cache-shared forks before the budget runs out;
|
||||
* 10 is generous for legitimate parallel research while making
|
||||
* pathological loops obvious. Anything not listed falls back to
|
||||
* Per-agent default total caps. Anything not listed falls back to
|
||||
* FALLBACK_LIMIT.
|
||||
*
|
||||
* `verification` was 5 historically; that turned out to be too tight
|
||||
* for legitimate multi-PR work (one verification per PR ⇒ 5 PRs and
|
||||
* the gate fires). Raised to 12. The streak gate below is the real
|
||||
* defence against verifier loops, so we can afford a generous total.
|
||||
*
|
||||
* `fork` stays at 10 — coordinator-research-fork-on sessions that
|
||||
* fail to converge could burn through dozens of cache-shared forks
|
||||
* before the budget runs out, and they don't have a verdict-style
|
||||
* PASS/FAIL signal that would let the streak gate help.
|
||||
*/
|
||||
const DEFAULT_LIMITS: Readonly<Record<string, number>> = {
|
||||
verification: 5,
|
||||
verification: 12,
|
||||
fork: 10,
|
||||
}
|
||||
|
||||
/**
|
||||
* Number of consecutive FAILs (no PASS in between) that trips the
|
||||
* streak gate. 3 is a strong loop signal — one or two FAILs in a row
|
||||
* are normal during fix-and-retry; three with no progress means the
|
||||
* approach probably needs to change.
|
||||
*/
|
||||
const STREAK_FAIL_THRESHOLD = 3
|
||||
|
||||
function envLimitFor(agentType: string): number | undefined {
|
||||
// Translate `code-reviewer` → CLAUDE_CODE_AGENT_LIMIT_CODE_REVIEWER
|
||||
const envName = `CLAUDE_CODE_AGENT_LIMIT_${agentType
|
||||
@ -63,13 +90,25 @@ export function isLimiterDisabled(): boolean {
|
||||
return isEnvTruthy(process.env.CLAUDE_CODE_AGENT_LIMITER_OFF)
|
||||
}
|
||||
|
||||
export type Verdict = 'PASS' | 'FAIL' | 'UNKNOWN'
|
||||
|
||||
export type InvocationCheckResult = {
|
||||
/** Total invocations of this type in this session, including this one. */
|
||||
count: number
|
||||
/** Cap currently in effect for this type. */
|
||||
limit: number
|
||||
/** Current consecutive-FAIL streak before this invocation runs. */
|
||||
failStreak: number
|
||||
/** Streak threshold currently in effect. */
|
||||
failStreakLimit: number
|
||||
/** True when this invocation pushes the count past the cap. */
|
||||
capped: boolean
|
||||
/**
|
||||
* Reason this invocation was capped. 'total' means the cumulative
|
||||
* total cap fired; 'streak' means consecutive FAILs tripped the
|
||||
* fast-fail gate; 'none' means not capped.
|
||||
*/
|
||||
cappedReason: 'none' | 'total' | 'streak'
|
||||
/**
|
||||
* True when this invocation is the LAST one allowed before the cap
|
||||
* fires. (count === limit and !capped). Callers can use this to surface
|
||||
@ -79,30 +118,137 @@ export type InvocationCheckResult = {
|
||||
nearLimit: boolean
|
||||
}
|
||||
|
||||
function getOrCreateCounters(sessionId: SessionId): SessionCounters {
|
||||
let counters = STATE.get(sessionId)
|
||||
if (!counters) {
|
||||
counters = { total: new Map(), failStreak: new Map() }
|
||||
STATE.set(sessionId, counters)
|
||||
}
|
||||
return counters
|
||||
}
|
||||
|
||||
/**
|
||||
* Increment the counter for `agentType` in the current session and
|
||||
* return the post-increment count plus whether the cap was crossed.
|
||||
* return the post-increment count plus whether either gate fired.
|
||||
*
|
||||
* Pure side effect: bumps the counter even when capped, so repeated
|
||||
* over-cap calls keep the count growing for analytics. Callers should
|
||||
* throw when `capped` is true.
|
||||
*
|
||||
* The streak gate fires *before* incrementing the total (it is the
|
||||
* earlier defence): if the consecutive-FAIL streak already meets the
|
||||
* threshold when this call begins, we cap immediately. Otherwise we
|
||||
* apply the total cap as before.
|
||||
*/
|
||||
export function noteInvocation(agentType: string): InvocationCheckResult {
|
||||
const sessionId = getSessionId()
|
||||
let counters = STATE.get(sessionId)
|
||||
if (!counters) {
|
||||
counters = new Map<string, number>()
|
||||
STATE.set(sessionId, counters)
|
||||
}
|
||||
const next = (counters.get(agentType) ?? 0) + 1
|
||||
counters.set(agentType, next)
|
||||
const counters = getOrCreateCounters(sessionId)
|
||||
const next = (counters.total.get(agentType) ?? 0) + 1
|
||||
counters.total.set(agentType, next)
|
||||
|
||||
const limit = getLimitFor(agentType)
|
||||
const capped = next > limit
|
||||
const failStreak = counters.failStreak.get(agentType) ?? 0
|
||||
const failStreakLimit = STREAK_FAIL_THRESHOLD
|
||||
|
||||
// Streak gate: an existing run of consecutive FAILs at or above the
|
||||
// threshold means the previous attempts already failed and the parent
|
||||
// is about to retry without success in between. Cap before doing more
|
||||
// work. (failStreak is read pre-increment; the current invocation has
|
||||
// not had its outcome recorded yet, so `>=` is correct.)
|
||||
const streakCapped = failStreak >= failStreakLimit
|
||||
const totalCapped = next > limit
|
||||
|
||||
const capped = streakCapped || totalCapped
|
||||
const cappedReason: 'none' | 'total' | 'streak' = streakCapped
|
||||
? 'streak'
|
||||
: totalCapped
|
||||
? 'total'
|
||||
: 'none'
|
||||
|
||||
// nearLimit fires once, on the last allowed invocation (next === limit).
|
||||
// Skip on small caps where it would coincide with the very first call:
|
||||
// a "you're near the limit" reminder on call #1 is noise, not signal.
|
||||
const nearLimit = !capped && next === limit && limit >= 2
|
||||
return { count: next, limit, capped, nearLimit }
|
||||
|
||||
return {
|
||||
count: next,
|
||||
limit,
|
||||
failStreak,
|
||||
failStreakLimit,
|
||||
capped,
|
||||
cappedReason,
|
||||
nearLimit,
|
||||
}
|
||||
}
|
||||
|
||||
/**
|
||||
* Record the outcome of a subagent invocation. PASS clears the
|
||||
* consecutive-FAIL streak; FAIL extends it; UNKNOWN treats the run
|
||||
* conservatively as PASS so subagents that don't emit a verdict are
|
||||
* not punished. Total count is unaffected — that is the
|
||||
* cumulative-cap concern.
|
||||
*/
|
||||
export function noteOutcome(agentType: string, verdict: Verdict): void {
|
||||
const sessionId = getSessionId()
|
||||
const counters = getOrCreateCounters(sessionId)
|
||||
|
||||
if (verdict === 'FAIL') {
|
||||
counters.failStreak.set(agentType, (counters.failStreak.get(agentType) ?? 0) + 1)
|
||||
} else {
|
||||
// PASS or UNKNOWN — break the streak. UNKNOWN is treated as a
|
||||
// soft-PASS to avoid penalising subagents that simply don't emit a
|
||||
// structured verdict (e.g. one-shot research / Explore). The total
|
||||
// cap still bounds them.
|
||||
counters.failStreak.set(agentType, 0)
|
||||
}
|
||||
}
|
||||
|
||||
/**
|
||||
* Heuristic verdict parser for subagent text output. Looks for an
|
||||
* explicit "VERDICT: PASS|FAIL|PARTIAL|CHANGES_NEEDED" line, common in
|
||||
* the verification / code-reviewer / security-reviewer skills, plus a
|
||||
* few well-defined synonyms. Anything else returns UNKNOWN.
|
||||
*
|
||||
* Conservative on purpose: PASS-shaped outputs that omit the keyword
|
||||
* should not extend the FAIL streak. The verifier loops we want to
|
||||
* catch always end in an explicit FAIL/CHANGES_NEEDED, so missing the
|
||||
* keyword on a PASS-shaped output is a safe miss; missing it on a
|
||||
* FAIL-shaped output would be a false PASS, but in practice the
|
||||
* verifier skill always prints the verdict line.
|
||||
*/
|
||||
export function parseVerdict(text: string | null | undefined): Verdict {
|
||||
if (!text) return 'UNKNOWN'
|
||||
|
||||
// Take the last 4 KB — verdict lines live near the end of the report
|
||||
// and scanning the whole transcript wastes time on huge outputs.
|
||||
const tail = text.length > 4096 ? text.slice(text.length - 4096) : text
|
||||
|
||||
// Match VERDICT: <token> on a line of its own (the documented format).
|
||||
// Allow optional markdown bullets / leading whitespace.
|
||||
const verdictLine = /(?:^|\n)[\s>*\-#]*VERDICT\s*:\s*([A-Z_/]+)/i.exec(tail)
|
||||
if (verdictLine) {
|
||||
const token = verdictLine[1]!.toUpperCase()
|
||||
if (token === 'PASS' || token === 'APPROVE') return 'PASS'
|
||||
if (
|
||||
token === 'FAIL' ||
|
||||
token === 'PARTIAL' ||
|
||||
token === 'CHANGES_NEEDED' ||
|
||||
token === 'UNCONFIRMED'
|
||||
) {
|
||||
return 'FAIL'
|
||||
}
|
||||
return 'UNKNOWN'
|
||||
}
|
||||
|
||||
// Common standalone summary lines used by some skills.
|
||||
if (/(?:^|\n)\s*SECURITY\s*:\s*PASS\b/i.test(tail)) return 'PASS'
|
||||
if (/(?:^|\n)\s*SECURITY\s*:\s*CHANGES_NEEDED\b/i.test(tail)) return 'FAIL'
|
||||
if (/(?:^|\n)\s*PLAN_REVIEW\s*:\s*APPROVE\b/i.test(tail)) return 'PASS'
|
||||
if (/(?:^|\n)\s*PLAN_REVIEW\s*:\s*CHANGES_NEEDED\b/i.test(tail)) return 'FAIL'
|
||||
if (/(?:^|\n)\s*ROOT CAUSE\s*:\s*FOUND\b/i.test(tail)) return 'PASS'
|
||||
if (/(?:^|\n)\s*ROOT CAUSE\s*:\s*UNCONFIRMED\b/i.test(tail)) return 'FAIL'
|
||||
|
||||
return 'UNKNOWN'
|
||||
}
|
||||
|
||||
/**
|
||||
@ -131,9 +277,9 @@ export function formatNearLimitWarning(
|
||||
}
|
||||
|
||||
/**
|
||||
* Format the user-facing error string when an invocation goes over cap.
|
||||
* Kept separate so call sites can compose it into Tool errors without
|
||||
* re-importing AGENT_TOOL_NAME etc.
|
||||
* Format the user-facing error string when an invocation goes over a
|
||||
* cap. Differentiates between the total cap and the consecutive-FAIL
|
||||
* streak so the user can see *which* gate fired and act accordingly.
|
||||
*/
|
||||
export function formatLimitExceededMessage(
|
||||
agentType: string,
|
||||
@ -142,6 +288,19 @@ export function formatLimitExceededMessage(
|
||||
const envName = `CLAUDE_CODE_AGENT_LIMIT_${agentType
|
||||
.replace(/[-/]/g, '_')
|
||||
.toUpperCase()}`
|
||||
|
||||
if (result.cappedReason === 'streak') {
|
||||
return (
|
||||
`Subagent '${agentType}' has reported FAIL ${result.failStreak} times in a row ` +
|
||||
`(streak cap is ${result.failStreakLimit}). Stop and consult the user — ` +
|
||||
`repeated FAIL/CHANGES_NEEDED outcomes without a PASS in between are the ` +
|
||||
`verifier-loop pattern this gate exists to catch. A PASS verdict from this ` +
|
||||
`agent type clears the streak. If the user authorises more attempts, raise ` +
|
||||
`the total cap via ${envName}=N or disable this guard with ` +
|
||||
`CLAUDE_CODE_AGENT_LIMITER_OFF=1.`
|
||||
)
|
||||
}
|
||||
|
||||
return (
|
||||
`Subagent '${agentType}' has been invoked ${result.count} times in this session, ` +
|
||||
`exceeding the cap of ${result.limit}. Stop and consult the user before invoking it again — ` +
|
||||
@ -159,6 +318,8 @@ export function _resetLimiterState(): void {
|
||||
/** Test helper. */
|
||||
export function _getLimiterStateSnapshot(
|
||||
sessionId: SessionId,
|
||||
): ReadonlyMap<string, number> | undefined {
|
||||
return STATE.get(sessionId)
|
||||
): { total: ReadonlyMap<string, number>; failStreak: ReadonlyMap<string, number> } | undefined {
|
||||
const c = STATE.get(sessionId)
|
||||
if (!c) return undefined
|
||||
return { total: c.total, failStreak: c.failStreak }
|
||||
}
|
||||
|
||||
Loading…
x
Reference in New Issue
Block a user