diff --git a/src/goals/goalEvaluator.test.ts b/src/goals/goalEvaluator.test.ts index ade3dbb7..15008382 100644 --- a/src/goals/goalEvaluator.test.ts +++ b/src/goals/goalEvaluator.test.ts @@ -134,6 +134,95 @@ describe('goalEvaluator', () => { ) }) + test('marks complete from finished task evidence without waiting for the evaluator', async () => { + setThreadGoal('thread-eval-complete-tasks', { + objective: 'build a todo app and review it', + now: 1_000, + }) + + const decision = await evaluateThreadGoalAfterTurn({ + threadId: 'thread-eval-complete-tasks', + messages: [ + createUserMessage({ + content: [ + { + type: 'tool_result', + tool_use_id: 'task-create-1', + content: 'Task #1 created successfully: Build app', + } as unknown as BetaContentBlock, + ], + }), + createAssistantMessage({ + content: [ + { + type: 'tool_use', + id: 'task-update-1', + name: 'TaskUpdate', + input: { taskId: '1', status: 'completed' }, + } as unknown as BetaContentBlock, + ], + }), + ], + assistantMessages: [ + createAssistantMessage({ + content: [ + { + type: 'text', + text: '## 完成总结\n\n已成功创建 Todo 应用。TypeScript 编译、ESLint PASS (0 errors)、Vite 生产构建均通过。', + }, + ], + }), + ], + signal: new AbortController().signal, + now: 3_000, + evaluate: async () => { + throw new Error('completion evidence should not call evaluator') + }, + }) + + expect(decision.action).toBe('complete') + expect(getThreadGoal('thread-eval-complete-tasks')?.status).toBe('complete') + }) + + test('continues instead of hanging when the evaluator times out', async () => { + setThreadGoal('thread-eval-timeout', { + objective: 'finish after external proof', + now: 1_000, + }) + const previous = process.env.CLAUDE_CODE_GOAL_EVALUATOR_TIMEOUT_MS + process.env.CLAUDE_CODE_GOAL_EVALUATOR_TIMEOUT_MS = '5' + try { + const decision = await evaluateThreadGoalAfterTurn({ + threadId: 'thread-eval-timeout', + messages: [ + createAssistantMessage({ + content: [{ type: 'text', text: 'Still checking.' }], + }), + ], + assistantMessages: [], + signal: new AbortController().signal, + now: 3_000, + evaluate: async ({ signal }) => + new Promise((resolve, reject) => { + signal.addEventListener('abort', () => + reject(new Error('aborted by timeout')), + ) + }), + }) + + expect(decision.action).toBe('continue') + if (decision.action === 'continue') { + expect(decision.reason).toContain('timed out') + } + } finally { + if (previous === undefined) { + delete process.env.CLAUDE_CODE_GOAL_EVALUATOR_TIMEOUT_MS + } else { + process.env.CLAUDE_CODE_GOAL_EVALUATOR_TIMEOUT_MS = previous + } + } + }) + test('continues an active goal before evaluating when tasks are still incomplete', async () => { setThreadGoal('thread-eval-open-task', { objective: 'finish all task-list work', diff --git a/src/goals/goalEvaluator.ts b/src/goals/goalEvaluator.ts index 1553a20a..22b4ef5f 100644 --- a/src/goals/goalEvaluator.ts +++ b/src/goals/goalEvaluator.ts @@ -5,6 +5,7 @@ import type { import type { QuerySource } from '../constants/querySource.js' import type { AssistantMessage, Message } from '../types/message.js' import { extractTextContent } from '../utils/messages.js' +import { createCombinedAbortSignal } from '../utils/combinedAbortSignal.js' import { getSmallFastModel } from '../utils/model/model.js' import { safeParseJSON } from '../utils/json.js' import { sideQuery } from '../utils/sideQuery.js' @@ -38,6 +39,7 @@ type EvaluateFn = (input: { }) => Promise const DEFAULT_MAX_CONTINUATIONS = 500 +const DEFAULT_EVALUATOR_TIMEOUT_MS = 45_000 export async function evaluateThreadGoalAfterTurn(input: { threadId: string @@ -97,12 +99,29 @@ export async function evaluateThreadGoalAfterTurn(input: { } } + const localCompletionReason = inferCompletionFromTaskEvidence( + [...input.messages, ...input.assistantMessages], + taskState, + ) + if (localCompletionReason) { + const completed = + markThreadGoalComplete(input.threadId, { + reason: localCompletionReason, + now, + }) ?? accounted + return { + action: 'complete', + goal: completed, + reason: localCompletionReason, + } + } + const transcript = formatTranscript([ ...input.messages, ...input.assistantMessages, ]) const evaluator = input.evaluate ?? evaluateGoalCompletion - const evaluation = await evaluator({ + const evaluation = await evaluateWithTimeout(evaluator, { goal: accounted, transcript, signal: input.signal, @@ -284,7 +303,10 @@ type TaskSummary = { status: string } -function summarizeTaskState(messages: Message[]): { incomplete: TaskSummary[] } { +function summarizeTaskState(messages: Message[]): { + tasks: TaskSummary[] + incomplete: TaskSummary[] +} { const tasks = new Map() for (const message of messages) { @@ -325,8 +347,10 @@ function summarizeTaskState(messages: Message[]): { incomplete: TaskSummary[] } } } + const allTasks = [...tasks.values()] return { - incomplete: [...tasks.values()].filter(task => + tasks: allTasks, + incomplete: allTasks.filter(task => task.status === 'pending' || task.status === 'in_progress', ), } @@ -368,3 +392,94 @@ function getMaxContinuations(): number { ? parsed : DEFAULT_MAX_CONTINUATIONS } + +async function evaluateWithTimeout( + evaluator: EvaluateFn, + input: { + goal: ThreadGoal + transcript: string + signal: AbortSignal + querySource?: QuerySource + }, +): Promise { + const timeoutMs = getEvaluatorTimeoutMs() + const { signal, cleanup } = createCombinedAbortSignal(input.signal, { + timeoutMs, + }) + try { + return await evaluator({ + ...input, + signal, + }) + } catch (error) { + if (input.signal.aborted) throw error + if (signal.aborted) { + return { + complete: false, + reason: `The goal completion evaluator timed out after ${Math.round( + timeoutMs / 1000, + )}s.`, + } + } + throw error + } finally { + cleanup() + } +} + +function getEvaluatorTimeoutMs(): number { + const raw = process.env.CLAUDE_CODE_GOAL_EVALUATOR_TIMEOUT_MS + if (!raw) return DEFAULT_EVALUATOR_TIMEOUT_MS + const parsed = Number.parseInt(raw, 10) + return Number.isFinite(parsed) && parsed > 0 + ? parsed + : DEFAULT_EVALUATOR_TIMEOUT_MS +} + +function inferCompletionFromTaskEvidence( + messages: Message[], + taskState: { tasks: TaskSummary[]; incomplete: TaskSummary[] }, +): string | null { + if (taskState.tasks.length === 0 || taskState.incomplete.length > 0) { + return null + } + + const text = latestAssistantVisibleText(messages) + if (!text) return null + if (looksLikeFailureOrIncomplete(text)) return null + if (!looksLikeCompletionSummary(text)) return null + + return `All ${taskState.tasks.length} tracked task(s) are complete and the final assistant response reports completion.` +} + +function latestAssistantVisibleText(messages: Message[]): string { + for (let i = messages.length - 1; i >= 0; i--) { + const message = messages[i] + if (message?.type !== 'assistant') continue + const text = assistantVisibleText(message.message.content).trim() + if (text) return text + } + return '' +} + +function looksLikeCompletionSummary(text: string): boolean { + return ( + /完成总结|目标已完成|已成功|全部验[证證]通过|均通过|构建成功|检查结果|代码审查结果/.test( + text, + ) || + /\b(completion summary|completed|complete|successfully|all tests passed|build passed|review complete|ready)\b/i.test( + text, + ) + ) +} + +function looksLikeFailureOrIncomplete(text: string): boolean { + return ( + /未完成|尚未完成|没有完成|失败|未通过|阻塞|(?:存在|有|出现|发现).{0,6}错误/.test( + text, + ) || + /\b(incomplete|not complete|not completed|not all tests passed|tests? did not pass|failed|failing|failure|blocked|errors? found|has errors?)\b/i.test( + text, + ) + ) +}