From 96ce12d84b26cc5a62d832d72df4bedee49c6f67 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=E7=A8=8B=E5=BA=8F=E5=91=98=E9=98=BF=E6=B1=9F=28Relakkes?= =?UTF-8?q?=29?= Date: Tue, 23 Jun 2026 19:05:54 +0800 Subject: [PATCH] fix(runtime): keep goal loop alive on evaluator errors Goal prompt-hook evaluator failures now become blocking continuation feedback instead of non-blocking hook errors, so an active goal is treated as incomplete and the loop keeps working. Tested: bun test src/utils/hooks/execPromptHook.test.ts src/query/stopHooks.test.ts src/goals/goalState.test.ts Tested: bun run check:server Scope-risk: moderate Confidence: high --- src/utils/hooks.ts | 1 + src/utils/hooks/execPromptHook.test.ts | 114 +++++++++++++++++++++++++ src/utils/hooks/execPromptHook.ts | 43 ++++++++++ 3 files changed, 158 insertions(+) create mode 100644 src/utils/hooks/execPromptHook.test.ts diff --git a/src/utils/hooks.ts b/src/utils/hooks.ts index 5b3e519b..d75ce8bb 100644 --- a/src/utils/hooks.ts +++ b/src/utils/hooks.ts @@ -2769,6 +2769,7 @@ async function* executeHooks({ ) yield { preventContinuation: true, + blockingError: result.blockingError, stopReason: result.stopReason, } } diff --git a/src/utils/hooks/execPromptHook.test.ts b/src/utils/hooks/execPromptHook.test.ts new file mode 100644 index 00000000..00c6fe38 --- /dev/null +++ b/src/utils/hooks/execPromptHook.test.ts @@ -0,0 +1,114 @@ +import { beforeEach, describe, expect, mock, test } from 'bun:test' +import type { ToolUseContext } from '../../Tool.js' +import type { PromptHook } from '../settings/types.js' + +const queryModelWithoutStreamingMock = mock(async () => ({ + message: { + content: [{ type: 'text', text: 'not json' }], + }, +})) + +mock.module('../../services/api/claude.js', () => ({ + queryModelWithoutStreaming: queryModelWithoutStreamingMock, +})) + +const { execPromptHook } = await import('./execPromptHook.js') + +function makeContext(): ToolUseContext { + let responseLength = 0 + return { + abortController: new AbortController(), + agentId: undefined, + getAppState: () => ({ + toolPermissionContext: { mode: 'default' }, + }), + options: { + tools: [], + }, + setResponseLength: updater => { + responseLength = updater(responseLength) + }, + } as unknown as ToolUseContext +} + +const goalHook = { + type: 'prompt', + prompt: [ + '', + '', + 'ship the goal', + '', + ].join('\n'), + timeout: 1, +} satisfies PromptHook + +const ordinaryHook = { + type: 'prompt', + prompt: 'ordinary prompt hook', + timeout: 1, +} satisfies PromptHook + +describe('execPromptHook goal failures', () => { + beforeEach(() => { + queryModelWithoutStreamingMock.mockClear() + queryModelWithoutStreamingMock.mockImplementation(async () => ({ + message: { + content: [{ type: 'text', text: 'not json' }], + }, + })) + }) + + test('turns invalid /goal evaluator JSON into blocking continuation feedback', async () => { + const result = await execPromptHook( + goalHook, + 'Stop', + 'Stop', + '{}', + new AbortController().signal, + makeContext(), + ) + + expect(result.outcome).toBe('blocking') + expect(result.preventContinuation).toBe(true) + expect(result.blockingError?.command).toContain('') + expect(result.blockingError?.blockingError).toContain('response was not valid JSON') + expect(result.blockingError?.blockingError).toContain('continue working toward it') + }) + + test('keeps ordinary prompt hook invalid JSON as a non-blocking hook error', async () => { + const result = await execPromptHook( + ordinaryHook, + 'Stop', + 'Stop', + '{}', + new AbortController().signal, + makeContext(), + ) + + expect(result.outcome).toBe('non_blocking_error') + expect(result.preventContinuation).toBeUndefined() + expect(result.blockingError).toBeUndefined() + expect(result.message?.attachment.type).toBe('hook_non_blocking_error') + }) + + test('turns /goal evaluator schema failures into blocking continuation feedback', async () => { + queryModelWithoutStreamingMock.mockImplementation(async () => ({ + message: { + content: [{ type: 'text', text: '{"ok":"not boolean"}' }], + }, + })) + + const result = await execPromptHook( + goalHook, + 'Stop', + 'Stop', + '{}', + new AbortController().signal, + makeContext(), + ) + + expect(result.outcome).toBe('blocking') + expect(result.preventContinuation).toBe(true) + expect(result.blockingError?.blockingError).toContain('expected schema') + }) +}) diff --git a/src/utils/hooks/execPromptHook.ts b/src/utils/hooks/execPromptHook.ts index eb6843fe..f5bb9cc1 100644 --- a/src/utils/hooks/execPromptHook.ts +++ b/src/utils/hooks/execPromptHook.ts @@ -3,6 +3,7 @@ import type { HookEvent } from 'src/entrypoints/agentSdkTypes.js' import { queryModelWithoutStreaming } from '../../services/api/claude.js' import type { ToolUseContext } from '../../Tool.js' import type { Message } from '../../types/message.js' +import { isGoalPromptHookCommand } from '../../goals/goalState.js' import { createAttachmentMessage } from '../attachments.js' import { createCombinedAbortSignal } from '../combinedAbortSignal.js' import { logForDebugging } from '../debug.js' @@ -15,6 +16,25 @@ import type { PromptHook } from '../settings/types.js' import { asSystemPrompt } from '../systemPromptType.js' import { addArgumentsToPrompt, hookResponseSchema } from './hookHelpers.js' +function goalHookFailureResult( + hook: PromptHook, + reason: string, +): HookResult | null { + if (!isGoalPromptHookCommand(hook.prompt)) return null + + const blockingError = `Goal evaluator failed: ${reason}. Treat the goal as incomplete and continue working toward it.` + return { + hook, + outcome: 'blocking', + blockingError: { + blockingError, + command: hook.prompt, + }, + preventContinuation: true, + stopReason: blockingError, + } +} + /** * Execute a prompt-based hook using an LLM */ @@ -115,6 +135,12 @@ Your response must be a JSON object matching one of the following schemas: logForDebugging( `Hooks: error parsing response as JSON: ${fullResponse}`, ) + const goalFailure = goalHookFailureResult( + hook, + 'response was not valid JSON', + ) + if (goalFailure) return goalFailure + return { hook, outcome: 'non_blocking_error', @@ -135,6 +161,12 @@ Your response must be a JSON object matching one of the following schemas: logForDebugging( `Hooks: model response does not conform to expected schema: ${parsed.error.message}`, ) + const goalFailure = goalHookFailureResult( + hook, + `response did not match the expected schema (${parsed.error.message})`, + ) + if (goalFailure) return goalFailure + return { hook, outcome: 'non_blocking_error', @@ -184,6 +216,11 @@ Your response must be a JSON object matching one of the following schemas: cleanupSignal() if (combinedSignal.aborted) { + const goalFailure = signal.aborted + ? null + : goalHookFailureResult(hook, 'evaluation timed out') + if (goalFailure) return goalFailure + return { hook, outcome: 'cancelled', @@ -194,6 +231,12 @@ Your response must be a JSON object matching one of the following schemas: } catch (error) { const errorMsg = errorMessage(error) logForDebugging(`Hooks: Prompt hook error: ${errorMsg}`) + const goalFailure = goalHookFailureResult( + hook, + errorMsg || 'evaluation failed', + ) + if (goalFailure) return goalFailure + return { hook, outcome: 'non_blocking_error',