mirror of
https://github.com/NanmiCoder/cc-haha
synced 2026-07-16 13:03:31 +08:00
fix(runtime): keep goal loop alive on evaluator errors
Goal prompt-hook evaluator failures now become blocking continuation feedback instead of non-blocking hook errors, so an active goal is treated as incomplete and the loop keeps working. Tested: bun test src/utils/hooks/execPromptHook.test.ts src/query/stopHooks.test.ts src/goals/goalState.test.ts Tested: bun run check:server Scope-risk: moderate Confidence: high
This commit is contained in:
parent
be8b004c36
commit
96ce12d84b
@ -2769,6 +2769,7 @@ async function* executeHooks({
|
||||
)
|
||||
yield {
|
||||
preventContinuation: true,
|
||||
blockingError: result.blockingError,
|
||||
stopReason: result.stopReason,
|
||||
}
|
||||
}
|
||||
|
||||
114
src/utils/hooks/execPromptHook.test.ts
Normal file
114
src/utils/hooks/execPromptHook.test.ts
Normal file
@ -0,0 +1,114 @@
|
||||
import { beforeEach, describe, expect, mock, test } from 'bun:test'
|
||||
import type { ToolUseContext } from '../../Tool.js'
|
||||
import type { PromptHook } from '../settings/types.js'
|
||||
|
||||
const queryModelWithoutStreamingMock = mock(async () => ({
|
||||
message: {
|
||||
content: [{ type: 'text', text: 'not json' }],
|
||||
},
|
||||
}))
|
||||
|
||||
mock.module('../../services/api/claude.js', () => ({
|
||||
queryModelWithoutStreaming: queryModelWithoutStreamingMock,
|
||||
}))
|
||||
|
||||
const { execPromptHook } = await import('./execPromptHook.js')
|
||||
|
||||
function makeContext(): ToolUseContext {
|
||||
let responseLength = 0
|
||||
return {
|
||||
abortController: new AbortController(),
|
||||
agentId: undefined,
|
||||
getAppState: () => ({
|
||||
toolPermissionContext: { mode: 'default' },
|
||||
}),
|
||||
options: {
|
||||
tools: [],
|
||||
},
|
||||
setResponseLength: updater => {
|
||||
responseLength = updater(responseLength)
|
||||
},
|
||||
} as unknown as ToolUseContext
|
||||
}
|
||||
|
||||
const goalHook = {
|
||||
type: 'prompt',
|
||||
prompt: [
|
||||
'<cc-haha-goal-hook>',
|
||||
'<goal-objective>',
|
||||
'ship the goal',
|
||||
'</goal-objective>',
|
||||
].join('\n'),
|
||||
timeout: 1,
|
||||
} satisfies PromptHook
|
||||
|
||||
const ordinaryHook = {
|
||||
type: 'prompt',
|
||||
prompt: 'ordinary prompt hook',
|
||||
timeout: 1,
|
||||
} satisfies PromptHook
|
||||
|
||||
describe('execPromptHook goal failures', () => {
|
||||
beforeEach(() => {
|
||||
queryModelWithoutStreamingMock.mockClear()
|
||||
queryModelWithoutStreamingMock.mockImplementation(async () => ({
|
||||
message: {
|
||||
content: [{ type: 'text', text: 'not json' }],
|
||||
},
|
||||
}))
|
||||
})
|
||||
|
||||
test('turns invalid /goal evaluator JSON into blocking continuation feedback', async () => {
|
||||
const result = await execPromptHook(
|
||||
goalHook,
|
||||
'Stop',
|
||||
'Stop',
|
||||
'{}',
|
||||
new AbortController().signal,
|
||||
makeContext(),
|
||||
)
|
||||
|
||||
expect(result.outcome).toBe('blocking')
|
||||
expect(result.preventContinuation).toBe(true)
|
||||
expect(result.blockingError?.command).toContain('<cc-haha-goal-hook>')
|
||||
expect(result.blockingError?.blockingError).toContain('response was not valid JSON')
|
||||
expect(result.blockingError?.blockingError).toContain('continue working toward it')
|
||||
})
|
||||
|
||||
test('keeps ordinary prompt hook invalid JSON as a non-blocking hook error', async () => {
|
||||
const result = await execPromptHook(
|
||||
ordinaryHook,
|
||||
'Stop',
|
||||
'Stop',
|
||||
'{}',
|
||||
new AbortController().signal,
|
||||
makeContext(),
|
||||
)
|
||||
|
||||
expect(result.outcome).toBe('non_blocking_error')
|
||||
expect(result.preventContinuation).toBeUndefined()
|
||||
expect(result.blockingError).toBeUndefined()
|
||||
expect(result.message?.attachment.type).toBe('hook_non_blocking_error')
|
||||
})
|
||||
|
||||
test('turns /goal evaluator schema failures into blocking continuation feedback', async () => {
|
||||
queryModelWithoutStreamingMock.mockImplementation(async () => ({
|
||||
message: {
|
||||
content: [{ type: 'text', text: '{"ok":"not boolean"}' }],
|
||||
},
|
||||
}))
|
||||
|
||||
const result = await execPromptHook(
|
||||
goalHook,
|
||||
'Stop',
|
||||
'Stop',
|
||||
'{}',
|
||||
new AbortController().signal,
|
||||
makeContext(),
|
||||
)
|
||||
|
||||
expect(result.outcome).toBe('blocking')
|
||||
expect(result.preventContinuation).toBe(true)
|
||||
expect(result.blockingError?.blockingError).toContain('expected schema')
|
||||
})
|
||||
})
|
||||
@ -3,6 +3,7 @@ import type { HookEvent } from 'src/entrypoints/agentSdkTypes.js'
|
||||
import { queryModelWithoutStreaming } from '../../services/api/claude.js'
|
||||
import type { ToolUseContext } from '../../Tool.js'
|
||||
import type { Message } from '../../types/message.js'
|
||||
import { isGoalPromptHookCommand } from '../../goals/goalState.js'
|
||||
import { createAttachmentMessage } from '../attachments.js'
|
||||
import { createCombinedAbortSignal } from '../combinedAbortSignal.js'
|
||||
import { logForDebugging } from '../debug.js'
|
||||
@ -15,6 +16,25 @@ import type { PromptHook } from '../settings/types.js'
|
||||
import { asSystemPrompt } from '../systemPromptType.js'
|
||||
import { addArgumentsToPrompt, hookResponseSchema } from './hookHelpers.js'
|
||||
|
||||
function goalHookFailureResult(
|
||||
hook: PromptHook,
|
||||
reason: string,
|
||||
): HookResult | null {
|
||||
if (!isGoalPromptHookCommand(hook.prompt)) return null
|
||||
|
||||
const blockingError = `Goal evaluator failed: ${reason}. Treat the goal as incomplete and continue working toward it.`
|
||||
return {
|
||||
hook,
|
||||
outcome: 'blocking',
|
||||
blockingError: {
|
||||
blockingError,
|
||||
command: hook.prompt,
|
||||
},
|
||||
preventContinuation: true,
|
||||
stopReason: blockingError,
|
||||
}
|
||||
}
|
||||
|
||||
/**
|
||||
* Execute a prompt-based hook using an LLM
|
||||
*/
|
||||
@ -115,6 +135,12 @@ Your response must be a JSON object matching one of the following schemas:
|
||||
logForDebugging(
|
||||
`Hooks: error parsing response as JSON: ${fullResponse}`,
|
||||
)
|
||||
const goalFailure = goalHookFailureResult(
|
||||
hook,
|
||||
'response was not valid JSON',
|
||||
)
|
||||
if (goalFailure) return goalFailure
|
||||
|
||||
return {
|
||||
hook,
|
||||
outcome: 'non_blocking_error',
|
||||
@ -135,6 +161,12 @@ Your response must be a JSON object matching one of the following schemas:
|
||||
logForDebugging(
|
||||
`Hooks: model response does not conform to expected schema: ${parsed.error.message}`,
|
||||
)
|
||||
const goalFailure = goalHookFailureResult(
|
||||
hook,
|
||||
`response did not match the expected schema (${parsed.error.message})`,
|
||||
)
|
||||
if (goalFailure) return goalFailure
|
||||
|
||||
return {
|
||||
hook,
|
||||
outcome: 'non_blocking_error',
|
||||
@ -184,6 +216,11 @@ Your response must be a JSON object matching one of the following schemas:
|
||||
cleanupSignal()
|
||||
|
||||
if (combinedSignal.aborted) {
|
||||
const goalFailure = signal.aborted
|
||||
? null
|
||||
: goalHookFailureResult(hook, 'evaluation timed out')
|
||||
if (goalFailure) return goalFailure
|
||||
|
||||
return {
|
||||
hook,
|
||||
outcome: 'cancelled',
|
||||
@ -194,6 +231,12 @@ Your response must be a JSON object matching one of the following schemas:
|
||||
} catch (error) {
|
||||
const errorMsg = errorMessage(error)
|
||||
logForDebugging(`Hooks: Prompt hook error: ${errorMsg}`)
|
||||
const goalFailure = goalHookFailureResult(
|
||||
hook,
|
||||
errorMsg || 'evaluation failed',
|
||||
)
|
||||
if (goalFailure) return goalFailure
|
||||
|
||||
return {
|
||||
hook,
|
||||
outcome: 'non_blocking_error',
|
||||
|
||||
Loading…
x
Reference in New Issue
Block a user