From cf7e48e5405590948ab8d6803b9ba230d89aa326 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?=E7=A8=8B=E5=BA=8F=E5=91=98=E9=98=BF=E6=B1=9F=28Relakkes?=
 =?UTF-8?q?=29?= <relakkes@gmail.com>
Date: Thu, 11 Jun 2026 00:36:39 +0800
Subject: [PATCH] fix(desktop): restore visual selection history cards

Restore visual-selection prompts from persisted transcript history as annotated screenshot attachments instead of exposing the model-facing selector prompt after reopening a session.

Tested: cd desktop && bun run test -- --run src/stores/chatStore.test.ts -t "restores visual selection history" --reporter verbose
Tested: bun run check:desktop
Confidence: high
Scope-risk: narrow
---
 desktop/src/stores/chatStore.test.ts | 50 ++++++++++++++++++++++
 desktop/src/stores/chatStore.ts      | 64 +++++++++++++++++++++++++++-
 2 files changed, 112 insertions(+), 2 deletions(-)

diff --git a/desktop/src/stores/chatStore.test.ts b/desktop/src/stores/chatStore.test.ts
index 47276030..ced3ef43 100644
--- a/desktop/src/stores/chatStore.test.ts
+++ b/desktop/src/stores/chatStore.test.ts
@@ -567,6 +567,56 @@ describe('chatStore history mapping', () => {
     ])
   })
 
+  it('restores visual selection history as annotated screenshot attachment without exposing model prompt', () => {
+    const modelPrompt = [
+      '请根据截图中编号 1 的蓝色标注修改本地前端。',
+      '目标元素：<time>',
+      'Selector：#root > main > section > ol > li:nth-of-type(1) > article > div:nth-of-type(1) > time',
+      'DOM 路径：body:nth-child(2) > div:nth-child(1) > main:nth-child(1) > section:nth-child(1) > ol:nth-child(4) > li:nth-child(1) > article:nth-child(1) > div:nth-child(3) > time:nth-child(2)',
+      '页面标题：Todo Desk Board',
+      '页面 URL：http://127.0.0.1:47931/',
+      '当前文本：06/10 21:12',
+      '用户注释：',
+      '这里的时间加上年份',
+      '请优先依据截图里的编号标注定位元素，selector 只作为辅助线索。',
+    ].join('\n')
+
+    const mapped = mapHistoryMessagesToUiMessages([
+      {
+        id: 'selection-user-1',
+        type: 'user',
+        timestamp: '2026-06-10T16:20:00.000Z',
+        content: [
+          { type: 'text', text: modelPrompt },
+          {
+            type: 'image',
+            source: {
+              media_type: 'image/png',
+              data: 'SELECTIONPNG',
+            },
+          },
+        ],
+      } as MessageEntry,
+    ])
+
+    expect(mapped).toMatchObject([
+      {
+        id: 'selection-user-1',
+        type: 'user_text',
+        content: '',
+        modelContent: modelPrompt,
+        attachments: [{
+          type: 'image',
+          name: '<time>',
+          data: 'data:image/png;base64,SELECTIONPNG',
+          mimeType: 'image/png',
+          note: '这里的时间加上年份',
+          quote: '#root > main > section > ol > li:nth-of-type(1) > article > div:nth-of-type(1) > time',
+        }],
+      },
+    ])
+  })
+
   it('restores /goal local command output from transcript history', () => {
     const messages: MessageEntry[] = [
       {
diff --git a/desktop/src/stores/chatStore.ts b/desktop/src/stores/chatStore.ts
index d237ef94..0803ce73 100644
--- a/desktop/src/stores/chatStore.ts
+++ b/desktop/src/stores/chatStore.ts
@@ -1955,6 +1955,14 @@ function isTeammateMessage(text: string): boolean {
 const SIMPLE_IMAGE_SOURCE_RE = /^\[Image source: (.+)\]$/
 const DETAILED_IMAGE_SOURCE_RE = /^\[Image: source: (.+?)(?:, original \d+x\d+, displayed at \d+x\d+\. Multiply coordinates by \d+(?:\.\d+)? to map to original image\.)?\]$/
 const IMAGE_RESIZE_METADATA_RE = /^\[Image: original \d+x\d+, displayed at \d+x\d+\. Multiply coordinates by \d+(?:\.\d+)? to map to original image\.\]$/
+const VISUAL_SELECTION_PROMPT_HEADER = '请根据截图中编号 1 的蓝色标注修改本地前端。'
+const VISUAL_SELECTION_PROMPT_FOOTER = '请优先依据截图里的编号标注定位元素，selector 只作为辅助线索。'
+
+type VisualSelectionHistoryDisplay = {
+  displayName: string
+  selector?: string
+  note?: string
+}
 
 function getHistoryImageMediaType(block: UserHistoryBlock): string {
   const mediaType = block.source?.media_type ?? block.mimeType ?? block.media_type
@@ -1981,6 +1989,50 @@ function isGeneratedImageMetadataText(text: string): boolean {
   return Boolean(extractImageMetadataSourcePath(text)) || IMAGE_RESIZE_METADATA_RE.test(text.trim())
 }
 
+function parseVisualSelectionHistoryPrompt(text: string): VisualSelectionHistoryDisplay | null {
+  const lines = text.replace(/\r\n?/g, '\n').split('\n')
+  if (lines[0]?.trim() !== VISUAL_SELECTION_PROMPT_HEADER) return null
+
+  let displayName: string | undefined
+  let selector: string | undefined
+  let note: string | undefined
+
+  for (let index = 1; index < lines.length; index += 1) {
+    const line = lines[index]?.trim() ?? ''
+    if (line.startsWith('目标元素：')) {
+      displayName = line.slice('目标元素：'.length).trim()
+    } else if (line.startsWith('Selector：')) {
+      selector = line.slice('Selector：'.length).trim()
+    } else if (line === '用户注释：') {
+      const noteLines: string[] = []
+      for (let noteIndex = index + 1; noteIndex < lines.length; noteIndex += 1) {
+        const noteLine = lines[noteIndex] ?? ''
+        if (noteLine.trim() === VISUAL_SELECTION_PROMPT_FOOTER) break
+        noteLines.push(noteLine)
+      }
+      const trimmedNote = noteLines.join('\n').trim()
+      note = trimmedNote || undefined
+      break
+    }
+  }
+
+  return displayName
+    ? {
+        displayName,
+        ...(selector ? { selector } : {}),
+        ...(note ? { note } : {}),
+      }
+    : null
+}
+
+function applyVisualSelectionHistoryDisplay(attachments: UIAttachment[], display: VisualSelectionHistoryDisplay): void {
+  const imageAttachment = attachments.find((attachment) => attachment.type === 'image')
+  if (!imageAttachment) return
+  imageAttachment.name = display.displayName
+  if (display.selector) imageAttachment.quote = display.selector
+  if (display.note) imageAttachment.note = display.note
+}
+
 function normalizeHistoryImageAttachment(block: UserHistoryBlock): UIAttachment {
   const mediaType = getHistoryImageMediaType(block)
   return {
@@ -2745,13 +2797,21 @@ export function mapHistoryMessagesToUiMessages(
       if (visibleTextParts.length > 0 || attachments.length > 0) {
         const visibleText = visibleTextParts.join('\n')
         const modelText = modelTextParts.join('\n')
+        const visualSelectionDisplay =
+          msg.type === 'user' && hasImageBlock
+            ? parseVisualSelectionHistoryPrompt(modelText)
+            : null
+        if (visualSelectionDisplay) {
+          applyVisualSelectionHistoryDisplay(attachments, visualSelectionDisplay)
+        }
         const parsed = extractLeadingFileReferences(visibleText)
-        const modelContent = modelText !== visibleText ? modelText : parsed.modelContent
+        const userContent = visualSelectionDisplay ? '' : parsed.content
+        const modelContent = visualSelectionDisplay || modelText !== visibleText ? modelText : parsed.modelContent
         const allAttachments = [...(parsed.attachments ?? []), ...attachments]
         uiMessages.push({
           id: msg.id || nextId(),
           type: 'user_text',
-          content: parsed.content,
+          content: userContent,
           ...(msg.id ? { transcriptMessageId: msg.id } : {}),
           ...(modelContent ? { modelContent } : {}),
           attachments: allAttachments.length > 0 ? allAttachments : undefined,