fix: send preview selections as annotated chat turns

Element selection from the local preview should preserve the visual target
the user picked and send the confirmed request directly to chat. The
screenshot now captures the visible viewport before drawing the marker, and
the selection event sends a model-facing prompt while the chat UI shows only
the annotated image and compact element label.

Constraint: No issue number was provided for this task.
Constraint: Keep the preview-agent protocol shape unchanged.
Rejected: Prefill the composer with the selection note | it keeps the old extra confirmation step and exposes implementation text in the input.
Rejected: Full-document annotation coordinates | body-relative captures drift from the viewport selection users see.
Confidence: high
Scope-risk: moderate
Directive: Do not route confirmed preview selections back through composer prefill without rechecking the Codex-style direct-send flow.
Tested: cd desktop && bun run test --run src/preview-agent/screenshot.test.ts src/lib/previewEvents.test.ts src/lib/selectionComposer.test.ts src/components/chat/AttachmentGallery.test.tsx src/stores/chatStore.test.ts
Tested: cd desktop && bun run lint
Tested: cd desktop && bun run build:preview-agent
Tested: bun run check:desktop
Tested: git diff --check
Not-tested: Real Tauri webview click-through smoke; in-app Browser automation timed out while local Vite HTTP smoke returned 200.

desktop/src/components/browser/BrowserSurface.tsx

@@ -102,10 +102,11 @@ export function BrowserSurface({ sessionId }: { sessionId: string }) {
           previewBridge.navigate(session.url)
         }}
       />
-      <div className="flex items-center gap-1 border-b px-2 py-1">
+      <div className="flex h-10 items-center gap-1 border-b border-[var(--color-border)] bg-[var(--color-surface-container-lowest)] px-2">
         <button
           aria-label="截图"
-          className="rounded p-1 hover:bg-muted"
+          title="截图"
+          className="inline-flex h-8 w-8 items-center justify-center rounded-full border border-transparent text-[var(--color-text-secondary)] transition-colors hover:border-[var(--color-border)] hover:bg-[var(--color-surface-container-low)] hover:text-[var(--color-text-primary)] focus-visible:outline-none focus-visible:ring-2 focus-visible:ring-[var(--color-brand)]"
           onClick={() => previewBridge.eval(`window.__PREVIEW_BRIDGE__?.handleHostRaw('{"v":1,"type":"capture","kind":"full"}')`)}
         >
           <Camera size={16} />
@@ -113,7 +114,13 @@ export function BrowserSurface({ sessionId }: { sessionId: string }) {
         <button
           aria-label="选择元素"
           aria-pressed={Boolean(session.pickerActive)}
-          className={`rounded p-1 hover:bg-muted ${session.pickerActive ? 'bg-muted text-primary' : ''}`}
+          title="选择元素"
+          className={[
+            'inline-flex h-8 w-8 items-center justify-center rounded-full border transition-colors focus-visible:outline-none focus-visible:ring-2 focus-visible:ring-[var(--color-brand)]',
+            session.pickerActive
+              ? 'border-[var(--color-brand)]/45 bg-[var(--color-surface-selected)] text-[var(--color-brand)]'
+              : 'border-transparent text-[var(--color-text-secondary)] hover:border-[var(--color-border)] hover:bg-[var(--color-surface-container-low)] hover:text-[var(--color-text-primary)]',
+          ].join(' ')}
           onClick={() => {
             const cur = useBrowserPanelStore.getState().bySession[sessionId]
             const next = !cur?.pickerActive

desktop/src/components/chat/AttachmentGallery.test.tsx

@@ -64,4 +64,25 @@ describe('AttachmentGallery', () => {
 
     expect(onRemove).toHaveBeenCalledWith('selection-1')
   })
+
+  it('shows a compact element chip for annotated selection images and exposes the note on hover', () => {
+    const view = render(
+      <AttachmentGallery
+        variant="message"
+        attachments={[{
+          id: 'preview-selection',
+          type: 'image',
+          name: '<h1>',
+          data: 'data:image/png;base64,AAAA',
+          note: '这个标题更轻一点',
+        }]}
+      />,
+    )
+
+    expect(view.getByRole('button', { name: 'Open <h1>' })).toBeTruthy()
+    const noteChip = view.getByLabelText('Selection note: 这个标题更轻一点')
+    expect(noteChip.textContent).toContain('<h1>')
+    expect(noteChip.getAttribute('title')).toBe('这个标题更轻一点')
+    expect(document.body.textContent).not.toContain('这个标题更轻一点')
+  })
 })

desktop/src/components/chat/AttachmentGallery.tsx

@@ -45,13 +45,15 @@ export function AttachmentGallery({ attachments, variant = 'message', onRemove }
         {attachments.map((attachment, index) => {
           if (attachment.type === 'image' && (attachment.previewUrl || attachment.data)) {
             const src = attachment.previewUrl || attachment.data || ''
+            const hasSelectionNote = !isComposer && !!attachment.note
             return (
               <div
                 key={attachment.id || `${attachment.name}-${index}`}
-                className={isComposer ? 'group relative' : ''}
+                className={isComposer ? 'group relative' : 'flex max-w-full flex-col items-end gap-1.5'}
               >
                 <button
                   type="button"
+                  aria-label={`Open ${attachment.name}`}
                   onClick={() => setActiveImageIndex(images.findIndex((image) => image.src === src))}
                   className={
                     isComposer
@@ -69,6 +71,25 @@ export function AttachmentGallery({ attachments, variant = 'message', onRemove }
                     }
                   />
                 </button>
+                {hasSelectionNote && (
+                  <span
+                    aria-label={`Selection note: ${attachment.note}`}
+                    title={attachment.note}
+                    tabIndex={0}
+                    className={[
+                      'inline-flex h-7 max-w-[260px] items-center gap-1.5 rounded-full border',
+                      'border-[var(--color-border)] bg-[var(--color-surface-container-low)] px-2.5',
+                      'text-[12px] font-medium leading-none text-[var(--color-text-primary)] shadow-[0_1px_2px_rgba(0,0,0,0.04)]',
+                      'transition-colors hover:border-[var(--color-brand)]/45 hover:bg-[var(--color-surface-container)]',
+                      'focus-visible:outline-none focus-visible:ring-2 focus-visible:ring-[var(--color-brand)] focus-visible:ring-offset-2',
+                    ].join(' ')}
+                  >
+                    <span className="material-symbols-outlined text-[15px] text-[var(--color-text-tertiary)]">
+                      ads_click
+                    </span>
+                    <span className="min-w-0 truncate">{attachment.name}</span>
+                  </span>
+                )}
                 {onRemove && attachment.id && (
                   <button
                     type="button"

desktop/src/lib/previewEvents.test.ts

@@ -1,17 +1,32 @@
-import { describe, expect, it, vi } from 'vitest'
+import { beforeEach, describe, expect, it, vi } from 'vitest'
 
 const listeners: Record<string, (e: { payload: string }) => void> = {}
 vi.mock('@tauri-apps/api/event', () => ({
   listen: (name: string, cb: (e: { payload: string }) => void) => { listeners[name] = cb; return Promise.resolve(() => {}) },
 }))
 
-const prefill = vi.hoisted(() => vi.fn())
-vi.mock('../stores/chatStore', () => ({ useChatStore: { getState: () => ({ queueComposerPrefill: prefill }) } }))
+const { prefill, sendMessage } = vi.hoisted(() => ({
+  prefill: vi.fn(),
+  sendMessage: vi.fn(),
+}))
+vi.mock('../stores/chatStore', () => ({
+  useChatStore: {
+    getState: () => ({
+      queueComposerPrefill: prefill,
+      sendMessage,
+    }),
+  },
+}))
 
 import { subscribePreviewEvents } from './previewEvents'
 import { useBrowserPanelStore } from '../stores/browserPanelStore'
 
 describe('subscribePreviewEvents', () => {
+  beforeEach(() => {
+    prefill.mockClear()
+    sendMessage.mockClear()
+  })
+
   it('routes navigated event to the store', async () => {
     useBrowserPanelStore.getState().open('s1', 'http://x/a')
     await subscribePreviewEvents('s1')
@@ -27,14 +42,25 @@ describe('subscribePreviewEvents', () => {
     }))
   })
 
-  it('selection event prefills composer with text + annotated screenshot', async () => {
+  it('selection event sends a chat turn directly with hidden prompt text + annotated screenshot', async () => {
     await subscribePreviewEvents('s1')
     const payload = { pageUrl: 'http://x/', element: { selector: '#t', tag: 'h1', classes: [] }, change: { description: '改一下' }, screenshot: { dataUrl: 'data:image/png;base64,AAAA', kind: 'element' } }
     listeners['preview://event']!({ payload: JSON.stringify({ v: 1, type: 'selection', payload }) })
-    expect(prefill).toHaveBeenCalledWith('s1', expect.objectContaining({
-      text: expect.stringContaining('改一下'),
-      attachments: [expect.objectContaining({ type: 'image', data: 'data:image/png;base64,AAAA' })],
-    }))
+    expect(prefill).not.toHaveBeenCalled()
+    expect(sendMessage).toHaveBeenCalledWith(
+      's1',
+      expect.stringContaining('改一下'),
+      [expect.objectContaining({
+        type: 'image',
+        name: '<h1>',
+        data: 'data:image/png;base64,AAAA',
+        note: '改一下',
+      })],
+      expect.objectContaining({
+        hideDisplayContent: true,
+        displayAttachments: [expect.objectContaining({ name: '<h1>', note: '改一下' })],
+      }),
+    )
   })
 
   it('selection event resets pickerActive on the session', async () => {

desktop/src/lib/previewEvents.ts

@@ -1,7 +1,7 @@
 import { listen } from '@tauri-apps/api/event'
 import { useBrowserPanelStore } from '../stores/browserPanelStore'
 import { useChatStore } from '../stores/chatStore'
-import { buildSelectionComposerText, type SelectionPayload } from './selectionComposer'
+import { buildSelectionDirectMessage, type SelectionPayload } from './selectionComposer'
 
 function kindLabel(kind?: string): string {
   if (kind === 'viewport') return 'viewport'
@@ -27,11 +27,21 @@ export async function subscribePreviewEvents(sessionId: string): Promise<() => v
       store.setPicker(sessionId, false)
       const p = msg.payload as (SelectionPayload & { screenshot?: { dataUrl?: string; kind?: string } }) | undefined
       if (!p || typeof p !== 'object' || !p.element) return
-      useChatStore.getState().queueComposerPrefill(sessionId, {
-        text: buildSelectionComposerText(p),
-        attachments: p.screenshot?.dataUrl
-          ? [{ type: 'image', name: 'selection.png', mimeType: 'image/png', data: p.screenshot.dataUrl }]
-          : [],
+      const selection = buildSelectionDirectMessage(p)
+      const attachments = p.screenshot?.dataUrl
+        ? [{
+            type: 'image' as const,
+            name: selection.displayName,
+            mimeType: 'image/png',
+            data: p.screenshot.dataUrl,
+            note: selection.note,
+            quote: p.element.selector,
+          }]
+        : []
+      useChatStore.getState().sendMessage(sessionId, selection.modelText, attachments, {
+        displayContent: selection.displayName,
+        displayAttachments: attachments,
+        hideDisplayContent: attachments.length > 0,
       })
     }
     else if (msg.type === 'picker-exited') {

desktop/src/lib/selectionComposer.test.ts

@@ -1,5 +1,5 @@
 import { expect, it } from 'vitest'
-import { buildSelectionComposerText } from './selectionComposer'
+import { buildSelectionComposerText, buildSelectionDirectMessage } from './selectionComposer'
 
 it('renders only the user instruction + concrete changes (no selector/DOM/page noise)', () => {
   const text = buildSelectionComposerText({
@@ -24,3 +24,25 @@ it('returns empty text when there is no description or change (image alone is th
   })
   expect(text).toBe('')
 })
+
+it('builds a model prompt while keeping the visible selection label compact', () => {
+  const message = buildSelectionDirectMessage({
+    pageUrl: 'http://localhost:5174/',
+    sourceHint: 'Todo preview',
+    element: {
+      selector: '#todo-title',
+      tag: 'h1',
+      text: 'Todo 管理',
+      classes: ['title'],
+      boundingBox: { x: 10, y: 20, w: 300, h: 80 },
+    } as never,
+    change: { description: '这个标题更轻一点' } as never,
+  })
+
+  expect(message.modelText).toContain('截图中编号 1')
+  expect(message.modelText).toContain('<h1>')
+  expect(message.modelText).toContain('#todo-title')
+  expect(message.modelText).toContain('这个标题更轻一点')
+  expect(message.displayName).toBe('<h1>')
+  expect(message.note).toBe('这个标题更轻一点')
+})

desktop/src/lib/selectionComposer.ts

@@ -7,6 +7,12 @@ export type SelectionPayload = {
   change?: EditDiff & { description?: string }
 }
 
+export type SelectionDirectMessage = {
+  modelText: string
+  displayName: string
+  note?: string
+}
+
 /**
  * 选中元素的「引用」由随附的圈选标注截图承载（截图里已把元素圈出来）。
  * 这里只产出用户的修改意见 + 具体改动（人话），**不再**把 selector / DOM 定位 /
@@ -24,3 +30,36 @@ export function buildSelectionComposerText(p: SelectionPayload): string {
   if (c?.fontFamily) lines.push(`- 字体：${c.fontFamily.from} → ${c.fontFamily.to}`)
   return lines.join('\n')
 }
+
+function formatElementLabel(element: ElementMetadata): string {
+  const tag = element.tag || 'element'
+  return `<${tag}>`
+}
+
+export function buildSelectionDirectMessage(p: SelectionPayload): SelectionDirectMessage {
+  const displayName = formatElementLabel(p.element)
+  const note = buildSelectionComposerText(p)
+  const lines = [
+    '请根据截图中编号 1 的蓝色标注修改本地前端。',
+    `目标元素：${displayName}`,
+    `Selector：${p.element.selector}`,
+  ]
+
+  if (p.element.nthPath) lines.push(`DOM 路径：${p.element.nthPath}`)
+  if (p.sourceHint) lines.push(`页面标题：${p.sourceHint}`)
+  if (p.pageUrl) lines.push(`页面 URL：${p.pageUrl}`)
+  if (p.element.text) lines.push(`当前文本：${p.element.text}`)
+  if (note) {
+    lines.push('用户注释：')
+    lines.push(note)
+  } else {
+    lines.push('用户没有提供额外注释；请只依据截图中的选中元素理解修改目标。')
+  }
+  lines.push('请优先依据截图里的编号标注定位元素，selector 只作为辅助线索。')
+
+  return {
+    modelText: lines.join('\n'),
+    displayName,
+    note: note || undefined,
+  }
+}

desktop/src/preview-agent/screenshot.test.ts

@@ -59,7 +59,7 @@ describe('captureToDataUrl', () => {
 })
 
 describe('captureAnnotatedRegion', () => {
-  it('calls html2canvas with document.body and scale:1', async () => {
+  it('captures the visible viewport with scale:1', async () => {
     const ctx = makeMockCtx()
     html2canvasMock.mockResolvedValue({
       getContext: () => ctx as unknown as CanvasRenderingContext2D,
@@ -76,7 +76,11 @@ describe('captureAnnotatedRegion', () => {
 
     await captureAnnotatedRegion(el, 1)
 
-    expect(html2canvasMock).toHaveBeenCalledWith(document.body, expect.objectContaining({ scale: 1 }))
+    expect(html2canvasMock).toHaveBeenCalledWith(document.documentElement, expect.objectContaining({
+      width: window.innerWidth,
+      height: window.innerHeight,
+      scale: 1,
+    }))
   })
 
   it('draws the annotation on the captured canvas (stroke and fillText called)', async () => {
@@ -100,6 +104,44 @@ describe('captureAnnotatedRegion', () => {
     expect(ctx.fillText).toHaveBeenCalled()
   })
 
+  it('draws the selected element at viewport coordinates in the captured viewport', async () => {
+    const ctx = makeMockCtx()
+    html2canvasMock.mockResolvedValue({
+      getContext: () => ctx as unknown as CanvasRenderingContext2D,
+      toDataURL: () => 'data:image/png;base64,RAW',
+      width: window.innerWidth,
+      height: window.innerHeight,
+    })
+    const el = document.createElement('input')
+    vi.spyOn(el, 'getBoundingClientRect').mockReturnValue({
+      left: 100, top: 50, width: 80, height: 40,
+      right: 180, bottom: 90, x: 100, y: 50,
+      toJSON: () => ({}),
+    } as DOMRect)
+    vi.spyOn(document.body, 'getBoundingClientRect').mockReturnValue({
+      left: 0, top: -200, width: 1000, height: 2000,
+      right: 1000, bottom: 1800, x: 0, y: -200,
+      toJSON: () => ({}),
+    } as DOMRect)
+
+    await captureAnnotatedRegion(el, 1)
+
+    const options = html2canvasMock.mock.calls[0]?.[1] as Record<string, unknown>
+    expect(html2canvasMock.mock.calls[0]?.[0]).toBe(document.documentElement)
+    expect(options).toMatchObject({
+      x: window.scrollX,
+      y: window.scrollY,
+      width: window.innerWidth,
+      height: window.innerHeight,
+      windowWidth: window.innerWidth,
+      windowHeight: window.innerHeight,
+      scale: 1,
+    })
+    const badgeCall = ctx.arc.mock.calls[0]
+    expect(badgeCall?.[0]).toBe(140)
+    expect(badgeCall?.[1]).toBe(50)
+  })
+
   it('returns the compressed wrapper of the canvas dataURL', async () => {
     const ctx = makeMockCtx()
     html2canvasMock.mockResolvedValue({

desktop/src/preview-agent/screenshot.ts

@@ -16,12 +16,24 @@ export async function captureToDataUrl(kind: CaptureKind, element?: Element): Pr
   return compressDataUrl(canvas.toDataURL('image/png'))
 }
 
-/** Full-page screenshot with the picked element's region annotated (blue box + numbered badge). 图4 */
+/** Viewport screenshot with the picked element's region annotated (blue box + numbered badge). 图4 */
 export async function captureAnnotatedRegion(el: Element, label = 1): Promise<string> {
   const elementRect = el.getBoundingClientRect()
-  const bodyRect = document.body.getBoundingClientRect()
-  const canvas = await html2canvas(document.body, { useCORS: true, logging: false, scale: 1 })
+  const viewportRect = { left: 0, top: 0, width: window.innerWidth, height: window.innerHeight }
+  const canvas = await html2canvas(document.documentElement, {
+    useCORS: true,
+    logging: false,
+    scale: 1,
+    x: window.scrollX,
+    y: window.scrollY,
+    width: window.innerWidth,
+    height: window.innerHeight,
+    windowWidth: window.innerWidth,
+    windowHeight: window.innerHeight,
+    scrollX: window.scrollX,
+    scrollY: window.scrollY,
+  })
   const ctx = canvas.getContext('2d')
-  if (ctx) drawAnnotation(ctx, computeAnnotationRect(elementRect, bodyRect, 1), label)
+  if (ctx) drawAnnotation(ctx, computeAnnotationRect(elementRect, viewportRect, 1), label)
   return compressDataUrl(canvas.toDataURL('image/png'))
 }

desktop/src/stores/chatStore.test.ts

@@ -1183,6 +1183,82 @@ describe('chatStore history mapping', () => {
     )
   })
 
+  it('can send a visual selection turn without rendering the full model prompt as user text', () => {
+    useChatStore.setState({
+      sessions: {
+        [TEST_SESSION_ID]: {
+          messages: [],
+          chatState: 'idle',
+          connectionState: 'connected',
+          streamingText: '',
+          streamingToolInput: '',
+          activeToolUseId: null,
+          activeToolName: null,
+          activeThinkingId: null,
+          pendingPermission: null,
+          pendingComputerUsePermission: null,
+          tokenUsage: { input_tokens: 0, output_tokens: 0 },
+          elapsedSeconds: 0,
+          statusVerb: '',
+          slashCommands: [],
+          agentTaskNotifications: {},
+          elapsedTimer: null,
+        },
+      },
+    })
+
+    useChatStore.getState().sendMessage(
+      TEST_SESSION_ID,
+      '请根据截图中编号 1 的 <h1> 修改：这个标题更轻一点',
+      [{
+        type: 'image',
+        name: '<h1>',
+        data: 'data:image/png;base64,AAAA',
+        mimeType: 'image/png',
+        note: '这个标题更轻一点',
+      }],
+      {
+        hideDisplayContent: true,
+        displayAttachments: [{
+          type: 'image',
+          name: '<h1>',
+          data: 'data:image/png;base64,AAAA',
+          mimeType: 'image/png',
+          note: '这个标题更轻一点',
+        }],
+      },
+    )
+
+    expect(useChatStore.getState().sessions[TEST_SESSION_ID]?.messages).toMatchObject([
+      {
+        type: 'user_text',
+        content: '',
+        modelContent: '请根据截图中编号 1 的 <h1> 修改：这个标题更轻一点',
+        attachments: [{
+          type: 'image',
+          name: '<h1>',
+          data: 'data:image/png;base64,AAAA',
+          mimeType: 'image/png',
+          note: '这个标题更轻一点',
+        }],
+      },
+    ])
+    expect(sendMock).toHaveBeenCalledWith(
+      TEST_SESSION_ID,
+      {
+        type: 'user_message',
+        content: '请根据截图中编号 1 的 <h1> 修改：这个标题更轻一点',
+        attachments: [{
+          type: 'image',
+          name: '<h1>',
+          data: 'data:image/png;base64,AAAA',
+          mimeType: 'image/png',
+          note: '这个标题更轻一点',
+        }],
+      },
+    )
+  })
+
   it('stores server-materialized attachment prefixes for rewind matching', () => {
     useChatStore.setState({
       sessions: {

desktop/src/stores/chatStore.ts

@@ -134,7 +134,7 @@ type ChatStore = {
     sessionId: string,
     content: string,
     attachments?: AttachmentRef[],
-    options?: { displayContent?: string; displayAttachments?: AttachmentRef[] },
+    options?: { displayContent?: string; displayAttachments?: AttachmentRef[]; hideDisplayContent?: boolean },
   ) => void
   respondToPermission: (
     sessionId: string,
@@ -878,10 +878,13 @@ export const useChatStore = create<ChatStore>((set, get) => ({
   },
 
   sendMessage: (sessionId, content, attachments, options) => {
-    const userFacingContent =
-      options?.displayContent?.trim() || content.trim()
-    const modelFacingContent = buildModelContent(content, attachments)
     const isMemberSession = !!useTeamStore.getState().getMemberBySessionId(sessionId)
+    const hideDisplayContent = !isMemberSession && options?.hideDisplayContent === true
+    const userFacingContent =
+      hideDisplayContent
+        ? ''
+        : options?.displayContent?.trim() || content.trim()
+    const modelFacingContent = buildModelContent(content, attachments)
     const visibleAttachments = options?.displayAttachments ?? attachments
     const uiAttachments: UIAttachment[] | undefined =
       visibleAttachments && visibleAttachments.length > 0
@@ -910,7 +913,7 @@ export const useChatStore = create<ChatStore>((set, get) => ({
     }
 
     if (!isMemberSession) {
-      updateOptimisticSessionTitle(sessionId, userFacingContent)
+      updateOptimisticSessionTitle(sessionId, userFacingContent || content.trim())
     }
 
     set((s) => {