fix(desktop): selection captures full-page screenshot with annotated region (图4)

Co-Authored-By: Claude Sonnet 4.6 <noreply@anthropic.com>

desktop/src/preview-agent/annotate.test.ts

@@ -0,0 +1,87 @@
+import { describe, expect, it, vi } from 'vitest'
+import { computeAnnotationRect, drawAnnotation } from './annotate'
+
+// ---------------------------------------------------------------------------
+// computeAnnotationRect
+// ---------------------------------------------------------------------------
+
+describe('computeAnnotationRect', () => {
+  it('(a) maps element rect relative to body rect at scale 1', () => {
+    const box = computeAnnotationRect(
+      { left: 100, top: 50, width: 80, height: 40 },
+      { left: 0, top: 0, width: 1000, height: 2000 },
+      1,
+    )
+    expect(box).toEqual({ x: 100, y: 50, w: 80, h: 40 })
+  })
+
+  it('(b) cancels scroll — scrolled body with top:-200 gives correct page-offset y', () => {
+    // body rect top = -200 means page has scrolled 200px down
+    const box = computeAnnotationRect(
+      { left: 100, top: 50, width: 80, height: 40 },
+      { left: 0, top: -200, width: 1000, height: 2000 },
+      1,
+    )
+    expect(box).toEqual({ x: 100, y: 250, w: 80, h: 40 })
+  })
+
+  it('(c) scale 2 doubles all four values', () => {
+    const box = computeAnnotationRect(
+      { left: 100, top: 50, width: 80, height: 40 },
+      { left: 0, top: 0, width: 1000, height: 2000 },
+      2,
+    )
+    expect(box).toEqual({ x: 200, y: 100, w: 160, h: 80 })
+  })
+})
+
+// ---------------------------------------------------------------------------
+// drawAnnotation — mock 2d context
+// ---------------------------------------------------------------------------
+
+function makeMockCtx() {
+  return {
+    // mutable props
+    lineWidth: 0,
+    strokeStyle: '',
+    fillStyle: '',
+    font: '',
+    textAlign: '' as CanvasRenderingContext2D['textAlign'],
+    textBaseline: '' as CanvasRenderingContext2D['textBaseline'],
+    // mock functions
+    save: vi.fn(),
+    restore: vi.fn(),
+    beginPath: vi.fn(),
+    moveTo: vi.fn(),
+    arcTo: vi.fn(),
+    arc: vi.fn(),
+    closePath: vi.fn(),
+    stroke: vi.fn(),
+    fill: vi.fn(),
+    fillText: vi.fn(),
+  }
+}
+
+describe('drawAnnotation', () => {
+  it('draws the selection outline and badge with expected calls', () => {
+    const ctx = makeMockCtx()
+    drawAnnotation(ctx as unknown as CanvasRenderingContext2D, { x: 10, y: 20, w: 30, h: 40 }, 1)
+
+    // selection outline must be stroked
+    expect(ctx.stroke).toHaveBeenCalled()
+
+    // badge: arc(cx, cy, radius, ...) — cx = x + w/2 = 10 + 15 = 25, cy = y = 20, radius = 13
+    expect(ctx.arc).toHaveBeenCalled()
+    const arcCall = ctx.arc.mock.calls[0]
+    expect(arcCall?.[0]).toBe(25)   // cx
+    expect(arcCall?.[1]).toBe(20)   // cy
+    expect(arcCall?.[2]).toBe(13)   // radius
+
+    // fillText('1', cx, cy)
+    expect(ctx.fillText).toHaveBeenCalled()
+    const textCall = ctx.fillText.mock.calls[0]
+    expect(textCall?.[0]).toBe('1')
+    expect(textCall?.[1]).toBe(25)
+    expect(textCall?.[2]).toBe(20)
+  })
+})

desktop/src/preview-agent/annotate.ts

@@ -0,0 +1,51 @@
+export type Box = { x: number; y: number; w: number; h: number }
+export type RectLike = { left: number; top: number; width: number; height: number }
+
+/**
+ * Map an element's viewport rect into the html2canvas full-body capture's pixel coords.
+ * Both rects are viewport-relative, so subtracting the body's rect yields the element's
+ * offset within the captured body (scroll cancels out). Multiply by the capture scale.
+ */
+export function computeAnnotationRect(elementRect: RectLike, bodyRect: RectLike, scale = 1): Box {
+  return {
+    x: (elementRect.left - bodyRect.left) * scale,
+    y: (elementRect.top - bodyRect.top) * scale,
+    w: elementRect.width * scale,
+    h: elementRect.height * scale,
+  }
+}
+
+const ACCENT = '#2f7bff'
+
+function roundRectPath(ctx: CanvasRenderingContext2D, x: number, y: number, w: number, h: number, r: number): void {
+  const rr = Math.max(0, Math.min(r, w / 2, h / 2))
+  ctx.beginPath()
+  ctx.moveTo(x + rr, y)
+  ctx.arcTo(x + w, y, x + w, y + h, rr)
+  ctx.arcTo(x + w, y + h, x, y + h, rr)
+  ctx.arcTo(x, y + h, x, y, rr)
+  ctx.arcTo(x, y, x + w, y, rr)
+  ctx.closePath()
+}
+
+/** Draw a blue rounded selection box + a numbered badge centered on the top edge (图4). */
+export function drawAnnotation(ctx: CanvasRenderingContext2D, box: Box, label: number | string = 1, scale = 1): void {
+  ctx.save()
+  ctx.lineWidth = Math.max(2, Math.round(3 * scale))
+  ctx.strokeStyle = ACCENT
+  roundRectPath(ctx, box.x, box.y, box.w, box.h, 6 * scale)
+  ctx.stroke()
+  const cx = box.x + box.w / 2
+  const cy = box.y
+  const radius = 13 * scale
+  ctx.beginPath()
+  ctx.arc(cx, cy, radius, 0, Math.PI * 2)
+  ctx.fillStyle = ACCENT
+  ctx.fill()
+  ctx.fillStyle = '#ffffff'
+  ctx.font = `bold ${Math.round(14 * scale)}px sans-serif`
+  ctx.textAlign = 'center'
+  ctx.textBaseline = 'middle'
+  ctx.fillText(String(label), cx, cy)
+  ctx.restore()
+}

desktop/src/preview-agent/index.ts

@@ -1,5 +1,5 @@
 import { createBridge } from './bridge'
-import { captureToDataUrl } from './screenshot'
+import { captureToDataUrl, captureAnnotatedRegion } from './screenshot'
 import { createPicker } from './picker'
 import { buildElementMetadata } from './metadata'
 
@@ -26,14 +26,14 @@ import { buildElementMetadata } from './metadata'
 
   const emitSelection = async (el: Element) => {
     try {
-      const dataUrl = await captureToDataUrl('element', el)
+      const dataUrl = await captureAnnotatedRegion(el)
       bridge.send({
         type: 'selection',
         payload: {
           pageUrl: window.location.href,
           sourceHint: document.title || undefined,
           element: buildElementMetadata(el),
-          screenshot: { dataUrl, kind: 'element' },
+          screenshot: { dataUrl, kind: 'region' },
         },
       })
     } catch (e) { bridge.reportError(String(e)) }

desktop/src/preview-agent/screenshot.test.ts

@@ -2,11 +2,32 @@ import { describe, expect, it, vi, beforeEach } from 'vitest'
 
 const html2canvasMock = vi.fn()
 vi.mock('html2canvas', () => ({ default: (...args: unknown[]) => html2canvasMock(...args) }))
-vi.mock('../lib/imageCompress', () => ({ compressDataUrl: vi.fn(async (d: string) => `compressed:${d}`) }))
+vi.mock('../lib/imageCompress', () => ({ compressDataUrl: vi.fn(async (d: string) => `c:${d}`) }))
 
-import { captureToDataUrl } from './screenshot'
+import { captureToDataUrl, captureAnnotatedRegion } from './screenshot'
 import { compressDataUrl } from '../lib/imageCompress'
 
+function makeMockCtx() {
+  return {
+    lineWidth: 0,
+    strokeStyle: '',
+    fillStyle: '',
+    font: '',
+    textAlign: '' as CanvasRenderingContext2D['textAlign'],
+    textBaseline: '' as CanvasRenderingContext2D['textBaseline'],
+    save: vi.fn(),
+    restore: vi.fn(),
+    beginPath: vi.fn(),
+    moveTo: vi.fn(),
+    arcTo: vi.fn(),
+    arc: vi.fn(),
+    closePath: vi.fn(),
+    stroke: vi.fn(),
+    fill: vi.fn(),
+    fillText: vi.fn(),
+  }
+}
+
 beforeEach(() => {
   html2canvasMock.mockReset()
   html2canvasMock.mockResolvedValue({ toDataURL: () => 'data:image/png;base64,RAW' })
@@ -18,7 +39,7 @@ describe('captureToDataUrl', () => {
     const out = await captureToDataUrl('full')
     expect(html2canvasMock).toHaveBeenCalledWith(document.body, expect.any(Object))
     expect(compressDataUrl).toHaveBeenCalledWith('data:image/png;base64,RAW')
-    expect(out).toBe('compressed:data:image/png;base64,RAW')
+    expect(out).toBe('c:data:image/png;base64,RAW')
   })
   it('captures the given element for element kind', async () => {
     const el = document.createElement('div')
@@ -36,3 +57,67 @@ describe('captureToDataUrl', () => {
     expect(opts.windowWidth).toBe(window.innerWidth)
   })
 })
+
+describe('captureAnnotatedRegion', () => {
+  it('calls html2canvas with document.body and scale:1', async () => {
+    const ctx = makeMockCtx()
+    html2canvasMock.mockResolvedValue({
+      getContext: () => ctx as unknown as CanvasRenderingContext2D,
+      toDataURL: () => 'data:image/png;base64,RAW',
+      width: 1000,
+      height: 2000,
+    })
+    const el = document.createElement('div')
+    vi.spyOn(el, 'getBoundingClientRect').mockReturnValue({
+      left: 100, top: 50, width: 80, height: 40,
+      right: 180, bottom: 90, x: 100, y: 50,
+      toJSON: () => ({}),
+    } as DOMRect)
+
+    await captureAnnotatedRegion(el, 1)
+
+    expect(html2canvasMock).toHaveBeenCalledWith(document.body, expect.objectContaining({ scale: 1 }))
+  })
+
+  it('draws the annotation on the captured canvas (stroke and fillText called)', async () => {
+    const ctx = makeMockCtx()
+    html2canvasMock.mockResolvedValue({
+      getContext: () => ctx as unknown as CanvasRenderingContext2D,
+      toDataURL: () => 'data:image/png;base64,RAW',
+      width: 1000,
+      height: 2000,
+    })
+    const el = document.createElement('div')
+    vi.spyOn(el, 'getBoundingClientRect').mockReturnValue({
+      left: 100, top: 50, width: 80, height: 40,
+      right: 180, bottom: 90, x: 100, y: 50,
+      toJSON: () => ({}),
+    } as DOMRect)
+
+    await captureAnnotatedRegion(el, 1)
+
+    expect(ctx.stroke).toHaveBeenCalled()
+    expect(ctx.fillText).toHaveBeenCalled()
+  })
+
+  it('returns the compressed wrapper of the canvas dataURL', async () => {
+    const ctx = makeMockCtx()
+    html2canvasMock.mockResolvedValue({
+      getContext: () => ctx as unknown as CanvasRenderingContext2D,
+      toDataURL: () => 'data:image/png;base64,RAW',
+      width: 1000,
+      height: 2000,
+    })
+    const el = document.createElement('div')
+    vi.spyOn(el, 'getBoundingClientRect').mockReturnValue({
+      left: 0, top: 0, width: 50, height: 50,
+      right: 50, bottom: 50, x: 0, y: 0,
+      toJSON: () => ({}),
+    } as DOMRect)
+
+    const result = await captureAnnotatedRegion(el, 1)
+
+    expect(compressDataUrl).toHaveBeenCalledWith('data:image/png;base64,RAW')
+    expect(result).toBe('c:data:image/png;base64,RAW')
+  })
+})

desktop/src/preview-agent/screenshot.ts

@@ -1,5 +1,6 @@
 import html2canvas from 'html2canvas'
 import { compressDataUrl } from '../lib/imageCompress'
+import { computeAnnotationRect, drawAnnotation } from './annotate'
 
 export type CaptureKind = 'full' | 'viewport' | 'element'
 
@@ -14,3 +15,13 @@ export async function captureToDataUrl(kind: CaptureKind, element?: Element): Pr
   })
   return compressDataUrl(canvas.toDataURL('image/png'))
 }
+
+/** Full-page screenshot with the picked element's region annotated (blue box + numbered badge). 图4 */
+export async function captureAnnotatedRegion(el: Element, label = 1): Promise<string> {
+  const elementRect = el.getBoundingClientRect()
+  const bodyRect = document.body.getBoundingClientRect()
+  const canvas = await html2canvas(document.body, { useCORS: true, logging: false, scale: 1 })
+  const ctx = canvas.getContext('2d')
+  if (ctx) drawAnnotation(ctx, computeAnnotationRect(elementRect, bodyRect, 1), label)
+  return compressDataUrl(canvas.toDataURL('image/png'))
+}