From 737eb581b5c4bb977dde8e9e95da001e6aa52b96 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Micha=C5=82=20Pierzcha=C5=82a?= Date: Thu, 30 Jul 2026 11:55:14 +0200 Subject: [PATCH 1/5] refactor: simplify screenshot diff output --- .../screenshot-diff-non-text.test.ts | 137 ------ .../__tests__/screenshot-diff-ocr.test.ts | 131 ----- .../__tests__/screenshot-diff.test.ts | 11 - .../screenshot-diff-non-text.ts | 446 ------------------ src/screenshot-diff/screenshot-diff-ocr.ts | 376 --------------- .../screenshot-diff-region-split.ts | 43 +- .../screenshot-diff-region-types.ts | 6 - .../screenshot-diff-regions.ts | 185 +------- src/screenshot-diff/screenshot-diff.ts | 46 +- src/utils/__tests__/output.test.ts | 77 +-- src/utils/output.ts | 148 +----- website/docs/docs/commands.md | 4 +- 12 files changed, 25 insertions(+), 1585 deletions(-) delete mode 100644 src/screenshot-diff/__tests__/screenshot-diff-non-text.test.ts delete mode 100644 src/screenshot-diff/__tests__/screenshot-diff-ocr.test.ts delete mode 100644 src/screenshot-diff/screenshot-diff-non-text.ts delete mode 100644 src/screenshot-diff/screenshot-diff-ocr.ts diff --git a/src/screenshot-diff/__tests__/screenshot-diff-non-text.test.ts b/src/screenshot-diff/__tests__/screenshot-diff-non-text.test.ts deleted file mode 100644 index b7cd9dabc7..0000000000 --- a/src/screenshot-diff/__tests__/screenshot-diff-non-text.test.ts +++ /dev/null @@ -1,137 +0,0 @@ -import assert from 'node:assert/strict'; -import { test } from 'vitest'; -import { summarizeNonTextDiffDeltas } from '../screenshot-diff-non-text.ts'; -import { normalizedRect } from '../../utils/screenshot-geometry.ts'; - -function paintMaskRect( - mask: Uint8Array, - imageWidth: number, - rect: { x: number; y: number; width: number; height: number }, -): void { - for (let y = rect.y; y < rect.y + rect.height; y += 1) { - for (let x = rect.x; x < rect.x + rect.width; x += 1) { - mask[y * imageWidth + x] = 1; - } - } -} - -test('summarizeNonTextDiffDeltas masks OCR text and reports leading icon residuals', () => { - const width = 220; - const height = 120; - const diffMask = new Uint8Array(width * height); - paintMaskRect(diffMask, width, { x: 20, y: 30, width: 20, height: 20 }); - paintMaskRect(diffMask, width, { x: 70, y: 32, width: 48, height: 12 }); - - const deltas = summarizeNonTextDiffDeltas({ - diffMask, - width, - height, - regions: [ - { - index: 1, - rect: { x: 0, y: 20, width: 180, height: 50 }, - normalizedRect: normalizedRect({ x: 0, y: 16.67, width: 81.82, height: 41.67 }), - differentPixels: 976, - shareOfDiffPercentage: 100, - densityPercentage: 10.84, - shape: 'horizontal-band', - size: 'medium', - location: 'center', - averageBaselineColorHex: '#000000', - averageCurrentColorHex: '#ffffff', - baselineLuminance: 0, - currentLuminance: 255, - dominantChange: 'brighter', - }, - ], - ocr: { - provider: 'tesseract', - baselineBlocks: 1, - currentBlocks: 1, - baselineBlocksRaw: [], - currentBlocksRaw: [ - { - text: 'Wi-Fi', - confidence: 90, - rect: { x: 68, y: 28, width: 60, height: 24 }, - normalizedRect: normalizedRect({ x: 30.91, y: 23.33, width: 27.27, height: 20 }), - }, - ], - matches: [], - }, - }); - - assert.equal(deltas.length, 1); - assert.equal(deltas[0]?.regionIndex, 1); - assert.equal(deltas[0]?.slot, 'leading'); - assert.equal(deltas[0]?.likelyKind, 'icon'); - assert.deepEqual(deltas[0]?.rect, { x: 20, y: 30, width: 20, height: 20 }); - assert.equal(deltas[0]?.nearestText, 'Wi-Fi'); -}); - -test('summarizeNonTextDiffDeltas uses overlapping baseline text when current OCR misses a row', () => { - const width = 220; - const height = 120; - const diffMask = new Uint8Array(width * height); - paintMaskRect(diffMask, width, { x: 20, y: 30, width: 20, height: 20 }); - - const deltas = summarizeNonTextDiffDeltas({ - diffMask, - width, - height, - regions: [], - ocr: { - provider: 'tesseract', - baselineBlocks: 1, - currentBlocks: 0, - baselineBlocksRaw: [ - { - text: 'Wi-Fi', - confidence: 90, - rect: { x: 68, y: 28, width: 60, height: 24 }, - normalizedRect: normalizedRect({ x: 30.91, y: 23.33, width: 27.27, height: 20 }), - }, - ], - currentBlocksRaw: [], - matches: [], - }, - }); - - assert.equal(deltas.length, 1); - assert.equal(deltas[0]?.slot, 'leading'); - assert.equal(deltas[0]?.likelyKind, 'icon'); - assert.equal(deltas[0]?.nearestText, 'Wi-Fi'); -}); - -test('summarizeNonTextDiffDeltas omits broad background residuals', () => { - const width = 220; - const height = 120; - const diffMask = new Uint8Array(width * height); - paintMaskRect(diffMask, width, { x: 10, y: 30, width: 180, height: 40 }); - - const deltas = summarizeNonTextDiffDeltas({ - diffMask, - width, - height, - regions: [ - { - index: 1, - rect: { x: 10, y: 30, width: 180, height: 40 }, - normalizedRect: normalizedRect({ x: 4.55, y: 25, width: 81.82, height: 33.33 }), - differentPixels: 7200, - shareOfDiffPercentage: 100, - densityPercentage: 100, - shape: 'large-area', - size: 'large', - location: 'center', - averageBaselineColorHex: '#000000', - averageCurrentColorHex: '#ffffff', - baselineLuminance: 0, - currentLuminance: 255, - dominantChange: 'brighter', - }, - ], - }); - - assert.deepEqual(deltas, []); -}); diff --git a/src/screenshot-diff/__tests__/screenshot-diff-ocr.test.ts b/src/screenshot-diff/__tests__/screenshot-diff-ocr.test.ts deleted file mode 100644 index 71e9d893e6..0000000000 --- a/src/screenshot-diff/__tests__/screenshot-diff-ocr.test.ts +++ /dev/null @@ -1,131 +0,0 @@ -import assert from 'node:assert/strict'; -import fs from 'node:fs'; -import os from 'node:os'; -import path from 'node:path'; -import { test } from 'vitest'; -import { - matchOcrBlocks, - parseTesseractTsv, - summarizeScreenshotOcr, - summarizeOcrMovementClusters, -} from '../screenshot-diff-ocr.ts'; -import { normalizedRect } from '../../utils/screenshot-geometry.ts'; - -test('parseTesseractTsv groups word rows into text line blocks', () => { - const blocks = parseTesseractTsv( - [ - 'level\tpage_num\tblock_num\tpar_num\tline_num\tword_num\tleft\ttop\twidth\theight\tconf\ttext', - '5\t1\t1\t1\t1\t1\t100\t200\t40\t20\t96\tAirplane', - '5\t1\t1\t1\t1\t2\t150\t200\t30\t20\t94\tMode', - '5\t1\t1\t1\t1\t3\t300\t200\t90\t20\t92\tDisconnected', - '5\t1\t1\t1\t2\t1\t100\t240\t50\t20\t90\tWi-Fi', - '5\t1\t1\t1\t3\t1\t100\t280\t10\t20\t-1\t', - ].join('\n'), - 400, - 800, - ); - - assert.equal(blocks.length, 3); - assert.deepEqual(blocks[0], { - text: 'Airplane Mode', - confidence: 95, - rect: { x: 100, y: 200, width: 80, height: 20 }, - normalizedRect: normalizedRect({ x: 25, y: 25, width: 20, height: 2.5 }), - }); - assert.deepEqual(blocks[1], { - text: 'Disconnected', - confidence: 92, - rect: { x: 300, y: 200, width: 90, height: 20 }, - normalizedRect: normalizedRect({ x: 75, y: 25, width: 22.5, height: 2.5 }), - }); - assert.deepEqual(blocks[2], { - text: 'Wi-Fi', - confidence: 90, - rect: { x: 100, y: 240, width: 50, height: 20 }, - normalizedRect: normalizedRect({ x: 25, y: 30, width: 12.5, height: 2.5 }), - }); -}); - -test('matchOcrBlocks reports movement and OCR bbox size change', () => { - const matches = matchOcrBlocks( - [ - { - text: 'Wi-Fi', - confidence: 96, - rect: { x: 100, y: 200, width: 50, height: 20 }, - normalizedRect: normalizedRect({ x: 25, y: 25, width: 12.5, height: 2.5 }), - }, - ], - [ - { - text: 'Wi-Fi', - confidence: 94, - rect: { x: 112, y: 192, width: 60, height: 20 }, - normalizedRect: normalizedRect({ x: 28, y: 24, width: 15, height: 2.5 }), - }, - ], - ); - - assert.equal(matches.length, 1); - assert.deepEqual(matches[0]?.delta, { x: 12, y: -8, width: 10, height: 0 }); - assert.equal(matches[0]?.possibleTextMetricMismatch, true); -}); - -test('summarizeOcrMovementClusters groups repeated x-axis text movement', () => { - const clusters = summarizeOcrMovementClusters([ - { - text: 'Wi-Fi', - baselineRect: { x: 100, y: 200, width: 50, height: 20 }, - currentRect: { x: 286, y: 120, width: 50, height: 20 }, - delta: { x: 186, y: -80, width: 0, height: 0 }, - confidence: 96, - possibleTextMetricMismatch: false, - }, - { - text: 'Bluetooth', - baselineRect: { x: 100, y: 260, width: 90, height: 20 }, - currentRect: { x: 284, y: 190, width: 90, height: 20 }, - delta: { x: 184, y: -70, width: 0, height: 0 }, - confidence: 90, - possibleTextMetricMismatch: false, - }, - { - text: 'Search', - baselineRect: { x: 100, y: 500, width: 90, height: 20 }, - currentRect: { x: 52, y: 560, width: 90, height: 20 }, - delta: { x: -48, y: 60, width: 0, height: 0 }, - confidence: 94, - possibleTextMetricMismatch: false, - }, - ]); - - assert.equal(clusters.length, 1); - assert.deepEqual(clusters[0]?.texts, ['Wi-Fi', 'Bluetooth']); - assert.deepEqual(clusters[0]?.xRange, { min: 184, max: 186 }); - assert.deepEqual(clusters[0]?.yRange, { min: -80, max: -70 }); -}); - -test('summarizeScreenshotOcr returns undefined when tesseract exits non-zero', async () => { - const dir = fs.mkdtempSync(path.join(os.tmpdir(), 'ocr-test-')); - const binDir = path.join(dir, 'bin'); - const fakeTesseract = path.join(binDir, 'tesseract'); - const originalPath = process.env.PATH; - fs.mkdirSync(binDir, { recursive: true }); - fs.writeFileSync(fakeTesseract, '#!/bin/sh\nexit 2\n'); - fs.chmodSync(fakeTesseract, 0o755); - process.env.PATH = `${binDir}${path.delimiter}${originalPath ?? ''}`; - - try { - const result = await summarizeScreenshotOcr({ - baselinePath: path.join(dir, 'baseline.png'), - currentPath: path.join(dir, 'current.png'), - width: 100, - height: 100, - }); - assert.equal(result, undefined); - } finally { - if (originalPath === undefined) delete process.env.PATH; - else process.env.PATH = originalPath; - fs.rmSync(dir, { recursive: true, force: true }); - } -}); diff --git a/src/screenshot-diff/__tests__/screenshot-diff.test.ts b/src/screenshot-diff/__tests__/screenshot-diff.test.ts index 9e8ea8c674..85818895f9 100644 --- a/src/screenshot-diff/__tests__/screenshot-diff.test.ts +++ b/src/screenshot-diff/__tests__/screenshot-diff.test.ts @@ -148,15 +148,6 @@ test('changed pixels are summarized into nearby diff regions', async () => { assert.equal(result.regions?.[0]?.differentPixels, 32); assert.equal(result.regions?.[0]?.shareOfDiffPercentage, 66.67); assert.deepEqual(result.regions?.[0]?.normalizedRect, { x: 5, y: 10, width: 30, height: 20 }); - assert.equal(result.regions?.[0]?.densityPercentage, 66.67); - assert.equal(result.regions?.[0]?.shape, 'horizontal-band'); - assert.equal(result.regions?.[0]?.size, 'large'); - assert.equal(result.regions?.[0]?.averageBaselineColorHex, '#000000'); - assert.equal(result.regions?.[0]?.averageCurrentColorHex, '#ffffff'); - assert.equal(result.regions?.[0]?.baselineLuminance, 0); - assert.equal(result.regions?.[0]?.currentLuminance, 255); - assert.equal(result.regions?.[0]?.location, 'top-left'); - assert.equal(result.regions?.[0]?.dominantChange, 'brighter'); assert.deepEqual(result.regions?.[1]?.rect, { x: 30, y: 15, width: 4, height: 4 }); const diffPng = PNG.sync.read(fs.readFileSync(diffOut)); @@ -310,8 +301,6 @@ test('dimension mismatch returns expected vs actual sizes', async () => { assert.equal(result.mismatchPercentage, 100); assert.equal(result.diffPath, undefined, 'diffPath should not be set for dimension mismatch'); assert.equal(result.regions, undefined); - assert.equal(result.ocr, undefined); - assert.equal(result.nonTextDeltas, undefined); assert.deepEqual(result.dimensionMismatch, { expected: { width: 10, height: 20 }, actual: { width: 15, height: 25 }, diff --git a/src/screenshot-diff/screenshot-diff-non-text.ts b/src/screenshot-diff/screenshot-diff-non-text.ts deleted file mode 100644 index 4f4912d291..0000000000 --- a/src/screenshot-diff/screenshot-diff-non-text.ts +++ /dev/null @@ -1,446 +0,0 @@ -import type { Rect } from '@agent-device/kernel/snapshot'; -import { findConnectedMaskComponents } from './screenshot-diff-components.ts'; -import type { ScreenshotOcrAnalysis, ScreenshotOcrBlock } from './screenshot-diff-ocr.ts'; -import type { ScreenshotDiffRegion } from './screenshot-diff-regions.ts'; -import { - clamp, - expandRect, - intersectArea, - rectCenter, - squaredDistance, - unionRects, - type ImageDimensions, -} from '../utils/screenshot-geometry.ts'; - -export type ScreenshotNonTextDelta = { - index: number; - regionIndex?: number; - slot: 'leading' | 'trailing' | 'background' | 'separator' | 'unknown'; - likelyKind: 'icon' | 'toggle' | 'chevron' | 'separator' | 'visual'; - rect: Rect; - nearestText?: string; -}; - -type NonTextKind = ScreenshotNonTextDelta['likelyKind'] | 'background'; - -const MAX_NON_TEXT_DELTAS = 12; -const OCR_MASK_PADDING_PX = 8; -const MIN_COMPONENT_PIXELS = 24; -const MIN_COMPONENT_SIDE = 3; -const MERGE_GAP_PX = 10; -const MIN_CONTENT_Y_RATIO = 0.08; -// Non-text hints classify residual geometry relative to the screenshot size. -// Aspect/density checks describe common UI glyph shapes rather than app-specific elements. -const SEPARATOR_MAX_THICKNESS_PX = 3; -const SEPARATOR_MIN_WIDTH_RATIO = 0.12; -const BACKGROUND_SLOT_WIDTH_RATIO = 0.4; -const UNKNOWN_BACKGROUND_SLOT_WIDTH_RATIO = 0.35; -const LARGE_RESIDUAL_WIDTH_RATIO = 0.25; -const LARGE_RESIDUAL_HEIGHT_RATIO = 0.06; -const TOGGLE_MIN_ASPECT_RATIO = 1.5; -const TOGGLE_MAX_ASPECT_RATIO = 3.8; -const TOGGLE_MIN_DENSITY_RATIO = 0.35; -const CHEVRON_MAX_WIDTH_RATIO = 0.06; -const CHEVRON_MAX_HEIGHT_RATIO = 0.04; -const ICON_MIN_ASPECT_RATIO = 0.55; -const ICON_MAX_ASPECT_RATIO = 1.8; -const LARGE_RESIDUAL_SCORE_PENALTY = -35; -const REGION_OVERLAP_SCORE = 20; -const MAX_PIXEL_COUNT_SCORE = 20; -const PIXELS_PER_SCORE_POINT = 200; -const KIND_SCORE = { - icon: 90, - toggle: 90, - chevron: 75, - separator: 45, - visual: 35, - background: 10, -} satisfies Record; -const SLOT_SCORE = { - leading: 20, - trailing: 20, - separator: 10, - unknown: 0, - background: -30, -} satisfies Record; - -type MutableComponent = { - minX: number; - minY: number; - maxX: number; - maxY: number; - differentPixels: number; -}; - -type ScoredNonTextDelta = Omit & { - likelyKind: NonTextKind; - score: number; -}; - -type OcrRow = { - rect: Rect; - blocks: ScreenshotOcrBlock[]; -}; - -export function summarizeNonTextDiffDeltas(params: { - diffMask: Uint8Array; - width: number; - height: number; - regions: ScreenshotDiffRegion[]; - ocr?: ScreenshotOcrAnalysis; - maxDeltas?: number; -}): ScreenshotNonTextDelta[] { - const maskedDiff = maskOcrText(params.diffMask, params.width, params.height, params.ocr); - const rawComponents = findConnectedComponents(maskedDiff, params.width, params.height); - const mergedComponents = mergeNearbyComponents(rawComponents, MERGE_GAP_PX); - const currentRows = groupOcrRows(params.ocr?.currentBlocksRaw ?? []); - const baselineRows = groupOcrRows(params.ocr?.baselineBlocksRaw ?? []); - return ( - mergedComponents - .filter(hasUsefulComponentSize) - .map((component) => toNonTextDelta(component, params, currentRows, baselineRows)) - // Status bars and top chrome tend to produce noisy residuals around time, - // signal, and battery text; changed regions still report that area. - .filter((delta) => delta.rect.y >= params.height * MIN_CONTENT_Y_RATIO) - .filter(hasAgentFacingKind) - .sort((left, right) => right.score - left.score) - .slice(0, Math.max(0, params.maxDeltas ?? MAX_NON_TEXT_DELTAS)) - .map((delta, index) => toPublicNonTextDelta(delta, index + 1)) - ); -} - -function maskOcrText( - diffMask: Uint8Array, - width: number, - height: number, - ocr: ScreenshotOcrAnalysis | undefined, -): Uint8Array { - const maskedDiff = new Uint8Array(diffMask); - if (!ocr) return maskedDiff; - for (const block of [...ocr.baselineBlocksRaw, ...ocr.currentBlocksRaw]) { - clearRect(maskedDiff, width, height, expandRect(block.rect, OCR_MASK_PADDING_PX)); - } - return maskedDiff; -} - -function findConnectedComponents( - mask: Uint8Array, - width: number, - height: number, -): MutableComponent[] { - return findConnectedMaskComponents({ - mask, - width, - height, - hooks: { - create: (pixelIndex) => createComponent(pixelIndex, width), - visit: (component, pixelIndex) => addPixelToComponent(component, pixelIndex, width), - }, - }); -} - -function createComponent(pixelIndex: number, width: number): MutableComponent { - const startX = pixelIndex % width; - const startY = Math.floor(pixelIndex / width); - return { - minX: startX, - minY: startY, - maxX: startX, - maxY: startY, - differentPixels: 0, - }; -} - -function addPixelToComponent(component: MutableComponent, pixelIndex: number, width: number): void { - const x = pixelIndex % width; - const y = Math.floor(pixelIndex / width); - component.minX = Math.min(component.minX, x); - component.minY = Math.min(component.minY, y); - component.maxX = Math.max(component.maxX, x); - component.maxY = Math.max(component.maxY, y); - component.differentPixels += 1; -} - -function mergeNearbyComponents(components: MutableComponent[], gapPx: number): MutableComponent[] { - const merged: MutableComponent[] = []; - for (const component of components.sort( - (left, right) => left.minY - right.minY || left.minX - right.minX, - )) { - const existing = merged.find((candidate) => componentsAreNear(candidate, component, gapPx)); - if (!existing) { - merged.push({ ...component }); - continue; - } - existing.minX = Math.min(existing.minX, component.minX); - existing.minY = Math.min(existing.minY, component.minY); - existing.maxX = Math.max(existing.maxX, component.maxX); - existing.maxY = Math.max(existing.maxY, component.maxY); - existing.differentPixels += component.differentPixels; - } - return merged; -} - -function toNonTextDelta( - component: MutableComponent, - params: { - width: number; - height: number; - regions: ScreenshotDiffRegion[]; - }, - currentRows: OcrRow[], - baselineRows: OcrRow[], -): ScoredNonTextDelta { - const rect = componentToRect(component); - const regionIndex = findContainingRegionIndex(rect, params.regions); - const textAnchor = findTextAnchor(rect, currentRows, baselineRows); - const slot = classifySlot(rect, textAnchor?.block.rect, params.width); - const likelyKind = classifyLikelyKind(rect, slot, component.differentPixels, params); - const scoreParams = { - ...(regionIndex ? { regionIndex } : {}), - slot, - likelyKind, - rect, - }; - return { - ...(regionIndex ? { regionIndex } : {}), - slot, - likelyKind, - rect, - ...(textAnchor ? { nearestText: cleanOcrAnchorText(textAnchor.block.text) } : {}), - score: scoreNonTextDelta(scoreParams, component.differentPixels, params), - }; -} - -function toPublicNonTextDelta( - delta: ScoredNonTextDelta & { likelyKind: ScreenshotNonTextDelta['likelyKind'] }, - index: number, -): ScreenshotNonTextDelta { - return { - index, - ...(delta.regionIndex ? { regionIndex: delta.regionIndex } : {}), - slot: delta.slot, - likelyKind: delta.likelyKind, - rect: delta.rect, - ...(delta.nearestText ? { nearestText: delta.nearestText } : {}), - }; -} - -function classifySlot( - rect: Rect, - nearestTextRect: Rect | undefined, - imageWidth: number, -): ScreenshotNonTextDelta['slot'] { - if ( - rect.height <= SEPARATOR_MAX_THICKNESS_PX && - rect.width >= imageWidth * SEPARATOR_MIN_WIDTH_RATIO - ) { - return 'separator'; - } - if (!nearestTextRect) { - if (rect.width >= imageWidth * BACKGROUND_SLOT_WIDTH_RATIO) return 'background'; - return 'unknown'; - } - if (rect.width >= imageWidth * BACKGROUND_SLOT_WIDTH_RATIO) return 'background'; - const rectCenterX = rect.x + rect.width / 2; - const textCenterX = nearestTextRect.x + nearestTextRect.width / 2; - if (rectCenterX < textCenterX - nearestTextRect.width / 2) return 'leading'; - if (rectCenterX > textCenterX + nearestTextRect.width / 2) return 'trailing'; - return rect.width >= imageWidth * UNKNOWN_BACKGROUND_SLOT_WIDTH_RATIO ? 'background' : 'unknown'; -} - -function classifyLikelyKind( - rect: Rect, - slot: ScreenshotNonTextDelta['slot'], - differentPixels: number, - image: ImageDimensions, -): NonTextKind { - const aspect = rect.width / rect.height; - const density = differentPixels / (rect.width * rect.height); - if (slot === 'separator') return 'separator'; - if (slot === 'background') return 'background'; - if ( - slot === 'trailing' && - aspect >= TOGGLE_MIN_ASPECT_RATIO && - aspect <= TOGGLE_MAX_ASPECT_RATIO && - density >= TOGGLE_MIN_DENSITY_RATIO - ) { - return 'toggle'; - } - if ( - slot === 'trailing' && - rect.width <= image.width * CHEVRON_MAX_WIDTH_RATIO && - rect.height <= image.height * CHEVRON_MAX_HEIGHT_RATIO - ) { - return 'chevron'; - } - if (slot === 'leading' && aspect >= ICON_MIN_ASPECT_RATIO && aspect <= ICON_MAX_ASPECT_RATIO) { - return 'icon'; - } - if (isLargeResidual(rect, image)) return 'background'; - return 'visual'; -} - -function hasAgentFacingKind( - delta: ScoredNonTextDelta, -): delta is ScoredNonTextDelta & { likelyKind: ScreenshotNonTextDelta['likelyKind'] } { - return delta.likelyKind !== 'background'; -} - -function scoreNonTextDelta( - delta: { - regionIndex?: number; - slot: ScreenshotNonTextDelta['slot']; - likelyKind: NonTextKind; - rect: Rect; - }, - differentPixels: number, - image: ImageDimensions, -): number { - const sizePenalty = isLargeResidual(delta.rect, image) ? LARGE_RESIDUAL_SCORE_PENALTY : 0; - const regionScore = delta.regionIndex ? REGION_OVERLAP_SCORE : 0; - return ( - KIND_SCORE[delta.likelyKind] + - SLOT_SCORE[delta.slot] + - regionScore + - sizePenalty + - Math.min(MAX_PIXEL_COUNT_SCORE, differentPixels / PIXELS_PER_SCORE_POINT) - ); -} - -function isLargeResidual(rect: Rect, image: ImageDimensions): boolean { - return ( - rect.width >= image.width * LARGE_RESIDUAL_WIDTH_RATIO || - rect.height >= image.height * LARGE_RESIDUAL_HEIGHT_RATIO - ); -} - -function findContainingRegionIndex( - rect: Rect, - regions: ScreenshotDiffRegion[], -): number | undefined { - let bestRegion: ScreenshotDiffRegion | undefined; - let bestOverlap = 0; - for (const region of regions) { - const overlap = intersectArea(rect, region.rect); - if (overlap <= bestOverlap) continue; - bestOverlap = overlap; - bestRegion = region; - } - return bestRegion?.index; -} - -function findTextAnchor( - rect: Rect, - currentRows: OcrRow[], - baselineRows: OcrRow[], -): { block: ScreenshotOcrBlock; distance: number } | undefined { - const currentRow = findOverlappingRow(rect, currentRows); - if (currentRow) return findNearestText(rect, currentRow.blocks); - const baselineRow = findOverlappingRow(rect, baselineRows); - return baselineRow ? findNearestText(rect, baselineRow.blocks) : undefined; -} - -function findOverlappingRow(rect: Rect, rows: OcrRow[]): OcrRow | undefined { - let bestRow: OcrRow | undefined; - let bestOverlap = 0; - for (const row of rows) { - const overlap = verticalOverlap(rect, row.rect); - if (overlap <= bestOverlap) continue; - bestOverlap = overlap; - bestRow = row; - } - return bestRow; -} - -function groupOcrRows(blocks: ScreenshotOcrBlock[]): OcrRow[] { - const rows: OcrRow[] = []; - for (const block of [...blocks].sort((left, right) => left.rect.y - right.rect.y)) { - const row = rows.find((candidate) => blocksShareRow(candidate.rect, block.rect)); - if (!row) { - rows.push({ rect: block.rect, blocks: [block] }); - continue; - } - row.blocks.push(block); - row.blocks.sort((left, right) => left.rect.x - right.rect.x); - row.rect = unionRects([row.rect, block.rect]); - } - return rows; -} - -function blocksShareRow(left: Rect, right: Rect): boolean { - const overlap = verticalOverlap(left, right); - if (overlap > 0) return true; - const centerDistance = Math.abs(rectCenter(left).y - rectCenter(right).y); - return centerDistance <= Math.max(left.height, right.height) * 0.5; -} - -function findNearestText( - rect: Rect, - textBlocks: ScreenshotOcrBlock[], -): { block: ScreenshotOcrBlock; distance: number } | undefined { - let nearest: { block: ScreenshotOcrBlock; distance: number } | undefined; - const center = rectCenter(rect); - for (const block of textBlocks) { - const distance = Math.sqrt(squaredDistance(center, rectCenter(block.rect))); - if (nearest && distance >= nearest.distance) continue; - nearest = { block, distance }; - } - return nearest; -} - -function cleanOcrAnchorText(text: string): string { - return text - .trim() - .replace(/^[^\p{L}\p{N}]+/u, '') - .replace(/^\p{L}\s+/u, ''); -} - -function hasUsefulComponentSize(component: MutableComponent): boolean { - const rect = componentToRect(component); - return ( - component.differentPixels >= MIN_COMPONENT_PIXELS && - rect.width >= MIN_COMPONENT_SIDE && - rect.height >= MIN_COMPONENT_SIDE - ); -} - -function componentToRect(component: MutableComponent): Rect { - return { - x: component.minX, - y: component.minY, - width: component.maxX - component.minX + 1, - height: component.maxY - component.minY + 1, - }; -} - -function clearRect(mask: Uint8Array, width: number, height: number, rect: Rect): void { - const minX = clamp(Math.floor(rect.x), 0, width - 1); - const minY = clamp(Math.floor(rect.y), 0, height - 1); - const maxX = clamp(Math.ceil(rect.x + rect.width), 0, width); - const maxY = clamp(Math.ceil(rect.y + rect.height), 0, height); - for (let y = minY; y < maxY; y += 1) { - for (let x = minX; x < maxX; x += 1) { - mask[y * width + x] = 0; - } - } -} - -function componentsAreNear( - left: MutableComponent, - right: MutableComponent, - gapPx: number, -): boolean { - return ( - left.minX - gapPx <= right.maxX && - right.minX - gapPx <= left.maxX && - left.minY - gapPx <= right.maxY && - right.minY - gapPx <= left.maxY - ); -} - -function verticalOverlap(left: Rect, right: Rect): number { - return Math.max( - 0, - Math.min(left.y + left.height, right.y + right.height) - Math.max(left.y, right.y), - ); -} diff --git a/src/screenshot-diff/screenshot-diff-ocr.ts b/src/screenshot-diff/screenshot-diff-ocr.ts deleted file mode 100644 index cf4fb61a12..0000000000 --- a/src/screenshot-diff/screenshot-diff-ocr.ts +++ /dev/null @@ -1,376 +0,0 @@ -import type { Rect } from '@agent-device/kernel/snapshot'; -import { runCmd, whichCmd } from '../utils/exec.ts'; - -export type MovementRange = { min: number; max: number }; -import { - normalizedRect, - rectCenter, - squaredDistance, - unionRects, - type NormalizedPoint, - type NormalizedRect, -} from '../utils/screenshot-geometry.ts'; - -export type ScreenshotOcrBlock = { - text: string; - confidence: number; - rect: Rect; - normalizedRect: NormalizedRect; -}; - -export type ScreenshotOcrTextMatch = { - text: string; - baselineRect: Rect; - currentRect: Rect; - delta: Rect; - confidence: number; - possibleTextMetricMismatch: boolean; -}; - -export type ScreenshotOcrMovementCluster = { - texts: string[]; - xRange: MovementRange; - yRange: MovementRange; -}; - -export type ScreenshotOcrSummary = { - provider: 'tesseract'; - baselineBlocks: number; - currentBlocks: number; - matches: ScreenshotOcrTextMatch[]; - movementClusters?: ScreenshotOcrMovementCluster[]; -}; - -export type ScreenshotOcrAnalysis = ScreenshotOcrSummary & { - baselineBlocksRaw: ScreenshotOcrBlock[]; - currentBlocksRaw: ScreenshotOcrBlock[]; -}; - -type TesseractWord = { - key: string; - text: string; - confidence: number; - rect: Rect; -}; - -const OCR_TIMEOUT_MS = 10_000; -const MAX_OCR_MATCHES = 12; -const MAX_MOVEMENT_CLUSTERS = 4; -const MIN_CLUSTERED_MATCHES = 2; -const MOVEMENT_CLUSTER_MAX_X_SPREAD_PX = 32; -const MOVEMENT_CLUSTER_MAX_Y_SPREAD_PX = 60; -// OCR text matching uses small generic movement/shape thresholds; the fixed gap -// is only a floor before falling back to word-height-relative spacing. -const MIN_MEANINGFUL_DELTA_PX = 2; -const MIN_SEGMENT_GAP_PX = 48; -const TEXT_WIDTH_MISMATCH_RATIO = 0.08; -const TEXT_HEIGHT_MISMATCH_RATIO = 0.12; - -export async function summarizeScreenshotOcr(params: { - baselinePath: string; - currentPath: string; - width: number; - height: number; -}): Promise { - if (!(await whichCmd('tesseract'))) return undefined; - - try { - const [baselineResult, currentResult] = await Promise.all([ - runTesseractTsv(params.baselinePath), - runTesseractTsv(params.currentPath), - ]); - if (baselineResult.exitCode !== 0 || currentResult.exitCode !== 0) return undefined; - - const baselineBlocks = parseTesseractTsv(baselineResult.stdout, params.width, params.height); - const currentBlocks = parseTesseractTsv(currentResult.stdout, params.width, params.height); - const matches = matchOcrBlocks(baselineBlocks, currentBlocks); - const movementClusters = summarizeOcrMovementClusters(matches); - if (baselineBlocks.length === 0 && currentBlocks.length === 0) return undefined; - - return { - provider: 'tesseract', - baselineBlocks: baselineBlocks.length, - currentBlocks: currentBlocks.length, - baselineBlocksRaw: baselineBlocks, - currentBlocksRaw: currentBlocks, - matches, - ...(movementClusters.length > 0 ? { movementClusters } : {}), - }; - } catch { - return undefined; - } -} - -export function parseTesseractTsv( - tsv: string, - imageWidth: number, - imageHeight: number, -): ScreenshotOcrBlock[] { - const [headerLine, ...lines] = tsv.split(/\r?\n/); - if (!headerLine) return []; - - const headers = headerLine.split('\t'); - const indexByName = new Map(headers.map((header, index) => [header, index])); - const words: TesseractWord[] = []; - for (const line of lines) { - if (!line.trim()) continue; - const values = line.split('\t'); - const level = readTsvNumber(values, indexByName, 'level'); - const rawText = readTsvString(values, indexByName, 'text').trim(); - const confidence = readTsvNumber(values, indexByName, 'conf'); - // Tesseract TSV uses level=5 for word rows; higher-level rows are page/block/line containers. - if (level !== 5 || !isMeaningfulText(rawText) || confidence < 0) continue; - - const left = readTsvNumber(values, indexByName, 'left'); - const top = readTsvNumber(values, indexByName, 'top'); - const width = readTsvNumber(values, indexByName, 'width'); - const height = readTsvNumber(values, indexByName, 'height'); - if (width <= 0 || height <= 0) continue; - - words.push({ - key: [ - readTsvString(values, indexByName, 'page_num'), - readTsvString(values, indexByName, 'block_num'), - readTsvString(values, indexByName, 'par_num'), - readTsvString(values, indexByName, 'line_num'), - ].join(':'), - text: rawText, - confidence, - rect: { x: left, y: top, width, height }, - }); - } - - const wordsByLine = new Map(); - for (const word of words) { - const existing = wordsByLine.get(word.key); - if (existing) existing.push(word); - else wordsByLine.set(word.key, [word]); - } - - return Array.from(wordsByLine.values()) - .flatMap((lineWords) => splitLineWordsIntoSegments(lineWords)) - .map((segmentWords) => toOcrBlock(segmentWords, imageWidth, imageHeight)) - .filter((block): block is ScreenshotOcrBlock => block !== null); -} - -export function matchOcrBlocks( - baselineBlocks: ScreenshotOcrBlock[], - currentBlocks: ScreenshotOcrBlock[], -): ScreenshotOcrTextMatch[] { - const usedCurrent = new Set(); - const matches: ScreenshotOcrTextMatch[] = []; - - for (const baselineBlock of baselineBlocks) { - const normalizedText = normalizeTextForMatching(baselineBlock.text); - const currentIndex = findBestCurrentMatch( - baselineBlock, - normalizedText, - currentBlocks, - usedCurrent, - ); - if (currentIndex === null) continue; - usedCurrent.add(currentIndex); - - const currentBlock = currentBlocks[currentIndex]!; - const match = toOcrTextMatch(baselineBlock, currentBlock); - if (!hasMeaningfulOcrDelta(match)) continue; - matches.push(match); - } - - return matches - .sort((left, right) => scoreOcrMatch(right) - scoreOcrMatch(left)) - .slice(0, MAX_OCR_MATCHES); -} - -function runTesseractTsv(imagePath: string): ReturnType { - return runCmd('tesseract', [imagePath, 'stdout', '-l', 'eng', 'tsv'], { - allowFailure: true, - timeoutMs: OCR_TIMEOUT_MS, - }); -} - -function toOcrBlock( - words: TesseractWord[], - imageWidth: number, - imageHeight: number, -): ScreenshotOcrBlock | null { - if (words.length === 0) return null; - const sortedWords = [...words].sort((left, right) => left.rect.x - right.rect.x); - const rect = unionRects(sortedWords.map((word) => word.rect)); - const confidence = Math.round(average(sortedWords.map((word) => word.confidence)) * 100) / 100; - return { - text: sortedWords.map((word) => word.text).join(' '), - confidence, - rect, - normalizedRect: normalizedRect({ - x: roundPercentage(rect.x / imageWidth), - y: roundPercentage(rect.y / imageHeight), - width: roundPercentage(rect.width / imageWidth), - height: roundPercentage(rect.height / imageHeight), - }), - }; -} - -function splitLineWordsIntoSegments(words: TesseractWord[]): TesseractWord[][] { - const sortedWords = [...words].sort((left, right) => left.rect.x - right.rect.x); - const segments: TesseractWord[][] = []; - let currentSegment: TesseractWord[] = []; - for (const word of sortedWords) { - const previousWord = currentSegment.at(-1); - if (!previousWord) { - currentSegment.push(word); - continue; - } - - const gap = word.rect.x - (previousWord.rect.x + previousWord.rect.width); - const height = Math.max(previousWord.rect.height, word.rect.height); - if (gap > Math.max(MIN_SEGMENT_GAP_PX, height * 2.5)) { - segments.push(currentSegment); - currentSegment = [word]; - continue; - } - currentSegment.push(word); - } - if (currentSegment.length > 0) segments.push(currentSegment); - return segments; -} - -function findBestCurrentMatch( - baselineBlock: ScreenshotOcrBlock, - normalizedText: string, - currentBlocks: ScreenshotOcrBlock[], - usedCurrent: Set, -): number | null { - let bestIndex: number | null = null; - let bestDistance = Number.POSITIVE_INFINITY; - for (let index = 0; index < currentBlocks.length; index += 1) { - if (usedCurrent.has(index)) continue; - const currentBlock = currentBlocks[index]!; - if (normalizeTextForMatching(currentBlock.text) !== normalizedText) continue; - // Centers are in normalized [0..100] space; compare like-for-like. - const baselineCenter: NormalizedPoint = rectCenter(baselineBlock.normalizedRect); - const currentCenter: NormalizedPoint = rectCenter(currentBlock.normalizedRect); - const distance = squaredDistance(baselineCenter, currentCenter); - if (distance >= bestDistance) continue; - bestIndex = index; - bestDistance = distance; - } - return bestIndex; -} - -function toOcrTextMatch( - baselineBlock: ScreenshotOcrBlock, - currentBlock: ScreenshotOcrBlock, -): ScreenshotOcrTextMatch { - const delta = { - x: currentBlock.rect.x - baselineBlock.rect.x, - y: currentBlock.rect.y - baselineBlock.rect.y, - width: currentBlock.rect.width - baselineBlock.rect.width, - height: currentBlock.rect.height - baselineBlock.rect.height, - }; - const widthRatio = roundRatio(currentBlock.rect.width / baselineBlock.rect.width); - const heightRatio = roundRatio(currentBlock.rect.height / baselineBlock.rect.height); - const possibleTextMetricMismatch = - Math.abs(widthRatio - 1) >= TEXT_WIDTH_MISMATCH_RATIO || - Math.abs(heightRatio - 1) >= TEXT_HEIGHT_MISMATCH_RATIO; - return { - text: baselineBlock.text, - baselineRect: baselineBlock.rect, - currentRect: currentBlock.rect, - delta, - confidence: Math.round(Math.min(baselineBlock.confidence, currentBlock.confidence) * 100) / 100, - possibleTextMetricMismatch, - }; -} - -function hasMeaningfulOcrDelta(match: ScreenshotOcrTextMatch): boolean { - return ( - Math.abs(match.delta.x) >= MIN_MEANINGFUL_DELTA_PX || - Math.abs(match.delta.y) >= MIN_MEANINGFUL_DELTA_PX || - Math.abs(match.delta.width) >= MIN_MEANINGFUL_DELTA_PX || - Math.abs(match.delta.height) >= MIN_MEANINGFUL_DELTA_PX || - match.possibleTextMetricMismatch - ); -} - -function scoreOcrMatch(match: ScreenshotOcrTextMatch): number { - return ( - Math.abs(match.delta.x) + - Math.abs(match.delta.y) + - Math.abs(match.delta.width) + - Math.abs(match.delta.height) + - (match.possibleTextMetricMismatch ? 25 : 0) - ); -} - -export function summarizeOcrMovementClusters( - matches: ScreenshotOcrTextMatch[], -): ScreenshotOcrMovementCluster[] { - const clusters: ScreenshotOcrTextMatch[][] = []; - for (const match of [...matches].sort( - (left, right) => left.currentRect.y - right.currentRect.y, - )) { - const cluster = clusters.find( - (candidate) => - Math.abs(match.delta.x - average(candidate.map((item) => item.delta.x))) <= - MOVEMENT_CLUSTER_MAX_X_SPREAD_PX, - ); - if (cluster) cluster.push(match); - else clusters.push([match]); - } - - return clusters - .filter((cluster) => cluster.length >= MIN_CLUSTERED_MATCHES) - .map(toMovementCluster) - .filter( - (cluster) => cluster.yRange.max - cluster.yRange.min <= MOVEMENT_CLUSTER_MAX_Y_SPREAD_PX, - ) - .sort((left, right) => scoreMovementCluster(right) - scoreMovementCluster(left)) - .slice(0, MAX_MOVEMENT_CLUSTERS); -} - -function toMovementCluster(matches: ScreenshotOcrTextMatch[]): ScreenshotOcrMovementCluster { - const xDeltas = matches.map((match) => match.delta.x); - const yDeltas = matches.map((match) => match.delta.y); - return { - texts: matches.map((match) => match.text), - xRange: { min: Math.min(...xDeltas), max: Math.max(...xDeltas) }, - yRange: { min: Math.min(...yDeltas), max: Math.max(...yDeltas) }, - }; -} - -function scoreMovementCluster(cluster: ScreenshotOcrMovementCluster): number { - const averageX = (cluster.xRange.min + cluster.xRange.max) / 2; - const averageY = (cluster.yRange.min + cluster.yRange.max) / 2; - return Math.abs(averageX) * 2 + Math.abs(averageY); -} - -function readTsvString(values: string[], indexByName: Map, name: string): string { - const index = indexByName.get(name); - return index === undefined ? '' : (values[index] ?? ''); -} - -function readTsvNumber(values: string[], indexByName: Map, name: string): number { - const value = Number(readTsvString(values, indexByName, name)); - return Number.isFinite(value) ? value : 0; -} - -function isMeaningfulText(text: string): boolean { - return /[\p{L}\p{N}]/u.test(text); -} - -function normalizeTextForMatching(text: string): string { - return text.trim().replace(/\s+/g, ' ').toLowerCase(); -} - -function average(values: number[]): number { - return values.reduce((sum, value) => sum + value, 0) / values.length; -} - -function roundPercentage(ratio: number): number { - return Math.round(ratio * 100 * 100) / 100; -} - -function roundRatio(ratio: number): number { - return Math.round(ratio * 1000) / 1000; -} diff --git a/src/screenshot-diff/screenshot-diff-region-split.ts b/src/screenshot-diff/screenshot-diff-region-split.ts index bb537b37de..a6b67274a6 100644 --- a/src/screenshot-diff/screenshot-diff-region-split.ts +++ b/src/screenshot-diff/screenshot-diff-region-split.ts @@ -1,4 +1,3 @@ -import type { PNG } from '../utils/png.ts'; import type { MutableDiffRegion } from './screenshot-diff-region-types.ts'; // Region splitting is based on screen-relative heights so it works on phone, @@ -14,15 +13,11 @@ const ROW_SMOOTHING_RADIUS = 3; export function splitLargeDiffRegions( regions: MutableDiffRegion[], - params: { diffMask: Uint8Array; baseline: PNG; current: PNG }, + params: { diffMask: Uint8Array; width: number; height: number }, ): MutableDiffRegion[] { return regions.flatMap((region) => - shouldSplitRegion(region, params.baseline.width, params.baseline.height) - ? splitRegionByHorizontalDensity( - region, - params, - minSplitSegmentHeight(params.baseline.height), - ) + shouldSplitRegion(region, params.width, params.height) + ? splitRegionByHorizontalDensity(region, params, minSplitSegmentHeight(params.height)) : [region], ); } @@ -42,10 +37,10 @@ function shouldSplitRegion( function splitRegionByHorizontalDensity( region: MutableDiffRegion, - params: { diffMask: Uint8Array; baseline: PNG; current: PNG }, + params: { diffMask: Uint8Array; width: number }, minSegmentHeight: number, ): MutableDiffRegion[] { - const rowCounts = measureRowDiffCounts(region, params.diffMask, params.baseline.width); + const rowCounts = measureRowDiffCounts(region, params.diffMask, params.width); const smoothed = smoothCounts(rowCounts); const lowDensityBands = findLowDensityBands( smoothed, @@ -146,15 +141,15 @@ function buildRegionSlice( region: MutableDiffRegion, minY: number, maxY: number, - params: { diffMask: Uint8Array; baseline: PNG; current: PNG }, + params: { diffMask: Uint8Array; width: number }, ): MutableDiffRegion | null { let slice: MutableDiffRegion | null = null; for (let y = minY; y <= maxY; y += 1) { for (let x = region.minX; x <= region.maxX; x += 1) { - const pixelIndex = y * params.baseline.width + x; + const pixelIndex = y * params.width + x; if (params.diffMask[pixelIndex] !== 1) continue; slice ??= createEmptyRegion(x, y); - addPixelToSlice(slice, pixelIndex, x, y, params.baseline, params.current); + addPixelToSlice(slice, x, y); } } return slice; @@ -167,33 +162,13 @@ function createEmptyRegion(x: number, y: number): MutableDiffRegion { maxX: x, maxY: y, differentPixels: 0, - baselineRed: 0, - baselineGreen: 0, - baselineBlue: 0, - currentRed: 0, - currentGreen: 0, - currentBlue: 0, }; } -function addPixelToSlice( - slice: MutableDiffRegion, - pixelIndex: number, - x: number, - y: number, - baseline: PNG, - current: PNG, -): void { - const dataIndex = pixelIndex * 4; +function addPixelToSlice(slice: MutableDiffRegion, x: number, y: number): void { slice.minX = Math.min(slice.minX, x); slice.minY = Math.min(slice.minY, y); slice.maxX = Math.max(slice.maxX, x); slice.maxY = Math.max(slice.maxY, y); slice.differentPixels += 1; - slice.baselineRed += baseline.data[dataIndex]!; - slice.baselineGreen += baseline.data[dataIndex + 1]!; - slice.baselineBlue += baseline.data[dataIndex + 2]!; - slice.currentRed += current.data[dataIndex]!; - slice.currentGreen += current.data[dataIndex + 1]!; - slice.currentBlue += current.data[dataIndex + 2]!; } diff --git a/src/screenshot-diff/screenshot-diff-region-types.ts b/src/screenshot-diff/screenshot-diff-region-types.ts index 3e42e45a23..645e06f290 100644 --- a/src/screenshot-diff/screenshot-diff-region-types.ts +++ b/src/screenshot-diff/screenshot-diff-region-types.ts @@ -4,10 +4,4 @@ export type MutableDiffRegion = { maxX: number; maxY: number; differentPixels: number; - baselineRed: number; - baselineGreen: number; - baselineBlue: number; - currentRed: number; - currentGreen: number; - currentBlue: number; }; diff --git a/src/screenshot-diff/screenshot-diff-regions.ts b/src/screenshot-diff/screenshot-diff-regions.ts index 1a45d6a37f..ade8165db4 100644 --- a/src/screenshot-diff/screenshot-diff-regions.ts +++ b/src/screenshot-diff/screenshot-diff-regions.ts @@ -1,31 +1,15 @@ -import type { PNG } from '../utils/png.ts'; import type { Rect } from '@agent-device/kernel/snapshot'; import { normalizedRect, type NormalizedRect } from '../utils/screenshot-geometry.ts'; import { findConnectedMaskComponents } from './screenshot-diff-components.ts'; import { splitLargeDiffRegions } from './screenshot-diff-region-split.ts'; import type { MutableDiffRegion } from './screenshot-diff-region-types.ts'; -type ScreenshotDiffColor = { - r: number; - g: number; - b: number; -}; - export type ScreenshotDiffRegion = { index: number; rect: Rect; normalizedRect: NormalizedRect; differentPixels: number; shareOfDiffPercentage: number; - densityPercentage: number; - shape: 'compact' | 'horizontal-band' | 'vertical-band' | 'large-area'; - size: 'small' | 'medium' | 'large'; - location: string; - averageBaselineColorHex: string; - averageCurrentColorHex: string; - baselineLuminance: number; - currentLuminance: number; - dominantChange: 'brighter' | 'darker' | 'color-shift' | 'mixed'; currentOverlayMatches?: ScreenshotDiffRegionOverlayMatch[]; }; @@ -39,20 +23,10 @@ export type ScreenshotDiffRegionOverlayMatch = { const DEFAULT_MAX_DIFF_REGIONS = 8; const REGION_MERGE_GAP_PX = 12; const MAX_REGIONS_TO_MERGE = 2000; -// These region labels are coarse, screen-relative buckets for agent guidance, -// not tuned to a specific screenshot size or app layout. -const DOMINANT_CHANGE_MIN_CHANNEL_DELTA = 12; -const LARGE_AREA_MIN_WIDTH_RATIO = 0.55; -const LARGE_AREA_MIN_HEIGHT_RATIO = 0.12; -const BAND_MIN_ASPECT_RATIO = 2.5; -const LARGE_REGION_MIN_AREA_RATIO = 0.04; -const MEDIUM_REGION_MIN_AREA_RATIO = 0.01; - export function summarizeDiffRegions(params: { diffMask: Uint8Array; - baseline: PNG; - current: PNG; - totalPixels: number; + width: number; + height: number; differentPixels: number; }): ScreenshotDiffRegion[] { const rawRegions = findConnectedDiffRegions(params); @@ -74,9 +48,8 @@ export function summarizeDiffRegions(params: { .slice(0, DEFAULT_MAX_DIFF_REGIONS) .map((region, index) => toScreenshotDiffRegion(region, index + 1, { - width: params.baseline.width, - height: params.baseline.height, - totalPixels: params.totalPixels, + width: params.width, + height: params.height, differentPixels: params.differentPixels, }), ); @@ -84,18 +57,17 @@ export function summarizeDiffRegions(params: { function findConnectedDiffRegions(params: { diffMask: Uint8Array; - baseline: PNG; - current: PNG; + width: number; + height: number; }): MutableDiffRegion[] { - const { diffMask, baseline, current } = params; - const { width, height } = baseline; + const { diffMask, width, height } = params; return findConnectedMaskComponents({ mask: diffMask, width, height, hooks: { create: (pixelIndex) => createDiffRegion(pixelIndex, width), - visit: (region, pixelIndex) => addPixelToRegion(region, pixelIndex, width, baseline, current), + visit: (region, pixelIndex) => addPixelToRegion(region, pixelIndex, width), }, }); } @@ -109,36 +81,17 @@ function createDiffRegion(pixelIndex: number, width: number): MutableDiffRegion maxX: startX, maxY: startY, differentPixels: 0, - baselineRed: 0, - baselineGreen: 0, - baselineBlue: 0, - currentRed: 0, - currentGreen: 0, - currentBlue: 0, }; } -function addPixelToRegion( - region: MutableDiffRegion, - pixelIndex: number, - width: number, - baseline: PNG, - current: PNG, -): void { +function addPixelToRegion(region: MutableDiffRegion, pixelIndex: number, width: number): void { const x = pixelIndex % width; const y = Math.floor(pixelIndex / width); - const dataIndex = pixelIndex * 4; region.minX = Math.min(region.minX, x); region.minY = Math.min(region.minY, y); region.maxX = Math.max(region.maxX, x); region.maxY = Math.max(region.maxY, y); region.differentPixels += 1; - region.baselineRed += baseline.data[dataIndex]!; - region.baselineGreen += baseline.data[dataIndex + 1]!; - region.baselineBlue += baseline.data[dataIndex + 2]!; - region.currentRed += current.data[dataIndex]!; - region.currentGreen += current.data[dataIndex + 1]!; - region.currentBlue += current.data[dataIndex + 2]!; } function mergeNearbyRegions(regions: MutableDiffRegion[], gapPx: number): MutableDiffRegion[] { @@ -173,18 +126,12 @@ function mergeRegionInto(target: MutableDiffRegion, source: MutableDiffRegion): target.maxX = Math.max(target.maxX, source.maxX); target.maxY = Math.max(target.maxY, source.maxY); target.differentPixels += source.differentPixels; - target.baselineRed += source.baselineRed; - target.baselineGreen += source.baselineGreen; - target.baselineBlue += source.baselineBlue; - target.currentRed += source.currentRed; - target.currentGreen += source.currentGreen; - target.currentBlue += source.currentBlue; } function toScreenshotDiffRegion( region: MutableDiffRegion, index: number, - image: { width: number; height: number; totalPixels: number; differentPixels: number }, + image: { width: number; height: number; differentPixels: number }, ): ScreenshotDiffRegion { const rect = { x: region.minX, @@ -192,30 +139,6 @@ function toScreenshotDiffRegion( width: region.maxX - region.minX + 1, height: region.maxY - region.minY + 1, }; - const center = { - x: Math.round(region.minX + rect.width / 2), - y: Math.round(region.minY + rect.height / 2), - }; - const averageBaselineColor = averageRegionColor( - region.baselineRed, - region.baselineGreen, - region.baselineBlue, - region.differentPixels, - ); - const averageCurrentColor = averageRegionColor( - region.currentRed, - region.currentGreen, - region.currentBlue, - region.differentPixels, - ); - const regionArea = rect.width * rect.height; - const densityPercentage = roundPercentage(region.differentPixels / regionArea); - const baselineLuminance = Math.round(luminance(averageBaselineColor)); - const currentLuminance = Math.round(luminance(averageCurrentColor)); - const shape = describeRegionShape(rect, image.width, image.height); - const size = describeRegionSize(regionArea, image.totalPixels); - const dominantChange = describeDominantChange(averageBaselineColor, averageCurrentColor); - const location = describeRegionLocation(center, image.width, image.height); return { index, rect, @@ -227,97 +150,9 @@ function toScreenshotDiffRegion( }), differentPixels: region.differentPixels, shareOfDiffPercentage: roundPercentage(region.differentPixels / image.differentPixels), - densityPercentage, - shape, - size, - location, - averageBaselineColorHex: toHexColor(averageBaselineColor), - averageCurrentColorHex: toHexColor(averageCurrentColor), - baselineLuminance, - currentLuminance, - dominantChange, }; } -function averageRegionColor( - red: number, - green: number, - blue: number, - pixels: number, -): ScreenshotDiffColor { - return { - r: Math.round(red / pixels), - g: Math.round(green / pixels), - b: Math.round(blue / pixels), - }; -} - -function describeRegionLocation( - center: { x: number; y: number }, - width: number, - height: number, -): string { - const horizontal = - center.x < width / 3 ? 'left' : center.x > (width * 2) / 3 ? 'right' : 'center'; - const vertical = - center.y < height / 3 ? 'top' : center.y > (height * 2) / 3 ? 'bottom' : 'middle'; - return horizontal === 'center' && vertical === 'middle' ? 'center' : `${vertical}-${horizontal}`; -} - -function describeDominantChange( - baseline: ScreenshotDiffColor, - current: ScreenshotDiffColor, -): ScreenshotDiffRegion['dominantChange'] { - const baselineLuminance = luminance(baseline); - const currentLuminance = luminance(current); - const luminanceDelta = currentLuminance - baselineLuminance; - if (Math.abs(luminanceDelta) >= DOMINANT_CHANGE_MIN_CHANNEL_DELTA) { - return luminanceDelta > 0 ? 'brighter' : 'darker'; - } - - const maxChannelDelta = Math.max( - Math.abs(current.r - baseline.r), - Math.abs(current.g - baseline.g), - Math.abs(current.b - baseline.b), - ); - return maxChannelDelta >= DOMINANT_CHANGE_MIN_CHANNEL_DELTA ? 'color-shift' : 'mixed'; -} - -function describeRegionShape( - rect: { width: number; height: number }, - imageWidth: number, - imageHeight: number, -): ScreenshotDiffRegion['shape'] { - if ( - rect.width >= imageWidth * LARGE_AREA_MIN_WIDTH_RATIO && - rect.height >= imageHeight * LARGE_AREA_MIN_HEIGHT_RATIO - ) { - return 'large-area'; - } - if (rect.width >= rect.height * BAND_MIN_ASPECT_RATIO) return 'horizontal-band'; - if (rect.height >= rect.width * BAND_MIN_ASPECT_RATIO) return 'vertical-band'; - return 'compact'; -} - -function describeRegionSize(regionArea: number, totalPixels: number): ScreenshotDiffRegion['size'] { - const areaRatio = regionArea / totalPixels; - if (areaRatio >= LARGE_REGION_MIN_AREA_RATIO) return 'large'; - if (areaRatio >= MEDIUM_REGION_MIN_AREA_RATIO) return 'medium'; - return 'small'; -} - -function luminance(color: ScreenshotDiffColor): number { - return color.r * 0.2126 + color.g * 0.7152 + color.b * 0.0722; -} - -function toHexColor(color: ScreenshotDiffColor): string { - return `#${toHexChannel(color.r)}${toHexChannel(color.g)}${toHexChannel(color.b)}`; -} - -function toHexChannel(value: number): string { - return value.toString(16).padStart(2, '0'); -} - function roundPercentage(ratio: number): number { return Math.round(ratio * 100 * 100) / 100; } diff --git a/src/screenshot-diff/screenshot-diff.ts b/src/screenshot-diff/screenshot-diff.ts index fdd00c7230..7e5263e0d1 100644 --- a/src/screenshot-diff/screenshot-diff.ts +++ b/src/screenshot-diff/screenshot-diff.ts @@ -8,11 +8,6 @@ import { encodePngAsync, } from '../utils/png-worker-client.ts'; import { annotateDiffRegions } from './screenshot-diff-region-overlay.ts'; -import { - summarizeNonTextDiffDeltas, - type ScreenshotNonTextDelta, -} from './screenshot-diff-non-text.ts'; -import { summarizeScreenshotOcr, type ScreenshotOcrSummary } from './screenshot-diff-ocr.ts'; import { summarizeDiffRegions, type ScreenshotDiffRegion } from './screenshot-diff-regions.ts'; import type { ImageDimensions } from '../utils/screenshot-geometry.ts'; @@ -31,8 +26,6 @@ export type ScreenshotDiffResult = { regions?: ScreenshotDiffRegion[]; currentOverlayPath?: string; currentOverlayRefCount?: number; - ocr?: ScreenshotOcrSummary; - nonTextDeltas?: ScreenshotNonTextDelta[]; }; export type ScreenshotDiffOptions = { @@ -104,9 +97,8 @@ export async function compareScreenshots( differentPixels > 0 ? summarizeDiffRegions({ diffMask, - baseline, - current, - totalPixels, + width: baseline.width, + height: baseline.height, differentPixels, }) : []; @@ -120,38 +112,6 @@ export async function compareScreenshots( await removeStaleDiffOutput(options.outputPath); } - const ocrAnalysis = - differentPixels > 0 - ? await summarizeScreenshotOcr({ - baselinePath, - currentPath, - width: baseline.width, - height: baseline.height, - }) - : undefined; - const shouldIncludeOcr = - ocrAnalysis && - (ocrAnalysis.matches.length > 0 || (ocrAnalysis.movementClusters?.length ?? 0) > 0); - const ocr = shouldIncludeOcr - ? { - provider: ocrAnalysis.provider, - baselineBlocks: ocrAnalysis.baselineBlocks, - currentBlocks: ocrAnalysis.currentBlocks, - matches: ocrAnalysis.matches, - ...(ocrAnalysis.movementClusters ? { movementClusters: ocrAnalysis.movementClusters } : {}), - } - : undefined; - const nonTextDeltas = - differentPixels > 0 && ocrAnalysis - ? summarizeNonTextDiffDeltas({ - diffMask, - width: baseline.width, - height: baseline.height, - regions, - ocr: ocrAnalysis, - }) - : []; - // Round to 2 decimal places: multiply percentage by 100 before rounding, // then divide back. e.g. 0.12345 → 12.345% → round(1234.5)/100 → 12.35% const mismatchPercentage = @@ -160,8 +120,6 @@ export async function compareScreenshots( return { ...(differentPixels > 0 && diffOutputPath ? { diffPath: diffOutputPath } : {}), ...(regions.length > 0 ? { regions } : {}), - ...(ocr ? { ocr } : {}), - ...(nonTextDeltas.length > 0 ? { nonTextDeltas } : {}), totalPixels, differentPixels, mismatchPercentage, diff --git a/src/utils/__tests__/output.test.ts b/src/utils/__tests__/output.test.ts index a65d8ea018..616b61a32b 100644 --- a/src/utils/__tests__/output.test.ts +++ b/src/utils/__tests__/output.test.ts @@ -1577,15 +1577,6 @@ test('formatScreenshotDiffText renders mismatch with pixel counts without color' normalizedRect: normalizedRect({ x: 10, y: 20, width: 100, height: 40 }), differentPixels: 350, shareOfDiffPercentage: 70, - densityPercentage: 8.75, - shape: 'horizontal-band', - size: 'medium', - location: 'top-left', - averageBaselineColorHex: '#141414', - averageCurrentColorHex: '#dcdcdc', - baselineLuminance: 20, - currentLuminance: 220, - dominantChange: 'brighter', currentOverlayMatches: [ { ref: 'e1', @@ -1596,45 +1587,6 @@ test('formatScreenshotDiffText renders mismatch with pixel counts without color' ], }, ], - ocr: { - provider: 'tesseract', - baselineBlocks: 2, - currentBlocks: 2, - matches: [ - { - text: 'Wi-Fi', - baselineRect: { x: 120, y: 320, width: 60, height: 22 }, - currentRect: { x: 130, y: 332, width: 70, height: 22 }, - delta: { x: 10, y: 12, width: 10, height: 0 }, - confidence: 94, - possibleTextMetricMismatch: true, - }, - ], - movementClusters: [ - { - texts: ['Wi-Fi', 'Bluetooth'], - xRange: { min: 10, max: 12 }, - yRange: { min: 10, max: 14 }, - }, - ], - }, - nonTextDeltas: [ - { - index: 1, - regionIndex: 1, - slot: 'leading', - likelyKind: 'icon', - rect: { x: 80, y: 318, width: 30, height: 30 }, - nearestText: 'Wi-Fi', - }, - { - index: 2, - regionIndex: 1, - slot: 'separator', - likelyKind: 'separator', - rect: { x: 90, y: 360, width: 120, height: 2 }, - }, - ], }), ); assert.match(text, /✗ 5% pixels differ/); @@ -1642,36 +1594,9 @@ test('formatScreenshotDiffText renders mismatch with pixel counts without color' assert.match(text, /Current overlay:/); assert.match(text, /diff\.current-overlay\.png \(1 refs\)/); assert.match(text, /500 different \/ 10000 total pixels/); - assert.match(text, /Hints:/); - assert.match( - text, - /text movement cluster: "Wi-Fi", "Bluetooth" dx=\+10\.\.\+12px dy=\+10\.\.\+14px/, - ); - assert.match(text, /non-text controls: icon near "Wi-Fi" r1/); - assert.match(text, /non-text boundaries: separator r1/); assert.match(text, /Changed regions:/); - assert.match(text, /1\. top-left x=10 y=20 100x40, 70% of diff, change=brighter/); - assert.match( - text, - /size=medium shape=horizontal-band density=8\.75% avgColor=#141414->#dcdcdc luminance=20->220/, - ); + assert.match(text, /1\. x=10 y=20 100x40, 70% of diff/); assert.match(text, /overlaps @e1 "Continue", 12% of region/); - assert.match( - text, - /OCR text deltas \(tesseract; baselineBlocks=2 currentBlocks=2; showing 1\/1; px\):/, - ); - assert.match( - text, - /item \| text \| movePx \| sizeDeltaPx \| bboxBaseline \| bboxCurrent \| confidence \| issueHint/, - ); - assert.match( - text, - /1 \| "Wi-Fi" \| \+10,\+12 \| \+10,0 \| x=120,y=320,w=60,h=22 \| x=130,y=332,w=70,h=22 \| 94 \| ocr-bbox-size-change/, - ); - assert.match(text, /Non-text visual deltas \(showing 2\/2; px\):/); - assert.match(text, /item \| region \| slot \| kind \| bboxCurrent \| nearestText/); - assert.match(text, /1 \| r1 \| leading \| icon \| x=80,y=318,w=30,h=30 \| "Wi-Fi"/); - assert.match(text, /2 \| r1 \| separator \| separator \| x=90,y=360,w=120,h=2 \| -/); assert.equal(text.includes('\x1b['), false); }); diff --git a/src/utils/output.ts b/src/utils/output.ts index d6a4c186c6..68dea1f54a 100644 --- a/src/utils/output.ts +++ b/src/utils/output.ts @@ -10,12 +10,10 @@ import { buildSnapshotDisplayLines, formatSnapshotLine } from '../snapshot/snaps import { isSnapshotBackend, usesMobileSnapshotPresentation, - type Rect, type SnapshotNode, type SnapshotUnchanged, type SnapshotVisibility, } from '@agent-device/kernel/snapshot'; -import type { MovementRange } from '../screenshot-diff/screenshot-diff-ocr.ts'; import type { ScreenshotDiffResult } from '../screenshot-diff/screenshot-diff.ts'; import type { ScreenshotDiffRegion } from '../screenshot-diff/screenshot-diff-regions.ts'; import { styleText } from 'node:util'; @@ -323,10 +321,7 @@ export function formatScreenshotDiffText(data: ScreenshotDiffResult): string { if (!match && !dimensionMismatch) { lines.push(...formatScreenshotDiffPixelCountLines(data, useColor)); - lines.push(...formatScreenshotDiffHintLines(data, useColor)); lines.push(...formatScreenshotDiffRegionLines(data, useColor)); - lines.push(...formatScreenshotDiffOcrLines(data, useColor)); - lines.push(...formatScreenshotDiffNonTextLines(data, useColor)); } return `${lines.join('\n')}\n`; @@ -395,12 +390,6 @@ function formatScreenshotDiffPixelCountLines( return [` ${diffCount} different / ${totalPixels} total pixels`]; } -function formatScreenshotDiffHintLines(data: ScreenshotDiffResult, useColor: boolean): string[] { - const hints = formatScreenshotDiffHints(data); - if (hints.length === 0) return []; - return [` ${formatMuted('Hints:', useColor)}`, ...hints.map((hint) => ` - ${hint}`)]; -} - function formatScreenshotDiffRegionLines(data: ScreenshotDiffResult, useColor: boolean): string[] { const regions = Array.isArray(data.regions) ? data.regions : []; if (regions.length === 0) return []; @@ -419,15 +408,10 @@ function formatScreenshotDiffRegionEntryLines(region: ScreenshotDiffRegion): str : String(region.shareOfDiffPercentage); const rect = region.rect; const lines = [ - ` ${region.index}. ${region.location} x=${rect.x} y=${rect.y} ` + - `${rect.width}x${rect.height}, ${share}% of diff, change=${region.dominantChange}`, + ` ${region.index}. x=${rect.x} y=${rect.y} ${rect.width}x${rect.height}, ` + + `${share}% of diff`, ]; - const detailLine = formatScreenshotRegionDetails(region); - if (detailLine) { - lines.push(` ${detailLine}`); - } - const bestMatch = region.currentOverlayMatches?.[0]; if (bestMatch) { const label = bestMatch.label ? ` "${bestMatch.label}"` : ''; @@ -440,134 +424,6 @@ function formatScreenshotDiffRegionEntryLines(region: ScreenshotDiffRegion): str return lines; } -function formatScreenshotDiffOcrLines(data: ScreenshotDiffResult, useColor: boolean): string[] { - const ocrMatches = data.ocr?.matches ?? []; - if (ocrMatches.length === 0) return []; - - const shownOcrMatches = ocrMatches.slice(0, 8); - const lines = [ - ` ${formatMuted( - `OCR text deltas (${data.ocr?.provider}; baselineBlocks=${data.ocr?.baselineBlocks} ` + - `currentBlocks=${data.ocr?.currentBlocks}; showing ${shownOcrMatches.length}/${ocrMatches.length}; px):`, - useColor, - )}`, - ` ${formatMuted( - 'item | text | movePx | sizeDeltaPx | bboxBaseline | bboxCurrent | confidence | issueHint', - useColor, - )}`, - ]; - - for (const [index, ocrMatch] of shownOcrMatches.entries()) { - const delta = ocrMatch.delta; - lines.push( - ` ${index + 1} | ${JSON.stringify(ocrMatch.text)} | ` + - `${formatSignedPixels(delta.x)},${formatSignedPixels(delta.y)} | ` + - `${formatSignedPixels(delta.width)},${formatSignedPixels(delta.height)} | ` + - `${formatRect(ocrMatch.baselineRect)} | ${formatRect(ocrMatch.currentRect)} | ` + - `${ocrMatch.confidence} | ` + - `${ocrMatch.possibleTextMetricMismatch ? 'ocr-bbox-size-change' : '-'}`, - ); - } - - return lines; -} - -function formatScreenshotDiffNonTextLines(data: ScreenshotDiffResult, useColor: boolean): string[] { - const nonTextDeltas = data.nonTextDeltas ?? []; - if (nonTextDeltas.length === 0) return []; - - const shownNonTextDeltas = nonTextDeltas.slice(0, 8); - const lines = [ - ` ${formatMuted( - `Non-text visual deltas (showing ${shownNonTextDeltas.length}/${nonTextDeltas.length}; px):`, - useColor, - )}`, - ` ${formatMuted('item | region | slot | kind | bboxCurrent | nearestText', useColor)}`, - ]; - - for (const delta of shownNonTextDeltas) { - lines.push( - ` ${delta.index} | ${delta.regionIndex ? `r${delta.regionIndex}` : '-'} | ` + - `${delta.slot} | ${delta.likelyKind} | ${formatRect(delta.rect)} | ` + - `${delta.nearestText ? JSON.stringify(delta.nearestText) : '-'}`, - ); - } - - return lines; -} - -function formatRect(rect: Rect): string { - return `x=${rect.x},y=${rect.y},w=${rect.width},h=${rect.height}`; -} - -function formatSignedPixels(value: number): string { - return value > 0 ? `+${value}` : String(value); -} - -function formatScreenshotDiffHints(data: ScreenshotDiffResult): string[] { - const hints: string[] = []; - const clusters = data.ocr?.movementClusters ?? []; - for (const cluster of clusters.slice(0, 2)) { - hints.push( - `text movement cluster: ${formatQuotedList(cluster.texts)} dx=${formatRange(cluster.xRange)}px ` + - `dy=${formatRange(cluster.yRange)}px`, - ); - } - - const controlDeltas = (data.nonTextDeltas ?? []) - .filter((delta) => ['icon', 'toggle', 'chevron'].includes(delta.likelyKind)) - .slice(0, 3); - if (controlDeltas.length > 0) { - hints.push(`non-text controls: ${controlDeltas.map(formatNonTextHint).join('; ')}`); - } - - const boundaryDeltas = (data.nonTextDeltas ?? []) - .filter((delta) => delta.likelyKind === 'separator') - .slice(0, 2); - if (boundaryDeltas.length > 0) { - hints.push(`non-text boundaries: ${boundaryDeltas.map(formatNonTextHint).join('; ')}`); - } - - return hints.slice(0, 6); -} - -function formatNonTextHint(delta: { - likelyKind: string; - nearestText?: string; - regionIndex?: number; -}): string { - const anchor = delta.nearestText ? ` near ${JSON.stringify(delta.nearestText)}` : ''; - const region = delta.regionIndex ? ` r${delta.regionIndex}` : ''; - return `${delta.likelyKind}${anchor}${region}`; -} - -function formatRange(range: MovementRange): string { - return range.min === range.max - ? formatSignedPixels(range.min) - : `${formatSignedPixels(range.min)}..${formatSignedPixels(range.max)}`; -} - -function formatQuotedList(values: string[]): string { - const shown = values.slice(0, 4).map((value) => JSON.stringify(value)); - const suffix = values.length > shown.length ? ` +${values.length - shown.length} more` : ''; - return `${shown.join(', ')}${suffix}`; -} - -function formatScreenshotRegionDetails(region: ScreenshotDiffRegion): string | null { - const details = [ - region.size ? `size=${region.size}` : null, - region.shape ? `shape=${region.shape}` : null, - typeof region.densityPercentage === 'number' ? `density=${region.densityPercentage}%` : null, - region.averageBaselineColorHex && region.averageCurrentColorHex - ? `avgColor=${region.averageBaselineColorHex}->${region.averageCurrentColorHex}` - : null, - typeof region.baselineLuminance === 'number' && typeof region.currentLuminance === 'number' - ? `luminance=${region.baselineLuminance}->${region.currentLuminance}` - : null, - ].filter((entry): entry is string => entry !== null); - return details.length > 0 ? details.join(' ') : null; -} - function toRelativePath(filePath: string): string { const cwd = process.cwd(); const relativePath = path.relative(cwd, filePath); diff --git a/website/docs/docs/commands.md b/website/docs/docs/commands.md index 100e6f7065..7dcabb821b 100644 --- a/website/docs/docs/commands.md +++ b/website/docs/docs/commands.md @@ -852,9 +852,7 @@ agent-device record stop # Stop active recording - `screenshot --overlay-refs` captures a fresh full snapshot and burns visible `@eN` refs plus their target rectangles into the saved PNG. - `screenshot --normalize-status-bar` temporarily normalizes iOS simulator status-bar chrome for deterministic screenshot baselines; ordinary screenshots leave the simulator's current chrome visible. - `screenshot --max-size --overlay-refs` writes a smaller image and draws refs for that final image size; avoid very small max sizes when text, icons, or labels need to remain readable. -- `diff screenshot` compares the current live screenshot to `--baseline`, or compares `--baseline` to an optional saved `current.png` path without requiring an active session, then prints ranked changed regions with screen-space rectangles, shape, size, density, average color, and luminance, and writes a diff PNG with a light grayscale current-screen context, red-tinted changed pixels, and outlined changed regions when `--out` is provided. Live iOS simulator diffs normalize status-bar chrome by default; use `screenshot --normalize-status-bar` when capturing reusable baselines. JSON also includes normalized bounds. -- If `tesseract` is installed, `diff screenshot` also adds best-effort OCR text deltas, movement clusters, and bbox size-change hints to the text and JSON output. OCR improves descriptions only; it does not change the pixel comparison or the diff PNG. -- When OCR is available, `diff screenshot` also reports best-effort non-text visual deltas by masking OCR text boxes out of the diff and clustering remaining residuals. These are hints for icons, controls, and separators, not semantic icon recognition. +- `diff screenshot` compares the current live screenshot to `--baseline`, or compares `--baseline` to an optional saved `current.png` path without requiring an active session. Its text output reports ranked changed regions with screen-space rectangles, changed-pixel counts, and each region's share of the diff; JSON also includes normalized rectangles. It writes a diff PNG with a light grayscale current-screen context, red-tinted changed pixels, and outlined changed regions when `--out` is provided. Live iOS simulator diffs normalize status-bar chrome by default; use `screenshot --normalize-status-bar` when capturing reusable baselines. - `diff screenshot --overlay-refs` additionally writes a separate current-screen overlay guide for live captures without using that annotated image for the pixel comparison. If current-screen refs intersect changed regions, the output lists the best ref matches under those regions. Saved-image comparisons do not have live accessibility refs, so `--overlay-refs` is unavailable when a `current.png` path is provided. - In `--json` mode, each overlay ref also includes a screenshot-space `center` point for coordinate fallback like `press `. - Burned-in touch overlays are exported only on macOS hosts, because the overlay pipeline depends on Swift + AVFoundation helpers. From a6d662329a5f6b4bded7df594415212bf1054696 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Micha=C5=82=20Pierzcha=C5=82a?= Date: Thu, 30 Jul 2026 12:07:39 +0200 Subject: [PATCH 2/5] test: update screenshot diff cli output --- src/__tests__/cli-diff.test.ts | 5 +---- 1 file changed, 1 insertion(+), 4 deletions(-) diff --git a/src/__tests__/cli-diff.test.ts b/src/__tests__/cli-diff.test.ts index ddfeedcae5..f89d528d97 100644 --- a/src/__tests__/cli-diff.test.ts +++ b/src/__tests__/cli-diff.test.ts @@ -382,10 +382,7 @@ describe('cli diff commands', () => { assert.match(result.stdout, /Diff image:/); assert.match(result.stdout, /Current overlay:/); assert.match(result.stdout, /diff\.current-overlay\.png \(1 refs\)/); - assert.match( - result.stdout, - /size=large shape=large-area density=100% avgColor=#000000->#ffffff luminance=0->255/, - ); + assert.match(result.stdout, /1\. x=0 y=0 10x10, 100% of diff/); assert.match(result.stdout, /overlaps @e1 "Continue", 12% of region/); } finally { fs.rmSync(dir, { recursive: true, force: true }); From dcf11e89484ffa302b5ea578c097b164e574571e Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Micha=C5=82=20Pierzcha=C5=82a?= Date: Thu, 30 Jul 2026 12:31:13 +0200 Subject: [PATCH 3/5] chore: remove unused screenshot geometry helpers --- src/utils/screenshot-geometry.ts | 46 +------------------------------- 1 file changed, 1 insertion(+), 45 deletions(-) diff --git a/src/utils/screenshot-geometry.ts b/src/utils/screenshot-geometry.ts index 6341801c2e..789b2383fc 100644 --- a/src/utils/screenshot-geometry.ts +++ b/src/utils/screenshot-geometry.ts @@ -1,9 +1,8 @@ -import type { Point, Rect } from '@agent-device/kernel/snapshot'; +import type { Rect } from '@agent-device/kernel/snapshot'; export type ImageDimensions = { width: number; height: number }; declare const normalizedRectBrand: unique symbol; -declare const normalizedPointBrand: unique symbol; /** * A rect whose coordinates are normalized percentages [0..100] of the screenshot @@ -12,36 +11,10 @@ declare const normalizedPointBrand: unique symbol; */ export type NormalizedRect = Rect & { readonly [normalizedRectBrand]: 'normalized-rect' }; -/** A point in normalized [0..100] screenshot-image space. */ -export type NormalizedPoint = Point & { readonly [normalizedPointBrand]: 'normalized-point' }; - export function normalizedRect(rect: Rect): NormalizedRect { return rect as NormalizedRect; } -export function unionRects(rects: Rect[]): Rect { - let minX = Number.POSITIVE_INFINITY; - let minY = Number.POSITIVE_INFINITY; - let maxX = Number.NEGATIVE_INFINITY; - let maxY = Number.NEGATIVE_INFINITY; - for (const rect of rects) { - minX = Math.min(minX, rect.x); - minY = Math.min(minY, rect.y); - maxX = Math.max(maxX, rect.x + rect.width); - maxY = Math.max(maxY, rect.y + rect.height); - } - return { x: minX, y: minY, width: maxX - minX, height: maxY - minY }; -} - -export function expandRect(rect: Rect, padding: number): Rect { - return { - x: rect.x - padding, - y: rect.y - padding, - width: rect.width + padding * 2, - height: rect.height + padding * 2, - }; -} - export function intersectArea(left: Rect, right: Rect): number { const minX = Math.max(left.x, right.x); const minY = Math.max(left.y, right.y); @@ -50,20 +23,3 @@ export function intersectArea(left: Rect, right: Rect): number { if (maxX <= minX || maxY <= minY) return 0; return (maxX - minX) * (maxY - minY); } - -export function rectCenter(rect: NormalizedRect): NormalizedPoint; -export function rectCenter(rect: Rect): Point; -export function rectCenter(rect: Rect): Point { - return { x: rect.x + rect.width / 2, y: rect.y + rect.height / 2 }; -} - -export function squaredDistance( - left: { x: number; y: number }, - right: { x: number; y: number }, -): number { - return (left.x - right.x) ** 2 + (left.y - right.y) ** 2; -} - -export function clamp(value: number, min: number, max: number): number { - return Math.min(Math.max(value, min), max); -} From 0b4656b225eeaf242c47e5ba67da8d14407a938c Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Micha=C5=82=20Pierzcha=C5=82a?= Date: Thu, 30 Jul 2026 13:18:25 +0200 Subject: [PATCH 4/5] fix: version screenshot diff result contract --- CHANGELOG.md | 1 + src/__tests__/cli-diff.test.ts | 1 + .../__tests__/screenshot-diff.test.ts | 12 ++ .../screenshot-diff-region-split.ts | 43 ++++- .../screenshot-diff-region-types.ts | 6 + .../screenshot-diff-regions.ts | 172 +++++++++++++++++- src/screenshot-diff/screenshot-diff.ts | 17 +- src/utils/__tests__/output.test.ts | 16 ++ website/docs/docs/commands.md | 2 +- 9 files changed, 246 insertions(+), 24 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index 4b71b4a984..1914657491 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -2,6 +2,7 @@ ## Unreleased +- Breaking: `diff screenshot` structured results now carry `schemaVersion: 2`. The retired `ocr` and `nonTextDeltas` payloads are no longer emitted; use the baseline/current images and diff artifact with vision for qualitative interpretation. Existing pixel counts and region fields remain available. - Breaking: removed the deprecated `--session-locked` and `--session-lock-conflicts` flags. Use `--session-lock reject|strip` instead; passing either old flag now fails with `Unknown flag: ... Use --session-lock reject|strip instead.` - Breaking: removed the `replay export --format` flag. `replay export` always writes Maestro YAML. - Breaking: removed the unused `LeaseAllocatePayload`, `LeaseHeartbeatPayload`, and `LeaseReleasePayload` type exports from `agent-device/contracts`. Lease request metadata is fully described by `DaemonRequestMeta`. diff --git a/src/__tests__/cli-diff.test.ts b/src/__tests__/cli-diff.test.ts index f89d528d97..86850e0336 100644 --- a/src/__tests__/cli-diff.test.ts +++ b/src/__tests__/cli-diff.test.ts @@ -202,6 +202,7 @@ describe('cli diff commands', () => { const payload = JSON.parse(result.stdout); assert.equal(payload.success, true); assert.equal(payload.data.match, true); + assert.equal(payload.data.schemaVersion, 2); assert.equal(payload.data.differentPixels, 0); assert.equal(payload.data.totalPixels, 100); assert.equal(payload.data.mismatchPercentage, 0); diff --git a/src/screenshot-diff/__tests__/screenshot-diff.test.ts b/src/screenshot-diff/__tests__/screenshot-diff.test.ts index 85818895f9..6c1f9910ce 100644 --- a/src/screenshot-diff/__tests__/screenshot-diff.test.ts +++ b/src/screenshot-diff/__tests__/screenshot-diff.test.ts @@ -74,6 +74,9 @@ test('identical images produce match: true with 0% mismatch', async () => { assert.equal(result.differentPixels, 0); assert.equal(result.mismatchPercentage, 0); assert.equal(result.totalPixels, 100); + assert.equal(result.schemaVersion, 2); + assert.equal(Object.hasOwn(result, 'ocr'), false); + assert.equal(Object.hasOwn(result, 'nonTextDeltas'), false); assert.equal(result.dimensionMismatch, undefined); assert.equal(result.diffPath, undefined, 'diffPath should not be set when images match'); // No diff image should be written when images match @@ -148,6 +151,15 @@ test('changed pixels are summarized into nearby diff regions', async () => { assert.equal(result.regions?.[0]?.differentPixels, 32); assert.equal(result.regions?.[0]?.shareOfDiffPercentage, 66.67); assert.deepEqual(result.regions?.[0]?.normalizedRect, { x: 5, y: 10, width: 30, height: 20 }); + assert.equal(result.regions?.[0]?.densityPercentage, 66.67); + assert.equal(result.regions?.[0]?.shape, 'horizontal-band'); + assert.equal(result.regions?.[0]?.size, 'large'); + assert.equal(result.regions?.[0]?.averageBaselineColorHex, '#000000'); + assert.equal(result.regions?.[0]?.averageCurrentColorHex, '#ffffff'); + assert.equal(result.regions?.[0]?.baselineLuminance, 0); + assert.equal(result.regions?.[0]?.currentLuminance, 255); + assert.equal(result.regions?.[0]?.location, 'top-left'); + assert.equal(result.regions?.[0]?.dominantChange, 'brighter'); assert.deepEqual(result.regions?.[1]?.rect, { x: 30, y: 15, width: 4, height: 4 }); const diffPng = PNG.sync.read(fs.readFileSync(diffOut)); diff --git a/src/screenshot-diff/screenshot-diff-region-split.ts b/src/screenshot-diff/screenshot-diff-region-split.ts index a6b67274a6..bb537b37de 100644 --- a/src/screenshot-diff/screenshot-diff-region-split.ts +++ b/src/screenshot-diff/screenshot-diff-region-split.ts @@ -1,3 +1,4 @@ +import type { PNG } from '../utils/png.ts'; import type { MutableDiffRegion } from './screenshot-diff-region-types.ts'; // Region splitting is based on screen-relative heights so it works on phone, @@ -13,11 +14,15 @@ const ROW_SMOOTHING_RADIUS = 3; export function splitLargeDiffRegions( regions: MutableDiffRegion[], - params: { diffMask: Uint8Array; width: number; height: number }, + params: { diffMask: Uint8Array; baseline: PNG; current: PNG }, ): MutableDiffRegion[] { return regions.flatMap((region) => - shouldSplitRegion(region, params.width, params.height) - ? splitRegionByHorizontalDensity(region, params, minSplitSegmentHeight(params.height)) + shouldSplitRegion(region, params.baseline.width, params.baseline.height) + ? splitRegionByHorizontalDensity( + region, + params, + minSplitSegmentHeight(params.baseline.height), + ) : [region], ); } @@ -37,10 +42,10 @@ function shouldSplitRegion( function splitRegionByHorizontalDensity( region: MutableDiffRegion, - params: { diffMask: Uint8Array; width: number }, + params: { diffMask: Uint8Array; baseline: PNG; current: PNG }, minSegmentHeight: number, ): MutableDiffRegion[] { - const rowCounts = measureRowDiffCounts(region, params.diffMask, params.width); + const rowCounts = measureRowDiffCounts(region, params.diffMask, params.baseline.width); const smoothed = smoothCounts(rowCounts); const lowDensityBands = findLowDensityBands( smoothed, @@ -141,15 +146,15 @@ function buildRegionSlice( region: MutableDiffRegion, minY: number, maxY: number, - params: { diffMask: Uint8Array; width: number }, + params: { diffMask: Uint8Array; baseline: PNG; current: PNG }, ): MutableDiffRegion | null { let slice: MutableDiffRegion | null = null; for (let y = minY; y <= maxY; y += 1) { for (let x = region.minX; x <= region.maxX; x += 1) { - const pixelIndex = y * params.width + x; + const pixelIndex = y * params.baseline.width + x; if (params.diffMask[pixelIndex] !== 1) continue; slice ??= createEmptyRegion(x, y); - addPixelToSlice(slice, x, y); + addPixelToSlice(slice, pixelIndex, x, y, params.baseline, params.current); } } return slice; @@ -162,13 +167,33 @@ function createEmptyRegion(x: number, y: number): MutableDiffRegion { maxX: x, maxY: y, differentPixels: 0, + baselineRed: 0, + baselineGreen: 0, + baselineBlue: 0, + currentRed: 0, + currentGreen: 0, + currentBlue: 0, }; } -function addPixelToSlice(slice: MutableDiffRegion, x: number, y: number): void { +function addPixelToSlice( + slice: MutableDiffRegion, + pixelIndex: number, + x: number, + y: number, + baseline: PNG, + current: PNG, +): void { + const dataIndex = pixelIndex * 4; slice.minX = Math.min(slice.minX, x); slice.minY = Math.min(slice.minY, y); slice.maxX = Math.max(slice.maxX, x); slice.maxY = Math.max(slice.maxY, y); slice.differentPixels += 1; + slice.baselineRed += baseline.data[dataIndex]!; + slice.baselineGreen += baseline.data[dataIndex + 1]!; + slice.baselineBlue += baseline.data[dataIndex + 2]!; + slice.currentRed += current.data[dataIndex]!; + slice.currentGreen += current.data[dataIndex + 1]!; + slice.currentBlue += current.data[dataIndex + 2]!; } diff --git a/src/screenshot-diff/screenshot-diff-region-types.ts b/src/screenshot-diff/screenshot-diff-region-types.ts index 645e06f290..3e42e45a23 100644 --- a/src/screenshot-diff/screenshot-diff-region-types.ts +++ b/src/screenshot-diff/screenshot-diff-region-types.ts @@ -4,4 +4,10 @@ export type MutableDiffRegion = { maxX: number; maxY: number; differentPixels: number; + baselineRed: number; + baselineGreen: number; + baselineBlue: number; + currentRed: number; + currentGreen: number; + currentBlue: number; }; diff --git a/src/screenshot-diff/screenshot-diff-regions.ts b/src/screenshot-diff/screenshot-diff-regions.ts index ade8165db4..ee464b9465 100644 --- a/src/screenshot-diff/screenshot-diff-regions.ts +++ b/src/screenshot-diff/screenshot-diff-regions.ts @@ -1,15 +1,31 @@ import type { Rect } from '@agent-device/kernel/snapshot'; +import type { PNG } from '../utils/png.ts'; import { normalizedRect, type NormalizedRect } from '../utils/screenshot-geometry.ts'; import { findConnectedMaskComponents } from './screenshot-diff-components.ts'; import { splitLargeDiffRegions } from './screenshot-diff-region-split.ts'; import type { MutableDiffRegion } from './screenshot-diff-region-types.ts'; +type ScreenshotDiffColor = { + r: number; + g: number; + b: number; +}; + export type ScreenshotDiffRegion = { index: number; rect: Rect; normalizedRect: NormalizedRect; differentPixels: number; shareOfDiffPercentage: number; + densityPercentage: number; + shape: 'compact' | 'horizontal-band' | 'vertical-band' | 'large-area'; + size: 'small' | 'medium' | 'large'; + location: string; + averageBaselineColorHex: string; + averageCurrentColorHex: string; + baselineLuminance: number; + currentLuminance: number; + dominantChange: 'brighter' | 'darker' | 'color-shift' | 'mixed'; currentOverlayMatches?: ScreenshotDiffRegionOverlayMatch[]; }; @@ -23,10 +39,17 @@ export type ScreenshotDiffRegionOverlayMatch = { const DEFAULT_MAX_DIFF_REGIONS = 8; const REGION_MERGE_GAP_PX = 12; const MAX_REGIONS_TO_MERGE = 2000; +const DOMINANT_CHANGE_MIN_CHANNEL_DELTA = 12; +const LARGE_AREA_MIN_WIDTH_RATIO = 0.55; +const LARGE_AREA_MIN_HEIGHT_RATIO = 0.12; +const BAND_MIN_ASPECT_RATIO = 2.5; +const LARGE_REGION_MIN_AREA_RATIO = 0.04; +const MEDIUM_REGION_MIN_AREA_RATIO = 0.01; export function summarizeDiffRegions(params: { diffMask: Uint8Array; - width: number; - height: number; + baseline: PNG; + current: PNG; + totalPixels: number; differentPixels: number; }): ScreenshotDiffRegion[] { const rawRegions = findConnectedDiffRegions(params); @@ -48,8 +71,9 @@ export function summarizeDiffRegions(params: { .slice(0, DEFAULT_MAX_DIFF_REGIONS) .map((region, index) => toScreenshotDiffRegion(region, index + 1, { - width: params.width, - height: params.height, + width: params.baseline.width, + height: params.baseline.height, + totalPixels: params.totalPixels, differentPixels: params.differentPixels, }), ); @@ -57,17 +81,18 @@ export function summarizeDiffRegions(params: { function findConnectedDiffRegions(params: { diffMask: Uint8Array; - width: number; - height: number; + baseline: PNG; + current: PNG; }): MutableDiffRegion[] { - const { diffMask, width, height } = params; + const { diffMask, baseline, current } = params; + const { width, height } = baseline; return findConnectedMaskComponents({ mask: diffMask, width, height, hooks: { create: (pixelIndex) => createDiffRegion(pixelIndex, width), - visit: (region, pixelIndex) => addPixelToRegion(region, pixelIndex, width), + visit: (region, pixelIndex) => addPixelToRegion(region, pixelIndex, width, baseline, current), }, }); } @@ -81,17 +106,36 @@ function createDiffRegion(pixelIndex: number, width: number): MutableDiffRegion maxX: startX, maxY: startY, differentPixels: 0, + baselineRed: 0, + baselineGreen: 0, + baselineBlue: 0, + currentRed: 0, + currentGreen: 0, + currentBlue: 0, }; } -function addPixelToRegion(region: MutableDiffRegion, pixelIndex: number, width: number): void { +function addPixelToRegion( + region: MutableDiffRegion, + pixelIndex: number, + width: number, + baseline: PNG, + current: PNG, +): void { const x = pixelIndex % width; const y = Math.floor(pixelIndex / width); + const dataIndex = pixelIndex * 4; region.minX = Math.min(region.minX, x); region.minY = Math.min(region.minY, y); region.maxX = Math.max(region.maxX, x); region.maxY = Math.max(region.maxY, y); region.differentPixels += 1; + region.baselineRed += baseline.data[dataIndex]!; + region.baselineGreen += baseline.data[dataIndex + 1]!; + region.baselineBlue += baseline.data[dataIndex + 2]!; + region.currentRed += current.data[dataIndex]!; + region.currentGreen += current.data[dataIndex + 1]!; + region.currentBlue += current.data[dataIndex + 2]!; } function mergeNearbyRegions(regions: MutableDiffRegion[], gapPx: number): MutableDiffRegion[] { @@ -126,12 +170,18 @@ function mergeRegionInto(target: MutableDiffRegion, source: MutableDiffRegion): target.maxX = Math.max(target.maxX, source.maxX); target.maxY = Math.max(target.maxY, source.maxY); target.differentPixels += source.differentPixels; + target.baselineRed += source.baselineRed; + target.baselineGreen += source.baselineGreen; + target.baselineBlue += source.baselineBlue; + target.currentRed += source.currentRed; + target.currentGreen += source.currentGreen; + target.currentBlue += source.currentBlue; } function toScreenshotDiffRegion( region: MutableDiffRegion, index: number, - image: { width: number; height: number; differentPixels: number }, + image: { width: number; height: number; totalPixels: number; differentPixels: number }, ): ScreenshotDiffRegion { const rect = { x: region.minX, @@ -139,6 +189,23 @@ function toScreenshotDiffRegion( width: region.maxX - region.minX + 1, height: region.maxY - region.minY + 1, }; + const center = { + x: Math.round(region.minX + rect.width / 2), + y: Math.round(region.minY + rect.height / 2), + }; + const averageBaselineColor = averageRegionColor( + region.baselineRed, + region.baselineGreen, + region.baselineBlue, + region.differentPixels, + ); + const averageCurrentColor = averageRegionColor( + region.currentRed, + region.currentGreen, + region.currentBlue, + region.differentPixels, + ); + const regionArea = rect.width * rect.height; return { index, rect, @@ -150,9 +217,94 @@ function toScreenshotDiffRegion( }), differentPixels: region.differentPixels, shareOfDiffPercentage: roundPercentage(region.differentPixels / image.differentPixels), + densityPercentage: roundPercentage(region.differentPixels / regionArea), + shape: describeRegionShape(rect, image.width, image.height), + size: describeRegionSize(regionArea, image.totalPixels), + location: describeRegionLocation(center, image.width, image.height), + averageBaselineColorHex: toHexColor(averageBaselineColor), + averageCurrentColorHex: toHexColor(averageCurrentColor), + baselineLuminance: Math.round(luminance(averageBaselineColor)), + currentLuminance: Math.round(luminance(averageCurrentColor)), + dominantChange: describeDominantChange(averageBaselineColor, averageCurrentColor), + }; +} + +function averageRegionColor( + red: number, + green: number, + blue: number, + pixels: number, +): ScreenshotDiffColor { + return { + r: Math.round(red / pixels), + g: Math.round(green / pixels), + b: Math.round(blue / pixels), }; } +function describeRegionLocation( + center: { x: number; y: number }, + width: number, + height: number, +): string { + const horizontal = + center.x < width / 3 ? 'left' : center.x > (width * 2) / 3 ? 'right' : 'center'; + const vertical = + center.y < height / 3 ? 'top' : center.y > (height * 2) / 3 ? 'bottom' : 'middle'; + return horizontal === 'center' && vertical === 'middle' ? 'center' : `${vertical}-${horizontal}`; +} + +function describeDominantChange( + baseline: ScreenshotDiffColor, + current: ScreenshotDiffColor, +): ScreenshotDiffRegion['dominantChange'] { + const luminanceDelta = luminance(current) - luminance(baseline); + if (Math.abs(luminanceDelta) >= DOMINANT_CHANGE_MIN_CHANNEL_DELTA) { + return luminanceDelta > 0 ? 'brighter' : 'darker'; + } + const maxChannelDelta = Math.max( + Math.abs(current.r - baseline.r), + Math.abs(current.g - baseline.g), + Math.abs(current.b - baseline.b), + ); + return maxChannelDelta >= DOMINANT_CHANGE_MIN_CHANNEL_DELTA ? 'color-shift' : 'mixed'; +} + +function describeRegionShape( + rect: { width: number; height: number }, + imageWidth: number, + imageHeight: number, +): ScreenshotDiffRegion['shape'] { + if ( + rect.width >= imageWidth * LARGE_AREA_MIN_WIDTH_RATIO && + rect.height >= imageHeight * LARGE_AREA_MIN_HEIGHT_RATIO + ) { + return 'large-area'; + } + if (rect.width >= rect.height * BAND_MIN_ASPECT_RATIO) return 'horizontal-band'; + if (rect.height >= rect.width * BAND_MIN_ASPECT_RATIO) return 'vertical-band'; + return 'compact'; +} + +function describeRegionSize(regionArea: number, totalPixels: number): ScreenshotDiffRegion['size'] { + const areaRatio = regionArea / totalPixels; + if (areaRatio >= LARGE_REGION_MIN_AREA_RATIO) return 'large'; + if (areaRatio >= MEDIUM_REGION_MIN_AREA_RATIO) return 'medium'; + return 'small'; +} + +function luminance(color: ScreenshotDiffColor): number { + return color.r * 0.2126 + color.g * 0.7152 + color.b * 0.0722; +} + +function toHexColor(color: ScreenshotDiffColor): string { + return `#${toHexChannel(color.r)}${toHexChannel(color.g)}${toHexChannel(color.b)}`; +} + +function toHexChannel(value: number): string { + return value.toString(16).padStart(2, '0'); +} + function roundPercentage(ratio: number): number { return Math.round(ratio * 100 * 100) / 100; } diff --git a/src/screenshot-diff/screenshot-diff.ts b/src/screenshot-diff/screenshot-diff.ts index 7e5263e0d1..8ea268b1ea 100644 --- a/src/screenshot-diff/screenshot-diff.ts +++ b/src/screenshot-diff/screenshot-diff.ts @@ -17,6 +17,12 @@ export type ScreenshotDimensionMismatch = { }; export type ScreenshotDiffResult = { + /** + * Version of the structured screenshot-diff result. Version 2 removes the + * retired analysis payloads while preserving the established pixel and + * region contracts. + */ + schemaVersion: 2; diffPath?: string; totalPixels: number; differentPixels: number; @@ -71,6 +77,7 @@ export async function compareScreenshots( await removeStaleDiffOutput(options.outputPath); return { match: false, + schemaVersion: 2, mismatchPercentage: 100, totalPixels, differentPixels: totalPixels, @@ -86,10 +93,10 @@ export async function compareScreenshots( // Per-pixel comparison is CPU-heavy for full-resolution screenshots, so it // runs on the PNG worker thread (with an in-process synchronous fallback). const { diffData, diffMask, differentPixels } = await computeScreenshotDiffPixelsAsync({ - width: baseline.width, - height: baseline.height, baselineData: baseline.data, currentData: current.data, + width: baseline.width, + height: baseline.height, maxColorDistance, }); @@ -97,8 +104,9 @@ export async function compareScreenshots( differentPixels > 0 ? summarizeDiffRegions({ diffMask, - width: baseline.width, - height: baseline.height, + baseline, + current, + totalPixels, differentPixels, }) : []; @@ -118,6 +126,7 @@ export async function compareScreenshots( totalPixels > 0 ? Math.round((differentPixels / totalPixels) * 100 * 100) / 100 : 0; return { + schemaVersion: 2, ...(differentPixels > 0 && diffOutputPath ? { diffPath: diffOutputPath } : {}), ...(regions.length > 0 ? { regions } : {}), totalPixels, diff --git a/src/utils/__tests__/output.test.ts b/src/utils/__tests__/output.test.ts index 616b61a32b..c669205f3a 100644 --- a/src/utils/__tests__/output.test.ts +++ b/src/utils/__tests__/output.test.ts @@ -1550,6 +1550,7 @@ function withColor(fn: () => T): T { test('formatScreenshotDiffText renders match success without color', () => { const text = withNoColor(() => formatScreenshotDiffText({ + schemaVersion: 2, match: true, differentPixels: 0, totalPixels: 100, @@ -1563,6 +1564,7 @@ test('formatScreenshotDiffText renders match success without color', () => { test('formatScreenshotDiffText renders mismatch with pixel counts without color', () => { const text = withNoColor(() => formatScreenshotDiffText({ + schemaVersion: 2, match: false, differentPixels: 500, totalPixels: 10000, @@ -1577,6 +1579,15 @@ test('formatScreenshotDiffText renders mismatch with pixel counts without color' normalizedRect: normalizedRect({ x: 10, y: 20, width: 100, height: 40 }), differentPixels: 350, shareOfDiffPercentage: 70, + densityPercentage: 87.5, + shape: 'horizontal-band', + size: 'small', + location: 'top-left', + averageBaselineColorHex: '#000000', + averageCurrentColorHex: '#ffffff', + baselineLuminance: 0, + currentLuminance: 255, + dominantChange: 'brighter', currentOverlayMatches: [ { ref: 'e1', @@ -1603,6 +1614,7 @@ test('formatScreenshotDiffText renders mismatch with pixel counts without color' test('formatScreenshotDiffText renders dimension mismatch', () => { const text = withNoColor(() => formatScreenshotDiffText({ + schemaVersion: 2, match: false, differentPixels: 100, totalPixels: 100, @@ -1623,6 +1635,7 @@ test('formatScreenshotDiffText renders diff path relative to cwd', () => { const cwd = process.cwd(); const text = withNoColor(() => formatScreenshotDiffText({ + schemaVersion: 2, match: false, differentPixels: 10, totalPixels: 100, @@ -1641,6 +1654,7 @@ test('formatScreenshotDiffText keeps absolute diff path outside cwd', () => { const diffPath = path.join(siblingDir, 'diff.png'); const text = withNoColor(() => formatScreenshotDiffText({ + schemaVersion: 2, match: false, differentPixels: 10, totalPixels: 100, @@ -1655,6 +1669,7 @@ test('formatScreenshotDiffText keeps absolute diff path outside cwd', () => { test('formatScreenshotDiffText uses ANSI colors when enabled', () => { const text = withColor(() => formatScreenshotDiffText({ + schemaVersion: 2, match: false, differentPixels: 10, totalPixels: 100, @@ -1670,6 +1685,7 @@ test('formatScreenshotDiffText uses ANSI colors when enabled', () => { test('formatScreenshotDiffText does not show diff path when images match', () => { const text = withNoColor(() => formatScreenshotDiffText({ + schemaVersion: 2, match: true, differentPixels: 0, totalPixels: 100, diff --git a/website/docs/docs/commands.md b/website/docs/docs/commands.md index 7dcabb821b..441d029a2a 100644 --- a/website/docs/docs/commands.md +++ b/website/docs/docs/commands.md @@ -852,7 +852,7 @@ agent-device record stop # Stop active recording - `screenshot --overlay-refs` captures a fresh full snapshot and burns visible `@eN` refs plus their target rectangles into the saved PNG. - `screenshot --normalize-status-bar` temporarily normalizes iOS simulator status-bar chrome for deterministic screenshot baselines; ordinary screenshots leave the simulator's current chrome visible. - `screenshot --max-size --overlay-refs` writes a smaller image and draws refs for that final image size; avoid very small max sizes when text, icons, or labels need to remain readable. -- `diff screenshot` compares the current live screenshot to `--baseline`, or compares `--baseline` to an optional saved `current.png` path without requiring an active session. Its text output reports ranked changed regions with screen-space rectangles, changed-pixel counts, and each region's share of the diff; JSON also includes normalized rectangles. It writes a diff PNG with a light grayscale current-screen context, red-tinted changed pixels, and outlined changed regions when `--out` is provided. Live iOS simulator diffs normalize status-bar chrome by default; use `screenshot --normalize-status-bar` when capturing reusable baselines. +- `diff screenshot` compares the current live screenshot to `--baseline`, or compares `--baseline` to an optional saved `current.png` path without requiring an active session. Its text output reports ranked changed regions with screen-space rectangles, changed-pixel counts, and each region's share of the diff; JSON also includes normalized rectangles. Structured results carry `schemaVersion: 2`; the earlier `ocr` and `nonTextDeltas` analysis payloads are retired, so use the baseline/current images and diff artifact with vision for qualitative interpretation. It writes a diff PNG with a light grayscale current-screen context, red-tinted changed pixels, and outlined changed regions when `--out` is provided. Live iOS simulator diffs normalize status-bar chrome by default; use `screenshot --normalize-status-bar` when capturing reusable baselines. - `diff screenshot --overlay-refs` additionally writes a separate current-screen overlay guide for live captures without using that annotated image for the pixel comparison. If current-screen refs intersect changed regions, the output lists the best ref matches under those regions. Saved-image comparisons do not have live accessibility refs, so `--overlay-refs` is unavailable when a `current.png` path is provided. - In `--json` mode, each overlay ref also includes a screenshot-space `center` point for coordinate fallback like `press `. - Burned-in touch overlays are exported only on macOS hosts, because the overlay pipeline depends on Swift + AVFoundation helpers. From 016ccc1229121b54a7a6590aecdf0e40191f78f3 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Micha=C5=82=20Pierzcha=C5=82a?= Date: Thu, 30 Jul 2026 14:20:25 +0200 Subject: [PATCH 5/5] fix: preserve screenshot diff result compatibility --- CHANGELOG.md | 2 +- src/__tests__/cli-diff.test.ts | 4 +- .../__tests__/screenshot-diff.test.ts | 3 +- src/screenshot-diff/screenshot-diff.ts | 41 +++++++++++++++---- src/utils/__tests__/output.test.ts | 7 ---- website/docs/docs/commands.md | 2 +- 6 files changed, 40 insertions(+), 19 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index 1914657491..e3cb588103 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -2,7 +2,7 @@ ## Unreleased -- Breaking: `diff screenshot` structured results now carry `schemaVersion: 2`. The retired `ocr` and `nonTextDeltas` payloads are no longer emitted; use the baseline/current images and diff artifact with vision for qualitative interpretation. Existing pixel counts and region fields remain available. +- `diff screenshot` no longer runs the retired best-effort OCR and non-text analyzers. Their optional `ocr` and `nonTextDeltas` fields remain in the result type for source compatibility but are no longer emitted; use the baseline/current images and diff artifact with vision for qualitative interpretation. - Breaking: removed the deprecated `--session-locked` and `--session-lock-conflicts` flags. Use `--session-lock reject|strip` instead; passing either old flag now fails with `Unknown flag: ... Use --session-lock reject|strip instead.` - Breaking: removed the `replay export --format` flag. `replay export` always writes Maestro YAML. - Breaking: removed the unused `LeaseAllocatePayload`, `LeaseHeartbeatPayload`, and `LeaseReleasePayload` type exports from `agent-device/contracts`. Lease request metadata is fully described by `DaemonRequestMeta`. diff --git a/src/__tests__/cli-diff.test.ts b/src/__tests__/cli-diff.test.ts index 86850e0336..c1fe7dc3f0 100644 --- a/src/__tests__/cli-diff.test.ts +++ b/src/__tests__/cli-diff.test.ts @@ -202,7 +202,9 @@ describe('cli diff commands', () => { const payload = JSON.parse(result.stdout); assert.equal(payload.success, true); assert.equal(payload.data.match, true); - assert.equal(payload.data.schemaVersion, 2); + assert.equal(Object.hasOwn(payload.data, 'schemaVersion'), false); + assert.equal(Object.hasOwn(payload.data, 'ocr'), false); + assert.equal(Object.hasOwn(payload.data, 'nonTextDeltas'), false); assert.equal(payload.data.differentPixels, 0); assert.equal(payload.data.totalPixels, 100); assert.equal(payload.data.mismatchPercentage, 0); diff --git a/src/screenshot-diff/__tests__/screenshot-diff.test.ts b/src/screenshot-diff/__tests__/screenshot-diff.test.ts index 6c1f9910ce..e50889d15e 100644 --- a/src/screenshot-diff/__tests__/screenshot-diff.test.ts +++ b/src/screenshot-diff/__tests__/screenshot-diff.test.ts @@ -74,7 +74,8 @@ test('identical images produce match: true with 0% mismatch', async () => { assert.equal(result.differentPixels, 0); assert.equal(result.mismatchPercentage, 0); assert.equal(result.totalPixels, 100); - assert.equal(result.schemaVersion, 2); + assert.equal(result.ocr, undefined); + assert.equal(result.nonTextDeltas, undefined); assert.equal(Object.hasOwn(result, 'ocr'), false); assert.equal(Object.hasOwn(result, 'nonTextDeltas'), false); assert.equal(result.dimensionMismatch, undefined); diff --git a/src/screenshot-diff/screenshot-diff.ts b/src/screenshot-diff/screenshot-diff.ts index 8ea268b1ea..328360c551 100644 --- a/src/screenshot-diff/screenshot-diff.ts +++ b/src/screenshot-diff/screenshot-diff.ts @@ -1,6 +1,7 @@ import { promises as fs } from 'node:fs'; import path from 'node:path'; import { AppError } from '@agent-device/kernel/errors'; +import type { Rect } from '@agent-device/kernel/snapshot'; import { PNG } from '../utils/png.ts'; import { computeScreenshotDiffPixelsAsync, @@ -16,13 +17,35 @@ export type ScreenshotDimensionMismatch = { actual: ImageDimensions; }; +type ScreenshotOcrSummary = { + provider: 'tesseract'; + baselineBlocks: number; + currentBlocks: number; + matches: Array<{ + text: string; + baselineRect: Rect; + currentRect: Rect; + delta: Rect; + confidence: number; + possibleTextMetricMismatch: boolean; + }>; + movementClusters?: Array<{ + texts: string[]; + xRange: { min: number; max: number }; + yRange: { min: number; max: number }; + }>; +}; + +type ScreenshotNonTextDelta = { + index: number; + regionIndex?: number; + slot: 'leading' | 'trailing' | 'background' | 'separator' | 'unknown'; + likelyKind: 'icon' | 'toggle' | 'chevron' | 'separator' | 'visual'; + rect: Rect; + nearestText?: string; +}; + export type ScreenshotDiffResult = { - /** - * Version of the structured screenshot-diff result. Version 2 removes the - * retired analysis payloads while preserving the established pixel and - * region contracts. - */ - schemaVersion: 2; diffPath?: string; totalPixels: number; differentPixels: number; @@ -32,6 +55,10 @@ export type ScreenshotDiffResult = { regions?: ScreenshotDiffRegion[]; currentOverlayPath?: string; currentOverlayRefCount?: number; + /** @deprecated Retained for source compatibility; OCR analysis is no longer emitted. */ + ocr?: ScreenshotOcrSummary; + /** @deprecated Retained for source compatibility; non-text analysis is no longer emitted. */ + nonTextDeltas?: ScreenshotNonTextDelta[]; }; export type ScreenshotDiffOptions = { @@ -77,7 +104,6 @@ export async function compareScreenshots( await removeStaleDiffOutput(options.outputPath); return { match: false, - schemaVersion: 2, mismatchPercentage: 100, totalPixels, differentPixels: totalPixels, @@ -126,7 +152,6 @@ export async function compareScreenshots( totalPixels > 0 ? Math.round((differentPixels / totalPixels) * 100 * 100) / 100 : 0; return { - schemaVersion: 2, ...(differentPixels > 0 && diffOutputPath ? { diffPath: diffOutputPath } : {}), ...(regions.length > 0 ? { regions } : {}), totalPixels, diff --git a/src/utils/__tests__/output.test.ts b/src/utils/__tests__/output.test.ts index c669205f3a..054fa28732 100644 --- a/src/utils/__tests__/output.test.ts +++ b/src/utils/__tests__/output.test.ts @@ -1550,7 +1550,6 @@ function withColor(fn: () => T): T { test('formatScreenshotDiffText renders match success without color', () => { const text = withNoColor(() => formatScreenshotDiffText({ - schemaVersion: 2, match: true, differentPixels: 0, totalPixels: 100, @@ -1564,7 +1563,6 @@ test('formatScreenshotDiffText renders match success without color', () => { test('formatScreenshotDiffText renders mismatch with pixel counts without color', () => { const text = withNoColor(() => formatScreenshotDiffText({ - schemaVersion: 2, match: false, differentPixels: 500, totalPixels: 10000, @@ -1614,7 +1612,6 @@ test('formatScreenshotDiffText renders mismatch with pixel counts without color' test('formatScreenshotDiffText renders dimension mismatch', () => { const text = withNoColor(() => formatScreenshotDiffText({ - schemaVersion: 2, match: false, differentPixels: 100, totalPixels: 100, @@ -1635,7 +1632,6 @@ test('formatScreenshotDiffText renders diff path relative to cwd', () => { const cwd = process.cwd(); const text = withNoColor(() => formatScreenshotDiffText({ - schemaVersion: 2, match: false, differentPixels: 10, totalPixels: 100, @@ -1654,7 +1650,6 @@ test('formatScreenshotDiffText keeps absolute diff path outside cwd', () => { const diffPath = path.join(siblingDir, 'diff.png'); const text = withNoColor(() => formatScreenshotDiffText({ - schemaVersion: 2, match: false, differentPixels: 10, totalPixels: 100, @@ -1669,7 +1664,6 @@ test('formatScreenshotDiffText keeps absolute diff path outside cwd', () => { test('formatScreenshotDiffText uses ANSI colors when enabled', () => { const text = withColor(() => formatScreenshotDiffText({ - schemaVersion: 2, match: false, differentPixels: 10, totalPixels: 100, @@ -1685,7 +1679,6 @@ test('formatScreenshotDiffText uses ANSI colors when enabled', () => { test('formatScreenshotDiffText does not show diff path when images match', () => { const text = withNoColor(() => formatScreenshotDiffText({ - schemaVersion: 2, match: true, differentPixels: 0, totalPixels: 100, diff --git a/website/docs/docs/commands.md b/website/docs/docs/commands.md index 441d029a2a..dc15526e2f 100644 --- a/website/docs/docs/commands.md +++ b/website/docs/docs/commands.md @@ -852,7 +852,7 @@ agent-device record stop # Stop active recording - `screenshot --overlay-refs` captures a fresh full snapshot and burns visible `@eN` refs plus their target rectangles into the saved PNG. - `screenshot --normalize-status-bar` temporarily normalizes iOS simulator status-bar chrome for deterministic screenshot baselines; ordinary screenshots leave the simulator's current chrome visible. - `screenshot --max-size --overlay-refs` writes a smaller image and draws refs for that final image size; avoid very small max sizes when text, icons, or labels need to remain readable. -- `diff screenshot` compares the current live screenshot to `--baseline`, or compares `--baseline` to an optional saved `current.png` path without requiring an active session. Its text output reports ranked changed regions with screen-space rectangles, changed-pixel counts, and each region's share of the diff; JSON also includes normalized rectangles. Structured results carry `schemaVersion: 2`; the earlier `ocr` and `nonTextDeltas` analysis payloads are retired, so use the baseline/current images and diff artifact with vision for qualitative interpretation. It writes a diff PNG with a light grayscale current-screen context, red-tinted changed pixels, and outlined changed regions when `--out` is provided. Live iOS simulator diffs normalize status-bar chrome by default; use `screenshot --normalize-status-bar` when capturing reusable baselines. +- `diff screenshot` compares the current live screenshot to `--baseline`, or compares `--baseline` to an optional saved `current.png` path without requiring an active session. Its text output reports ranked changed regions with screen-space rectangles, changed-pixel counts, and each region's share of the diff; JSON also includes normalized rectangles. The earlier best-effort `ocr` and `nonTextDeltas` analyzers are retired; their optional result fields remain for source compatibility but are no longer emitted, so use the baseline/current images and diff artifact with vision for qualitative interpretation. It writes a diff PNG with a light grayscale current-screen context, red-tinted changed pixels, and outlined changed regions when `--out` is provided. Live iOS simulator diffs normalize status-bar chrome by default; use `screenshot --normalize-status-bar` when capturing reusable baselines. - `diff screenshot --overlay-refs` additionally writes a separate current-screen overlay guide for live captures without using that annotated image for the pixel comparison. If current-screen refs intersect changed regions, the output lists the best ref matches under those regions. Saved-image comparisons do not have live accessibility refs, so `--overlay-refs` is unavailable when a `current.png` path is provided. - In `--json` mode, each overlay ref also includes a screenshot-space `center` point for coordinate fallback like `press `. - Burned-in touch overlays are exported only on macOS hosts, because the overlay pipeline depends on Swift + AVFoundation helpers.