diff --git a/package-lock.json b/package-lock.json index 84b1f4bc06..be85e472e2 100644 --- a/package-lock.json +++ b/package-lock.json @@ -26,13 +26,17 @@ "@cloudflare/vitest-pool-workers": "^0.16.10", "@tktco/node-actionlint": "^1.6.0", "@types/node": "^24.10.1", + "@types/pixelmatch": "^5.2.6", + "@types/pngjs": "^6.0.5", "@vitest/coverage-v8": "^4.1.7", "drizzle-kit": "^0.31.7", "git-cliff": "^2.13.1", "github-actionlint": "^1.7.12", "node-addon-api": "^8.5.0", "node-gyp": "^12.1.0", + "pixelmatch": "^7.2.0", "playwright": "^1.56.1", + "pngjs": "^7.0.0", "tsx": "^4.22.4", "typescript": "^5.9.3", "vitest": "^4.1.7", @@ -5718,6 +5722,26 @@ "undici-types": "~7.16.0" } }, + "node_modules/@types/pixelmatch": { + "version": "5.2.6", + "resolved": "https://registry.npmjs.org/@types/pixelmatch/-/pixelmatch-5.2.6.tgz", + "integrity": "sha512-wC83uexE5KGuUODn6zkm9gMzTwdY5L0chiK+VrKcDfEjzxh1uadlWTvOmAbCpnM9zx/Ww3f8uKlYQVnO/TrqVg==", + "dev": true, + "license": "MIT", + "dependencies": { + "@types/node": "*" + } + }, + "node_modules/@types/pngjs": { + "version": "6.0.5", + "resolved": "https://registry.npmjs.org/@types/pngjs/-/pngjs-6.0.5.tgz", + "integrity": "sha512-0k5eKfrA83JOZPppLtS2C7OUtyNAl2wKNxfyYl9Q5g9lPkgBl/9hNyAu6HuEH2J4XmIv2znEpkDd0SaZVxW6iQ==", + "dev": true, + "license": "MIT", + "dependencies": { + "@types/node": "*" + } + }, "node_modules/@types/react": { "version": "19.2.15", "resolved": "https://registry.npmjs.org/@types/react/-/react-19.2.15.tgz", @@ -11011,6 +11035,19 @@ "node": ">= 6" } }, + "node_modules/pixelmatch": { + "version": "7.2.0", + "resolved": "https://registry.npmjs.org/pixelmatch/-/pixelmatch-7.2.0.tgz", + "integrity": "sha512-xhcb4yHu9sM/G7foGzoLtXYcC0zHEaOXXjRKhGup0fw78Nf2Tkiapv4EQyMzrbcmQPsllAI7DbFY2UT7PlI9Pg==", + "dev": true, + "license": "ISC", + "dependencies": { + "pngjs": "^7.0.0" + }, + "bin": { + "pixelmatch": "bin/pixelmatch" + } + }, "node_modules/pkce-challenge": { "version": "5.0.1", "resolved": "https://registry.npmjs.org/pkce-challenge/-/pkce-challenge-5.0.1.tgz", @@ -11067,6 +11104,16 @@ "node": "^8.16.0 || ^10.6.0 || >=11.0.0" } }, + "node_modules/pngjs": { + "version": "7.0.0", + "resolved": "https://registry.npmjs.org/pngjs/-/pngjs-7.0.0.tgz", + "integrity": "sha512-LKWqWJRhstyYo9pGvgor/ivk2w94eSjE3RGVuzLGlr3NmD8bf7RcYGze1mNdEHRP6TRP6rMuDHk5t44hnTRyow==", + "dev": true, + "license": "MIT", + "engines": { + "node": ">=14.19.0" + } + }, "node_modules/postcss": { "version": "8.5.15", "resolved": "https://registry.npmjs.org/postcss/-/postcss-8.5.15.tgz", diff --git a/package.json b/package.json index 0ef2f27cd8..f1fefea49f 100644 --- a/package.json +++ b/package.json @@ -72,6 +72,8 @@ "@cloudflare/vitest-pool-workers": "^0.16.10", "@tktco/node-actionlint": "^1.6.0", "@types/node": "^24.10.1", + "@types/pixelmatch": "^5.2.6", + "@types/pngjs": "^6.0.5", "@vitest/coverage-v8": "^4.1.7", "drizzle-kit": "^0.31.7", "git-cliff": "^2.13.1", @@ -79,6 +81,8 @@ "node-addon-api": "^8.5.0", "node-gyp": "^12.1.0", "playwright": "^1.56.1", + "pixelmatch": "^7.2.0", + "pngjs": "^7.0.0", "tsx": "^4.22.4", "typescript": "^5.9.3", "vitest": "^4.1.7", diff --git a/src/visual-agent/visual-diff.ts b/src/visual-agent/visual-diff.ts new file mode 100644 index 0000000000..18a0394d5f --- /dev/null +++ b/src/visual-agent/visual-diff.ts @@ -0,0 +1,177 @@ +/** + * Agent-path visual diff utilities (Node `Buffer` + PNG decode). + * Must not be imported from the Worker entry (`src/index.ts`) or MCP bin bundle. + */ +import pixelmatch from "pixelmatch"; +import { PNG } from "pngjs"; + +export type VisualRouteStatus = "changed" | "unchanged" | "new" | "removed"; + +export type VisualDiffOptions = { + /** Pixelmatch anti-alias tolerance (0–1). Default 0.1. */ + threshold?: number; + /** Routes below this changed-pixel % are treated as unchanged noise. Default 0.05. */ + changeThresholdPercent?: number; + /** Include diff PNG bytes for changed routes. Default true. */ + includeDiffImage?: boolean; +}; + +export type VisualRouteComparison = { + route: string; + status: VisualRouteStatus; + changedPixelPercent: number | null; + width: number | null; + height: number | null; + diffImagePng: Buffer | null; +}; + +export type VisualDiffSummary = { + generatedAt: string; + routes: VisualRouteComparison[]; + changedCount: number; + unchangedCount: number; + newCount: number; + removedCount: number; + overallChangedPixelPercent: number; + summary: string; +}; + +const DEFAULT_THRESHOLD = 0.1; +const DEFAULT_CHANGE_THRESHOLD_PERCENT = 0.05; + +function decodePng(buffer: Buffer): PNG { + return PNG.sync.read(buffer); +} + +function changedPercent(diffPixels: number, width: number, height: number): number { + const total = width * height; + return roundPercent((diffPixels / Math.max(total, 1)) * 100); +} + +function roundPercent(value: number): number { + return Math.round(value * 10_000) / 10_000; +} + +function resolveOptions(options: VisualDiffOptions | undefined) { + return { + threshold: options?.threshold ?? DEFAULT_THRESHOLD, + changeThresholdPercent: options?.changeThresholdPercent ?? DEFAULT_CHANGE_THRESHOLD_PERCENT, + includeDiffImage: options?.includeDiffImage ?? true, + }; +} + +function comparePair(route: string, before: Buffer, after: Buffer, options: VisualDiffOptions | undefined): VisualRouteComparison { + const resolved = resolveOptions(options); + const beforeImage = decodePng(before); + const afterImage = decodePng(after); + if (beforeImage.width !== afterImage.width || beforeImage.height !== afterImage.height) { + return { + route, + status: "changed", + changedPixelPercent: 100, + width: Math.max(beforeImage.width, afterImage.width), + height: Math.max(beforeImage.height, afterImage.height), + diffImagePng: null, + }; + } + + const { width, height } = beforeImage; + const diff = new PNG({ width, height }); + const diffPixels = pixelmatch(beforeImage.data, afterImage.data, diff.data, width, height, { + threshold: resolved.threshold, + includeAA: true, + }); + const changedPixelPercent = changedPercent(diffPixels, width, height); + const status = changedPixelPercent >= resolved.changeThresholdPercent ? "changed" : "unchanged"; + return { + route, + status, + changedPixelPercent, + width, + height, + diffImagePng: status === "changed" && resolved.includeDiffImage ? PNG.sync.write(diff) : null, + }; +} + +export function compareRouteScreenshots(args: { + route: string; + before?: Buffer | null | undefined; + after?: Buffer | null | undefined; + options?: VisualDiffOptions; +}): VisualRouteComparison { + const { route, before, after, options } = args; + if (!before && !after) { + return { route, status: "unchanged", changedPixelPercent: 0, width: null, height: null, diffImagePng: null }; + } + if (!before && after) { + const afterImage = decodePng(after); + return { + route, + status: "new", + changedPixelPercent: null, + width: afterImage.width, + height: afterImage.height, + diffImagePng: null, + }; + } + if (before && !after) { + const beforeImage = decodePng(before); + return { + route, + status: "removed", + changedPixelPercent: null, + width: beforeImage.width, + height: beforeImage.height, + diffImagePng: null, + }; + } + return comparePair(route, before!, after!, options); +} + +export function compareVisualCaptureSets(args: { + before: Record; + after: Record; + options?: VisualDiffOptions; +}): VisualDiffSummary { + const routes = [...new Set([...Object.keys(args.before), ...Object.keys(args.after)])].sort((left, right) => left.localeCompare(right)); + const comparisons = routes.map((route) => { + const input: { + route: string; + before?: Buffer; + after?: Buffer; + options?: VisualDiffOptions; + } = { route }; + if (args.before[route]) input.before = args.before[route]; + if (args.after[route]) input.after = args.after[route]; + if (args.options) input.options = args.options; + return compareRouteScreenshots(input); + }); + + const changed = comparisons.filter((entry) => entry.status === "changed"); + const unchanged = comparisons.filter((entry) => entry.status === "unchanged"); + const added = comparisons.filter((entry) => entry.status === "new"); + const removed = comparisons.filter((entry) => entry.status === "removed"); + const measurable = comparisons.filter( + (entry): entry is VisualRouteComparison & { changedPixelPercent: number } => entry.changedPixelPercent !== null, + ); + const overallChangedPixelPercent = + measurable.length > 0 + ? roundPercent(measurable.reduce((sum, entry) => sum + entry.changedPixelPercent, 0) / measurable.length) + : 0; + + const summary = + changed.length > 0 + ? `${changed.length} route(s) changed (${overallChangedPixelPercent}% avg changed pixels); ${unchanged.length} unchanged, ${added.length} new, ${removed.length} removed.` + : `${unchanged.length} route(s) unchanged; ${added.length} new, ${removed.length} removed.`; + + return { + generatedAt: new Date().toISOString(), + routes: comparisons, + changedCount: changed.length, + unchangedCount: unchanged.length, + newCount: added.length, + removedCount: removed.length, + overallChangedPixelPercent, + summary, + }; +} diff --git a/test/unit/visual-diff.test.ts b/test/unit/visual-diff.test.ts new file mode 100644 index 0000000000..56fd1100cd --- /dev/null +++ b/test/unit/visual-diff.test.ts @@ -0,0 +1,205 @@ +import { PNG } from "pngjs"; +import { describe, expect, it } from "vitest"; +import { compareRouteScreenshots, compareVisualCaptureSets } from "../../src/visual-agent/visual-diff"; + +function createSolidPng(width: number, height: number, rgba: [number, number, number, number]): Buffer { + const png = new PNG({ width, height }); + for (let y = 0; y < height; y += 1) { + for (let x = 0; x < width; x += 1) { + const idx = (width * y + x) << 2; + png.data[idx] = rgba[0]; + png.data[idx + 1] = rgba[1]; + png.data[idx + 2] = rgba[2]; + png.data[idx + 3] = rgba[3]; + } + } + return PNG.sync.write(png); +} + +function createCheckerPng(width: number, height: number): Buffer { + const png = new PNG({ width, height }); + for (let y = 0; y < height; y += 1) { + for (let x = 0; x < width; x += 1) { + const idx = (width * y + x) << 2; + const light = (x + y) % 2 === 0; + png.data[idx] = light ? 240 : 20; + png.data[idx + 1] = light ? 240 : 20; + png.data[idx + 2] = light ? 240 : 20; + png.data[idx + 3] = 255; + } + } + return PNG.sync.write(png); +} + +describe("visual diff quantification", () => { + it("marks identical routes unchanged without a diff image", () => { + const png = createSolidPng(32, 24, [10, 20, 30, 255]); + const result = compareRouteScreenshots({ route: "/app", before: png, after: png }); + expect(result).toMatchObject({ status: "unchanged", changedPixelPercent: 0, diffImagePng: null }); + }); + + it("flags real visual changes with a diff image and changed-pixel percentage", () => { + const before = createSolidPng(40, 30, [255, 255, 255, 255]); + const after = createSolidPng(40, 30, [0, 0, 0, 255]); + const result = compareRouteScreenshots({ route: "/app", before, after }); + expect(result.status).toBe("changed"); + expect(result.changedPixelPercent).toBe(100); + expect(result.diffImagePng).toBeInstanceOf(Buffer); + expect(result.diffImagePng?.length).toBeGreaterThan(0); + }); + + it("suppresses sub-threshold noise as unchanged", () => { + const before = createSolidPng(100, 100, [250, 250, 250, 255]); + const afterPng = new PNG({ width: 100, height: 100 }); + afterPng.data.set(PNG.sync.read(before).data); + afterPng.data[400] = 240; + const after = PNG.sync.write(afterPng); + const noisy = compareRouteScreenshots({ + route: "/app", + before, + after, + options: { changeThresholdPercent: 1 }, + }); + expect(noisy.status).toBe("unchanged"); + expect((noisy.changedPixelPercent ?? 0)).toBeLessThan(1); + }); + + it("classifies new and removed routes", () => { + const beforeOnly = createSolidPng(20, 20, [100, 100, 100, 255]); + const afterOnly = createSolidPng(20, 20, [200, 200, 200, 255]); + expect(compareRouteScreenshots({ route: "/removed", before: beforeOnly, after: null })).toMatchObject({ + status: "removed", + changedPixelPercent: null, + }); + expect(compareRouteScreenshots({ route: "/new", before: null, after: afterOnly })).toMatchObject({ + status: "new", + changedPixelPercent: null, + }); + }); + + it("treats missing before/after captures as unchanged", () => { + expect(compareRouteScreenshots({ route: "/empty", before: null, after: null })).toMatchObject({ + status: "unchanged", + changedPixelPercent: 0, + diffImagePng: null, + }); + }); + + it("summarizes mixed route sets with overall changed-pixel average", () => { + const unchanged = createSolidPng(20, 20, [10, 10, 10, 255]); + const beforeChanged = createSolidPng(20, 20, [255, 0, 0, 255]); + const afterChanged = createSolidPng(20, 20, [0, 255, 0, 255]); + const summary = compareVisualCaptureSets({ + before: { + "/unchanged": unchanged, + "/changed": beforeChanged, + "/removed-only": createSolidPng(10, 10, [1, 2, 3, 255]), + }, + after: { + "/unchanged": unchanged, + "/changed": afterChanged, + "/new-only": createCheckerPng(10, 10), + }, + }); + + expect(summary.changedCount).toBe(1); + expect(summary.unchangedCount).toBe(1); + expect(summary.newCount).toBe(1); + expect(summary.removedCount).toBe(1); + expect(summary.routes.find((entry) => entry.route === "/changed")).toMatchObject({ status: "changed" }); + expect(summary.routes.find((entry) => entry.route === "/unchanged")).toMatchObject({ status: "unchanged" }); + expect(summary.summary).toMatch(/1 route\(s\) changed/i); + expect(summary.overallChangedPixelPercent).toBeGreaterThan(0); + }); + + it("treats dimension mismatches as changed", () => { + const before = createSolidPng(30, 20, [255, 255, 255, 255]); + const after = createSolidPng(40, 20, [255, 255, 255, 255]); + const result = compareRouteScreenshots({ route: "/app", before, after }); + expect(result).toMatchObject({ status: "changed", changedPixelPercent: 100, diffImagePng: null }); + }); + + it("treats height-only dimension mismatches as changed", () => { + const before = createSolidPng(20, 20, [255, 255, 255, 255]); + const after = createSolidPng(20, 30, [255, 255, 255, 255]); + const result = compareRouteScreenshots({ route: "/app", before, after }); + expect(result).toMatchObject({ + status: "changed", + changedPixelPercent: 100, + width: 20, + height: 30, + diffImagePng: null, + }); + }); + + it("honors a custom pixelmatch threshold option", () => { + const before = createSolidPng(10, 10, [255, 0, 0, 255]); + const after = createSolidPng(10, 10, [0, 255, 0, 255]); + const result = compareRouteScreenshots({ + route: "/app", + before, + after, + options: { threshold: 0.2 }, + }); + expect(result.status).toBe("changed"); + expect(result.diffImagePng).toBeInstanceOf(Buffer); + }); + + it("can omit diff images when requested", () => { + const before = createSolidPng(10, 10, [255, 0, 0, 255]); + const after = createSolidPng(10, 10, [0, 255, 0, 255]); + const result = compareRouteScreenshots({ + route: "/app", + before, + after, + options: { includeDiffImage: false }, + }); + expect(result.status).toBe("changed"); + expect(result.diffImagePng).toBeNull(); + }); + + it("reports an unchanged-only summary when no routes materially change", () => { + const png = createSolidPng(16, 16, [120, 120, 120, 255]); + const summary = compareVisualCaptureSets({ + before: { "/stable": png }, + after: { "/stable": png }, + }); + expect(summary.changedCount).toBe(0); + expect(summary.overallChangedPixelPercent).toBe(0); + expect(summary.summary).toMatch(/1 route\(s\) unchanged; 0 new, 0 removed/i); + }); + + it("handles new-only capture sets without measurable changed-pixel averages", () => { + const summary = compareVisualCaptureSets({ + before: {}, + after: { "/new-only": createSolidPng(12, 12, [1, 2, 3, 255]) }, + }); + expect(summary.changedCount).toBe(0); + expect(summary.newCount).toBe(1); + expect(summary.overallChangedPixelPercent).toBe(0); + expect(summary.summary).toMatch(/0 route\(s\) unchanged; 1 new, 0 removed/i); + }); + + it("forwards diff options to per-route comparisons", () => { + const png = createSolidPng(20, 20, [255, 255, 255, 255]); + const tweaked = createSolidPng(20, 20, [254, 255, 255, 255]); + const summary = compareVisualCaptureSets({ + before: { "/app": png }, + after: { "/app": tweaked }, + options: { changeThresholdPercent: 100, includeDiffImage: false }, + }); + expect(summary.routes[0]).toMatchObject({ status: "unchanged", diffImagePng: null }); + }); + + it("summarizes removed-only capture sets without measurable pixel deltas", () => { + const beforeOnly = createSolidPng(12, 12, [4, 5, 6, 255]); + const summary = compareVisualCaptureSets({ + before: { "/gone": beforeOnly }, + after: {}, + }); + expect(summary.removedCount).toBe(1); + expect(summary.changedCount).toBe(0); + expect(summary.overallChangedPixelPercent).toBe(0); + expect(summary.summary).toMatch(/0 route\(s\) unchanged; 0 new, 1 removed/i); + }); +}); diff --git a/test/unit/worker-entry-boundary.test.ts b/test/unit/worker-entry-boundary.test.ts new file mode 100644 index 0000000000..443c52e38f --- /dev/null +++ b/test/unit/worker-entry-boundary.test.ts @@ -0,0 +1,90 @@ +import { readFileSync, statSync } from "node:fs"; +import { dirname, join, resolve } from "node:path"; +import { fileURLToPath } from "node:url"; +import { describe, expect, it } from "vitest"; + +const root = resolve(dirname(fileURLToPath(import.meta.url)), "../.."); +const srcRoot = join(root, "src"); + +const WORKER_ENTRY = join(srcRoot, "index.ts"); +const MCP_BIN = join(root, "packages/gittensory-mcp/bin/gittensory-mcp.js"); + +const FORBIDDEN_PATH = /(?:^|\/)visual-agent\//; +const FORBIDDEN_IDENTIFIERS = /\b(?:pixelmatch|pngjs|visual-diff)\b/; + +function resolveLocalImport(fromFile: string, specifier: string): string | null { + if (!specifier.startsWith(".")) return null; + const base = dirname(fromFile); + const candidates = [ + join(base, specifier), + join(base, `${specifier}.ts`), + join(base, `${specifier}.tsx`), + join(base, specifier, "index.ts"), + ]; + for (const candidate of candidates) { + try { + statSync(candidate); + return candidate; + } catch { + // try next candidate + } + } + return null; +} + +function parseImportSpecifiers(filePath: string): string[] { + const content = readFileSync(filePath, "utf8"); + const specifiers = new Set(); + for (const match of content.matchAll(/(?:import|export)\s+[\s\S]*?\sfrom\s+["']([^"']+)["']/g)) { + specifiers.add(match[1]!); + } + for (const match of content.matchAll(/import\s*\(\s*["']([^"']+)["']\s*\)/g)) { + specifiers.add(match[1]!); + } + return [...specifiers]; +} + +function collectReachableSources(entryFile: string): string[] { + const queue = [entryFile]; + const seen = new Set(); + while (queue.length > 0) { + const file = queue.pop()!; + if (seen.has(file)) continue; + seen.add(file); + for (const specifier of parseImportSpecifiers(file)) { + const resolved = resolveLocalImport(file, specifier); + if (resolved && resolved.startsWith(srcRoot) && !seen.has(resolved)) { + queue.push(resolved); + } + } + } + return [...seen].sort(); +} + +function relativeToRoot(path: string): string { + return path.replace(`${root}/`, ""); +} + +describe("worker entry boundary", () => { + it("does not import visual-agent modules from the Worker bundle entry", () => { + const reachable = collectReachableSources(WORKER_ENTRY).map(relativeToRoot); + const forbidden = reachable.filter((path) => FORBIDDEN_PATH.test(path)); + expect(forbidden, `worker entry must not reach agent-only modules: ${forbidden.join(", ")}`).toEqual([]); + }); + + it("does not reference pixelmatch, pngjs, or visual-diff in worker-reachable source", () => { + const hits = collectReachableSources(WORKER_ENTRY) + .map((file) => { + const content = readFileSync(file, "utf8"); + return FORBIDDEN_IDENTIFIERS.test(content) ? relativeToRoot(file) : null; + }) + .filter((entry): entry is string => entry !== null); + expect(hits, `worker-reachable files must not mention Node-only visual diff deps: ${hits.join(", ")}`).toEqual([]); + }); + + it("does not reference visual diff modules in the published MCP bin bundle", () => { + const content = readFileSync(MCP_BIN, "utf8"); + expect(content).not.toMatch(FORBIDDEN_IDENTIFIERS); + expect(content).not.toMatch(/visual-agent/); + }); +});