feat(recording): choose the frames a contact sheet shows

packages/capture-kit/src/png-changed-pixel-ratio.test.ts

@@ -0,0 +1,47 @@
+import { describe, expect, test } from 'vitest';
+import { computePngChangedPixelRatio } from './png-changed-pixel-ratio.ts';
+import { BLACK, paintPng, solidPng, WHITE } from './png-pixels.fixtures.ts';
+
+function pixels(png: { width: number; height: number; data: Buffer }) {
+  return { width: png.width, height: png.height, data: png.data };
+}
+
+describe('computePngChangedPixelRatio', () => {
+  test('reports nothing moved for identical frames', () => {
+    expect(
+      computePngChangedPixelRatio(pixels(solidPng(4, 4, BLACK)), pixels(solidPng(4, 4, BLACK))),
+    ).toEqual({ status: 'compared', changedPixelRatio: 0 });
+  });
+
+  test('counts a pixel whose blue channel alone moved', () => {
+    const blue = paintPng(
+      solidPng(4, 4, BLACK),
+      { x: 0, y: 0, width: 1, height: 1 },
+      [0, 0, 9, 255],
+    );
+    expect(computePngChangedPixelRatio(pixels(solidPng(4, 4, BLACK)), pixels(blue))).toEqual({
+      status: 'compared',
+      changedPixelRatio: 1 / 16,
+    });
+  });
+
+  test('ignores alpha, which never reaches the recorded screen', () => {
+    const transparent = solidPng(4, 4, BLACK);
+    transparent.data[3] = 0;
+    expect(computePngChangedPixelRatio(pixels(solidPng(4, 4, BLACK)), pixels(transparent))).toEqual(
+      { status: 'compared', changedPixelRatio: 0 },
+    );
+  });
+
+  test('counts every pixel when the whole frame moved', () => {
+    expect(
+      computePngChangedPixelRatio(pixels(solidPng(4, 4, BLACK)), pixels(solidPng(4, 4, WHITE))),
+    ).toEqual({ status: 'compared', changedPixelRatio: 1 });
+  });
+
+  test('refuses a ratio across frames of different shapes', () => {
+    expect(
+      computePngChangedPixelRatio(pixels(solidPng(4, 4, BLACK)), pixels(solidPng(4, 8, BLACK))),
+    ).toEqual({ status: 'dimension_mismatch' });
+  });
+});

packages/capture-kit/src/png-changed-pixel-ratio.ts

@@ -0,0 +1,47 @@
+import type { PngRgbImage } from './png-rgb-difference.ts';
+
+export type { PngRgbImage };
+
+export type PngChangedPixelRatioResult =
+  | { readonly status: 'compared'; readonly changedPixelRatio: number }
+  | { readonly status: 'dimension_mismatch' };
+
+/**
+ * The share of pixels whose color moved at all between two decoded PNGs.
+ *
+ * A pixel counts as changed when any of its RGB channels differs, ignoring alpha, so a
+ * translucent-layer fade that leaves the composite untouched is not reported. This is a
+ * coverage metric, not a magnitude one: `computePngRgbDifference` answers how far the colors
+ * moved, and this answers how much of the frame moved. A recording that only shifts a 1 px
+ * progress line scores a ratio near zero either way.
+ *
+ * Dimensions must match. Callers comparing frames of different sizes are asking a different
+ * question, and returning a made-up ratio would hide it.
+ */
+export function computePngChangedPixelRatio(
+  first: PngRgbImage,
+  second: PngRgbImage,
+): PngChangedPixelRatioResult {
+  if (first.width !== second.width || first.height !== second.height) {
+    return { status: 'dimension_mismatch' };
+  }
+
+  const totalPixels = first.width * first.height;
+  if (totalPixels === 0) return { status: 'compared', changedPixelRatio: 0 };
+  if (first.data.length !== second.data.length) {
+    return { status: 'dimension_mismatch' };
+  }
+
+  let changedPixels = 0;
+  for (let offset = 0; offset + 3 < first.data.length; offset += 4) {
+    if (
+      first.data[offset] !== second.data[offset] ||
+      first.data[offset + 1] !== second.data[offset + 1] ||
+      first.data[offset + 2] !== second.data[offset + 2]
+    ) {
+      changedPixels += 1;
+    }
+  }
+
+  return { status: 'compared', changedPixelRatio: changedPixels / totalPixels };
+}

packages/capture-kit/src/png-pixels.fixtures.ts

@@ -1,21 +1,43 @@
 import { PNG } from './png.ts';
 
 /**
- * Decoded-frame builders for tests that reason about pixels: a solid fill is enough to name what a
- * test claims a frame contains without shipping a PNG fixture through the repo.
+ * Decoded-frame builders for tests that reason about pixels: a solid fill and one painted rectangle
+ * name exactly what a test claims changed between two frames.
  */
 
 export type Rgba = readonly [number, number, number, number];
+export type Rectangle = Readonly<{ x: number; y: number; width: number; height: number }>;
 
 export const BLACK: Rgba = [0, 0, 0, 255];
+export const WHITE: Rgba = [255, 255, 255, 255];
+export const RED: Rgba = [255, 0, 0, 255];
 
 export function solidPng(width: number, height: number, color: Rgba = BLACK): PNG {
-  const png = new PNG({ width, height });
-  for (let offset = 0; offset < png.data.length; offset += 4) {
-    png.data[offset] = color[0];
-    png.data[offset + 1] = color[1];
-    png.data[offset + 2] = color[2];
-    png.data[offset + 3] = color[3];
+  return fillPng(new PNG({ width, height }), () => true, color);
+}
+
+/** Paints one rectangle of `color` onto a copy of `source`, leaving the source untouched. */
+export function paintPng(source: PNG, rectangle: Rectangle, color: Rgba): PNG {
+  const copy = solidPng(source.width, source.height);
+  source.data.copy(copy.data);
+  const inside = (column: number, row: number) =>
+    column >= rectangle.x &&
+    column < rectangle.x + rectangle.width &&
+    row >= rectangle.y &&
+    row < rectangle.y + rectangle.height;
+  return fillPng(copy, inside, color);
+}
+
+function fillPng(png: PNG, paint: (column: number, row: number) => boolean, color: Rgba): PNG {
+  for (let row = 0; row < png.height; row += 1) {
+    for (let column = 0; column < png.width; column += 1) {
+      if (!paint(column, row)) continue;
+      const offset = (row * png.width + column) * 4;
+      png.data[offset] = color[0];
+      png.data[offset + 1] = color[1];
+      png.data[offset + 2] = color[2];
+      png.data[offset + 3] = color[3];
+    }
   }
   return png;
 }

packages/capture-kit/src/recording/contact-sheet-plan.test.ts

@@ -0,0 +1,69 @@
+import { describe, expect, test } from 'vitest';
+import { CONTACT_SHEET_DURATION_REASON } from './contact-sheet-report.ts';
+import { AppError } from '@agent-device/kernel/errors';
+import {
+  CONTACT_SHEET_SAMPLE_INTERVAL_MS,
+  MAX_CONTACT_SHEET_SAMPLED_FRAMES,
+  planContactSheetSampleTimes,
+} from './contact-sheet-plan.ts';
+
+const VIDEO = '/tmp/recording.mp4';
+
+function reasonOf(action: () => unknown): unknown {
+  try {
+    action();
+  } catch (error) {
+    return error instanceof AppError ? error.details?.reason : `not an AppError: ${String(error)}`;
+  }
+  return 'no error thrown';
+}
+
+describe('planContactSheetSampleTimes', () => {
+  test('names both endpoints of a clip no longer than one sample step', () => {
+    expect(planContactSheetSampleTimes(0, VIDEO)).toEqual({ durationMs: 0, timesMs: [0] });
+    // One step is not one frame: a clip can change between its first and last presentation sample,
+    // and the sheet promises its ending whatever the clip's length.
+    expect(planContactSheetSampleTimes(CONTACT_SHEET_SAMPLE_INTERVAL_MS, VIDEO)).toEqual({
+      durationMs: CONTACT_SHEET_SAMPLE_INTERVAL_MS,
+      timesMs: [0, CONTACT_SHEET_SAMPLE_INTERVAL_MS],
+    });
+    expect(planContactSheetSampleTimes(100, VIDEO)).toEqual({ durationMs: 100, timesMs: [0, 100] });
+  });
+
+  test('samples every step while the clip is shorter than the cap', () => {
+    expect(planContactSheetSampleTimes(1_000, VIDEO).timesMs).toEqual([0, 250, 500, 750, 1000]);
+  });
+
+  test('stretches the same grid over a long clip instead of adding samples', () => {
+    const hour = planContactSheetSampleTimes(3_600_000, VIDEO);
+
+    expect(hour.timesMs).toHaveLength(MAX_CONTACT_SHEET_SAMPLED_FRAMES);
+    expect(hour.timesMs[0]).toBe(0);
+    expect(hour.timesMs.at(-1)).toBe(3_600_000);
+    expect(new Set(hour.timesMs).size).toBe(MAX_CONTACT_SHEET_SAMPLED_FRAMES);
+    hour.timesMs.forEach((time, index) => {
+      if (index > 0) expect(time).toBeGreaterThan(hour.timesMs[index - 1]!);
+    });
+  });
+
+  test('keeps both endpoints, which is what a coverage claim rests on', () => {
+    for (const durationMs of [5_000, 41_000, 900_000, 7_200_000]) {
+      const plan = planContactSheetSampleTimes(durationMs, VIDEO);
+      expect(plan.durationMs).toBe(durationMs);
+      expect(plan.timesMs[0]).toBe(0);
+      expect(plan.timesMs.at(-1)).toBe(durationMs);
+    }
+  });
+
+  test('refuses to plan over a timeline the container cannot name', () => {
+    expect(reasonOf(() => planContactSheetSampleTimes(undefined, VIDEO))).toBe(
+      CONTACT_SHEET_DURATION_REASON,
+    );
+    expect(reasonOf(() => planContactSheetSampleTimes(Number.NaN, VIDEO))).toBe(
+      CONTACT_SHEET_DURATION_REASON,
+    );
+    expect(reasonOf(() => planContactSheetSampleTimes(-1, VIDEO))).toBe(
+      CONTACT_SHEET_DURATION_REASON,
+    );
+  });
+});

packages/capture-kit/src/recording/contact-sheet-plan.ts

@@ -0,0 +1,60 @@
+import { CONTACT_SHEET_DURATION_REASON } from './contact-sheet-report.ts';
+import { AppError } from '@agent-device/kernel/errors';
+
+/** Spacing between requested sample times, in milliseconds. */
+export const CONTACT_SHEET_SAMPLE_INTERVAL_MS = 250;
+/** Sample times one sheet asks the decoder for, however long the clip runs. */
+export const MAX_CONTACT_SHEET_SAMPLED_FRAMES = 48;
+
+/**
+ * The times to sample from a clip of `durationMs`, spread evenly across the whole timeline.
+ *
+ * Coverage is the point: the grid always names the start and the end, and it never grows with the
+ * recording. A 5s take samples 21 times; a 2h take samples the same 48 times across its length, so
+ * a long recording costs the same decode budget as a short one and still shows both endpoints.
+ *
+ * A grid is a promise with a hole in it. A flash that opens and closes entirely between two sample
+ * times is not in the returned frames and no threshold can recover it; that is why the sheet
+ * reports how many times it sampled rather than claiming to have reviewed the footage.
+ *
+ * A clip whose timeline cannot be read is refused rather than guessed at. Sampling an unknown
+ * length would mean either walking the whole file or drawing a grid over a duration the container
+ * never claimed, and a sheet built that way could not say what it covered.
+ */
+export type ContactSheetSamplePlan = Readonly<{
+  /** Timeline the grid was planned over, in milliseconds. */
+  durationMs: number;
+  timesMs: readonly number[];
+}>;
+
+export function planContactSheetSampleTimes(
+  durationMs: number | undefined,
+  videoPath: string,
+): ContactSheetSamplePlan {
+  if (durationMs === undefined || !Number.isFinite(durationMs) || durationMs < 0) {
+    throw new AppError(
+      'COMMAND_FAILED',
+      `Cannot plan a contact sheet: the video timeline of ${videoPath} could not be read`,
+      {
+        reason: CONTACT_SHEET_DURATION_REASON,
+        videoPath,
+        hint: 'Retry once the recording finished finalizing; a clip whose container is still being written has no readable duration.',
+      },
+    );
+  }
+  if (durationMs <= CONTACT_SHEET_SAMPLE_INTERVAL_MS) {
+    // A clip shorter than one sampling interval still ends somewhere, and the sheet always promises
+    // its last frame, so the closing sample is asked for even when it is the only other one.
+    return { durationMs, timesMs: durationMs > 0 ? [0, Math.round(durationMs)] : [0] };
+  }
+
+  const count = Math.min(
+    MAX_CONTACT_SHEET_SAMPLED_FRAMES,
+    Math.floor(durationMs / CONTACT_SHEET_SAMPLE_INTERVAL_MS) + 1,
+  );
+  const step = durationMs / (count - 1);
+  return {
+    durationMs,
+    timesMs: Array.from({ length: count }, (_, index) => Math.round(index * step)),
+  };
+}

packages/capture-kit/src/recording/contact-sheet-report.ts

@@ -5,6 +5,8 @@
 
 /** The host cannot extract frames at all: frame decoding is Apple AVFoundation tooling. */
 export const CONTACT_SHEET_UNSUPPORTED_HOST_REASON = 'contact_sheet_unsupported_host';
+/** The MP4 timeline could not be read, so no bounded sample grid can be planned. */
+export const CONTACT_SHEET_DURATION_REASON = 'contact_sheet_duration_unknown';
 /** Frame extraction ran and failed, rather than returning fewer frames. */
 export const CONTACT_SHEET_EXTRACTION_REASON = 'contact_sheet_frame_extraction_failed';
 /** Extraction returned nothing usable, so there is no sheet to draw. */

packages/capture-kit/src/recording/contact-sheet-selection.test.ts

@@ -0,0 +1,98 @@
+import { describe, expect, test } from 'vitest';
+import { paintPng, RED, solidPng } from '../png-pixels.fixtures.ts';
+import {
+  MAX_CONTACT_SHEET_CELLS,
+  selectContactSheetCells,
+  type ContactSheetSample,
+} from './contact-sheet-selection.ts';
+
+const FRAME_SIZE = 10;
+
+function sample(timeMs: number, changedPixels: number, offset = 0): ContactSheetSample {
+  return {
+    timeMs,
+    image: paintPng(
+      solidPng(FRAME_SIZE, FRAME_SIZE),
+      { x: offset, y: 0, width: changedPixels, height: 1 },
+      RED,
+    ),
+  };
+}
+
+function distinctSample(index: number): ContactSheetSample {
+  // Frames of a different shape always count as fully changed, which keeps this fixture honest
+  // without asking it to invent pixels a video decoder would have had to produce.
+  return { timeMs: index * 100, image: solidPng(FRAME_SIZE + index, FRAME_SIZE) };
+}
+
+describe('selectContactSheetCells', () => {
+  test('returns nothing for no samples', () => {
+    expect(selectContactSheetCells([])).toEqual({ cells: [], keptCellCount: 0, thinned: false });
+  });
+
+  test('keeps the only frame it was given', () => {
+    const selection = selectContactSheetCells([sample(0, 0)]);
+
+    expect(selection.cells).toHaveLength(1);
+    expect(selection.cells[0]).toMatchObject({ timeMs: 0, changedPixelRatio: 1 });
+    expect(selection.thinned).toBe(false);
+  });
+
+  test('ends with the final frame even when nothing visibly moved', () => {
+    const selection = selectContactSheetCells([sample(0, 0), sample(250, 0), sample(500, 0)], 0.04);
+
+    expect(selection.cells.map((cell) => cell.timeMs)).toEqual([0, 500]);
+    expect(selection.cells.at(-1)?.changedPixelRatio).toBe(0);
+  });
+
+  test('measures against the last kept cell so small changes add up', () => {
+    const selection = selectContactSheetCells(
+      [
+        sample(0, 0),
+        // 1% of the frame: too small on its own, and it does not become the baseline.
+        sample(250, 1),
+        // 10% of the frame, but only 9% against the previous sample.
+        sample(500, FRAME_SIZE),
+      ],
+      0.1,
+    );
+
+    expect(selection.cells.map((cell) => cell.timeMs)).toEqual([0, 500]);
+    expect(selection.cells[1]?.changedPixelRatio).toBeCloseTo(0.1, 10);
+  });
+
+  test('keeps a frame whose shape changed', () => {
+    const selection = selectContactSheetCells(
+      [sample(0, 0), { timeMs: 250, image: solidPng(20, FRAME_SIZE) }],
+      0.5,
+    );
+
+    expect(selection.cells.map((cell) => cell.timeMs)).toEqual([0, 250]);
+    expect(selection.cells[1]?.changedPixelRatio).toBe(1);
+  });
+
+  test('thins evenly across the kept sequence and keeps both ends', () => {
+    const samples = Array.from({ length: 30 }, (_, index) => distinctSample(index));
+
+    const selection = selectContactSheetCells(samples, 0.04, 4);
+
+    expect(selection.keptCellCount).toBe(30);
+    expect(selection.thinned).toBe(true);
+    expect(selection.cells).toHaveLength(4);
+    expect(selection.cells[0]?.timeMs).toBe(0);
+    expect(selection.cells.at(-1)?.timeMs).toBe(2_900);
+    const times = selection.cells.map((cell) => cell.timeMs);
+    expect(new Set(times).size).toBe(times.length);
+  });
+
+  test('prints every kept cell while the count stays under the sheet cap', () => {
+    const samples = Array.from({ length: MAX_CONTACT_SHEET_CELLS }, (_, index) =>
+      distinctSample(index),
+    );
+
+    const selection = selectContactSheetCells(samples);
+
+    expect(selection.thinned).toBe(false);
+    expect(selection.cells).toHaveLength(MAX_CONTACT_SHEET_CELLS);
+  });
+});