fix(server): stop storing tool image bytes no client reads

apps/server/src/assets/AssetAccess.ts

@@ -1,7 +1,6 @@
 import type { AssetResource } from "@t3tools/contracts";
 import {
   AssetAttachmentNotFoundError,
-  PROVIDER_SEND_TURN_MAX_IMAGE_BYTES,
   AssetGitHubMediaUrlValidationError,
   AssetPreviewTypeValidationError,
   AssetProjectFaviconInspectionError,
@@ -33,7 +32,7 @@ import {
 } from "@t3tools/shared/imageDimensions";
 import { githubMediaFetchUrl, githubMediaFileName } from "@t3tools/shared/githubMedia";
 import { PROJECT_FAVICON_FALLBACK_MARKER } from "@t3tools/shared/projectFavicon";
-import { toolOutputImages } from "@t3tools/shared/toolOutput";
+import { MAX_TOOL_OUTPUT_IMAGE_BASE64_LENGTH, toolOutputImages } from "@t3tools/shared/toolOutput";
 import * as Clock from "effect/Clock";
 import * as Crypto from "effect/Crypto";
 import * as Effect from "effect/Effect";
@@ -216,9 +215,6 @@ const optionOnNotFound = <A, R>(
     }),
   );
 
-// The largest image a provider turn accepts, as base64 (4 characters per 3 bytes).
-const MAX_TOOL_OUTPUT_IMAGE_BASE64_LENGTH = Math.ceil(PROVIDER_SEND_TURN_MAX_IMAGE_BYTES / 3) * 4;
-
 /**
  * Decodes one image a tool returned inline; null when the stored item has no
  * such image, or it is larger than a provider turn accepts.

apps/server/src/orchestration-v2/ProviderEventIngestor.test.ts

@@ -27,6 +27,7 @@ import * as Layer from "effect/Layer";
 import * as Stream from "effect/Stream";
 import * as TestClock from "effect/testing/TestClock";
 
+import { toolOutputImages } from "@t3tools/shared/toolOutput";
 import * as SqlitePersistence from "../persistence/Sqlite.ts";
 import * as EventSink from "./EventSink.ts";
 import * as EventStore from "./EventStore.ts";
@@ -1169,6 +1170,97 @@ layer("ProviderEventIngestorV2", (it) => {
     }),
   );
 
+  it.effect("stores tool image bytes only where a tool-output-image asset serves them", () =>
+    Effect.gen(function* () {
+      const now = yield* DateTime.now;
+      const eventSink = yield* EventSink.EventSinkV2;
+      const eventStore = yield* EventStore.EventStoreV2;
+      const projectionStore = yield* ProjectionStore.ProjectionStoreV2;
+      const ingestor = yield* ProviderEventIngestor.ProviderEventIngestorV2;
+      const idAllocator = yield* IdAllocator.IdAllocatorV2;
+      const threadEvent = yield* threadCreatedEvent(now);
+      const providerSessionId = yield* idAllocator.allocate.providerSession({
+        providerInstanceId: modelSelection.instanceId,
+        threadId: threadEvent.threadId,
+      });
+      const readBase64 = Buffer.alloc(30_000, 7).toString("base64");
+      const screenshotBase64 = Buffer.alloc(20_000, 9).toString("base64");
+      const toolItem = (
+        id: string,
+        ordinal: number,
+        toolName: string,
+        output: unknown,
+      ): OrchestrationV2TurnItem => ({
+        id: TurnItemId.make(id),
+        threadId: threadEvent.threadId,
+        runId: null,
+        nodeId: null,
+        providerThreadId: null,
+        providerTurnId: null,
+        nativeItemRef: null,
+        parentItemId: null,
+        ordinal,
+        status: "completed",
+        title: toolName,
+        startedAt: now,
+        completedAt: now,
+        updatedAt: now,
+        type: "dynamic_tool",
+        toolName,
+        input: {},
+        output,
+      });
+      const read = toolItem("turn-item:read-image", 1, "Read", {
+        type: "image",
+        file: { base64: readBase64, type: "image/png", originalSize: 30_000 },
+      });
+      const screenshot = toolItem("turn-item:screenshot", 2, "mcp__t3-code__device_screenshot", {
+        content: [
+          {
+            type: "image",
+            source: { type: "base64", media_type: "image/png", data: screenshotBase64 },
+          },
+        ],
+      });
+
+      yield* eventSink.write({ events: [threadEvent] });
+      for (const turnItem of [read, screenshot]) {
+        yield* ingestor.ingestNormalized({
+          providerSessionId,
+          providerInstanceId: modelSelection.instanceId,
+          threadId: threadEvent.threadId,
+          event: { type: "turn_item.updated", driver: CODEX_DRIVER, turnItem },
+        });
+      }
+
+      const storedEvents = yield* eventStore
+        .read({ threadId: threadEvent.threadId, eventType: "turn-item.updated" })
+        .pipe(Stream.runCollect);
+      const storedJson = JSON.stringify(Array.from(storedEvents, (stored) => stored.event));
+      const projectedRead = yield* projectionStore.getTurnItem({
+        threadId: threadEvent.threadId,
+        itemId: read.id,
+      });
+      const projectedScreenshot = yield* projectionStore.getTurnItem({
+        threadId: threadEvent.threadId,
+        itemId: screenshot.id,
+      });
+
+      assert.equal(storedJson.includes(readBase64), false);
+      assert.equal(storedJson.includes(screenshotBase64), true);
+      assert.deepEqual(projectedRead?.type === "dynamic_tool" ? projectedRead.output : null, {
+        type: "image",
+        file: { type: "image/png", originalSize: 30_000, sizeBytes: 30_000 },
+      });
+      assert.deepEqual(
+        toolOutputImages(
+          projectedScreenshot?.type === "dynamic_tool" ? projectedScreenshot.output : null,
+        ),
+        [{ mimeType: "image/png", data: screenshotBase64 }],
+      );
+    }),
+  );
+
   it.effect("routes provider-owned child artifacts to their child app thread", () =>
     Effect.gen(function* () {
       const now = yield* DateTime.now;

apps/server/src/orchestration-v2/ProviderEventIngestor.ts

@@ -33,6 +33,7 @@ import * as IdAllocator from "./IdAllocator.ts";
 import { ProviderAdapterV2Event } from "./ProviderAdapter.ts";
 import { makeProviderFailureTurnItem } from "./ProviderFailure.ts";
 import * as ThreadCommandExecutor from "./ThreadCommandExecutor.ts";
+import { stripUnservedToolOutputImageBytes } from "./toolOutputImageBytes.ts";
 
 export class ProviderEventNormalizeError extends Schema.TaggedError<ProviderEventNormalizeError>()(
   "ProviderEventNormalizeError",
@@ -465,7 +466,7 @@ export const layer: Layer.Layer<
               yield* makeDomainEvent(input, {
                 type: "turn-item.updated",
                 threadId: input.event.turnItem.threadId,
-                payload: input.event.turnItem,
+                payload: stripUnservedToolOutputImageBytes(input.event.turnItem),
                 runId: input.event.turnItem.runId,
                 nodeId: input.event.turnItem.nodeId,
               }),

apps/server/src/orchestration-v2/toolOutputImageBytes.test.ts

@@ -0,0 +1,234 @@
+import { NodeId, ThreadId, TurnItemId, type OrchestrationV2TurnItem } from "@t3tools/contracts";
+import {
+  MAX_TOOL_OUTPUT_IMAGE_BASE64_LENGTH,
+  MAX_TOOL_OUTPUT_IMAGES,
+  toolOutputImages,
+} from "@t3tools/shared/toolOutput";
+import * as DateTime from "effect/DateTime";
+import { describe, expect, it } from "vite-plus/test";
+
+import { stripUnservedToolOutputImageBytes } from "./toolOutputImageBytes.ts";
+
+const PNG_BYTES = 3_000;
+const PNG_BASE64 = Buffer.alloc(PNG_BYTES, 7).toString("base64");
+// 3,001 bytes encode with `=` padding, which must not count as data.
+const PADDED_BASE64 = Buffer.alloc(PNG_BYTES + 1, 9).toString("base64");
+
+const now = DateTime.makeUnsafe("2026-10-06T00:00:00.000Z");
+
+const itemBase = {
+  id: TurnItemId.make("turn-item:tool"),
+  threadId: ThreadId.make("thread:tool"),
+  runId: null,
+  nodeId: NodeId.make("node:tool"),
+  providerThreadId: null,
+  providerTurnId: null,
+  nativeItemRef: null,
+  parentItemId: null,
+  ordinal: 1,
+  status: "completed",
+  title: null,
+  startedAt: now,
+  completedAt: now,
+  updatedAt: now,
+} as const;
+
+function toolItem(output: unknown, toolName = "Read"): OrchestrationV2TurnItem {
+  return {
+    ...itemBase,
+    type: "dynamic_tool",
+    toolName,
+    input: { file_path: "/tmp/shot.png" },
+    output,
+  };
+}
+
+function outputOf(item: OrchestrationV2TurnItem): unknown {
+  return item.type === "dynamic_tool" ? item.output : undefined;
+}
+
+describe("stripUnservedToolOutputImageBytes", () => {
+  it("replaces the base64 of Claude's structured Read result with its measured size", () => {
+    const item = toolItem({
+      type: "image",
+      file: {
+        base64: PNG_BASE64,
+        type: "image/png",
+        originalSize: PNG_BYTES,
+        dimensions: { originalWidth: 180, originalHeight: 180 },
+      },
+    });
+
+    expect(outputOf(stripUnservedToolOutputImageBytes(item))).toEqual({
+      type: "image",
+      file: {
+        type: "image/png",
+        originalSize: PNG_BYTES,
+        dimensions: { originalWidth: 180, originalHeight: 180 },
+        sizeBytes: PNG_BYTES,
+      },
+    });
+  });
+
+  it("strips ACP image shapes: Grok's native ImageContent and an MCP result behind an error", () => {
+    const item = toolItem({
+      rawOutput: { type: "ReadFile", ImageContent: { data: PNG_BASE64, mime_type: "image/png" } },
+      failed: {
+        error: "partial result",
+        result: [{ type: "image", data: PADDED_BASE64, mimeType: "image/png" }],
+      },
+    });
+
+    expect(outputOf(stripUnservedToolOutputImageBytes(item))).toEqual({
+      rawOutput: {
+        type: "ReadFile",
+        ImageContent: { mime_type: "image/png", sizeBytes: PNG_BYTES },
+      },
+      failed: {
+        error: "partial result",
+        result: [{ type: "image", mimeType: "image/png", sizeBytes: PNG_BYTES + 1 }],
+      },
+    });
+  });
+
+  it("strips a Cursor image part that names no mime type", () => {
+    const item = toolItem({
+      content: [{ text: { text: "captured" } }, { image: { data: PNG_BASE64 } }],
+    });
+
+    expect(outputOf(stripUnservedToolOutputImageBytes(item))).toEqual({
+      content: [{ text: { text: "captured" } }, { image: { sizeBytes: PNG_BYTES } }],
+    });
+  });
+
+  it("keeps the screenshots a tool-output-image asset serves", () => {
+    const claudeImage = {
+      type: "image",
+      source: { type: "base64", media_type: "image/png", data: PNG_BASE64 },
+    };
+    const mcpImage = { type: "image", data: PADDED_BASE64, mimeType: "image/jpeg" };
+    const item = toolItem(
+      { content: [{ type: "text", text: "captured" }, claudeImage, mcpImage] },
+      "mcp__t3-code__device_screenshot",
+    );
+
+    const stripped = stripUnservedToolOutputImageBytes(item);
+
+    expect(stripped).toBe(item);
+    expect(toolOutputImages(outputOf(stripped))).toEqual([
+      { mimeType: "image/png", data: PNG_BASE64 },
+      { mimeType: "image/jpeg", data: PADDED_BASE64 },
+    ]);
+  });
+
+  it("strips only the images past the last one an asset can serve", () => {
+    const images = Array.from({ length: MAX_TOOL_OUTPUT_IMAGES + 2 }, () => ({
+      type: "image",
+      data: PNG_BASE64,
+      mimeType: "image/png",
+    }));
+
+    const output = outputOf(stripUnservedToolOutputImageBytes(toolItem(images)));
+
+    expect(Array.isArray(output) ? output.slice(0, MAX_TOOL_OUTPUT_IMAGES) : null).toEqual(
+      images.slice(0, MAX_TOOL_OUTPUT_IMAGES),
+    );
+    expect(Array.isArray(output) ? output.slice(MAX_TOOL_OUTPUT_IMAGES) : null).toEqual([
+      { type: "image", mimeType: "image/png", sizeBytes: PNG_BYTES },
+      { type: "image", mimeType: "image/png", sizeBytes: PNG_BYTES },
+    ]);
+  });
+
+  it("strips a served-position image larger than an asset may serve", () => {
+    const oversized = "A".repeat(MAX_TOOL_OUTPUT_IMAGE_BASE64_LENGTH + 4);
+    const servable = { type: "image", data: PNG_BASE64, mimeType: "image/png" };
+    const item = toolItem({
+      content: [
+        { type: "image", source: { type: "base64", media_type: "image/png", data: oversized } },
+        servable,
+      ],
+    });
+
+    const output = outputOf(stripUnservedToolOutputImageBytes(item)) as {
+      content: ReadonlyArray<unknown>;
+    };
+
+    expect(output.content).toEqual([
+      {
+        type: "image",
+        source: {
+          type: "base64",
+          media_type: "image/png",
+          sizeBytes: ((MAX_TOOL_OUTPUT_IMAGE_BASE64_LENGTH + 4) / 4) * 3,
+        },
+      },
+      servable,
+    ]);
+    expect(output.content[1]).toBe(servable);
+  });
+
+  it("keeps image fields whose value is not a base64 body", () => {
+    const item = toolItem({
+      nested: {
+        byId: { type: "image", data: "asset-id-42" },
+        svg: {
+          mimeType: "image/svg+xml",
+          data: `<svg xmlns='http://www.w3.org/2000/svg'>${"<circle r='4'/>".repeat(8)}</svg>`,
+        },
+        dataUrl: { type: "image", data: `data:image/png;base64,${PNG_BASE64}` },
+        remote: { type: "image", source: { type: "url", url: "https://example.test/shot.png" } },
+        tiny: { type: "image", data: "AA==", mimeType: "image/png" },
+        notImage: { mimeType: "application/pdf", data: PNG_BASE64 },
+      },
+    });
+
+    expect(stripUnservedToolOutputImageBytes(item)).toBe(item);
+  });
+
+  it("measures the size instead of trusting one the provider supplied", () => {
+    const item = toolItem({
+      type: "image",
+      file: { base64: PNG_BASE64, type: "image/png", sizeBytes: 0 },
+    });
+
+    expect(outputOf(stripUnservedToolOutputImageBytes(item))).toEqual({
+      type: "image",
+      file: { type: "image/png", sizeBytes: PNG_BYTES },
+    });
+  });
+
+  it("returns items without image bytes by reference", () => {
+    const nested = { files: [{ path: "src/app.ts" }] };
+    const item = toolItem({ stdout: "data: 42", data: PNG_BASE64, nested });
+    const command: OrchestrationV2TurnItem = {
+      ...itemBase,
+      type: "command_execution",
+      input: "cat shot.png | base64",
+      output: PNG_BASE64,
+    };
+
+    const stripped = stripUnservedToolOutputImageBytes(item);
+
+    expect(stripped).toBe(item);
+    expect(outputOf(stripped)).toBe(outputOf(item));
+    expect(stripUnservedToolOutputImageBytes(command)).toBe(command);
+    expect(stripUnservedToolOutputImageBytes(toolItem("plain text output"))).toEqual(
+      toolItem("plain text output"),
+    );
+  });
+
+  it("does not mutate the provider's item and shares untouched siblings", () => {
+    const untouched = { type: "text", text: "captured" };
+    const output = {
+      error: "partial result",
+      result: [untouched, { type: "image", data: PNG_BASE64, mimeType: "image/png" }],
+    };
+    const item = toolItem(output);
+    const snapshot = structuredClone(output);
+
+    const stripped = outputOf(stripUnservedToolOutputImageBytes(item)) as typeof output;
+
+    expect(outputOf(item)).toEqual(snapshot);
+    expect(stripped.result[0]).toBe(untouched);
+  });
+});