Skip to content

Commit 278fdce

Browse files
t3dotggclaude
andauthored
test(server): data regression test now fails when recent fetch savings regress (#25795)
Co-authored-by: Claude Opus 5.5 <noreply@anthropic.com>
1 parent 0b02b21 commit 278fdce

4 files changed

Lines changed: 249 additions & 85 deletions

File tree

‎apps/server/integration/TransferBudgetReport.integration.ts‎

Lines changed: 57 additions & 18 deletions
Original file line numberDiff line numberDiff line change
@@ -11,7 +11,11 @@ import {
1111
TRANSFER_MEASURED_MCP_RESULT_BYTES,
1212
TRANSFER_MEASURED_TOOLS,
1313
} from "./fixtures/transferBudget.ts";
14-
import { LONG_TURN_WORK_ROWS } from "./TransferBudgetV2Fixture.integration.ts";
14+
import {
15+
LONG_TURN_WORK_ROWS,
16+
SHELL_IDLE_SUBAGENTS,
17+
SHELL_SETTLED_THREADS,
18+
} from "./TransferBudgetV2Fixture.integration.ts";
1519

1620
export interface WebSocketCatchUpMeasurement extends WebSocketTransferTotals {
1721
readonly mode: "replay" | "snapshot";
@@ -36,6 +40,12 @@ export interface TransferBudgetRun {
3640
/** The compact bounded snapshot web and mobile load when the long-turn thread opens. */
3741
readonly longTurnColdOpen: TimelineHttpMeasurement;
3842
readonly longTurnHistoryPage: TimelineHttpMeasurement;
43+
/** The same reads as an older client that only sends `compactTurnItems`, to show the savings. */
44+
readonly longTurnColdOpenLegacy: TimelineHttpMeasurement;
45+
readonly longTurnHistoryPageLegacy: TimelineHttpMeasurement;
46+
/** The cold-connect shell snapshot after the settled-thread fleet is written. */
47+
readonly fleetShell: TimelineHttpMeasurement;
48+
readonly fleetShellLegacy: TimelineHttpMeasurement;
3949
}
4050

4151
interface ProviderTransferBudget {
@@ -46,11 +56,12 @@ interface ProviderTransferBudget {
4656
readonly measuredTurnWebSocketMessages: number;
4757
}
4858

59+
// The measured turn decodes to about 22 KB; without live body dedupe its repeated answer makes it 26 KB.
4960
const TRANSFER_BUDGET = {
5061
totalWireBytes: 7_000,
5162
threadSnapshotWireBytes: 5_000,
5263
measuredTurnWebSocketWireBytes: 2_000,
53-
measuredTurnWebSocketDecodedBytes: 30_000,
64+
measuredTurnWebSocketDecodedBytes: 25_000,
5465
measuredTurnWebSocketMessages: 8,
5566
} satisfies ProviderTransferBudget;
5667

@@ -59,15 +70,20 @@ const TRANSFER_BUDGETS: Readonly<Record<string, ProviderTransferBudget>> = {
5970
claudeAgent: TRANSFER_BUDGET,
6071
};
6172

62-
// Decoded bytes, not wire bytes: phones parse and keep the decoded JSON. About
63-
// 30% above the observed 92 KB; without row budgets inside user turns the cold
64-
// open carried all 920 rows (1.07 MB). Kept apart from TRANSFER_BUDGET because
65-
// the PR comment validates that object's keys.
73+
// Decoded bytes, not wire bytes: phones parse and keep the decoded JSON. About 7% above the
74+
// observed 103 KB, so a cold open without body dedupe (117 KB), the ID table, or row budgets
75+
// (all 920 rows) fails. Kept apart from TRANSFER_BUDGET because the PR comment validates its keys.
6676
const LONG_TURN_BUDGET = {
67-
decodedBytes: 120_000,
77+
decodedBytes: 110_000,
6878
rows: 75,
6979
};
7080

81+
// About 30% above the observed 42 KB. Without the shell opt-ins it carries all 142 rows (217 KB).
82+
const FLEET_SHELL_BUDGET = {
83+
decodedBytes: 55_000,
84+
rows: 27,
85+
};
86+
7187
function totalWireBytes(run: TransferBudgetRun): number {
7288
return run.threadSnapshot.wireBytes + run.measuredTurnWebSocket.wireBytes;
7389
}
@@ -160,21 +176,28 @@ function webSocketRows(
160176
];
161177
}
162178

163-
function longTurnRows(
179+
function timelineRows(
164180
provider: ProviderDriverKind,
165181
phase: string,
166182
measured: TimelineHttpMeasurement,
183+
budget: { readonly decodedBytes: number; readonly rows: number },
167184
): string[] {
168185
return [
169186
infoRow(provider, phase, "HTTP wire", measured.http.wireBytes),
170-
row(
171-
provider,
172-
phase,
173-
"HTTP decoded",
174-
measured.http.decodedBodyBytes,
175-
LONG_TURN_BUDGET.decodedBytes,
176-
),
177-
row(provider, phase, "rows", measured.rows, LONG_TURN_BUDGET.rows, String),
187+
row(provider, phase, "HTTP decoded", measured.http.decodedBodyBytes, budget.decodedBytes),
188+
row(provider, phase, "rows", measured.rows, budget.rows, String),
189+
];
190+
}
191+
192+
function legacyRows(
193+
provider: ProviderDriverKind,
194+
phase: string,
195+
measured: TimelineHttpMeasurement,
196+
): string[] {
197+
return [
198+
infoRow(provider, `${phase}, older client`, "HTTP wire", measured.http.wireBytes),
199+
infoRow(provider, `${phase}, older client`, "HTTP decoded", measured.http.decodedBodyBytes),
200+
infoRow(provider, `${phase}, older client`, "rows", measured.rows, String),
178201
];
179202
}
180203

@@ -216,6 +239,12 @@ export function transferBudgetViolations(runs: ReadonlyArray<TransferBudgetRun>)
216239
LONG_TURN_BUDGET.decodedBytes,
217240
],
218241
["long-turn history page rows", run.longTurnHistoryPage.rows, LONG_TURN_BUDGET.rows],
242+
[
243+
"fleet shell snapshot decoded bytes",
244+
run.fleetShell.http.decodedBodyBytes,
245+
FLEET_SHELL_BUDGET.decodedBytes,
246+
],
247+
["fleet shell snapshot rows", run.fleetShell.rows, FLEET_SHELL_BUDGET.rows],
219248
] as const;
220249
for (const [metric, observed, maximum] of checks) {
221250
if (observed > maximum) {
@@ -234,6 +263,7 @@ export function formatTransferBudgetReport(runs: ReadonlyArray<TransferBudgetRun
234263
"The measured turn is observed on three sockets at once: one with only the thread subscription (the capped rows), one with only the shell subscription, and a second client holding both. Server egress is the sum of the three. After the turn the second client disconnects and new sockets resubscribe from the cursor it held before the turn, which is the cursor a backgrounded phone would hold. SQL statements are the SQL client's per-statement spans, counted across V2 persistence and the HTTP/WebSocket handlers.",
235264
`Scenario: ${TRANSFER_HISTORY_TURN_COUNT} historical turns with ${TRANSFER_HISTORY_TOOLS_PER_TURN} command tools and one retained ${formatBytes(TRANSFER_HISTORY_MCP_RESULT_BYTES)} MCP result each, followed by one measured turn with ${TRANSFER_MEASURED_TOOLS} command tools and a retained ${formatBytes(TRANSFER_MEASURED_MCP_RESULT_BYTES)} MCP result. Synthetic V2 domain events exercise persistence, wire projection, and production HTTP/subscription handlers; this does not measure provider adapter ingestion. Payloads contain no user data.`,
236265
`The long-turn rows open a second thread through the compact bounded snapshot web and mobile use, then load one older page. Its last user turn holds ${LONG_TURN_WORK_ROWS} rows of reasoning, commands, file changes, MCP calls, progress messages, checkpoints, and subagents.`,
266+
`The fleet shell rows read the cold-connect shell after ${SHELL_SETTLED_THREADS} settled threads and ${SHELL_IDLE_SUBAGENTS} idle subagent children are added. Budgeted reads send the opt-ins current clients send; "older client" rows show the same read without the opt-ins added since row budgets.`,
237267
"",
238268
"| Provider | Total thread wire | Budget | Result |",
239269
"| --- | ---: | ---: | --- |",
@@ -311,8 +341,17 @@ export function formatTransferBudgetReport(runs: ReadonlyArray<TransferBudgetRun
311341
run.reconnectShell,
312342
),
313343
infoRow(run.provider, "reconnect", "SQL statements", run.reconnectSqlStatements, String),
314-
...longTurnRows(run.provider, "long-turn cold open", run.longTurnColdOpen),
315-
...longTurnRows(run.provider, "long-turn history page", run.longTurnHistoryPage),
344+
...timelineRows(run.provider, "long-turn cold open", run.longTurnColdOpen, LONG_TURN_BUDGET),
345+
...legacyRows(run.provider, "long-turn cold open", run.longTurnColdOpenLegacy),
346+
...timelineRows(
347+
run.provider,
348+
"long-turn history page",
349+
run.longTurnHistoryPage,
350+
LONG_TURN_BUDGET,
351+
),
352+
...legacyRows(run.provider, "long-turn history page", run.longTurnHistoryPageLegacy),
353+
...timelineRows(run.provider, "fleet shell snapshot", run.fleetShell, FLEET_SHELL_BUDGET),
354+
...legacyRows(run.provider, "fleet shell snapshot", run.fleetShellLegacy),
316355
);
317356
}
318357

0 commit comments

Comments
 (0)