Skip to content
Closed
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
189 changes: 189 additions & 0 deletions test/harness/replayingCapiProxy.test.ts
Original file line number Diff line number Diff line change
Expand Up @@ -546,6 +546,107 @@ Always include PINEAPPLE_COCONUT_42.
expect(toolMessage?.content).toBe("Tool 'report_intent' does not exist.");
});

test("strips view line-number prefixes from view tool results", async () => {
const requestBody = JSON.stringify({
messages: [
{ role: "user", content: "Read the file" },
{
role: "assistant",
tool_calls: [
{
id: "tc1",
type: "function",
function: {
name: "view",
arguments: '{"path":"lines.txt","view_range":[2,4]}',
},
},
{
id: "tc2",
type: "function",
function: { name: "sql", arguments: '{"query":"SELECT 1"}' },
},
{
id: "tc3",
type: "function",
function: { name: "view", arguments: '{"path":"crlf.txt"}' },
},
],
},
{
role: "tool",
tool_call_id: "tc1",
content: "2. line2\n3. line3\n4. line4",
},
{
role: "tool",
tool_call_id: "tc2",
content: "1. INSERT\n2. INSERT",
},
{
role: "tool",
tool_call_id: "tc3",
content: "1. first\r\n2. second",
},
],
});
const responseBody = JSON.stringify({
choices: [{ message: { role: "assistant", content: "Done" } }],
});

const outputPath = await createProxy([
{ url: "/chat/completions", requestBody, responseBody },
]);

const result = await readYamlOutput(outputPath);
const toolMessages = result.conversations[0].messages.filter(
(m) => m.role === "tool",
);
expect(toolMessages.map((message) => message.content)).toEqual([
"line2\nline3\nline4",
// Only `view` results carried line-number prefixes, so other tools that
// legitimately return numbered lines must be left alone.
"1. INSERT\n2. INSERT",
"first\r\nsecond",
]);
});

test("stops stripping view line numbers at the first non-consecutive line", async () => {
const requestBody = JSON.stringify({
messages: [
{ role: "user", content: "Read the file" },
{
role: "assistant",
tool_calls: [
{
id: "tc1",
type: "function",
function: { name: "view", arguments: '{"path":"steps.md"}' },
},
],
},
{
role: "tool",
tool_call_id: "tc1",
content: "1. first step\n1. second step",
},
],
});
const responseBody = JSON.stringify({
choices: [{ message: { role: "assistant", content: "Done" } }],
});

const outputPath = await createProxy([
{ url: "/chat/completions", requestBody, responseBody },
]);

const result = await readYamlOutput(outputPath);
const toolMessage = result.conversations[0].messages.find(
(m) => m.role === "tool",
);
expect(toolMessage?.content).toBe("first step\n1. second step");
});

test("normalizes interrupted tool execution results", async () => {
const requestBody = JSON.stringify({
messages: [
Expand Down Expand Up @@ -1151,6 +1252,94 @@ Always include PINEAPPLE_COCONUT_42.
}
});

test("matches view results recorded before line numbers were removed", async () => {
const cachePath = path.join(tempDir, "cache.yaml");
// Snapshots now store the raw file content that current runtimes send.
const cacheContent = yaml.stringify({
models: ["test-model"],
conversations: [
{
messages: [
{ role: "system", content: "${system}" },
{ role: "user", content: "Read the file" },
{
role: "assistant",
tool_calls: [
{
id: "toolcall_0",
type: "function",
function: {
name: "view",
arguments: '{"path":"greeting.txt"}',
},
},
],
},
{
role: "tool",
tool_call_id: "toolcall_0",
content: "Hello\nWorld",
},
{ role: "assistant", content: "Done" },
],
},
],
} satisfies NormalizedData);
await writeFile(cachePath, cacheContent);

const proxy = new ReplayingCapiProxy(
"http://localhost:9999",
cachePath,
workDir,
);
const proxyUrl = await proxy.start();

try {
for (const viewResult of [
// Older runtimes prefixed each line with its line number.
"1. Hello\n2. World",
// Current runtimes return the raw content.
"Hello\nWorld",
]) {
const response = await makeRequest(proxyUrl, "/chat/completions", {
body: {
model: "test-model",
messages: [
{ role: "system", content: "System prompt" },
{ role: "user", content: "Read the file" },
{
role: "assistant",
tool_calls: [
{
id: "runtime-call-id",
type: "function",
function: {
name: "view",
arguments: '{"path":"greeting.txt"}',
},
},
],
},
{
role: "tool",
tool_call_id: "runtime-call-id",
content: viewResult,
},
],
},
});

expect(response.status).toBe(200);
expect(
(JSON.parse(response.body) as ChatCompletion).choices[0].message
.content,
).toBe("Done");
}
} finally {
await proxy.stop();
}
});

test("expands workdir placeholder in cached response", async () => {
const cachePath = path.join(tempDir, "cache.yaml");
const cacheContent = yaml.stringify({
Expand Down
31 changes: 31 additions & 0 deletions test/harness/replayingCapiProxy.ts
Original file line number Diff line number Diff line change
Expand Up @@ -132,6 +132,7 @@ export class ReplayingCapiProxy extends CapturingHttpProxy {
{ toolName: "*", normalizer: normalizeAvailableToolNames },
{ toolName: "*", normalizer: normalizeBackgroundAgentStartMessage },
{ toolName: "read_agent", normalizer: normalizeReadAgentResult },
{ toolName: "view", normalizer: normalizeViewLineNumbers },
];

/**
Expand Down Expand Up @@ -1554,6 +1555,36 @@ function normalizeAvailableToolNames(result: string): string {
);
}

// The runtime used to prefix every line of `view` output with `N. ` (the line
// number). copilot-agent-runtime#13802 shipped the winning experiment branch and
// made raw file content the unconditional behavior, so captures recorded against
// older runtimes carry the prefixes while current runtimes do not. Strip a leading
// run of consecutively numbered lines so both forms replay against the same
// snapshot. Trailing content such as the truncation notice is left untouched
// because it was never numbered.
function normalizeViewLineNumbers(result: string): string {
const lines = result.split("\n");
let expected: number | undefined;
let index = 0;
for (; index < lines.length; index++) {
// Lines of a CRLF file keep their carriage return after splitting on "\n".
const carriageReturn = lines[index].endsWith("\r");
const line = carriageReturn ? lines[index].slice(0, -1) : lines[index];
const match = /^(\d+)\.(?: (.*))?$/.exec(line);
if (!match) break;
const lineNumber = Number(match[1]);
if (expected === undefined) {
if (lineNumber < 1) break;
} else if (lineNumber !== expected) {
break;
}
expected = lineNumber + 1;
lines[index] = `${match[2] ?? ""}${carriageReturn ? "\r" : ""}`;
}

return index === 0 ? result : lines.join("\n").trimEnd();
}

function normalizeInterruptedToolResult(result: string): string {
return result.replace(
/^(?:Failed to execute `[^`]+` tool(?: with arguments: [\s\S]*?)? due to error: (?:Error: )?Session aborted|<shell context is being reconfigured; retry the command>)$/,
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -54,6 +54,6 @@ conversations:
arguments: '{"path":"${workdir}/new_file.txt"}'
- role: tool
tool_call_id: toolcall_2
content: 1. Created by test
content: Created by test
- role: assistant
content: ✓ Done! Created `new_file.txt` with content "Created by test" and confirmed the content matches.
Original file line number Diff line number Diff line change
Expand Up @@ -57,9 +57,8 @@ conversations:
- role: tool
tool_call_id: toolcall_2
content: |-
1. Hi Universe
2. Goodbye World
3.
Hi Universe
Goodbye World
- role: assistant
content: |-
Done! The file now contains:
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -45,9 +45,9 @@ conversations:
- role: tool
tool_call_id: toolcall_1
content: |-
2. line2
3. line3
4. line4
line2
line3
line4
- role: assistant
content: |-
Lines 2 through 4 of 'lines.txt' contain:
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -25,6 +25,6 @@ conversations:
task.
- role: tool
tool_call_id: toolcall_1
content: 1. I am in the client cwd
content: I am in the client cwd
- role: assistant
content: 'The file `marker.txt` says: "I am in the client cwd"'
Original file line number Diff line number Diff line change
Expand Up @@ -44,7 +44,7 @@ conversations:
task.
- role: tool
tool_call_id: toolcall_1
content: 1. Hello World
content: Hello World
- role: assistant
content: |-
The file 'hello.txt' contains:
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -44,7 +44,7 @@ conversations:
task.
- role: tool
tool_call_id: toolcall_1
content: 1. test data
content: test data
- role: assistant
content: |-
The file `data.txt` contains:
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -15,6 +15,6 @@ conversations:
arguments: '{"path":"order.txt"}'
- role: tool
tool_call_id: toolcall_0
content: 1. ORDER_CONTENT_42
content: ORDER_CONTENT_42
- role: assistant
content: The number in 'order.txt' is **42**.
Original file line number Diff line number Diff line change
Expand Up @@ -44,7 +44,7 @@ conversations:
task.
- role: tool
tool_call_id: toolcall_1
content: 1. Testing both hooks!
content: Testing both hooks!
- role: assistant
content: |-
The file **both.txt** contains:
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -44,7 +44,7 @@ conversations:
task.
- role: tool
tool_call_id: toolcall_1
content: 1. World from the test!
content: World from the test!
- role: assistant
content: |-
The file `world.txt` contains:
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -44,7 +44,7 @@ conversations:
task.
- role: tool
tool_call_id: toolcall_1
content: 1. Hello from the test!
content: Hello from the test!
- role: assistant
content: |-
The file **hello.txt** contains:
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -44,7 +44,7 @@ conversations:
task.
- role: tool
tool_call_id: toolcall_1
content: 1. Testing both hooks!
content: Testing both hooks!
- role: assistant
content: |-
The file **both.txt** contains:
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -44,7 +44,7 @@ conversations:
task.
- role: tool
tool_call_id: toolcall_1
content: 1. Testing both hooks!
content: Testing both hooks!
- role: assistant
content: |-
The file **both.txt** contains:
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -44,7 +44,7 @@ conversations:
task.
- role: tool
tool_call_id: toolcall_1
content: 1. World from the test!
content: World from the test!
- role: assistant
content: |-
The file `world.txt` contains:
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -44,7 +44,7 @@ conversations:
task.
- role: tool
tool_call_id: toolcall_1
content: 1. Hello from the test!
content: Hello from the test!
- role: assistant
content: |-
The file **hello.txt** contains:
Expand Down
Loading
Loading