diff --git a/e2e/README.md b/e2e/README.md index 06d7b7a574..9e83da193c 100644 --- a/e2e/README.md +++ b/e2e/README.md @@ -148,6 +148,44 @@ previous step's heading gone), not by sleeping. 5. The conversation gets its generated name, which is neither the question nor the answer, and the sidebar shows it. +## The Bot's computer and approvals + +The `computer-*` and `approvals-*` journeys: + +| Journey | What a person does | +| --- | --- | +| `computer-browse` | Asks a Bot to read a page. The Computer opens on its live screen; the answer comes from the page; Activity lists the page with its time, and still does after a reload; the conversation folds the browsing into one line that opens to its steps. | +| `computer-take-control` | Opens the Bot's screen full size, takes control, goes somewhere from the address bar, back, forward, reload, opens a tab and closes it, is asked before clearing the browser, and hands it back. Stops and resets the Bot's browser in Settings → Computer. | +| `computer-terminal` | Asks a Bot to run a command: Activity shows it with what it printed. Takes control of the Terminal, types a command, hands it back. | +| `computer-form` | Asks a Bot to fill in and send a form from one message, its fields and checkbox in one step: every action runs, the form arrives filled in, and Activity lists each with its time. | +| `computer-sign-in` | A Bot reaches a password field and hands it over (no standing answer offered). The person takes control, types the password into the Bot's screen, hands back and answers "I did it myself"; the Bot carries on from the signed-in page. | +| `approvals-allow-once` | A Bot paying asks first. Allow once places that order and the reply arrives whole; the next order asks again, and waits on the Bots page under Needs you too; Deny there places nothing and the Bot says so. | +| `approvals-this-conversation` | Allow for this conversation: the next order there goes through with no card, the banner says what is allowed, Turn off makes the next order ask again. | +| `approvals-always` | Always allow here: the Bot pays without asking in its next conversation; the rule is on the Bot's Approvals page, where Remove takes it away. | +| `approvals-settings` | Settings → Approvals: Extra vigilant, Auto-review and commands on this computer, kept across a reload; a rule for every Bot added, changed, removed, and cancelled. | + +Two things they do that other journeys may want to copy: + +- **The run's own website.** The pages a Bot reads, the form it fills in and the checkout it pays at + are `e2e/site/site.ts`, served on `127.0.0.1:46600` while such a journey runs, never a third-party + site. The Bot's browser, in the computer's container, reaches it at + `http://host.docker.internal:46600` (Docker Desktop carries that to this machine's loopback). The + port is fixed whatever `E2E_PORT_OFFSET` says, because the address is in what the person types and + so in every fixture. The site counts the orders placed and the forms sent, so a journey checks what + the Bot did, not only what it said. +- **A Bot of its own for each journey** (`makeBot`, a fixture in `journeys/computer-steps.ts`), made + through the API as setup and deleted when the journey ends, pass or fail. A Bot's browser numbers + its page snapshots from its first, and a recorded click carries that number: on a Bot another + journey had browsed with, it would be out of date. A new Bot's browser starts from the beginning + whichever journeys ran first, an answer or a rule one journey gives stays with its Bot, and the + journeys after it see only the Bots they expect. + +A reply is waited for by what it says (`replyAfter` in `journeys/computer-steps.ts`): a turn that +uses the computer is several runs, and between two of them the last one has finished with a sentence +like "I'm opening the page". It is then checked on screen word for word, once, as the Markdown renders. +Each conversation is also waited for until it has its name, so the naming call is made, and recorded, +inside the journey that caused it. + ## What is and isn't under aimock Every model call on this path goes to aimock: `OPENAI_BASE_URL` and `ANTHROPIC_BASE_URL` point at it diff --git a/e2e/fixtures/approvals-allow-once/openai-2026-11-12T21-23-24-430Z-dafbf779.json b/e2e/fixtures/approvals-allow-once/openai-2026-11-12T21-23-24-430Z-dafbf779.json new file mode 100644 index 0000000000..2a1aa43082 --- /dev/null +++ b/e2e/fixtures/approvals-allow-once/openai-2026-11-12T21-23-24-430Z-dafbf779.json @@ -0,0 +1,31 @@ +{ + "fixtures": [ + { + "match": { + "userMessage": "You choose which capabilities to load for the message below. You are not answering it.\n\nCapabilities available:\n- research-public-web: Research the public web. Search current public sources and write a cited brief.\n- find-a-document: Find a document. Search the connected document sources for a file and read what it says.\n- find-a-notion-page: Find a Notion page. Search Notion for a page and read what it says.\n- skill-creator: Write a skill. Interview me about a skill I want, then write it and save it as a new skill.\n- bot-creator: Make a Bot. Work out what a new Bot should be, then put it in front of you to create.\n\nMessage:\nOpen http://host.docker.internal:46600/shop/tomato and place the order for the tomato seeds. Then tell me the order number in one short sentence.\n\nReply with only JSON: {\"skills\": [\"\", ...]}.\nChoose every capability that might be needed, including ones you are only somewhat sure about.\nChoosing one that turns out to be unnecessary costs almost nothing. Failing to choose one that\nwas needed means the work cannot be done at all, because its tools will not be loaded. When in\ndoubt, include it. Use an empty list only when the message plainly needs none of them.", + "model": "gpt-5.6-terra", + "turnIndex": 0, + "hasToolResult": false + }, + "response": { + "content": "{\"skills\":[]}", + "usage": { + "prompt_tokens": 279, + "completion_tokens": 51, + "total_tokens": 330, + "prompt_tokens_details": { + "cached_tokens": 0, + "cache_write_tokens": 0, + "audio_tokens": 0 + }, + "completion_tokens_details": { + "reasoning_tokens": 35, + "audio_tokens": 0, + "accepted_prediction_tokens": 0, + "rejected_prediction_tokens": 0 + } + } + } + } + ] +} \ No newline at end of file diff --git a/e2e/fixtures/approvals-allow-once/openai-2026-11-12T21-23-27-267Z-06a18622.json b/e2e/fixtures/approvals-allow-once/openai-2026-11-12T21-23-27-267Z-06a18622.json new file mode 100644 index 0000000000..125b89844e --- /dev/null +++ b/e2e/fixtures/approvals-allow-once/openai-2026-11-12T21-23-27-267Z-06a18622.json @@ -0,0 +1,77 @@ +{ + "fixtures": [ + { + "match": { + "userMessage": "Open http://host.docker.internal:46600/shop/tomato and place the order for the tomato seeds. Then tell me the order number in one short sentence.", + "model": "gpt-5.6-terra", + "turnIndex": 0, + "hasToolResult": false + }, + "response": { + "content": "I’m opening the Fernhill Garden Supply tomato page to order the tomato seeds.", + "toolCalls": [ + { + "name": "computer_navigate", + "arguments": "{\"url\":\"http://host.docker.internal:46600/shop/tomato\"}", + "id": "call_J9oN5NgeOjL0RJqNNAplkAMt" + } + ] + }, + "recordedTimings": { + "ttftMs": 994, + "interChunkDelaysMs": [ + 28, + 475, + 227, + 4, + 4, + 3, + 4, + 5, + 3, + 5, + 9, + 10, + 3, + 8, + 0, + 0, + 11, + 5, + 5, + 3, + 0, + 149, + 5, + 3, + 2, + 4, + 2, + 0, + 0, + 0, + 8, + 13, + 7, + 3, + 2, + 2, + 0, + 0, + 12, + 10, + 7, + 15, + 179, + 12, + 547 + ], + "totalDurationMs": 2788 + }, + "metadata": { + "systemHash": "800ac84f", + "toolsHash": "6d7140c9" + } + } + ] +} \ No newline at end of file diff --git a/e2e/fixtures/approvals-allow-once/openai-2026-11-12T21-23-30-946Z-c3ee8020.json b/e2e/fixtures/approvals-allow-once/openai-2026-11-12T21-23-30-946Z-c3ee8020.json new file mode 100644 index 0000000000..a41616af83 --- /dev/null +++ b/e2e/fixtures/approvals-allow-once/openai-2026-11-12T21-23-30-946Z-c3ee8020.json @@ -0,0 +1,39 @@ +{ + "fixtures": [ + { + "match": { + "userMessage": "Open http://host.docker.internal:46600/shop/tomato and place the order for the tomato seeds. Then tell me the order number in one short sentence.", + "model": "gpt-5.6-terra", + "turnIndex": 2, + "hasToolResult": true + }, + "response": { + "toolCalls": [ + { + "name": "computer_snapshot", + "arguments": "{}", + "id": "call_Hbh3csZHpnbxW3QFeLZhupBT" + } + ] + }, + "recordedTimings": { + "ttftMs": 877, + "interChunkDelaysMs": [ + 21, + 360, + 416, + 5, + 0, + 160, + 0, + 175 + ], + "totalDurationMs": 2014 + }, + "metadata": { + "systemHash": "800ac84f", + "toolsHash": "6d7140c9" + } + } + ] +} \ No newline at end of file diff --git a/e2e/fixtures/approvals-allow-once/openai-2026-11-12T21-23-31-407Z-703999ab.json b/e2e/fixtures/approvals-allow-once/openai-2026-11-12T21-23-31-407Z-703999ab.json new file mode 100644 index 0000000000..fb03b0cd8b --- /dev/null +++ b/e2e/fixtures/approvals-allow-once/openai-2026-11-12T21-23-31-407Z-703999ab.json @@ -0,0 +1,34 @@ +{ + "fixtures": [ + { + "match": { + "userMessage": "Asked: Open http://host.docker.internal:46600/shop/tomato and place the order for the tomato seeds. Then tell me the order number in one short sentence.", + "model": "gpt-5.6-terra", + "turnIndex": 0, + "hasToolResult": false + }, + "response": { + "content": "Tomato Seeds Order", + "usage": { + "prompt_tokens": 155, + "completion_tokens": 18, + "total_tokens": 173, + "prompt_tokens_details": { + "cached_tokens": 0, + "cache_write_tokens": 0, + "audio_tokens": 0 + }, + "completion_tokens_details": { + "reasoning_tokens": 5, + "audio_tokens": 0, + "accepted_prediction_tokens": 0, + "rejected_prediction_tokens": 0 + } + } + }, + "metadata": { + "systemHash": "dedd9ec9" + } + } + ] +} \ No newline at end of file diff --git a/e2e/fixtures/approvals-allow-once/openai-2026-11-12T21-23-33-506Z-4e008cab.json b/e2e/fixtures/approvals-allow-once/openai-2026-11-12T21-23-33-506Z-4e008cab.json new file mode 100644 index 0000000000..16db88c292 --- /dev/null +++ b/e2e/fixtures/approvals-allow-once/openai-2026-11-12T21-23-33-506Z-4e008cab.json @@ -0,0 +1,49 @@ +{ + "fixtures": [ + { + "match": { + "userMessage": "Open http://host.docker.internal:46600/shop/tomato and place the order for the tomato seeds. Then tell me the order number in one short sentence.", + "model": "gpt-5.6-terra", + "turnIndex": 3, + "hasToolResult": true + }, + "response": { + "toolCalls": [ + { + "name": "computer_click", + "arguments": "{\"ref\":\"e13\",\"snapshotId\":2}", + "id": "call_cDQKqnn7ubgsG4kqN1d5dzGz" + } + ] + }, + "recordedTimings": { + "ttftMs": 817, + "interChunkDelaysMs": [ + 34, + 577, + 314, + 3, + 2, + 3, + 2, + 129, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 14, + 1, + 175 + ], + "totalDurationMs": 2071 + }, + "metadata": { + "systemHash": "800ac84f", + "toolsHash": "6d7140c9" + } + } + ] +} \ No newline at end of file diff --git a/e2e/fixtures/approvals-allow-once/openai-2026-11-12T21-23-37-355Z-6770049b.json b/e2e/fixtures/approvals-allow-once/openai-2026-11-12T21-23-37-355Z-6770049b.json new file mode 100644 index 0000000000..28c635c7cd --- /dev/null +++ b/e2e/fixtures/approvals-allow-once/openai-2026-11-12T21-23-37-355Z-6770049b.json @@ -0,0 +1,37 @@ +{ + "fixtures": [ + { + "match": { + "userMessage": "Open http://host.docker.internal:46600/shop/tomato and place the order for the tomato seeds. Then tell me the order number in one short sentence.", + "model": "gpt-5.6-terra", + "turnIndex": 4, + "hasToolResult": true + }, + "response": { + "toolCalls": [ + { + "name": "computer_read", + "arguments": "{}", + "id": "call_TkTm7ggRQkAbACpKn2ammmlx" + } + ] + }, + "recordedTimings": { + "ttftMs": 724, + "interChunkDelaysMs": [ + 2, + 545, + 3, + 189, + 8, + 194 + ], + "totalDurationMs": 1665 + }, + "metadata": { + "systemHash": "b244db5b", + "toolsHash": "88b34f59" + } + } + ] +} \ No newline at end of file diff --git a/e2e/fixtures/approvals-allow-once/openai-2026-11-12T21-23-39-112Z-f8c74f04.json b/e2e/fixtures/approvals-allow-once/openai-2026-11-12T21-23-39-112Z-f8c74f04.json new file mode 100644 index 0000000000..9545635cca --- /dev/null +++ b/e2e/fixtures/approvals-allow-once/openai-2026-11-12T21-23-39-112Z-f8c74f04.json @@ -0,0 +1,46 @@ +{ + "fixtures": [ + { + "match": { + "userMessage": "Open http://host.docker.internal:46600/shop/tomato and place the order for the tomato seeds. Then tell me the order number in one short sentence.", + "model": "gpt-5.6-terra", + "turnIndex": 5, + "hasToolResult": true + }, + "response": { + "content": "Your tomato-seed order number is **FH-2041**." + }, + "recordedTimings": { + "ttftMs": 863, + "interChunkDelaysMs": [ + 39, + 387, + 6, + 0, + 4, + 26, + 0, + 3, + 0, + 26, + 6, + 3, + 4, + 0, + 1, + 0, + 0, + 157, + 2, + 0, + 196 + ], + "totalDurationMs": 1723 + }, + "metadata": { + "systemHash": "b244db5b", + "toolsHash": "88b34f59" + } + } + ] +} \ No newline at end of file diff --git a/e2e/fixtures/approvals-allow-once/openai-2026-11-12T21-23-42-140Z-24626ff5.json b/e2e/fixtures/approvals-allow-once/openai-2026-11-12T21-23-42-140Z-24626ff5.json new file mode 100644 index 0000000000..889c11818f --- /dev/null +++ b/e2e/fixtures/approvals-allow-once/openai-2026-11-12T21-23-42-140Z-24626ff5.json @@ -0,0 +1,31 @@ +{ + "fixtures": [ + { + "match": { + "userMessage": "You choose which capabilities to load for the message below. You are not answering it.\n\nCapabilities available:\n- research-public-web: Research the public web. Search current public sources and write a cited brief.\n- find-a-document: Find a document. Search the connected document sources for a file and read what it says.\n- find-a-notion-page: Find a Notion page. Search Notion for a page and read what it says.\n- skill-creator: Write a skill. Interview me about a skill I want, then write it and save it as a new skill.\n- bot-creator: Make a Bot. Work out what a new Bot should be, then put it in front of you to create.\n\nMessage:\nNow order the basil seeds too, from http://host.docker.internal:46600/shop/basil, and tell me the order number.\n\nReply with only JSON: {\"skills\": [\"\", ...]}.\nChoose every capability that might be needed, including ones you are only somewhat sure about.\nChoosing one that turns out to be unnecessary costs almost nothing. Failing to choose one that\nwas needed means the work cannot be done at all, because its tools will not be loaded. When in\ndoubt, include it. Use an empty list only when the message plainly needs none of them.", + "model": "gpt-5.6-terra", + "turnIndex": 0, + "hasToolResult": false + }, + "response": { + "content": "{\"skills\":[\"research-public-web\"]}", + "usage": { + "prompt_tokens": 273, + "completion_tokens": 62, + "total_tokens": 335, + "prompt_tokens_details": { + "cached_tokens": 0, + "cache_write_tokens": 0, + "audio_tokens": 0 + }, + "completion_tokens_details": { + "reasoning_tokens": 42, + "audio_tokens": 0, + "accepted_prediction_tokens": 0, + "rejected_prediction_tokens": 0 + } + } + } + } + ] +} \ No newline at end of file diff --git a/e2e/fixtures/approvals-allow-once/openai-2026-11-12T21-23-43-840Z-74524717.json b/e2e/fixtures/approvals-allow-once/openai-2026-11-12T21-23-43-840Z-74524717.json new file mode 100644 index 0000000000..62ecdd61db --- /dev/null +++ b/e2e/fixtures/approvals-allow-once/openai-2026-11-12T21-23-43-840Z-74524717.json @@ -0,0 +1,73 @@ +{ + "fixtures": [ + { + "match": { + "userMessage": "Now order the basil seeds too, from http://host.docker.internal:46600/shop/basil, and tell me the order number.", + "model": "gpt-5.6-terra", + "turnIndex": 6, + "hasToolResult": false + }, + "response": { + "content": "I’m opening the Fernhill Garden Supply basil page to place the order.", + "toolCalls": [ + { + "name": "computer_navigate", + "arguments": "{\"url\":\"http://host.docker.internal:46600/shop/basil\"}", + "id": "call_SmsQ3wGePJXzRGZaEViVXNRm" + } + ] + }, + "recordedTimings": { + "ttftMs": 725, + "interChunkDelaysMs": [ + 35, + 364, + 0, + 0, + 1, + 2, + 0, + 0, + 21, + 8, + 2, + 2, + 2, + 3, + 2, + 0, + 20, + 2, + 81, + 3, + 3, + 6, + 0, + 0, + 0, + 0, + 0, + 16, + 2, + 6, + 6, + 0, + 0, + 0, + 25, + 10, + 1, + 6, + 138, + 3, + 169 + ], + "totalDurationMs": 1664 + }, + "metadata": { + "systemHash": "800ac84f", + "toolsHash": "6d7140c9" + } + } + ] +} \ No newline at end of file diff --git a/e2e/fixtures/approvals-allow-once/openai-2026-11-12T21-23-46-253Z-d5d64697.json b/e2e/fixtures/approvals-allow-once/openai-2026-11-12T21-23-46-253Z-d5d64697.json new file mode 100644 index 0000000000..763280d003 --- /dev/null +++ b/e2e/fixtures/approvals-allow-once/openai-2026-11-12T21-23-46-253Z-d5d64697.json @@ -0,0 +1,37 @@ +{ + "fixtures": [ + { + "match": { + "userMessage": "Now order the basil seeds too, from http://host.docker.internal:46600/shop/basil, and tell me the order number.", + "model": "gpt-5.6-terra", + "turnIndex": 8, + "hasToolResult": true + }, + "response": { + "toolCalls": [ + { + "name": "computer_snapshot", + "arguments": "{}", + "id": "call_4JZ1Tf4C8ZkDa5i2jfzk49Tl" + } + ] + }, + "recordedTimings": { + "ttftMs": 764, + "interChunkDelaysMs": [ + 19, + 364, + 6, + 330, + 12, + 178 + ], + "totalDurationMs": 1673 + }, + "metadata": { + "systemHash": "800ac84f", + "toolsHash": "6d7140c9" + } + } + ] +} \ No newline at end of file diff --git a/e2e/fixtures/approvals-allow-once/openai-2026-11-12T21-23-48-700Z-afffc6be.json b/e2e/fixtures/approvals-allow-once/openai-2026-11-12T21-23-48-700Z-afffc6be.json new file mode 100644 index 0000000000..64ed3fb6ca --- /dev/null +++ b/e2e/fixtures/approvals-allow-once/openai-2026-11-12T21-23-48-700Z-afffc6be.json @@ -0,0 +1,49 @@ +{ + "fixtures": [ + { + "match": { + "userMessage": "Now order the basil seeds too, from http://host.docker.internal:46600/shop/basil, and tell me the order number.", + "model": "gpt-5.6-terra", + "turnIndex": 9, + "hasToolResult": true + }, + "response": { + "toolCalls": [ + { + "name": "computer_click", + "arguments": "{\"ref\":\"f2e13\",\"snapshotId\":4}", + "id": "call_uD8VjsLrNAmHyqUH9APeF7jB" + } + ] + }, + "recordedTimings": { + "ttftMs": 785, + "interChunkDelaysMs": [ + 37, + 504, + 8, + 2, + 0, + 0, + 2, + 29, + 1, + 6, + 7, + 3, + 4, + 28, + 20, + 76, + 3, + 183 + ], + "totalDurationMs": 1698 + }, + "metadata": { + "systemHash": "800ac84f", + "toolsHash": "6d7140c9" + } + } + ] +} \ No newline at end of file diff --git a/e2e/fixtures/approvals-allow-once/openai-2026-11-12T21-23-53-747Z-c9cd304c.json b/e2e/fixtures/approvals-allow-once/openai-2026-11-12T21-23-53-747Z-c9cd304c.json new file mode 100644 index 0000000000..054c7275bf --- /dev/null +++ b/e2e/fixtures/approvals-allow-once/openai-2026-11-12T21-23-53-747Z-c9cd304c.json @@ -0,0 +1,48 @@ +{ + "fixtures": [ + { + "match": { + "userMessage": "Now order the basil seeds too, from http://host.docker.internal:46600/shop/basil, and tell me the order number.", + "model": "gpt-5.6-terra", + "turnIndex": 10, + "hasToolResult": true + }, + "response": { + "content": "I did not place the basil-seed order because you declined the purchase approval." + }, + "recordedTimings": { + "ttftMs": 957, + "interChunkDelaysMs": [ + 4, + 393, + 7, + 5, + 4, + 2, + 4, + 0, + 0, + 13, + 39, + 7, + 7, + 7, + 2, + 0, + 0, + 134, + 0, + 60, + 5, + 1, + 198 + ], + "totalDurationMs": 1849 + }, + "metadata": { + "systemHash": "b244db5b", + "toolsHash": "88b34f59" + } + } + ] +} \ No newline at end of file diff --git a/e2e/fixtures/approvals-always/openai-2026-11-12T21-23-58-254Z-99f82469.json b/e2e/fixtures/approvals-always/openai-2026-11-12T21-23-58-254Z-99f82469.json new file mode 100644 index 0000000000..031777a0ca --- /dev/null +++ b/e2e/fixtures/approvals-always/openai-2026-11-12T21-23-58-254Z-99f82469.json @@ -0,0 +1,31 @@ +{ + "fixtures": [ + { + "match": { + "userMessage": "You choose which capabilities to load for the message below. You are not answering it.\n\nCapabilities available:\n- research-public-web: Research the public web. Search current public sources and write a cited brief.\n- find-a-document: Find a document. Search the connected document sources for a file and read what it says.\n- find-a-notion-page: Find a Notion page. Search Notion for a page and read what it says.\n- skill-creator: Write a skill. Interview me about a skill I want, then write it and save it as a new skill.\n- bot-creator: Make a Bot. Work out what a new Bot should be, then put it in front of you to create.\n\nMessage:\nOrder the marigold seeds from http://host.docker.internal:46600/shop/marigold for me, then reply with just the order number.\n\nReply with only JSON: {\"skills\": [\"\", ...]}.\nChoose every capability that might be needed, including ones you are only somewhat sure about.\nChoosing one that turns out to be unnecessary costs almost nothing. Failing to choose one that\nwas needed means the work cannot be done at all, because its tools will not be loaded. When in\ndoubt, include it. Use an empty list only when the message plainly needs none of them.", + "model": "gpt-5.6-terra", + "turnIndex": 0, + "hasToolResult": false + }, + "response": { + "content": "{\"skills\":[\"research-public-web\"]}", + "usage": { + "prompt_tokens": 277, + "completion_tokens": 56, + "total_tokens": 333, + "prompt_tokens_details": { + "cached_tokens": 0, + "cache_write_tokens": 0, + "audio_tokens": 0 + }, + "completion_tokens_details": { + "reasoning_tokens": 36, + "audio_tokens": 0, + "accepted_prediction_tokens": 0, + "rejected_prediction_tokens": 0 + } + } + } + } + ] +} \ No newline at end of file diff --git a/e2e/fixtures/approvals-always/openai-2026-11-12T21-24-00-546Z-6fd46802.json b/e2e/fixtures/approvals-always/openai-2026-11-12T21-24-00-546Z-6fd46802.json new file mode 100644 index 0000000000..c8a85c0216 --- /dev/null +++ b/e2e/fixtures/approvals-always/openai-2026-11-12T21-24-00-546Z-6fd46802.json @@ -0,0 +1,80 @@ +{ + "fixtures": [ + { + "match": { + "userMessage": "Order the marigold seeds from http://host.docker.internal:46600/shop/marigold for me, then reply with just the order number.", + "model": "gpt-5.6-terra", + "turnIndex": 0, + "hasToolResult": false + }, + "response": { + "content": "I’m opening Fernhill Garden Supply’s marigold seed page to place the order.", + "toolCalls": [ + { + "name": "computer_navigate", + "arguments": "{\"url\":\"http://host.docker.internal:46600/shop/marigold\"}", + "id": "call_7H58iVS4Itc5S8FyteN8eMWx" + } + ] + }, + "recordedTimings": { + "ttftMs": 776, + "interChunkDelaysMs": [ + 5, + 418, + 421, + 0, + 2, + 0, + 0, + 0, + 22, + 34, + 2, + 6, + 2, + 0, + 0, + 3, + 19, + 13, + 0, + 24, + 5, + 5, + 0, + 67, + 3, + 3, + 2, + 0, + 0, + 0, + 16, + 3, + 9, + 4, + 4, + 4, + 0, + 0, + 17, + 3, + 5, + 3, + 4, + 2, + 28, + 91, + 13, + 209 + ], + "totalDurationMs": 2247 + }, + "metadata": { + "systemHash": "d034e2a7", + "toolsHash": "b3e8252a" + } + } + ] +} \ No newline at end of file diff --git a/e2e/fixtures/approvals-always/openai-2026-11-12T21-24-02-511Z-d0821a8e.json b/e2e/fixtures/approvals-always/openai-2026-11-12T21-24-02-511Z-d0821a8e.json new file mode 100644 index 0000000000..7eaa7bc2de --- /dev/null +++ b/e2e/fixtures/approvals-always/openai-2026-11-12T21-24-02-511Z-d0821a8e.json @@ -0,0 +1,34 @@ +{ + "fixtures": [ + { + "match": { + "userMessage": "Asked: Order the marigold seeds from http://host.docker.internal:46600/shop/marigold for me, then reply with just the order number.", + "model": "gpt-5.6-terra", + "turnIndex": 0, + "hasToolResult": false + }, + "response": { + "content": "Marigold Seed Order Request", + "usage": { + "prompt_tokens": 155, + "completion_tokens": 20, + "total_tokens": 175, + "prompt_tokens_details": { + "cached_tokens": 0, + "cache_write_tokens": 0, + "audio_tokens": 0 + }, + "completion_tokens_details": { + "reasoning_tokens": 5, + "audio_tokens": 0, + "accepted_prediction_tokens": 0, + "rejected_prediction_tokens": 0 + } + } + }, + "metadata": { + "systemHash": "dedd9ec9" + } + } + ] +} \ No newline at end of file diff --git a/e2e/fixtures/approvals-always/openai-2026-11-12T21-24-05-109Z-2adb8e6e.json b/e2e/fixtures/approvals-always/openai-2026-11-12T21-24-05-109Z-2adb8e6e.json new file mode 100644 index 0000000000..0a27a16b5c --- /dev/null +++ b/e2e/fixtures/approvals-always/openai-2026-11-12T21-24-05-109Z-2adb8e6e.json @@ -0,0 +1,39 @@ +{ + "fixtures": [ + { + "match": { + "userMessage": "Order the marigold seeds from http://host.docker.internal:46600/shop/marigold for me, then reply with just the order number.", + "model": "gpt-5.6-terra", + "turnIndex": 2, + "hasToolResult": true + }, + "response": { + "toolCalls": [ + { + "name": "computer_snapshot", + "arguments": "{}", + "id": "call_2MV6qFGcVzXgUMNM5LDn8xZS" + } + ] + }, + "recordedTimings": { + "ttftMs": 1092, + "interChunkDelaysMs": [ + 23, + 289, + 249, + 0, + 5, + 138, + 4, + 149 + ], + "totalDurationMs": 1949 + }, + "metadata": { + "systemHash": "d034e2a7", + "toolsHash": "b3e8252a" + } + } + ] +} \ No newline at end of file diff --git a/e2e/fixtures/approvals-always/openai-2026-11-12T21-24-07-522Z-d80b8228.json b/e2e/fixtures/approvals-always/openai-2026-11-12T21-24-07-522Z-d80b8228.json new file mode 100644 index 0000000000..adb08a1b69 --- /dev/null +++ b/e2e/fixtures/approvals-always/openai-2026-11-12T21-24-07-522Z-d80b8228.json @@ -0,0 +1,47 @@ +{ + "fixtures": [ + { + "match": { + "userMessage": "Order the marigold seeds from http://host.docker.internal:46600/shop/marigold for me, then reply with just the order number.", + "model": "gpt-5.6-terra", + "turnIndex": 3, + "hasToolResult": true + }, + "response": { + "toolCalls": [ + { + "name": "computer_click", + "arguments": "{\"ref\":\"e13\",\"snapshotId\":2}", + "id": "call_B6qrQ51lfKL5QFYksIoROw0E" + } + ] + }, + "recordedTimings": { + "ttftMs": 891, + "interChunkDelaysMs": [ + 21, + 461, + 3, + 1, + 7, + 3, + 0, + 29, + 13, + 5, + 2, + 22, + 66, + 59, + 31, + 148 + ], + "totalDurationMs": 1762 + }, + "metadata": { + "systemHash": "d034e2a7", + "toolsHash": "b3e8252a" + } + } + ] +} \ No newline at end of file diff --git a/e2e/fixtures/approvals-always/openai-2026-11-12T21-24-13-357Z-a9c6e63c.json b/e2e/fixtures/approvals-always/openai-2026-11-12T21-24-13-357Z-a9c6e63c.json new file mode 100644 index 0000000000..a172bdb3b5 --- /dev/null +++ b/e2e/fixtures/approvals-always/openai-2026-11-12T21-24-13-357Z-a9c6e63c.json @@ -0,0 +1,37 @@ +{ + "fixtures": [ + { + "match": { + "userMessage": "Order the marigold seeds from http://host.docker.internal:46600/shop/marigold for me, then reply with just the order number.", + "model": "gpt-5.6-terra", + "turnIndex": 4, + "hasToolResult": true + }, + "response": { + "toolCalls": [ + { + "name": "computer_read", + "arguments": "{}", + "id": "call_kxgSUqZ7exl9qAVqfVufV5bz" + } + ] + }, + "recordedTimings": { + "ttftMs": 911, + "interChunkDelaysMs": [ + 21, + 505, + 2, + 138, + 2, + 224 + ], + "totalDurationMs": 1803 + }, + "metadata": { + "systemHash": "86bf5553", + "toolsHash": "128a9f57" + } + } + ] +} \ No newline at end of file diff --git a/e2e/fixtures/approvals-always/openai-2026-11-12T21-24-15-444Z-447fbc10.json b/e2e/fixtures/approvals-always/openai-2026-11-12T21-24-15-444Z-447fbc10.json new file mode 100644 index 0000000000..3480b5dfab --- /dev/null +++ b/e2e/fixtures/approvals-always/openai-2026-11-12T21-24-15-444Z-447fbc10.json @@ -0,0 +1,36 @@ +{ + "fixtures": [ + { + "match": { + "userMessage": "Order the marigold seeds from http://host.docker.internal:46600/shop/marigold for me, then reply with just the order number.", + "model": "gpt-5.6-terra", + "turnIndex": 5, + "hasToolResult": true + }, + "response": { + "content": "FH-2043" + }, + "recordedTimings": { + "ttftMs": 620, + "interChunkDelaysMs": [ + 5, + 1054, + 9, + 3, + 4, + 16, + 0, + 70, + 2, + 22, + 233 + ], + "totalDurationMs": 2038 + }, + "metadata": { + "systemHash": "86bf5553", + "toolsHash": "128a9f57" + } + } + ] +} \ No newline at end of file diff --git a/e2e/fixtures/approvals-always/openai-2026-11-12T21-24-20-366Z-2bc0af26.json b/e2e/fixtures/approvals-always/openai-2026-11-12T21-24-20-366Z-2bc0af26.json new file mode 100644 index 0000000000..a83e4e6e15 --- /dev/null +++ b/e2e/fixtures/approvals-always/openai-2026-11-12T21-24-20-366Z-2bc0af26.json @@ -0,0 +1,31 @@ +{ + "fixtures": [ + { + "match": { + "userMessage": "You choose which capabilities to load for the message below. You are not answering it.\n\nCapabilities available:\n- research-public-web: Research the public web. Search current public sources and write a cited brief.\n- find-a-document: Find a document. Search the connected document sources for a file and read what it says.\n- find-a-notion-page: Find a Notion page. Search Notion for a page and read what it says.\n- skill-creator: Write a skill. Interview me about a skill I want, then write it and save it as a new skill.\n- bot-creator: Make a Bot. Work out what a new Bot should be, then put it in front of you to create.\n\nMessage:\nOrder the tomato seeds from http://host.docker.internal:46600/shop/tomato for me, then reply with just the order number.\n\nReply with only JSON: {\"skills\": [\"\", ...]}.\nChoose every capability that might be needed, including ones you are only somewhat sure about.\nChoosing one that turns out to be unnecessary costs almost nothing. Failing to choose one that\nwas needed means the work cannot be done at all, because its tools will not be loaded. When in\ndoubt, include it. Use an empty list only when the message plainly needs none of them.", + "model": "gpt-5.6-terra", + "turnIndex": 0, + "hasToolResult": false + }, + "response": { + "content": "{\"skills\":[\"research-public-web\"]}", + "usage": { + "prompt_tokens": 274, + "completion_tokens": 77, + "total_tokens": 351, + "prompt_tokens_details": { + "cached_tokens": 0, + "cache_write_tokens": 0, + "audio_tokens": 0 + }, + "completion_tokens_details": { + "reasoning_tokens": 57, + "audio_tokens": 0, + "accepted_prediction_tokens": 0, + "rejected_prediction_tokens": 0 + } + } + } + } + ] +} \ No newline at end of file diff --git a/e2e/fixtures/approvals-always/openai-2026-11-12T21-24-22-155Z-edd4afc1.json b/e2e/fixtures/approvals-always/openai-2026-11-12T21-24-22-155Z-edd4afc1.json new file mode 100644 index 0000000000..583311859d --- /dev/null +++ b/e2e/fixtures/approvals-always/openai-2026-11-12T21-24-22-155Z-edd4afc1.json @@ -0,0 +1,75 @@ +{ + "fixtures": [ + { + "match": { + "userMessage": "Order the tomato seeds from http://host.docker.internal:46600/shop/tomato for me, then reply with just the order number.", + "model": "gpt-5.6-terra", + "turnIndex": 0, + "hasToolResult": false + }, + "response": { + "content": "I’m opening the tomato-seed page and will place the order.", + "toolCalls": [ + { + "name": "computer_navigate", + "arguments": "{\"url\":\"http://host.docker.internal:46600/shop/tomato\"}", + "id": "call_vLVWWXlAKwkEjir3xenVn7cn" + } + ] + }, + "recordedTimings": { + "ttftMs": 719, + "interChunkDelaysMs": [ + 34, + 235, + 237, + 3, + 3, + 3, + 3, + 0, + 0, + 3, + 0, + 10, + 33, + 3, + 5, + 5, + 4, + 0, + 0, + 96, + 6, + 3, + 5, + 1, + 3, + 1, + 4, + 3, + 3, + 0, + 2, + 0, + 0, + 4, + 5, + 4, + 1, + 0, + 19, + 6, + 99, + 3, + 207 + ], + "totalDurationMs": 1775 + }, + "metadata": { + "systemHash": "d034e2a7", + "toolsHash": "b3e8252a" + } + } + ] +} \ No newline at end of file diff --git a/e2e/fixtures/approvals-always/openai-2026-11-12T21-24-23-822Z-4725fbcf.json b/e2e/fixtures/approvals-always/openai-2026-11-12T21-24-23-822Z-4725fbcf.json new file mode 100644 index 0000000000..7775d8c683 --- /dev/null +++ b/e2e/fixtures/approvals-always/openai-2026-11-12T21-24-23-822Z-4725fbcf.json @@ -0,0 +1,34 @@ +{ + "fixtures": [ + { + "match": { + "userMessage": "Asked: Order the tomato seeds from http://host.docker.internal:46600/shop/tomato for me, then reply with just the order number.", + "model": "gpt-5.6-terra", + "turnIndex": 0, + "hasToolResult": false + }, + "response": { + "content": "Tomato Seed Order Request", + "usage": { + "prompt_tokens": 148, + "completion_tokens": 30, + "total_tokens": 178, + "prompt_tokens_details": { + "cached_tokens": 0, + "cache_write_tokens": 0, + "audio_tokens": 0 + }, + "completion_tokens_details": { + "reasoning_tokens": 16, + "audio_tokens": 0, + "accepted_prediction_tokens": 0, + "rejected_prediction_tokens": 0 + } + } + }, + "metadata": { + "systemHash": "dedd9ec9" + } + } + ] +} \ No newline at end of file diff --git a/e2e/fixtures/approvals-always/openai-2026-11-12T21-24-24-859Z-7005acbf.json b/e2e/fixtures/approvals-always/openai-2026-11-12T21-24-24-859Z-7005acbf.json new file mode 100644 index 0000000000..850153548e --- /dev/null +++ b/e2e/fixtures/approvals-always/openai-2026-11-12T21-24-24-859Z-7005acbf.json @@ -0,0 +1,39 @@ +{ + "fixtures": [ + { + "match": { + "userMessage": "Order the tomato seeds from http://host.docker.internal:46600/shop/tomato for me, then reply with just the order number.", + "model": "gpt-5.6-terra", + "turnIndex": 2, + "hasToolResult": true + }, + "response": { + "toolCalls": [ + { + "name": "computer_snapshot", + "arguments": "{}", + "id": "call_aBHqzmoMXdi6WgOYDtR4UR1Y" + } + ] + }, + "recordedTimings": { + "ttftMs": 796, + "interChunkDelaysMs": [ + 5, + 293, + 382, + 0, + 0, + 99, + 0, + 204 + ], + "totalDurationMs": 1779 + }, + "metadata": { + "systemHash": "d034e2a7", + "toolsHash": "b3e8252a" + } + } + ] +} \ No newline at end of file diff --git a/e2e/fixtures/approvals-always/openai-2026-11-12T21-24-27-759Z-6034f708.json b/e2e/fixtures/approvals-always/openai-2026-11-12T21-24-27-759Z-6034f708.json new file mode 100644 index 0000000000..210448749f --- /dev/null +++ b/e2e/fixtures/approvals-always/openai-2026-11-12T21-24-27-759Z-6034f708.json @@ -0,0 +1,49 @@ +{ + "fixtures": [ + { + "match": { + "userMessage": "Order the tomato seeds from http://host.docker.internal:46600/shop/tomato for me, then reply with just the order number.", + "model": "gpt-5.6-terra", + "turnIndex": 3, + "hasToolResult": true + }, + "response": { + "toolCalls": [ + { + "name": "computer_click", + "arguments": "{\"ref\":\"f2e13\",\"snapshotId\":4}", + "id": "call_Sgcgzh4IlVTygRu3PpIV2BUx" + } + ] + }, + "recordedTimings": { + "ttftMs": 828, + "interChunkDelaysMs": [ + 21, + 417, + 7, + 0, + 4, + 0, + 0, + 18, + 5, + 1, + 3, + 6, + 5, + 5, + 22, + 137, + 5, + 390 + ], + "totalDurationMs": 1874 + }, + "metadata": { + "systemHash": "d034e2a7", + "toolsHash": "b3e8252a" + } + } + ] +} \ No newline at end of file diff --git a/e2e/fixtures/approvals-always/openai-2026-11-12T21-24-30-367Z-cf4b024b.json b/e2e/fixtures/approvals-always/openai-2026-11-12T21-24-30-367Z-cf4b024b.json new file mode 100644 index 0000000000..27039ec22d --- /dev/null +++ b/e2e/fixtures/approvals-always/openai-2026-11-12T21-24-30-367Z-cf4b024b.json @@ -0,0 +1,37 @@ +{ + "fixtures": [ + { + "match": { + "userMessage": "Order the tomato seeds from http://host.docker.internal:46600/shop/tomato for me, then reply with just the order number.", + "model": "gpt-5.6-terra", + "turnIndex": 4, + "hasToolResult": true + }, + "response": { + "toolCalls": [ + { + "name": "computer_read", + "arguments": "{}", + "id": "call_i1NBR4Kfzhd4ojeoEpQCtpBj" + } + ] + }, + "recordedTimings": { + "ttftMs": 811, + "interChunkDelaysMs": [ + 15, + 389, + 3, + 128, + 13, + 199 + ], + "totalDurationMs": 1558 + }, + "metadata": { + "systemHash": "d034e2a7", + "toolsHash": "b3e8252a" + } + } + ] +} \ No newline at end of file diff --git a/e2e/fixtures/approvals-always/openai-2026-11-12T21-24-32-178Z-f2911fc2.json b/e2e/fixtures/approvals-always/openai-2026-11-12T21-24-32-178Z-f2911fc2.json new file mode 100644 index 0000000000..ce0ef1c8b9 --- /dev/null +++ b/e2e/fixtures/approvals-always/openai-2026-11-12T21-24-32-178Z-f2911fc2.json @@ -0,0 +1,36 @@ +{ + "fixtures": [ + { + "match": { + "userMessage": "Order the tomato seeds from http://host.docker.internal:46600/shop/tomato for me, then reply with just the order number.", + "model": "gpt-5.6-terra", + "turnIndex": 5, + "hasToolResult": true + }, + "response": { + "content": "FH-2041" + }, + "recordedTimings": { + "ttftMs": 655, + "interChunkDelaysMs": [ + 18, + 362, + 2, + 4, + 2, + 12, + 1, + 108, + 6, + 1, + 128 + ], + "totalDurationMs": 1299 + }, + "metadata": { + "systemHash": "d034e2a7", + "toolsHash": "b3e8252a" + } + } + ] +} \ No newline at end of file diff --git a/e2e/fixtures/approvals-this-conversation/openai-2026-11-12T21-25-40-583Z-0c4a393a.json b/e2e/fixtures/approvals-this-conversation/openai-2026-11-12T21-25-40-583Z-0c4a393a.json new file mode 100644 index 0000000000..2d5ef6504f --- /dev/null +++ b/e2e/fixtures/approvals-this-conversation/openai-2026-11-12T21-25-40-583Z-0c4a393a.json @@ -0,0 +1,31 @@ +{ + "fixtures": [ + { + "match": { + "userMessage": "You choose which capabilities to load for the message below. You are not answering it.\n\nCapabilities available:\n- research-public-web: Research the public web. Search current public sources and write a cited brief.\n- find-a-document: Find a document. Search the connected document sources for a file and read what it says.\n- find-a-notion-page: Find a Notion page. Search Notion for a page and read what it says.\n- skill-creator: Write a skill. Interview me about a skill I want, then write it and save it as a new skill.\n- bot-creator: Make a Bot. Work out what a new Bot should be, then put it in front of you to create.\n\nMessage:\nPlease buy the sunflower seeds at http://host.docker.internal:46600/shop/sunflower and give me the order number in one sentence.\n\nReply with only JSON: {\"skills\": [\"\", ...]}.\nChoose every capability that might be needed, including ones you are only somewhat sure about.\nChoosing one that turns out to be unnecessary costs almost nothing. Failing to choose one that\nwas needed means the work cannot be done at all, because its tools will not be loaded. When in\ndoubt, include it. Use an empty list only when the message plainly needs none of them.", + "model": "gpt-5.6-terra", + "turnIndex": 0, + "hasToolResult": false + }, + "response": { + "content": "{\"skills\":[\"research-public-web\"]}", + "usage": { + "prompt_tokens": 274, + "completion_tokens": 78, + "total_tokens": 352, + "prompt_tokens_details": { + "cached_tokens": 0, + "cache_write_tokens": 0, + "audio_tokens": 0 + }, + "completion_tokens_details": { + "reasoning_tokens": 58, + "audio_tokens": 0, + "accepted_prediction_tokens": 0, + "rejected_prediction_tokens": 0 + } + } + } + } + ] +} \ No newline at end of file diff --git a/e2e/fixtures/approvals-this-conversation/openai-2026-11-12T21-25-43-303Z-1a8dbde7.json b/e2e/fixtures/approvals-this-conversation/openai-2026-11-12T21-25-43-303Z-1a8dbde7.json new file mode 100644 index 0000000000..57387c78b7 --- /dev/null +++ b/e2e/fixtures/approvals-this-conversation/openai-2026-11-12T21-25-43-303Z-1a8dbde7.json @@ -0,0 +1,81 @@ +{ + "fixtures": [ + { + "match": { + "userMessage": "Please buy the sunflower seeds at http://host.docker.internal:46600/shop/sunflower and give me the order number in one sentence.", + "model": "gpt-5.6-terra", + "turnIndex": 0, + "hasToolResult": false + }, + "response": { + "content": "I’m opening Fernhill Garden Supply’s sunflower listing to purchase the seeds and retrieve the order number.", + "toolCalls": [ + { + "name": "computer_navigate", + "arguments": "{\"url\":\"http://host.docker.internal:46600/shop/sunflower\"}", + "id": "call_kMnQ4ymZZjqPFS59sykKr0qa" + } + ] + }, + "recordedTimings": { + "ttftMs": 1108, + "interChunkDelaysMs": [ + 20, + 466, + 183, + 6, + 0, + 1, + 0, + 0, + 16, + 39, + 2, + 5, + 0, + 26, + 38, + 33, + 70, + 10, + 31, + 6, + 2, + 3, + 4, + 0, + 0, + 100, + 2, + 2, + 8, + 3, + 1, + 0, + 4, + 0, + 1, + 0, + 7, + 0, + 0, + 12, + 1, + 7, + 0, + 2, + 0, + 21, + 115, + 0, + 168 + ], + "totalDurationMs": 2523 + }, + "metadata": { + "systemHash": "67ebaa8c", + "toolsHash": "4a353f4d" + } + } + ] +} \ No newline at end of file diff --git a/e2e/fixtures/approvals-this-conversation/openai-2026-11-12T21-25-45-278Z-abe31f9d.json b/e2e/fixtures/approvals-this-conversation/openai-2026-11-12T21-25-45-278Z-abe31f9d.json new file mode 100644 index 0000000000..bf23dbdd08 --- /dev/null +++ b/e2e/fixtures/approvals-this-conversation/openai-2026-11-12T21-25-45-278Z-abe31f9d.json @@ -0,0 +1,34 @@ +{ + "fixtures": [ + { + "match": { + "userMessage": "Asked: Please buy the sunflower seeds at http://host.docker.internal:46600/shop/sunflower and give me the order number in one sentence.", + "model": "gpt-5.6-terra", + "turnIndex": 0, + "hasToolResult": false + }, + "response": { + "content": "Sunflower Seed Purchase Request", + "usage": { + "prompt_tokens": 154, + "completion_tokens": 19, + "total_tokens": 173, + "prompt_tokens_details": { + "cached_tokens": 0, + "cache_write_tokens": 0, + "audio_tokens": 0 + }, + "completion_tokens_details": { + "reasoning_tokens": 5, + "audio_tokens": 0, + "accepted_prediction_tokens": 0, + "rejected_prediction_tokens": 0 + } + } + }, + "metadata": { + "systemHash": "dedd9ec9" + } + } + ] +} \ No newline at end of file diff --git a/e2e/fixtures/approvals-this-conversation/openai-2026-11-12T21-25-47-234Z-33bdd2b1.json b/e2e/fixtures/approvals-this-conversation/openai-2026-11-12T21-25-47-234Z-33bdd2b1.json new file mode 100644 index 0000000000..c0569a0b01 --- /dev/null +++ b/e2e/fixtures/approvals-this-conversation/openai-2026-11-12T21-25-47-234Z-33bdd2b1.json @@ -0,0 +1,39 @@ +{ + "fixtures": [ + { + "match": { + "userMessage": "Please buy the sunflower seeds at http://host.docker.internal:46600/shop/sunflower and give me the order number in one sentence.", + "model": "gpt-5.6-terra", + "turnIndex": 2, + "hasToolResult": true + }, + "response": { + "toolCalls": [ + { + "name": "computer_snapshot", + "arguments": "{}", + "id": "call_ohOxoXAi35J05eE4uYc4VoMP" + } + ] + }, + "recordedTimings": { + "ttftMs": 1526, + "interChunkDelaysMs": [ + 20, + 359, + 289, + 46, + 22, + 101, + 10, + 212 + ], + "totalDurationMs": 2585 + }, + "metadata": { + "systemHash": "67ebaa8c", + "toolsHash": "4a353f4d" + } + } + ] +} \ No newline at end of file diff --git a/e2e/fixtures/approvals-this-conversation/openai-2026-11-12T21-25-49-411Z-80081cab.json b/e2e/fixtures/approvals-this-conversation/openai-2026-11-12T21-25-49-411Z-80081cab.json new file mode 100644 index 0000000000..4512aa85e0 --- /dev/null +++ b/e2e/fixtures/approvals-this-conversation/openai-2026-11-12T21-25-49-411Z-80081cab.json @@ -0,0 +1,47 @@ +{ + "fixtures": [ + { + "match": { + "userMessage": "Please buy the sunflower seeds at http://host.docker.internal:46600/shop/sunflower and give me the order number in one sentence.", + "model": "gpt-5.6-terra", + "turnIndex": 3, + "hasToolResult": true + }, + "response": { + "toolCalls": [ + { + "name": "computer_click", + "arguments": "{\"ref\":\"e13\",\"snapshotId\":2}", + "id": "call_EVVsbPowyyaKzFB0xQD5bqao" + } + ] + }, + "recordedTimings": { + "ttftMs": 848, + "interChunkDelaysMs": [ + 16, + 434, + 3, + 10, + 3, + 3, + 24, + 10, + 9, + 3, + 26, + 27, + 66, + 18, + 23, + 179 + ], + "totalDurationMs": 1702 + }, + "metadata": { + "systemHash": "67ebaa8c", + "toolsHash": "4a353f4d" + } + } + ] +} \ No newline at end of file diff --git a/e2e/fixtures/approvals-this-conversation/openai-2026-11-12T21-25-54-238Z-cf91e491.json b/e2e/fixtures/approvals-this-conversation/openai-2026-11-12T21-25-54-238Z-cf91e491.json new file mode 100644 index 0000000000..291e79f30b --- /dev/null +++ b/e2e/fixtures/approvals-this-conversation/openai-2026-11-12T21-25-54-238Z-cf91e491.json @@ -0,0 +1,37 @@ +{ + "fixtures": [ + { + "match": { + "userMessage": "Please buy the sunflower seeds at http://host.docker.internal:46600/shop/sunflower and give me the order number in one sentence.", + "model": "gpt-5.6-terra", + "turnIndex": 4, + "hasToolResult": true + }, + "response": { + "toolCalls": [ + { + "name": "computer_read", + "arguments": "{}", + "id": "call_XCEr3FwNq9bb6rXs09mWY5Nm" + } + ] + }, + "recordedTimings": { + "ttftMs": 660, + "interChunkDelaysMs": [ + 35, + 339, + 6, + 117, + 18, + 216 + ], + "totalDurationMs": 1391 + }, + "metadata": { + "systemHash": "d77dcfa1", + "toolsHash": "e503a007" + } + } + ] +} \ No newline at end of file diff --git a/e2e/fixtures/approvals-this-conversation/openai-2026-11-12T21-25-56-501Z-2fd79d71.json b/e2e/fixtures/approvals-this-conversation/openai-2026-11-12T21-25-56-501Z-2fd79d71.json new file mode 100644 index 0000000000..cfd12f02ff --- /dev/null +++ b/e2e/fixtures/approvals-this-conversation/openai-2026-11-12T21-25-56-501Z-2fd79d71.json @@ -0,0 +1,49 @@ +{ + "fixtures": [ + { + "match": { + "userMessage": "Please buy the sunflower seeds at http://host.docker.internal:46600/shop/sunflower and give me the order number in one sentence.", + "model": "gpt-5.6-terra", + "turnIndex": 5, + "hasToolResult": true + }, + "response": { + "content": "Your Giant sunflower seeds order is confirmed; order number **FH-2044**." + }, + "recordedTimings": { + "ttftMs": 1108, + "interChunkDelaysMs": [ + 15, + 442, + 2, + 0, + 2, + 6, + 43, + 1, + 4, + 42, + 0, + 20, + 49, + 4, + 5, + 0, + 2, + 2, + 0, + 0, + 144, + 16, + 58, + 257 + ], + "totalDurationMs": 2222 + }, + "metadata": { + "systemHash": "d77dcfa1", + "toolsHash": "e503a007" + } + } + ] +} \ No newline at end of file diff --git a/e2e/fixtures/approvals-this-conversation/openai-2026-11-12T21-26-02-566Z-e26c3e3a.json b/e2e/fixtures/approvals-this-conversation/openai-2026-11-12T21-26-02-566Z-e26c3e3a.json new file mode 100644 index 0000000000..6b5fd00a1c --- /dev/null +++ b/e2e/fixtures/approvals-this-conversation/openai-2026-11-12T21-26-02-566Z-e26c3e3a.json @@ -0,0 +1,31 @@ +{ + "fixtures": [ + { + "match": { + "userMessage": "You choose which capabilities to load for the message below. You are not answering it.\n\nCapabilities available:\n- research-public-web: Research the public web. Search current public sources and write a cited brief.\n- find-a-document: Find a document. Search the connected document sources for a file and read what it says.\n- find-a-notion-page: Find a Notion page. Search Notion for a page and read what it says.\n- skill-creator: Write a skill. Interview me about a skill I want, then write it and save it as a new skill.\n- bot-creator: Make a Bot. Work out what a new Bot should be, then put it in front of you to create.\n\nMessage:\nGreat. Please also buy the lavender seeds at http://host.docker.internal:46600/shop/lavender and give me that order number too.\n\nReply with only JSON: {\"skills\": [\"\", ...]}.\nChoose every capability that might be needed, including ones you are only somewhat sure about.\nChoosing one that turns out to be unnecessary costs almost nothing. Failing to choose one that\nwas needed means the work cannot be done at all, because its tools will not be loaded. When in\ndoubt, include it. Use an empty list only when the message plainly needs none of them.", + "model": "gpt-5.6-terra", + "turnIndex": 0, + "hasToolResult": false + }, + "response": { + "content": "{\"skills\":[\"research-public-web\"]}", + "usage": { + "prompt_tokens": 275, + "completion_tokens": 128, + "total_tokens": 403, + "prompt_tokens_details": { + "cached_tokens": 0, + "cache_write_tokens": 0, + "audio_tokens": 0 + }, + "completion_tokens_details": { + "reasoning_tokens": 108, + "audio_tokens": 0, + "accepted_prediction_tokens": 0, + "rejected_prediction_tokens": 0 + } + } + } + } + ] +} \ No newline at end of file diff --git a/e2e/fixtures/approvals-this-conversation/openai-2026-11-12T21-26-04-747Z-b145eac9.json b/e2e/fixtures/approvals-this-conversation/openai-2026-11-12T21-26-04-747Z-b145eac9.json new file mode 100644 index 0000000000..7775df7618 --- /dev/null +++ b/e2e/fixtures/approvals-this-conversation/openai-2026-11-12T21-26-04-747Z-b145eac9.json @@ -0,0 +1,81 @@ +{ + "fixtures": [ + { + "match": { + "userMessage": "Great. Please also buy the lavender seeds at http://host.docker.internal:46600/shop/lavender and give me that order number too.", + "model": "gpt-5.6-terra", + "turnIndex": 6, + "hasToolResult": false + }, + "response": { + "content": "I’m opening Fernhill Garden Supply’s lavender checkout to place the additional order and get its number.", + "toolCalls": [ + { + "name": "computer_navigate", + "arguments": "{\"url\":\"http://host.docker.internal:46600/shop/lavender\"}", + "id": "call_3vDW4PRw3wEKMvjUNJ8jNhoG" + } + ] + }, + "recordedTimings": { + "ttftMs": 1059, + "interChunkDelaysMs": [ + 34, + 323, + 151, + 0, + 3, + 3, + 2, + 3, + 3, + 5, + 0, + 0, + 0, + 10, + 3, + 2, + 7, + 3, + 28, + 3, + 3, + 5, + 1, + 0, + 0, + 116, + 4, + 2, + 2, + 2, + 6, + 0, + 2, + 1, + 0, + 0, + 0, + 13, + 0, + 11, + 6, + 3, + 4, + 0, + 0, + 16, + 119, + 2, + 197 + ], + "totalDurationMs": 2157 + }, + "metadata": { + "systemHash": "67ebaa8c", + "toolsHash": "4a353f4d" + } + } + ] +} \ No newline at end of file diff --git a/e2e/fixtures/approvals-this-conversation/openai-2026-11-12T21-26-06-877Z-3bab232c.json b/e2e/fixtures/approvals-this-conversation/openai-2026-11-12T21-26-06-877Z-3bab232c.json new file mode 100644 index 0000000000..6d606cca76 --- /dev/null +++ b/e2e/fixtures/approvals-this-conversation/openai-2026-11-12T21-26-06-877Z-3bab232c.json @@ -0,0 +1,37 @@ +{ + "fixtures": [ + { + "match": { + "userMessage": "Great. Please also buy the lavender seeds at http://host.docker.internal:46600/shop/lavender and give me that order number too.", + "model": "gpt-5.6-terra", + "turnIndex": 8, + "hasToolResult": true + }, + "response": { + "toolCalls": [ + { + "name": "computer_snapshot", + "arguments": "{}", + "id": "call_tCQSPXRdH6bBaUyDKXDJO3Na" + } + ] + }, + "recordedTimings": { + "ttftMs": 769, + "interChunkDelaysMs": [ + 30, + 388, + 12, + 115, + 7, + 156 + ], + "totalDurationMs": 1477 + }, + "metadata": { + "systemHash": "67ebaa8c", + "toolsHash": "4a353f4d" + } + } + ] +} \ No newline at end of file diff --git a/e2e/fixtures/approvals-this-conversation/openai-2026-11-12T21-26-09-088Z-90332b4f.json b/e2e/fixtures/approvals-this-conversation/openai-2026-11-12T21-26-09-088Z-90332b4f.json new file mode 100644 index 0000000000..e4ced83283 --- /dev/null +++ b/e2e/fixtures/approvals-this-conversation/openai-2026-11-12T21-26-09-088Z-90332b4f.json @@ -0,0 +1,49 @@ +{ + "fixtures": [ + { + "match": { + "userMessage": "Great. Please also buy the lavender seeds at http://host.docker.internal:46600/shop/lavender and give me that order number too.", + "model": "gpt-5.6-terra", + "turnIndex": 9, + "hasToolResult": true + }, + "response": { + "toolCalls": [ + { + "name": "computer_click", + "arguments": "{\"ref\":\"f2e13\",\"snapshotId\":4}", + "id": "call_cEFYyxh6ITFsnC7SVV7p1tD7" + } + ] + }, + "recordedTimings": { + "ttftMs": 805, + "interChunkDelaysMs": [ + 7, + 406, + 4, + 4, + 5, + 0, + 15, + 2, + 7, + 4, + 13, + 0, + 0, + 16, + 9, + 138, + 11, + 117 + ], + "totalDurationMs": 1563 + }, + "metadata": { + "systemHash": "67ebaa8c", + "toolsHash": "4a353f4d" + } + } + ] +} \ No newline at end of file diff --git a/e2e/fixtures/approvals-this-conversation/openai-2026-11-12T21-26-11-200Z-1ac24d31.json b/e2e/fixtures/approvals-this-conversation/openai-2026-11-12T21-26-11-200Z-1ac24d31.json new file mode 100644 index 0000000000..ca45ff4b76 --- /dev/null +++ b/e2e/fixtures/approvals-this-conversation/openai-2026-11-12T21-26-11-200Z-1ac24d31.json @@ -0,0 +1,37 @@ +{ + "fixtures": [ + { + "match": { + "userMessage": "Great. Please also buy the lavender seeds at http://host.docker.internal:46600/shop/lavender and give me that order number too.", + "model": "gpt-5.6-terra", + "turnIndex": 10, + "hasToolResult": true + }, + "response": { + "toolCalls": [ + { + "name": "computer_read", + "arguments": "{}", + "id": "call_R9HKXqB3RmptysBg2NGQRoRZ" + } + ] + }, + "recordedTimings": { + "ttftMs": 690, + "interChunkDelaysMs": [ + 35, + 389, + 3, + 145, + 5, + 212 + ], + "totalDurationMs": 1479 + }, + "metadata": { + "systemHash": "67ebaa8c", + "toolsHash": "4a353f4d" + } + } + ] +} \ No newline at end of file diff --git a/e2e/fixtures/approvals-this-conversation/openai-2026-11-12T21-26-13-288Z-80582db6.json b/e2e/fixtures/approvals-this-conversation/openai-2026-11-12T21-26-13-288Z-80582db6.json new file mode 100644 index 0000000000..35bf39a8ef --- /dev/null +++ b/e2e/fixtures/approvals-this-conversation/openai-2026-11-12T21-26-13-288Z-80582db6.json @@ -0,0 +1,49 @@ +{ + "fixtures": [ + { + "match": { + "userMessage": "Great. Please also buy the lavender seeds at http://host.docker.internal:46600/shop/lavender and give me that order number too.", + "model": "gpt-5.6-terra", + "turnIndex": 11, + "hasToolResult": true + }, + "response": { + "content": "Your English lavender seeds order is confirmed; order number **FH-2045**." + }, + "recordedTimings": { + "ttftMs": 778, + "interChunkDelaysMs": [ + 16, + 316, + 4, + 14, + 7, + 0, + 0, + 0, + 2, + 20, + 4, + 6, + 2, + 0, + 0, + 3, + 0, + 25, + 3, + 29, + 111, + 1, + 3, + 262 + ], + "totalDurationMs": 1606 + }, + "metadata": { + "systemHash": "67ebaa8c", + "toolsHash": "4a353f4d" + } + } + ] +} \ No newline at end of file diff --git a/e2e/fixtures/approvals-this-conversation/openai-2026-11-12T21-26-17-063Z-dc36ec1f.json b/e2e/fixtures/approvals-this-conversation/openai-2026-11-12T21-26-17-063Z-dc36ec1f.json new file mode 100644 index 0000000000..462ac2daf3 --- /dev/null +++ b/e2e/fixtures/approvals-this-conversation/openai-2026-11-12T21-26-17-063Z-dc36ec1f.json @@ -0,0 +1,31 @@ +{ + "fixtures": [ + { + "match": { + "userMessage": "You choose which capabilities to load for the message below. You are not answering it.\n\nCapabilities available:\n- research-public-web: Research the public web. Search current public sources and write a cited brief.\n- find-a-document: Find a document. Search the connected document sources for a file and read what it says.\n- find-a-notion-page: Find a Notion page. Search Notion for a page and read what it says.\n- skill-creator: Write a skill. Interview me about a skill I want, then write it and save it as a new skill.\n- bot-creator: Make a Bot. Work out what a new Bot should be, then put it in front of you to create.\n\nMessage:\nOne more, please: the chamomile seeds at http://host.docker.internal:46600/shop/chamomile, with the order number.\n\nReply with only JSON: {\"skills\": [\"\", ...]}.\nChoose every capability that might be needed, including ones you are only somewhat sure about.\nChoosing one that turns out to be unnecessary costs almost nothing. Failing to choose one that\nwas needed means the work cannot be done at all, because its tools will not be loaded. When in\ndoubt, include it. Use an empty list only when the message plainly needs none of them.", + "model": "gpt-5.6-terra", + "turnIndex": 0, + "hasToolResult": false + }, + "response": { + "content": "{\"skills\":[\"find-a-document\",\"find-a-notion-page\"]}", + "usage": { + "prompt_tokens": 274, + "completion_tokens": 113, + "total_tokens": 387, + "prompt_tokens_details": { + "cached_tokens": 0, + "cache_write_tokens": 0, + "audio_tokens": 0 + }, + "completion_tokens_details": { + "reasoning_tokens": 87, + "audio_tokens": 0, + "accepted_prediction_tokens": 0, + "rejected_prediction_tokens": 0 + } + } + } + } + ] +} \ No newline at end of file diff --git a/e2e/fixtures/approvals-this-conversation/openai-2026-11-12T21-26-19-878Z-6241233b.json b/e2e/fixtures/approvals-this-conversation/openai-2026-11-12T21-26-19-878Z-6241233b.json new file mode 100644 index 0000000000..a3870c5c3c --- /dev/null +++ b/e2e/fixtures/approvals-this-conversation/openai-2026-11-12T21-26-19-878Z-6241233b.json @@ -0,0 +1,81 @@ +{ + "fixtures": [ + { + "match": { + "userMessage": "One more, please: the chamomile seeds at http://host.docker.internal:46600/shop/chamomile, with the order number.", + "model": "gpt-5.6-terra", + "turnIndex": 12, + "hasToolResult": false + }, + "response": { + "content": "I’m opening Fernhill Garden Supply’s chamomile checkout to place the order and retrieve its number.", + "toolCalls": [ + { + "name": "computer_navigate", + "arguments": "{\"url\":\"http://host.docker.internal:46600/shop/chamomile\"}", + "id": "call_Pnc8iMDUhxWqPfQTLf6dedXv" + } + ] + }, + "recordedTimings": { + "ttftMs": 846, + "interChunkDelaysMs": [ + 13, + 503, + 342, + 7, + 7, + 9, + 9, + 12, + 10, + 8, + 17, + 3, + 21, + 23, + 4, + 9, + 15, + 18, + 13, + 15, + 9, + 13, + 13, + 19, + 20, + 4, + 7, + 10, + 13, + 41, + 61, + 299, + 7, + 28, + 12, + 23, + 19, + 15, + 9, + 10, + 4, + 14, + 16, + 22, + 9, + 6, + 9, + 6, + 74 + ], + "totalDurationMs": 2696 + }, + "metadata": { + "systemHash": "d36ffef6", + "toolsHash": "4954077c" + } + } + ] +} \ No newline at end of file diff --git a/e2e/fixtures/approvals-this-conversation/openai-2026-11-12T21-26-22-167Z-5181b308.json b/e2e/fixtures/approvals-this-conversation/openai-2026-11-12T21-26-22-167Z-5181b308.json new file mode 100644 index 0000000000..f82036449a --- /dev/null +++ b/e2e/fixtures/approvals-this-conversation/openai-2026-11-12T21-26-22-167Z-5181b308.json @@ -0,0 +1,37 @@ +{ + "fixtures": [ + { + "match": { + "userMessage": "One more, please: the chamomile seeds at http://host.docker.internal:46600/shop/chamomile, with the order number.", + "model": "gpt-5.6-terra", + "turnIndex": 14, + "hasToolResult": true + }, + "response": { + "toolCalls": [ + { + "name": "computer_snapshot", + "arguments": "{}", + "id": "call_dhLJH40VrToEk4WW3IlXqR7v" + } + ] + }, + "recordedTimings": { + "ttftMs": 671, + "interChunkDelaysMs": [ + 22, + 349, + 6, + 134, + 6, + 110 + ], + "totalDurationMs": 1298 + }, + "metadata": { + "systemHash": "d36ffef6", + "toolsHash": "4954077c" + } + } + ] +} \ No newline at end of file diff --git a/e2e/fixtures/approvals-this-conversation/openai-2026-11-12T21-26-24-690Z-3dbfa551.json b/e2e/fixtures/approvals-this-conversation/openai-2026-11-12T21-26-24-690Z-3dbfa551.json new file mode 100644 index 0000000000..10f7b7aa73 --- /dev/null +++ b/e2e/fixtures/approvals-this-conversation/openai-2026-11-12T21-26-24-690Z-3dbfa551.json @@ -0,0 +1,49 @@ +{ + "fixtures": [ + { + "match": { + "userMessage": "One more, please: the chamomile seeds at http://host.docker.internal:46600/shop/chamomile, with the order number.", + "model": "gpt-5.6-terra", + "turnIndex": 15, + "hasToolResult": true + }, + "response": { + "toolCalls": [ + { + "name": "computer_click", + "arguments": "{\"ref\":\"f4e13\",\"snapshotId\":6}", + "id": "call_yw3WnBH3mGt2ft3bLJFCCgNk" + } + ] + }, + "recordedTimings": { + "ttftMs": 761, + "interChunkDelaysMs": [ + 24, + 407, + 6, + 6, + 0, + 0, + 19, + 6, + 8, + 4, + 0, + 0, + 0, + 22, + 8, + 125, + 0, + 451 + ], + "totalDurationMs": 1847 + }, + "metadata": { + "systemHash": "d36ffef6", + "toolsHash": "4954077c" + } + } + ] +} \ No newline at end of file diff --git a/e2e/fixtures/approvals-this-conversation/openai-2026-11-12T21-26-28-777Z-50a3785b.json b/e2e/fixtures/approvals-this-conversation/openai-2026-11-12T21-26-28-777Z-50a3785b.json new file mode 100644 index 0000000000..ac6ac585e9 --- /dev/null +++ b/e2e/fixtures/approvals-this-conversation/openai-2026-11-12T21-26-28-777Z-50a3785b.json @@ -0,0 +1,37 @@ +{ + "fixtures": [ + { + "match": { + "userMessage": "One more, please: the chamomile seeds at http://host.docker.internal:46600/shop/chamomile, with the order number.", + "model": "gpt-5.6-terra", + "turnIndex": 16, + "hasToolResult": true + }, + "response": { + "toolCalls": [ + { + "name": "computer_read", + "arguments": "{}", + "id": "call_9bt0jr2HnfMUVjPEWAvE1rd9" + } + ] + }, + "recordedTimings": { + "ttftMs": 742, + "interChunkDelaysMs": [ + 8, + 412, + 3, + 137, + 13, + 132 + ], + "totalDurationMs": 1447 + }, + "metadata": { + "systemHash": "a9084f6a", + "toolsHash": "161314cb" + } + } + ] +} \ No newline at end of file diff --git a/e2e/fixtures/approvals-this-conversation/openai-2026-11-12T21-26-30-457Z-b1831a4c.json b/e2e/fixtures/approvals-this-conversation/openai-2026-11-12T21-26-30-457Z-b1831a4c.json new file mode 100644 index 0000000000..f13f85885d --- /dev/null +++ b/e2e/fixtures/approvals-this-conversation/openai-2026-11-12T21-26-30-457Z-b1831a4c.json @@ -0,0 +1,50 @@ +{ + "fixtures": [ + { + "match": { + "userMessage": "One more, please: the chamomile seeds at http://host.docker.internal:46600/shop/chamomile, with the order number.", + "model": "gpt-5.6-terra", + "turnIndex": 17, + "hasToolResult": true + }, + "response": { + "content": "Your German chamomile seeds order is confirmed; order number **FH-2046**." + }, + "recordedTimings": { + "ttftMs": 758, + "interChunkDelaysMs": [ + 20, + 333, + 2, + 1, + 4, + 0, + 0, + 21, + 6, + 5, + 3, + 1, + 3, + 0, + 0, + 23, + 7, + 7, + 6, + 9, + 39, + 95, + 3, + 2, + 137 + ], + "totalDurationMs": 1485 + }, + "metadata": { + "systemHash": "a9084f6a", + "toolsHash": "161314cb" + } + } + ] +} \ No newline at end of file diff --git a/e2e/fixtures/computer-browse/openai-2026-11-12T21-26-34-921Z-cde5f065.json b/e2e/fixtures/computer-browse/openai-2026-11-12T21-26-34-921Z-cde5f065.json new file mode 100644 index 0000000000..a94f9deffc --- /dev/null +++ b/e2e/fixtures/computer-browse/openai-2026-11-12T21-26-34-921Z-cde5f065.json @@ -0,0 +1,31 @@ +{ + "fixtures": [ + { + "match": { + "userMessage": "You choose which capabilities to load for the message below. You are not answering it.\n\nCapabilities available:\n- research-public-web: Research the public web. Search current public sources and write a cited brief.\n- find-a-document: Find a document. Search the connected document sources for a file and read what it says.\n- find-a-notion-page: Find a Notion page. Search Notion for a page and read what it says.\n- skill-creator: Write a skill. Interview me about a skill I want, then write it and save it as a new skill.\n- bot-creator: Make a Bot. Work out what a new Bot should be, then put it in front of you to create.\n\nMessage:\nOpen http://host.docker.internal:46600/ and tell me what time the greenhouse opens on Saturdays. Answer in one short sentence.\n\nReply with only JSON: {\"skills\": [\"\", ...]}.\nChoose every capability that might be needed, including ones you are only somewhat sure about.\nChoosing one that turns out to be unnecessary costs almost nothing. Failing to choose one that\nwas needed means the work cannot be done at all, because its tools will not be loaded. When in\ndoubt, include it. Use an empty list only when the message plainly needs none of them.", + "model": "gpt-5.6-terra", + "turnIndex": 0, + "hasToolResult": false + }, + "response": { + "content": "{\"skills\":[\"research-public-web\"]}", + "usage": { + "prompt_tokens": 273, + "completion_tokens": 59, + "total_tokens": 332, + "prompt_tokens_details": { + "cached_tokens": 0, + "cache_write_tokens": 0, + "audio_tokens": 0 + }, + "completion_tokens_details": { + "reasoning_tokens": 39, + "audio_tokens": 0, + "accepted_prediction_tokens": 0, + "rejected_prediction_tokens": 0 + } + } + } + } + ] +} \ No newline at end of file diff --git a/e2e/fixtures/computer-browse/openai-2026-11-12T21-26-36-427Z-eaa30c54.json b/e2e/fixtures/computer-browse/openai-2026-11-12T21-26-36-427Z-eaa30c54.json new file mode 100644 index 0000000000..0b6311cbab --- /dev/null +++ b/e2e/fixtures/computer-browse/openai-2026-11-12T21-26-36-427Z-eaa30c54.json @@ -0,0 +1,34 @@ +{ + "fixtures": [ + { + "match": { + "userMessage": "Asked: Open http://host.docker.internal:46600/ and tell me what time the greenhouse opens on Saturdays. Answer in one short sentence.", + "model": "gpt-5.6-terra", + "turnIndex": 0, + "hasToolResult": false + }, + "response": { + "content": "Greenhouse Saturday Opening Hours", + "usage": { + "prompt_tokens": 130, + "completion_tokens": 8, + "total_tokens": 138, + "prompt_tokens_details": { + "cached_tokens": 0, + "cache_write_tokens": 0, + "audio_tokens": 0 + }, + "completion_tokens_details": { + "reasoning_tokens": 0, + "audio_tokens": 0, + "accepted_prediction_tokens": 0, + "rejected_prediction_tokens": 0 + } + } + }, + "metadata": { + "systemHash": "dedd9ec9" + } + } + ] +} \ No newline at end of file diff --git a/e2e/fixtures/computer-browse/openai-2026-11-12T21-26-37-710Z-51de2fa9.json b/e2e/fixtures/computer-browse/openai-2026-11-12T21-26-37-710Z-51de2fa9.json new file mode 100644 index 0000000000..a1652d7bf7 --- /dev/null +++ b/e2e/fixtures/computer-browse/openai-2026-11-12T21-26-37-710Z-51de2fa9.json @@ -0,0 +1,72 @@ +{ + "fixtures": [ + { + "match": { + "userMessage": "Open http://host.docker.internal:46600/ and tell me what time the greenhouse opens on Saturdays. Answer in one short sentence.", + "model": "gpt-5.6-terra", + "turnIndex": 0, + "hasToolResult": false + }, + "response": { + "content": "I’m opening the garden shop site to check its Saturday greenhouse hours.", + "toolCalls": [ + { + "name": "computer_navigate", + "arguments": "{\"url\":\"http://host.docker.internal:46600/\"}", + "id": "call_a1QYBqvS0OC2LdWNT26ELkC2" + } + ] + }, + "recordedTimings": { + "ttftMs": 729, + "interChunkDelaysMs": [ + 10, + 523, + 304, + 1, + 8, + 4, + 0, + 7, + 4, + 6, + 5, + 7, + 406, + 22, + 30, + 22, + 11, + 17, + 4, + 16, + 60, + 13, + 7, + 11, + 4, + 28, + 25, + 6, + 11, + 25, + 50, + 61, + 7, + 61, + 28, + 46, + 14, + 20, + 11, + 150 + ], + "totalDurationMs": 2774 + }, + "metadata": { + "systemHash": "136e55bc", + "toolsHash": "a02cfa3f" + } + } + ] +} \ No newline at end of file diff --git a/e2e/fixtures/computer-browse/openai-2026-11-12T21-26-39-851Z-9c4775c5.json b/e2e/fixtures/computer-browse/openai-2026-11-12T21-26-39-851Z-9c4775c5.json new file mode 100644 index 0000000000..bf4c02a40c --- /dev/null +++ b/e2e/fixtures/computer-browse/openai-2026-11-12T21-26-39-851Z-9c4775c5.json @@ -0,0 +1,45 @@ +{ + "fixtures": [ + { + "match": { + "userMessage": "Open http://host.docker.internal:46600/ and tell me what time the greenhouse opens on Saturdays. Answer in one short sentence.", + "model": "gpt-5.6-terra", + "turnIndex": 2, + "hasToolResult": true + }, + "response": { + "content": "The greenhouse opens at **7:40 am on Saturdays**." + }, + "recordedTimings": { + "ttftMs": 721, + "interChunkDelaysMs": [ + 27, + 419, + 2, + 4, + 0, + 2, + 0, + 25, + 6, + 1, + 2, + 3, + 2, + 0, + 33, + 10, + 151, + 6, + 2, + 172 + ], + "totalDurationMs": 1588 + }, + "metadata": { + "systemHash": "136e55bc", + "toolsHash": "a02cfa3f" + } + } + ] +} \ No newline at end of file diff --git a/e2e/fixtures/computer-form/openai-2026-11-12T21-34-16-806Z-22682346.json b/e2e/fixtures/computer-form/openai-2026-11-12T21-34-16-806Z-22682346.json new file mode 100644 index 0000000000..5a0e97f525 --- /dev/null +++ b/e2e/fixtures/computer-form/openai-2026-11-12T21-34-16-806Z-22682346.json @@ -0,0 +1,31 @@ +{ + "fixtures": [ + { + "match": { + "userMessage": "You choose which capabilities to load for the message below. You are not answering it.\n\nCapabilities available:\n- research-public-web: Research the public web. Search current public sources and write a cited brief.\n- find-a-document: Find a document. Search the connected document sources for a file and read what it says.\n- find-a-notion-page: Find a Notion page. Search Notion for a page and read what it says.\n- skill-creator: Write a skill. Interview me about a skill I want, then write it and save it as a new skill.\n- bot-creator: Make a Bot. Work out what a new Bot should be, then put it in front of you to create.\n\nMessage:\nGo to http://host.docker.internal:46600/volunteer and fill in the volunteer form for me in one go: full name Robin Okafor, email robin.okafor@example.org, preferred shift Saturday morning, and tick the code of conduct box. Fill in all three fields and tick the box together, in a single step from one snapshot. Then submit it and tell me the reference number in one plain sentence.\n\nReply with only JSON: {\"skills\": [\"\", ...]}.\nChoose every capability that might be needed, including ones you are only somewhat sure about.\nChoosing one that turns out to be unnecessary costs almost nothing. Failing to choose one that\nwas needed means the work cannot be done at all, because its tools will not be loaded. When in\ndoubt, include it. Use an empty list only when the message plainly needs none of them.", + "model": "gpt-5.6-terra", + "turnIndex": 0, + "hasToolResult": false + }, + "response": { + "content": "{\"skills\":[]}", + "usage": { + "prompt_tokens": 332, + "completion_tokens": 59, + "total_tokens": 391, + "prompt_tokens_details": { + "cached_tokens": 0, + "cache_write_tokens": 0, + "audio_tokens": 0 + }, + "completion_tokens_details": { + "reasoning_tokens": 43, + "audio_tokens": 0, + "accepted_prediction_tokens": 0, + "rejected_prediction_tokens": 0 + } + } + } + } + ] +} \ No newline at end of file diff --git a/e2e/fixtures/computer-form/openai-2026-11-12T21-34-19-025Z-c111ed02.json b/e2e/fixtures/computer-form/openai-2026-11-12T21-34-19-025Z-c111ed02.json new file mode 100644 index 0000000000..2d04529b25 --- /dev/null +++ b/e2e/fixtures/computer-form/openai-2026-11-12T21-34-19-025Z-c111ed02.json @@ -0,0 +1,73 @@ +{ + "fixtures": [ + { + "match": { + "userMessage": "Go to http://host.docker.internal:46600/volunteer and fill in the volunteer form for me in one go: full name Robin Okafor, email robin.okafor@example.org, preferred shift Saturday morning, and tick the code of conduct box. Fill in all three fields and tick the box together, in a single step from one snapshot. Then submit it and tell me the reference number in one plain sentence.", + "model": "gpt-5.6-terra", + "turnIndex": 0, + "hasToolResult": false + }, + "response": { + "content": "I’m opening the volunteer form and will complete the requested details.", + "toolCalls": [ + { + "name": "computer_navigate", + "arguments": "{\"url\":\"http://host.docker.internal:46600/volunteer\"}", + "id": "call_0szWOYa5xj2R0ky4x9YTMIV6" + } + ] + }, + "recordedTimings": { + "ttftMs": 840, + "interChunkDelaysMs": [ + 5, + 569, + 163, + 0, + 5, + 2, + 3, + 4, + 2, + 2, + 5, + 2, + 1, + 0, + 0, + 0, + 5, + 23, + 111, + 5, + 3, + 5, + 0, + 0, + 2, + 0, + 0, + 9, + 3, + 1, + 0, + 3, + 1, + 4, + 1, + 29, + 0, + 3, + 133, + 8, + 194 + ], + "totalDurationMs": 2146 + }, + "metadata": { + "systemHash": "7a8bda2d", + "toolsHash": "6d7140c9" + } + } + ] +} \ No newline at end of file diff --git a/e2e/fixtures/computer-form/openai-2026-11-12T21-34-21-549Z-a0e599e7.json b/e2e/fixtures/computer-form/openai-2026-11-12T21-34-21-549Z-a0e599e7.json new file mode 100644 index 0000000000..4f7e4fe38f --- /dev/null +++ b/e2e/fixtures/computer-form/openai-2026-11-12T21-34-21-549Z-a0e599e7.json @@ -0,0 +1,37 @@ +{ + "fixtures": [ + { + "match": { + "userMessage": "Go to http://host.docker.internal:46600/volunteer and fill in the volunteer form for me in one go: full name Robin Okafor, email robin.okafor@example.org, preferred shift Saturday morning, and tick the code of conduct box. Fill in all three fields and tick the box together, in a single step from one snapshot. Then submit it and tell me the reference number in one plain sentence.", + "model": "gpt-5.6-terra", + "turnIndex": 2, + "hasToolResult": true + }, + "response": { + "toolCalls": [ + { + "name": "computer_snapshot", + "arguments": "{}", + "id": "call_fwrmGRiAXrrRCXA112CubKQT" + } + ] + }, + "recordedTimings": { + "ttftMs": 679, + "interChunkDelaysMs": [ + 28, + 861, + 9, + 143, + 0, + 211 + ], + "totalDurationMs": 1931 + }, + "metadata": { + "systemHash": "7a8bda2d", + "toolsHash": "6d7140c9" + } + } + ] +} \ No newline at end of file diff --git a/e2e/fixtures/computer-form/openai-2026-11-12T21-34-22-806Z-a2a3e138.json b/e2e/fixtures/computer-form/openai-2026-11-12T21-34-22-806Z-a2a3e138.json new file mode 100644 index 0000000000..1363d3b36d --- /dev/null +++ b/e2e/fixtures/computer-form/openai-2026-11-12T21-34-22-806Z-a2a3e138.json @@ -0,0 +1,34 @@ +{ + "fixtures": [ + { + "match": { + "userMessage": "Asked: Go to http://host.docker.internal:46600/volunteer and fill in the volunteer form for me in one go: full name Robin Okafor, email robin.okafor@example.org, preferred shift Saturday morning, and tick the code of conduct box. Fill in all three fields and tick the box together, in a single step from one snapshot. Then submit it and tell me the reference number in one plain sentence.", + "model": "gpt-5.6-terra", + "turnIndex": 0, + "hasToolResult": false + }, + "response": { + "content": "Volunteer Form Submission", + "usage": { + "prompt_tokens": 205, + "completion_tokens": 28, + "total_tokens": 233, + "prompt_tokens_details": { + "cached_tokens": 0, + "cache_write_tokens": 0, + "audio_tokens": 0 + }, + "completion_tokens_details": { + "reasoning_tokens": 16, + "audio_tokens": 0, + "accepted_prediction_tokens": 0, + "rejected_prediction_tokens": 0 + } + } + }, + "metadata": { + "systemHash": "dedd9ec9" + } + } + ] +} \ No newline at end of file diff --git a/e2e/fixtures/computer-form/openai-2026-11-12T21-34-25-268Z-9c27282c.json b/e2e/fixtures/computer-form/openai-2026-11-12T21-34-25-268Z-9c27282c.json new file mode 100644 index 0000000000..46186377bf --- /dev/null +++ b/e2e/fixtures/computer-form/openai-2026-11-12T21-34-25-268Z-9c27282c.json @@ -0,0 +1,66 @@ +{ + "fixtures": [ + { + "match": { + "userMessage": "Go to http://host.docker.internal:46600/volunteer and fill in the volunteer form for me in one go: full name Robin Okafor, email robin.okafor@example.org, preferred shift Saturday morning, and tick the code of conduct box. Fill in all three fields and tick the box together, in a single step from one snapshot. Then submit it and tell me the reference number in one plain sentence.", + "model": "gpt-5.6-terra", + "turnIndex": 3, + "hasToolResult": true + }, + "response": { + "toolCalls": [ + { + "name": "computer_type", + "arguments": "{\"ref\":\"e12\",\"snapshotId\":2,\"text\":\"Robin Okafor\",\"submit\":false}", + "id": "call_NjYZbndRshsjnUvbeLu0XBCY" + }, + { + "name": "computer_type", + "arguments": "{\"ref\":\"e14\",\"snapshotId\":2,\"text\":\"robin.okafor@example.org\",\"submit\":false}", + "id": "call_5S9TmZtwEhi5e9X4rFUs5OaM" + }, + { + "name": "computer_type", + "arguments": "{\"ref\":\"e16\",\"snapshotId\":2,\"text\":\"Saturday morning\",\"submit\":false}", + "id": "call_YHnRJsbmXy9W9mmHYzj1HBlK" + }, + { + "name": "computer_click", + "arguments": "{\"ref\":\"e18\",\"snapshotId\":2}", + "id": "call_ehfwuawtzwZzyz04Fm4MqhAS" + } + ] + }, + "recordedTimings": { + "ttftMs": 822, + "interChunkDelaysMs": [ + 21, + 271, + 1124, + 31, + 7, + 21, + 7, + 2, + 3, + 1, + 2, + 0, + 4, + 1, + 0, + 0, + 0, + 4, + 0, + 109 + ], + "totalDurationMs": 2430 + }, + "metadata": { + "systemHash": "7a8bda2d", + "toolsHash": "6d7140c9" + } + } + ] +} \ No newline at end of file diff --git a/e2e/fixtures/computer-form/openai-2026-11-12T21-34-29-037Z-c182b7e6.json b/e2e/fixtures/computer-form/openai-2026-11-12T21-34-29-037Z-c182b7e6.json new file mode 100644 index 0000000000..d57a71f6ec --- /dev/null +++ b/e2e/fixtures/computer-form/openai-2026-11-12T21-34-29-037Z-c182b7e6.json @@ -0,0 +1,39 @@ +{ + "fixtures": [ + { + "match": { + "userMessage": "Go to http://host.docker.internal:46600/volunteer and fill in the volunteer form for me in one go: full name Robin Okafor, email robin.okafor@example.org, preferred shift Saturday morning, and tick the code of conduct box. Fill in all three fields and tick the box together, in a single step from one snapshot. Then submit it and tell me the reference number in one plain sentence.", + "model": "gpt-5.6-terra", + "turnIndex": 7, + "hasToolResult": true + }, + "response": { + "toolCalls": [ + { + "name": "computer_snapshot", + "arguments": "{}", + "id": "call_zqlpYSFUqy2cENMsRXr4MVK1" + } + ] + }, + "recordedTimings": { + "ttftMs": 953, + "interChunkDelaysMs": [ + 9, + 305, + 920, + 6, + 16, + 137, + 10, + 160 + ], + "totalDurationMs": 2516 + }, + "metadata": { + "systemHash": "7a8bda2d", + "toolsHash": "6d7140c9" + } + } + ] +} \ No newline at end of file diff --git a/e2e/fixtures/computer-form/openai-2026-11-12T21-34-31-440Z-13d5acd9.json b/e2e/fixtures/computer-form/openai-2026-11-12T21-34-31-440Z-13d5acd9.json new file mode 100644 index 0000000000..a69790bfb6 --- /dev/null +++ b/e2e/fixtures/computer-form/openai-2026-11-12T21-34-31-440Z-13d5acd9.json @@ -0,0 +1,47 @@ +{ + "fixtures": [ + { + "match": { + "userMessage": "Go to http://host.docker.internal:46600/volunteer and fill in the volunteer form for me in one go: full name Robin Okafor, email robin.okafor@example.org, preferred shift Saturday morning, and tick the code of conduct box. Fill in all three fields and tick the box together, in a single step from one snapshot. Then submit it and tell me the reference number in one plain sentence.", + "model": "gpt-5.6-terra", + "turnIndex": 8, + "hasToolResult": true + }, + "response": { + "toolCalls": [ + { + "name": "computer_click", + "arguments": "{\"ref\":\"e19\",\"snapshotId\":3}", + "id": "call_iQRjtNmUQWYEeMVICR92tPMZ" + } + ] + }, + "recordedTimings": { + "ttftMs": 943, + "interChunkDelaysMs": [ + 7, + 392, + 9, + 5, + 0, + 3, + 38, + 11, + 8, + 6, + 24, + 35, + 44, + 66, + 7, + 140 + ], + "totalDurationMs": 1738 + }, + "metadata": { + "systemHash": "7a8bda2d", + "toolsHash": "6d7140c9" + } + } + ] +} \ No newline at end of file diff --git a/e2e/fixtures/computer-form/openai-2026-11-12T21-34-34-191Z-d2bf85a1.json b/e2e/fixtures/computer-form/openai-2026-11-12T21-34-34-191Z-d2bf85a1.json new file mode 100644 index 0000000000..f3f5be679a --- /dev/null +++ b/e2e/fixtures/computer-form/openai-2026-11-12T21-34-34-191Z-d2bf85a1.json @@ -0,0 +1,37 @@ +{ + "fixtures": [ + { + "match": { + "userMessage": "Go to http://host.docker.internal:46600/volunteer and fill in the volunteer form for me in one go: full name Robin Okafor, email robin.okafor@example.org, preferred shift Saturday morning, and tick the code of conduct box. Fill in all three fields and tick the box together, in a single step from one snapshot. Then submit it and tell me the reference number in one plain sentence.", + "model": "gpt-5.6-terra", + "turnIndex": 9, + "hasToolResult": true + }, + "response": { + "toolCalls": [ + { + "name": "computer_read", + "arguments": "{}", + "id": "call_Au9FTfUyYALuLSnUE09UjeJw" + } + ] + }, + "recordedTimings": { + "ttftMs": 936, + "interChunkDelaysMs": [ + 5, + 736, + 2, + 155, + 30, + 153 + ], + "totalDurationMs": 2017 + }, + "metadata": { + "systemHash": "7a8bda2d", + "toolsHash": "6d7140c9" + } + } + ] +} \ No newline at end of file diff --git a/e2e/fixtures/computer-form/openai-2026-11-12T21-34-36-675Z-d2427cb2.json b/e2e/fixtures/computer-form/openai-2026-11-12T21-34-36-675Z-d2427cb2.json new file mode 100644 index 0000000000..bc310ea68d --- /dev/null +++ b/e2e/fixtures/computer-form/openai-2026-11-12T21-34-36-675Z-d2427cb2.json @@ -0,0 +1,47 @@ +{ + "fixtures": [ + { + "match": { + "userMessage": "Go to http://host.docker.internal:46600/volunteer and fill in the volunteer form for me in one go: full name Robin Okafor, email robin.okafor@example.org, preferred shift Saturday morning, and tick the code of conduct box. Fill in all three fields and tick the box together, in a single step from one snapshot. Then submit it and tell me the reference number in one plain sentence.", + "model": "gpt-5.6-terra", + "turnIndex": 10, + "hasToolResult": true + }, + "response": { + "content": "Your volunteer application was submitted with reference number **VOL-3172**." + }, + "recordedTimings": { + "ttftMs": 1141, + "interChunkDelaysMs": [ + 160, + 239, + 14, + 5, + 1, + 6, + 0, + 12, + 0, + 0, + 0, + 24, + 3, + 4, + 6, + 3, + 3, + 4, + 158, + 12, + 15, + 145 + ], + "totalDurationMs": 1955 + }, + "metadata": { + "systemHash": "7a8bda2d", + "toolsHash": "6d7140c9" + } + } + ] +} \ No newline at end of file diff --git a/e2e/fixtures/computer-sign-in/openai-2026-11-12T22-04-03-762Z-51852a1f.json b/e2e/fixtures/computer-sign-in/openai-2026-11-12T22-04-03-762Z-51852a1f.json new file mode 100644 index 0000000000..bf16051368 --- /dev/null +++ b/e2e/fixtures/computer-sign-in/openai-2026-11-12T22-04-03-762Z-51852a1f.json @@ -0,0 +1,31 @@ +{ + "fixtures": [ + { + "match": { + "userMessage": "You choose which capabilities to load for the message below. You are not answering it.\n\nCapabilities available:\n- research-public-web: Research the public web. Search current public sources and write a cited brief.\n- find-a-document: Find a document. Search the connected document sources for a file and read what it says.\n- find-a-notion-page: Find a Notion page. Search Notion for a page and read what it says.\n- skill-creator: Write a skill. Interview me about a skill I want, then write it and save it as a new skill.\n- bot-creator: Make a Bot. Work out what a new Bot should be, then put it in front of you to create.\n\nMessage:\nOpen http://host.docker.internal:46600/members, fill in my username robin.okafor, then hand me the browser so I can type my password myself. After I hand it back, tell me my seed allowance in one plain sentence.\n\nReply with only JSON: {\"skills\": [\"\", ...]}.\nChoose every capability that might be needed, including ones you are only somewhat sure about.\nChoosing one that turns out to be unnecessary costs almost nothing. Failing to choose one that\nwas needed means the work cannot be done at all, because its tools will not be loaded. When in\ndoubt, include it. Use an empty list only when the message plainly needs none of them.", + "model": "gpt-5.6-terra", + "turnIndex": 0, + "hasToolResult": false + }, + "response": { + "content": "{\"skills\": []}", + "usage": { + "prompt_tokens": 296, + "completion_tokens": 36, + "total_tokens": 332, + "prompt_tokens_details": { + "cached_tokens": 0, + "cache_write_tokens": 0, + "audio_tokens": 0 + }, + "completion_tokens_details": { + "reasoning_tokens": 19, + "audio_tokens": 0, + "accepted_prediction_tokens": 0, + "rejected_prediction_tokens": 0 + } + } + } + } + ] +} \ No newline at end of file diff --git a/e2e/fixtures/computer-sign-in/openai-2026-11-12T22-04-06-906Z-8f1ed8be.json b/e2e/fixtures/computer-sign-in/openai-2026-11-12T22-04-06-906Z-8f1ed8be.json new file mode 100644 index 0000000000..acdbd4aa8d --- /dev/null +++ b/e2e/fixtures/computer-sign-in/openai-2026-11-12T22-04-06-906Z-8f1ed8be.json @@ -0,0 +1,82 @@ +{ + "fixtures": [ + { + "match": { + "userMessage": "Open http://host.docker.internal:46600/members, fill in my username robin.okafor, then hand me the browser so I can type my password myself. After I hand it back, tell me my seed allowance in one plain sentence.", + "model": "gpt-5.6-terra", + "turnIndex": 0, + "hasToolResult": false + }, + "response": { + "content": "I’m opening the Fernhill Garden Supply members’ area and will enter your username before handing over for your password.", + "toolCalls": [ + { + "name": "computer_navigate", + "arguments": "{\"url\":\"http://host.docker.internal:46600/members\"}", + "id": "call_A5SZ23mruV2sf4bNfkL7brHH" + } + ] + }, + "recordedTimings": { + "ttftMs": 772, + "interChunkDelaysMs": [ + 19, + 1335, + 177, + 3, + 0, + 0, + 0, + 17, + 2, + 38, + 0, + 0, + 28, + 5, + 0, + 29, + 15, + 6, + 2, + 1, + 3, + 0, + 6, + 40, + 3, + 1, + 4, + 0, + 132, + 5, + 2, + 4, + 5, + 0, + 0, + 0, + 0, + 8, + 13, + 4, + 4, + 3, + 5, + 2, + 0, + 4, + 8, + 114, + 0, + 183 + ], + "totalDurationMs": 3002 + }, + "metadata": { + "systemHash": "1499f2d6", + "toolsHash": "6d7140c9" + } + } + ] +} \ No newline at end of file diff --git a/e2e/fixtures/computer-sign-in/openai-2026-11-12T22-04-10-721Z-e84a2793.json b/e2e/fixtures/computer-sign-in/openai-2026-11-12T22-04-10-721Z-e84a2793.json new file mode 100644 index 0000000000..584c80f1e0 --- /dev/null +++ b/e2e/fixtures/computer-sign-in/openai-2026-11-12T22-04-10-721Z-e84a2793.json @@ -0,0 +1,37 @@ +{ + "fixtures": [ + { + "match": { + "userMessage": "Open http://host.docker.internal:46600/members, fill in my username robin.okafor, then hand me the browser so I can type my password myself. After I hand it back, tell me my seed allowance in one plain sentence.", + "model": "gpt-5.6-terra", + "turnIndex": 2, + "hasToolResult": true + }, + "response": { + "toolCalls": [ + { + "name": "computer_snapshot", + "arguments": "{}", + "id": "call_VKyxVmqeixvDDPqMvENES1BN" + } + ] + }, + "recordedTimings": { + "ttftMs": 1400, + "interChunkDelaysMs": [ + 7, + 398, + 16, + 180, + 12, + 201 + ], + "totalDurationMs": 2214 + }, + "metadata": { + "systemHash": "1499f2d6", + "toolsHash": "6d7140c9" + } + } + ] +} \ No newline at end of file diff --git a/e2e/fixtures/computer-sign-in/openai-2026-11-12T22-04-10-731Z-8b2fb702.json b/e2e/fixtures/computer-sign-in/openai-2026-11-12T22-04-10-731Z-8b2fb702.json new file mode 100644 index 0000000000..e065a95a62 --- /dev/null +++ b/e2e/fixtures/computer-sign-in/openai-2026-11-12T22-04-10-731Z-8b2fb702.json @@ -0,0 +1,34 @@ +{ + "fixtures": [ + { + "match": { + "userMessage": "Asked: Open http://host.docker.internal:46600/members, fill in my username robin.okafor, then hand me the browser so I can type my password myself. After I hand it back, tell me my seed allowance in one plain sentence.", + "model": "gpt-5.6-terra", + "turnIndex": 0, + "hasToolResult": false + }, + "response": { + "content": "Member Login and Seed Allowance", + "usage": { + "prompt_tokens": 179, + "completion_tokens": 35, + "total_tokens": 214, + "prompt_tokens_details": { + "cached_tokens": 0, + "cache_write_tokens": 0, + "audio_tokens": 0 + }, + "completion_tokens_details": { + "reasoning_tokens": 20, + "audio_tokens": 0, + "accepted_prediction_tokens": 0, + "rejected_prediction_tokens": 0 + } + } + }, + "metadata": { + "systemHash": "dedd9ec9" + } + } + ] +} \ No newline at end of file diff --git a/e2e/fixtures/computer-sign-in/openai-2026-11-12T22-04-13-422Z-41a7e89e.json b/e2e/fixtures/computer-sign-in/openai-2026-11-12T22-04-13-422Z-41a7e89e.json new file mode 100644 index 0000000000..53c903faab --- /dev/null +++ b/e2e/fixtures/computer-sign-in/openai-2026-11-12T22-04-13-422Z-41a7e89e.json @@ -0,0 +1,59 @@ +{ + "fixtures": [ + { + "match": { + "userMessage": "Open http://host.docker.internal:46600/members, fill in my username robin.okafor, then hand me the browser so I can type my password myself. After I hand it back, tell me my seed allowance in one plain sentence.", + "model": "gpt-5.6-terra", + "turnIndex": 3, + "hasToolResult": true + }, + "response": { + "toolCalls": [ + { + "name": "computer_type", + "arguments": "{\"ref\":\"e12\",\"snapshotId\":2,\"text\":\"robin.okafor\",\"submit\":false}", + "id": "call_B4wwQiRJZjnanLgkE9Yua97m" + } + ] + }, + "recordedTimings": { + "ttftMs": 1163, + "interChunkDelaysMs": [ + 24, + 306, + 5, + 3, + 4, + 4, + 19, + 9, + 2, + 3, + 3, + 4, + 6, + 11, + 0, + 6, + 9, + 20, + 8, + 5, + 6, + 5, + 12, + 51, + 39, + 59, + 0, + 300 + ], + "totalDurationMs": 2086 + }, + "metadata": { + "systemHash": "1499f2d6", + "toolsHash": "6d7140c9" + } + } + ] +} \ No newline at end of file diff --git a/e2e/fixtures/computer-sign-in/openai-2026-11-12T22-04-16-101Z-d2d7a905.json b/e2e/fixtures/computer-sign-in/openai-2026-11-12T22-04-16-101Z-d2d7a905.json new file mode 100644 index 0000000000..5ccad67523 --- /dev/null +++ b/e2e/fixtures/computer-sign-in/openai-2026-11-12T22-04-16-101Z-d2d7a905.json @@ -0,0 +1,47 @@ +{ + "fixtures": [ + { + "match": { + "userMessage": "Open http://host.docker.internal:46600/members, fill in my username robin.okafor, then hand me the browser so I can type my password myself. After I hand it back, tell me my seed allowance in one plain sentence.", + "model": "gpt-5.6-terra", + "turnIndex": 4, + "hasToolResult": true + }, + "response": { + "toolCalls": [ + { + "name": "computer_click", + "arguments": "{\"ref\":\"e14\",\"snapshotId\":2}", + "id": "call_AnaIKFMajr7dOGsC08OlylhO" + } + ] + }, + "recordedTimings": { + "ttftMs": 768, + "interChunkDelaysMs": [ + 6, + 511, + 8, + 11, + 2, + 3, + 22, + 7, + 9, + 5, + 5, + 2, + 9, + 143, + 6, + 199 + ], + "totalDurationMs": 1716 + }, + "metadata": { + "systemHash": "1499f2d6", + "toolsHash": "6d7140c9" + } + } + ] +} \ No newline at end of file diff --git a/e2e/fixtures/computer-sign-in/openai-2026-11-12T22-04-23-065Z-f5304e36.json b/e2e/fixtures/computer-sign-in/openai-2026-11-12T22-04-23-065Z-f5304e36.json new file mode 100644 index 0000000000..5f43c28937 --- /dev/null +++ b/e2e/fixtures/computer-sign-in/openai-2026-11-12T22-04-23-065Z-f5304e36.json @@ -0,0 +1,39 @@ +{ + "fixtures": [ + { + "match": { + "userMessage": "Open http://host.docker.internal:46600/members, fill in my username robin.okafor, then hand me the browser so I can type my password myself. After I hand it back, tell me my seed allowance in one plain sentence.", + "model": "gpt-5.6-terra", + "turnIndex": 5, + "hasToolResult": true + }, + "response": { + "toolCalls": [ + { + "name": "computer_snapshot", + "arguments": "{}", + "id": "call_5joTTAcBmu5mctR8BoblDLTd" + } + ] + }, + "recordedTimings": { + "ttftMs": 823, + "interChunkDelaysMs": [ + 3, + 352, + 334, + 3, + 43, + 102, + 7, + 212 + ], + "totalDurationMs": 1879 + }, + "metadata": { + "systemHash": "e4af14ab", + "toolsHash": "88b34f59" + } + } + ] +} \ No newline at end of file diff --git a/e2e/fixtures/computer-sign-in/openai-2026-11-12T22-04-25-624Z-993825de.json b/e2e/fixtures/computer-sign-in/openai-2026-11-12T22-04-25-624Z-993825de.json new file mode 100644 index 0000000000..4b1d2a36dc --- /dev/null +++ b/e2e/fixtures/computer-sign-in/openai-2026-11-12T22-04-25-624Z-993825de.json @@ -0,0 +1,37 @@ +{ + "fixtures": [ + { + "match": { + "userMessage": "Open http://host.docker.internal:46600/members, fill in my username robin.okafor, then hand me the browser so I can type my password myself. After I hand it back, tell me my seed allowance in one plain sentence.", + "model": "gpt-5.6-terra", + "turnIndex": 6, + "hasToolResult": true + }, + "response": { + "toolCalls": [ + { + "name": "computer_read", + "arguments": "{}", + "id": "call_2SUEiDuOTo0aMZdDsvPS0jdN" + } + ] + }, + "recordedTimings": { + "ttftMs": 1039, + "interChunkDelaysMs": [ + 3, + 283, + 2, + 722, + 23, + 237 + ], + "totalDurationMs": 2309 + }, + "metadata": { + "systemHash": "e4af14ab", + "toolsHash": "88b34f59" + } + } + ] +} \ No newline at end of file diff --git a/e2e/fixtures/computer-sign-in/openai-2026-11-12T22-04-27-719Z-57a9f298.json b/e2e/fixtures/computer-sign-in/openai-2026-11-12T22-04-27-719Z-57a9f298.json new file mode 100644 index 0000000000..a81af6603a --- /dev/null +++ b/e2e/fixtures/computer-sign-in/openai-2026-11-12T22-04-27-719Z-57a9f298.json @@ -0,0 +1,43 @@ +{ + "fixtures": [ + { + "match": { + "userMessage": "Open http://host.docker.internal:46600/members, fill in my username robin.okafor, then hand me the browser so I can type my password myself. After I hand it back, tell me my seed allowance in one plain sentence.", + "model": "gpt-5.6-terra", + "turnIndex": 7, + "hasToolResult": true + }, + "response": { + "content": "Your seed allowance is **12 packets this season**." + }, + "recordedTimings": { + "ttftMs": 1227, + "interChunkDelaysMs": [ + 2, + 234, + 6, + 5, + 5, + 33, + 3, + 11, + 10, + 3, + 10, + 12, + 38, + 86, + 13, + 2, + 7, + 172 + ], + "totalDurationMs": 1879 + }, + "metadata": { + "systemHash": "e4af14ab", + "toolsHash": "88b34f59" + } + } + ] +} \ No newline at end of file diff --git a/e2e/fixtures/computer-take-control/openai-2026-11-12T21-34-41-231Z-4d6b05c7.json b/e2e/fixtures/computer-take-control/openai-2026-11-12T21-34-41-231Z-4d6b05c7.json new file mode 100644 index 0000000000..df925adaef --- /dev/null +++ b/e2e/fixtures/computer-take-control/openai-2026-11-12T21-34-41-231Z-4d6b05c7.json @@ -0,0 +1,31 @@ +{ + "fixtures": [ + { + "match": { + "userMessage": "You choose which capabilities to load for the message below. You are not answering it.\n\nCapabilities available:\n- research-public-web: Research the public web. Search current public sources and write a cited brief.\n- find-a-document: Find a document. Search the connected document sources for a file and read what it says.\n- find-a-notion-page: Find a Notion page. Search Notion for a page and read what it says.\n- skill-creator: Write a skill. Interview me about a skill I want, then write it and save it as a new skill.\n- bot-creator: Make a Bot. Work out what a new Bot should be, then put it in front of you to create.\n\nMessage:\nOpen http://host.docker.internal:46600/about and tell me who founded the shop, in one short sentence.\n\nReply with only JSON: {\"skills\": [\"\", ...]}.\nChoose every capability that might be needed, including ones you are only somewhat sure about.\nChoosing one that turns out to be unnecessary costs almost nothing. Failing to choose one that\nwas needed means the work cannot be done at all, because its tools will not be loaded. When in\ndoubt, include it. Use an empty list only when the message plainly needs none of them.", + "model": "gpt-5.6-terra", + "turnIndex": 0, + "hasToolResult": false + }, + "response": { + "content": "{\"skills\":[\"research-public-web\"]}", + "usage": { + "prompt_tokens": 269, + "completion_tokens": 57, + "total_tokens": 326, + "prompt_tokens_details": { + "cached_tokens": 0, + "cache_write_tokens": 0, + "audio_tokens": 0 + }, + "completion_tokens_details": { + "reasoning_tokens": 37, + "audio_tokens": 0, + "accepted_prediction_tokens": 0, + "rejected_prediction_tokens": 0 + } + } + } + } + ] +} \ No newline at end of file diff --git a/e2e/fixtures/computer-take-control/openai-2026-11-12T21-34-43-864Z-f9dc6cee.json b/e2e/fixtures/computer-take-control/openai-2026-11-12T21-34-43-864Z-f9dc6cee.json new file mode 100644 index 0000000000..3432b77f3f --- /dev/null +++ b/e2e/fixtures/computer-take-control/openai-2026-11-12T21-34-43-864Z-f9dc6cee.json @@ -0,0 +1,34 @@ +{ + "fixtures": [ + { + "match": { + "userMessage": "Asked: Open http://host.docker.internal:46600/about and tell me who founded the shop, in one short sentence.", + "model": "gpt-5.6-terra", + "turnIndex": 0, + "hasToolResult": false + }, + "response": { + "content": "Shop Founder Information", + "usage": { + "prompt_tokens": 126, + "completion_tokens": 22, + "total_tokens": 148, + "prompt_tokens_details": { + "cached_tokens": 0, + "cache_write_tokens": 0, + "audio_tokens": 0 + }, + "completion_tokens_details": { + "reasoning_tokens": 10, + "audio_tokens": 0, + "accepted_prediction_tokens": 0, + "rejected_prediction_tokens": 0 + } + } + }, + "metadata": { + "systemHash": "dedd9ec9" + } + } + ] +} \ No newline at end of file diff --git a/e2e/fixtures/computer-take-control/openai-2026-11-12T21-34-44-234Z-32cc65f6.json b/e2e/fixtures/computer-take-control/openai-2026-11-12T21-34-44-234Z-32cc65f6.json new file mode 100644 index 0000000000..1bd52e359f --- /dev/null +++ b/e2e/fixtures/computer-take-control/openai-2026-11-12T21-34-44-234Z-32cc65f6.json @@ -0,0 +1,71 @@ +{ + "fixtures": [ + { + "match": { + "userMessage": "Open http://host.docker.internal:46600/about and tell me who founded the shop, in one short sentence.", + "model": "gpt-5.6-terra", + "turnIndex": 0, + "hasToolResult": false + }, + "response": { + "content": "I’m opening the shop’s About page to check its founder.", + "toolCalls": [ + { + "name": "computer_navigate", + "arguments": "{\"url\":\"http://host.docker.internal:46600/about\"}", + "id": "call_A73lL8x33JCVNmQFSXYaX5Ei" + } + ] + }, + "recordedTimings": { + "ttftMs": 1151, + "interChunkDelaysMs": [ + 3, + 488, + 391, + 6, + 18, + 6, + 11, + 5, + 12, + 10, + 22, + 15, + 15, + 9, + 12, + 16, + 2, + 2, + 8, + 5, + 9, + 6, + 5, + 8, + 19, + 45, + 25, + 357, + 14, + 11, + 21, + 22, + 27, + 5, + 4, + 7, + 2, + 5, + 41 + ], + "totalDurationMs": 2840 + }, + "metadata": { + "systemHash": "2222390f", + "toolsHash": "259d7636" + } + } + ] +} \ No newline at end of file diff --git a/e2e/fixtures/computer-take-control/openai-2026-11-12T21-34-48-708Z-66fed028.json b/e2e/fixtures/computer-take-control/openai-2026-11-12T21-34-48-708Z-66fed028.json new file mode 100644 index 0000000000..26064650c4 --- /dev/null +++ b/e2e/fixtures/computer-take-control/openai-2026-11-12T21-34-48-708Z-66fed028.json @@ -0,0 +1,45 @@ +{ + "fixtures": [ + { + "match": { + "userMessage": "Open http://host.docker.internal:46600/about and tell me who founded the shop, in one short sentence.", + "model": "gpt-5.6-terra", + "turnIndex": 2, + "hasToolResult": true + }, + "response": { + "content": "Fernhill Garden Supply was founded by Ines Calloway." + }, + "recordedTimings": { + "ttftMs": 725, + "interChunkDelaysMs": [ + 4, + 1798, + 2, + 3, + 32, + 0, + 5, + 1, + 6, + 2, + 0, + 0, + 24, + 9, + 4, + 11, + 122, + 5, + 0, + 234 + ], + "totalDurationMs": 2987 + }, + "metadata": { + "systemHash": "2222390f", + "toolsHash": "259d7636" + } + } + ] +} \ No newline at end of file diff --git a/e2e/fixtures/computer-terminal/openai-2026-11-12T21-34-53-287Z-906c657d.json b/e2e/fixtures/computer-terminal/openai-2026-11-12T21-34-53-287Z-906c657d.json new file mode 100644 index 0000000000..7f6d664673 --- /dev/null +++ b/e2e/fixtures/computer-terminal/openai-2026-11-12T21-34-53-287Z-906c657d.json @@ -0,0 +1,31 @@ +{ + "fixtures": [ + { + "match": { + "userMessage": "You choose which capabilities to load for the message below. You are not answering it.\n\nCapabilities available:\n- research-public-web: Research the public web. Search current public sources and write a cited brief.\n- find-a-document: Find a document. Search the connected document sources for a file and read what it says.\n- find-a-notion-page: Find a Notion page. Search Notion for a page and read what it says.\n- skill-creator: Write a skill. Interview me about a skill I want, then write it and save it as a new skill.\n- bot-creator: Make a Bot. Work out what a new Bot should be, then put it in front of you to create.\n\nMessage:\nIn your terminal, run the command echo fernhill-$((40+2)) and tell me what it printed, in one plain sentence with no formatting.\n\nReply with only JSON: {\"skills\": [\"\", ...]}.\nChoose every capability that might be needed, including ones you are only somewhat sure about.\nChoosing one that turns out to be unnecessary costs almost nothing. Failing to choose one that\nwas needed means the work cannot be done at all, because its tools will not be loaded. When in\ndoubt, include it. Use an empty list only when the message plainly needs none of them.", + "model": "gpt-5.6-terra", + "turnIndex": 0, + "hasToolResult": false + }, + "response": { + "content": "{\"skills\":[]}", + "usage": { + "prompt_tokens": 276, + "completion_tokens": 34, + "total_tokens": 310, + "prompt_tokens_details": { + "cached_tokens": 0, + "cache_write_tokens": 0, + "audio_tokens": 0 + }, + "completion_tokens_details": { + "reasoning_tokens": 18, + "audio_tokens": 0, + "accepted_prediction_tokens": 0, + "rejected_prediction_tokens": 0 + } + } + } + } + ] +} \ No newline at end of file diff --git a/e2e/fixtures/computer-terminal/openai-2026-11-12T21-34-55-238Z-7c172fc6.json b/e2e/fixtures/computer-terminal/openai-2026-11-12T21-34-55-238Z-7c172fc6.json new file mode 100644 index 0000000000..f8a8a881aa --- /dev/null +++ b/e2e/fixtures/computer-terminal/openai-2026-11-12T21-34-55-238Z-7c172fc6.json @@ -0,0 +1,51 @@ +{ + "fixtures": [ + { + "match": { + "userMessage": "In your terminal, run the command echo fernhill-$((40+2)) and tell me what it printed, in one plain sentence with no formatting.", + "model": "gpt-5.6-terra", + "turnIndex": 0, + "hasToolResult": false + }, + "response": { + "toolCalls": [ + { + "name": "computer_run_command", + "arguments": "{\"command\":\"echo fernhill-$((40+2))\"}", + "id": "call_tqHgHodmWv3LRRNsCEkmON6E" + } + ] + }, + "recordedTimings": { + "ttftMs": 953, + "interChunkDelaysMs": [ + 11, + 400, + 148, + 9, + 2, + 2, + 1, + 14, + 3, + 7, + 11, + 4, + 12, + 9, + 14, + 5, + 25, + 134, + 10, + 142 + ], + "totalDurationMs": 1916 + }, + "metadata": { + "systemHash": "8712e5f3", + "toolsHash": "fb6fd309" + } + } + ] +} \ No newline at end of file diff --git a/e2e/fixtures/computer-terminal/openai-2026-11-12T21-34-55-299Z-65196775.json b/e2e/fixtures/computer-terminal/openai-2026-11-12T21-34-55-299Z-65196775.json new file mode 100644 index 0000000000..35d7800d01 --- /dev/null +++ b/e2e/fixtures/computer-terminal/openai-2026-11-12T21-34-55-299Z-65196775.json @@ -0,0 +1,34 @@ +{ + "fixtures": [ + { + "match": { + "userMessage": "Asked: In your terminal, run the command echo fernhill-$((40+2)) and tell me what it printed, in one plain sentence with no formatting.", + "model": "gpt-5.6-terra", + "turnIndex": 0, + "hasToolResult": false + }, + "response": { + "content": "Terminal arithmetic command output", + "usage": { + "prompt_tokens": 133, + "completion_tokens": 36, + "total_tokens": 169, + "prompt_tokens_details": { + "cached_tokens": 0, + "cache_write_tokens": 0, + "audio_tokens": 0 + }, + "completion_tokens_details": { + "reasoning_tokens": 23, + "audio_tokens": 0, + "accepted_prediction_tokens": 0, + "rejected_prediction_tokens": 0 + } + } + }, + "metadata": { + "systemHash": "dedd9ec9" + } + } + ] +} \ No newline at end of file diff --git a/e2e/fixtures/computer-terminal/openai-2026-11-12T21-34-57-895Z-c86e9f58.json b/e2e/fixtures/computer-terminal/openai-2026-11-12T21-34-57-895Z-c86e9f58.json new file mode 100644 index 0000000000..6191832c7f --- /dev/null +++ b/e2e/fixtures/computer-terminal/openai-2026-11-12T21-34-57-895Z-c86e9f58.json @@ -0,0 +1,39 @@ +{ + "fixtures": [ + { + "match": { + "userMessage": "In your terminal, run the command echo fernhill-$((40+2)) and tell me what it printed, in one plain sentence with no formatting.", + "model": "gpt-5.6-terra", + "turnIndex": 1, + "hasToolResult": true + }, + "response": { + "content": "It printed fernhill-42." + }, + "recordedTimings": { + "ttftMs": 1208, + "interChunkDelaysMs": [ + 4, + 330, + 5, + 5, + 3, + 2, + 0, + 0, + 0, + 16, + 158, + 3, + 0, + 105 + ], + "totalDurationMs": 1839 + }, + "metadata": { + "systemHash": "8712e5f3", + "toolsHash": "fb6fd309" + } + } + ] +} \ No newline at end of file diff --git a/e2e/harness/config.ts b/e2e/harness/config.ts index 43a34e8deb..ad05713c5b 100644 --- a/e2e/harness/config.ts +++ b/e2e/harness/config.ts @@ -21,19 +21,40 @@ export const LOGS = join(ARTIFACTS, "logs"); /** Recorded fixtures, one directory per journey. */ export const FIXTURES = join(E2E, "fixtures"); -export const COMPOSE_PROJECT = "openbot-e2e"; +/** + * A second run on the same machine, from another checkout, gets a stack of its own with + * `E2E_PORT_OFFSET=1000` (or 2000, …): every port below moves up by it, and the compose project is + * `openbot-e2e-1000`, so neither run sees, reuses or stops the other's stack. A multiple of 1000, + * so one run's ports never land on another's. Unset, the run is on 46xxx in `openbot-e2e`. + */ +const PORT_OFFSET = portOffset(process.env.E2E_PORT_OFFSET); + +function portOffset(value: string | undefined): number { + if (!value) return 0; + const offset = Number(value); + if (!Number.isInteger(offset) || offset < 0 || offset % 1000 !== 0) { + throw new Error( + `E2E_PORT_OFFSET must be a multiple of 1000, such as 1000 or 2000; it is "${value}".`, + ); + } + return offset; +} + +export const COMPOSE_PROJECT = PORT_OFFSET + ? `openbot-e2e-${PORT_OFFSET}` + : "openbot-e2e"; export const PORTS = { - server: 46001, - app: 46010, - computer: 46100, - browser: 46101, - bot: 46200, - langgraph: 46201, - aimock: 46400, - composio: 46410, - postgres: 46432, - supervisor: 46500, + server: 46001 + PORT_OFFSET, + app: 46010 + PORT_OFFSET, + computer: 46100 + PORT_OFFSET, + browser: 46101 + PORT_OFFSET, + bot: 46200 + PORT_OFFSET, + langgraph: 46201 + PORT_OFFSET, + aimock: 46400 + PORT_OFFSET, + composio: 46410 + PORT_OFFSET, + postgres: 46432 + PORT_OFFSET, + supervisor: 46500 + PORT_OFFSET, } as const; export const APP_URL = `http://127.0.0.1:${PORTS.app}`; diff --git a/e2e/journeys/approval-steps.ts b/e2e/journeys/approval-steps.ts new file mode 100644 index 0000000000..7139d031ce --- /dev/null +++ b/e2e/journeys/approval-steps.ts @@ -0,0 +1,76 @@ +import type { Locator, Page } from "@playwright/test"; +import { expect } from "../harness/test"; +import { runsSoFar, transcript } from "./computer-steps"; + +/** + * What the approval journeys share: the card a Bot's payment waits on, answering it, and the banner + * that lists what a conversation allows. + * + * Paying is one of the actions that always asks (server/src/approvals/policy.ts), and a checkout's + * Place order button is how a Bot pays on a website. The shop is the run's own (`e2e/site`), and it + * counts the orders actually placed, so an answer is checked by what happened as well as by what + * the Bot said. After an answer the server carries the action out and continues the conversation + * itself; the reply it streams is checked word for word on screen, once, because a reply after an + * approval once arrived garbled. + */ + +/** Why a payment asks, as its card says. */ +export const PAYS = "Paying, buying or moving money asks you first."; + +/** Every answer a payment's card offers, in order. */ +export const CHOICES = [ + "Allow once", + "Allow for this conversation", + "Always allow here", + "Deny", +] as const; +/** The answers a step handed to the person offers: they do it, or nobody does. */ +export const HANDED_OVER = ["I did it myself", "Don't do it"] as const; +export type Choice = (typeof CHOICES)[number] | (typeof HANDED_OVER)[number]; + +/** The approval card in the conversation, while it waits on the person. */ +export function approvalCard(page: Page): Locator { + return transcript(page).getByRole("region", { + name: "Waiting for your approval", + }); +} + +/** The card for a payment on Place order, waiting, with every answer on it. */ +export async function expectAsksToPay(page: Page) { + const card = approvalCard(page); + await expect(card).toBeVisible({ timeout: 120_000 }); + await expect(card).toContainText(PAYS); + await expect(card).toContainText("Place order"); + await expect(card.getByText("Waiting on you")).toBeVisible(); + for (const choice of CHOICES) { + await expect(card.getByRole("radio", { name: choice })).toBeVisible(); + } +} + +/** + * Pick an answer on the card and send it with the button, which is named for the answer picked. + * Returns the mark (`runsSoFar`) the reply that follows is waited for from. + */ +export async function answer( + page: Page, + channelId: string, + choice: Choice, +): Promise { + const card = approvalCard(page); + const mark = await runsSoFar(page, channelId); + // The radio is visually hidden inside its row; a person picks the row. + const radio = card.getByRole("radio", { name: choice }); + await card + .locator("label") + .filter({ has: page.getByRole("radio", { name: choice }) }) + .click({ timeout: 30_000 }); + await expect(radio).toBeChecked(); + await card.getByRole("button", { name: choice }).click({ timeout: 30_000 }); + await expect(approvalCard(page)).toHaveCount(0, { timeout: 60_000 }); + return mark; +} + +/** The banner above the composer that lists what this conversation allows. */ +export function allowances(page: Page): Locator { + return page.getByRole("region", { name: "Allowed in this conversation" }); +} diff --git a/e2e/journeys/approvals-allow-once.journey.ts b/e2e/journeys/approvals-allow-once.journey.ts new file mode 100644 index 0000000000..500d93eb1a --- /dev/null +++ b/e2e/journeys/approvals-allow-once.journey.ts @@ -0,0 +1,105 @@ +import { ITEMS, SITE, SITE_HOST } from "../site/site"; +import { answer, approvalCard, expectAsksToPay, PAYS } from "./approval-steps"; +import { + expect, + expectWholeReply, + named, + newConversation, + replyAfter, + runsSoFar, + send, + serveSite, + test, + turnOver, +} from "./computer-steps"; + +/** + * APPROVALS, ALLOW ONCE AND DENY. A Bot about to pay asks first; Allow once places that order and + * no other, so the next order asks again. That one also waits on the Bots page under Needs you, + * the same request as the card; Deny there places nothing, and the Bot says so. + */ + +const site = serveSite(); + +test("Allow once places one order and the next one asks again; Deny buys nothing", async ({ + page, + shot, + makeBot, +}) => { + test.setTimeout(400_000); + const botId = await makeBot( + "Fernhill Buyer", + "Buy garden supplies for me on the Fernhill Garden Supply website. Keep replies short.", + ); + const before = site().orders.length; + const ordered = () => site().orders.slice(before); + let channelId = ""; + + await test.step("the Bot asks before it pays", async () => { + await newConversation(page, botId); + channelId = await send( + page, + `Open ${SITE}/shop/tomato and place the order for the tomato seeds. Then tell me the order number in one short sentence.`, + ); + await expectAsksToPay(page); + expect(ordered(), "nothing is bought before the person answers").toEqual( + [], + ); + await shot("asks-before-paying"); + }); + + await test.step("Allow once: the order is placed, and the reply arrives whole", async () => { + const mark = await answer(page, channelId, "Allow once"); + const reply = await replyAfter(page, channelId, mark, /FH-2041/); + await turnOver(page); + await expectWholeReply(page, reply); + await named(page, channelId); + expect(ordered()).toEqual(["tomato"]); + await shot("allowed-once"); + }); + + await test.step("the next order asks again, on the Bots page too; Deny it there", async () => { + await send( + page, + `Now order the basil seeds too, from ${SITE}/shop/basil, and tell me the order number.`, + ); + await expectAsksToPay(page); + await shot("asks-again"); + const mark = await runsSoFar(page, channelId); + + // The same request waits on the Bots page, under Needs you, and can be answered there. + await page.getByRole("link", { name: /^Bots/ }).click(); + const board = page.getByRole("main"); + await expect( + board.getByRole("heading", { name: "Waiting on you" }), + ).toBeVisible(); + await expect( + board.getByRole("button", { name: /^Click the button “Place order”/ }), + ).toContainText(SITE_HOST); + await expect(board.getByText(PAYS)).toBeVisible(); + // Allow and Deny on the row; the standing answers in its menu. + const more = board.getByRole("button", { + name: "More for click the button “Place order”", + }); + await more.click(); + for (const choice of ["Allow for this conversation", "Always allow here"]) { + await expect(page.getByRole("menuitem", { name: choice })).toBeVisible(); + } + await shot("waiting-on-the-bots-page"); + await page.keyboard.press("Escape"); + await expect(page.getByRole("menu")).toHaveCount(0); + await board.getByRole("button", { name: "Deny", exact: true }).click(); + await expect( + board.getByRole("heading", { name: "Waiting on you" }), + ).toHaveCount(0); + + // Answered there, the conversation's card is gone and the Bot has said what came of it. + await page.goto(`/channel/${channelId}`); + const reply = await replyAfter(page, channelId, mark, /declin/i); + await expect(approvalCard(page)).toHaveCount(0); + await expectWholeReply(page, reply); + expect(reply).not.toContain(ITEMS.basil.order); + expect(ordered(), "a denied order is not placed").toEqual(["tomato"]); + await shot("denied"); + }); +}); diff --git a/e2e/journeys/approvals-always.journey.ts b/e2e/journeys/approvals-always.journey.ts new file mode 100644 index 0000000000..cc84d11579 --- /dev/null +++ b/e2e/journeys/approvals-always.journey.ts @@ -0,0 +1,91 @@ +import { SITE, SITE_HOST } from "../site/site"; +import { + allowances, + answer, + approvalCard, + expectAsksToPay, +} from "./approval-steps"; +import { + expect, + expectWholeReply, + named, + newConversation, + replyAfter, + send, + serveSite, + settingsRow, + test, + turnOver, +} from "./computer-steps"; + +/** + * APPROVALS, ALWAYS ALLOW HERE. The first order asks; always allowed, the Bot pays without asking + * in its next conversation too, because the answer is a rule. The rule is on the Bot's Approvals + * page, where the person can remove it. + */ + +const site = serveSite(); + +test("Always allow here lets the Bot pay in its next conversation, until the rule is removed", async ({ + page, + shot, + makeBot, +}) => { + test.setTimeout(500_000); + const botId = await makeBot( + "Fernhill Regular", + "Keep my garden stocked by ordering from the Fernhill Garden Supply website. Keep replies short.", + ); + const before = site().orders.length; + const ordered = () => site().orders.slice(before); + + await test.step("the first order asks; always allow it here", async () => { + await newConversation(page, botId); + const channelId = await send( + page, + `Order the marigold seeds from ${SITE}/shop/marigold for me, then reply with just the order number.`, + ); + await expectAsksToPay(page); + const mark = await answer(page, channelId, "Always allow here"); + const reply = await replyAfter(page, channelId, mark, /FH-2043/); + await turnOver(page); + await expectWholeReply(page, reply); + await named(page, channelId); + expect(ordered()).toEqual(["marigold"]); + // Always is a rule, not this conversation's answer: there is no banner for it here. + await expect(allowances(page)).toHaveCount(0); + await shot("always-allowed"); + }); + + await test.step("in a new conversation, the Bot pays without asking", async () => { + await newConversation(page, botId); + const channelId = await send( + page, + `Order the tomato seeds from ${SITE}/shop/tomato for me, then reply with just the order number.`, + ); + const reply = await replyAfter(page, channelId, 0, /FH-2041/); + await turnOver(page); + await expectWholeReply(page, reply); + await named(page, channelId); + await expect(approvalCard(page)).toHaveCount(0); + expect(ordered()).toEqual(["marigold", "tomato"]); + await shot("paid-without-asking"); + }); + + await test.step("the rule is on the Bot's Approvals page, and Remove takes it away", async () => { + await page.goto(`/bots/${botId}`); + await page.getByRole("link", { name: /^Approvals/ }).click(); + await expect( + page.getByRole("heading", { name: "Approvals", level: 1 }), + ).toBeVisible(); + const rule = settingsRow(page, SITE_HOST); + await expect(rule).toHaveCount(1); + await expect( + rule.getByRole("combobox", { name: /^Behaviour for / }), + ).toContainText("Take action without asking"); + await shot("bot-approvals-rule"); + await rule.getByRole("button", { name: "Remove" }).click(); + await expect(page.getByText("No rules for this Bot")).toBeVisible(); + await shot("rule-removed"); + }); +}); diff --git a/e2e/journeys/approvals-settings.journey.ts b/e2e/journeys/approvals-settings.journey.ts new file mode 100644 index 0000000000..964ea31c6a --- /dev/null +++ b/e2e/journeys/approvals-settings.journey.ts @@ -0,0 +1,140 @@ +import { expect, test } from "../harness/test"; +import { settingsRow } from "./computer-steps"; + +/** + * APPROVALS, THE SETTINGS. Settings → Approvals: Extra vigilant, Auto-review and how commands on + * this computer are handled, each kept across a reload; and rules for every Bot, added, changed and + * removed. No Bot runs here. + */ + +// The switches are the person's, for every Bot. Put back after the journey, even one that failed +// half way, so no journey after it meets an Extra vigilant it did not ask for. +test.afterEach(async ({ page }) => { + await page.request.patch("/api/approvals/preferences", { + data: { enabled: false, autoReview: false, hostCommands: "ask" }, + }); +}); + +test("the Approvals settings: Extra vigilant, Auto-review, commands, and rules for every Bot", async ({ + page, + shot, +}) => { + await test.step("open Settings → Approvals", async () => { + await page.goto("/"); + await page.getByRole("button", { name: "Sam Rivera" }).click(); + await page.getByRole("menuitem", { name: "Settings" }).click(); + await page.getByRole("link", { name: "Approvals" }).click(); + await expect( + page.getByRole("heading", { name: "Approvals", level: 1 }), + ).toBeVisible(); + await expect(page.getByText("No rules yet")).toBeVisible(); + await shot("approvals-settings"); + }); + + const vigilant = page.getByRole("switch", { name: "Extra vigilant" }); + const review = page.getByRole("switch", { name: "Auto-review" }); + const commands = page.getByRole("combobox", { + name: "Commands on your computer", + }); + + await test.step("switch Extra vigilant and Auto-review on, and they stay on", async () => { + await expect(vigilant).not.toBeChecked(); + await expect(review).not.toBeChecked(); + await vigilant.click(); + await expect(vigilant).toBeChecked(); + await review.click(); + await expect(review).toBeChecked(); + await page.reload(); + await expect(vigilant).toBeChecked(); + await expect(review).toBeChecked(); + await shot("switched-on"); + }); + + await test.step("choose how commands on this computer are handled", async () => { + await expect(commands).toContainText("Ask every time"); + await commands.click(); + await page.getByRole("option", { name: "Never" }).click(); + await expect(commands).toContainText("Never"); + await page.reload(); + await expect(commands).toContainText("Never"); + await shot("commands-never"); + }); + + await test.step("put them back", async () => { + await vigilant.click(); + await expect(vigilant).not.toBeChecked(); + await review.click(); + await expect(review).not.toBeChecked(); + await commands.click(); + await page.getByRole("option", { name: "Ask every time" }).click(); + await expect(commands).toContainText("Ask every time"); + await page.reload(); + await expect(vigilant).not.toBeChecked(); + await expect(review).not.toBeChecked(); + await expect(commands).toContainText("Ask every time"); + }); + + await test.step("add a rule for every Bot, change it, and remove it", async () => { + await page.getByRole("button", { name: /^Add a rule/ }).click(); + const dialog = page.getByRole("dialog", { name: "Add a rule" }); + await expect(dialog).toBeVisible(); + const save = dialog.getByRole("button", { name: "Save rule" }); + await expect(save).toBeDisabled(); + await dialog + .getByRole("textbox", { name: "Tool or app" }) + .fill("computer_click"); + await dialog.getByRole("textbox", { name: "Kind of action" }).fill("*"); + await dialog + .getByRole("textbox", { name: "Target" }) + .fill("*.fernhill.example"); + await dialog.getByRole("combobox", { name: "Behaviour" }).click(); + await page.getByRole("option", { name: "Hand off to you" }).click(); + await shot("add-rule"); + await save.click(); + await expect(dialog).toBeHidden(); + + const rule = settingsRow(page, "*.fernhill.example"); + await expect(rule).toHaveCount(1); + const behaviour = rule.getByRole("combobox", { name: /^Behaviour for / }); + await expect(behaviour).toContainText("Hand off to you"); + await expect(page.getByText("No rules yet")).toHaveCount(0); + await shot("rule-added"); + + await behaviour.click(); + await page + .getByRole("option", { name: "Ask before taking action" }) + .click(); + await expect(behaviour).toContainText("Ask before taking action"); + await page.reload(); + await expect( + settingsRow(page, "*.fernhill.example").getByRole("combobox", { + name: /^Behaviour for /, + }), + ).toContainText("Ask before taking action"); + + await settingsRow(page, "*.fernhill.example") + .getByRole("button", { name: "Remove" }) + .click(); + await expect(page.getByText("No rules yet")).toBeVisible(); + await shot("rule-removed"); + }); + + await test.step("Cancel leaves a rule unsaved", async () => { + await page.getByRole("button", { name: /^Add a rule/ }).click(); + const dialog = page.getByRole("dialog", { name: "Add a rule" }); + await dialog + .getByRole("textbox", { name: "Tool or app" }) + .fill("computer_type"); + await dialog.getByRole("button", { name: "Cancel" }).click(); + await expect(dialog).toBeHidden(); + await expect(page.getByText("No rules yet")).toBeVisible(); + }); + + await test.step("the old Approvals address opens these settings", async () => { + await page.goto("/approvals"); + await expect(page).toHaveURL(/\/settings\/approvals$/); + await expect( + page.getByRole("heading", { name: "Approvals", level: 1 }), + ).toBeVisible(); + }); +}); diff --git a/e2e/journeys/approvals-this-conversation.journey.ts b/e2e/journeys/approvals-this-conversation.journey.ts new file mode 100644 index 0000000000..76ffba3de5 --- /dev/null +++ b/e2e/journeys/approvals-this-conversation.journey.ts @@ -0,0 +1,105 @@ +import { SITE, SITE_HOST } from "../site/site"; +import { + allowances, + answer, + approvalCard, + expectAsksToPay, +} from "./approval-steps"; +import { + expect, + expectWholeReply, + named, + newConversation, + replyAfter, + runsSoFar, + send, + serveSite, + test, + turnOver, +} from "./computer-steps"; + +/** + * APPROVALS, ALLOW FOR THIS CONVERSATION. The first order asks; allowed for the conversation, the + * next order there goes through with no card, and the banner above the composer says what is + * allowed. Turned off there, the next order asks again. + */ + +const site = serveSite(); + +test("Allow for this conversation lets the next order through, until it is turned off", async ({ + page, + shot, + makeBot, +}) => { + test.setTimeout(500_000); + const botId = await makeBot( + "Fernhill Shopper", + "Order seeds for me from the Fernhill Garden Supply website. Keep replies short.", + ); + const before = site().orders.length; + const ordered = () => site().orders.slice(before); + let channelId = ""; + + await test.step("the first order asks; allow it for this conversation", async () => { + await newConversation(page, botId); + channelId = await send( + page, + `Please buy the sunflower seeds at ${SITE}/shop/sunflower and give me the order number in one sentence.`, + ); + await expectAsksToPay(page); + await shot("asks"); + const mark = await answer(page, channelId, "Allow for this conversation"); + const reply = await replyAfter(page, channelId, mark, /FH-2044/); + await turnOver(page); + await expectWholeReply(page, reply); + await named(page, channelId); + expect(ordered()).toEqual(["sunflower"]); + }); + + const allowed = allowances(page); + const turnOff = allowed.getByRole("button", { name: /^Turn off: / }); + + await test.step("the banner says what this conversation allows", async () => { + await expect(allowed).toBeVisible(); + await expect(allowed).toContainText( + `Paying on ${SITE_HOST} is allowed in this conversation.`, + ); + await expect(turnOff).toHaveCount(1); + await shot("allowed-for-conversation"); + }); + + await test.step("the second order goes through with no card", async () => { + const mark = await runsSoFar(page, channelId); + await send( + page, + `Great. Please also buy the lavender seeds at ${SITE}/shop/lavender and give me that order number too.`, + ); + const reply = await replyAfter(page, channelId, mark, /FH-2045/); + await turnOver(page); + await expectWholeReply(page, reply); + await expect(approvalCard(page)).toHaveCount(0); + expect(ordered()).toEqual(["sunflower", "lavender"]); + await shot("second-order-no-card"); + }); + + await test.step("Turn off, and the next order asks again", async () => { + await turnOff.click(); + await expect(allowed).toHaveCount(0); + await shot("turned-off"); + await send( + page, + `One more, please: the chamomile seeds at ${SITE}/shop/chamomile, with the order number.`, + ); + await expectAsksToPay(page); + expect(ordered()).toEqual(["sunflower", "lavender"]); + await shot("asks-after-turn-off"); + const mark = await answer(page, channelId, "Allow once"); + const reply = await replyAfter(page, channelId, mark, /FH-2046/); + await turnOver(page); + await expectWholeReply(page, reply); + expect(ordered()).toEqual(["sunflower", "lavender", "chamomile"]); + // Allowed once, not for the conversation: the banner does not come back. + await expect(allowed).toHaveCount(0); + await shot("allowed-once-after-turn-off"); + }); +}); diff --git a/e2e/journeys/computer-browse.journey.ts b/e2e/journeys/computer-browse.journey.ts new file mode 100644 index 0000000000..4d6e7ddd69 --- /dev/null +++ b/e2e/journeys/computer-browse.journey.ts @@ -0,0 +1,103 @@ +import { SITE, SITE_HOST } from "../site/site"; +import { + activityRows, + CLOCK, + computerPanel, + expect, + expectWholeReply, + named, + newConversation, + openComputer, + replyAfter, + send, + serveSite, + test, + transcript, + turnOver, +} from "./computer-steps"; + +/** + * THE BOT'S COMPUTER, BROWSING. A Bot opens a page, reads it and answers from it. The Computer opens + * beside the conversation on the Bot's screen as it browses, Activity lists the page it opened with + * the time, the conversation folds the browsing into one line that opens to its steps, and a reload + * keeps Activity. + */ + +serveSite(); + +test("a Bot opens a page, reads it and answers, live on its computer", async ({ + page, + shot, + makeBot, +}) => { + test.setTimeout(300_000); + const botId = await makeBot( + "Fernhill Scout", + "Look things up on garden shop websites for me and answer briefly.", + ); + let channelId = ""; + + await test.step("ask the Bot to read a page", async () => { + await newConversation(page, botId); + channelId = await send( + page, + `Open ${SITE}/ and tell me what time the greenhouse opens on Saturdays. Answer in one short sentence.`, + ); + await shot("asked"); + }); + + await test.step("the Computer opens on the Bot's screen as it browses", async () => { + const panel = await openComputer(page); + await expect(panel.getByRole("tab", { name: "Screen" })).toHaveAttribute( + "aria-selected", + "true", + ); + await expect( + panel.getByRole("img", { name: "What the Bot is looking at" }), + ).toBeVisible({ timeout: 120_000 }); + // The card reads as a browser window: where it is, as an address bar says it. + await expect(panel.getByText(SITE_HOST, { exact: true })).toBeVisible(); + await expect(panel.getByText("Fernhill Scout has control")).toBeVisible(); + await shot("computer-live"); + }); + + await test.step("the answer comes from the page", async () => { + await turnOver(page); + const reply = await replyAfter(page, channelId, 0, /7:40/); + await expectWholeReply(page, reply); + await named(page, channelId); + await shot("answer"); + }); + + await test.step("Activity lists the page it opened, with the time", async () => { + const opened = activityRows(page).filter({ hasText: "Opened page" }); + await expect(opened).toHaveCount(1); + await expect(opened).toContainText(`${SITE}/`); + await expect(opened).toContainText("Fernhill Garden Supply"); + await expect(opened.locator("time")).toHaveText(CLOCK); + await expect(computerPanel(page).getByText(/^Nothing yet/)).toHaveCount(0); + await shot("activity"); + }); + + await test.step("the conversation folds the browsing into one line that opens to its steps", async () => { + const fold = transcript(page).getByText("Browsed host.docker.internal", { + exact: true, + }); + await expect(fold).toBeVisible(); + await fold.click(); + await expect( + transcript(page).getByRole("link", { name: "Fernhill Garden Supply" }), + ).toHaveAttribute("href", `${SITE}/`); + await shot("browsing-unfolded"); + }); + + await test.step("after a reload, Activity still lists what was done", async () => { + await page.reload(); + const panel = await openComputer(page); + const opened = activityRows(page).filter({ hasText: "Opened page" }); + await expect(opened).toHaveCount(1); + await expect(opened).toContainText(`${SITE}/`); + await expect(panel.getByText(/^Nothing yet/)).toHaveCount(0); + await shot("activity-after-reload"); + }); +}); diff --git a/e2e/journeys/computer-form.journey.ts b/e2e/journeys/computer-form.journey.ts new file mode 100644 index 0000000000..7d2e7be3cb --- /dev/null +++ b/e2e/journeys/computer-form.journey.ts @@ -0,0 +1,90 @@ +import { SITE } from "../site/site"; +import { + activityRows, + CLOCK, + expect, + expectWholeReply, + named, + newConversation, + openComputer, + replyAfter, + send, + serveSite, + test, + turnOver, +} from "./computer-steps"; + +/** + * THE BOT'S COMPUTER, MANY STEPS IN ONE GO. A Bot fills in a form and sends it from one request: + * every field and click runs, the form arrives filled in, and each step is listed in Activity with + * its time, with none stopped short. + */ + +const site = serveSite(); + +test("a Bot fills in a form in one go, and every step shows in Activity", async ({ + page, + shot, + makeBot, +}) => { + test.setTimeout(400_000); + const botId = await makeBot( + "Fernhill Helper", + "Fill in forms on garden shop websites for me. Keep replies short.", + ); + const before = site().applications.length; + let channelId = ""; + + await test.step("ask for a form to be filled in and sent", async () => { + await newConversation(page, botId); + channelId = await send( + page, + `Go to ${SITE}/volunteer and fill in the volunteer form for me in one go: full name Robin Okafor, email robin.okafor@example.org, preferred shift Saturday morning, and tick the code of conduct box. Fill in all three fields and tick the box together, in a single step from one snapshot. Then submit it and tell me the reference number in one plain sentence.`, + ); + await openComputer(page); + await shot("asked"); + }); + + await test.step("every action runs, and the form arrives filled in", async () => { + await turnOver(page); + const reply = await replyAfter(page, channelId, 0, /VOL-3172/); + await expectWholeReply(page, reply); + await named(page, channelId); + expect(site().applications.slice(before)).toEqual([ + { + fullName: "Robin Okafor", + email: "robin.okafor@example.org", + shift: "Saturday morning", + agreed: true, + }, + ]); + await shot("sent"); + }); + + await test.step("Activity lists every step, each with its time", async () => { + const rows = activityRows(page); + for (const field of ["Full name", "Email", "Preferred shift"]) { + await expect( + rows.filter({ hasText: "Filled in field" }).filter({ hasText: field }), + ).toHaveCount(1); + } + await expect( + rows + .filter({ hasText: "Clicked on page" }) + .filter({ hasText: "I agree to the volunteer code of conduct" }), + ).toHaveCount(1); + await expect( + rows + .filter({ hasText: "Clicked on page" }) + .filter({ hasText: "Submit application" }), + ).toHaveCount(1); + // No step stopped short: none failed, none was left without its time. + await expect(rows.filter({ hasText: "It did not work." })).toHaveCount(0); + const count = await rows.count(); + expect(count).toBeGreaterThanOrEqual(6); + for (let index = 0; index < count; index += 1) { + await expect(rows.nth(index).locator("time")).toHaveText(CLOCK); + } + await shot("activity"); + }); +}); diff --git a/e2e/journeys/computer-sign-in.journey.ts b/e2e/journeys/computer-sign-in.journey.ts new file mode 100644 index 0000000000..9e0e0dd1ff --- /dev/null +++ b/e2e/journeys/computer-sign-in.journey.ts @@ -0,0 +1,108 @@ +import { MEMBER, SITE } from "../site/site"; +import { answer, approvalCard, HANDED_OVER } from "./approval-steps"; +import { + activityRows, + expect, + expectWholeReply, + named, + newConversation, + openComputer, + replyAfter, + send, + serveSite, + test, + turnOver, +} from "./computer-steps"; + +/** + * THE BOT'S COMPUTER, A SIGN-IN HANDED TO THE PERSON. A password is the person's to type, so a Bot + * that reaches a password field hands that step over (server/src/approvals/policy.ts, a credential + * typed on a website). The person takes control of the Bot's browser, types the password into the + * page themselves, hands the browser back and answers "I did it myself"; the Bot carries on from the + * signed-in page. The password goes from the person's keyboard to the page, never to the Bot. + */ + +const site = serveSite(); + +const HANDED = "needs you to type it, so it's yours to do."; + +test("a password is handed to the person, who signs in and hands the browser back", async ({ + page, + shot, + makeBot, +}) => { + test.setTimeout(400_000); + const botId = await makeBot( + "Fernhill Member", + "Look things up in my Fernhill Garden Supply members' area. Keep replies short.", + ); + const before = site().signIns.length; + let channelId = ""; + + await test.step("the Bot fills in the username and hands the password to the person", async () => { + await newConversation(page, botId); + channelId = await send( + page, + `Open ${SITE}/members, fill in my username ${MEMBER.username}, then hand me the browser so I can type my password myself. After I hand it back, tell me my seed allowance in one plain sentence.`, + ); + const card = approvalCard(page); + await expect(card).toBeVisible({ timeout: 120_000 }); + await expect(card).toContainText("Password"); + await expect(card).toContainText(HANDED); + for (const choice of HANDED_OVER) { + await expect(card.getByRole("radio", { name: choice })).toBeVisible(); + } + // Nobody can allow a password for the person: there is no standing answer to give. + await expect( + card.getByRole("radio", { name: "Always allow here" }), + ).toHaveCount(0); + await expect( + activityRows(page) + .filter({ hasText: "Filled in field" }) + .filter({ hasText: "Username" }), + ).toHaveCount(1); + expect(site().signIns.slice(before)).toEqual([]); + await shot("password-handed-over"); + }); + + const view = page.getByRole("dialog", { name: "The Bot's screen" }); + + await test.step("take control and sign in on the Bot's screen", async () => { + const panel = await openComputer(page); + await panel.getByRole("button", { name: "Take control" }).click(); + await expect(view).toBeVisible(); + await expect(view.getByRole("textbox", { name: "Address" })).toHaveValue( + `${SITE}/members`, + ); + const screen = view.getByRole("application"); + await expect(screen).toBeVisible(); + // The username field has the focus: Tab to the password, type it, and send the form. + await screen.press("Tab"); + await screen.pressSequentially(MEMBER.password); + await screen.press("Enter"); + await expect(view.getByRole("tab", { selected: true })).toHaveText( + "Members area", + ); + expect(site().signIns.slice(before)).toEqual([ + { username: MEMBER.username, ok: true }, + ]); + await shot("signed-in-by-the-person"); + await view.getByRole("button", { name: "Hand back" }).click(); + await expect( + view.getByRole("button", { name: "Take control" }), + ).toBeVisible(); + await view.getByRole("button", { name: "Close the Bot's screen" }).click(); + await expect(view).toBeHidden(); + }); + + await test.step("I did it myself: the Bot carries on from the signed-in page", async () => { + const mark = await answer(page, channelId, "I did it myself"); + const reply = await replyAfter(page, channelId, mark, /12 packets/); + await turnOver(page); + await expectWholeReply(page, reply); + await named(page, channelId); + // The Bot read the page the person signed in to; it did not sign in again. + expect(site().signIns.slice(before)).toHaveLength(1); + await shot("carried-on"); + }); +}); diff --git a/e2e/journeys/computer-steps.ts b/e2e/journeys/computer-steps.ts new file mode 100644 index 0000000000..e73c92422c --- /dev/null +++ b/e2e/journeys/computer-steps.ts @@ -0,0 +1,281 @@ +import type { APIRequestContext, Locator, Page } from "@playwright/test"; +import { test as base, expect } from "../harness/test"; +import { type Site, startSite } from "../site/site"; + +/** + * What the computer and approvals journeys do over and over: make a Bot of their own, talk to it, + * and read back what a run streamed. + */ + +async function createBot( + request: APIRequestContext, + name: string, + roleDescription: string, +): Promise { + const response = await request.post("/api/agents", { + data: { name, title: "", roleDescription, visibility: "private" }, + }); + expect(response.status(), await response.text()).toBe(201); + const { agent } = (await response.json()) as { agent: { id: string } }; + return agent.id; +} + +/** + * The harness's `test`, with `makeBot`: A BOT OF ITS OWN FOR EACH JOURNEY, deleted when it ends. + * + * A Bot's browser numbers its page snapshots from the first one it takes, and a click a fixture + * recorded carries that number. On a Bot another journey had already browsed with, the numbers would + * start further on and the recorded click would be out of date. A new Bot's browser starts at the + * beginning every time, whichever journeys ran first, and whatever a journey allows (a rule, an + * answer for a conversation) stays with that Bot. Deleted afterwards, pass or fail, so the journeys + * after it see the Bots they expect. + * + * Made through the API, as setup: making a Bot on screen is the Bots journeys' to test. It runs in + * OpenBot on the deployment's model, as a Bot made from New Bot does, so aimock answers it. + */ +export const test = base.extend<{ + makeBot: (name: string, roleDescription: string) => Promise; +}>({ + makeBot: async ({ page }, use) => { + const made: string[] = []; + await use(async (name, roleDescription) => { + const id = await createBot(page.request, name, roleDescription); + made.push(id); + return id; + }); + for (const id of made) { + await page.request.delete(`/api/agents/${encodeURIComponent(id)}`); + } + }, +}); +export { expect }; + +/** A new conversation with one Bot, from its own address, ready to type in. */ +export async function newConversation(page: Page, botId: string) { + await page.goto(`/channel/new?agent=${encodeURIComponent(botId)}`); + await expect(page.getByRole("textbox", { name: "Message" })).toBeVisible(); +} + +/** Send a message and wait for the conversation it lands in. Returns the conversation's id. */ +export async function send(page: Page, text: string): Promise { + const message = page.getByRole("textbox", { name: "Message" }); + await message.fill(text); + await page.getByRole("button", { name: "Send message" }).click(); + await expect(page).toHaveURL(/\/channel\/channel_[0-9a-f-]+/); + await expect(transcript(page).getByText(text, { exact: true })).toBeVisible(); + return new URL(page.url()).pathname.split("/").pop() ?? ""; +} + +/** The conversation's messages, as a person reads them. */ +export function transcript(page: Page): Locator { + return page.getByRole("region", { name: "Messages" }).getByRole("log"); +} + +type ThreadEvent = { + type: string; + runId?: string; + messageId?: string; + delta?: string; +}; + +async function threadOf(page: Page, channelId: string): Promise { + const response = await page.request.get(`/api/channels/${channelId}`); + expect(response.ok()).toBe(true); + return ((await response.json()) as { channel: { threadId: string } }).channel + .threadId; +} + +/** Every event of the conversation's runs, from Intelligence through the runtime. */ +export async function threadEvents( + page: Page, + channelId: string, +): Promise { + const threadId = await threadOf(page, channelId); + const response = await page.request.get( + `/api/copilotkit/threads/${threadId}/events`, + ); + // Intelligence may not have the thread's events yet just after a run. + if (!response.ok()) return []; + return ((await response.json()) as { events?: ThreadEvent[] }).events ?? []; +} + +/** One run of the conversation: whether it has finished, and the text it streamed. */ +type Run = { finished: boolean; text: string }; + +function runsOf(events: ThreadEvent[]): Run[] { + const runs: { finished: boolean; text: string[] }[] = []; + for (const event of events) { + if (event.type === "RUN_STARTED") runs.push({ finished: false, text: [] }); + const run = runs.at(-1); + if (!run) continue; + if (event.type === "TEXT_MESSAGE_CONTENT") run.text.push(event.delta ?? ""); + if (event.type === "RUN_FINISHED" || event.type === "RUN_ERROR") + run.finished = true; + } + return runs.map((run) => ({ + finished: run.finished, + text: run.text.join("").trim(), + })); +} + +/** How many runs the conversation has had so far: a mark to wait for the next reply from. */ +export async function runsSoFar( + page: Page, + channelId: string, +): Promise { + return runsOf(await threadEvents(page, channelId)).length; +} + +/** + * The Bot's reply, once its turn is over: the text the last run that said anything streamed, + * joined from its `TEXT_MESSAGE_CONTENT` deltas. + * + * Over means a run started after `since` (a mark from `runsSoFar`), the last run has finished, and + * its reply says `until`. A turn is several runs when the Bot uses its computer, one for each step + * it takes, and between two of them the last run has finished with a sentence like "I'm opening the + * page": `until` is what tells the answer from that. After an approval the server continues the + * conversation with runs of its own. + */ +export async function replyAfter( + page: Page, + channelId: string, + since: number, + until: RegExp, +): Promise { + let reply = ""; + let previous: string | null = null; + await expect + .poll( + async () => { + const runs = runsOf(await threadEvents(page, channelId)); + if (runs.length <= since || !runs.at(-1)?.finished) return false; + reply = + runs + .slice(since) + .map((run) => run.text) + .filter(Boolean) + .at(-1) ?? ""; + /* + * The same reply on two reads in a row. Read the moment its run has finished, the stored + * events have been seen to lack a piece of it for a moment ("…order is conf**FH-2044**.", + * the screen whole), so one read is not yet what was streamed. + */ + const settled = reply === previous; + previous = reply; + return settled && until.test(reply); + }, + { message: "the Bot's turn ends with a reply", timeout: 180_000 }, + ) + .toBe(true); + return reply; +} + +/** + * What a reply streamed in Markdown reads as on screen: the words, without the marks that style them. + * `**FH-2041**` is drawn as FH-2041 in bold, a `[link](https://…)` as its words. + */ +export function asRendered(markdown: string): string { + return markdown + .replace(/\[([^\]]+)\]\([^)]*\)/g, "$1") + .replace(/(\*\*|__)(.+?)\1/g, "$2") + .replace(/(^|[^\w*])[*_]([^*_\n]+)[*_](?=$|[^\w*])/g, "$1$2") + .replace(/`([^`]+)`/g, "$1") + .replace(/\s+/g, " ") + .trim(); +} + +/** + * The reply on screen is exactly what was streamed, whole and once: not cut short, not doubled, + * not two answers run together. `exact` text, so a garbled or interleaved reply fails. Compared as + * it renders (`asRendered`), since the transcript draws the Markdown the Bot wrote. + */ +export async function expectWholeReply(page: Page, reply: string) { + const shown = transcript(page).getByText(asRendered(reply), { exact: true }); + await expect(shown).toHaveCount(1); + await expect(shown).toBeVisible(); +} + +/** The Bot's turn is over: it can no longer be stopped. */ +export async function turnOver(page: Page) { + await expect(page.getByRole("button", { name: "Stop the Bot" })).toBeHidden({ + timeout: 120_000, + }); +} + +/** The Computer, open beside the conversation. */ +export function computerPanel(page: Page): Locator { + return page.getByRole("region", { name: "Computer sidebar" }); +} + +/** + * Open the Computer beside the conversation, unless it is open already. Its button in the header + * says which (`aria-expanded`): the Computer opens itself when the Bot starts browsing, and an + * address with `watch` opens on it. + */ +export async function openComputer(page: Page): Promise { + const toggle = page.getByRole("button", { name: /^(Open|Close) Computer$/ }); + await expect(toggle).toBeEnabled(); + if ((await toggle.getAttribute("aria-expanded")) !== "true") { + await toggle.click(); + } + const panel = computerPanel(page); + await expect(panel).toBeVisible(); + return panel; +} + +/** + * The rows of Activity, under the Computer's screen, newest first: one per step the Bot took on its + * computer. The list is the only one in the Computer. + */ +export function activityRows(page: Page): Locator { + return computerPanel(page).getByRole("list").getByRole("listitem"); +} + +/** + * A row on a settings page (an `Item`), found by what it says. Rows have no role of their own, so + * the row is the item that holds the text, and its buttons are found by role inside it. + */ +export function settingsRow(page: Page, text: string): Locator { + return page.locator('[data-slot="item"]').filter({ hasText: text }); +} + +/** + * The run's own website (`e2e/site`), served for the journey in this file: started before its + * test and stopped after it. Returns the site, to read the orders and applications it took. + */ +export function serveSite(): () => Site { + let site: Site | undefined; + test.beforeAll(async () => { + site = await startSite(); + }); + test.afterAll(async () => { + await site?.stop(); + }); + return () => { + if (!site) throw new Error("The test site is not running."); + return site; + }; +} + +/** A time as Activity prints one beside a step: "12:57:38 PM", or "12:57:38". */ +export const CLOCK = /\d{1,2}:\d{2}:\d{2}/; + +/** + * The conversation has its generated name. OpenBot names a conversation on a loop after its first + * answer; waited for here so the naming call is made inside the journey that caused it, and a + * recording files its fixture with that journey rather than the next one. + */ +export async function named(page: Page, channelId: string) { + await expect + .poll( + async () => { + const response = await page.request.get("/api/channels"); + const { channels } = (await response.json()) as { + channels: { id: string; summary?: string | null }[]; + }; + return channels.find((channel) => channel.id === channelId)?.summary; + }, + { message: "OpenBot names the conversation", timeout: 60_000 }, + ) + .toBeTruthy(); +} diff --git a/e2e/journeys/computer-take-control.journey.ts b/e2e/journeys/computer-take-control.journey.ts new file mode 100644 index 0000000000..a71cc0bbff --- /dev/null +++ b/e2e/journeys/computer-take-control.journey.ts @@ -0,0 +1,192 @@ +import type { Page } from "@playwright/test"; +import { SITE } from "../site/site"; +import { + activityRows, + computerPanel, + expect, + expectWholeReply, + named, + newConversation, + openComputer, + replyAfter, + send, + serveSite, + settingsRow, + test, + turnOver, +} from "./computer-steps"; + +/** + * THE BOT'S COMPUTER, TAKEN OVER. After a Bot has browsed, the person opens its screen full size, + * takes control, goes somewhere from the address bar, goes back, forward and reloads, opens a tab + * and closes it, is asked before clearing the browser, and hands it back. Settings → Computer lists + * the Bot's browser, which they stop and reset. + */ + +serveSite(); + +/** The Bot's screen, opened full size over the conversation. */ +const fullSize = (page: Page) => + page.getByRole("dialog", { name: "The Bot's screen" }); + +test("the person takes the Bot's browser over, browses, and hands it back", async ({ + page, + shot, + makeBot, +}) => { + test.setTimeout(300_000); + const botId = await makeBot( + "Fernhill Guide", + "Show me around garden shop websites. Keep answers to one sentence.", + ); + let channelId = ""; + + await test.step("the Bot opens a page", async () => { + await newConversation(page, botId); + channelId = await send( + page, + `Open ${SITE}/about and tell me who founded the shop, in one short sentence.`, + ); + await turnOver(page); + const reply = await replyAfter(page, channelId, 0, /Ines Calloway/); + await expectWholeReply(page, reply); + await named(page, channelId); + }); + + const view = fullSize(page); + const address = view.getByRole("textbox", { name: "Address" }); + const tabs = view.getByRole("tablist", { name: "Tabs" }); + + await test.step("the full-size screen shows where the Bot is, and nothing in it acts", async () => { + const panel = await openComputer(page); + await panel + .getByRole("button", { name: "Open the Bot's screen full size" }) + .click(); + await expect(view).toBeVisible(); + await expect(address).toHaveValue(`${SITE}/about`); + await expect(tabs.getByRole("tab", { selected: true })).toHaveText( + "About Fernhill Garden Supply", + ); + for (const name of ["Back", "Forward", "Reload"]) { + await expect( + view.getByRole("button", { name, exact: true }), + ).toBeDisabled(); + } + await expect(view.getByText("Take control to record.")).toBeVisible(); + await shot("full-size-bot-has-control"); + }); + + await test.step("take control", async () => { + await view.getByRole("button", { name: "Take control" }).click(); + await expect( + view.getByText("You have control. Click and type on the page."), + ).toBeVisible(); + await expect(view.getByRole("button", { name: "Hand back" })).toBeEnabled(); + await expect(view.getByRole("application", { name: /./ })).toBeVisible(); + for (const name of ["Back", "Forward", "Reload"]) { + await expect( + view.getByRole("button", { name, exact: true }), + ).toBeEnabled(); + } + await shot("you-have-control"); + }); + + await test.step("go somewhere from the address bar, back, forward and reload", async () => { + await address.fill(`${SITE}/`); + await address.press("Enter"); + await expect(tabs.getByRole("tab", { selected: true })).toHaveText( + "Fernhill Garden Supply", + ); + await expect(address).toHaveValue(`${SITE}/`); + await shot("went-home"); + + await view.getByRole("button", { name: "Back", exact: true }).click(); + await expect(address).toHaveValue(`${SITE}/about`); + await view.getByRole("button", { name: "Forward", exact: true }).click(); + await expect(address).toHaveValue(`${SITE}/`); + await view.getByRole("button", { name: "Reload", exact: true }).click(); + await expect(address).toHaveValue(`${SITE}/`); + await expect(tabs.getByRole("tab", { selected: true })).toHaveText( + "Fernhill Garden Supply", + ); + }); + + await test.step("open a new tab, browse in it, and close it", async () => { + await view.getByRole("button", { name: "New tab" }).click(); + await expect(tabs.getByRole("tab")).toHaveCount(2); + // A new tab takes the keyboard into its address, as a desktop browser's does. + await expect(address).toBeFocused(); + await address.fill(`${SITE}/volunteer`); + await address.press("Enter"); + await expect(tabs.getByRole("tab", { selected: true })).toHaveText( + "Volunteer at Fernhill", + ); + await shot("two-tabs"); + + // Delete on a tab closes it (the tab says so in `aria-keyshortcuts`), as its × does. + await tabs.getByRole("tab", { selected: true }).press("Delete"); + await expect(tabs.getByRole("tab")).toHaveCount(1); + await expect(tabs.getByRole("tab", { selected: true })).toHaveText( + "Fernhill Garden Supply", + ); + await expect(address).toHaveValue(`${SITE}/`); + await shot("tab-closed"); + }); + + await test.step("Clear browser data asks first, and Cancel keeps it", async () => { + await view.getByRole("button", { name: "Clear browser data" }).click(); + const confirm = page.getByRole("dialog", { name: "Clear this browser?" }); + await expect(confirm).toBeVisible(); + await shot("clear-browser-asks"); + await confirm.getByRole("button", { name: "Cancel" }).click(); + await expect(confirm).toBeHidden(); + await expect(address).toHaveValue(`${SITE}/`); + }); + + await test.step("hand back", async () => { + await view.getByRole("button", { name: "Hand back" }).click(); + await expect( + view.getByRole("button", { name: "Take control" }), + ).toBeVisible(); + for (const name of ["Back", "Forward", "Reload"]) { + await expect( + view.getByRole("button", { name, exact: true }), + ).toBeDisabled(); + } + await shot("handed-back"); + await view.getByRole("button", { name: "Close the Bot's screen" }).click(); + await expect(view).toBeHidden(); + await expect( + computerPanel(page).getByText("Fernhill Guide has control"), + ).toBeVisible(); + // What the person did is theirs, not the Bot's: Activity still lists only the Bot's one page. + await expect( + activityRows(page).filter({ hasText: "Opened page" }), + ).toHaveCount(1); + }); + + await test.step("Settings → Computer lists the Bot's browser, to stop or reset", async () => { + await page.goto("/settings/computer"); + await expect( + page.getByRole("heading", { name: "Your Bots' browsers" }), + ).toBeVisible(); + const row = settingsRow(page, "Fernhill Guide"); + await expect(row).toContainText("Browser running since"); + await shot("settings-browsers"); + + await row.getByRole("button", { name: "Stop browser" }).click(); + await expect(row).toContainText("No browser running"); + + await row.getByRole("button", { name: "Reset" }).click(); + const confirm = page.getByRole("dialog", { + name: "Reset Fernhill Guide's browser?", + }); + await expect(confirm).toContainText("You can't undo this."); + await shot("reset-asks"); + await confirm.getByRole("button", { name: "Reset it" }).click(); + await expect(page.getByRole("dialog")).toHaveCount(0); + // Reset deletes the Bot's browser: it leaves the list until the Bot next opens a page. + await expect(row).toHaveCount(0); + await shot("reset-done"); + }); +}); diff --git a/e2e/journeys/computer-terminal.journey.ts b/e2e/journeys/computer-terminal.journey.ts new file mode 100644 index 0000000000..5e10da3920 --- /dev/null +++ b/e2e/journeys/computer-terminal.journey.ts @@ -0,0 +1,90 @@ +import { + activityRows, + CLOCK, + computerPanel, + expect, + expectWholeReply, + named, + newConversation, + openComputer, + replyAfter, + send, + test, + turnOver, +} from "./computer-steps"; + +/** + * THE BOT'S COMPUTER, ITS TERMINAL. A Bot runs a command; Activity shows it with what it printed, + * and the Terminal tab shows the Bot's live shell. The person takes control of the terminal, types a + * command of their own, and hands it back. + */ + +test("a Bot runs a command in its terminal, and the person types in it too", async ({ + page, + shot, + makeBot, +}) => { + test.setTimeout(300_000); + const botId = await makeBot( + "Fernhill Tinkerer", + "Run small shell commands for me and report what they print.", + ); + let channelId = ""; + + await test.step("the Bot runs a command", async () => { + await newConversation(page, botId); + channelId = await send( + page, + "In your terminal, run the command echo fernhill-$((40+2)) and tell me what it printed, in one plain sentence with no formatting.", + ); + await turnOver(page); + const reply = await replyAfter(page, channelId, 0, /fernhill-42/); + await expectWholeReply(page, reply); + await named(page, channelId); + }); + + const terminal = computerPanel(page).getByRole("region", { + name: /^The Bot's terminal/, + }); + + await test.step("the Computer shows the command in Activity and the terminal", async () => { + const panel = await openComputer(page); + await panel.getByRole("tab", { name: "Terminal" }).click(); + await expect(panel.getByRole("tab", { name: "Terminal" })).toHaveAttribute( + "aria-selected", + "true", + ); + await expect(terminal).toBeVisible(); + await expect(panel.getByText("The Bot has control.")).toBeVisible(); + const ran = activityRows(page).filter({ hasText: "echo fernhill-" }); + await expect(ran).toHaveCount(1); + await expect(ran).toContainText("fernhill-42"); + await expect(ran.locator("time")).toHaveText(CLOCK); + await shot("terminal-and-activity"); + }); + + await test.step("take control of the terminal, type, and hand it back", async () => { + const panel = computerPanel(page); + await panel.getByRole("button", { name: "Take control" }).click(); + await expect( + panel.getByText("of the terminal and the screen."), + ).toBeVisible(); + await expect( + panel.getByRole("status").filter({ hasText: "You have control" }), + ).toBeVisible(); + const typing = panel.getByRole("region", { + name: "The Bot's terminal. You have control. Type here.", + }); + await expect(typing).toBeVisible(); + await typing.click(); + await page.keyboard.type("echo typed-by-$((6*7))"); + await page.keyboard.press("Enter"); + await expect(typing).toContainText("typed-by-42"); + await shot("person-typed"); + + await panel.getByRole("button", { name: "Hand back" }).click(); + await expect(panel.getByText("The Bot has control.")).toBeVisible(); + await expect(terminal).toHaveAccessibleName("The Bot's terminal"); + await shot("terminal-handed-back"); + }); +}); diff --git a/e2e/journeys/settings.journey.ts b/e2e/journeys/settings.journey.ts index 9eeedb2d2d..90586e9319 100644 --- a/e2e/journeys/settings.journey.ts +++ b/e2e/journeys/settings.journey.ts @@ -470,7 +470,14 @@ test("Activity: views, search, sort, refresh and a CSV export", async ({ await test.step("search", async () => { const search = main.getByRole("searchbox", { name: "Search events" }); await search.fill("openai-api-key"); - await expect(saved.first()).toBeVisible(); + // Search covers the events loaded, the newest 50 at first. After journeys that make a lot of + // events (a Bot browsing does), the core loop's key is further back: load older ones until it is. + const older = main.getByRole("button", { name: "Show older events" }); + await expect(async () => { + if ((await saved.count()) === 0 && (await older.isVisible())) + await older.click(); + await expect(saved.first()).toBeVisible({ timeout: 1_000 }); + }).toPass({ timeout: 30_000 }); await expect( table.getByRole("row").filter({ hasText: "Turn on an app" }), ).toHaveCount(0); diff --git a/e2e/site/site.ts b/e2e/site/site.ts new file mode 100644 index 0000000000..2661e4593f --- /dev/null +++ b/e2e/site/site.ts @@ -0,0 +1,290 @@ +import { createServer, type Server } from "node:http"; + +/** + * A small website the Bots browse in the computer journeys: a garden shop with a page to read, a + * second page to go to, a form to fill in and a checkout to pay at. + * + * SERVED BY THE RUN, NOT A THIRD-PARTY SITE. A real site changes, is slow, or blocks a headless + * browser, and a journey that reads it is not the same twice. This one answers the same bytes every + * time, so the page a Bot reads in replay is the page it read when the fixtures were recorded. + * + * WHERE THE BOT REACHES IT. The Bot's browser runs in the computer's container, which reaches this + * machine as `host.docker.internal` (docker-compose.yml gives the computer that name). Docker + * Desktop carries it to this machine's loopback, so the site listens on 127.0.0.1 only. + * + * A FIXED PORT, whatever `E2E_PORT_OFFSET` says, because the address is in what the person asks + * the Bot ("open http://host.docker.internal:46600/…"), and that is what a fixture is matched on. A + * port that moved with the offset would make every recording answer one offset only. + * + * WHAT IT REMEMBERS. Every application sent and every order placed, so a journey can tell an action + * the Bot actually took from one it only said it took: Allow once places one order, Deny places none. + */ + +export const SITE_PORT = 46600; +/** The site as the Bot's browser addresses it. */ +export const SITE = `http://host.docker.internal:${SITE_PORT}`; +/** The host part a page's address shows, in the address bar and on an approval card. */ +export const SITE_HOST = `host.docker.internal:${SITE_PORT}`; + +/** What the shop sells, by the address of its checkout page. */ +export const ITEMS = { + tomato: { name: "Heirloom tomato seeds", price: "$4.50", order: "FH-2041" }, + basil: { name: "Genovese basil seeds", price: "$3.25", order: "FH-2042" }, + marigold: { name: "French marigold seeds", price: "$2.75", order: "FH-2043" }, + sunflower: { + name: "Giant sunflower seeds", + price: "$3.80", + order: "FH-2044", + }, + lavender: { + name: "English lavender seeds", + price: "$5.10", + order: "FH-2045", + }, + chamomile: { + name: "German chamomile seeds", + price: "$2.95", + order: "FH-2046", + }, +} as const; +export type Item = keyof typeof ITEMS; + +export type Application = { + fullName: string; + email: string; + shift: string; + agreed: boolean; +}; + +/** The member who signs in to the members' area, and the password only the person knows. */ +export const MEMBER = { username: "robin.okafor", password: "marigold-7731" }; + +export type SignIn = { username: string; ok: boolean }; + +export type Site = { + /** Orders placed, oldest first, by item. */ + orders: Item[]; + /** Volunteer applications sent, oldest first. */ + applications: Application[]; + /** Sign-ins to the members' area, oldest first, and whether the password was right. */ + signIns: SignIn[]; + stop: () => Promise; +}; + +function page(title: string, body: string): string { + return ` + + + + +${title} + + + +
Fernhill Garden Supply   HomeAbout usVolunteer
+
+${body} +
+ +`; +} + +const HOME = page( + "Fernhill Garden Supply", + `

Fernhill Garden Supply

+

Seeds, soil and seedlings for city gardens since 1987.

+

Opening hours

+
    +
  • Weekdays: 8:00 am to 6:00 pm
    • +
  • Saturdays: the greenhouse opens at 7:40 am, and the shop at 9:00 am
    • +
  • Sundays: closed
    • +
+

Ask for Marguerite at the potting bench for advice on herbs.

`, +); + +const ABOUT = page( + "About Fernhill Garden Supply", + `

About us

+

Fernhill Garden Supply was founded in 1987 by Ines Calloway, in a former tram depot on Wren Street.

`, +); + +const VOLUNTEER = page( + "Volunteer at Fernhill", + `

Volunteer at Fernhill

+

Help in the greenhouse on a weekend morning.

+
+ + + + + + + + +
`, +); + +function applicationReceived(application: Application): string { + return page( + "Application received", + `

Application received

+

Thank you, ${escapeHtml(application.fullName)}. Your reference is VOL-3172.

+

We will email ${escapeHtml(application.email)} about your ${escapeHtml(application.shift)} shift.

`, + ); +} + +/** + * The members' sign-in. The Bot can fill in the username; the password is the person's, typed by + * them in the Bot's browser. The username field has the focus when the page opens, so a person who + * takes control reaches the password with Tab. + */ +const MEMBERS = page( + "Members sign in", + `

Members sign in

+

Sign in to see your seed allowance.

+
+ + + + + +
`, +); + +function membersArea(username: string): string { + return page( + "Members area", + `

Welcome back

+

Signed in as ${escapeHtml(username)}. Your seed allowance is 12 packets this season.

`, + ); +} + +const WRONG_PASSWORD = page( + "Members sign in", + "

Members sign in

That password is not right.

", +); + +function checkout(item: Item): string { + const { name, price } = ITEMS[item]; + return page( + `Checkout: ${name}`, + `

Checkout

+

Order summary: ${name}, 1 packet.

+

Total: ${price}

+

Paid with the card on file ending 4242.

+
+ +
`, + ); +} + +function orderConfirmed(item: Item): string { + const { name, order } = ITEMS[item]; + return page( + "Order confirmed", + `

Order confirmed

+

Your order ${order} for ${name} is placed. It will be ready to collect tomorrow.

`, + ); +} + +function escapeHtml(text: string): string { + return text.replace( + /[&<>"']/g, + (character) => + ({ "&": "&", "<": "<", ">": ">", '"': """, "'": "'" })[ + character + ] ?? character, + ); +} + +function isItem(value: string): value is Item { + return Object.hasOwn(ITEMS, value); +} + +/** Start the site on 127.0.0.1:46600. Fails if the port is taken, rather than serve someone else's pages. */ +export async function startSite(): Promise { + const orders: Item[] = []; + const applications: Application[] = []; + const signIns: SignIn[] = []; + const server: Server = createServer((request, response) => { + const url = new URL(request.url ?? "/", "http://site.invalid"); + const send = (status: number, html: string) => { + response.writeHead(status, { + "content-type": "text/html; charset=utf-8", + "cache-control": "no-store", + }); + response.end(html); + }; + const shop = /^\/shop\/([a-z]+)$/.exec(url.pathname)?.[1]; + if (request.method === "GET" && url.pathname === "/") + return send(200, HOME); + if (request.method === "GET" && url.pathname === "/about") + return send(200, ABOUT); + if (url.pathname === "/members" && request.method === "GET") + return send(200, MEMBERS); + if (url.pathname === "/volunteer" && request.method === "GET") + return send(200, VOLUNTEER); + if (shop && isItem(shop) && request.method === "GET") + return send(200, checkout(shop)); + if ( + request.method === "POST" && + (url.pathname === "/volunteer" || + url.pathname === "/members" || + (shop && isItem(shop))) + ) { + let body = ""; + request.setEncoding("utf8"); + request.on("data", (chunk: string) => { + body += chunk; + }); + request.on("end", () => { + if (shop && isItem(shop)) { + orders.push(shop); + return send(200, orderConfirmed(shop)); + } + const form = new URLSearchParams(body); + if (url.pathname === "/members") { + const username = form.get("username") ?? ""; + const ok = + username === MEMBER.username && + form.get("password") === MEMBER.password; + signIns.push({ username, ok }); + return send(200, ok ? membersArea(username) : WRONG_PASSWORD); + } + const application: Application = { + fullName: form.get("fullName") ?? "", + email: form.get("email") ?? "", + shift: form.get("shift") ?? "", + agreed: form.get("agree") === "yes", + }; + applications.push(application); + send(200, applicationReceived(application)); + }); + return; + } + send(404, page("Not found", "

Not found

")); + }); + await new Promise((settle, fail) => { + server.once("error", fail); + server.listen(SITE_PORT, "127.0.0.1", () => settle()); + }); + return { + orders, + applications, + signIns, + stop: () => + new Promise((settle) => { + server.closeAllConnections(); + server.close(() => settle()); + }), + }; +}