From 12211ff5720cd6ab9ffc5d8a7aa66cee8006bd20 Mon Sep 17 00:00:00 2001 From: David McKay Date: Fri, 13 Nov 2026 21:00:21 -0800 Subject: [PATCH 1/2] Start a computer that is off when a person opens its screen Nothing the Screen tab reads starts a computer, so one that was off stayed off however long a person looked at it, and the only way to see the Bot's browser was to ask the Bot to do something. The screen beside a conversation now says it was opened (POST /api/computers/:botId/wake) when it finds the computer off, and reads "Starting ... computer" while it comes up. Once, for what the screen found when it opened: a computer that goes off later in front of a screen left open stays off, and a card in a turn never asks, because the Bot's action starts that one. When the call is not answered the screen says the computer is off, as before. Also words for two audit events, a computer turned off for being idle and started again, on the Audit and Activity pages. --- app/src/components/audit/audit-cells.tsx | 3 + app/src/components/computer/computer-view.tsx | 41 +++++++++- app/src/lib/computers/screen.ts | 31 ++++++++ app/tests/computer-calm-start.test.tsx | 76 ++++++++++++++++++- shared/audit-words.ts | 2 + 5 files changed, 149 insertions(+), 4 deletions(-) diff --git a/app/src/components/audit/audit-cells.tsx b/app/src/components/audit/audit-cells.tsx index 96d986405e..0ca2cd664b 100644 --- a/app/src/components/audit/audit-cells.tsx +++ b/app/src/components/audit/audit-cells.tsx @@ -1178,6 +1178,9 @@ const DECISIONS: Record = { "computer.browser_cleared": "A person cleared the browser", "computer.updated": "The computer was updated", "computer.stopped": "A person pressed stop", + // Not "Allowed": nobody asked. The computer was idle, and its files and sign-ins are kept. + "computer.suspended": "Turned off while idle", + "computer.resumed": "Started again", "component.granted": "Granted to this Bot", "component.revoked": "Taken away from this Bot", diff --git a/app/src/components/computer/computer-view.tsx b/app/src/components/computer/computer-view.tsx index d87b8801ec..777e2cb811 100644 --- a/app/src/components/computer/computer-view.tsx +++ b/app/src/components/computer/computer-view.tsx @@ -16,6 +16,7 @@ import { import { readPageFrame, readScreenshot, + startComputer, type Screenshot, } from "@/lib/computers/screen"; import { useComputerControl } from "@/lib/computers/use-control"; @@ -576,8 +577,44 @@ export function ComputerView({ /** The fresher of the two answers: the poll that just ran, else the status read. */ const computerState: ComputerState | undefined = polledState ?? status.data?.state; - /** In a turn, an absent computer is one the Bot's action is about to start, not one left off. */ - const asleep = computerState === "absent" && !toolCallId; + /* + * OPENING THE SCREEN STARTS A COMPUTER THAT IS OFF. Once, for the state the screen was opened on. + * + * Nothing this view reads starts a computer, so one that was off stayed off however long a person + * looked at it, and the only way to see the Bot's browser was to ask the Bot to do something. A + * person opening the screen is asking to see the computer, so the server is told, and starts it. + * + * Only the screen beside a conversation, which a person opens: a card in a turn is drawn by + * scrolling past it, and its computer is started by the Bot's action. And only for what the screen + * found when it opened. A computer that goes off later, in front of a person who left the screen + * open, stays off: nobody opened anything, and starting it again would keep every computer with a + * screen left open running for ever. + */ + const openedOn = useRef(null); + const [startAsked, setStartAsked] = useState(false); + // A picture is a computer that is up, whatever the state reads have or have not said yet. + const hasFrame = shot !== null; + useEffect(() => { + if (toolCallId || settled || !visualVisible) return; + if (!computerState && !hasFrame) return; + if (openedOn.current === computerId) return; + openedOn.current = computerId; + if (computerState !== "absent") return; + setStartAsked(true); + void startComputer(computerId).then((state) => { + // Not started (an answer of off, or none): the screen says it is off, which it is. + if (state === null || state === "absent") setStartAsked(false); + }); + }, [computerId, computerState, hasFrame, settled, toolCallId, visualVisible]); + useEffect(() => { + // Up, or on its way: what the server says is the news again, and a later "off" is one. + if (computerState && computerState !== "absent") setStartAsked(false); + }, [computerState]); + /** + * Off, and staying off until the Bot uses it. In a turn, an absent computer is one the Bot's action + * is about to start, and one this screen has just asked for is on its way. + */ + const asleep = computerState === "absent" && !toolCallId && !startAsked; /** Waiting for a first frame, for a reason that can end. A computer that is off waits for the Bot. */ const waitingForFrame = !settled && visualVisible && shot === null && !asleep; diff --git a/app/src/lib/computers/screen.ts b/app/src/lib/computers/screen.ts index a8117b0139..3368a40227 100644 --- a/app/src/lib/computers/screen.ts +++ b/app/src/lib/computers/screen.ts @@ -65,6 +65,37 @@ export async function readScreenshot( } } +/** + * Say the Bot's screen was opened, which starts a computer that is off. + * + * The reads above never start one: a conversation makes them too, and a computer that started for + * every look would never stay off. Opening the screen is a person asking to see the computer, so + * that is said on its own. The server answers at once with the computer's state (`starting` for one + * it has just started) and does not wait for it to come up. + * + * Fails closed: nothing when the server could not be asked or has no such route, and the screen then + * says the computer is off, which it is. + */ +export async function startComputer( + computerId: string, +): Promise { + try { + const response = await tryClient( + `/api/computers/${encodeURIComponent(computerId)}/wake`, + { method: "POST" }, + ); + if (!response.ok) return null; + const body = (await response.json().catch(() => null)) as { + state?: unknown; + } | null; + return COMPUTER_STATES.includes(body?.state as ComputerState) + ? (body?.state as ComputerState) + : null; + } catch { + return null; + } +} + /** The server's "not ready yet" answer, or nothing when the body is something else. */ function notReadyOf(body: unknown): ScreenshotRead["notReady"] { if (!body || typeof body !== "object" || Array.isArray(body)) return; diff --git a/app/tests/computer-calm-start.test.tsx b/app/tests/computer-calm-start.test.tsx index a1c9c63b5a..d17db53e33 100644 --- a/app/tests/computer-calm-start.test.tsx +++ b/app/tests/computer-calm-start.test.tsx @@ -250,6 +250,66 @@ test("a computer that is off, watched from the side panel, says it starts when t expect(view.queryByText(/can't show the screen/i)).toBeNull(); }); +/** A server that keeps a computer off until it is asked to start it, and counts the asking. */ +function serveComputerThatStartsWhenAsked() { + const asked = { times: 0 }; + const state = () => (asked.times > 0 ? "starting" : "absent"); + globalThis.fetch = (async (input, init) => { + const path = String(input); + if (path.endsWith("/wake") && init?.method === "POST") { + asked.times += 1; + return Response.json({ botId: "sales", state: "starting" }); + } + if (path.endsWith("/status")) + return Response.json({ botId: "sales", state: state() }); + if (path.endsWith("/screenshot")) + return Response.json({ ready: false, state: state() }); + if (path.endsWith("/control")) return Response.json(control); + return new Response("not found", { status: 404 }); + }) as typeof fetch; + return asked; +} + +test("opening the screen beside a conversation starts a computer that is off, once", async () => { + const asked = serveComputerThatStartsWhenAsked(); + + const view = renderWithQueries( + , + ); + + expect( + await view.findByText( + "Starting Sales's computer. The first start takes about a minute.", + ), + ).toBeDefined(); + // Many more polls: it was asked for once, and is never called off while it comes up. + await act(async () => { + await new Promise((resolve) => setTimeout(resolve, 120)); + }); + expect(asked.times).toBe(1); + expect(view.queryByText(/is off/)).toBeNull(); +}); + +test("a card in a turn does not start a computer that is off: the Bot's action does", async () => { + const asked = serveComputerThatStartsWhenAsked(); + + const view = renderWithQueries( + , + ); + + await view.findByText(/^Starting Sales's computer/); + await act(async () => { + await new Promise((resolve) => setTimeout(resolve, 80)); + }); + expect(asked.times).toBe(0); +}); + test("a live stream that drops shows a quiet badge, not the Retry panel", async () => { HTMLCanvasElement.prototype.getContext = (() => ({ drawImage: () => undefined, @@ -309,10 +369,17 @@ test("a computer that was off and then fails is not called off for ever", async expect(view.queryByText(/is off/)).toBeNull(); }); -test("a computer that goes to sleep stops showing its last frame", async () => { +test("a computer that goes to sleep stops showing its last frame, and is not started again", async () => { let shots = 0; + let asked = 0; serve({ - "/status": () => Response.json({ botId: "sales", state: "absent" }), + "/wake": () => { + asked += 1; + return Response.json({ botId: "sales", state: "starting" }); + }, + // Up while its first frame is taken, as a computer that can be photographed is. + "/status": () => + Response.json({ botId: "sales", state: shots < 2 ? "ready" : "absent" }), "/screenshot": () => shots++ === 0 ? frame() @@ -332,4 +399,9 @@ test("a computer that goes to sleep stops showing its last frame", async () => { ).toBeDefined(); expect(view.queryByAltText("What the Bot is looking at")).toBeNull(); }); + // Nobody opened anything: the screen was already open when the computer went off. + await act(async () => { + await new Promise((resolve) => setTimeout(resolve, 80)); + }); + expect(asked).toBe(0); }); diff --git a/shared/audit-words.ts b/shared/audit-words.ts index b9c0e472f2..0936924c0d 100644 --- a/shared/audit-words.ts +++ b/shared/audit-words.ts @@ -120,6 +120,8 @@ export const AUDIT_EVENT_WORDS: Readonly> = { "computer.reset": "Reset the computer", "computer.browser_cleared": "Clear the browser", "computer.updated": "Update the computer", + "computer.suspended": "Turn off an idle computer", + "computer.resumed": "Start a computer again", "computer.policy_loaded": "Start the computer", "computer.policy_changed": "Change the boundaries", "computer.policy_translated": "Write a boundary from a description", From cf8bd9db30b701a9fc8840c5b653ba8813a1714a Mon Sep 17 00:00:00 2001 From: David McKay Date: Fri, 13 Nov 2026 22:16:13 -0800 Subject: [PATCH 2/2] Say a computer is starting only once the server says so The screen read Starting while its call to start the computer was out, so a server that has no such call showed Starting for a moment and then off. It now says off until the answer is that the computer is starting, and a call that is refused changes nothing a person sees. --- app/src/components/computer/computer-view.tsx | 8 ++-- app/tests/computer-calm-start.test.tsx | 46 +++++++++++++++++++ 2 files changed, 51 insertions(+), 3 deletions(-) diff --git a/app/src/components/computer/computer-view.tsx b/app/src/components/computer/computer-view.tsx index 777e2cb811..c43b558153 100644 --- a/app/src/components/computer/computer-view.tsx +++ b/app/src/components/computer/computer-view.tsx @@ -600,10 +600,12 @@ export function ComputerView({ if (openedOn.current === computerId) return; openedOn.current = computerId; if (computerState !== "absent") return; - setStartAsked(true); + /* + * "Starting" only once the server says it is. Until then, and for a server that has no such + * call or refuses it, the screen says what it always said: the computer is off. + */ void startComputer(computerId).then((state) => { - // Not started (an answer of off, or none): the screen says it is off, which it is. - if (state === null || state === "absent") setStartAsked(false); + if (state === "starting" || state === "ready") setStartAsked(true); }); }, [computerId, computerState, hasFrame, settled, toolCallId, visualVisible]); useEffect(() => { diff --git a/app/tests/computer-calm-start.test.tsx b/app/tests/computer-calm-start.test.tsx index d17db53e33..ef87601c4f 100644 --- a/app/tests/computer-calm-start.test.tsx +++ b/app/tests/computer-calm-start.test.tsx @@ -310,6 +310,52 @@ test("a card in a turn does not start a computer that is off: the Bot's action d expect(asked.times).toBe(0); }); +test("a server with no way to start a computer leaves the screen as it was: off, never starting, no problem", async () => { + let asked = 0; + const errors: unknown[] = []; + const originalError = console.error; + const originalWarn = console.warn; + console.error = (...args: unknown[]) => void errors.push(args); + console.warn = (...args: unknown[]) => void errors.push(args); + globalThis.fetch = (async (input) => { + const path = String(input); + if (path.endsWith("/wake")) { + asked += 1; + // Slow enough that whatever the screen says while it waits is seen. + await new Promise((resolve) => setTimeout(resolve, 150)); + return Response.json({ error: "Not found." }, { status: 404 }); + } + if (path.endsWith("/status")) + return Response.json({ botId: "sales", state: "absent" }); + if (path.endsWith("/screenshot")) + return Response.json({ ready: false, state: "absent" }); + if (path.endsWith("/control")) return Response.json(control); + return new Response("not found", { status: 404 }); + }) as typeof fetch; + + try { + const view = renderWithQueries( + , + ); + const off = "Sales's computer is off. It starts when the Bot next uses it."; + await view.findByText(off); + // While the call is out, once it is refused, and for many polls after: off, and nothing else. + for (let pass = 0; pass < 30; pass++) { + await act(async () => { + await new Promise((resolve) => setTimeout(resolve, 10)); + }); + expect(view.getByText(off)).toBeDefined(); + expect(view.queryByText(/^Starting/)).toBeNull(); + expect(view.queryByText(/can't show the screen/i)).toBeNull(); + } + expect(asked).toBe(1); + expect(errors).toEqual([]); + } finally { + console.error = originalError; + console.warn = originalWarn; + } +}); + test("a live stream that drops shows a quiet badge, not the Retry panel", async () => { HTMLCanvasElement.prototype.getContext = (() => ({ drawImage: () => undefined,