Fix seven Activity, screen and approval bugs the computer journeys found

agent-computer/src/bot-browser-host.ts

@@ -458,6 +458,8 @@ export function createBotBrowserHost(options: BotBrowserHostOptions) {
             ? await bar.navigate(browser, navigate.value)
             : ({ ok: true } as const);
         if (!done.ok) return json({ error: done.error }, done.status);
+        // The tab now in front, fitted at once, so the screen never shows it short (see the live screen's `attach`).
+        if (tabs?.ok) await view.fitWindow(view.active());
         const after = profiles.bot(botId);
         return json(after ? await stateOf(botId, after) : closedState);
       }

agent-computer/src/index.ts

@@ -1024,6 +1024,15 @@ serve<StreamData & TerminalSocketData>({
         const attach = async () => {
           const target = await currentPage(ws.data.botId);
           if (target === casting) return;
+          /*
+           * THE WINDOW FITTED TO THE PAGE THE SCREEN IS ABOUT TO SHOW, every time it starts showing one. A tab laid out
+           * again (in front again after a person closed the one before it) takes what its window has then, and a window
+           * fitted for another tab can have less: Chromium's warning bar for `--no-sandbox` sits in a Bot's first tab and
+           * is only made room for when that tab is laid out again, so it came back 56 pixels short and the screen showed
+           * a black band under the page, after Hand back too. Fitted here, from the tab as it is now laid out, it holds
+           * the whole page. A Bot's browser only: a person's is fitted as its tabs open, and local Chrome is theirs.
+           */
+          await profiles.bot(ws.data.botId)?.fitWindow(target);
           const cast = await startScreencast(target, send);
           // The claim stops a cast it refuses, so a launch that lost the screen leaks nothing.
           if (!(await claim.install(cast))) return;
@@ -1307,7 +1316,8 @@ serve<StreamData & TerminalSocketData>({
         return json(state);
       }
 
-      // The Bot asking for one value it must not be told. It has already focused the field.
+      // The Bot asking for one value it must not be told, naming the field by its ref. It does not click it first:
+      // the value is put into that field below, when the person sends it.
       if (url.pathname === "/control/secret" && request.method === "POST") {
         const body = (await request.json().catch(() => null)) as {
           label?: unknown;

agent-computer/src/profiles.ts

@@ -83,6 +83,7 @@ import { seatIdsIn } from "./profile-listing";
 import { findRecorder, isRecorderPage, type Recorder } from "./recorder";
 import {
   fitWindowsToViewport,
+  fitWindowToViewport,
   holdFilterCredentials,
   keepRecorderCurrent,
   reachRecorder,
@@ -489,6 +490,8 @@ export type OpenBotBrowser = {
    * `fitWindowsToViewport`). Not at launch: a Bot's browser nobody works stays exactly as it launches.
    */
   fitWindows(): Promise<void>;
+  /** Fit the window to this tab now, as it is laid out (see `fitWindowToViewport`). */
+  fitWindow(page: Page): Promise<void>;
 };
 
 /** A person's open browser, for the browser routes. */
@@ -556,6 +559,7 @@ export function createProfiles(
       pin(page: Page): void;
       session(): Promise<CDPSession>;
       fitWindows(): Promise<void>;
+      fitWindow(page: Page): Promise<void>;
     };
   };
 
@@ -1251,6 +1255,10 @@ export function createProfiles(
               fitted ??= fitWindowsToViewport(context, VIEWPORT, botId);
               return fitted;
             },
+            fitWindow: (shown: Page) =>
+              local
+                ? Promise.resolve()
+                : fitWindowToViewport(shown, VIEWPORT, botId),
           },
         };
         touchTab(page);
@@ -1532,6 +1540,7 @@ export function createProfiles(
             .pages()
             .filter((page) => !page.isClosed() && bot.own(page)),
         fitWindows: bot.fitWindows,
+        fitWindow: bot.fitWindow,
       };
     },
 

agent-computer/src/recorder-in-browser.ts

@@ -121,30 +121,41 @@ export async function fitWindowsToViewport(
   viewport: { width: number; height: number },
   botId: string,
 ): Promise<void> {
-  const fit = async (page: Page) => {
-    const session = await context.newCDPSession(page);
+  context.on("page", (page) => void fitWindowToViewport(page, viewport, botId));
+  await Promise.all(
+    context.pages().map((page) => fitWindowToViewport(page, viewport, botId)),
+  );
+}
+
+/**
+ * Make the window hold exactly this tab's page, now. Fitting is measured from the tab as it is laid
+ * out at that moment, so fitting again corrects a window fitted for a tab laid out differently.
+ */
+export async function fitWindowToViewport(
+  page: Page,
+  viewport: { width: number; height: number },
+  botId: string,
+): Promise<void> {
+  try {
+    const session = await page.context().newCDPSession(page);
     try {
       const { windowId } = await session.send("Browser.getWindowForTarget");
       await session.send("Browser.setContentsSize", { windowId, ...viewport });
     } finally {
       await session.detach().catch(() => undefined);
     }
-  };
-  const fitting = (page: Page) =>
-    fit(page).catch((error: unknown) => {
-      // A tab that closed while it was being fitted needs nothing; anything else is said, because
-      // the live screen then shows less than the whole page.
-      if (page.isClosed()) return;
-      console.info(
-        JSON.stringify({
-          type: "computer-window-not-fitted",
-          botId,
-          error: error instanceof Error ? error.message : String(error),
-        }),
-      );
-    });
-  context.on("page", (page) => void fitting(page));
-  await Promise.all(context.pages().map(fitting));
+  } catch (error) {
+    // A tab that closed while it was being fitted needs nothing; anything else is said, because
+    // the live screen then shows less than the whole page.
+    if (page.isClosed()) return;
+    console.info(
+      JSON.stringify({
+        type: "computer-window-not-fitted",
+        botId,
+        error: error instanceof Error ? error.message : String(error),
+      }),
+    );
+  }
 }
 
 /**

app/src/components/approvals/inline-approval.tsx

@@ -10,7 +10,7 @@ import {
   approvalInboxOptions,
   decideApproval,
 } from "@/lib/approvals";
-import { noteWatched } from "@/lib/computers/activity";
+import { noteAnswered } from "@/lib/computers/activity";
 import { EASE_OUT } from "@/lib/motion";
 
 /**
@@ -80,7 +80,7 @@ export function InlineApproval({
     onSuccess: (_, choice) => {
       setAnswered(choice);
       // Answered here, so the step's entry in Activity says when, even on a page opened after it was asked.
-      if (toolCallId) noteWatched(toolCallId);
+      if (toolCallId) noteAnswered(toolCallId);
       return cache.invalidateQueries({ queryKey: ["approvals"] });
     },
   });

app/src/components/computer/activity-log.tsx

@@ -9,8 +9,7 @@
  *
  * Read from the conversation, so it is the same whichever way a call ran: in the browser, at the
  * server's gate, or after a person allowed it. A conversation opened again, after a reload too,
- * shows what was already done in it. What this page watched happen says when; what was done before
- * the page opened it does not, because when it ran is not something the conversation says. The
+ * shows what was already done in it, and when: each step's answer says when it finished. The
  * record is still the audit trail on the server; this is the window beside the screen.
  */
 import { ownToolLabel } from "@openbot/shared/tool-name";
@@ -28,6 +27,7 @@ import {
 } from "@tabler/icons-react";
 import { useSyncExternalStore } from "react";
 import {
+  type AllowedDecision,
   activityFor,
   type BrowserStepName,
   type ComputerActivity,
@@ -62,6 +62,23 @@ const BROWSER_ICONS: Record<BrowserStepName, typeof IconWorld> = {
   computer_scroll: IconArrowsVertical,
 };
 
+/** How the person allowed a step on its card, as the card says it once answered. */
+const ALLOWED: Record<AllowedDecision, string> = {
+  allow_once: "Allowed once.",
+  allow_conversation: "Allowed for this conversation.",
+  allow_always: "Always allowed here.",
+};
+
+/** The line under a step the person allowed on its card, as one they declined has its own. */
+function AllowedLine({ allowed }: { allowed: AllowedDecision | undefined }) {
+  if (!allowed) return null;
+  return (
+    <p className="mt-1 pl-5 text-muted-foreground text-xs">
+      {ALLOWED[allowed]}
+    </p>
+  );
+}
+
 function timeOf(at: number): string {
   return new Date(at).toLocaleTimeString(undefined, {
     hour: "2-digit",
@@ -108,6 +125,7 @@ function CommandEntry({
         })}
         truncated={entry.truncated === true}
       />
+      <AllowedLine allowed={entry.allowed} />
     </li>
   );
 }
@@ -161,6 +179,7 @@ function BrowserEntry({ entry }: { entry: ComputerActivity }) {
             : "It did not work."}
         </p>
       ) : null}
+      <AllowedLine allowed={entry.allowed} />
     </li>
   );
 }
@@ -222,6 +241,7 @@ function Entry({ entry }: { entry: ComputerActivity }) {
           />
         </div>
       )}
+      <AllowedLine allowed={entry.allowed} />
     </li>
   );
 }

app/src/components/manage/computers-page.tsx

@@ -68,6 +68,29 @@ function isOwnBrowser(computer: ComputerProfile): boolean {
   return computer.botId !== undefined && isPersonBrowserId(computer.botId);
 }
 
+/**
+ * What a confirmation is about, kept while its dialog closes.
+ *
+ * The dialog fades out after Cancel or Reset it, and its words are drawn on every frame of that.
+ * Cleared on close, the row went with it, and the title fell back to its words for no row: "Reset the
+ * computer?" flashed up as the dialog left after "Reset Fernhill Guide's browser?" was answered. So
+ * closing only closes, and what the dialog names is replaced the next time it asks.
+ */
+function useConfirmation() {
+  const [asking, setAsking] = useState<{
+    about: ComputerProfile;
+    open: boolean;
+  } | null>(null);
+  return {
+    /** The row asked about, still named while the dialog closes. */
+    about: asking?.about ?? null,
+    open: asking?.open === true,
+    ask: (about: ComputerProfile) => setAsking({ about, open: true }),
+    close: () =>
+      setAsking((current) => (current ? { ...current, open: false } : null)),
+  };
+}
+
 /**
  * `children` are sections only the team edition's Admin adds below the page's own (the admin's
  * controls for these). The personal edition's Settings renders this page with none.
@@ -97,10 +120,11 @@ export function ComputersPage({
   /** The row currently running a stop/reset request. */
   const [busy, setBusy] = useState<string | null>(null);
   /** Reset deletes the browser profiles, so it requires confirmation. */
-  const [confirming, setConfirming] = useState<ComputerProfile | null>(null);
+  const reset = useConfirmation();
+  const confirming = reset.about;
   /** Updating restarts a running computer mid-task, so it is confirmed too. */
-  const [confirmingUpdate, setConfirmingUpdate] =
-    useState<ComputerProfile | null>(null);
+  const update = useConfirmation();
+  const confirmingUpdate = update.about;
   const queryClient = useQueryClient();
   const nameFor = useBotNames();
   /*
@@ -156,8 +180,8 @@ export function ComputersPage({
     action: "stop" | "reset" | "update",
   ) => {
     setBusy(rowId(computer));
-    setConfirming(null);
-    setConfirmingUpdate(null);
+    reset.close();
+    update.close();
     setState.mutate(
       {
         action,
@@ -347,7 +371,7 @@ export function ComputersPage({
                       {computer.updateAvailable ? (
                         <Button
                           disabled={busy === rowId(computer)}
-                          onClick={() => setConfirmingUpdate(computer)}
+                          onClick={() => update.ask(computer)}
                           size="sm"
                           variant="outline"
                         >
@@ -377,7 +401,7 @@ export function ComputersPage({
                       ) : (
                         <Button
                           disabled={busy === rowId(computer)}
-                          onClick={() => setConfirming(computer)}
+                          onClick={() => reset.ask(computer)}
                           size="sm"
                           variant="outline"
                         >
@@ -402,9 +426,9 @@ export function ComputersPage({
        */}
       <Dialog
         onOpenChange={(open) => {
-          if (!open) setConfirming(null);
+          if (!open) reset.close();
         }}
-        open={confirming !== null}
+        open={reset.open}
       >
         <DialogContent>
           <DialogHeader>
@@ -425,22 +449,20 @@ export function ComputersPage({
             </DialogDescription>
           </DialogHeader>
           <DialogFooter>
-            <Button
-              onClick={() => setConfirming(null)}
-              size="sm"
-              variant="ghost"
-            >
+            <Button onClick={reset.close} size="sm" variant="ghost">
               Cancel
             </Button>
             <Button
-              disabled={confirming !== null && busy === rowId(confirming)}
+              disabled={
+                reset.open && confirming !== null && busy === rowId(confirming)
+              }
               onClick={() => {
                 if (confirming) void run(confirming, "reset");
               }}
               size="sm"
               variant="destructive"
             >
-              {confirming !== null && busy === rowId(confirming)
+              {reset.open && confirming !== null && busy === rowId(confirming)
                 ? "Resetting…"
                 : "Reset it"}
             </Button>
@@ -456,9 +478,9 @@ export function ComputersPage({
 
       <Dialog
         onOpenChange={(open) => {
-          if (!open) setConfirmingUpdate(null);
+          if (!open) update.close();
         }}
-        open={confirmingUpdate !== null}
+        open={update.open}
       >
         <DialogContent>
           <DialogHeader>
@@ -472,23 +494,23 @@ export function ComputersPage({
             </DialogDescription>
           </DialogHeader>
           <DialogFooter>
-            <Button
-              onClick={() => setConfirmingUpdate(null)}
-              size="sm"
-              variant="ghost"
-            >
+            <Button onClick={update.close} size="sm" variant="ghost">
               Cancel
             </Button>
             <Button
               disabled={
-                confirmingUpdate !== null && busy === rowId(confirmingUpdate)
+                update.open &&
+                confirmingUpdate !== null &&
+                busy === rowId(confirmingUpdate)
               }
               onClick={() => {
                 if (confirmingUpdate) void run(confirmingUpdate, "update");
               }}
               size="sm"
             >
-              {confirmingUpdate !== null && busy === rowId(confirmingUpdate)
+              {update.open &&
+              confirmingUpdate !== null &&
+              busy === rowId(confirmingUpdate)
                 ? "Updating…"
                 : "Update it"}
             </Button>

app/src/lib/approvals.ts

@@ -65,6 +65,8 @@ export type ApprovalInboxData = {
     createdAt: string;
     /** When the person (or a rule) decided it; null while it waits. */
     decidedAt?: string | null;
+    /** What the person answered; null while it waits. */
+    decision?: ApprovalDecision | null;
     action: {
       botId: string;
       toolRef: string;