From 78756ec7bebccbafb99649a58dee8cc2e7a0889f Mon Sep 17 00:00:00 2001 From: David McKay Date: Mon, 9 Nov 2026 16:27:54 -0800 Subject: [PATCH 1/5] Start the big setup downloads on the first screen, before Install is clicked When the Welcome screen shows, OpenBot Desktop now downloads what this computer's install will need, side by side, into a cache: the Podman engine machine image (937 MB on a Mac, 252 MB on Windows), Podman's installer and Compose, the deployment files, and, when Docker or Podman already answers, the default Bot's images straight into that engine. Nothing is installed and nothing asks for a password before Install. The machine is built from the downloaded image with `podman machine init --image `, which takes the compressed .raw.zst / .tar.zst as served. The image is asked of downloads.copilotkit.ai first and quay.io after it, and the mirror sync now copies quay.io images, so the release's image list names the three pinned machine images. Every download resumes from a partial file after a quit (HTTP Range), is checked against its pinned SHA-256 before it is kept, and is downloaded once even when Install asks for it while it runs. With less free disk than the back end's 40 GiB plus the downloads, nothing starts and the screen says how much is free and how much is needed. Welcome, the Account step and the Install screen show one plain line with a bar while they run. --- CHANGELOG.md | 11 + desktop/src-tauri/src/acquire.rs | 49 +- desktop/src-tauri/src/deployment.rs | 5 + desktop/src-tauri/src/early_downloads.rs | 1241 +++++++++++++++++ .../src-tauri/src/early_downloads/tests.rs | 855 ++++++++++++ desktop/src-tauri/src/harness.rs | 18 + desktop/src-tauri/src/install.rs | 106 +- desktop/src-tauri/src/lib.rs | 1 + desktop/src-tauri/src/main.rs | 91 +- desktop/src-tauri/src/registry.rs | 2 +- desktop/src-tauri/src/release_images.rs | 6 + desktop/src-tauri/src/windows.rs | 2 +- desktop/src/App.tsx | 17 + desktop/src/CopilotKitAccount.tsx | 4 + desktop/src/EarlyDownloads.test.tsx | 204 +++ desktop/src/EarlyDownloads.tsx | 135 ++ desktop/src/Welcome.tsx | 5 + desktop/src/styles.css | 6 + docs/releasing.md | 9 +- scripts/downloads-images.test.ts | 27 +- scripts/downloads-images.ts | 37 +- 21 files changed, 2769 insertions(+), 62 deletions(-) create mode 100644 desktop/src-tauri/src/early_downloads.rs create mode 100644 desktop/src-tauri/src/early_downloads/tests.rs create mode 100644 desktop/src/EarlyDownloads.test.tsx create mode 100644 desktop/src/EarlyDownloads.tsx diff --git a/CHANGELOG.md b/CHANGELOG.md index 35850aeb8a..ee18a1dc68 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -10,6 +10,17 @@ Newest first. `Unreleased` is what is on `main` and not yet tagged. ### OpenBot Desktop +- **The big downloads start on the first screen, before you click Install.** Nothing used to + download until Install, and then one thing at a time: on a new Mac, Podman, then the 937 MB + engine image with no progress shown, then 1.1 GB of OpenBot's software. Now the engine image, + Podman's installer, the deployment and, when Docker or Podman already runs, OpenBot's software + start downloading as soon as the first screen shows, side by side, while you sign in and enter + your Mac password. Each screen says how far they are ("Downloading OpenBot (1.2 of 2.9 GB, about + 3 minutes left)."). A download stopped by quitting carries on from where it stopped next time, + and every file is checked against its pinned digest before it is used. Podman builds its machine + from the downloaded image instead of fetching its own. With less free disk than OpenBot needs, + the first screen says how much is free and how much is needed, and the downloads start once + there is room. Nothing is installed before you are asked. - **Sign-in comes first, and macOS asks for your password only after OpenBot says why.** On a new Mac the Account step waited for Install to put Podman in before any browser sign-in opened, and the "osascript wants to make changes" password box came up by itself over it. The CopilotKit diff --git a/desktop/src-tauri/src/acquire.rs b/desktop/src-tauri/src/acquire.rs index 4941c15abf..48bb325caf 100644 --- a/desktop/src-tauri/src/acquire.rs +++ b/desktop/src-tauri/src/acquire.rs @@ -296,14 +296,24 @@ pub fn machine_exists() -> Result { /// `local_eval::Failure::RootlessPodman`). After the switch OpenBot stays in the rootless storage /// its database is in (see `connection_keeping_openbot`), and Intelligence runs on the rootful /// engine of the same machine. -pub fn create_machine(cpus: u32, memory_mib: u32, disk_gib: u32) -> StepOutcome { - create_machine_with(cpus, memory_mib, disk_gib, machine_exists, podman) +/// +/// `image` is the machine image already downloaded (`early_downloads`), which Podman builds the +/// machine from instead of pulling its own from quay.io: `--image` takes a local path, and unpacks a +/// compressed one itself. `None` leaves the pull to Podman, as before. +pub fn create_machine( + cpus: u32, + memory_mib: u32, + disk_gib: u32, + image: Option<&Path>, +) -> StepOutcome { + create_machine_with(cpus, memory_mib, disk_gib, image, machine_exists, podman) } fn create_machine_with( cpus: u32, memory_mib: u32, disk_gib: u32, + image: Option<&Path>, exists: impl FnOnce() -> Result, mut run: impl FnMut(&[&str]) -> Result, ) -> StepOutcome { @@ -314,7 +324,10 @@ fn create_machine_with( Ok(false) => {} Err(error) => return StepOutcome::stopped(Step::CreateMachine, &error), } - let args = create_machine_args(cpus, memory_mib, disk_gib, cfg!(target_os = "windows")); + let mut args = create_machine_args(cpus, memory_mib, disk_gib, cfg!(target_os = "windows")); + if let Some(image) = image { + args.extend(["--image".to_string(), image.to_string_lossy().into_owned()]); + } let refs = args.iter().map(String::as_str).collect::>(); match run(&refs) { Ok(_) => StepOutcome::went(Step::CreateMachine, format!("{MACHINE} created.")), @@ -783,6 +796,7 @@ mod tests { 2, 4096, 20, + None, || machine_exists_with(|| Ok(listing.into())), |_args| { init_called = true; @@ -804,6 +818,7 @@ mod tests { 2, 4096, 20, + None, || { machine_exists_with(|| { Err("podman machine list exited with status 125; stdout: denied".into()) @@ -841,6 +856,7 @@ mod tests { 2, 4096, 20, + None, || machine_exists_with(|| Ok(listing.into())), |_args| panic!("machine init must not run after malformed list output"), ); @@ -863,6 +879,7 @@ mod tests { 4, 8192, 64, + None, || Ok(false), |args| { captured = args.iter().map(|arg| (*arg).to_string()).collect(); @@ -877,6 +894,31 @@ mod tests { ); } + /// The machine image the first screen downloaded is what the machine is built from: Podman is + /// given the file, and pulls nothing from quay.io (see `early_downloads`). + #[test] + fn a_downloaded_machine_image_is_what_the_machine_is_built_from() { + let image = std::path::Path::new("/cache/podman-machine.aarch64.applehv.raw.zst"); + let mut captured = Vec::new(); + let result = create_machine_with( + 4, + 6144, + 60, + Some(image), + || Ok(false), + |args| { + captured = args.iter().map(|arg| (*arg).to_string()).collect(); + Ok(String::new()) + }, + ); + assert!(result.ok, "{result:?}"); + assert_eq!(&captured[..3], ["machine", "init", "openbot"]); + assert_eq!( + &captured[captured.len() - 2..], + ["--image", "/cache/podman-machine.aarch64.applehv.raw.zst"] + ); + } + #[test] fn windows_machine_init_enables_user_mode_networking() { assert_eq!( @@ -931,6 +973,7 @@ mod tests { 4, 6144, 60, + None, || machine_exists_with(|| Ok(listing.into())), |args| panic!("nothing runs against an existing machine: {args:?}"), ); diff --git a/desktop/src-tauri/src/deployment.rs b/desktop/src-tauri/src/deployment.rs index 300b7d1eb9..417ae6026f 100644 --- a/desktop/src-tauri/src/deployment.rs +++ b/desktop/src-tauri/src/deployment.rs @@ -593,6 +593,11 @@ const WAITS: &[std::time::Duration] = &RETRY_DELAYS; #[cfg(test)] const WAITS: &[std::time::Duration] = &[std::time::Duration::ZERO; RETRY_DELAYS.len()]; +/// The waits between attempts at a download: `RETRY_DELAYS`, or none in this crate's tests. +pub fn retry_waits() -> &'static [std::time::Duration] { + WAITS +} + /// How long connecting to the server may take, before anything is asked of it. const CONNECT_TIMEOUT: std::time::Duration = std::time::Duration::from_secs(15); diff --git a/desktop/src-tauri/src/early_downloads.rs b/desktop/src-tauri/src/early_downloads.rs new file mode 100644 index 0000000000..a440687085 --- /dev/null +++ b/desktop/src-tauri/src/early_downloads.rs @@ -0,0 +1,1241 @@ +/*! +The big downloads, started when the first setup screen shows, before anybody clicks Install. + +Nothing used to download until Install was clicked, and then one item after another. On a new Mac +that was Podman's package, the password, then the 937 MB engine machine image from quay.io with no +progress shown, then 1.1 GB of OpenBot's images. None of those bytes needs an engine, a password, a +sign-in or the licence to download, only to be used. So when Welcome shows, this starts them all at +once, into a cache on disk, and Install finds them there: + +- **The engine machine image.** Podman's own `machine init` pulls it from quay.io itself, silently. + Here it is downloaded by OpenBot, with progress, and the machine is built from the local file + (`podman machine init --image `, which takes a local path and unpacks a `.zst` itself: + podman `pkg/machine/shim/diskpull` and `pkg/machine/compression`). Only for an installation that + will create OpenBot's machine with the Podman 6.1 the pins below belong to. +- **Podman's installer and Compose**, the files `install.rs` installs, through the same `fetch`. +- **The deployment** (`docker-compose.yml` and the image list), laid out as Install lays it out. +- **OpenBot's images**, pulled into an engine that already answers (Docker Desktop, or a Podman + machine that is running), with Install's own pull. + +Nothing here asks for anything. The Podman package is downloaded, never installed: installing it +needs the person's password, and that waits for Install and the explained Continue +(`install::GoAhead`). + +EVERY FILE IS PINNED, AND RESUMES. A download goes to `.partial` and becomes `` only once +its SHA-256 is the pinned one. An app quit part-way leaves the partial file, and the next try asks +for the rest (`Range`, RFC 9110 section 14) and hashes the bytes it already has first. A server that +answers the whole file instead, or a partial file that hashes wrong at the end, starts again from +nothing. downloads.copilotkit.ai is asked first, once, and the original source after it, the way +`deployment::get` does. + +ONE DOWNLOAD PER FILE. Install and the early downloads ask for the same files. Each file has a lock, +so Install waits for a download already running rather than starting a second, and then finds the +file verified. + +THE DISK IS CHECKED FIRST. The back end's CLI wants 40 GiB free (`local_eval::MIN_ENGINE_FREE_DISK_BYTES`), +and on a Mac or Windows that space is taken from this computer's disk as the engine's virtual disk +grows. k3s's own line (evicting under 5% free) is measured inside that virtual disk, not here. With +less free than the back end needs and the downloads together, nothing starts, the window says so with +both numbers, and the check runs again every few seconds until there is room. +*/ + +use std::collections::BTreeMap; +use std::fs::File; +use std::io::{Read, Seek, Write}; +use std::path::{Path, PathBuf}; +use std::sync::{Arc, Mutex, MutexGuard, OnceLock}; +use std::time::{Duration, Instant}; + +use serde::Serialize; +use sha2::{Digest, Sha256}; + +use crate::engine::{Address, Engine}; + +/// The `podman/machine-os` release the machine images below come from, which is the minor version +/// of the Podman this app installs (`install::PODMAN`). Podman pulls `machine-os:`. +pub const MACHINE_OS: &str = "6.1"; + +/// One engine machine image: the disk artifact `quay.io/podman/machine-os:6.1` names for one +/// platform, read on 2026-11-09. `manifest` is the artifact's manifest, which is what the mirror +/// copies; `blob` is its one layer, the compressed disk, which is what is downloaded. +#[derive(Clone, Copy, Debug, PartialEq, Eq)] +pub struct MachineImage { + /// The file name the artifact gives the disk (`org.opencontainers.image.title`). + pub file: &'static str, + pub manifest: &'static str, + pub blob: &'static str, + pub size: u64, +} + +/// The repository the machine images are in, on quay.io and in downloads.copilotkit.ai's copy. +pub const MACHINE_OS_REPOSITORY: &str = "podman/machine-os"; + +/// The machine image Podman would pull on this kind of computer, or `None` where OpenBot never +/// creates a machine from one: Linux has no machine, and machine-os has no `applehv` disk for an +/// Intel Mac. Windows machines are WSL distributions, so theirs is the WSL root file system. +pub fn machine_image_for(os: &str, arch: &str) -> Option { + match (os, arch) { + ("macos", "aarch64") => Some(MachineImage { + file: "podman-machine.aarch64.applehv.raw.zst", + manifest: "sha256:e4b416b1758cb94b477d0c27e265b9180e8d52ee529df9276d726206ea69b883", + blob: "sha256:cd711cd582d301f11616281f0b0098de7cad26b050b7317b8bd3f3ac667c0d21", + size: 937_318_069, + }), + ("windows", "x86_64") => Some(MachineImage { + file: "podman-machine.x86_64.wsl.tar.zst", + manifest: "sha256:b3f5e2810eecb8778a6dc09283868c1255a06d7c156312c387616d41500d4175", + blob: "sha256:81ce79567e5aa7a1a122035b7ad14bd8e0dae8be009276427800e2cc62d753c7", + size: 251_517_341, + }), + ("windows", "aarch64") => Some(MachineImage { + file: "podman-machine.aarch64.wsl.tar.zst", + manifest: "sha256:80a2eb6bb3345c955d5feea5a43dfa6bfff02b0e54b013c1455ad55b23f4e999", + blob: "sha256:1e0107fb11ae52c9b5011ed9ec9661cc5e99b15efe22556eec721b5abced7394", + size: 236_832_353, + }), + _ => None, + } +} + +/// This computer's machine image, if it has one. +pub fn machine_image() -> Option { + machine_image_for(std::env::consts::OS, crate::windows::machine_arch()) +} + +/// Every machine image reference downloads.copilotkit.ai copies (`release_images`), by the +/// artifact manifest's digest. +pub fn machine_image_references() -> Vec { + [ + ("macos", "aarch64"), + ("windows", "x86_64"), + ("windows", "aarch64"), + ] + .iter() + .filter_map(|(os, arch)| machine_image_for(os, arch)) + .map(|image| format!("quay.io/{MACHINE_OS_REPOSITORY}@{}", image.manifest)) + .collect() +} + +impl MachineImage { + /// Where to download it from: downloads.copilotkit.ai's copy, then quay.io. + pub fn sources(&self) -> Vec { + vec![ + format!( + "{}/v2/quay.io/{MACHINE_OS_REPOSITORY}/blobs/{}", + crate::deployment::DOWNLOADS, + self.blob + ), + format!( + "https://quay.io/v2/{MACHINE_OS_REPOSITORY}/blobs/{}", + self.blob + ), + ] + } + + pub fn sha256(&self) -> &'static str { + self.blob.trim_start_matches("sha256:") + } +} + +/// Whether a Podman reporting `version` (`podman version 6.1.1`, `podman.exe version 6.1.1`) pulls +/// the machine-os release the pins are from. Another Podman pulls its own, and is left to. +pub fn podman_takes_pinned_machine_image(version: &str) -> bool { + version + .split_whitespace() + .last() + .is_some_and(|number| number.starts_with(&format!("{MACHINE_OS}."))) +} + +// --------------------------------------------------------------------------------------------- +// Downloading one pinned file. +// --------------------------------------------------------------------------------------------- + +/// A file to download and the SHA-256 it must have. +#[derive(Clone, Debug, PartialEq, Eq)] +pub struct Pinned { + /// Its name on disk. Named, not taken from a URL, so a redirect cannot choose it. + pub name: String, + /// Lower-case hex. + pub sha256: String, + /// Its size, when it is known before the download answers. + pub size: Option, + /// Asked in order: the first once, the last with `RETRY_DELAYS` between attempts. + pub sources: Vec, +} + +/// What `fetch` is told as the bytes on disk grow, and the size once it is known. +pub trait Progress: Sync { + fn size(&self, _size: u64) {} + fn bytes(&self, _bytes: u64) {} +} + +impl Progress for () {} + +/// The part of a download not yet finished, beside where the file goes. +pub fn partial_path(into: &Path, name: &str) -> PathBuf { + into.join(format!("{name}.partial")) +} + +/// One lock per file, so two downloads of the same file never run at once. +fn file_lock(path: &Path) -> Arc> { + static LOCKS: OnceLock>>>> = OnceLock::new(); + LOCKS + .get_or_init(Default::default) + .lock() + .unwrap_or_else(|held| held.into_inner()) + .entry(path.to_path_buf()) + .or_default() + .clone() +} + +fn hold(lock: &Mutex<()>) -> MutexGuard<'_, ()> { + lock.lock().unwrap_or_else(|held| held.into_inner()) +} + +/// The SHA-256 of a file, read in pieces: a machine image is most of a gigabyte. +pub fn file_sha256(path: &Path) -> std::io::Result { + let mut file = File::open(path)?; + let mut hasher = Sha256::new(); + let mut buffer = vec![0u8; 1 << 20]; + loop { + let read = file.read(&mut buffer)?; + if read == 0 { + break; + } + hasher.update(&buffer[..read]); + } + Ok(hex(&hasher.finalize())) +} + +fn hex(bytes: &[u8]) -> String { + bytes.iter().map(|byte| format!("{byte:02x}")).collect() +} + +/// `pinned` in `into`, downloaded if it is not there already with its pinned digest. +/// +/// A file there with the wrong digest is replaced: a half-written download far more often than an +/// attack, and either way never used. Waits for a download of the same file already running. +pub fn fetch(pinned: &Pinned, into: &Path, progress: &dyn Progress) -> Result { + fetch_waiting(pinned, into, progress, crate::deployment::retry_waits()) +} + +/// `fetch`, waiting `waits` between attempts at the last source. +pub fn fetch_waiting( + pinned: &Pinned, + into: &Path, + progress: &dyn Progress, + waits: &[Duration], +) -> Result { + let path = into.join(&pinned.name); + let lock = file_lock(&path); + let _only = hold(&lock); + + if let Some(size) = pinned.size { + progress.size(size); + } + if path.is_file() { + if file_sha256(&path).ok().as_deref() == Some(pinned.sha256.as_str()) { + let size = std::fs::metadata(&path).map(|meta| meta.len()).unwrap_or(0); + progress.size(size); + progress.bytes(size); + return Ok(path); + } + log::warn!( + "[download] {} is not the file pinned, downloading it again", + path.display() + ); + let _ = std::fs::remove_file(&path); + } + std::fs::create_dir_all(into) + .map_err(|error| format!("could not make {}: {error}", into.display()))?; + + let partial = partial_path(into, &pinned.name); + let mut last_error = String::from("there was nowhere to download it from"); + for (index, source) in pinned.sources.iter().enumerate() { + let last = index + 1 == pinned.sources.len(); + let waits: &[Duration] = if last { waits } else { &[] }; + match download_resuming(source, &partial, pinned, progress, waits) { + Ok(()) => { + std::fs::rename(&partial, &path) + .map_err(|error| format!("could not keep {}: {error}", path.display()))?; + return Ok(path); + } + Err(error) => { + if !last { + log::warn!( + "[download] {error}; downloading {} instead", + pinned.sources[index + 1] + ); + } + last_error = error; + } + } + } + Err(last_error) +} + +/// The bytes of a partial file so far, hashed, so a download can carry on from them. +struct Resumed { + file: File, + hasher: Sha256, + bytes: u64, +} + +impl Resumed { + fn open(partial: &Path) -> Result { + let mut file = std::fs::OpenOptions::new() + .create(true) + .truncate(false) + .read(true) + .write(true) + .open(partial) + .map_err(|error| format!("could not write {}: {error}", partial.display()))?; + let mut hasher = Sha256::new(); + let mut bytes = 0u64; + let mut buffer = vec![0u8; 1 << 20]; + loop { + let read = file + .read(&mut buffer) + .map_err(|error| format!("could not read {}: {error}", partial.display()))?; + if read == 0 { + break; + } + hasher.update(&buffer[..read]); + bytes += read as u64; + } + Ok(Self { + file, + hasher, + bytes, + }) + } + + /// Back to nothing: the server sent the whole file, or what was there was not the file. + fn restart(&mut self) -> Result<(), String> { + self.file + .set_len(0) + .and_then(|()| self.file.rewind()) + .map_err(|error| format!("could not start the download again: {error}"))?; + self.hasher = Sha256::new(); + self.bytes = 0; + Ok(()) + } +} + +/// Why one attempt failed, and so whether another could succeed. +enum Failed { + ForNow(String), + ForGood(String), +} + +/// `source` into `partial`, carrying on from what is there, with an attempt more than `waits`. +/// Done when the whole file is there with the pinned digest. +fn download_resuming( + source: &str, + partial: &Path, + pinned: &Pinned, + progress: &dyn Progress, + waits: &[Duration], +) -> Result<(), String> { + let mut attempts = 1; + loop { + let result = attempt(source, partial, pinned, progress); + match result { + Ok(()) => return Ok(()), + Err(Failed::ForGood(error)) => return Err(error), + Err(Failed::ForNow(error)) => match waits.get(attempts - 1) { + Some(wait) => { + log::warn!( + "[download] attempt {attempts} of {}: {error}; trying again", + waits.len() + 1 + ); + std::thread::sleep(*wait); + attempts += 1; + } + None if attempts == 1 => return Err(error), + None => return Err(format!("{error} ({attempts} attempts)")), + }, + } + } +} + +/// How long connecting may take, and how long a download may go without a byte arriving: the same +/// as `deployment::get`'s, which has no limit on the whole download, only on silence. +const CONNECT_TIMEOUT: Duration = Duration::from_secs(15); +const STALL_TIMEOUT: Duration = Duration::from_secs(30); + +fn client() -> Result { + reqwest::blocking::Client::builder() + .user_agent("openbot-desktop") + .connect_timeout(CONNECT_TIMEOUT) + .timeout(STALL_TIMEOUT) + .build() + .map_err(|error| Failed::ForGood(format!("could not prepare the download: {error}"))) +} + +fn attempt( + source: &str, + partial: &Path, + pinned: &Pinned, + progress: &dyn Progress, +) -> Result<(), Failed> { + let mut have = Resumed::open(partial).map_err(Failed::ForGood)?; + let client = client()?; + let mut token: Option = None; + let mut response = loop { + let mut request = client.get(source); + if have.bytes > 0 { + request = request.header(reqwest::header::RANGE, format!("bytes={}-", have.bytes)); + } + if let Some(token) = &token { + request = request.bearer_auth(token); + } + let response = request + .send() + .map_err(|error| Failed::ForNow(format!("could not reach {source}: {error}")))?; + // A registry answers an anonymous download with the token to ask for (quay.io does). + if response.status() == reqwest::StatusCode::UNAUTHORIZED && token.is_none() { + token = Some(anonymous_token(&client, &response).ok_or_else(|| { + Failed::ForGood(format!( + "{source} asked for credentials OpenBot does not have" + )) + })?); + continue; + } + break response; + }; + + let status = response.status(); + if status == reqwest::StatusCode::RANGE_NOT_SATISFIABLE { + // What is there is as long as the file or longer, and was not the file. + have.restart().map_err(Failed::ForGood)?; + return Err(Failed::ForNow(format!( + "{source} had nothing after the part already downloaded" + ))); + } + if !status.is_success() { + let error = format!("{source} answered {status}"); + return Err( + if status.is_server_error() || status == reqwest::StatusCode::TOO_MANY_REQUESTS { + Failed::ForNow(error) + } else { + Failed::ForGood(error) + }, + ); + } + let resumes = status == reqwest::StatusCode::PARTIAL_CONTENT + && response + .headers() + .get(reqwest::header::CONTENT_RANGE) + .and_then(|range| range.to_str().ok()) + .is_some_and(|range| range.starts_with(&format!("bytes {}-", have.bytes))); + if !resumes && have.bytes > 0 { + // The whole file, from the start: what was there is replaced. + log::info!( + "[download] {source} sent the whole file, so {} starts again", + pinned.name + ); + have.restart().map_err(Failed::ForGood)?; + } + have.file + .seek(std::io::SeekFrom::Start(have.bytes)) + .map_err(|error| Failed::ForGood(format!("could not write the download: {error}")))?; + if pinned.size.is_none() { + if let Some(length) = response.content_length() { + progress.size(have.bytes + length); + } + } + progress.bytes(have.bytes); + + let mut buffer = vec![0u8; 256 * 1024]; + loop { + let read = match response.read(&mut buffer) { + Ok(0) => break, + Ok(read) => read, + Err(error) => { + let _ = have.file.flush(); + return Err(Failed::ForNow(format!( + "the download stopped after {} bytes: {error}", + have.bytes + ))); + } + }; + have.file + .write_all(&buffer[..read]) + .map_err(|error| Failed::ForGood(format!("could not write the download: {error}")))?; + have.hasher.update(&buffer[..read]); + have.bytes += read as u64; + progress.bytes(have.bytes); + if pinned.size.is_some_and(|size| have.bytes > size) { + have.restart().map_err(Failed::ForGood)?; + return Err(Failed::ForGood(format!( + "{source} sent more than the {} bytes {} has", + pinned.size.unwrap_or_default(), + pinned.name + ))); + } + } + have.file + .flush() + .map_err(|error| Failed::ForGood(format!("could not write the download: {error}")))?; + let got = hex(&have.hasher.clone().finalize()); + if got != pinned.sha256 { + have.restart().map_err(Failed::ForGood)?; + return Err(Failed::ForGood(format!( + "{} from {source} is not what OpenBot expected: sha256 {got}, expected {}", + pinned.name, pinned.sha256 + ))); + } + Ok(()) +} + +/// The anonymous token a registry's `WWW-Authenticate: Bearer` challenge offers, as an engine asks +/// for it (https://distribution.github.io/distribution/spec/auth/token/). See `registry`. +fn anonymous_token( + client: &reqwest::blocking::Client, + response: &reqwest::blocking::Response, +) -> Option { + let challenge = response + .headers() + .get(reqwest::header::WWW_AUTHENTICATE)? + .to_str() + .ok()?; + #[derive(serde::Deserialize)] + struct Token { + token: Option, + access_token: Option, + } + let answer: Token = client + .get(crate::registry::token_url(challenge)?) + .send() + .ok()? + .json() + .ok()?; + answer.token.or(answer.access_token) +} + +// --------------------------------------------------------------------------------------------- +// What the window shows. +// --------------------------------------------------------------------------------------------- + +/// Where the early downloads are, as the setup window says it. +#[derive(Clone, Debug, Default, PartialEq, Eq, Serialize)] +#[serde(rename_all = "camelCase")] +pub struct Summary { + pub state: State, + /// Downloaded so far, over every download. + pub bytes: u64, + /// Every download together, once each one's size is known. + #[serde(skip_serializing_if = "Option::is_none")] + pub size: Option, + #[serde(skip_serializing_if = "Option::is_none")] + pub bytes_per_second: Option, + /// Free on this computer's disk, and what setup needs, when the disk is short. + #[serde(skip_serializing_if = "Option::is_none")] + pub free: Option, + #[serde(skip_serializing_if = "Option::is_none")] + pub needed: Option, +} + +#[derive(Clone, Copy, Debug, Default, PartialEq, Eq, Serialize)] +#[serde(rename_all = "kebab-case")] +pub enum State { + /// Not started, or working out what to download. + #[default] + Idle, + /// Not enough free disk; waiting for some. + Short, + Downloading, + /// Everything planned is on disk, or in the engine. + Done, + /// Something did not download. Install downloads it again when it gets there. + Failed, + /// There is nothing to download early: this installation has everything already. + Nothing, +} + +/// One download in the table. +#[derive(Clone, Debug, Default)] +struct Entry { + size: Option, + bytes: u64, + finished: bool, + failed: bool, +} + +/// The table the window's line is made from, and the speed it is measured at. +#[derive(Default)] +struct Table { + state: State, + entries: BTreeMap<&'static str, Entry>, + /// What OpenBot's image pulls report: Install shows the same while it waits for them. + images: Option, + free: Option, + needed: Option, + speed: Speed, +} + +fn table() -> MutexGuard<'static, Table> { + static TABLE: OnceLock> = OnceLock::new(); + TABLE + .get_or_init(Default::default) + .lock() + .unwrap_or_else(|held| held.into_inner()) +} + +/// Download speed over the last few seconds. +#[derive(Default)] +struct Speed { + samples: std::collections::VecDeque<(Instant, u64)>, +} + +impl Speed { + const WINDOW: Duration = Duration::from_secs(8); + + fn observe(&mut self, now: Instant, bytes: u64) -> Option { + self.samples.push_back((now, bytes)); + while self + .samples + .front() + .is_some_and(|(at, _)| now.saturating_duration_since(*at) > Self::WINDOW) + { + self.samples.pop_front(); + } + let (first_at, first_bytes) = *self.samples.front()?; + let seconds = now.saturating_duration_since(first_at).as_secs_f64(); + (seconds >= 1.0).then(|| (bytes.saturating_sub(first_bytes) as f64 / seconds) as u64) + } +} + +impl Table { + fn summary(&mut self, now: Instant) -> Summary { + let bytes = self.entries.values().map(|entry| entry.bytes).sum::(); + let size = self + .entries + .values() + .map(|entry| entry.size) + .sum::>() + .filter(|size| *size > 0); + let bytes_per_second = self.speed.observe(now, bytes); + Summary { + state: self.state, + bytes, + size, + bytes_per_second: bytes_per_second.filter(|_| self.state == State::Downloading), + free: self.free.filter(|_| self.state == State::Short), + needed: self.needed.filter(|_| self.state == State::Short), + } + } + + /// Done or failed once every download has finished. + fn settle(&mut self) { + if self.state != State::Downloading || self.entries.values().any(|entry| !entry.finished) { + return; + } + self.state = if self.entries.values().any(|entry| entry.failed) { + State::Failed + } else { + State::Done + }; + } +} + +/// Where the early downloads are now. +pub fn summary() -> Summary { + table().summary(Instant::now()) +} + +/// One download's progress, into the table. +struct Tracked(&'static str); + +impl Progress for Tracked { + fn size(&self, size: u64) { + table().entries.entry(self.0).or_default().size = Some(size); + } + fn bytes(&self, bytes: u64) { + table().entries.entry(self.0).or_default().bytes = bytes; + } +} + +fn track(key: &'static str, size: Option) { + let mut table = table(); + let entry = table.entries.entry(key).or_default(); + if entry.size.is_none() { + entry.size = size; + } +} + +fn finish(key: &'static str, outcome: Result<(), String>) { + let mut table = table(); + let entry = table.entries.entry(key).or_default(); + entry.finished = true; + if let Err(error) = outcome { + log::warn!("[early downloads] {key}: {error}"); + entry.failed = true; + } else { + // Done is all of it, whatever the size was thought to be. + entry.size = Some(entry.size.unwrap_or(entry.bytes).max(entry.bytes)); + entry.bytes = entry.size.unwrap_or(entry.bytes); + log::info!("[early downloads] {key} is ready"); + } + table.settle(); +} + +// --------------------------------------------------------------------------------------------- +// What to download. +// --------------------------------------------------------------------------------------------- + +/// What this computer has, read once when the first screen shows. +#[derive(Clone, Debug, PartialEq, Eq)] +pub struct Facts { + pub os: &'static str, + /// The engine Install will use, as `install::choose` decides it. + pub choice: crate::install::Choice, + /// The Podman on this computer answers `--version` with this, when there is one. + pub podman_version: Option, + /// OpenBot's Podman machine is already there. + pub machine_exists: bool, + /// An engine answers now, so images can go straight into it. + pub answering: Option
, + /// This folder has finished an installation or a start before. + pub installed_before: bool, + pub machine_image: Option, +} + +/// What to download early. +#[derive(Clone, Debug, Default, PartialEq, Eq)] +pub struct Plan { + pub machine_image: Option, + pub podman: bool, + pub compose: bool, + pub deployment: bool, + /// The engine to pull OpenBot's images into. + pub images: Option
, +} + +impl Plan { + pub fn is_empty(&self) -> bool { + self == &Plan::default() + } + + /// About how much these downloads add to the disk, for the free-space check. The images and + /// Podman's files are estimates measured on 2026-11-09: their sizes are known only once their + /// downloads answer. + pub fn disk_bytes(&self) -> u64 { + const MB: u64 = 1_000_000; + self.machine_image.map_or(0, |image| image.size) + + if self.podman { 80 * MB } else { 0 } + + if self.compose { 35 * MB } else { 0 } + + if self.images.is_some() { 1_200 * MB } else { 0 } + } +} + +/// What to download early on a computer with these `facts`. +/// +/// Nothing for an installation that finished before: its machine and images are there. Otherwise +/// the deployment always, and: +/// +/// - Podman's installer and Compose when Install will install Podman (macOS and Windows; Linux's +/// Podman comes from its own packages); +/// - the machine image when Install will create OpenBot's machine with a Podman that pulls the +/// pinned machine-os release; +/// - OpenBot's images when an engine answers already. +pub fn plan(facts: &Facts) -> Plan { + use crate::install::Choice; + if facts.installed_before { + return Plan::default(); + } + let desktop = matches!(facts.os, "macos" | "windows"); + let installs_podman = desktop && facts.choice == Choice::InstallPodman; + let creates_machine = desktop + && match facts.choice { + Choice::InstallPodman => true, + Choice::UsePodman => { + !facts.machine_exists + && facts + .podman_version + .as_deref() + .is_some_and(podman_takes_pinned_machine_image) + } + _ => false, + }; + Plan { + machine_image: facts.machine_image.filter(|_| creates_machine), + podman: installs_podman, + compose: installs_podman, + deployment: true, + images: facts.answering.clone(), + } +} + +/// This computer's facts, for the installation in `root`. +fn facts_here(root: &Path) -> Facts { + let saved = crate::preparation::saved_engine(root); + let choice = crate::install::choice_here(saved); + let podman_version = crate::engine::program(Engine::Podman).and_then(|_| { + let output = crate::engine::tool(Engine::Podman) + .arg("--version") + .output() + .ok()?; + output + .status + .success() + .then(|| String::from_utf8_lossy(&output.stdout).trim().to_string()) + }); + let machine_exists = choice == crate::install::Choice::UsePodman + && crate::acquire::machine_exists().unwrap_or(true); + let status = crate::engine::detect_saved(saved); + let answering = status + .address + .filter(|_| status.responding) + .and_then(|address| address.pin().ok()); + Facts { + os: std::env::consts::OS, + choice, + podman_version, + machine_exists, + answering, + installed_before: crate::preparation::installed_before(root), + machine_image: machine_image(), + } +} + +// --------------------------------------------------------------------------------------------- +// Running them. +// --------------------------------------------------------------------------------------------- + +/// Where downloaded engine files are kept: the folder `install.rs` keeps Podman and Compose in. +pub fn cache() -> PathBuf { + crate::acquire::download_dir(&crate::stack::default_root()) +} + +/// What the running app does with the early downloads: tell the window, and record a pull. +pub struct Hooks { + pub said: Box, + pub pulled: Box, +} + +/// Held while the deployment is laid out, by the early downloads or by Install. +pub fn hold_deployment() -> MutexGuard<'static, ()> { + static DEPLOYMENT: Mutex<()> = Mutex::new(()); + hold(&DEPLOYMENT) +} + +/// Held while OpenBot's images are pulled early. +fn images_lock() -> &'static Mutex<()> { + static IMAGES: Mutex<()> = Mutex::new(()); + &IMAGES +} + +/// Start the early downloads for the installation in `root`, once per run of the app. Answers at +/// once; the downloads run on threads of their own and say where they are through `hooks`. +pub fn start(root: PathBuf, hooks: Hooks) { + static STARTED: std::sync::atomic::AtomicBool = std::sync::atomic::AtomicBool::new(false); + if STARTED.swap(true, std::sync::atomic::Ordering::SeqCst) { + return; + } + let hooks = Arc::new(hooks); + std::thread::spawn(move || { + let facts = facts_here(&root); + let plan = plan(&facts); + log::info!("[early downloads] {facts:?} plans {plan:?}"); + let cache = cache(); + if plan.machine_image.is_none() { + // A machine image that is no longer needed is not kept: it is most of a gigabyte. + forget_machine_image_in(&cache); + } + let needed = needed_for(&plan); + let jobs = jobs(&root, &plan, &cache, &hooks); + let said = |summary| (hooks.said)(summary); + run(jobs, needed, || free_space(&cache), &said, DISK_RECHECK); + }); +} + +/// How long between two looks at the free disk, while it is short. +const DISK_RECHECK: Duration = Duration::from_secs(10); + +/// The free disk OpenBot needs for `plan`: the back end's 40 GiB, and the downloads themselves. +pub fn needed_for(plan: &Plan) -> u64 { + crate::local_eval::MIN_ENGINE_FREE_DISK_BYTES + plan.disk_bytes() +} + +/// One early download: its row in the table, its size when it is known, and the work. +pub struct Job { + key: &'static str, + size: Option, + /// Further rows this job finishes itself. + also: Vec<&'static str>, + work: Box Result<(), String> + Send>, +} + +impl Job { + /// A download of `pinned` into `into`, in the table as `key`. + pub fn file(key: &'static str, pinned: Pinned, into: PathBuf) -> Self { + Self { + key, + size: pinned.size, + also: Vec::new(), + work: Box::new(move || fetch(&pinned, &into, &Tracked(key)).map(drop)), + } + } +} + +/// The jobs for `plan`. +fn jobs(root: &Path, plan: &Plan, cache: &Path, hooks: &Arc) -> Vec { + let mut jobs = Vec::new(); + if let Some(image) = plan.machine_image { + jobs.push(Job::file( + "machine-image", + machine_pinned(&image), + cache.to_path_buf(), + )); + } + if plan.podman { + let cache = cache.to_path_buf(); + jobs.push(Job { + key: "podman", + size: None, + also: Vec::new(), + work: Box::new(move || { + crate::install::fetch_podman_installer(&cache, &Tracked("podman")).map(drop) + }), + }); + } + if plan.compose { + let cache = cache.to_path_buf(); + jobs.push(Job { + key: "compose", + size: None, + also: Vec::new(), + work: Box::new(move || { + crate::install::fetch_compose(&cache, &Tracked("compose")).map(drop) + }), + }); + } + if plan.deployment { + let root = root.to_path_buf(); + match plan.images.clone() { + // The deployment is under a megabyte: a row so "done" waits for it, with no bytes. + None => jobs.push(Job { + key: "deployment", + size: Some(0), + also: Vec::new(), + work: Box::new(move || lay_out_deployment(&root)), + }), + // The images are named by the deployment's image list, so they come after it, and + // Install waits for them from the start (`wait_for_images`). + Some(engine) => { + let hooks = hooks.clone(); + jobs.push(Job { + key: "images", + size: None, + also: vec!["deployment"], + work: Box::new(move || { + let _only = hold(images_lock()); + let laid_out = lay_out_deployment(&root); + let ok = laid_out.is_ok(); + finish("deployment", laid_out); + if !ok { + return Err("the deployment did not download".into()); + } + pull_images(&engine, &root, &hooks) + }), + }); + } + } + } + jobs +} + +/// Run `jobs` once `free` says there is `needed` free, all at once, saying where they are through +/// `said` while they run. With less free, nothing starts: the window is told both numbers, and +/// `free` is asked again every `recheck`. +pub fn run( + jobs: Vec, + needed: u64, + free: impl Fn() -> Option, + said: &(dyn Fn(Summary) + Sync), + recheck: Duration, +) { + if jobs.is_empty() { + table().state = State::Nothing; + said(summary()); + return; + } + loop { + match free() { + Some(free) if free < needed => { + { + let mut table = table(); + table.state = State::Short; + table.free = Some(free); + table.needed = Some(needed); + } + log::warn!( + "[early downloads] {free} bytes free, {needed} needed: waiting for space" + ); + said(summary()); + std::thread::sleep(recheck); + } + _ => break, + } + } + + { + let mut table = table(); + table.state = State::Downloading; + table.free = None; + table.needed = None; + for job in &jobs { + for key in std::iter::once(job.key).chain(job.also.iter().copied()) { + let entry = table.entries.entry(key).or_default(); + entry.size = entry + .size + .or(if key == job.key { job.size } else { Some(0) }); + } + } + } + let ticking = std::sync::atomic::AtomicBool::new(true); + std::thread::scope(|scope| { + let workers: Vec<_> = jobs + .into_iter() + .map(|job| { + scope.spawn(move || { + let key = job.key; + let outcome = std::panic::catch_unwind(std::panic::AssertUnwindSafe(job.work)) + .unwrap_or_else(|_| Err("the download stopped unexpectedly".into())); + finish(key, outcome); + }) + }) + .collect(); + let ticking = &ticking; + scope.spawn(move || { + while ticking.load(std::sync::atomic::Ordering::SeqCst) { + said(summary()); + std::thread::sleep(Duration::from_millis(500)); + } + }); + for worker in workers { + let _ = worker.join(); + } + ticking.store(false, std::sync::atomic::Ordering::SeqCst); + }); + table().settle(); + said(summary()); +} + +fn machine_pinned(image: &MachineImage) -> Pinned { + Pinned { + name: image.file.to_string(), + sha256: image.sha256().to_string(), + size: Some(image.size), + sources: image.sources(), + } +} + +/// Lay the latest deployment out in `root`, as Install's deployment step does, unless it is there. +fn lay_out_deployment(root: &Path) -> Result<(), String> { + let _only = hold_deployment(); + // An installation on another release is Start's to move (`upgrade`), never this. + if crate::deployment::installed(root).is_some() { + return Ok(()); + } + let version = crate::deployment_release::resolve_version(root)?; + if crate::deployment::needs_fetch(root, &version) { + match crate::deployment::pinned().filter(|pin| pin.version == version) { + Some(pin) => crate::deployment::fetch_pinned(root, &pin)?, + None => crate::deployment::fetch(root, &version)?, + } + } + Ok(()) +} + +/// The images Install pulls for the default Bot, from the deployment laid out in `root`, asked of +/// Compose as Install asks it. Compose reads the release from a folder that is not an +/// installation, so nothing is decided or recorded for `root` before Install (`release_images` +/// reads a release the same way). +pub fn default_bot_images(engine: &Address, root: &Path) -> Result, String> { + let harness = crate::harness::HarnessChoice { + id: crate::harness::DEFAULT_HARNESS.into(), + agent_url: None, + }; + let nowhere = root.join(".openbot-early-downloads-nowhere"); + crate::preparation::installation_pull_set(engine, &nowhere, root, Some(&harness)).map_err( + |problem| { + format!( + "{}{}", + problem.said, + problem + .detail + .map(|detail| format!(" ({detail})")) + .unwrap_or_default() + ) + }, + ) +} + +/// OpenBot's images for the default Bot, pulled into `engine` with Install's own pull. +fn pull_images(engine: &Address, root: &Path, hooks: &Hooks) -> Result<(), String> { + let images = default_bot_images(engine, root)?; + let present = { + let engine = engine.clone(); + move |image: &str| crate::preparation::image_present(&engine, image).unwrap_or(false) + }; + let sizes = crate::pull_metrics::size_images(&images, present); + crate::pull_metrics::pull_images_limited( + &images, + crate::pull_metrics::IMAGE_PULLS_AT_ONCE, + sizes, + |_, image, on_layer| { + crate::pull_metrics::pull_image(engine, image, on_layer, |metrics| { + (hooks.pulled)(metrics) + }) + }, + || Ok(()), + |downloads| { + let mut table = table(); + let entry = table.entries.entry("images").or_default(); + entry.bytes = downloads.bytes; + entry.size = downloads.size; + table.images = Some(downloads.clone()); + }, + ) + .map_err(|problem| problem.said) +} + +/// Wait for OpenBot's images to finish pulling early, if they are, saying where they are while it +/// waits. Install's own pull then finds them there. +pub fn wait_for_images(said: impl Fn(&crate::pull_metrics::Downloads)) { + let lock = images_lock(); + let mut pacer = crate::pull_metrics::Pacer::default(); + loop { + if let Ok(_done) = lock.try_lock() { + return; + } + let images = table().images.clone(); + if let Some(downloads) = images { + if let Some(paced) = pacer.offer(Instant::now(), &downloads) { + said(&paced); + } + } + std::thread::sleep(Duration::from_millis(250)); + } +} + +/// The machine image for `podman machine init --image`, downloaded and checked, or `None` when +/// Podman should pull its own: no image for this computer, a Podman that pulls another machine-os +/// release, or a download that failed (said in the log). Waits for the early download if it is +/// running, saying how far it is. +pub fn machine_image_for_init( + said: impl Fn(crate::pull_metrics::Downloads) + Sync, +) -> Option { + let image = machine_image()?; + let version = crate::engine::tool(Engine::Podman) + .arg("--version") + .output() + .ok() + .map(|output| String::from_utf8_lossy(&output.stdout).trim().to_string())?; + if !podman_takes_pinned_machine_image(&version) { + log::info!("[early downloads] {version} pulls its own machine image"); + return None; + } + match machine_image_waiting(&machine_pinned(&image), &cache(), said) { + Ok(path) => Some(path), + Err(error) => { + log::warn!( + "[early downloads] the machine image did not download: {error}; Podman pulls its own" + ); + None + } + } +} + +/// `pinned` into `cache`, saying how far the download is every quarter of a second, whoever runs +/// it: this call, or the early download it waits for. +pub fn machine_image_waiting( + pinned: &Pinned, + cache: &Path, + said: impl Fn(crate::pull_metrics::Downloads) + Sync, +) -> Result { + let reporting = std::sync::atomic::AtomicBool::new(true); + track("machine-image", pinned.size); + std::thread::scope(|scope| { + let (reporting, said) = (&reporting, &said); + scope.spawn(move || { + let mut pacer = crate::pull_metrics::Pacer::default(); + while reporting.load(std::sync::atomic::Ordering::SeqCst) { + let entry = table().entries.get("machine-image").cloned(); + if let Some(entry) = entry { + // One file: its bytes, with no count of items beside them. + let downloads = crate::pull_metrics::Downloads { + done: 0, + total: 0, + bytes: entry.bytes, + size: entry.size, + bytes_per_second: None, + }; + if let Some(paced) = pacer.offer(Instant::now(), &downloads) { + said(paced); + } + } + std::thread::sleep(Duration::from_millis(250)); + } + }); + let fetched = fetch(pinned, cache, &Tracked("machine-image")); + reporting.store(false, std::sync::atomic::Ordering::SeqCst); + fetched + }) +} + +/// The machine is created: its image is in the machine now, and the download is not kept. +pub fn forget_machine_image() { + forget_machine_image_in(&cache()); +} + +fn forget_machine_image_in(cache: &Path) { + let Some(image) = machine_image() else { + return; + }; + for path in [cache.join(image.file), partial_path(cache, image.file)] { + match std::fs::remove_file(&path) { + Ok(()) => log::info!("[early downloads] removed {}", path.display()), + Err(error) if error.kind() == std::io::ErrorKind::NotFound => {} + Err(error) => log::warn!( + "[early downloads] could not remove {}: {error}", + path.display() + ), + } + } +} + +// --------------------------------------------------------------------------------------------- +// Free disk. +// --------------------------------------------------------------------------------------------- + +/// The bytes free to this account on the disk that holds `path`, or the nearest folder above it +/// that exists. `None` when it cannot be read, which is not judged. +pub fn free_space(path: &Path) -> Option { + let existing = path.ancestors().find(|folder| folder.exists())?; + free_space_at(existing) +} + +#[cfg(unix)] +fn free_space_at(path: &Path) -> Option { + use std::os::unix::ffi::OsStrExt; + let path = std::ffi::CString::new(path.as_os_str().as_bytes()).ok()?; + let mut stats: libc::statvfs = unsafe { std::mem::zeroed() }; + // SAFETY: `path` is NUL-terminated and outlives the call; `stats` is a statvfs it may write. + if unsafe { libc::statvfs(path.as_ptr(), &mut stats) } != 0 { + return None; + } + #[allow(clippy::unnecessary_cast)] + Some(stats.f_bavail as u64 * stats.f_frsize as u64) +} + +#[cfg(windows)] +fn free_space_at(path: &Path) -> Option { + crate::windows::free_space(&path.to_string_lossy()) +} + +#[cfg(test)] +pub(crate) mod tests; diff --git a/desktop/src-tauri/src/early_downloads/tests.rs b/desktop/src-tauri/src/early_downloads/tests.rs new file mode 100644 index 0000000000..cda32c2b6c --- /dev/null +++ b/desktop/src-tauri/src/early_downloads/tests.rs @@ -0,0 +1,855 @@ +use super::*; +use crate::test_support::temp_root; +use std::io::{BufRead, BufReader}; +use std::net::{TcpListener, TcpStream}; +use std::sync::atomic::{AtomicUsize, Ordering}; + +/// One request as the server saw it. +#[derive(Clone, Debug)] +pub(crate) struct Seen { + pub(crate) path: String, + pub(crate) range: Option, + authorization: Option, +} + +/// What the server answers a request with. +pub(crate) enum Reply { + /// `status`, these headers, then `body`. With `cut`, only that many bytes of the body are sent + /// before the connection closes, under a `Content-Length` for the whole of it. + Send { + status: &'static str, + headers: Vec<(String, String)>, + body: Vec, + cut: Option, + }, +} + +/// A loopback HTTP/1.1 server that answers each request with `answer`, and keeps what it was asked. +pub(crate) struct Server { + address: String, + seen: Arc>>, + body_bytes: Arc, + /// Loopback ports are shared with the port tests: held while this server listens. + _ports: MutexGuard<'static, ()>, +} + +impl Server { + pub(crate) fn start(answer: impl Fn(&Seen, usize) -> Reply + Send + Sync + 'static) -> Self { + let ports = crate::test_support::hold_ports(); + let listener = TcpListener::bind("127.0.0.1:0").unwrap(); + let address = format!("http://{}", listener.local_addr().unwrap()); + let seen = Arc::new(Mutex::new(Vec::new())); + let body_bytes = Arc::new(AtomicUsize::new(0)); + let answer = Arc::new(answer); + { + let (seen, body_bytes) = (seen.clone(), body_bytes.clone()); + std::thread::spawn(move || { + for stream in listener.incoming().flatten() { + let (seen, body_bytes, answer) = + (seen.clone(), body_bytes.clone(), answer.clone()); + std::thread::spawn(move || serve(stream, &seen, &body_bytes, &*answer)); + } + }); + } + Self { + address, + seen, + body_bytes, + _ports: ports, + } + } + + pub(crate) fn url(&self, path: &str) -> String { + format!("{}{path}", self.address) + } + + pub(crate) fn seen(&self) -> Vec { + self.seen.lock().unwrap().clone() + } + + /// Body bytes sent, over every answer. + pub(crate) fn sent(&self) -> usize { + self.body_bytes.load(Ordering::SeqCst) + } +} + +fn serve( + stream: TcpStream, + seen: &Mutex>, + body_bytes: &AtomicUsize, + answer: &(dyn Fn(&Seen, usize) -> Reply + Send + Sync), +) { + let mut reader = BufReader::new(stream.try_clone().unwrap()); + let mut first = String::new(); + if reader.read_line(&mut first).is_err() || first.is_empty() { + return; + } + let path = first + .split_whitespace() + .nth(1) + .unwrap_or_default() + .to_string(); + let (mut range, mut authorization) = (None, None); + loop { + let mut line = String::new(); + if reader.read_line(&mut line).is_err() || line.trim().is_empty() { + break; + } + if let Some((name, value)) = line.split_once(':') { + match name.trim().to_ascii_lowercase().as_str() { + "range" => range = Some(value.trim().to_string()), + "authorization" => authorization = Some(value.trim().to_string()), + _ => {} + } + } + } + let request = Seen { + path, + range, + authorization, + }; + let index = { + let mut seen = seen.lock().unwrap(); + seen.push(request.clone()); + seen.len() - 1 + }; + let Reply::Send { + status, + headers, + body, + cut, + } = answer(&request, index); + let mut stream = stream; + let mut head = format!( + "HTTP/1.1 {status}\r\nContent-Length: {}\r\nConnection: close\r\n", + body.len() + ); + for (name, value) in headers { + head.push_str(&format!("{name}: {value}\r\n")); + } + head.push_str("\r\n"); + let _ = stream.write_all(head.as_bytes()); + let sent = cut.unwrap_or(body.len()).min(body.len()); + let _ = stream.write_all(&body[..sent]); + body_bytes.fetch_add(sent, Ordering::SeqCst); + let _ = stream.flush(); + let _ = stream.shutdown(std::net::Shutdown::Both); +} + +/// `body` from the start, or from where a `Range: bytes=N-` asks, as a server that resumes does. +pub(crate) fn ranged(request: &Seen, body: &[u8]) -> Reply { + match request + .range + .as_deref() + .and_then(|range| range.strip_prefix("bytes=")) + .and_then(|range| range.strip_suffix('-')) + .and_then(|from| from.parse::().ok()) + { + Some(from) if from < body.len() => Reply::Send { + status: "206 Partial Content", + headers: vec![( + "Content-Range".into(), + format!("bytes {from}-{}/{}", body.len() - 1, body.len()), + )], + body: body[from..].to_vec(), + cut: None, + }, + Some(_) => Reply::Send { + status: "416 Range Not Satisfiable", + headers: vec![], + body: vec![], + cut: None, + }, + None => whole(body, None), + } +} + +pub(crate) fn whole(body: &[u8], cut: Option) -> Reply { + Reply::Send { + status: "200 OK", + headers: vec![], + body: body.to_vec(), + cut, + } +} + +fn refused(status: &'static str) -> Reply { + Reply::Send { + status, + headers: vec![], + body: vec![], + cut: None, + } +} + +/// A file of `size` bytes that is not all one byte, and the pin for it. +pub(crate) fn file(size: usize) -> (Vec, String) { + let body: Vec = (0..size).map(|n| (n * 7 % 251) as u8).collect(); + let sha = hex(&Sha256::digest(&body)); + (body, sha) +} + +fn pinned(name: &str, sha256: &str, size: usize, sources: Vec) -> Pinned { + Pinned { + name: name.into(), + sha256: sha256.into(), + size: Some(size as u64), + sources, + } +} + +const NO_WAIT: [Duration; 0] = []; + +/// The table is one per process. Tests that read it run one at a time, each from an empty one. +fn table_to_myself() -> MutexGuard<'static, ()> { + static TABLE_TEST: Mutex<()> = Mutex::new(()); + let held = TABLE_TEST.lock().unwrap_or_else(|held| held.into_inner()); + *table() = Table::default(); + held +} + +/// The app was quit part-way through a download. The next launch asks only for the rest, carries +/// on hashing from the bytes already there, and keeps the file only once it is the pinned one. +#[test] +fn a_download_stopped_part_way_carries_on_from_the_bytes_already_there() { + let (body, sha) = file(300_000); + let served = body.clone(); + let server = Server::start(move |request, index| match index { + // The first launch: the connection drops two fifths of the way through. + 0 => whole(&served, Some(120_000)), + _ => ranged(request, &served), + }); + let cache = temp_root("early-resume"); + let pin = pinned("disk.raw.zst", &sha, body.len(), vec![server.url("/disk")]); + + fetch_waiting(&pin, &cache, &(), &NO_WAIT).expect_err("the first download was cut off"); + assert!( + !cache.join("disk.raw.zst").exists(), + "an unfinished file is never kept" + ); + assert_eq!( + std::fs::metadata(partial_path(&cache, "disk.raw.zst")) + .unwrap() + .len(), + 120_000 + ); + + // The next launch. + let path = fetch_waiting(&pin, &cache, &(), &NO_WAIT).expect("the rest of the download"); + assert_eq!(std::fs::read(&path).unwrap(), body); + assert_eq!(file_sha256(&path).unwrap(), sha); + assert!(!partial_path(&cache, "disk.raw.zst").exists()); + let seen = server.seen(); + assert_eq!(seen[1].range.as_deref(), Some("bytes=120000-"), "{seen:?}"); + // Nothing was downloaded twice. + assert_eq!(server.sent(), body.len()); + let _ = std::fs::remove_dir_all(&cache); +} + +/// A server that answers a range with the whole file starts the download again from nothing, +/// rather than appending the whole file to the part already there. +#[test] +fn a_server_that_sends_the_whole_file_again_starts_the_download_again() { + let (body, sha) = file(200_000); + let served = body.clone(); + let server = Server::start(move |_, _| whole(&served, None)); + let cache = temp_root("early-whole-again"); + std::fs::create_dir_all(&cache).unwrap(); + std::fs::write(partial_path(&cache, "f"), &body[..50_000]).unwrap(); + + let path = fetch_waiting( + &pinned("f", &sha, body.len(), vec![server.url("/f")]), + &cache, + &(), + &NO_WAIT, + ) + .expect("the whole file"); + assert_eq!(std::fs::read(&path).unwrap(), body); + assert_eq!(server.seen()[0].range.as_deref(), Some("bytes=50000-")); + let _ = std::fs::remove_dir_all(&cache); +} + +/// The part already there was not the start of the file (a disk error, another file). The digest +/// check at the end refuses it, nothing is kept under the file's name, and the next try downloads +/// the whole file. +#[test] +fn a_partial_download_that_is_not_the_file_is_refused_then_downloaded_whole() { + let (body, sha) = file(200_000); + let served = body.clone(); + let server = Server::start(move |request, _| ranged(request, &served)); + let cache = temp_root("early-wrong-partial"); + std::fs::create_dir_all(&cache).unwrap(); + std::fs::write(partial_path(&cache, "f"), vec![0u8; 80_000]).unwrap(); + let pin = pinned("f", &sha, body.len(), vec![server.url("/f")]); + + let refused = fetch_waiting(&pin, &cache, &(), &NO_WAIT).expect_err("a wrong file"); + assert!( + refused.contains("is not what OpenBot expected"), + "{refused}" + ); + assert!(!cache.join("f").exists()); + + let path = fetch_waiting(&pin, &cache, &(), &NO_WAIT).expect("the file, downloaded whole"); + assert_eq!(file_sha256(&path).unwrap(), sha); + let seen = server.seen(); + assert_eq!(seen[0].range.as_deref(), Some("bytes=80000-")); + assert_eq!(seen[1].range, None, "{seen:?}"); + let _ = std::fs::remove_dir_all(&cache); +} + +/// A file already there with its pinned digest is used as it is: no request at all. +#[test] +fn a_file_already_downloaded_is_not_downloaded_again() { + let (body, sha) = file(10_000); + let server = Server::start(|_, _| refused("500 Internal Server Error")); + let cache = temp_root("early-already"); + std::fs::create_dir_all(&cache).unwrap(); + std::fs::write(cache.join("f"), &body).unwrap(); + fetch_waiting( + &pinned("f", &sha, body.len(), vec![server.url("/f")]), + &cache, + &(), + &NO_WAIT, + ) + .expect("the file already here"); + assert!(server.seen().is_empty()); + let _ = std::fs::remove_dir_all(&cache); +} + +/// downloads.copilotkit.ai first, once; the file's own source when the copy is not there. +#[test] +fn the_mirror_is_asked_first_and_the_source_after_it() { + let (body, sha) = file(20_000); + let served = body.clone(); + let server = Server::start(move |request, _| { + if request.path.starts_with("/mirror/") { + refused("403 Forbidden") + } else { + whole(&served, None) + } + }); + let cache = temp_root("early-mirror"); + fetch_waiting( + &pinned( + "f", + &sha, + body.len(), + vec![server.url("/mirror/f"), server.url("/source/f")], + ), + &cache, + &(), + &NO_WAIT, + ) + .expect("the file from its source"); + let paths: Vec = server.seen().into_iter().map(|seen| seen.path).collect(); + assert_eq!(paths, ["/mirror/f", "/source/f"]); + let _ = std::fs::remove_dir_all(&cache); +} + +/// quay.io answers an anonymous blob download with the token to ask for, as every registry does. +#[test] +fn a_registry_that_asks_for_a_token_is_given_an_anonymous_one() { + let (body, sha) = file(20_000); + let served = body.clone(); + let realm = Arc::new(Mutex::new(String::new())); + let server = { + let realm = realm.clone(); + Server::start(move |request, _| { + if request.path.starts_with("/token") { + return Reply::Send { + status: "200 OK", + headers: vec![("Content-Type".into(), "application/json".into())], + body: br#"{"token":"anonymous-token"}"#.to_vec(), + cut: None, + }; + } + if request.authorization.as_deref() != Some("Bearer anonymous-token") { + return Reply::Send { + status: "401 Unauthorized", + headers: vec![( + "WWW-Authenticate".into(), + format!( + r#"Bearer realm="{}/token",service="quay.io",scope="repository:podman/machine-os:pull""#, + realm.lock().unwrap() + ), + )], + body: vec![], + cut: None, + }; + } + whole(&served, None) + }) + }; + *realm.lock().unwrap() = server.address.clone(); + let cache = temp_root("early-token"); + fetch_waiting( + &pinned( + "f", + &sha, + body.len(), + vec![server.url("/v2/podman/machine-os/blobs/sha256:x")], + ), + &cache, + &(), + &NO_WAIT, + ) + .expect("the blob, with the token"); + let seen = server.seen(); + assert!(seen + .iter() + .any(|seen| seen.path.starts_with("/token?") && seen.path.contains("scope=repository"))); + let _ = std::fs::remove_dir_all(&cache); +} + +/// Install asks for a file the first screen is already downloading. It waits for that download +/// and uses it: the file is downloaded once. +#[test] +fn two_asks_for_one_file_download_it_once() { + let (body, sha) = file(400_000); + let served = body.clone(); + let server = Server::start(move |_, _| { + std::thread::sleep(Duration::from_millis(300)); + whole(&served, None) + }); + let cache = temp_root("early-once"); + let pin = pinned("f", &sha, body.len(), vec![server.url("/f")]); + std::thread::scope(|scope| { + let first = scope.spawn(|| fetch_waiting(&pin, &cache, &(), &NO_WAIT)); + let second = scope.spawn(|| fetch_waiting(&pin, &cache, &(), &NO_WAIT)); + first.join().unwrap().expect("the first ask"); + second.join().unwrap().expect("the second ask"); + }); + assert_eq!(server.seen().len(), 1); + let _ = std::fs::remove_dir_all(&cache); +} + +/// With less free disk than the back end and the downloads need, nothing downloads: the window +/// is told how much is free and how much is needed. Once there is room, the downloads start. +#[test] +fn with_too_little_free_disk_nothing_downloads_until_there_is_room() { + let _table = table_to_myself(); + let (body, sha) = file(50_000); + let served = body.clone(); + let server = Server::start(move |_, _| whole(&served, None)); + let cache = temp_root("early-disk"); + let looks = AtomicUsize::new(0); + let said = Mutex::new(Vec::::new()); + let needed = needed_for(&Plan { + machine_image: machine_image_for("macos", "aarch64"), + deployment: true, + ..Plan::default() + }); + // 40 GiB for the back end, and the machine image. + assert_eq!(needed, 40 * 1024 * 1024 * 1024 + 937_318_069); + run( + vec![Job::file( + "machine-image", + pinned("f", &sha, body.len(), vec![server.url("/f")]), + cache.clone(), + )], + needed, + || { + // Nothing is asked of the server while the disk is short. + assert!(server.seen().is_empty()); + let look = looks.fetch_add(1, Ordering::SeqCst); + Some(if look < 2 { 12_000_000_000 } else { needed }) + }, + &|summary| said.lock().unwrap().push(summary), + Duration::ZERO, + ); + let said = said.into_inner().unwrap(); + assert_eq!( + said[0], + Summary { + state: State::Short, + free: Some(12_000_000_000), + needed: Some(needed), + ..Summary::default() + } + ); + let last = said.last().unwrap(); + assert_eq!(last.state, State::Done, "{said:?}"); + assert_eq!(last.bytes, body.len() as u64); + assert_eq!(last.size, Some(body.len() as u64)); + assert_eq!(file_sha256(&cache.join("f")).unwrap(), sha); + let _ = std::fs::remove_dir_all(&cache); +} + +/// The free space this computer reports for a real folder, as `df` would: some, and not more than +/// the disk holds. +#[test] +fn free_space_is_read_from_the_disk_that_holds_the_folder() { + let folder = temp_root("early-free").join("not-yet").join("made"); + let free = free_space(&folder).expect("the free space of the nearest folder that exists"); + assert!(free > 0); +} + +/// A download that fails is said as failed, and the others still finish. +#[test] +fn a_download_that_fails_is_said_and_the_others_finish() { + let _table = table_to_myself(); + let (body, sha) = file(30_000); + let served = body.clone(); + let server = Server::start(move |request, _| { + if request.path == "/missing" { + refused("404 Not Found") + } else { + whole(&served, None) + } + }); + let cache = temp_root("early-fails"); + let said = Mutex::new(Vec::::new()); + run( + vec![ + Job::file( + "podman", + pinned("missing", &sha, body.len(), vec![server.url("/missing")]), + cache.clone(), + ), + Job::file( + "compose", + pinned("there", &sha, body.len(), vec![server.url("/there")]), + cache.clone(), + ), + ], + 0, + || None, + &|summary| said.lock().unwrap().push(summary), + Duration::ZERO, + ); + assert_eq!( + said.into_inner().unwrap().last().unwrap().state, + State::Failed + ); + assert!(cache.join("there").is_file()); + let _ = std::fs::remove_dir_all(&cache); +} + +/// Install's machine step waits for the machine image the first screen started, saying how far it +/// is on its own row, and gets the verified file to build the machine from. +#[test] +fn the_machine_step_waits_for_the_image_and_gets_the_verified_file() { + let _table = table_to_myself(); + let (body, sha) = file(600_000); + let served = body.clone(); + let server = Server::start(move |request, _| { + std::thread::sleep(Duration::from_millis(400)); + ranged(request, &served) + }); + let cache = temp_root("early-machine"); + let pin = pinned( + "podman-machine.aarch64.applehv.raw.zst", + &sha, + body.len(), + vec![server.url("/blob")], + ); + let rows = Mutex::new(Vec::new()); + std::thread::scope(|scope| { + // The first screen's download, already running. + let early = scope.spawn(|| fetch(&pin, &cache, &Tracked("machine-image"))); + std::thread::sleep(Duration::from_millis(100)); + let path = machine_image_waiting(&pin, &cache, |downloads| { + rows.lock().unwrap().push(downloads) + }) + .expect("the machine image"); + assert_eq!(file_sha256(&path).unwrap(), sha); + assert_eq!(path, cache.join("podman-machine.aarch64.applehv.raw.zst")); + early.join().unwrap().expect("the early download"); + }); + let rows = rows.into_inner().unwrap(); + assert!(!rows.is_empty()); + assert!(rows.iter().all(|row| row.size == Some(body.len() as u64))); + assert_eq!(server.seen().len(), 1, "downloaded once"); + let _ = std::fs::remove_dir_all(&cache); +} + +/// What each kind of computer downloads before Install is clicked. +#[test] +fn what_downloads_early_depends_on_the_engine_install_will_use() { + use crate::install::Choice; + let mac = machine_image_for("macos", "aarch64"); + let docker = Address::new(Engine::Docker, None); + let facts = + |os, choice, podman: Option<&str>, machine_exists, answering: Option<&Address>| Facts { + os, + choice, + podman_version: podman.map(str::to_string), + machine_exists, + answering: answering.cloned(), + installed_before: false, + machine_image: if os == "macos" { mac } else { None }, + }; + // A new Mac with no engine: everything Install will need. + assert_eq!( + plan(&facts("macos", Choice::InstallPodman, None, false, None)), + Plan { + machine_image: mac, + podman: true, + compose: true, + deployment: true, + images: None, + } + ); + // Docker Desktop answering: the images go straight into it, and no engine is downloaded. + assert_eq!( + plan(&facts( + "macos", + Choice::UseDocker, + None, + false, + Some(&docker) + )), + Plan { + deployment: true, + images: Some(docker.clone()), + ..Plan::default() + } + ); + // Podman 6.1 is here without OpenBot's machine: the machine image only. + assert_eq!( + plan(&facts( + "macos", + Choice::UsePodman, + Some("podman version 6.1.1"), + false, + None + )), + Plan { + machine_image: mac, + deployment: true, + ..Plan::default() + } + ); + // Another Podman pulls its own machine-os, and OpenBot's machine already there needs none. + for (version, exists) in [ + ("podman version 5.7.0", false), + ("podman version 6.1.1", true), + ] { + assert_eq!( + plan(&facts( + "macos", + Choice::UsePodman, + Some(version), + exists, + None + )) + .machine_image, + None, + "{version} {exists}" + ); + } + // A Docker Desktop that is stopped is not opened before Install. + assert_eq!( + plan(&facts( + "macos", + Choice::OpenDockerDesktop, + None, + false, + None + )), + Plan { + deployment: true, + ..Plan::default() + } + ); + // Linux's Podman comes from its own packages, and it has no machine. + assert_eq!( + plan(&facts("linux", Choice::InstallPodman, None, false, None)), + Plan { + deployment: true, + ..Plan::default() + } + ); + // An installation that finished before needs nothing early. + let mut done = facts("macos", Choice::InstallPodman, None, false, None); + done.installed_before = true; + assert!(plan(&done).is_empty()); +} + +/// The machine images are pinned for the Podman this app installs. Moving `install::PODMAN` to +/// another minor release means pinning that release's machine images. +#[test] +fn the_machine_images_belong_to_the_podman_this_app_installs() { + assert!( + crate::install::PODMAN.starts_with(&format!("{MACHINE_OS}.")), + "Podman {} pulls machine-os {}: pin its images", + crate::install::PODMAN, + crate::install::PODMAN + ); + assert!(podman_takes_pinned_machine_image("podman version 6.1.1")); + assert!(podman_takes_pinned_machine_image( + "podman.exe version 6.1.0" + )); + assert!(!podman_takes_pinned_machine_image("podman version 5.7.1")); + assert!(!podman_takes_pinned_machine_image("podman version 6.10.0")); + assert_eq!(machine_image_for("macos", "x86_64"), None); + assert_eq!(machine_image_for("linux", "x86_64"), None); + let mac = machine_image_for("macos", "aarch64").unwrap(); + assert_eq!( + mac.sources(), + [ + "https://downloads.copilotkit.ai/v2/quay.io/podman/machine-os/blobs/sha256:cd711cd582d301f11616281f0b0098de7cad26b050b7317b8bd3f3ac667c0d21", + "https://quay.io/v2/podman/machine-os/blobs/sha256:cd711cd582d301f11616281f0b0098de7cad26b050b7317b8bd3f3ac667c0d21", + ] + ); + assert_eq!(machine_image_references().len(), 3); +} + +/** +The machine is built from the downloaded image, with nothing asked of quay.io. + +Runs the real `podman machine init --image` against the pinned applehv image, with every proxy +pointed at a closed loopback port, so any download Podman tried would fail the init. Needs Podman +6.1 and the image on disk: `OPENBOT_LIVE_MACHINE_IMAGE=`. +The machine is removed afterwards. +*/ +#[test] +#[ignore = "runs podman machine init with the 937 MB machine image"] +fn a_machine_is_built_from_the_downloaded_image_without_asking_quay() { + let image = PathBuf::from( + std::env::var("OPENBOT_LIVE_MACHINE_IMAGE").expect("OPENBOT_LIVE_MACHINE_IMAGE"), + ); + let pinned = machine_image_for("macos", "aarch64").unwrap(); + assert_eq!(file_sha256(&image).unwrap(), pinned.sha256()); + let name = format!("openbot-early-{}", std::process::id()); + let closed = "http://127.0.0.1:9"; + let podman = |args: &[&str]| { + std::process::Command::new("podman") + .args(args) + .env("HTTP_PROXY", closed) + .env("HTTPS_PROXY", closed) + .env("http_proxy", closed) + .env("https_proxy", closed) + .env_remove("NO_PROXY") + .env_remove("no_proxy") + .output() + .unwrap() + }; + // `machine init` makes the new machine the default connection, and `rm` hands that on to + // another: the person's own default is put back afterwards. + let default_connection = || { + let listed = podman(&[ + "system", + "connection", + "list", + "--format", + "{{.Name}} {{.Default}}", + ]); + String::from_utf8_lossy(&listed.stdout) + .lines() + .find_map(|line| line.strip_suffix(" true").map(str::to_string)) + }; + let default_before = default_connection(); + let image_arg = image.to_string_lossy().into_owned(); + let init = podman(&[ + "machine", + "init", + &name, + "--cpus", + "2", + "--memory", + "2048", + "--disk-size", + "12", + "--rootful", + "--image", + &image_arg, + ]); + let listed = podman(&["machine", "inspect", &name]); + let removed = podman(&["machine", "rm", "--force", &name]); + if let Some(before) = + default_before.filter(|before| Some(before) != default_connection().as_ref()) + { + podman(&["system", "connection", "default", &before]); + } + assert!( + init.status.success(), + "init: {}{}", + String::from_utf8_lossy(&init.stdout), + String::from_utf8_lossy(&init.stderr) + ); + let said = String::from_utf8_lossy(&init.stdout); + assert!( + !said.contains("Looking up Podman Machine image"), + "Podman looked the image up: {said}" + ); + assert!(listed.status.success()); + assert!(removed.status.success()); +} + +/** +The real machine image, from the real sources: downloads.copilotkit.ai (which answers 403 until the +next sync copies it), then quay.io with its anonymous token and its CDN's redirect. Stopped part-way, +as a quit would, then carried on from the bytes on disk and checked against the pin. + +The Windows x86_64 image, 252 MB, so it runs on any computer: `cargo test -- --ignored live_machine`. +*/ +#[test] +#[ignore = "downloads 252 MB from quay.io"] +fn live_machine_image_downloads_from_quay_and_carries_on_after_a_stop() { + struct StopAt(u64); + impl Progress for StopAt { + fn bytes(&self, bytes: u64) { + assert!(bytes < self.0, "stopped part-way"); + } + } + let image = machine_image_for("windows", "x86_64").unwrap(); + let pinned = machine_pinned(&image); + let cache = temp_root("early-live-quay"); + let stopped = std::panic::catch_unwind(|| fetch(&pinned, &cache, &StopAt(60_000_000))); + assert!(stopped.is_err()); + let partial = std::fs::metadata(partial_path(&cache, image.file)) + .unwrap() + .len(); + assert!(partial >= 60_000_000, "{partial}"); + let started = Instant::now(); + let path = fetch(&pinned, &cache, &()).expect("the rest, from quay.io"); + eprintln!( + "carried on from {partial} bytes, finished in {:.1} s", + started.elapsed().as_secs_f64() + ); + assert_eq!(file_sha256(&path).unwrap(), image.sha256()); + let _ = std::fs::remove_dir_all(&cache); +} + +/** +The images the first screen pulls are the ones Install pulls for the default Bot, worked out without +an installation: nothing is recorded in the folder before Install. Runs `docker compose config` +against this repository's Compose file and a release's image list. +*/ +#[test] +#[ignore = "runs docker compose"] +fn live_the_first_screen_pulls_the_images_install_would() { + let root = temp_root("early-images"); + std::fs::create_dir_all(&root).unwrap(); + let repository = Path::new(env!("CARGO_MANIFEST_DIR")).join("../.."); + std::fs::copy( + repository.join("docker-compose.yml"), + root.join("docker-compose.yml"), + ) + .unwrap(); + std::fs::copy( + Path::new(env!("CARGO_MANIFEST_DIR")) + .join("test-fixtures/release-images/container-images-v0.3.3.json"), + crate::deployment::images_path(&root), + ) + .unwrap(); + let docker = Address::new(Engine::Docker, None); + let early = default_bot_images(&docker, &root).expect("the default Bot's images"); + // Nothing of an installation's was decided or written. + let written: Vec = std::fs::read_dir(&root) + .unwrap() + .map(|entry| entry.unwrap().file_name().to_string_lossy().into_owned()) + .collect(); + assert_eq!(written.len(), 2, "{written:?}"); + let harness = crate::harness::HarnessChoice { + id: crate::harness::DEFAULT_HARNESS.into(), + agent_url: None, + }; + let install = crate::preparation::installation_pull_set(&docker, &root, &root, Some(&harness)) + .expect("Install's images"); + assert_eq!(early, install); + assert!(early.iter().any(|image| image.contains("openbot-server@"))); + eprintln!("{early:#?}"); + let _ = std::fs::remove_dir_all(&root); +} diff --git a/desktop/src-tauri/src/harness.rs b/desktop/src-tauri/src/harness.rs index d736c19798..4c8061e583 100644 --- a/desktop/src-tauri/src/harness.rs +++ b/desktop/src-tauri/src/harness.rs @@ -137,6 +137,10 @@ fn byo_remote_ag_ui_url(value: &str) -> Result { } } +/// The Bot a first setup installs, as the setup window names it (`DEFAULT_HARNESS` in +/// desktop/src/harness.ts). The early downloads pull its images before Install is clicked. +pub const DEFAULT_HARNESS: &str = "langgraph"; + /// The list, ranked as the build doc ranks it: stars first, with downloads as the sanity check, /// because each misleads alone. /// @@ -302,6 +306,20 @@ where mod tests { use crate::test_support::temp_root; + /// The window installs this Bot on Install's click, and the early downloads pull its images + /// before that click: the two must name the same one. + #[test] + fn the_default_bot_is_the_one_the_setup_window_installs() { + let window = include_str!("../../src/harness.ts"); + assert!( + window.contains(&format!("DEFAULT_HARNESS = \"{}\"", super::DEFAULT_HARNESS)), + "desktop/src/harness.ts DEFAULT_HARNESS must equal harness::DEFAULT_HARNESS" + ); + assert!(super::catalogue() + .iter() + .any(|row| row.id == super::DEFAULT_HARNESS && row.image.is_some())); + } + /// Both plans name a Bot that exists and can actually use them. #[test] fn each_plan_names_a_bot_that_exists() { diff --git a/desktop/src-tauri/src/install.rs b/desktop/src-tauri/src/install.rs index 1e9b6719e3..b3f1e34cc7 100644 --- a/desktop/src-tauri/src/install.rs +++ b/desktop/src-tauri/src/install.rs @@ -47,8 +47,6 @@ use std::path::{Path, PathBuf}; use std::sync::{Condvar, Mutex}; use std::time::Duration; -use sha2::{Digest, Sha256}; - use crate::engine::{self, Engine, StepLine}; use crate::problem::Problem; @@ -164,38 +162,58 @@ pub fn compose_download() -> Result { /// /// A file already there with the right digest is kept, so a retry after a failed install is not a /// second download. A file already there with the wrong one is replaced: that is a half-written -/// download far more often than it is an attack, and either way it must not be run. +/// download far more often than it is an attack, and either way it must not be run. The download +/// itself is the early downloads' (`early_downloads::fetch`): from downloads.copilotkit.ai first, +/// resumed where an earlier one stopped, and waited for when the first screen already started it. fn fetch_verified(download: &Download, into: &Path) -> Result { - let path = into.join(download.file); - if let Ok(existing) = std::fs::read(&path) { - if digest_of(&existing) == download.sha256 { - return Ok(path); - } - } - - let body = crate::deployment::get(&download.url).map_err(|error| { + fetch_pinned(download, into, &()).map_err(|error| { Problem::with( "OpenBot could not download the software it needs to run. Check the internet \ connection and try again.", format!("{}: {error}", download.url), ) - })?; + }) +} - let got = digest_of(&body); - if got != download.sha256 { - return Err(Problem::with( - "What OpenBot downloaded is not what it was expecting, so it has not been run. Try \ - again.", - format!( - "{} from {}: expected sha256 {}, got {got}", - download.file, download.url, download.sha256 - ), - )); - } +fn fetch_pinned( + download: &Download, + into: &Path, + progress: &dyn crate::early_downloads::Progress, +) -> Result { + let pinned = crate::early_downloads::Pinned { + name: download.file.to_string(), + sha256: download.sha256.to_string(), + size: None, + sources: crate::deployment::mirrored(&download.url) + .into_iter() + .chain(std::iter::once(download.url.clone())) + .collect(), + }; + crate::early_downloads::fetch(&pinned, into, progress) +} - std::fs::create_dir_all(into).map_err(|error| unwritable(into, &error.to_string()))?; - std::fs::write(&path, &body).map_err(|error| unwritable(&path, &error.to_string()))?; - Ok(path) +/// Podman's installer for this computer, into `into`, before Install needs it. Never installed here. +pub fn fetch_podman_installer( + into: &Path, + progress: &dyn crate::early_downloads::Progress, +) -> Result { + fetch_pinned( + &podman_download().map_err(|problem| problem.said)?, + into, + progress, + ) +} + +/// Compose for this computer, into `into`, before Install places it. +pub fn fetch_compose( + into: &Path, + progress: &dyn crate::early_downloads::Progress, +) -> Result { + fetch_pinned( + &compose_download().map_err(|problem| problem.said)?, + into, + progress, + ) } /// One sentence for every "could not write here", because the person's fix is the same each time. @@ -207,7 +225,9 @@ fn unwritable(path: &Path, error: &str) -> Problem { ) } +#[cfg(test)] fn digest_of(bytes: &[u8]) -> String { + use sha2::{Digest, Sha256}; let mut hasher = Sha256::new(); hasher.update(bytes); hasher @@ -474,7 +494,7 @@ impl Found { /// What the install step does to have an engine answering. #[derive(Debug, Clone, Copy, PartialEq, Eq)] -enum Choice { +pub enum Choice { /// Docker answers now. UseDocker, /// Docker Desktop is installed and stopped, and opening it shows nothing that needs the person. @@ -520,6 +540,11 @@ fn choose(found: Found, saved: Option) -> Choice { } } +/// What Install would do for an engine on this computer now. The early downloads fetch what it needs. +pub fn choice_here(saved: Option) -> Choice { + choose(Found::here(saved), saved) +} + fn install_engine_with( cache: &Path, found: Found, @@ -1193,6 +1218,33 @@ mod tests { ); } + /// A Podman installer whose download was cut off, by a dropped connection or by quitting the app, + /// carries on from the bytes already on disk at the next try, and is checked whole before it is + /// kept. Before the early downloads, every try downloaded the whole file again into memory. + #[test] + fn a_download_cut_off_part_way_carries_on_from_where_it_stopped() { + use crate::early_downloads::tests::{file, ranged, whole, Server}; + let (body, sha256) = file(400_000); + let served = body.clone(); + let server = Server::start(move |request, index| match index { + 0 => whole(&served, Some(150_000)), + _ => ranged(request, &served), + }); + let dir = temp_root("install-resume"); + let download = Download { + url: server.url("/podman-installer-macos-arm64.pkg"), + sha256: Box::leak(sha256.into_boxed_str()), + file: "podman-installer-macos-arm64.pkg", + }; + let path = fetch_verified(&download, &dir).expect("the installer, carried on"); + assert_eq!(std::fs::read(&path).unwrap(), body); + let seen = server.seen(); + assert_eq!(seen.len(), 2, "{seen:?}"); + assert_eq!(seen[1].range.as_deref(), Some("bytes=150000-"), "{seen:?}"); + assert_eq!(server.sent(), body.len(), "nothing was downloaded twice"); + let _ = std::fs::remove_dir_all(&dir); + } + #[test] fn a_file_whose_digest_is_wrong_is_never_returned_to_be_run() { let dir = temp_root("digest"); diff --git a/desktop/src-tauri/src/lib.rs b/desktop/src-tauri/src/lib.rs index d7e4b0f442..54565e7f06 100644 --- a/desktop/src-tauri/src/lib.rs +++ b/desktop/src-tauri/src/lib.rs @@ -8,6 +8,7 @@ pub mod deployment; pub mod deployment_release; pub mod docker_config; pub mod docker_desktop; +pub mod early_downloads; pub mod engine; pub mod env; pub mod harness; diff --git a/desktop/src-tauri/src/main.rs b/desktop/src-tauri/src/main.rs index 9c0726767e..d8a90be377 100644 --- a/desktop/src-tauri/src/main.rs +++ b/desktop/src-tauri/src/main.rs @@ -18,8 +18,9 @@ mod zoom; mod test_support; use openbot_desktop_lib::{ - acquire, app_update, deployment, deployment_release, docker_config, engine, env as openbot_env, - harness, host_access, install, local_eval, local_eval_state, logging, preparation, + acquire, app_update, deployment, deployment_release, docker_config, early_downloads, engine, + env as openbot_env, harness, host_access, install, local_eval, local_eval_state, logging, + preparation, problem::{LeftoverReset, Problem}, provider, pull_metrics, quiet, recorder_external, recorder_host, recorder_install, stack, telemetry, tray, troubleshooting, upgrade, windows as win, @@ -543,6 +544,10 @@ fn forget_running_logged() { .unwrap_or_else(|held| held.into_inner()) = None; } +/// Each running step's latest line, so a download added to its row keeps what the row says. +static RUNNING_SAID: Mutex> = + Mutex::new(std::collections::BTreeMap::new()); + fn report_running( app: &tauri::AppHandle, step: &str, @@ -559,6 +564,10 @@ fn report_running_line( line: engine::StepLine, download_bytes: Option, ) { + RUNNING_SAID + .lock() + .unwrap_or_else(|held| held.into_inner()) + .insert(step.to_owned(), line.clone()); let engine::StepLine { said: detail, elsewhere, @@ -641,6 +650,34 @@ fn allow_podman_install() -> bool { /// Install's running image downloads: the step's sentence, the overall bytes as `downloadBytes` /// for anything that reads only that, and the per-image detail. +/// A running step's download, on the row it is already showing: the engine machine image while +/// Install waits for it. The row keeps its line; only the download is added. +fn report_running_downloads( + app: &tauri::AppHandle, + step: &str, + downloads: pull_metrics::Downloads, +) { + let line = RUNNING_SAID + .lock() + .unwrap_or_else(|held| held.into_inner()) + .get(step) + .cloned() + .unwrap_or_else(|| engine::StepLine::here("")); + let _ = app.emit( + "setup:progress", + Progress { + step: step.into(), + ok: true, + detail: line.said, + running: true, + download_bytes: (downloads.bytes > 0).then_some(downloads.bytes), + downloads: Some(downloads), + elsewhere: line.elsewhere, + asks: line.asks, + }, + ); +} + fn report_downloads( app: &tauri::AppHandle, downloads: pull_metrics::Downloads, @@ -1681,6 +1718,30 @@ async fn prepare_engine(app: tauri::AppHandle) -> Result early_downloads::Summary { + early_downloads::summary() +} + /// Complete all local software acquisition before either authentication screen is available. #[tauri::command] async fn prepare_installation( @@ -1781,6 +1842,9 @@ async fn prepare_installation_inner( "Checking which local software to download.", None, ); + // The first screen may have started pulling these already: wait for it, showing its + // download, and the pulls below find them there. + early_downloads::wait_for_images(|downloads| report_downloads(&handle, downloads.clone())); // Only what this installation runs: see `installation_pull_set`. let images = preparation::installation_pull_set(&address, &root, &root, harness.as_ref())?; // Several pulls run at once. Stop is checked before each pull starts; running pulls finish. @@ -2017,7 +2081,24 @@ async fn engine_ready( }, None, ); - let created = acquire::create_machine(4, 6144, 60); + // The machine image the first screen started downloading: waited for with its download shown + // on this row, and the machine built from it. Without one, Podman pulls its own. On a blocking + // thread, as the deployment's download is: a blocking HTTP client must not be dropped here. + let progress_app = app.clone(); + let image = tauri::async_runtime::spawn_blocking(move || { + early_downloads::machine_image_for_init(|downloads| { + report_running_downloads(&progress_app, "create-machine", downloads) + }) + }) + .await + .unwrap_or_else(|error| { + log::warn!("[early downloads] the machine image wait did not run: {error}"); + None + }); + let created = acquire::create_machine(4, 6144, 60, image.as_deref()); + if created.ok && image.is_some() { + early_downloads::forget_machine_image(); + } if cfg!(target_os = "windows") { bring_to_front(app, "the engine machine is prepared"); } @@ -2121,6 +2202,8 @@ async fn deployment_ready( let target = root.to_path_buf(); let handle = app.clone(); let version = tauri::async_runtime::spawn_blocking(move || { + // The first screen may be laying the same deployment out: one at a time. + let _laying_out = early_downloads::hold_deployment(); let version = deployment_release::resolve_version(&target)?; if deployment::needs_fetch(&target, &version) { report_running( @@ -8757,6 +8840,8 @@ fn main() { .plugin(outside_links_guard()) .manage(Shell::default()) .invoke_handler(tauri::generate_handler![ + start_early_downloads, + early_downloads_now, record_setup_event, bring_setup_to_front, allow_podman_install, diff --git a/desktop/src-tauri/src/registry.rs b/desktop/src-tauri/src/registry.rs index b47116c12a..78f95161a4 100644 --- a/desktop/src-tauri/src/registry.rs +++ b/desktop/src-tauri/src/registry.rs @@ -170,7 +170,7 @@ fn read_manifest(body: &[u8], architecture: &str) -> Result, Option Option { +pub(crate) fn token_url(challenge: &str) -> Option { let rest = challenge.trim().strip_prefix("Bearer ")?; let mut realm = None; let mut query = Vec::new(); diff --git a/desktop/src-tauri/src/release_images.rs b/desktop/src-tauri/src/release_images.rs index 498143de44..55144642d1 100644 --- a/desktop/src-tauri/src/release_images.rs +++ b/desktop/src-tauri/src/release_images.rs @@ -15,6 +15,8 @@ it cannot name one image fewer than the app pulls: (`repository:release`, then the index that tag serves). And the cluster's own images, which the host's engine pulls when the CLI has k3d create the cluster (`local_images::cli_cluster_images`): the k3s node image and k3d's proxy and tools, each at the digest the CLI gives k3d. +- The engine machine images on quay.io that the first setup screen downloads before Install, so + Podman builds OpenBot's machine from a local file (`early_downloads::machine_image_references`). The release publishes the list as `desktop-images.json` (`examples/release_images.rs`). */ @@ -124,6 +126,7 @@ pub fn release_images(compose: &Path, manifest: &Path, cli: &str) -> Result Option { /// The bytes free to this account on the volume that holds `folder`. #[cfg(target_os = "windows")] -fn free_space(folder: &str) -> Option { +pub(crate) fn free_space(folder: &str) -> Option { use std::os::windows::ffi::OsStrExt; let folder = folder.strip_prefix(r"\\?\").unwrap_or(folder); let wide: Vec = std::ffi::OsStr::new(folder) diff --git a/desktop/src/App.tsx b/desktop/src/App.tsx index 9047b21cb5..664f1ff29f 100644 --- a/desktop/src/App.tsx +++ b/desktop/src/App.tsx @@ -3,6 +3,7 @@ import { listen } from "@tauri-apps/api/event"; import { useCallback, useEffect, useRef, useState } from "react"; import { CheckForUpdates } from "./CheckForUpdates"; import { CopilotKitAccount } from "./CopilotKitAccount"; +import { EarlyDownloadsLine } from "./EarlyDownloads"; import { useWindowFitsContent } from "./fitWindow"; import { DEFAULT_HARNESS, type HarnessChoice } from "./harness"; import { isHttpEndpointUrl } from "./http-endpoint-url"; @@ -175,6 +176,16 @@ function Setup() { const [running, setRunning] = useState(false); const visibleSetupStep = checkingResume || blockerFailure || blocker || running ? null : step; + /* + * The first screen starts the big downloads, without a click: the person opened OpenBot's setup + * to install it. The native side downloads only what this computer's install will need, once per + * run of the app (`early_downloads.rs`). + */ + const onFirstScreen = visibleSetupStep === "welcome"; + useEffect(() => { + if (!onFirstScreen || !root.trim()) return; + invoke("start_early_downloads", { root: root.trim() }).catch(() => {}); + }, [onFirstScreen, root]); const lastViewedStep = useRef(null); useEffect(() => { if (visibleSetupStep === lastViewedStep.current) return; @@ -812,6 +823,10 @@ function Setup() { {goAhead && } {/* Intelligence sets itself up alongside, once the install has Docker answering. */} + {/* The first screen's downloads, until a row of Install's own shows the one it waits on. */} + {!steps.some((row) => row.running && row.downloads) && ( + + )} {displayedFailure && } {!busy && (
@@ -866,6 +881,8 @@ function Setup() { engine={engineNow(steps)} // Once signed in, the step explains the password prompt Install waits to raise. goAhead={goAhead} + // The downloads the first screen started, while they still run. + downloads={} onDone={(acceptedVersion) => { if (acceptedVersion) local.acceptTerms(acceptedVersion); // Intelligence sets itself up once the install has Docker answering; see above. The diff --git a/desktop/src/CopilotKitAccount.tsx b/desktop/src/CopilotKitAccount.tsx index 77b50e3d0e..6c7b080afe 100644 --- a/desktop/src/CopilotKitAccount.tsx +++ b/desktop/src/CopilotKitAccount.tsx @@ -85,6 +85,7 @@ export function CopilotKitAccount({ handOff = null, engine = null, goAhead = null, + downloads = null, }: { /** The installation folder the apps are connected for. */ root: string; @@ -105,6 +106,8 @@ export function CopilotKitAccount({ * offers Continue. */ goAhead?: string | null; + /** The downloads the first screen started, while they still run behind this step. */ + downloads?: ReactNode; /** Signed in and agreed: the version accepted here, or null when nothing needed accepting. */ onDone: (acceptedVersion: string | null) => void; /** A failure from setup itself, shown inside this card rather than under it. */ @@ -391,6 +394,7 @@ export function CopilotKitAccount({ {seen.at === "failed" && (

{seen.said}

)} + {downloads} {failure}
diff --git a/desktop/src/EarlyDownloads.test.tsx b/desktop/src/EarlyDownloads.test.tsx new file mode 100644 index 0000000000..fd58c398cd --- /dev/null +++ b/desktop/src/EarlyDownloads.test.tsx @@ -0,0 +1,204 @@ +import { afterAll, afterEach, beforeAll, expect, mock, test } from "bun:test"; +import { GlobalRegistrator } from "@happy-dom/global-registrator"; +import { act, cleanup, render } from "@testing-library/react"; +import userEvent from "@testing-library/user-event"; +import type { EarlyDownloads } from "./EarlyDownloads"; + +let invokeCalls: Array<{ command: string; args?: unknown }> = []; +const downloadListeners = new Set< + (event: { payload: EarlyDownloads }) => void +>(); + +/** A first run on a computer with nothing installed: every startup question answered plainly. */ +async function answer(command: string): Promise { + switch (command) { + case "windows_restart_pending": + return false; + case "default_root": + return "/tmp/openbot-early-downloads"; + case "already_configured": + return { + values: {}, + saved: { + modelApiKeys: { openai: false, anthropic: false }, + modelSessions: { openai: false, anthropic: false }, + }, + }; + case "already_running": + case "take_ai_connection_change": + return false; + case "early_downloads_now": + return { state: "idle", bytes: 0 }; + // Install and the sign-in stay in progress: the account step is where the person is. + case "prepare_installation": + case "sign_in_to_copilotkit": + return new Promise(() => {}); + default: + return null; + } +} + +mock.module("@tauri-apps/api/core", () => ({ + invoke: (command: string, args?: unknown) => { + invokeCalls.push({ command, args }); + return answer(command); + }, +})); + +mock.module("@tauri-apps/api/event", () => ({ + listen: async ( + name: string, + listener: (event: { payload: EarlyDownloads }) => void, + ) => { + if (name === "setup:downloads") downloadListeners.add(listener); + return () => downloadListeners.delete(listener); + }, +})); + +mock.module("./Mark", () => ({ + Mark: ({ name }: { name: string }) => {name}, + ButtonMark: () => null, +})); + +const { App } = await import("./App"); +const { earlyDownloadsSaid } = await import("./EarlyDownloads"); + +beforeAll(() => GlobalRegistrator.register()); +afterEach(() => { + cleanup(); + invokeCalls = []; + downloadListeners.clear(); +}); +afterAll(() => GlobalRegistrator.unregister()); + +async function said(payload: EarlyDownloads) { + await act(async () => { + for (const listener of downloadListeners) listener({ payload }); + }); +} + +function called(command: string) { + return invokeCalls.filter((call) => call.command === command); +} + +const GB = 1_000_000_000; + +test("the big downloads start when the first screen shows, before Install is clicked or anyone signs in", async () => { + let view!: ReturnType; + await act(async () => { + view = render(); + }); + await view.findByRole("heading", { name: "Set up OpenBot" }); + + // Nothing was clicked: the first screen alone started them, for the installation's folder. + expect(called("start_early_downloads")).toEqual([ + { + command: "start_early_downloads", + args: { root: "/tmp/openbot-early-downloads" }, + }, + ]); + expect(called("prepare_installation")).toEqual([]); + expect(called("sign_in_to_copilotkit")).toEqual([]); + + // The first screen says how far they are, in plain words. + await said({ + state: "downloading", + bytes: 1.2 * GB, + size: 2.9 * GB, + bytesPerSecond: 10_000_000, + }); + expect(view.getByRole("status").textContent).toBe( + "Downloading OpenBot (1.2 of 2.9 GB, about 3 minutes left).", + ); + const bar = view.getByRole("progressbar", { name: "Download progress" }); + expect(bar.getAttribute("value")).toBe(String(1.2 * GB)); + expect(bar.getAttribute("max")).toBe(String(2.9 * GB)); + + // Install moves on to the CopilotKit sign-in, and the downloads keep saying where they are. + await userEvent.click(view.getByRole("button", { name: "Install" })); + await view.findByRole("heading", { name: /CopilotKit/ }); + await said({ + state: "downloading", + bytes: 2.0 * GB, + size: 2.9 * GB, + bytesPerSecond: 10_000_000, + }); + expect( + view.getByText( + "Downloading OpenBot (2.0 of 2.9 GB, about 2 minutes left).", + ), + ).toBeTruthy(); + // They were started once, by the first screen. + expect(called("start_early_downloads")).toHaveLength(1); +}); + +test("a disk too full to download to is said plainly, with what is free and what is needed", async () => { + let view!: ReturnType; + await act(async () => { + view = render(); + }); + await view.findByRole("heading", { name: "Set up OpenBot" }); + await said({ + state: "short", + bytes: 0, + free: 14.6 * GB, + needed: 44.3 * GB, + }); + const line = view.getByRole("status"); + expect(line.textContent).toBe( + "This computer has 15 GB of free space, and OpenBot needs about 44 GB. Free up some space and the download starts by itself.", + ); + // Nothing is downloading, so nothing spins and there is no bar. + expect(line.querySelector(".step-spinner")).toBeNull(); + expect(view.queryByRole("progressbar")).toBeNull(); +}); + +test("the line says what is known, and only the first screen says when it is all downloaded", () => { + expect(earlyDownloadsSaid({ state: "downloading", bytes: 0 }, true)).toBe( + "Downloading OpenBot.", + ); + expect( + earlyDownloadsSaid({ state: "downloading", bytes: 0.4 * GB }, true), + ).toBe("Downloading OpenBot (0.4 GB so far)."); + expect( + earlyDownloadsSaid( + { + state: "downloading", + bytes: 450_000_000, + size: 937_000_000, + bytesPerSecond: 20_000_000, + }, + false, + ), + ).toBe("Downloading OpenBot (450 of 937 MB, less than a minute left)."); + expect( + earlyDownloadsSaid( + { + state: "downloading", + bytes: GB, + size: 2 * GB, + bytesPerSecond: 12_500_000, + }, + false, + ), + ).toBe("Downloading OpenBot (1.0 of 2.0 GB, about a minute left)."); + expect(earlyDownloadsSaid({ state: "done", bytes: GB, size: GB }, true)).toBe( + "OpenBot is downloaded and ready to install.", + ); + expect( + earlyDownloadsSaid({ state: "done", bytes: GB, size: GB }, false), + ).toBe(null); + expect(earlyDownloadsSaid({ state: "failed", bytes: 0 }, true)).toBe( + "Some downloads didn’t finish. Install tries them again.", + ); + expect(earlyDownloadsSaid({ state: "nothing", bytes: 0 }, true)).toBe(null); + // No em dashes, and nothing technical, in anything it says. + for (const state of ["downloading", "short", "done", "failed"] as const) { + const line = + earlyDownloadsSaid( + { state, bytes: GB, size: 2 * GB, free: GB, needed: 44 * GB }, + true, + ) ?? ""; + expect(line).not.toMatch(/—|Podman|machine|image|quay|engine/i); + } +}); diff --git a/desktop/src/EarlyDownloads.tsx b/desktop/src/EarlyDownloads.tsx new file mode 100644 index 0000000000..14402a62db --- /dev/null +++ b/desktop/src/EarlyDownloads.tsx @@ -0,0 +1,135 @@ +import { invoke } from "@tauri-apps/api/core"; +import { listen } from "@tauri-apps/api/event"; +import { useEffect, useState } from "react"; +import { StatusLine } from "./SetupProgress"; + +/** + * The big downloads that start when the first screen shows, before Install is clicked + * (`early_downloads.rs`): the engine machine image, Podman and Compose, the deployment and + * OpenBot's images. One line says where they are, on every setup screen they run behind. + */ +export type EarlyDownloads = { + state: "idle" | "short" | "downloading" | "done" | "failed" | "nothing"; + /** Downloaded so far, over every download. */ + bytes: number; + /** Every download together, once each one's size is known. */ + size?: number; + bytesPerSecond?: number; + /** Free on this computer's disk, and what setup needs, when the disk is short. */ + free?: number; + needed?: number; +}; + +/** Where the early downloads are, as the native side last said. */ +export function useEarlyDownloads(): EarlyDownloads | null { + const [downloads, setDownloads] = useState(null); + useEffect(() => { + let active = true; + const stop = listen("setup:downloads", (event) => { + if (active) setDownloads(event.payload); + }); + // Downloads that started before this screen was drawn. + invoke("early_downloads_now") + .then((now) => { + if (active) setDownloads((seen) => seen ?? now); + }) + .catch(() => {}); + return () => { + active = false; + stop.then((unlisten) => unlisten()).catch(() => {}); + }; + }, []); + return downloads; +} + +const GB = 1_000_000_000; +const MB = 1_000_000; + +/** "1.2 of 2.9 GB", or "450 of 937 MB": both in the unit of the whole. */ +export function ofSize(bytes: number, size: number): string { + if (size >= GB) + return `${(bytes / GB).toFixed(1)} of ${(size / GB).toFixed(1)} GB`; + return `${Math.round(bytes / MB)} of ${Math.round(size / MB)} MB`; +} + +/** "about 3 minutes left", from what is left and the speed it is arriving at. */ +export function timeLeft(seconds: number): string { + if (seconds < 60) return "less than a minute left"; + if (seconds < 90) return "about a minute left"; + return `about ${Math.round(seconds / 60)} minutes left`; +} + +function gigabytes(bytes: number): string { + return `${Math.max(1, Math.round(bytes / GB))} GB`; +} + +/** + * The line for `downloads`, or null when there is nothing to say. The first screen also says when + * everything is downloaded, or when something will download at Install instead; later screens say + * only what is still going on, since Install's own rows take over from there. + */ +export function earlyDownloadsSaid( + downloads: EarlyDownloads, + firstScreen: boolean, +): string | null { + const { state, bytes, size, bytesPerSecond, free, needed } = downloads; + switch (state) { + case "downloading": { + if (size == null || size <= 0) { + return bytes > 0 + ? `Downloading OpenBot (${(bytes / GB).toFixed(1)} GB so far).` + : "Downloading OpenBot."; + } + const parts = [ofSize(Math.min(bytes, size), size)]; + if (bytesPerSecond != null && bytesPerSecond > 0 && bytes < size) { + parts.push(timeLeft((size - bytes) / bytesPerSecond)); + } + return `Downloading OpenBot (${parts.join(", ")}).`; + } + case "short": + return free != null && needed != null + ? `This computer has ${gigabytes(free)} of free space, and OpenBot needs about ${gigabytes(needed)}. Free up some space and the download starts by itself.` + : null; + case "done": + return firstScreen ? "OpenBot is downloaded and ready to install." : null; + case "failed": + return firstScreen + ? "Some downloads didn’t finish. Install tries them again." + : null; + default: + return null; + } +} + +/** The early downloads' line, with a bar while they download. Nothing when there is nothing to say. */ +export function EarlyDownloadsLine({ + firstScreen = false, +}: { + /** The Welcome screen, where the downloads start. */ + firstScreen?: boolean; +}) { + const downloads = useEarlyDownloads(); + if (!downloads) return null; + const said = earlyDownloadsSaid(downloads, firstScreen); + if (!said) return null; + const downloading = downloads.state === "downloading"; + const known = downloading && downloads.size != null && downloads.size > 0; + return ( +
+ {said} + {downloading && ( +
+ {known ? ( + + ) : ( + + )} +
+ )} +
+ ); +} diff --git a/desktop/src/Welcome.tsx b/desktop/src/Welcome.tsx index 361855379b..af2a65da80 100644 --- a/desktop/src/Welcome.tsx +++ b/desktop/src/Welcome.tsx @@ -1,4 +1,5 @@ import type { ReactNode } from "react"; +import { EarlyDownloadsLine } from "./EarlyDownloads"; import { ExternalLink } from "./ExternalLink"; /** What OpenBot Desktop sends, and what it never does, in full. */ @@ -15,6 +16,9 @@ export const USAGE_DATA_DOC = * * The usage-data notice is here because this is the first screen, and the CopilotKit sign-in that * links the data to an account is the next. Settings > General turns it off. + * + * The big downloads start while this screen shows, before Install is clicked, and its line says how + * far they are (`EarlyDownloadsLine`). */ export function Welcome({ onInstall, @@ -40,6 +44,7 @@ export function Welcome({ Install
+

OpenBot sends CopilotKit usage data, linked to your CopilotKit account. It never sends your conversations, files or keys. You can turn this off diff --git a/desktop/src/styles.css b/desktop/src/styles.css index e609b3b10c..911e6df912 100644 --- a/desktop/src/styles.css +++ b/desktop/src/styles.css @@ -1406,6 +1406,12 @@ a:hover { flex: none; } +/* The downloads the first screen started: their line, and its bar under it. */ +.early-downloads .downloads { + display: grid; + margin-top: 8px; +} + /* The Status window: one card per part, each with its memory and its actions. */ .status-part { border: 1px solid var(--border); diff --git a/docs/releasing.md b/docs/releasing.md index 51392e24f7..bccd7500f8 100644 --- a/docs/releasing.md +++ b/docs/releasing.md @@ -629,7 +629,10 @@ it: they hash what they receive. A key that is not there answers 403, not 404 (C Docker Hub's images are copied the same way under `v2/docker.io//`, official images under `library/` (`v2/docker.io/library/busybox`), so a client uses `downloads.copilotkit.ai/docker.io` as -the mirror for `docker.io`. Images on any other registry are left where they are. +the mirror for `docker.io`. quay.io's are copied under `v2/quay.io//`: the Podman engine machine +images, which the app downloads itself from the first setup screen as plain blobs +(`v2/quay.io/podman/machine-os/blobs/`, then quay.io), and builds the machine from +(`desktop/src-tauri/src/early_downloads.rs`). Images on any other registry are left where they are. Which images: every image the release's `container-images.json` pins, all of them, and every image `desktop-images.json` on the release names. The `image list` job of the publish workflow writes that @@ -639,7 +642,9 @@ list with the app's own code (`desktop/src-tauri/src/release_images.rs`, run as every Bot the app can install (`pgvector` included), the two sign-in images, the bundled CLI's back end images (its runtime images at its release's tag, its pinned ones and the Kubernetes system images), and the cluster's own images that the host's engine pulls when k3d creates the cluster (the -k3s node image and k3d's proxy and tools, at the digests the CLI gives k3d). A release published +k3s node image and k3d's proxy and tools, at the digests the CLI gives k3d), and the engine machine +images Podman's machine is built from (`quay.io/podman/machine-os`, by the digests +`early_downloads.rs` pins). A release published before this carries no list, and only its `container-images.json` images are copied; `DOWNLOADS_IMAGES_FILE` names a list to use instead. Images are read with [crane](https://github.com/google/go-containerregistry) (a pinned release, checked by diff --git a/scripts/downloads-images.test.ts b/scripts/downloads-images.test.ts index 0d0a4570d6..0fc9c344b6 100644 --- a/scripts/downloads-images.test.ts +++ b/scripts/downloads-images.test.ts @@ -268,6 +268,8 @@ test("a release's images are copied under the registry's paths, byte for byte an const pgvector = push(fake, "pgvector/pgvector", "pg17", "docker.io"); const busybox = push(fake, "library/busybox", "1.37", "docker.io"); const k3s = push(fake, "rancher/k3s", "v1.32.5-k3s1", "docker.io"); + // quay.io's: the engine machine image the app downloads before Install, named by digest. + const machineOs = push(fake, "podman/machine-os", "6.1", "quay.io"); const list = { openbot: "v0.3.4", intelligence: "0.16.0-rc.9", @@ -277,7 +279,8 @@ test("a release's images are copied under the registry's paths, byte for byte an "ghcr.io/copilotkit/intelligence/app-api:0.16.0-rc.9", `ghcr.io/copilotkit/openbot-server@${server.digest}`, "pgvector/pgvector:pg17", - "quay.io/someone/else:1", + `quay.io/podman/machine-os@${machineOs.digest}`, + "mcr.microsoft.com/someone/else:1", ], }; // Every image the release pins is copied, listed in desktop-images.json or not. @@ -312,7 +315,7 @@ test("a release's images are copied under the registry's paths, byte for byte an // Each image's top manifest, every manifest its index names and the referrers of each are asked // for through the address before the run succeeds. expect(first.out).toContain( - `All 6 images for v0.3.4 answer through ${edge.url}, every manifest and referrers object included.`, + `All 7 images for v0.3.4 answer through ${edge.url}, every manifest and referrers object included.`, ); expect(edge.requests).toContain( `GET v2/copilotkit/openbot-supervisor/manifests/${supervisor.digest}`, @@ -349,6 +352,8 @@ test("a release's images are copied under the registry's paths, byte for byte an ...manifests("docker.io/library/busybox", busybox), "v2/docker.io/library/busybox/manifests/1.37", ...manifests("docker.io/rancher/k3s", k3s), + // quay.io's under quay.io/. + ...manifests("quay.io/podman/machine-os", machineOs), // Named only by container-images.json. ...manifests("copilotkit/openbot-supervisor", supervisor), ]) { @@ -372,18 +377,18 @@ test("a release's images are copied under the registry's paths, byte for byte an cacheControl: TAG, sha256: true, }); - // Six images, each an index and four manifests (three platforms and an attestation), and three + // Seven images, each an index and four manifests (three platforms and an attestation), and three // tags; a referrers object for each manifest. expect( Object.keys(bucket).filter((key) => key.includes("/manifests/")), - ).toHaveLength(33); + ).toHaveLength(38); expect( Object.keys(bucket).filter((key) => key.includes("/referrers/")), - ).toHaveLength(30); + ).toHaveLength(35); // A config and two layers for each of the four manifests, an image. expect( Object.keys(bucket).filter((key) => key.includes("/blobs/")), - ).toHaveLength(72); + ).toHaveLength(84); expect(bucket["v2/"]).toEqual({ content: "{}", contentType: "application/json", @@ -441,12 +446,12 @@ test("a release's images are copied under the registry's paths, byte for byte an expect( bucket["tools/compose/v5.5.1/docker-compose-darwin-aarch64"]?.content, ).toBe("compose arm64"); - // Only ghcr.io and Docker Hub are copied. + // Only ghcr.io, Docker Hub and quay.io are copied. expect( - fake.calls().some((call) => call.startsWith("crane") && /quay/.test(call)), + fake.calls().some((call) => call.startsWith("crane") && /mcr/.test(call)), ).toBe(false); expect(first.out).toContain( - "quay.io/someone/else:1 is not on ghcr.io or Docker Hub; it is left to its own registry.", + "mcr.microsoft.com/someone/else:1 is not on ghcr.io, Docker Hub or quay.io; it is left to its own registry.", ); // The tags and the ping are invalidated together, after the last upload, and every new object // fetched once through the address. @@ -531,7 +536,7 @@ test("a release's images are copied under the registry's paths, byte for byte an const refused = await run(fake, env); expect(refused.code).toBe(1); expect(refused.out).toContain( - `::error::1 of the objects for the 6 images for v0.3.4 do not answer through ${edge.url}: ${edge.url}/${stale} (403)`, + `::error::1 of the objects for the 7 images for v0.3.4 do not answer through ${edge.url}: ${edge.url}/${stale} (403)`, ); // The check alone, as a person runs it after a release: no AWS, no registry. const checked = (bucketless: Record) => @@ -550,7 +555,7 @@ test("a release's images are copied under the registry's paths, byte for byte an const whole = await checked({}); expect(whole.code).toBe(0); expect(whole.out).toContain( - `All 6 images for v0.3.4 answer through ${edge.url}, every manifest and referrers object included.`, + `All 7 images for v0.3.4 answer through ${edge.url}, every manifest and referrers object included.`, ); expect(fake.calls().filter((call) => /^(aws|crane) /.test(call)).length).toBe( awsBefore, diff --git a/scripts/downloads-images.ts b/scripts/downloads-images.ts index 6d42f1b519..a0358c3c21 100644 --- a/scripts/downloads-images.ts +++ b/scripts/downloads-images.ts @@ -6,8 +6,9 @@ * Run by .github/workflows/downloads-sync.yml after scripts/downloads-sync.sh, for the same * release, and by hand. The release's `desktop-images.json` says which images: the Desktop app's * own code writes it when the release is published (desktop/src-tauri/src/release_images.rs), so - * it names exactly what the app pulls. Images on ghcr.io and Docker Hub are copied; any other registry is - * left as it is. + * it names exactly what the app pulls. Images on ghcr.io, Docker Hub and quay.io are copied; any other + * registry is left as it is. quay.io holds the engine machine images Podman builds its machine from, + * which the app downloads before Install (desktop/src-tauri/src/early_downloads.rs). * * What it writes, in the bucket behind the CloudFront distribution: a read-only copy of the * registry's own paths (the OCI Distribution API, GET only), so the address works as a registry: @@ -21,8 +22,9 @@ * v2//referrers/sha256: an empty image index for each of those manifests, so the * referrers API Docker 29 asks with every pull answers 200; * cached for five minutes - * is the path on ghcr.io unchanged, such as copilotkit/openbot-server, or for Docker Hub - * docker.io/ and its path, official images under library/ (docker.io/library/busybox). The whole + * is the path on ghcr.io unchanged, such as copilotkit/openbot-server, for Docker Hub + * docker.io/ and its path, official images under library/ (docker.io/library/busybox), and for + * quay.io quay.io/ and its path (quay.io/podman/machine-os). The whole * index is copied because Docker 29 fetches every manifest it names, attestations included, and a * 403 for one fails the pull. An image named by digest whose referrers are there already is whole, * and is not read again. @@ -117,7 +119,8 @@ export function imageRef( const hosted = first.includes(".") || first.includes(":") || first === "localhost"; const host = hosted ? first : "docker.io"; - if (host !== "ghcr.io" && !DOCKER_HUB.has(host)) return undefined; + if (host !== "ghcr.io" && host !== "quay.io" && !DOCKER_HUB.has(host)) + return undefined; let rest = hosted ? reference.slice(slash + 1) : reference; let digest: string | undefined; const at = rest.indexOf("@"); @@ -142,14 +145,20 @@ export function imageRef( const where = host === "ghcr.io" ? { name: `ghcr.io/${rest}`, path: rest, read: `${source}/${rest}` } - : (() => { - const repo = rest.includes("/") ? rest : `library/${rest}`; - return { - name: `docker.io/${repo}`, - path: `docker.io/${repo}`, - read: `docker.io/${repo}`, - }; - })(); + : host === "quay.io" + ? { + name: `quay.io/${rest}`, + path: `quay.io/${rest}`, + read: `quay.io/${rest}`, + } + : (() => { + const repo = rest.includes("/") ? rest : `library/${rest}`; + return { + name: `docker.io/${repo}`, + path: `docker.io/${repo}`, + read: `docker.io/${repo}`, + }; + })(); // A digest names the image exactly; a tag beside it is only a label, as it is to a client. return digest ? { ...where, digest } : { ...where, tag }; } @@ -702,7 +711,7 @@ async function copy(options: Copy): Promise { } for (const image of elsewhere) console.log( - `${image} is not on ghcr.io or Docker Hub; it is left to its own registry.`, + `${image} is not on ghcr.io, Docker Hub or quay.io; it is left to its own registry.`, ); // What S3 holds for a key: its SHA-256 as S3 computed it on an upload sent with one, its From c9178cbb90975181479e1c42ecd02bed70429cdd Mon Sep 17 00:00:00 2001 From: David McKay Date: Mon, 9 Nov 2026 16:29:33 -0800 Subject: [PATCH 2/5] Say what setup is doing in plain words, and what's installing behind each step The setup window named Podman, the engine machine, WSL and the back end's versions while it waited, and a tester on the first step could not tell whether setup was stuck. Each line now says what OpenBot is doing in plain words. The native side sends two registers per line: the window shows the plain one, and the Desktop log keeps the technical detail beside it, with the back end's releases and the step an update stopped at. The Account and AI steps show one row for the install running behind them: what it is doing, step N of 4, the time so far, and the time left while it downloads. It sits under the step's form in a slot of its own height: on Windows the back end's download card appeared above the AI step's endpoint form while the person typed, the fields jumped about 100px and the window grew and moved. A real-browser test holds the form and the page still as the download comes and goes. --- .github/workflows/ci.yml | 8 + CHANGELOG.md | 26 +- desktop/src-tauri/src/engine.rs | 67 +++ desktop/src-tauri/src/install.rs | 152 +++++-- desktop/src-tauri/src/main.rs | 109 +++-- desktop/src/App.test.tsx | 556 +++++++++++++++++++++--- desktop/src/App.tsx | 64 ++- desktop/src/CopilotKitAccount.test.tsx | 14 +- desktop/src/CopilotKitAccount.tsx | 43 +- desktop/src/LocalIntelligence.test.tsx | 83 ++-- desktop/src/LocalIntelligence.tsx | 123 ++++-- desktop/src/PodmanGoAhead.tsx | 14 +- desktop/src/ProviderPicker.tsx | 4 +- desktop/src/SetupProgress.test.tsx | 26 +- desktop/src/SetupProgress.tsx | 215 ++++++++- desktop/src/Welcome.tsx | 4 +- desktop/src/localEvalCli.ts | 52 ++- desktop/src/localReadiness.ts | 8 +- desktop/src/setupLayout.browser.test.ts | 280 ++++++++++++ desktop/src/styles.css | 32 ++ 20 files changed, 1588 insertions(+), 292 deletions(-) create mode 100644 desktop/src/setupLayout.browser.test.ts diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index 83f4ca5151..7a47cf7d6e 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -168,6 +168,14 @@ jobs: bun test app/tests/tool-line-reduced-motion.test.tsx app/tests/onboarding-window-buttons.test.tsx app/tests/onboarding-orb-room.test.tsx app/tests/transcript-rail-clearance.test.tsx app/tests/a2ui-frame.test.tsx env: OPENBOT_APP_BROWSER: '1' + # The desktop setup window's, for the same reason: the AI step's form holding still while the + # install behind it comes and goes. It bundles the window from the desktop's own dependencies. + - run: bun install --frozen-lockfile + working-directory: desktop + - run: bun test src/setupLayout.browser.test.ts + working-directory: desktop + env: + OPENBOT_APP_BROWSER: '1' # Headed is the default for hosted computers, so it is tested as one: the computer starting its # own Xvfb, and the signing again with cross-site frames in processes of their own. - run: bun test tests/headed-browser.test.ts diff --git a/CHANGELOG.md b/CHANGELOG.md index 35850aeb8a..c17919597b 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -10,24 +10,34 @@ Newest first. `Unreleased` is what is on `main` and not yet tagged. ### OpenBot Desktop +- **Setup says what it's doing in plain words, and what's installing behind every step.** The + setup window said "Preparing the engine machine", "Installing Podman", "Waiting for the back end" + and "Updating the back end to 0.16.0-rc.10", and a tester waiting on the first step couldn't tell + whether it was stuck. Each line now says what OpenBot is doing for you ("Preparing this computer + to run OpenBot. This takes a few minutes the first time.", "Getting OpenBot ready", "Updating + OpenBot"), and the Desktop log keeps the technical detail beside it. The Account and AI steps + show one row for the install running behind them: what it's doing, step N of 4, the time so far, + and the time left while it downloads. That row has its own slot under the step's form, so the AI + step's fields no longer jump when a download starts or ends, and the window no longer grows or + moves. - **Sign-in comes first, and macOS asks for your password only after OpenBot says why.** On a new Mac the Account step waited for Install to put Podman in before any browser sign-in opened, and the "osascript wants to make changes" password box came up by itself over it. The CopilotKit sign-in now opens in your browser as soon as the Account step opens: the CLI's sign-in needs no engine, and only the license agreement waits for one. Podman downloads meanwhile. Once you're - signed in, the step says "OpenBot installs Podman, which runs OpenBot's back end on this - computer. macOS asks for your password once to install it." and the password box comes up only - when you choose Continue. It now says "OpenBot needs your password to install Podman, which runs - OpenBot's back end." Closing it without a password offers Continue again instead of stopping + signed in, the step says "OpenBot needs one more piece of software to run on this computer. + macOS asks for your password once to install it." and the password box comes up only when you + choose Continue. It now says "OpenBot needs your password to install the software it runs on." + Closing it without a password offers Continue again instead of stopping Install. Windows and Linux install as before. - **A back end update that was stopped partway no longer breaks every later Start.** Quitting while the back end updated could leave it half updated, and every Start after that failed with "Local update-quiesce failed (exit 1)". OpenBot now lets an update finish once it has begun - changing the back end ("Finishing the back end update" on the Shutting down screen), repairs a + changing the back end ("Finishing the OpenBot update" on the Shutting down screen), repairs a back end an earlier update left half done, rolls back one Helm left mid-upgrade, uncordons a - back end node left unschedulable, and tries the update once more. While it runs, the row names - the step it is waiting on and for how long, and a failed database job is said as soon as it - fails. A failed update says the step it stopped at, and that your data is kept. + back end node left unschedulable, and tries the update once more. While it runs, the row says + what the step it is waiting on is doing and for how long, and a failed part of the update is said + as soon as it fails. A failed update says where it stopped, and that your data is kept. - **The Account step says when it is waiting for Install to set up the engine.** On a new Mac it said "Checking your CopilotKit account…" for minutes while Install put Podman in, because the license agreement check needs a running engine. It now says "You're signed in. OpenBot is setting diff --git a/desktop/src-tauri/src/engine.rs b/desktop/src-tauri/src/engine.rs index a9385f7210..42d4bd450c 100644 --- a/desktop/src-tauri/src/engine.rs +++ b/desktop/src-tauri/src/engine.rs @@ -47,11 +47,17 @@ use serde::{Deserialize, Serialize}; /// /// `asks` when the step waits for the person's go-ahead in OpenBot's own window before it raises /// such a prompt (`install::GoAhead`). The window explains what comes next, with one button. +/// +/// `said` is what the person reads, in plain words: what OpenBot is doing for them. `detail` is the +/// same moment in the log's words, with the names a person never needs on screen (Podman, the +/// engine machine, WSL). The window shows only `said`; the log keeps both (`log_entry`). A tester +/// read "Preparing the engine machine" for minutes and could not tell whether setup was stuck. #[derive(Clone, Debug, PartialEq, Eq)] pub struct StepLine { pub said: String, pub elsewhere: bool, pub asks: bool, + pub detail: Option, } impl StepLine { @@ -61,6 +67,7 @@ impl StepLine { said: said.into(), elsewhere: false, asks: false, + detail: None, } } @@ -70,6 +77,7 @@ impl StepLine { said: said.into(), elsewhere: true, asks: false, + detail: None, } } @@ -79,10 +87,69 @@ impl StepLine { said: said.into(), elsewhere: false, asks: true, + detail: None, + } + } + + /// The technical words for the log, beside the plain line the window shows. + pub fn logged(mut self, detail: impl Into) -> Self { + self.detail = Some(detail.into()); + self + } + + /// What the Desktop log writes for this line: the technical detail when there is one, then + /// what the window said, so the log reads as the window did and loses nothing. + pub fn log_entry(&self) -> String { + match &self.detail { + Some(detail) => format!("{detail} (window: {})", self.said), + None => self.said.clone(), } } } +/// Preparing the machine OpenBot's containers run in (`podman machine init`). Minutes the first +/// time: it downloads the machine's disk. +/// +/// On Windows, Podman's own installer for WSL comes first when WSL is missing: its dialog, then +/// Windows' administrator prompt, then a restart. Said before it happens, on whichever screen the +/// person is on: those windows come to the front over OpenBot's. +pub fn preparing_machine_line(os: &str) -> StepLine { + if os == "windows" { + StepLine::elsewhere( + "Preparing this computer to run OpenBot. If a window asks to install a Windows \ + feature OpenBot needs, allow it. Windows then asks for your approval and a restart.", + ) + .logged( + "Preparing the engine machine (podman machine init). If Windows Subsystem for Linux \ + is missing, Podman offers to install it. Windows then asks for administrator approval \ + and a restart.", + ) + } else { + StepLine::here( + "Preparing this computer to run OpenBot. This takes a few minutes the first time.", + ) + .logged("Preparing the engine machine (podman machine init).") + } +} + +/// Starting that machine (`podman machine start`): at install, and on a start after a restart. +pub fn starting_machine_line() -> StepLine { + StepLine::here("Starting the software OpenBot runs on.") + .logged("Starting the engine machine (podman machine start).") +} + +/// Waiting for the engine to answer once its machine is up. +pub fn checking_engine_line() -> StepLine { + StepLine::here("Checking that the software OpenBot runs on is ready.") + .logged("Waiting for the engine to answer.") +} + +/// The engine answers: what the row says once it is done. +pub fn engine_ready_line() -> StepLine { + StepLine::here("The software OpenBot runs on is ready.") + .logged("The container engine is answering.") +} + /// Which engine is in use, because the answer changes what is mounted and what is reported. #[derive(Clone, Copy, Debug, PartialEq, Eq, Serialize, Deserialize)] #[serde(rename_all = "kebab-case")] diff --git a/desktop/src-tauri/src/install.rs b/desktop/src-tauri/src/install.rs index 1e9b6719e3..8a6670bad6 100644 --- a/desktop/src-tauri/src/install.rs +++ b/desktop/src-tauri/src/install.rs @@ -37,7 +37,7 @@ //! to want one: it is the same dialog every other installer raises. It is never raised unexplained: //! the package downloads while the person signs in, then the step waits (`GoAhead`) while OpenBot's //! window says what comes next and why, and raises the prompt only once they choose Continue. The -//! prompt itself names OpenBot and Podman (`PODMAN_PASSWORD_PROMPT`). +//! prompt itself names OpenBot and says why, in plain words (`PODMAN_PASSWORD_PROMPT`). //! - **Linux.** Podman there is not a binary but a set of them (`conmon`, `crun`, `netavark`, //! `slirp4netns`), wired to the distribution's own paths, so downloading one file would produce //! something that runs nothing. The distribution's package manager installs it, through @@ -262,19 +262,32 @@ pub fn install_engine_observed( ) } +/* + * The person reads "the software OpenBot runs on", never "Podman": a person setting OpenBot up + * does not know what Podman is, and does not need to. The log names it (`StepLine::logged`). + */ + /// The step row's line while Podman's package downloads, before anything needs the person. -pub const PODMAN_DOWNLOADING: &str = "Downloading Podman."; +pub fn podman_downloading() -> StepLine { + StepLine::here("Downloading the software OpenBot runs on.").logged("Downloading Podman.") +} /// The step row's line while it waits for the person's go-ahead to install Podman. The window /// shows its own explanation and Continue beside it. -pub const PODMAN_GO_AHEAD: &str = "Podman is downloaded and ready to install."; +pub fn podman_go_ahead() -> StepLine { + StepLine::asks("The software OpenBot runs on is downloaded and ready to install.") + .logged("Podman is downloaded and ready to install; waiting for Continue.") +} /// The same wait, after the person closed macOS's password prompt without a password. -pub const PODMAN_DECLINED: &str = "macOS didn't get your password, so Podman isn't installed yet."; +pub fn podman_declined() -> StepLine { + StepLine::asks("macOS didn't get your password, so the software isn't installed yet.") + .logged("The password prompt was closed, so Podman isn't installed yet; asking again.") +} /// What macOS's password prompt says, under the name of the process that raised it. pub const PODMAN_PASSWORD_PROMPT: &str = - "OpenBot needs your password to install Podman, which runs OpenBot's back end."; + "OpenBot needs your password to install the software it runs on."; /** The person's go-ahead for a step that raises the system's password prompt. @@ -340,8 +353,10 @@ impl GoAhead { } if cancelled() { *state = Asked::Nothing; - return Err(Problem::plain( - "OpenBot stopped before installing Podman. Start again when you're ready.", + return Err(Problem::with( + "OpenBot stopped before installing the software it runs on. Start again when \ + you're ready.", + "stopped while waiting for Continue to install Podman", )); } state = self @@ -387,19 +402,19 @@ fn install_after_go_ahead( mut prompt: impl FnMut(&str) -> Prompted, back: &dyn Fn(), ) -> Result<(), Problem> { - progress(StepLine::here(PODMAN_DOWNLOADING)); + progress(podman_downloading()); let package = fetch()?; let script = podman_install_script(&package); - let mut asking = PODMAN_GO_AHEAD; + let mut asking = podman_go_ahead(); loop { - gate.wait(|| progress(StepLine::asks(asking)), cancelled)?; + gate.wait(|| progress(asking.clone()), cancelled)?; progress(podman_install_said("macos")); let prompted = prompt(&script); // The prompt was in front of OpenBot's window, however it ended. back(); match prompted { Prompted::Installed => return Ok(()), - Prompted::Declined => asking = PODMAN_DECLINED, + Prompted::Declined => asking = podman_declined(), Prompted::Failed(problem) => return Err(problem), } } @@ -428,15 +443,21 @@ fn applescript_string(text: &str) -> String { /// macOS asks for the person's password, because the package writes to `/opt/podman`. Windows /// asks for nothing here: the MSI installs for this account alone. Linux asks through `pkexec`. /// A prompt is a window of the system's own, in front of OpenBot's, so its line is `elsewhere`. -fn podman_install_said(os: &str) -> StepLine { +pub fn podman_install_said(os: &str) -> StepLine { match os { "macos" => StepLine::elsewhere( - "Installing Podman. Enter your Mac password when macOS asks for it.", - ), + "Installing the software OpenBot runs on. Enter your Mac password when macOS asks \ + for it.", + ) + .logged("Installing Podman's package; macOS asks for the password."), "linux" => StepLine::elsewhere( - "Installing Podman. Enter your password when your system asks for it.", - ), - _ => StepLine::here("Installing Podman."), + "Installing the software OpenBot runs on. Enter your password when your system asks \ + for it.", + ) + .logged("Installing Podman with pkexec; the system asks for the password."), + _ => { + StepLine::here("Installing the software OpenBot runs on.").logged("Installing Podman.") + } } } @@ -1479,10 +1500,76 @@ esac assert!(line.said.contains("password"), "{os}"); assert!(line.elsewhere, "{os}"); } - assert_eq!( + let windows = podman_install_said("windows"); + assert_eq!(windows.said, "Installing the software OpenBot runs on."); + assert!(!windows.elsewhere); + } + + /// Words a person setting OpenBot up doesn't know. The screen never says them; the log does. + const TECHNICAL: &[&str] = &[ + "Podman", + "WSL", + "Windows Subsystem for Linux", + "engine machine", + "engine", + "machine", + "k3s", + "Helm", + "Compose", + ]; + + /// A tester read "Preparing the engine machine" and "Installing Podman" for minutes and could + /// not tell whether setup was stuck. Every line setup shows while it prepares this computer is + /// in plain words; the log keeps the technical names beside it. + #[test] + fn every_line_shown_while_preparing_this_computer_is_plain_and_the_log_keeps_the_detail() { + let lines = [ + podman_downloading(), + podman_go_ahead(), + podman_declined(), + podman_install_said("macos"), + podman_install_said("linux"), podman_install_said("windows"), - StepLine::here("Installing Podman.") - ); + engine::preparing_machine_line("macos"), + engine::preparing_machine_line("windows"), + engine::preparing_machine_line("linux"), + engine::starting_machine_line(), + engine::checking_engine_line(), + engine::engine_ready_line(), + ]; + let shown = lines + .iter() + .map(|line| line.said.as_str()) + .chain([PODMAN_PASSWORD_PROMPT]); + for said in shown { + for word in TECHNICAL { + assert!( + !said.to_lowercase().contains(&word.to_lowercase()), + "{word:?} is on screen: {said:?}" + ); + } + } + for line in &lines { + let detail = line.detail.as_deref().expect("the log keeps the detail"); + let entry = line.log_entry(); + assert!(entry.contains(detail), "{entry}"); + assert!(entry.contains(&line.said), "{entry}"); + } + let log = lines + .iter() + .map(StepLine::log_entry) + .collect::>() + .join("\n"); + for kept in [ + "Podman", + "engine machine", + "Windows Subsystem for Linux", + "podman machine init", + "podman machine start", + "Waiting for the engine to answer.", + ] { + assert!(log.contains(kept), "{kept:?} is not in the log:\n{log}"); + } } /// Where Podman's package lands: the app's cache, which has a space in its path on macOS. @@ -1570,10 +1657,11 @@ esac assert_eq!( seen.all(), [ - "here: Downloading Podman.", + "here: Downloading the software OpenBot runs on.", "downloaded", - "asks: Podman is downloaded and ready to install.", - "elsewhere: Installing Podman. Enter your Mac password when macOS asks for it.", + "asks: The software OpenBot runs on is downloaded and ready to install.", + "elsewhere: Installing the software OpenBot runs on. Enter your Mac password when \ + macOS asks for it.", "prompt", "back", ] @@ -1583,8 +1671,7 @@ esac [concat!( r#"do shell script "/usr/sbin/installer -pkg '/Users/a b/Library/Caches/OpenBot/"#, r#"podman-installer-macos-arm64.pkg' -target /" with administrator privileges "#, - r#"with prompt "OpenBot needs your password to install Podman, which runs "#, - r#"OpenBot's back end.""# + r#"with prompt "OpenBot needs your password to install the software it runs on.""# )] ); } @@ -1619,13 +1706,15 @@ esac assert_eq!( seen.all(), [ - "here: Downloading Podman.", - "asks: Podman is downloaded and ready to install.", - "elsewhere: Installing Podman. Enter your Mac password when macOS asks for it.", + "here: Downloading the software OpenBot runs on.", + "asks: The software OpenBot runs on is downloaded and ready to install.", + "elsewhere: Installing the software OpenBot runs on. Enter your Mac password when \ + macOS asks for it.", "prompt", "back", - "asks: macOS didn't get your password, so Podman isn't installed yet.", - "elsewhere: Installing Podman. Enter your Mac password when macOS asks for it.", + "asks: macOS didn't get your password, so the software isn't installed yet.", + "elsewhere: Installing the software OpenBot runs on. Enter your Mac password when \ + macOS asks for it.", "prompt", "back", ] @@ -1655,7 +1744,8 @@ esac .unwrap_err(); assert_eq!( failed.said, - "OpenBot stopped before installing Podman. Start again when you're ready." + "OpenBot stopped before installing the software it runs on. Start again when you're \ + ready." ); assert!( !seen.all().contains(&"prompt".to_owned()), diff --git a/desktop/src-tauri/src/main.rs b/desktop/src-tauri/src/main.rs index 9c0726767e..d1b73ab85e 100644 --- a/desktop/src-tauri/src/main.rs +++ b/desktop/src-tauri/src/main.rs @@ -398,10 +398,10 @@ async fn wake_installation_engine( }, || acquire::machine_exists().unwrap_or(false), || { - report_running( + report_running_line( &progress_app, "start-machine", - "Starting the engine machine.", + engine::starting_machine_line(), None, ); let started = acquire::start_machine(); @@ -499,12 +499,23 @@ fn report( ok: bool, detail: impl Into, ) { - let detail = detail.into(); + report_line(app, step, ok, engine::StepLine::here(detail)); +} + +/// A finished step's line, said plainly in the window, with its technical detail in the log only. +fn report_line( + app: &tauri::AppHandle, + step: &str, + ok: bool, + line: engine::StepLine, +) { + let entry = line.log_entry(); + let detail = line.said; // Every row setup shows goes in the log too, so the file reads as the window did. if ok { - log::info!("[step {step}] {detail}"); + log::info!("[step {step}] {entry}"); } else { - log::error!("[step {step}] failed: {detail}"); + log::error!("[step {step}] failed: {entry}"); } forget_running_logged(); if !ok { @@ -559,18 +570,21 @@ fn report_running_line( line: engine::StepLine, download_bytes: Option, ) { + // The window shows the plain line; the log keeps the technical detail beside it. + let entry = line.log_entry(); let engine::StepLine { said: detail, elsewhere, asks, + detail: _, } = line; { let mut logged = RUNNING_LOGGED .lock() .unwrap_or_else(|held| held.into_inner()); - let now = (step.to_owned(), detail.clone()); + let now = (step.to_owned(), entry.clone()); if logged.as_ref() != Some(&now) { - log::info!("[step {step}] {detail}"); + log::info!("[step {step}] {entry}"); *logged = Some(now); } } @@ -2001,20 +2015,11 @@ async fn engine_ready( // One at a time, and each only if the last one worked. Written as a loop over an array once, // which ran all three before the first was checked: a failed `machine init` was still followed // by `machine start`. + // On Windows, Podman's own installer for WSL may come first: see `preparing_machine_line`. report_running_line( app, "create-machine", - if cfg!(target_os = "windows") { - // Podman's own installer for WSL, when WSL is missing: its dialog, then Windows' own - // administrator prompt, then a restart. Said before it happens rather than after, and - // on whichever screen the person is on: those windows come to the front over OpenBot's. - engine::StepLine::elsewhere( - "Preparing the engine machine. If Windows Subsystem for Linux is missing, Podman \ - offers to install it. Windows then asks for administrator approval and a restart.", - ) - } else { - engine::StepLine::here("Preparing the engine machine.") - }, + engine::preparing_machine_line(std::env::consts::OS), None, ); let created = acquire::create_machine(4, 6144, 60); @@ -2026,19 +2031,14 @@ async fn engine_ready( return Err(created.problem()); } - report_running(app, "start-machine", "Starting the engine machine.", None); + report_running_line(app, "start-machine", engine::starting_machine_line(), None); let started = acquire::start_machine(); report(app, "start-machine", started.ok, started.said.clone()); if !started.ok { return Err(started.problem()); } - report_running( - app, - "health-gate", - "Waiting for the engine to answer.", - None, - ); + report_running_line(app, "health-gate", engine::checking_engine_line(), None); let gate = acquire::health_gate(&acquire::address()); report(app, "health-gate", gate.ok, gate.said.clone()); if !gate.ok { @@ -2058,7 +2058,7 @@ async fn engine_ready( ) })?; local_eval::use_engine(&address); - report(app, "engine", true, "The container engine is answering."); + report_line(app, "engine", true, engine::engine_ready_line()); Ok(address) } @@ -2115,7 +2115,13 @@ async fn deployment_ready( ); return Ok(()); } - report_running(app, "deployment", "Checking the OpenBot release.", None); + report_running_line( + app, + "deployment", + engine::StepLine::here("Checking which version of OpenBot to download.") + .logged("Checking the OpenBot release."), + None, + ); // Both release discovery and downloading use blocking HTTP. Keeping them in a blocking task // avoids dropping reqwest's runtime inside this async context. let target = root.to_path_buf(); @@ -6589,6 +6595,31 @@ async fn run_local_eval( Ok(report) } +/// The back end update last written to the log, so a status polled every few seconds says it once. +static UPDATE_LOGGED: Mutex> = Mutex::new(None); + +/// The release the back end is on, the one the window updates it to, and the CLI's step an earlier +/// update stopped at. The window says only "Updating OpenBot", and where it stopped in plain words: +/// the versions and the step's name are for whoever reads the log. +fn log_update_due(from: &str, to: &str, stopped_at: Option<&str>) { + let entry = update_due_entry(from, to, stopped_at); + let mut logged = UPDATE_LOGGED + .lock() + .unwrap_or_else(|held| held.into_inner()); + if logged.as_deref() != Some(entry.as_str()) { + log::info!("{entry}"); + *logged = Some(entry); + } +} + +fn update_due_entry(from: &str, to: &str, stopped_at: Option<&str>) -> String { + let mut entry = format!("[local update] the back end is on {from}, due an update to {to}"); + if let Some(step) = stopped_at { + entry.push_str(&format!("; the last update stopped at {step}")); + } + entry +} + /// What `run_local_eval` answers the window for one CLI call, from the installed CLI. fn local_eval_answer( root: &Path, @@ -6603,6 +6634,15 @@ fn local_eval_answer( if action == local_eval::Action::Status { report.update_to = local_eval::update_due(&report, local_eval::bundled_release().as_deref()); + // The window says "Updating OpenBot"; the log keeps the releases, once per update due, + // and the step an update that did not finish stopped at. + if let (Some(to), Some(stack)) = (&report.update_to, &report.stack) { + let stopped_at = stack + .unfinished_update + .as_ref() + .and_then(|unfinished| unfinished.failed_step.as_deref()); + log_update_due(&stack.installed_release, to, stopped_at); + } } Ok(report) } @@ -9490,6 +9530,21 @@ mod runtime_update_tests { #[cfg(test)] mod tests { + /// The window says "Updating OpenBot" and where an update stopped in plain words; the log + /// keeps the releases and the CLI's own name for the step. + #[test] + fn the_log_keeps_the_back_end_releases_and_the_step_an_update_stopped_at() { + assert_eq!( + update_due_entry("0.16.0-rc.9", "0.16.0-rc.10", Some("update-quiesce")), + "[local update] the back end is on 0.16.0-rc.9, due an update to 0.16.0-rc.10; the last \ + update stopped at update-quiesce" + ); + assert_eq!( + update_due_entry("0.16.0-rc.9", "0.16.0-rc.10", None), + "[local update] the back end is on 0.16.0-rc.9, due an update to 0.16.0-rc.10" + ); + } + #[test] fn only_an_app_that_makes_room_gets_the_overlay_title_bar() { let root = test_support::temp_root("titlebar-room"); diff --git a/desktop/src/App.test.tsx b/desktop/src/App.test.tsx index 22f97f5d70..7ac84f7daf 100644 --- a/desktop/src/App.test.tsx +++ b/desktop/src/App.test.tsx @@ -180,6 +180,27 @@ const failed = (step: string) => ({ running: false, }); +/** + * The background install's row, on a screen that asks for something else meanwhile: what it says, + * how far along it is, and whether it is working (its spinner) or waiting for the person. + */ +function expectBehind( + view: ReturnType, + at: number, + said: string, + { waiting = false }: { waiting?: boolean } = {}, +) { + const row = view.getByRole("listitem", { + name: "Installing OpenBot in the background", + }); + expect(row.querySelector(".detail")?.textContent).toBe(said); + expect(row.querySelector(".step-status")?.textContent).toContain( + `Step ${at} of 4`, + ); + expect(row.querySelector(".step-spinner") !== null).toBe(!waiting); + expect(row.textContent?.includes("Waiting for you")).toBe(waiting); +} + /** A status line that is still working: its words, with the setup rows' spinner beside them. */ function expectWorking(view: ReturnType, said: string) { const status = view @@ -207,14 +228,25 @@ test("installation shows its active stage immediately without an elapsed timer", expect( preparing.getByText("Checking the software OpenBot needs."), ).toBeTruthy(); - expect(preparing.getByText("In progress")).toBeTruthy(); + // Running, it says how far along the install is: the first of its four parts. + expect(preparing.getByText("Step 1 of 4")).toBeTruthy(); expect(preparing.queryByText("✓")).toBeNull(); expect(preparing.queryByText(/elapsed/)).toBeNull(); await act(async () => preparation.resolve()); - // Installed: on to the AI provider screen, with nothing left in progress. + // Installed: on to the AI provider screen. The install's rows are done; what is left, setting + // up the rest of OpenBot, is said on one row under the choice. await view.findByRole("heading", { name: "Connect your AI" }); - expect(view.queryByText("In progress")).toBeNull(); - expect(view.queryByText(/elapsed/)).toBeNull(); + expect( + view.queryByRole("listitem", { name: "Preparing this computer" }), + ).toBeNull(); + expect( + view.queryByRole("listitem", { name: "Installing OpenBot" }), + ).toBeNull(); + expectBehind( + view, + 3, + "Setting up OpenBot on this computer. This takes a few minutes the first time.", + ); }); test("installation failure stops active indicators and retry begins fresh", async () => { @@ -242,7 +274,7 @@ test("installation failure stops active indicators and retry begins fresh", asyn expect(view.getByRole("alert").textContent).toContain( "The download was interrupted.", ); - expect(view.queryByText("In progress") === null).toBe(true); + expect(view.queryByText(/In progress|Step \d of \d/) === null).toBe(true); expect(view.queryByText(/elapsed/)).toBeNull(); expect( within( @@ -253,7 +285,7 @@ test("installation failure stops active indicators and retry begins fresh", asyn expect(view.queryByRole("heading", { name: "Connect your AI" })).toBeNull(); const secondAttempt = await retry(); expect(view.queryByText("Failed")).toBeNull(); - expect(view.getByText("In progress")).toBeTruthy(); + expect(view.getByText("Step 1 of 4")).toBeTruthy(); expect(view.queryByRole("alert")).toBeNull(); await act(async () => secondAttempt.resolve()); await view.findByRole("heading", { name: "Connect your AI" }); @@ -398,14 +430,20 @@ async function installLocal(view: Awaited>) { } /** Words about the software underneath, which no local screen says. */ -// Docker and Podman are not here: they are the engine a person may already know, and their prompts -// and agreements show windows of their own, so a screen names them when it waits on one. See -// `plainInstallSteps`. Intelligence is: a screen names the part of OpenBot, the back end, instead. +// Docker is not here: Docker Desktop is an app a person may already have, and its prompts and +// agreements show windows of its own, so a screen names it when it waits on one. See +// `plainInstallSteps`. Podman is: OpenBot installs it, and a person reads "the software OpenBot +// runs on" instead. The log names it. const JARGON = [ /\bCLI\b/, /Container engine/i, /Engine machine/i, /k3d/i, + /k3s/i, + /\bHelm\b/, + /Podman/, + /\bWSL\b/, + /Windows Subsystem for Linux/, /Intelligence/, ]; @@ -441,7 +479,7 @@ test("the first screen is Install, with nothing to choose", async () => { expect(view.getByRole("heading", { name: "Set up OpenBot" })).toBeTruthy(); expect( view.getByText( - "Installs the OpenBot app and backend on this computer. The first setup takes about 10 minutes.", + "This installs OpenBot on this computer. The first setup takes about 10 minutes.", ), ).toBeTruthy(); expect(buttons(view)).toEqual(["Install"]); @@ -490,7 +528,7 @@ test("Install prepares the default Bot in the default folder, with no Bot or fol // Names every part it installs, so a large download is expected. expect( view.getByText( - /installs OpenBot on this computer: the front end, the back end, the database/, + /installs OpenBot on this computer: the app, its database, your Bots/, ), ).toBeTruthy(); expect(view.queryByLabelText("Where OpenBot lives")).toBeNull(); @@ -532,12 +570,12 @@ test("Install prepares the default Bot in the default folder, with no Bot or fol await emitProgress({ step: "engine", ok: true, - detail: "The container engine is answering.", + detail: "The software OpenBot runs on is ready.", }); await emitProgress({ step: "create-machine", ok: true, - detail: "Preparing the engine machine.", + detail: "openbot created.", }); await emitProgress({ step: "images", @@ -565,7 +603,7 @@ test("Install prepares the default Bot in the default folder, with no Bot or fol const installing = within( view.getByRole("listitem", { name: "Installing OpenBot" }), ); - expect(installing.getByText("In progress")).toBeTruthy(); + expect(installing.getByText("Step 2 of 4")).toBeTruthy(); expect(installing.queryByText(/Downloading local software/)).toBeNull(); expect(installing.queryByRole("progressbar")).toBeNull(); await emitProgress({ @@ -712,12 +750,14 @@ test("the account step opens the CopilotKit sign-in at once, without waiting for step: "install-engine", ok: true, running: true, - detail: "Downloading Podman.", + detail: DOWNLOADING_SOFTWARE, }); - // The browser sign-in, first, with nothing asked of the engine before it. + // The browser sign-in, first, with nothing asked of the engine before it. Under it, what is + // installing meanwhile. await view.findByText("Finish signing in to CopilotKit in your browser…"); expect(view.getByText("WXYZ-2345")).toBeTruthy(); + expectBehind(view, 1, DOWNLOADING_SOFTWARE); const accountCalls = () => invokeCalls .map((call) => call.command) @@ -734,15 +774,19 @@ test("the account step opens the CopilotKit sign-in at once, without waiting for "You're signed in. OpenBot is setting up the software it runs on. The license agreement comes next.", ), ).toBeTruthy(); - expectWorking(view, "Downloading Podman."); + expectBehind(view, 1, DOWNLOADING_SOFTWARE); expect(accountCalls()[0]).toBe("sign_in_to_copilotkit"); expect(accountCalls()).toContain("local_eval_terms"); }); -// What Install says while it waits for Continue (`install::PODMAN_GO_AHEAD`), and once it has it. -const PODMAN_READY = "Podman is downloaded and ready to install."; +// What Install says as it downloads the software OpenBot runs on, while it waits for Continue, and +// once it has it (`install::podman_downloading`, `podman_go_ahead`, `podman_install_said`). +const DOWNLOADING_SOFTWARE = "Downloading the software OpenBot runs on."; +const PODMAN_READY = + "The software OpenBot runs on is downloaded and ready to install."; const PODMAN_PROMPTING = - "Installing Podman. Enter your Mac password when macOS asks for it."; + "Installing the software OpenBot runs on. Enter your Mac password when macOS asks for it."; +const PODMAN_HEADING = "Install the software OpenBot needs"; /** * macOS asked for the person's password over the Account step, mid-sign-in, before they knew what @@ -773,21 +817,20 @@ test("macOS asks for the password only after the person reads why and chooses Co asks: true, detail: PODMAN_READY, }); - expect(view.queryByRole("heading", { name: "Install Podman" })).toBeNull(); + expect(view.queryByRole("heading", { name: PODMAN_HEADING })).toBeNull(); expect(view.queryByRole("button", { name: "Continue" })).toBeNull(); + // The background install's row says it waits, and on what. + expectBehind(view, 1, PODMAN_READY, { waiting: true }); // Signed in: what happens next and why, with one button, and nothing raised until it is chosen. await act(async () => browser.resolve({ signedIn: true })); - await view.findByRole("heading", { name: "Install Podman" }); + await view.findByRole("heading", { name: PODMAN_HEADING }); expect( view.getByText( - "OpenBot installs Podman, which runs OpenBot's back end on this computer. macOS asks for your password once to install it.", + "OpenBot needs one more piece of software to run on this computer. macOS asks for your password once to install it.", ), ).toBeTruthy(); - const waiting = view - .getAllByRole("status") - .find((line) => line.textContent === PODMAN_READY); - expect(waiting?.querySelector(".step-spinner")).toBeNull(); + expectBehind(view, 1, PODMAN_READY, { waiting: true }); expect( view.getAllByRole("button").map((button) => button.textContent), ).toEqual(["Continue"]); @@ -806,9 +849,9 @@ test("macOS asks for the password only after the person reads why and chooses Co elsewhere: true, detail: PODMAN_PROMPTING, }); - expect(view.queryByRole("heading", { name: "Install Podman" })).toBeNull(); + expect(view.queryByRole("heading", { name: PODMAN_HEADING })).toBeNull(); expect(view.queryByRole("button", { name: "Continue" })).toBeNull(); - expectWorking(view, PODMAN_PROMPTING); + expectBehind(view, 1, PODMAN_PROMPTING); }); test("the account step waits for a container engine Install is still setting up, rather than failing", async () => { @@ -868,9 +911,10 @@ test("the account step says Docker Desktop is starting while Install waits on it elsewhere: true, detail: DOCKER_HAND_OFF, }); - expectWorking(view, DOCKER_HAND_OFF); + expectBehind(view, 1, DOCKER_HAND_OFF); // One line, saying what the wait is for. expect(view.queryByText("Checking your CopilotKit account…")).toBeNull(); + expect(view.getAllByText(DOCKER_HAND_OFF)).toHaveLength(1); // Docker answers: the line goes. Until the account check hears the engine, the step says what // Install is doing, as Install's own row does. @@ -881,10 +925,14 @@ test("the account step says Docker Desktop is starting while Install waits on it detail: "Docker Desktop is running.", }); expect(view.queryByText(DOCKER_HAND_OFF)).toBeNull(); - expectWorking(view, "Checking the software OpenBot needs."); + expectBehind(view, 1, "Checking the software OpenBot needs."); }); -test("a line about OpenBot's own work is said on the account step only while it waits on the engine", async () => { +/** + * The line about the install behind the account step stays whatever the step shows: signing in, + * waiting for the engine, the agreement. A person on any of them knows OpenBot is still installing. + */ +test("the install behind the account step is said on every part of it, signing in included", async () => { setupRootConfiguration("/tmp/hand-off-none", async () => emptyConfiguration(), ); @@ -901,22 +949,374 @@ test("a line about OpenBot's own work is said on the account step only while it step: "install-engine", ok: true, running: true, - detail: "Installing Podman.", + detail: "Installing the software OpenBot runs on.", }); - // What the account check waits on, in Install's own words. - expectWorking(view, "Installing Podman."); + // What the account check waits on, in Install's own words, said once. + expectBehind(view, 1, "Installing the software OpenBot runs on."); expect(view.queryByText("Checking your CopilotKit account…")).toBeNull(); + expect(view.queryAllByRole("status")).toHaveLength(0); - // Signed out, the step is about signing in. Install's own work is not said there. + // Signed out, the step is about signing in, and the install behind it is still said. engineAnswers = true; await view.findByRole( "button", { name: "Sign in to CopilotKit" }, { timeout: 5000 }, ); - expect(view.queryByText("Installing Podman.")).toBeNull(); + expectBehind(view, 1, "Installing the software OpenBot runs on."); }, 15000); +/* + * A tester waited minutes on the first setup step and could not tell whether it was stuck: the + * screen read "Preparing the engine machine", "Installing Podman", "Waiting for the back end", + * "Updating the back end to 0.16.0-rc.10". The product's rule: the log keeps the technical parts, + * and the screen says what OpenBot is doing in plain words. + * + * These walk setup as a person meets it, with the native side's lines word for word as it sends + * them, and read every screen. The log side is `install::tests::every_line_shown_while_preparing_ + * this_computer_is_plain_and_the_log_keeps_the_detail`: each of these lines is logged with the + * technical words beside it, and the back end's releases by `log_update_due`. + */ + +/** Words a person setting OpenBot up doesn't know, and the back end's versions. */ +const TECHNICAL = [ + /Podman/i, + /\bWSL\b/, + /Windows Subsystem for Linux/i, + /engine machine/i, + /\bengine\b/i, + /\bk3[sd]\b/i, + /\bHelm\b/, + /\bkubectl\b/i, + /\bback ?end\b/i, + /Intelligence/, + /\bCLI\b/, + /\bCompose\b/, + /0\.16\.0/, +]; + +/** The native side's lines while it prepares this computer (`engine.rs`, `install.rs`). */ +const NATIVE = { + checking: "Checking the software OpenBot needs.", + looking: "Looking for the software OpenBot runs on.", + downloading: DOWNLOADING_SOFTWARE, + goAhead: PODMAN_READY, + prompting: PODMAN_PROMPTING, + installingWindows: "Installing the software OpenBot runs on.", + // The row of a finished install keeps Install's own words: never shown, and checked here. + installed: "Podman 5.6.2 and Compose 2.40.3 installed.", + preparingMac: + "Preparing this computer to run OpenBot. This takes a few minutes the first time.", + preparingWindows: + "Preparing this computer to run OpenBot. If a window asks to install a Windows feature OpenBot needs, allow it. Windows then asks for your approval and a restart.", + starting: "Starting the software OpenBot runs on.", + answering: "Checking that the software OpenBot runs on is ready.", + ready: "The software OpenBot runs on is ready.", + version: "Checking which version of OpenBot to download.", + images: "Checking which local software to download.", +}; + +type Beat = { beat: string; text: string; behind: string | null }; + +/** The background install's row as a person reads it, less its ticking time: status | line. */ +function behindNow(view: ReturnType): string | null { + const row = view.queryByRole("listitem", { + name: "Installing OpenBot in the background", + }); + if (!row) return null; + const status = row + .querySelector(".step-status") + ?.textContent?.replace(/ · [\dms ]+ elapsed$/, ""); + return `${status} | ${row.querySelector(".detail")?.textContent ?? ""}`; +} + +function technicalOn(beats: Beat[]) { + return beats.flatMap(({ beat, text }) => + TECHNICAL.filter((word) => word.test(text)).map( + (word) => `${beat}: ${text.match(word)?.[0]}`, + ), + ); +} + +/** What each part of a first run said, from the first screen to OpenBot opening. */ +async function walkFirstRun(platform: "macOS" | "Windows"): Promise { + setupRootConfiguration(`/tmp/plain-walk-${platform}`, async () => + emptyConfiguration(), + ); + const preparation = deferred(); + const browser = deferred<{ signedIn: boolean }>(); + const setUp = deferred(); + let engineAnswers = false; + withLocalCli({ + prepare_installation: () => preparation.promise, + sign_in_to_copilotkit: () => browser.promise, + copilotkit_sign_in_page: () => ({ + url: "https://dashboard.copilotkit.ai/cli-auth/device?user_code=WXYZ-2345", + userCode: "WXYZ-2345", + }), + local_eval_terms: () => + engineAnswers + ? { ...ACCEPTED_TERMS, accepted: false } + : { status: "engine-not-ready" }, + allow_podman_install: () => true, + set_up_local_eval: () => setUp.promise, + local_eval_download: () => ({ done: 400_000_000, total: 1_600_000_000 }), + connect_local_eval: localConnection, + run_local_eval: localStatus("ready", "valid"), + }); + const beats: Beat[] = []; + const view = await renderApp(); + const seen = (beat: string) => + beats.push({ + beat, + text: view.container.textContent ?? "", + behind: behindNow(view), + }); + const running = (step: string, detail: string, more = {}) => + emitProgress({ step, ok: true, running: true, detail, ...more }); + const done = (step: string, detail: string) => + emitProgress({ step, ok: true, running: false, detail }); + seen("welcome"); + + await installLocal(view); + await view.findByText("Finish signing in to CopilotKit in your browser…"); + seen("signing in"); + await running("install-engine", NATIVE.looking); + seen("signing in, looking"); + await running("install-engine", NATIVE.downloading); + seen("signing in, downloading"); + if (platform === "macOS") { + await running("install-engine", NATIVE.goAhead, { asks: true }); + seen("signing in, ready to install"); + // Signed in: what macOS asks next, and why, with Continue. + await act(async () => browser.resolve({ signedIn: true })); + await view.findByRole("button", { name: "Continue" }); + seen("before the password prompt"); + await userEvent.click(view.getByRole("button", { name: "Continue" })); + await running("install-engine", NATIVE.prompting, { elsewhere: true }); + seen("password prompt"); + } else { + await running("install-engine", NATIVE.installingWindows); + seen("signing in, installing"); + await act(async () => browser.resolve({ signedIn: true })); + } + await done("install-engine", NATIVE.installed); + await running( + "create-machine", + platform === "macOS" ? NATIVE.preparingMac : NATIVE.preparingWindows, + { elsewhere: platform === "Windows" }, + ); + seen("preparing this computer"); + await done("create-machine", "openbot created."); + await running("start-machine", NATIVE.starting); + seen("starting the software"); + await done("start-machine", "openbot started."); + await running("health-gate", NATIVE.answering); + seen("checking the software"); + await done("health-gate", "The engine answers."); + await done("engine", NATIVE.ready); + engineAnswers = true; + await running("deployment", NATIVE.version); + seen("checking the version"); + await done("deployment", "v0.3.4 in /tmp/plain-walk"); + await running("images", NATIVE.images); + seen("checking what to download"); + await emitProgress({ + step: "images", + ok: true, + running: true, + detail: "Downloading local software.", + downloadBytes: 1_200_000_000, + downloads: { + done: 1, + total: 5, + bytes: 1_200_000_000, + size: 3_400_000_000, + bytesPerSecond: 9_000_000, + }, + }); + seen("downloading"); + + // The agreement, once the software it is checked on answers. + await view.findByRole("button", { name: "Accept" }, { timeout: 5000 }); + seen("agreement"); + await userEvent.click(view.getByRole("button", { name: "Accept" })); + await view.findByRole("heading", { name: "Installing OpenBot" }); + await emitProgress({ + step: "images", + ok: true, + running: true, + detail: "Downloading local software.", + downloadBytes: 3_000_000_000, + downloads: { + done: 4, + total: 5, + bytes: 3_000_000_000, + size: 3_400_000_000, + bytesPerSecond: 9_000_000, + }, + }); + seen("install"); + + await act(async () => preparation.resolve()); + await view.findByRole("heading", { name: "Connect your AI" }); + await waitFor(() => expect(localSequence()).toContain("set_up_local_eval")); + await waitFor( + () => expect(view.container.textContent).toContain("of 1.6 GB"), + { timeout: 3000 }, + ); + seen("AI, downloading the rest"); + await act(async () => setUp.resolve(localStatus("starting", "valid"))); + await waitFor(() => expect(localSequence()).toContain("run_local_eval")); + await act(async () => new Promise((resolve) => setTimeout(resolve, 50))); + seen("AI, installed"); + + await chooseOpenAi(view); + await expectHandover(); + seen("start"); + return beats; +} + +for (const platform of ["macOS", "Windows"] as const) { + test(`a first run on ${platform} says every line in plain words, and what is installing behind every screen`, async () => { + const beats = await walkFirstRun(platform); + + // No technical word on any screen, at any point. + expect(technicalOn(beats)).toEqual([]); + + // Behind the account and AI steps: one line, in plain words, saying what is installing and + // how far along it is. It changes as the install moves on. + const behind = Object.fromEntries( + beats.map(({ beat, behind }) => [beat, behind]), + ); + const mac = platform === "macOS"; + expect(behind).toEqual({ + welcome: null, + "signing in": `Step 1 of 4 | ${NATIVE.checking}`, + "signing in, looking": `Step 1 of 4 | ${NATIVE.looking}`, + "signing in, downloading": `Step 1 of 4 | ${NATIVE.downloading}`, + ...(mac + ? { + "signing in, ready to install": `Step 1 of 4 · Waiting for you | ${NATIVE.goAhead}`, + "before the password prompt": `Step 1 of 4 · Waiting for you | ${NATIVE.goAhead}`, + "password prompt": `Step 1 of 4 | ${NATIVE.prompting}`, + "preparing this computer": `Step 1 of 4 | ${NATIVE.preparingMac}`, + } + : { + "signing in, installing": `Step 1 of 4 | ${NATIVE.installingWindows}`, + "preparing this computer": `Step 1 of 4 | ${NATIVE.preparingWindows}`, + }), + "starting the software": `Step 1 of 4 | ${NATIVE.starting}`, + "checking the software": `Step 1 of 4 | ${NATIVE.answering}`, + "checking the version": `Step 2 of 4 | ${NATIVE.version}`, + "checking what to download": `Step 2 of 4 | ${NATIVE.images}`, + downloading: "Step 2 of 4 | Downloading OpenBot. About 4 minutes left.", + agreement: "Step 2 of 4 | Downloading OpenBot. About 4 minutes left.", + // The install is the screen itself here: its own rows say it. + install: null, + "AI, downloading the rest": + "Step 3 of 4 | Downloading the rest of OpenBot.", + "AI, installed": null, + start: null, + }); + // Once everything is installed, the AI step says so, and what comes next. + const installed = beats.find(({ beat }) => beat === "AI, installed"); + expect(installed?.text).toContain( + "OpenBot is installedCompleteOpenBot starts as soon as you connect your AI.", + ); + // The Install step's own rows say how far along, and the time left. + const install = beats.find(({ beat }) => beat === "install"); + expect(install?.text).toContain( + "Installing OpenBotStep 2 of 4Less than a minute left.", + ); + }); +} + +/** + * Every day: a start after a restart wakes the software OpenBot runs on, and a start after an app + * update updates what runs behind OpenBot. "Waiting for the back end" and "Updating the back end to + * 0.16.0-rc.9" are now "Getting OpenBot ready" and "Updating OpenBot", with how far along. + */ +test("a start that wakes the computer's software and updates OpenBot says it in plain words", async () => { + const after = pinnedCli.updateDidNotFinish; + const before = { + ...after, + readiness: "ready", + stack: { + ...after.stack, + step: "ready", + hasFailedStep: false, + updateInProgress: false, + unfinishedUpdate: null, + }, + }; + setupConnectedLocalStack("/tmp/plain-walk-start", before); + const wake = deferred(); + const update = deferred(); + let asked = 0; + let downloading = false; + withLocalCli({ + wake_engine: () => wake.promise, + local_eval_download: () => + downloading ? { done: 300_000_000, total: 900_000_000 } : null, + run_local_eval: (args: { action: string }) => { + if (args.action === "update") return update.promise; + asked += 1; + return asked === 1 ? before : after; + }, + }); + const beats: Beat[] = []; + const view = await renderApp(); + const seen = (beat: string) => + beats.push({ + beat, + text: view.container.textContent ?? "", + behind: behindNow(view), + }); + await view.findByRole("heading", { name: "Starting OpenBot" }); + await emitProgress({ + step: "start-machine", + ok: true, + running: true, + detail: NATIVE.starting, + }); + seen("waking the software"); + await emitProgress({ + step: "start-machine", + ok: true, + running: false, + detail: "openbot started.", + }); + await act(async () => wake.resolve(null)); + await waitFor(() => expect(localActions()).toContain("update")); + seen("updating"); + downloading = true; + await waitFor( + () => expect(view.container.textContent).toContain("of 900.0 MB"), + { timeout: 3000 }, + ); + seen("downloading the update"); + await act(async () => update.resolve(updateFailed)); + await view.findByRole("heading", { name: "OpenBot is running" }); + seen("running on the version it had"); + + // No technical word, and no version of what runs behind OpenBot, at any point. + expect(technicalOn(beats)).toEqual([]); + + // What each part said instead: the row that waits says how far along, and what it is doing. + const said = Object.fromEntries(beats.map(({ beat, text }) => [beat, text])); + expect(said["waking the software"]).toContain( + `Getting OpenBot readyStep 3 of 4 · 0s elapsedThis can take a few minutes.${NATIVE.starting}`, + ); + expect(said.updating).toContain( + "Getting OpenBot readyStep 3 of 4 · 0s elapsedUpdating OpenBot. Your Bots and data stay as they are. This usually takes a few minutes.", + ); + expect(said["downloading the update"]).toContain( + "Downloading the update.300.0 MB of 900.0 MB", + ); + expect(said["running on the version it had"]).toContain(KEPT_RELEASE); +}); + test("a finished install brings the window to the front for the next step", async () => { const { view, preparation } = await beginPendingInstallation(); // Nothing is the person's while it installs: the window stays where they left it. @@ -1041,9 +1441,17 @@ test("the local sequence starts when installation completes and survives the pro expect( invokeCalls.find((call) => call.command === "set_up_local_eval")?.args, ).toMatchObject({ root: "/tmp/local-eval-overlap" }); - // Working behind this screen says nothing on it: its progress has its own step. - expect(view.queryByText(/Setting up OpenBot/)).toBeNull(); - expect(view.queryByRole("alert", { name: "Back end" })).toBeNull(); + // Working behind this screen is said on one row, how far along included. Its own rows wait for + // their step. + expectBehind( + view, + 3, + "Setting up OpenBot on this computer. This takes a few minutes the first time.", + ); + expect( + view.queryByRole("listitem", { name: "Setting up OpenBot" }), + ).toBeNull(); + expect(view.queryByRole("alert", { name: "Setup" })).toBeNull(); // A local first run has nothing behind this screen to go back to. expect(view.queryByRole("button", { name: "Back" })).toBeNull(); await userEvent.click(await view.findByRole("radio", { name: /OpenAI/ })); @@ -1055,6 +1463,11 @@ test("the local sequence starts when installation completes and survives the pro ]), ); expect(view.queryByText(/Connecting OpenBot/)).toBeNull(); + expectBehind( + view, + 3, + "Setting up OpenBot on this computer. This takes a few minutes the first time.", + ); expect(invokeCalls.some((call) => call.command === "cancel_local_eval")).toBe( false, ); @@ -1077,7 +1490,7 @@ test("Continue during setup shows Getting ready, which starts and opens OpenBot await view.findByRole("heading", { name: "Getting ready" }); expect(stepLabel(view)).toBe("Start"); const row = view.getByRole("listitem", { name: "Setting up OpenBot" }); - expect(row.textContent).toContain("In progress"); + expect(row.textContent).toContain("Step 1 of 4"); expectPlain(view); await act(async () => setUp.resolve(localStatus("starting", "valid"))); await expectHandover(); @@ -1202,9 +1615,13 @@ test("a Claude plan signed in on the provider screen reaches Start with its toke await userEvent.click( view.getByRole("button", { name: "Finish signing in" }), ); - // A tick for a second, then on by itself. - expect((await view.findByText(/✓/)).closest("p")?.textContent).toContain( - "Signed in to Anthropic.", + // A tick for a second, then on by itself. (The install's row under it has a tick of its own.) + await waitFor(() => + expect( + view + .getAllByText(/✓/) + .map((tick) => tick.closest("p")?.textContent ?? ""), + ).toContainEqual(expect.stringContaining("Signed in to Anthropic.")), ); await expectHandover(4000); expect(getStartStackPayload()).toMatchObject({ @@ -1251,7 +1668,7 @@ test("Continue after Intelligence is ready starts OpenBot with the chosen model" await installLocal(view); await view.findByRole("heading", { name: "Connect your AI" }); await waitFor(() => expect(localSequence()).toContain("run_local_eval")); - expect(view.queryByRole("alert", { name: "Back end" })).toBeNull(); + expect(view.queryByRole("alert", { name: "Setup" })).toBeNull(); expect(localSequence()).not.toContain("start_stack"); await chooseOpenAi(view); await expectHandover(); @@ -1324,8 +1741,8 @@ test("a blocked setup stops at its row in plain words and Retry resumes there", await installLocal(view); await view.findByRole("heading", { name: "Connect your AI" }); // A stop interrupts the screen in front at once, in a card with its reason and Retry. - const alert = await view.findByRole("alert", { name: "Back end" }); - expect(alert.textContent).toContain("Back end setup stopped"); + const alert = await view.findByRole("alert", { name: "Setup" }); + expect(alert.textContent).toContain("Setup stopped"); expect(alert.textContent).toContain( "The back end cannot start until the software it runs on is running.", ); @@ -1504,7 +1921,7 @@ test("Quit shows Shutting down with each part's row as it starts and ends", asyn rowStatus(view, "Stopping OpenBot").getByText("In progress"), ).toBeTruthy(); expect( - view.queryByRole("listitem", { name: "Stopping the back end" }), + view.queryByRole("listitem", { name: "Stopping OpenBot's services" }), ).toBeNull(); // Nothing to press: the app exits by itself. expect(view.queryAllByRole("button")).toHaveLength(0); @@ -1517,12 +1934,12 @@ test("Quit shows Shutting down with each part's row as it starts and ends", asyn rowStatus(view, "Stopping OpenBot").getByText("Complete"), ).toBeTruthy(); expect( - rowStatus(view, "Stopping the back end").getByText("In progress"), + rowStatus(view, "Stopping OpenBot's services").getByText("In progress"), ).toBeTruthy(); // An older copy arriving late never undoes a newer one. await emitQuit({ seq: 3, steps: [complete("stop-openbot")] }); expect( - rowStatus(view, "Stopping the back end").getByText("In progress"), + rowStatus(view, "Stopping OpenBot's services").getByText("In progress"), ).toBeTruthy(); await emitQuit({ @@ -1530,7 +1947,7 @@ test("Quit shows Shutting down with each part's row as it starts and ends", asyn steps: [complete("stop-openbot"), failed("stop-intelligence")], }); expect( - rowStatus(view, "Stopping the back end").getByText("Failed"), + rowStatus(view, "Stopping OpenBot's services").getByText("Failed"), ).toBeTruthy(); expect(view.getByRole("status").textContent).toBe( "Part of it could not be stopped. OpenBot says what happened the next time it opens.", @@ -1571,7 +1988,7 @@ test("a window sent to the Shutting down page catches up, then follows the rows" steps: [complete("stop-openbot"), complete("stop-intelligence")], }); expect( - rowStatus(view, "Stopping the back end").getByText("Complete"), + rowStatus(view, "Stopping OpenBot's services").getByText("Complete"), ).toBeTruthy(); // Only the catch-up call; nothing that starts or stops anything. expect(invokeCalls.map((call) => call.command)).toEqual(["quit_status"]); @@ -1761,8 +2178,8 @@ test("relaunch with Docker Desktop quit opens it, says so, and starts once it an await view.findByRole("heading", { name: "Starting OpenBot" }); await emitDockerHandOff(); expectWorking(view, DOCKER_HAND_OFF); - const row = view.getByRole("listitem", { name: "Waiting for the back end" }); - expect(row.textContent).toContain("In progress"); + const row = view.getByRole("listitem", { name: "Getting OpenBot ready" }); + expect(row.textContent).toContain("Step 3 of 4"); expect(localSequence()).toEqual([]); // Docker answers: the hand-off goes, and the back end and OpenBot start. await emitDockerSettled(true, "Docker Desktop is running."); @@ -1781,9 +2198,10 @@ test("relaunch with Docker Desktop quit opens it, says so, and starts once it an ).toBeNull(); }); -test("relaunch with the engine machine stopped says it is starting it, in setup's words", async () => { +test("relaunch with the engine machine stopped says it is starting it, in plain words", async () => { // Seen on Windows: Podman's machine was stopped, and for the ten seconds it took to start the // screen said only "Waiting for the back end", while the log said "Starting the engine machine". + // The screen says it plainly now (`engine::starting_machine_line`); the log keeps the machine. setupConnectedLocalStack( "/tmp/local-eval-machine-stopped", localStatus("ready", "valid"), @@ -1799,16 +2217,16 @@ test("relaunch with the engine machine stopped says it is starting it, in setup' step: "start-machine", ok: true, running: true, - detail: "Starting the engine machine.", + detail: "Starting the software OpenBot runs on.", }); - expectWorking(view, "Starting the engine machine."); + expectWorking(view, "Starting the software OpenBot runs on."); await emitProgress({ step: "start-machine", ok: true, running: false, detail: "The engine machine is running.", }); - expect(view.queryByText("Starting the engine machine.")).toBeNull(); + expect(view.queryByText("Starting the software OpenBot runs on.")).toBeNull(); await act(async () => machine.resolve(null)); await view.findByRole("heading", { name: "OpenBot is running" }); }); @@ -2062,10 +2480,15 @@ test("the line under Start's rows says the upgrade step running now, never one t invokeCalls.some((call) => call.command === "upgrade_and_start"), ).toBe(true), ); + // The update's one row: its line, with its spinner and time so far. const lines = () => - [...view.container.querySelectorAll(".status-line")].map( - (line) => line.textContent, - ); + [ + ...view.container.querySelectorAll( + '[aria-label="Update progress"] .detail', + ), + ].map((line) => line.textContent); + const updatingRow = () => + within(view.getByRole("listitem", { name: "Updating OpenBot" })); const release = (detail: string) => ({ step: "release", ok: true, @@ -2145,9 +2568,14 @@ test("the line under Start's rows says the upgrade step running now, never one t expect( view.getByRole("heading", { name: "Starting OpenBot" }), ).toBeTruthy(); - // One line, the step running now, with the spinner. + // One line, the step running now, with the spinner and the time so far. expect(lines()).toEqual([said]); - expectWorking(view, said); + expect( + view + .getByRole("listitem", { name: "Updating OpenBot" }) + .querySelector(".step-spinner"), + ).not.toBeNull(); + expect(updatingRow().getByText(/elapsed/)).toBeTruthy(); } // Nothing an earlier step said is left on the screen. for (const finished of [ @@ -2281,7 +2709,7 @@ const pinnedCli = JSON.parse( ), ); const KEPT_RELEASE = - "The back end is running 0.16.0-rc.5. Downloading 0.16.0-rc.9 did not finish, so OpenBot tries again the next time it starts."; + "The OpenBot update didn't finish downloading, so OpenBot is running the version it had. It tries again the next time it starts."; /** `local update` as the CLI answers when it stops at the image download again. */ const updateFailed = { diff --git a/desktop/src/App.tsx b/desktop/src/App.tsx index 9047b21cb5..34b0831fa6 100644 --- a/desktop/src/App.tsx +++ b/desktop/src/App.tsx @@ -32,7 +32,9 @@ import { engineNow, goAheadNow, handOffNow, + BehindSlot, INSTALL_BLURB, + installingBehind, type Progress, plainInstallSteps, SetupProgress, @@ -141,6 +143,8 @@ function Setup() { const [backEndNotice, setBackEndNotice] = useState(null); const [step, setStep] = useState("welcome"); const [steps, setSteps] = useState([]); + // When the install now running began: its row on the other screens counts from here. + const [installSince, setInstallSince] = useState(null); const [busy, setBusy] = useState(false); const [checkingResume, setCheckingResume] = useState(true); const savedLaunch = useRef(null); @@ -419,6 +423,7 @@ function Setup() { setBusy(true); setFailure(null); setRecoveryFailure(null); + setInstallSince(Date.now()); setSteps([ { step: "engine", @@ -728,6 +733,21 @@ function Setup() { // on says why, with Continue. See `install::GoAhead`. const goAhead = goAheadNow(steps); + /* + * What is installing behind a screen that asks the person for something else, as one row: the + * account step and the AI step. It says what OpenBot is doing, how far along it is and for how + * long, so the screen never looks stuck while minutes of install go on behind it. + */ + const behind = (said: { waitingFor?: string; done?: string }) => + installingBehind({ + steps, + installing: preparation?.status === "preparing", + installed: installationReady, + since: installSince, + backEnd: local, + ...said, + }); + // Quit outranks every screen: the app is closing, whatever was on it. const quitting = useQuitStatus(false); if (quitting) return ; @@ -866,6 +886,10 @@ function Setup() { engine={engineNow(steps)} // Once signed in, the step explains the password prompt Install waits to raise. goAhead={goAhead} + behind={behind({ + waitingFor: + "The rest of the install starts after you sign in and accept the license agreement.", + })} onDone={(acceptedVersion) => { if (acceptedVersion) local.acceptTerms(acceptedVersion); // Intelligence sets itself up once the install has Docker answering; see above. The @@ -892,7 +916,20 @@ function Setup() { chosen={model} returning={returningToInstallation} busy={busy} - status={} + status={ + returningToInstallation ? ( + + ) : ( + + + + + ) + } onInstallRequired={(problem, choice) => { // The record is from an earlier run. Install again, keeping the provider chosen. localInstall.current = null; @@ -944,7 +981,23 @@ function Setup() { row.running, ) : undefined; - const updatingLine = updating && {updating.detail}; + // One row while it updates, with its download's bar once there is one: "Installing OpenBot" sat + // unchanged for minutes while the update's software downloaded. + const updatingLine = updating && ( + row.step === "images" && row.running) + ?.downloads, + }, + ]} + /> + ); // Docker Desktop opened, or the engine machine started, before the back end starts, as after a // restart: said under the rows. const handOff = engineNow(steps); @@ -956,7 +1009,7 @@ function Setup() {

{localStarting ? "Starting OpenBot" : "OpenBot is stopped"}

{localStarting - ? "Starting the back end first, then the rest of OpenBot." + ? "OpenBot is getting ready and opens by itself. This usually takes a minute or two." : "Your installation and saved connections are ready to reopen."}

@@ -1126,6 +1179,11 @@ function Setup() { ); } +/** The background install's row, when there is one. */ +function BehindRow({ row }: { row: Progress | null }) { + return row && ; +} + function settleProgress(steps: Progress[], problem?: Problem): Progress[] { return steps.map((step) => step.running diff --git a/desktop/src/CopilotKitAccount.test.tsx b/desktop/src/CopilotKitAccount.test.tsx index 0813e223ed..4160b5d300 100644 --- a/desktop/src/CopilotKitAccount.test.tsx +++ b/desktop/src/CopilotKitAccount.test.tsx @@ -269,8 +269,12 @@ test("a closed password prompt is asked again, with Continue", async () => { goAhead={goAhead} /> ); - const screen = render(account("Podman is downloaded and ready to install.")); - await screen.findByRole("heading", { name: "Install Podman" }); + const screen = render( + account("The software OpenBot runs on is downloaded and ready to install."), + ); + await screen.findByRole("heading", { + name: "Install the software OpenBot needs", + }); await userEvent.click(screen.getByRole("button", { name: "Continue" })); expect( (screen.getByRole("button", { name: "Continue" }) as HTMLButtonElement) @@ -278,11 +282,13 @@ test("a closed password prompt is asked again, with Continue", async () => { ).toBe(true); screen.rerender( - account("macOS didn't get your password, so Podman isn't installed yet."), + account( + "macOS didn't get your password, so the software isn't installed yet.", + ), ); expect( screen.getByText( - "macOS didn't get your password, so Podman isn't installed yet.", + "macOS didn't get your password, so the software isn't installed yet.", ), ).toBeTruthy(); const again = screen.getByRole("button", { diff --git a/desktop/src/CopilotKitAccount.tsx b/desktop/src/CopilotKitAccount.tsx index 77b50e3d0e..5c8fb937bc 100644 --- a/desktop/src/CopilotKitAccount.tsx +++ b/desktop/src/CopilotKitAccount.tsx @@ -16,10 +16,19 @@ import { signInToCopilotKit, } from "./localEvalCli"; import { engineTooSmallSaid } from "./localReadiness"; -import { ContinueToPodman, PODMAN_EXPLAINED } from "./PodmanGoAhead"; +import { + ContinueToPodman, + PODMAN_EXPLAINED, + PODMAN_HEADING, +} from "./PodmanGoAhead"; import { InlineFailure, type Problem } from "./Problem"; import { Sections } from "./Sections"; -import { StatusLine } from "./SetupProgress"; +import { + BehindSlot, + type Progress, + SetupProgress, + StatusLine, +} from "./SetupProgress"; import { Command } from "./WindowsBlocker"; type Seen = @@ -85,6 +94,7 @@ export function CopilotKitAccount({ handOff = null, engine = null, goAhead = null, + behind, }: { /** The installation folder the apps are connected for. */ root: string; @@ -105,6 +115,13 @@ export function CopilotKitAccount({ * offers Continue. */ goAhead?: string | null; + /** + * What is installing behind this step, as one row: what it is doing, how far along, and for how + * long (`installingBehind`). Shown under every part of the step, so the person always knows + * OpenBot is still installing. It carries the engine's line and any hand-off, so those are not + * said twice. Its slot is held even while it is null; without it, there is none. + */ + behind?: Progress | null; /** Signed in and agreed: the version accepted here, or null when nothing needed accepting. */ onDone: (acceptedVersion: string | null) => void; /** A failure from setup itself, shown inside this card rather than under it. */ @@ -287,7 +304,7 @@ export function CopilotKitAccount({

{asking - ? "Install Podman" + ? PODMAN_HEADING : agreement ? "Accept the license agreement" : connecting @@ -351,15 +368,17 @@ export function CopilotKitAccount({ Install waits for the person instead, it says that, and Continue goes on to the prompt. */} {asking ? ( <> - {goAhead} + {!behind && {goAhead}} ) : waitingForEngine ? ( - - {engine ?? handOff ?? "Preparing this computer…"} - + !behind && ( + + {engine ?? handOff ?? "Preparing this computer…"} + + ) ) : ( - handOff && {handOff} + handOff && !behind && {handOff} )} {seen.at === "checking" && !handOff && !asking && ( Checking your CopilotKit account… @@ -432,6 +451,14 @@ export function CopilotKitAccount({ )}

+ {/* Last, under whatever the step asks: OpenBot is still installing meanwhile. */} + {behind !== undefined && ( + + {behind && ( + + )} + + )} ); } diff --git a/desktop/src/LocalIntelligence.test.tsx b/desktop/src/LocalIntelligence.test.tsx index fcf4c4305b..3597fdc3bf 100644 --- a/desktop/src/LocalIntelligence.test.tsx +++ b/desktop/src/LocalIntelligence.test.tsx @@ -184,7 +184,7 @@ function row(screen: ReturnType, name: string) { const SETUP = "Setting up OpenBot"; const CONNECT = "Connecting OpenBot"; -const WAIT = "Waiting for the back end"; +const WAIT = "Getting OpenBot ready"; const START = "Starting OpenBot"; test("the whole sequence runs by itself: set up, connect, wait for ready and valid, start", async () => { @@ -238,7 +238,7 @@ test("a first setup shows its download on its row and on the screen in front of }; const screen = view({ prepare: true }); await waitFor(() => - expect(row(screen, SETUP).textContent).toContain("In progress"), + expect(row(screen, SETUP).textContent).toContain("Step 1 of 4"), ); arrived = { done: 245_000_000, total: 690_000_000 }; await waitFor( @@ -247,16 +247,14 @@ test("a first setup shows its download on its row and on the screen in front of { timeout: 3_000 }, ); const setting = within(row(screen, SETUP)); - expect(setting.getByText("Downloading the back end.")).toBeTruthy(); + expect(setting.getByText("Downloading the rest of OpenBot.")).toBeTruthy(); const bar = setting.getByRole("progressbar", { name: "Download progress" }); expect(bar.getAttribute("value")).toBe("245000000"); expect(bar.getAttribute("max")).toBe("690000000"); // The screen in front (Install, or Connect your AI) shows the same download while it runs. - const behind = within( - screen.getByRole("list", { name: "Back end download" }), - ); + const behind = within(screen.getByRole("list", { name: "OpenBot download" })); expect( - behind.getByRole("listitem", { name: "Downloading the back end" }) + behind.getByRole("listitem", { name: "Downloading the rest of OpenBot" }) .textContent, ).toContain("245.0 MB of 690.0 MB"); expect( @@ -278,7 +276,7 @@ test("a first setup shows its download on its row and on the screen in front of await waitFor(() => expect(screen.queryByRole("progressbar")).toBeNull(), { timeout: 3_000, }); - expect(screen.queryByRole("list", { name: "Back end download" })).toBeNull(); + expect(screen.queryByRole("list", { name: "OpenBot download" })).toBeNull(); }); test("a recorded, connected stack skips straight to waiting", async () => { @@ -327,7 +325,7 @@ test("a back end an older app installed is updated to this app's release before // One plain line while it runs, on the back end's own row. await waitFor(() => expect(row(screen, WAIT).textContent).toContain( - "Updating the back end to 0.16.0-rc.9. Its data stays.", + "Updating OpenBot. Your Bots and data stay as they are. This usually takes a few minutes.", ), ); expect(started).toEqual([]); @@ -344,7 +342,7 @@ test("a back end an older app installed is updated to this app's release before )?.args, ).toMatchObject({ stack: "local-abc123" }); expect( - screen.queryByRole("listitem", { name: "Updating the back end" }), + screen.queryByRole("listitem", { name: "Updating OpenBot" }), ).toBeNull(); }); @@ -443,8 +441,8 @@ test("an update that fails says so and starts OpenBot on the release the back en "run_local_eval:update", "run_local_eval:status", ]); - expect(row(screen, "Updating the back end").textContent).toContain( - "The back end could not be updated to 0.16.0-rc.9. Your data is kept, and OpenBot tries again the next time it starts.", + expect(row(screen, "Updating OpenBot").textContent).toContain( + "OpenBot couldn't finish updating, so it runs the version it had. Your data is kept, and OpenBot tries again the next time it starts.", ); expect(calls).toBe(2); cleanup(); @@ -455,7 +453,8 @@ test("an update that fails says so and starts OpenBot on the release the back en /** * Found on OpenBot Desktop, 0.16.0-rc.9 to rc.10: every Start after an interrupted update failed * with the same generic line, while the CLI had recorded exactly where it stopped. The row now - * names that step from the record, in the CLI's own words, and says the data is kept. + * says where it stopped from the record, in plain words, and that the data is kept. The step's own + * name is the log's. */ test("an update that fails says the step it stopped at and that the data is kept", async () => { const stoppedAt = { @@ -491,8 +490,8 @@ test("an update that fails says the step it stopped at and that the data is kept }; const screen = view({ stack: "local-abc123", connected: true }); await waitFor(() => - expect(row(screen, "Updating the back end").textContent).toContain( - `The back end could not be updated to 0.16.0-rc.10. It stopped at "Stopping the back end's services" (update-quiesce). Your data is kept, and OpenBot tries again the next time it starts.`, + expect(row(screen, "Updating OpenBot").textContent).toContain( + "OpenBot couldn't finish updating, so it runs the version it had. It stopped while pausing OpenBot's services. Your data is kept, and OpenBot tries again the next time it starts.", ), ); }); @@ -502,6 +501,9 @@ test("an update that fails says the step it stopped at and that the data is kept * else on the row, while the CLI waited on its databases and Jobs (up to ten minutes each) and * logged nothing until each ended. The row now says which step it is in, and for how long. */ +const UPDATING_SAID = + "Updating OpenBot. Your Bots and data stay as they are. This usually takes a few minutes."; + test("a running update says which step it is waiting on, and for how long", async () => { const updating = deferred(); invokeHandler = async (command, args) => { @@ -517,19 +519,18 @@ test("a running update says which step it is waiting on, and for how long", asyn }; const screen = view({ stack: "local-abc123", connected: true }); await waitFor(() => - expect(row(screen, WAIT).textContent).toContain( - "Updating the back end to 0.16.0-rc.10. Its data stays.", - ), + expect(row(screen, WAIT).textContent).toContain(UPDATING_SAID), ); updateStep = { step: "dependencies-wait", seconds: 130 }; await waitFor( () => expect(row(screen, WAIT).textContent).toContain( - "Updating the back end to 0.16.0-rc.10. Its data stays. Waiting for databases and storage, 2m 10s.", + `${UPDATING_SAID} Waiting for where OpenBot keeps your data, 2m 10s so far.`, ), { timeout: 3_000 }, ); - // In the chart's Jobs: the one it waits on, and a failed one at once, not ten minutes later. + // In the chart's Jobs: what they are for (their names are the log's), and a failed one at once, + // not ten minutes later. updateStep = { step: "jobs", seconds: 5, @@ -538,7 +539,7 @@ test("a running update says which step it is waiting on, and for how long", asyn await waitFor( () => expect(row(screen, WAIT).textContent).toContain( - "Running the database jobs (cpki-bootstrap-ef56ab78), 5s.", + `${UPDATING_SAID} Updating your data, 5s so far.`, ), { timeout: 3_000 }, ); @@ -551,7 +552,7 @@ test("a running update says which step it is waiting on, and for how long", asyn await waitFor( () => expect(row(screen, WAIT).textContent).toContain( - "Running the database jobs, 40s. The back end's job cpki-bootstrap-ef56ab78 failed, so this update will stop. The back end's tool waits up to 10 minutes for it first. Your data is kept.", + "Updating your data, 40s so far. Part of the update didn't work, so it will stop. OpenBot waits up to 10 minutes for it first. Your data is kept.", ), { timeout: 3_000 }, ); @@ -567,7 +568,7 @@ test("a running update says which step it is waiting on, and for how long", asyn updating.resolve(onRelease("0.16.0-rc.10", null)); await waitFor(() => expect(row(screen, WAIT).textContent).not.toContain( - "Waiting for databases", + "Waiting for where OpenBot keeps", ), ); }); @@ -621,7 +622,7 @@ test("after Quit, the unfinished update starts the stack, and the release it had apiUrl: "http://localhost:18082", gatewayWsUrl: "ws://localhost:14401", notice: - "The back end is running 0.16.0-rc.5. Downloading 0.16.0-rc.9 did not finish, so OpenBot tries again the next time it starts.", + "The OpenBot update didn't finish downloading, so OpenBot is running the version it had. It tries again the next time it starts.", }, ]), ); @@ -671,7 +672,7 @@ test("a retried update shows its download, and a stall keeps the line that it di apiUrl: "http://localhost:18082", gatewayWsUrl: "ws://localhost:14401", notice: - "The back end is running 0.16.0-rc.5. Downloading 0.16.0-rc.9 did not finish, so OpenBot tries again the next time it starts.", + "The OpenBot update didn't finish downloading, so OpenBot is running the version it had. It tries again the next time it starts.", }, ]), ); @@ -707,7 +708,7 @@ test("an update says how much of its download has arrived, so a slow line reads }); await waitFor(() => expect(row(screen, WAIT).textContent).toContain( - "Updating the back end to 0.16.0-rc.9. Its data stays.", + "Updating OpenBot. Your Bots and data stay as they are. This usually takes a few minutes.", ), ); arrived = { done: 48_200_000, total: 147_000_000 }; @@ -718,7 +719,7 @@ test("an update says how much of its download has arrived, so a slow line reads ); // The same bar Install shows, driven by the bytes, out of the whole download. const waiting = within(row(screen, WAIT)); - expect(waiting.getByText("Downloading the back end update.")).toBeTruthy(); + expect(waiting.getByText("Downloading the update.")).toBeTruthy(); const bar = waiting.getByRole("progressbar", { name: "Download progress" }); expect(bar.getAttribute("value")).toBe("48200000"); expect(bar.getAttribute("max")).toBe("147000000"); @@ -727,7 +728,7 @@ test("an update says how much of its download has arrived, so a slow line reads await waitFor( () => expect(row(screen, WAIT).textContent).toContain( - "Updating the back end to 0.16.0-rc.9. Its data stays.", + "Updating OpenBot. Your Bots and data stay as they are. This usually takes a few minutes.", ), { timeout: 3_000 }, ); @@ -766,8 +767,8 @@ test("an update whose download stopped says so, and that the next start continue }, }); await waitFor(() => expect(started).toEqual([startableUrls])); - expect(row(screen, "Updating the back end").textContent).toContain( - "The back end update stopped downloading, so the back end stays on 0.16.0-rc.5. OpenBot continues the download where it left off the next time it starts.", + expect(row(screen, "Updating OpenBot").textContent).toContain( + "The update stopped downloading, so OpenBot runs the version it had. It picks up the download the next time it starts.", ); expect(calls).toBe(2); }); @@ -1007,7 +1008,7 @@ test("an acceptance setup does not take is said once, not offered again", async }); const screen = view(); await userEvent.click(await screen.findByRole("button", { name: "Accept" })); - await screen.findByText(/The back end did not take the acceptance/); + await screen.findByText(/OpenBot couldn't record your acceptance/); expect(screen.queryByRole("button", { name: "Accept" })).toBeNull(); expect( invokeCalls.filter((call) => call.command === "set_up_local_eval"), @@ -1121,7 +1122,9 @@ test("polling is bounded, and says so when Intelligence never became ready", asy connected: true, pollDelays: [0, 0], }); - await screen.findByText("The back end did not report ready in time."); + await screen.findByText( + "OpenBot didn't get ready in time. Retry to check again.", + ); expect(statusCalls()).toHaveLength(3); expect(screen.getByRole("button", { name: "Retry" })).toBeTruthy(); }); @@ -1344,8 +1347,8 @@ test("a start that fails says OpenBot did not start, not that Intelligence stopp }); const card = await screen.findByRole("alert", { name: "OpenBot" }); expect(card.textContent).toContain("OpenBot did not start"); - expect(card.textContent).toContain("The back end is ready"); - expect(card.textContent).not.toContain("Back end setup stopped"); + expect(card.textContent).toContain("Everything else is ready"); + expect(card.textContent).not.toContain("Setup stopped"); expect(within(card).getByRole("button", { name: "Retry" })).toBeTruthy(); }); @@ -1419,7 +1422,7 @@ test("prepare runs to ready and waits for the go-ahead, which starts at once", a await waitFor(() => expect(row(screen, WAIT).textContent).toContain("Complete"), ); - expect(screen.queryByRole("alert", { name: "Back end" })).toBeNull(); + expect(screen.queryByRole("alert", { name: "Setup" })).toBeNull(); expect(screen.queryByRole("listitem", { name: START })).toBeNull(); expect(started).toEqual([]); // No Start OpenBot once it has run: the go-ahead comes from the screen in front. @@ -1435,9 +1438,9 @@ test("the screen in front hears nothing while Intelligence works, and a stop int }); const working = view({ prepare: true }); await waitFor(() => - expect(row(working, SETUP).textContent).toContain("In progress"), + expect(row(working, SETUP).textContent).toContain("Step 1 of 4"), ); - expect(working.queryByRole("alert", { name: "Back end" })).toBeNull(); + expect(working.queryByRole("alert", { name: "Setup" })).toBeNull(); cleanup(); invokeCalls = []; @@ -1448,8 +1451,8 @@ test("the screen in front hears nothing while Intelligence works, and a stop int cancel_local_eval: [null], }); const failed = view({ prepare: true }); - const stopped = await failed.findByRole("alert", { name: "Back end" }); - expect(stopped.textContent).toContain("Back end setup stopped"); + const stopped = await failed.findByRole("alert", { name: "Setup" }); + expect(stopped.textContent).toContain("Setup stopped"); expect(stopped.textContent).toContain("A port the back end needs is in use."); expect(within(stopped).getByRole("button", { name: "Retry" })).toBeTruthy(); // The logs to paste into an AI assistant, beside the card that says it stopped. @@ -1468,7 +1471,7 @@ test("the screen in front hears nothing while Intelligence works, and a stop int cancel_local_eval: [null], }); const waiting = view({ prepare: true }); - const needs = await waiting.findByRole("alert", { name: "Back end" }); + const needs = await waiting.findByRole("alert", { name: "Setup" }); expect(needs.textContent).toContain("Setup needs you"); expect( await within(needs).findByRole("button", { name: "Accept" }), diff --git a/desktop/src/LocalIntelligence.tsx b/desktop/src/LocalIntelligence.tsx index 4e4ebd6f3b..8b40e6fa8f 100644 --- a/desktop/src/LocalIntelligence.tsx +++ b/desktop/src/LocalIntelligence.tsx @@ -35,6 +35,7 @@ import { type Progress, SetupProgress, StatusLine, + timeLeft, } from "./SetupProgress"; type Call = { attempt: string; answer: Promise }; @@ -75,13 +76,25 @@ const unreadable: Problem = { type Phase = "setup" | "connect" | "wait" | "start"; +/* + * In the person's words: the back end, its releases and the CLI are the log's business. Every row + * says what OpenBot is doing for them, and the running one how far along it is ("Step 3 of 4"). + */ const PHASES: [Phase, string][] = [ ["setup", "Setting up OpenBot"], ["connect", "Connecting OpenBot"], - ["wait", "Waiting for the back end"], + ["wait", "Getting OpenBot ready"], ["start", "Starting OpenBot"], ]; +/** What a running row says when nothing more particular is happening: roughly how long it takes. */ +const TAKES: Record = { + setup: "This takes a few minutes the first time.", + connect: "", + wait: "This can take a few minutes.", + start: "", +}; + /** The readiness sentence for one CLI answer or failure category. */ function said(intelligence: LocalReport | LocalFailure): string { return localReadiness({ @@ -153,18 +166,17 @@ const idle: State = { }; /** The row an update that failed is said on, under the back end's. */ -const UPDATING = "Updating the back end"; +const UPDATING = "Updating OpenBot"; /** - * An update that failed, and where it stopped once the CLI's record says (`failedStep`). Its data - * is kept either way: `local update` never deletes it, and the next Start repairs what a stopped - * update left and tries again. + * An update that failed, and where it stopped once the CLI's record says (`failedStep`), in plain + * words. The step's own name and the releases are in the log (`log_update_due`). Its data is kept + * either way: `local update` never deletes it, and the next Start repairs what a stopped update left + * and tries again. */ -function updateFailedSaid(to: string, step: string | null): string { - const where = step - ? ` It stopped at "${updateStepSaid(step)}" (${step}).` - : ""; - return `The back end could not be updated to ${to}.${where} Your data is kept, and OpenBot tries again the next time it starts.`; +function updateFailedSaid(step: string | null): string { + const where = step ? ` It stopped while ${updateStepSaid(step)}.` : ""; + return `OpenBot couldn't finish updating, so it runs the version it had.${where} Your data is kept, and OpenBot tries again the next time it starts.`; } export type LocalSequence = State & { @@ -283,6 +295,8 @@ export function useLocalSequence({ let following = true; let shown = false; let next: ReturnType | null = null; + // The first answer with bytes in it, so the speed so far can say how long is left. + let first: { at: number; bytes: number } | null = null; /* * Only while the run it reports on is current. Halting it (the window closed, or Cancel) only asks the CLI to stop, * and the call it followed settles once the CLI has, so a poll that waited for that went on asking every second from @@ -297,10 +311,21 @@ export function useLocalSequence({ if (!current()) return; if (found) { shown = true; + const at = Date.now(); + first ??= { at, bytes: found.done }; + const seconds = (at - first.at) / 1000; + const download: Downloads = { + bytes: found.done, + size: found.total, + ...(seconds >= 3 && found.done > first.bytes + ? { bytesPerSecond: (found.done - first.bytes) / seconds } + : {}), + }; + const left = timeLeft(download); // The row's bar and "48.0 MB of 147.0 MB", as Install shows its own download. update({ - detail: `${saying}.`, - download: { bytes: found.done, size: found.total }, + detail: left ? `${saying}. ${left}` : `${saying}.`, + download, }); } else if (shown) { shown = false; @@ -344,18 +369,18 @@ export function useLocalSequence({ */ async function updateIntelligence( id: string, - to: string, - from: string, token: number, ): Promise { - const updating = `Updating the back end to ${to}. Its data stays.`; + // The releases it updates from and to are in the log (`log_update_due`), not on the screen. + const updating = + "Updating OpenBot. Your Bots and data stay as they are. This usually takes a few minutes."; update({ detail: updating, download: null }); const started = runLocalEval("update", id); attempt.current = started.attempt; const unfollow = followDownload( started.attempt, token, - "Downloading the back end update", + "Downloading the update", updating, ); const unfollowSteps = followUpdateSteps(started.attempt, token, updating); @@ -380,8 +405,8 @@ export function useLocalSequence({ updateFailed: updated ? null : stalled - ? `The back end update stopped downloading, so the back end stays on ${from}. OpenBot continues the download where it left off the next time it starts.` - : updateFailedSaid(to, null), + ? "The update stopped downloading, so OpenBot runs the version it had. It picks up the download the next time it starts." + : updateFailedSaid(null), }); return true; } @@ -405,12 +430,12 @@ export function useLocalSequence({ .then( (found) => { if (!current() || !found || now.current.download) return; - const job = found.job ? ` (${found.job})` : ""; + // The Job's name is in the log (`watch_jobs`): the row says what the step is for. const failed = found.failedJob - ? ` The back end's job ${found.failedJob} failed, so this update will stop. The back end's tool waits up to 10 minutes for it first. Your data is kept.` + ? " Part of the update didn't work, so it will stop. OpenBot waits up to 10 minutes for it first. Your data is kept." : ""; update({ - detail: `${updating} ${found.said}${job}, ${formatElapsed(found.seconds)}.${failed}`, + detail: `${updating} ${found.said}, ${formatElapsed(found.seconds)} so far.${failed}`, }); }, () => {}, @@ -481,16 +506,9 @@ export function useLocalSequence({ ) { updatedIntelligence = true; woken = found.readiness === "stopped"; - const going = await updateIntelligence( - id, - found.updateTo, - found.stack.installedRelease, - token, - ); + const going = await updateIntelligence(id, token); if (!going) return null; - if ( - now.current.updateFailed === updateFailedSaid(found.updateTo, null) - ) { + if (now.current.updateFailed === updateFailedSaid(null)) { failedTo = found.updateTo; } continue; @@ -498,7 +516,7 @@ export function useLocalSequence({ // Where the update that just failed stopped, from the record the CLI kept of it. const failedStep = unfinished?.failedStep; if (failedTo && failedStep) { - update({ updateFailed: updateFailedSaid(failedTo, failedStep) }); + update({ updateFailed: updateFailedSaid(failedStep) }); } failedTo = null; if (isReady(found)) { @@ -541,7 +559,7 @@ export function useLocalSequence({ if (tries >= pollDelays.length) { update({ blocked: { - said: "The back end did not report ready in time.", + said: "OpenBot didn't get ready in time. Retry to check again.", reason: null, }, }); @@ -609,7 +627,7 @@ export function useLocalSequence({ const unfollow = followDownload( started.attempt, token, - "Downloading the back end", + "Downloading the rest of OpenBot", null, ); const report = await call( @@ -738,9 +756,10 @@ export function useLocalSequence({ step: label, ok: true, running: true, - detail: state.detail ?? "", + detail: state.detail ?? TAKES[phase], downloads: state.download ?? undefined, startedAt: state.since ?? undefined, + position: { at: index + 1, of: PHASES.length }, } : stopped ? { step: label, ok: false, detail: state.blocked?.said ?? "" } @@ -782,7 +801,7 @@ export function LocalIntelligence({ const { at, running, stopped, problem } = sequence; return ( -
+
{handOff && {handOff}} @@ -815,21 +834,30 @@ export function LocalIntelligence({ * at once, because finding out only at that later step would waste the minutes since: one they * clear themselves (signing in, accepting the agreement) with its action, anything else with Retry. */ -export function LocalStatus({ sequence }: { sequence: LocalSequence }) { +export function LocalStatus({ + sequence, + download = true, +}: { + sequence: LocalSequence; + /** Its download's row too. Off where the background install's row says it already. */ + download?: boolean; +}) { if (!sequence.stopped) { - // The back end downloads behind the screen in front: say how far, so the wait is visible. - if (!sequence.running || !sequence.download) return null; + // The rest of OpenBot downloads behind the screen in front: say how far, so the wait is visible. + if (!download || !sequence.running || !sequence.download) return null; return ( @@ -842,14 +870,14 @@ export function LocalStatus({ sequence }: { sequence: LocalSequence }) {

{needs ? "Setup needs you" : openBot ? "OpenBot did not start" - : "Back end setup stopped"} + : "Setup stopped"}

{needs ? ( @@ -857,7 +885,7 @@ export function LocalStatus({ sequence }: { sequence: LocalSequence }) { <>

{openBot - ? "The back end is ready. Retry starts OpenBot again." + ? "Everything else is ready. Retry starts OpenBot again." : (sequence.blocked?.said ?? sequence.problem?.said)}

@@ -958,7 +986,7 @@ function SignInToCopilotKit({ onSignedIn }: { onSignedIn: () => void }) { return (
-

OpenBot's back end needs your CopilotKit account.

+

OpenBot needs your CopilotKit account to finish setting up.

{waiting ? (

- The back end did not take the acceptance from OpenBot. If a coding - agent or a script opened OpenBot, quit it, open it yourself, and - accept again. + OpenBot couldn't record your acceptance. If a coding agent or a script + opened OpenBot, quit it, open it yourself, and accept again.

+ {/* Under the form, never above it: what runs behind this step comes and goes while the + person types here, and nothing above the fields may move. See `BehindSlot`. */} + {status}
); } diff --git a/desktop/src/SetupProgress.test.tsx b/desktop/src/SetupProgress.test.tsx index 93fab1833b..b25a616dfe 100644 --- a/desktop/src/SetupProgress.test.tsx +++ b/desktop/src/SetupProgress.test.tsx @@ -42,7 +42,8 @@ test("install progress is the bytes downloaded out of the whole download, and di const row = within( view.getByRole("listitem", { name: "Installing OpenBot" }), ); - expect(row.getByText("In progress")).toBeTruthy(); + // Running, it says how far along the install is. + expect(row.getByText("Step 2 of 4")).toBeTruthy(); expect(row.queryByRole("progressbar")).toBeNull(); for (const [done, bytes, said] of [ @@ -159,15 +160,18 @@ test("a download whose size is not known yet says so, and its bar does not guess ).toBeTruthy(); }); +const UPDATING = + "Updating OpenBot. Your Bots and data stay as they are. This usually takes a few minutes."; + test("a row with bytes and no items says only the bytes", () => { const view = render( { />, ); const row = within( - view.getByRole("listitem", { name: "Waiting for the back end" }), + view.getByRole("listitem", { name: "Getting OpenBot ready" }), ); - expect( - row.getByText("Updating the back end to 0.16.0-rc.9. Its data stays."), - ).toBeTruthy(); + expect(row.getByText(UPDATING)).toBeTruthy(); expect(row.getByText("207.0 MB of 315.0 MB")).toBeTruthy(); const bar = row.getByRole("progressbar", { name: "Download progress" }); expect(bar.getAttribute("value")).toBe("207000000"); @@ -188,9 +190,13 @@ test("a row with bytes and no items says only the bytes", () => { test("the install blurb names the parts of OpenBot it installs, in two sentences at most", () => { expect(INSTALL_BLURB).toContain( - "This installs OpenBot on this computer: the front end, the back end, the database, your Bots, and the computers they work on.", + "This installs OpenBot on this computer: the app, its database, your Bots, and the computers they work on.", ); expect(INSTALL_BLURB.split(/(?<=\.)\s/).length).toBeLessThanOrEqual(2); - // A person installing OpenBot knows its parts, not the product names behind them. - expect(INSTALL_BLURB).not.toMatch(/—|agent|Intelligence/); + // A person installing OpenBot knows its parts, not the product names behind them, and how long + // the wait is. + expect(INSTALL_BLURB).not.toMatch( + /—|agent|Intelligence|back ?end|front ?end/i, + ); + expect(INSTALL_BLURB).toContain("about 10 minutes"); }); diff --git a/desktop/src/SetupProgress.tsx b/desktop/src/SetupProgress.tsx index 26e861d38b..62c707227c 100644 --- a/desktop/src/SetupProgress.tsx +++ b/desktop/src/SetupProgress.tsx @@ -2,7 +2,7 @@ import { type ReactNode, useEffect, useState } from "react"; /** What the Install step says while it downloads OpenBot's software. */ export const INSTALL_BLURB = - "This installs OpenBot on this computer: the front end, the back end, the database, your Bots, and the computers they work on. It's a large download the first time, so you might want to grab a coffee or tea."; + "This installs OpenBot on this computer: the app, its database, your Bots, and the computers they work on. The first install takes about 10 minutes, so you might want to grab a coffee or tea."; /** * A download while it runs: Install's images (`pull_metrics::Downloads`), or the back end's update @@ -39,6 +39,13 @@ export type Progress = { * prompt (`install::GoAhead`). See `goAheadNow`. */ asks?: boolean; + /** + * Where this step is among the parts of what it belongs to, said while it runs: "Step 2 of 4". + * A row list shows the parts done and the one running; this says how many are left. + */ + position?: { at: number; of: number }; + /** The step waits on the person, not on work: no spinner, and its status says so. */ + waiting?: boolean; }; /** @@ -201,6 +208,7 @@ export function plainInstallSteps( detail: preparingNow?.detail ?? "", downloadBytes: preparingNow?.downloadBytes, downloads: preparingNow?.downloads, + position: { at: 1, of: INSTALL_PARTS }, }); } if (rest.length > 0 || (installing && !stillPreparing && !preparingFailed)) { @@ -217,9 +225,12 @@ export function plainInstallSteps( step: "Installing OpenBot", ok: !installingFailed, running: installingNow, - detail: pulling ? "" : (restNow?.detail ?? ""), + detail: pulling + ? (timeLeft(restNow?.downloads) ?? "") + : (restNow?.detail ?? ""), downloadBytes: pulling ? undefined : restNow?.downloadBytes, downloads: restNow?.downloads, + position: { at: 2, of: INSTALL_PARTS }, }); } return rows; @@ -232,22 +243,44 @@ function SetupProgressRow({ step: Progress; showElapsed: boolean; }) { - const running = step.running ?? false; + const waiting = step.waiting ?? false; + const running = !waiting && (step.running ?? false); const elapsed = useElapsed(running && showElapsed, step.startedAt); const downloads = running ? step.downloads : undefined; + const at = + step.position && (running || waiting) + ? `Step ${step.position.at} of ${step.position.of}` + : null; + const status = waiting + ? at + ? `${at} · Waiting for you` + : "Waiting for you" + : running + ? (at ?? "In progress") + : step.ok + ? "Complete" + : "Failed"; return (

    • Set up OpenBot

    - Installs the OpenBot app and backend on this computer. The first setup - takes about 10 minutes. + This installs OpenBot on this computer. The first setup takes about 10 + minutes.