diff --git a/CHANGELOG.md b/CHANGELOG.md index b306eb039b..de878b9df7 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -30,6 +30,24 @@ backup taken before the upgrade instead. Three things change for an existing dep ### OpenBot Desktop +- **OpenBot's own software downloads on the first screen too, before there is anything to run it.** + On a new Mac or PC, or with Docker Desktop not running, OpenBot's images (about 1.1 GB for the + default Bot) waited for the engine to start, then downloaded behind it. They now download from + the first screen with everything else, into the same cache, checked against their digests and + carried on after a quit. Once the engine answers, Install loads them into it, so the step after + it no longer downloads them. Anything that didn't download or load is downloaded then, as before. +- **The big downloads start on the first screen, before you click Install.** Nothing used to + download until Install, and then one thing at a time: on a new Mac, Podman, then the 937 MB + engine image with no progress shown, then 1.1 GB of OpenBot's software. Now the engine image, + Podman's installer, the deployment and, when Docker or Podman already runs, OpenBot's software + start downloading as soon as the first screen shows, side by side, while you sign in and enter + your Mac password. The first screen says how far they are ("Downloading OpenBot. About 3 minutes + left.", with the bytes under a bar), and the row for what's installing in the background carries + on with it. A download stopped by quitting carries on from where it stopped next time, + and every file is checked against its pinned digest before it is used. Podman builds its machine + from the downloaded image instead of fetching its own. With less free disk than OpenBot needs, + the first screen says how much is free and how much is needed, and the downloads start once + there is room. Nothing is installed before you are asked. - **An update downloads before you click Update.** As soon as a new version of OpenBot is out, OpenBot Desktop downloads what it runs in the background while you keep working: the parts for the Bots you use, and the back end's when it changes too, kept until you update. Clicking Update diff --git a/desktop/src-tauri/Cargo.lock b/desktop/src-tauri/Cargo.lock index 95bfd30a86..d0061d0c1d 100644 --- a/desktop/src-tauri/Cargo.lock +++ b/desktop/src-tauri/Cargo.lock @@ -296,6 +296,15 @@ dependencies = [ "generic-array", ] +[[package]] +name = "block-buffer" +version = "0.12.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "d2f6c7dbe95a6ed67ad9f18e57daf93a2f034c524b99fd2b76d18fdfeb6660aa" +dependencies = [ + "hybrid-array", +] + [[package]] name = "block2" version = "0.6.2" @@ -522,8 +531,10 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "1aa79e62e7697b8e29b513a68abacf485adcd1fe8284a4316c5ae868e6633327" dependencies = [ "iana-time-zone", + "js-sys", "num-traits", "serde", + "wasm-bindgen", "windows-link 0.2.1", ] @@ -546,6 +557,12 @@ dependencies = [ "crossbeam-utils", ] +[[package]] +name = "const-oid" +version = "0.10.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "be95d4a884358a21a199689323c29e9256b4a1b315e22f183d155422041d3a9a" + [[package]] name = "const-random" version = "0.1.18" @@ -566,6 +583,27 @@ dependencies = [ "tiny-keccak", ] +[[package]] +name = "const_format" +version = "0.2.37" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "a3decb4d6965448659923544a634a0571ba44cede4e11ef15a5a317126840454" +dependencies = [ + "const_format_proc_macros", + "konst", +] + +[[package]] +name = "const_format_proc_macros" +version = "0.2.37" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "941b570269a642506f08cb5b688c04bfa6cfd41a26f8bcdc49db8fd00c0ab659" +dependencies = [ + "proc-macro2", + "quote", + "unicode-xid", +] + [[package]] name = "cookie" version = "0.18.2" @@ -684,6 +722,15 @@ dependencies = [ "typenum", ] +[[package]] +name = "crypto-common" +version = "0.2.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "ce6e4c961d6cd6c9a86db418387425e8bdeaf05b3c8bc1411e6dca4c252f1453" +dependencies = [ + "hybrid-array", +] + [[package]] name = "cssparser" version = "0.37.0" @@ -713,14 +760,38 @@ version = "1.0.13" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "914a755b7c2d4af2bdcff7ce1739e2db9a1b81a9b07123d8015786ae03c0980d" +[[package]] +name = "darling" +version = "0.20.11" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "fc7f46116c46ff9ab3eb1597a45688b6715c6e628b5c133e288e709a29bcb4ee" +dependencies = [ + "darling_core 0.20.11", + "darling_macro 0.20.11", +] + [[package]] name = "darling" version = "0.23.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "25ae13da2f202d56bd7f91c25fba009e7717a1e4a1cc98a76d844b65ae912e9d" dependencies = [ - "darling_core", - "darling_macro", + "darling_core 0.23.0", + "darling_macro 0.23.0", +] + +[[package]] +name = "darling_core" +version = "0.20.11" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "0d00b9596d185e565c2207a0b01f8bd1a135483d02d9b7b0a54b11da8d53412e" +dependencies = [ + "fnv", + "ident_case", + "proc-macro2", + "quote", + "strsim", + "syn 2.0.119", ] [[package]] @@ -736,13 +807,24 @@ dependencies = [ "syn 2.0.119", ] +[[package]] +name = "darling_macro" +version = "0.20.11" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "fc34b93ccb385b40dc71c6fceac4b2ad23662c7eeb248cf10d529b7e055b6ead" +dependencies = [ + "darling_core 0.20.11", + "quote", + "syn 2.0.119", +] + [[package]] name = "darling_macro" version = "0.23.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "ac3984ec7bd6cfa798e62b4a642426a5be0e68f9401cfc2a01e3fa9ea2fcdb8d" dependencies = [ - "darling_core", + "darling_core 0.23.0", "quote", "syn 2.0.119", ] @@ -809,6 +891,37 @@ dependencies = [ "syn 2.0.119", ] +[[package]] +name = "derive_builder" +version = "0.20.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "507dfb09ea8b7fa618fcf76e953f4f5e192547945816d5358edffe39f6f94947" +dependencies = [ + "derive_builder_macro", +] + +[[package]] +name = "derive_builder_core" +version = "0.20.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "2d5bcf7b024d6835cfb3d473887cd966994907effbe9227e8c8219824d06c4e8" +dependencies = [ + "darling 0.20.11", + "proc-macro2", + "quote", + "syn 2.0.119", +] + +[[package]] +name = "derive_builder_macro" +version = "0.20.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "ab63b0e2bf4d5928aff72e83a7dace85d7bba5fe12dcc3c5a572d78caffd3f3c" +dependencies = [ + "derive_builder_core", + "syn 2.0.119", +] + [[package]] name = "derive_more" version = "2.1.1" @@ -836,8 +949,19 @@ version = "0.10.7" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "9ed9a281f7bc9b7576e61468ba615a66a5c8cfdff42420a70aa82701a3b1e292" dependencies = [ - "block-buffer", - "crypto-common", + "block-buffer 0.10.4", + "crypto-common 0.1.7", +] + +[[package]] +name = "digest" +version = "0.11.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "f1dd6dbb5841937940781866fa1281a1ff7bd3bf827091440879f9994983d5c2" +dependencies = [ + "block-buffer 0.12.1", + "const-oid", + "crypto-common 0.2.2", ] [[package]] @@ -1439,6 +1563,17 @@ dependencies = [ "wasm-bindgen", ] +[[package]] +name = "getset" +version = "0.1.7" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "6cf442baaabe4213ce7d1239afc26c039180b6456da2cededa316ae2c8a77a77" +dependencies = [ + "proc-macro2", + "quote", + "syn 2.0.119", +] + [[package]] name = "gio" version = "0.18.4" @@ -1649,6 +1784,15 @@ dependencies = [ "itoa", ] +[[package]] +name = "http-auth" +version = "0.1.10" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "150fa4a9462ef926824cf4519c84ed652ca8f4fbae34cb8af045b5cbcaf98822" +dependencies = [ + "memchr", +] + [[package]] name = "http-body" version = "1.1.0" @@ -1678,6 +1822,15 @@ version = "1.10.1" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "6dbf3de79e51f3d586ab4cb9d5c3e2c14aa28ed23d180cf89b4df0454a69cc87" +[[package]] +name = "hybrid-array" +version = "0.4.15" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "27f864f10dfb56725ce5ce5472bc52252c8f93a4ab86327122cebf62c5f59a17" +dependencies = [ + "typenum", +] + [[package]] name = "hyper" version = "1.11.1" @@ -2152,6 +2305,21 @@ dependencies = [ "serde", ] +[[package]] +name = "konst" +version = "0.2.20" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "128133ed7824fcd73d6e7b17957c5eb7bacb885649bd8c69708b2331a10bcefb" +dependencies = [ + "konst_macro_rules", +] + +[[package]] +name = "konst_macro_rules" +version = "0.2.19" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "a4933f3f57a8e9d9da04db23fb153356ecaf00cbd14aee46279c33dc80925c37" + [[package]] name = "lazy_static" version = "1.5.0" @@ -2727,6 +2895,60 @@ dependencies = [ "objc2-foundation", ] +[[package]] +name = "oci-client" +version = "0.18.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "ddc7848d781053285280fcd35aab24c9c0a677e26db4aa4209bd5db57467de0f" +dependencies = [ + "base64 0.22.1", + "bytes", + "chrono", + "futures-util", + "hex", + "http", + "http-auth", + "oci-spec", + "olpc-cjson", + "regex", + "reqwest 0.13.4", + "serde", + "serde_json", + "sha2 0.11.1", + "thiserror 2.0.20", + "tokio", + "tracing", + "unicase", +] + +[[package]] +name = "oci-spec" +version = "0.10.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "3df6f876ad774d6a676f7e968f5c3edacc32f90e65fe680a8b686235396556fb" +dependencies = [ + "const_format", + "derive_builder", + "getset", + "regex", + "serde", + "serde_json", + "strum", + "strum_macros", + "thiserror 2.0.20", +] + +[[package]] +name = "olpc-cjson" +version = "0.1.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "696183c9b5fe81a7715d074fd632e8bd46f4ccc0231a3ed7fc580a80de5f7083" +dependencies = [ + "serde", + "serde_json", + "unicode-normalization", +] + [[package]] name = "once_cell" version = "1.21.4" @@ -2759,14 +2981,16 @@ dependencies = [ "objc2-avf-audio", "objc2-foundation", "objc2-speech", + "oci-client", "portable-pty", "rand 0.9.5", "regex", "reqwest 0.12.28", + "rustls", "semver", "serde", "serde_json", - "sha2", + "sha2 0.10.9", "socket2", "tar", "tauri", @@ -3433,6 +3657,7 @@ dependencies = [ "rustls-platform-verifier", "serde", "serde_json", + "serde_urlencoded", "sync_wrapper", "tokio", "tokio-rustls", @@ -3870,7 +4095,7 @@ version = "3.22.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "8705578779c2b6bd90d84d66eb2e206b708b1a4d7b9f17641b293545bf1c7e46" dependencies = [ - "darling", + "darling 0.23.0", "proc-macro2", "quote", "syn 2.0.119", @@ -3926,7 +4151,18 @@ checksum = "a7507d819769d01a365ab707794a4084392c824f54a7a6a7862f8c3d0892b283" dependencies = [ "cfg-if", "cpufeatures 0.2.17", - "digest", + "digest 0.10.7", +] + +[[package]] +name = "sha2" +version = "0.11.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "47d7069beb7d6ac7b9acd1039986e73443f24234f41074da099d6f994ac9ad19" +dependencies = [ + "cfg-if", + "cpufeatures 0.3.1", + "digest 0.11.3", ] [[package]] @@ -4133,6 +4369,24 @@ version = "0.11.1" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "7da8b5736845d9f2fcb837ea5d9e2628564b3b043a70948a3f0b778838c5fb4f" +[[package]] +name = "strum" +version = "0.27.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "af23d6f6c1a224baef9d3f61e287d2761385a5b88fdab4eb4c6f11aeb54c4bcf" + +[[package]] +name = "strum_macros" +version = "0.27.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "7695ce3845ea4b33927c055a39dc438a45b059f7c1b3d91d38d10355fb8cbca7" +dependencies = [ + "heck 0.5.0", + "proc-macro2", + "quote", + "syn 2.0.119", +] + [[package]] name = "subtle" version = "2.6.1" @@ -4396,7 +4650,7 @@ dependencies = [ "semver", "serde", "serde_json", - "sha2", + "sha2 0.10.9", "syn 2.0.119", "tauri-utils", "thiserror 2.0.20", @@ -4898,9 +5152,21 @@ dependencies = [ "mio", "pin-project-lite", "socket2", + "tokio-macros", "windows-sys 0.61.2", ] +[[package]] +name = "tokio-macros" +version = "2.7.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "78773a2a397f451582ce068015985c33193cf6dea8b74d2a639fe457b2f07b0e" +dependencies = [ + "proc-macro2", + "quote", + "syn 3.0.5", +] + [[package]] name = "tokio-rustls" version = "0.26.5" @@ -5071,6 +5337,7 @@ version = "0.1.44" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "63e71662fa4b2a2c3a26f570f037eb95bb1f85397f3cd8076caed2f026a6d100" dependencies = [ + "log", "pin-project-lite", "tracing-attributes", "tracing-core", @@ -5153,18 +5420,39 @@ dependencies = [ "windows-sys 0.61.2", ] +[[package]] +name = "unicase" +version = "2.10.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "357cc3acc6a036009fd6c973ed009037c732d60d0b4f6c673e9041497482a28f" + [[package]] name = "unicode-ident" version = "1.0.24" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "e6e4313cd5fcd3dad5cafa179702e2b244f760991f45397d14d4ebf38247da75" +[[package]] +name = "unicode-normalization" +version = "0.1.26" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "75cd3fb03d5288b3ba9a1ee103ac7e9cbb59f0b03d64f8ddfcc04dd42758a85d" +dependencies = [ + "tinyvec", +] + [[package]] name = "unicode-segmentation" version = "1.13.3" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "c6f5d3c3b1bf09027a88a6bc961fc00497d651009560b5463668dc81b0fa87a8" +[[package]] +name = "unicode-xid" +version = "0.2.6" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "ebc1c04c71510c7f702b52b7c350734c9ff1295c464a03335b00bb84fc54f853" + [[package]] name = "untrusted" version = "0.9.0" @@ -6051,7 +6339,7 @@ dependencies = [ "objc2-web-kit", "percent-encoding", "raw-window-handle", - "sha2", + "sha2 0.10.9", "soup3", "tao-macros", "thiserror 2.0.20", diff --git a/desktop/src-tauri/Cargo.toml b/desktop/src-tauri/Cargo.toml index dc4866ea79..70b4346f2c 100644 --- a/desktop/src-tauri/Cargo.toml +++ b/desktop/src-tauri/Cargo.toml @@ -38,6 +38,14 @@ sha2 = "0.10" tar = "0.4" # Comparing the app's release with its deployment's. See `follow.rs`. semver = "1" +# OpenBot's images downloaded before any engine answers: the registry's manifests, read as the OCI +# Distribution spec says, for this computer's platform. The ORAS project's client. See +# `early_images.rs`. Its blobs come through `early_downloads::fetch`, which resumes and checks them. +# No TLS provider of its own: rustls's ring, which reqwest 0.12 already builds, is the one used. +oci-client = { version = "0.18", default-features = false, features = ["rustls-tls-no-provider"] } +# The TLS provider the client above needs installed, as tauri-plugin-updater installs it: ring, +# which this build already has. The version reqwest and the updater already use. +rustls = { version = "0.23", default-features = false, features = ["ring"] } # Read Bun's release ZIP in-process; no unzip or PowerShell installation is required. zip = { version = "2.4.2", default-features = false, features = ["deflate-flate2", "flate2"] } # With `deep-link`, a second launch's `openbot://` link reaches the running OpenBot. See `notice.rs`. diff --git a/desktop/src-tauri/src/acquire.rs b/desktop/src-tauri/src/acquire.rs index 9b797c7896..865c4fb345 100644 --- a/desktop/src-tauri/src/acquire.rs +++ b/desktop/src-tauri/src/acquire.rs @@ -296,14 +296,24 @@ pub fn machine_exists() -> Result { /// `local_eval::Failure::RootlessPodman`). After the switch OpenBot stays in the rootless storage /// its database is in (see `connection_keeping_openbot`), and Intelligence runs on the rootful /// engine of the same machine. -pub fn create_machine(cpus: u32, memory_mib: u32, disk_gib: u32) -> StepOutcome { - create_machine_with(cpus, memory_mib, disk_gib, machine_exists, podman) +/// +/// `image` is the machine image already downloaded (`early_downloads`), which Podman builds the +/// machine from instead of pulling its own from quay.io: `--image` takes a local path, and unpacks a +/// compressed one itself. `None` leaves the pull to Podman, as before. +pub fn create_machine( + cpus: u32, + memory_mib: u32, + disk_gib: u32, + image: Option<&Path>, +) -> StepOutcome { + create_machine_with(cpus, memory_mib, disk_gib, image, machine_exists, podman) } fn create_machine_with( cpus: u32, memory_mib: u32, disk_gib: u32, + image: Option<&Path>, exists: impl FnOnce() -> Result, mut run: impl FnMut(&[&str]) -> Result, ) -> StepOutcome { @@ -314,7 +324,10 @@ fn create_machine_with( Ok(false) => {} Err(error) => return StepOutcome::stopped(Step::CreateMachine, &error), } - let args = create_machine_args(cpus, memory_mib, disk_gib, cfg!(target_os = "windows")); + let mut args = create_machine_args(cpus, memory_mib, disk_gib, cfg!(target_os = "windows")); + if let Some(image) = image { + args.extend(["--image".to_string(), image.to_string_lossy().into_owned()]); + } let refs = args.iter().map(String::as_str).collect::>(); match run(&refs) { Ok(_) => StepOutcome::went(Step::CreateMachine, format!("{MACHINE} created.")), @@ -793,6 +806,7 @@ mod tests { 2, 4096, 20, + None, || machine_exists_with(|| Ok(listing.into())), |_args| { init_called = true; @@ -814,6 +828,7 @@ mod tests { 2, 4096, 20, + None, || { machine_exists_with(|| { Err("podman machine list exited with status 125; stdout: denied".into()) @@ -851,6 +866,7 @@ mod tests { 2, 4096, 20, + None, || machine_exists_with(|| Ok(listing.into())), |_args| panic!("machine init must not run after malformed list output"), ); @@ -873,6 +889,7 @@ mod tests { 4, 8192, 64, + None, || Ok(false), |args| { captured = args.iter().map(|arg| (*arg).to_string()).collect(); @@ -887,6 +904,31 @@ mod tests { ); } + /// The machine image the first screen downloaded is what the machine is built from: Podman is + /// given the file, and pulls nothing from quay.io (see `early_downloads`). + #[test] + fn a_downloaded_machine_image_is_what_the_machine_is_built_from() { + let image = std::path::Path::new("/cache/podman-machine.aarch64.applehv.raw.zst"); + let mut captured = Vec::new(); + let result = create_machine_with( + 4, + 6144, + 60, + Some(image), + || Ok(false), + |args| { + captured = args.iter().map(|arg| (*arg).to_string()).collect(); + Ok(String::new()) + }, + ); + assert!(result.ok, "{result:?}"); + assert_eq!(&captured[..3], ["machine", "init", "openbot"]); + assert_eq!( + &captured[captured.len() - 2..], + ["--image", "/cache/podman-machine.aarch64.applehv.raw.zst"] + ); + } + #[test] fn windows_machine_init_enables_user_mode_networking() { assert_eq!( @@ -941,6 +983,7 @@ mod tests { 4, 6144, 60, + None, || machine_exists_with(|| Ok(listing.into())), |args| panic!("nothing runs against an existing machine: {args:?}"), ); diff --git a/desktop/src-tauri/src/deployment.rs b/desktop/src-tauri/src/deployment.rs index 300b7d1eb9..417ae6026f 100644 --- a/desktop/src-tauri/src/deployment.rs +++ b/desktop/src-tauri/src/deployment.rs @@ -593,6 +593,11 @@ const WAITS: &[std::time::Duration] = &RETRY_DELAYS; #[cfg(test)] const WAITS: &[std::time::Duration] = &[std::time::Duration::ZERO; RETRY_DELAYS.len()]; +/// The waits between attempts at a download: `RETRY_DELAYS`, or none in this crate's tests. +pub fn retry_waits() -> &'static [std::time::Duration] { + WAITS +} + /// How long connecting to the server may take, before anything is asked of it. const CONNECT_TIMEOUT: std::time::Duration = std::time::Duration::from_secs(15); diff --git a/desktop/src-tauri/src/desktop_stall.rs b/desktop/src-tauri/src/desktop_stall.rs index 6b06fd2eb9..8a21687f72 100644 --- a/desktop/src-tauri/src/desktop_stall.rs +++ b/desktop/src-tauri/src/desktop_stall.rs @@ -70,6 +70,23 @@ fn row_running_at( } } +/// A download the running row `step` waits on brought in more bytes: the machine image or OpenBot's +/// images the first screen started (`early_downloads`). Its clock restarts, as a moving download +/// restarts the clock of the CLI step it holds up (`follow_cli`). +pub fn moved(app: &tauri::AppHandle, step: &str) { + moved_at(app, step, Instant::now()); +} + +fn moved_at(app: &tauri::AppHandle, step: &str, now: Instant) { + let Some(stalls) = app.try_state::() else { + return; + }; + let mut steps = lock(&stalls.steps); + if steps.timing(telemetry::StallPart::Start) == Some(step) { + steps.moved(telemetry::StallPart::Start, now); + } +} + /// A setup row finished, or failed. pub fn row_done(app: &tauri::AppHandle, step: &str) { let Some(stalls) = app.try_state::() else { @@ -450,6 +467,27 @@ mod tests { assert!(sink.next(Duration::from_secs(2)).is_none(), "said twice"); } + /// A step that waits on a download the first screen started is not stuck while its bytes come + /// in: each one restarts its clock. Once they stop coming, it is said as any step is. + #[test] + fn a_step_whose_download_keeps_moving_is_not_recorded() { + let sink = Sink::start(); + let app = app("stall-moving", true, &sink); + let app = app.handle(); + let start = Instant::now(); + row_running_at(app, "installation", "Finishing the installation.", start); + // Ten minutes of a download moving under a step that typically takes 4 seconds. + for second in (0..=600).step_by(5) { + moved_at(app, "installation", at(start, second)); + tick(app, at(start, second), Some(true)); + } + assert!(app.state::().queued().is_empty()); + // A row that is not the one being timed restarts nothing. + moved_at(app, "services", at(start, 700)); + tick(app, at(start, 730), Some(true)); + assert_eq!(app.state::().queued().len(), 1); + } + /// Minimised or hidden, the page may be held back by the system: nothing it misses is a stall. #[test] fn a_minimised_or_hidden_window_records_nothing() { diff --git a/desktop/src-tauri/src/early_downloads.rs b/desktop/src-tauri/src/early_downloads.rs new file mode 100644 index 0000000000..e00a7079b9 --- /dev/null +++ b/desktop/src-tauri/src/early_downloads.rs @@ -0,0 +1,1389 @@ +/*! +The big downloads, started when the first setup screen shows, before anybody clicks Install. + +Nothing used to download until Install was clicked, and then one item after another. On a new Mac +that was Podman's package, the password, then the 937 MB engine machine image from quay.io with no +progress shown, then 1.1 GB of OpenBot's images. None of those bytes needs an engine, a password, a +sign-in or the licence to download, only to be used. So when Welcome shows, this starts them all at +once, into a cache on disk, and Install finds them there: + +- **The engine machine image.** Podman's own `machine init` pulls it from quay.io itself, silently. + Here it is downloaded by OpenBot, with progress, and the machine is built from the local file + (`podman machine init --image `, which takes a local path and unpacks a `.zst` itself: + podman `pkg/machine/shim/diskpull` and `pkg/machine/compression`). Only for an installation that + will create OpenBot's machine with the Podman 6.1 the pins below belong to. +- **Podman's installer and Compose**, the files `install.rs` installs, through the same `fetch`. +- **The deployment** (`docker-compose.yml` and the image list), laid out as Install lays it out. +- **OpenBot's images**, pulled into an engine that already answers (Docker Desktop, or a Podman + machine that is running), with Install's own pull. With no engine answering yet, they download + into the cache instead, and Install loads them into the engine once it does (`early_images`). + +Nothing here asks for anything. The Podman package is downloaded, never installed: installing it +needs the person's password, and that waits for Install and the explained Continue +(`install::GoAhead`). + +EVERY FILE IS PINNED, AND RESUMES. A download goes to `.partial` and becomes `` only once +its SHA-256 is the pinned one. An app quit part-way leaves the partial file, and the next try asks +for the rest (`Range`, RFC 9110 section 14) and hashes the bytes it already has first. A server that +answers the whole file instead, or a partial file that hashes wrong at the end, starts again from +nothing. downloads.copilotkit.ai is asked first, once, and the original source after it, the way +`deployment::get` does. + +ONE DOWNLOAD PER FILE. Install and the early downloads ask for the same files. Each file has a lock, +so Install waits for a download already running rather than starting a second, and then finds the +file verified. + +THE DISK IS CHECKED FIRST. The back end's CLI wants 40 GiB free (`local_eval::MIN_ENGINE_FREE_DISK_BYTES`), +and on a Mac or Windows that space is taken from this computer's disk as the engine's virtual disk +grows. k3s's own line (evicting under 5% free) is measured inside that virtual disk, not here. With +less free than the back end needs and the downloads together, nothing starts, the window says so with +both numbers, and the check runs again every few seconds until there is room. +*/ + +use std::collections::BTreeMap; +use std::fs::File; +use std::io::{Read, Seek, Write}; +use std::path::{Path, PathBuf}; +use std::sync::{Arc, Mutex, MutexGuard, OnceLock}; +use std::time::{Duration, Instant}; + +use serde::Serialize; +use sha2::{Digest, Sha256}; + +use crate::engine::{Address, Engine}; + +/// The `podman/machine-os` release the machine images below come from, which is the minor version +/// of the Podman this app installs (`install::PODMAN`). Podman pulls `machine-os:`. +pub const MACHINE_OS: &str = "6.1"; + +/// One engine machine image: the disk artifact `quay.io/podman/machine-os:6.1` names for one +/// platform, read on 2026-11-09. `manifest` is the artifact's manifest, which is what the mirror +/// copies; `blob` is its one layer, the compressed disk, which is what is downloaded. +#[derive(Clone, Copy, Debug, PartialEq, Eq)] +pub struct MachineImage { + /// The file name the artifact gives the disk (`org.opencontainers.image.title`). + pub file: &'static str, + pub manifest: &'static str, + pub blob: &'static str, + pub size: u64, +} + +/// The repository the machine images are in, on quay.io and in downloads.copilotkit.ai's copy. +pub const MACHINE_OS_REPOSITORY: &str = "podman/machine-os"; + +/// The machine image Podman would pull on this kind of computer, or `None` where OpenBot never +/// creates a machine from one: Linux has no machine, and machine-os has no `applehv` disk for an +/// Intel Mac. Windows machines are WSL distributions, so theirs is the WSL root file system. +pub fn machine_image_for(os: &str, arch: &str) -> Option { + match (os, arch) { + ("macos", "aarch64") => Some(MachineImage { + file: "podman-machine.aarch64.applehv.raw.zst", + manifest: "sha256:e4b416b1758cb94b477d0c27e265b9180e8d52ee529df9276d726206ea69b883", + blob: "sha256:cd711cd582d301f11616281f0b0098de7cad26b050b7317b8bd3f3ac667c0d21", + size: 937_318_069, + }), + ("windows", "x86_64") => Some(MachineImage { + file: "podman-machine.x86_64.wsl.tar.zst", + manifest: "sha256:b3f5e2810eecb8778a6dc09283868c1255a06d7c156312c387616d41500d4175", + blob: "sha256:81ce79567e5aa7a1a122035b7ad14bd8e0dae8be009276427800e2cc62d753c7", + size: 251_517_341, + }), + ("windows", "aarch64") => Some(MachineImage { + file: "podman-machine.aarch64.wsl.tar.zst", + manifest: "sha256:80a2eb6bb3345c955d5feea5a43dfa6bfff02b0e54b013c1455ad55b23f4e999", + blob: "sha256:1e0107fb11ae52c9b5011ed9ec9661cc5e99b15efe22556eec721b5abced7394", + size: 236_832_353, + }), + _ => None, + } +} + +/// This computer's machine image, if it has one. +pub fn machine_image() -> Option { + machine_image_for(std::env::consts::OS, crate::windows::machine_arch()) +} + +/// Every machine image reference downloads.copilotkit.ai copies (`release_images`), by the +/// artifact manifest's digest. +pub fn machine_image_references() -> Vec { + [ + ("macos", "aarch64"), + ("windows", "x86_64"), + ("windows", "aarch64"), + ] + .iter() + .filter_map(|(os, arch)| machine_image_for(os, arch)) + .map(|image| format!("quay.io/{MACHINE_OS_REPOSITORY}@{}", image.manifest)) + .collect() +} + +impl MachineImage { + /// Where to download it from: downloads.copilotkit.ai's copy, then quay.io. + pub fn sources(&self) -> Vec { + vec![ + format!( + "{}/v2/quay.io/{MACHINE_OS_REPOSITORY}/blobs/{}", + crate::deployment::DOWNLOADS, + self.blob + ), + format!( + "https://quay.io/v2/{MACHINE_OS_REPOSITORY}/blobs/{}", + self.blob + ), + ] + } + + pub fn sha256(&self) -> &'static str { + self.blob.trim_start_matches("sha256:") + } +} + +/// Whether a Podman reporting `version` (`podman version 6.1.1`, `podman.exe version 6.1.1`) pulls +/// the machine-os release the pins are from. Another Podman pulls its own, and is left to. +pub fn podman_takes_pinned_machine_image(version: &str) -> bool { + version + .split_whitespace() + .last() + .is_some_and(|number| number.starts_with(&format!("{MACHINE_OS}."))) +} + +// --------------------------------------------------------------------------------------------- +// Downloading one pinned file. +// --------------------------------------------------------------------------------------------- + +/// A file to download and the SHA-256 it must have. +#[derive(Clone, Debug, PartialEq, Eq)] +pub struct Pinned { + /// Its name on disk. Named, not taken from a URL, so a redirect cannot choose it. + pub name: String, + /// Lower-case hex. + pub sha256: String, + /// Its size, when it is known before the download answers. + pub size: Option, + /// Asked in order: the first once, the last with `RETRY_DELAYS` between attempts. + pub sources: Vec, +} + +/// What `fetch` is told as the bytes on disk grow, and the size once it is known. +pub trait Progress: Sync { + fn size(&self, _size: u64) {} + fn bytes(&self, _bytes: u64) {} +} + +impl Progress for () {} + +/// The part of a download not yet finished, beside where the file goes. +pub fn partial_path(into: &Path, name: &str) -> PathBuf { + into.join(format!("{name}.partial")) +} + +/// One lock per file, so two downloads of the same file never run at once. +fn file_lock(path: &Path) -> Arc> { + static LOCKS: OnceLock>>>> = OnceLock::new(); + LOCKS + .get_or_init(Default::default) + .lock() + .unwrap_or_else(|held| held.into_inner()) + .entry(path.to_path_buf()) + .or_default() + .clone() +} + +fn hold(lock: &Mutex<()>) -> MutexGuard<'_, ()> { + lock.lock().unwrap_or_else(|held| held.into_inner()) +} + +/// The SHA-256 of a file, read in pieces: a machine image is most of a gigabyte. +pub fn file_sha256(path: &Path) -> std::io::Result { + let mut file = File::open(path)?; + let mut hasher = Sha256::new(); + let mut buffer = vec![0u8; 1 << 20]; + loop { + let read = file.read(&mut buffer)?; + if read == 0 { + break; + } + hasher.update(&buffer[..read]); + } + Ok(hex(&hasher.finalize())) +} + +fn hex(bytes: &[u8]) -> String { + bytes.iter().map(|byte| format!("{byte:02x}")).collect() +} + +/// `pinned` in `into`, downloaded if it is not there already with its pinned digest. +/// +/// A file there with the wrong digest is replaced: a half-written download far more often than an +/// attack, and either way never used. Waits for a download of the same file already running. +pub fn fetch(pinned: &Pinned, into: &Path, progress: &dyn Progress) -> Result { + fetch_waiting(pinned, into, progress, crate::deployment::retry_waits()) +} + +/// `fetch`, waiting `waits` between attempts at the last source. +pub fn fetch_waiting( + pinned: &Pinned, + into: &Path, + progress: &dyn Progress, + waits: &[Duration], +) -> Result { + let path = into.join(&pinned.name); + let lock = file_lock(&path); + let _only = hold(&lock); + + if let Some(size) = pinned.size { + progress.size(size); + } + if path.is_file() { + if file_sha256(&path).ok().as_deref() == Some(pinned.sha256.as_str()) { + let size = std::fs::metadata(&path).map(|meta| meta.len()).unwrap_or(0); + progress.size(size); + progress.bytes(size); + return Ok(path); + } + log::warn!( + "[download] {} is not the file pinned, downloading it again", + path.display() + ); + let _ = std::fs::remove_file(&path); + } + std::fs::create_dir_all(into) + .map_err(|error| format!("could not make {}: {error}", into.display()))?; + + let partial = partial_path(into, &pinned.name); + let mut last_error = String::from("there was nowhere to download it from"); + for (index, source) in pinned.sources.iter().enumerate() { + let last = index + 1 == pinned.sources.len(); + let waits: &[Duration] = if last { waits } else { &[] }; + match download_resuming(source, &partial, pinned, progress, waits) { + Ok(()) => { + std::fs::rename(&partial, &path) + .map_err(|error| format!("could not keep {}: {error}", path.display()))?; + return Ok(path); + } + Err(error) => { + if !last { + log::warn!( + "[download] {error}; downloading {} instead", + pinned.sources[index + 1] + ); + } + last_error = error; + } + } + } + Err(last_error) +} + +/// The bytes of a partial file so far, hashed, so a download can carry on from them. +struct Resumed { + file: File, + hasher: Sha256, + bytes: u64, +} + +impl Resumed { + fn open(partial: &Path) -> Result { + let mut file = std::fs::OpenOptions::new() + .create(true) + .truncate(false) + .read(true) + .write(true) + .open(partial) + .map_err(|error| format!("could not write {}: {error}", partial.display()))?; + let mut hasher = Sha256::new(); + let mut bytes = 0u64; + let mut buffer = vec![0u8; 1 << 20]; + loop { + let read = file + .read(&mut buffer) + .map_err(|error| format!("could not read {}: {error}", partial.display()))?; + if read == 0 { + break; + } + hasher.update(&buffer[..read]); + bytes += read as u64; + } + Ok(Self { + file, + hasher, + bytes, + }) + } + + /// Back to nothing: the server sent the whole file, or what was there was not the file. + fn restart(&mut self) -> Result<(), String> { + self.file + .set_len(0) + .and_then(|()| self.file.rewind()) + .map_err(|error| format!("could not start the download again: {error}"))?; + self.hasher = Sha256::new(); + self.bytes = 0; + Ok(()) + } +} + +/// Why one attempt failed, and so whether another could succeed. +enum Failed { + ForNow(String), + ForGood(String), +} + +/// `source` into `partial`, carrying on from what is there, with an attempt more than `waits`. +/// Done when the whole file is there with the pinned digest. +fn download_resuming( + source: &str, + partial: &Path, + pinned: &Pinned, + progress: &dyn Progress, + waits: &[Duration], +) -> Result<(), String> { + let mut attempts = 1; + loop { + let result = attempt(source, partial, pinned, progress); + match result { + Ok(()) => return Ok(()), + Err(Failed::ForGood(error)) => return Err(error), + Err(Failed::ForNow(error)) => match waits.get(attempts - 1) { + Some(wait) => { + log::warn!( + "[download] attempt {attempts} of {}: {error}; trying again", + waits.len() + 1 + ); + std::thread::sleep(*wait); + attempts += 1; + } + None if attempts == 1 => return Err(error), + None => return Err(format!("{error} ({attempts} attempts)")), + }, + } + } +} + +/// How long connecting may take, and how long a download may go without a byte arriving: the same +/// as `deployment::get`'s, which has no limit on the whole download, only on silence. +const CONNECT_TIMEOUT: Duration = Duration::from_secs(15); +const STALL_TIMEOUT: Duration = Duration::from_secs(30); + +fn client() -> Result { + reqwest::blocking::Client::builder() + .user_agent("openbot-desktop") + .connect_timeout(CONNECT_TIMEOUT) + .timeout(STALL_TIMEOUT) + .build() + .map_err(|error| Failed::ForGood(format!("could not prepare the download: {error}"))) +} + +fn attempt( + source: &str, + partial: &Path, + pinned: &Pinned, + progress: &dyn Progress, +) -> Result<(), Failed> { + let mut have = Resumed::open(partial).map_err(Failed::ForGood)?; + let client = client()?; + let mut token: Option = None; + let mut response = loop { + let mut request = client.get(source); + if have.bytes > 0 { + request = request.header(reqwest::header::RANGE, format!("bytes={}-", have.bytes)); + } + if let Some(token) = &token { + request = request.bearer_auth(token); + } + let response = request + .send() + .map_err(|error| Failed::ForNow(format!("could not reach {source}: {error}")))?; + // A registry answers an anonymous download with the token to ask for (quay.io does). + if response.status() == reqwest::StatusCode::UNAUTHORIZED && token.is_none() { + token = Some(anonymous_token(&client, &response).ok_or_else(|| { + Failed::ForGood(format!( + "{source} asked for credentials OpenBot does not have" + )) + })?); + continue; + } + break response; + }; + + let status = response.status(); + if status == reqwest::StatusCode::RANGE_NOT_SATISFIABLE { + // What is there is as long as the file or longer, and was not the file. + have.restart().map_err(Failed::ForGood)?; + return Err(Failed::ForNow(format!( + "{source} had nothing after the part already downloaded" + ))); + } + if !status.is_success() { + let error = format!("{source} answered {status}"); + return Err( + if status.is_server_error() || status == reqwest::StatusCode::TOO_MANY_REQUESTS { + Failed::ForNow(error) + } else { + Failed::ForGood(error) + }, + ); + } + let resumes = status == reqwest::StatusCode::PARTIAL_CONTENT + && response + .headers() + .get(reqwest::header::CONTENT_RANGE) + .and_then(|range| range.to_str().ok()) + .is_some_and(|range| range.starts_with(&format!("bytes {}-", have.bytes))); + if !resumes && have.bytes > 0 { + // The whole file, from the start: what was there is replaced. + log::info!( + "[download] {source} sent the whole file, so {} starts again", + pinned.name + ); + have.restart().map_err(Failed::ForGood)?; + } + if have.bytes > 0 { + log::info!( + "[download] {} from {source}, carrying on after {} bytes", + pinned.name, + have.bytes + ); + } else { + log::info!("[download] {} from {source}", pinned.name); + } + have.file + .seek(std::io::SeekFrom::Start(have.bytes)) + .map_err(|error| Failed::ForGood(format!("could not write the download: {error}")))?; + if pinned.size.is_none() { + if let Some(length) = response.content_length() { + progress.size(have.bytes + length); + } + } + progress.bytes(have.bytes); + + let mut buffer = vec![0u8; 256 * 1024]; + loop { + let read = match response.read(&mut buffer) { + Ok(0) => break, + Ok(read) => read, + Err(error) => { + let _ = have.file.flush(); + return Err(Failed::ForNow(format!( + "the download stopped after {} bytes: {error}", + have.bytes + ))); + } + }; + have.file + .write_all(&buffer[..read]) + .map_err(|error| Failed::ForGood(format!("could not write the download: {error}")))?; + have.hasher.update(&buffer[..read]); + have.bytes += read as u64; + progress.bytes(have.bytes); + if pinned.size.is_some_and(|size| have.bytes > size) { + have.restart().map_err(Failed::ForGood)?; + return Err(Failed::ForGood(format!( + "{source} sent more than the {} bytes {} has", + pinned.size.unwrap_or_default(), + pinned.name + ))); + } + } + have.file + .flush() + .map_err(|error| Failed::ForGood(format!("could not write the download: {error}")))?; + let got = hex(&have.hasher.clone().finalize()); + if got == pinned.sha256 { + log::info!( + "[download] {}: {} bytes, sha256 {got} as pinned", + pinned.name, + have.bytes + ); + } + if got != pinned.sha256 { + have.restart().map_err(Failed::ForGood)?; + return Err(Failed::ForGood(format!( + "{} from {source} is not what OpenBot expected: sha256 {got}, expected {}", + pinned.name, pinned.sha256 + ))); + } + Ok(()) +} + +/// The anonymous token a registry's `WWW-Authenticate: Bearer` challenge offers, as an engine asks +/// for it (https://distribution.github.io/distribution/spec/auth/token/). See `registry`. +fn anonymous_token( + client: &reqwest::blocking::Client, + response: &reqwest::blocking::Response, +) -> Option { + let challenge = response + .headers() + .get(reqwest::header::WWW_AUTHENTICATE)? + .to_str() + .ok()?; + #[derive(serde::Deserialize)] + struct Token { + token: Option, + access_token: Option, + } + let answer: Token = client + .get(crate::registry::token_url(challenge)?) + .send() + .ok()? + .json() + .ok()?; + answer.token.or(answer.access_token) +} + +// --------------------------------------------------------------------------------------------- +// What the window shows. +// --------------------------------------------------------------------------------------------- + +/// Where the early downloads are, as the setup window says it. +#[derive(Clone, Debug, Default, PartialEq, Eq, Serialize)] +#[serde(rename_all = "camelCase")] +pub struct Summary { + pub state: State, + /// Downloaded so far, over every download. + pub bytes: u64, + /// Every download together, once each one's size is known. + #[serde(skip_serializing_if = "Option::is_none")] + pub size: Option, + #[serde(skip_serializing_if = "Option::is_none")] + pub bytes_per_second: Option, + /// Free on this computer's disk, and what setup needs, when the disk is short. + #[serde(skip_serializing_if = "Option::is_none")] + pub free: Option, + #[serde(skip_serializing_if = "Option::is_none")] + pub needed: Option, +} + +#[derive(Clone, Copy, Debug, Default, PartialEq, Eq, Serialize)] +#[serde(rename_all = "kebab-case")] +pub enum State { + /// Not started, or working out what to download. + #[default] + Idle, + /// Not enough free disk; waiting for some. + Short, + Downloading, + /// Everything planned is on disk, or in the engine. + Done, + /// Something did not download. Install downloads it again when it gets there. + Failed, + /// There is nothing to download early: this installation has everything already. + Nothing, +} + +/// One download in the table. +#[derive(Clone, Debug, Default)] +struct Entry { + size: Option, + bytes: u64, + finished: bool, + failed: bool, +} + +/// The table the window's line is made from, and the speed it is measured at. +#[derive(Default)] +struct Table { + state: State, + entries: BTreeMap<&'static str, Entry>, + /// What OpenBot's image pulls report: Install shows the same while it waits for them. + images: Option, + free: Option, + needed: Option, + speed: Speed, +} + +fn table() -> MutexGuard<'static, Table> { + static TABLE: OnceLock> = OnceLock::new(); + TABLE + .get_or_init(Default::default) + .lock() + .unwrap_or_else(|held| held.into_inner()) +} + +/// Download speed over the last few seconds. +#[derive(Default)] +struct Speed { + samples: std::collections::VecDeque<(Instant, u64)>, +} + +impl Speed { + const WINDOW: Duration = Duration::from_secs(8); + + fn observe(&mut self, now: Instant, bytes: u64) -> Option { + self.samples.push_back((now, bytes)); + while self + .samples + .front() + .is_some_and(|(at, _)| now.saturating_duration_since(*at) > Self::WINDOW) + { + self.samples.pop_front(); + } + let (first_at, first_bytes) = *self.samples.front()?; + let seconds = now.saturating_duration_since(first_at).as_secs_f64(); + (seconds >= 1.0).then(|| (bytes.saturating_sub(first_bytes) as f64 / seconds) as u64) + } +} + +impl Table { + fn summary(&mut self, now: Instant) -> Summary { + let bytes = self.entries.values().map(|entry| entry.bytes).sum::(); + let size = self + .entries + .values() + .map(|entry| entry.size) + .sum::>() + .filter(|size| *size > 0); + let bytes_per_second = self.speed.observe(now, bytes); + Summary { + state: self.state, + bytes, + size, + bytes_per_second: bytes_per_second.filter(|_| self.state == State::Downloading), + free: self.free.filter(|_| self.state == State::Short), + needed: self.needed.filter(|_| self.state == State::Short), + } + } + + /// Done or failed once every download has finished. + fn settle(&mut self) { + if self.state != State::Downloading || self.entries.values().any(|entry| !entry.finished) { + return; + } + self.state = if self.entries.values().any(|entry| entry.failed) { + State::Failed + } else { + State::Done + }; + } +} + +/// Where the early downloads are now. +pub fn summary() -> Summary { + table().summary(Instant::now()) +} + +/// One download's progress, into the table. +pub(crate) struct Tracked(pub(crate) &'static str); + +impl Progress for Tracked { + fn size(&self, size: u64) { + table().entries.entry(self.0).or_default().size = Some(size); + } + fn bytes(&self, bytes: u64) { + table().entries.entry(self.0).or_default().bytes = bytes; + } +} + +fn track(key: &'static str, size: Option) { + let mut table = table(); + let entry = table.entries.entry(key).or_default(); + if entry.size.is_none() { + entry.size = size; + } +} + +fn finish(key: &'static str, outcome: Result<(), String>) { + let mut table = table(); + let entry = table.entries.entry(key).or_default(); + entry.finished = true; + if let Err(error) = outcome { + log::warn!("[early downloads] {key}: {error}"); + entry.failed = true; + } else { + // Done is all of it, whatever the size was thought to be. + entry.size = Some(entry.size.unwrap_or(entry.bytes).max(entry.bytes)); + entry.bytes = entry.size.unwrap_or(entry.bytes); + log::info!("[early downloads] {key} is ready"); + } + table.settle(); +} + +// --------------------------------------------------------------------------------------------- +// What to download. +// --------------------------------------------------------------------------------------------- + +/// What this computer has, read once when the first screen shows. +#[derive(Clone, Debug, PartialEq, Eq)] +pub struct Facts { + pub os: &'static str, + /// The engine Install will use, as `install::choose` decides it. + pub choice: crate::install::Choice, + /// The Podman on this computer answers `--version` with this, when there is one. + pub podman_version: Option, + /// OpenBot's Podman machine is already there. + pub machine_exists: bool, + /// An engine answers now, so images can go straight into it. + pub answering: Option
, + /// This folder has finished an installation or a start before. + pub installed_before: bool, + pub machine_image: Option, +} + +/// What to download early. +#[derive(Clone, Debug, Default, PartialEq, Eq)] +pub struct Plan { + pub machine_image: Option, + pub podman: bool, + pub compose: bool, + pub deployment: bool, + /// The engine to pull OpenBot's images into. + pub images: Option
, + /// With no engine answering, OpenBot's images downloaded into the cache, to load once one + /// does (`early_images`), and the Compose that names them. + pub offline_images: Option, +} + +/// Which Compose names OpenBot's images when no engine answers: `config --images` reads files. +#[derive(Clone, Copy, Debug, PartialEq, Eq)] +pub enum NamedBy { + /// OpenBot's own standalone Compose, downloaded with Podman's installer. + OwnCompose, + /// Docker Desktop's, which answers `compose config` while Docker Desktop is stopped. + Docker, +} + +impl Plan { + pub fn is_empty(&self) -> bool { + self == &Plan::default() + } + + /// About how much these downloads add to the disk, for the free-space check. The images and + /// Podman's files are estimates measured on 2026-11-09: their sizes are known only once their + /// downloads answer. + pub fn disk_bytes(&self) -> u64 { + const MB: u64 = 1_000_000; + self.machine_image.map_or(0, |image| image.size) + + if self.podman { 80 * MB } else { 0 } + + if self.compose { 35 * MB } else { 0 } + + if self.images.is_some() || self.offline_images.is_some() { + 1_200 * MB + } else { + 0 + } + } +} + +/// What to download early on a computer with these `facts`. +/// +/// Nothing for an installation that finished before: its machine and images are there. Otherwise +/// the deployment always, and: +/// +/// - Podman's installer and Compose when Install will install Podman (macOS and Windows; Linux's +/// Podman comes from its own packages); +/// - the machine image when Install will create OpenBot's machine with a Podman that pulls the +/// pinned machine-os release; +/// - OpenBot's images when an engine answers already. +pub fn plan(facts: &Facts) -> Plan { + use crate::install::Choice; + if facts.installed_before { + return Plan::default(); + } + let desktop = matches!(facts.os, "macos" | "windows"); + // A Podman too old for OpenBot is updated with the same installer (`engine_versions::ensure`). + let too_old = facts + .podman_version + .as_deref() + .and_then(crate::engine_versions::Version::find) + .is_some_and(|version| version < crate::engine_versions::PODMAN); + let installs_podman = desktop + && (facts.choice == Choice::InstallPodman + || (facts.choice == Choice::UsePodman && too_old)); + let creates_machine = desktop + && match facts.choice { + Choice::InstallPodman => true, + Choice::UsePodman => { + !facts.machine_exists + && facts + .podman_version + .as_deref() + .is_some_and(podman_takes_pinned_machine_image) + } + _ => false, + }; + // No engine answers: OpenBot's images download now, to load once one does. Docker Desktop + // stopped names them with its own Compose; anything else with the one OpenBot downloads. + let offline_images = (desktop && facts.answering.is_none()) + .then_some(match facts.choice { + Choice::InstallPodman | Choice::UsePodman => Some(NamedBy::OwnCompose), + Choice::OpenDockerDesktop => Some(NamedBy::Docker), + _ => None, + }) + .flatten(); + Plan { + machine_image: facts.machine_image.filter(|_| creates_machine), + podman: installs_podman, + compose: installs_podman || offline_images == Some(NamedBy::OwnCompose), + deployment: true, + images: facts.answering.clone(), + offline_images, + } +} + +/// This computer's facts, for the installation in `root`. +fn facts_here(root: &Path) -> Facts { + let saved = crate::preparation::saved_engine(root); + let choice = crate::install::choice_here(saved); + let podman_version = crate::engine::program(Engine::Podman).and_then(|_| { + let output = crate::engine::tool(Engine::Podman) + .arg("--version") + .output() + .ok()?; + output + .status + .success() + .then(|| String::from_utf8_lossy(&output.stdout).trim().to_string()) + }); + let machine_exists = choice == crate::install::Choice::UsePodman + && crate::acquire::machine_exists().unwrap_or(true); + let status = crate::engine::detect_saved(saved); + let answering = status + .address + .filter(|_| status.responding) + .and_then(|address| address.pin().ok()); + Facts { + os: std::env::consts::OS, + choice, + podman_version, + machine_exists, + answering, + installed_before: crate::preparation::installed_before(root), + machine_image: machine_image(), + } +} + +// --------------------------------------------------------------------------------------------- +// Running them. +// --------------------------------------------------------------------------------------------- + +/// Where downloaded engine files are kept: the folder `install.rs` keeps Podman and Compose in. +pub fn cache() -> PathBuf { + crate::acquire::download_dir(&crate::stack::default_root()) +} + +/// What the running app does with the early downloads: tell the window, and record a pull. +pub struct Hooks { + pub said: Box, + pub pulled: Box, +} + +/// Held while the deployment is laid out, by the early downloads or by Install. +pub fn hold_deployment() -> MutexGuard<'static, ()> { + static DEPLOYMENT: Mutex<()> = Mutex::new(()); + hold(&DEPLOYMENT) +} + +/// Held while OpenBot's images are pulled early. +fn images_lock() -> &'static Mutex<()> { + static IMAGES: Mutex<()> = Mutex::new(()); + &IMAGES +} + +/// Start the early downloads for the installation in `root`, once per run of the app. Answers at +/// once; the downloads run on threads of their own and say where they are through `hooks`. +pub fn start(root: PathBuf, hooks: Hooks) { + static STARTED: std::sync::atomic::AtomicBool = std::sync::atomic::AtomicBool::new(false); + if STARTED.swap(true, std::sync::atomic::Ordering::SeqCst) { + return; + } + let hooks = Arc::new(hooks); + std::thread::spawn(move || { + let facts = facts_here(&root); + let plan = plan(&facts); + log::info!("[early downloads] {facts:?} plans {plan:?}"); + let cache = cache(); + if plan.machine_image.is_none() { + // A machine image that is no longer needed is not kept: it is most of a gigabyte. + forget_machine_image_in(&cache); + } + let needed = needed_for(&plan); + let jobs = jobs(&root, &plan, &cache, &hooks); + let said = |summary| (hooks.said)(summary); + run(jobs, needed, || free_space(&cache), &said, DISK_RECHECK); + }); +} + +/// How long between two looks at the free disk, while it is short. +const DISK_RECHECK: Duration = Duration::from_secs(10); + +/// The free disk OpenBot needs for `plan`: the back end's 40 GiB, and the downloads themselves. +pub fn needed_for(plan: &Plan) -> u64 { + crate::local_eval::MIN_ENGINE_FREE_DISK_BYTES + plan.disk_bytes() +} + +/// One early download: its row in the table, its size when it is known, and the work. +pub struct Job { + key: &'static str, + size: Option, + /// Further rows this job finishes itself. + also: Vec<&'static str>, + work: Box Result<(), String> + Send>, +} + +impl Job { + /// A download of `pinned` into `into`, in the table as `key`. + pub fn file(key: &'static str, pinned: Pinned, into: PathBuf) -> Self { + Self { + key, + size: pinned.size, + also: Vec::new(), + work: Box::new(move || fetch(&pinned, &into, &Tracked(key)).map(drop)), + } + } +} + +/// The jobs for `plan`. +fn jobs(root: &Path, plan: &Plan, cache: &Path, hooks: &Arc) -> Vec { + let mut jobs = Vec::new(); + if let Some(image) = plan.machine_image { + jobs.push(Job::file( + "machine-image", + machine_pinned(&image), + cache.to_path_buf(), + )); + } + if plan.podman { + let cache = cache.to_path_buf(); + jobs.push(Job { + key: "podman", + size: None, + also: Vec::new(), + work: Box::new(move || { + crate::install::fetch_podman_installer(&cache, &Tracked("podman")).map(drop) + }), + }); + } + if plan.compose { + let cache = cache.to_path_buf(); + jobs.push(Job { + key: "compose", + size: None, + also: Vec::new(), + work: Box::new(move || { + crate::install::fetch_compose(&cache, &Tracked("compose")).map(drop) + }), + }); + } + if let (true, Some(named_by)) = (plan.deployment, plan.offline_images) { + let root = root.to_path_buf(); + let cache = cache.to_path_buf(); + jobs.push(Job { + key: "images", + size: None, + also: vec!["deployment"], + work: Box::new(move || { + let _only = hold(images_lock()); + let laid_out = lay_out_deployment(&root); + let ok = laid_out.is_ok(); + finish("deployment", laid_out); + if !ok { + return Err("the deployment did not download".into()); + } + download_offline_images(&root, &cache, named_by) + }), + }); + return jobs; + } + if plan.deployment { + let root = root.to_path_buf(); + match plan.images.clone() { + // The deployment is under a megabyte: a row so "done" waits for it, with no bytes. + None => jobs.push(Job { + key: "deployment", + size: Some(0), + also: Vec::new(), + work: Box::new(move || lay_out_deployment(&root)), + }), + // The images are named by the deployment's image list, so they come after it, and + // Install waits for them from the start (`wait_for_images`). + Some(engine) => { + let hooks = hooks.clone(); + jobs.push(Job { + key: "images", + size: None, + also: vec!["deployment"], + work: Box::new(move || { + let _only = hold(images_lock()); + let laid_out = lay_out_deployment(&root); + let ok = laid_out.is_ok(); + finish("deployment", laid_out); + if !ok { + return Err("the deployment did not download".into()); + } + pull_images(&engine, &root, &hooks) + }), + }); + } + } + } + jobs +} + +/// Run `jobs` once `free` says there is `needed` free, all at once, saying where they are through +/// `said` while they run. With less free, nothing starts: the window is told both numbers, and +/// `free` is asked again every `recheck`. +pub fn run( + jobs: Vec, + needed: u64, + free: impl Fn() -> Option, + said: &(dyn Fn(Summary) + Sync), + recheck: Duration, +) { + if jobs.is_empty() { + table().state = State::Nothing; + said(summary()); + return; + } + loop { + match free() { + Some(free) if free < needed => { + { + let mut table = table(); + table.state = State::Short; + table.free = Some(free); + table.needed = Some(needed); + } + log::warn!( + "[early downloads] {free} bytes free, {needed} needed: waiting for space" + ); + said(summary()); + std::thread::sleep(recheck); + } + _ => break, + } + } + + log::info!( + "[early downloads] downloading before Install: {}", + jobs.iter() + .map(|job| job.key) + .collect::>() + .join(", ") + ); + { + let mut table = table(); + table.state = State::Downloading; + table.free = None; + table.needed = None; + for job in &jobs { + for key in std::iter::once(job.key).chain(job.also.iter().copied()) { + let entry = table.entries.entry(key).or_default(); + entry.size = entry + .size + .or(if key == job.key { job.size } else { Some(0) }); + } + } + } + let ticking = std::sync::atomic::AtomicBool::new(true); + std::thread::scope(|scope| { + let workers: Vec<_> = jobs + .into_iter() + .map(|job| { + scope.spawn(move || { + let key = job.key; + let outcome = std::panic::catch_unwind(std::panic::AssertUnwindSafe(job.work)) + .unwrap_or_else(|_| Err("the download stopped unexpectedly".into())); + finish(key, outcome); + }) + }) + .collect(); + let ticking = &ticking; + scope.spawn(move || { + while ticking.load(std::sync::atomic::Ordering::SeqCst) { + said(summary()); + std::thread::sleep(Duration::from_millis(500)); + } + }); + for worker in workers { + let _ = worker.join(); + } + ticking.store(false, std::sync::atomic::Ordering::SeqCst); + }); + table().settle(); + said(summary()); +} + +fn machine_pinned(image: &MachineImage) -> Pinned { + Pinned { + name: image.file.to_string(), + sha256: image.sha256().to_string(), + size: Some(image.size), + sources: image.sources(), + } +} + +/// Lay the latest deployment out in `root`, as Install's deployment step does, unless it is there. +fn lay_out_deployment(root: &Path) -> Result<(), String> { + let _only = hold_deployment(); + // An installation on another release is Start's to move (`upgrade`), never this. + if crate::deployment::installed(root).is_some() { + return Ok(()); + } + let version = crate::deployment_release::resolve_version(root)?; + if crate::deployment::needs_fetch(root, &version) { + match crate::deployment::pinned().filter(|pin| pin.version == version) { + Some(pin) => crate::deployment::fetch_pinned(root, &pin)?, + None => crate::deployment::fetch(root, &version)?, + } + } + Ok(()) +} + +/// The images Install pulls for the default Bot, from the deployment laid out in `root`, asked of +/// Compose as Install asks it. Compose reads the release from a folder that is not an +/// installation, so nothing is decided or recorded for `root` before Install (`release_images` +/// reads a release the same way). +pub fn default_bot_images(engine: &Address, root: &Path) -> Result, String> { + let harness = crate::harness::HarnessChoice { + id: crate::harness::DEFAULT_HARNESS.into(), + agent_url: None, + }; + let nowhere = root.join(".openbot-early-downloads-nowhere"); + crate::preparation::installation_pull_set(engine, &nowhere, root, Some(&harness)).map_err( + |problem| { + format!( + "{}{}", + problem.said, + problem + .detail + .map(|detail| format!(" ({detail})")) + .unwrap_or_default() + ) + }, + ) +} + +/// OpenBot's images for the default Bot, named by the Compose `named_by` says, downloaded into the +/// cache for Install to load once the engine answers (`early_images`). +fn download_offline_images(root: &Path, cache: &Path, named_by: NamedBy) -> Result<(), String> { + let images = match named_by { + NamedBy::Docker => default_bot_images(&Address::new(Engine::Docker, None), root)?, + NamedBy::OwnCompose => { + let compose = crate::install::runnable_compose(cache, &())?; + let harness = crate::harness::HarnessChoice { + id: crate::harness::DEFAULT_HARNESS.into(), + agent_url: None, + }; + let nowhere = root.join(".openbot-early-downloads-nowhere"); + crate::preparation::installation_pull_set_by( + |settings| crate::stack::standalone_compose_command(&compose, &nowhere, settings), + &nowhere, + root, + Some(&harness), + ) + .map_err(|problem| { + format!( + "{}{}", + problem.said, + problem + .detail + .map(|detail| format!(" ({detail})")) + .unwrap_or_default() + ) + })? + } + }; + log::info!( + "[early images] downloading {} before any engine answers", + images.join(", ") + ); + crate::early_images::download( + &images, + &crate::early_images::Registries::openbot(), + crate::registry::host_architecture(), + &crate::early_images::folder(cache), + &Tracked("images"), + ) + .map(drop) +} + +/// Load the images downloaded before the engine answered into `engine`, once their download has +/// finished, saying how far it is through `said`. Install's pull then finds them there. +pub fn load_offline_images(engine: &Address, said: impl Fn(crate::pull_metrics::Downloads)) { + let _only = hold(images_lock()); + let loaded = + crate::early_images::load_downloaded(engine, &crate::early_images::folder(&cache()), said); + if loaded > 0 { + log::info!("[early images] {loaded} images loaded into the engine"); + } +} + +/// OpenBot's images for the default Bot, pulled into `engine` with Install's own pull. +fn pull_images(engine: &Address, root: &Path, hooks: &Hooks) -> Result<(), String> { + let images = default_bot_images(engine, root)?; + let present = { + let engine = engine.clone(); + move |image: &str| crate::preparation::image_present(&engine, image).unwrap_or(false) + }; + let sizes = crate::pull_metrics::size_images(&images, present); + crate::pull_metrics::pull_images_limited( + &images, + crate::pull_metrics::IMAGE_PULLS_AT_ONCE, + sizes, + |_, image, on_layer| { + crate::pull_metrics::pull_image(engine, image, on_layer, |metrics| { + (hooks.pulled)(metrics) + }) + }, + || Ok(()), + |downloads| { + let mut table = table(); + let entry = table.entries.entry("images").or_default(); + entry.bytes = downloads.bytes; + entry.size = downloads.size; + table.images = Some(downloads.clone()); + }, + ) + .map_err(|problem| problem.said) +} + +/// Wait for OpenBot's images to finish pulling early, if they are, saying where they are while it +/// waits. Install's own pull then finds them there. +pub fn wait_for_images(said: impl Fn(&crate::pull_metrics::Downloads)) { + let lock = images_lock(); + let mut pacer = crate::pull_metrics::Pacer::default(); + loop { + if let Ok(_done) = lock.try_lock() { + return; + } + // The early pull's own report, or the bytes of images downloading for an engine to come. + let images = { + let table = table(); + table.images.clone().or_else(|| { + table + .entries + .get("images") + .map(|entry| crate::pull_metrics::Downloads { + done: 0, + total: 0, + bytes: entry.bytes, + size: entry.size, + bytes_per_second: None, + }) + }) + }; + if let Some(downloads) = images { + if let Some(paced) = pacer.offer(Instant::now(), &downloads) { + said(&paced); + } + } + std::thread::sleep(Duration::from_millis(250)); + } +} + +/// The machine image for `podman machine init --image`, downloaded and checked, or `None` when +/// Podman should pull its own: no image for this computer, a Podman that pulls another machine-os +/// release, or a download that failed (said in the log). Waits for the early download if it is +/// running, saying how far it is. +pub fn machine_image_for_init( + said: impl Fn(crate::pull_metrics::Downloads) + Sync, +) -> Option { + let image = machine_image()?; + let version = crate::engine::tool(Engine::Podman) + .arg("--version") + .output() + .ok() + .map(|output| String::from_utf8_lossy(&output.stdout).trim().to_string())?; + if !podman_takes_pinned_machine_image(&version) { + log::info!("[early downloads] {version} pulls its own machine image"); + return None; + } + match machine_image_waiting(&machine_pinned(&image), &cache(), said) { + Ok(path) => Some(path), + Err(error) => { + log::warn!( + "[early downloads] the machine image did not download: {error}; Podman pulls its own" + ); + None + } + } +} + +/// `pinned` into `cache`, saying how far the download is every quarter of a second, whoever runs +/// it: this call, or the early download it waits for. +pub fn machine_image_waiting( + pinned: &Pinned, + cache: &Path, + said: impl Fn(crate::pull_metrics::Downloads) + Sync, +) -> Result { + let reporting = std::sync::atomic::AtomicBool::new(true); + track("machine-image", pinned.size); + std::thread::scope(|scope| { + let (reporting, said) = (&reporting, &said); + scope.spawn(move || { + let mut pacer = crate::pull_metrics::Pacer::default(); + while reporting.load(std::sync::atomic::Ordering::SeqCst) { + let entry = table().entries.get("machine-image").cloned(); + if let Some(entry) = entry { + // One file: its bytes, with no count of items beside them. + let downloads = crate::pull_metrics::Downloads { + done: 0, + total: 0, + bytes: entry.bytes, + size: entry.size, + bytes_per_second: None, + }; + if let Some(paced) = pacer.offer(Instant::now(), &downloads) { + said(paced); + } + } + std::thread::sleep(Duration::from_millis(250)); + } + }); + let fetched = fetch(pinned, cache, &Tracked("machine-image")); + reporting.store(false, std::sync::atomic::Ordering::SeqCst); + fetched + }) +} + +/// The machine is created: its image is in the machine now, and the download is not kept. +pub fn forget_machine_image() { + forget_machine_image_in(&cache()); +} + +fn forget_machine_image_in(cache: &Path) { + let Some(image) = machine_image() else { + return; + }; + for path in [cache.join(image.file), partial_path(cache, image.file)] { + match std::fs::remove_file(&path) { + Ok(()) => log::info!("[early downloads] removed {}", path.display()), + Err(error) if error.kind() == std::io::ErrorKind::NotFound => {} + Err(error) => log::warn!( + "[early downloads] could not remove {}: {error}", + path.display() + ), + } + } +} + +// --------------------------------------------------------------------------------------------- +// Free disk. +// --------------------------------------------------------------------------------------------- + +/// The bytes free to this account on the disk that holds `path`, or the nearest folder above it +/// that exists. `None` when it cannot be read, which is not judged. +pub fn free_space(path: &Path) -> Option { + let existing = path.ancestors().find(|folder| folder.exists())?; + free_space_at(existing) +} + +#[cfg(unix)] +fn free_space_at(path: &Path) -> Option { + use std::os::unix::ffi::OsStrExt; + let path = std::ffi::CString::new(path.as_os_str().as_bytes()).ok()?; + let mut stats: libc::statvfs = unsafe { std::mem::zeroed() }; + // SAFETY: `path` is NUL-terminated and outlives the call; `stats` is a statvfs it may write. + if unsafe { libc::statvfs(path.as_ptr(), &mut stats) } != 0 { + return None; + } + #[allow(clippy::unnecessary_cast)] + Some(stats.f_bavail as u64 * stats.f_frsize as u64) +} + +#[cfg(windows)] +fn free_space_at(path: &Path) -> Option { + crate::windows::free_space(&path.to_string_lossy()) +} + +#[cfg(test)] +pub(crate) mod tests; diff --git a/desktop/src-tauri/src/early_downloads/tests.rs b/desktop/src-tauri/src/early_downloads/tests.rs new file mode 100644 index 0000000000..2c69f56dc4 --- /dev/null +++ b/desktop/src-tauri/src/early_downloads/tests.rs @@ -0,0 +1,934 @@ +use super::*; +use crate::test_support::temp_root; +use std::io::{BufRead, BufReader}; +use std::net::{TcpListener, TcpStream}; +use std::sync::atomic::{AtomicUsize, Ordering}; + +/// One request as the server saw it. +#[derive(Clone, Debug)] +pub(crate) struct Seen { + pub(crate) path: String, + pub(crate) range: Option, + authorization: Option, +} + +/// What the server answers a request with. +pub(crate) enum Reply { + /// `status`, these headers, then `body`. With `cut`, only that many bytes of the body are sent + /// before the connection closes, under a `Content-Length` for the whole of it. + Send { + status: &'static str, + headers: Vec<(String, String)>, + body: Vec, + cut: Option, + }, +} + +/// A loopback HTTP/1.1 server that answers each request with `answer`, and keeps what it was asked. +pub(crate) struct Server { + address: String, + seen: Arc>>, + body_bytes: Arc, + /// Loopback ports are shared with the port tests: held while this server listens. + _ports: MutexGuard<'static, ()>, +} + +impl Server { + pub(crate) fn start(answer: impl Fn(&Seen, usize) -> Reply + Send + Sync + 'static) -> Self { + let ports = crate::test_support::hold_ports(); + let listener = below_the_ephemeral_ports(); + let address = format!("http://{}", listener.local_addr().unwrap()); + let seen = Arc::new(Mutex::new(Vec::new())); + let body_bytes = Arc::new(AtomicUsize::new(0)); + let answer = Arc::new(answer); + { + let (seen, body_bytes) = (seen.clone(), body_bytes.clone()); + std::thread::spawn(move || { + for stream in listener.incoming().flatten() { + let (seen, body_bytes, answer) = + (seen.clone(), body_bytes.clone(), answer.clone()); + std::thread::spawn(move || serve(stream, &seen, &body_bytes, &*answer)); + } + }); + } + Self { + address, + seen, + body_bytes, + _ports: ports, + } + } + + pub(crate) fn url(&self, path: &str) -> String { + format!("{}{path}", self.address) + } + + pub(crate) fn seen(&self) -> Vec { + self.seen.lock().unwrap().clone() + } + + /// Body bytes sent, over every answer. + pub(crate) fn sent(&self) -> usize { + self.body_bytes.load(Ordering::SeqCst) + } +} + +/** +A listener on a port below the range the system hands out for port 0. + +This server closes each connection first, so each one leaves its port in TIME_WAIT for a while after +the test. On a port the system hands out, that broke `env`'s port tests, which bind port 0 and then +probe the port they got without SO_REUSEADDR: the probe found it taken. Below that range, nothing +asks the system for it. +*/ +fn below_the_ephemeral_ports() -> TcpListener { + let start = 20_000 + (std::process::id() % 8_000) as u16; + (0..2_000u16) + .map(|step| 20_000 + (start - 20_000 + step * 7) % 10_000) + .find_map(|port| TcpListener::bind(("127.0.0.1", port)).ok()) + .expect("a free port from 20000 to 29999") +} + +fn serve( + stream: TcpStream, + seen: &Mutex>, + body_bytes: &AtomicUsize, + answer: &(dyn Fn(&Seen, usize) -> Reply + Send + Sync), +) { + let mut reader = BufReader::new(stream.try_clone().unwrap()); + let mut first = String::new(); + if reader.read_line(&mut first).is_err() || first.is_empty() { + return; + } + let path = first + .split_whitespace() + .nth(1) + .unwrap_or_default() + .to_string(); + let (mut range, mut authorization) = (None, None); + loop { + let mut line = String::new(); + if reader.read_line(&mut line).is_err() || line.trim().is_empty() { + break; + } + if let Some((name, value)) = line.split_once(':') { + match name.trim().to_ascii_lowercase().as_str() { + "range" => range = Some(value.trim().to_string()), + "authorization" => authorization = Some(value.trim().to_string()), + _ => {} + } + } + } + let request = Seen { + path, + range, + authorization, + }; + let index = { + let mut seen = seen.lock().unwrap(); + seen.push(request.clone()); + seen.len() - 1 + }; + let Reply::Send { + status, + headers, + body, + cut, + } = answer(&request, index); + let mut stream = stream; + let mut head = format!( + "HTTP/1.1 {status}\r\nContent-Length: {}\r\nConnection: close\r\n", + body.len() + ); + for (name, value) in headers { + head.push_str(&format!("{name}: {value}\r\n")); + } + head.push_str("\r\n"); + let _ = stream.write_all(head.as_bytes()); + let sent = cut.unwrap_or(body.len()).min(body.len()); + let _ = stream.write_all(&body[..sent]); + body_bytes.fetch_add(sent, Ordering::SeqCst); + let _ = stream.flush(); + let _ = stream.shutdown(std::net::Shutdown::Both); +} + +/// `body` from the start, or from where a `Range: bytes=N-` asks, as a server that resumes does. +pub(crate) fn ranged(request: &Seen, body: &[u8]) -> Reply { + match request + .range + .as_deref() + .and_then(|range| range.strip_prefix("bytes=")) + .and_then(|range| range.strip_suffix('-')) + .and_then(|from| from.parse::().ok()) + { + Some(from) if from < body.len() => Reply::Send { + status: "206 Partial Content", + headers: vec![( + "Content-Range".into(), + format!("bytes {from}-{}/{}", body.len() - 1, body.len()), + )], + body: body[from..].to_vec(), + cut: None, + }, + Some(_) => Reply::Send { + status: "416 Range Not Satisfiable", + headers: vec![], + body: vec![], + cut: None, + }, + None => whole(body, None), + } +} + +pub(crate) fn whole(body: &[u8], cut: Option) -> Reply { + Reply::Send { + status: "200 OK", + headers: vec![], + body: body.to_vec(), + cut, + } +} + +fn refused(status: &'static str) -> Reply { + Reply::Send { + status, + headers: vec![], + body: vec![], + cut: None, + } +} + +/// A file of `size` bytes that is not all one byte, and the pin for it. +pub(crate) fn file(size: usize) -> (Vec, String) { + let body: Vec = (0..size).map(|n| (n * 7 % 251) as u8).collect(); + let sha = hex(&Sha256::digest(&body)); + (body, sha) +} + +fn pinned(name: &str, sha256: &str, size: usize, sources: Vec) -> Pinned { + Pinned { + name: name.into(), + sha256: sha256.into(), + size: Some(size as u64), + sources, + } +} + +const NO_WAIT: [Duration; 0] = []; + +/// The table is one per process. Tests that read it run one at a time, each from an empty one. +fn table_to_myself() -> MutexGuard<'static, ()> { + static TABLE_TEST: Mutex<()> = Mutex::new(()); + let held = TABLE_TEST.lock().unwrap_or_else(|held| held.into_inner()); + *table() = Table::default(); + held +} + +/// The app was quit part-way through a download. The next launch asks only for the rest, carries +/// on hashing from the bytes already there, and keeps the file only once it is the pinned one. +#[test] +fn a_download_stopped_part_way_carries_on_from_the_bytes_already_there() { + let (body, sha) = file(300_000); + let served = body.clone(); + let server = Server::start(move |request, index| match index { + // The first launch: the connection drops two fifths of the way through. + 0 => whole(&served, Some(120_000)), + _ => ranged(request, &served), + }); + let cache = temp_root("early-resume"); + let pin = pinned("disk.raw.zst", &sha, body.len(), vec![server.url("/disk")]); + + fetch_waiting(&pin, &cache, &(), &NO_WAIT).expect_err("the first download was cut off"); + assert!( + !cache.join("disk.raw.zst").exists(), + "an unfinished file is never kept" + ); + assert_eq!( + std::fs::metadata(partial_path(&cache, "disk.raw.zst")) + .unwrap() + .len(), + 120_000 + ); + + // The next launch. + let path = fetch_waiting(&pin, &cache, &(), &NO_WAIT).expect("the rest of the download"); + assert_eq!(std::fs::read(&path).unwrap(), body); + assert_eq!(file_sha256(&path).unwrap(), sha); + assert!(!partial_path(&cache, "disk.raw.zst").exists()); + let seen = server.seen(); + assert_eq!(seen[1].range.as_deref(), Some("bytes=120000-"), "{seen:?}"); + // Nothing was downloaded twice. + assert_eq!(server.sent(), body.len()); + let _ = std::fs::remove_dir_all(&cache); +} + +/// A server that answers a range with the whole file starts the download again from nothing, +/// rather than appending the whole file to the part already there. +#[test] +fn a_server_that_sends_the_whole_file_again_starts_the_download_again() { + let (body, sha) = file(200_000); + let served = body.clone(); + let server = Server::start(move |_, _| whole(&served, None)); + let cache = temp_root("early-whole-again"); + std::fs::create_dir_all(&cache).unwrap(); + std::fs::write(partial_path(&cache, "f"), &body[..50_000]).unwrap(); + + let path = fetch_waiting( + &pinned("f", &sha, body.len(), vec![server.url("/f")]), + &cache, + &(), + &NO_WAIT, + ) + .expect("the whole file"); + assert_eq!(std::fs::read(&path).unwrap(), body); + assert_eq!(server.seen()[0].range.as_deref(), Some("bytes=50000-")); + let _ = std::fs::remove_dir_all(&cache); +} + +/// The part already there was not the start of the file (a disk error, another file). The digest +/// check at the end refuses it, nothing is kept under the file's name, and the next try downloads +/// the whole file. +#[test] +fn a_partial_download_that_is_not_the_file_is_refused_then_downloaded_whole() { + let (body, sha) = file(200_000); + let served = body.clone(); + let server = Server::start(move |request, _| ranged(request, &served)); + let cache = temp_root("early-wrong-partial"); + std::fs::create_dir_all(&cache).unwrap(); + std::fs::write(partial_path(&cache, "f"), vec![0u8; 80_000]).unwrap(); + let pin = pinned("f", &sha, body.len(), vec![server.url("/f")]); + + let refused = fetch_waiting(&pin, &cache, &(), &NO_WAIT).expect_err("a wrong file"); + assert!( + refused.contains("is not what OpenBot expected"), + "{refused}" + ); + assert!(!cache.join("f").exists()); + + let path = fetch_waiting(&pin, &cache, &(), &NO_WAIT).expect("the file, downloaded whole"); + assert_eq!(file_sha256(&path).unwrap(), sha); + let seen = server.seen(); + assert_eq!(seen[0].range.as_deref(), Some("bytes=80000-")); + assert_eq!(seen[1].range, None, "{seen:?}"); + let _ = std::fs::remove_dir_all(&cache); +} + +/// A file already there with its pinned digest is used as it is: no request at all. +#[test] +fn a_file_already_downloaded_is_not_downloaded_again() { + let (body, sha) = file(10_000); + let server = Server::start(|_, _| refused("500 Internal Server Error")); + let cache = temp_root("early-already"); + std::fs::create_dir_all(&cache).unwrap(); + std::fs::write(cache.join("f"), &body).unwrap(); + fetch_waiting( + &pinned("f", &sha, body.len(), vec![server.url("/f")]), + &cache, + &(), + &NO_WAIT, + ) + .expect("the file already here"); + assert!(server.seen().is_empty()); + let _ = std::fs::remove_dir_all(&cache); +} + +/// downloads.copilotkit.ai first, once; the file's own source when the copy is not there. +#[test] +fn the_mirror_is_asked_first_and_the_source_after_it() { + let (body, sha) = file(20_000); + let served = body.clone(); + let server = Server::start(move |request, _| { + if request.path.starts_with("/mirror/") { + refused("403 Forbidden") + } else { + whole(&served, None) + } + }); + let cache = temp_root("early-mirror"); + fetch_waiting( + &pinned( + "f", + &sha, + body.len(), + vec![server.url("/mirror/f"), server.url("/source/f")], + ), + &cache, + &(), + &NO_WAIT, + ) + .expect("the file from its source"); + let paths: Vec = server.seen().into_iter().map(|seen| seen.path).collect(); + assert_eq!(paths, ["/mirror/f", "/source/f"]); + let _ = std::fs::remove_dir_all(&cache); +} + +/// quay.io answers an anonymous blob download with the token to ask for, as every registry does. +#[test] +fn a_registry_that_asks_for_a_token_is_given_an_anonymous_one() { + let (body, sha) = file(20_000); + let served = body.clone(); + let realm = Arc::new(Mutex::new(String::new())); + let server = { + let realm = realm.clone(); + Server::start(move |request, _| { + if request.path.starts_with("/token") { + return Reply::Send { + status: "200 OK", + headers: vec![("Content-Type".into(), "application/json".into())], + body: br#"{"token":"anonymous-token"}"#.to_vec(), + cut: None, + }; + } + if request.authorization.as_deref() != Some("Bearer anonymous-token") { + return Reply::Send { + status: "401 Unauthorized", + headers: vec![( + "WWW-Authenticate".into(), + format!( + r#"Bearer realm="{}/token",service="quay.io",scope="repository:podman/machine-os:pull""#, + realm.lock().unwrap() + ), + )], + body: vec![], + cut: None, + }; + } + whole(&served, None) + }) + }; + *realm.lock().unwrap() = server.address.clone(); + let cache = temp_root("early-token"); + fetch_waiting( + &pinned( + "f", + &sha, + body.len(), + vec![server.url("/v2/podman/machine-os/blobs/sha256:x")], + ), + &cache, + &(), + &NO_WAIT, + ) + .expect("the blob, with the token"); + let seen = server.seen(); + assert!(seen + .iter() + .any(|seen| seen.path.starts_with("/token?") && seen.path.contains("scope=repository"))); + let _ = std::fs::remove_dir_all(&cache); +} + +/// Install asks for a file the first screen is already downloading. It waits for that download +/// and uses it: the file is downloaded once. +#[test] +fn two_asks_for_one_file_download_it_once() { + let (body, sha) = file(400_000); + let served = body.clone(); + let server = Server::start(move |_, _| { + std::thread::sleep(Duration::from_millis(300)); + whole(&served, None) + }); + let cache = temp_root("early-once"); + let pin = pinned("f", &sha, body.len(), vec![server.url("/f")]); + std::thread::scope(|scope| { + let first = scope.spawn(|| fetch_waiting(&pin, &cache, &(), &NO_WAIT)); + let second = scope.spawn(|| fetch_waiting(&pin, &cache, &(), &NO_WAIT)); + first.join().unwrap().expect("the first ask"); + second.join().unwrap().expect("the second ask"); + }); + assert_eq!(server.seen().len(), 1); + let _ = std::fs::remove_dir_all(&cache); +} + +/// With less free disk than the back end and the downloads need, nothing downloads: the window +/// is told how much is free and how much is needed. Once there is room, the downloads start. +#[test] +fn with_too_little_free_disk_nothing_downloads_until_there_is_room() { + let _table = table_to_myself(); + let (body, sha) = file(50_000); + let served = body.clone(); + let server = Server::start(move |_, _| whole(&served, None)); + let cache = temp_root("early-disk"); + let looks = AtomicUsize::new(0); + let said = Mutex::new(Vec::::new()); + let needed = needed_for(&Plan { + machine_image: machine_image_for("macos", "aarch64"), + deployment: true, + ..Plan::default() + }); + // 40 GiB for the back end, and the machine image. + assert_eq!(needed, 40 * 1024 * 1024 * 1024 + 937_318_069); + run( + vec![Job::file( + "machine-image", + pinned("f", &sha, body.len(), vec![server.url("/f")]), + cache.clone(), + )], + needed, + || { + // Nothing is asked of the server while the disk is short. + assert!(server.seen().is_empty()); + let look = looks.fetch_add(1, Ordering::SeqCst); + Some(if look < 2 { 12_000_000_000 } else { needed }) + }, + &|summary| said.lock().unwrap().push(summary), + Duration::ZERO, + ); + let said = said.into_inner().unwrap(); + assert_eq!( + said[0], + Summary { + state: State::Short, + free: Some(12_000_000_000), + needed: Some(needed), + ..Summary::default() + } + ); + let last = said.last().unwrap(); + assert_eq!(last.state, State::Done, "{said:?}"); + assert_eq!(last.bytes, body.len() as u64); + assert_eq!(last.size, Some(body.len() as u64)); + assert_eq!(file_sha256(&cache.join("f")).unwrap(), sha); + let _ = std::fs::remove_dir_all(&cache); +} + +/// The free space this computer reports for a real folder, as `df` would: some, and not more than +/// the disk holds. +#[test] +fn free_space_is_read_from_the_disk_that_holds_the_folder() { + let folder = temp_root("early-free").join("not-yet").join("made"); + let free = free_space(&folder).expect("the free space of the nearest folder that exists"); + assert!(free > 0); +} + +/// A download that fails is said as failed, and the others still finish. +#[test] +fn a_download_that_fails_is_said_and_the_others_finish() { + let _table = table_to_myself(); + let (body, sha) = file(30_000); + let served = body.clone(); + let server = Server::start(move |request, _| { + if request.path == "/missing" { + refused("404 Not Found") + } else { + whole(&served, None) + } + }); + let cache = temp_root("early-fails"); + let said = Mutex::new(Vec::::new()); + run( + vec![ + Job::file( + "podman", + pinned("missing", &sha, body.len(), vec![server.url("/missing")]), + cache.clone(), + ), + Job::file( + "compose", + pinned("there", &sha, body.len(), vec![server.url("/there")]), + cache.clone(), + ), + ], + 0, + || None, + &|summary| said.lock().unwrap().push(summary), + Duration::ZERO, + ); + assert_eq!( + said.into_inner().unwrap().last().unwrap().state, + State::Failed + ); + assert!(cache.join("there").is_file()); + let _ = std::fs::remove_dir_all(&cache); +} + +/// Install's machine step waits for the machine image the first screen started, saying how far it +/// is on its own row, and gets the verified file to build the machine from. +#[test] +fn the_machine_step_waits_for_the_image_and_gets_the_verified_file() { + let _table = table_to_myself(); + let (body, sha) = file(600_000); + let served = body.clone(); + let server = Server::start(move |request, _| { + std::thread::sleep(Duration::from_millis(400)); + ranged(request, &served) + }); + let cache = temp_root("early-machine"); + let pin = pinned( + "podman-machine.aarch64.applehv.raw.zst", + &sha, + body.len(), + vec![server.url("/blob")], + ); + let rows = Mutex::new(Vec::new()); + std::thread::scope(|scope| { + // The first screen's download, already running. + let early = scope.spawn(|| fetch(&pin, &cache, &Tracked("machine-image"))); + std::thread::sleep(Duration::from_millis(100)); + let path = machine_image_waiting(&pin, &cache, |downloads| { + rows.lock().unwrap().push(downloads) + }) + .expect("the machine image"); + assert_eq!(file_sha256(&path).unwrap(), sha); + assert_eq!(path, cache.join("podman-machine.aarch64.applehv.raw.zst")); + early.join().unwrap().expect("the early download"); + }); + let rows = rows.into_inner().unwrap(); + assert!(!rows.is_empty()); + assert!(rows.iter().all(|row| row.size == Some(body.len() as u64))); + assert_eq!(server.seen().len(), 1, "downloaded once"); + let _ = std::fs::remove_dir_all(&cache); +} + +/// What each kind of computer downloads before Install is clicked. +#[test] +fn what_downloads_early_depends_on_the_engine_install_will_use() { + use crate::install::Choice; + let mac = machine_image_for("macos", "aarch64"); + let docker = Address::new(Engine::Docker, None); + let facts = + |os, choice, podman: Option<&str>, machine_exists, answering: Option<&Address>| Facts { + os, + choice, + podman_version: podman.map(str::to_string), + machine_exists, + answering: answering.cloned(), + installed_before: false, + machine_image: if os == "macos" { mac } else { None }, + }; + // A new Mac with no engine: everything Install will need, OpenBot's images included, named by + // the Compose OpenBot downloads, to load once the engine answers. + assert_eq!( + plan(&facts("macos", Choice::InstallPodman, None, false, None)), + Plan { + machine_image: mac, + podman: true, + compose: true, + deployment: true, + images: None, + offline_images: Some(NamedBy::OwnCompose), + } + ); + // Docker Desktop answering: the images go straight into it, and no engine is downloaded. + assert_eq!( + plan(&facts( + "macos", + Choice::UseDocker, + None, + false, + Some(&docker) + )), + Plan { + deployment: true, + images: Some(docker.clone()), + ..Plan::default() + } + ); + // Podman 6.1 is here without OpenBot's machine: the machine image only. + assert_eq!( + plan(&facts( + "macos", + Choice::UsePodman, + Some("podman version 6.1.1"), + false, + None + )), + Plan { + machine_image: mac, + compose: true, + deployment: true, + offline_images: Some(NamedBy::OwnCompose), + ..Plan::default() + } + ); + // Another Podman pulls its own machine-os, and OpenBot's machine already there needs none. + for (version, exists) in [ + ("podman version 5.7.0", false), + ("podman version 6.1.1", true), + ] { + assert_eq!( + plan(&facts( + "macos", + Choice::UsePodman, + Some(version), + exists, + None + )) + .machine_image, + None, + "{version} {exists}" + ); + } + // A Podman too old for OpenBot is updated over with OpenBot's installer: it downloads early too. + let old = plan(&facts( + "macos", + Choice::UsePodman, + Some("podman version 5.7.0"), + true, + None, + )); + assert!(old.podman, "{old:?}"); + assert_eq!(old.machine_image, None); + // A Docker Desktop that is stopped is not opened before Install: its Compose names the images, + // which download to load once it answers. + assert_eq!( + plan(&facts( + "macos", + Choice::OpenDockerDesktop, + None, + false, + None + )), + Plan { + deployment: true, + offline_images: Some(NamedBy::Docker), + ..Plan::default() + } + ); + // A new Windows computer is the same as a new Mac, with the WSL image. + let windows = plan(&facts("windows", Choice::InstallPodman, None, false, None)); + assert_eq!(windows.offline_images, Some(NamedBy::OwnCompose)); + assert!(windows.compose && windows.podman); + // Linux's Podman comes from its own packages, and it has no machine. + assert_eq!( + plan(&facts("linux", Choice::InstallPodman, None, false, None)), + Plan { + deployment: true, + ..Plan::default() + } + ); + // An installation that finished before needs nothing early. + let mut done = facts("macos", Choice::InstallPodman, None, false, None); + done.installed_before = true; + assert!(plan(&done).is_empty()); +} + +/// The machine images are pinned for the Podman this app installs. Moving `install::PODMAN` to +/// another minor release means pinning that release's machine images. +#[test] +fn the_machine_images_belong_to_the_podman_this_app_installs() { + assert!( + crate::install::PODMAN.starts_with(&format!("{MACHINE_OS}.")), + "Podman {} pulls machine-os {}: pin its images", + crate::install::PODMAN, + crate::install::PODMAN + ); + assert!(podman_takes_pinned_machine_image("podman version 6.1.1")); + assert!(podman_takes_pinned_machine_image( + "podman.exe version 6.1.0" + )); + assert!(!podman_takes_pinned_machine_image("podman version 5.7.1")); + assert!(!podman_takes_pinned_machine_image("podman version 6.10.0")); + assert_eq!(machine_image_for("macos", "x86_64"), None); + assert_eq!(machine_image_for("linux", "x86_64"), None); + let mac = machine_image_for("macos", "aarch64").unwrap(); + assert_eq!( + mac.sources(), + [ + "https://downloads.copilotkit.ai/v2/quay.io/podman/machine-os/blobs/sha256:cd711cd582d301f11616281f0b0098de7cad26b050b7317b8bd3f3ac667c0d21", + "https://quay.io/v2/podman/machine-os/blobs/sha256:cd711cd582d301f11616281f0b0098de7cad26b050b7317b8bd3f3ac667c0d21", + ] + ); + assert_eq!(machine_image_references().len(), 3); +} + +/** +The machine is built from the downloaded image, with nothing asked of quay.io. + +Runs the real `podman machine init --image` against the pinned applehv image, with every proxy +pointed at a closed loopback port, so any download Podman tried would fail the init. Needs Podman +6.1 and the image on disk: `OPENBOT_LIVE_MACHINE_IMAGE=`. +The machine is removed afterwards. +*/ +#[test] +#[ignore = "runs podman machine init with the 937 MB machine image"] +fn a_machine_is_built_from_the_downloaded_image_without_asking_quay() { + let image = PathBuf::from( + std::env::var("OPENBOT_LIVE_MACHINE_IMAGE").expect("OPENBOT_LIVE_MACHINE_IMAGE"), + ); + let pinned = machine_image_for("macos", "aarch64").unwrap(); + assert_eq!(file_sha256(&image).unwrap(), pinned.sha256()); + let name = format!("openbot-early-{}", std::process::id()); + let closed = "http://127.0.0.1:9"; + let podman = |args: &[&str]| { + std::process::Command::new("podman") + .args(args) + .env("HTTP_PROXY", closed) + .env("HTTPS_PROXY", closed) + .env("http_proxy", closed) + .env("https_proxy", closed) + .env_remove("NO_PROXY") + .env_remove("no_proxy") + .output() + .unwrap() + }; + // `machine init` makes the new machine the default connection, and `rm` hands that on to + // another: the person's own default is put back afterwards. + let default_connection = || { + let listed = podman(&[ + "system", + "connection", + "list", + "--format", + "{{.Name}} {{.Default}}", + ]); + String::from_utf8_lossy(&listed.stdout) + .lines() + .find_map(|line| line.strip_suffix(" true").map(str::to_string)) + }; + let default_before = default_connection(); + let image_arg = image.to_string_lossy().into_owned(); + let init = podman(&[ + "machine", + "init", + &name, + "--cpus", + "2", + "--memory", + "2048", + "--disk-size", + "12", + "--rootful", + "--image", + &image_arg, + ]); + let listed = podman(&["machine", "inspect", &name]); + let removed = podman(&["machine", "rm", "--force", &name]); + if let Some(before) = + default_before.filter(|before| Some(before) != default_connection().as_ref()) + { + podman(&["system", "connection", "default", &before]); + } + assert!( + init.status.success(), + "init: {}{}", + String::from_utf8_lossy(&init.stdout), + String::from_utf8_lossy(&init.stderr) + ); + let said = String::from_utf8_lossy(&init.stdout); + assert!( + !said.contains("Looking up Podman Machine image"), + "Podman looked the image up: {said}" + ); + assert!(listed.status.success()); + assert!(removed.status.success()); +} + +/** +The real machine image, from the real sources: downloads.copilotkit.ai (which answers 403 until the +next sync copies it), then quay.io with its anonymous token and its CDN's redirect. Stopped part-way, +as a quit would, then carried on from the bytes on disk and checked against the pin. + +The Windows x86_64 image, 252 MB, so it runs on any computer: `cargo test -- --ignored live_machine`. +*/ +#[test] +#[ignore = "downloads 252 MB from quay.io"] +fn live_machine_image_downloads_from_quay_and_carries_on_after_a_stop() { + struct StopAt(u64); + impl Progress for StopAt { + fn bytes(&self, bytes: u64) { + assert!(bytes < self.0, "stopped part-way"); + } + } + let image = machine_image_for("windows", "x86_64").unwrap(); + let pinned = machine_pinned(&image); + let cache = temp_root("early-live-quay"); + let stopped = std::panic::catch_unwind(|| fetch(&pinned, &cache, &StopAt(60_000_000))); + assert!(stopped.is_err()); + let partial = std::fs::metadata(partial_path(&cache, image.file)) + .unwrap() + .len(); + assert!(partial >= 60_000_000, "{partial}"); + let started = Instant::now(); + let path = fetch(&pinned, &cache, &()).expect("the rest, from quay.io"); + eprintln!( + "carried on from {partial} bytes, finished in {:.1} s", + started.elapsed().as_secs_f64() + ); + assert_eq!(file_sha256(&path).unwrap(), image.sha256()); + let _ = std::fs::remove_dir_all(&cache); +} + +/** +The images the first screen pulls are the ones Install pulls for the default Bot, worked out without +an installation: nothing is recorded in the folder before Install. Runs `docker compose config` +against this repository's Compose file and a release's image list. +*/ +#[test] +#[ignore = "runs docker compose"] +fn live_the_first_screen_pulls_the_images_install_would() { + let root = temp_root("early-images"); + std::fs::create_dir_all(&root).unwrap(); + let repository = Path::new(env!("CARGO_MANIFEST_DIR")).join("../.."); + std::fs::copy( + repository.join("docker-compose.yml"), + root.join("docker-compose.yml"), + ) + .unwrap(); + std::fs::copy( + Path::new(env!("CARGO_MANIFEST_DIR")) + .join("test-fixtures/release-images/container-images-v0.3.3.json"), + crate::deployment::images_path(&root), + ) + .unwrap(); + let docker = Address::new(Engine::Docker, None); + let early = default_bot_images(&docker, &root).expect("the default Bot's images"); + // Nothing of an installation's was decided or written. + let written: Vec = std::fs::read_dir(&root) + .unwrap() + .map(|entry| entry.unwrap().file_name().to_string_lossy().into_owned()) + .collect(); + assert_eq!(written.len(), 2, "{written:?}"); + let harness = crate::harness::HarnessChoice { + id: crate::harness::DEFAULT_HARNESS.into(), + agent_url: None, + }; + let install = crate::preparation::installation_pull_set(&docker, &root, &root, Some(&harness)) + .expect("Install's images"); + assert_eq!(early, install); + assert!(early.iter().any(|image| image.contains("openbot-server@"))); + eprintln!("{early:#?}"); + let _ = std::fs::remove_dir_all(&root); +} + +/** +With no engine on the computer, OpenBot's own Compose, downloaded and run on its own, names the same +images Install pulls for the default Bot. Downloads Compose (about 30 MB) and runs `docker compose` +for Install's list to compare with. +*/ +#[test] +#[ignore = "downloads Compose and runs docker compose"] +fn live_openbots_own_compose_names_the_images_with_no_engine() { + let root = temp_root("early-own-compose"); + std::fs::create_dir_all(&root).unwrap(); + let repository = Path::new(env!("CARGO_MANIFEST_DIR")).join("../.."); + std::fs::copy( + repository.join("docker-compose.yml"), + root.join("docker-compose.yml"), + ) + .unwrap(); + std::fs::copy( + Path::new(env!("CARGO_MANIFEST_DIR")) + .join("test-fixtures/release-images/container-images-v0.3.3.json"), + crate::deployment::images_path(&root), + ) + .unwrap(); + let cache = temp_root("early-own-compose-cache"); + let compose = crate::install::runnable_compose(&cache, &()).expect("Compose"); + let harness = crate::harness::HarnessChoice { + id: crate::harness::DEFAULT_HARNESS.into(), + agent_url: None, + }; + let nowhere = root.join(".openbot-early-downloads-nowhere"); + let own = crate::preparation::installation_pull_set_by( + |settings| crate::stack::standalone_compose_command(&compose, &nowhere, settings), + &nowhere, + &root, + Some(&harness), + ) + .expect("the images, from OpenBot's own Compose"); + let docker = default_bot_images(&Address::new(Engine::Docker, None), &root).expect("Docker's"); + assert_eq!(own, docker); + assert_eq!(own.len(), 5, "{own:#?}"); + let _ = std::fs::remove_dir_all(&root); + let _ = std::fs::remove_dir_all(&cache); +} diff --git a/desktop/src-tauri/src/early_images.rs b/desktop/src-tauri/src/early_images.rs new file mode 100644 index 0000000000..200764f84d --- /dev/null +++ b/desktop/src-tauri/src/early_images.rs @@ -0,0 +1,625 @@ +/*! +OpenBot's own images, downloaded before any engine answers, then loaded into the engine once it does. + +On a computer with no engine yet (a new Mac or PC, or a Docker Desktop that is not running), the +first setup screen used to download Podman and the machine image early, but OpenBot's images (about +1.1 GB for the default Bot) still waited for the engine to start, and then downloaded one after +another behind it. Their bytes need no engine. So the first screen downloads them too: + +- **Which images.** The ones Install pulls for the default Bot (`preparation::installation_pull_set`), + asked of OpenBot's own standalone Compose when there is no engine to ask + (`stack::standalone_compose_command`), or of Docker's Compose when Docker Desktop is installed but + stopped. `config --images` reads files and reaches no engine. +- **The manifests**, read with `oci-client` (the ORAS project's OCI Distribution client) for this + computer's platform (`linux/`), from downloads.copilotkit.ai's copy first and the image's + own registry after it, as `image_mirror` asks an engine to. A manifest is checked against its + digest by the client. +- **The blobs**, every layer and config, through `early_downloads::fetch`: resumed after a quit, + checked against the digest the manifest names, from the mirror first. One copy of each blob, + shared by the images that use it. + +**Loading.** Once the engine answers, Install loads each image it does not have yet as an OCI image +layout streamed into `docker load` / `podman load` (both read an OCI layout's tar: moby +`image/tarexport` and containers/image `oci-archive`), then deletes the download. Install's own pull +runs after it as before. It finds the image's layers and config already in the engine, so it moves +only the manifests that give the image its pinned name: the containerd store finds the blobs in its +content store, the classic store sees the image's config and skips the layers (moby +`distribution/pull_v2.go`), and Podman reuses layers it holds by their compressed digest. Anything +that did not load is downloaded by that pull, as it always was. +*/ + +use std::collections::BTreeMap; +use std::io::Write; +use std::path::{Path, PathBuf}; +use std::sync::Mutex; + +use serde::{Deserialize, Serialize}; +use sha2::{Digest, Sha256}; + +use crate::early_downloads::{fetch, Pinned, Progress}; +use crate::engine::Address; +use crate::image_mirror::Mirror; + +/// Where downloaded images are kept, under the early downloads' cache. +pub fn folder(cache: &Path) -> PathBuf { + cache.join("images") +} + +fn blobs(folder: &Path) -> PathBuf { + folder.join("blobs").join("sha256") +} + +/// One part of an image: its digest, `sha256:`, and size. +#[derive(Clone, Debug, PartialEq, Eq, Serialize, Deserialize)] +pub struct Blob { + pub digest: String, + pub size: u64, +} + +impl Blob { + fn hex(&self) -> &str { + self.digest.trim_start_matches("sha256:") + } +} + +/// One image downloaded for this computer's platform: its manifest and every blob it names. +#[derive(Clone, Debug, PartialEq, Eq, Serialize, Deserialize)] +pub struct Downloaded { + /// The reference Install pulls it by. + pub image: String, + pub manifest: Blob, + #[serde(rename = "mediaType")] + pub media_type: String, + pub config: Blob, + pub layers: Vec, +} + +impl Downloaded { + fn parts(&self) -> impl Iterator { + std::iter::once(&self.config).chain(self.layers.iter()) + } +} + +/// Where a registry is, and how to ask it: production asks over https; the tests over http on +/// loopback. +#[derive(Clone, Debug)] +pub struct Registries { + pub mirror: Mirror, + pub https: bool, +} + +impl Registries { + /// downloads.copilotkit.ai for ghcr.io and Docker Hub, over https. + pub fn openbot() -> Self { + Self { + mirror: Mirror::openbot(), + https: true, + } + } + + fn scheme(&self) -> &'static str { + if self.https { + "https" + } else { + "http" + } + } + + /// Where to ask for `image`: the mirror's copy when it keeps one, then the image's own. + fn sources(&self, image: &str) -> Vec { + self.mirror + .mirrored(image) + .into_iter() + .chain(std::iter::once(image.to_string())) + .collect() + } + + /// The URL of `digest`'s blob in the repository `image` names. + fn blob_url(&self, image: &str, digest: &str) -> Option { + let reference = crate::registry::parse_reference(image)?; + Some(format!( + "{}://{}/v2/{}/blobs/{digest}", + self.scheme(), + reference.host, + reference.repository + )) + } +} + +const INDEX_TYPES: [&str; 2] = [ + "application/vnd.oci.image.index.v1+json", + "application/vnd.docker.distribution.manifest.list.v2+json", +]; +const MANIFEST_TYPES: [&str; 2] = [ + "application/vnd.oci.image.manifest.v1+json", + "application/vnd.docker.distribution.manifest.v2+json", +]; + +#[derive(Deserialize)] +struct Document { + #[serde(rename = "mediaType", default)] + media_type: Option, + #[serde(default)] + manifests: Vec, + config: Option, + #[serde(default)] + layers: Vec, +} + +#[derive(Deserialize)] +struct Entry { + digest: String, + #[serde(rename = "mediaType", default)] + media_type: Option, + platform: Option, +} + +#[derive(Deserialize)] +struct Platform { + os: String, + architecture: String, +} + +/// The entry for `linux/` in an index, or the only Linux image there is. An +/// attestation names `unknown/unknown` and is never an image. +fn pick<'a>(entries: &'a [Entry], architecture: &str) -> Option<&'a Entry> { + let linux: Vec<&Entry> = entries + .iter() + .filter(|entry| { + entry.platform.as_ref().is_some_and(|platform| { + platform.os == "linux" && platform.architecture != "unknown" + }) + }) + .collect(); + linux + .iter() + .copied() + .find(|entry| { + entry + .platform + .as_ref() + .is_some_and(|platform| platform.architecture == architecture) + }) + .or_else(|| (linux.len() == 1).then(|| linux[0])) +} + +/// The manifest at `reference`, as its registry serves it, checked against its digest by the +/// client when the reference names one. +async fn manifest( + client: &oci_client::Client, + reference: &str, +) -> Result<(Vec, String), String> { + let parsed: oci_client::Reference = reference + .parse() + .map_err(|error| format!("{reference} is not an image reference: {error}"))?; + let accepted: Vec<&str> = INDEX_TYPES + .iter() + .chain(MANIFEST_TYPES.iter()) + .copied() + .collect(); + let (body, digest) = client + .pull_manifest_raw( + &parsed, + &oci_client::secrets::RegistryAuth::Anonymous, + &accepted, + ) + .await + .map_err(|error| format!("{reference}: {error}"))?; + Ok((body.to_vec(), digest)) +} + +/// `image`'s manifest for `architecture`, from the first of its sources that gives one. +async fn resolve( + client: &oci_client::Client, + registries: &Registries, + image: &str, + architecture: &str, +) -> Result<(String, Vec, String, Document), String> { + let mut last = format!("no source for {image}"); + for source in registries.sources(image) { + match resolve_at(client, &source, architecture).await { + Ok((bytes, digest, document)) => return Ok((source, bytes, digest, document)), + Err(error) => { + log::warn!("[early images] {error}"); + last = error; + } + } + } + Err(last) +} + +async fn resolve_at( + client: &oci_client::Client, + source: &str, + architecture: &str, +) -> Result<(Vec, String, Document), String> { + let (bytes, digest) = manifest(client, source).await?; + let document: Document = serde_json::from_slice(&bytes) + .map_err(|error| format!("{source}: the manifest is not readable: {error}"))?; + if document.manifests.is_empty() { + return Ok((bytes, digest, document)); + } + let entry = pick(&document.manifests, architecture) + .ok_or_else(|| format!("{source} has no image for linux/{architecture}"))?; + let name = source.split('@').next().unwrap_or(source); + // A tag's repository: everything before the tag, which follows the last path part's colon. + let repository = match name.rsplit_once('/') { + Some((before, last)) => { + format!("{before}/{}", last.split(':').next().unwrap_or(last)) + } + None => name.split(':').next().unwrap_or(name).to_string(), + }; + let platform = format!("{repository}@{}", entry.digest); + let (bytes, digest) = manifest(client, &platform).await?; + let mut document: Document = serde_json::from_slice(&bytes) + .map_err(|error| format!("{platform}: the manifest is not readable: {error}"))?; + if document.media_type.is_none() { + document.media_type = entry.media_type.clone(); + } + Ok((bytes, digest, document)) +} + +fn sha256_hex(bytes: &[u8]) -> String { + Sha256::digest(bytes) + .iter() + .map(|byte| format!("{byte:02x}")) + .collect() +} + +/// The record of one downloaded image, by a name made from its reference. +fn record_path(folder: &Path, image: &str) -> PathBuf { + folder.join(format!("{}.json", &sha256_hex(image.as_bytes())[..16])) +} + +/// Every image recorded as downloaded in `folder`. +pub fn downloaded(folder: &Path) -> Vec { + let Ok(entries) = std::fs::read_dir(folder) else { + return Vec::new(); + }; + let mut found: Vec = entries + .flatten() + .filter(|entry| entry.path().extension().is_some_and(|ext| ext == "json")) + .filter_map(|entry| serde_json::from_slice(&std::fs::read(entry.path()).ok()?).ok()) + .collect(); + found.sort_by(|a, b| a.image.cmp(&b.image)); + found +} + +/// Every byte downloaded so far, over every blob, for one progress line. +struct Together<'a> { + parent: &'a dyn Progress, + bytes: Mutex>, +} + +impl Together<'_> { + fn blob(&self, hex: String) -> impl Progress + '_ { + struct One<'b, 'a> { + together: &'b Together<'a>, + hex: String, + } + impl Progress for One<'_, '_> { + fn bytes(&self, bytes: u64) { + let total = { + let mut all = self + .together + .bytes + .lock() + .unwrap_or_else(|held| held.into_inner()); + all.insert(self.hex.clone(), bytes); + all.values().sum::() + }; + self.together.parent.bytes(total); + } + } + One { + together: self, + hex, + } + } +} + +/// How many blobs download at once. +const BLOBS_AT_ONCE: usize = 4; + +/** +Download `images` for `architecture` into `folder`: each one's manifest, then every blob they +name, at most `BLOBS_AT_ONCE` at a time. `progress` hears the size of them all once every manifest +is read, then the bytes so far. An image already recorded with every blob there is not asked for +again. Answers with what was downloaded; an image that could not be is left to Install's pull. +*/ +pub fn download( + images: &[String], + registries: &Registries, + architecture: &str, + folder: &Path, + progress: &dyn Progress, +) -> Result, String> { + std::fs::create_dir_all(blobs(folder)) + .map_err(|error| format!("could not make {}: {error}", folder.display()))?; + // The client's TLS needs a provider installed for the process: ring, as the updater installs it. + // Installed once; a second install answers an error that changes nothing. + let _ = rustls::crypto::ring::default_provider().install_default(); + let client = oci_client::Client::new(oci_client::client::ClientConfig { + protocol: if registries.https { + oci_client::client::ClientProtocol::Https + } else { + oci_client::client::ClientProtocol::Http + }, + user_agent: "openbot-desktop", + connect_timeout: Some(std::time::Duration::from_secs(15)), + read_timeout: Some(std::time::Duration::from_secs(30)), + platform_resolver: None, + ..Default::default() + }); + + let mut wanted: Vec<(Downloaded, String)> = Vec::new(); + for image in images { + let resolved = + tauri::async_runtime::block_on(resolve(&client, registries, image, architecture)); + let (source, bytes, digest, document) = match resolved { + Ok(resolved) => resolved, + Err(error) => { + log::warn!("[early images] {image} is left to Install: {error}"); + continue; + } + }; + let hex = digest.trim_start_matches("sha256:").to_string(); + if sha256_hex(&bytes) != hex { + log::warn!("[early images] {image}: its manifest is not the one its digest names"); + continue; + } + let Some(config) = document.config else { + log::warn!("[early images] {image}: its manifest names no config"); + continue; + }; + std::fs::write(blobs(folder).join(&hex), &bytes) + .map_err(|error| format!("could not write {image}'s manifest: {error}"))?; + let record = Downloaded { + image: image.clone(), + manifest: Blob { + digest: format!("sha256:{hex}"), + size: bytes.len() as u64, + }, + media_type: document + .media_type + .unwrap_or_else(|| MANIFEST_TYPES[0].to_string()), + config, + layers: document.layers, + }; + wanted.push((record, source)); + } + + // Each blob once, from where its image was found, then the image's own registry. + let mut parts: BTreeMap)> = BTreeMap::new(); + for (record, source) in &wanted { + for blob in record.parts() { + parts.entry(blob.hex().to_string()).or_insert_with(|| { + let sources = [source.as_str(), record.image.as_str()] + .iter() + .filter_map(|from| registries.blob_url(from, &blob.digest)) + .collect::>(); + (blob.clone(), sources) + }); + } + } + progress.size(parts.values().map(|(blob, _)| blob.size).sum()); + let together = Together { + parent: progress, + bytes: Mutex::new(BTreeMap::new()), + }; + let queue = Mutex::new(parts.into_iter().collect::>()); + let failed = Mutex::new(Vec::::new()); + std::thread::scope(|scope| { + for _ in 0..BLOBS_AT_ONCE { + scope.spawn(|| loop { + let Some((hex, (blob, sources))) = + queue.lock().unwrap_or_else(|held| held.into_inner()).pop() + else { + return; + }; + let pinned = Pinned { + name: hex.clone(), + sha256: hex.clone(), + size: Some(blob.size), + sources, + }; + if let Err(error) = fetch(&pinned, &blobs(folder), &together.blob(hex.clone())) { + log::warn!("[early images] blob {hex}: {error}"); + failed + .lock() + .unwrap_or_else(|held| held.into_inner()) + .push(hex); + } + }); + } + }); + let failed = failed.into_inner().unwrap_or_default(); + let mut done = Vec::new(); + for (record, _) in wanted { + if record + .parts() + .any(|blob| failed.iter().any(|hex| hex == blob.hex())) + { + log::warn!("[early images] {} is left to Install", record.image); + continue; + } + let path = record_path(folder, &record.image); + let text = serde_json::to_vec_pretty(&record).map_err(|error| error.to_string())?; + std::fs::write(&path, text) + .map_err(|error| format!("could not write {}: {error}", path.display()))?; + log::info!("[early images] {} is downloaded", record.image); + done.push(record); + } + if done.is_empty() && !images.is_empty() { + return Err("no image downloaded".into()); + } + Ok(done) +} + +/// The tag an image is loaded with, as the engine names it once it is pulled (`image_mirror`): a +/// digest-pinned reference is tagged `sha256-`. +fn loaded_name(image: &str) -> String { + let name = crate::image_mirror::local_name(image); + match crate::registry::parse_reference(&name) { + Some(reference) if reference.host == "registry-1.docker.io" => { + format!("docker.io/{}:{}", reference.repository, reference.reference) + } + _ => name, + } +} + +/// The OCI image layout for `record`, as a tar, written to `into`: `oci-layout`, an `index.json` +/// naming the platform's manifest, and every blob it needs from `folder`. +pub fn write_layout(record: &Downloaded, folder: &Path, into: impl Write) -> Result<(), String> { + let name = loaded_name(&record.image); + let tag = name.rsplit(':').next().unwrap_or("latest").to_string(); + let index = serde_json::json!({ + "schemaVersion": 2, + "mediaType": "application/vnd.oci.image.index.v1+json", + "manifests": [{ + "mediaType": record.media_type, + "digest": record.manifest.digest, + "size": record.manifest.size, + "annotations": { + "io.containerd.image.name": name, + "org.opencontainers.image.ref.name": tag, + }, + }], + }); + let mut tar = tar::Builder::new(into); + let mut add = |path: &str, bytes: &[u8]| -> Result<(), String> { + let mut header = tar::Header::new_gnu(); + header.set_size(bytes.len() as u64); + header.set_mode(0o644); + header.set_cksum(); + tar.append_data(&mut header, path, bytes) + .map_err(|error| format!("could not write the image's layout: {error}")) + }; + add("oci-layout", br#"{"imageLayoutVersion":"1.0.0"}"#)?; + add( + "index.json", + &serde_json::to_vec(&index).map_err(|error| error.to_string())?, + )?; + for blob in std::iter::once(&record.manifest).chain(record.parts()) { + let path = blobs(folder).join(blob.hex()); + let mut file = std::fs::File::open(&path) + .map_err(|error| format!("could not read {}: {error}", path.display()))?; + tar.append_file(format!("blobs/sha256/{}", blob.hex()), &mut file) + .map_err(|error| format!("could not write the image's layout: {error}"))?; + } + tar.into_inner() + .and_then(|mut into| into.flush()) + .map_err(|error| format!("could not write the image's layout: {error}")) +} + +/// Load `record` into `engine`: its OCI layout streamed into `load`. +pub fn load(engine: &Address, record: &Downloaded, folder: &Path) -> Result { + let mut child = engine + .command() + .arg("load") + .stdin(std::process::Stdio::piped()) + .stdout(std::process::Stdio::piped()) + .stderr(std::process::Stdio::piped()) + .spawn() + .map_err(|error| format!("could not run load: {error}"))?; + let stdin = child + .stdin + .take() + .ok_or_else(|| "load took no input".to_string())?; + let written = std::thread::scope(|scope| { + let writer = scope.spawn(|| write_layout(record, folder, stdin)); + let output = child.wait_with_output(); + (writer.join(), output) + }); + let output = written + .1 + .map_err(|error| format!("load did not finish: {error}"))?; + let said = format!( + "{}{}", + String::from_utf8_lossy(&output.stdout).trim(), + String::from_utf8_lossy(&output.stderr).trim() + ); + if !output.status.success() { + return Err(format!("load stopped: {said}")); + } + match written.0 { + Ok(Ok(())) => Ok(said), + Ok(Err(error)) => Err(error), + Err(_) => Err("the layout could not be written".into()), + } +} + +/** +Load every image downloaded in `folder` that `engine` does not have yet, saying how far it is through +`said` (`done` of `total` images, over the bytes they hold), then delete the downloads. An image that +does not load is left to Install's pull. Answers with how many loaded. +*/ +pub fn load_downloaded( + engine: &Address, + folder: &Path, + said: impl Fn(crate::pull_metrics::Downloads), +) -> usize { + let records = downloaded(folder); + if records.is_empty() { + return 0; + } + let bytes: u64 = records + .iter() + .flat_map(|record| record.parts().map(|blob| blob.size)) + .sum(); + let total = records.len(); + let mut loaded = 0; + for (at, record) in records.iter().enumerate() { + said(crate::pull_metrics::Downloads { + done: at, + total, + bytes, + size: Some(bytes), + bytes_per_second: None, + }); + if crate::preparation::image_present(engine, &record.image).unwrap_or(false) { + continue; + } + let started = std::time::Instant::now(); + match load(engine, record, folder) { + Ok(answer) => { + loaded += 1; + log::info!( + "[early images] loaded {} in {:.1} s: {answer}", + record.image, + started.elapsed().as_secs_f64() + ); + } + Err(error) => log::warn!( + "[early images] {} did not load, so Install pulls it: {error}", + record.image + ), + } + } + said(crate::pull_metrics::Downloads { + done: total, + total, + bytes, + size: Some(bytes), + bytes_per_second: None, + }); + forget(folder); + loaded +} + +/// The downloaded images are in the engine now, or will come from Install's pull: not kept. +pub fn forget(folder: &Path) { + if folder.file_name().is_some_and(|name| name == "images") { + if let Err(error) = std::fs::remove_dir_all(folder) { + if error.kind() != std::io::ErrorKind::NotFound { + log::warn!( + "[early images] could not remove {}: {error}", + folder.display() + ); + } + } + } +} + +#[cfg(test)] +mod tests; diff --git a/desktop/src-tauri/src/early_images/tests.rs b/desktop/src-tauri/src/early_images/tests.rs new file mode 100644 index 0000000000..92f8ddc33b --- /dev/null +++ b/desktop/src-tauri/src/early_images/tests.rs @@ -0,0 +1,373 @@ +use super::*; +use crate::early_downloads::tests::{Reply, Server}; +use crate::test_support::temp_root; +use std::io::Read; + +fn digest_of(bytes: &[u8]) -> String { + format!("sha256:{}", sha256_hex(bytes)) +} + +/// A registry with one image on two platforms and an attestation, as ghcr.io serves OpenBot's: an +/// index, a manifest per platform, and each one's config and layers. +struct Published { + index: Vec, + manifests: BTreeMap>, + blobs: BTreeMap>, + /// The arm64 image's manifest, config and layers, by digest. + arm64: (String, String, Vec), +} + +fn published() -> Published { + let mut manifests = BTreeMap::new(); + let mut blobs = BTreeMap::new(); + let mut entries = Vec::new(); + let mut arm64 = (String::new(), String::new(), Vec::new()); + for (n, architecture) in ["amd64", "arm64", "unknown"].iter().enumerate() { + let config = format!(r#"{{"architecture":"{architecture}","os":"linux"}}"#).into_bytes(); + let layers: Vec> = (0..2) + .map(|layer| { + (0..40_000 + layer * 7_000) + .map(|byte| ((byte * (n + 3) + layer) % 251) as u8) + .collect() + }) + .collect(); + let manifest = serde_json::to_vec(&serde_json::json!({ + "schemaVersion": 2, + "mediaType": MANIFEST_TYPES[0], + "config": {"mediaType": "application/vnd.oci.image.config.v1+json", "digest": digest_of(&config), "size": config.len()}, + "layers": layers.iter().map(|layer| serde_json::json!({ + "mediaType": "application/vnd.oci.image.layer.v1.tar+gzip", + "digest": digest_of(layer), + "size": layer.len(), + })).collect::>(), + })) + .unwrap(); + let digest = digest_of(&manifest); + if *architecture == "arm64" { + arm64 = ( + digest.clone(), + digest_of(&config), + layers.iter().map(|layer| digest_of(layer)).collect(), + ); + } + entries.push(serde_json::json!({ + "mediaType": MANIFEST_TYPES[0], + "digest": digest, + "size": manifest.len(), + "platform": {"os": if *architecture == "unknown" { "unknown" } else { "linux" }, "architecture": architecture}, + })); + manifests.insert(digest, manifest); + blobs.insert(digest_of(&config), config); + for layer in layers { + blobs.insert(digest_of(&layer), layer); + } + } + let index = serde_json::to_vec(&serde_json::json!({ + "schemaVersion": 2, + "mediaType": INDEX_TYPES[0], + "manifests": entries, + })) + .unwrap(); + Published { + index, + manifests, + blobs, + arm64, + } +} + +fn manifest_reply(bytes: &[u8], media_type: &str) -> Reply { + Reply::Send { + status: "200 OK", + headers: vec![ + ("Content-Type".into(), media_type.into()), + ("Docker-Content-Digest".into(), digest_of(bytes)), + ], + body: bytes.to_vec(), + cut: None, + } +} + +fn not_here() -> Reply { + Reply::Send { + status: "403 Forbidden", + headers: vec![], + body: vec![], + cut: None, + } +} + +/// Serves `published` as the image `copilotkit/openbot-server`, at `/v2/copilotkit/...` as its own +/// registry, and under `/v2/m/` as the mirror's copy, which lacks the arm64 image's second layer. +fn registry(published: Published) -> (Server, String) { + let published = std::sync::Arc::new(published); + let missing = published.arm64.2[1].clone(); + let server = Server::start(move |request, _| { + let path = request.path.as_str(); + if path == "/v2/" { + return manifest_reply(b"{}", "application/json"); + } + let (mirror, rest) = match path.strip_prefix("/v2/m/") { + Some(rest) => (true, rest), + None => (false, path.trim_start_matches("/v2/")), + }; + let Some(rest) = rest.strip_prefix("copilotkit/openbot-server/") else { + return not_here(); + }; + if let Some(reference) = rest.strip_prefix("manifests/") { + if reference == digest_of(&published.index) { + return manifest_reply(&published.index, INDEX_TYPES[0]); + } + return match published.manifests.get(reference) { + Some(bytes) => manifest_reply(bytes, MANIFEST_TYPES[0]), + None => not_here(), + }; + } + if let Some(digest) = rest.strip_prefix("blobs/") { + if mirror && digest == missing { + return not_here(); + } + return match published.blobs.get(digest) { + Some(bytes) => crate::early_downloads::tests::ranged(request, bytes), + None => not_here(), + }; + } + not_here() + }); + let host = server + .url("") + .trim_start_matches("http://") + .trim_end_matches('/') + .to_string(); + (server, host) +} + +struct Seen { + size: Mutex>, + bytes: Mutex, +} + +impl Progress for Seen { + fn size(&self, size: u64) { + *self.size.lock().unwrap() = Some(size); + } + fn bytes(&self, bytes: u64) { + let mut seen = self.bytes.lock().unwrap(); + assert!(bytes >= *seen, "the bytes went back from {seen} to {bytes}"); + *seen = bytes; + } +} + +/** +An image Install pulls by its index digest is downloaded with no engine: the arm64 manifest from +the mirror, every blob checked against its digest, the one the mirror lacks from the image's own +registry. Its OCI layout is what `docker load` and `podman load` read: `oci-layout`, an index naming +the platform's manifest under the name the engine pulls it by, and every blob byte for byte. +*/ +#[test] +fn an_image_downloads_with_no_engine_and_is_laid_out_for_load() { + let published = published(); + let (index, arm64, blobs_served) = ( + digest_of(&published.index), + published.arm64.clone(), + published.blobs.clone(), + ); + let (server, host) = registry(published); + let image = format!("{host}/copilotkit/openbot-server@{index}"); + let registries = Registries { + mirror: Mirror { + mirror: format!("{host}/m"), + upstream: host.clone(), + }, + https: false, + }; + let folder = temp_root("early-images").join("images"); + let seen = Seen { + size: Mutex::new(None), + bytes: Mutex::new(0), + }; + + let done = download( + std::slice::from_ref(&image), + ®istries, + "arm64", + &folder, + &seen, + ) + .expect("the image"); + assert_eq!(done.len(), 1); + let record = &done[0]; + assert_eq!(record.image, image); + assert_eq!(record.manifest.digest, arm64.0); + assert_eq!(record.config.digest, arm64.1); + assert_eq!( + record + .layers + .iter() + .map(|layer| layer.digest.clone()) + .collect::>(), + arm64.2 + ); + // The size of every part was known before the bytes, and every byte came in. + let size: u64 = record.parts().map(|blob| blob.size).sum(); + assert_eq!(*seen.size.lock().unwrap(), Some(size)); + assert_eq!(*seen.bytes.lock().unwrap(), size); + // Recorded, so Install finds it to load. + assert_eq!(downloaded(&folder), done); + // The manifest from the mirror; the layer it lacks from the image's own registry. + let paths: Vec = server.seen().into_iter().map(|seen| seen.path).collect(); + assert!( + paths.contains(&format!( + "/v2/m/copilotkit/openbot-server/manifests/{index}" + )), + "{paths:?}" + ); + assert!( + paths.contains(&format!( + "/v2/copilotkit/openbot-server/blobs/{}", + arm64.2[1] + )), + "{paths:?}" + ); + assert!( + !paths + .iter() + .any(|path| path.contains("/manifests/") && !path.contains("/m/")), + "{paths:?}" + ); + + let mut tar = Vec::new(); + write_layout(record, &folder, &mut tar).expect("the layout"); + let mut entries: BTreeMap> = BTreeMap::new(); + for entry in tar::Archive::new(tar.as_slice()).entries().unwrap() { + let mut entry = entry.unwrap(); + let path = entry.path().unwrap().to_string_lossy().into_owned(); + let mut bytes = Vec::new(); + entry.read_to_end(&mut bytes).unwrap(); + entries.insert(path, bytes); + } + assert_eq!( + serde_json::from_slice::(&entries["oci-layout"]).unwrap(), + serde_json::json!({"imageLayoutVersion": "1.0.0"}) + ); + let laid: serde_json::Value = serde_json::from_slice(&entries["index.json"]).unwrap(); + assert_eq!(laid["manifests"].as_array().unwrap().len(), 1); + assert_eq!(laid["manifests"][0]["digest"], arm64.0.as_str()); + assert_eq!( + laid["manifests"][0]["annotations"]["io.containerd.image.name"], + format!( + "{host}/copilotkit/openbot-server:{}", + index.replace(':', "-") + ) + .as_str() + ); + for digest in std::iter::once(&arm64.1).chain(arm64.2.iter()) { + let path = format!("blobs/sha256/{}", digest.trim_start_matches("sha256:")); + assert_eq!(entries[&path], blobs_served[digest], "{path}"); + } + assert_eq!( + digest_of(&entries[&format!("blobs/sha256/{}", arm64.0.trim_start_matches("sha256:"))]), + arm64.0 + ); + // Only this platform's image: nothing of the other's. + assert_eq!(entries.len(), 2 + 1 + 1 + arm64.2.len()); + + // Loaded, the downloads are not kept. + forget(&folder); + assert!(!folder.exists()); +} + +/// Docker Hub's images are named as an engine names them once loaded. +#[test] +fn a_loaded_image_has_the_name_its_pull_would_give_it() { + assert_eq!( + loaded_name("ghcr.io/copilotkit/openbot-server@sha256:5ce6f82ab8c90810cb45c86c2f5e9cdf4e73bfee1aef13713c9076a0883fd54d"), + "ghcr.io/copilotkit/openbot-server:sha256-5ce6f82ab8c90810cb45c86c2f5e9cdf4e73bfee1aef13713c9076a0883fd54d" + ); + assert_eq!( + loaded_name("pgvector/pgvector:pg17"), + "docker.io/pgvector/pgvector:pg17" + ); +} + +/** +OpenBot's images downloaded with no engine, loaded into a real one, then Install's own pull: it +finds the image's layers and config there and moves only manifests. + +`OPENBOT_LIVE_LOAD_ENGINE=docker`, or `podman:` for a Podman machine's (a throwaway one: +`podman machine init --image`). The image is an earlier release's, which the engine must not have; +it is removed afterwards. `OPENBOT_LIVE_LOAD_SKIP=1` leaves the load out, as Install was before: +then the pull moves the whole image, and this fails. +*/ +#[test] +#[ignore = "downloads an OpenBot image and loads it into a real engine"] +fn live_an_image_downloaded_before_the_engine_loads_and_its_pull_moves_no_layers() { + let engine = match std::env::var("OPENBOT_LIVE_LOAD_ENGINE").as_deref() { + Ok("docker") => Address::new(crate::engine::Engine::Docker, None), + Ok(podman) if podman.starts_with("podman:") => Address::new( + crate::engine::Engine::Podman, + Some(podman.trim_start_matches("podman:").to_string()), + ), + _ => panic!("OPENBOT_LIVE_LOAD_ENGINE=docker or podman:"), + }; + // v0.3.3's LangGraph AG-UI image, about 83 MB. + let image = "ghcr.io/copilotkit/openbot-agent-langgraph-agui@sha256:7ae4c62fed073506a0a709776640cbb55030da6c8a928603ff649d5d891936fa"; + assert!( + !crate::preparation::image_present(&engine, image).unwrap(), + "the engine already has {image}" + ); + let folder = temp_root("early-images-live").join("images"); + let started = std::time::Instant::now(); + let done = download( + &[image.to_string()], + &Registries::openbot(), + crate::registry::host_architecture(), + &folder, + &(), + ) + .expect("the image, with no engine"); + let bytes: u64 = done[0].parts().map(|blob| blob.size).sum(); + eprintln!( + "downloaded {} in {:.1} s, {bytes} bytes", + done[0].manifest.digest, + started.elapsed().as_secs_f64() + ); + if std::env::var("OPENBOT_LIVE_LOAD_SKIP").as_deref() != Ok("1") { + let started = std::time::Instant::now(); + let said = load(&engine, &done[0], &folder).expect("loaded"); + eprintln!("loaded in {:.1} s: {said}", started.elapsed().as_secs_f64()); + } + + // Install's own pull, by the pinned reference, from the mirror first. + let pulled = Mutex::new(0u64); + let started = std::time::Instant::now(); + crate::pull_metrics::pull_image( + &engine, + image, + |_| {}, + |metrics| *pulled.lock().unwrap() = metrics.bytes.unwrap_or(0), + ) + .expect("the pull after the load"); + let pulled = *pulled.lock().unwrap(); + eprintln!( + "pulled after the load in {:.1} s, {pulled} bytes of layers", + started.elapsed().as_secs_f64() + ); + assert!(crate::preparation::image_present(&engine, image).unwrap()); + + for name in [ + image.to_string(), + loaded_name(image), + crate::image_mirror::local_name(image), + ] { + let _ = engine + .command() + .args(["image", "rm", "--force", &name]) + .output(); + } + forget(&folder); + assert!( + pulled < bytes / 20, + "the pull moved {pulled} of the image's {bytes} bytes" + ); +} diff --git a/desktop/src-tauri/src/harness.rs b/desktop/src-tauri/src/harness.rs index d736c19798..4c8061e583 100644 --- a/desktop/src-tauri/src/harness.rs +++ b/desktop/src-tauri/src/harness.rs @@ -137,6 +137,10 @@ fn byo_remote_ag_ui_url(value: &str) -> Result { } } +/// The Bot a first setup installs, as the setup window names it (`DEFAULT_HARNESS` in +/// desktop/src/harness.ts). The early downloads pull its images before Install is clicked. +pub const DEFAULT_HARNESS: &str = "langgraph"; + /// The list, ranked as the build doc ranks it: stars first, with downloads as the sanity check, /// because each misleads alone. /// @@ -302,6 +306,20 @@ where mod tests { use crate::test_support::temp_root; + /// The window installs this Bot on Install's click, and the early downloads pull its images + /// before that click: the two must name the same one. + #[test] + fn the_default_bot_is_the_one_the_setup_window_installs() { + let window = include_str!("../../src/harness.ts"); + assert!( + window.contains(&format!("DEFAULT_HARNESS = \"{}\"", super::DEFAULT_HARNESS)), + "desktop/src/harness.ts DEFAULT_HARNESS must equal harness::DEFAULT_HARNESS" + ); + assert!(super::catalogue() + .iter() + .any(|row| row.id == super::DEFAULT_HARNESS && row.image.is_some())); + } + /// Both plans name a Bot that exists and can actually use them. #[test] fn each_plan_names_a_bot_that_exists() { diff --git a/desktop/src-tauri/src/image_mirror.rs b/desktop/src-tauri/src/image_mirror.rs index 7cea21e8fd..c392cc1552 100644 --- a/desktop/src-tauri/src/image_mirror.rs +++ b/desktop/src-tauri/src/image_mirror.rs @@ -147,7 +147,7 @@ fn docker_hub(image: &str) -> Option { /// An engine will not tag with a digest, so a digest-only reference is tagged `sha256-` after /// its digest: unique to the image, and never a tag anyone publishes. A reference with a tag keeps /// its tag. -fn local_name(image: &str) -> String { +pub(crate) fn local_name(image: &str) -> String { let (name, digest) = match image.split_once('@') { Some((name, digest)) => (name, Some(digest)), None => (image, None), diff --git a/desktop/src-tauri/src/install.rs b/desktop/src-tauri/src/install.rs index d818b6f299..7a11d150ae 100644 --- a/desktop/src-tauri/src/install.rs +++ b/desktop/src-tauri/src/install.rs @@ -48,8 +48,6 @@ use std::path::{Path, PathBuf}; use std::sync::{Condvar, Mutex}; use std::time::Duration; -use sha2::{Digest, Sha256}; - use crate::engine::{self, Engine, StepLine}; use crate::problem::Problem; @@ -163,38 +161,80 @@ pub fn compose_download() -> Result { /// /// A file already there with the right digest is kept, so a retry after a failed install is not a /// second download. A file already there with the wrong one is replaced: that is a half-written -/// download far more often than it is an attack, and either way it must not be run. +/// download far more often than it is an attack, and either way it must not be run. The download +/// itself is the early downloads' (`early_downloads::fetch`): from downloads.copilotkit.ai first, +/// resumed where an earlier one stopped, and waited for when the first screen already started it. fn fetch_verified(download: &Download, into: &Path) -> Result { - let path = into.join(download.file); - if let Ok(existing) = std::fs::read(&path) { - if digest_of(&existing) == download.sha256 { - return Ok(path); - } - } - - let body = crate::deployment::get(&download.url).map_err(|error| { + fetch_pinned(download, into, &()).map_err(|error| { Problem::with( "OpenBot could not download the software it needs to run. Check the internet \ connection and try again.", format!("{}: {error}", download.url), ) - })?; + }) +} - let got = digest_of(&body); - if got != download.sha256 { - return Err(Problem::with( - "What OpenBot downloaded is not what it was expecting, so it has not been run. Try \ - again.", - format!( - "{} from {}: expected sha256 {}, got {got}", - download.file, download.url, download.sha256 - ), - )); +fn fetch_pinned( + download: &Download, + into: &Path, + progress: &dyn crate::early_downloads::Progress, +) -> Result { + let pinned = crate::early_downloads::Pinned { + name: download.file.to_string(), + sha256: download.sha256.to_string(), + size: None, + sources: crate::deployment::mirrored(&download.url) + .into_iter() + .chain(std::iter::once(download.url.clone())) + .collect(), + }; + crate::early_downloads::fetch(&pinned, into, progress) +} + +/// Podman's installer for this computer, into `into`, before Install needs it. Never installed here. +pub fn fetch_podman_installer( + into: &Path, + progress: &dyn crate::early_downloads::Progress, +) -> Result { + fetch_pinned( + &podman_download().map_err(|problem| problem.said)?, + into, + progress, + ) +} + +/// Compose for this computer, ready to run on its own, for `config` before any engine is here: the +/// verified download, copied where nothing looks for a provider and made executable. +pub fn runnable_compose( + into: &Path, + progress: &dyn crate::early_downloads::Progress, +) -> Result { + let staged = fetch_compose(into, progress)?; + let folder = into.join("early-compose"); + std::fs::create_dir_all(&folder) + .map_err(|error| format!("could not make {}: {error}", folder.display()))?; + let named = folder.join(compose_provider_name()); + std::fs::copy(&staged, &named) + .map_err(|error| format!("could not copy Compose to {}: {error}", named.display()))?; + #[cfg(unix)] + { + use std::os::unix::fs::PermissionsExt; + std::fs::set_permissions(&named, std::fs::Permissions::from_mode(0o755)) + .map_err(|error| format!("chmod 755 {}: {error}", named.display()))?; } + Ok(named) +} - std::fs::create_dir_all(into).map_err(|error| unwritable(into, &error.to_string()))?; - std::fs::write(&path, &body).map_err(|error| unwritable(&path, &error.to_string()))?; - Ok(path) +/// Compose for this computer, into `into`, before Install places it. +pub fn fetch_compose( + into: &Path, + progress: &dyn crate::early_downloads::Progress, +) -> Result { + fetch_pinned( + &compose_download().map_err(|problem| problem.said)?, + into, + progress, + ) } /// One sentence for every "could not write here", because the person's fix is the same each time. @@ -206,7 +246,9 @@ fn unwritable(path: &Path, error: &str) -> Problem { ) } +#[cfg(test)] fn digest_of(bytes: &[u8]) -> String { + use sha2::{Digest, Sha256}; let mut hasher = Sha256::new(); hasher.update(bytes); hasher @@ -520,7 +562,7 @@ impl Found { /// What the install step does to have an engine answering. #[derive(Debug, Clone, Copy, PartialEq, Eq)] -enum Choice { +pub enum Choice { /// Docker answers now. UseDocker, /// Docker Desktop is installed and stopped, and opening it shows nothing that needs the person. @@ -566,6 +608,11 @@ fn choose(found: Found, saved: Option) -> Choice { } } +/// What Install would do for an engine on this computer now. The early downloads fetch what it needs. +pub fn choice_here(saved: Option) -> Choice { + choose(Found::here(saved), saved) +} + fn install_engine_with( cache: &Path, found: Found, @@ -1310,6 +1357,33 @@ mod tests { ); } + /// A Podman installer whose download was cut off, by a dropped connection or by quitting the app, + /// carries on from the bytes already on disk at the next try, and is checked whole before it is + /// kept. Before the early downloads, every try downloaded the whole file again into memory. + #[test] + fn a_download_cut_off_part_way_carries_on_from_where_it_stopped() { + use crate::early_downloads::tests::{file, ranged, whole, Server}; + let (body, sha256) = file(400_000); + let served = body.clone(); + let server = Server::start(move |request, index| match index { + 0 => whole(&served, Some(150_000)), + _ => ranged(request, &served), + }); + let dir = temp_root("install-resume"); + let download = Download { + url: server.url("/podman-installer-macos-arm64.pkg"), + sha256: Box::leak(sha256.into_boxed_str()), + file: "podman-installer-macos-arm64.pkg", + }; + let path = fetch_verified(&download, &dir).expect("the installer, carried on"); + assert_eq!(std::fs::read(&path).unwrap(), body); + let seen = server.seen(); + assert_eq!(seen.len(), 2, "{seen:?}"); + assert_eq!(seen[1].range.as_deref(), Some("bytes=150000-"), "{seen:?}"); + assert_eq!(server.sent(), body.len(), "nothing was downloaded twice"); + let _ = std::fs::remove_dir_all(&dir); + } + #[test] fn a_file_whose_digest_is_wrong_is_never_returned_to_be_run() { let dir = temp_root("digest"); diff --git a/desktop/src-tauri/src/lib.rs b/desktop/src-tauri/src/lib.rs index b3556da616..39f25ee363 100644 --- a/desktop/src-tauri/src/lib.rs +++ b/desktop/src-tauri/src/lib.rs @@ -9,6 +9,8 @@ pub mod deployment_release; pub mod docker_config; pub mod docker_desktop; pub mod docker_desktop_update; +pub mod early_downloads; +pub mod early_images; pub mod engine; pub mod engine_versions; pub mod env; diff --git a/desktop/src-tauri/src/main.rs b/desktop/src-tauri/src/main.rs index 0879bdcffe..c5e0f0fb05 100644 --- a/desktop/src-tauri/src/main.rs +++ b/desktop/src-tauri/src/main.rs @@ -19,9 +19,9 @@ mod zoom; mod test_support; use openbot_desktop_lib::{ - acquire, app_update, deployment, deployment_release, docker_config, engine, engine_versions, - env as openbot_env, harness, host_access, install, local_eval, local_eval_state, logging, - preparation, + acquire, app_update, deployment, deployment_release, docker_config, early_downloads, engine, + engine_versions, env as openbot_env, harness, host_access, install, local_eval, + local_eval_state, logging, preparation, problem::{LeftoverReset, Problem}, provider, pull_metrics, quiet, recorder_external, recorder_host, recorder_install, stack, telemetry, tray, troubleshooting, update_ready, upgrade, windows as win, @@ -710,6 +710,10 @@ fn forget_running_logged() { .unwrap_or_else(|held| held.into_inner()) = None; } +/// Each running step's latest line, so a download added to its row keeps what the row says. +static RUNNING_SAID: Mutex> = + Mutex::new(std::collections::BTreeMap::new()); + fn report_running( app: &tauri::AppHandle, step: &str, @@ -751,6 +755,10 @@ fn report_running_line( line: engine::StepLine, download_bytes: Option, ) { + RUNNING_SAID + .lock() + .unwrap_or_else(|held| held.into_inner()) + .insert(step.to_owned(), line.clone()); // The window shows the plain line; the log keeps the technical detail beside it. let entry = line.log_entry(); let engine::StepLine { @@ -942,10 +950,54 @@ fn allow_podman_install() -> bool { /// Install's running image downloads: the step's sentence, the overall bytes as `downloadBytes` /// for anything that reads only that, and the per-image detail. +/// A running step's download, on the row it is already showing: the engine machine image while +/// Install waits for it. The row keeps its line; only the download is added. +fn report_running_downloads( + app: &tauri::AppHandle, + step: &str, + downloads: pull_metrics::Downloads, +) { + let line = RUNNING_SAID + .lock() + .unwrap_or_else(|held| held.into_inner()) + .get(step) + .cloned() + .unwrap_or_else(|| engine::StepLine::here("")); + // Bytes moving restart the step's clock, so a long download is not a step that looks stuck. + if downloads_moved(step, downloads.bytes) { + desktop_stall::moved(app, step); + } + emit_progress( + app, + Progress { + step: step.into(), + ok: true, + detail: line.said, + running: true, + download_bytes: (downloads.bytes > 0).then_some(downloads.bytes), + downloads: Some(downloads), + elsewhere: line.elsewhere, + asks: line.asks, + }, + ); +} + +/// Whether `step`'s download has more bytes than the last time it said. +fn downloads_moved(step: &str, bytes: u64) -> bool { + static SEEN: Mutex> = + Mutex::new(std::collections::BTreeMap::new()); + let mut seen = SEEN.lock().unwrap_or_else(|held| held.into_inner()); + let before = seen.insert(step.to_owned(), bytes); + before.is_none_or(|before| bytes > before) +} + fn report_downloads( app: &tauri::AppHandle, downloads: pull_metrics::Downloads, ) { + if downloads_moved("images", downloads.bytes) { + desktop_stall::moved(app, "images"); + } emit_progress( app, Progress { @@ -2218,6 +2270,30 @@ async fn prepare_engine(app: tauri::AppHandle) -> Result early_downloads::Summary { + early_downloads::summary() +} + /// Complete all local software acquisition before either authentication screen is available. #[tauri::command] async fn prepare_installation( @@ -2333,6 +2409,13 @@ async fn prepare_installation_inner( "images", "Checking what OpenBot needs to download.", ); + // The first screen may have started pulling these already: wait for it, showing its + // download, and the pulls below find them there. + early_downloads::wait_for_images(|downloads| report_downloads(&handle, downloads.clone())); + // Downloaded before the engine answered: loaded into it now, so the pulls below find them. + early_downloads::load_offline_images(&address, |downloads| { + report_downloads(&handle, downloads) + }); // Only what this installation runs: see `installation_pull_set`. let images = preparation::installation_pull_set(&address, &root, &root, harness.as_ref())?; // Several pulls run at once. Stop is checked before each pull starts; running pulls finish. @@ -2639,7 +2722,39 @@ async fn engine_answering( engine::preparing_machine_line(std::env::consts::OS), None, ); - let created = acquire::create_machine(4, 6144, 60); + // The machine image the first screen started downloading: waited for with its download shown + // on this row, and the machine built from it. Without one, Podman pulls its own. On a blocking + // thread, as the deployment's download is: a blocking HTTP client must not be dropped here. + let progress_app = app.clone(); + let image = tauri::async_runtime::spawn_blocking(move || { + early_downloads::machine_image_for_init(|downloads| { + report_running_downloads(&progress_app, "create-machine", downloads) + }) + }) + .await + .unwrap_or_else(|error| { + log::warn!("[early downloads] the machine image wait did not run: {error}"); + None + }); + log::info!( + "[early downloads] creating the machine {}", + image.as_deref().map_or_else( + || "with the image Podman pulls itself".to_string(), + |image| format!("from {}", image.display()) + ) + ); + // Downloaded: the row says the machine's own line again, without the download, since that line + // is what a Windows prompt during `machine init` needs said. + report_running_line( + app, + "create-machine", + engine::preparing_machine_line(std::env::consts::OS), + None, + ); + let created = acquire::create_machine(4, 6144, 60, image.as_deref()); + if created.ok && image.is_some() { + early_downloads::forget_machine_image(); + } if cfg!(target_os = "windows") { bring_to_front(app, "the engine machine is prepared"); } @@ -2744,6 +2859,8 @@ async fn deployment_ready( let target = root.to_path_buf(); let handle = app.clone(); let version = tauri::async_runtime::spawn_blocking(move || { + // The first screen may be laying the same deployment out: one at a time. + let _laying_out = early_downloads::hold_deployment(); let version = deployment_release::resolve_version(&target)?; if deployment::needs_fetch(&target, &version) { report_running_line( @@ -9731,6 +9848,8 @@ fn main() { .plugin(outside_links_guard()) .manage(Shell::default()) .invoke_handler(tauri::generate_handler![ + start_early_downloads, + early_downloads_now, record_setup_event, bring_setup_to_front, allow_podman_install, diff --git a/desktop/src-tauri/src/preparation.rs b/desktop/src-tauri/src/preparation.rs index aa1e71b80c..2aa10c8206 100644 --- a/desktop/src-tauri/src/preparation.rs +++ b/desktop/src-tauri/src/preparation.rs @@ -325,6 +325,23 @@ pub fn installation_pull_set( root: &Path, release: &Path, harness: Option<&HarnessChoice>, +) -> Result, Problem> { + installation_pull_set_by( + |settings| stack::compose_command_for(address, root, settings), + root, + release, + harness, + ) +} + +/// [`installation_pull_set`], with the Compose to ask made by `compose` from the image settings: +/// an engine's, or before there is one, OpenBot's own standalone Compose +/// (`stack::standalone_compose_command`). +pub fn installation_pull_set_by( + compose: impl FnOnce(&stack::Secrets) -> std::process::Command, + root: &Path, + release: &Path, + harness: Option<&HarnessChoice>, ) -> Result, Problem> { let picked = crate::harness::picked(harness, release).map_err(Problem::from)?; let settings = image_settings(release, picked.as_ref())?; @@ -332,7 +349,7 @@ pub fn installation_pull_set( .as_ref() .and_then(|picked| picked.installed_port()) .is_some(); - let mut images = stack::installation_images_in(address, root, release, installed, &settings)?; + let mut images = stack::installation_images_by(compose(&settings), root, release, installed)?; for published in [ crate::plan::SIGN_IN_IMAGE, crate::plan::CHATGPT_SIGN_IN_IMAGE, diff --git a/desktop/src-tauri/src/registry.rs b/desktop/src-tauri/src/registry.rs index 234837524f..f95219a465 100644 --- a/desktop/src-tauri/src/registry.rs +++ b/desktop/src-tauri/src/registry.rs @@ -170,7 +170,7 @@ fn read_manifest(body: &[u8], architecture: &str) -> Result, Option Option { +pub(crate) fn token_url(challenge: &str) -> Option { let rest = challenge.trim().strip_prefix("Bearer ")?; let mut realm = None; let mut query = Vec::new(); diff --git a/desktop/src-tauri/src/release_images.rs b/desktop/src-tauri/src/release_images.rs index 498143de44..55144642d1 100644 --- a/desktop/src-tauri/src/release_images.rs +++ b/desktop/src-tauri/src/release_images.rs @@ -15,6 +15,8 @@ it cannot name one image fewer than the app pulls: (`repository:release`, then the index that tag serves). And the cluster's own images, which the host's engine pulls when the CLI has k3d create the cluster (`local_images::cli_cluster_images`): the k3s node image and k3d's proxy and tools, each at the digest the CLI gives k3d. +- The engine machine images on quay.io that the first setup screen downloads before Install, so + Podman builds OpenBot's machine from a local file (`early_downloads::machine_image_references`). The release publishes the list as `desktop-images.json` (`examples/release_images.rs`). */ @@ -124,6 +126,7 @@ pub fn release_images(compose: &Path, manifest: &Path, cli: &str) -> Result; /// below only `-p`, so every caller's arguments stay exactly as they were. A project that could not /// be decided, because the engine could not be asked, is `DESKTOP_PROJECT`: never the folder's /// name, which would be adopting whatever else runs under it. +/// The engine's Compose for the installation in `root`, with `secrets` in its environment. +pub fn compose_command_for(engine: &Address, root: &Path, secrets: &Secrets) -> Command { + compose_command(engine, root, secrets) +} + fn compose_command(engine: &Address, root: &Path, secrets: &Secrets) -> Command { let project = project(engine, root).unwrap_or_else(|_| DESKTOP_PROJECT.to_owned()); let mut command = engine.command(); @@ -1010,10 +1015,34 @@ pub fn installation_images_in( release: &Path, harness: bool, settings: &Secrets, +) -> Result, Problem> { + installation_images_by( + compose_command(engine, root, settings), + root, + release, + harness, + ) +} + +/// A Compose of OpenBot's own that runs without any engine, for `config`: the standalone binary +/// `install.rs` downloads (`docker-compose`), with the same settings as `compose_command`. Before +/// Podman is on a computer, this is the only Compose there is. +pub fn standalone_compose_command(compose: &Path, root: &Path, secrets: &Secrets) -> Command { + let mut command = crate::quiet::command(compose); + command.current_dir(root).envs(secrets); + command.env("COMPOSE_PROJECT_NAME", DESKTOP_PROJECT); + command +} + +/// [`installation_images_in`], with the Compose `command` to ask given. +pub fn installation_images_by( + mut command: Command, + root: &Path, + release: &Path, + harness: bool, ) -> Result, Problem> { let mut requested = selected_services(harness); requested.push("migrate"); - let mut command = compose_command(engine, root, settings); if release != root { // Compose reads the release's file from where it runs, and `.env` from there too, so the // installation's own settings are named explicitly. diff --git a/desktop/src-tauri/src/windows.rs b/desktop/src-tauri/src/windows.rs index 628902c445..4adc403d16 100644 --- a/desktop/src-tauri/src/windows.rs +++ b/desktop/src-tauri/src/windows.rs @@ -1032,7 +1032,7 @@ fn distribution_folder(registry: &str, distribution: &str) -> Option { /// The bytes free to this account on the volume that holds `folder`. #[cfg(target_os = "windows")] -fn free_space(folder: &str) -> Option { +pub(crate) fn free_space(folder: &str) -> Option { use std::os::windows::ffi::OsStrExt; let folder = folder.strip_prefix(r"\\?\").unwrap_or(folder); let wide: Vec = std::ffi::OsStr::new(folder) diff --git a/desktop/src/App.tsx b/desktop/src/App.tsx index 22c46267b2..4c7048e74a 100644 --- a/desktop/src/App.tsx +++ b/desktop/src/App.tsx @@ -3,6 +3,11 @@ import { listen } from "@tauri-apps/api/event"; import { useCallback, useEffect, useRef, useState } from "react"; import { CheckForUpdates } from "./CheckForUpdates"; import { CopilotKitAccount } from "./CopilotKitAccount"; +import { + EarlyDownloadsLine, + earlyDownload, + useEarlyDownloads, +} from "./EarlyDownloads"; import { DEFAULT_HARNESS, type HarnessChoice } from "./harness"; import { isHttpEndpointUrl } from "./http-endpoint-url"; import { @@ -183,6 +188,16 @@ function Setup() { const [running, setRunning] = useState(false); const visibleSetupStep = checkingResume || blockerFailure || blocker || running ? null : step; + /* + * The first screen starts the big downloads, without a click: the person opened OpenBot's setup + * to install it. The native side downloads only what this computer's install will need, once per + * run of the app (`early_downloads.rs`). + */ + const onFirstScreen = visibleSetupStep === "welcome"; + useEffect(() => { + if (!onFirstScreen || !root.trim()) return; + invoke("start_early_downloads", { root: root.trim() }).catch(() => {}); + }, [onFirstScreen, root]); const lastViewedStep = useRef(null); useEffect(() => { if (visibleSetupStep === lastViewedStep.current) return; @@ -769,6 +784,8 @@ function Setup() { * account step and the AI step. It says what OpenBot is doing, how far along it is and for how * long, so the screen never looks stuck while minutes of install go on behind it. */ + // The downloads the first screen started, which the background row shows while they run. + const earlyDownloads = useEarlyDownloads(); const behind = (said: { waitingFor?: string; done?: string }) => installingBehind({ steps, @@ -776,6 +793,7 @@ function Setup() { installed: installationReady, since: installSince, backEnd: local, + early: earlyDownload(earlyDownloads), ...said, }); @@ -868,6 +886,10 @@ function Setup() { {/* The person closed macOS's password prompt: said, with Try again. */} {goAhead && } + {/* The first screen's downloads, until a row of Install's own shows the one it waits on. */} + {!steps.some((row) => row.running && row.downloads) && ( + + )} {displayedFailure && } {!busy && (
diff --git a/desktop/src/EarlyDownloads.test.tsx b/desktop/src/EarlyDownloads.test.tsx new file mode 100644 index 0000000000..93b5649065 --- /dev/null +++ b/desktop/src/EarlyDownloads.test.tsx @@ -0,0 +1,327 @@ +import { + afterAll, + afterEach, + beforeAll, + expect, + mock, + setSystemTime, + test, +} from "bun:test"; +import { GlobalRegistrator } from "@happy-dom/global-registrator"; +import { act, cleanup, render, within } from "@testing-library/react"; +import userEvent from "@testing-library/user-event"; +import type { EarlyDownloads } from "./EarlyDownloads"; + +let invokeCalls: Array<{ command: string; args?: unknown }> = []; +const downloadListeners = new Set< + (event: { payload: EarlyDownloads }) => void +>(); + +/** A first run on a computer with nothing installed: every startup question answered plainly. */ +async function answer(command: string): Promise { + switch (command) { + case "windows_restart_pending": + return false; + case "default_root": + return "/tmp/openbot-early-downloads"; + case "already_configured": + return { + values: {}, + saved: { + modelApiKeys: { openai: false, anthropic: false }, + modelSessions: { openai: false, anthropic: false }, + }, + }; + case "already_running": + case "take_ai_connection_change": + return false; + case "early_downloads_now": + return { state: "idle", bytes: 0 }; + // Install and the sign-in stay in progress: the account step is where the person is. + case "prepare_installation": + case "sign_in_to_copilotkit": + return new Promise(() => {}); + default: + return null; + } +} + +mock.module("@tauri-apps/api/core", () => ({ + invoke: (command: string, args?: unknown) => { + invokeCalls.push({ command, args }); + return answer(command); + }, +})); + +mock.module("@tauri-apps/api/event", () => ({ + listen: async ( + name: string, + listener: (event: { payload: EarlyDownloads }) => void, + ) => { + if (name === "setup:downloads") downloadListeners.add(listener); + return () => downloadListeners.delete(listener); + }, +})); + +mock.module("./Mark", () => ({ + Mark: ({ name }: { name: string }) => {name}, + ButtonMark: () => null, +})); + +const { App } = await import("./App"); +const { earlyDownloadsSaid } = await import("./EarlyDownloads"); + +beforeAll(() => GlobalRegistrator.register()); +afterEach(() => { + cleanup(); + setSystemTime(); + invokeCalls = []; + downloadListeners.clear(); +}); +afterAll(() => GlobalRegistrator.unregister()); + +async function said(payload: EarlyDownloads) { + await act(async () => { + for (const listener of downloadListeners) listener({ payload }); + }); +} + +/** + * The same report again once it has run long enough for its speed to say a time from: no time left + * is said before 3 seconds and 5 MB (`STEADY_SECONDS`, `STEADY_BYTES` in `journey.ts`). + */ +async function steadied(payload: EarlyDownloads) { + setSystemTime(new Date(Date.now() + 3_500)); + // A new report, as the native side sends one twice a second. + await said({ ...payload }); +} + +function called(command: string) { + return invokeCalls.filter((call) => call.command === command); +} + +const GB = 1_000_000_000; + +test("the big downloads start when the first screen shows, before Install is clicked or anyone signs in", async () => { + let view!: ReturnType; + await act(async () => { + view = render(); + }); + await view.findByRole("heading", { name: "Set up OpenBot" }); + + // Nothing was clicked: the first screen alone started them, for the installation's folder. + expect(called("start_early_downloads")).toEqual([ + { + command: "start_early_downloads", + args: { root: "/tmp/openbot-early-downloads" }, + }, + ]); + expect(called("prepare_installation")).toEqual([]); + expect(called("sign_in_to_copilotkit")).toEqual([]); + + // The first screen says how far they are in the background row's words, with the bytes under + // its bar, and the time left once the speed is steady. + const welcome = { + state: "downloading" as const, + bytes: 1.2 * GB, + size: 2.9 * GB, + bytesPerSecond: 10_000_000, + }; + await said(welcome); + expect(view.getByRole("status").textContent).toBe("Downloading OpenBot."); + await steadied(welcome); + expect(view.getByRole("status").textContent).toBe( + "Downloading OpenBot. About 3 minutes left.", + ); + const bar = view.getByRole("progressbar", { name: "Download progress" }); + expect(Number(bar.getAttribute("value"))).toBeCloseTo(1.2 / 2.9, 3); + expect(bar.getAttribute("aria-valuetext")).toBe("1.2 GB of 2.9 GB"); + + // Install moves on to the CopilotKit sign-in. The row for what is installing in the background + // carries the download on, in the same words, and its bar goes on from where it was. + await userEvent.click(view.getByRole("button", { name: "Install" })); + await view.findByRole("heading", { name: "Self-contained OpenBot" }); + const account = { ...welcome, bytes: 2.0 * GB }; + await said(account); + await steadied(account); + const row = within( + view.getByRole("listitem", { + name: "Installing OpenBot in the background", + }), + ); + expect( + row.getByText("Downloading OpenBot. About 2 minutes left."), + ).toBeTruthy(); + const carried = row.getByRole("progressbar", { name: "Download progress" }); + expect(Number(carried.getAttribute("value"))).toBeCloseTo(2.0 / 2.9, 3); + expect(carried.getAttribute("aria-valuetext")).toBe("2.0 GB of 2.9 GB"); + // They were started once, by the first screen. + expect(called("start_early_downloads")).toHaveLength(1); +}); + +test("a disk too full to download to is said plainly, with what is free and what is needed", async () => { + let view!: ReturnType; + await act(async () => { + view = render(); + }); + await view.findByRole("heading", { name: "Set up OpenBot" }); + await said({ + state: "short", + bytes: 0, + free: 14.6 * GB, + needed: 44.3 * GB, + }); + const line = view.getByRole("status"); + expect(line.textContent).toBe( + "This computer has 15 GB of free space, and OpenBot needs about 44 GB. Free up some space and the download starts by itself.", + ); + // Nothing is downloading, so nothing spins and there is no bar. + expect(line.querySelector(".step-spinner")).toBeNull(); + expect(view.queryByRole("progressbar")).toBeNull(); +}); + +test("the line says what is known, and only the first screen says when it is all downloaded", () => { + const downloading = { + state: "downloading" as const, + bytes: GB, + size: 2 * GB, + }; + expect(earlyDownloadsSaid(downloading, true)).toBe("Downloading OpenBot."); + expect( + earlyDownloadsSaid(downloading, false, { left: 40, slow: false }), + ).toBe("Downloading OpenBot. Less than a minute left."); + expect( + earlyDownloadsSaid(downloading, false, { left: 80, slow: false }), + ).toBe("Downloading OpenBot. About a minute left."); + expect( + earlyDownloadsSaid(downloading, false, { left: null, slow: true }), + ).toBe("Downloading OpenBot. Taking longer than usual, still working."); + expect(earlyDownloadsSaid({ state: "done", bytes: GB, size: GB }, true)).toBe( + "OpenBot is downloaded and ready to install.", + ); + expect( + earlyDownloadsSaid({ state: "done", bytes: GB, size: GB }, false), + ).toBe(null); + expect(earlyDownloadsSaid({ state: "failed", bytes: 0 }, true)).toBe( + "Some downloads didn’t finish. Install tries them again.", + ); + expect(earlyDownloadsSaid({ state: "nothing", bytes: 0 }, true)).toBe(null); + // No em dashes, and nothing technical, in anything it says. + for (const state of ["downloading", "short", "done", "failed"] as const) { + const line = + earlyDownloadsSaid( + { state, bytes: GB, size: 2 * GB, free: GB, needed: 44 * GB }, + true, + { left: 200, slow: false }, + ) ?? ""; + expect(line).not.toMatch(/—|Podman|machine|image|quay|engine/i); + } +}); + +test("the background row carries the first screen's download as one bar, and keeps what the person has to see", async () => { + const { installingBehind } = await import("./SetupProgress"); + const backEnd = { + at: null, + running: false, + stopped: false, + ready: false, + detail: null, + download: null, + since: null, + }; + const early = { bytes: 1.5 * GB, size: 2.9 * GB, bytesPerSecond: 10_000_000 }; + const own = { + done: 0, + total: 0, + bytes: 600_000_000, + size: 937_000_000, + bytesPerSecond: 20_000_000, + }; + const behind = ( + step: Record, + first: { bytes: number; size?: number } | null = early, + ) => + installingBehind({ + steps: [ + { step: "engine", ok: true, detail: "", running: false }, + step as never, + ], + installing: true, + installed: false, + backEnd, + early: first, + }); + + // Install is checking what it needs: the row is the first screen's download. + const checking = behind({ + step: "install-engine", + ok: true, + running: true, + detail: "Checking the software OpenBot needs.", + }); + expect(checking?.detail).toBe("Downloading OpenBot."); + expect(checking?.downloads).toEqual(early); + + // Install waiting on the machine image while the first screen's downloads still run: the same + // one bar over all of them, so it never goes back to the machine image's own. + const waitingOnTheMachine = behind({ + step: "create-machine", + ok: true, + running: true, + detail: "Preparing this computer to run OpenBot.", + downloads: own, + }); + expect(waitingOnTheMachine?.detail).toBe("Downloading OpenBot."); + expect(waitingOnTheMachine?.downloads).toEqual(early); + + // Once those are done, a download of Install's own is the row's. + const machine = behind( + { + step: "create-machine", + ok: true, + running: true, + detail: "Preparing this computer to run OpenBot.", + downloads: own, + }, + null, + ); + expect(machine?.detail).toBe("Downloading OpenBot."); + expect(machine?.downloads).toEqual(own); + + // The OS asking in its own window, explained in plain words: the row keeps them. + const prompting = behind({ + step: "install-engine", + ok: true, + running: true, + elsewhere: true, + detail: "Enter your Mac password when macOS asks for it.", + }); + expect(prompting?.detail).toBe( + "Enter your Mac password when macOS asks for it.", + ); + expect(prompting?.downloads).toBeUndefined(); + + // A closed password prompt: the row says only what it waits for. + const declined = behind({ + step: "install-engine", + ok: true, + running: true, + asks: true, + detail: "macOS didn't get your password.", + }); + expect(declined?.detail).toBe("Waiting for your password."); + expect(declined?.waiting).toBe(true); + + // Nothing downloading early: the row says what Install is doing, as before. + const plain = behind( + { + step: "create-machine", + ok: true, + running: true, + detail: "Preparing this computer to run OpenBot.", + }, + null, + ); + expect(plain?.detail).toBe("Preparing this computer to run OpenBot."); + expect(plain?.downloads).toBeUndefined(); +}); diff --git a/desktop/src/EarlyDownloads.tsx b/desktop/src/EarlyDownloads.tsx new file mode 100644 index 0000000000..8bcc06fc08 --- /dev/null +++ b/desktop/src/EarlyDownloads.tsx @@ -0,0 +1,138 @@ +import { invoke } from "@tauri-apps/api/core"; +import { listen } from "@tauri-apps/api/event"; +import { useEffect, useState } from "react"; +import { leftSaid } from "./journey"; +import { + type Downloads, + DownloadProgress, + StatusLine, + useDownloadWhere, +} from "./SetupProgress"; + +/** + * The big downloads that start when the first screen shows, before Install is clicked + * (`early_downloads.rs`): the engine machine image, Podman and Compose, the deployment and + * OpenBot's images. One line says where they are, on every setup screen they run behind. + */ +export type EarlyDownloads = { + state: "idle" | "short" | "downloading" | "done" | "failed" | "nothing"; + /** Downloaded so far, over every download. */ + bytes: number; + /** Every download together, once each one's size is known. */ + size?: number; + bytesPerSecond?: number; + /** Free on this computer's disk, and what setup needs, when the disk is short. */ + free?: number; + needed?: number; +}; + +/** Where the early downloads are, as the native side last said. */ +export function useEarlyDownloads(): EarlyDownloads | null { + const [downloads, setDownloads] = useState(null); + useEffect(() => { + let active = true; + const stop = listen("setup:downloads", (event) => { + if (active) setDownloads(event.payload); + }); + // Downloads that started before this screen was drawn. + invoke("early_downloads_now") + .then((now) => { + if (active) setDownloads((seen) => seen ?? now); + }) + .catch(() => {}); + return () => { + active = false; + stop.then((unlisten) => unlisten()).catch(() => {}); + }; + }, []); + return downloads; +} + +const GB = 1_000_000_000; + +/** The early downloads as a download the setup rows can show, while they run; null otherwise. */ +export function earlyDownload( + downloads: EarlyDownloads | null, +): Downloads | null { + if (downloads?.state !== "downloading") return null; + return { + bytes: downloads.bytes, + size: downloads.size, + bytesPerSecond: downloads.bytesPerSecond, + }; +} + +function gigabytes(bytes: number): string { + return `${Math.max(1, Math.round(bytes / GB))} GB`; +} + +/** + * The line for `downloads`, or null when there is nothing to say. In the words of the background + * install's row: "Downloading OpenBot. About 3 minutes left.", the time left as that row works it + * out (`left`, in seconds; null while it is not known, `slow` when it is taking longer than usual). + * The first screen also says when everything is downloaded, or when something will download at + * Install instead; later screens say only what is still going on, since Install's own rows take + * over from there. + */ +export function earlyDownloadsSaid( + downloads: EarlyDownloads, + firstScreen: boolean, + time: { left: number | null; slow: boolean } | null = null, +): string | null { + const { state, free, needed } = downloads; + switch (state) { + case "downloading": { + if (time?.slow) return `Downloading OpenBot. ${leftSaid(null)}.`; + return time?.left != null + ? `Downloading OpenBot. ${leftSaid(time.left)}.` + : "Downloading OpenBot."; + } + case "short": + return free != null && needed != null + ? `This computer has ${gigabytes(free)} of free space, and OpenBot needs about ${gigabytes(needed)}. Free up some space and the download starts by itself.` + : null; + case "done": + return firstScreen ? "OpenBot is downloaded and ready to install." : null; + case "failed": + return firstScreen + ? "Some downloads didn’t finish. Install tries them again." + : null; + default: + return null; + } +} + +/** + * The early downloads' line, with the background row's bar while they download: one bar over every + * download, that only moves forward. Nothing when there is nothing to say. + */ +export function EarlyDownloadsLine({ + firstScreen = false, +}: { + /** + * The Welcome screen, where the downloads start. There the line keeps the height of its tallest + * state (a line, a bar and the bytes) whatever it says, and when it says nothing: on Windows F + * the usage-data text under it moved up about 40px when the download finished. + */ + firstScreen?: boolean; +}) { + const downloads = useEarlyDownloads(); + const download = earlyDownload(downloads); + const where = useDownloadWhere(download); + const said = downloads + ? earlyDownloadsSaid(downloads, firstScreen, where) + : null; + if (!said && !firstScreen) return null; + return ( +
+ {said && {said}} + {said && download && where && ( + + )} +
+ ); +} diff --git a/desktop/src/SetupProgress.tsx b/desktop/src/SetupProgress.tsx index 1a4624250e..6cf8afe742 100644 --- a/desktop/src/SetupProgress.tsx +++ b/desktop/src/SetupProgress.tsx @@ -345,7 +345,7 @@ function SetupProgressRow({ * Install shows its download from the start, items and all. The back end's update shows one only * once it has something to say: an update with nothing to download is not a download. */ -function showsDownload({ bytes, size, total }: Downloads): boolean { +export function showsDownload({ bytes, size, total }: Downloads): boolean { return (total ?? 0) > 0 || bytes > 0 || (size ?? 0) > 0; } @@ -355,7 +355,9 @@ function showsDownload({ bytes, size, total }: Downloads): boolean { * byte came in, so the part unpacking now moves the bar and the time left, and the parts that * finished say how long one takes here. */ -function useDownloadWhere(downloads: Downloads | null): DownloadWhere | null { +export function useDownloadWhere( + downloads: Downloads | null, +): DownloadWhere | null { const [, tick] = useState(0); const seen = useRef(null); const now = Date.now(); @@ -405,7 +407,7 @@ function downloadSaid(step: Progress, where: DownloadWhere): string { * do on 1.0.0-rc.1, and on Windows F, with its last bytes uncounted, "1.1 GB of 1.1 GB · 2 of 5 * parts left" sat under a full bar with no time left, and looked stuck. */ -function DownloadProgress({ +export function DownloadProgress({ downloads, where, }: { @@ -536,6 +538,10 @@ export type BackEndNow = { * `installing` and `installed` are OpenBot's own install (`prepare_installation`), whose rows are * `steps`; the back end's setup follows it (`backEnd`). `waitingFor` is what the rest waits on when * it waits on the person; `done` is what the row says once everything is installed. + * + * `early` is the download the first screen started before Install was clicked (`EarlyDownloads`), + * while it runs. The row shows it whenever the step Install is on shows no download of its own, so + * the bytes arriving behind a step that waits on the person are never hidden. */ export function installingBehind({ steps, @@ -545,6 +551,7 @@ export function installingBehind({ backEnd, waitingFor, done, + early, }: { steps: Progress[]; installing: boolean; @@ -553,6 +560,7 @@ export function installingBehind({ backEnd: BackEndNow; waitingFor?: string; done?: string; + early?: Downloads | null; }): Progress | null { const row = ( at: number, @@ -571,20 +579,32 @@ export function installingBehind({ const preparing = now ? PREPARING.has(now.step) : !steps.some((step) => !PREPARING.has(step.step)); + const own = + now?.downloads && showsDownload(now.downloads) ? now.downloads : null; + const meanwhile = early && showsDownload(early) ? early : null; if (preparing) { // A closed password prompt has its own card, with Try again: the row says only what it // waits for. const waiting = now?.asks === true; - return row( - 1, - waiting - ? "Waiting for your password." - : now?.detail || "Preparing this computer to run OpenBot.", - { waiting, startedAt: since ?? undefined }, - ); + if (waiting) + return row(1, "Waiting for your password.", { + waiting, + startedAt: since ?? undefined, + }); + // The first screen's downloads while they run, the machine it runs on among them, so one bar + // carries them all and only moves forward; then a download of Install's own. A prompt in + // another window keeps its words. + const downloading = now?.elsewhere ? null : (meanwhile ?? own); + if (downloading) + return row(1, "Downloading OpenBot.", { + downloads: downloading, + startedAt: since ?? undefined, + }); + return row(1, now?.detail || "Preparing this computer to run OpenBot.", { + startedAt: since ?? undefined, + }); } - const downloads = - now?.downloads && showsDownload(now.downloads) ? now.downloads : null; + const downloads = meanwhile ?? own; return row( 2, downloads diff --git a/desktop/src/Welcome.tsx b/desktop/src/Welcome.tsx index 813f625076..3bcd670c39 100644 --- a/desktop/src/Welcome.tsx +++ b/desktop/src/Welcome.tsx @@ -1,4 +1,5 @@ import type { ReactNode } from "react"; +import { EarlyDownloadsLine } from "./EarlyDownloads"; import { ExternalLink } from "./ExternalLink"; /** What OpenBot Desktop sends, and what it never does, in full. */ @@ -15,6 +16,9 @@ export const USAGE_DATA_DOC = * * The usage-data notice is here because this is the first screen, and the CopilotKit sign-in that * links the data to an account is the next. Settings > General turns it off. + * + * The big downloads start while this screen shows, before Install is clicked, and its line says how + * far they are (`EarlyDownloadsLine`). */ export function Welcome({ onInstall, @@ -40,6 +44,7 @@ export function Welcome({ Install
+

OpenBot sends CopilotKit usage data, linked to your CopilotKit account. It never sends your conversations, files or keys. You can turn this off diff --git a/desktop/src/setupLayout.browser.test.ts b/desktop/src/setupLayout.browser.test.ts index 88ea219b00..9aeed709a4 100644 --- a/desktop/src/setupLayout.browser.test.ts +++ b/desktop/src/setupLayout.browser.test.ts @@ -416,6 +416,131 @@ describe.skipIf(!asked)("a setup step's foot", () => { } }); +/* + * Seen on the Account step while the person signs in: its Cancel sits above the foot, the foot's + * own actions row is empty, and the background row sat flush against Cancel with no gap at all. The + * row sits the card's one gap (24px) under the step's button on every step, Cancel included. + */ +describe.skipIf(!asked)("the background row under the step's button", () => { + for (const [width, height] of [ + [1000, 800], + [720, 560], + ] as const) { + test(`at ${width}x${height}, is one card gap below Cancel, Accept and Continue`, async () => { + /** The gap from the button's bottom to the background row's top, the card scrolled to its end. */ + const gap = async (page: Page, name: string) => { + await page.waitForTimeout(600); + return page.evaluate((label) => { + const sheet = document.querySelector("main > .sheet") as HTMLElement; + sheet.scrollTop = sheet.scrollHeight; + const button = [...document.querySelectorAll("button")].find( + (found) => found.textContent?.trim() === label, + ); + const row = document.querySelector(".behind-slot > .steps"); + if (!button || !row) return null; + return Math.round( + row.getBoundingClientRect().top - + button.getBoundingClientRect().bottom, + ); + }, name); + }; + const scenes: Array<[string, string, string]> = [ + ["#signin", "Self-contained OpenBot", "Cancel"], + ["#licence", "Accept the license agreement", "Accept"], + ["", "Connect your AI", "Continue"], + ]; + for (const [hash, heading, button] of scenes) { + const page = await browser.newPage({ viewport: { width, height } }); + try { + await open(page, hash); + await page + .getByRole("button", { name: "Install", exact: true }) + .click(); + await page.getByRole("heading", { name: heading }).waitFor(); + expect({ button, gap: await gap(page, button) }).toEqual({ + button, + gap: 24, + }); + } finally { + await page.close(); + } + } + }, 40_000); + } +}); + +/* + * Seen on Windows F: on the first screen the download line shrank about 40px when the download + * finished ("Downloading OpenBot." with its bar and bytes, then "OpenBot is downloaded and ready to + * install."), and the usage-data text under it moved up. The line holds its tallest height in every + * state, and before it says anything. + */ +describe.skipIf(!asked)("the first screen while its downloads run", () => { + for (const [width, height] of [ + [1000, 800], + [720, 560], + ] as const) { + test(`at ${width}x${height}, the text under the download line stays where it is`, async () => { + const page = await browser.newPage({ viewport: { width, height } }); + try { + await open(page); + await page.getByRole("heading", { name: "Set up OpenBot" }).waitFor(); + await page.waitForTimeout(400); + const usage = async () => + Math.round( + ( + await page + .getByText("OpenBot sends CopilotKit usage data", { + exact: false, + }) + .boundingBox() + )?.y ?? -1, + ); + const say = async (payload: Record) => { + await page.evaluate((sent) => { + ( + globalThis as unknown as { + __fake: { emit: (name: string, payload: unknown) => void }; + } + ).__fake.emit("setup:downloads", sent); + }, payload); + await page.waitForTimeout(300); + }; + const GB = 1_000_000_000; + const seen: Array<[string, number]> = [["nothing yet", await usage()]]; + for (const [state, payload] of [ + ["starting", { state: "downloading", bytes: 0 }], + [ + "downloading", + { + state: "downloading", + bytes: 0.15 * GB, + size: 0.31 * GB, + bytesPerSecond: 9_000_000, + }, + ], + ["done", { state: "done", bytes: 0.31 * GB, size: 0.31 * GB }], + ["failed", { state: "failed", bytes: 0.2 * GB, size: 0.31 * GB }], + [ + "short of disk", + { state: "short", bytes: 0, free: 14.6 * GB, needed: 44.3 * GB }, + ], + ] as const) { + await say(payload); + seen.push([state, await usage()]); + } + const first = seen[0]?.[1]; + expect(first).toBeGreaterThan(0); + expect(seen.map(([state, y]) => [state, y - (first ?? 0)])).toEqual( + seen.map(([state]) => [state, 0]), + ); + } finally { + await page.close(); + } + }, 30_000); + } +}); + describe.skipIf(!asked)("the AI step while OpenBot installs behind it", () => { for (const [width, height] of [ [1000, 800], diff --git a/desktop/src/styles.css b/desktop/src/styles.css index 7e900adda4..d6bb16a7c2 100644 --- a/desktop/src/styles.css +++ b/desktop/src/styles.css @@ -1516,6 +1516,27 @@ a:hover { flex: none; } +/* + * On the first screen the line keeps the height of its tallest state, the line, the bar and the + * bytes under it, whatever it says: the usage-data text under it never moves (`EarlyDownloadsLine`). + * Its own box, so the line's top margin is inside the height it holds. + */ +.early-downloads.first-screen { + display: flow-root; + min-height: 72px; +} + +/* The downloads the first screen started: their line, and its bar under it. */ +.early-downloads .downloads { + display: grid; + gap: 6px; + margin-top: 8px; + color: var(--text-secondary); + font-size: var(--text-sm); + line-height: 20px; + font-variant-numeric: tabular-nums; +} + /* The Status window: one card per part, each with its memory and its actions. */ .status-part { border: 1px solid var(--border); @@ -1738,6 +1759,14 @@ main > .sheet > .step-footer > .row:first-child:empty { padding-top: 0; } +/* + * With its own actions above the foot, as Cancel is while the account step signs in, the foot's + * empty row still holds the card's one gap above the background row. It sat flush against Cancel. + */ +main > .sheet > .step-footer > .row:first-child:empty:has(+ .behind-slot) { + padding-top: 24px; +} + @media (max-width: 580px) { main > .sheet > .step-footer { bottom: -24px; diff --git a/docs/releasing.md b/docs/releasing.md index 9c7ea8d3e8..1779abfcca 100644 --- a/docs/releasing.md +++ b/docs/releasing.md @@ -637,7 +637,10 @@ it: they hash what they receive. A key that is not there answers 403, not 404 (C Docker Hub's images are copied the same way under `v2/docker.io//`, official images under `library/` (`v2/docker.io/library/busybox`), so a client uses `downloads.copilotkit.ai/docker.io` as -the mirror for `docker.io`. Images on any other registry are left where they are. +the mirror for `docker.io`. quay.io's are copied under `v2/quay.io//`: the Podman engine machine +images, which the app downloads itself from the first setup screen as plain blobs +(`v2/quay.io/podman/machine-os/blobs/`, then quay.io), and builds the machine from +(`desktop/src-tauri/src/early_downloads.rs`). Images on any other registry are left where they are. Which images: every image the release's `container-images.json` pins, all of them, and every image `desktop-images.json` on the release names. The `image list` job of the publish workflow writes that @@ -647,7 +650,9 @@ list with the app's own code (`desktop/src-tauri/src/release_images.rs`, run as every Bot the app can install (`pgvector` included), the two sign-in images, the bundled CLI's back end images (its runtime images at its release's tag, its pinned ones and the Kubernetes system images), and the cluster's own images that the host's engine pulls when k3d creates the cluster (the -k3s node image and k3d's proxy and tools, at the digests the CLI gives k3d). A release published +k3s node image and k3d's proxy and tools, at the digests the CLI gives k3d), and the engine machine +images Podman's machine is built from (`quay.io/podman/machine-os`, by the digests +`early_downloads.rs` pins). A release published before this carries no list, and only its `container-images.json` images are copied; `DOWNLOADS_IMAGES_FILE` names a list to use instead. Images are read with [crane](https://github.com/google/go-containerregistry) (a pinned release, checked by diff --git a/scripts/downloads-images.test.ts b/scripts/downloads-images.test.ts index 0d0a4570d6..0fc9c344b6 100644 --- a/scripts/downloads-images.test.ts +++ b/scripts/downloads-images.test.ts @@ -268,6 +268,8 @@ test("a release's images are copied under the registry's paths, byte for byte an const pgvector = push(fake, "pgvector/pgvector", "pg17", "docker.io"); const busybox = push(fake, "library/busybox", "1.37", "docker.io"); const k3s = push(fake, "rancher/k3s", "v1.32.5-k3s1", "docker.io"); + // quay.io's: the engine machine image the app downloads before Install, named by digest. + const machineOs = push(fake, "podman/machine-os", "6.1", "quay.io"); const list = { openbot: "v0.3.4", intelligence: "0.16.0-rc.9", @@ -277,7 +279,8 @@ test("a release's images are copied under the registry's paths, byte for byte an "ghcr.io/copilotkit/intelligence/app-api:0.16.0-rc.9", `ghcr.io/copilotkit/openbot-server@${server.digest}`, "pgvector/pgvector:pg17", - "quay.io/someone/else:1", + `quay.io/podman/machine-os@${machineOs.digest}`, + "mcr.microsoft.com/someone/else:1", ], }; // Every image the release pins is copied, listed in desktop-images.json or not. @@ -312,7 +315,7 @@ test("a release's images are copied under the registry's paths, byte for byte an // Each image's top manifest, every manifest its index names and the referrers of each are asked // for through the address before the run succeeds. expect(first.out).toContain( - `All 6 images for v0.3.4 answer through ${edge.url}, every manifest and referrers object included.`, + `All 7 images for v0.3.4 answer through ${edge.url}, every manifest and referrers object included.`, ); expect(edge.requests).toContain( `GET v2/copilotkit/openbot-supervisor/manifests/${supervisor.digest}`, @@ -349,6 +352,8 @@ test("a release's images are copied under the registry's paths, byte for byte an ...manifests("docker.io/library/busybox", busybox), "v2/docker.io/library/busybox/manifests/1.37", ...manifests("docker.io/rancher/k3s", k3s), + // quay.io's under quay.io/. + ...manifests("quay.io/podman/machine-os", machineOs), // Named only by container-images.json. ...manifests("copilotkit/openbot-supervisor", supervisor), ]) { @@ -372,18 +377,18 @@ test("a release's images are copied under the registry's paths, byte for byte an cacheControl: TAG, sha256: true, }); - // Six images, each an index and four manifests (three platforms and an attestation), and three + // Seven images, each an index and four manifests (three platforms and an attestation), and three // tags; a referrers object for each manifest. expect( Object.keys(bucket).filter((key) => key.includes("/manifests/")), - ).toHaveLength(33); + ).toHaveLength(38); expect( Object.keys(bucket).filter((key) => key.includes("/referrers/")), - ).toHaveLength(30); + ).toHaveLength(35); // A config and two layers for each of the four manifests, an image. expect( Object.keys(bucket).filter((key) => key.includes("/blobs/")), - ).toHaveLength(72); + ).toHaveLength(84); expect(bucket["v2/"]).toEqual({ content: "{}", contentType: "application/json", @@ -441,12 +446,12 @@ test("a release's images are copied under the registry's paths, byte for byte an expect( bucket["tools/compose/v5.5.1/docker-compose-darwin-aarch64"]?.content, ).toBe("compose arm64"); - // Only ghcr.io and Docker Hub are copied. + // Only ghcr.io, Docker Hub and quay.io are copied. expect( - fake.calls().some((call) => call.startsWith("crane") && /quay/.test(call)), + fake.calls().some((call) => call.startsWith("crane") && /mcr/.test(call)), ).toBe(false); expect(first.out).toContain( - "quay.io/someone/else:1 is not on ghcr.io or Docker Hub; it is left to its own registry.", + "mcr.microsoft.com/someone/else:1 is not on ghcr.io, Docker Hub or quay.io; it is left to its own registry.", ); // The tags and the ping are invalidated together, after the last upload, and every new object // fetched once through the address. @@ -531,7 +536,7 @@ test("a release's images are copied under the registry's paths, byte for byte an const refused = await run(fake, env); expect(refused.code).toBe(1); expect(refused.out).toContain( - `::error::1 of the objects for the 6 images for v0.3.4 do not answer through ${edge.url}: ${edge.url}/${stale} (403)`, + `::error::1 of the objects for the 7 images for v0.3.4 do not answer through ${edge.url}: ${edge.url}/${stale} (403)`, ); // The check alone, as a person runs it after a release: no AWS, no registry. const checked = (bucketless: Record) => @@ -550,7 +555,7 @@ test("a release's images are copied under the registry's paths, byte for byte an const whole = await checked({}); expect(whole.code).toBe(0); expect(whole.out).toContain( - `All 6 images for v0.3.4 answer through ${edge.url}, every manifest and referrers object included.`, + `All 7 images for v0.3.4 answer through ${edge.url}, every manifest and referrers object included.`, ); expect(fake.calls().filter((call) => /^(aws|crane) /.test(call)).length).toBe( awsBefore, diff --git a/scripts/downloads-images.ts b/scripts/downloads-images.ts index 6d42f1b519..a0358c3c21 100644 --- a/scripts/downloads-images.ts +++ b/scripts/downloads-images.ts @@ -6,8 +6,9 @@ * Run by .github/workflows/downloads-sync.yml after scripts/downloads-sync.sh, for the same * release, and by hand. The release's `desktop-images.json` says which images: the Desktop app's * own code writes it when the release is published (desktop/src-tauri/src/release_images.rs), so - * it names exactly what the app pulls. Images on ghcr.io and Docker Hub are copied; any other registry is - * left as it is. + * it names exactly what the app pulls. Images on ghcr.io, Docker Hub and quay.io are copied; any other + * registry is left as it is. quay.io holds the engine machine images Podman builds its machine from, + * which the app downloads before Install (desktop/src-tauri/src/early_downloads.rs). * * What it writes, in the bucket behind the CloudFront distribution: a read-only copy of the * registry's own paths (the OCI Distribution API, GET only), so the address works as a registry: @@ -21,8 +22,9 @@ * v2//referrers/sha256: an empty image index for each of those manifests, so the * referrers API Docker 29 asks with every pull answers 200; * cached for five minutes - * is the path on ghcr.io unchanged, such as copilotkit/openbot-server, or for Docker Hub - * docker.io/ and its path, official images under library/ (docker.io/library/busybox). The whole + * is the path on ghcr.io unchanged, such as copilotkit/openbot-server, for Docker Hub + * docker.io/ and its path, official images under library/ (docker.io/library/busybox), and for + * quay.io quay.io/ and its path (quay.io/podman/machine-os). The whole * index is copied because Docker 29 fetches every manifest it names, attestations included, and a * 403 for one fails the pull. An image named by digest whose referrers are there already is whole, * and is not read again. @@ -117,7 +119,8 @@ export function imageRef( const hosted = first.includes(".") || first.includes(":") || first === "localhost"; const host = hosted ? first : "docker.io"; - if (host !== "ghcr.io" && !DOCKER_HUB.has(host)) return undefined; + if (host !== "ghcr.io" && host !== "quay.io" && !DOCKER_HUB.has(host)) + return undefined; let rest = hosted ? reference.slice(slash + 1) : reference; let digest: string | undefined; const at = rest.indexOf("@"); @@ -142,14 +145,20 @@ export function imageRef( const where = host === "ghcr.io" ? { name: `ghcr.io/${rest}`, path: rest, read: `${source}/${rest}` } - : (() => { - const repo = rest.includes("/") ? rest : `library/${rest}`; - return { - name: `docker.io/${repo}`, - path: `docker.io/${repo}`, - read: `docker.io/${repo}`, - }; - })(); + : host === "quay.io" + ? { + name: `quay.io/${rest}`, + path: `quay.io/${rest}`, + read: `quay.io/${rest}`, + } + : (() => { + const repo = rest.includes("/") ? rest : `library/${rest}`; + return { + name: `docker.io/${repo}`, + path: `docker.io/${repo}`, + read: `docker.io/${repo}`, + }; + })(); // A digest names the image exactly; a tag beside it is only a label, as it is to a client. return digest ? { ...where, digest } : { ...where, tag }; } @@ -702,7 +711,7 @@ async function copy(options: Copy): Promise { } for (const image of elsewhere) console.log( - `${image} is not on ghcr.io or Docker Hub; it is left to its own registry.`, + `${image} is not on ghcr.io, Docker Hub or quay.io; it is left to its own registry.`, ); // What S3 holds for a key: its SHA-256 as S3 computed it on an upload sent with one, its