diff --git a/CHANGELOG.md b/CHANGELOG.md index b75de103f9..c70ae2f58b 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -20,6 +20,18 @@ Newest first. `Unreleased` is what is on `main` and not yet tagged. Downloads sync now copies a prerelease asked for by tag (its images and its own files, never `latest/`), and each scheduled run copies the images of every prerelease newer than Latest from the last 30 days. Drafts are never copied. +- **A back end start that fails on a nearly full disk says so, with the numbers.** Relaunching a + used install whose back end's disk was nearly full said only "The back end stopped with an error + partway through." Each start, update and setup now checks the engine first and logs what it has + as `[local check]`, and a start that fails is checked again with the back end asked whether it is + short of disk. The card then says, for example, "The back end's disk is nearly full. Podman has + 30.5 GB free of 1006.8 GB. OpenBot needs 40 GB. Free space on the drive that holds OpenBot's + engine (C:), then Retry." On Windows, free disk is measured on that drive, which can run out + while the machine itself shows hundreds of GB free, and the card never suggests + `podman machine set`, which WSL machines refuse. In the few minutes after space was freed, while + the back end still reports low disk, a start that fails, or a Retry that finds the back end + running but not healthy, says "The back end is recovering from low disk space. Wait a few + minutes, then Retry." where Retry said "The back end is running but not healthy." ## 1.0.0 diff --git a/desktop/src-tauri/Cargo.lock b/desktop/src-tauri/Cargo.lock index 027563c7ca..6d4236a9d5 100644 --- a/desktop/src-tauri/Cargo.lock +++ b/desktop/src-tauri/Cargo.lock @@ -2780,6 +2780,7 @@ dependencies = [ "tauri-plugin-updater", "time", "windows 0.62.2", + "windows-registry", "zip 2.4.2", ] diff --git a/desktop/src-tauri/Cargo.toml b/desktop/src-tauri/Cargo.toml index 1e680ce755..b1e3a8bee4 100644 --- a/desktop/src-tauri/Cargo.toml +++ b/desktop/src-tauri/Cargo.toml @@ -84,7 +84,10 @@ objc2-av-foundation = { version = "0.3", default-features = false, features = [" # Narration on Windows: the SAPI in-process recognizer. See `speech/sapi.rs`. The version Tauri uses. [target.'cfg(windows)'.dependencies] -windows = { version = "0.62", features = ["Win32_Foundation", "Win32_Globalization", "Win32_Media_Speech", "Win32_System_Com", "Data_Xml_Dom", "UI_Notifications", "Win32_UI_WindowsAndMessaging", "Win32_System_Threading"] } +windows = { version = "0.62", features = ["Win32_Foundation", "Win32_Globalization", "Win32_Media_Speech", "Win32_System_Com", "Data_Xml_Dom", "UI_Notifications", "Win32_UI_WindowsAndMessaging", "Win32_System_Threading", "Win32_Storage_FileSystem"] } +# Where WSL keeps a Podman machine's disk, so its drive's free space is checked before Start. See +# `windows::podman_machine_drive`. The version reqwest already uses. +windows-registry = "0.6" [dev-dependencies] tauri = { version = "2", features = ["test"] } diff --git a/desktop/src-tauri/src/local_eval.rs b/desktop/src-tauri/src/local_eval.rs index 4b9ce3a051..57bdd8cdfd 100644 --- a/desktop/src-tauri/src/local_eval.rs +++ b/desktop/src-tauri/src/local_eval.rs @@ -248,6 +248,8 @@ pub struct Report { /// from a back end that is down (see `UnfinishedUpdate`). pub checks_passed: bool, /// With `Reason::ResourcesShort`: which resources the CLI found short, and on which engine. + /// With `Readiness::Degraded`: a disk OpenBot found short, or recovering, behind a back end that + /// runs but does not answer (`checked_when_unhealthy`). #[serde(skip_serializing_if = "Option::is_none")] pub shortfall: Option, } @@ -269,6 +271,25 @@ pub struct Shortfall { pub memory_gb: Option, #[serde(skip_serializing_if = "Option::is_none")] pub disk_gb: Option, + /// The back end's own disk is nearly full: its node reports `DiskPressure`, or so little is + /// free that k3s removes the back end's images (`EngineCheck::nearly_full`). OpenBot's own + /// check after a start that failed, never the CLI's. + #[serde(skip_serializing_if = "std::ops::Not::not")] + pub nearly_full: bool, + /// How large that disk is, in GiB to one decimal, beside `disk_gb`, when it is nearly full. + #[serde(skip_serializing_if = "Option::is_none")] + pub disk_size_gb: Option, + /// The node still reports `DiskPressure` with the disk above both lines: k3s keeps the + /// condition for five minutes after space is freed, and the back end recovers by itself. + #[serde(skip_serializing_if = "std::ops::Not::not")] + pub recovering: bool, + /// Podman on Windows, whose machine runs in WSL: `podman machine set` changes none of its + /// resources there, and its disk is freed on the drive that holds it. + #[serde(skip_serializing_if = "std::ops::Not::not")] + pub wsl: bool, + /// That drive, `C:`, when WSL says where the machine's disk is. + #[serde(skip_serializing_if = "Option::is_none")] + pub drive: Option, } /// A number the CLI printed, to one decimal and rounded down as it rounds, without a trailing `.0`. @@ -280,7 +301,7 @@ fn shown(value: f64) -> String { impl Shortfall { fn named(&self) -> bool { - self.cpu || self.memory || self.disk + self.cpu || self.memory || self.disk || self.recovering } } @@ -845,6 +866,11 @@ pub enum Terms { cpu: bool, memory: bool, disk: bool, + /// Podman on Windows, in WSL (`Shortfall::wsl`), and the drive that holds its disk. + #[serde(skip_serializing_if = "std::ops::Not::not")] + wsl: bool, + #[serde(skip_serializing_if = "Option::is_none")] + drive: Option, }, } @@ -1198,17 +1224,28 @@ pub fn parse_engine_size(output: &str) -> Option<(u64, u64)> { fields.next().is_none().then_some((cpus, memory)) } -/// Free bytes from `df -Pk /`: the fourth field of its last line, in KiB, as the CLI reads it. +/// A filesystem as `df -Pk` reports it, in bytes. +#[derive(Clone, Copy, Debug, PartialEq, Eq)] +pub struct Disk { + pub size: u64, + pub free: u64, +} + +/// Size and free bytes from `df -Pk /`: the second and fourth fields of its last line, in KiB, as +/// the CLI reads them. +pub fn parse_disk(output: &str) -> Option { + let mut fields = output.trim().lines().last()?.split_whitespace(); + let size: u64 = fields.nth(1)?.parse().ok()?; + let free: u64 = fields.nth(1)?.parse().ok()?; + Some(Disk { + size: size * 1024, + free: free * 1024, + }) +} + +/// Free bytes from `df -Pk /`, as the CLI reads them. pub fn parse_free_disk(output: &str) -> Option { - let kib: u64 = output - .trim() - .lines() - .last()? - .split_whitespace() - .nth(3)? - .parse() - .ok()?; - Some(kib * 1024) + parse_disk(output).map(|disk| disk.free) } /// The engine's CPUs and memory from its `info`. @@ -1228,11 +1265,11 @@ fn engine_size(address: &crate::engine::Address) -> Option<(u64, u64)> { } /** -Free disk inside a container, the way the CLI measures it: `df -Pk /` in a container of an image the -engine already has, so nothing is downloaded. An engine with no image that has a shell gives no -answer, and disk is then not judged. +The engine's disk inside a container, the way the CLI measures it: `df -Pk /` in a container of an +image the engine already has, so nothing is downloaded. An engine with no image that has a shell +gives no answer, and disk is then not judged. */ -fn engine_free_disk(address: &crate::engine::Address) -> Option { +fn engine_disk(address: &crate::engine::Address) -> Option { let images = address .command() .args(["image", "ls", "--quiet", "--no-trunc"]) @@ -1263,17 +1300,209 @@ fn engine_free_disk(address: &crate::engine::Address) -> Option { "df -Pk /", ]) .output(); - let free = measured + let disk = measured .ok() .filter(|output| output.status.success()) - .and_then(|output| parse_free_disk(&String::from_utf8_lossy(&output.stdout))); - if free.is_some() { - return free; + .and_then(|output| parse_disk(&String::from_utf8_lossy(&output.stdout))); + if disk.is_some() { + return disk; } } None } +#[cfg(test)] +thread_local! { + /// What Windows says of the drive that holds the Podman machine's disk, for the tests: Some + /// stands in for `windows::podman_machine_drive` on this thread. + static MACHINE_DRIVE: std::cell::RefCell>> = + const { std::cell::RefCell::new(None) }; +} + +/// The drive that holds the Podman machine `address` names, on Windows (`podman_machine_drive`). +fn machine_drive(address: &crate::engine::Address) -> Option { + if address.engine != crate::engine::Engine::Podman { + return None; + } + #[cfg(test)] + if let Some(said) = MACHINE_DRIVE.with(|drive| drive.borrow().clone()) { + return said; + } + let connection = address + .connection + .clone() + .or_else(|| address.pin().ok().and_then(|pinned| pinned.connection))?; + crate::windows::podman_machine_drive(crate::acquire::machine_of(&connection)) +} + +/** +What OpenBot measured of the engine the back end runs on (`[local check]` in the log): its CPUs and +memory, its disk as a container sees it, on Windows the drive that holds a Podman machine's disk, and +after a call that failed, whether the back end's node says it is short of disk. +*/ +#[derive(Clone, Debug, Default, PartialEq, Eq)] +struct EngineCheck { + podman: bool, + /// CPUs and bytes of memory, from the engine's `info`. + size: Option<(u64, u64)>, + disk: Option, + drive: Option, + /// `DiskPressure` on the back end's node (`local_images::Node::disk_pressure`). + pressure: Option, +} + +/// Bytes as GiB to one decimal, as the CLI rounds them. +fn gib(bytes: u64) -> String { + shown(bytes as f64 / GIB as f64) +} + +impl EngineCheck { + fn of(address: &crate::engine::Address) -> Self { + Self { + podman: address.engine == crate::engine::Engine::Podman, + size: engine_size(address), + disk: engine_disk(address), + drive: machine_drive(address), + pressure: None, + } + } + + /// The free disk that counts: the smaller of what a container sees and what the drive under + /// the machine's disk has. Either can run out first. + fn free_disk(&self) -> Option { + [ + self.disk.map(|disk| disk.free), + self.drive.as_ref().map(|drive| drive.free), + ] + .into_iter() + .flatten() + .min() + } + + /** + The back end's own disk is nearly full, by k3s's lines rather than the CLI's 40 GiB: its node + says `DiskPressure`, which k3s raises below 5% free (`evictionHard`), or less than 15% is free, + where the kubelet starts removing the images it is not running (`imageGCHighThresholdPercent` + 85). A relaunched back end runs none yet, so that is when it loses them. + */ + fn nearly_full(&self) -> bool { + self.pressure == Some(true) && !self.recovering() || self.below_image_line() + } + + /// Less than 15% of the node's disk free: the kubelet removes images it is not running. + fn below_image_line(&self) -> bool { + self.disk + .is_some_and(|disk| u128::from(disk.free) * 100 < u128::from(disk.size) * 15) + } + + /// Below the CLI's 40 GiB, or below the kubelet's image line. + fn low_on_disk(&self) -> bool { + self.free_disk() + .is_some_and(|free| free < MIN_ENGINE_FREE_DISK_BYTES) + || self.below_image_line() + } + + /** + The node still says `DiskPressure`, but the disk is above both lines: space was freed, and k3s + keeps the condition for five minutes after (`evictionPressureTransitionPeriod`). The back end + recovers by itself, and a Retry in those minutes only fails again. + */ + fn recovering(&self) -> bool { + self.pressure == Some(true) && self.free_disk().is_some() && !self.low_on_disk() + } + + /// Podman on Windows, whose machine runs in WSL. + fn wsl(&self) -> bool { + self.podman && (cfg!(target_os = "windows") || self.drive.is_some()) + } + + /// What is short, with the numbers. Anything not measured is not judged. + fn shortfall(&self) -> Shortfall { + let (cpus, memory) = self.size.unwrap_or((u64::MAX, u64::MAX)); + let free = self.free_disk(); + let mut short = engine_shortfall(cpus, memory, free); + short.podman = self.podman; + short.wsl = self.wsl(); + short.recovering = self.recovering(); + short.nearly_full = self.nearly_full(); + short.disk |= short.nearly_full; + if short.disk { + short.disk_gb = free.map(gib); + short.disk_size_gb = self + .disk + .filter(|_| short.nearly_full) + .map(|disk| gib(disk.size)); + } + short.drive = self.drive.as_ref().and_then(|drive| drive.drive.clone()); + short + } + + /// `4 CPUs and 5.8 GiB of memory, 30.5 GiB free of 983.2 GiB in a container, 120.0 GiB free on + /// C:, no disk pressure`, then what is short. + fn said(&self) -> String { + let mut said = vec![match self.size { + Some((cpus, memory)) => format!("{cpus} CPUs and {} GiB of memory", gib(memory)), + None => "CPUs and memory not measured".to_owned(), + }]; + said.push(match self.disk { + Some(disk) => format!( + "{} GiB free of {} GiB in a container", + gib(disk.free), + gib(disk.size) + ), + None => "free disk not measured".to_owned(), + }); + if let Some(drive) = &self.drive { + let on = drive + .drive + .as_deref() + .unwrap_or("the drive that holds its disk"); + said.push(format!("{} GiB free on {on}", gib(drive.free))); + } + match self.pressure { + Some(true) => said.push("the node reports disk pressure".to_owned()), + Some(false) => said.push("no disk pressure".to_owned()), + None => {} + } + let short = self.shortfall(); + let mut verdict: Vec<&str> = Vec::new(); + if short.cpu { + verdict.push("CPUs"); + } + if short.memory { + verdict.push("memory"); + } + if short.disk { + verdict.push(if short.nearly_full { + "disk (nearly full)" + } else { + "disk (OpenBot needs 40 GiB free)" + }); + } + let verdict = if short.recovering && verdict.is_empty() { + "recovering: the node still reports disk pressure with the disk above both lines" + .to_owned() + } else if !verdict.is_empty() { + format!("short of {}", verdict.join(", ")) + } else if self.size.is_none() && self.free_disk().is_none() { + "not measured".to_owned() + } else { + "enough".to_owned() + }; + format!("{}; {verdict}", said.join(", ")) + } +} + +/// Log a check: `[local check] before local start: ...`. A shortfall is a warning. +fn log_check(when: &str, check: &EngineCheck) { + let level = if check.shortfall().named() { + log::Level::Warn + } else { + log::Level::Info + }; + log::log!(level, "[local check] {when}: {}", check.said()); +} + /** The onboarding check of the engine, before the account screen asks about the agreement: `Some(Terms::EngineTooSmall)` for an engine below `MIN_ENGINE_CPUS`, `MIN_ENGINE_MEMORY_BYTES` or @@ -1282,39 +1511,108 @@ here; the CLI's own check still runs at Start. */ pub fn engine_size_terms() -> Option { let address = cli_engine()?; - let (cpus, memory) = engine_size(&address)?; - let free_disk = engine_free_disk(&address); - let short = engine_shortfall(cpus, memory, free_disk); + let check = EngineCheck::of(&address); + let (cpus, memory) = check.size?; + let short = engine_shortfall(cpus, memory, check.free_disk()); if !short.named() { return None; } - log::warn!( - "[local terms] the engine has {cpus} CPUs, {memory} bytes of memory and {free_disk:?} bytes of free disk" - ); + log_check("before local setup", &check); Some(Terms::EngineTooSmall { - podman: address.engine == crate::engine::Engine::Podman, + podman: check.podman, cpu: short.cpu, memory: short.memory, disk: short.disk, + wsl: check.wsl(), + drive: check.drive.and_then(|drive| drive.drive), }) } /** A shortfall whose CLI error did not say which resource, measured here instead: CPUs and memory from -the engine's `info` against the CLI's 4 and 4 GiB, and free disk inside a container against its 40 -GiB. +the engine's `info` against the CLI's 4 and 4 GiB, and free disk inside a container, and on the drive +under a Podman machine's disk, against its 40 GiB. */ -fn measured_shortfall(mut shortfall: Shortfall) -> Shortfall { +fn measured_shortfall(shortfall: Shortfall) -> Shortfall { let Some(address) = cli_engine() else { return shortfall; }; - if let Some((cpus, memory)) = engine_size(&address) { - shortfall.cpu = cpus < 4; - shortfall.memory = memory < 4 * GIB; + let measured = EngineCheck::of(&address).shortfall(); + Shortfall { + podman: shortfall.podman, + ..measured + } +} + +/** +The engine checked around a call that starts the back end, `local start`, `local update` or `local +setup`. + +Before it, the check is only logged: an existing back end can run on less than the CLI's 40 GiB, and +the CLI enforces that only at a first setup. After a call that failed, it is checked again, with the +back end's node asked whether it is short of disk, and anything short becomes the report's reason +with its numbers, so the card names it and says where to raise or free it. A failure the CLI already +named, or one OpenBot stopped, keeps its own words. +*/ +fn checked_call( + command: &str, + stack: Option<&StackId>, + call: impl FnOnce() -> Result, +) -> Result { + let Some(address) = cli_engine() else { + return call(); + }; + log_check(&format!("before {command}"), &EngineCheck::of(&address)); + let result = call(); + let unexplained = match &result { + Ok(report) => { + report.outcome == Outcome::Failed + && matches!( + report.reason, + None | Some(Reason::LocalCommandFailed | Reason::Unrecognized) + ) + } + // The back end ran and stopped without a report OpenBot can use. One OpenBot stopped, or + // that never ran, is not about the engine. + Err(failure) => matches!( + failure, + Failure::Timeout + | Failure::OutputLimit + | Failure::InvalidJson + | Failure::UnexpectedExit + ), + }; + if !unexplained { + return result; + } + let mut check = EngineCheck::of(&address); + check.pressure = node_disk_pressure(stack); + log_check(&format!("after {command} failed"), &check); + let short = check.shortfall(); + match result { + Ok(mut report) if short.named() => { + report.reason = Some(Reason::ResourcesShort); + report.shortfall = Some(short); + Ok(report) + } + other => other, + } +} + +/// `DiskPressure` on `stack`'s node, or on any back end node on the engine when no stack is named +/// (a setup's failed report names none). +fn node_disk_pressure(stack: Option<&StackId>) -> Option { + let engine = recorded_engine(); + let command = || engine_command(engine.as_ref()); + let limits = crate::local_images::Limits { + call: Duration::from_secs(20), + ..crate::local_images::Limits::standard() + }; + match stack { + Some(stack) => crate::local_images::Node::for_stack(&command, stack.as_str()) + .disk_pressure(limits.call), + None => crate::local_images::any_disk_pressure(&command, limits), } - shortfall.disk = - engine_free_disk(&address).is_some_and(|free| free < MIN_ENGINE_FREE_DISK_BYTES); - shortfall } /// The version-1 envelope every `local` command shares: version, command, and a status that @@ -2918,6 +3216,55 @@ pub(crate) fn run_with( limits: (Limits, Limits), cancel: &Cancel, images_limits: crate::local_images::Limits, +) -> Result { + let call = || run_downloading(cli, attempt, action, stack, limits, cancel, images_limits); + match action { + // The calls that start the back end: the engine is checked around them. + Action::Start | Action::Update => { + checked_call(&format!("local {}", action.as_str()), Some(stack), call) + } + Action::Status => call().map(|report| checked_when_unhealthy(report, stack)), + Action::Stop => call(), + } +} + +/** +A back end that runs but does not answer (`local status` says `degraded`), checked as a failed start +is. After a start that failed on a full disk the node stays up, so Retry reads its status rather +than starting it again, and k3s keeps its `DiskPressure` for minutes after space is freed. A disk +that is short, or a node recovering from one, becomes the report's `shortfall`, and the card says +which; anything else leaves the report as it was. Not when every check passed: an update the CLI +keeps on record is `degraded` too, with the back end answering. +*/ +fn checked_when_unhealthy(mut report: Report, stack: &StackId) -> Report { + if report.outcome != Outcome::Completed + || report.readiness != Readiness::Degraded + || report.checks_passed + { + return report; + } + let Some(address) = cli_engine() else { + return report; + }; + let mut check = EngineCheck::of(&address); + check.pressure = node_disk_pressure(Some(stack)); + log_check("local status, running but not healthy", &check); + let short = check.shortfall(); + if short.disk || short.recovering { + report.shortfall = Some(short); + } + report +} + +/// `run_with`, without the engine check. +fn run_downloading( + cli: &PinnedCli, + attempt: &str, + action: Action, + stack: &StackId, + limits: (Limits, Limits), + cancel: &Cancel, + images_limits: crate::local_images::Limits, ) -> Result { // A CLI whose images cannot be read runs exactly as before, with nothing asked of it first. let downloaded = match (action, cli_images_of(cli)) { @@ -3130,6 +3477,30 @@ pub(crate) fn setup_with( limits: (Limits, Limits), cancel: &Cancel, images_limits: crate::local_images::Limits, +) -> Result { + // The engine is checked around it, as around `local start` (`checked_call`). + checked_call("local setup", None, || { + setup_downloading( + cli, + attempt, + installation, + accepted, + limits, + cancel, + images_limits, + ) + }) +} + +/// `setup_with`, without the engine check. +fn setup_downloading( + cli: &PinnedCli, + attempt: &str, + installation: &std::path::Path, + accepted: Option<&EulaVersion>, + limits: (Limits, Limits), + cancel: &Cancel, + images_limits: crate::local_images::Limits, ) -> Result { let downloaded = match cli_images_of(cli) { Some(images) => match stack_set_up_in(cli, installation, cancel)? { @@ -6695,11 +7066,25 @@ overlay 164506208 20481024 135620148 14% / podman: false, cpu: false, memory: false, - disk: true + disk: true, + wsl: false, + drive: None, }) .unwrap(), json!({ "status": "engine-too-small", "podman": false, "cpu": false, "memory": false, "disk": true }) ); + assert_eq!( + serde_json::to_value(Terms::EngineTooSmall { + podman: true, + cpu: false, + memory: false, + disk: true, + wsl: true, + drive: Some("C:".into()), + }) + .unwrap(), + json!({ "status": "engine-too-small", "podman": true, "cpu": false, "memory": false, "disk": true, "wsl": true, "drive": "C:" }) + ); } /// A new Windows computer installs Podman while the person signs in, and the CLI will not say @@ -6808,4 +7193,609 @@ overlay 164506208 20481024 135620148 14% / assert!(bundled_cli().is_none()); assert!(bundled_cli_root().is_some()); } + + /** + A container engine as `local start` meets it, put on PATH as `docker` or `podman`. It answers + what OpenBot asks before and after a start with what the real one prints, from files beside it: + `info --format` (`info`), `image ls --quiet --no-trunc` (`images`), `df -Pk /` in a container + (`df`), `kubectl get nodes -o json` in the stack's node (`nodes`) and `machine inspect` + (`inspect`). A node or a file that is not there fails as the engine does. + */ + const FAKE_ENGINE: &str = r#" +use std::io::Write; +fn main() { + let exe = std::env::current_exe().unwrap(); + let dir = exe.parent().unwrap().to_path_buf(); + let name = exe.file_stem().unwrap().to_string_lossy().into_owned(); + let mut args: Vec = std::env::args().skip(1).collect(); + let mut asked = std::fs::OpenOptions::new().create(true).append(true).open(dir.join("asked")).unwrap(); + writeln!(asked, "{} {}", name, args.join(" ")).unwrap(); + while matches!(args.first().map(String::as_str), Some("--connection" | "--url" | "--context" | "--host")) { + args.drain(..2.min(args.len())); + } + let a: Vec<&str> = args.iter().map(String::as_str).collect(); + let file = match a.as_slice() { + ["info", "--format", _] => "info", + ["image", "ls", "--quiet", "--no-trunc"] => "images", + ["run", "--rm", "--pull=never", "--network=none", "--entrypoint", "/bin/sh", _, "-c", "df -Pk /"] => "df", + ["exec", "k3d-local-abc123-server-0", "kubectl", "get", "nodes", ..] => "nodes", + ["machine", "inspect", "openbot"] => "inspect", + _ => "", + }; + if let Some(bytes) = (!file.is_empty()).then(|| std::fs::read(dir.join(file)).ok()).flatten() { + std::io::stdout().write_all(&bytes).unwrap(); + return; + } + if a.first() == Some(&"exec") { + let node = a.get(1).copied().unwrap_or_default(); + if name == "podman" { + eprintln!("Error: no container with name or ID \"{node}\" found: no such container"); + std::process::exit(125); + } + eprintln!("Error response from daemon: No such container: {node}"); + std::process::exit(1); + } + eprintln!("Error: unknown command {a:?}"); + std::process::exit(125); +} +"#; + + /// Every entry logged in this process. Only a test that runs alone in its process + /// (`isolated_process`) installs it. + static LOGGED: std::sync::Mutex> = std::sync::Mutex::new(Vec::new()); + + struct Keep; + + impl log::Log for Keep { + fn enabled(&self, _: &log::Metadata) -> bool { + true + } + fn log(&self, record: &log::Record) { + LOGGED + .lock() + .unwrap() + .push(format!("{} {}", record.level(), record.args())); + } + fn flush(&self) {} + } + + /// The `[local check]` entries logged since the last call, taken. + fn checks_logged() -> Vec { + std::mem::take(&mut *LOGGED.lock().unwrap()) + .into_iter() + .filter(|entry| entry.contains("[local check]")) + .collect() + } + + /// `docker info --format '{{.NCPU}} {{.MemTotal}}'` on a Mac's Docker Desktop with 10 CPUs and + /// 20.5 GiB, and `podman info --format '{{.Host.CPUs}} {{.Host.MemTotal}}'` on OpenBot's own + /// Podman machine, 4 CPUs and 6144 MiB. + const DOCKER_INFO: &str = "10 22011707392\n"; + const PODMAN_INFO: &str = "4 6262661120\n"; + + /// `df -Pk /` in a container, as busybox prints it: a disk of `size` KiB with `free` KiB free. + fn df(size: u64, free: u64) -> String { + let used = size - free - size / 20; + format!( + "Filesystem 1024-blocks Used Available Capacity Mounted on\noverlay {size} {used} {free} {}% /\n", + used * 100 / (used + free) + ) + } + + const KIB_PER_GIB: u64 = 1024 * 1024; + /// OpenBot's Podman machine on WSL: a sparse virtual disk of about a terabyte. + const WSL_DISK: u64 = 1_055_762_868; + + /// `kubectl get nodes -o json` in the stack's node, as k3s v1.31.5 printed it, with its + /// `DiskPressure` condition as the kubelet sets it when `pressure`. + fn nodes(pressure: bool) -> String { + let mut nodes: Value = serde_json::from_str(include_str!( + "../test-fixtures/local-eval/k3s-v1.31.5-get-nodes.json" + )) + .unwrap(); + if pressure { + for condition in nodes["items"][0]["status"]["conditions"] + .as_array_mut() + .unwrap() + { + if condition["type"] == "DiskPressure" { + condition["status"] = json!("True"); + condition["reason"] = json!("KubeletHasDiskPressure"); + condition["message"] = json!("kubelet has disk pressure"); + } + } + } + serde_json::to_string_pretty(&nodes).unwrap() + } + + /// `podman machine inspect openbot` for OpenBot's own rootful machine on WSL, as Podman 5 + /// prints it on Windows. + const WSL_INSPECT: &str = r#"[ + { + "ConfigDir": { + "Path": "C:\\Users\\me\\.config\\containers\\podman\\machine\\wsl" + }, + "ConnectionInfo": { + "PodmanSocket": null, + "PodmanPipe": { + "Path": "\\\\.\\pipe\\podman-openbot" + } + }, + "Created": "2026-11-02T09:14:03.4719262-08:00", + "LastUp": "2026-11-09T08:02:51.1063417-08:00", + "Name": "openbot", + "Resources": { + "CPUs": 4, + "DiskSize": 100, + "Memory": 6144, + "USBs": [] + }, + "SSHConfig": { + "IdentityPath": "C:\\Users\\me\\.local\\share\\containers\\podman\\machine\\machine", + "Port": 61842, + "RemoteUsername": "user" + }, + "State": "running", + "UserModeNetworking": true, + "Rootful": true, + "Rosetta": false + } +] +"#; + + /// `local start` as the pinned CLI (4.34.0-pre.647a03b) reports a start that stopped in + /// `stack-services-ready`, the step that waits for the back end's deployments. + fn start_failed() -> String { + json!({ + "version": 1, + "command": "local start", + "status": "failed", + "error": { + "code": "LOCAL_COMMAND_FAILED", + "message": "Local stack-services-ready failed (exit 1). Run `copilotkit local status` and retry the failed step." + }, + "nextAction": { + "reason": "local_command_failed", + "summary": "The local command did not finish. Check local status for the recorded failed step, then resume the command. No automatic rollback was requested.", + "instructions": ["Inspect the recorded stack with npx --yes copilotkit@4.34.0-pre.647a03b local status --json."], + "resumeCommand": "npx --yes copilotkit@4.34.0-pre.647a03b local start --json", + "url": null, "linkLabel": null, "linkFile": null, + "caveats": [], "requiredEnvVars": [], "artifacts": [] + }, + "diagnostics": [] + }) + .to_string() + } + + fn start_completed() -> String { + let mut done = envelope(); + done["command"] = json!("local start"); + done.to_string() + } + + /** + An engine and a CLI for `local start` through `run_installed`, the window's own path. The engine + is `engine` on PATH answering with `answers` (`FAKE_ENGINE`), and named to the CLI as OpenBot + names it; the CLI is pinned as a developer pins one. Windows' answer for the drive that holds a + Podman machine's disk is `drive`. Sets process environment: only for a test in its own process. + */ + struct StartLab { + fixture: Fixture, + bin: PathBuf, + } + + impl StartLab { + fn new(label: &str) -> Self { + let fixture = Fixture::new(label, "0\n{}"); + let bin = fixture.root.join("bin"); + std::fs::create_dir_all(&bin).unwrap(); + let source = fixture.root.join("engine.rs"); + std::fs::write(&source, FAKE_ENGINE).unwrap(); + for engine in ["docker", "podman"] { + compile_fixture( + &source, + &bin.join(format!("{engine}{}", std::env::consts::EXE_SUFFIX)), + ); + } + std::env::set_var( + "PATH", + std::env::join_paths(std::iter::once(bin.clone()).chain(std::env::split_paths( + &std::env::var_os("PATH").unwrap_or_default(), + ))) + .unwrap(), + ); + std::env::set_var(DEVELOPER_PROGRAM, &fixture.program); + std::env::set_var(DEVELOPER_PROGRAM_SHA256, sha256(&fixture.program)); + let _ = log::set_boxed_logger(Box::new(Keep)); + log::set_max_level(log::LevelFilter::Info); + Self { fixture, bin } + } + + /// The engine as it answers now, and how `local start` ends. + fn engine( + &self, + engine: crate::engine::Engine, + info: &str, + df: &str, + nodes: Option<&str>, + drive: Option, + ) { + let write = |file: &str, said: Option<&str>| match said { + Some(said) => std::fs::write(self.bin.join(file), said).unwrap(), + None => { + let _ = std::fs::remove_file(self.bin.join(file)); + } + }; + write("info", Some(info)); + // `image ls --quiet --no-trunc`: the k3d tools image, which has a shell. + write( + "images", + Some("sha256:3a68d8567335f56c168c36b53059c5dd4dfe23c2edbf4d9251e9ed3170dfacd8\n"), + ); + write("df", Some(df)); + write("nodes", nodes); + write("inspect", Some(WSL_INSPECT)); + use_engine(&match engine { + crate::engine::Engine::Docker => crate::engine::Address::new(engine, None), + crate::engine::Engine::Podman => { + crate::engine::Address::new(engine, Some("openbot-root".into())) + } + }); + MACHINE_DRIVE.with(|said| *said.borrow_mut() = Some(drive)); + } + + fn start(&self, ends: &str) -> Report { + self.run(Action::Start, ends) + } + + /// `action` through the window's path, answered with `ends`. + fn run(&self, action: Action, ends: &str) -> Report { + std::fs::write(self.fixture.root.join("scenario"), ends).unwrap(); + let stack = StackId::parse("local-abc123").unwrap(); + let report = run_installed("checked", action, &stack).unwrap(); + assert_eq!( + self.fixture.argv().unwrap(), + format!("local\n{}\n--stack\nlocal-abc123\n--json", action.as_str()) + ); + report + } + } + + /// Windows' answer for drive C: with `tenths` tenths of a GiB free, and a little more. + fn drive(tenths: u64) -> Option { + Some(crate::windows::MachineDrive { + drive: Some("C:".into()), + free: tenths * GIB / 10 + GIB / 100, + }) + } + + /** + 1.0.0 on Windows, a used install on OpenBot's own Podman machine with 30.5 GiB free inside it: + Quit, then a relaunch. The CLI checks resources only at a first setup, and an existing back end + can run on less than its 40 GiB, so a start that finishes is let through. The log warns, with the + numbers, and nothing else changes. + */ + #[test] + fn a_used_install_short_of_disk_still_starts_and_the_log_warns() { + if crate::test_support::isolated_process( + "local_eval::tests::a_used_install_short_of_disk_still_starts_and_the_log_warns", + ) { + return; + } + let lab = StartLab::new("start-short-of-disk-warns"); + lab.engine( + crate::engine::Engine::Podman, + PODMAN_INFO, + &df(WSL_DISK, 30 * KIB_PER_GIB + KIB_PER_GIB / 2), + Some(&nodes(false)), + drive(946), + ); + let report = lab.start(&format!("0\n{}", start_completed())); + let wire = serde_json::to_value(&report).unwrap(); + assert_eq!(wire["outcome"], json!("completed")); + assert_eq!(wire["reason"], Value::Null); + assert!(wire.get("shortfall").is_none(), "{wire}"); + assert_eq!( + checks_logged(), + ["WARN [local check] before local start: 4 CPUs and 5.8 GiB of memory, 30.5 GiB free of 1006.8 GiB in a container, 94.6 GiB free on C:; short of disk (nearly full)"] + ); + } + + /** + The same start, failed: the card said only "The back end stopped with an error partway through" + while k3s, short of disk, had removed 13 of the back end's 15 images. A start that fails is + checked again with the node asked, and the report names the disk with its numbers, for Podman on + Windows, and for Docker short of the CLI's 40 GiB or with its node under disk pressure. + */ + #[test] + fn a_start_that_fails_short_of_disk_names_the_disk_with_its_numbers() { + if crate::test_support::isolated_process( + "local_eval::tests::a_start_that_fails_short_of_disk_names_the_disk_with_its_numbers", + ) { + return; + } + let lab = StartLab::new("start-short-of-disk-named"); + let failed = format!("1\n{}", start_failed()); + + lab.engine( + crate::engine::Engine::Podman, + PODMAN_INFO, + &df(WSL_DISK, 30 * KIB_PER_GIB + KIB_PER_GIB / 2), + Some(&nodes(true)), + drive(946), + ); + let wire = serde_json::to_value(lab.start(&failed)).unwrap(); + assert_eq!(wire["outcome"], json!("failed")); + assert_eq!(wire["reason"], json!("resources-short")); + assert_eq!( + wire["shortfall"], + json!({ + "podman": true, "cpu": false, "memory": false, "disk": true, + "diskGb": "30.5", "nearlyFull": true, "diskSizeGb": "1006.8", + "wsl": true, "drive": "C:" + }) + ); + assert_eq!( + checks_logged(), + [ + "WARN [local check] before local start: 4 CPUs and 5.8 GiB of memory, 30.5 GiB free of 1006.8 GiB in a container, 94.6 GiB free on C:; short of disk (nearly full)", + "WARN [local check] after local start failed: 4 CPUs and 5.8 GiB of memory, 30.5 GiB free of 1006.8 GiB in a container, 94.6 GiB free on C:, the node reports disk pressure; short of disk (nearly full)", + ] + ); + + // Docker Desktop with 30 GiB of a 64 GiB disk free: below the CLI's 40 GiB. + lab.engine( + crate::engine::Engine::Docker, + DOCKER_INFO, + &df(64 * KIB_PER_GIB, 30 * KIB_PER_GIB), + Some(&nodes(false)), + None, + ); + let wire = serde_json::to_value(lab.start(&failed)).unwrap(); + assert_eq!(wire["reason"], json!("resources-short")); + assert_eq!( + wire["shortfall"], + json!({ "podman": false, "cpu": false, "memory": false, "disk": true, "diskGb": "30" }) + ); + assert_eq!( + checks_logged().last().unwrap(), + "WARN [local check] after local start failed: 10 CPUs and 20.5 GiB of memory, 30 GiB free of 64 GiB in a container, no disk pressure; short of disk (OpenBot needs 40 GiB free)" + ); + + // Docker on a 1.8 TiB disk with 160 GiB free, its node under DiskPressure: nearly full + // by k3s's own line, though above the CLI's 40 GiB. + lab.engine( + crate::engine::Engine::Docker, + DOCKER_INFO, + &df(1_916_694_008, 160 * KIB_PER_GIB), + Some(&nodes(true)), + None, + ); + let wire = serde_json::to_value(lab.start(&failed)).unwrap(); + assert_eq!(wire["reason"], json!("resources-short")); + assert_eq!( + wire["shortfall"], + json!({ + "podman": false, "cpu": false, "memory": false, "disk": true, + "diskGb": "160", "nearlyFull": true, "diskSizeGb": "1827.9" + }) + ); + checks_logged(); + } + + /** + Windows, once space was freed: k3s keeps its DiskPressure condition for five minutes after, and + a Retry in those minutes failed again, three times over six minutes. A start that fails while + the node still says DiskPressure, with the disk above both lines, says the back end is + recovering, so the card can say to wait rather than to free more space. + */ + #[test] + fn a_start_that_fails_while_the_node_recovers_from_low_disk_says_so() { + if crate::test_support::isolated_process( + "local_eval::tests::a_start_that_fails_while_the_node_recovers_from_low_disk_says_so", + ) { + return; + } + let lab = StartLab::new("start-recovering"); + lab.engine( + crate::engine::Engine::Podman, + PODMAN_INFO, + &df(WSL_DISK, 941 * KIB_PER_GIB), + Some(&nodes(true)), + drive(1200), + ); + let wire = serde_json::to_value(lab.start(&format!("1\n{}", start_failed()))).unwrap(); + assert_eq!(wire["reason"], json!("resources-short")); + assert_eq!( + wire["shortfall"], + json!({ + "podman": true, "cpu": false, "memory": false, "disk": false, + "recovering": true, "wsl": true, "drive": "C:" + }) + ); + assert_eq!( + checks_logged().last().unwrap(), + "WARN [local check] after local start failed: 4 CPUs and 5.8 GiB of memory, 941 GiB free of 1006.8 GiB in a container, 120 GiB free on C:, the node reports disk pressure; recovering: the node still reports disk pressure with the disk above both lines" + ); + } + + /** + On Windows the machine's disk is a sparse virtual disk of about a terabyte, so a container sees + 941 GiB free while the drive that holds it has 30. The drive is what runs out: it is checked, + the smaller figure counts, and the card says to free space on that drive. The account step's + check before a first setup reads the drive too. + */ + #[test] + fn on_windows_the_drive_under_podmans_disk_is_the_free_disk_that_counts() { + if crate::test_support::isolated_process( + "local_eval::tests::on_windows_the_drive_under_podmans_disk_is_the_free_disk_that_counts", + ) { + return; + } + let lab = StartLab::new("start-windows-drive"); + lab.engine( + crate::engine::Engine::Podman, + PODMAN_INFO, + &df(WSL_DISK, 941 * KIB_PER_GIB), + Some(&nodes(false)), + drive(300), + ); + let wire = serde_json::to_value(lab.start(&format!("1\n{}", start_failed()))).unwrap(); + assert_eq!(wire["reason"], json!("resources-short")); + assert_eq!( + wire["shortfall"], + json!({ + "podman": true, "cpu": false, "memory": false, "disk": true, + "diskGb": "30", "wsl": true, "drive": "C:" + }) + ); + assert_eq!( + checks_logged(), + [ + "WARN [local check] before local start: 4 CPUs and 5.8 GiB of memory, 941 GiB free of 1006.8 GiB in a container, 30 GiB free on C:; short of disk (OpenBot needs 40 GiB free)", + "WARN [local check] after local start failed: 4 CPUs and 5.8 GiB of memory, 941 GiB free of 1006.8 GiB in a container, 30 GiB free on C:, no disk pressure; short of disk (OpenBot needs 40 GiB free)", + ] + ); + assert_eq!( + serde_json::to_value(engine_size_terms()).unwrap(), + json!({ + "status": "engine-too-small", "podman": true, "cpu": false, "memory": false, + "disk": true, "wsl": true, "drive": "C:" + }) + ); + } + + /** + OpenBot's own Podman machine, 4 CPUs and 6144 MiB, with room on its disk and its drive, passes: + a start that finishes says so in the log, and one that fails for some other reason keeps its + own words rather than being blamed on resources. The account step lets it through. + */ + #[test] + fn openbots_own_podman_machine_with_room_passes_the_check() { + if crate::test_support::isolated_process( + "local_eval::tests::openbots_own_podman_machine_with_room_passes_the_check", + ) { + return; + } + let lab = StartLab::new("start-roomy-podman"); + lab.engine( + crate::engine::Engine::Podman, + PODMAN_INFO, + &df(WSL_DISK, 941 * KIB_PER_GIB), + Some(&nodes(false)), + drive(1200), + ); + let started = + serde_json::to_value(lab.start(&format!("0\n{}", start_completed()))).unwrap(); + assert_eq!(started["outcome"], json!("completed")); + assert!(started.get("shortfall").is_none(), "{started}"); + let failed = serde_json::to_value(lab.start(&format!("1\n{}", start_failed()))).unwrap(); + assert_eq!(failed["reason"], json!("local-command-failed")); + assert!(failed.get("shortfall").is_none(), "{failed}"); + let enough = "4 CPUs and 5.8 GiB of memory, 941 GiB free of 1006.8 GiB in a container, 120 GiB free on C:"; + assert_eq!( + checks_logged(), + [ + format!("INFO [local check] before local start: {enough}; enough"), + format!("INFO [local check] before local start: {enough}; enough"), + format!("INFO [local check] after local start failed: {enough}, no disk pressure; enough"), + ] + ); + assert_eq!(engine_size_terms(), None); + } + + /// `local status` for a back end left up after a start that failed: its node runs, and the + /// API and gateway do not answer. `passed` when every check passed, as with an update on record. + fn status_degraded(passed: bool) -> String { + let mut status = canonical("degraded", "valid", Value::Null); + status["readiness"]["checks"] = json!([ + { "name": "api", "ok": passed }, + { "name": "gateway", "ok": passed }, + { "name": "dashboard", "ok": true } + ]); + format!("0\n{status}") + } + + /** + Windows, after space was freed: Retry read the back end's status, which said `degraded`, and the + card said "The back end is running but not healthy." twice over two minutes while the node still + reported DiskPressure. A back end that runs but does not answer is checked as a failed start is: + recovering, nearly full, or nothing to say, in which case the report is as it was. + */ + #[test] + fn a_back_end_running_but_not_healthy_is_checked_for_its_disk() { + if crate::test_support::isolated_process( + "local_eval::tests::a_back_end_running_but_not_healthy_is_checked_for_its_disk", + ) { + return; + } + let lab = StartLab::new("status-degraded-checked"); + let degraded = status_degraded(false); + + // Space freed, the node still under DiskPressure. + lab.engine( + crate::engine::Engine::Podman, + PODMAN_INFO, + &df(WSL_DISK, 941 * KIB_PER_GIB), + Some(&nodes(true)), + drive(1200), + ); + let wire = serde_json::to_value(lab.run(Action::Status, °raded)).unwrap(); + assert_eq!(wire["readiness"], json!("degraded")); + assert_eq!(wire["reason"], Value::Null); + assert_eq!( + wire["shortfall"], + json!({ + "podman": true, "cpu": false, "memory": false, "disk": false, + "recovering": true, "wsl": true, "drive": "C:" + }) + ); + assert_eq!( + checks_logged(), + ["WARN [local check] local status, running but not healthy: 4 CPUs and 5.8 GiB of memory, 941 GiB free of 1006.8 GiB in a container, 120 GiB free on C:, the node reports disk pressure; recovering: the node still reports disk pressure with the disk above both lines"] + ); + + // Still nearly full. + lab.engine( + crate::engine::Engine::Podman, + PODMAN_INFO, + &df(WSL_DISK, 30 * KIB_PER_GIB + KIB_PER_GIB / 2), + Some(&nodes(true)), + drive(946), + ); + let wire = serde_json::to_value(lab.run(Action::Status, °raded)).unwrap(); + assert_eq!( + wire["shortfall"], + json!({ + "podman": true, "cpu": false, "memory": false, "disk": true, + "diskGb": "30.5", "nearlyFull": true, "diskSizeGb": "1006.8", + "wsl": true, "drive": "C:" + }) + ); + + // Room on the disk and no pressure: not the disk, and the report is as it was. + lab.engine( + crate::engine::Engine::Podman, + PODMAN_INFO, + &df(WSL_DISK, 941 * KIB_PER_GIB), + Some(&nodes(false)), + drive(1200), + ); + let wire = serde_json::to_value(lab.run(Action::Status, °raded)).unwrap(); + assert_eq!(wire["readiness"], json!("degraded")); + assert!(wire.get("shortfall").is_none(), "{wire}"); + checks_logged(); + + // Every check passed: an update on record, not a back end that does not answer. + lab.engine( + crate::engine::Engine::Podman, + PODMAN_INFO, + &df(WSL_DISK, 30 * KIB_PER_GIB), + Some(&nodes(true)), + drive(946), + ); + let wire = serde_json::to_value(lab.run(Action::Status, &status_degraded(true))).unwrap(); + assert!(wire.get("shortfall").is_none(), "{wire}"); + assert_eq!(checks_logged(), Vec::::new()); + } } diff --git a/desktop/src-tauri/src/local_images.rs b/desktop/src-tauri/src/local_images.rs index 6c769c13b8..0ee66c198e 100644 --- a/desktop/src-tauri/src/local_images.rs +++ b/desktop/src-tauri/src/local_images.rs @@ -586,6 +586,28 @@ done"#; ok.then(|| lines(&out).map(str::to_owned).collect()) } + /** + Whether the node's kubelet says it is short of disk: its `DiskPressure` condition, from `kubectl + get nodes` run in the node, as the CLI runs its own `kubectl`. k3s raises it below 5% free and + removes images it is not running above 85% used, which is how a relaunch on a nearly full disk + loses the back end's images. None when the node does not answer. + */ + pub fn disk_pressure(&self, limit: Duration) -> Option { + let (ok, out) = self.exec( + &[ + "kubectl", + "get", + "nodes", + "--request-timeout=5s", + "-o", + "json", + ], + limit, + 4 << 20, + )?; + ok.then(|| disk_pressure_of(&out)).flatten() + } + /// The node's architecture, as an image index names it. fn architecture(&self, limits: Limits) -> Option<&'static str> { let (ok, out) = self.exec(&["uname", "-m"], limits.call, 64)?; @@ -676,6 +698,36 @@ done"#; } } +/// `DiskPressure` from `kubectl get nodes -o json`: true when any node has it, false when every +/// node that says has not, None when none says. +pub fn disk_pressure_of(json: &[u8]) -> Option { + let nodes: Value = serde_json::from_slice(json).ok()?; + let said: Vec = nodes + .get("items")? + .as_array()? + .iter() + .filter_map(|node| node.pointer("/status/conditions")?.as_array()) + .flatten() + .filter(|condition| condition.get("type").and_then(Value::as_str) == Some("DiskPressure")) + .filter_map(|condition| match condition.get("status")?.as_str()? { + "True" => Some(true), + "False" => Some(false), + _ => None, + }) + .collect(); + (!said.is_empty()).then(|| said.contains(&true)) +} + +/// What every k3d node on the engine says of `DiskPressure` (`Node::disk_pressure`): true when +/// any has it. The nodes share the engine's disk. +pub fn any_disk_pressure(engine: &(dyn Fn() -> Command + Sync), limits: Limits) -> Option { + let said: Vec = stack_nodes(engine, limits)? + .iter() + .filter_map(|stack| Node::for_stack(engine, stack).disk_pressure(limits.call)) + .collect(); + (!said.is_empty()).then(|| said.contains(&true)) +} + fn lines(bytes: &[u8]) -> impl Iterator { std::str::from_utf8(bytes) .unwrap_or_default() diff --git a/desktop/src-tauri/src/windows.rs b/desktop/src-tauri/src/windows.rs index 76f644fbe4..17aab8603c 100644 --- a/desktop/src-tauri/src/windows.rs +++ b/desktop/src-tauri/src/windows.rs @@ -952,11 +952,171 @@ pub fn restart() -> Result<(), Problem> { Err(Problem::plain("Only Windows restarts from here.")) } +/// The Windows drive that holds a Podman machine's disk, and how much is free on it. +#[derive(Clone, Debug, PartialEq, Eq)] +pub struct MachineDrive { + /// `C:`, or None when the disk's folder is not on a lettered drive. + pub drive: Option, + pub free: u64, +} + +/** +Where a Podman machine on WSL keeps its disk, and the space free on that drive. + +The machine's disk is a sparse virtual disk of about a terabyte, so the space a container sees free +inside it says nothing about the space left on Windows: a machine can show 941 GiB free on a drive +with 30. Podman imports each machine as the WSL distribution `podman-` +(`env.WithPodmanPrefix`), and WSL records the folder it holds the disk in as that distribution's +`BasePath`. None off Windows, and when WSL has no such distribution. +*/ +pub fn podman_machine_drive(machine: &str) -> Option { + #[cfg(target_os = "windows")] + { + machine_drive_in(LXSS, &podman_distribution(machine)) + } + #[cfg(not(target_os = "windows"))] + { + let _ = machine; + None + } +} + +/// The WSL distribution Podman makes for `machine`, as Podman names it (`env.WithPodmanPrefix`). +pub fn podman_distribution(machine: &str) -> String { + if machine.starts_with("podman") { + machine.to_owned() + } else { + format!("podman-{machine}") + } +} + +/// The drive a Windows path is on, `C:`, with or without the `\\?\` prefix WSL records it with. +pub fn drive_of(path: &str) -> Option { + let path = path.strip_prefix(r"\\?\").unwrap_or(path); + match path.as_bytes() { + [letter, b':', ..] if letter.is_ascii_alphabetic() => { + Some(format!("{}:", letter.to_ascii_uppercase() as char)) + } + _ => None, + } +} + +/// Where WSL records each distribution it has, for the person signed in. +#[cfg(target_os = "windows")] +const LXSS: &str = r"Software\Microsoft\Windows\CurrentVersion\Lxss"; + +#[cfg(target_os = "windows")] +fn machine_drive_in(registry: &str, distribution: &str) -> Option { + let folder = distribution_folder(registry, distribution)?; + let free = free_space(&folder)?; + Some(MachineDrive { + drive: drive_of(&folder), + free, + }) +} + +/// The `BasePath` WSL recorded for `distribution`, under `registry` in the person's own hive. +#[cfg(target_os = "windows")] +fn distribution_folder(registry: &str, distribution: &str) -> Option { + let lxss = windows_registry::CURRENT_USER.open(registry).ok()?; + let ids: Vec = lxss.keys().ok()?.collect(); + ids.iter().find_map(|id| { + let entry = lxss.open(id).ok()?; + let name = entry.get_string("DistributionName").ok()?; + if !name.eq_ignore_ascii_case(distribution) { + return None; + } + entry.get_string("BasePath").ok() + }) +} + +/// The bytes free to this account on the volume that holds `folder`. +#[cfg(target_os = "windows")] +fn free_space(folder: &str) -> Option { + use std::os::windows::ffi::OsStrExt; + let folder = folder.strip_prefix(r"\\?\").unwrap_or(folder); + let wide: Vec = std::ffi::OsStr::new(folder) + .encode_wide() + .chain(std::iter::once(0)) + .collect(); + let mut free = 0u64; + // SAFETY: `wide` is a NUL-terminated path that outlives the call, and `free` is a u64 it may + // write; the two totals are not asked for. + unsafe { + windows::Win32::Storage::FileSystem::GetDiskFreeSpaceExW( + windows::core::PCWSTR(wide.as_ptr()), + Some(&mut free), + None, + None, + ) + } + .ok()?; + Some(free) +} + #[cfg(test)] mod tests { use super::*; use crate::test_support::temp_root; + /// Podman names its machine's WSL distribution as `env.WithPodmanPrefix` does, and WSL records + /// its folder with the `\\?\` prefix or without. + #[test] + fn a_podman_machines_distribution_and_drive_are_named_as_podman_and_wsl_name_them() { + assert_eq!(podman_distribution("openbot"), "podman-openbot"); + assert_eq!( + podman_distribution("podman-machine-default"), + "podman-machine-default" + ); + for (path, drive) in [ + ( + r"\\?\C:\Users\me\.local\share\containers\podman\machine\wsl\wsldist\openbot", + Some("C:"), + ), + (r"d:\podman\wsldist\openbot", Some("D:")), + (r"\\server\share\wsldist\openbot", None), + ("", None), + ] { + assert_eq!(drive_of(path).as_deref(), drive, "{path}"); + } + } + + /// The machine's drive is read where WSL keeps it: a distribution's `BasePath` in the person's + /// own registry hive, here under a key of this test's own with the shape WSL writes, and the + /// free space of the real volume that folder is on. + #[cfg(target_os = "windows")] + #[test] + fn a_podman_machines_drive_is_read_from_the_folder_wsl_recorded() { + let registry = format!(r"Software\OpenBot\Tests\Lxss-{}", std::process::id()); + let folder = temp_root("wsl-distribution"); + std::fs::create_dir_all(&folder).unwrap(); + let entry = |id: &str, name: &str, base: &str| { + let key = windows_registry::CURRENT_USER + .create(format!(r"{registry}\{id}")) + .unwrap(); + key.set_string("DistributionName", name).unwrap(); + key.set_string("BasePath", base).unwrap(); + key.set_string("VhdFileName", "ext4.vhdx").unwrap(); + }; + entry( + "{0d4e1c9a-5f0b-4a57-9d3e-1a2b3c4d5e6f}", + "Ubuntu", + r"Z:\elsewhere", + ); + entry( + "{7c1f2e3d-9a8b-4c5d-8e7f-6a5b4c3d2e1f}", + "podman-openbot", + &format!(r"\\?\{}", folder.display()), + ); + let found = machine_drive_in(®istry, "podman-openbot"); + let missing = machine_drive_in(®istry, "podman-other"); + let _ = windows_registry::CURRENT_USER.remove_tree(®istry); + let found = found.expect("the machine's folder and its volume were read"); + assert_eq!(found.drive, drive_of(&folder.display().to_string())); + assert!(found.free > 0, "{found:?}"); + assert_eq!(missing, None); + } + const PROBE_OUTPUTS: [&str; 6] = [ "hypervisor=True\nfirmware=False\n", "True\n", diff --git a/desktop/src-tauri/test-fixtures/local-eval/k3s-v1.31.5-get-nodes.json b/desktop/src-tauri/test-fixtures/local-eval/k3s-v1.31.5-get-nodes.json new file mode 100644 index 0000000000..00faaafbfe --- /dev/null +++ b/desktop/src-tauri/test-fixtures/local-eval/k3s-v1.31.5-get-nodes.json @@ -0,0 +1,264 @@ +{ + "apiVersion": "v1", + "items": [ + { + "apiVersion": "v1", + "kind": "Node", + "metadata": { + "annotations": { + "alpha.kubernetes.io/provided-node-ip": "172.27.0.2", + "flannel.alpha.coreos.com/backend-data": "{\"VNI\":1,\"VtepMAC\":\"52:3a:21:98:27:b8\"}", + "flannel.alpha.coreos.com/backend-type": "vxlan", + "flannel.alpha.coreos.com/kube-subnet-manager": "true", + "flannel.alpha.coreos.com/public-ip": "172.27.0.2", + "k3s.io/hostname": "k3d-local-e0cf90debea88453ad454b93-server-0", + "k3s.io/internal-ip": "172.27.0.2", + "k3s.io/node-args": "[\"server\",\"--disable\",\"traefik\",\"--tls-san\",\"127.0.0.1\"]", + "k3s.io/node-config-hash": "MXCMP4EPPR7HGKB2OXMJELW7CFJQH6IT4ZP54RPNISYBXKEKV6WA====", + "k3s.io/node-env": "{\"K3S_KUBECONFIG_OUTPUT\":\"/output/kubeconfig.yaml\",\"K3S_TOKEN\":\"********\"}", + "node.alpha.kubernetes.io/ttl": "0", + "volumes.kubernetes.io/controller-managed-attach-detach": "true" + }, + "creationTimestamp": "2026-11-07T09:46:42Z", + "finalizers": ["wrangler.cattle.io/node"], + "labels": { + "beta.kubernetes.io/arch": "arm64", + "beta.kubernetes.io/instance-type": "k3s", + "beta.kubernetes.io/os": "linux", + "kubernetes.io/arch": "arm64", + "kubernetes.io/hostname": "k3d-local-e0cf90debea88453ad454b93-server-0", + "kubernetes.io/os": "linux", + "node-role.kubernetes.io/control-plane": "true", + "node-role.kubernetes.io/master": "true", + "node.kubernetes.io/instance-type": "k3s" + }, + "name": "k3d-local-e0cf90debea88453ad454b93-server-0", + "resourceVersion": "105240", + "uid": "71b0e4ba-c3f1-4a91-b493-88743fc7f270" + }, + "spec": { + "podCIDR": "10.42.0.0/24", + "podCIDRs": ["10.42.0.0/24"], + "providerID": "k3s://k3d-local-e0cf90debea88453ad454b93-server-0" + }, + "status": { + "addresses": [ + { + "address": "172.27.0.2", + "type": "InternalIP" + }, + { + "address": "k3d-local-e0cf90debea88453ad454b93-server-0", + "type": "Hostname" + } + ], + "allocatable": { + "cpu": "18", + "ephemeral-storage": "1864559929521", + "hugepages-1Gi": "0", + "hugepages-2Mi": "0", + "hugepages-32Mi": "0", + "hugepages-64Ki": "0", + "memory": "16353252Ki", + "pods": "110" + }, + "capacity": { + "cpu": "18", + "ephemeral-storage": "1916694008Ki", + "hugepages-1Gi": "0", + "hugepages-2Mi": "0", + "hugepages-32Mi": "0", + "hugepages-64Ki": "0", + "memory": "16353252Ki", + "pods": "110" + }, + "conditions": [ + { + "lastHeartbeatTime": "2026-11-09T12:03:04Z", + "lastTransitionTime": "2026-11-08T05:15:10Z", + "message": "kubelet has sufficient memory available", + "reason": "KubeletHasSufficientMemory", + "status": "False", + "type": "MemoryPressure" + }, + { + "lastHeartbeatTime": "2026-11-09T12:03:04Z", + "lastTransitionTime": "2026-11-08T05:15:10Z", + "message": "kubelet has no disk pressure", + "reason": "KubeletHasNoDiskPressure", + "status": "False", + "type": "DiskPressure" + }, + { + "lastHeartbeatTime": "2026-11-09T12:03:04Z", + "lastTransitionTime": "2026-11-08T05:15:10Z", + "message": "kubelet has sufficient PID available", + "reason": "KubeletHasSufficientPID", + "status": "False", + "type": "PIDPressure" + }, + { + "lastHeartbeatTime": "2026-11-09T12:03:04Z", + "lastTransitionTime": "2026-11-08T05:15:10Z", + "message": "kubelet is posting ready status", + "reason": "KubeletReady", + "status": "True", + "type": "Ready" + } + ], + "daemonEndpoints": { + "kubeletEndpoint": { + "Port": 10250 + } + }, + "images": [ + { + "names": [ + "ghcr.io/copilotkit/intelligence/app-api@sha256:060ae0f883786612d064ebbb8d081b83006e9e538a936b74e383ecd3dd81a844" + ], + "sizeBytes": 147210711 + }, + { + "names": [ + "ghcr.io/copilotkit/intelligence/app-api@sha256:9ee3006d92a656a72be43f9bda0e0738aec910f3b8098aba8e029f28061e85b7" + ], + "sizeBytes": 147206537 + }, + { + "names": [ + "ghcr.io/copilotkit/intelligence/postgres-pgvector@sha256:3ec79ad28b8ba356734e1f29d757d5e1f5f4104360e2d285a63db1bb6a0cfe28" + ], + "sizeBytes": 108486056 + }, + { + "names": [ + "ghcr.io/copilotkit/intelligence/postgres-pgvector@sha256:f98d6b61014200ab2b946720c84d4693ecd2f68abbe7d00779a57729021d03de" + ], + "sizeBytes": 108486053 + }, + { + "names": [ + "ghcr.io/copilotkit/intelligence/db-migrations@sha256:ba649b6358d6e74d84bd6673dd4c2bbfd924715f7cf7402d3ba76e50572242df" + ], + "sizeBytes": 75121018 + }, + { + "names": [ + "ghcr.io/copilotkit/intelligence/db-migrations@sha256:9c2eedb0f1dbd140d26a8542693c4e73e76338b9264630724e1dcb767014c858" + ], + "sizeBytes": 75120614 + }, + { + "names": [ + "docker.io/rancher/klipper-helm@sha256:73ff7ef399717ba8339559054557bd427bdafb47db112165a8c0c358d1ca0283", + "docker.io/rancher/klipper-helm:v0.9.3-build20241008" + ], + "sizeBytes": 65893717 + }, + { + "names": [ + "docker.io/bitnamilegacy/redis@sha256:a25b5d07a14ec13730022c7cd9bab6308d55ccd86b74af7315553c17be884889" + ], + "sizeBytes": 50512298 + }, + { + "names": [ + "ghcr.io/copilotkit/intelligence/local-object-storage-init@sha256:1c2a56bcd21e6ef33b77ddefec998ba09b026c339af1c2827d312ee3df7b7280", + "ghcr.io/copilotkit/intelligence/local-object-storage@sha256:1c2a56bcd21e6ef33b77ddefec998ba09b026c339af1c2827d312ee3df7b7280" + ], + "sizeBytes": 49169049 + }, + { + "names": [ + "docker.io/rancher/mirrored-library-traefik@sha256:25df7bff0b414867f16a74c571c0dc84920404e45cc7780976cec77809230e09", + "docker.io/rancher/mirrored-library-traefik:2.11.18" + ], + "sizeBytes": 46474327 + }, + { + "names": [ + "ghcr.io/copilotkit/intelligence/app-frontend@sha256:3432ef79640850e6db3b4418b60e3c85987268aaeac72c82e4f9eec62e6d73e0" + ], + "sizeBytes": 27977381 + }, + { + "names": [ + "ghcr.io/copilotkit/intelligence/app-frontend@sha256:ac554fdffa0eff4ee185642ac4dc7f8ee3fd4e0f479f6f279d1499807587f130" + ], + "sizeBytes": 27977374 + }, + { + "names": [ + "ghcr.io/copilotkit/intelligence/realtime-gateway@sha256:c005749db55b2a415e7c09254d673ccf582faea9d583848f27caa9c574b018f4" + ], + "sizeBytes": 21753907 + }, + { + "names": [ + "ghcr.io/copilotkit/intelligence/realtime-gateway@sha256:5131c1da79bb9237d9171aa4cb19643425d53a6154ada3de98b21a8caee800c8" + ], + "sizeBytes": 21753888 + }, + { + "names": [ + "docker.io/rancher/mirrored-coredns-coredns@sha256:82979ddf442c593027a57239ad90616deb874e90c365d1a96ad508c2104bdea5", + "docker.io/rancher/mirrored-coredns-coredns:1.12.0" + ], + "sizeBytes": 19148178 + }, + { + "names": [ + "docker.io/rancher/mirrored-metrics-server@sha256:dccf8474fb910fef261d31d9483d7e4c1df7b86cf4d638fb6a7d7c88bd51600a", + "docker.io/rancher/mirrored-metrics-server:v0.7.2" + ], + "sizeBytes": 17862421 + }, + { + "names": [ + "docker.io/rancher/local-path-provisioner@sha256:9b914881170048f80ae9302f36e5b99b4a6b18af73a38adc1c66d12f65d360be", + "docker.io/rancher/local-path-provisioner:v0.0.30" + ], + "sizeBytes": 17807097 + }, + { + "names": [ + "docker.io/rancher/klipper-lb@sha256:dd380f5d89a52f2a07853ff17a6048f805c1f8113b50578f3efc3efb9bcf670a", + "docker.io/rancher/klipper-lb:v0.4.9" + ], + "sizeBytes": 5678915 + }, + { + "names": [ + "docker.io/rancher/mirrored-library-busybox@sha256:8a45424ddf949bbe9bb3231b05f9032a45da5cd036eb4867b511b00734756d6f", + "docker.io/rancher/mirrored-library-busybox:1.36.1" + ], + "sizeBytes": 1847165 + }, + { + "names": [ + "docker.io/rancher/mirrored-pause@sha256:74c4244427b7312c5b901fe0f67cbc53683d06f4f24c6faee65d4182bf0fa893", + "docker.io/rancher/mirrored-pause:3.6" + ], + "sizeBytes": 253243 + } + ], + "nodeInfo": { + "architecture": "arm64", + "bootID": "bda7140e-2723-49f3-ad3b-4bf341290510", + "containerRuntimeVersion": "containerd://1.7.23-k3s2", + "kernelVersion": "6.12.76-linuxkit", + "kubeProxyVersion": "v1.31.5+k3s1", + "kubeletVersion": "v1.31.5+k3s1", + "machineID": "", + "operatingSystem": "linux", + "osImage": "K3s v1.31.5+k3s1", + "systemUUID": "" + } + } + } + ], + "kind": "List", + "metadata": { + "resourceVersion": "" + } +} diff --git a/desktop/src/LocalIntelligence.test.tsx b/desktop/src/LocalIntelligence.test.tsx index 60f7b5a2da..3e8e9bf321 100644 --- a/desktop/src/LocalIntelligence.test.tsx +++ b/desktop/src/LocalIntelligence.test.tsx @@ -1101,6 +1101,89 @@ test("a stopped stack is started, then polled", async () => { ]); }); +/** + * 1.0.0 on Windows: a used install relaunched with its back end's disk nearly full said only "The + * back end stopped with an error partway through." The start the window runs on a stopped stack now + * comes back from `local_eval::checked_call` naming the disk, as `local_eval.rs` reports it. + */ +test("a relaunch whose start fails on a nearly full disk names the disk on its row, with Retry", async () => { + const stopped = report({ readiness: "stopped", licence: "valid" }); + invokeHandler = async (command, args) => { + const action = (args as { action?: string }).action; + if (command === "run_local_eval" && action === "start") { + return report({ + outcome: "failed", + stack: null, + reason: "resources-short", + shortfall: { + podman: true, + cpu: false, + memory: false, + disk: true, + diskGb: "30.5", + nearlyFull: true, + diskSizeGb: "1006.8", + wsl: true, + drive: "C:", + }, + }); + } + if (command === "run_local_eval") return stopped; + throw new Error(`unexpected ${command}`); + }; + const screen = view({ stack: "local-abc123", connected: true }); + await screen.findByRole("button", { name: "Retry" }); + const waiting = row(screen, WAIT).textContent; + expect(waiting).toContain( + "The back end's disk is nearly full. Podman has 30.5 GB free of 1006.8 GB. OpenBot needs 40 GB. Free space on the drive that holds OpenBot's engine (C:), then Retry.", + ); + expect(waiting).not.toContain("partway through"); + expect(commands()).toEqual(["run_local_eval:status", "run_local_eval:start"]); +}); + +/** + * Windows, after space was freed: Retry reads the status of the back end the failed start left + * running, which says `degraded`, and the row said only "The back end is running but not healthy." + * while k3s still reported low disk. `local_eval::checked_when_unhealthy` now says why. + */ +test("a back end running but not healthy after low disk says it is recovering, and Retry asks again", async () => { + const recovering = report({ + readiness: "degraded", + licence: "valid", + shortfall: { + podman: true, + cpu: false, + memory: false, + disk: false, + recovering: true, + wsl: true, + drive: "C:", + }, + }); + invokeHandler = answers({ run_local_eval: [recovering, ready] }); + const started: Startable[] = []; + const screen = view({ + stack: "local-abc123", + connected: true, + onStart: async (startable) => { + started.push(startable); + return "started"; + }, + }); + const retry = await screen.findByRole("button", { name: "Retry" }); + const waiting = row(screen, WAIT).textContent; + expect(waiting).toContain( + "The back end is recovering from low disk space. Wait a few minutes, then Retry.", + ); + expect(waiting).not.toContain("running but not healthy"); + await userEvent.click(retry); + await waitFor(() => expect(started).toEqual([startableUrls])); + expect(commands()).toEqual([ + "run_local_eval:status", + "run_local_eval:status", + ]); +}); + test("a start that fails stops at its row, and Retry starts again without asking the CLI", async () => { invokeHandler = answers({ run_local_eval: [ready] }); let starts = 0; diff --git a/desktop/src/localEvalCli.ts b/desktop/src/localEvalCli.ts index dff5e699c3..16396780de 100644 --- a/desktop/src/localEvalCli.ts +++ b/desktop/src/localEvalCli.ts @@ -90,11 +90,29 @@ export type Shortfall = { cpus?: string; memoryGb?: string; diskGb?: string; + /** + * OpenBot's own check after a start that failed: the back end's disk is nearly full, and how + * large it is beside `diskGb`. + */ + nearlyFull?: boolean; + diskSizeGb?: string; + /** + * The node still reports low disk with the disk above both lines: space was freed, and the + * back end recovers by itself within minutes. + */ + recovering?: boolean; + /** Podman on Windows, in WSL: `podman machine set` changes nothing there. */ + wsl?: boolean; + /** The drive that holds its machine's disk, `C:`. */ + drive?: string; }; export type LocalReport = { outcome: LocalOutcome; - /** With the reason `resources-short`: which resources the CLI found short. */ + /** + * With the reason `resources-short`: which resources the CLI found short. With readiness + * `degraded`: a disk OpenBot found short, or recovering, behind a back end that does not answer. + */ shortfall?: Shortfall; stack: null | { id: string; diff --git a/desktop/src/localReadiness.test.ts b/desktop/src/localReadiness.test.ts index 602bee84c1..4b53af50df 100644 --- a/desktop/src/localReadiness.test.ts +++ b/desktop/src/localReadiness.test.ts @@ -8,7 +8,11 @@ import { localFailures, type Shortfall, } from "./localEvalCli"; -import { localReadiness, type ReadinessInput } from "./localReadiness"; +import { + engineTooSmallSaid, + localReadiness, + type ReadinessInput, +} from "./localReadiness"; const reasons: LocalReason[] = [ "authentication-required", @@ -227,6 +231,111 @@ test("specific wording and actions", () => { ).toBe( "Docker has 3.5 free CPUs and 12 GB of free disk. OpenBot needs 4 free CPUs and 40 GB of free disk. Raise the limits in Docker Desktop > Settings > Resources, or free space in Docker, then Retry.", ); + // OpenBot's own check after a start that failed (`local_eval::checked_call`), as it reports + // each case. A nearly full disk is said as k3s's, beside the 40 GB only when it is below it. + expect( + short({ + podman: true, + disk: true, + diskGb: "30.5", + nearlyFull: true, + diskSizeGb: "1006.8", + wsl: true, + drive: "C:", + }).said, + ).toBe( + "The back end's disk is nearly full. Podman has 30.5 GB free of 1006.8 GB. OpenBot needs 40 GB. Free space on the drive that holds OpenBot's engine (C:), then Retry.", + ); + expect( + short({ podman: true, disk: true, diskGb: "30", wsl: true, drive: "C:" }) + .said, + ).toBe( + "Podman has 30 GB of free disk. OpenBot needs 40 GB. Free space on the drive that holds OpenBot's engine (C:), then Retry.", + ); + // On Windows, never `podman machine set --disk-size`: Podman refuses it for WSL machines, even + // when WSL did not say which drive holds the disk. + expect(short({ podman: true, disk: true, wsl: true }).said).toBe( + "Podman is out of disk space. OpenBot needs 40 GB free. Free space on the drive that holds OpenBot's engine, then Retry.", + ); + expect( + short({ + disk: true, + diskGb: "160", + nearlyFull: true, + diskSizeGb: "1827.9", + }).said, + ).toBe( + "The back end's disk is nearly full. Docker has 160 GB free of 1827.9 GB. Raise the disk limit in Docker Desktop > Settings > Resources, or free space in Docker, then Retry.", + ); + expect( + short({ + podman: true, + disk: true, + diskGb: "8", + nearlyFull: true, + diskSizeGb: "60", + }).said, + ).toBe( + "The back end's disk is nearly full. Podman has 8 GB free of 60 GB. OpenBot needs 40 GB. Stop your Podman machine, run podman machine set --disk-size 160, start it again, then Retry.", + ); + // Space was freed and k3s still says the disk is low: wait, rather than free more. + expect( + short({ podman: true, recovering: true, wsl: true, drive: "C:" }), + ).toEqual({ + state: "blocked", + said: "The back end is recovering from low disk space. Wait a few minutes, then Retry.", + action: "retry", + }); + // A back end that runs but does not answer, with OpenBot's check of its disk, and without. + const degraded = (shortfall?: Partial) => + said( + report({ + readiness: "degraded", + licence: "valid", + ...(shortfall + ? { + shortfall: { + podman: true, + cpu: false, + memory: false, + disk: false, + ...shortfall, + }, + } + : {}), + }), + ); + expect(degraded({ recovering: true, wsl: true, drive: "C:" })).toEqual({ + state: "blocked", + said: "The back end is recovering from low disk space. Wait a few minutes, then Retry.", + action: "restart-intelligence", + }); + expect( + degraded({ + disk: true, + diskGb: "30.5", + nearlyFull: true, + diskSizeGb: "1006.8", + wsl: true, + drive: "C:", + }).said, + ).toBe( + "The back end's disk is nearly full. Podman has 30.5 GB free of 1006.8 GB. OpenBot needs 40 GB. Free space on the drive that holds OpenBot's engine (C:), then Retry.", + ); + expect(degraded().said).toBe("The back end is running but not healthy."); + // The account step's check before a first setup, on Windows. + expect( + engineTooSmallSaid({ + podman: true, + cpu: false, + memory: false, + disk: true, + wsl: true, + drive: "C:", + }), + ).toBe( + "Podman is out of disk space. OpenBot needs 40 GB free. Free space on the drive that holds OpenBot's engine (C:), then Try again.", + ); // A report that does not say which is still about resources, never "partway through". expect( said(report({ outcome: "failed", reason: "resources-short" })).said, diff --git a/desktop/src/localReadiness.ts b/desktop/src/localReadiness.ts index 8b3a1aa3dd..e525221d6c 100644 --- a/desktop/src/localReadiness.ts +++ b/desktop/src/localReadiness.ts @@ -134,6 +134,10 @@ function shortfallSaid( const short = (["cpu", "memory", "disk"] as const).filter( (resource) => shortfall[resource], ); + // Space was freed, and k3s still says the disk is low for a few minutes after. + if (shortfall.recovering && short.length === 0) { + return `The back end is recovering from low disk space. Wait a few minutes, then ${then}.`; + } if (short.length === 0) { const fix = shortfall.podman ? "Stop your Podman machine, raise them with podman machine set, start it again" @@ -141,8 +145,19 @@ function shortfallSaid( return `${engine} doesn't have enough CPUs, memory or disk space for the back end. ${fix}, then ${then}.`; } const fix = shortfall.podman - ? `Stop your Podman machine, run podman machine set ${short.map((resource) => podmanFlag[resource]).join(" ")}, start it again` + ? podmanFix(short, shortfall) : `Raise the ${short.length === 1 ? `${limitName[short[0]]} limit` : "limits"} in Docker Desktop > Settings > Resources${shortfall.disk ? ", or free space in Docker" : ""}`; + // OpenBot's own check after a start that failed: k3s itself is short of disk. + if (shortfall.nearlyFull && short.length === 1) { + const free = shortfall.diskGb + ? ` ${engine} has ${shortfall.diskGb} GB free${shortfall.diskSizeGb ? ` of ${shortfall.diskSizeGb} GB` : ""}.` + : ""; + const needs40 = + shortfall.diskGb && Number(shortfall.diskGb) < 40 + ? ` OpenBot needs ${needed.disk}.` + : ""; + return `The back end's disk is nearly full.${free}${needs40} ${fix}, then ${then}.`; + } // The CLI's own numbers, when its error gave one for every resource that is short. const measured: Record = { cpu: shortfall.cpus && `${shortfall.cpus} free CPUs`, @@ -159,7 +174,31 @@ function shortfallSaid( : short.length === 1 && short[0] === "disk" ? `${engine} is out of disk space. OpenBot needs ${needs.disk}.` : `${engine} doesn't have enough ${listed(short.map((resource) => resourceName[resource]))}. OpenBot needs ${listed(short.map((resource) => needs[resource]))}.`; - return `${said} ${fix}, then ${then}.`; + const full = shortfall.nearlyFull + ? "The back end's disk is nearly full. " + : ""; + return `${full}${said} ${fix}, then ${then}.`; +} + +/** + * Where Podman raises what is short. On Windows its machine runs in WSL, where `podman machine set` + * refuses to change its disk ("changing disk size not supported for WSL machines"), and the disk + * is a sparse virtual disk on `drive`, so space is freed on that drive instead. + */ +function podmanFix(short: Resource[], shortfall: Shortfall): string { + const raised = short.filter( + (resource) => !(shortfall.wsl && resource === "disk"), + ); + const set = + raised.length > 0 && + `Stop your Podman machine, run podman machine set ${raised.map((resource) => podmanFlag[resource]).join(" ")}, start it again`; + const freed = + shortfall.wsl && + short.includes("disk") && + `free space on the drive that holds OpenBot's engine${shortfall.drive ? ` (${shortfall.drive})` : ""}`; + if (set && freed) return `${set}, and ${freed}`; + if (freed) return freed.charAt(0).toUpperCase() + freed.slice(1); + return set || ""; } // What the CLI requires free, beside its own numbers: "Docker has 0 GB of free disk. OpenBot @@ -351,6 +390,13 @@ function intelligenceRow( return blocked("The back end is stopped.", "start-intelligence"); } if (intelligence.readiness === "degraded") { + // OpenBot's check of a back end that runs but does not answer: a full disk, or one recovering. + if (intelligence.shortfall) { + return blocked( + resourcesShortSaid(intelligence.shortfall), + "restart-intelligence", + ); + } return blocked( "The back end is running but not healthy.", "restart-intelligence",