From 4a7bfb147011d030cba23e03dce321d7f378e32f Mon Sep 17 00:00:00 2001 From: David McKay Date: Mon, 9 Nov 2026 04:19:48 -0800 Subject: [PATCH 1/4] Name a nearly full disk when a back end start fails, and check the drive under Podman on Windows A used install relaunched with its back end's disk nearly full showed only "The back end stopped with an error partway through." The CLI checks resources only at a first setup, and the Desktop's own engine check ran only at the account step, so nothing looked at the disk on a relaunch while k3s, short of disk, removed the back end's images. Each local start, update and setup now checks the engine first and logs it as "[local check]" with the numbers and a verdict. That check only warns: an existing back end can run on less than the CLI's 40 GiB. When the call fails, the engine is checked again with the back end's node asked for its DiskPressure condition, and a shortfall becomes the report's reason with its numbers. The disk counts as short below 40 GiB free, or when it is nearly full by k3s's own lines (DiskPressure, or under 15% free, where the kubelet removes images it is not running). On Windows the Podman machine's disk is a sparse virtual disk of about a terabyte, so a container sees hundreds of GiB free while the drive under it can have 30. The drive that holds the machine's WSL distribution (its BasePath) is measured too, and the smaller figure counts, at the account step and around each start. The card then says to free space on that drive or in Podman, since podman machine set does not change a WSL machine's disk. --- CHANGELOG.md | 8 + desktop/src-tauri/Cargo.lock | 1 + desktop/src-tauri/Cargo.toml | 5 +- desktop/src-tauri/src/local_eval.rs | 850 +++++++++++++++++- desktop/src-tauri/src/local_images.rs | 52 ++ desktop/src-tauri/src/windows.rs | 160 ++++ .../local-eval/k3s-v1.31.5-get-nodes.json | 268 ++++++ desktop/src/LocalIntelligence.test.tsx | 39 + desktop/src/localEvalCli.ts | 8 + desktop/src/localReadiness.test.ts | 62 +- desktop/src/localReadiness.ts | 36 +- 11 files changed, 1450 insertions(+), 39 deletions(-) create mode 100644 desktop/src-tauri/test-fixtures/local-eval/k3s-v1.31.5-get-nodes.json diff --git a/CHANGELOG.md b/CHANGELOG.md index b75de103f9..b57f51b55f 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -20,6 +20,14 @@ Newest first. `Unreleased` is what is on `main` and not yet tagged. Downloads sync now copies a prerelease asked for by tag (its images and its own files, never `latest/`), and each scheduled run copies the images of every prerelease newer than Latest from the last 30 days. Drafts are never copied. +- **A back end start that fails on a nearly full disk says so, with the numbers.** Relaunching a + used install whose back end's disk was nearly full said only "The back end stopped with an error + partway through." Each start, update and setup now checks the engine first and logs what it has + as `[local check]`, and a start that fails is checked again with the back end asked whether it is + short of disk. The card then says, for example, "The back end's disk is nearly full. Podman has + 30.5 GB free of 1006.8 GB. OpenBot needs 40 GB. Free space on drive C: or in Podman, then Retry." + On Windows, free disk is measured on the drive that holds Podman's machine, which can run out + while the machine itself shows hundreds of GB free. ## 1.0.0 diff --git a/desktop/src-tauri/Cargo.lock b/desktop/src-tauri/Cargo.lock index 027563c7ca..6d4236a9d5 100644 --- a/desktop/src-tauri/Cargo.lock +++ b/desktop/src-tauri/Cargo.lock @@ -2780,6 +2780,7 @@ dependencies = [ "tauri-plugin-updater", "time", "windows 0.62.2", + "windows-registry", "zip 2.4.2", ] diff --git a/desktop/src-tauri/Cargo.toml b/desktop/src-tauri/Cargo.toml index 1e680ce755..b1e3a8bee4 100644 --- a/desktop/src-tauri/Cargo.toml +++ b/desktop/src-tauri/Cargo.toml @@ -84,7 +84,10 @@ objc2-av-foundation = { version = "0.3", default-features = false, features = [" # Narration on Windows: the SAPI in-process recognizer. See `speech/sapi.rs`. The version Tauri uses. [target.'cfg(windows)'.dependencies] -windows = { version = "0.62", features = ["Win32_Foundation", "Win32_Globalization", "Win32_Media_Speech", "Win32_System_Com", "Data_Xml_Dom", "UI_Notifications", "Win32_UI_WindowsAndMessaging", "Win32_System_Threading"] } +windows = { version = "0.62", features = ["Win32_Foundation", "Win32_Globalization", "Win32_Media_Speech", "Win32_System_Com", "Data_Xml_Dom", "UI_Notifications", "Win32_UI_WindowsAndMessaging", "Win32_System_Threading", "Win32_Storage_FileSystem"] } +# Where WSL keeps a Podman machine's disk, so its drive's free space is checked before Start. See +# `windows::podman_machine_drive`. The version reqwest already uses. +windows-registry = "0.6" [dev-dependencies] tauri = { version = "2", features = ["test"] } diff --git a/desktop/src-tauri/src/local_eval.rs b/desktop/src-tauri/src/local_eval.rs index 4b9ce3a051..ddf9f280c5 100644 --- a/desktop/src-tauri/src/local_eval.rs +++ b/desktop/src-tauri/src/local_eval.rs @@ -269,6 +269,18 @@ pub struct Shortfall { pub memory_gb: Option, #[serde(skip_serializing_if = "Option::is_none")] pub disk_gb: Option, + /// The back end's own disk is nearly full: its node reports `DiskPressure`, or so little is + /// free that k3s removes the back end's images (`EngineCheck::nearly_full`). OpenBot's own + /// check after a start that failed, never the CLI's. + #[serde(skip_serializing_if = "std::ops::Not::not")] + pub nearly_full: bool, + /// How large that disk is, in GiB to one decimal, beside `disk_gb`, when it is nearly full. + #[serde(skip_serializing_if = "Option::is_none")] + pub disk_size_gb: Option, + /// Podman on Windows: the drive that holds its machine's disk, `C:`. Space is freed there, not + /// raised with `podman machine set`. + #[serde(skip_serializing_if = "Option::is_none")] + pub drive: Option, } /// A number the CLI printed, to one decimal and rounded down as it rounds, without a trailing `.0`. @@ -845,6 +857,9 @@ pub enum Terms { cpu: bool, memory: bool, disk: bool, + /// Podman on Windows: the drive that holds its machine's disk (`Shortfall::drive`). + #[serde(skip_serializing_if = "Option::is_none")] + drive: Option, }, } @@ -1198,17 +1213,28 @@ pub fn parse_engine_size(output: &str) -> Option<(u64, u64)> { fields.next().is_none().then_some((cpus, memory)) } -/// Free bytes from `df -Pk /`: the fourth field of its last line, in KiB, as the CLI reads it. +/// A filesystem as `df -Pk` reports it, in bytes. +#[derive(Clone, Copy, Debug, PartialEq, Eq)] +pub struct Disk { + pub size: u64, + pub free: u64, +} + +/// Size and free bytes from `df -Pk /`: the second and fourth fields of its last line, in KiB, as +/// the CLI reads them. +pub fn parse_disk(output: &str) -> Option { + let mut fields = output.trim().lines().last()?.split_whitespace(); + let size: u64 = fields.nth(1)?.parse().ok()?; + let free: u64 = fields.nth(1)?.parse().ok()?; + Some(Disk { + size: size * 1024, + free: free * 1024, + }) +} + +/// Free bytes from `df -Pk /`, as the CLI reads them. pub fn parse_free_disk(output: &str) -> Option { - let kib: u64 = output - .trim() - .lines() - .last()? - .split_whitespace() - .nth(3)? - .parse() - .ok()?; - Some(kib * 1024) + parse_disk(output).map(|disk| disk.free) } /// The engine's CPUs and memory from its `info`. @@ -1228,11 +1254,11 @@ fn engine_size(address: &crate::engine::Address) -> Option<(u64, u64)> { } /** -Free disk inside a container, the way the CLI measures it: `df -Pk /` in a container of an image the -engine already has, so nothing is downloaded. An engine with no image that has a shell gives no -answer, and disk is then not judged. +The engine's disk inside a container, the way the CLI measures it: `df -Pk /` in a container of an +image the engine already has, so nothing is downloaded. An engine with no image that has a shell +gives no answer, and disk is then not judged. */ -fn engine_free_disk(address: &crate::engine::Address) -> Option { +fn engine_disk(address: &crate::engine::Address) -> Option { let images = address .command() .args(["image", "ls", "--quiet", "--no-trunc"]) @@ -1263,17 +1289,180 @@ fn engine_free_disk(address: &crate::engine::Address) -> Option { "df -Pk /", ]) .output(); - let free = measured + let disk = measured .ok() .filter(|output| output.status.success()) - .and_then(|output| parse_free_disk(&String::from_utf8_lossy(&output.stdout))); - if free.is_some() { - return free; + .and_then(|output| parse_disk(&String::from_utf8_lossy(&output.stdout))); + if disk.is_some() { + return disk; } } None } +#[cfg(test)] +thread_local! { + /// What Windows says of the drive that holds the Podman machine's disk, for the tests: Some + /// stands in for `windows::podman_machine_drive` on this thread. + static MACHINE_DRIVE: std::cell::RefCell>> = + const { std::cell::RefCell::new(None) }; +} + +/// The drive that holds the Podman machine `address` names, on Windows (`podman_machine_drive`). +fn machine_drive(address: &crate::engine::Address) -> Option { + if address.engine != crate::engine::Engine::Podman { + return None; + } + #[cfg(test)] + if let Some(said) = MACHINE_DRIVE.with(|drive| drive.borrow().clone()) { + return said; + } + let connection = address + .connection + .clone() + .or_else(|| address.pin().ok().and_then(|pinned| pinned.connection))?; + crate::windows::podman_machine_drive(crate::acquire::machine_of(&connection)) +} + +/** +What OpenBot measured of the engine the back end runs on (`[local check]` in the log): its CPUs and +memory, its disk as a container sees it, on Windows the drive that holds a Podman machine's disk, and +after a call that failed, whether the back end's node says it is short of disk. +*/ +#[derive(Clone, Debug, Default, PartialEq, Eq)] +struct EngineCheck { + podman: bool, + /// CPUs and bytes of memory, from the engine's `info`. + size: Option<(u64, u64)>, + disk: Option, + drive: Option, + /// `DiskPressure` on the back end's node (`local_images::Node::disk_pressure`). + pressure: Option, +} + +/// Bytes as GiB to one decimal, as the CLI rounds them. +fn gib(bytes: u64) -> String { + shown(bytes as f64 / GIB as f64) +} + +impl EngineCheck { + fn of(address: &crate::engine::Address) -> Self { + Self { + podman: address.engine == crate::engine::Engine::Podman, + size: engine_size(address), + disk: engine_disk(address), + drive: machine_drive(address), + pressure: None, + } + } + + /// The free disk that counts: the smaller of what a container sees and what the drive under + /// the machine's disk has. Either can run out first. + fn free_disk(&self) -> Option { + [ + self.disk.map(|disk| disk.free), + self.drive.as_ref().map(|drive| drive.free), + ] + .into_iter() + .flatten() + .min() + } + + /** + The back end's own disk is nearly full, by k3s's lines rather than the CLI's 40 GiB: its node + says `DiskPressure`, which k3s raises below 5% free (`evictionHard`), or less than 15% is free, + where the kubelet starts removing the images it is not running (`imageGCHighThresholdPercent` + 85). A relaunched back end runs none yet, so that is when it loses them. + */ + fn nearly_full(&self) -> bool { + self.pressure == Some(true) + || self + .disk + .is_some_and(|disk| u128::from(disk.free) * 100 < u128::from(disk.size) * 15) + } + + /// What is short, with the numbers. Anything not measured is not judged. + fn shortfall(&self) -> Shortfall { + let (cpus, memory) = self.size.unwrap_or((u64::MAX, u64::MAX)); + let free = self.free_disk(); + let mut short = engine_shortfall(cpus, memory, free); + short.podman = self.podman; + short.nearly_full = self.nearly_full(); + short.disk |= short.nearly_full; + if short.disk { + short.disk_gb = free.map(gib); + short.disk_size_gb = self + .disk + .filter(|_| short.nearly_full) + .map(|disk| gib(disk.size)); + } + short.drive = self.drive.as_ref().and_then(|drive| drive.drive.clone()); + short + } + + /// `4 CPUs and 5.8 GiB of memory, 30.5 GiB free of 983.2 GiB in a container, 120.0 GiB free on + /// C:, no disk pressure`, then what is short. + fn said(&self) -> String { + let mut said = vec![match self.size { + Some((cpus, memory)) => format!("{cpus} CPUs and {} GiB of memory", gib(memory)), + None => "CPUs and memory not measured".to_owned(), + }]; + said.push(match self.disk { + Some(disk) => format!( + "{} GiB free of {} GiB in a container", + gib(disk.free), + gib(disk.size) + ), + None => "free disk not measured".to_owned(), + }); + if let Some(drive) = &self.drive { + let on = drive + .drive + .as_deref() + .unwrap_or("the drive that holds its disk"); + said.push(format!("{} GiB free on {on}", gib(drive.free))); + } + match self.pressure { + Some(true) => said.push("the node reports disk pressure".to_owned()), + Some(false) => said.push("no disk pressure".to_owned()), + None => {} + } + let short = self.shortfall(); + let mut verdict: Vec<&str> = Vec::new(); + if short.cpu { + verdict.push("CPUs"); + } + if short.memory { + verdict.push("memory"); + } + if short.disk { + verdict.push(if short.nearly_full { + "disk (nearly full)" + } else { + "disk (OpenBot needs 40 GiB free)" + }); + } + let verdict = if !verdict.is_empty() { + format!("short of {}", verdict.join(", ")) + } else if self.size.is_none() && self.free_disk().is_none() { + "not measured".to_owned() + } else { + "enough".to_owned() + }; + format!("{}; {verdict}", said.join(", ")) + } +} + +/// Log a check: `[local check] before local start: ...`. A shortfall is a warning. +fn log_check(when: &str, check: &EngineCheck) { + let level = if check.shortfall().named() { + log::Level::Warn + } else { + log::Level::Info + }; + log::log!(level, "[local check] {when}: {}", check.said()); +} + /** The onboarding check of the engine, before the account screen asks about the agreement: `Some(Terms::EngineTooSmall)` for an engine below `MIN_ENGINE_CPUS`, `MIN_ENGINE_MEMORY_BYTES` or @@ -1282,39 +1471,107 @@ here; the CLI's own check still runs at Start. */ pub fn engine_size_terms() -> Option { let address = cli_engine()?; - let (cpus, memory) = engine_size(&address)?; - let free_disk = engine_free_disk(&address); - let short = engine_shortfall(cpus, memory, free_disk); + let check = EngineCheck::of(&address); + let (cpus, memory) = check.size?; + let short = engine_shortfall(cpus, memory, check.free_disk()); if !short.named() { return None; } - log::warn!( - "[local terms] the engine has {cpus} CPUs, {memory} bytes of memory and {free_disk:?} bytes of free disk" - ); + log_check("before local setup", &check); Some(Terms::EngineTooSmall { - podman: address.engine == crate::engine::Engine::Podman, + podman: check.podman, cpu: short.cpu, memory: short.memory, disk: short.disk, + drive: check.drive.and_then(|drive| drive.drive), }) } /** A shortfall whose CLI error did not say which resource, measured here instead: CPUs and memory from -the engine's `info` against the CLI's 4 and 4 GiB, and free disk inside a container against its 40 -GiB. +the engine's `info` against the CLI's 4 and 4 GiB, and free disk inside a container, and on the drive +under a Podman machine's disk, against its 40 GiB. */ -fn measured_shortfall(mut shortfall: Shortfall) -> Shortfall { +fn measured_shortfall(shortfall: Shortfall) -> Shortfall { let Some(address) = cli_engine() else { return shortfall; }; - if let Some((cpus, memory)) = engine_size(&address) { - shortfall.cpu = cpus < 4; - shortfall.memory = memory < 4 * GIB; + let measured = EngineCheck::of(&address).shortfall(); + Shortfall { + podman: shortfall.podman, + ..measured + } +} + +/** +The engine checked around a call that starts the back end, `local start`, `local update` or `local +setup`. + +Before it, the check is only logged: an existing back end can run on less than the CLI's 40 GiB, and +the CLI enforces that only at a first setup. After a call that failed, it is checked again, with the +back end's node asked whether it is short of disk, and anything short becomes the report's reason +with its numbers, so the card names it and says where to raise or free it. A failure the CLI already +named, or one OpenBot stopped, keeps its own words. +*/ +fn checked_call( + command: &str, + stack: Option<&StackId>, + call: impl FnOnce() -> Result, +) -> Result { + let Some(address) = cli_engine() else { + return call(); + }; + log_check(&format!("before {command}"), &EngineCheck::of(&address)); + let result = call(); + let unexplained = match &result { + Ok(report) => { + report.outcome == Outcome::Failed + && matches!( + report.reason, + None | Some(Reason::LocalCommandFailed | Reason::Unrecognized) + ) + } + // The back end ran and stopped without a report OpenBot can use. One OpenBot stopped, or + // that never ran, is not about the engine. + Err(failure) => matches!( + failure, + Failure::Timeout + | Failure::OutputLimit + | Failure::InvalidJson + | Failure::UnexpectedExit + ), + }; + if !unexplained { + return result; + } + let mut check = EngineCheck::of(&address); + check.pressure = node_disk_pressure(stack); + log_check(&format!("after {command} failed"), &check); + let short = check.shortfall(); + match result { + Ok(mut report) if short.named() => { + report.reason = Some(Reason::ResourcesShort); + report.shortfall = Some(short); + Ok(report) + } + other => other, + } +} + +/// `DiskPressure` on `stack`'s node, or on any back end node on the engine when no stack is named +/// (a setup's failed report names none). +fn node_disk_pressure(stack: Option<&StackId>) -> Option { + let engine = recorded_engine(); + let command = || engine_command(engine.as_ref()); + let limits = crate::local_images::Limits { + call: Duration::from_secs(20), + ..crate::local_images::Limits::standard() + }; + match stack { + Some(stack) => crate::local_images::Node::for_stack(&command, stack.as_str()) + .disk_pressure(limits.call), + None => crate::local_images::any_disk_pressure(&command, limits), } - shortfall.disk = - engine_free_disk(&address).is_some_and(|free| free < MIN_ENGINE_FREE_DISK_BYTES); - shortfall } /// The version-1 envelope every `local` command shares: version, command, and a status that @@ -2918,6 +3175,26 @@ pub(crate) fn run_with( limits: (Limits, Limits), cancel: &Cancel, images_limits: crate::local_images::Limits, +) -> Result { + let call = || run_downloading(cli, attempt, action, stack, limits, cancel, images_limits); + match action { + // The calls that start the back end: the engine is checked around them. + Action::Start | Action::Update => { + checked_call(&format!("local {}", action.as_str()), Some(stack), call) + } + Action::Status | Action::Stop => call(), + } +} + +/// `run_with`, without the engine check. +fn run_downloading( + cli: &PinnedCli, + attempt: &str, + action: Action, + stack: &StackId, + limits: (Limits, Limits), + cancel: &Cancel, + images_limits: crate::local_images::Limits, ) -> Result { // A CLI whose images cannot be read runs exactly as before, with nothing asked of it first. let downloaded = match (action, cli_images_of(cli)) { @@ -3130,6 +3407,30 @@ pub(crate) fn setup_with( limits: (Limits, Limits), cancel: &Cancel, images_limits: crate::local_images::Limits, +) -> Result { + // The engine is checked around it, as around `local start` (`checked_call`). + checked_call("local setup", None, || { + setup_downloading( + cli, + attempt, + installation, + accepted, + limits, + cancel, + images_limits, + ) + }) +} + +/// `setup_with`, without the engine check. +fn setup_downloading( + cli: &PinnedCli, + attempt: &str, + installation: &std::path::Path, + accepted: Option<&EulaVersion>, + limits: (Limits, Limits), + cancel: &Cancel, + images_limits: crate::local_images::Limits, ) -> Result { let downloaded = match cli_images_of(cli) { Some(images) => match stack_set_up_in(cli, installation, cancel)? { @@ -6695,11 +6996,23 @@ overlay 164506208 20481024 135620148 14% / podman: false, cpu: false, memory: false, - disk: true + disk: true, + drive: None, }) .unwrap(), json!({ "status": "engine-too-small", "podman": false, "cpu": false, "memory": false, "disk": true }) ); + assert_eq!( + serde_json::to_value(Terms::EngineTooSmall { + podman: true, + cpu: false, + memory: false, + disk: true, + drive: Some("C:".into()), + }) + .unwrap(), + json!({ "status": "engine-too-small", "podman": true, "cpu": false, "memory": false, "disk": true, "drive": "C:" }) + ); } /// A new Windows computer installs Podman while the person signs in, and the CLI will not say @@ -6808,4 +7121,471 @@ overlay 164506208 20481024 135620148 14% / assert!(bundled_cli().is_none()); assert!(bundled_cli_root().is_some()); } + + /** + A container engine as `local start` meets it, put on PATH as `docker` or `podman`. It answers + what OpenBot asks before and after a start with what the real one prints, from files beside it: + `info --format` (`info`), `image ls --quiet --no-trunc` (`images`), `df -Pk /` in a container + (`df`), `kubectl get nodes -o json` in the stack's node (`nodes`) and `machine inspect` + (`inspect`). A node or a file that is not there fails as the engine does. + */ + const FAKE_ENGINE: &str = r#" +use std::io::Write; +fn main() { + let exe = std::env::current_exe().unwrap(); + let dir = exe.parent().unwrap().to_path_buf(); + let name = exe.file_stem().unwrap().to_string_lossy().into_owned(); + let mut args: Vec = std::env::args().skip(1).collect(); + let mut asked = std::fs::OpenOptions::new().create(true).append(true).open(dir.join("asked")).unwrap(); + writeln!(asked, "{} {}", name, args.join(" ")).unwrap(); + while matches!(args.first().map(String::as_str), Some("--connection" | "--url" | "--context" | "--host")) { + args.drain(..2.min(args.len())); + } + let a: Vec<&str> = args.iter().map(String::as_str).collect(); + let file = match a.as_slice() { + ["info", "--format", _] => "info", + ["image", "ls", "--quiet", "--no-trunc"] => "images", + ["run", "--rm", "--pull=never", "--network=none", "--entrypoint", "/bin/sh", _, "-c", "df -Pk /"] => "df", + ["exec", "k3d-local-abc123-server-0", "kubectl", "get", "nodes", ..] => "nodes", + ["machine", "inspect", "openbot"] => "inspect", + _ => "", + }; + if let Some(bytes) = (!file.is_empty()).then(|| std::fs::read(dir.join(file)).ok()).flatten() { + std::io::stdout().write_all(&bytes).unwrap(); + return; + } + if a.first() == Some(&"exec") { + let node = a.get(1).copied().unwrap_or_default(); + if name == "podman" { + eprintln!("Error: no container with name or ID \"{node}\" found: no such container"); + std::process::exit(125); + } + eprintln!("Error response from daemon: No such container: {node}"); + std::process::exit(1); + } + eprintln!("Error: unknown command {a:?}"); + std::process::exit(125); +} +"#; + + /// Every entry logged in this process. Only a test that runs alone in its process + /// (`isolated_process`) installs it. + static LOGGED: std::sync::Mutex> = std::sync::Mutex::new(Vec::new()); + + struct Keep; + + impl log::Log for Keep { + fn enabled(&self, _: &log::Metadata) -> bool { + true + } + fn log(&self, record: &log::Record) { + LOGGED + .lock() + .unwrap() + .push(format!("{} {}", record.level(), record.args())); + } + fn flush(&self) {} + } + + /// The `[local check]` entries logged since the last call, taken. + fn checks_logged() -> Vec { + std::mem::take(&mut *LOGGED.lock().unwrap()) + .into_iter() + .filter(|entry| entry.contains("[local check]")) + .collect() + } + + /// `docker info --format '{{.NCPU}} {{.MemTotal}}'` on a Mac's Docker Desktop with 10 CPUs and + /// 20.5 GiB, and `podman info --format '{{.Host.CPUs}} {{.Host.MemTotal}}'` on OpenBot's own + /// Podman machine, 4 CPUs and 6144 MiB. + const DOCKER_INFO: &str = "10 22011707392\n"; + const PODMAN_INFO: &str = "4 6262661120\n"; + + /// `df -Pk /` in a container, as busybox prints it: a disk of `size` KiB with `free` KiB free. + fn df(size: u64, free: u64) -> String { + let used = size - free - size / 20; + format!( + "Filesystem 1024-blocks Used Available Capacity Mounted on\noverlay {size} {used} {free} {}% /\n", + used * 100 / (used + free) + ) + } + + const KIB_PER_GIB: u64 = 1024 * 1024; + /// OpenBot's Podman machine on WSL: a sparse virtual disk of about a terabyte. + const WSL_DISK: u64 = 1_055_762_868; + + /// `kubectl get nodes -o json` in the stack's node, as k3s v1.31.5 printed it, with its + /// `DiskPressure` condition as the kubelet sets it when `pressure`. + fn nodes(pressure: bool) -> String { + let mut nodes: Value = serde_json::from_str(include_str!( + "../test-fixtures/local-eval/k3s-v1.31.5-get-nodes.json" + )) + .unwrap(); + if pressure { + for condition in nodes["items"][0]["status"]["conditions"] + .as_array_mut() + .unwrap() + { + if condition["type"] == "DiskPressure" { + condition["status"] = json!("True"); + condition["reason"] = json!("KubeletHasDiskPressure"); + condition["message"] = json!("kubelet has disk pressure"); + } + } + } + serde_json::to_string_pretty(&nodes).unwrap() + } + + /// `podman machine inspect openbot` for OpenBot's own rootful machine on WSL, as Podman 5 + /// prints it on Windows. + const WSL_INSPECT: &str = r#"[ + { + "ConfigDir": { + "Path": "C:\\Users\\me\\.config\\containers\\podman\\machine\\wsl" + }, + "ConnectionInfo": { + "PodmanSocket": null, + "PodmanPipe": { + "Path": "\\\\.\\pipe\\podman-openbot" + } + }, + "Created": "2026-11-02T09:14:03.4719262-08:00", + "LastUp": "2026-11-09T08:02:51.1063417-08:00", + "Name": "openbot", + "Resources": { + "CPUs": 4, + "DiskSize": 100, + "Memory": 6144, + "USBs": [] + }, + "SSHConfig": { + "IdentityPath": "C:\\Users\\me\\.local\\share\\containers\\podman\\machine\\machine", + "Port": 61842, + "RemoteUsername": "user" + }, + "State": "running", + "UserModeNetworking": true, + "Rootful": true, + "Rosetta": false + } +] +"#; + + /// `local start` as the pinned CLI (4.34.0-pre.647a03b) reports a start that stopped in + /// `stack-services-ready`, the step that waits for the back end's deployments. + fn start_failed() -> String { + json!({ + "version": 1, + "command": "local start", + "status": "failed", + "error": { + "code": "LOCAL_COMMAND_FAILED", + "message": "Local stack-services-ready failed (exit 1). Run `copilotkit local status` and retry the failed step." + }, + "nextAction": { + "reason": "local_command_failed", + "summary": "The local command did not finish. Check local status for the recorded failed step, then resume the command. No automatic rollback was requested.", + "instructions": ["Inspect the recorded stack with npx --yes copilotkit@4.34.0-pre.647a03b local status --json."], + "resumeCommand": "npx --yes copilotkit@4.34.0-pre.647a03b local start --json", + "url": null, "linkLabel": null, "linkFile": null, + "caveats": [], "requiredEnvVars": [], "artifacts": [] + }, + "diagnostics": [] + }) + .to_string() + } + + fn start_completed() -> String { + let mut done = envelope(); + done["command"] = json!("local start"); + done.to_string() + } + + /** + An engine and a CLI for `local start` through `run_installed`, the window's own path. The engine + is `engine` on PATH answering with `answers` (`FAKE_ENGINE`), and named to the CLI as OpenBot + names it; the CLI is pinned as a developer pins one. Windows' answer for the drive that holds a + Podman machine's disk is `drive`. Sets process environment: only for a test in its own process. + */ + struct StartLab { + fixture: Fixture, + bin: PathBuf, + } + + impl StartLab { + fn new(label: &str) -> Self { + let fixture = Fixture::new(label, "0\n{}"); + let bin = fixture.root.join("bin"); + std::fs::create_dir_all(&bin).unwrap(); + let source = fixture.root.join("engine.rs"); + std::fs::write(&source, FAKE_ENGINE).unwrap(); + for engine in ["docker", "podman"] { + compile_fixture( + &source, + &bin.join(format!("{engine}{}", std::env::consts::EXE_SUFFIX)), + ); + } + std::env::set_var( + "PATH", + std::env::join_paths(std::iter::once(bin.clone()).chain(std::env::split_paths( + &std::env::var_os("PATH").unwrap_or_default(), + ))) + .unwrap(), + ); + std::env::set_var(DEVELOPER_PROGRAM, &fixture.program); + std::env::set_var(DEVELOPER_PROGRAM_SHA256, sha256(&fixture.program)); + let _ = log::set_boxed_logger(Box::new(Keep)); + log::set_max_level(log::LevelFilter::Info); + Self { fixture, bin } + } + + /// The engine as it answers now, and how `local start` ends. + fn engine( + &self, + engine: crate::engine::Engine, + info: &str, + df: &str, + nodes: Option<&str>, + drive: Option, + ) { + let write = |file: &str, said: Option<&str>| match said { + Some(said) => std::fs::write(self.bin.join(file), said).unwrap(), + None => { + let _ = std::fs::remove_file(self.bin.join(file)); + } + }; + write("info", Some(info)); + // `image ls --quiet --no-trunc`: the k3d tools image, which has a shell. + write( + "images", + Some("sha256:3a68d8567335f56c168c36b53059c5dd4dfe23c2edbf4d9251e9ed3170dfacd8\n"), + ); + write("df", Some(df)); + write("nodes", nodes); + write("inspect", Some(WSL_INSPECT)); + use_engine(&match engine { + crate::engine::Engine::Docker => crate::engine::Address::new(engine, None), + crate::engine::Engine::Podman => { + crate::engine::Address::new(engine, Some("openbot-root".into())) + } + }); + MACHINE_DRIVE.with(|said| *said.borrow_mut() = Some(drive)); + } + + fn start(&self, ends: &str) -> Report { + std::fs::write(self.fixture.root.join("scenario"), ends).unwrap(); + let stack = StackId::parse("local-abc123").unwrap(); + let report = run_installed("start-checked", Action::Start, &stack).unwrap(); + assert_eq!( + self.fixture.argv().unwrap(), + "local\nstart\n--stack\nlocal-abc123\n--json" + ); + report + } + } + + /// Windows' answer for drive C: with `tenths` tenths of a GiB free, and a little more. + fn drive(tenths: u64) -> Option { + Some(crate::windows::MachineDrive { + drive: Some("C:".into()), + free: tenths * GIB / 10 + GIB / 100, + }) + } + + /** + 1.0.0 on Windows, a used install on OpenBot's own Podman machine with 30.5 GiB free inside it: + Quit, then a relaunch. The CLI checks resources only at a first setup, and an existing back end + can run on less than its 40 GiB, so a start that finishes is let through. The log warns, with the + numbers, and nothing else changes. + */ + #[test] + fn a_used_install_short_of_disk_still_starts_and_the_log_warns() { + if crate::test_support::isolated_process( + "local_eval::tests::a_used_install_short_of_disk_still_starts_and_the_log_warns", + ) { + return; + } + let lab = StartLab::new("start-short-of-disk-warns"); + lab.engine( + crate::engine::Engine::Podman, + PODMAN_INFO, + &df(WSL_DISK, 30 * KIB_PER_GIB + KIB_PER_GIB / 2), + Some(&nodes(false)), + drive(946), + ); + let report = lab.start(&format!("0\n{}", start_completed())); + let wire = serde_json::to_value(&report).unwrap(); + assert_eq!(wire["outcome"], json!("completed")); + assert_eq!(wire["reason"], Value::Null); + assert!(wire.get("shortfall").is_none(), "{wire}"); + assert_eq!( + checks_logged(), + ["WARN [local check] before local start: 4 CPUs and 5.8 GiB of memory, 30.5 GiB free of 1006.8 GiB in a container, 94.6 GiB free on C:; short of disk (nearly full)"] + ); + } + + /** + The same start, failed: the card said only "The back end stopped with an error partway through" + while k3s, short of disk, had removed 13 of the back end's 15 images. A start that fails is + checked again with the node asked, and the report names the disk with its numbers, for Podman on + Windows, and for Docker short of the CLI's 40 GiB or with its node under disk pressure. + */ + #[test] + fn a_start_that_fails_short_of_disk_names_the_disk_with_its_numbers() { + if crate::test_support::isolated_process( + "local_eval::tests::a_start_that_fails_short_of_disk_names_the_disk_with_its_numbers", + ) { + return; + } + let lab = StartLab::new("start-short-of-disk-named"); + let failed = format!("1\n{}", start_failed()); + + lab.engine( + crate::engine::Engine::Podman, + PODMAN_INFO, + &df(WSL_DISK, 30 * KIB_PER_GIB + KIB_PER_GIB / 2), + Some(&nodes(true)), + drive(946), + ); + let wire = serde_json::to_value(lab.start(&failed)).unwrap(); + assert_eq!(wire["outcome"], json!("failed")); + assert_eq!(wire["reason"], json!("resources-short")); + assert_eq!( + wire["shortfall"], + json!({ + "podman": true, "cpu": false, "memory": false, "disk": true, + "diskGb": "30.5", "nearlyFull": true, "diskSizeGb": "1006.8", "drive": "C:" + }) + ); + assert_eq!( + checks_logged(), + [ + "WARN [local check] before local start: 4 CPUs and 5.8 GiB of memory, 30.5 GiB free of 1006.8 GiB in a container, 94.6 GiB free on C:; short of disk (nearly full)", + "WARN [local check] after local start failed: 4 CPUs and 5.8 GiB of memory, 30.5 GiB free of 1006.8 GiB in a container, 94.6 GiB free on C:, the node reports disk pressure; short of disk (nearly full)", + ] + ); + + // Docker Desktop with 30 GiB of a 64 GiB disk free: below the CLI's 40 GiB. + lab.engine( + crate::engine::Engine::Docker, + DOCKER_INFO, + &df(64 * KIB_PER_GIB, 30 * KIB_PER_GIB), + Some(&nodes(false)), + None, + ); + let wire = serde_json::to_value(lab.start(&failed)).unwrap(); + assert_eq!(wire["reason"], json!("resources-short")); + assert_eq!( + wire["shortfall"], + json!({ "podman": false, "cpu": false, "memory": false, "disk": true, "diskGb": "30" }) + ); + assert_eq!( + checks_logged().last().unwrap(), + "WARN [local check] after local start failed: 10 CPUs and 20.5 GiB of memory, 30 GiB free of 64 GiB in a container, no disk pressure; short of disk (OpenBot needs 40 GiB free)" + ); + + // 300 GiB free of a 1.8 TiB disk is above both lines, but the node still says + // DiskPressure: k3s holds it for five minutes after the kubelet freed space. + lab.engine( + crate::engine::Engine::Docker, + DOCKER_INFO, + &df(1_916_694_008, 300 * KIB_PER_GIB), + Some(&nodes(true)), + None, + ); + let wire = serde_json::to_value(lab.start(&failed)).unwrap(); + assert_eq!(wire["reason"], json!("resources-short")); + assert_eq!( + wire["shortfall"], + json!({ + "podman": false, "cpu": false, "memory": false, "disk": true, + "diskGb": "300", "nearlyFull": true, "diskSizeGb": "1827.9" + }) + ); + } + + /** + On Windows the machine's disk is a sparse virtual disk of about a terabyte, so a container sees + 941 GiB free while the drive that holds it has 30. The drive is what runs out: it is checked, + the smaller figure counts, and the card says to free space on that drive. The account step's + check before a first setup reads the drive too. + */ + #[test] + fn on_windows_the_drive_under_podmans_disk_is_the_free_disk_that_counts() { + if crate::test_support::isolated_process( + "local_eval::tests::on_windows_the_drive_under_podmans_disk_is_the_free_disk_that_counts", + ) { + return; + } + let lab = StartLab::new("start-windows-drive"); + lab.engine( + crate::engine::Engine::Podman, + PODMAN_INFO, + &df(WSL_DISK, 941 * KIB_PER_GIB), + Some(&nodes(false)), + drive(300), + ); + let wire = serde_json::to_value(lab.start(&format!("1\n{}", start_failed()))).unwrap(); + assert_eq!(wire["reason"], json!("resources-short")); + assert_eq!( + wire["shortfall"], + json!({ + "podman": true, "cpu": false, "memory": false, "disk": true, + "diskGb": "30", "drive": "C:" + }) + ); + assert_eq!( + checks_logged(), + [ + "WARN [local check] before local start: 4 CPUs and 5.8 GiB of memory, 941 GiB free of 1006.8 GiB in a container, 30 GiB free on C:; short of disk (OpenBot needs 40 GiB free)", + "WARN [local check] after local start failed: 4 CPUs and 5.8 GiB of memory, 941 GiB free of 1006.8 GiB in a container, 30 GiB free on C:, no disk pressure; short of disk (OpenBot needs 40 GiB free)", + ] + ); + assert_eq!( + serde_json::to_value(engine_size_terms()).unwrap(), + json!({ + "status": "engine-too-small", "podman": true, "cpu": false, "memory": false, + "disk": true, "drive": "C:" + }) + ); + } + + /** + OpenBot's own Podman machine, 4 CPUs and 6144 MiB, with room on its disk and its drive, passes: + a start that finishes says so in the log, and one that fails for some other reason keeps its + own words rather than being blamed on resources. The account step lets it through. + */ + #[test] + fn openbots_own_podman_machine_with_room_passes_the_check() { + if crate::test_support::isolated_process( + "local_eval::tests::openbots_own_podman_machine_with_room_passes_the_check", + ) { + return; + } + let lab = StartLab::new("start-roomy-podman"); + lab.engine( + crate::engine::Engine::Podman, + PODMAN_INFO, + &df(WSL_DISK, 941 * KIB_PER_GIB), + Some(&nodes(false)), + drive(1200), + ); + let started = + serde_json::to_value(lab.start(&format!("0\n{}", start_completed()))).unwrap(); + assert_eq!(started["outcome"], json!("completed")); + assert!(started.get("shortfall").is_none(), "{started}"); + let failed = serde_json::to_value(lab.start(&format!("1\n{}", start_failed()))).unwrap(); + assert_eq!(failed["reason"], json!("local-command-failed")); + assert!(failed.get("shortfall").is_none(), "{failed}"); + let enough = "4 CPUs and 5.8 GiB of memory, 941 GiB free of 1006.8 GiB in a container, 120 GiB free on C:"; + assert_eq!( + checks_logged(), + [ + format!("INFO [local check] before local start: {enough}; enough"), + format!("INFO [local check] before local start: {enough}; enough"), + format!("INFO [local check] after local start failed: {enough}, no disk pressure; enough"), + ] + ); + assert_eq!(engine_size_terms(), None); + } } diff --git a/desktop/src-tauri/src/local_images.rs b/desktop/src-tauri/src/local_images.rs index 6c769c13b8..0ee66c198e 100644 --- a/desktop/src-tauri/src/local_images.rs +++ b/desktop/src-tauri/src/local_images.rs @@ -586,6 +586,28 @@ done"#; ok.then(|| lines(&out).map(str::to_owned).collect()) } + /** + Whether the node's kubelet says it is short of disk: its `DiskPressure` condition, from `kubectl + get nodes` run in the node, as the CLI runs its own `kubectl`. k3s raises it below 5% free and + removes images it is not running above 85% used, which is how a relaunch on a nearly full disk + loses the back end's images. None when the node does not answer. + */ + pub fn disk_pressure(&self, limit: Duration) -> Option { + let (ok, out) = self.exec( + &[ + "kubectl", + "get", + "nodes", + "--request-timeout=5s", + "-o", + "json", + ], + limit, + 4 << 20, + )?; + ok.then(|| disk_pressure_of(&out)).flatten() + } + /// The node's architecture, as an image index names it. fn architecture(&self, limits: Limits) -> Option<&'static str> { let (ok, out) = self.exec(&["uname", "-m"], limits.call, 64)?; @@ -676,6 +698,36 @@ done"#; } } +/// `DiskPressure` from `kubectl get nodes -o json`: true when any node has it, false when every +/// node that says has not, None when none says. +pub fn disk_pressure_of(json: &[u8]) -> Option { + let nodes: Value = serde_json::from_slice(json).ok()?; + let said: Vec = nodes + .get("items")? + .as_array()? + .iter() + .filter_map(|node| node.pointer("/status/conditions")?.as_array()) + .flatten() + .filter(|condition| condition.get("type").and_then(Value::as_str) == Some("DiskPressure")) + .filter_map(|condition| match condition.get("status")?.as_str()? { + "True" => Some(true), + "False" => Some(false), + _ => None, + }) + .collect(); + (!said.is_empty()).then(|| said.contains(&true)) +} + +/// What every k3d node on the engine says of `DiskPressure` (`Node::disk_pressure`): true when +/// any has it. The nodes share the engine's disk. +pub fn any_disk_pressure(engine: &(dyn Fn() -> Command + Sync), limits: Limits) -> Option { + let said: Vec = stack_nodes(engine, limits)? + .iter() + .filter_map(|stack| Node::for_stack(engine, stack).disk_pressure(limits.call)) + .collect(); + (!said.is_empty()).then(|| said.contains(&true)) +} + fn lines(bytes: &[u8]) -> impl Iterator { std::str::from_utf8(bytes) .unwrap_or_default() diff --git a/desktop/src-tauri/src/windows.rs b/desktop/src-tauri/src/windows.rs index 76f644fbe4..17aab8603c 100644 --- a/desktop/src-tauri/src/windows.rs +++ b/desktop/src-tauri/src/windows.rs @@ -952,11 +952,171 @@ pub fn restart() -> Result<(), Problem> { Err(Problem::plain("Only Windows restarts from here.")) } +/// The Windows drive that holds a Podman machine's disk, and how much is free on it. +#[derive(Clone, Debug, PartialEq, Eq)] +pub struct MachineDrive { + /// `C:`, or None when the disk's folder is not on a lettered drive. + pub drive: Option, + pub free: u64, +} + +/** +Where a Podman machine on WSL keeps its disk, and the space free on that drive. + +The machine's disk is a sparse virtual disk of about a terabyte, so the space a container sees free +inside it says nothing about the space left on Windows: a machine can show 941 GiB free on a drive +with 30. Podman imports each machine as the WSL distribution `podman-` +(`env.WithPodmanPrefix`), and WSL records the folder it holds the disk in as that distribution's +`BasePath`. None off Windows, and when WSL has no such distribution. +*/ +pub fn podman_machine_drive(machine: &str) -> Option { + #[cfg(target_os = "windows")] + { + machine_drive_in(LXSS, &podman_distribution(machine)) + } + #[cfg(not(target_os = "windows"))] + { + let _ = machine; + None + } +} + +/// The WSL distribution Podman makes for `machine`, as Podman names it (`env.WithPodmanPrefix`). +pub fn podman_distribution(machine: &str) -> String { + if machine.starts_with("podman") { + machine.to_owned() + } else { + format!("podman-{machine}") + } +} + +/// The drive a Windows path is on, `C:`, with or without the `\\?\` prefix WSL records it with. +pub fn drive_of(path: &str) -> Option { + let path = path.strip_prefix(r"\\?\").unwrap_or(path); + match path.as_bytes() { + [letter, b':', ..] if letter.is_ascii_alphabetic() => { + Some(format!("{}:", letter.to_ascii_uppercase() as char)) + } + _ => None, + } +} + +/// Where WSL records each distribution it has, for the person signed in. +#[cfg(target_os = "windows")] +const LXSS: &str = r"Software\Microsoft\Windows\CurrentVersion\Lxss"; + +#[cfg(target_os = "windows")] +fn machine_drive_in(registry: &str, distribution: &str) -> Option { + let folder = distribution_folder(registry, distribution)?; + let free = free_space(&folder)?; + Some(MachineDrive { + drive: drive_of(&folder), + free, + }) +} + +/// The `BasePath` WSL recorded for `distribution`, under `registry` in the person's own hive. +#[cfg(target_os = "windows")] +fn distribution_folder(registry: &str, distribution: &str) -> Option { + let lxss = windows_registry::CURRENT_USER.open(registry).ok()?; + let ids: Vec = lxss.keys().ok()?.collect(); + ids.iter().find_map(|id| { + let entry = lxss.open(id).ok()?; + let name = entry.get_string("DistributionName").ok()?; + if !name.eq_ignore_ascii_case(distribution) { + return None; + } + entry.get_string("BasePath").ok() + }) +} + +/// The bytes free to this account on the volume that holds `folder`. +#[cfg(target_os = "windows")] +fn free_space(folder: &str) -> Option { + use std::os::windows::ffi::OsStrExt; + let folder = folder.strip_prefix(r"\\?\").unwrap_or(folder); + let wide: Vec = std::ffi::OsStr::new(folder) + .encode_wide() + .chain(std::iter::once(0)) + .collect(); + let mut free = 0u64; + // SAFETY: `wide` is a NUL-terminated path that outlives the call, and `free` is a u64 it may + // write; the two totals are not asked for. + unsafe { + windows::Win32::Storage::FileSystem::GetDiskFreeSpaceExW( + windows::core::PCWSTR(wide.as_ptr()), + Some(&mut free), + None, + None, + ) + } + .ok()?; + Some(free) +} + #[cfg(test)] mod tests { use super::*; use crate::test_support::temp_root; + /// Podman names its machine's WSL distribution as `env.WithPodmanPrefix` does, and WSL records + /// its folder with the `\\?\` prefix or without. + #[test] + fn a_podman_machines_distribution_and_drive_are_named_as_podman_and_wsl_name_them() { + assert_eq!(podman_distribution("openbot"), "podman-openbot"); + assert_eq!( + podman_distribution("podman-machine-default"), + "podman-machine-default" + ); + for (path, drive) in [ + ( + r"\\?\C:\Users\me\.local\share\containers\podman\machine\wsl\wsldist\openbot", + Some("C:"), + ), + (r"d:\podman\wsldist\openbot", Some("D:")), + (r"\\server\share\wsldist\openbot", None), + ("", None), + ] { + assert_eq!(drive_of(path).as_deref(), drive, "{path}"); + } + } + + /// The machine's drive is read where WSL keeps it: a distribution's `BasePath` in the person's + /// own registry hive, here under a key of this test's own with the shape WSL writes, and the + /// free space of the real volume that folder is on. + #[cfg(target_os = "windows")] + #[test] + fn a_podman_machines_drive_is_read_from_the_folder_wsl_recorded() { + let registry = format!(r"Software\OpenBot\Tests\Lxss-{}", std::process::id()); + let folder = temp_root("wsl-distribution"); + std::fs::create_dir_all(&folder).unwrap(); + let entry = |id: &str, name: &str, base: &str| { + let key = windows_registry::CURRENT_USER + .create(format!(r"{registry}\{id}")) + .unwrap(); + key.set_string("DistributionName", name).unwrap(); + key.set_string("BasePath", base).unwrap(); + key.set_string("VhdFileName", "ext4.vhdx").unwrap(); + }; + entry( + "{0d4e1c9a-5f0b-4a57-9d3e-1a2b3c4d5e6f}", + "Ubuntu", + r"Z:\elsewhere", + ); + entry( + "{7c1f2e3d-9a8b-4c5d-8e7f-6a5b4c3d2e1f}", + "podman-openbot", + &format!(r"\\?\{}", folder.display()), + ); + let found = machine_drive_in(®istry, "podman-openbot"); + let missing = machine_drive_in(®istry, "podman-other"); + let _ = windows_registry::CURRENT_USER.remove_tree(®istry); + let found = found.expect("the machine's folder and its volume were read"); + assert_eq!(found.drive, drive_of(&folder.display().to_string())); + assert!(found.free > 0, "{found:?}"); + assert_eq!(missing, None); + } + const PROBE_OUTPUTS: [&str; 6] = [ "hypervisor=True\nfirmware=False\n", "True\n", diff --git a/desktop/src-tauri/test-fixtures/local-eval/k3s-v1.31.5-get-nodes.json b/desktop/src-tauri/test-fixtures/local-eval/k3s-v1.31.5-get-nodes.json new file mode 100644 index 0000000000..5ba5339b63 --- /dev/null +++ b/desktop/src-tauri/test-fixtures/local-eval/k3s-v1.31.5-get-nodes.json @@ -0,0 +1,268 @@ +{ + "apiVersion": "v1", + "items": [ + { + "apiVersion": "v1", + "kind": "Node", + "metadata": { + "annotations": { + "alpha.kubernetes.io/provided-node-ip": "172.27.0.2", + "flannel.alpha.coreos.com/backend-data": "{\"VNI\":1,\"VtepMAC\":\"52:3a:21:98:27:b8\"}", + "flannel.alpha.coreos.com/backend-type": "vxlan", + "flannel.alpha.coreos.com/kube-subnet-manager": "true", + "flannel.alpha.coreos.com/public-ip": "172.27.0.2", + "k3s.io/hostname": "k3d-local-e0cf90debea88453ad454b93-server-0", + "k3s.io/internal-ip": "172.27.0.2", + "k3s.io/node-args": "[\"server\",\"--disable\",\"traefik\",\"--tls-san\",\"127.0.0.1\"]", + "k3s.io/node-config-hash": "MXCMP4EPPR7HGKB2OXMJELW7CFJQH6IT4ZP54RPNISYBXKEKV6WA====", + "k3s.io/node-env": "{\"K3S_KUBECONFIG_OUTPUT\":\"/output/kubeconfig.yaml\",\"K3S_TOKEN\":\"********\"}", + "node.alpha.kubernetes.io/ttl": "0", + "volumes.kubernetes.io/controller-managed-attach-detach": "true" + }, + "creationTimestamp": "2026-11-07T09:46:42Z", + "finalizers": [ + "wrangler.cattle.io/node" + ], + "labels": { + "beta.kubernetes.io/arch": "arm64", + "beta.kubernetes.io/instance-type": "k3s", + "beta.kubernetes.io/os": "linux", + "kubernetes.io/arch": "arm64", + "kubernetes.io/hostname": "k3d-local-e0cf90debea88453ad454b93-server-0", + "kubernetes.io/os": "linux", + "node-role.kubernetes.io/control-plane": "true", + "node-role.kubernetes.io/master": "true", + "node.kubernetes.io/instance-type": "k3s" + }, + "name": "k3d-local-e0cf90debea88453ad454b93-server-0", + "resourceVersion": "105240", + "uid": "71b0e4ba-c3f1-4a91-b493-88743fc7f270" + }, + "spec": { + "podCIDR": "10.42.0.0/24", + "podCIDRs": [ + "10.42.0.0/24" + ], + "providerID": "k3s://k3d-local-e0cf90debea88453ad454b93-server-0" + }, + "status": { + "addresses": [ + { + "address": "172.27.0.2", + "type": "InternalIP" + }, + { + "address": "k3d-local-e0cf90debea88453ad454b93-server-0", + "type": "Hostname" + } + ], + "allocatable": { + "cpu": "18", + "ephemeral-storage": "1864559929521", + "hugepages-1Gi": "0", + "hugepages-2Mi": "0", + "hugepages-32Mi": "0", + "hugepages-64Ki": "0", + "memory": "16353252Ki", + "pods": "110" + }, + "capacity": { + "cpu": "18", + "ephemeral-storage": "1916694008Ki", + "hugepages-1Gi": "0", + "hugepages-2Mi": "0", + "hugepages-32Mi": "0", + "hugepages-64Ki": "0", + "memory": "16353252Ki", + "pods": "110" + }, + "conditions": [ + { + "lastHeartbeatTime": "2026-11-09T12:03:04Z", + "lastTransitionTime": "2026-11-08T05:15:10Z", + "message": "kubelet has sufficient memory available", + "reason": "KubeletHasSufficientMemory", + "status": "False", + "type": "MemoryPressure" + }, + { + "lastHeartbeatTime": "2026-11-09T12:03:04Z", + "lastTransitionTime": "2026-11-08T05:15:10Z", + "message": "kubelet has no disk pressure", + "reason": "KubeletHasNoDiskPressure", + "status": "False", + "type": "DiskPressure" + }, + { + "lastHeartbeatTime": "2026-11-09T12:03:04Z", + "lastTransitionTime": "2026-11-08T05:15:10Z", + "message": "kubelet has sufficient PID available", + "reason": "KubeletHasSufficientPID", + "status": "False", + "type": "PIDPressure" + }, + { + "lastHeartbeatTime": "2026-11-09T12:03:04Z", + "lastTransitionTime": "2026-11-08T05:15:10Z", + "message": "kubelet is posting ready status", + "reason": "KubeletReady", + "status": "True", + "type": "Ready" + } + ], + "daemonEndpoints": { + "kubeletEndpoint": { + "Port": 10250 + } + }, + "images": [ + { + "names": [ + "ghcr.io/copilotkit/intelligence/app-api@sha256:060ae0f883786612d064ebbb8d081b83006e9e538a936b74e383ecd3dd81a844" + ], + "sizeBytes": 147210711 + }, + { + "names": [ + "ghcr.io/copilotkit/intelligence/app-api@sha256:9ee3006d92a656a72be43f9bda0e0738aec910f3b8098aba8e029f28061e85b7" + ], + "sizeBytes": 147206537 + }, + { + "names": [ + "ghcr.io/copilotkit/intelligence/postgres-pgvector@sha256:3ec79ad28b8ba356734e1f29d757d5e1f5f4104360e2d285a63db1bb6a0cfe28" + ], + "sizeBytes": 108486056 + }, + { + "names": [ + "ghcr.io/copilotkit/intelligence/postgres-pgvector@sha256:f98d6b61014200ab2b946720c84d4693ecd2f68abbe7d00779a57729021d03de" + ], + "sizeBytes": 108486053 + }, + { + "names": [ + "ghcr.io/copilotkit/intelligence/db-migrations@sha256:ba649b6358d6e74d84bd6673dd4c2bbfd924715f7cf7402d3ba76e50572242df" + ], + "sizeBytes": 75121018 + }, + { + "names": [ + "ghcr.io/copilotkit/intelligence/db-migrations@sha256:9c2eedb0f1dbd140d26a8542693c4e73e76338b9264630724e1dcb767014c858" + ], + "sizeBytes": 75120614 + }, + { + "names": [ + "docker.io/rancher/klipper-helm@sha256:73ff7ef399717ba8339559054557bd427bdafb47db112165a8c0c358d1ca0283", + "docker.io/rancher/klipper-helm:v0.9.3-build20241008" + ], + "sizeBytes": 65893717 + }, + { + "names": [ + "docker.io/bitnamilegacy/redis@sha256:a25b5d07a14ec13730022c7cd9bab6308d55ccd86b74af7315553c17be884889" + ], + "sizeBytes": 50512298 + }, + { + "names": [ + "ghcr.io/copilotkit/intelligence/local-object-storage-init@sha256:1c2a56bcd21e6ef33b77ddefec998ba09b026c339af1c2827d312ee3df7b7280", + "ghcr.io/copilotkit/intelligence/local-object-storage@sha256:1c2a56bcd21e6ef33b77ddefec998ba09b026c339af1c2827d312ee3df7b7280" + ], + "sizeBytes": 49169049 + }, + { + "names": [ + "docker.io/rancher/mirrored-library-traefik@sha256:25df7bff0b414867f16a74c571c0dc84920404e45cc7780976cec77809230e09", + "docker.io/rancher/mirrored-library-traefik:2.11.18" + ], + "sizeBytes": 46474327 + }, + { + "names": [ + "ghcr.io/copilotkit/intelligence/app-frontend@sha256:3432ef79640850e6db3b4418b60e3c85987268aaeac72c82e4f9eec62e6d73e0" + ], + "sizeBytes": 27977381 + }, + { + "names": [ + "ghcr.io/copilotkit/intelligence/app-frontend@sha256:ac554fdffa0eff4ee185642ac4dc7f8ee3fd4e0f479f6f279d1499807587f130" + ], + "sizeBytes": 27977374 + }, + { + "names": [ + "ghcr.io/copilotkit/intelligence/realtime-gateway@sha256:c005749db55b2a415e7c09254d673ccf582faea9d583848f27caa9c574b018f4" + ], + "sizeBytes": 21753907 + }, + { + "names": [ + "ghcr.io/copilotkit/intelligence/realtime-gateway@sha256:5131c1da79bb9237d9171aa4cb19643425d53a6154ada3de98b21a8caee800c8" + ], + "sizeBytes": 21753888 + }, + { + "names": [ + "docker.io/rancher/mirrored-coredns-coredns@sha256:82979ddf442c593027a57239ad90616deb874e90c365d1a96ad508c2104bdea5", + "docker.io/rancher/mirrored-coredns-coredns:1.12.0" + ], + "sizeBytes": 19148178 + }, + { + "names": [ + "docker.io/rancher/mirrored-metrics-server@sha256:dccf8474fb910fef261d31d9483d7e4c1df7b86cf4d638fb6a7d7c88bd51600a", + "docker.io/rancher/mirrored-metrics-server:v0.7.2" + ], + "sizeBytes": 17862421 + }, + { + "names": [ + "docker.io/rancher/local-path-provisioner@sha256:9b914881170048f80ae9302f36e5b99b4a6b18af73a38adc1c66d12f65d360be", + "docker.io/rancher/local-path-provisioner:v0.0.30" + ], + "sizeBytes": 17807097 + }, + { + "names": [ + "docker.io/rancher/klipper-lb@sha256:dd380f5d89a52f2a07853ff17a6048f805c1f8113b50578f3efc3efb9bcf670a", + "docker.io/rancher/klipper-lb:v0.4.9" + ], + "sizeBytes": 5678915 + }, + { + "names": [ + "docker.io/rancher/mirrored-library-busybox@sha256:8a45424ddf949bbe9bb3231b05f9032a45da5cd036eb4867b511b00734756d6f", + "docker.io/rancher/mirrored-library-busybox:1.36.1" + ], + "sizeBytes": 1847165 + }, + { + "names": [ + "docker.io/rancher/mirrored-pause@sha256:74c4244427b7312c5b901fe0f67cbc53683d06f4f24c6faee65d4182bf0fa893", + "docker.io/rancher/mirrored-pause:3.6" + ], + "sizeBytes": 253243 + } + ], + "nodeInfo": { + "architecture": "arm64", + "bootID": "bda7140e-2723-49f3-ad3b-4bf341290510", + "containerRuntimeVersion": "containerd://1.7.23-k3s2", + "kernelVersion": "6.12.76-linuxkit", + "kubeProxyVersion": "v1.31.5+k3s1", + "kubeletVersion": "v1.31.5+k3s1", + "machineID": "", + "operatingSystem": "linux", + "osImage": "K3s v1.31.5+k3s1", + "systemUUID": "" + } + } + } + ], + "kind": "List", + "metadata": { + "resourceVersion": "" + } +} diff --git a/desktop/src/LocalIntelligence.test.tsx b/desktop/src/LocalIntelligence.test.tsx index 60f7b5a2da..0259497c69 100644 --- a/desktop/src/LocalIntelligence.test.tsx +++ b/desktop/src/LocalIntelligence.test.tsx @@ -1101,6 +1101,45 @@ test("a stopped stack is started, then polled", async () => { ]); }); +/** + * 1.0.0 on Windows: a used install relaunched with its back end's disk nearly full said only "The + * back end stopped with an error partway through." The start the window runs on a stopped stack now + * comes back from `local_eval::checked_call` naming the disk, as `local_eval.rs` reports it. + */ +test("a relaunch whose start fails on a nearly full disk names the disk on its row, with Retry", async () => { + const stopped = report({ readiness: "stopped", licence: "valid" }); + invokeHandler = async (command, args) => { + const action = (args as { action?: string }).action; + if (command === "run_local_eval" && action === "start") { + return report({ + outcome: "failed", + stack: null, + reason: "resources-short", + shortfall: { + podman: true, + cpu: false, + memory: false, + disk: true, + diskGb: "30.5", + nearlyFull: true, + diskSizeGb: "1006.8", + drive: "C:", + }, + }); + } + if (command === "run_local_eval") return stopped; + throw new Error(`unexpected ${command}`); + }; + const screen = view({ stack: "local-abc123", connected: true }); + await screen.findByRole("button", { name: "Retry" }); + const waiting = row(screen, WAIT).textContent; + expect(waiting).toContain( + "The back end's disk is nearly full. Podman has 30.5 GB free of 1006.8 GB. OpenBot needs 40 GB. Free space on drive C: or in Podman, then Retry.", + ); + expect(waiting).not.toContain("partway through"); + expect(commands()).toEqual(["run_local_eval:status", "run_local_eval:start"]); +}); + test("a start that fails stops at its row, and Retry starts again without asking the CLI", async () => { invokeHandler = answers({ run_local_eval: [ready] }); let starts = 0; diff --git a/desktop/src/localEvalCli.ts b/desktop/src/localEvalCli.ts index dff5e699c3..854f945fc0 100644 --- a/desktop/src/localEvalCli.ts +++ b/desktop/src/localEvalCli.ts @@ -90,6 +90,14 @@ export type Shortfall = { cpus?: string; memoryGb?: string; diskGb?: string; + /** + * OpenBot's own check after a start that failed: the back end's disk is nearly full, and how + * large it is beside `diskGb`. + */ + nearlyFull?: boolean; + diskSizeGb?: string; + /** Podman on Windows: the drive that holds its machine's disk, `C:`. */ + drive?: string; }; export type LocalReport = { diff --git a/desktop/src/localReadiness.test.ts b/desktop/src/localReadiness.test.ts index 602bee84c1..9e737211ff 100644 --- a/desktop/src/localReadiness.test.ts +++ b/desktop/src/localReadiness.test.ts @@ -8,7 +8,11 @@ import { localFailures, type Shortfall, } from "./localEvalCli"; -import { localReadiness, type ReadinessInput } from "./localReadiness"; +import { + engineTooSmallSaid, + localReadiness, + type ReadinessInput, +} from "./localReadiness"; const reasons: LocalReason[] = [ "authentication-required", @@ -227,6 +231,62 @@ test("specific wording and actions", () => { ).toBe( "Docker has 3.5 free CPUs and 12 GB of free disk. OpenBot needs 4 free CPUs and 40 GB of free disk. Raise the limits in Docker Desktop > Settings > Resources, or free space in Docker, then Retry.", ); + // OpenBot's own check after a start that failed (`local_eval::checked_call`), as it reports + // each case. A nearly full disk is said as k3s's, beside the 40 GB only when it is below it. + expect( + short({ + podman: true, + disk: true, + diskGb: "30.5", + nearlyFull: true, + diskSizeGb: "1006.8", + drive: "C:", + }).said, + ).toBe( + "The back end's disk is nearly full. Podman has 30.5 GB free of 1006.8 GB. OpenBot needs 40 GB. Free space on drive C: or in Podman, then Retry.", + ); + expect( + short({ podman: true, disk: true, diskGb: "30", drive: "C:" }).said, + ).toBe( + "Podman has 30 GB of free disk. OpenBot needs 40 GB. Free space on drive C: or in Podman, then Retry.", + ); + expect( + short({ + disk: true, + diskGb: "300", + nearlyFull: true, + diskSizeGb: "1827.9", + }).said, + ).toBe( + "The back end's disk is nearly full. Docker has 300 GB free of 1827.9 GB. Raise the disk limit in Docker Desktop > Settings > Resources, or free space in Docker, then Retry.", + ); + expect( + short({ + podman: true, + disk: true, + diskGb: "8", + nearlyFull: true, + diskSizeGb: "60", + }).said, + ).toBe( + "The back end's disk is nearly full. Podman has 8 GB free of 60 GB. OpenBot needs 40 GB. Stop your Podman machine, run podman machine set --disk-size 160, start it again, then Retry.", + ); + // Podman on Windows short of CPUs too: those are raised, the drive's space is freed. + expect(short({ podman: true, cpu: true, disk: true, drive: "C:" }).said).toBe( + "Podman doesn't have enough CPUs and disk space. OpenBot needs 4 free CPUs and 40 GB free. Stop your Podman machine, run podman machine set --cpus 6, start it again, and free space on drive C: or in Podman, then Retry.", + ); + // The account step's check before a first setup, on Windows. + expect( + engineTooSmallSaid({ + podman: true, + cpu: false, + memory: false, + disk: true, + drive: "C:", + }), + ).toBe( + "Podman is out of disk space. OpenBot needs 40 GB free. Free space on drive C: or in Podman, then Try again.", + ); // A report that does not say which is still about resources, never "partway through". expect( said(report({ outcome: "failed", reason: "resources-short" })).said, diff --git a/desktop/src/localReadiness.ts b/desktop/src/localReadiness.ts index 8b3a1aa3dd..825fc0b2fc 100644 --- a/desktop/src/localReadiness.ts +++ b/desktop/src/localReadiness.ts @@ -141,8 +141,19 @@ function shortfallSaid( return `${engine} doesn't have enough CPUs, memory or disk space for the back end. ${fix}, then ${then}.`; } const fix = shortfall.podman - ? `Stop your Podman machine, run podman machine set ${short.map((resource) => podmanFlag[resource]).join(" ")}, start it again` + ? podmanFix(short, shortfall.drive) : `Raise the ${short.length === 1 ? `${limitName[short[0]]} limit` : "limits"} in Docker Desktop > Settings > Resources${shortfall.disk ? ", or free space in Docker" : ""}`; + // OpenBot's own check after a start that failed: k3s itself is short of disk. + if (shortfall.nearlyFull && short.length === 1) { + const free = shortfall.diskGb + ? ` ${engine} has ${shortfall.diskGb} GB free${shortfall.diskSizeGb ? ` of ${shortfall.diskSizeGb} GB` : ""}.` + : ""; + const needs40 = + shortfall.diskGb && Number(shortfall.diskGb) < 40 + ? ` OpenBot needs ${needed.disk}.` + : ""; + return `The back end's disk is nearly full.${free}${needs40} ${fix}, then ${then}.`; + } // The CLI's own numbers, when its error gave one for every resource that is short. const measured: Record = { cpu: shortfall.cpus && `${shortfall.cpus} free CPUs`, @@ -159,7 +170,28 @@ function shortfallSaid( : short.length === 1 && short[0] === "disk" ? `${engine} is out of disk space. OpenBot needs ${needs.disk}.` : `${engine} doesn't have enough ${listed(short.map((resource) => resourceName[resource]))}. OpenBot needs ${listed(short.map((resource) => needs[resource]))}.`; - return `${said} ${fix}, then ${then}.`; + const full = shortfall.nearlyFull + ? "The back end's disk is nearly full. " + : ""; + return `${full}${said} ${fix}, then ${then}.`; +} + +/** + * Where Podman raises what is short. On Windows its machine's disk is a sparse virtual disk on + * `drive`, and `podman machine set` does not change it, so space is freed there instead. + */ +function podmanFix(short: Resource[], drive: string | undefined): string { + const raised = short.filter((resource) => !(drive && resource === "disk")); + const set = + raised.length > 0 && + `Stop your Podman machine, run podman machine set ${raised.map((resource) => podmanFlag[resource]).join(" ")}, start it again`; + const freed = + drive && + short.includes("disk") && + `free space on drive ${drive} or in Podman`; + if (set && freed) return `${set}, and ${freed}`; + if (freed) return freed.charAt(0).toUpperCase() + freed.slice(1); + return set || ""; } // What the CLI requires free, beside its own numbers: "Docker has 0 GB of free disk. OpenBot From 9abc2e2eccd09786d9bd8f0597721f92e16c58cc Mon Sep 17 00:00:00 2001 From: David McKay Date: Mon, 9 Nov 2026 04:21:17 -0800 Subject: [PATCH 2/4] Format the captured k3s node fixture as biome does biome format checks JSON fixtures too. The content is the same kubectl output; only its indentation changes. --- .../local-eval/k3s-v1.31.5-get-nodes.json | 524 +++++++++--------- 1 file changed, 260 insertions(+), 264 deletions(-) diff --git a/desktop/src-tauri/test-fixtures/local-eval/k3s-v1.31.5-get-nodes.json b/desktop/src-tauri/test-fixtures/local-eval/k3s-v1.31.5-get-nodes.json index 5ba5339b63..00faaafbfe 100644 --- a/desktop/src-tauri/test-fixtures/local-eval/k3s-v1.31.5-get-nodes.json +++ b/desktop/src-tauri/test-fixtures/local-eval/k3s-v1.31.5-get-nodes.json @@ -1,268 +1,264 @@ { - "apiVersion": "v1", - "items": [ - { - "apiVersion": "v1", - "kind": "Node", - "metadata": { - "annotations": { - "alpha.kubernetes.io/provided-node-ip": "172.27.0.2", - "flannel.alpha.coreos.com/backend-data": "{\"VNI\":1,\"VtepMAC\":\"52:3a:21:98:27:b8\"}", - "flannel.alpha.coreos.com/backend-type": "vxlan", - "flannel.alpha.coreos.com/kube-subnet-manager": "true", - "flannel.alpha.coreos.com/public-ip": "172.27.0.2", - "k3s.io/hostname": "k3d-local-e0cf90debea88453ad454b93-server-0", - "k3s.io/internal-ip": "172.27.0.2", - "k3s.io/node-args": "[\"server\",\"--disable\",\"traefik\",\"--tls-san\",\"127.0.0.1\"]", - "k3s.io/node-config-hash": "MXCMP4EPPR7HGKB2OXMJELW7CFJQH6IT4ZP54RPNISYBXKEKV6WA====", - "k3s.io/node-env": "{\"K3S_KUBECONFIG_OUTPUT\":\"/output/kubeconfig.yaml\",\"K3S_TOKEN\":\"********\"}", - "node.alpha.kubernetes.io/ttl": "0", - "volumes.kubernetes.io/controller-managed-attach-detach": "true" - }, - "creationTimestamp": "2026-11-07T09:46:42Z", - "finalizers": [ - "wrangler.cattle.io/node" - ], - "labels": { - "beta.kubernetes.io/arch": "arm64", - "beta.kubernetes.io/instance-type": "k3s", - "beta.kubernetes.io/os": "linux", - "kubernetes.io/arch": "arm64", - "kubernetes.io/hostname": "k3d-local-e0cf90debea88453ad454b93-server-0", - "kubernetes.io/os": "linux", - "node-role.kubernetes.io/control-plane": "true", - "node-role.kubernetes.io/master": "true", - "node.kubernetes.io/instance-type": "k3s" - }, - "name": "k3d-local-e0cf90debea88453ad454b93-server-0", - "resourceVersion": "105240", - "uid": "71b0e4ba-c3f1-4a91-b493-88743fc7f270" - }, - "spec": { - "podCIDR": "10.42.0.0/24", - "podCIDRs": [ - "10.42.0.0/24" - ], - "providerID": "k3s://k3d-local-e0cf90debea88453ad454b93-server-0" - }, - "status": { - "addresses": [ - { - "address": "172.27.0.2", - "type": "InternalIP" - }, - { - "address": "k3d-local-e0cf90debea88453ad454b93-server-0", - "type": "Hostname" - } - ], - "allocatable": { - "cpu": "18", - "ephemeral-storage": "1864559929521", - "hugepages-1Gi": "0", - "hugepages-2Mi": "0", - "hugepages-32Mi": "0", - "hugepages-64Ki": "0", - "memory": "16353252Ki", - "pods": "110" - }, - "capacity": { - "cpu": "18", - "ephemeral-storage": "1916694008Ki", - "hugepages-1Gi": "0", - "hugepages-2Mi": "0", - "hugepages-32Mi": "0", - "hugepages-64Ki": "0", - "memory": "16353252Ki", - "pods": "110" - }, - "conditions": [ - { - "lastHeartbeatTime": "2026-11-09T12:03:04Z", - "lastTransitionTime": "2026-11-08T05:15:10Z", - "message": "kubelet has sufficient memory available", - "reason": "KubeletHasSufficientMemory", - "status": "False", - "type": "MemoryPressure" - }, - { - "lastHeartbeatTime": "2026-11-09T12:03:04Z", - "lastTransitionTime": "2026-11-08T05:15:10Z", - "message": "kubelet has no disk pressure", - "reason": "KubeletHasNoDiskPressure", - "status": "False", - "type": "DiskPressure" - }, - { - "lastHeartbeatTime": "2026-11-09T12:03:04Z", - "lastTransitionTime": "2026-11-08T05:15:10Z", - "message": "kubelet has sufficient PID available", - "reason": "KubeletHasSufficientPID", - "status": "False", - "type": "PIDPressure" - }, - { - "lastHeartbeatTime": "2026-11-09T12:03:04Z", - "lastTransitionTime": "2026-11-08T05:15:10Z", - "message": "kubelet is posting ready status", - "reason": "KubeletReady", - "status": "True", - "type": "Ready" - } - ], - "daemonEndpoints": { - "kubeletEndpoint": { - "Port": 10250 - } - }, - "images": [ - { - "names": [ - "ghcr.io/copilotkit/intelligence/app-api@sha256:060ae0f883786612d064ebbb8d081b83006e9e538a936b74e383ecd3dd81a844" - ], - "sizeBytes": 147210711 - }, - { - "names": [ - "ghcr.io/copilotkit/intelligence/app-api@sha256:9ee3006d92a656a72be43f9bda0e0738aec910f3b8098aba8e029f28061e85b7" - ], - "sizeBytes": 147206537 - }, - { - "names": [ - "ghcr.io/copilotkit/intelligence/postgres-pgvector@sha256:3ec79ad28b8ba356734e1f29d757d5e1f5f4104360e2d285a63db1bb6a0cfe28" - ], - "sizeBytes": 108486056 - }, - { - "names": [ - "ghcr.io/copilotkit/intelligence/postgres-pgvector@sha256:f98d6b61014200ab2b946720c84d4693ecd2f68abbe7d00779a57729021d03de" - ], - "sizeBytes": 108486053 - }, - { - "names": [ - "ghcr.io/copilotkit/intelligence/db-migrations@sha256:ba649b6358d6e74d84bd6673dd4c2bbfd924715f7cf7402d3ba76e50572242df" - ], - "sizeBytes": 75121018 - }, - { - "names": [ - "ghcr.io/copilotkit/intelligence/db-migrations@sha256:9c2eedb0f1dbd140d26a8542693c4e73e76338b9264630724e1dcb767014c858" - ], - "sizeBytes": 75120614 - }, - { - "names": [ - "docker.io/rancher/klipper-helm@sha256:73ff7ef399717ba8339559054557bd427bdafb47db112165a8c0c358d1ca0283", - "docker.io/rancher/klipper-helm:v0.9.3-build20241008" - ], - "sizeBytes": 65893717 - }, - { - "names": [ - "docker.io/bitnamilegacy/redis@sha256:a25b5d07a14ec13730022c7cd9bab6308d55ccd86b74af7315553c17be884889" - ], - "sizeBytes": 50512298 - }, - { - "names": [ - "ghcr.io/copilotkit/intelligence/local-object-storage-init@sha256:1c2a56bcd21e6ef33b77ddefec998ba09b026c339af1c2827d312ee3df7b7280", - "ghcr.io/copilotkit/intelligence/local-object-storage@sha256:1c2a56bcd21e6ef33b77ddefec998ba09b026c339af1c2827d312ee3df7b7280" - ], - "sizeBytes": 49169049 - }, - { - "names": [ - "docker.io/rancher/mirrored-library-traefik@sha256:25df7bff0b414867f16a74c571c0dc84920404e45cc7780976cec77809230e09", - "docker.io/rancher/mirrored-library-traefik:2.11.18" - ], - "sizeBytes": 46474327 - }, - { - "names": [ - "ghcr.io/copilotkit/intelligence/app-frontend@sha256:3432ef79640850e6db3b4418b60e3c85987268aaeac72c82e4f9eec62e6d73e0" - ], - "sizeBytes": 27977381 - }, - { - "names": [ - "ghcr.io/copilotkit/intelligence/app-frontend@sha256:ac554fdffa0eff4ee185642ac4dc7f8ee3fd4e0f479f6f279d1499807587f130" - ], - "sizeBytes": 27977374 - }, - { - "names": [ - "ghcr.io/copilotkit/intelligence/realtime-gateway@sha256:c005749db55b2a415e7c09254d673ccf582faea9d583848f27caa9c574b018f4" - ], - "sizeBytes": 21753907 - }, - { - "names": [ - "ghcr.io/copilotkit/intelligence/realtime-gateway@sha256:5131c1da79bb9237d9171aa4cb19643425d53a6154ada3de98b21a8caee800c8" - ], - "sizeBytes": 21753888 - }, - { - "names": [ - "docker.io/rancher/mirrored-coredns-coredns@sha256:82979ddf442c593027a57239ad90616deb874e90c365d1a96ad508c2104bdea5", - "docker.io/rancher/mirrored-coredns-coredns:1.12.0" - ], - "sizeBytes": 19148178 - }, - { - "names": [ - "docker.io/rancher/mirrored-metrics-server@sha256:dccf8474fb910fef261d31d9483d7e4c1df7b86cf4d638fb6a7d7c88bd51600a", - "docker.io/rancher/mirrored-metrics-server:v0.7.2" - ], - "sizeBytes": 17862421 - }, - { - "names": [ - "docker.io/rancher/local-path-provisioner@sha256:9b914881170048f80ae9302f36e5b99b4a6b18af73a38adc1c66d12f65d360be", - "docker.io/rancher/local-path-provisioner:v0.0.30" - ], - "sizeBytes": 17807097 - }, - { - "names": [ - "docker.io/rancher/klipper-lb@sha256:dd380f5d89a52f2a07853ff17a6048f805c1f8113b50578f3efc3efb9bcf670a", - "docker.io/rancher/klipper-lb:v0.4.9" - ], - "sizeBytes": 5678915 - }, - { - "names": [ - "docker.io/rancher/mirrored-library-busybox@sha256:8a45424ddf949bbe9bb3231b05f9032a45da5cd036eb4867b511b00734756d6f", - "docker.io/rancher/mirrored-library-busybox:1.36.1" - ], - "sizeBytes": 1847165 - }, - { - "names": [ - "docker.io/rancher/mirrored-pause@sha256:74c4244427b7312c5b901fe0f67cbc53683d06f4f24c6faee65d4182bf0fa893", - "docker.io/rancher/mirrored-pause:3.6" - ], - "sizeBytes": 253243 - } - ], - "nodeInfo": { - "architecture": "arm64", - "bootID": "bda7140e-2723-49f3-ad3b-4bf341290510", - "containerRuntimeVersion": "containerd://1.7.23-k3s2", - "kernelVersion": "6.12.76-linuxkit", - "kubeProxyVersion": "v1.31.5+k3s1", - "kubeletVersion": "v1.31.5+k3s1", - "machineID": "", - "operatingSystem": "linux", - "osImage": "K3s v1.31.5+k3s1", - "systemUUID": "" - } - } + "apiVersion": "v1", + "items": [ + { + "apiVersion": "v1", + "kind": "Node", + "metadata": { + "annotations": { + "alpha.kubernetes.io/provided-node-ip": "172.27.0.2", + "flannel.alpha.coreos.com/backend-data": "{\"VNI\":1,\"VtepMAC\":\"52:3a:21:98:27:b8\"}", + "flannel.alpha.coreos.com/backend-type": "vxlan", + "flannel.alpha.coreos.com/kube-subnet-manager": "true", + "flannel.alpha.coreos.com/public-ip": "172.27.0.2", + "k3s.io/hostname": "k3d-local-e0cf90debea88453ad454b93-server-0", + "k3s.io/internal-ip": "172.27.0.2", + "k3s.io/node-args": "[\"server\",\"--disable\",\"traefik\",\"--tls-san\",\"127.0.0.1\"]", + "k3s.io/node-config-hash": "MXCMP4EPPR7HGKB2OXMJELW7CFJQH6IT4ZP54RPNISYBXKEKV6WA====", + "k3s.io/node-env": "{\"K3S_KUBECONFIG_OUTPUT\":\"/output/kubeconfig.yaml\",\"K3S_TOKEN\":\"********\"}", + "node.alpha.kubernetes.io/ttl": "0", + "volumes.kubernetes.io/controller-managed-attach-detach": "true" + }, + "creationTimestamp": "2026-11-07T09:46:42Z", + "finalizers": ["wrangler.cattle.io/node"], + "labels": { + "beta.kubernetes.io/arch": "arm64", + "beta.kubernetes.io/instance-type": "k3s", + "beta.kubernetes.io/os": "linux", + "kubernetes.io/arch": "arm64", + "kubernetes.io/hostname": "k3d-local-e0cf90debea88453ad454b93-server-0", + "kubernetes.io/os": "linux", + "node-role.kubernetes.io/control-plane": "true", + "node-role.kubernetes.io/master": "true", + "node.kubernetes.io/instance-type": "k3s" + }, + "name": "k3d-local-e0cf90debea88453ad454b93-server-0", + "resourceVersion": "105240", + "uid": "71b0e4ba-c3f1-4a91-b493-88743fc7f270" + }, + "spec": { + "podCIDR": "10.42.0.0/24", + "podCIDRs": ["10.42.0.0/24"], + "providerID": "k3s://k3d-local-e0cf90debea88453ad454b93-server-0" + }, + "status": { + "addresses": [ + { + "address": "172.27.0.2", + "type": "InternalIP" + }, + { + "address": "k3d-local-e0cf90debea88453ad454b93-server-0", + "type": "Hostname" + } + ], + "allocatable": { + "cpu": "18", + "ephemeral-storage": "1864559929521", + "hugepages-1Gi": "0", + "hugepages-2Mi": "0", + "hugepages-32Mi": "0", + "hugepages-64Ki": "0", + "memory": "16353252Ki", + "pods": "110" + }, + "capacity": { + "cpu": "18", + "ephemeral-storage": "1916694008Ki", + "hugepages-1Gi": "0", + "hugepages-2Mi": "0", + "hugepages-32Mi": "0", + "hugepages-64Ki": "0", + "memory": "16353252Ki", + "pods": "110" + }, + "conditions": [ + { + "lastHeartbeatTime": "2026-11-09T12:03:04Z", + "lastTransitionTime": "2026-11-08T05:15:10Z", + "message": "kubelet has sufficient memory available", + "reason": "KubeletHasSufficientMemory", + "status": "False", + "type": "MemoryPressure" + }, + { + "lastHeartbeatTime": "2026-11-09T12:03:04Z", + "lastTransitionTime": "2026-11-08T05:15:10Z", + "message": "kubelet has no disk pressure", + "reason": "KubeletHasNoDiskPressure", + "status": "False", + "type": "DiskPressure" + }, + { + "lastHeartbeatTime": "2026-11-09T12:03:04Z", + "lastTransitionTime": "2026-11-08T05:15:10Z", + "message": "kubelet has sufficient PID available", + "reason": "KubeletHasSufficientPID", + "status": "False", + "type": "PIDPressure" + }, + { + "lastHeartbeatTime": "2026-11-09T12:03:04Z", + "lastTransitionTime": "2026-11-08T05:15:10Z", + "message": "kubelet is posting ready status", + "reason": "KubeletReady", + "status": "True", + "type": "Ready" + } + ], + "daemonEndpoints": { + "kubeletEndpoint": { + "Port": 10250 + } + }, + "images": [ + { + "names": [ + "ghcr.io/copilotkit/intelligence/app-api@sha256:060ae0f883786612d064ebbb8d081b83006e9e538a936b74e383ecd3dd81a844" + ], + "sizeBytes": 147210711 + }, + { + "names": [ + "ghcr.io/copilotkit/intelligence/app-api@sha256:9ee3006d92a656a72be43f9bda0e0738aec910f3b8098aba8e029f28061e85b7" + ], + "sizeBytes": 147206537 + }, + { + "names": [ + "ghcr.io/copilotkit/intelligence/postgres-pgvector@sha256:3ec79ad28b8ba356734e1f29d757d5e1f5f4104360e2d285a63db1bb6a0cfe28" + ], + "sizeBytes": 108486056 + }, + { + "names": [ + "ghcr.io/copilotkit/intelligence/postgres-pgvector@sha256:f98d6b61014200ab2b946720c84d4693ecd2f68abbe7d00779a57729021d03de" + ], + "sizeBytes": 108486053 + }, + { + "names": [ + "ghcr.io/copilotkit/intelligence/db-migrations@sha256:ba649b6358d6e74d84bd6673dd4c2bbfd924715f7cf7402d3ba76e50572242df" + ], + "sizeBytes": 75121018 + }, + { + "names": [ + "ghcr.io/copilotkit/intelligence/db-migrations@sha256:9c2eedb0f1dbd140d26a8542693c4e73e76338b9264630724e1dcb767014c858" + ], + "sizeBytes": 75120614 + }, + { + "names": [ + "docker.io/rancher/klipper-helm@sha256:73ff7ef399717ba8339559054557bd427bdafb47db112165a8c0c358d1ca0283", + "docker.io/rancher/klipper-helm:v0.9.3-build20241008" + ], + "sizeBytes": 65893717 + }, + { + "names": [ + "docker.io/bitnamilegacy/redis@sha256:a25b5d07a14ec13730022c7cd9bab6308d55ccd86b74af7315553c17be884889" + ], + "sizeBytes": 50512298 + }, + { + "names": [ + "ghcr.io/copilotkit/intelligence/local-object-storage-init@sha256:1c2a56bcd21e6ef33b77ddefec998ba09b026c339af1c2827d312ee3df7b7280", + "ghcr.io/copilotkit/intelligence/local-object-storage@sha256:1c2a56bcd21e6ef33b77ddefec998ba09b026c339af1c2827d312ee3df7b7280" + ], + "sizeBytes": 49169049 + }, + { + "names": [ + "docker.io/rancher/mirrored-library-traefik@sha256:25df7bff0b414867f16a74c571c0dc84920404e45cc7780976cec77809230e09", + "docker.io/rancher/mirrored-library-traefik:2.11.18" + ], + "sizeBytes": 46474327 + }, + { + "names": [ + "ghcr.io/copilotkit/intelligence/app-frontend@sha256:3432ef79640850e6db3b4418b60e3c85987268aaeac72c82e4f9eec62e6d73e0" + ], + "sizeBytes": 27977381 + }, + { + "names": [ + "ghcr.io/copilotkit/intelligence/app-frontend@sha256:ac554fdffa0eff4ee185642ac4dc7f8ee3fd4e0f479f6f279d1499807587f130" + ], + "sizeBytes": 27977374 + }, + { + "names": [ + "ghcr.io/copilotkit/intelligence/realtime-gateway@sha256:c005749db55b2a415e7c09254d673ccf582faea9d583848f27caa9c574b018f4" + ], + "sizeBytes": 21753907 + }, + { + "names": [ + "ghcr.io/copilotkit/intelligence/realtime-gateway@sha256:5131c1da79bb9237d9171aa4cb19643425d53a6154ada3de98b21a8caee800c8" + ], + "sizeBytes": 21753888 + }, + { + "names": [ + "docker.io/rancher/mirrored-coredns-coredns@sha256:82979ddf442c593027a57239ad90616deb874e90c365d1a96ad508c2104bdea5", + "docker.io/rancher/mirrored-coredns-coredns:1.12.0" + ], + "sizeBytes": 19148178 + }, + { + "names": [ + "docker.io/rancher/mirrored-metrics-server@sha256:dccf8474fb910fef261d31d9483d7e4c1df7b86cf4d638fb6a7d7c88bd51600a", + "docker.io/rancher/mirrored-metrics-server:v0.7.2" + ], + "sizeBytes": 17862421 + }, + { + "names": [ + "docker.io/rancher/local-path-provisioner@sha256:9b914881170048f80ae9302f36e5b99b4a6b18af73a38adc1c66d12f65d360be", + "docker.io/rancher/local-path-provisioner:v0.0.30" + ], + "sizeBytes": 17807097 + }, + { + "names": [ + "docker.io/rancher/klipper-lb@sha256:dd380f5d89a52f2a07853ff17a6048f805c1f8113b50578f3efc3efb9bcf670a", + "docker.io/rancher/klipper-lb:v0.4.9" + ], + "sizeBytes": 5678915 + }, + { + "names": [ + "docker.io/rancher/mirrored-library-busybox@sha256:8a45424ddf949bbe9bb3231b05f9032a45da5cd036eb4867b511b00734756d6f", + "docker.io/rancher/mirrored-library-busybox:1.36.1" + ], + "sizeBytes": 1847165 + }, + { + "names": [ + "docker.io/rancher/mirrored-pause@sha256:74c4244427b7312c5b901fe0f67cbc53683d06f4f24c6faee65d4182bf0fa893", + "docker.io/rancher/mirrored-pause:3.6" + ], + "sizeBytes": 253243 + } + ], + "nodeInfo": { + "architecture": "arm64", + "bootID": "bda7140e-2723-49f3-ad3b-4bf341290510", + "containerRuntimeVersion": "containerd://1.7.23-k3s2", + "kernelVersion": "6.12.76-linuxkit", + "kubeProxyVersion": "v1.31.5+k3s1", + "kubeletVersion": "v1.31.5+k3s1", + "machineID": "", + "operatingSystem": "linux", + "osImage": "K3s v1.31.5+k3s1", + "systemUUID": "" } - ], - "kind": "List", - "metadata": { - "resourceVersion": "" + } } + ], + "kind": "List", + "metadata": { + "resourceVersion": "" + } } From 222404e9cab7541b4d339da44b173673fa9f0a36 Mon Sep 17 00:00:00 2001 From: David McKay Date: Mon, 9 Nov 2026 04:28:43 -0800 Subject: [PATCH 3/4] Say to free the engine's drive on Windows, and that a back end recovering from low disk needs a few minutes Podman refuses to change a WSL machine's disk ("changing disk size not supported for WSL machines"), so on Windows the card never suggests podman machine set for it, even when WSL did not say which drive holds the disk. It says to free space on the drive that holds OpenBot's engine, with the drive when known. After space is freed, k3s keeps its DiskPressure condition for about five minutes, and a Retry in those minutes failed again. A start that fails while the node still reports DiskPressure, with the disk above both lines, now says "The back end is recovering from low disk space. Wait a few minutes, then Retry." instead of asking for more space. --- CHANGELOG.md | 9 +- desktop/src-tauri/src/local_eval.rs | 113 +++++++++++++++++++++---- desktop/src/LocalIntelligence.test.tsx | 3 +- desktop/src/localEvalCli.ts | 9 +- desktop/src/localReadiness.test.ts | 32 ++++--- desktop/src/localReadiness.ts | 21 +++-- 6 files changed, 148 insertions(+), 39 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index b57f51b55f..8d3dd671e8 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -25,9 +25,12 @@ Newest first. `Unreleased` is what is on `main` and not yet tagged. partway through." Each start, update and setup now checks the engine first and logs what it has as `[local check]`, and a start that fails is checked again with the back end asked whether it is short of disk. The card then says, for example, "The back end's disk is nearly full. Podman has - 30.5 GB free of 1006.8 GB. OpenBot needs 40 GB. Free space on drive C: or in Podman, then Retry." - On Windows, free disk is measured on the drive that holds Podman's machine, which can run out - while the machine itself shows hundreds of GB free. + 30.5 GB free of 1006.8 GB. OpenBot needs 40 GB. Free space on the drive that holds OpenBot's + engine (C:), then Retry." On Windows, free disk is measured on that drive, which can run out + while the machine itself shows hundreds of GB free, and the card never suggests + `podman machine set`, which WSL machines refuse. A start that fails in the few minutes after + space was freed, while the back end still reports low disk, says "The back end is recovering + from low disk space. Wait a few minutes, then Retry." ## 1.0.0 diff --git a/desktop/src-tauri/src/local_eval.rs b/desktop/src-tauri/src/local_eval.rs index ddf9f280c5..9a27c3a6e6 100644 --- a/desktop/src-tauri/src/local_eval.rs +++ b/desktop/src-tauri/src/local_eval.rs @@ -277,8 +277,15 @@ pub struct Shortfall { /// How large that disk is, in GiB to one decimal, beside `disk_gb`, when it is nearly full. #[serde(skip_serializing_if = "Option::is_none")] pub disk_size_gb: Option, - /// Podman on Windows: the drive that holds its machine's disk, `C:`. Space is freed there, not - /// raised with `podman machine set`. + /// The node still reports `DiskPressure` with the disk above both lines: k3s keeps the + /// condition for five minutes after space is freed, and the back end recovers by itself. + #[serde(skip_serializing_if = "std::ops::Not::not")] + pub recovering: bool, + /// Podman on Windows, whose machine runs in WSL: `podman machine set` changes none of its + /// resources there, and its disk is freed on the drive that holds it. + #[serde(skip_serializing_if = "std::ops::Not::not")] + pub wsl: bool, + /// That drive, `C:`, when WSL says where the machine's disk is. #[serde(skip_serializing_if = "Option::is_none")] pub drive: Option, } @@ -292,7 +299,7 @@ fn shown(value: f64) -> String { impl Shortfall { fn named(&self) -> bool { - self.cpu || self.memory || self.disk + self.cpu || self.memory || self.disk || self.recovering } } @@ -857,7 +864,9 @@ pub enum Terms { cpu: bool, memory: bool, disk: bool, - /// Podman on Windows: the drive that holds its machine's disk (`Shortfall::drive`). + /// Podman on Windows, in WSL (`Shortfall::wsl`), and the drive that holds its disk. + #[serde(skip_serializing_if = "std::ops::Not::not")] + wsl: bool, #[serde(skip_serializing_if = "Option::is_none")] drive: Option, }, @@ -1375,10 +1384,34 @@ impl EngineCheck { 85). A relaunched back end runs none yet, so that is when it loses them. */ fn nearly_full(&self) -> bool { - self.pressure == Some(true) - || self - .disk - .is_some_and(|disk| u128::from(disk.free) * 100 < u128::from(disk.size) * 15) + self.pressure == Some(true) && !self.recovering() || self.below_image_line() + } + + /// Less than 15% of the node's disk free: the kubelet removes images it is not running. + fn below_image_line(&self) -> bool { + self.disk + .is_some_and(|disk| u128::from(disk.free) * 100 < u128::from(disk.size) * 15) + } + + /// Below the CLI's 40 GiB, or below the kubelet's image line. + fn low_on_disk(&self) -> bool { + self.free_disk() + .is_some_and(|free| free < MIN_ENGINE_FREE_DISK_BYTES) + || self.below_image_line() + } + + /** + The node still says `DiskPressure`, but the disk is above both lines: space was freed, and k3s + keeps the condition for five minutes after (`evictionPressureTransitionPeriod`). The back end + recovers by itself, and a Retry in those minutes only fails again. + */ + fn recovering(&self) -> bool { + self.pressure == Some(true) && self.free_disk().is_some() && !self.low_on_disk() + } + + /// Podman on Windows, whose machine runs in WSL. + fn wsl(&self) -> bool { + self.podman && (cfg!(target_os = "windows") || self.drive.is_some()) } /// What is short, with the numbers. Anything not measured is not judged. @@ -1387,6 +1420,8 @@ impl EngineCheck { let free = self.free_disk(); let mut short = engine_shortfall(cpus, memory, free); short.podman = self.podman; + short.wsl = self.wsl(); + short.recovering = self.recovering(); short.nearly_full = self.nearly_full(); short.disk |= short.nearly_full; if short.disk { @@ -1442,7 +1477,10 @@ impl EngineCheck { "disk (OpenBot needs 40 GiB free)" }); } - let verdict = if !verdict.is_empty() { + let verdict = if short.recovering && verdict.is_empty() { + "recovering: the node still reports disk pressure with the disk above both lines" + .to_owned() + } else if !verdict.is_empty() { format!("short of {}", verdict.join(", ")) } else if self.size.is_none() && self.free_disk().is_none() { "not measured".to_owned() @@ -1483,6 +1521,7 @@ pub fn engine_size_terms() -> Option { cpu: short.cpu, memory: short.memory, disk: short.disk, + wsl: check.wsl(), drive: check.drive.and_then(|drive| drive.drive), }) } @@ -6997,6 +7036,7 @@ overlay 164506208 20481024 135620148 14% / cpu: false, memory: false, disk: true, + wsl: false, drive: None, }) .unwrap(), @@ -7008,10 +7048,11 @@ overlay 164506208 20481024 135620148 14% / cpu: false, memory: false, disk: true, + wsl: true, drive: Some("C:".into()), }) .unwrap(), - json!({ "status": "engine-too-small", "podman": true, "cpu": false, "memory": false, "disk": true, "drive": "C:" }) + json!({ "status": "engine-too-small", "podman": true, "cpu": false, "memory": false, "disk": true, "wsl": true, "drive": "C:" }) ); } @@ -7454,7 +7495,8 @@ fn main() { wire["shortfall"], json!({ "podman": true, "cpu": false, "memory": false, "disk": true, - "diskGb": "30.5", "nearlyFull": true, "diskSizeGb": "1006.8", "drive": "C:" + "diskGb": "30.5", "nearlyFull": true, "diskSizeGb": "1006.8", + "wsl": true, "drive": "C:" }) ); assert_eq!( @@ -7484,12 +7526,12 @@ fn main() { "WARN [local check] after local start failed: 10 CPUs and 20.5 GiB of memory, 30 GiB free of 64 GiB in a container, no disk pressure; short of disk (OpenBot needs 40 GiB free)" ); - // 300 GiB free of a 1.8 TiB disk is above both lines, but the node still says - // DiskPressure: k3s holds it for five minutes after the kubelet freed space. + // Docker on a 1.8 TiB disk with 160 GiB free, its node under DiskPressure: nearly full + // by k3s's own line, though above the CLI's 40 GiB. lab.engine( crate::engine::Engine::Docker, DOCKER_INFO, - &df(1_916_694_008, 300 * KIB_PER_GIB), + &df(1_916_694_008, 160 * KIB_PER_GIB), Some(&nodes(true)), None, ); @@ -7499,9 +7541,46 @@ fn main() { wire["shortfall"], json!({ "podman": false, "cpu": false, "memory": false, "disk": true, - "diskGb": "300", "nearlyFull": true, "diskSizeGb": "1827.9" + "diskGb": "160", "nearlyFull": true, "diskSizeGb": "1827.9" }) ); + checks_logged(); + } + + /** + Windows, once space was freed: k3s keeps its DiskPressure condition for five minutes after, and + a Retry in those minutes failed again, three times over six minutes. A start that fails while + the node still says DiskPressure, with the disk above both lines, says the back end is + recovering, so the card can say to wait rather than to free more space. + */ + #[test] + fn a_start_that_fails_while_the_node_recovers_from_low_disk_says_so() { + if crate::test_support::isolated_process( + "local_eval::tests::a_start_that_fails_while_the_node_recovers_from_low_disk_says_so", + ) { + return; + } + let lab = StartLab::new("start-recovering"); + lab.engine( + crate::engine::Engine::Podman, + PODMAN_INFO, + &df(WSL_DISK, 941 * KIB_PER_GIB), + Some(&nodes(true)), + drive(1200), + ); + let wire = serde_json::to_value(lab.start(&format!("1\n{}", start_failed()))).unwrap(); + assert_eq!(wire["reason"], json!("resources-short")); + assert_eq!( + wire["shortfall"], + json!({ + "podman": true, "cpu": false, "memory": false, "disk": false, + "recovering": true, "wsl": true, "drive": "C:" + }) + ); + assert_eq!( + checks_logged().last().unwrap(), + "WARN [local check] after local start failed: 4 CPUs and 5.8 GiB of memory, 941 GiB free of 1006.8 GiB in a container, 120 GiB free on C:, the node reports disk pressure; recovering: the node still reports disk pressure with the disk above both lines" + ); } /** @@ -7531,7 +7610,7 @@ fn main() { wire["shortfall"], json!({ "podman": true, "cpu": false, "memory": false, "disk": true, - "diskGb": "30", "drive": "C:" + "diskGb": "30", "wsl": true, "drive": "C:" }) ); assert_eq!( @@ -7545,7 +7624,7 @@ fn main() { serde_json::to_value(engine_size_terms()).unwrap(), json!({ "status": "engine-too-small", "podman": true, "cpu": false, "memory": false, - "disk": true, "drive": "C:" + "disk": true, "wsl": true, "drive": "C:" }) ); } diff --git a/desktop/src/LocalIntelligence.test.tsx b/desktop/src/LocalIntelligence.test.tsx index 0259497c69..89dbacca01 100644 --- a/desktop/src/LocalIntelligence.test.tsx +++ b/desktop/src/LocalIntelligence.test.tsx @@ -1123,6 +1123,7 @@ test("a relaunch whose start fails on a nearly full disk names the disk on its r diskGb: "30.5", nearlyFull: true, diskSizeGb: "1006.8", + wsl: true, drive: "C:", }, }); @@ -1134,7 +1135,7 @@ test("a relaunch whose start fails on a nearly full disk names the disk on its r await screen.findByRole("button", { name: "Retry" }); const waiting = row(screen, WAIT).textContent; expect(waiting).toContain( - "The back end's disk is nearly full. Podman has 30.5 GB free of 1006.8 GB. OpenBot needs 40 GB. Free space on drive C: or in Podman, then Retry.", + "The back end's disk is nearly full. Podman has 30.5 GB free of 1006.8 GB. OpenBot needs 40 GB. Free space on the drive that holds OpenBot's engine (C:), then Retry.", ); expect(waiting).not.toContain("partway through"); expect(commands()).toEqual(["run_local_eval:status", "run_local_eval:start"]); diff --git a/desktop/src/localEvalCli.ts b/desktop/src/localEvalCli.ts index 854f945fc0..5933875245 100644 --- a/desktop/src/localEvalCli.ts +++ b/desktop/src/localEvalCli.ts @@ -96,7 +96,14 @@ export type Shortfall = { */ nearlyFull?: boolean; diskSizeGb?: string; - /** Podman on Windows: the drive that holds its machine's disk, `C:`. */ + /** + * The node still reports low disk with the disk above both lines: space was freed, and the + * back end recovers by itself within minutes. + */ + recovering?: boolean; + /** Podman on Windows, in WSL: `podman machine set` changes nothing there. */ + wsl?: boolean; + /** The drive that holds its machine's disk, `C:`. */ drive?: string; }; diff --git a/desktop/src/localReadiness.test.ts b/desktop/src/localReadiness.test.ts index 9e737211ff..ed55e14640 100644 --- a/desktop/src/localReadiness.test.ts +++ b/desktop/src/localReadiness.test.ts @@ -240,25 +240,32 @@ test("specific wording and actions", () => { diskGb: "30.5", nearlyFull: true, diskSizeGb: "1006.8", + wsl: true, drive: "C:", }).said, ).toBe( - "The back end's disk is nearly full. Podman has 30.5 GB free of 1006.8 GB. OpenBot needs 40 GB. Free space on drive C: or in Podman, then Retry.", + "The back end's disk is nearly full. Podman has 30.5 GB free of 1006.8 GB. OpenBot needs 40 GB. Free space on the drive that holds OpenBot's engine (C:), then Retry.", ); expect( - short({ podman: true, disk: true, diskGb: "30", drive: "C:" }).said, + short({ podman: true, disk: true, diskGb: "30", wsl: true, drive: "C:" }) + .said, ).toBe( - "Podman has 30 GB of free disk. OpenBot needs 40 GB. Free space on drive C: or in Podman, then Retry.", + "Podman has 30 GB of free disk. OpenBot needs 40 GB. Free space on the drive that holds OpenBot's engine (C:), then Retry.", + ); + // On Windows, never `podman machine set --disk-size`: Podman refuses it for WSL machines, even + // when WSL did not say which drive holds the disk. + expect(short({ podman: true, disk: true, wsl: true }).said).toBe( + "Podman is out of disk space. OpenBot needs 40 GB free. Free space on the drive that holds OpenBot's engine, then Retry.", ); expect( short({ disk: true, - diskGb: "300", + diskGb: "160", nearlyFull: true, diskSizeGb: "1827.9", }).said, ).toBe( - "The back end's disk is nearly full. Docker has 300 GB free of 1827.9 GB. Raise the disk limit in Docker Desktop > Settings > Resources, or free space in Docker, then Retry.", + "The back end's disk is nearly full. Docker has 160 GB free of 1827.9 GB. Raise the disk limit in Docker Desktop > Settings > Resources, or free space in Docker, then Retry.", ); expect( short({ @@ -271,10 +278,14 @@ test("specific wording and actions", () => { ).toBe( "The back end's disk is nearly full. Podman has 8 GB free of 60 GB. OpenBot needs 40 GB. Stop your Podman machine, run podman machine set --disk-size 160, start it again, then Retry.", ); - // Podman on Windows short of CPUs too: those are raised, the drive's space is freed. - expect(short({ podman: true, cpu: true, disk: true, drive: "C:" }).said).toBe( - "Podman doesn't have enough CPUs and disk space. OpenBot needs 4 free CPUs and 40 GB free. Stop your Podman machine, run podman machine set --cpus 6, start it again, and free space on drive C: or in Podman, then Retry.", - ); + // Space was freed and k3s still says the disk is low: wait, rather than free more. + expect( + short({ podman: true, recovering: true, wsl: true, drive: "C:" }), + ).toEqual({ + state: "blocked", + said: "The back end is recovering from low disk space. Wait a few minutes, then Retry.", + action: "retry", + }); // The account step's check before a first setup, on Windows. expect( engineTooSmallSaid({ @@ -282,10 +293,11 @@ test("specific wording and actions", () => { cpu: false, memory: false, disk: true, + wsl: true, drive: "C:", }), ).toBe( - "Podman is out of disk space. OpenBot needs 40 GB free. Free space on drive C: or in Podman, then Try again.", + "Podman is out of disk space. OpenBot needs 40 GB free. Free space on the drive that holds OpenBot's engine (C:), then Try again.", ); // A report that does not say which is still about resources, never "partway through". expect( diff --git a/desktop/src/localReadiness.ts b/desktop/src/localReadiness.ts index 825fc0b2fc..08b0211c1c 100644 --- a/desktop/src/localReadiness.ts +++ b/desktop/src/localReadiness.ts @@ -134,6 +134,10 @@ function shortfallSaid( const short = (["cpu", "memory", "disk"] as const).filter( (resource) => shortfall[resource], ); + // Space was freed, and k3s still says the disk is low for a few minutes after. + if (shortfall.recovering && short.length === 0) { + return `The back end is recovering from low disk space. Wait a few minutes, then ${then}.`; + } if (short.length === 0) { const fix = shortfall.podman ? "Stop your Podman machine, raise them with podman machine set, start it again" @@ -141,7 +145,7 @@ function shortfallSaid( return `${engine} doesn't have enough CPUs, memory or disk space for the back end. ${fix}, then ${then}.`; } const fix = shortfall.podman - ? podmanFix(short, shortfall.drive) + ? podmanFix(short, shortfall) : `Raise the ${short.length === 1 ? `${limitName[short[0]]} limit` : "limits"} in Docker Desktop > Settings > Resources${shortfall.disk ? ", or free space in Docker" : ""}`; // OpenBot's own check after a start that failed: k3s itself is short of disk. if (shortfall.nearlyFull && short.length === 1) { @@ -177,18 +181,21 @@ function shortfallSaid( } /** - * Where Podman raises what is short. On Windows its machine's disk is a sparse virtual disk on - * `drive`, and `podman machine set` does not change it, so space is freed there instead. + * Where Podman raises what is short. On Windows its machine runs in WSL, where `podman machine set` + * refuses to change its disk ("changing disk size not supported for WSL machines"), and the disk + * is a sparse virtual disk on `drive`, so space is freed on that drive instead. */ -function podmanFix(short: Resource[], drive: string | undefined): string { - const raised = short.filter((resource) => !(drive && resource === "disk")); +function podmanFix(short: Resource[], shortfall: Shortfall): string { + const raised = short.filter( + (resource) => !(shortfall.wsl && resource === "disk"), + ); const set = raised.length > 0 && `Stop your Podman machine, run podman machine set ${raised.map((resource) => podmanFlag[resource]).join(" ")}, start it again`; const freed = - drive && + shortfall.wsl && short.includes("disk") && - `free space on drive ${drive} or in Podman`; + `free space on the drive that holds OpenBot's engine${shortfall.drive ? ` (${shortfall.drive})` : ""}`; if (set && freed) return `${set}, and ${freed}`; if (freed) return freed.charAt(0).toUpperCase() + freed.slice(1); return set || ""; From a3a43dcbe9ea7f75407ded4fd4aa403fd45c0c2d Mon Sep 17 00:00:00 2001 From: David McKay Date: Mon, 9 Nov 2026 05:04:50 -0800 Subject: [PATCH 4/4] Check the disk behind a back end that runs but is not healthy After a start that failed on a full disk, the back end's node stays up. Retry then reads its status, which says degraded, and the row said only "The back end is running but not healthy." while k3s still reported DiskPressure. No start ran, so the check around a start never did either. A status that says degraded, with checks failing, is now checked as a failed start is: free disk, the drive under Podman's machine on Windows, and the node's DiskPressure. A disk that is still short says the back end's disk is nearly full, with the numbers. A node still under DiskPressure with room on the disk says the back end is recovering and to wait a few minutes. Anything else keeps "The back end is running but not healthy." A status whose checks all passed, as with an update on record, is not checked. --- CHANGELOG.md | 7 +- desktop/src-tauri/src/local_eval.rs | 137 ++++++++++++++++++++++++- desktop/src/LocalIntelligence.test.tsx | 43 ++++++++ desktop/src/localEvalCli.ts | 5 +- desktop/src/localReadiness.test.ts | 37 +++++++ desktop/src/localReadiness.ts | 7 ++ 6 files changed, 229 insertions(+), 7 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index 8d3dd671e8..c70ae2f58b 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -28,9 +28,10 @@ Newest first. `Unreleased` is what is on `main` and not yet tagged. 30.5 GB free of 1006.8 GB. OpenBot needs 40 GB. Free space on the drive that holds OpenBot's engine (C:), then Retry." On Windows, free disk is measured on that drive, which can run out while the machine itself shows hundreds of GB free, and the card never suggests - `podman machine set`, which WSL machines refuse. A start that fails in the few minutes after - space was freed, while the back end still reports low disk, says "The back end is recovering - from low disk space. Wait a few minutes, then Retry." + `podman machine set`, which WSL machines refuse. In the few minutes after space was freed, while + the back end still reports low disk, a start that fails, or a Retry that finds the back end + running but not healthy, says "The back end is recovering from low disk space. Wait a few + minutes, then Retry." where Retry said "The back end is running but not healthy." ## 1.0.0 diff --git a/desktop/src-tauri/src/local_eval.rs b/desktop/src-tauri/src/local_eval.rs index 9a27c3a6e6..57bdd8cdfd 100644 --- a/desktop/src-tauri/src/local_eval.rs +++ b/desktop/src-tauri/src/local_eval.rs @@ -248,6 +248,8 @@ pub struct Report { /// from a back end that is down (see `UnfinishedUpdate`). pub checks_passed: bool, /// With `Reason::ResourcesShort`: which resources the CLI found short, and on which engine. + /// With `Readiness::Degraded`: a disk OpenBot found short, or recovering, behind a back end that + /// runs but does not answer (`checked_when_unhealthy`). #[serde(skip_serializing_if = "Option::is_none")] pub shortfall: Option, } @@ -3221,10 +3223,39 @@ pub(crate) fn run_with( Action::Start | Action::Update => { checked_call(&format!("local {}", action.as_str()), Some(stack), call) } - Action::Status | Action::Stop => call(), + Action::Status => call().map(|report| checked_when_unhealthy(report, stack)), + Action::Stop => call(), } } +/** +A back end that runs but does not answer (`local status` says `degraded`), checked as a failed start +is. After a start that failed on a full disk the node stays up, so Retry reads its status rather +than starting it again, and k3s keeps its `DiskPressure` for minutes after space is freed. A disk +that is short, or a node recovering from one, becomes the report's `shortfall`, and the card says +which; anything else leaves the report as it was. Not when every check passed: an update the CLI +keeps on record is `degraded` too, with the back end answering. +*/ +fn checked_when_unhealthy(mut report: Report, stack: &StackId) -> Report { + if report.outcome != Outcome::Completed + || report.readiness != Readiness::Degraded + || report.checks_passed + { + return report; + } + let Some(address) = cli_engine() else { + return report; + }; + let mut check = EngineCheck::of(&address); + check.pressure = node_disk_pressure(Some(stack)); + log_check("local status, running but not healthy", &check); + let short = check.shortfall(); + if short.disk || short.recovering { + report.shortfall = Some(short); + } + report +} + /// `run_with`, without the engine check. fn run_downloading( cli: &PinnedCli, @@ -7414,12 +7445,17 @@ fn main() { } fn start(&self, ends: &str) -> Report { + self.run(Action::Start, ends) + } + + /// `action` through the window's path, answered with `ends`. + fn run(&self, action: Action, ends: &str) -> Report { std::fs::write(self.fixture.root.join("scenario"), ends).unwrap(); let stack = StackId::parse("local-abc123").unwrap(); - let report = run_installed("start-checked", Action::Start, &stack).unwrap(); + let report = run_installed("checked", action, &stack).unwrap(); assert_eq!( self.fixture.argv().unwrap(), - "local\nstart\n--stack\nlocal-abc123\n--json" + format!("local\n{}\n--stack\nlocal-abc123\n--json", action.as_str()) ); report } @@ -7667,4 +7703,99 @@ fn main() { ); assert_eq!(engine_size_terms(), None); } + + /// `local status` for a back end left up after a start that failed: its node runs, and the + /// API and gateway do not answer. `passed` when every check passed, as with an update on record. + fn status_degraded(passed: bool) -> String { + let mut status = canonical("degraded", "valid", Value::Null); + status["readiness"]["checks"] = json!([ + { "name": "api", "ok": passed }, + { "name": "gateway", "ok": passed }, + { "name": "dashboard", "ok": true } + ]); + format!("0\n{status}") + } + + /** + Windows, after space was freed: Retry read the back end's status, which said `degraded`, and the + card said "The back end is running but not healthy." twice over two minutes while the node still + reported DiskPressure. A back end that runs but does not answer is checked as a failed start is: + recovering, nearly full, or nothing to say, in which case the report is as it was. + */ + #[test] + fn a_back_end_running_but_not_healthy_is_checked_for_its_disk() { + if crate::test_support::isolated_process( + "local_eval::tests::a_back_end_running_but_not_healthy_is_checked_for_its_disk", + ) { + return; + } + let lab = StartLab::new("status-degraded-checked"); + let degraded = status_degraded(false); + + // Space freed, the node still under DiskPressure. + lab.engine( + crate::engine::Engine::Podman, + PODMAN_INFO, + &df(WSL_DISK, 941 * KIB_PER_GIB), + Some(&nodes(true)), + drive(1200), + ); + let wire = serde_json::to_value(lab.run(Action::Status, °raded)).unwrap(); + assert_eq!(wire["readiness"], json!("degraded")); + assert_eq!(wire["reason"], Value::Null); + assert_eq!( + wire["shortfall"], + json!({ + "podman": true, "cpu": false, "memory": false, "disk": false, + "recovering": true, "wsl": true, "drive": "C:" + }) + ); + assert_eq!( + checks_logged(), + ["WARN [local check] local status, running but not healthy: 4 CPUs and 5.8 GiB of memory, 941 GiB free of 1006.8 GiB in a container, 120 GiB free on C:, the node reports disk pressure; recovering: the node still reports disk pressure with the disk above both lines"] + ); + + // Still nearly full. + lab.engine( + crate::engine::Engine::Podman, + PODMAN_INFO, + &df(WSL_DISK, 30 * KIB_PER_GIB + KIB_PER_GIB / 2), + Some(&nodes(true)), + drive(946), + ); + let wire = serde_json::to_value(lab.run(Action::Status, °raded)).unwrap(); + assert_eq!( + wire["shortfall"], + json!({ + "podman": true, "cpu": false, "memory": false, "disk": true, + "diskGb": "30.5", "nearlyFull": true, "diskSizeGb": "1006.8", + "wsl": true, "drive": "C:" + }) + ); + + // Room on the disk and no pressure: not the disk, and the report is as it was. + lab.engine( + crate::engine::Engine::Podman, + PODMAN_INFO, + &df(WSL_DISK, 941 * KIB_PER_GIB), + Some(&nodes(false)), + drive(1200), + ); + let wire = serde_json::to_value(lab.run(Action::Status, °raded)).unwrap(); + assert_eq!(wire["readiness"], json!("degraded")); + assert!(wire.get("shortfall").is_none(), "{wire}"); + checks_logged(); + + // Every check passed: an update on record, not a back end that does not answer. + lab.engine( + crate::engine::Engine::Podman, + PODMAN_INFO, + &df(WSL_DISK, 30 * KIB_PER_GIB), + Some(&nodes(true)), + drive(946), + ); + let wire = serde_json::to_value(lab.run(Action::Status, &status_degraded(true))).unwrap(); + assert!(wire.get("shortfall").is_none(), "{wire}"); + assert_eq!(checks_logged(), Vec::::new()); + } } diff --git a/desktop/src/LocalIntelligence.test.tsx b/desktop/src/LocalIntelligence.test.tsx index 89dbacca01..3e8e9bf321 100644 --- a/desktop/src/LocalIntelligence.test.tsx +++ b/desktop/src/LocalIntelligence.test.tsx @@ -1141,6 +1141,49 @@ test("a relaunch whose start fails on a nearly full disk names the disk on its r expect(commands()).toEqual(["run_local_eval:status", "run_local_eval:start"]); }); +/** + * Windows, after space was freed: Retry reads the status of the back end the failed start left + * running, which says `degraded`, and the row said only "The back end is running but not healthy." + * while k3s still reported low disk. `local_eval::checked_when_unhealthy` now says why. + */ +test("a back end running but not healthy after low disk says it is recovering, and Retry asks again", async () => { + const recovering = report({ + readiness: "degraded", + licence: "valid", + shortfall: { + podman: true, + cpu: false, + memory: false, + disk: false, + recovering: true, + wsl: true, + drive: "C:", + }, + }); + invokeHandler = answers({ run_local_eval: [recovering, ready] }); + const started: Startable[] = []; + const screen = view({ + stack: "local-abc123", + connected: true, + onStart: async (startable) => { + started.push(startable); + return "started"; + }, + }); + const retry = await screen.findByRole("button", { name: "Retry" }); + const waiting = row(screen, WAIT).textContent; + expect(waiting).toContain( + "The back end is recovering from low disk space. Wait a few minutes, then Retry.", + ); + expect(waiting).not.toContain("running but not healthy"); + await userEvent.click(retry); + await waitFor(() => expect(started).toEqual([startableUrls])); + expect(commands()).toEqual([ + "run_local_eval:status", + "run_local_eval:status", + ]); +}); + test("a start that fails stops at its row, and Retry starts again without asking the CLI", async () => { invokeHandler = answers({ run_local_eval: [ready] }); let starts = 0; diff --git a/desktop/src/localEvalCli.ts b/desktop/src/localEvalCli.ts index 5933875245..16396780de 100644 --- a/desktop/src/localEvalCli.ts +++ b/desktop/src/localEvalCli.ts @@ -109,7 +109,10 @@ export type Shortfall = { export type LocalReport = { outcome: LocalOutcome; - /** With the reason `resources-short`: which resources the CLI found short. */ + /** + * With the reason `resources-short`: which resources the CLI found short. With readiness + * `degraded`: a disk OpenBot found short, or recovering, behind a back end that does not answer. + */ shortfall?: Shortfall; stack: null | { id: string; diff --git a/desktop/src/localReadiness.test.ts b/desktop/src/localReadiness.test.ts index ed55e14640..4b53af50df 100644 --- a/desktop/src/localReadiness.test.ts +++ b/desktop/src/localReadiness.test.ts @@ -286,6 +286,43 @@ test("specific wording and actions", () => { said: "The back end is recovering from low disk space. Wait a few minutes, then Retry.", action: "retry", }); + // A back end that runs but does not answer, with OpenBot's check of its disk, and without. + const degraded = (shortfall?: Partial) => + said( + report({ + readiness: "degraded", + licence: "valid", + ...(shortfall + ? { + shortfall: { + podman: true, + cpu: false, + memory: false, + disk: false, + ...shortfall, + }, + } + : {}), + }), + ); + expect(degraded({ recovering: true, wsl: true, drive: "C:" })).toEqual({ + state: "blocked", + said: "The back end is recovering from low disk space. Wait a few minutes, then Retry.", + action: "restart-intelligence", + }); + expect( + degraded({ + disk: true, + diskGb: "30.5", + nearlyFull: true, + diskSizeGb: "1006.8", + wsl: true, + drive: "C:", + }).said, + ).toBe( + "The back end's disk is nearly full. Podman has 30.5 GB free of 1006.8 GB. OpenBot needs 40 GB. Free space on the drive that holds OpenBot's engine (C:), then Retry.", + ); + expect(degraded().said).toBe("The back end is running but not healthy."); // The account step's check before a first setup, on Windows. expect( engineTooSmallSaid({ diff --git a/desktop/src/localReadiness.ts b/desktop/src/localReadiness.ts index 08b0211c1c..e525221d6c 100644 --- a/desktop/src/localReadiness.ts +++ b/desktop/src/localReadiness.ts @@ -390,6 +390,13 @@ function intelligenceRow( return blocked("The back end is stopped.", "start-intelligence"); } if (intelligence.readiness === "degraded") { + // OpenBot's check of a back end that runs but does not answer: a full disk, or one recovering. + if (intelligence.shortfall) { + return blocked( + resourcesShortSaid(intelligence.shortfall), + "restart-intelligence", + ); + } return blocked( "The back end is running but not healthy.", "restart-intelligence",