diff --git a/CHANGELOG.md b/CHANGELOG.md index 4b1e179866..dcd2d95ce0 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -72,6 +72,14 @@ deployment: and the end of its error output. A setup that stopped used to leave nothing in the log. The card says whether the back end stopped with an error partway, quit without saying why, or could not be run, where all three said "The back end could not start." +- **A Docker short of disk, CPUs or memory says which, and where to raise it.** A setup the back + end refuses for its resources said "The back end stopped with an error partway through." It now + names the resource with the back end's own numbers: a full Docker disk says "Docker has 0 GB of + free disk. OpenBot needs 40 GB. Raise the disk limit in Docker Desktop > Settings > Resources, or + free space in Docker, then Retry.", and too few free CPUs or too little free memory say so the + same way. On Podman it names `podman machine set` instead. The CopilotKit account step now checks the engine first, and one + with fewer than 6 CPUs, less than 8 GB of memory or less than 40 GB of free disk stops there with + the same advice, before Start. - When OpenBot Desktop is signed in with a model Learning can't use, Learn now in Settings → Learning says "Learning can't run on the model you signed in with. To use it, sign OpenBot in with an OpenAI or Anthropic API key, or a Claude or ChatGPT plan.", where it told you to give the back diff --git a/desktop/src-tauri/src/learning_model.rs b/desktop/src-tauri/src/learning_model.rs index 227f1cbf6f..ccce39c07b 100644 --- a/desktop/src-tauri/src/learning_model.rs +++ b/desktop/src-tauri/src/learning_model.rs @@ -468,6 +468,7 @@ mod tests { licence_expires_at: None, update_to: None, checks_passed: readiness == crate::local_eval::Readiness::Ready, + shortfall: None, } } diff --git a/desktop/src-tauri/src/local_eval.rs b/desktop/src-tauri/src/local_eval.rs index 6e3ab2d695..5ae8982687 100644 --- a/desktop/src-tauri/src/local_eval.rs +++ b/desktop/src-tauri/src/local_eval.rs @@ -193,6 +193,10 @@ pub enum Reason { DashboardUnavailable, /// `login`: the CLI could not open the browser. Its fallback is a code approved in a terminal. BrowserOpenFailed, + /// The CLI's own resource check: the engine has too few free CPUs, or too little free memory + /// or disk. Read from `error.code`, since its `nextAction.reason` is only + /// `local_command_failed`. Which of them is short is the report's `shortfall`. + ResourcesShort, /// A reason this build does not know. Treated as blocking, never as success. Unrecognized, } @@ -243,6 +247,41 @@ pub struct Report { /// on record, however well the release it has answers. This is how the window tells that apart /// from a back end that is down (see `UnfinishedUpdate`). pub checks_passed: bool, + /// With `Reason::ResourcesShort`: which resources the CLI found short, and on which engine. + #[serde(skip_serializing_if = "Option::is_none")] + pub shortfall: Option, +} + +/// Which of the engine's resources are below what local setup needs, and whether the engine is +/// Podman, so the window names the right resource and the right settings. +#[derive(Clone, Debug, Default, PartialEq, Eq, Serialize)] +#[serde(rename_all = "camelCase")] +pub struct Shortfall { + pub podman: bool, + pub cpu: bool, + pub memory: bool, + pub disk: bool, + /// What the CLI measured free, as it said it, to one decimal: CPUs, and GiB of memory and of + /// disk. Absent when its error did not say. + #[serde(skip_serializing_if = "Option::is_none")] + pub cpus: Option, + #[serde(skip_serializing_if = "Option::is_none")] + pub memory_gb: Option, + #[serde(skip_serializing_if = "Option::is_none")] + pub disk_gb: Option, +} + +/// A number the CLI printed, to one decimal and rounded down as it rounds, without a trailing `.0`. +fn shown(value: f64) -> String { + let tenths = (value.max(0.0) * 10.0).floor() / 10.0; + let said = format!("{tenths:.1}"); + said.strip_suffix(".0").map(str::to_owned).unwrap_or(said) +} + +impl Shortfall { + fn named(&self) -> bool { + self.cpu || self.memory || self.disk + } } /// What `local connect` answered. The settings it wrote stay in the installation's `.env`; their @@ -600,15 +639,28 @@ fn report(command: &str, exit_code: i32, stdout: &[u8]) -> Result Result /// Only the CLI's reason for stopping, never its summary, instructions or resume command. fn reason(outcome: Outcome, wire: &Map) -> Result, Failure> { + if outcome != Outcome::Completed && resource_shortfall_code(wire).is_some() { + return Ok(Some(Reason::ResourcesShort)); + } match (outcome, wire.get("nextAction")) { (Outcome::Completed, _) | (_, None | Some(Value::Null)) => Ok(None), (_, Some(next)) => match object(next)?.get("reason") { @@ -1026,6 +1089,234 @@ fn reason(outcome: Outcome, wire: &Map) -> Result, } } +/** +The `error.code` the CLI gives when its resource check fails (`LocalResourceShortfallError`): too +few free CPUs, too little free memory or too little free storage in the engine. A probe that could +not measure (`LOCAL_DOCKER_RESOURCES_UNMEASURABLE`) is not a shortfall and stays a plain failure. +*/ +const RESOURCES_SHORT: &str = "LOCAL_DOCKER_RESOURCES_SHORT"; +const MEMORY_SHORT: &str = "LOCAL_DOCKER_MEMORY_SHORT"; + +fn resource_shortfall_code(wire: &Map) -> Option<&str> { + wire.get("error") + .and_then(|error| error.get("code")) + .and_then(Value::as_str) + .filter(|code| [RESOURCES_SHORT, MEMORY_SHORT].contains(code)) +} + +/** +Which resources the CLI found short, from its own error. `LOCAL_DOCKER_MEMORY_SHORT` is memory +alone. `LOCAL_DOCKER_RESOURCES_SHORT` says each measure beside what it requires: "Docker available +resources: 9.8 CPUs (requires 4), 18.9 GiB RAM available (requires 4), 0 GiB free storage (requires +40)." Only these numbers are read; the rest of the message is not kept. +*/ +fn shortfall_of(wire: &Map, podman: bool) -> Option { + let code = resource_shortfall_code(wire)?; + let mut shortfall = Shortfall { + podman, + ..Shortfall::default() + }; + let message = wire + .get("error") + .and_then(|error| error.get("message")) + .and_then(Value::as_str) + .unwrap_or(""); + if code == MEMORY_SHORT { + shortfall.memory = true; + let available = regex::Regex::new(r"([\d.]+) GiB available to new containers") + .expect("a valid pattern"); + shortfall.memory_gb = available + .captures(message) + .and_then(|found| found[1].parse::().ok()) + .map(shown); + return Some(shortfall); + } + let measures = regex::Regex::new( + r"(-?[\d.]+) CPUs \(requires ([\d.]+)\), (-?[\d.]+) GiB RAM available \(requires ([\d.]+)\), (-?[\d.]+) GiB free storage \(requires ([\d.]+)\)", + ) + .expect("a valid pattern"); + if let Some(found) = measures.captures(message) { + let number = |at: usize| found[at].parse::().ok(); + let below = |have: usize, need: usize| match (number(have), number(need)) { + (Some(have), Some(need)) => have < need, + _ => false, + }; + shortfall.cpu = below(1, 2); + shortfall.memory = below(3, 4); + shortfall.disk = below(5, 6); + shortfall.cpus = number(1).map(shown); + shortfall.memory_gb = number(3).map(shown); + shortfall.disk_gb = number(5).map(shown); + } + Some(shortfall) +} + +/// The engine the CLI runs on, when the app has said. +fn cli_engine() -> Option { + ENGINE + .lock() + .unwrap_or_else(|poisoned| poisoned.into_inner()) + .clone() +} + +fn on_podman() -> bool { + cli_engine().is_some_and(|address| address.engine == crate::engine::Engine::Podman) +} + +const GIB: u64 = 1024 * 1024 * 1024; + +/** +The smallest engine OpenBot asks for before local setup: 6 CPUs, 8 GB of memory and 40 GiB of free +disk. + +The CLI's own check wants 4 free CPUs and 4 GiB of free memory inside a container, measured while +OpenBot's own containers run, so an engine of exactly that size fails it at Start. Memory is +compared at 7 GiB, since an engine set to 8 GB reports a little under 8 GiB. Free disk is the CLI's +own 40 GiB, measured as it measures it, inside a container. +*/ +pub const MIN_ENGINE_CPUS: u64 = 6; +pub const MIN_ENGINE_MEMORY_BYTES: u64 = 7 * GIB; +pub const MIN_ENGINE_FREE_DISK_BYTES: u64 = 40 * GIB; + +/// What an engine of this size is short of for local setup. Free disk that could not be measured +/// is not judged. +pub fn engine_shortfall(cpus: u64, memory_bytes: u64, free_disk_bytes: Option) -> Shortfall { + Shortfall { + cpu: cpus < MIN_ENGINE_CPUS, + memory: memory_bytes < MIN_ENGINE_MEMORY_BYTES, + disk: free_disk_bytes.is_some_and(|free| free < MIN_ENGINE_FREE_DISK_BYTES), + ..Shortfall::default() + } +} + +/// The engine's CPUs and memory as its `info` reports them: `NCPU MemTotal` on Docker, +/// `Host.CPUs Host.MemTotal` on Podman. Anything else is no answer. +pub fn parse_engine_size(output: &str) -> Option<(u64, u64)> { + let mut fields = output.split_whitespace(); + let cpus = fields.next()?.parse().ok()?; + let memory = fields.next()?.parse().ok()?; + fields.next().is_none().then_some((cpus, memory)) +} + +/// Free bytes from `df -Pk /`: the fourth field of its last line, in KiB, as the CLI reads it. +pub fn parse_free_disk(output: &str) -> Option { + let kib: u64 = output + .trim() + .lines() + .last()? + .split_whitespace() + .nth(3)? + .parse() + .ok()?; + Some(kib * 1024) +} + +/// The engine's CPUs and memory from its `info`. +fn engine_size(address: &crate::engine::Address) -> Option<(u64, u64)> { + let format = if address.engine == crate::engine::Engine::Podman { + "{{.Host.CPUs}} {{.Host.MemTotal}}" + } else { + "{{.NCPU}} {{.MemTotal}}" + }; + let output = address + .command() + .args(["info", "--format", format]) + .output() + .ok() + .filter(|output| output.status.success())?; + parse_engine_size(&String::from_utf8_lossy(&output.stdout)) +} + +/** +Free disk inside a container, the way the CLI measures it: `df -Pk /` in a container of an image the +engine already has, so nothing is downloaded. An engine with no image that has a shell gives no +answer, and disk is then not judged. +*/ +fn engine_free_disk(address: &crate::engine::Address) -> Option { + let images = address + .command() + .args(["image", "ls", "--quiet", "--no-trunc"]) + .output() + .ok() + .filter(|output| output.status.success())?; + let images = String::from_utf8_lossy(&images.stdout).into_owned(); + let mut tried = Vec::new(); + for image in images.lines().map(str::trim).filter(|id| !id.is_empty()) { + if tried.contains(&image) { + continue; + } + if tried.len() == 3 { + break; + } + tried.push(image); + let measured = address + .command() + .args([ + "run", + "--rm", + "--pull=never", + "--network=none", + "--entrypoint", + "/bin/sh", + image, + "-c", + "df -Pk /", + ]) + .output(); + let free = measured + .ok() + .filter(|output| output.status.success()) + .and_then(|output| parse_free_disk(&String::from_utf8_lossy(&output.stdout))); + if free.is_some() { + return free; + } + } + None +} + +/** +The onboarding check of the engine, before the account screen asks about the agreement: +`Some(Terms::EngineTooSmall)` for an engine below `MIN_ENGINE_CPUS`, `MIN_ENGINE_MEMORY_BYTES` or +`MIN_ENGINE_FREE_DISK_BYTES` free. An engine that is not known yet or does not answer is not judged +here; the CLI's own check still runs at Start. +*/ +pub fn engine_size_terms() -> Option { + let address = cli_engine()?; + let (cpus, memory) = engine_size(&address)?; + let free_disk = engine_free_disk(&address); + let short = engine_shortfall(cpus, memory, free_disk); + if !short.named() { + return None; + } + log::warn!( + "[local terms] the engine has {cpus} CPUs, {memory} bytes of memory and {free_disk:?} bytes of free disk" + ); + Some(Terms::EngineTooSmall { + podman: address.engine == crate::engine::Engine::Podman, + cpu: short.cpu, + memory: short.memory, + disk: short.disk, + }) +} + +/** +A shortfall whose CLI error did not say which resource, measured here instead: CPUs and memory from +the engine's `info` against the CLI's 4 and 4 GiB, and free disk inside a container against its 40 +GiB. +*/ +fn measured_shortfall(mut shortfall: Shortfall) -> Shortfall { + let Some(address) = cli_engine() else { + return shortfall; + }; + if let Some((cpus, memory)) = engine_size(&address) { + shortfall.cpu = cpus < 4; + shortfall.memory = memory < 4 * GIB; + } + shortfall.disk = + engine_free_disk(&address).is_some_and(|free| free < MIN_ENGINE_FREE_DISK_BYTES); + shortfall +} + /// The version-1 envelope every `local` command shares: version, command, and a status that /// agrees with the exit code. fn parse_envelope( @@ -6172,6 +6463,225 @@ fn main() {{ ); } + /// The CLI's report of a failed resource check, as the 4.34.0-pre.647a03b CLI prints it. + fn refused_for_resources(code: &str, message: &str) -> Vec { + json!({ + "version": 1, + "command": "local setup", + "status": "failed", + "error": { "code": code, "message": message }, + "nextAction": { + "reason": "local_command_failed", + "summary": "The local command did not finish. Check local status for the recorded failed step, then resume the command. No automatic rollback was requested.", + "instructions": ["Inspect the recorded stack with npx --yes copilotkit@4.34.0-pre.647a03b local status --json."], + "resumeCommand": "npx --yes copilotkit@4.34.0-pre.647a03b local setup --json", + "url": null, "linkLabel": null, "linkFile": null, + "caveats": [], "requiredEnvVars": [], "artifacts": [] + }, + "diagnostics": [] + }) + .to_string() + .into_bytes() + } + + /// A fresh Mac failed at Start with "stopped with an error partway through": Docker had 10 + /// CPUs and 20.5 GiB, but its virtual disk was full, against the CLI's 40 GiB free. The CLI's + /// `nextAction.reason` is only `local_command_failed`; its `error` says which resource is short, + /// and the window is told that one. + #[test] + fn a_setup_the_cli_refused_for_resources_names_the_resource() { + let none = Shortfall::default(); + let numbers = |cpus: &str, memory: &str, disk: &str| Shortfall { + cpus: Some(cpus.into()), + memory_gb: Some(memory.into()), + disk_gb: Some(disk.into()), + ..Shortfall::default() + }; + for (code, message, short) in [ + ( + "LOCAL_DOCKER_RESOURCES_SHORT", + "Docker available resources: 9.871234 CPUs (requires 4), 18.9 GiB RAM available (requires 4), 0 GiB free storage (requires 40). Stop other container workloads or increase Docker resources, then retry local setup.", + Shortfall { disk: true, ..numbers("9.8", "18.9", "0") }, + ), + ( + "LOCAL_DOCKER_RESOURCES_SHORT", + "Docker available resources: 3.71 CPUs (requires 4), 5.2 GiB RAM available (requires 4), 51 GiB free storage (requires 40). Stop other container workloads or increase Docker resources, then retry local setup.", + Shortfall { cpu: true, ..numbers("3.7", "5.2", "51") }, + ), + ( + "LOCAL_DOCKER_RESOURCES_SHORT", + "Docker available resources: 3.5 CPUs (requires 4), 2.9 GiB RAM available (requires 4), 12 GiB free storage (requires 40). Docker memory: 3.8 GiB total, 2.9 GiB available to new containers (requires 4 GiB available). Docker keeps part of its memory limit for itself, so raise Docker's memory limit by at least 3 GiB (Docker Desktop: Settings > Resources > Memory), or stop other containers, raise the CPUs or storage named above, then retry local setup.", + Shortfall { cpu: true, memory: true, disk: true, ..numbers("3.5", "2.9", "12") }, + ), + ( + "LOCAL_DOCKER_MEMORY_SHORT", + "Docker memory: 7.6 GiB total, 2.9 GiB available to new containers (requires 4 GiB available). Docker keeps part of its memory limit for itself, so raise Docker's memory limit by at least 3 GiB (Docker Desktop: Settings > Resources > Memory), or stop other containers, then retry local setup.", + Shortfall { memory: true, memory_gb: Some("2.9".into()), ..none.clone() }, + ), + ] { + let report = normalize_setup(1, &refused_for_resources(code, message)).unwrap(); + assert_eq!(report.outcome, Outcome::Failed); + assert_eq!(report.reason, Some(Reason::ResourcesShort), "{message}"); + assert_eq!(report.shortfall, Some(short), "{message}"); + } + let disk = normalize_setup( + 1, + &refused_for_resources( + "LOCAL_DOCKER_RESOURCES_SHORT", + "Docker available resources: 9.871234 CPUs (requires 4), 18.9 GiB RAM available (requires 4), 0 GiB free storage (requires 40). Stop other container workloads or increase Docker resources, then retry local setup.", + ), + ) + .unwrap(); + let wire = serde_json::to_value(&disk).unwrap(); + assert_eq!(wire["reason"], json!("resources-short")); + assert_eq!( + wire["shortfall"], + json!({ + "podman": false, "cpu": false, "memory": false, "disk": true, + "cpus": "9.8", "memoryGb": "18.9", "diskGb": "0" + }) + ); + // A probe that measured nothing is not a shortfall the person can fix in Settings. + let unmeasurable = normalize_setup( + 1, + &refused_for_resources( + "LOCAL_DOCKER_RESOURCES_UNMEASURABLE", + "Docker available memory or storage could not be measured. Check the container engine, then retry local setup.", + ), + ) + .unwrap(); + assert_eq!(unmeasurable.reason, Some(Reason::LocalCommandFailed)); + assert_eq!(unmeasurable.shortfall, None); + assert!(serde_json::to_value(&unmeasurable) + .unwrap() + .get("shortfall") + .is_none()); + } + + /// The onboarding check asks for 6 CPUs, 8 GB and 40 GiB of free disk, so an engine that would + /// fail the CLI's check at Start, with OpenBot's own containers running beside it, is caught + /// first. Free disk is measured as the CLI measures it, with `df -Pk /` in a container. + #[test] + fn a_small_or_full_engine_is_caught_before_local_setup() { + const GB: u64 = 1_000_000_000; + let roomy = Some(100 * GB); + // `docker info --format '{{.NCPU}} {{.MemTotal}}'` on Docker Desktop. + for (said, free, cpu, memory, disk) in [ + // 4 CPUs and 8 GB, as Docker Desktop reports an 8 GB limit. + ( + "4 8217473024 +", + roomy, + true, + false, + false, + ), + ( + "6 4113829888 +", + roomy, + false, + true, + false, + ), + // 10 CPUs and 20.5 GiB with a full disk: the case that happened. + ( + "10 22011707392 +", + Some(0), + false, + false, + true, + ), + ( + "10 22011707392 +", + Some(39 * GIB), + false, + false, + true, + ), + // 6 CPUs, 8 GB and 40 GiB free, and a larger one, are enough. + ( + "6 8217473024 +", + Some(40 * GIB), + false, + false, + false, + ), + ( + "18 16745730048 +", + roomy, + false, + false, + false, + ), + // Free disk that could not be measured is not judged. + ( + "18 16745730048 +", + None, + false, + false, + false, + ), + ] { + let (cpus, bytes) = parse_engine_size(said).unwrap(); + let short = engine_shortfall(cpus, bytes, free); + assert_eq!( + (short.cpu, short.memory, short.disk), + (cpu, memory, disk), + "{said} {free:?}" + ); + } + assert!(!engine_shortfall(MIN_ENGINE_CPUS, 8 * GB, roomy).named()); + for unread in [ + "", + "4", + "four 8217473024", + "4 8217473024 extra", + "-1 8217473024", + ] { + assert_eq!(parse_engine_size(unread), None, "{unread:?}"); + } + // `df -Pk /` inside a container on a full Docker Desktop disk, and on a roomy one. + assert_eq!( + parse_free_disk( + "Filesystem 1024-blocks Used Available Capacity Mounted on +overlay 100597760 100597760 0 100% / +" + ), + Some(0) + ); + assert_eq!( + parse_free_disk( + "Filesystem 1024-blocks Used Available Capacity Mounted on +overlay 164506208 20481024 135620148 14% / +" + ), + Some(135_620_148 * 1024) + ); + assert_eq!( + parse_free_disk( + "df: /: No such file or directory +" + ), + None + ); + assert_eq!( + serde_json::to_value(Terms::EngineTooSmall { + podman: false, + cpu: false, + memory: false, + disk: true + }) + .unwrap(), + json!({ "status": "engine-too-small", "podman": false, "cpu": false, "memory": false, "disk": true }) + ); + } + /// A new Windows computer installs Podman while the person signs in, and the CLI will not say /// anything about the agreement until an engine answers. Those answers, as the CLI gave them on /// Windows, are "not yet", which the account screen waits out, not a failure it shows. diff --git a/desktop/src-tauri/src/main.rs b/desktop/src-tauri/src/main.rs index f98b300c5c..aac206a173 100644 --- a/desktop/src-tauri/src/main.rs +++ b/desktop/src-tauri/src/main.rs @@ -6416,6 +6416,10 @@ async fn local_eval_terms( tauri::async_runtime::spawn_blocking(move || { // The CLI checks its engine before it answers. See `know_cli_engine`. know_cli_engine(&root); + // An engine too small for local setup is said here, before Start fails on it. + if let Some(too_small) = local_eval::engine_size_terms() { + return Ok(too_small); + } local_eval::terms_installed(&attempt) }) .await @@ -11697,6 +11701,7 @@ mod tests { licence_expires_at: None, update_to: None, checks_passed: false, + shortfall: None, } } diff --git a/desktop/src/CopilotKitAccount.test.tsx b/desktop/src/CopilotKitAccount.test.tsx index 2834cbfcfe..4a52407e45 100644 --- a/desktop/src/CopilotKitAccount.test.tsx +++ b/desktop/src/CopilotKitAccount.test.tsx @@ -69,6 +69,33 @@ async function startSigningIn() { return screen; } +/** OpenBot's own engine check: a Docker with a full disk is caught here, before Start fails on it. */ +test("an engine short of disk says how to raise it, and Try again checks again", async () => { + invokeHandler = async ({ command }) => { + if (command === "local_eval_terms") + return { + status: "engine-too-small", + podman: false, + cpu: false, + memory: false, + disk: true, + }; + throw new Error(`unexpected ${command}`); + }; + const screen = render( + {}} />, + ); + await screen.findByText( + "Docker is out of disk space. OpenBot needs 40 GB free. Raise the disk limit in Docker Desktop > Settings > Resources, or free space in Docker, then Try again.", + ); + await userEvent.click(screen.getByRole("button", { name: "Try again" })); + await waitFor(() => + expect( + invokeCalls.filter((call) => call.command === "local_eval_terms"), + ).toHaveLength(2), + ); +}); + test("a sign-in waiting on the browser shows its code and a way back to the page", async () => { waitingSignIn(); const screen = await startSigningIn(); diff --git a/desktop/src/CopilotKitAccount.tsx b/desktop/src/CopilotKitAccount.tsx index a7d2c7819d..c1b4c4d472 100644 --- a/desktop/src/CopilotKitAccount.tsx +++ b/desktop/src/CopilotKitAccount.tsx @@ -15,6 +15,7 @@ import { localEvalTerms, signInToCopilotKit, } from "./localEvalCli"; +import { engineTooSmallSaid } from "./localReadiness"; import { InlineFailure, type Problem } from "./Problem"; import { Sections } from "./Sections"; import { StatusLine } from "./SetupProgress"; @@ -133,7 +134,9 @@ export function CopilotKitAccount({ } } waitingSince.current = null; - if (terms.status === "signed-out") setSeen({ at: "signed-out" }); + if (terms.status === "engine-too-small") + setSeen({ at: "failed", said: engineTooSmallSaid(terms) }); + else if (terms.status === "signed-out") setSeen({ at: "signed-out" }); else if (terms.status === "required" && !terms.accepted) setSeen({ at: "agreement", terms }); else if (terms.status === "unavailable") diff --git a/desktop/src/LocalIntelligence.test.tsx b/desktop/src/LocalIntelligence.test.tsx index 16e48f0bbd..60f7b5a2da 100644 --- a/desktop/src/LocalIntelligence.test.tsx +++ b/desktop/src/LocalIntelligence.test.tsx @@ -1,6 +1,6 @@ +import { afterAll, afterEach, beforeAll, expect, mock, test } from "bun:test"; import { readFileSync } from "node:fs"; import { join } from "node:path"; -import { afterAll, afterEach, beforeAll, expect, mock, test } from "bun:test"; import { GlobalRegistrator } from "@happy-dom/global-registrator"; import { cleanup, render, waitFor, within } from "@testing-library/react"; import userEvent from "@testing-library/user-event"; @@ -708,6 +708,44 @@ test("setup that needs a CopilotKit sign-in offers it in the window, then carrie expect(commands().filter((c) => c === "set_up_local_eval")).toHaveLength(2); }); +/** + * A fresh Mac stopped at Start with "The back end stopped with an error partway through": Docker's + * disk was full, and the CLI's resource check failed. Its report, as `local_eval.rs` reads the + * CLI's `LOCAL_DOCKER_RESOURCES_SHORT` with "0 GiB free storage (requires 40)", now says the disk + * is short and where to raise it. + */ +test("a setup refused for a full Docker disk says so, with Retry", async () => { + invokeHandler = answers({ + set_up_local_eval: [ + report({ + outcome: "failed", + stack: null, + reason: "resources-short", + // As `local_eval.rs` reads the rc.10 CLI's "Docker available resources: 9.871234 CPUs + // (requires 4), 18.9 GiB RAM available (requires 4), 0 GiB free storage (requires 40)." + shortfall: { + podman: false, + cpu: false, + memory: false, + disk: true, + cpus: "9.8", + memoryGb: "18.9", + diskGb: "0", + }, + }), + ], + local_eval_download: [null], + cancel_local_eval: [null], + }); + const screen = view(); + await screen.findByRole("button", { name: "Retry" }); + const setup = row(screen, SETUP).textContent; + expect(setup).toContain( + "Docker has 0 GB of free disk. OpenBot needs 40 GB. Raise the disk limit in Docker Desktop > Settings > Resources, or free space in Docker, then Retry.", + ); + expect(setup).not.toContain("partway through"); +}); + test("a CopilotKit sign-in that does not finish says so and can be tried again", async () => { invokeHandler = answers({ set_up_local_eval: [blockedOnSignIn], diff --git a/desktop/src/localEvalCli.ts b/desktop/src/localEvalCli.ts index 4df8c341e9..dff5e699c3 100644 --- a/desktop/src/localEvalCli.ts +++ b/desktop/src/localEvalCli.ts @@ -55,6 +55,8 @@ export type LocalReason = // `local login` only; the menu says these as a fixed sentence. | "dashboard-unavailable" | "browser-open-failed" + // The CLI's resource check failed: the report's `shortfall` says which resource is short. + | "resources-short" | "unrecognized"; /** `local_eval::Readiness`: the CLI's own end-to-end check. Running pods are not this. */ @@ -78,8 +80,22 @@ export const localLicences = [ ] as const; export type LocalLicence = (typeof localLicences)[number]; +/** `local_eval::Shortfall`: which of the engine's resources are short, and whether it is Podman. */ +export type Shortfall = { + podman: boolean; + cpu: boolean; + memory: boolean; + disk: boolean; + /** What the CLI measured free, as it said it: CPUs, and GB of memory and of disk. */ + cpus?: string; + memoryGb?: string; + diskGb?: string; +}; + export type LocalReport = { outcome: LocalOutcome; + /** With the reason `resources-short`: which resources the CLI found short. */ + shortfall?: Shortfall; stack: null | { id: string; installedRelease: string; @@ -301,7 +317,9 @@ export type LocalTerms = } | { status: "not-required" | "unavailable" | "signed-out" } /** No container engine answers yet, so the CLI says nothing about the account. */ - | { status: "engine-not-ready" }; + | { status: "engine-not-ready" } + /** OpenBot's own check: the engine has fewer CPUs or less memory than local setup needs. */ + | ({ status: "engine-too-small" } & Shortfall); export function localEvalTerms(): { attempt: string; diff --git a/desktop/src/localReadiness.test.ts b/desktop/src/localReadiness.test.ts index 6881f8c40a..602bee84c1 100644 --- a/desktop/src/localReadiness.test.ts +++ b/desktop/src/localReadiness.test.ts @@ -6,6 +6,7 @@ import { type LocalReason, type LocalReport, localFailures, + type Shortfall, } from "./localEvalCli"; import { localReadiness, type ReadinessInput } from "./localReadiness"; @@ -27,6 +28,7 @@ const reasons: LocalReason[] = [ "confirmation-required", "dashboard-unavailable", "browser-open-failed", + "resources-short", "unrecognized", ]; @@ -86,7 +88,7 @@ test("every failure gives a plain, non-empty sentence without its own name or In }); test("the reason and failure lists in this test match the source", () => { - expect(new Set(reasons).size).toBe(18); + expect(new Set(reasons).size).toBe(19); expect(localFailures.length).toBe(18); }); @@ -173,6 +175,64 @@ test("specific wording and actions", () => { ).toBe( "The back end stopped with an error partway through. OpenBot's log has the details.", ); + // The CLI's resource check: which resource is short and where to raise it, on each engine. + const short = (shortfall: Partial) => + said( + report({ + outcome: "failed", + reason: "resources-short", + shortfall: { + podman: false, + cpu: false, + memory: false, + disk: false, + ...shortfall, + }, + }), + ); + expect(short({ disk: true })).toEqual({ + state: "blocked", + said: "Docker is out of disk space. OpenBot needs 40 GB free. Raise the disk limit in Docker Desktop > Settings > Resources, or free space in Docker, then Retry.", + action: "retry", + }); + expect(short({ cpu: true }).said).toBe( + "Docker doesn't have enough CPUs. OpenBot needs 4 free CPUs. Raise the CPU limit in Docker Desktop > Settings > Resources, then Retry.", + ); + expect(short({ memory: true }).said).toBe( + "Docker doesn't have enough memory. OpenBot needs 4 GB of free memory. Raise the memory limit in Docker Desktop > Settings > Resources, then Retry.", + ); + expect(short({ cpu: true, memory: true, disk: true }).said).toBe( + "Docker doesn't have enough CPUs, memory and disk space. OpenBot needs 4 free CPUs, 4 GB of free memory and 40 GB free. Raise the limits in Docker Desktop > Settings > Resources, or free space in Docker, then Retry.", + ); + expect(short({ podman: true, disk: true }).said).toBe( + "Podman is out of disk space. OpenBot needs 40 GB free. Stop your Podman machine, run podman machine set --disk-size 160, start it again, then Retry.", + ); + // With the CLI's own numbers, as `local_eval.rs` reads them from its error. + expect( + short({ disk: true, cpus: "9.8", memoryGb: "18.9", diskGb: "0" }).said, + ).toBe( + "Docker has 0 GB of free disk. OpenBot needs 40 GB. Raise the disk limit in Docker Desktop > Settings > Resources, or free space in Docker, then Retry.", + ); + expect( + short({ cpu: true, cpus: "3.7", memoryGb: "5.2", diskGb: "51" }).said, + ).toBe( + "Docker has 3.7 free CPUs. OpenBot needs 4. Raise the CPU limit in Docker Desktop > Settings > Resources, then Retry.", + ); + expect(short({ memory: true, memoryGb: "2.9" }).said).toBe( + "Docker has 2.9 GB of free memory. OpenBot needs 4 GB. Raise the memory limit in Docker Desktop > Settings > Resources, then Retry.", + ); + expect( + short({ cpu: true, disk: true, cpus: "3.5", memoryGb: "6", diskGb: "12" }) + .said, + ).toBe( + "Docker has 3.5 free CPUs and 12 GB of free disk. OpenBot needs 4 free CPUs and 40 GB of free disk. Raise the limits in Docker Desktop > Settings > Resources, or free space in Docker, then Retry.", + ); + // A report that does not say which is still about resources, never "partway through". + expect( + said(report({ outcome: "failed", reason: "resources-short" })).said, + ).toBe( + "Docker doesn't have enough CPUs, memory or disk space for the back end. Raise them in Docker Desktop > Settings > Resources, then Retry.", + ); expect(said("unavailable").action).toBeUndefined(); expect(said("unverified").action).toBe("repair"); expect(said("invalid-json")).toEqual({ diff --git a/desktop/src/localReadiness.ts b/desktop/src/localReadiness.ts index fc3c1cd289..a92e1fc4d7 100644 --- a/desktop/src/localReadiness.ts +++ b/desktop/src/localReadiness.ts @@ -16,6 +16,7 @@ import { type LocalReason, licenceExpiresSoon, onInstalledRelease, + type Shortfall, } from "./localEvalCli"; /** @@ -95,6 +96,110 @@ const quitUnexplained = blocked( "restart-intelligence", ); +type Resource = "cpu" | "memory" | "disk"; + +function listed(words: string[]): string { + return words.length < 2 + ? (words[0] ?? "") + : `${words.slice(0, -1).join(", ")} and ${words[words.length - 1]}`; +} + +const resourceName: Record = { + cpu: "CPUs", + memory: "memory", + disk: "disk space", +}; +const limitName: Record = { + cpu: "CPU", + memory: "memory", + disk: "disk", +}; +// What `podman machine set` raises each to, for the minimum below. +const podmanFlag: Record = { + cpu: "--cpus 6", + memory: "--memory 8192", + disk: "--disk-size 160", +}; + +/** + * Which resource is short and where to raise it. `needs` is what OpenBot needs of each: free, at + * Start, as the CLI measures it, or in total, before it. Disk alone is "out of disk space". + */ +function shortfallSaid( + shortfall: Shortfall, + needs: Record, + then: string, +): string { + const engine = shortfall.podman ? "Podman" : "Docker"; + const short = (["cpu", "memory", "disk"] as const).filter( + (resource) => shortfall[resource], + ); + if (short.length === 0) { + const fix = shortfall.podman + ? "Stop your Podman machine, raise them with podman machine set, start it again" + : "Raise them in Docker Desktop > Settings > Resources"; + return `${engine} doesn't have enough CPUs, memory or disk space for the back end. ${fix}, then ${then}.`; + } + const fix = shortfall.podman + ? `Stop your Podman machine, run podman machine set ${short.map((resource) => podmanFlag[resource]).join(" ")}, start it again` + : `Raise the ${short.length === 1 ? `${limitName[short[0]]} limit` : "limits"} in Docker Desktop > Settings > Resources${shortfall.disk ? ", or free space in Docker" : ""}`; + // The CLI's own numbers, when its error gave one for every resource that is short. + const measured: Record = { + cpu: shortfall.cpus && `${shortfall.cpus} free CPUs`, + memory: shortfall.memoryGb && `${shortfall.memoryGb} GB of free memory`, + disk: shortfall.diskGb && `${shortfall.diskGb} GB of free disk`, + }; + const has = short.map((resource) => measured[resource]); + const said = has.every((said) => said !== undefined) + ? `${engine} has ${listed(has as string[])}. OpenBot needs ${ + short.length === 1 + ? needed[short[0]] + : listed(short.map((resource) => neededFully[resource])) + }.` + : short.length === 1 && short[0] === "disk" + ? `${engine} is out of disk space. OpenBot needs ${needs.disk}.` + : `${engine} doesn't have enough ${listed(short.map((resource) => resourceName[resource]))}. OpenBot needs ${listed(short.map((resource) => needs[resource]))}.`; + return `${said} ${fix}, then ${then}.`; +} + +// What the CLI requires free, beside its own numbers: "Docker has 0 GB of free disk. OpenBot +// needs 40 GB." +const needed: Record = { + cpu: "4", + memory: "4 GB", + disk: "40 GB", +}; +const neededFully: Record = { + cpu: "4 free CPUs", + memory: "4 GB of free memory", + disk: "40 GB of free disk", +}; + +/** + * The CLI's resource check at Start: 4 free CPUs, 4 GB of free memory and 40 GB of free disk, with + * OpenBot's own containers running. + */ +export function resourcesShortSaid(shortfall: Shortfall): string { + return shortfallSaid( + shortfall, + { + cpu: "4 free CPUs", + memory: "4 GB of free memory", + disk: "40 GB free", + }, + "Retry", + ); +} + +/** OpenBot's own check before Start: the engine's totals, with room for its own containers. */ +export function engineTooSmallSaid(shortfall: Shortfall): string { + return shortfallSaid( + shortfall, + { cpu: "6 CPUs", memory: "8 GB of memory", disk: "40 GB free" }, + "Try again", + ); +} + // Compiler-enforced: a new upstream reason or failure breaks the build here. const reasonRows: Record = { "authentication-required": blocked("Sign in to CopilotKit.", "sign-in"), @@ -133,6 +238,16 @@ const reasonRows: Record = { "retry", ), "browser-open-failed": blocked("Your browser did not open.", "retry"), + // Said with the resource when the report names it: see `intelligenceRow`. + "resources-short": blocked( + resourcesShortSaid({ + podman: false, + cpu: false, + memory: false, + disk: false, + }), + "retry", + ), unrecognized: couldNotStart, }; @@ -222,6 +337,9 @@ function intelligenceRow( : "The back end is ready.", }; } + if (intelligence.reason === "resources-short" && intelligence.shortfall) { + return blocked(resourcesShortSaid(intelligence.shortfall), "retry"); + } if (intelligence.reason !== null) return reasonRows[intelligence.reason]; if (intelligence.outcome !== "completed") return couldNotStart; const licence = intelligence.licence;