diff --git a/.env.example b/.env.example index de474e33d1..ccf14d683f 100644 --- a/.env.example +++ b/.env.example @@ -320,7 +320,9 @@ REDIS_URL=redis://redis:6379 # REQUIRED for the self-host review # REDIS_MEM_LIMIT=512m # core app (always runs) # POSTGRES_MEM_LIMIT=2g # --profile postgres / --profile pgbouncer # QDRANT_MEM_LIMIT=2g # --profile qdrant -# OLLAMA_MEM_LIMIT=8g # --profile ollama; raise this before pulling a large local model +# OLLAMA_MEM_LIMIT=20g # --profile ollama; raise this before pulling a large local model +# # (default sized for an embed model + one vision model resident +# # together, #4335 -- raise further for additional/larger models) # PROMETHEUS_MEM_LIMIT=1g # --profile observability # LOKI_MEM_LIMIT=1g # --profile observability # TEMPO_MEM_LIMIT=1g # --profile observability @@ -558,6 +560,13 @@ REDIS_URL=redis://redis:6379 # REQUIRED for the self-host review # OLLAMA_AI_API_KEY= # OLLAMA_AI_MODEL=llama3.1 # +# Ollama concurrency/residency (--profile ollama, docker-compose.yml; #4327/#4335). Defaults sized for a +# single GPU running an embed model + one vision model concurrently -- see the comment above the ollama +# service in docker-compose.yml for the full reasoning. +# OLLAMA_NUM_PARALLEL=2 # concurrent requests per loaded model +# OLLAMA_MAX_LOADED_MODELS=2 # distinct models kept resident at once +# OLLAMA_KEEP_ALIVE=30m # how long an idle model stays loaded before eviction +# # Generic OpenAI-compatible reviewer (AI_PROVIDER=openai-compatible). Defaults: # OPENAI_COMPATIBLE_AI_BASE_URL=http://localhost:11434/v1, OPENAI_COMPATIBLE_AI_MODEL=llama3.1. # OPENAI_COMPATIBLE_AI_BASE_URL=http://localhost:11434/v1 diff --git a/docker-compose.yml b/docker-compose.yml index 91036dc0d9..8ab7100efb 100644 --- a/docker-compose.yml +++ b/docker-compose.yml @@ -348,12 +348,26 @@ services: timeout: 5s start_period: 15s retries: 5 - # Ollama can load multi-GB models into memory; default high enough for common 7-8B quantized models but - # cap it so a large model pull can't exhaust host RAM and take down the core review pipeline (#1828). + # Concurrency/residency tuning for a shared embed+vision GPU deployment (#4327/#4335). Defaults sized + # for a single 24GB-class card running bge-m3 (embeddings) alongside one vision model concurrently -- + # NUM_PARALLEL=2 keeps worst-case VRAM (resident weights + per-slot KV cache) well under 24GB; raising + # it on the vision model without also raising MAX_LOADED_MODELS/testing risks per-slot KV cache + # exhausting VRAM under burst load (latency collapse, not a hard OOM crash -- Ollama backpressures via + # its request queue instead). MAX_LOADED_MODELS pins exactly {embed model, vision model} resident so a + # stray third pull mid-burst can't force an unload/reload cycle on either one you actually need. + environment: + OLLAMA_NUM_PARALLEL: "${OLLAMA_NUM_PARALLEL:-2}" + OLLAMA_MAX_LOADED_MODELS: "${OLLAMA_MAX_LOADED_MODELS:-2}" + # Reviews land in bursts, not constant load -- keep both models warm across typical gaps between + # bursts instead of Ollama's 5m default forcing a cold-start reload every time. + OLLAMA_KEEP_ALIVE: "${OLLAMA_KEEP_ALIVE:-30m}" + # Ollama can load multi-GB models into memory; raised from the single-embedder-era default (8g) now that + # a vision model (#4335) shares this container's memory ceiling alongside bge-m3 -- still capped so a + # runaway pull can't exhaust host RAM and take down the core review pipeline (#1828). deploy: resources: limits: - memory: "${OLLAMA_MEM_LIMIT:-8g}" + memory: "${OLLAMA_MEM_LIMIT:-20g}" # ── Browserless / visual review (--profile visual-review) ────────────────── # Headless Chromium for automated before/after PR screenshot capture (src/review/visual/**). diff --git a/test/unit/selfhost-compose-resource-limits.test.ts b/test/unit/selfhost-compose-resource-limits.test.ts index 142e6e643b..ef38b72aa6 100644 --- a/test/unit/selfhost-compose-resource-limits.test.ts +++ b/test/unit/selfhost-compose-resource-limits.test.ts @@ -19,7 +19,7 @@ describe("docker-compose.yml — per-service memory limits (#1828, #2495, #3893) redis: "${REDIS_MEM_LIMIT:-512m}", postgres: "${POSTGRES_MEM_LIMIT:-2g}", qdrant: "${QDRANT_MEM_LIMIT:-2g}", - ollama: "${OLLAMA_MEM_LIMIT:-8g}", + ollama: "${OLLAMA_MEM_LIMIT:-20g}", // raised from 8g (#4335): a vision model now shares this ceiling with bge-m3 prometheus: "${PROMETHEUS_MEM_LIMIT:-1g}", loki: "${LOKI_MEM_LIMIT:-1g}", tempo: "${TEMPO_MEM_LIMIT:-1g}", @@ -58,3 +58,31 @@ describe("docker-compose.yml — per-service memory limits (#1828, #2495, #3893) } }); }); + +// Concurrency/residency tuning for a shared embed+vision GPU deployment (#4327/#4335) — separate describe +// block from the memory-limit checks above since these are `environment:` entries, not `deploy.resources`. +describe("docker-compose.yml — ollama concurrency/residency env vars (#4327, #4335)", () => { + const EXPECTED_OLLAMA_ENV: Record = { + OLLAMA_NUM_PARALLEL: "${OLLAMA_NUM_PARALLEL:-2}", + OLLAMA_MAX_LOADED_MODELS: "${OLLAMA_MAX_LOADED_MODELS:-2}", + OLLAMA_KEEP_ALIVE: "${OLLAMA_KEEP_ALIVE:-30m}", + }; + + it("sets an operator-overridable default for every ollama concurrency/residency variable", () => { + const compose = readYaml("docker-compose.yml"); + const services = (compose.services as Record>) ?? {}; + const environment = (services.ollama?.environment as Record) ?? {}; + + for (const [key, expected] of Object.entries(EXPECTED_OLLAMA_ENV)) { + expect(environment[key], key).toBe(expected); + } + }); + + it("documents every ollama concurrency/residency override variable in .env.example", () => { + const env = readFileSync(".env.example", "utf8"); + + for (const key of Object.keys(EXPECTED_OLLAMA_ENV)) { + expect(env, key).toContain(key); + } + }); +});