feat(selfhost): tune Ollama concurrency for a shared embed+vision GPU (#4327)

.env.example

@@ -320,7 +320,9 @@ REDIS_URL=redis://redis:6379                # REQUIRED for the self-host review
 # REDIS_MEM_LIMIT=512m                       # core app (always runs)
 # POSTGRES_MEM_LIMIT=2g                      # --profile postgres / --profile pgbouncer
 # QDRANT_MEM_LIMIT=2g                        # --profile qdrant
-# OLLAMA_MEM_LIMIT=8g                        # --profile ollama; raise this before pulling a large local model
+# OLLAMA_MEM_LIMIT=20g                       # --profile ollama; raise this before pulling a large local model
+#                                            #   (default sized for an embed model + one vision model resident
+#                                            #   together, #4335 -- raise further for additional/larger models)
 # PROMETHEUS_MEM_LIMIT=1g                    # --profile observability
 # LOKI_MEM_LIMIT=1g                          # --profile observability
 # TEMPO_MEM_LIMIT=1g                         # --profile observability
@@ -558,6 +560,13 @@ REDIS_URL=redis://redis:6379                # REQUIRED for the self-host review
 # OLLAMA_AI_API_KEY=
 # OLLAMA_AI_MODEL=llama3.1
 #
+# Ollama concurrency/residency (--profile ollama, docker-compose.yml; #4327/#4335). Defaults sized for a
+# single GPU running an embed model + one vision model concurrently -- see the comment above the ollama
+# service in docker-compose.yml for the full reasoning.
+# OLLAMA_NUM_PARALLEL=2                      # concurrent requests per loaded model
+# OLLAMA_MAX_LOADED_MODELS=2                 # distinct models kept resident at once
+# OLLAMA_KEEP_ALIVE=30m                      # how long an idle model stays loaded before eviction
+#
 # Generic OpenAI-compatible reviewer (AI_PROVIDER=openai-compatible). Defaults:
 # OPENAI_COMPATIBLE_AI_BASE_URL=http://localhost:11434/v1, OPENAI_COMPATIBLE_AI_MODEL=llama3.1.
 # OPENAI_COMPATIBLE_AI_BASE_URL=http://localhost:11434/v1

docker-compose.yml

@@ -348,12 +348,26 @@ services:
       timeout: 5s
       start_period: 15s
       retries: 5
-    # Ollama can load multi-GB models into memory; default high enough for common 7-8B quantized models but
-    # cap it so a large model pull can't exhaust host RAM and take down the core review pipeline (#1828).
+    # Concurrency/residency tuning for a shared embed+vision GPU deployment (#4327/#4335). Defaults sized
+    # for a single 24GB-class card running bge-m3 (embeddings) alongside one vision model concurrently --
+    # NUM_PARALLEL=2 keeps worst-case VRAM (resident weights + per-slot KV cache) well under 24GB; raising
+    # it on the vision model without also raising MAX_LOADED_MODELS/testing risks per-slot KV cache
+    # exhausting VRAM under burst load (latency collapse, not a hard OOM crash -- Ollama backpressures via
+    # its request queue instead). MAX_LOADED_MODELS pins exactly {embed model, vision model} resident so a
+    # stray third pull mid-burst can't force an unload/reload cycle on either one you actually need.
+    environment:
+      OLLAMA_NUM_PARALLEL: "${OLLAMA_NUM_PARALLEL:-2}"
+      OLLAMA_MAX_LOADED_MODELS: "${OLLAMA_MAX_LOADED_MODELS:-2}"
+      # Reviews land in bursts, not constant load -- keep both models warm across typical gaps between
+      # bursts instead of Ollama's 5m default forcing a cold-start reload every time.
+      OLLAMA_KEEP_ALIVE: "${OLLAMA_KEEP_ALIVE:-30m}"
+    # Ollama can load multi-GB models into memory; raised from the single-embedder-era default (8g) now that
+    # a vision model (#4335) shares this container's memory ceiling alongside bge-m3 -- still capped so a
+    # runaway pull can't exhaust host RAM and take down the core review pipeline (#1828).
     deploy:
       resources:
         limits:
-          memory: "${OLLAMA_MEM_LIMIT:-8g}"
+          memory: "${OLLAMA_MEM_LIMIT:-20g}"
 
   # ── Browserless / visual review (--profile visual-review) ──────────────────
   # Headless Chromium for automated before/after PR screenshot capture (src/review/visual/**).

test/unit/selfhost-compose-resource-limits.test.ts

@@ -19,7 +19,7 @@ describe("docker-compose.yml — per-service memory limits (#1828, #2495, #3893)
     redis: "${REDIS_MEM_LIMIT:-512m}",
     postgres: "${POSTGRES_MEM_LIMIT:-2g}",
     qdrant: "${QDRANT_MEM_LIMIT:-2g}",
-    ollama: "${OLLAMA_MEM_LIMIT:-8g}",
+    ollama: "${OLLAMA_MEM_LIMIT:-20g}", // raised from 8g (#4335): a vision model now shares this ceiling with bge-m3
     prometheus: "${PROMETHEUS_MEM_LIMIT:-1g}",
     loki: "${LOKI_MEM_LIMIT:-1g}",
     tempo: "${TEMPO_MEM_LIMIT:-1g}",
@@ -58,3 +58,31 @@ describe("docker-compose.yml — per-service memory limits (#1828, #2495, #3893)
     }
   });
 });
+
+// Concurrency/residency tuning for a shared embed+vision GPU deployment (#4327/#4335) — separate describe
+// block from the memory-limit checks above since these are `environment:` entries, not `deploy.resources`.
+describe("docker-compose.yml — ollama concurrency/residency env vars (#4327, #4335)", () => {
+  const EXPECTED_OLLAMA_ENV: Record<string, string> = {
+    OLLAMA_NUM_PARALLEL: "${OLLAMA_NUM_PARALLEL:-2}",
+    OLLAMA_MAX_LOADED_MODELS: "${OLLAMA_MAX_LOADED_MODELS:-2}",
+    OLLAMA_KEEP_ALIVE: "${OLLAMA_KEEP_ALIVE:-30m}",
+  };
+
+  it("sets an operator-overridable default for every ollama concurrency/residency variable", () => {
+    const compose = readYaml("docker-compose.yml");
+    const services = (compose.services as Record<string, Record<string, unknown>>) ?? {};
+    const environment = (services.ollama?.environment as Record<string, unknown>) ?? {};
+
+    for (const [key, expected] of Object.entries(EXPECTED_OLLAMA_ENV)) {
+      expect(environment[key], key).toBe(expected);
+    }
+  });
+
+  it("documents every ollama concurrency/residency override variable in .env.example", () => {
+    const env = readFileSync(".env.example", "utf8");
+
+    for (const key of Object.keys(EXPECTED_OLLAMA_ENV)) {
+      expect(env, key).toContain(key);
+    }
+  });
+});