diff --git a/.env.example b/.env.example index c1bb760ea6..8e53a488b1 100644 --- a/.env.example +++ b/.env.example @@ -165,6 +165,13 @@ GITTENSORY_REVIEW_DRAFT=false # PGVECTOR_ENABLED=false # set true only when using the Postgres pgvector table for RAG. # # Leave false when QDRANT_URL is set; Qdrant remains the preferred # # dedicated vector store for review context at scale. +# PGPOOL_MAX=10 # max concurrent Postgres connections in the pool shared by every +# # HTTP handler AND every queue worker's own DB traffic. 10 (this +# # default) is fine for a small/idle instance; raise it if you +# # register many repos or run higher QUEUE_CONCURRENCY, and you see +# # request/job latency climb without Postgres itself under pressure +# # (check GittensoryPostgresConnectionPressure and the app's own +# # request-latency metrics to tell those two cases apart). REDIS_URL=redis://redis:6379 # REQUIRED for the self-host review runtime. The default compose stack # # starts Redis automatically; override for an external Redis. # GITTENSORY_IMAGE=ghcr.io/jsonbored/gittensory-selfhost:latest # image used by scripts/deploy-selfhost-image.sh; diff --git a/apps/gittensory-ui/src/routes/docs.self-hosting-backup-scaling.tsx b/apps/gittensory-ui/src/routes/docs.self-hosting-backup-scaling.tsx index ed012253fb..422d53b5f2 100644 --- a/apps/gittensory-ui/src/routes/docs.self-hosting-backup-scaling.tsx +++ b/apps/gittensory-ui/src/routes/docs.self-hosting-backup-scaling.tsx @@ -88,6 +88,16 @@ REDIS_URL=redis://redis:6379 QDRANT_URL=http://qdrant:6333`} /> +

+ PgBouncer pools connections between instances and Postgres. Each app instance still + opens its own connection pool to whatever it's pointed at (PgBouncer or Postgres directly), + shared by every HTTP handler and queue worker in that instance — set PGPOOL_MAX{" "} + (default 10) if a single instance needs more headroom than that under real concurrency (many + registered repos, higher QUEUE_CONCURRENCY). Raise it gradually and watch for{" "} + GittensoryPostgresConnectionPressure: that alert means you're approaching + Postgres's own max_connections, a different ceiling than this per-instance pool + size. +

One-time SQLite to Postgres copy

diff --git a/src/selfhost/queue-common.ts b/src/selfhost/queue-common.ts index 141e5bf48a..d3fc47f809 100644 --- a/src/selfhost/queue-common.ts +++ b/src/selfhost/queue-common.ts @@ -568,6 +568,17 @@ export function queueStartupJitterMinJobs(): number { return parsePositiveIntEnv("QUEUE_STARTUP_JITTER_MIN_JOBS", { min: 0, fallback: DEFAULT_STARTUP_JITTER_MIN_JOBS }); } +// The Postgres pool (src/server.ts's buildPostgresBackend) is shared by every HTTP handler AND every queue +// worker's own DB traffic, including jobs that fan out several concurrent writes (e.g. +// hydrateMergedPullRequestFiles). 10 (pg's own hardcoded default, made explicit here rather than left +// implicit) is fine for a small/idle instance but can bottleneck the app on its own connection pool -- +// well before Postgres's own max_connections or the GittensoryPostgresConnectionPressure alert would fire +// -- once webhook bursts and fan-out jobs overlap at real volume. PGPOOL_MAX lets an operator raise this +// without a code change (#audit-rate-headroom). +export function resolvePostgresPoolMax(): number { + return parsePositiveIntEnv("PGPOOL_MAX", { min: 1, fallback: 10 }); +} + export function deterministicJitterMs(seed: string, maxJitterMs: number): number { if (!Number.isFinite(maxJitterMs) || maxJitterMs <= 0) return 0; let h = 2166136261; diff --git a/src/server.ts b/src/server.ts index 83c9a4d17c..ec5c21e8c5 100644 --- a/src/server.ts +++ b/src/server.ts @@ -48,6 +48,7 @@ import { runSelfHostMigrations } from "./selfhost/migrate"; import { createPgAdapter } from "./selfhost/pg-adapter"; import { createPgQueue } from "./selfhost/pg-queue"; import { createPgVectorize, initPgVectorize } from "./selfhost/pg-vectorize"; +import { resolvePostgresPoolMax } from "./selfhost/queue-common"; import { createSqliteQueue } from "./selfhost/sqlite-queue"; import { createSqliteVectorize } from "./selfhost/vectorize"; import { createFsBlobStore } from "./selfhost/blob-store"; @@ -204,7 +205,7 @@ async function buildPostgresBackend( await waitForPostgres(url); const pg = (await import("pg")).default; pg.types.setTypeParser(20, (v: string) => Number.parseInt(v, 10)); // int8 (COUNT) → number, like D1 - const pool = new pg.Pool({ connectionString: url }); + const pool = new pg.Pool({ connectionString: url, max: resolvePostgresPoolMax() }); const db = createPgAdapter(pool); const queue = createPgQueue(pool, consume); await queue.init(); diff --git a/test/unit/selfhost-queue-common.test.ts b/test/unit/selfhost-queue-common.test.ts index 6e150cc494..0780eba652 100644 --- a/test/unit/selfhost-queue-common.test.ts +++ b/test/unit/selfhost-queue-common.test.ts @@ -30,6 +30,7 @@ import { queueSnapshotFromBinding, queueStartupJitterMinJobs, queueStartupJitterMs, + resolvePostgresPoolMax, scheduledEnqueueDelaySeconds, scheduledEnqueueJitterMs, } from "../../src/selfhost/queue-common"; @@ -1169,3 +1170,26 @@ describe("parsePositiveIntEnv", () => { expect(warn).not.toHaveBeenCalled(); }); }); + +describe("resolvePostgresPoolMax (#audit-rate-headroom)", () => { + afterEach(() => { + delete process.env.PGPOOL_MAX; + }); + + it("defaults to 10 (pg's own hardcoded default, made explicit) when PGPOOL_MAX is unset", () => { + delete process.env.PGPOOL_MAX; + expect(resolvePostgresPoolMax()).toBe(10); + }); + + it("honors a valid PGPOOL_MAX override", () => { + process.env.PGPOOL_MAX = "25"; + expect(resolvePostgresPoolMax()).toBe(25); + }); + + it("falls back to the default for an invalid value, so a typo never disables pooling entirely", () => { + process.env.PGPOOL_MAX = "not-a-number"; + const warn = vi.spyOn(console, "warn").mockImplementation(() => {}); + expect(resolvePostgresPoolMax()).toBe(10); + expect(warn).toHaveBeenCalledOnce(); + }); +});