diff --git a/.env.example b/.env.example index aa61778589..424be8605c 100644 --- a/.env.example +++ b/.env.example @@ -240,6 +240,10 @@ LOOPOVER_REVIEW_DRAFT=false # # scopes dashboard sign-in, not the shared MCP token's per-repo reach. # PORT=8787 # DATABASE_PATH=/data/loopover.sqlite # SQLite file on the mounted data volume; all migrations auto-apply +# LOOPOVER_REPO_CONFIG_DIR=/config # in-container dir the app reads per-repo .loopover.yml files from +# COMPOSE_PROJECT_NAME=loopover # Docker Compose's own project name; also labels the log stream +# # Promtail ships to Loki. Change it to run two stacks on one host. +# TZ=UTC # timezone passed to n8n (--profile workflows) for cron schedules # POSTGRES_PASSWORD=change-this-long-random-value # used by the --profile postgres / --profile pgbouncer services # DATABASE_URL= # set to postgres://user:pw@host:5432/db to use Postgres instead of # # SQLite (shared DB → multi-instance). Overrides DATABASE_PATH. @@ -348,6 +352,7 @@ REDIS_URL=redis://redis:6379 # REQUIRED for the self-host review # REDIS_MEM_LIMIT=512m # core app (always runs) # POSTGRES_MEM_LIMIT=2g # --profile postgres / --profile pgbouncer # QDRANT_MEM_LIMIT=2g # --profile qdrant +# REES_MEM_LIMIT=512m # --profile rees (the review-enrichment service) # OLLAMA_MEM_LIMIT=20g # --profile ollama; raise this before pulling a large local model # # (default sized for an embed model + one vision model resident # # together, #4335 -- raise further for additional/larger models) @@ -377,6 +382,15 @@ REDIS_URL=redis://redis:6379 # REQUIRED for the self-host review # acknowledged backup once you've wired Litestream (or an # equivalent) above. Default false (warning shown). +# --- Snapshot backups (--profile backup; the backup + backup-exporter sidecars) --- +# Separate from Litestream above: this runs scripts/backup.sh on a loop and exports the result as metrics. +# BACKUP_RETAIN=7 # how many snapshots to keep before pruning the oldest +# BACKUP_INTERVAL_SECONDS=86400 # seconds between runs (default 86400 = daily) +# Opt-in restore drill (scripts/verify-backup.sh): both are empty by default, which skips the drill. Set the +# pair together to restore each snapshot into a scratch database and prove it is actually restorable. +# VERIFY_RESTORE_SCRATCH= # non-empty enables the drill +# LOOPOVER_VERIFY_SCRATCH_DATABASE_URL= # throwaway scratch DB the drill restores into (never your live DB) + # --- n8n workflow automation (--profile workflows) --- # N8N_PASSWORD=changeme # REQUIRED at runtime when using --profile workflows # N8N_USER=admin @@ -519,6 +533,10 @@ REDIS_URL=redis://redis:6379 # REQUIRED for the self-host review # prometheus/rules/, routing in alertmanager/alertmanager.yml — silent until you fill in a receiver) + # Loki + Promtail (ship every container's logs to Loki) + Grafana (dashboards for metrics AND logs). # GRAFANA_ADMIN_PASSWORD= # REQUIRED at runtime when using --profile observability; generate a strong value +# PROMETHEUS_RETENTION_TIME=90d # metrics history window (a full quarter for trend/capacity review). +# # Deliberately longer than Loki's 14d / Tempo's 7d: the TSDB +# # compresses samples to ~1-2 bytes each, so metrics cost far less +# # disk per retained day than raw logs or span trees (#1828). # GRAFANA_LOCAL_SMOKE_PASSWORD= # optional local-only fallback for smoke tests; never expose Grafana with this # # Maintainer dashboards (in addition to the infra dashboard): @@ -527,6 +545,7 @@ REDIS_URL=redis://redis:6379 # REQUIRED for the self-host review # • "Resource hub" — links to every integrated service. # GRAFANA_REPORTING_EXPORT_INTERVAL_SECONDS=30 # refresh cadence for the redacted reporting SQLite export # LOOPOVER_REPORTING_SOURCE_DB=/appdb/loopover.sqlite # if DATABASE_PATH=/data/custom.sqlite, set /appdb/custom.sqlite +# LOOPOVER_REPORTING_DB=/reporting/loopover-reporting.sqlite # where the redacted export is written (Grafana reads this) # # AMS (loopover-miner) ledger dashboards — only useful when a miner ALSO runs on this same host (see # packages/loopover-miner/docs/observability.md). Requires --profile ams-observability, separate from the @@ -535,6 +554,13 @@ REDIS_URL=redis://redis:6379 # REQUIRED for the self-host review # snapshot (free-form attempt_log_events.reason/.payload_json dropped) into the same reporting volume. # LOOPOVER_MINER_CONFIG_DIR=~/.config/loopover-miner # host dir the exporter mounts read-only at /ams-ledgers # LOOPOVER_AMS_REPORTING_EXPORT_INTERVAL_SECONDS=30 # refresh cadence for the redacted AMS ledger exports +# Path overrides for the same exporter — in-container paths, not host paths. The source pair lives under the +# read-only /ams-ledgers mount above; the reporting pair is written into the shared reporting volume. Override +# only if your miner names its ledgers differently. +# LOOPOVER_AMS_ATTEMPT_LOG_SOURCE_DB=/ams-ledgers/attempt-log.sqlite3 +# LOOPOVER_AMS_PREDICTION_LEDGER_SOURCE_DB=/ams-ledgers/prediction-ledger.sqlite3 +# LOOPOVER_AMS_ATTEMPT_LOG_REPORTING_DB=/reporting/ams-attempt-log.sqlite +# LOOPOVER_AMS_PREDICTION_LEDGER_REPORTING_DB=/reporting/ams-prediction-ledger.sqlite # # Claude usage telemetry → OTEL collector → Prometheus → the Claude usage dashboard. OFF by default. # CLAUDE_CODE_ENABLE_TELEMETRY=1 # enable; needs --profile observability (starts the otel-collector) @@ -557,7 +583,11 @@ REDIS_URL=redis://redis:6379 # REQUIRED for the self-host review # # images bake LOOPOVER_VERSION=gittensory-selfhost@, so do # # not override SENTRY_RELEASE for those images. # OTEL_METRIC_EXPORT_INTERVAL=10000 # ms between metric exports (default 10s here; CLI default is 60s) +# OTEL_METRICS_EXPORTER=otlp # metrics-side counterpart to OTEL_TRACES_EXPORTER; empty = no app metrics # OTEL_EXPORTER_OTLP_ENDPOINT=http://otel-collector:4318 # override only for an external collector +# OTEL_EXPORTER_OTLP_PROTOCOL=http/protobuf # wire protocol for the endpoint above (http/protobuf | grpc) +# OTEL_EXPORTER_OTLP_METRICS_TEMPORALITY_PREFERENCE=cumulative # cumulative is what Prometheus expects; change +# # only for a backend that requires delta temporality # OTEL_SERVICE_NAME=gittensory-selfhost # # Live upstream PR/issue census (the GitHub data source). Install is automatic (GF_INSTALL_PLUGINS); add the @@ -603,6 +633,10 @@ REDIS_URL=redis://redis:6379 # REQUIRED for the self-host review # runtime-only: set AI_PROVIDER plus the provider-specific auth below. Set INSTALL_AI_CLIS=false only for a # custom minimal local build that will never use the subscription CLI providers. # INSTALL_AI_CLIS=true +# INSTALL_VISUAL_REVIEW=false # sibling build arg: installs puppeteer-core for a LOCAL build. +# # Official release images already bundle it, so this only matters +# # when you build the image yourself and want LOOPOVER_REVIEW_SCREENSHOTS +# # (the --profile visual-review browserless service is the alternative). # # Deprecated shared AI_* knobs are intentionally rejected at startup: # AI_BASE_URL, AI_API_KEY, AI_MODEL, AI_EFFORT, AI_TIMEOUT_MS. Use the explicit diff --git a/test/unit/docker-compose-env-example-parity.test.ts b/test/unit/docker-compose-env-example-parity.test.ts new file mode 100644 index 0000000000..bd5ab0db0c --- /dev/null +++ b/test/unit/docker-compose-env-example-parity.test.ts @@ -0,0 +1,80 @@ +import { readFileSync } from "node:fs"; +import { join } from "node:path"; + +import { describe, expect, it } from "vitest"; + +// Drift guard for #5814: docker-compose.yml's own header calls .env.example "the exhaustive reference", but +// ~18 variables compose actually interpolates were documented nowhere — the whole --profile backup block, the +// AMS exporter path overrides, the OTEL metrics-side siblings, PROMETHEUS_RETENTION_TIME, REES_MEM_LIMIT. This +// is a distinct check from `npm run selfhost:env-reference`, which scans process.env reads under +// src/selfhost/** and never looks at docker-compose.yml or .env.example at all. +// +// Every compose variable must be documented in .env.example (live `NAME=` or commented `# NAME=`) or, for the +// Docker-secrets `*_FILE` vars deliberately kept out of the sample file, in secrets/README.md's table. + +const ROOT = process.cwd(); +const composeText = readFileSync(join(ROOT, "docker-compose.yml"), "utf8"); +const envExampleText = readFileSync(join(ROOT, ".env.example"), "utf8"); +const secretsReadmeText = readFileSync(join(ROOT, "secrets/README.md"), "utf8"); + +/** + * `${VAR}` / `${VAR:-default}` / `${VAR:?err}` that compose really interpolates. + * + * The negative lookbehind is load-bearing: compose escapes a literal `$` as `$$`, so `$${GF_SECURITY_ADMIN_PASSWORD:-}` + * is passed through to the container's shell and resolved there — it is NOT a compose variable, and demanding + * `.env.example` document it would be wrong. + */ +function composeInterpolatedVars(compose: string): string[] { + return [...new Set([...compose.matchAll(/(? m[1]!))].sort(); +} + +/** Live `NAME=` and commented `# NAME=` reference lines alike — both count as documented. */ +function documentedInEnvExample(envExample: string): Set { + return new Set([...envExample.matchAll(/^\s*#?\s*([A-Z_][A-Z0-9_]*)=/gm)].map((m) => m[1]!)); +} + +/** The `*_FILE` vars in secrets/README.md's table, intentionally documented there instead of .env.example. */ +function documentedInSecretsReadme(secretsReadme: string): Set { + return new Set([...secretsReadme.matchAll(/\b([A-Z_][A-Z0-9_]*_FILE)\b/g)].map((m) => m[1]!)); +} + +function undocumentedComposeVars(compose: string, envExample: string, secretsReadme: string): string[] { + const documented = documentedInEnvExample(envExample); + const secrets = documentedInSecretsReadme(secretsReadme); + return composeInterpolatedVars(compose).filter((name) => !documented.has(name) && !secrets.has(name)); +} + +describe("docker-compose.yml ↔ .env.example parity (#5814)", () => { + it("finds the compose variables at all (guards the parser itself, so the check can't pass vacuously)", () => { + const vars = composeInterpolatedVars(composeText); + expect(vars.length).toBeGreaterThan(50); + expect(vars).toContain("PROMETHEUS_RETENTION_TIME"); + expect(documentedInEnvExample(envExampleText).size).toBeGreaterThan(50); + expect(documentedInSecretsReadme(secretsReadmeText)).toContain("GITHUB_APP_PRIVATE_KEY_FILE"); + }); + + it("ignores $$-escaped shell variables, which compose never interpolates", () => { + // `$${FOO}` reaches the container's shell as `${FOO}`; only the un-escaped `${BAR}` is a compose var. + expect(composeInterpolatedVars('a: "$${FOO:-x}"\nb: "${BAR:-y}"')).toEqual(["BAR"]); + }); + + it("INVARIANT: every compose variable is documented in .env.example or secrets/README.md", () => { + expect(undocumentedComposeVars(composeText, envExampleText, secretsReadmeText)).toEqual([]); + }); + + it("catches a real gap: a variable dropped from .env.example is reported (proves the guard works)", () => { + // Strip PROMETHEUS_RETENTION_TIME's documentation line — the check must notice. + const stripped = envExampleText + .split("\n") + .filter((line) => !/^\s*#?\s*PROMETHEUS_RETENTION_TIME=/.test(line)) + .join("\n"); + expect(undocumentedComposeVars(composeText, stripped, secretsReadmeText)).toEqual(["PROMETHEUS_RETENTION_TIME"]); + }); + + it("counts a *_FILE secret as documented via secrets/README.md, not .env.example", () => { + const compose = 'x: "${GITHUB_APP_PRIVATE_KEY_FILE}"'; + expect(undocumentedComposeVars(compose, "", secretsReadmeText)).toEqual([]); + // …and reports it when it is documented in neither. + expect(undocumentedComposeVars(compose, "", "")).toEqual(["GITHUB_APP_PRIVATE_KEY_FILE"]); + }); +});