diff --git a/apps/gittensory-ui/src/routes/docs.self-hosting-operations.tsx b/apps/gittensory-ui/src/routes/docs.self-hosting-operations.tsx index b1ac9f8404..5c469dacec 100644 --- a/apps/gittensory-ui/src/routes/docs.self-hosting-operations.tsx +++ b/apps/gittensory-ui/src/routes/docs.self-hosting-operations.tsx @@ -78,12 +78,22 @@ review_context_fetch_failed`} The observability profile starts Prometheus, Alertmanager, Loki, Promtail, and Grafana with dashboards for infra, review activity, and AI usage.

+

+ Postgres installs also expose database internals through the bundled Postgres exporter: + connection pressure, lock waits, long transactions, deadlocks, database/table growth, dead + tuples, autovacuum activity, and backup freshness. Backup freshness appears when the{" "} + backup profile is active. +

When OpenTelemetry and Sentry are enabled, job audit logs and Sentry events include trace_id/span_id fields so an operator can jump from a failed job or issue to the matching trace in Grafana or Tempo.

- +

Sentry tracing

@@ -129,6 +139,10 @@ review_context_fetch_failed`}

  • Webhook deliveries are recent and have 2xx responses.
  • AI usage matches expected review volume and model/effort choices.
  • REES and RAG failures are visible and bounded.
    • +
  • + Postgres connections, lock waits, slow transactions, dead tuples, and table growth are + stable. +
  • Backups are recent and restore-tested.
    • diff --git a/docker-compose.yml b/docker-compose.yml index cebc8d1d1f..00801edb2d 100644 --- a/docker-compose.yml +++ b/docker-compose.yml @@ -162,6 +162,22 @@ services: DEFAULT_POOL_SIZE: "20" AUTH_TYPE: md5 + # Postgres internals exporter. Starts with the Postgres profiles so Prometheus can scrape it when + # observability is enabled, without starting against a missing DB on SQLite-only installs. + postgres-exporter: + image: quay.io/prometheuscommunity/postgres-exporter:v0.20.0 + restart: unless-stopped + profiles: ["postgres", "pgbouncer"] + depends_on: + postgres: + condition: service_healthy + environment: + DATA_SOURCE_URI: "postgres:5432/gittensory?sslmode=disable" + DATA_SOURCE_USER: gittensory + DATA_SOURCE_PASS: ${POSTGRES_PASSWORD:-CHANGEME} + expose: + - "9187" + # ── Qdrant (--profile qdrant) ───────────────────────────────────────────── # Dedicated vector database for RAG — replaces the built-in sqlite-vec / pgvector when # QDRANT_URL=http://qdrant:6333 is set. Scales to millions of vectors with ANN search. @@ -528,6 +544,26 @@ services: command: - "apk add --no-cache sqlite postgresql16-client curl >/dev/null 2>&1 && while true; do sh /backup.sh || echo '[backup] run failed'; sleep ${BACKUP_INTERVAL_SECONDS:-86400}; done" + # Read-only backup freshness exporter. It serves file-age metrics from the retained backup volume so + # Prometheus can alert on missing or stale database backups without Grafana reading the live app DB. + backup-exporter: + image: alpine:3.20 + restart: unless-stopped + profiles: ["backup"] + volumes: + - gittensory-backups:/backups:ro + - ./scripts/backup-metrics.sh:/backup-metrics.sh:ro + expose: + - "9101" + command: + - /bin/sh + - -c + - "sh /backup-metrics.sh" + healthcheck: + test: ["CMD-SHELL", "wget -qO- http://127.0.0.1:9101/metrics | grep -q '^gittensory_backup_latest_timestamp_seconds'"] + interval: 30s + retries: 5 + volumes: gittensory-data: gittensory-pg: diff --git a/grafana/dashboards/gittensory.json b/grafana/dashboards/gittensory.json index 9aa83989f2..039f85abab 100644 --- a/grafana/dashboards/gittensory.json +++ b/grafana/dashboards/gittensory.json @@ -1623,6 +1623,501 @@ ], "title": "Queue Rate-Limit Deferrals", "type": "timeseries" + }, + { + "collapsed": false, + "gridPos": { + "h": 1, + "w": 24, + "x": 0, + "y": 107 + }, + "id": 118, + "title": "Postgres & Backups", + "type": "row" + }, + { + "datasource": { + "type": "prometheus", + "uid": "${DS_PROMETHEUS}" + }, + "fieldConfig": { + "defaults": { + "color": { "mode": "thresholds" }, + "thresholds": { + "mode": "absolute", + "steps": [ + { "color": "red", "value": null }, + { "color": "green", "value": 1 } + ] + }, + "unit": "short" + } + }, + "gridPos": { + "h": 4, + "w": 6, + "x": 0, + "y": 108 + }, + "id": 119, + "options": { + "colorMode": "background", + "graphMode": "none", + "justifyMode": "auto", + "orientation": "auto", + "reduceOptions": { + "calcs": ["lastNotNull"], + "fields": "", + "values": false + }, + "textMode": "auto" + }, + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "${DS_PROMETHEUS}" + }, + "expr": "pg_up or vector(0)", + "legendFormat": "up", + "refId": "A" + } + ], + "title": "Postgres Exporter Up", + "type": "stat" + }, + { + "datasource": { + "type": "prometheus", + "uid": "${DS_PROMETHEUS}" + }, + "fieldConfig": { + "defaults": { + "color": { "mode": "thresholds" }, + "thresholds": { + "mode": "absolute", + "steps": [ + { "color": "green", "value": null }, + { "color": "yellow", "value": 60 }, + { "color": "red", "value": 120 } + ] + }, + "unit": "short" + } + }, + "gridPos": { + "h": 4, + "w": 6, + "x": 6, + "y": 108 + }, + "id": 120, + "options": { + "colorMode": "background", + "graphMode": "area", + "justifyMode": "auto", + "orientation": "auto", + "reduceOptions": { + "calcs": ["lastNotNull"], + "fields": "", + "values": false + }, + "textMode": "auto" + }, + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "${DS_PROMETHEUS}" + }, + "expr": "sum(pg_stat_activity_count{datname=\"gittensory\"}) or vector(0)", + "legendFormat": "connections", + "refId": "A" + } + ], + "title": "Postgres Connections", + "type": "stat" + }, + { + "datasource": { + "type": "prometheus", + "uid": "${DS_PROMETHEUS}" + }, + "fieldConfig": { + "defaults": { + "color": { "mode": "thresholds" }, + "thresholds": { + "mode": "absolute", + "steps": [{ "color": "green", "value": null }] + }, + "unit": "decbytes" + } + }, + "gridPos": { + "h": 4, + "w": 6, + "x": 12, + "y": 108 + }, + "id": 121, + "options": { + "colorMode": "background", + "graphMode": "area", + "justifyMode": "auto", + "orientation": "auto", + "reduceOptions": { + "calcs": ["lastNotNull"], + "fields": "", + "values": false + }, + "textMode": "auto" + }, + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "${DS_PROMETHEUS}" + }, + "expr": "pg_database_size_bytes{datname=\"gittensory\"} or vector(0)", + "legendFormat": "size", + "refId": "A" + } + ], + "title": "Postgres DB Size", + "type": "stat" + }, + { + "datasource": { + "type": "prometheus", + "uid": "${DS_PROMETHEUS}" + }, + "fieldConfig": { + "defaults": { + "color": { "mode": "thresholds" }, + "thresholds": { + "mode": "absolute", + "steps": [ + { "color": "green", "value": null }, + { "color": "yellow", "value": 86400 }, + { "color": "red", "value": 93600 } + ] + }, + "unit": "s" + } + }, + "gridPos": { + "h": 4, + "w": 6, + "x": 18, + "y": 108 + }, + "id": 122, + "options": { + "colorMode": "background", + "graphMode": "area", + "justifyMode": "auto", + "orientation": "auto", + "reduceOptions": { + "calcs": ["lastNotNull"], + "fields": "", + "values": false + }, + "textMode": "auto" + }, + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "${DS_PROMETHEUS}" + }, + "expr": "(time() - max(gittensory_backup_latest_timestamp_seconds{target=~\"postgres|sqlite\"})) and max(gittensory_backup_latest_timestamp_seconds{target=~\"postgres|sqlite\"}) > 0", + "legendFormat": "age", + "refId": "A" + } + ], + "title": "Latest DB Backup Age", + "type": "stat" + }, + { + "datasource": { + "type": "prometheus", + "uid": "${DS_PROMETHEUS}" + }, + "fieldConfig": { + "defaults": { + "color": { "mode": "palette-classic" }, + "custom": { + "fillOpacity": 10, + "lineWidth": 2 + }, + "unit": "short" + } + }, + "gridPos": { + "h": 8, + "w": 12, + "x": 0, + "y": 112 + }, + "id": 123, + "options": { + "legend": { + "calcs": ["lastNotNull"], + "displayMode": "table", + "placement": "bottom" + }, + "tooltip": { + "mode": "multi", + "sort": "desc" + } + }, + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "${DS_PROMETHEUS}" + }, + "expr": "sum by (state) (pg_stat_activity_count{datname=\"gittensory\"}) or vector(0)", + "legendFormat": "{{state}}", + "refId": "A" + } + ], + "title": "Postgres Connections by State", + "type": "timeseries" + }, + { + "datasource": { + "type": "prometheus", + "uid": "${DS_PROMETHEUS}" + }, + "fieldConfig": { + "defaults": { + "color": { "mode": "palette-classic" }, + "custom": { + "fillOpacity": 10, + "lineWidth": 2 + }, + "unit": "short" + } + }, + "gridPos": { + "h": 8, + "w": 12, + "x": 12, + "y": 112 + }, + "id": 124, + "options": { + "legend": { + "calcs": ["lastNotNull"], + "displayMode": "table", + "placement": "bottom" + }, + "tooltip": { + "mode": "multi", + "sort": "desc" + } + }, + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "${DS_PROMETHEUS}" + }, + "expr": "sum(pg_stat_activity_count{datname=\"gittensory\", wait_event_type=\"Lock\"}) or vector(0)", + "legendFormat": "lock waits", + "refId": "A" + }, + { + "datasource": { + "type": "prometheus", + "uid": "${DS_PROMETHEUS}" + }, + "expr": "max(pg_stat_activity_max_tx_duration{datname=\"gittensory\"}) or vector(0)", + "legendFormat": "max transaction seconds", + "refId": "B" + }, + { + "datasource": { + "type": "prometheus", + "uid": "${DS_PROMETHEUS}" + }, + "expr": "rate(pg_stat_database_deadlocks{datname=\"gittensory\"}[5m]) or vector(0)", + "legendFormat": "deadlocks/s", + "refId": "C" + } + ], + "title": "Postgres Locks & Slow Transactions", + "type": "timeseries" + }, + { + "datasource": { + "type": "prometheus", + "uid": "${DS_PROMETHEUS}" + }, + "fieldConfig": { + "defaults": { + "color": { "mode": "palette-classic" }, + "custom": { + "fillOpacity": 10, + "lineWidth": 2 + }, + "unit": "short" + } + }, + "gridPos": { + "h": 8, + "w": 12, + "x": 0, + "y": 120 + }, + "id": 125, + "options": { + "legend": { + "calcs": ["lastNotNull"], + "displayMode": "table", + "placement": "bottom" + }, + "tooltip": { + "mode": "multi", + "sort": "desc" + } + }, + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "${DS_PROMETHEUS}" + }, + "expr": "pg_database_size_bytes{datname=\"gittensory\"} or vector(0)", + "legendFormat": "database bytes", + "refId": "A" + }, + { + "datasource": { + "type": "prometheus", + "uid": "${DS_PROMETHEUS}" + }, + "expr": "topk(10, pg_stat_user_tables_n_live_tup{datname=\"gittensory\"}) or vector(0)", + "legendFormat": "{{relname}} live rows", + "refId": "B" + } + ], + "title": "Postgres Size & Table Growth", + "type": "timeseries" + }, + { + "datasource": { + "type": "prometheus", + "uid": "${DS_PROMETHEUS}" + }, + "fieldConfig": { + "defaults": { + "color": { "mode": "palette-classic" }, + "custom": { + "fillOpacity": 10, + "lineWidth": 2 + }, + "unit": "short" + } + }, + "gridPos": { + "h": 8, + "w": 12, + "x": 12, + "y": 120 + }, + "id": 126, + "options": { + "legend": { + "calcs": ["lastNotNull"], + "displayMode": "table", + "placement": "bottom" + }, + "tooltip": { + "mode": "multi", + "sort": "desc" + } + }, + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "${DS_PROMETHEUS}" + }, + "expr": "topk(10, pg_stat_user_tables_n_dead_tup{datname=\"gittensory\"}) or vector(0)", + "legendFormat": "{{relname}} dead rows", + "refId": "A" + }, + { + "datasource": { + "type": "prometheus", + "uid": "${DS_PROMETHEUS}" + }, + "expr": "sum by (relname) (increase(pg_stat_user_tables_autovacuum_count{datname=\"gittensory\"}[1h])) or vector(0)", + "legendFormat": "{{relname}} autovacuum/hour", + "refId": "B" + } + ], + "title": "Dead Tuples / Autovacuum", + "type": "timeseries" + }, + { + "datasource": { + "type": "prometheus", + "uid": "${DS_PROMETHEUS}" + }, + "fieldConfig": { + "defaults": { + "color": { "mode": "palette-classic" }, + "custom": { + "fillOpacity": 10, + "lineWidth": 2 + }, + "unit": "short" + } + }, + "gridPos": { + "h": 8, + "w": 24, + "x": 0, + "y": 128 + }, + "id": 127, + "options": { + "legend": { + "calcs": ["lastNotNull"], + "displayMode": "table", + "placement": "bottom" + }, + "tooltip": { + "mode": "multi", + "sort": "desc" + } + }, + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "${DS_PROMETHEUS}" + }, + "expr": "(time() - gittensory_backup_latest_timestamp_seconds{target=~\"postgres|sqlite|qdrant\"}) and gittensory_backup_latest_timestamp_seconds{target=~\"postgres|sqlite|qdrant\"} > 0", + "legendFormat": "{{target}} age seconds", + "refId": "A" + }, + { + "datasource": { + "type": "prometheus", + "uid": "${DS_PROMETHEUS}" + }, + "expr": "gittensory_backup_files{target=~\"postgres|sqlite|qdrant\"} or vector(0)", + "legendFormat": "{{target}} files", + "refId": "B" + } + ], + "title": "Backup Freshness", + "type": "timeseries" } ], "refresh": "30s", diff --git a/prometheus/prometheus.yml b/prometheus/prometheus.yml index 25b85de73d..6608cdd059 100644 --- a/prometheus/prometheus.yml +++ b/prometheus/prometheus.yml @@ -23,6 +23,20 @@ scrape_configs: scrape_interval: 15s scrape_timeout: 10s + # Postgres internals from prometheus-community/postgres_exporter. The exporter service starts only with + # the Postgres profiles; SQLite-only installs show this target as absent/down without firing alerts. + - job_name: postgres + static_configs: + - targets: ["postgres-exporter:9187"] + scrape_interval: 30s + + # Backup freshness from the read-only backup-exporter sidecar. The metrics exist only when the backup + # profile is active, which keeps backup alerts opt-in with the backup feature. + - job_name: gittensory-backup + static_configs: + - targets: ["backup-exporter:9101"] + scrape_interval: 60s + # Claude Code usage telemetry, re-exposed by the OTEL collector's Prometheus exporter (--profile # observability + CLAUDE_CODE_ENABLE_TELEMETRY=1). An absent collector just shows the target as down. - job_name: claude-code-otel diff --git a/prometheus/rules/alerts.yml b/prometheus/rules/alerts.yml index 212300cf70..20c85902f4 100644 --- a/prometheus/rules/alerts.yml +++ b/prometheus/rules/alerts.yml @@ -130,6 +130,120 @@ groups: description: "GitHub rate-limit deferrals exceeded 0.05/s for 10m. The queue is protecting the bucket but throughput is GitHub-budget bound." runbook: "Use gittensory_jobs_rate_limit_* labels (kind/key_scope/job_type) to identify the stuck class. Prefer reducing duplicate webhook/regate work or adding cache coverage before raising worker concurrency." + # ── Postgres database + backup freshness ─────────────────────────────────── + - name: gittensory-postgres + rules: + - alert: GittensoryPostgresConnectionPressure + # Postgres exporter is present only for Postgres-backed installs. Fire when the app DB + # consumes more than 80% of max_connections for 10m; use PgBouncer or lower app concurrency. + expr: | + ( + sum(pg_stat_activity_count{datname="gittensory"}) + / + max(pg_settings_max_connections) + ) > 0.80 + for: 10m + labels: + severity: warning + annotations: + summary: "Postgres connection usage is above 80%" + description: "The gittensory database is using {{ $value | humanizePercentage }} of max_connections for over 10m." + runbook: "Enable the pgbouncer profile or reduce self-host worker/app concurrency. Check the Postgres Connections panel by state before increasing max_connections." + + - alert: GittensoryPostgresLockWaits + # Wait-event labels are emitted by the postgres_exporter stat_activity collector. Any sustained + # lock wait means a write path is blocked and queue processing can appear stuck. + expr: sum(pg_stat_activity_count{datname="gittensory", wait_event_type="Lock"}) > 0 + for: 5m + labels: + severity: warning + annotations: + summary: "Postgres sessions are waiting on locks" + description: "{{ $value | printf \"%.0f\" }} gittensory database session(s) have waited on Postgres locks for over 5m." + runbook: "Inspect the Postgres Locks and Slow Transactions panels. Look for long transactions, migration jobs, or stuck writers before restarting workers." + + - alert: GittensoryPostgresSlowTransaction + # Long active transactions are the self-host proxy for slow/problem queries without requiring + # pg_stat_statements or superuser-only setup. + expr: max(pg_stat_activity_max_tx_duration{datname="gittensory"}) > 120 + for: 5m + labels: + severity: warning + annotations: + summary: "Postgres transaction has been active for over 2m" + description: "The longest active gittensory database transaction is {{ $value | printf \"%.0f\" }}s old." + runbook: "Check app logs around DB-heavy jobs and review whether queue workers are saturating Postgres. A single long transaction can block cleanup or migrations." + + - alert: GittensoryPostgresDeadlocks + # Deadlocks should be effectively zero; even a low rate points at competing write paths. + expr: rate(pg_stat_database_deadlocks{datname="gittensory"}[5m]) > 0 + for: 2m + labels: + severity: warning + annotations: + summary: "Postgres deadlocks detected" + description: "Postgres deadlocks are occurring for the gittensory database." + runbook: "Correlate with deploys and queue write paths. Repeated deadlocks usually mean a transaction ordering bug or too much parallel write pressure." + + - alert: GittensoryPostgresDatabaseGrowingFast + # Growth over a 6h window smooths normal small-table churn while catching runaway queue/audit growth. + # 262144 bytes/s is roughly 5.4 GiB over 6h. + expr: deriv(pg_database_size_bytes{datname="gittensory"}[6h]) > 262144 + for: 30m + labels: + severity: warning + annotations: + summary: "Postgres database is growing quickly" + description: "The gittensory database is growing at about {{ $value | humanize }} bytes/s over a 6h trend window." + runbook: "Open the Postgres Table Growth panel and inspect queue/audit/event tables first. Confirm retention and backup volume capacity before raising traffic." + + - alert: GittensoryPostgresDeadTuplesHigh + # Autovacuum pressure: high dead tuple ratio and enough absolute dead rows to avoid tiny-table noise. + expr: | + ( + sum(pg_stat_user_tables_n_dead_tup{datname="gittensory"}) + / + clamp_min( + sum(pg_stat_user_tables_n_live_tup{datname="gittensory"} + pg_stat_user_tables_n_dead_tup{datname="gittensory"}), + 1 + ) + ) > 0.20 + and + sum(pg_stat_user_tables_n_dead_tup{datname="gittensory"}) > 10000 + for: 30m + labels: + severity: warning + annotations: + summary: "Postgres dead tuples are high" + description: "More than 20% of table tuples are dead and at least 10k dead tuples remain for over 30m." + runbook: "Check the Dead Tuples / Autovacuum panel. If autovacuum counts are flat while dead tuples rise, reduce write pressure or tune autovacuum for the largest tables." + + - alert: GittensoryBackupMissing + # Fires only when the backup-exporter service is running (backup profile enabled) and no retained + # DB backup exists yet. Qdrant backups are best-effort and not included in this DB backup alert. + expr: gittensory_backup_files{target=~"postgres|sqlite"} == 0 + for: 2h + labels: + severity: warning + annotations: + summary: "No retained gittensory database backup found" + description: "The backup profile is exposing metrics, but {{ $labels.target }} has no retained backup file after 2h." + runbook: "Run `docker compose --profile backup run --rm backup sh /backup.sh` and inspect the gittensory-backups volume. For Postgres, confirm DATABASE_URL is available to the backup service." + + - alert: GittensoryBackupStale + # Default backup loop is daily. 26h allows one missed scrape/restart window before warning. + expr: | + (time() - gittensory_backup_latest_timestamp_seconds{target=~"postgres|sqlite"} > 93600) + and + gittensory_backup_latest_timestamp_seconds{target=~"postgres|sqlite"} > 0 + for: 30m + labels: + severity: warning + annotations: + summary: "gittensory database backup is stale" + description: "The newest retained {{ $labels.target }} backup is older than 26h." + runbook: "Check `docker compose --profile backup logs backup` and run the backup command manually. Pair this with the restore verification drill before deleting old backups." + # ── Qdrant vector backend ────────────────────────────────────────────────── - name: gittensory-qdrant rules: diff --git a/scripts/backup-metrics.sh b/scripts/backup-metrics.sh new file mode 100644 index 0000000000..ad5fb6d621 --- /dev/null +++ b/scripts/backup-metrics.sh @@ -0,0 +1,75 @@ +#!/bin/sh +# Expose retained backup freshness from the gittensory-backups volume in Prometheus text format. +# Intended for the backup-exporter compose sidecar; it never touches the live database. +set -eu + +ROOT=${BACKUP_ROOT:-/backups} +OUT=${BACKUP_METRICS_DIR:-/metrics} +FILE=${BACKUP_METRICS_FILE:-$OUT/metrics} +INTERVAL=${BACKUP_METRICS_INTERVAL_SECONDS:-30} +PORT=${BACKUP_METRICS_PORT:-9101} + +case "$INTERVAL" in + ''|*[!0-9]*) INTERVAL=30 ;; +esac + +latest_timestamp() { + dir=$1 + if [ ! -d "$dir" ]; then + echo 0 + return + fi + newest=$( + find "$dir" -type f ! -name '*.tmp' -exec sh -c ' + for path do + stat -c "%Y" "$path" 2>/dev/null || stat -f "%m" "$path" 2>/dev/null || true + done + ' sh {} + 2>/dev/null | sort -nr | head -1 || true + ) + case "$newest" in + ''|*[!0-9]*) echo 0 ;; + *) echo "$newest" ;; + esac +} + +file_count() { + dir=$1 + if [ ! -d "$dir" ]; then + echo 0 + return + fi + find "$dir" -type f ! -name '*.tmp' 2>/dev/null | wc -l | tr -d ' ' +} + +write_metrics() { + mkdir -p "$OUT" + tmp="$FILE.tmp" + { + echo "# HELP gittensory_backup_latest_timestamp_seconds Unix timestamp of the newest retained self-host backup file by target." + echo "# TYPE gittensory_backup_latest_timestamp_seconds gauge" + for target in postgres sqlite qdrant; do + echo "gittensory_backup_latest_timestamp_seconds{target=\"$target\"} $(latest_timestamp "$ROOT/$target")" + done + echo "# HELP gittensory_backup_files Retained self-host backup files by target." + echo "# TYPE gittensory_backup_files gauge" + for target in postgres sqlite qdrant; do + echo "gittensory_backup_files{target=\"$target\"} $(file_count "$ROOT/$target")" + done + } > "$tmp" + mv "$tmp" "$FILE" +} + +if [ "${BACKUP_METRICS_ONCE:-}" = "1" ]; then + write_metrics + exit 0 +fi + +write_metrics +httpd -f -p "$PORT" -h "$OUT" & +server=$! +trap 'kill "$server" 2>/dev/null || true' INT TERM EXIT + +while true; do + write_metrics + sleep "$INTERVAL" +done diff --git a/test/unit/selfhost-backup-metrics-script.test.ts b/test/unit/selfhost-backup-metrics-script.test.ts new file mode 100644 index 0000000000..794375e2be --- /dev/null +++ b/test/unit/selfhost-backup-metrics-script.test.ts @@ -0,0 +1,73 @@ +import { execFileSync } from "node:child_process"; +import { mkdirSync, mkdtempSync, readFileSync, rmSync, utimesSync, writeFileSync } from "node:fs"; +import { tmpdir } from "node:os"; +import { join } from "node:path"; +import { afterEach, describe, expect, it } from "vitest"; + +const tmpRoots: string[] = []; + +function tmpRoot(): string { + const dir = mkdtempSync(join(tmpdir(), "gittensory-backup-metrics-")); + tmpRoots.push(dir); + return dir; +} + +function writeBackup(root: string, target: "postgres" | "sqlite" | "qdrant", name: string, timestamp: string): number { + const dir = join(root, target); + mkdirSync(dir, { recursive: true }); + const file = join(dir, name); + writeFileSync(file, `${target} backup\n`); + const when = new Date(timestamp); + utimesSync(file, when, when); + return Math.floor(when.getTime() / 1000); +} + +function runExporterOnce(root: string): string { + const out = join(root, "metrics"); + execFileSync("sh", ["scripts/backup-metrics.sh"], { + cwd: process.cwd(), + env: { + ...process.env, + BACKUP_ROOT: root, + BACKUP_METRICS_DIR: out, + BACKUP_METRICS_ONCE: "1", + }, + }); + return readFileSync(join(out, "metrics"), "utf8"); +} + +afterEach(() => { + for (const dir of tmpRoots.splice(0)) rmSync(dir, { force: true, recursive: true }); +}); + +describe("backup-metrics.sh", () => { + it("exports newest retained backup timestamps and file counts by target", () => { + const root = tmpRoot(); + writeBackup(root, "postgres", "gittensory-older.dump", "2026-07-01T01:00:00Z"); + const newestPostgres = writeBackup(root, "postgres", "gittensory-newer.dump", "2026-07-01T02:00:00Z"); + const sqlite = writeBackup(root, "sqlite", "gittensory.sqlite.gz", "2026-07-01T03:00:00Z"); + const qdrant = writeBackup(root, "qdrant", "snapshot", "2026-07-01T04:00:00Z"); + + const metrics = runExporterOnce(root); + + expect(metrics).toContain(`gittensory_backup_latest_timestamp_seconds{target="postgres"} ${newestPostgres}`); + expect(metrics).toContain(`gittensory_backup_latest_timestamp_seconds{target="sqlite"} ${sqlite}`); + expect(metrics).toContain(`gittensory_backup_latest_timestamp_seconds{target="qdrant"} ${qdrant}`); + expect(metrics).toContain('gittensory_backup_files{target="postgres"} 2'); + expect(metrics).toContain('gittensory_backup_files{target="sqlite"} 1'); + expect(metrics).toContain('gittensory_backup_files{target="qdrant"} 1'); + }); + + it("exports zeroes for missing backup directories instead of failing the scrape", () => { + const root = tmpRoot(); + + const metrics = runExporterOnce(root); + + expect(metrics).toContain('gittensory_backup_latest_timestamp_seconds{target="postgres"} 0'); + expect(metrics).toContain('gittensory_backup_latest_timestamp_seconds{target="sqlite"} 0'); + expect(metrics).toContain('gittensory_backup_latest_timestamp_seconds{target="qdrant"} 0'); + expect(metrics).toContain('gittensory_backup_files{target="postgres"} 0'); + expect(metrics).toContain('gittensory_backup_files{target="sqlite"} 0'); + expect(metrics).toContain('gittensory_backup_files{target="qdrant"} 0'); + }); +}); diff --git a/test/unit/selfhost-grafana-dashboard.test.ts b/test/unit/selfhost-grafana-dashboard.test.ts index cd3b1830db..6f58b78021 100644 --- a/test/unit/selfhost-grafana-dashboard.test.ts +++ b/test/unit/selfhost-grafana-dashboard.test.ts @@ -13,6 +13,7 @@ type DashboardTarget = { type DashboardPanel = { id?: number; + title?: string; targets?: DashboardTarget[]; }; @@ -106,6 +107,45 @@ describe("Gittensory Self-Host Grafana dashboard", () => { expect(alerts).toContain("sum by (kind, key_scope, job_type) (rate(gittensory_jobs_rate_limit_admission_deferred_total[5m])) > 0.05"); expect(alerts).toContain("sum by (kind, key_scope, job_type) (rate(gittensory_jobs_rate_limit_budget_deferred_total[5m])) > 0.05"); }); + + it("surfaces Postgres internals and backup freshness panels", () => { + const dashboard = readDashboard(selfhostDashboardPath); + const targets = dashboard.panels.flatMap((panel) => panel.targets ?? []); + const titles = dashboard.panels.map((panel) => panel.title); + + expect(titles).toEqual(expect.arrayContaining(["Postgres & Backups", "Postgres Connections by State", "Postgres Locks & Slow Transactions", "Postgres Size & Table Growth", "Dead Tuples / Autovacuum", "Backup Freshness"])); + expect(targets.some((target) => target.expr === 'pg_up or vector(0)')).toBe(true); + expect(targets.some((target) => target.expr === 'sum(pg_stat_activity_count{datname="gittensory"}) or vector(0)')).toBe(true); + expect(targets.some((target) => target.expr === 'sum by (state) (pg_stat_activity_count{datname="gittensory"}) or vector(0)')).toBe(true); + expect(targets.some((target) => target.expr === 'sum(pg_stat_activity_count{datname="gittensory", wait_event_type="Lock"}) or vector(0)')).toBe(true); + expect(targets.some((target) => target.expr === 'max(pg_stat_activity_max_tx_duration{datname="gittensory"}) or vector(0)')).toBe(true); + expect(targets.some((target) => target.expr === 'pg_database_size_bytes{datname="gittensory"} or vector(0)')).toBe(true); + expect(targets.some((target) => target.expr === 'topk(10, pg_stat_user_tables_n_live_tup{datname="gittensory"}) or vector(0)')).toBe(true); + expect(targets.some((target) => target.expr === 'topk(10, pg_stat_user_tables_n_dead_tup{datname="gittensory"}) or vector(0)')).toBe(true); + expect(targets.some((target) => target.expr === 'sum by (relname) (increase(pg_stat_user_tables_autovacuum_count{datname="gittensory"}[1h])) or vector(0)')).toBe(true); + expect(targets.some((target) => target.expr === 'gittensory_backup_files{target=~"postgres|sqlite|qdrant"} or vector(0)')).toBe(true); + }); + + it("ships Postgres and backup alerts for the same dashboarded failure modes", () => { + const alerts = readFileSync(selfhostAlertsPath, "utf8"); + + expect(alerts).toContain("alert: GittensoryPostgresConnectionPressure"); + expect(alerts).toContain('sum(pg_stat_activity_count{datname="gittensory"})'); + expect(alerts).toContain("alert: GittensoryPostgresLockWaits"); + expect(alerts).toContain('pg_stat_activity_count{datname="gittensory", wait_event_type="Lock"}'); + expect(alerts).toContain("alert: GittensoryPostgresSlowTransaction"); + expect(alerts).toContain('pg_stat_activity_max_tx_duration{datname="gittensory"}'); + expect(alerts).toContain("alert: GittensoryPostgresDeadlocks"); + expect(alerts).toContain('pg_stat_database_deadlocks{datname="gittensory"}'); + expect(alerts).toContain("alert: GittensoryPostgresDatabaseGrowingFast"); + expect(alerts).toContain('deriv(pg_database_size_bytes{datname="gittensory"}[6h]) > 262144'); + expect(alerts).toContain("alert: GittensoryPostgresDeadTuplesHigh"); + expect(alerts).toContain('pg_stat_user_tables_n_dead_tup{datname="gittensory"}'); + expect(alerts).toContain("alert: GittensoryBackupMissing"); + expect(alerts).toContain('gittensory_backup_files{target=~"postgres|sqlite"} == 0'); + expect(alerts).toContain("alert: GittensoryBackupStale"); + expect(alerts).toContain('time() - gittensory_backup_latest_timestamp_seconds{target=~"postgres|sqlite"} > 93600'); + }); }); describe("maintainer Reviews & PRs Grafana dashboard", () => { diff --git a/test/unit/selfhost-observability-config.test.ts b/test/unit/selfhost-observability-config.test.ts index c720732e31..1fa78d5d8f 100644 --- a/test/unit/selfhost-observability-config.test.ts +++ b/test/unit/selfhost-observability-config.test.ts @@ -88,4 +88,47 @@ describe("self-host observability trace config", () => { expect(script).toContain("gittensory-selfhost-smoke"); expect(script).toContain("selfhost.observability.smoke"); }); + + it("wires Postgres and backup exporters without starting them on SQLite-only observability", () => { + const compose = record(readYaml("docker-compose.yml")); + const services = record(compose.services); + const postgresExporter = record(services["postgres-exporter"]); + const backupExporter = record(services["backup-exporter"]); + const prometheus = record(readYaml("prometheus/prometheus.yml")); + const scrapeConfigs = prometheus.scrape_configs as Array>; + + expect(postgresExporter.image).toBe("quay.io/prometheuscommunity/postgres-exporter:v0.20.0"); + expect(postgresExporter.profiles).toEqual(["postgres", "pgbouncer"]); + expect(postgresExporter.depends_on?.postgres).toEqual({ condition: "service_healthy" }); + expect(postgresExporter.environment).toMatchObject({ + DATA_SOURCE_URI: "postgres:5432/gittensory?sslmode=disable", + DATA_SOURCE_USER: "gittensory", + DATA_SOURCE_PASS: "${POSTGRES_PASSWORD:-CHANGEME}", + }); + + expect(backupExporter.profiles).toEqual(["backup"]); + expect(backupExporter.volumes).toEqual( + expect.arrayContaining([ + "gittensory-backups:/backups:ro", + "./scripts/backup-metrics.sh:/backup-metrics.sh:ro", + ]), + ); + expect(backupExporter.healthcheck?.test).toEqual([ + "CMD-SHELL", + "wget -qO- http://127.0.0.1:9101/metrics | grep -q '^gittensory_backup_latest_timestamp_seconds'", + ]); + + expect(scrapeConfigs).toEqual( + expect.arrayContaining([ + expect.objectContaining({ + job_name: "postgres", + static_configs: [{ targets: ["postgres-exporter:9187"] }], + }), + expect.objectContaining({ + job_name: "gittensory-backup", + static_configs: [{ targets: ["backup-exporter:9101"] }], + }), + ]), + ); + }); });