From 667d2ceef999ef890265e3fed49a29605b5d0a18 Mon Sep 17 00:00:00 2001 From: JSONbored <49853598+JSONbored@users.noreply.github.com> Date: Wed, 1 Jul 2026 01:44:21 -0700 Subject: [PATCH 1/3] feat(selfhost): add a backup restore/validation drill (verify-backup.sh) The backup profile writes Postgres pg_dump / SQLite backups but nothing checked they are restorable. Add scripts/verify-backup.sh, run on demand via the backup service, which verifies the newest backup (or an explicit file) WITHOUT touching the live database: - Postgres .dump: `pg_restore --list` must parse the archive and find a non-empty table of contents. - SQLite .sqlite.gz: gzip integrity + `PRAGMA integrity_check` on a temp copy. - Opt-in scratch restore (VERIFY_RESTORE_SCRATCH=1 + a dedicated scratch DB URL): restores the dump into a throwaway database and sanity-counts tables. It refuses to run when the scratch URL is empty or equals the live source, so a destructive restore cannot hit production by accident. Mounts the script into the backup service and passes the scratch env through; documents the drill (and what a healthy run looks like) in the backup docs page. Covered by test/unit/selfhost-verify-backup-script.test.ts (fake pg_restore / psql / sqlite3, real gzip): validation pass/fail, the scratch guards, the scratch happy path, explicit-file mode, and the SQLite integrity/gzip failures. --- .../docs.self-hosting-backup-scaling.tsx | 35 +++ docker-compose.yml | 7 + scripts/verify-backup.sh | 137 ++++++++++ .../selfhost-verify-backup-script.test.ts | 233 ++++++++++++++++++ 4 files changed, 412 insertions(+) create mode 100644 scripts/verify-backup.sh create mode 100644 test/unit/selfhost-verify-backup-script.test.ts diff --git a/apps/gittensory-ui/src/routes/docs.self-hosting-backup-scaling.tsx b/apps/gittensory-ui/src/routes/docs.self-hosting-backup-scaling.tsx index bd2853b3d5..a52017cdb0 100644 --- a/apps/gittensory-ui/src/routes/docs.self-hosting-backup-scaling.tsx +++ b/apps/gittensory-ui/src/routes/docs.self-hosting-backup-scaling.tsx @@ -110,6 +110,41 @@ npm run selfhost:postgres:migrate -- --sqlite /data/gittensory.sqlite --postgres
  • Confirm recent review rows and job state are present.
    • +

    Verify a backup is restorable

    +

    + The backup profile ships verify-backup.sh, which checks the newest + backup without touching the live database: Postgres .dump archives with{" "} + pg_restore --list, and SQLite .sqlite.gz backups with a gzip and{" "} + integrity_check pass. Run it against the newest backup, or a specific file: +

    + .dump`} + /> +

    + A healthy run ends with [verify] postgres archive OK: … (N TOC entries) (or{" "} + [verify] sqlite backup OK), then [verify] complete, and exits 0. + Corruption, a missing backup, or an empty archive exits non-zero with a{" "} + [verify] reason. +

    +

    + To prove a dump actually restores, opt into a scratch restore into a throwaway{" "} + database — never the live one: +

    + + + The scratch restore runs pg_restore --clean against{" "} + GITTENSORY_VERIFY_SCRATCH_DATABASE_URL, so point it at a dedicated database you + can afford to drop. The script refuses to run when that URL equals the live backup source. + +

    After scaling, revisit Operations and{" "} Security because network and credential diff --git a/docker-compose.yml b/docker-compose.yml index cebc8d1d1f..568f84faa7 100644 --- a/docker-compose.yml +++ b/docker-compose.yml @@ -509,6 +509,8 @@ services: # Active database backup (Postgres pg_dump or WAL-safe SQLite online backup) + a Qdrant snapshot, on a loop # (default daily), kept in the gittensory-backups volume with retention. Run on demand: # docker compose --profile backup run --rm backup sh /backup.sh + # Verify the newest backup is restorable (pg_restore --list / SQLite integrity_check; opt-in scratch restore): + # docker compose --profile backup run --rm backup sh /verify-backup.sh backup: image: alpine:3.20 restart: unless-stopped @@ -519,11 +521,16 @@ services: GITTENSORY_BACKUP_SOURCE_DATABASE_URL: "${GITTENSORY_BACKUP_SOURCE_DATABASE_URL:-}" QDRANT_URL: ${QDRANT_URL:-http://qdrant:6333} BACKUP_RETAIN: ${BACKUP_RETAIN:-7} + # Opt-in restore drill (verify-backup.sh): restore the newest dump into a THROWAWAY database, never the + # live one. Both must be set, and the scratch URL must differ from the backup source. + VERIFY_RESTORE_SCRATCH: "${VERIFY_RESTORE_SCRATCH:-}" + GITTENSORY_VERIFY_SCRATCH_DATABASE_URL: "${GITTENSORY_VERIFY_SCRATCH_DATABASE_URL:-}" volumes: # RW (not :ro) so SQLite's online-backup can use the WAL index; the script only ever reads the DB. - gittensory-data:/data - gittensory-backups:/backups - ./scripts/backup.sh:/backup.sh:ro + - ./scripts/verify-backup.sh:/verify-backup.sh:ro entrypoint: ["/bin/sh", "-c"] command: - "apk add --no-cache sqlite postgresql16-client curl >/dev/null 2>&1 && while true; do sh /backup.sh || echo '[backup] run failed'; sleep ${BACKUP_INTERVAL_SECONDS:-86400}; done" diff --git a/scripts/verify-backup.sh b/scripts/verify-backup.sh new file mode 100644 index 0000000000..a43e63bca8 --- /dev/null +++ b/scripts/verify-backup.sh @@ -0,0 +1,137 @@ +#!/bin/sh +# Self-host backup verification: check the newest backup produced by backup.sh WITHOUT touching the live +# database. Postgres `.dump` archives are validated with `pg_restore --list` (a readable custom-format dump +# whose table of contents is non-empty); SQLite `.sqlite.gz` backups are gzip- and integrity-checked in a +# temp copy. An OPT-IN scratch restore (VERIFY_RESTORE_SCRATCH=1 + a dedicated scratch DB URL) additionally +# restores the Postgres dump into a throwaway database and runs a sanity query — it refuses to touch the live +# database. Run on demand (newest backup, or a specific file): +# docker compose --profile backup run --rm backup sh /verify-backup.sh +# docker compose --profile backup run --rm backup sh /verify-backup.sh /backups/postgres/gittensory-.dump +set -eu + +OUT=${BACKUP_OUT_DIR:-/backups} +PG_DB="${GITTENSORY_BACKUP_SOURCE_DATABASE_URL:-${DATABASE_URL:-}}" +TARGET="${1:-}" + +verify_postgres() { + dump="$1" + if [ ! -s "$dump" ]; then + echo "[verify] missing or empty Postgres dump: $dump" >&2 + return 1 + fi + if ! command -v pg_restore >/dev/null 2>&1; then + echo "[verify] pg_restore not found; cannot verify Postgres backup" >&2 + return 1 + fi + # 1) Structural validation (non-destructive): the archive must be a readable custom-format dump whose table + # of contents holds at least one entry. A truncated or corrupt dump fails here. + toc="$(pg_restore --list "$dump" 2>&1)" || { + echo "[verify] pg_restore --list failed for $dump:" >&2 + printf '%s\n' "$toc" | head -3 >&2 + return 1 + } + entries="$(printf '%s\n' "$toc" | grep -cvE '^;|^[[:space:]]*$' || true)" + if [ "${entries:-0}" -lt 1 ]; then + echo "[verify] $dump has an empty table of contents" >&2 + return 1 + fi + echo "[verify] postgres archive OK: $dump ($entries TOC entries)" + + # 2) Optional scratch restore smoke (opt-in, guarded): restore into a THROWAWAY database and sanity-check. + # Never runs against the live database — the scratch URL must be set explicitly and differ from the source. + [ "${VERIFY_RESTORE_SCRATCH:-}" = "1" ] || return 0 + scratch="${GITTENSORY_VERIFY_SCRATCH_DATABASE_URL:-}" + case "$scratch" in + postgres://* | postgresql://*) : ;; + *) + echo "[verify] VERIFY_RESTORE_SCRATCH=1 needs GITTENSORY_VERIFY_SCRATCH_DATABASE_URL=postgres://… (a dedicated scratch database, never the live one)" >&2 + return 1 + ;; + esac + if [ "$scratch" = "$PG_DB" ]; then + echo "[verify] refusing scratch restore: GITTENSORY_VERIFY_SCRATCH_DATABASE_URL must differ from the live backup source" >&2 + return 1 + fi + if ! command -v psql >/dev/null 2>&1; then + echo "[verify] psql not found; cannot run the scratch restore smoke" >&2 + return 1 + fi + echo "[verify] restoring $dump into the scratch database…" + if ! pg_restore --clean --if-exists --no-owner --no-privileges --dbname "$scratch" "$dump" >/dev/null 2>&1; then + echo "[verify] scratch restore failed for $dump" >&2 + return 1 + fi + tables="$(psql "$scratch" -X -q -t -A -v ON_ERROR_STOP=1 -c "SELECT count(*) FROM information_schema.tables WHERE table_schema = 'public'")" || { + echo "[verify] scratch sanity query failed" >&2 + return 1 + } + if [ "${tables:-0}" -lt 1 ]; then + echo "[verify] scratch restore produced no tables" >&2 + return 1 + fi + echo "[verify] scratch restore OK: $tables tables restored" +} + +verify_sqlite() { + gz="$1" + if [ ! -s "$gz" ]; then + echo "[verify] missing or empty SQLite backup: $gz" >&2 + return 1 + fi + if ! gzip -t "$gz" 2>/dev/null; then + echo "[verify] gzip integrity check failed for $gz" >&2 + return 1 + fi + if ! command -v sqlite3 >/dev/null 2>&1; then + echo "[verify] sqlite3 not found; verified gzip integrity only for $gz" + return 0 + fi + tmp="$(mktemp)" + if ! gzip -dc "$gz" >"$tmp" 2>/dev/null; then + rm -f "$tmp" + echo "[verify] failed to decompress $gz" >&2 + return 1 + fi + result="$(sqlite3 "$tmp" 'PRAGMA integrity_check;' 2>/dev/null | head -1 || true)" + rm -f "$tmp" + if [ "$result" != "ok" ]; then + echo "[verify] sqlite integrity_check failed for $gz (${result:-no output})" >&2 + return 1 + fi + echo "[verify] sqlite backup OK: $gz" +} + +# An explicit file argument wins; otherwise verify the newest backup for the active database type. +if [ -n "$TARGET" ]; then + case "$TARGET" in + *.dump) verify_postgres "$TARGET" ;; + *.sqlite.gz) verify_sqlite "$TARGET" ;; + *) + echo "[verify] unrecognized backup file: $TARGET (expected *.dump or *.sqlite.gz)" >&2 + exit 1 + ;; + esac + echo "[verify] complete" + exit 0 +fi + +case "$PG_DB" in + postgres://* | postgresql://*) + dump="$(ls -1t "$OUT"/postgres/*.dump 2>/dev/null | head -1 || true)" + if [ -z "$dump" ]; then + echo "[verify] no Postgres .dump found in $OUT/postgres" >&2 + exit 1 + fi + verify_postgres "$dump" + ;; + *) + gz="$(ls -1t "$OUT"/sqlite/*.sqlite.gz 2>/dev/null | head -1 || true)" + if [ -z "$gz" ]; then + echo "[verify] no SQLite backup found in $OUT/sqlite" >&2 + exit 1 + fi + verify_sqlite "$gz" + ;; +esac + +echo "[verify] complete" diff --git a/test/unit/selfhost-verify-backup-script.test.ts b/test/unit/selfhost-verify-backup-script.test.ts new file mode 100644 index 0000000000..e8693255f2 --- /dev/null +++ b/test/unit/selfhost-verify-backup-script.test.ts @@ -0,0 +1,233 @@ +import { execFileSync } from "node:child_process"; +import { chmodSync, mkdirSync, mkdtempSync, rmSync, writeFileSync } from "node:fs"; +import { tmpdir } from "node:os"; +import { join } from "node:path"; +import { gzipSync } from "node:zlib"; +import { afterEach, describe, expect, it } from "vitest"; + +const tmpRoots: string[] = []; + +function tmpRoot(): string { + const dir = mkdtempSync(join(tmpdir(), "gittensory-verify-")); + tmpRoots.push(dir); + return dir; +} + +afterEach(() => { + for (const dir of tmpRoots.splice(0)) rmSync(dir, { force: true, recursive: true }); +}); + +// A well-formed dump contains GOODDUMP; anything else makes the fake pg_restore fail as if the archive were +// truncated. `--list` prints a header (`;`-prefixed) plus two TOC entry lines; a restore just exits 0. +const PG_RESTORE = `#!/bin/sh +mode=list +dump= +while [ "$#" -gt 0 ]; do + case "$1" in + --list) mode=list; shift ;; + --dbname) mode=restore; shift 2 ;; + --clean|--if-exists|--no-owner|--no-privileges) shift ;; + -*) shift ;; + *) dump="$1"; shift ;; + esac +done +if ! grep -q GOODDUMP "$dump" 2>/dev/null; then + echo "pg_restore: error: could not read from input file: end of file" >&2 + exit 1 +fi +if [ "$mode" = list ]; then + printf ';\\n; Archive created\\n;\\n215; 1259 16385 TABLE public pull_requests owner\\n216; 1259 16400 TABLE public advisories owner\\n' +fi +exit 0 +`; +const PSQL = `#!/bin/sh +echo "\${FAKE_PSQL_TABLE_COUNT:-3}" +`; +const SQLITE3 = `#!/bin/sh +echo "\${FAKE_SQLITE_INTEGRITY:-ok}" +`; + +function fakeBin(root: string, bins: Record): string { + const bin = join(root, "bin"); + mkdirSync(bin, { recursive: true }); + for (const [name, body] of Object.entries(bins)) { + const path = join(bin, name); + writeFileSync(path, body); + chmodSync(path, 0o755); + } + return bin; +} + +function writePgDump(root: string, name: string, valid = true): string { + const dir = join(root, "backups", "postgres"); + mkdirSync(dir, { recursive: true }); + const path = join(dir, name); + writeFileSync(path, valid ? "PGDMP GOODDUMP payload" : "truncated garbage"); + return path; +} + +function writeSqliteGz(root: string, name: string, body = "fake sqlite db", gzip = true): string { + const dir = join(root, "backups", "sqlite"); + mkdirSync(dir, { recursive: true }); + const path = join(dir, name); + writeFileSync(path, gzip ? gzipSync(Buffer.from(body)) : Buffer.from(body)); + return path; +} + +function runVerify( + root: string, + args: string[], + env: Record, + bins: Record, +): { status: number; out: string } { + const bin = fakeBin(root, bins); + try { + const stdout = execFileSync("sh", ["scripts/verify-backup.sh", ...args], { + cwd: process.cwd(), + encoding: "utf8", + env: { + ...process.env, + PATH: `${bin}:${process.env.PATH ?? ""}`, + BACKUP_OUT_DIR: join(root, "backups"), + GITTENSORY_BACKUP_SOURCE_DATABASE_URL: "", + DATABASE_URL: "", + VERIFY_RESTORE_SCRATCH: "", + GITTENSORY_VERIFY_SCRATCH_DATABASE_URL: "", + ...env, + }, + }); + return { status: 0, out: stdout }; + } catch (err) { + const e = err as { status?: number; stdout?: string; stderr?: string }; + return { status: e.status ?? 1, out: `${e.stdout ?? ""}${e.stderr ?? ""}` }; + } +} + +describe("self-host verify-backup script", () => { + it("validates the newest Postgres dump with pg_restore --list", () => { + const root = tmpRoot(); + writePgDump(root, "gittensory-20240101T000000Z.dump", true); + + const r = runVerify(root, [], { GITTENSORY_BACKUP_SOURCE_DATABASE_URL: "postgres://u:p@h/db" }, { pg_restore: PG_RESTORE }); + + expect(r.status).toBe(0); + expect(r.out).toContain("postgres archive OK"); + expect(r.out).toContain("2 TOC entries"); + expect(r.out).toContain("[verify] complete"); + }); + + it("fails when the Postgres dump is unreadable", () => { + const root = tmpRoot(); + writePgDump(root, "gittensory-bad.dump", false); + + const r = runVerify(root, [], { GITTENSORY_BACKUP_SOURCE_DATABASE_URL: "postgres://u:p@h/db" }, { pg_restore: PG_RESTORE }); + + expect(r.status).toBe(1); + expect(r.out).toContain("pg_restore --list failed"); + }); + + it("fails when no Postgres dump is present", () => { + const root = tmpRoot(); + mkdirSync(join(root, "backups", "postgres"), { recursive: true }); + + const r = runVerify(root, [], { GITTENSORY_BACKUP_SOURCE_DATABASE_URL: "postgres://u:p@h/db" }, { pg_restore: PG_RESTORE }); + + expect(r.status).toBe(1); + expect(r.out).toContain("no Postgres .dump found"); + }); + + it("refuses the opt-in scratch restore when no scratch URL is configured", () => { + const root = tmpRoot(); + writePgDump(root, "gittensory-a.dump", true); + + const r = runVerify( + root, + [], + { GITTENSORY_BACKUP_SOURCE_DATABASE_URL: "postgres://u:p@h/live", VERIFY_RESTORE_SCRATCH: "1" }, + { pg_restore: PG_RESTORE, psql: PSQL }, + ); + + expect(r.status).toBe(1); + expect(r.out).toContain("needs GITTENSORY_VERIFY_SCRATCH_DATABASE_URL"); + }); + + it("refuses the scratch restore when the scratch URL is the live database", () => { + const root = tmpRoot(); + writePgDump(root, "gittensory-a.dump", true); + const live = "postgres://u:p@h/live"; + + const r = runVerify( + root, + [], + { + GITTENSORY_BACKUP_SOURCE_DATABASE_URL: live, + VERIFY_RESTORE_SCRATCH: "1", + GITTENSORY_VERIFY_SCRATCH_DATABASE_URL: live, + }, + { pg_restore: PG_RESTORE, psql: PSQL }, + ); + + expect(r.status).toBe(1); + expect(r.out).toContain("must differ from the live backup source"); + }); + + it("runs the guarded scratch restore into a throwaway database and sanity-checks it", () => { + const root = tmpRoot(); + writePgDump(root, "gittensory-a.dump", true); + + const r = runVerify( + root, + [], + { + GITTENSORY_BACKUP_SOURCE_DATABASE_URL: "postgres://u:p@h/live", + VERIFY_RESTORE_SCRATCH: "1", + GITTENSORY_VERIFY_SCRATCH_DATABASE_URL: "postgres://u:p@h/scratch", + FAKE_PSQL_TABLE_COUNT: "42", + }, + { pg_restore: PG_RESTORE, psql: PSQL }, + ); + + expect(r.status).toBe(0); + expect(r.out).toContain("scratch restore OK: 42 tables"); + }); + + it("verifies an explicit dump path argument", () => { + const root = tmpRoot(); + const target = writePgDump(root, "chosen.dump", true); + + const r = runVerify(root, [target], {}, { pg_restore: PG_RESTORE }); + + expect(r.status).toBe(0); + expect(r.out).toContain("postgres archive OK"); + }); + + it("validates the newest SQLite backup with an integrity check", () => { + const root = tmpRoot(); + writeSqliteGz(root, "gittensory-20240101T000000Z.sqlite.gz"); + + const r = runVerify(root, [], {}, { sqlite3: SQLITE3 }); + + expect(r.status).toBe(0); + expect(r.out).toContain("sqlite backup OK"); + }); + + it("fails when the SQLite backup fails its integrity check", () => { + const root = tmpRoot(); + writeSqliteGz(root, "gittensory-a.sqlite.gz"); + + const r = runVerify(root, [], { FAKE_SQLITE_INTEGRITY: "malformed database disk image" }, { sqlite3: SQLITE3 }); + + expect(r.status).toBe(1); + expect(r.out).toContain("sqlite integrity_check failed"); + }); + + it("fails when the SQLite backup is not valid gzip", () => { + const root = tmpRoot(); + writeSqliteGz(root, "gittensory-a.sqlite.gz", "not gzip at all", false); + + const r = runVerify(root, [], {}, { sqlite3: SQLITE3 }); + + expect(r.status).toBe(1); + expect(r.out).toContain("gzip integrity check failed"); + }); +}); From 66ae905d9e82bb4ee75e3ef8cd962c0d0833b74c Mon Sep 17 00:00:00 2001 From: JSONbored <49853598+JSONbored@users.noreply.github.com> Date: Wed, 1 Jul 2026 01:56:02 -0700 Subject: [PATCH 2/3] fix(selfhost): make on-demand backup/verify-backup runs install their packages MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit `docker compose run --rm backup sh /verify-backup.sh` REPLACES the service's `command:`, not its `entrypoint:`. The old entrypoint (`/bin/sh -c`) took the package-install as part of `command:`, so an on-demand run skipped it entirely — worse, since `-c`'s extra positional args become the nested shell's $0/$1 rather than a script to execute, the override became a bare interactive `sh` that just hangs reading stdin (confirmed with a live `docker compose run`; the documented `sh /backup.sh` one-shot had the exact same pre-existing bug). Move the package install into the entrypoint and end it with `exec "$@"`, with a placeholder `sh` entrypoint arg so the real payload — the default loop or a `run` override — lands in "$@" starting at $1, not $0. Mirrors the inline entrypoint-script pattern this file already uses for grafana. Verified against real Docker: `docker compose --profile backup up -d backup` still completes its normal backup.sh cycle, and `docker compose run --rm backup sh /verify-backup.sh` now actually executes (previously hung) with pg_restore present and invoked (confirmed via a seeded dummy dump producing pg_restore's own parse error, not "command not found"). --- docker-compose.yml | 17 +++++++++++++++-- 1 file changed, 15 insertions(+), 2 deletions(-) diff --git a/docker-compose.yml b/docker-compose.yml index 568f84faa7..7ccbe0eeaf 100644 --- a/docker-compose.yml +++ b/docker-compose.yml @@ -531,9 +531,22 @@ services: - gittensory-backups:/backups - ./scripts/backup.sh:/backup.sh:ro - ./scripts/verify-backup.sh:/verify-backup.sh:ro - entrypoint: ["/bin/sh", "-c"] + # `docker compose run --rm backup sh /backup.sh` (or /verify-backup.sh) REPLACES `command:`, not + # `entrypoint:`, so the package install must live in the entrypoint or an on-demand run gets a bare + # container with no pg_restore/sqlite3/psql. The entrypoint installs packages once, then `exec "$@"` + # runs whatever command is in effect — the default loop below, or a `run` override. The trailing `sh` + # entrypoint arg is a placeholder $0 so the real payload lands in "$@" starting at $1, not $0. + entrypoint: + - /bin/sh + - -ec + - | + apk add --no-cache sqlite postgresql16-client curl >/dev/null 2>&1 + exec "$@" + - sh command: - - "apk add --no-cache sqlite postgresql16-client curl >/dev/null 2>&1 && while true; do sh /backup.sh || echo '[backup] run failed'; sleep ${BACKUP_INTERVAL_SECONDS:-86400}; done" + - sh + - -c + - "while true; do sh /backup.sh || echo '[backup] run failed'; sleep ${BACKUP_INTERVAL_SECONDS:-86400}; done" volumes: gittensory-data: From 7efaf83d6229793db2c59ae2ed60cadea727a9ef Mon Sep 17 00:00:00 2001 From: JSONbored <49853598+JSONbored@users.noreply.github.com> Date: Wed, 1 Jul 2026 02:11:15 -0700 Subject: [PATCH 3/3] fix(selfhost): close scratch-restore guard bypass via equivalent connection URLs MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit `[ "$scratch" = "$PG_DB" ]` compared connection strings byte-for-byte, so a scratch URL that is a DIFFERENTLY-SPELLED equivalent of the live backup source (postgresql:// vs postgres://, a host alias, an explicit vs default port) would pass the guard and let `pg_restore --clean` drop objects in the live database. Replace the string compare with an identity check: ask Postgres itself for `current_database() || '@' || pg_control_system().system_identifier` on both connections and compare THAT. system_identifier is a random 64-bit value fixed for the life of a cluster's data directory, independent of how the connection was dialed, and PUBLIC has EXECUTE on pg_control_system() by default (no privilege escalation needed) — verified against a real non-superuser role. Deliberately NOT network-address-based (e.g. inet_server_addr()): testing against real Postgres showed the same server can report different addresses across connections over different address families (IPv6 vs IPv4 loopback), which would have reopened a false "these differ" negative — the wrong direction for a safety guard. Any failure to fingerprint either side now aborts (fails closed) rather than assuming the databases differ. Validated two ways: - test/unit/selfhost-verify-backup-script.test.ts: a fake psql keyed by connection string simulates two differently-spelled URLs resolving to the SAME database (the exact reported bypass) plus fail-closed cases when either identity query fails. - A real end-to-end run against Postgres 16 (matching the backup image): refuses a byte-identical URL, refuses a differently-spelled equivalent of the same database, and correctly allows + completes a restore into a genuinely different database on the same cluster — with the live database's table confirmed untouched throughout. --- scripts/verify-backup.sh | 39 +++++- .../selfhost-verify-backup-script.test.ts | 128 ++++++++++++++++-- 2 files changed, 153 insertions(+), 14 deletions(-) diff --git a/scripts/verify-backup.sh b/scripts/verify-backup.sh index a43e63bca8..4c4b85d705 100644 --- a/scripts/verify-backup.sh +++ b/scripts/verify-backup.sh @@ -48,14 +48,45 @@ verify_postgres() { return 1 ;; esac - if [ "$scratch" = "$PG_DB" ]; then - echo "[verify] refusing scratch restore: GITTENSORY_VERIFY_SCRATCH_DATABASE_URL must differ from the live backup source" >&2 - return 1 - fi if ! command -v psql >/dev/null 2>&1; then echo "[verify] psql not found; cannot run the scratch restore smoke" >&2 return 1 fi + # Identity check, NOT a string comparison: a differently-spelled URL (postgres:// vs postgresql://, a host + # alias, an explicit vs default port) can still point at the SAME database, and a naive `[ "$scratch" = + # "$PG_DB" ]` misses that — letting `pg_restore --clean` drop live objects. Ask Postgres itself for the + # connection's actual identity instead of comparing the raw strings: `pg_control_system()`'s system_identifier + # is a random 64-bit value fixed for the life of that specific cluster's data directory (independent of how + # the connection was dialed — unlike a network-address fingerprint, e.g. inet_server_addr(), which can + # legitimately differ for the SAME server across connections over different address families, such as an IPv4 + # vs IPv6 loopback — a false "these differ" that would defeat the guard). Combined with current_database(), + # this correctly matches "same server AND same database" while still treating a different database name on + # the same cluster as distinct (a legitimate, common scratch-DB setup). No special privilege is required: + # PUBLIC has EXECUTE on pg_control_system() by default. Any failure to fingerprint EITHER side aborts (fail + # closed) rather than assuming the databases differ. + db_identity() { + psql "$1" -X -q -t -A -v ON_ERROR_STOP=1 \ + -c "SELECT current_database() || '@' || (SELECT system_identifier FROM pg_control_system())::text" \ + 2>/dev/null + } + scratch_identity="$(db_identity "$scratch")" || scratch_identity="" + if [ -z "$scratch_identity" ]; then + echo "[verify] could not connect to the scratch database to verify its identity; refusing to proceed" >&2 + return 1 + fi + case "$PG_DB" in + postgres://* | postgresql://*) + live_identity="$(db_identity "$PG_DB")" || live_identity="" + if [ -z "$live_identity" ]; then + echo "[verify] could not connect to the live backup source to verify its identity; refusing to proceed" >&2 + return 1 + fi + if [ "$scratch_identity" = "$live_identity" ]; then + echo "[verify] refusing scratch restore: the scratch URL resolves to the SAME database as the live backup source ($scratch_identity)" >&2 + return 1 + fi + ;; + esac echo "[verify] restoring $dump into the scratch database…" if ! pg_restore --clean --if-exists --no-owner --no-privileges --dbname "$scratch" "$dump" >/dev/null 2>&1; then echo "[verify] scratch restore failed for $dump" >&2 diff --git a/test/unit/selfhost-verify-backup-script.test.ts b/test/unit/selfhost-verify-backup-script.test.ts index e8693255f2..325555e1a5 100644 --- a/test/unit/selfhost-verify-backup-script.test.ts +++ b/test/unit/selfhost-verify-backup-script.test.ts @@ -40,13 +40,43 @@ if [ "$mode" = list ]; then fi exit 0 `; -const PSQL = `#!/bin/sh -echo "\${FAKE_PSQL_TABLE_COUNT:-3}" -`; const SQLITE3 = `#!/bin/sh echo "\${FAKE_SQLITE_INTEGRITY:-ok}" `; +// Builds a fake `psql` that distinguishes the scratch-restore guard's identity query (`current_database()`) +// from the post-restore table-count sanity query, and returns a caller-mapped identity per connection URL — +// lets tests simulate two DIFFERENTLY-SPELLED URLs resolving to the SAME actual database (or genuinely +// different ones), which is exactly the distinction the real db_identity() guard has to get right. A URL with +// no entry in `identities` makes the identity query fail (exit 1), modeling "could not connect/fingerprint". +function fakePsql(identities: Record, tableCount = "3"): string { + const cases = Object.entries(identities) + .map(([url, identity]) => ` "${url}") printf '%s\\n' "${identity}" ;;`) + .join("\n"); + return `#!/bin/sh +url="$1" +shift +sql="" +while [ "$#" -gt 0 ]; do + case "$1" in + -c) sql="$2"; shift 2 ;; + *) shift ;; + esac +done +case "$sql" in + *current_database*) + case "$url" in +${cases} + *) exit 1 ;; + esac + ;; + *) + printf '%s\\n' "${tableCount}" + ;; +esac +`; +} + function fakeBin(root: string, bins: Record): string { const bin = join(root, "bin"); mkdirSync(bin, { recursive: true }); @@ -144,14 +174,14 @@ describe("self-host verify-backup script", () => { root, [], { GITTENSORY_BACKUP_SOURCE_DATABASE_URL: "postgres://u:p@h/live", VERIFY_RESTORE_SCRATCH: "1" }, - { pg_restore: PG_RESTORE, psql: PSQL }, + { pg_restore: PG_RESTORE, psql: fakePsql({}) }, ); expect(r.status).toBe(1); expect(r.out).toContain("needs GITTENSORY_VERIFY_SCRATCH_DATABASE_URL"); }); - it("refuses the scratch restore when the scratch URL is the live database", () => { + it("refuses the scratch restore when the scratch URL is byte-for-byte the live database", () => { const root = tmpRoot(); writePgDump(root, "gittensory-a.dump", true); const live = "postgres://u:p@h/live"; @@ -164,14 +194,46 @@ describe("self-host verify-backup script", () => { VERIFY_RESTORE_SCRATCH: "1", GITTENSORY_VERIFY_SCRATCH_DATABASE_URL: live, }, - { pg_restore: PG_RESTORE, psql: PSQL }, + { pg_restore: PG_RESTORE, psql: fakePsql({ [live]: "same-cluster@10.0.0.5:5432/live" }) }, ); expect(r.status).toBe(1); - expect(r.out).toContain("must differ from the live backup source"); + expect(r.out).toContain("SAME database as the live backup source"); }); - it("runs the guarded scratch restore into a throwaway database and sanity-checks it", () => { + it("refuses the scratch restore when a DIFFERENTLY-SPELLED URL resolves to the SAME database (regression: naive string compare bypass)", () => { + const root = tmpRoot(); + writePgDump(root, "gittensory-a.dump", true); + // Same database, deliberately spelled differently: scheme (postgres vs postgresql) AND an explicit vs + // default port — a `[ "$scratch" = "$PG_DB" ]` string compare would wrongly treat these as distinct. + const live = "postgres://gittensory:pw@postgres/gittensory"; + const scratch = "postgresql://gittensory:pw@postgres:5432/gittensory"; + expect(scratch).not.toBe(live); + + const r = runVerify( + root, + [], + { + GITTENSORY_BACKUP_SOURCE_DATABASE_URL: live, + VERIFY_RESTORE_SCRATCH: "1", + GITTENSORY_VERIFY_SCRATCH_DATABASE_URL: scratch, + }, + { + pg_restore: PG_RESTORE, + // Both URLs resolve to the identical real connection identity, exactly as they would in production + // if they point at the same Postgres server/database despite the different spelling. + psql: fakePsql({ + [live]: "gittensory@10.0.0.5:5432", + [scratch]: "gittensory@10.0.0.5:5432", + }), + }, + ); + + expect(r.status).toBe(1); + expect(r.out).toContain("SAME database as the live backup source"); + }); + + it("refuses (fails closed) when the scratch database's identity cannot be determined", () => { const root = tmpRoot(); writePgDump(root, "gittensory-a.dump", true); @@ -182,9 +244,55 @@ describe("self-host verify-backup script", () => { GITTENSORY_BACKUP_SOURCE_DATABASE_URL: "postgres://u:p@h/live", VERIFY_RESTORE_SCRATCH: "1", GITTENSORY_VERIFY_SCRATCH_DATABASE_URL: "postgres://u:p@h/scratch", - FAKE_PSQL_TABLE_COUNT: "42", }, - { pg_restore: PG_RESTORE, psql: PSQL }, + // No identity entries at all: the scratch identity query fails, so the guard must abort rather than + // silently assume the databases differ. + { pg_restore: PG_RESTORE, psql: fakePsql({}) }, + ); + + expect(r.status).toBe(1); + expect(r.out).toContain("could not connect to the scratch database"); + }); + + it("refuses (fails closed) when the live database's identity cannot be determined", () => { + const root = tmpRoot(); + writePgDump(root, "gittensory-a.dump", true); + const scratch = "postgres://u:p@h/scratch"; + + const r = runVerify( + root, + [], + { + GITTENSORY_BACKUP_SOURCE_DATABASE_URL: "postgres://u:p@h/live", + VERIFY_RESTORE_SCRATCH: "1", + GITTENSORY_VERIFY_SCRATCH_DATABASE_URL: scratch, + }, + // Scratch resolves fine, but the live URL has no mapping — its identity query fails. + { pg_restore: PG_RESTORE, psql: fakePsql({ [scratch]: "gittensory@10.0.0.9:5432/scratch" }) }, + ); + + expect(r.status).toBe(1); + expect(r.out).toContain("could not connect to the live backup source"); + }); + + it("runs the guarded scratch restore into a throwaway database and sanity-checks it", () => { + const root = tmpRoot(); + writePgDump(root, "gittensory-a.dump", true); + const live = "postgres://u:p@h/live"; + const scratch = "postgres://u:p@h/scratch"; + + const r = runVerify( + root, + [], + { + GITTENSORY_BACKUP_SOURCE_DATABASE_URL: live, + VERIFY_RESTORE_SCRATCH: "1", + GITTENSORY_VERIFY_SCRATCH_DATABASE_URL: scratch, + }, + { + pg_restore: PG_RESTORE, + psql: fakePsql({ [live]: "gittensory@10.0.0.5:5432/live", [scratch]: "gittensory@10.0.0.5:5432/scratch" }, "42"), + }, ); expect(r.status).toBe(0);