diff --git a/.env.example b/.env.example index 4f0db2ba22..53ea3f9e8a 100644 --- a/.env.example +++ b/.env.example @@ -230,9 +230,24 @@ REDIS_URL=redis://redis:6379 # REQUIRED for the self-host review # # most installs never need to touch this. # QDRANT_URL= # set to http://qdrant:6333 to use Qdrant as the RAG vector store # # (--profile qdrant). Overrides the built-in sqlite-vec / pgvector. -# BROWSER_WS_ENDPOINT= # ws:// URL of a browserless/chrome instance for visual-review -# # screenshot capture. Unset = visual review is fully inert (no -# # screenshots, no error) — this feature is entirely optional. +# BROWSER_WS_ENDPOINT= # ws:// URL of a browserless/chrome-compatible instance for +# # visual-review screenshot capture. Unset = visual review is +# # fully inert (no screenshots, no error) — entirely optional. +# # Bundled option (--profile visual-review): set this to +# # ws://browserless:3000?token=${BROWSERLESS_TOKEN} using the +# # SAME value as BROWSERLESS_TOKEN below. Or point at your own +# # externally-run browserless/chromium instance instead. +# BROWSERLESS_TOKEN= # auth token for the bundled --profile visual-review browserless +# # container. Set a strong random value and embed the SAME +# # value in BROWSER_WS_ENDPOINT's ?token= above. Unset ⇒ the +# # container auto-generates a random token each boot that will +# # never match a hardcoded BROWSER_WS_ENDPOINT — set this. +# # Irrelevant if you point BROWSER_WS_ENDPOINT at an external +# # browserless instance you manage yourself. +# BROWSERLESS_CONCURRENT=2 # (--profile visual-review) max simultaneous Chromium sessions. +# BROWSERLESS_QUEUED=4 # (--profile visual-review) max requests waiting for a session. +# BROWSERLESS_TIMEOUT_MS=30000 # (--profile visual-review) per-session timeout. +# BROWSERLESS_MEM_LIMIT=2g # (--profile visual-review) container memory cap. # REVIEW_AUDIT_DIR= # persist visual-review screenshot PNGs to this filesystem path # # instead of re-rendering on demand. Unset = re-render each time. # # Only relevant when BROWSER_WS_ENDPOINT above is set. diff --git a/Dockerfile b/Dockerfile index 7f55c11199..77c7096640 100644 --- a/Dockerfile +++ b/Dockerfile @@ -52,7 +52,8 @@ RUN mkdir -p /home/node/.npm-global /home/node/.npm \ USER node RUN if [ "$INSTALL_AI_CLIS" = "true" ]; then npm install -g --foreground-scripts @anthropic-ai/claude-code@2.1.187 @openai/codex@0.142.0 && npm cache clean --force; fi USER root -# Optional: enable visual review via an external Chrome sidecar (e.g. `browserless/chrome:latest`). +# Optional: enable visual review via an external Chrome sidecar (docker-compose --profile visual-review +# bundles `ghcr.io/browserless/chromium:latest`, or point at your own browserless-compatible instance). # Build with `--build-arg INSTALL_VISUAL_REVIEW=true` then set BROWSER_WS_ENDPOINT= at runtime. ARG INSTALL_VISUAL_REVIEW=false COPY package*.json ./ diff --git a/docker-compose.yml b/docker-compose.yml index 728373d6d3..abc21083b1 100644 --- a/docker-compose.yml +++ b/docker-compose.yml @@ -15,6 +15,7 @@ # --profile pgbouncer PgBouncer connection pooler in front of Postgres # --profile qdrant Qdrant vector database for RAG # --profile ollama Local Ollama AI backend +# --profile visual-review Headless Chromium (browserless) for before/after PR screenshot capture # --profile litestream Continuous SQLite backup to S3/B2/R2 via Litestream # --profile caddy Caddy HTTPS terminator with auto-TLS (set DOMAIN= in .env) # --profile observability Prometheus + Alertmanager + Loki + Promtail + Grafana (pre-wired) @@ -139,6 +140,13 @@ services: pgbouncer: condition: service_healthy required: false + # Unlike postgres/qdrant above, this is NOT a crash-loop risk if lost: visual capture is used + # per-PR-event, long after boot, not at startup, and every capture call is already fail-safe + # (a network error just degrades to a placeholder, never a thrown error). This dependency exists + # purely so `docker compose --profile visual-review up` starts services in a sensible order. + browserless: + condition: service_healthy + required: false # Sane defaults so a runaway optional service (below) can't OOM-kill or starve the core review pipeline; # override via .env for a bigger/smaller host (#1828). deploy: @@ -323,6 +331,51 @@ services: limits: memory: "${OLLAMA_MEM_LIMIT:-8g}" + # ── Browserless / visual review (--profile visual-review) ────────────────── + # Headless Chromium for automated before/after PR screenshot capture (src/review/visual/**). + # Set BOTH of these in .env (browserless always requires a token; there is no way around setting + # it in two places — the server reads TOKEN below, the app embeds the same value in the client URL): + # BROWSERLESS_TOKEN= + # BROWSER_WS_ENDPOINT=ws://browserless:3000?token= + # Then rebuild the app image with --build-arg INSTALL_VISUAL_REVIEW=true (installs puppeteer-core; + # see Dockerfile) and set GITTENSORY_REVIEW_SCREENSHOTS=true. Unset/default = fully inert, no + # container, no screenshots, no error — this whole feature is opt-in end to end. + browserless: + image: ghcr.io/browserless/chromium:latest + restart: unless-stopped + <<: *default-logging + profiles: ["visual-review"] + environment: + # Deliberately a SOFT default (":-", not ":?") -- docker compose interpolates every service's env + # vars for the WHOLE FILE even when a service's profile isn't active, so a hard-required var here + # would break `docker compose up` for every self-hoster NOT using visual review. If left unset, + # browserless auto-generates its own random token per boot (see its startup log) that will never + # match whatever you hardcoded into BROWSER_WS_ENDPOINT below -- captures then just fail closed + # (graceful placeholder, not a crash) instead of authenticating. Set BROWSERLESS_TOKEN explicitly. + TOKEN: ${BROWSERLESS_TOKEN:-} + # Self-host reviews PRs one at a time in practice; keep this low so a stray retry storm can't + # spin up a dozen Chromium tabs. Raise via env if you genuinely need more parallel captures. + CONCURRENT: "${BROWSERLESS_CONCURRENT:-2}" + QUEUED: "${BROWSERLESS_QUEUED:-4}" + TIMEOUT: "${BROWSERLESS_TIMEOUT_MS:-30000}" + # Never published to the host — reached only over the internal docker network at ws://browserless:3000, + # same in-network-only posture as postgres/redis/qdrant. + expose: + - "3000" + # Chrome needs real shared memory or it crashes under load writing to /tmp instead of /dev/shm. + # Docker's 64 MiB default is far too small for this; 2g matches browserless's own documented minimum. + shm_size: "2g" + deploy: + resources: + limits: + memory: "${BROWSERLESS_MEM_LIMIT:-2g}" + healthcheck: + test: ["CMD", "curl", "-f", "http://127.0.0.1:3000/docs"] + interval: 10s + timeout: 5s + start_period: 15s + retries: 5 + # ── Litestream (--profile litestream) ───────────────────────────────────── # Continuous WAL backup of the SQLite DB to S3/B2/R2. Copy litestream.yml.example # → litestream.yml and fill in your bucket. Set LITESTREAM_* secrets in .env. diff --git a/scripts/smoke-selfhost.sh b/scripts/smoke-selfhost.sh index 32915802bf..b1e80ded51 100755 --- a/scripts/smoke-selfhost.sh +++ b/scripts/smoke-selfhost.sh @@ -15,6 +15,12 @@ # # Assert an event must NOT appear (e.g. no AI-CLI-missing warning, no failed relay registration): # SELFHOST_SMOKE_FORBID_EVENTS="selfhost_ai_cli_missing" ./scripts/smoke-selfhost.sh gittensory:selfhost-ci +# +# Visual review (#3608): also boots a browserless/chromium sidecar, wires BROWSER_WS_ENDPOINT + +# PUBLIC_SITE_ORIGIN automatically, and asserts the on-demand /gittensory/shot?url= route returns a real +# PNG -- proving captureShot() actually renders through the self-host stub end to end, not just that the +# app boots. The IMAGE under test must have been built with --build-arg INSTALL_VISUAL_REVIEW=true. +# SELFHOST_SMOKE_VISUAL_REVIEW=1 ./scripts/smoke-selfhost.sh gittensory:selfhost-ci-visual set -euo pipefail IMAGE="${1:?usage: smoke-selfhost.sh }" @@ -32,6 +38,8 @@ REDIS_NAME="${SELFHOST_SMOKE_REDIS_NAME:-gt-smoke-redis-$$}" APP_NAME="${SELFHOST_SMOKE_APP_NAME:-gt-smoke-app-$$}" PORT="${SELFHOST_SMOKE_PORT:-8787}" HEALTH_TIMEOUT_SECONDS="${SELFHOST_SMOKE_HEALTH_TIMEOUT_SECONDS:-90}" +VISUAL_REVIEW="${SELFHOST_SMOKE_VISUAL_REVIEW:-0}" +BROWSERLESS_NAME="${SELFHOST_SMOKE_BROWSERLESS_NAME:-gt-smoke-browserless-$$}" require_cmd() { if ! command -v "$1" >/dev/null 2>&1; then @@ -51,7 +59,7 @@ else fi cleanup() { - docker rm -f "$APP_NAME" "$REDIS_NAME" >/dev/null 2>&1 || true + docker rm -f "$APP_NAME" "$REDIS_NAME" "$BROWSERLESS_NAME" >/dev/null 2>&1 || true if [ "$NETWORK_OWNED" = "1" ]; then docker network rm "$NETWORK_NAME" >/dev/null 2>&1 || true fi @@ -79,6 +87,46 @@ if [ "$redis_ok" != "1" ]; then exit 1 fi +VISUAL_EXTRA_ENV_ARGS=() +if [ "$VISUAL_REVIEW" = "1" ]; then + echo "smoke-selfhost: booting browserless/chromium for visual-review mode" + BROWSERLESS_TOKEN_SMOKE="$(od -An -N16 -tx1 /dev/urandom | tr -d ' \n')" + docker run -d --name "$BROWSERLESS_NAME" --network "$NETWORK_NAME" --shm-size 2g \ + -e "TOKEN=${BROWSERLESS_TOKEN_SMOKE}" \ + ghcr.io/browserless/chromium:latest >/dev/null + browserless_ok=0 + for _ in $(seq 1 30); do + if docker exec "$BROWSERLESS_NAME" curl -sf "http://127.0.0.1:3000/docs" >/dev/null 2>&1; then + browserless_ok=1 + break + fi + sleep 2 + done + if [ "$browserless_ok" != "1" ]; then + echo "::error::$BROWSERLESS_NAME never became ready" >&2 + docker logs "$BROWSERLESS_NAME" >&2 || true + exit 1 + fi + # GITTENSORY_REVIEW_SCREENSHOTS must be on: the /gittensory/shot route itself 404s when it's off + # (deliberately "truly inert" by design, src/api/routes.ts), independent of BROWSER_WS_ENDPOINT. + # + # SMOKE_SHOT_TARGET must be a REAL, publicly resolvable URL, unlike this script's other *.example + # placeholder values -- those are only ever used as opaque header/origin STRINGS, never fetched. This + # one IS fetched: the browser inside the browserless container actually navigates to it, so a fake + # .example domain would fail DNS resolution and captureShot() would correctly (but uselessly, for this + # test) degrade to null -- silently turning the assertion below into a no-op rather than a real proof. + # Defaults to example.com (IANA-reserved for exactly this kind of testing, stable, minimal). isSafeHttpUrl + # (SSRF guard) also means this can't be pointed at a docker-internal hostname -- it must stay a real + # public host, matching how production actually uses this endpoint (real preview-deploy URLs, never + # internal addresses). Placed BEFORE the caller's own EXTRA_ENV_ARGS so an explicit override still wins. + SMOKE_SHOT_TARGET="${SELFHOST_SMOKE_VISUAL_TARGET_URL:-https://example.com}" + VISUAL_EXTRA_ENV_ARGS=( + -e "GITTENSORY_REVIEW_SCREENSHOTS=true" + -e "BROWSER_WS_ENDPOINT=ws://${BROWSERLESS_NAME}:3000?token=${BROWSERLESS_TOKEN_SMOKE}" + -e "PUBLIC_SITE_ORIGIN=${SMOKE_SHOT_TARGET}" + ) +fi + # Extra env, one KEY=VALUE per line -- turned into repeated -e flags. Deliberately whitespace/newline # separated (not comma) so values containing commas (e.g. AI_PROVIDER=claude-code,codex) are unambiguous. # NOT for multiline secrets like a PEM private key -- a newline inside a value is indistinguishable from @@ -106,6 +154,7 @@ docker run -d --name "$APP_NAME" --network "$NETWORK_NAME" -p "127.0.0.1:${PORT} -e "REDIS_URL=redis://${REDIS_NAME}:6379" \ -e "SELFHOST_SETUP_TOKEN=${SETUP_TOKEN}" \ -e "PUBLIC_API_ORIGIN=${SELFHOST_SMOKE_PUBLIC_API_ORIGIN:-https://selfhost-smoke.example}" \ + "${VISUAL_EXTRA_ENV_ARGS[@]}" \ "${EXTRA_ENV_ARGS[@]}" \ "${EXTRA_VOLUME_ARGS[@]}" \ "$IMAGE" >/dev/null @@ -130,6 +179,29 @@ curl -sf "http://127.0.0.1:${PORT}/health" | grep -q '"status":"ok"' curl -sf "http://127.0.0.1:${PORT}/ready" | grep -q '"ok":true' curl -sf "http://127.0.0.1:${PORT}/metrics" | grep -q 'gittensory_uptime_seconds' +if [ "$VISUAL_REVIEW" = "1" ]; then + echo "smoke-selfhost: checking /gittensory/shot renders a real PNG through the self-host browser stub" + SHOT_URL="http://127.0.0.1:${PORT}/gittensory/shot?url=$(printf '%s' "$SMOKE_SHOT_TARGET" | tr -d '\n')" + SHOT_HEADERS="$(curl -sf -D - -o /tmp/gt-smoke-shot.png "$SHOT_URL")" + echo "$SHOT_HEADERS" | grep -qi '^content-type: image/png' || { + echo "::error::/gittensory/shot did not return image/png" >&2 + echo "$SHOT_HEADERS" >&2 + docker logs "$APP_NAME" >&2 || true + docker logs "$BROWSERLESS_NAME" >&2 || true + exit 1 + } + SHOT_BYTES="$(wc -c &2 + docker logs "$APP_NAME" >&2 || true + exit 1 + fi + echo "smoke-selfhost: /gittensory/shot returned a real PNG (${SHOT_BYTES} bytes)" + rm -f /tmp/gt-smoke-shot.png +fi + LOGS="$(docker logs "$APP_NAME" 2>&1)" if [ -n "${SELFHOST_SMOKE_EXPECT_EVENTS:-}" ]; then