diff --git a/.env.example b/.env.example index 3d09e3ebe3..85bd9b9831 100644 --- a/.env.example +++ b/.env.example @@ -209,7 +209,8 @@ REDIS_URL=redis://redis:6379 # REQUIRED for the self-host review # The observability profile starts Prometheus (scrapes /metrics) + Alertmanager (alert rules in # prometheus/rules/, routing in alertmanager/alertmanager.yml — silent until you fill in a receiver) + # Loki + Promtail (ship every container's logs to Loki) + Grafana (dashboards for metrics AND logs). -# GRAFANA_ADMIN_PASSWORD=changeme # REQUIRED when using --profile observability; compose fails if unset +# GRAFANA_ADMIN_PASSWORD= # REQUIRED at runtime when using --profile observability; generate a strong value +# GRAFANA_LOCAL_SMOKE_PASSWORD= # optional local-only fallback for smoke tests; never expose Grafana with this # # Maintainer dashboards (in addition to the infra dashboard): # • "Reviews & PRs (maintainer)" — per-repo + combined PR/review analytics from a redacted reporting DB export. diff --git a/apps/gittensory-ui/src/routes/docs.self-hosting-troubleshooting.tsx b/apps/gittensory-ui/src/routes/docs.self-hosting-troubleshooting.tsx index b05f8b340e..190d11087d 100644 --- a/apps/gittensory-ui/src/routes/docs.self-hosting-troubleshooting.tsx +++ b/apps/gittensory-ui/src/routes/docs.self-hosting-troubleshooting.tsx @@ -121,6 +121,40 @@ rees_analyzer_config_invalid`} docker compose logs gittensory | grep selfhost_job_dead`} /> +

Grafana traces error or show no data

+

+ The trace path is app or smoke process → OTEL collector → Tempo → Grafana. Tempo is only + started by the observability profile, and app traces are only emitted when{" "} + OTEL_TRACES_EXPORTER includes otlp. +

+ + +

Readiness fails

&2 "Set GRAFANA_ADMIN_PASSWORD in .env before using --profile observability, or set GRAFANA_LOCAL_SMOKE_PASSWORD for local-only smoke tests." + exit 1 + fi + exec /run.sh reporting-exporter: image: alpine:3.20 @@ -417,6 +428,9 @@ services: image: otel/opentelemetry-collector-contrib:0.155.0 restart: unless-stopped profiles: ["observability"] + depends_on: + tempo: + condition: service_healthy # Mount our config at the image's default path so the stock entrypoint picks it up (no command override). volumes: - ./otel/otel-collector-config.yml:/etc/otelcol-contrib/config.yaml:ro @@ -432,6 +446,12 @@ services: volumes: - ./tempo/tempo.yaml:/etc/tempo/tempo.yaml:ro - tempo-data:/var/tempo + healthcheck: + test: ["CMD", "wget", "-qO-", "http://127.0.0.1:3200/ready"] + interval: 10s + timeout: 5s + start_period: 20s + retries: 12 # ── Tailscale (--profile tailscale) ─────────────────────────────────────── # Joins your tailnet so the stack is accessible via Tailscale IP/hostname — no diff --git a/package.json b/package.json index dc4f8a0401..ddb5e8bd41 100644 --- a/package.json +++ b/package.json @@ -56,6 +56,7 @@ "test:workers": "vitest run --config vitest.workers.config.ts", "test:coverage": "vitest run --coverage", "test:smoke:production": "node scripts/smoke-production.mjs", + "test:smoke:observability": "node scripts/smoke-observability-traces.mjs", "test:smoke:browser:install": "playwright install chromium", "test:smoke:browser": "node scripts/smoke-ui-browser.mjs", "test:ci": "git diff --check && npm run actionlint && npm run db:migrations:check && npm run typecheck && npm run test:coverage && npm run test:workers && npm run build:mcp && npm run test:mcp-pack && npm run rees:test && npm run ui:openapi:check && npm run ui:version-audit && npm run ui:lint && npm run ui:typecheck && npm run ui:test && npm run ui:build", diff --git a/scripts/smoke-observability-traces.mjs b/scripts/smoke-observability-traces.mjs new file mode 100755 index 0000000000..424a700071 --- /dev/null +++ b/scripts/smoke-observability-traces.mjs @@ -0,0 +1,80 @@ +#!/usr/bin/env node +import { randomBytes } from "node:crypto"; +import { execFileSync } from "node:child_process"; + +const composeService = process.env.SELFHOST_SERVICE ?? "gittensory"; +const timeoutMs = Number(process.env.OBSERVABILITY_SMOKE_TIMEOUT_MS ?? "30000"); +const pollIntervalMs = Number( + process.env.OBSERVABILITY_SMOKE_POLL_MS ?? "1000", +); +const traceId = randomBytes(16).toString("hex"); +const spanId = randomBytes(8).toString("hex"); + +await main(); + +async function main() { + if (!Number.isFinite(timeoutMs) || timeoutMs <= 0) + throw new Error("OBSERVABILITY_SMOKE_TIMEOUT_MS must be a positive number"); + if (!Number.isFinite(pollIntervalMs) || pollIntervalMs <= 0) + throw new Error("OBSERVABILITY_SMOKE_POLL_MS must be a positive number"); + + const script = ` +const traceId = ${JSON.stringify(traceId)}; +const spanId = ${JSON.stringify(spanId)}; +const start = BigInt(Date.now()) * 1000000n; +const body = { + resourceSpans: [{ + resource: { + attributes: [ + { key: "service.name", value: { stringValue: "gittensory-selfhost-smoke" } }, + { key: "deployment.environment.name", value: { stringValue: "selfhost-smoke" } } + ] + }, + scopeSpans: [{ + scope: { name: "gittensory-selfhost-smoke" }, + spans: [{ + traceId, + spanId, + name: "selfhost.observability.smoke", + kind: 1, + startTimeUnixNano: String(start), + endTimeUnixNano: String(start + 1000000n), + attributes: [{ key: "smoke.kind", value: { stringValue: "tempo" } }], + status: { code: 1 } + }] + }] + }] +}; +const push = await fetch("http://otel-collector:4318/v1/traces", { + method: "POST", + headers: { "content-type": "application/json" }, + body: JSON.stringify(body) +}); +if (!push.ok) throw new Error("collector rejected smoke trace: " + push.status + " " + await push.text()); +const deadline = Date.now() + ${JSON.stringify(timeoutMs)}; +let last = ""; +while (Date.now() <= deadline) { + const res = await fetch("http://tempo:3200/api/traces/" + traceId); + if (res.ok) { + const json = await res.json(); + if (JSON.stringify(json).includes("selfhost.observability.smoke")) { + console.log(JSON.stringify({ ok: true, traceId })); + process.exit(0); + } + last = "trace response did not contain smoke span"; + } else { + last = res.status + " " + await res.text(); + } + await new Promise((resolve) => setTimeout(resolve, ${JSON.stringify(pollIntervalMs)})); +} +throw new Error("tempo did not return smoke trace " + traceId + ": " + last.slice(0, 300)); +`; + + execFileSync( + "docker", + ["compose", "exec", "-T", composeService, "node", "-e", script], + { + stdio: "inherit", + }, + ); +} diff --git a/test/unit/selfhost-observability-config.test.ts b/test/unit/selfhost-observability-config.test.ts new file mode 100644 index 0000000000..c720732e31 --- /dev/null +++ b/test/unit/selfhost-observability-config.test.ts @@ -0,0 +1,91 @@ +import { readFileSync } from "node:fs"; +import { join } from "node:path"; +import { describe, expect, it } from "vitest"; +import { parse } from "yaml"; + +function readYaml(path: string): unknown { + return parse(readFileSync(join(process.cwd(), path), "utf8")); +} + +function record(value: unknown): Record { + expect(value).toBeTruthy(); + expect(typeof value).toBe("object"); + return value as Record; +} + +describe("self-host observability trace config", () => { + it("gates Tempo consumers without breaking the default Compose profile", () => { + const compose = record(readYaml("docker-compose.yml")); + const services = record(compose.services); + const tempo = record(services.tempo); + const grafana = record(services.grafana); + const collector = record(services["otel-collector"]); + + expect(tempo.healthcheck?.test).toEqual([ + "CMD", + "wget", + "-qO-", + "http://127.0.0.1:3200/ready", + ]); + expect(tempo.healthcheck?.start_period).toBe("20s"); + expect(tempo.healthcheck?.retries).toBe(12); + expect(grafana.depends_on?.tempo).toBeUndefined(); + expect(collector.depends_on?.tempo).toEqual({ + condition: "service_healthy", + }); + expect(grafana.environment?.GF_SECURITY_ADMIN_PASSWORD).toBe( + "${GRAFANA_ADMIN_PASSWORD:-${GRAFANA_LOCAL_SMOKE_PASSWORD:-}}", + ); + expect(JSON.stringify(grafana)).not.toContain("changeme"); + expect(grafana.entrypoint).toEqual( + expect.arrayContaining([ + expect.stringContaining("Set GRAFANA_ADMIN_PASSWORD"), + expect.stringContaining("exec /run.sh"), + ]), + ); + + for (const [name, service] of Object.entries(services)) { + const serviceRecord = record(service); + if (!serviceRecord.depends_on?.tempo) continue; + expect(serviceRecord.profiles, name).toContain("observability"); + expect(tempo.profiles, "tempo").toContain("observability"); + } + }); + + it("keeps the collector, Tempo, and Grafana data source on the same trace path", () => { + const collector = record(readYaml("otel/otel-collector-config.yml")); + const tempo = record(readYaml("tempo/tempo.yaml")); + const datasource = record( + readYaml("grafana/provisioning/datasources/tempo.yml"), + ); + + expect(record(collector.exporters)["otlp/tempo"].endpoint).toBe( + "tempo:4317", + ); + expect( + record(record(collector.service).pipelines).traces.exporters, + ).toEqual(["otlp/tempo"]); + expect( + record(record(record(record(tempo.distributor).receivers).otlp).protocols) + .grpc.endpoint, + ).toBe("0.0.0.0:4317"); + expect( + record(record(record(record(tempo.distributor).receivers).otlp).protocols) + .http.endpoint, + ).toBe("0.0.0.0:4318"); + expect(record(record(tempo.storage).trace).backend).toBe("local"); + expect(record(datasource.datasources?.[0]).url).toBe("http://tempo:3200"); + }); + + it("ships an operator smoke probe that verifies collector to Tempo retrieval", () => { + const script = readFileSync( + join(process.cwd(), "scripts/smoke-observability-traces.mjs"), + "utf8", + ); + + expect(script).toContain("http://otel-collector:4318/v1/traces"); + expect(script).toContain("http://tempo:3200/api/traces/"); + expect(script).toContain("gittensory-selfhost-smoke"); + expect(script).toContain("selfhost.observability.smoke"); + }); +});