fix(observability): harden self-host trace pipeline

.env.example

@@ -209,7 +209,8 @@ REDIS_URL=redis://redis:6379                # REQUIRED for the self-host review
 # The observability profile starts Prometheus (scrapes /metrics) + Alertmanager (alert rules in
 # prometheus/rules/, routing in alertmanager/alertmanager.yml — silent until you fill in a receiver) +
 # Loki + Promtail (ship every container's logs to Loki) + Grafana (dashboards for metrics AND logs).
-# GRAFANA_ADMIN_PASSWORD=changeme            # REQUIRED when using --profile observability; compose fails if unset
+# GRAFANA_ADMIN_PASSWORD=                    # REQUIRED at runtime when using --profile observability; generate a strong value
+# GRAFANA_LOCAL_SMOKE_PASSWORD=              # optional local-only fallback for smoke tests; never expose Grafana with this
 #
 # Maintainer dashboards (in addition to the infra dashboard):
 #   • "Reviews & PRs (maintainer)" — per-repo + combined PR/review analytics from a redacted reporting DB export.

apps/gittensory-ui/src/routes/docs.self-hosting-troubleshooting.tsx

@@ -121,6 +121,40 @@ rees_analyzer_config_invalid`}
 docker compose logs gittensory | grep selfhost_job_dead`}
       />
 
+      <h2>Grafana traces error or show no data</h2>
+      <p>
+        The trace path is app or smoke process → OTEL collector → Tempo → Grafana. Tempo is only
+        started by the observability profile, and app traces are only emitted when{" "}
+        <code>OTEL_TRACES_EXPORTER</code> includes <code>otlp</code>.
+      </p>
+      <CodeBlock
+        lang="bash"
+        code={`docker compose --profile observability ps tempo otel-collector grafana
+docker compose logs --tail=80 tempo otel-collector grafana
+
+# Send one synthetic span through the collector and read it back from Tempo.
+npm run test:smoke:observability`}
+      />
+      <ul>
+        <li>
+          If the smoke command fails at <code>otel-collector:4318/v1/traces</code>, the collector is
+          not reachable from the app container.
+        </li>
+        <li>
+          If it pushes successfully but cannot read{" "}
+          <code>tempo:3200/api/traces/&lt;trace_id&gt;</code>, Tempo is unhealthy, not ingesting, or
+          not sharing the Compose network.
+        </li>
+        <li>
+          If the smoke command passes but Grafana Explore fails, check the Tempo data source URL. It
+          should point at <code>http://tempo:3200</code>, not the OTLP ingest ports.
+        </li>
+        <li>
+          For a temporary live debugging run, set <code>OTEL_TRACES_SAMPLER_ARG=1</code> so every
+          root trace is sampled, then lower it again after diagnosis.
+        </li>
+      </ul>
+
       <h2>Readiness fails</h2>
       <FeatureRow
         items={[

docker-compose.yml

@@ -300,11 +300,22 @@ services:
       # Maintainer dashboards query only the redacted reporting export, never the live app DB.
       - grafana-reporting-data:/reporting:ro
     environment:
-      GF_SECURITY_ADMIN_PASSWORD: ${GRAFANA_ADMIN_PASSWORD:?Set GRAFANA_ADMIN_PASSWORD in .env before using --profile observability}
+      # Compose interpolates every service even when --profile observability is inactive, so the required-password
+      # check lives in entrypoint below. Runtime still fails closed unless the operator sets GRAFANA_ADMIN_PASSWORD.
+      GF_SECURITY_ADMIN_PASSWORD: ${GRAFANA_ADMIN_PASSWORD:-${GRAFANA_LOCAL_SMOKE_PASSWORD:-}}
       GF_USERS_ALLOW_SIGN_UP: "false"
       GF_INSTALL_PLUGINS: frser-sqlite-datasource,grafana-github-datasource
       # Read-only fine-grained PAT for the GitHub data source provisioning ($GITHUB_TOKEN expansion). From .env.
       GITHUB_TOKEN: "${GITHUB_TOKEN:-}"
+    entrypoint:
+      - /bin/sh
+      - -ec
+      - |
+        if [ -z "$${GF_SECURITY_ADMIN_PASSWORD:-}" ]; then
+          echo >&2 "Set GRAFANA_ADMIN_PASSWORD in .env before using --profile observability, or set GRAFANA_LOCAL_SMOKE_PASSWORD for local-only smoke tests."
+          exit 1
+        fi
+        exec /run.sh
 
   reporting-exporter:
     image: alpine:3.20
@@ -417,6 +428,9 @@ services:
     image: otel/opentelemetry-collector-contrib:0.155.0
     restart: unless-stopped
     profiles: ["observability"]
+    depends_on:
+      tempo:
+        condition: service_healthy
     # Mount our config at the image's default path so the stock entrypoint picks it up (no command override).
     volumes:
       - ./otel/otel-collector-config.yml:/etc/otelcol-contrib/config.yaml:ro
@@ -432,6 +446,12 @@ services:
     volumes:
       - ./tempo/tempo.yaml:/etc/tempo/tempo.yaml:ro
       - tempo-data:/var/tempo
+    healthcheck:
+      test: ["CMD", "wget", "-qO-", "http://127.0.0.1:3200/ready"]
+      interval: 10s
+      timeout: 5s
+      start_period: 20s
+      retries: 12
 
   # ── Tailscale (--profile tailscale) ───────────────────────────────────────
   # Joins your tailnet so the stack is accessible via Tailscale IP/hostname — no

package.json

@@ -56,6 +56,7 @@
     "test:workers": "vitest run --config vitest.workers.config.ts",
     "test:coverage": "vitest run --coverage",
     "test:smoke:production": "node scripts/smoke-production.mjs",
+    "test:smoke:observability": "node scripts/smoke-observability-traces.mjs",
     "test:smoke:browser:install": "playwright install chromium",
     "test:smoke:browser": "node scripts/smoke-ui-browser.mjs",
     "test:ci": "git diff --check && npm run actionlint && npm run db:migrations:check && npm run typecheck && npm run test:coverage && npm run test:workers && npm run build:mcp && npm run test:mcp-pack && npm run rees:test && npm run ui:openapi:check && npm run ui:version-audit && npm run ui:lint && npm run ui:typecheck && npm run ui:test && npm run ui:build",

scripts/smoke-observability-traces.mjs

@@ -0,0 +1,80 @@
+#!/usr/bin/env node
+import { randomBytes } from "node:crypto";
+import { execFileSync } from "node:child_process";
+
+const composeService = process.env.SELFHOST_SERVICE ?? "gittensory";
+const timeoutMs = Number(process.env.OBSERVABILITY_SMOKE_TIMEOUT_MS ?? "30000");
+const pollIntervalMs = Number(
+  process.env.OBSERVABILITY_SMOKE_POLL_MS ?? "1000",
+);
+const traceId = randomBytes(16).toString("hex");
+const spanId = randomBytes(8).toString("hex");
+
+await main();
+
+async function main() {
+  if (!Number.isFinite(timeoutMs) || timeoutMs <= 0)
+    throw new Error("OBSERVABILITY_SMOKE_TIMEOUT_MS must be a positive number");
+  if (!Number.isFinite(pollIntervalMs) || pollIntervalMs <= 0)
+    throw new Error("OBSERVABILITY_SMOKE_POLL_MS must be a positive number");
+
+  const script = `
+const traceId = ${JSON.stringify(traceId)};
+const spanId = ${JSON.stringify(spanId)};
+const start = BigInt(Date.now()) * 1000000n;
+const body = {
+  resourceSpans: [{
+    resource: {
+      attributes: [
+        { key: "service.name", value: { stringValue: "gittensory-selfhost-smoke" } },
+        { key: "deployment.environment.name", value: { stringValue: "selfhost-smoke" } }
+      ]
+    },
+    scopeSpans: [{
+      scope: { name: "gittensory-selfhost-smoke" },
+      spans: [{
+        traceId,
+        spanId,
+        name: "selfhost.observability.smoke",
+        kind: 1,
+        startTimeUnixNano: String(start),
+        endTimeUnixNano: String(start + 1000000n),
+        attributes: [{ key: "smoke.kind", value: { stringValue: "tempo" } }],
+        status: { code: 1 }
+      }]
+    }]
+  }]
+};
+const push = await fetch("http://otel-collector:4318/v1/traces", {
+  method: "POST",
+  headers: { "content-type": "application/json" },
+  body: JSON.stringify(body)
+});
+if (!push.ok) throw new Error("collector rejected smoke trace: " + push.status + " " + await push.text());
+const deadline = Date.now() + ${JSON.stringify(timeoutMs)};
+let last = "";
+while (Date.now() <= deadline) {
+  const res = await fetch("http://tempo:3200/api/traces/" + traceId);
+  if (res.ok) {
+    const json = await res.json();
+    if (JSON.stringify(json).includes("selfhost.observability.smoke")) {
+      console.log(JSON.stringify({ ok: true, traceId }));
+      process.exit(0);
+    }
+    last = "trace response did not contain smoke span";
+  } else {
+    last = res.status + " " + await res.text();
+  }
+  await new Promise((resolve) => setTimeout(resolve, ${JSON.stringify(pollIntervalMs)}));
+}
+throw new Error("tempo did not return smoke trace " + traceId + ": " + last.slice(0, 300));
+`;
+
+  execFileSync(
+    "docker",
+    ["compose", "exec", "-T", composeService, "node", "-e", script],
+    {
+      stdio: "inherit",
+    },
+  );
+}

test/unit/selfhost-observability-config.test.ts

@@ -0,0 +1,91 @@
+import { readFileSync } from "node:fs";
+import { join } from "node:path";
+import { describe, expect, it } from "vitest";
+import { parse } from "yaml";
+
+function readYaml(path: string): unknown {
+  return parse(readFileSync(join(process.cwd(), path), "utf8"));
+}
+
+function record(value: unknown): Record<string, any> {
+  expect(value).toBeTruthy();
+  expect(typeof value).toBe("object");
+  return value as Record<string, any>;
+}
+
+describe("self-host observability trace config", () => {
+  it("gates Tempo consumers without breaking the default Compose profile", () => {
+    const compose = record(readYaml("docker-compose.yml"));
+    const services = record(compose.services);
+    const tempo = record(services.tempo);
+    const grafana = record(services.grafana);
+    const collector = record(services["otel-collector"]);
+
+    expect(tempo.healthcheck?.test).toEqual([
+      "CMD",
+      "wget",
+      "-qO-",
+      "http://127.0.0.1:3200/ready",
+    ]);
+    expect(tempo.healthcheck?.start_period).toBe("20s");
+    expect(tempo.healthcheck?.retries).toBe(12);
+    expect(grafana.depends_on?.tempo).toBeUndefined();
+    expect(collector.depends_on?.tempo).toEqual({
+      condition: "service_healthy",
+    });
+    expect(grafana.environment?.GF_SECURITY_ADMIN_PASSWORD).toBe(
+      "${GRAFANA_ADMIN_PASSWORD:-${GRAFANA_LOCAL_SMOKE_PASSWORD:-}}",
+    );
+    expect(JSON.stringify(grafana)).not.toContain("changeme");
+    expect(grafana.entrypoint).toEqual(
+      expect.arrayContaining([
+        expect.stringContaining("Set GRAFANA_ADMIN_PASSWORD"),
+        expect.stringContaining("exec /run.sh"),
+      ]),
+    );
+
+    for (const [name, service] of Object.entries(services)) {
+      const serviceRecord = record(service);
+      if (!serviceRecord.depends_on?.tempo) continue;
+      expect(serviceRecord.profiles, name).toContain("observability");
+      expect(tempo.profiles, "tempo").toContain("observability");
+    }
+  });
+
+  it("keeps the collector, Tempo, and Grafana data source on the same trace path", () => {
+    const collector = record(readYaml("otel/otel-collector-config.yml"));
+    const tempo = record(readYaml("tempo/tempo.yaml"));
+    const datasource = record(
+      readYaml("grafana/provisioning/datasources/tempo.yml"),
+    );
+
+    expect(record(collector.exporters)["otlp/tempo"].endpoint).toBe(
+      "tempo:4317",
+    );
+    expect(
+      record(record(collector.service).pipelines).traces.exporters,
+    ).toEqual(["otlp/tempo"]);
+    expect(
+      record(record(record(record(tempo.distributor).receivers).otlp).protocols)
+        .grpc.endpoint,
+    ).toBe("0.0.0.0:4317");
+    expect(
+      record(record(record(record(tempo.distributor).receivers).otlp).protocols)
+        .http.endpoint,
+    ).toBe("0.0.0.0:4318");
+    expect(record(record(tempo.storage).trace).backend).toBe("local");
+    expect(record(datasource.datasources?.[0]).url).toBe("http://tempo:3200");
+  });
+
+  it("ships an operator smoke probe that verifies collector to Tempo retrieval", () => {
+    const script = readFileSync(
+      join(process.cwd(), "scripts/smoke-observability-traces.mjs"),
+      "utf8",
+    );
+
+    expect(script).toContain("http://otel-collector:4318/v1/traces");
+    expect(script).toContain("http://tempo:3200/api/traces/");
+    expect(script).toContain("gittensory-selfhost-smoke");
+    expect(script).toContain("selfhost.observability.smoke");
+  });
+});