test(observability): guard against today's two Grafana bug classes

test/unit/selfhost-grafana-datasource-uids.test.ts

@@ -0,0 +1,79 @@
+import { readFileSync, readdirSync } from "node:fs";
+import { join } from "node:path";
+import { parse } from "yaml";
+import { describe, expect, it } from "vitest";
+
+// REGRESSION guard (#orb-grafana-datasource-uid-crash, 2026-07-14): a self-host box's Grafana instance
+// persists provisioned datasources (including their `uid`) in its own long-lived sqlite database, separate
+// from this repo's git history. If a datasource's `uid` here is silently renamed without a corresponding
+// one-time migration of that already-deployed instance's own DB row, Grafana's datasource-provisioning
+// module hard-fails on boot ("Datasource provisioning error: data source not found") -- crash-looping the
+// ENTIRE Grafana container, not just showing one broken panel. This actually happened live: `sqlite.yml`'s
+// `LoopoverDB` datasource was renamed `gittensory-db` -> `loopover-db` as part of the loopover rebrand, but
+// an already-running box's Grafana DB still had it registered under the old uid, and the very next
+// container recreate crashed it outright. Fixed with a one-time `UPDATE data_source SET uid = ...` on that
+// instance (not a code change, and not reproducible from a fresh install). This test pins the uids so a
+// FUTURE accidental/silent rename here fails CI immediately -- deliberately renaming a datasource still
+// requires updating this test, which is the intended prompt to also plan the live-instance migration step.
+
+type DatasourceEntry = { name: string; type: string; uid: string };
+type DatasourceFile = { apiVersion: number; datasources: DatasourceEntry[] };
+
+function readDatasources(path: string): DatasourceEntry[] {
+  const parsed = parse(readFileSync(path, "utf8")) as DatasourceFile;
+  return parsed.datasources;
+}
+
+describe("LoopOver — Grafana provisioned datasource uids are pinned (#orb-grafana-datasource-uid-crash)", () => {
+  it("sqlite.yml: LoopoverDB stays uid loopover-db", () => {
+    const datasources = readDatasources("grafana/provisioning/datasources/sqlite.yml");
+    const loopoverDb = datasources.find((d) => d.name === "LoopoverDB");
+    expect(loopoverDb?.type).toBe("frser-sqlite-datasource");
+    expect(loopoverDb?.uid).toBe("loopover-db");
+  });
+
+  it("prometheus.yml/loki.yml/tempo.yml: core observability datasource uids are pinned", () => {
+    expect(readDatasources("grafana/provisioning/datasources/prometheus.yml")[0]).toMatchObject({ uid: "prometheus", type: "prometheus" });
+    expect(readDatasources("grafana/provisioning/datasources/loki.yml")[0]).toMatchObject({ uid: "loki", type: "loki" });
+    expect(readDatasources("grafana/provisioning/datasources/tempo.yml")[0]).toMatchObject({ uid: "tempo", type: "tempo" });
+  });
+
+  it("ams-ledgers.yml: redacted AMS reporting datasource uids are pinned", () => {
+    const datasources = readDatasources("grafana/provisioning/datasources/ams-ledgers.yml");
+    const byName = Object.fromEntries(datasources.map((d) => [d.name, d]));
+    expect(byName["AMS Attempt Log"]).toMatchObject({ uid: "ams-attempt-log", type: "frser-sqlite-datasource" });
+    expect(byName["AMS Prediction Ledger"]).toMatchObject({ uid: "ams-prediction-ledger", type: "frser-sqlite-datasource" });
+  });
+
+  it("every dashboard panel/template-variable datasource uid actually matches a provisioned datasource", () => {
+    const provisionedUids = new Set(
+      ["ams-ledgers.yml", "loki.yml", "prometheus.yml", "sqlite.yml", "tempo.yml"].flatMap((f) =>
+        readDatasources(`grafana/provisioning/datasources/${f}`).map((d) => d.uid),
+      ),
+    );
+    // ${DS_PROMETHEUS}/${DS_SENTRY} are dashboard-level template inputs Grafana resolves at import time,
+    // not literal provisioned uids -- excluded the same way every panel-level test in this repo already does.
+    provisionedUids.add("${DS_PROMETHEUS}");
+    provisionedUids.add("${DS_SENTRY}");
+    // "github" (github-prs.json, grafana-github-datasource plugin) is a KNOWN, accepted gap, not an
+    // oversight: that plugin needs a live GitHub PAT to configure, which must never be committed as
+    // provisioning YAML -- it's set up manually in the Grafana UI on each self-host instance instead, so
+    // github-prs.json genuinely has no git-tracked provisioning source of truth. Flagged separately, not
+    // silently ignored here.
+    provisionedUids.add("github");
+
+    const dashboardsDir = join(process.cwd(), "grafana/dashboards");
+    for (const file of readdirSync(dashboardsDir)) {
+      const dashboard = JSON.parse(readFileSync(join(dashboardsDir, file), "utf8")) as {
+        panels?: Array<{ datasource?: { uid?: string } }>;
+        templating?: { list?: Array<{ datasource?: { uid?: string } }> };
+      };
+      for (const panel of dashboard.panels ?? []) {
+        if (panel.datasource?.uid) expect(provisionedUids.has(panel.datasource.uid), `${file}: panel datasource uid "${panel.datasource.uid}"`).toBe(true);
+      }
+      for (const templateVar of dashboard.templating?.list ?? []) {
+        if (templateVar.datasource?.uid) expect(provisionedUids.has(templateVar.datasource.uid), `${file}: template var datasource uid "${templateVar.datasource.uid}"`).toBe(true);
+      }
+    }
+  });
+});

test/unit/selfhost-grafana-no-dollar-underscore-sentinel.test.ts

@@ -0,0 +1,51 @@
+import { readFileSync, readdirSync } from "node:fs";
+import { join } from "node:path";
+import { describe, expect, it } from "vitest";
+
+// REGRESSION guard (#orb-grafana-ai-usage-all-filter, 2026-07-14): a hand-rolled SQL "no filter" sentinel
+// must never itself look like one of Grafana's reserved `$__`-prefixed global macros (`$__all`, `$__from`,
+// `$__to`, `$__interval`, ...). Confirmed live against a real Grafana + frser-sqlite-datasource instance:
+// `${var:sqlstring}` does NOT sql-quote a value that starts with `$__` (Grafana treats it as a macro
+// reference, not literal data), so a query comparing a variable against a literal '$__all' sentinel gets a
+// raw, UNQUOTED token substituted in — which SQLite then misparses as its own `$__` named bind parameter,
+// silently returning zero rows or erroring outright, even with real underlying data. This shipped
+// undetected in ai-usage.json/maintainer-reviews.json/miner-usage.json until a live incident surfaced it
+// (see those files' own git history + selfhost-grafana-ai-usage-dashboard.test.ts's doc comment). A plain
+// string (e.g. `__ALL__`) sidesteps the ambiguity entirely and is the pattern every dashboard now uses.
+
+const dashboardsDir = join(process.cwd(), "grafana/dashboards");
+
+type TemplateVar = { name: string; allValue?: string; current?: { value?: string } };
+type Dashboard = { uid?: string; templating?: { list?: TemplateVar[] } };
+
+function dashboardFiles(): string[] {
+  return readdirSync(dashboardsDir).filter((f) => f.endsWith(".json"));
+}
+
+function readDashboard(file: string): { raw: string; parsed: Dashboard } {
+  const raw = readFileSync(join(dashboardsDir, file), "utf8");
+  return { raw, parsed: JSON.parse(raw) as Dashboard };
+}
+
+describe("LoopOver — Grafana dashboards never use a Grafana-reserved $__ macro as a hand-rolled sentinel", () => {
+  it.each(dashboardFiles())("%s: no templating variable's allValue/current.value starts with '$__'", (file) => {
+    const { parsed } = readDashboard(file);
+    for (const templateVar of parsed.templating?.list ?? []) {
+      if (typeof templateVar.allValue === "string") {
+        expect(templateVar.allValue.startsWith("$__"), `${file} templating var "${templateVar.name}" allValue`).toBe(false);
+      }
+      if (typeof templateVar.current?.value === "string") {
+        expect(templateVar.current.value.startsWith("$__"), `${file} templating var "${templateVar.name}" current.value`).toBe(false);
+      }
+    }
+  });
+
+  it.each(dashboardFiles())("%s: no panel query embeds a '$__' literal as a hand-rolled sentinel comparison", (file) => {
+    const { raw } = readDashboard(file);
+    // Grafana's own real macros ($__from/$__to/$__interval/etc.) are always used BARE, never inside a
+    // quoted string literal -- a quoted occurrence ('$__anything') is exactly the broken pattern this
+    // guards against, so only that shape is flagged (bare $__from/$__to usage elsewhere is expected/fine).
+    const quotedDollarUnderscoreLiteral = /'\$__[a-zA-Z_]*'/;
+    expect(quotedDollarUnderscoreLiteral.test(raw), file).toBe(false);
+  });
+});