feat(orb): surface fleet calibration analytics via dashboard + MCP

apps/gittensory-ui/src/routes/app.operator.tsx

@@ -24,6 +24,7 @@ type OperatorDashboardResponse = {
   noiseReduction: Array<{ label: string; value: number; spark: number[] }>;
   weeklyReport: string[];
   recommendationQuality?: RecommendationQualityReport;
+  fleetMetrics?: FleetMetrics;
   weeklyValueReport?: {
     freshness: { status: string; latestRollupDay?: string | null };
     warnings: string[];
@@ -32,6 +33,28 @@ type OperatorDashboardResponse = {
   upstreamDrift?: { status?: string } | null;
 };
 
+type FleetMetrics = {
+  windowDays: number;
+  instanceCount: number;
+  fleet: {
+    mergePrecision: number | null;
+    closePrecision: number | null;
+    fpRate: number | null;
+    reversalRate: number | null;
+    cycleP50Ms: number | null;
+    cycleP95Ms: number | null;
+  };
+  outliers: Array<{ instanceId: string; metric: string; value: number; fleetMedian: number }>;
+};
+
+const formatPct = (v: number | null): string => (v === null ? "—" : `${Math.round(v * 100)}%`);
+const formatMs = (v: number | null): string =>
+  v === null
+    ? "—"
+    : v >= 3_600_000
+      ? `${(v / 3_600_000).toFixed(1)}h`
+      : `${Math.round(v / 60_000)}m`;
+
 type RecommendationQualityReport = {
   windowDays: number;
   visibility: "operator_only";
@@ -145,6 +168,51 @@ function OperatorDashboard() {
             ))}
           </section>
 
+          {data.fleetMetrics && data.fleetMetrics.instanceCount > 0 ? (
+            <section className="rounded-token border border-border bg-transparent p-5">
+              <div className="flex items-center gap-2">
+                <BarChart3 className="size-4 text-mint" />
+                <h2 className="font-display text-token-lg font-semibold">Fleet health</h2>
+              </div>
+              <p className="mt-1 max-w-2xl text-token-xs text-muted-foreground">
+                Gate calibration aggregated (median) across {data.fleetMetrics.instanceCount}{" "}
+                self-hosted instance(s) over the last {data.fleetMetrics.windowDays} days.
+              </p>
+              <div className="mt-4 grid gap-3 sm:grid-cols-2 lg:grid-cols-3">
+                <Stat
+                  label="Merge precision"
+                  value={formatPct(data.fleetMetrics.fleet.mergePrecision)}
+                  hint="approve → merged"
+                />
+                <Stat
+                  label="Close precision"
+                  value={formatPct(data.fleetMetrics.fleet.closePrecision)}
+                  hint="block → closed"
+                />
+                <Stat
+                  label="False-positive rate"
+                  value={formatPct(data.fleetMetrics.fleet.fpRate)}
+                  hint="approve → reverted/closed"
+                />
+                <Stat
+                  label="Reversal rate"
+                  value={formatPct(data.fleetMetrics.fleet.reversalRate)}
+                  hint="humans overrode the gate"
+                />
+                <Stat
+                  label="Cycle time (p50)"
+                  value={formatMs(data.fleetMetrics.fleet.cycleP50Ms)}
+                  hint="decision → close"
+                />
+                <Stat
+                  label="Instance outliers"
+                  value={String(data.fleetMetrics.outliers.length)}
+                  hint="off the fleet median"
+                />
+              </div>
+            </section>
+          ) : null}
+
           {quality ? (
             <section className="rounded-token border border-border bg-transparent p-5">
               <div className="flex flex-wrap items-start justify-between gap-3">

src/mcp/server.ts

@@ -70,6 +70,7 @@ import { loadOrComputeBurdenForecastResponse } from "../services/burden-forecast
 import { buildMcpClientTelemetry } from "../services/client-telemetry";
 import { loadOrComputeRepoOutcomePatternsResponse } from "../services/repo-outcome-patterns";
 import { buildRepoOutcomeCalibration, outcomeCalibrationSummary } from "../services/outcome-calibration";
+import { computeFleetAnalytics } from "../orb/analytics";
 import { buildUnavailableQueueTrendReport } from "../services/queue-trends";
 import {
   applyMcpPlanningChoices,
@@ -145,6 +146,18 @@ const ownerRepoWindowShape = {
   windowDays: z.number().int().positive().optional(),
 };
 
+const windowOnlyShape = {
+  windowDays: z.number().int().positive().optional(),
+};
+
+const fleetAnalyticsOutputSchema = {
+  windowDays: z.number().optional(),
+  instanceCount: z.number().optional(),
+  fleet: z.unknown().optional(),
+  instances: z.array(z.unknown()).optional(),
+  outliers: z.array(z.unknown()).optional(),
+};
+
 const loginShape = {
   login: z.string().min(1),
 };
@@ -1055,6 +1068,17 @@ export class GittensoryMcp {
       async (input) => this.toolResult(await this.getOutcomeCalibration(input)),
     );
 
+    server.registerTool(
+      "gittensory_get_fleet_analytics",
+      {
+        description:
+          "Operator-only: aggregated gate-calibration analytics across the self-host fleet — median merge/close precision, false-positive + reversal rates, cycle-time percentiles, and per-instance outliers. Measurement only.",
+        inputSchema: windowOnlyShape,
+        outputSchema: fleetAnalyticsOutputSchema,
+      },
+      async (input) => this.toolResult(await this.getFleetAnalytics(input)),
+    );
+
     server.registerTool(
       "gittensory_get_contributor_profile",
       {
@@ -1923,6 +1947,25 @@ export class GittensoryMcp {
     };
   }
 
+  // Operator-only gate: the fleet view aggregates ALL self-hosters' calibration, so a session must be an
+  // operator; private-token / static identities are trusted (the same model as the other measurement tools).
+  private async requireOperatorAccess(): Promise<void> {
+    if (this.identity.kind !== "session") return;
+    const scope = await this.loadSessionAccessScope();
+    if (scope.operator) return;
+    throw new Error("Forbidden: operator authority is required for fleet analytics.");
+  }
+
+  private async getFleetAnalytics(input: { windowDays?: number | undefined }): Promise<ToolPayload> {
+    await this.requireOperatorAccess();
+    const report = await computeFleetAnalytics(this.env, input.windowDays !== undefined ? { windowDays: input.windowDays } : {});
+    const merge = report.fleet.mergePrecision !== null ? `${Math.round(report.fleet.mergePrecision * 100)}%` : "n/a";
+    return {
+      summary: `Fleet calibration over ${report.windowDays}d: ${report.instanceCount} instance(s), median merge precision ${merge}, ${report.outliers.length} outlier(s).`,
+      data: report as unknown as Record<string, unknown>,
+    };
+  }
+
   private async loadOpenQueueCounts(fullName: string): Promise<{ openIssues: number; openPullRequests: number }> {
     const [totals, openIssues, openPullRequests] = await Promise.all([
       getLatestRepoGithubTotalsSnapshot(this.env, fullName),

src/services/operator-dashboard.ts

@@ -25,6 +25,7 @@ import type {
   ScoringModelSnapshotRecord,
   WeeklyValueReport,
 } from "../types";
+import { computeFleetAnalytics, type FleetAnalytics } from "../orb/analytics";
 import { loadUpstreamStatus, type UpstreamStatus } from "../upstream/ruleset";
 import { nowIso } from "../utils/json";
 import { buildRecommendationQualityReport, type RecommendationQualityReport } from "./recommendation-quality-report";
@@ -57,6 +58,7 @@ export type OperatorDashboardPayload = {
   registry: RegistrySnapshot | null;
   scoringModel: ScoringModelSnapshotRecord | null;
   upstreamDrift: UpstreamStatus;
+  fleetMetrics: FleetAnalytics;
 };
 
 const USAGE_WINDOW_DAYS = 7;
@@ -79,6 +81,7 @@ export async function buildOperatorDashboardPayload(env: Env): Promise<OperatorD
     mcpCompatibilityAdoption,
     commandUsefulness,
     recommendationQuality,
+    fleetMetrics,
   ] = await Promise.all([
     listRepositories(env),
     listInstallations(env),
@@ -95,6 +98,7 @@ export async function buildOperatorDashboardPayload(env: Env): Promise<OperatorD
     summarizeMcpCompatibilityAdoption(env, usageSince),
     getCommandUsefulnessSummary(env),
     buildRecommendationQualityReport(env, { windowDays: 90 }),
+    computeFleetAnalytics(env, { windowDays: 90 }),
   ]);
   const weeklyValueReport = buildWeeklyValueReport({
     generatedAt: nowIso(),
@@ -145,6 +149,16 @@ export async function buildOperatorDashboardPayload(env: Env): Promise<OperatorD
         delta: "current health cache",
       },
       { label: "Rate-limit events", value: String(rateLimits.length), delta: "latest observations" },
+      {
+        label: "Fleet instances",
+        value: String(fleetMetrics.instanceCount),
+        delta: fleetMetrics.outliers.length > 0 ? `${fleetMetrics.outliers.length} outlier(s)` : "self-host fleet",
+      },
+      {
+        label: "Fleet merge precision",
+        value: fleetMetrics.fleet.mergePrecision !== null ? `${Math.round(fleetMetrics.fleet.mergePrecision * 100)}%` : "—",
+        delta: "median across the fleet",
+      },
     ],
     noiseReduction: [
       {
@@ -177,6 +191,7 @@ export async function buildOperatorDashboardPayload(env: Env): Promise<OperatorD
     registry,
     scoringModel: scoring,
     upstreamDrift,
+    fleetMetrics,
   };
 }
 

test/unit/mcp-fleet-analytics.test.ts

@@ -0,0 +1,60 @@
+import { Client } from "@modelcontextprotocol/sdk/client/index.js";
+import { InMemoryTransport } from "@modelcontextprotocol/sdk/inMemory.js";
+import { describe, expect, it } from "vitest";
+import { createSessionForGitHubUser, type AuthIdentity } from "../../src/auth/security";
+import { GittensoryMcp } from "../../src/mcp/server";
+import { createTestEnv } from "../helpers/d1";
+
+async function connect(env: Env, identity?: AuthIdentity): Promise<Client> {
+  const server = (identity ? new GittensoryMcp(env, identity) : new GittensoryMcp(env)).createServer();
+  const [clientTransport, serverTransport] = InMemoryTransport.createLinkedPair();
+  await server.connect(serverTransport);
+  const client = new Client({ name: "fleet-test", version: "0.1.0" }, { capabilities: {} });
+  await client.connect(clientTransport);
+  return client;
+}
+
+let seq = 0;
+async function seedMergeSignals(env: Env, instance: string, n: number): Promise<void> {
+  for (let i = 0; i < n; i++) {
+    await env.DB
+      .prepare(`INSERT INTO orb_signals (instance_id, repo_hash, pr_hash, gate_verdict, outcome, reversal_flag) VALUES (?, ?, ?, 'merge', 'merged', 'none')`)
+      .bind(instance, `repo${seq}`, `pr${seq++}`)
+      .run();
+  }
+}
+
+describe("gittensory_get_fleet_analytics MCP tool", () => {
+  it("returns fleet analytics for a trusted (non-session) identity, honoring windowDays", async () => {
+    const env = createTestEnv();
+    await seedMergeSignals(env, "inst1", 5); // ≥ MIN_DECIDED → counts toward the fleet
+    const result = await (await connect(env)).callTool({ name: "gittensory_get_fleet_analytics", arguments: { windowDays: 30 } });
+    expect(result.isError).toBeFalsy();
+    const data = result.structuredContent as { windowDays: number; instanceCount: number; fleet: { mergePrecision: number } };
+    expect(data.windowDays).toBe(30);
+    expect(data.instanceCount).toBe(1);
+    expect(data.fleet.mergePrecision).toBe(1);
+  });
+
+  it("returns an empty report (n/a summary) when there is no data and no windowDays", async () => {
+    const result = await (await connect(createTestEnv())).callTool({ name: "gittensory_get_fleet_analytics", arguments: {} });
+    expect(result.isError).toBeFalsy();
+    const data = result.structuredContent as { instanceCount: number };
+    expect(data.instanceCount).toBe(0);
+  });
+
+  it("allows an operator session", async () => {
+    const env = createTestEnv({ ADMIN_GITHUB_LOGINS: "boss" });
+    const { session } = await createSessionForGitHubUser(env, { login: "boss", id: 1 });
+    const result = await (await connect(env, { kind: "session", actor: "boss", session })).callTool({ name: "gittensory_get_fleet_analytics", arguments: {} });
+    expect(result.isError).toBeFalsy();
+  });
+
+  it("forbids a non-operator session", async () => {
+    const env = createTestEnv({ ADMIN_GITHUB_LOGINS: "boss" });
+    const { session } = await createSessionForGitHubUser(env, { login: "rando", id: 2 });
+    const result = await (await connect(env, { kind: "session", actor: "rando", session })).callTool({ name: "gittensory_get_fleet_analytics", arguments: {} });
+    expect(result.isError).toBeTruthy();
+    expect(JSON.stringify(result.content)).toMatch(/operator authority/i);
+  });
+});

test/unit/operator-dashboard.test.ts

@@ -23,6 +23,39 @@ describe("operator dashboard payload", () => {
     expect(payload.usageSummary).toMatchObject({ totalEvents: expect.any(Number), activeActors: expect.any(Number) });
     expect(payload.commandUsefulness.totals).toMatchObject({ feedbackCount: expect.any(Number) });
     expect(serialized).not.toMatch(FORBIDDEN_EXPORT_TERMS);
+    // Empty fleet → instanceCount 0, null precision card ("—"), no-outlier delta.
+    expect(payload.fleetMetrics.instanceCount).toBe(0);
+    expect(payload.metrics).toEqual(
+      expect.arrayContaining([
+        expect.objectContaining({ label: "Fleet instances", value: "0", delta: "self-host fleet" }),
+        expect.objectContaining({ label: "Fleet merge precision", value: "—" }),
+      ]),
+    );
+  });
+
+  it("surfaces populated fleet metrics + outliers from orb_signals", async () => {
+    const env = createTestEnv();
+    let n = 0;
+    const seed = async (instance: string, count: number, outcome: string): Promise<void> => {
+      for (let i = 0; i < count; i++) {
+        await env.DB
+          .prepare(`INSERT INTO orb_signals (instance_id, repo_hash, pr_hash, gate_verdict, outcome, reversal_flag) VALUES (?, ?, ?, 'merge', ?, 'none')`)
+          .bind(instance, `r${n}`, `p${n++}`, outcome)
+          .run();
+      }
+    };
+    await seed("good1", 5, "merged"); // precision 1.0
+    await seed("good2", 5, "merged"); // precision 1.0
+    await seed("bad", 5, "closed"); // precision 0.0 → outlier vs the median (1.0)
+    const payload = await buildOperatorDashboardPayload(env);
+    expect(payload.fleetMetrics.instanceCount).toBe(3);
+    expect(payload.fleetMetrics.outliers.map((o) => o.instanceId)).toContain("bad");
+    expect(payload.metrics).toEqual(
+      expect.arrayContaining([
+        expect.objectContaining({ label: "Fleet instances", value: "3", delta: "1 outlier(s)" }),
+        expect.objectContaining({ label: "Fleet merge precision", value: "100%" }),
+      ]),
+    );
   });
 
   it("picks the newest rollup day for adoption insights", () => {