diff --git a/packages/gittensory-engine/src/calibration-dashboard.ts b/packages/gittensory-engine/src/calibration-dashboard.ts new file mode 100644 index 0000000000..77413200fb --- /dev/null +++ b/packages/gittensory-engine/src/calibration-dashboard.ts @@ -0,0 +1,95 @@ +import { + DOCUMENTED_CALIBRATION_BASELINE, + type CalibrationSourceMetric, + type Phase7CalibrationLoopResult, +} from "./phase7-calibration-loop.js"; + +// Calibration dashboard view (#4261). A read-only projection of a Phase7CalibrationLoopResult +// (phase7-calibration-loop.ts, computePhase7CalibrationLoop) into a human-readable dashboard shape that a CLI table +// or a UI panel renders. Pure: it re-shapes an ALREADY-computed result and adds NO new calibration computation — so +// predicted-gate accuracy vs realized pr_outcome is presented, never recomputed here. Public-safe: only accuracies, +// sample sizes, freshness, and hold reasons are surfaced (no raw scores/rewards). + +export type CalibrationDashboardStatus = "on_track" | "below_baseline" | "insufficient_signal" | "disabled"; + +/** One labeled row in the dashboard: a metric name, its formatted value, and a short detail line. */ +export type CalibrationDashboardRow = { + label: string; + value: string; + detail: string; +}; + +/** The read-only dashboard projection of a calibration-loop result. */ +export type CalibrationDashboardView = { + status: CalibrationDashboardStatus; + headline: string; + rows: readonly CalibrationDashboardRow[]; + holdReasons: readonly string[]; +}; + +/** A whole-number percentage, or an em dash when there is no signal yet. */ +function formatPercent(value: number | null): string { + return value === null ? "—" : `${Math.round(value * 100)}%`; +} + +/** A signed percentage-point delta (e.g. "+6pts" / "-4pts"), or an em dash when unknown. */ +function formatDeltaPoints(value: number | null): string { + if (value === null) return "—"; + const points = Math.round(value * 100); + return `${points >= 0 ? "+" : ""}${points}pts`; +} + +function sourceRow(label: string, metric: CalibrationSourceMetric): CalibrationDashboardRow { + return { + label, + value: formatPercent(metric.accuracy), + detail: `n=${metric.sampleSize} · ${metric.fresh ? "fresh" : "stale"}`, + }; +} + +/** Classify the overall calibration state for the dashboard's headline banner. */ +export function resolveCalibrationDashboardStatus(result: Phase7CalibrationLoopResult): CalibrationDashboardStatus { + if (!result.enabled) return "disabled"; + if (result.combinedAccuracy === null) return "insufficient_signal"; + return result.combinedAccuracy >= result.baselineAccuracy ? "on_track" : "below_baseline"; +} + +/** + * Project a computed {@link Phase7CalibrationLoopResult} into a read-only dashboard view. Pure and deterministic; + * adds no computation of its own. `holdReasons` are surfaced verbatim so an operator can see why an autonomy + * increase is (or isn't) permitted. + */ +export function buildCalibrationDashboardView(result: Phase7CalibrationLoopResult): CalibrationDashboardView { + const status = resolveCalibrationDashboardStatus(result); + const rows: CalibrationDashboardRow[] = [ + { + label: "Combined accuracy", + value: formatPercent(result.combinedAccuracy), + detail: `baseline ${formatPercent(result.baselineAccuracy)}`, + }, + { + label: "Delta from baseline", + value: formatDeltaPoints(result.deltaFromBaseline), + detail: `documented baseline ${formatPercent(DOCUMENTED_CALIBRATION_BASELINE)}`, + }, + sourceRow("Historical replay", result.bySource.historical_replay), + sourceRow("PR outcome", result.bySource.pr_outcome), + { + label: "Replay harness", + value: result.replayHarnessStatus, + detail: result.replayHarnessHold ? "hold" : "ok", + }, + { + label: "Autonomy increase", + value: result.autonomyIncreasePermitted ? "permitted" : "held", + detail: result.replayRunDue ? "replay run due" : "up to date", + }, + ]; + const headline = + status === "disabled" + ? "Calibration loop disabled" + : status === "insufficient_signal" + ? "Insufficient signal to score calibration yet" + : `${formatPercent(result.combinedAccuracy)} combined (${formatDeltaPoints(result.deltaFromBaseline)} vs baseline)`; + return { status, headline, rows, holdReasons: [...result.holdReasons] }; +} diff --git a/packages/gittensory-engine/src/index.ts b/packages/gittensory-engine/src/index.ts index f6efac5963..0bd5176ac1 100644 --- a/packages/gittensory-engine/src/index.ts +++ b/packages/gittensory-engine/src/index.ts @@ -77,6 +77,13 @@ export { type PrOutcomeCalibrationInput, type ReplayHarnessStatus, } from "./phase7-calibration-loop.js"; +export { + buildCalibrationDashboardView, + resolveCalibrationDashboardStatus, + type CalibrationDashboardRow, + type CalibrationDashboardStatus, + type CalibrationDashboardView, +} from "./calibration-dashboard.js"; export { computeFindingSeverityCompositeCalibrationScore, ingestFindingSeverityCalibrationSignals, diff --git a/test/unit/calibration-dashboard.test.ts b/test/unit/calibration-dashboard.test.ts new file mode 100644 index 0000000000..0bf15b801a --- /dev/null +++ b/test/unit/calibration-dashboard.test.ts @@ -0,0 +1,102 @@ +import { describe, expect, it } from "vitest"; +import { + buildCalibrationDashboardView, + resolveCalibrationDashboardStatus, +} from "../../packages/gittensory-engine/src/index"; +import type { Phase7CalibrationLoopResult } from "../../packages/gittensory-engine/src/index"; + +function metric(accuracy: number | null, sampleSize: number, fresh: boolean) { + return { source: "pr_outcome" as const, accuracy, sampleSize, observedAt: "2026-01-01T00:00:00Z", fresh }; +} + +function makeResult(overrides: Partial = {}): Phase7CalibrationLoopResult { + return { + enabled: true, + baselineAccuracy: 0.62, + combinedAccuracy: 0.68, + deltaFromBaseline: 0.06, + weights: { historicalReplay: 0.5, prOutcome: 0.5 }, + bySource: { + historical_replay: { ...metric(0.7, 20, true), source: "historical_replay" }, + pr_outcome: metric(0.66, 12, false), + }, + replayHarnessHold: false, + replayHarnessStatus: "healthy", + autonomyIncreasePermitted: true, + holdReasons: [], + replayRunDue: false, + audit: { contributingSources: ["pr_outcome"], rejectedSources: [] }, + ...overrides, + }; +} + +describe("resolveCalibrationDashboardStatus (#4261)", () => { + it("classifies disabled / insufficient / on-track / below-baseline", () => { + expect(resolveCalibrationDashboardStatus(makeResult({ enabled: false }))).toBe("disabled"); + expect(resolveCalibrationDashboardStatus(makeResult({ combinedAccuracy: null }))).toBe("insufficient_signal"); + expect(resolveCalibrationDashboardStatus(makeResult({ combinedAccuracy: 0.68, baselineAccuracy: 0.62 }))).toBe( + "on_track", + ); + expect(resolveCalibrationDashboardStatus(makeResult({ combinedAccuracy: 0.55, baselineAccuracy: 0.62 }))).toBe( + "below_baseline", + ); + }); +}); + +describe("buildCalibrationDashboardView", () => { + it("projects an on-track result: headline, formatted rows, per-source freshness", () => { + const view = buildCalibrationDashboardView(makeResult()); + expect(view.status).toBe("on_track"); + expect(view.headline).toBe("68% combined (+6pts vs baseline)"); + const byLabel = Object.fromEntries(view.rows.map((r) => [r.label, r])); + expect(byLabel["Combined accuracy"]?.value).toBe("68%"); + expect(byLabel["Combined accuracy"]?.detail).toBe("baseline 62%"); + expect(byLabel["Delta from baseline"]?.value).toBe("+6pts"); + expect(byLabel["Historical replay"]?.detail).toBe("n=20 · fresh"); + expect(byLabel["PR outcome"]?.detail).toBe("n=12 · stale"); + expect(byLabel["Replay harness"]?.value).toBe("healthy"); + expect(byLabel["Replay harness"]?.detail).toBe("ok"); + expect(byLabel["Autonomy increase"]?.value).toBe("permitted"); + expect(byLabel["Autonomy increase"]?.detail).toBe("up to date"); + }); + + it("formats a below-baseline result with a negative delta", () => { + const view = buildCalibrationDashboardView( + makeResult({ combinedAccuracy: 0.55, deltaFromBaseline: -0.07, baselineAccuracy: 0.62 }), + ); + expect(view.status).toBe("below_baseline"); + expect(view.headline).toBe("55% combined (-7pts vs baseline)"); + const delta = view.rows.find((r) => r.label === "Delta from baseline"); + expect(delta?.value).toBe("-7pts"); + }); + + it("shows an em dash and an insufficient-signal headline when there is no combined accuracy", () => { + const view = buildCalibrationDashboardView(makeResult({ combinedAccuracy: null, deltaFromBaseline: null })); + expect(view.status).toBe("insufficient_signal"); + expect(view.headline).toBe("Insufficient signal to score calibration yet"); + const combined = view.rows.find((r) => r.label === "Combined accuracy"); + expect(combined?.value).toBe("—"); + expect(view.rows.find((r) => r.label === "Delta from baseline")?.value).toBe("—"); + }); + + it("reflects a disabled loop, a harness hold, a due replay run, and surfaces hold reasons", () => { + const view = buildCalibrationDashboardView( + makeResult({ + enabled: false, + replayHarnessHold: true, + replayHarnessStatus: "missing", + autonomyIncreasePermitted: false, + replayRunDue: true, + holdReasons: ["no_historical_replay_signal", "replay_run_stale"], + }), + ); + expect(view.status).toBe("disabled"); + expect(view.headline).toBe("Calibration loop disabled"); + const byLabel = Object.fromEntries(view.rows.map((r) => [r.label, r])); + expect(byLabel["Replay harness"]?.detail).toBe("hold"); + expect(byLabel["Replay harness"]?.value).toBe("missing"); + expect(byLabel["Autonomy increase"]?.value).toBe("held"); + expect(byLabel["Autonomy increase"]?.detail).toBe("replay run due"); + expect(view.holdReasons).toEqual(["no_historical_replay_signal", "replay_run_stale"]); + }); +});