Skip to content
Merged
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
95 changes: 95 additions & 0 deletions packages/gittensory-engine/src/calibration-dashboard.ts
Original file line number Diff line number Diff line change
@@ -0,0 +1,95 @@
import {
DOCUMENTED_CALIBRATION_BASELINE,
type CalibrationSourceMetric,
type Phase7CalibrationLoopResult,
} from "./phase7-calibration-loop.js";

// Calibration dashboard view (#4261). A read-only projection of a Phase7CalibrationLoopResult
// (phase7-calibration-loop.ts, computePhase7CalibrationLoop) into a human-readable dashboard shape that a CLI table
// or a UI panel renders. Pure: it re-shapes an ALREADY-computed result and adds NO new calibration computation — so
// predicted-gate accuracy vs realized pr_outcome is presented, never recomputed here. Public-safe: only accuracies,
// sample sizes, freshness, and hold reasons are surfaced (no raw scores/rewards).

export type CalibrationDashboardStatus = "on_track" | "below_baseline" | "insufficient_signal" | "disabled";

/** One labeled row in the dashboard: a metric name, its formatted value, and a short detail line. */
export type CalibrationDashboardRow = {
label: string;
value: string;
detail: string;
};

/** The read-only dashboard projection of a calibration-loop result. */
export type CalibrationDashboardView = {
status: CalibrationDashboardStatus;
headline: string;
rows: readonly CalibrationDashboardRow[];
holdReasons: readonly string[];
};

/** A whole-number percentage, or an em dash when there is no signal yet. */
function formatPercent(value: number | null): string {
return value === null ? "—" : `${Math.round(value * 100)}%`;
}

/** A signed percentage-point delta (e.g. "+6pts" / "-4pts"), or an em dash when unknown. */
function formatDeltaPoints(value: number | null): string {
if (value === null) return "—";
const points = Math.round(value * 100);
return `${points >= 0 ? "+" : ""}${points}pts`;
}

function sourceRow(label: string, metric: CalibrationSourceMetric): CalibrationDashboardRow {
return {
label,
value: formatPercent(metric.accuracy),
detail: `n=${metric.sampleSize} · ${metric.fresh ? "fresh" : "stale"}`,
};
}

/** Classify the overall calibration state for the dashboard's headline banner. */
export function resolveCalibrationDashboardStatus(result: Phase7CalibrationLoopResult): CalibrationDashboardStatus {
if (!result.enabled) return "disabled";
if (result.combinedAccuracy === null) return "insufficient_signal";
return result.combinedAccuracy >= result.baselineAccuracy ? "on_track" : "below_baseline";
}

/**
* Project a computed {@link Phase7CalibrationLoopResult} into a read-only dashboard view. Pure and deterministic;
* adds no computation of its own. `holdReasons` are surfaced verbatim so an operator can see why an autonomy
* increase is (or isn't) permitted.
*/
export function buildCalibrationDashboardView(result: Phase7CalibrationLoopResult): CalibrationDashboardView {
const status = resolveCalibrationDashboardStatus(result);
const rows: CalibrationDashboardRow[] = [
{
label: "Combined accuracy",
value: formatPercent(result.combinedAccuracy),
detail: `baseline ${formatPercent(result.baselineAccuracy)}`,
},
{
label: "Delta from baseline",
value: formatDeltaPoints(result.deltaFromBaseline),
detail: `documented baseline ${formatPercent(DOCUMENTED_CALIBRATION_BASELINE)}`,
},
sourceRow("Historical replay", result.bySource.historical_replay),
sourceRow("PR outcome", result.bySource.pr_outcome),
{
label: "Replay harness",
value: result.replayHarnessStatus,
detail: result.replayHarnessHold ? "hold" : "ok",
},
{
label: "Autonomy increase",
value: result.autonomyIncreasePermitted ? "permitted" : "held",
detail: result.replayRunDue ? "replay run due" : "up to date",
},
];
const headline =
status === "disabled"
? "Calibration loop disabled"
: status === "insufficient_signal"
? "Insufficient signal to score calibration yet"
: `${formatPercent(result.combinedAccuracy)} combined (${formatDeltaPoints(result.deltaFromBaseline)} vs baseline)`;
return { status, headline, rows, holdReasons: [...result.holdReasons] };
}
7 changes: 7 additions & 0 deletions packages/gittensory-engine/src/index.ts
Original file line number Diff line number Diff line change
Expand Up @@ -77,6 +77,13 @@ export {
type PrOutcomeCalibrationInput,
type ReplayHarnessStatus,
} from "./phase7-calibration-loop.js";
export {
buildCalibrationDashboardView,
resolveCalibrationDashboardStatus,
type CalibrationDashboardRow,
type CalibrationDashboardStatus,
type CalibrationDashboardView,
} from "./calibration-dashboard.js";
export {
computeFindingSeverityCompositeCalibrationScore,
ingestFindingSeverityCalibrationSignals,
Expand Down
102 changes: 102 additions & 0 deletions test/unit/calibration-dashboard.test.ts
Original file line number Diff line number Diff line change
@@ -0,0 +1,102 @@
import { describe, expect, it } from "vitest";
import {
buildCalibrationDashboardView,
resolveCalibrationDashboardStatus,
} from "../../packages/gittensory-engine/src/index";
import type { Phase7CalibrationLoopResult } from "../../packages/gittensory-engine/src/index";

function metric(accuracy: number | null, sampleSize: number, fresh: boolean) {
return { source: "pr_outcome" as const, accuracy, sampleSize, observedAt: "2026-01-01T00:00:00Z", fresh };
}

function makeResult(overrides: Partial<Phase7CalibrationLoopResult> = {}): Phase7CalibrationLoopResult {
return {
enabled: true,
baselineAccuracy: 0.62,
combinedAccuracy: 0.68,
deltaFromBaseline: 0.06,
weights: { historicalReplay: 0.5, prOutcome: 0.5 },
bySource: {
historical_replay: { ...metric(0.7, 20, true), source: "historical_replay" },
pr_outcome: metric(0.66, 12, false),
},
replayHarnessHold: false,
replayHarnessStatus: "healthy",
autonomyIncreasePermitted: true,
holdReasons: [],
replayRunDue: false,
audit: { contributingSources: ["pr_outcome"], rejectedSources: [] },
...overrides,
};
}

describe("resolveCalibrationDashboardStatus (#4261)", () => {
it("classifies disabled / insufficient / on-track / below-baseline", () => {
expect(resolveCalibrationDashboardStatus(makeResult({ enabled: false }))).toBe("disabled");
expect(resolveCalibrationDashboardStatus(makeResult({ combinedAccuracy: null }))).toBe("insufficient_signal");
expect(resolveCalibrationDashboardStatus(makeResult({ combinedAccuracy: 0.68, baselineAccuracy: 0.62 }))).toBe(
"on_track",
);
expect(resolveCalibrationDashboardStatus(makeResult({ combinedAccuracy: 0.55, baselineAccuracy: 0.62 }))).toBe(
"below_baseline",
);
});
});

describe("buildCalibrationDashboardView", () => {
it("projects an on-track result: headline, formatted rows, per-source freshness", () => {
const view = buildCalibrationDashboardView(makeResult());
expect(view.status).toBe("on_track");
expect(view.headline).toBe("68% combined (+6pts vs baseline)");
const byLabel = Object.fromEntries(view.rows.map((r) => [r.label, r]));
expect(byLabel["Combined accuracy"]?.value).toBe("68%");
expect(byLabel["Combined accuracy"]?.detail).toBe("baseline 62%");
expect(byLabel["Delta from baseline"]?.value).toBe("+6pts");
expect(byLabel["Historical replay"]?.detail).toBe("n=20 · fresh");
expect(byLabel["PR outcome"]?.detail).toBe("n=12 · stale");
expect(byLabel["Replay harness"]?.value).toBe("healthy");
expect(byLabel["Replay harness"]?.detail).toBe("ok");
expect(byLabel["Autonomy increase"]?.value).toBe("permitted");
expect(byLabel["Autonomy increase"]?.detail).toBe("up to date");
});

it("formats a below-baseline result with a negative delta", () => {
const view = buildCalibrationDashboardView(
makeResult({ combinedAccuracy: 0.55, deltaFromBaseline: -0.07, baselineAccuracy: 0.62 }),
);
expect(view.status).toBe("below_baseline");
expect(view.headline).toBe("55% combined (-7pts vs baseline)");
const delta = view.rows.find((r) => r.label === "Delta from baseline");
expect(delta?.value).toBe("-7pts");
});

it("shows an em dash and an insufficient-signal headline when there is no combined accuracy", () => {
const view = buildCalibrationDashboardView(makeResult({ combinedAccuracy: null, deltaFromBaseline: null }));
expect(view.status).toBe("insufficient_signal");
expect(view.headline).toBe("Insufficient signal to score calibration yet");
const combined = view.rows.find((r) => r.label === "Combined accuracy");
expect(combined?.value).toBe("—");
expect(view.rows.find((r) => r.label === "Delta from baseline")?.value).toBe("—");
});

it("reflects a disabled loop, a harness hold, a due replay run, and surfaces hold reasons", () => {
const view = buildCalibrationDashboardView(
makeResult({
enabled: false,
replayHarnessHold: true,
replayHarnessStatus: "missing",
autonomyIncreasePermitted: false,
replayRunDue: true,
holdReasons: ["no_historical_replay_signal", "replay_run_stale"],
}),
);
expect(view.status).toBe("disabled");
expect(view.headline).toBe("Calibration loop disabled");
const byLabel = Object.fromEntries(view.rows.map((r) => [r.label, r]));
expect(byLabel["Replay harness"]?.detail).toBe("hold");
expect(byLabel["Replay harness"]?.value).toBe("missing");
expect(byLabel["Autonomy increase"]?.value).toBe("held");
expect(byLabel["Autonomy increase"]?.detail).toBe("replay run due");
expect(view.holdReasons).toEqual(["no_historical_replay_signal", "replay_run_stale"]);
});
});