From 4379953412fbf79816c7c3fc33763421023a5453 Mon Sep 17 00:00:00 2001 From: Jeff <158072326+jeffrey701@users.noreply.github.com> Date: Thu, 9 Jul 2026 15:57:50 -0400 Subject: [PATCH] feat(miner-selfimprove): render prediction-calibration Prometheus metrics MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit A pure Prometheus text-exposition renderer for the miner's own predicted-gate accuracy — the miner-side counterpart to the server's src/selfhost/metrics.ts registry, scoped to calibration rather than the server's queue/webhook metrics. Design decision (the issue's open question — CLI vs HTTP, shared vs standalone): gittensory-miner is a local CLI, not a daemon, so this is an on-demand RENDERER a caller prints to stdout for its own scrape/cron setup, not an HTTP-served registry. It mirrors src/selfhost/metrics.ts's metric-naming (gittensory_miner_* _total) and HELP/TYPE/label-escaping conventions rather than importing across the package boundary. It stays a pure, side-effect-free engine function: the caller reads the prediction-ledger (packages/gittensory-miner/lib/prediction- ledger.js `readPredictions`, #4263) and passes the rows in — no data collection of its own. Counters: - gittensory_miner_predictions_total{conclusion="..."} — predictions recorded, one series per predicted conclusion. - gittensory_miner_prediction_correct_total / _incorrect_total — the confusion- matrix-shaped correct/incorrect counts, which only move for rows carrying a resolved outcome (unresolved rows count toward predictions_total only), so the surface is meaningful before outcome-pairing exists and grows once it does. - packages/gittensory-engine/src/miner-prediction-metrics.ts: the renderer + metric-name constants + the MinerPredictionMetricRow input type; the counters are documented in the module header. - packages/gittensory-engine/src/index.ts: barrel re-export. - test/unit/miner-prediction-metrics.test.ts: valid-exposition assertions over empty / unresolved / mixed-correctness / label-escaping fixture states. Closes #4264 --- packages/gittensory-engine/src/index.ts | 7 ++ .../src/miner-prediction-metrics.ts | 74 +++++++++++++++++++ test/unit/miner-prediction-metrics.test.ts | 65 ++++++++++++++++ 3 files changed, 146 insertions(+) create mode 100644 packages/gittensory-engine/src/miner-prediction-metrics.ts create mode 100644 test/unit/miner-prediction-metrics.test.ts diff --git a/packages/gittensory-engine/src/index.ts b/packages/gittensory-engine/src/index.ts index af7bd6f173..8e0871cd48 100644 --- a/packages/gittensory-engine/src/index.ts +++ b/packages/gittensory-engine/src/index.ts @@ -146,6 +146,13 @@ export { type MinerTelemetryOutcomeBucket, type NormalizedMinerTelemetryEvent, } from "./miner-telemetry.js"; +export { + MINER_PREDICTIONS_TOTAL, + MINER_PREDICTION_CORRECT_TOTAL, + MINER_PREDICTION_INCORRECT_TOTAL, + renderMinerPredictionMetrics, + type MinerPredictionMetricRow, +} from "./miner-prediction-metrics.js"; export { ATTEMPT_LOG_EVENT_TYPES, createAttemptLogBuffer, diff --git a/packages/gittensory-engine/src/miner-prediction-metrics.ts b/packages/gittensory-engine/src/miner-prediction-metrics.ts new file mode 100644 index 0000000000..0973bfdd77 --- /dev/null +++ b/packages/gittensory-engine/src/miner-prediction-metrics.ts @@ -0,0 +1,74 @@ +// Miner prediction-calibration metrics (#4264). A pure Prometheus text-exposition renderer for the miner's own +// predicted-gate accuracy, the miner-side counterpart to the server's src/selfhost/metrics.ts registry. It turns +// prediction-ledger rows (packages/gittensory-miner/lib/prediction-ledger.js `readPredictions`) — optionally +// joined with their realized outcome — into counters a future dashboard can scrape. +// +// Scoped as an on-demand RENDERER, not a live HTTP registry: gittensory-miner is a local CLI, not a daemon, so a +// caller renders this to stdout for its own scrape/cron setup and reads the ledger itself (no data collection of +// its own lives here — this stays a pure, side-effect-free function like the rest of gittensory-engine). It mirrors +// the metric-naming (`gittensory_miner_*_total`) and HELP/TYPE/label conventions of src/selfhost/metrics.ts rather +// than importing across the package boundary. +// +// Counters emitted: +// - `gittensory_miner_predictions_total{conclusion="..."}` — predictions recorded, one series per predicted +// conclusion (e.g. merge/close/hold). +// - `gittensory_miner_prediction_correct_total` — predictions whose realized outcome matched the prediction. +// - `gittensory_miner_prediction_incorrect_total` — predictions whose realized outcome differed. +// The correct/incorrect counters only move for rows carrying a resolved outcome; unresolved rows count toward +// `predictions_total` only, so the surface is meaningful before outcome-pairing exists and grows once it does. + +export const MINER_PREDICTIONS_TOTAL = "gittensory_miner_predictions_total"; +export const MINER_PREDICTION_CORRECT_TOTAL = "gittensory_miner_prediction_correct_total"; +export const MINER_PREDICTION_INCORRECT_TOTAL = "gittensory_miner_prediction_incorrect_total"; + +/** One prediction-ledger row for metrics: its predicted `conclusion`, plus an optional realized-outcome pairing + * (`correct`: true = matched, false = differed, null/undefined = not yet resolved). */ +export type MinerPredictionMetricRow = { + conclusion: string; + correct?: boolean | null; +}; + +/** Mirror src/selfhost/metrics.ts:204 — HELP text escapes backslash and newline. */ +function escapeHelpText(help: string): string { + return help.replace(/\\/g, "\\\\").replace(/\n/g, "\\n"); +} + +/** Prometheus label-value escaping (backslash, double-quote, newline), a correctness-complete superset of + * src/selfhost/metrics.ts:193's `"`-only escape so an arbitrary conclusion string can never break the line. */ +function escapeLabelValue(value: string): string { + return value.replace(/\\/g, "\\\\").replace(/"/g, '\\"').replace(/\n/g, "\\n"); +} + +/** + * Render prediction-calibration counters as Prometheus text-exposition format. Pure and side-effect-free: a caller + * supplies the ledger rows (joined with any resolved outcomes) and prints the result. Deterministic — conclusion + * series are emitted in sorted order. Always emits HELP/TYPE for every counter, so the surface is well-formed even + * for an empty ledger. + */ +export function renderMinerPredictionMetrics(rows: readonly MinerPredictionMetricRow[]): string { + const totalByConclusion = new Map(); + let correct = 0; + let incorrect = 0; + for (const row of rows) { + totalByConclusion.set(row.conclusion, (totalByConclusion.get(row.conclusion) ?? 0) + 1); + if (row.correct === true) correct += 1; + else if (row.correct === false) incorrect += 1; + } + + const lines: string[] = []; + lines.push(`# HELP ${MINER_PREDICTIONS_TOTAL} ${escapeHelpText("Gate-outcome predictions the miner has recorded, by predicted conclusion.")}`); + lines.push(`# TYPE ${MINER_PREDICTIONS_TOTAL} counter`); + for (const [conclusion, count] of [...totalByConclusion.entries()].sort((a, b) => a[0].localeCompare(b[0]))) { + lines.push(`${MINER_PREDICTIONS_TOTAL}{conclusion="${escapeLabelValue(conclusion)}"} ${count}`); + } + + lines.push(`# HELP ${MINER_PREDICTION_CORRECT_TOTAL} ${escapeHelpText("Predictions whose realized outcome matched the predicted conclusion.")}`); + lines.push(`# TYPE ${MINER_PREDICTION_CORRECT_TOTAL} counter`); + lines.push(`${MINER_PREDICTION_CORRECT_TOTAL} ${correct}`); + + lines.push(`# HELP ${MINER_PREDICTION_INCORRECT_TOTAL} ${escapeHelpText("Predictions whose realized outcome differed from the predicted conclusion.")}`); + lines.push(`# TYPE ${MINER_PREDICTION_INCORRECT_TOTAL} counter`); + lines.push(`${MINER_PREDICTION_INCORRECT_TOTAL} ${incorrect}`); + + return `${lines.join("\n")}\n`; +} diff --git a/test/unit/miner-prediction-metrics.test.ts b/test/unit/miner-prediction-metrics.test.ts new file mode 100644 index 0000000000..0b80ed75ea --- /dev/null +++ b/test/unit/miner-prediction-metrics.test.ts @@ -0,0 +1,65 @@ +import { describe, expect, it } from "vitest"; +import { + MINER_PREDICTIONS_TOTAL, + MINER_PREDICTION_CORRECT_TOTAL, + MINER_PREDICTION_INCORRECT_TOTAL, + renderMinerPredictionMetrics, +} from "../../packages/gittensory-engine/src/index"; + +/** Parse `name{labels} value` / `name value` data lines out of an exposition string, keyed for easy assertions. */ +function dataLines(text: string): Record { + const out: Record = {}; + for (const line of text.split("\n")) { + if (!line || line.startsWith("#")) continue; + const idx = line.lastIndexOf(" "); + out[line.slice(0, idx)] = line.slice(idx + 1); + } + return out; +} + +describe("miner prediction-calibration metrics (#4264)", () => { + it("re-exports the renderer and metric-name constants from the engine barrel", () => { + expect(typeof renderMinerPredictionMetrics).toBe("function"); + expect(MINER_PREDICTIONS_TOTAL).toBe("gittensory_miner_predictions_total"); + expect(MINER_PREDICTION_CORRECT_TOTAL).toBe("gittensory_miner_prediction_correct_total"); + expect(MINER_PREDICTION_INCORRECT_TOTAL).toBe("gittensory_miner_prediction_incorrect_total"); + }); + + it("emits well-formed HELP/TYPE and zeroed counters for an empty ledger", () => { + const text = renderMinerPredictionMetrics([]); + expect(text.endsWith("\n")).toBe(true); + expect(text).toContain(`# HELP ${MINER_PREDICTIONS_TOTAL} `); + expect(text).toContain(`# TYPE ${MINER_PREDICTIONS_TOTAL} counter`); + // no predictions_total series when empty; correct/incorrect are single zeroed lines + expect(text).not.toContain(`${MINER_PREDICTIONS_TOTAL}{`); + expect(dataLines(text)).toEqual({ + [MINER_PREDICTION_CORRECT_TOTAL]: "0", + [MINER_PREDICTION_INCORRECT_TOTAL]: "0", + }); + }); + + it("counts predictions per conclusion in sorted order and ignores unresolved rows for correct/incorrect", () => { + const text = renderMinerPredictionMetrics([ + { conclusion: "merge" }, + { conclusion: "merge", correct: true }, + { conclusion: "close", correct: false }, + { conclusion: "hold" }, // unresolved: counts toward total only + { conclusion: "merge", correct: null }, // explicit unresolved + ]); + const d = dataLines(text); + expect(d[`${MINER_PREDICTIONS_TOTAL}{conclusion="merge"}`]).toBe("3"); + expect(d[`${MINER_PREDICTIONS_TOTAL}{conclusion="close"}`]).toBe("1"); + expect(d[`${MINER_PREDICTIONS_TOTAL}{conclusion="hold"}`]).toBe("1"); + expect(d[MINER_PREDICTION_CORRECT_TOTAL]).toBe("1"); + expect(d[MINER_PREDICTION_INCORRECT_TOTAL]).toBe("1"); + + // deterministic: conclusion series are alphabetically sorted (close, hold, merge) + const order = [...text.matchAll(/conclusion="([^"]+)"/g)].map((m) => m[1]); + expect(order).toEqual(["close", "hold", "merge"]); + }); + + it("escapes backslashes, quotes, and newlines in a conclusion label value", () => { + const text = renderMinerPredictionMetrics([{ conclusion: 'we"ird\\\nvalue' }]); + expect(text).toContain(`${MINER_PREDICTIONS_TOTAL}{conclusion="we\\"ird\\\\\\nvalue"} 1`); + }); +});