From 0bdf5f2e7a7efa59a9e787878900a2f02e9bd7f1 Mon Sep 17 00:00:00 2001 From: JSONbored <49853598+JSONbored@users.noreply.github.com> Date: Sat, 11 Jul 2026 05:14:41 -0700 Subject: [PATCH] feat(review): personalize predicted-gate readiness by contributor calibration history (#2349) Extends buildPredictedGateVerdict to read a login's own predicted_gate_calibration_ledger track record (predicted-vs-real agreement) and nudge the returned readinessScore within a fixed +/-10-point clamp -- a strong track record tightens confidence, a weak one is more conservative. Cold start (never-seen or <5 samples) is byte-identical to today. Applied strictly downstream of evaluateGateCheck's blockers/conclusion/warnings, so personalization structurally cannot flip a hard blocker off or add/remove a finding -- it only ever touches the one numeric field. The raw calibration numbers are never echoed back in the verdict, wired into both real call sites (MCP predict_gate/explain_gate_disposition and the /v1/local/branch-analysis route). Closes #2349 --- packages/gittensory-engine/src/index.ts | 4 + .../gittensory-engine/src/predicted-gate.ts | 15 +- .../src/signals/contributor-calibration.ts | 61 ++++++++ src/api/routes.ts | 5 + src/mcp/server.ts | 5 + .../predicted-gate-calibration-ledger.ts | 43 +++++- test/unit/mcp-predict-gate.test.ts | 52 +++++++ .../predicted-gate-calibration-ledger.test.ts | 89 ++++++++++- test/unit/predicted-gate.test.ts | 145 +++++++++++++++++- test/unit/routes-remediation-plan.test.ts | 53 +++++++ 10 files changed, 465 insertions(+), 7 deletions(-) create mode 100644 packages/gittensory-engine/src/signals/contributor-calibration.ts diff --git a/packages/gittensory-engine/src/index.ts b/packages/gittensory-engine/src/index.ts index 8de49cc58b..e0deabcccf 100644 --- a/packages/gittensory-engine/src/index.ts +++ b/packages/gittensory-engine/src/index.ts @@ -498,10 +498,14 @@ export { buildPredictedGateVerdict, predictedGateNote, publicSafeFinding, + applyContributorCalibration, + MIN_CALIBRATION_SAMPLES, + MAX_READINESS_ADJUSTMENT, type GateCheckConclusion, type GatePolicyPack, type PredictedGateInput, type PredictedGateVerdict, + type ContributorCalibrationSignal, } from "./predicted-gate.js"; // Focus-manifest parse/compile core (#2280): shared by the maintainer review stack and the miner's // `.gittensory-miner.yml` goal-spec parser (see miner-goal-spec.ts for the parallel surface). diff --git a/packages/gittensory-engine/src/predicted-gate.ts b/packages/gittensory-engine/src/predicted-gate.ts index d16eb07b8b..eadc90c2ec 100644 --- a/packages/gittensory-engine/src/predicted-gate.ts +++ b/packages/gittensory-engine/src/predicted-gate.ts @@ -9,12 +9,14 @@ import { import { buildFocusManifestGuidance, type FocusManifest } from "./focus-manifest/guidance.js"; import { guardrailPathMatches, isGuardrailHit } from "./signals/change-guardrail.js"; import { resolveHardGuardrailGlobs } from "./review/guardrail-config.js"; +import { applyContributorCalibration, type ContributorCalibrationSignal } from "./signals/contributor-calibration.js"; import { sanitizePublicComment } from "./github/sanitize-public-comment.js"; import { GITTENSOR_HOME_URL } from "./github/constants.js"; import type { BountyRecord, GatePolicyPack, IssueRecord, PullRequestRecord, RepositoryRecord } from "./types/predicted-gate-types.js"; export type { GatePolicyPack } from "./types/predicted-gate-types.js"; export type { GateCheckConclusion } from "./advisory/gate-advisory.js"; +export { applyContributorCalibration, MIN_CALIBRATION_SAMPLES, MAX_READINESS_ADJUSTMENT, type ContributorCalibrationSignal } from "./signals/contributor-calibration.js"; // Opt-in funnel (#694): a non-Gittensor adopter running the `oss-anti-slop` pack learns that Gittensor pays // contributors for OSS work like this. Public-safe "earn" wording only (never reward/payout/score). @@ -136,6 +138,15 @@ export function buildPredictedGateVerdict(args: { * focus-manifest path policy and the path-gated pre-merge checks. Absent ⇒ only path-independent pre-merge * checks are predicted and the note discloses the gap (#11-13/#18). */ changedPaths?: string[] | undefined; + /** #2349: this login's own historical predict-vs-real agreement (predicted_gate_calibration_ledger), + * pre-aggregated by the caller -- this engine package never touches D1. Adjusts ONLY the returned + * readinessScore, strictly AFTER blockers/conclusion/warnings are finalized below, so personalization can + * never flip a hard blocker off or add/remove a finding; it can only narrow/widen the advisory confidence + * number within a fixed clamp (see applyContributorCalibration). `undefined`, `null`, or below the + * cold-start sample threshold ⇒ unweighted baseline, byte-identical to before this field existed. The raw + * calibration numbers are never echoed back in the returned verdict -- only their clamped, already-public + * downstream effect on readinessScore is. */ + contributorCalibration?: ContributorCalibrationSignal | null | undefined; }): PredictedGateVerdict { const { input, manifest, repo, issues, pullRequests } = args; const gate = manifest.gate; @@ -313,7 +324,9 @@ export function buildPredictedGateVerdict(args: { conclusion: evaluation.conclusion, title: sanitizePublicComment(evaluation.title), summary: sanitizePublicComment(evaluation.summary), - readinessScore: readiness.total, + // #2349: applied strictly downstream of `evaluation` (already finalized above) -- personalization can + // only nudge this number, never the blockers/conclusion/warnings that were just computed. + readinessScore: applyContributorCalibration(readiness.total, args.contributorCalibration), confirmedContributor: effectiveConfirmedContributor, blockers: evaluation.blockers.map((finding) => publicSafeFinding(finding)), warnings: evaluation.warnings.map((finding) => publicSafeFinding(finding)), diff --git a/packages/gittensory-engine/src/signals/contributor-calibration.ts b/packages/gittensory-engine/src/signals/contributor-calibration.ts new file mode 100644 index 0000000000..17fd77b340 --- /dev/null +++ b/packages/gittensory-engine/src/signals/contributor-calibration.ts @@ -0,0 +1,61 @@ +// #2349: personalizes buildPredictedGateVerdict's readiness/confidence output using a contributor/miner's OWN +// historical predict-vs-real agreement (predicted_gate_calibration_ledger, written by +// src/review/predicted-gate-calibration-ledger.ts). Pure and D1-free by design -- this package never touches a +// database; the caller (src/mcp/server.ts, src/api/routes.ts) reads and aggregates the ledger for one login, +// then hands this module a plain, already-computed signal. +// +// SAFETY BOUNDARY (mirrors the design note in both src/review/contributor-calibration.ts and +// src/review/predicted-gate-calibration-ledger.ts): buildPredictedGateVerdict calls applyContributorCalibration +// strictly AFTER evaluateGateCheck has already finalized conclusion/blockers/warnings, and threads through +// ONLY the numeric readinessScore. This function never receives blockers or conclusion, so it is structurally +// incapable of flipping a hard blocker off -- not just clamped by convention, but by construction. +// +// PRIVACY: the calibration signal (sampleSize, agreementRate) is consumed here and never echoed back in +// PredictedGateVerdict -- only its clamped, bounded DOWNSTREAM EFFECT (a shifted readinessScore, itself +// already a public-facing concept) is returned. src/signals/redaction.ts's "no raw per-actor trust signal, +// ever public" boundary is preserved because the raw numbers never reach the output at all. + +function clamp(value: number, min: number, max: number): number { + return Math.min(max, Math.max(min, value)); +} + +/** Below this many historical (prediction, real-decision) pairings, a login's track record is treated as + * cold-start (unweighted baseline) -- too few samples to distinguish signal from noise. */ +export const MIN_CALIBRATION_SAMPLES = 5; + +/** Maximum points (out of the 0-100 readinessScore scale) personalization may add or subtract. Deliberately + * small relative to the 100-point scale: this is a confidence nudge, not a re-scoring. */ +export const MAX_READINESS_ADJUSTMENT = 10; + +/** Agreement rate that maps to a zero adjustment -- a coin-flip predict-vs-real track record earns neither a + * bonus nor a penalty. */ +const NEUTRAL_AGREEMENT_RATE = 0.5; + +export type ContributorCalibrationSignal = { + /** How many (prediction, real-decision) pairings this login has in predicted_gate_calibration_ledger. */ + sampleSize: number; + /** Fraction of those pairings where the predicted action matched the real decision. Clamped to [0, 1] + * before use, so a malformed upstream aggregate can never push the adjustment past its own clamp. */ + agreementRate: number; +}; + +/** + * Adjusts a baseline readinessScore by a login's own predict-vs-real calibration history, clamped to + * +/-{@link MAX_READINESS_ADJUSTMENT} points and to the score's own [0, 100] range. + * + * Cold start -- no calibration signal, or fewer than {@link MIN_CALIBRATION_SAMPLES} pairings -- returns the + * baseline completely UNCHANGED: a never-seen (or barely-seen) actor gets no penalty and no bonus. A `null` + * baseline (no readiness score to begin with) stays `null` -- personalization never manufactures a score out + * of nothing. + */ +export function applyContributorCalibration( + baselineReadinessScore: number | null, + calibration: ContributorCalibrationSignal | null | undefined, +): number | null { + if (baselineReadinessScore === null) return null; + if (!calibration || calibration.sampleSize < MIN_CALIBRATION_SAMPLES) return baselineReadinessScore; + const agreementRate = clamp(calibration.agreementRate, 0, 1); + const rawAdjustment = (agreementRate - NEUTRAL_AGREEMENT_RATE) * 2 * MAX_READINESS_ADJUSTMENT; + const adjustment = clamp(rawAdjustment, -MAX_READINESS_ADJUSTMENT, MAX_READINESS_ADJUSTMENT); + return clamp(baselineReadinessScore + adjustment, 0, 100); +} diff --git a/src/api/routes.ts b/src/api/routes.ts index bb400e492d..0a8a47d7e3 100644 --- a/src/api/routes.ts +++ b/src/api/routes.ts @@ -251,6 +251,7 @@ import { buildPullRequestReviewability, type PullRequestReviewability } from ".. import { buildLocalBranchAnalysis, findCurrentBranchPullRequest } from "../signals/local-branch"; import { buildSlopAssessment, buildIssueSlopAssessment, SLOP_RUBRIC_MARKDOWN, ISSUE_SLOP_RUBRIC_MARKDOWN } from "../signals/slop"; import { buildPredictedGateVerdict } from "../rules/predicted-gate"; +import { computeContributorCalibration } from "../review/predicted-gate-calibration-ledger"; import { buildFocusManifestValidation } from "../services/focus-manifest-validation"; import { buildMaintainerActivationPreview, recommendedAdvisoryActivationSettings } from "../services/maintainer-activation"; import { buildRepoOutcomeCalibration } from "../services/outcome-calibration"; @@ -3056,6 +3057,9 @@ export function createApp() { // Pre-submission gate prediction: the SAME advisory + evaluateGateCheck the maintainer PR pipeline // runs, over a synthetic PR from this local branch, using ONLY the repo's PUBLIC .gittensory.yml gate // policy (never the maintainer's private DB settings). Self-scoped (requireContributorAccess above). + // #2349: this login's own predict-vs-real track record, personalizing ONLY the returned readinessScore + // (see buildPredictedGateVerdict's contributorCalibration doc comment for the safety boundary). + const contributorCalibration = await computeContributorCalibration(c.env, parsed.data.login); const predictedGate = buildPredictedGateVerdict({ input: { repoFullName: parsed.data.repoFullName, @@ -3075,6 +3079,7 @@ export function createApp() { // #11-13/#18: thread the local branch's changed PATHS (already in the request) so the predictor also // evaluates the focus-manifest path policy + path-gated pre-merge checks, matching the live gate. ...(parsed.data.changedFiles ? { changedPaths: parsed.data.changedFiles.map((file) => file.path) } : {}), + contributorCalibration, }); const response = { ...analysis, predictedGate, dataQuality: await loadRepoDataQuality(c.env, parsed.data.repoFullName) }; await persistSignal(c.env, "local-branch-analysis", `${parsed.data.login}:${parsed.data.repoFullName}:${parsed.data.branchName ?? parsed.data.headRef ?? "local"}`, parsed.data.repoFullName, response as unknown as Record, analysis.generatedAt); diff --git a/src/mcp/server.ts b/src/mcp/server.ts index 0b62ba8050..a320465755 100644 --- a/src/mcp/server.ts +++ b/src/mcp/server.ts @@ -164,6 +164,7 @@ import { simulateOpenPrPressure, type OpenPrPressureInput } from "../services/op import { buildFindingTaxonomyDocument, FINDING_TAXONOMY_URI } from "../review/finding-taxonomy"; import { buildEnrichmentAnalyzersTaxonomyDocument, ENRICHMENT_ANALYZERS_URI } from "../review/enrichment-analyzers-taxonomy"; import { recordPredictedGateCall } from "../review/predicted-gate-calls"; +import { computeContributorCalibration } from "../review/predicted-gate-calibration-ledger"; type AppContext = Context<{ Bindings: Env }>; type ToolPayload = { @@ -3032,6 +3033,9 @@ export class GittensoryMcp { // all, so skip the lookup there (keeps the prediction account-free for non-Gittensor adopters). const pack = manifest.gate.pack ?? "gittensor"; const confirmedContributor = pack === "oss-anti-slop" ? undefined : (await fetchGittensorContributorSnapshot(input.login)) !== null; + // #2349: this login's own predict-vs-real track record, personalizing ONLY the returned readinessScore + // (see buildPredictedGateVerdict's contributorCalibration doc comment for the safety boundary). + const contributorCalibration = await computeContributorCalibration(this.env, input.login); const verdict = buildPredictedGateVerdict({ input: { repoFullName, @@ -3049,6 +3053,7 @@ export class GittensoryMcp { issueQuality: issueQuality?.report, confirmedContributor, ...(input.changedPaths === undefined ? {} : { changedPaths: input.changedPaths }), + contributorCalibration, }); // #predicted-live-gate-agreement: record this call so a later real gate decision for the same // (repo, login) can be paired against it (src/review/predicted-gate-agreement.ts). Shared by BOTH diff --git a/src/review/predicted-gate-calibration-ledger.ts b/src/review/predicted-gate-calibration-ledger.ts index a461a1df6c..c0bdbbc9bd 100644 --- a/src/review/predicted-gate-calibration-ledger.ts +++ b/src/review/predicted-gate-calibration-ledger.ts @@ -24,14 +24,19 @@ // recordContributorGateDecision's own per-commit REPLACE semantics, deliberately: once this ledger records a // prediction-vs-outcome pairing, that pairing must never change underneath a future calibration reader. // -// THIS PR ONLY WRITES THE LEDGER. Nothing reads predicted_gate_calibration_ledger yet -- mirrors -// contributor_gate_history's (migrations/0126) own "write-only, nothing reads yet" precedent; the eventual -// #2349 consumer is explicit future work, deliberately deferred so a personalization-adjustment reader gets -// its own focused review. +// READ SIDE (#2349): computeContributorCalibration aggregates ONE login's full history into a plain +// {sampleSize, agreementRate} signal for buildPredictedGateVerdict's personalization input +// (packages/gittensory-engine/src/signals/contributor-calibration.ts). It is intentionally NOT gated by +// isSelfHostedReviewRuntime/isParityAuditEnabled the way the writer above is: the write-side flag controls +// whether this telemetry class is collected at all, but the read is a plain "use whatever rows already +// exist" query -- gating it too would make historical calibration data silently stop being read the moment +// the flag is toggled off, which is a surprising extra restriction nothing here asks for. When the flag was +// never on, the table is simply empty and the read naturally degrades to cold-start. import { isParityAuditEnabled } from "./parity-wire"; import { isSelfHostedReviewRuntime } from "../selfhost/review-runtime"; import { errorMessage, nowIso } from "../utils/json"; +import type { ContributorCalibrationSignal } from "../../packages/gittensory-engine/src/signals/contributor-calibration"; /** The minimal env shape the recorder needs -- mirrors parity-wire.ts's ParityRecorderEnv / contributor- * calibration.ts's ContributorCalibrationEnv exactly (same gate-accuracy telemetry family, same flag). */ @@ -124,3 +129,33 @@ export async function recordPredictedGateCalibration( console.warn(JSON.stringify({ event: "predicted_gate_calibration_write_error", project, message: errorMessage(error).slice(0, 200) })); } } + +/** + * Aggregate one login's full predicted_gate_calibration_ledger history into the plain signal + * {@link ContributorCalibrationSignal} that buildPredictedGateVerdict's `contributorCalibration` argument + * expects (#2349). A missing/blank login or a read failure both resolve to `null` -- the caller threads that + * straight into buildPredictedGateVerdict, whose cold-start handling treats `null` exactly like "never seen + * this actor": no penalty, no bonus. Best-effort and fail-safe: a read error is swallowed and logged, never + * thrown -- a calibration lookup must never break gate prediction. + */ +export async function computeContributorCalibration(env: PredictedGateCalibrationEnv, login: string | null | undefined): Promise { + const trimmed = login?.trim(); + if (!trimmed) return null; + try { + const row = await env.DB.prepare( + `SELECT COUNT(*) AS sampleSize, COALESCE(AVG(agreed), 0) AS agreementRate + FROM predicted_gate_calibration_ledger + WHERE login = ?`, + ) + .bind(trimmed) + .first<{ sampleSize: number; agreementRate: number }>(); + // COUNT(*)/AVG(...) with no GROUP BY always returns exactly one row, even over zero matches (COUNT: 0, + // AVG: NULL -> COALESCE: 0) -- .first()'s nullable return type is a TypeScript-level formality here, not + // a reachable runtime case for this query shape. + /* v8 ignore next */ + return row ? { sampleSize: row.sampleSize, agreementRate: row.agreementRate } : { sampleSize: 0, agreementRate: 0 }; + } catch (error) { + console.warn(JSON.stringify({ event: "contributor_calibration_read_error", message: errorMessage(error).slice(0, 200) })); + return null; + } +} diff --git a/test/unit/mcp-predict-gate.test.ts b/test/unit/mcp-predict-gate.test.ts index 5ec61f9812..6cd3e17d57 100644 --- a/test/unit/mcp-predict-gate.test.ts +++ b/test/unit/mcp-predict-gate.test.ts @@ -274,4 +274,56 @@ testExpectations: expect(await rawAll(env, "SELECT * FROM predicted_gate_calls")).toHaveLength(0); }); }); + + describe("personalized calibration wiring (#2349)", () => { + async function seedLedgerRow(env: Env, opts: { login: string; agreed: boolean; pullNumber: number }) { + const now = new Date().toISOString(); + await env.DB.prepare( + `INSERT INTO predicted_gate_calibration_ledger (id, login, project, target_id, predicted_action, real_decision, agreed, predicted_at, decided_at, created_at) + VALUES (?, ?, ?, ?, ?, ?, ?, ?, ?, ?)`, + ) + .bind(crypto.randomUUID(), opts.login, "acme/widgets", `acme/widgets#${opts.pullNumber}`, "merge", opts.agreed ? "merge" : "hold", opts.agreed ? 1 : 0, now, now, now) + .run(); + } + + it("a login with a weak predict-vs-real track record gets a LOWER readinessScore than an identical fresh login", async () => { + const env = createTestEnv(); + await upsertRepositoryFromGitHub(env, { name: "widgets", full_name: "acme/widgets" }); + await upsertRepoFocusManifest(env, "acme/widgets", { gate: { pack: "oss-anti-slop", duplicates: "block", linkedIssue: "advisory" } }, "repo_file"); + // 6 rows, mostly disagreements -- well above MIN_CALIBRATION_SAMPLES (5), well below neutral (50%). + for (let i = 0; i < 6; i++) await seedLedgerRow(env, { login: "shaky-miner", pullNumber: i, agreed: i === 0 }); + const client = await connect(env); + + const fresh = await client.callTool({ + name: "gittensory_predict_gate", + arguments: { login: "fresh-miner", owner: "acme", repo: "widgets", title: "Add retry to upload client" }, + }); + const weak = await client.callTool({ + name: "gittensory_predict_gate", + arguments: { login: "shaky-miner", owner: "acme", repo: "widgets", title: "Add retry to upload client" }, + }); + + const freshData = fresh.structuredContent as { readinessScore: number }; + const weakData = weak.structuredContent as { readinessScore: number }; + expect(typeof freshData.readinessScore).toBe("number"); + expect(weakData.readinessScore).toBeLessThan(freshData.readinessScore); + }); + + it("never echoes the raw calibration numbers (sampleSize, agreementRate) back to the caller", async () => { + const env = createTestEnv(); + await upsertRepositoryFromGitHub(env, { name: "widgets", full_name: "acme/widgets" }); + await upsertRepoFocusManifest(env, "acme/widgets", { gate: { pack: "oss-anti-slop", duplicates: "block", linkedIssue: "advisory" } }, "repo_file"); + for (let i = 0; i < 6; i++) await seedLedgerRow(env, { login: "shaky-miner", pullNumber: i, agreed: false }); + const client = await connect(env); + + const result = await client.callTool({ + name: "gittensory_predict_gate", + arguments: { login: "shaky-miner", owner: "acme", repo: "widgets", title: "Add retry to upload client" }, + }); + + const serialized = JSON.stringify(result.structuredContent); + expect(serialized).not.toContain("agreementRate"); + expect(serialized).not.toContain("sampleSize"); + }); + }); }); diff --git a/test/unit/predicted-gate-calibration-ledger.test.ts b/test/unit/predicted-gate-calibration-ledger.test.ts index 6229f10389..ea4d9f06ec 100644 --- a/test/unit/predicted-gate-calibration-ledger.test.ts +++ b/test/unit/predicted-gate-calibration-ledger.test.ts @@ -1,5 +1,5 @@ import { describe, expect, it, vi } from "vitest"; -import { recordPredictedGateCalibration } from "../../src/review/predicted-gate-calibration-ledger"; +import { computeContributorCalibration, recordPredictedGateCalibration } from "../../src/review/predicted-gate-calibration-ledger"; import { createTestEnv } from "../helpers/d1"; async function rawAll(env: Env, sql: string, ...binds: unknown[]): Promise[]> { @@ -16,6 +16,20 @@ async function seedPredicted(env: Env, opts: { login: string; project: string; a .run(); } +/** Inserts directly into predicted_gate_calibration_ledger, bypassing recordPredictedGateCalibration's + * correlation-window pairing logic -- gives computeContributorCalibration's tests full control over how many + * agreed/disagreed rows a login has, regardless of timing. */ +async function seedLedgerRow(env: Env, opts: { login: string; project?: string; pullNumber: number; agreed: boolean }) { + const project = opts.project ?? repoFullName; + const now = new Date().toISOString(); + await env.DB.prepare( + `INSERT INTO predicted_gate_calibration_ledger (id, login, project, target_id, predicted_action, real_decision, agreed, predicted_at, decided_at, created_at) + VALUES (?, ?, ?, ?, ?, ?, ?, ?, ?, ?)`, + ) + .bind(crypto.randomUUID(), opts.login, project, `${project}#${opts.pullNumber}`, "merge", opts.agreed ? "merge" : "hold", opts.agreed ? 1 : 0, now, now, now) + .run(); +} + const repoFullName = "owner/repo"; describe("recordPredictedGateCalibration — login-keyed predict-vs-live calibration ledger (#4517)", () => { @@ -170,3 +184,76 @@ describe("recordPredictedGateCalibration — login-keyed predict-vs-live calibra warn.mockRestore(); }); }); + +describe("computeContributorCalibration — per-login calibration read (#2349)", () => { + it("cold start: a login with no ledger rows gets sampleSize 0 / agreementRate 0, not null", async () => { + const env = createTestEnv(); + expect(await computeContributorCalibration(env, "octocat")).toEqual({ sampleSize: 0, agreementRate: 0 }); + }); + + it("aggregates a mix of agreed/disagreed rows into an accurate sampleSize and agreementRate", async () => { + const env = createTestEnv(); + await seedLedgerRow(env, { login: "octocat", pullNumber: 1, agreed: true }); + await seedLedgerRow(env, { login: "octocat", pullNumber: 2, agreed: true }); + await seedLedgerRow(env, { login: "octocat", pullNumber: 3, agreed: true }); + await seedLedgerRow(env, { login: "octocat", pullNumber: 4, agreed: false }); + await seedLedgerRow(env, { login: "octocat", pullNumber: 5, agreed: false }); + + const result = await computeContributorCalibration(env, "octocat"); + expect(result).toEqual({ sampleSize: 5, agreementRate: 0.6 }); + }); + + it("a perfect track record aggregates to agreementRate 1", async () => { + const env = createTestEnv(); + await seedLedgerRow(env, { login: "octocat", pullNumber: 1, agreed: true }); + await seedLedgerRow(env, { login: "octocat", pullNumber: 2, agreed: true }); + expect(await computeContributorCalibration(env, "octocat")).toEqual({ sampleSize: 2, agreementRate: 1 }); + }); + + it("scopes strictly per-login — a different login's history never leaks in", async () => { + const env = createTestEnv(); + await seedLedgerRow(env, { login: "octocat", pullNumber: 1, agreed: false }); + await seedLedgerRow(env, { login: "octocat", pullNumber: 2, agreed: false }); + await seedLedgerRow(env, { login: "someone-else", pullNumber: 1, agreed: true }); + + expect(await computeContributorCalibration(env, "octocat")).toEqual({ sampleSize: 2, agreementRate: 0 }); + expect(await computeContributorCalibration(env, "someone-else")).toEqual({ sampleSize: 1, agreementRate: 1 }); + }); + + it("aggregates across ALL of a login's history regardless of which repo each pairing came from", async () => { + const env = createTestEnv(); + await seedLedgerRow(env, { login: "octocat", project: "owner/repo-a", pullNumber: 1, agreed: true }); + await seedLedgerRow(env, { login: "octocat", project: "owner/repo-b", pullNumber: 1, agreed: false }); + expect(await computeContributorCalibration(env, "octocat")).toEqual({ sampleSize: 2, agreementRate: 0.5 }); + }); + + it("returns null for a missing, null, or blank login — nothing meaningful to look up", async () => { + const env = createTestEnv(); + await seedLedgerRow(env, { login: "octocat", pullNumber: 1, agreed: true }); + expect(await computeContributorCalibration(env, undefined)).toBeNull(); + expect(await computeContributorCalibration(env, null)).toBeNull(); + expect(await computeContributorCalibration(env, " ")).toBeNull(); + }); + + it("reads regardless of the self-hosted/parity-audit flag — unlike the writer, the read is not flag-gated", async () => { + const env = createTestEnv(); + delete env.SELFHOST_TRANSIENT_CACHE; // simulates the cloud worker, where the WRITER would have been skipped + await seedLedgerRow(env, { login: "octocat", pullNumber: 1, agreed: true }); + expect(await computeContributorCalibration(env, "octocat")).toEqual({ sampleSize: 1, agreementRate: 1 }); + }); + + it("fails safe: a read error resolves to null (logs, never throws)", async () => { + const env = createTestEnv(); + const realPrepare = env.DB.prepare.bind(env.DB); + env.DB.prepare = ((sql: string) => { + if (/SELECT[\s\S]*FROM[\s\S]*predicted_gate_calibration_ledger/i.test(sql)) throw new Error("d1 down"); + return realPrepare(sql); + }) as typeof env.DB.prepare; + const warn = vi.spyOn(console, "warn").mockImplementation(() => {}); + + await expect(computeContributorCalibration(env, "octocat")).resolves.toBeNull(); + + expect(warn.mock.calls.map((c) => String(c[0])).some((line) => line.includes("contributor_calibration_read_error"))).toBe(true); + warn.mockRestore(); + }); +}); diff --git a/test/unit/predicted-gate.test.ts b/test/unit/predicted-gate.test.ts index 417be2e712..d995e09cb0 100644 --- a/test/unit/predicted-gate.test.ts +++ b/test/unit/predicted-gate.test.ts @@ -1,5 +1,12 @@ import { describe, expect, it } from "vitest"; -import { buildPredictedGateVerdict, type PredictedGateInput } from "../../src/rules/predicted-gate"; +import { + buildPredictedGateVerdict, + applyContributorCalibration, + MIN_CALIBRATION_SAMPLES, + MAX_READINESS_ADJUSTMENT, + type PredictedGateInput, + type ContributorCalibrationSignal, +} from "../../src/rules/predicted-gate"; import { parseFocusManifest } from "../../src/signals/focus-manifest"; import type { IssueRecord, PullRequestRecord, RepositoryRecord } from "../../src/types"; @@ -29,6 +36,7 @@ function verdict(args: { input?: Partial; issues?: IssueRecord[]; pullRequests?: PullRequestRecord[]; + contributorCalibration?: ContributorCalibrationSignal | null | undefined; }) { return buildPredictedGateVerdict({ input: { ...BASE_INPUT, ...args.input }, @@ -37,6 +45,7 @@ function verdict(args: { issues: args.issues ?? [openIssue(7, "Uploads should retry on 5xx")], pullRequests: args.pullRequests ?? [], ...(args.changedPaths ? { changedPaths: args.changedPaths } : {}), + ...(args.contributorCalibration !== undefined ? { contributorCalibration: args.contributorCalibration } : {}), }); } @@ -543,3 +552,137 @@ describe("pack-aware prediction (#693)", () => { expect(result.blockers.some((b) => b.code === "missing_linked_issue")).toBe(true); }); }); + +describe("applyContributorCalibration (#2349)", () => { + it("cold start: undefined calibration returns the baseline unchanged", () => { + expect(applyContributorCalibration(70, undefined)).toBe(70); + }); + + it("cold start: null calibration returns the baseline unchanged", () => { + expect(applyContributorCalibration(70, null)).toBe(70); + }); + + it("cold start: sampleSize below MIN_CALIBRATION_SAMPLES returns the baseline unchanged, even with an extreme agreementRate", () => { + const belowThreshold: ContributorCalibrationSignal = { sampleSize: MIN_CALIBRATION_SAMPLES - 1, agreementRate: 1 }; + expect(applyContributorCalibration(70, belowThreshold)).toBe(70); + }); + + it("at exactly MIN_CALIBRATION_SAMPLES, the adjustment kicks in", () => { + const atThreshold: ContributorCalibrationSignal = { sampleSize: MIN_CALIBRATION_SAMPLES, agreementRate: 1 }; + expect(applyContributorCalibration(70, atThreshold)).toBe(70 + MAX_READINESS_ADJUSTMENT); + }); + + it("a strong track record (high agreement) nudges the score UP", () => { + const strong: ContributorCalibrationSignal = { sampleSize: 20, agreementRate: 0.9 }; + // (0.9 - 0.5) * 2 * 10 = 8 + expect(applyContributorCalibration(50, strong)).toBe(58); + }); + + it("a weak track record (low agreement) nudges the score DOWN", () => { + const weak: ContributorCalibrationSignal = { sampleSize: 20, agreementRate: 0.1 }; + // (0.1 - 0.5) * 2 * 10 = -8 + expect(applyContributorCalibration(50, weak)).toBe(42); + }); + + it("a coin-flip (50%) agreement rate is neutral: zero adjustment", () => { + const neutral: ContributorCalibrationSignal = { sampleSize: 50, agreementRate: 0.5 }; + expect(applyContributorCalibration(50, neutral)).toBe(50); + }); + + it("CLAMP BOUNDARY: a perfect (100%) track record never exceeds +MAX_READINESS_ADJUSTMENT", () => { + const perfect: ContributorCalibrationSignal = { sampleSize: 500, agreementRate: 1 }; + expect(applyContributorCalibration(50, perfect)).toBe(50 + MAX_READINESS_ADJUSTMENT); + }); + + it("CLAMP BOUNDARY: an always-wrong (0%) track record never exceeds -MAX_READINESS_ADJUSTMENT", () => { + const alwaysWrong: ContributorCalibrationSignal = { sampleSize: 500, agreementRate: 0 }; + expect(applyContributorCalibration(50, alwaysWrong)).toBe(50 - MAX_READINESS_ADJUSTMENT); + }); + + it("CLAMP BOUNDARY: a malformed agreementRate above 1 is clamped to 1 before scaling — never a larger-than-max adjustment", () => { + const malformed: ContributorCalibrationSignal = { sampleSize: 500, agreementRate: 1.7 }; + expect(applyContributorCalibration(50, malformed)).toBe(50 + MAX_READINESS_ADJUSTMENT); + }); + + it("CLAMP BOUNDARY: a malformed negative agreementRate is clamped to 0 before scaling — never a larger-than-max penalty", () => { + const malformed: ContributorCalibrationSignal = { sampleSize: 500, agreementRate: -0.4 }; + expect(applyContributorCalibration(50, malformed)).toBe(50 - MAX_READINESS_ADJUSTMENT); + }); + + it("CLAMP BOUNDARY: the outer [0, 100] score clamp still applies when a positive adjustment would overflow 100", () => { + const perfect: ContributorCalibrationSignal = { sampleSize: 500, agreementRate: 1 }; + expect(applyContributorCalibration(95, perfect)).toBe(100); + }); + + it("CLAMP BOUNDARY: the outer [0, 100] score clamp still applies when a negative adjustment would underflow 0", () => { + const alwaysWrong: ContributorCalibrationSignal = { sampleSize: 500, agreementRate: 0 }; + expect(applyContributorCalibration(5, alwaysWrong)).toBe(0); + }); + + it("a null baseline (no readiness score to begin with) stays null — personalization never manufactures a score", () => { + const perfect: ContributorCalibrationSignal = { sampleSize: 500, agreementRate: 1 }; + expect(applyContributorCalibration(null, perfect)).toBeNull(); + }); +}); + +describe("buildPredictedGateVerdict — personalized calibration wiring (#2349)", () => { + const CLEAN_GATE = { duplicates: "block", linkedIssue: "advisory" } as const; + const STRONG: ContributorCalibrationSignal = { sampleSize: 30, agreementRate: 0.95 }; + const WEAK: ContributorCalibrationSignal = { sampleSize: 30, agreementRate: 0.05 }; + + function baselineScore(): number { + const result = verdict({ gate: CLEAN_GATE }); + expect(typeof result.readinessScore).toBe("number"); + return result.readinessScore as number; + } + + it("omitting contributorCalibration is byte-identical to today (no field, no change)", () => { + const withoutField = verdict({ gate: CLEAN_GATE }); + const explicitUndefined = verdict({ gate: CLEAN_GATE, contributorCalibration: undefined }); + expect(explicitUndefined.readinessScore).toBe(withoutField.readinessScore); + }); + + it("a strong track record raises readinessScore, clamped to the baseline's own [0, 100] range", () => { + const base = baselineScore(); + const result = verdict({ gate: CLEAN_GATE, contributorCalibration: STRONG }); + // agreementRate 0.95 -> (0.95 - 0.5) * 2 * MAX_READINESS_ADJUSTMENT = 9 points, not the full max. + expect(result.readinessScore).toBe(Math.min(100, base + 9)); + }); + + it("a weak track record lowers readinessScore, clamped to the baseline's own [0, 100] range", () => { + const base = baselineScore(); + const result = verdict({ gate: CLEAN_GATE, contributorCalibration: WEAK }); + // agreementRate 0.05 -> (0.05 - 0.5) * 2 * MAX_READINESS_ADJUSTMENT = -9 points, not the full max. + expect(result.readinessScore).toBe(Math.max(0, base - 9)); + }); + + it("personalization changes readinessScore ONLY — conclusion, blockers, and warnings are byte-identical", () => { + const base = verdict({ gate: CLEAN_GATE }); + const personalized = verdict({ gate: CLEAN_GATE, contributorCalibration: STRONG }); + expect(personalized.conclusion).toBe(base.conclusion); + expect(personalized.blockers).toEqual(base.blockers); + expect(personalized.warnings).toEqual(base.warnings); + expect(personalized.readinessScore).not.toBe(base.readinessScore); + }); + + it("SAFETY: an extreme positive track record cannot bypass a real hard blocker (missing linked issue)", () => { + const result = verdict({ + gate: { linkedIssue: "block" }, + input: { body: "no issue here", linkedIssues: [] }, + issues: [], + // The strongest possible personalization signal — still must not move conclusion off "failure" or + // remove the blocker, because applyContributorCalibration only ever touches readinessScore. + contributorCalibration: { sampleSize: 10_000, agreementRate: 1 }, + }); + expect(result.conclusion).toBe("failure"); + expect(result.blockers.some((b) => b.code === "missing_linked_issue")).toBe(true); + }); + + it("SAFETY: the raw calibration numbers (sampleSize, agreementRate) are never echoed back in the verdict", () => { + const result = verdict({ gate: CLEAN_GATE, contributorCalibration: STRONG }); + const serialized = JSON.stringify(result); + expect(serialized).not.toContain("agreementRate"); + expect(serialized).not.toContain("sampleSize"); + expect(serialized).not.toContain("0.95"); + }); +}); diff --git a/test/unit/routes-remediation-plan.test.ts b/test/unit/routes-remediation-plan.test.ts index 9d6f1a57a7..3e24abf689 100644 --- a/test/unit/routes-remediation-plan.test.ts +++ b/test/unit/routes-remediation-plan.test.ts @@ -207,3 +207,56 @@ describe("local branch routes byte-cap ingestion bound", () => { }); } }); + +describe("branch-analysis route — personalized calibration wiring (#2349)", () => { + async function seedLedgerRow(env: Env, opts: { login: string; agreed: boolean; pullNumber: number }) { + const now = new Date().toISOString(); + await env.DB.prepare( + `INSERT INTO predicted_gate_calibration_ledger (id, login, project, target_id, predicted_action, real_decision, agreed, predicted_at, decided_at, created_at) + VALUES (?, ?, ?, ?, ?, ?, ?, ?, ?, ?)`, + ) + .bind(crypto.randomUUID(), opts.login, "miner/demo", `miner/demo#${opts.pullNumber}`, "merge", opts.agreed ? "merge" : "hold", opts.agreed ? 1 : 0, now, now, now) + .run(); + } + + it("a login with a weak predict-vs-real track record gets a LOWER predictedGate.readinessScore than a fresh login", async () => { + const app = createApp(); + const env = createTestEnv(); + await seedRepo(env, "miner", "demo", 301); + for (let i = 0; i < 6; i++) await seedLedgerRow(env, { login: "shaky-miner", pullNumber: i, agreed: i === 0 }); + + const freshResponse = await app.request( + BRANCH_ANALYSIS_PATH, + { method: "POST", headers: apiHeaders(env), body: JSON.stringify(branchPayload("fresh-miner", "miner/demo")) }, + env, + ); + const weakResponse = await app.request( + BRANCH_ANALYSIS_PATH, + { method: "POST", headers: apiHeaders(env), body: JSON.stringify(branchPayload("shaky-miner", "miner/demo")) }, + env, + ); + expect(freshResponse.status).toBe(200); + expect(weakResponse.status).toBe(200); + const fresh = (await freshResponse.json()) as { predictedGate: { readinessScore: number } }; + const weak = (await weakResponse.json()) as { predictedGate: { readinessScore: number } }; + expect(typeof fresh.predictedGate.readinessScore).toBe("number"); + expect(weak.predictedGate.readinessScore).toBeLessThan(fresh.predictedGate.readinessScore); + }); + + it("never echoes the raw calibration numbers (sampleSize, agreementRate) back in the response", async () => { + const app = createApp(); + const env = createTestEnv(); + await seedRepo(env, "miner", "demo", 301); + for (let i = 0; i < 6; i++) await seedLedgerRow(env, { login: "shaky-miner", pullNumber: i, agreed: false }); + + const response = await app.request( + BRANCH_ANALYSIS_PATH, + { method: "POST", headers: apiHeaders(env), body: JSON.stringify(branchPayload("shaky-miner", "miner/demo")) }, + env, + ); + expect(response.status).toBe(200); + const serialized = JSON.stringify(await response.json()); + expect(serialized).not.toContain("agreementRate"); + expect(serialized).not.toContain("sampleSize"); + }); +});