feat(agent): feed aggregate outcome quality into confidence

src/services/agent-orchestrator.ts

@@ -22,7 +22,7 @@ import {
 import { contributorRepoStatsFromGittensor, fetchGittensorContributorSnapshot } from "../gittensor/api";
 import { fetchPublicContributorProfile } from "../github/public";
 import { getOrCreateScoringModelSnapshot } from "../scoring/model";
-import { loadContributorDecisionPackForServing, repoDecisionFromPack, type ActionPortfolio, type ActionPortfolioBucketName, type ContributorDecisionPack, type DecisionAction, type RepoDecision } from "./decision-pack";
+import { loadContributorDecisionPackForServing, repoDecisionFromPack, type ActionPortfolio, type ActionPortfolioBucketName, type ContributorDecisionPack, type DecisionAction, type RepoDecision, type RepoOutcomeSummary } from "./decision-pack";
 import { loadOrComputeIssueQualityResponse } from "./issue-quality";
 import { summarizeAgentBundleWithAi } from "./ai-summaries";
 import { buildContributorFit, buildContributorOutcomeHistory, buildContributorProfile, buildContributorScoringProfile } from "../signals/engine";
@@ -630,18 +630,21 @@ function decisionPackEvidence(pack: ContributorDecisionPack, decision: RepoDecis
   const missingOfficialStats = !pack.profile.officialStats || pack.profile.source !== "gittensor_api";
   const missingRepoOutcome = !decision.outcome && !decision.roleContext.maintainerLane;
   const freshness = pack.freshness !== "fresh" ? pack.freshness : repoQuality.freshness;
+  const outcomeQuality = aggregateOutcomeQuality(decision.repoOutcomePatterns);
   const warnings = uniqueStrings([
     ...(pack.freshness === "rebuilding" ? ["Decision pack is stale; a background rebuild was enqueued."] : []),
     ...(pack.freshness === "stale" ? ["Decision pack is stale and no rebuild was enqueued."] : []),
     ...(pack.dataQuality.signalFidelity.status === "blocked" ? ["Signal fidelity is blocked for this decision pack."] : []),
     ...repoQuality.warnings,
     ...(missingOfficialStats ? ["Official Gittensor contributor stats were unavailable; confidence is reduced."] : []),
     ...(missingRepoOutcome ? ["No repo-specific official outcome row was available; confidence is reduced."] : []),
+    ...(outcomeQuality.warning ? [outcomeQuality.warning] : []),
   ]);
   const assumptions = uniqueStrings([
     ...(missingOfficialStats ? ["Contributor-level official stats are missing, so cached GitHub and registry data carry more weight."] : []),
     ...(missingRepoOutcome ? ["Repo-specific prior outcomes are missing, so queue, lane, and role heuristics carry more weight."] : []),
     ...(userSuppliedScenarioCount > 0 ? ["Pending-PR scenario projections include user-supplied assumptions."] : []),
+    ...(outcomeQuality.assumption ? [outcomeQuality.assumption] : []),
   ]);
   return {
     confidence: confidenceForDecisionPack(pack, decision, repoQuality, userSuppliedScenarioCount),
@@ -664,6 +667,13 @@ function decisionPackEvidence(pack: ContributorDecisionPack, decision: RepoDecis
         decision.outcome ? "fresh" : "missing",
         decision.outcome ? "Repo-specific contributor outcomes present." : "Repo-specific contributor outcomes missing.",
       ),
+      evidenceSource(
+        "aggregate_outcome_quality",
+        decision.repoOutcomePatterns ? "cached_repo_patterns" : null,
+        pack.generatedAt,
+        outcomeQuality.freshness,
+        outcomeQuality.sourceSummary,
+      ),
       ...(pack.openPrMonitor
         ? [evidenceSource("open_pr_monitor", "cached_github_data", pack.openPrMonitor.generatedAt, pack.freshness === "fresh" ? "fresh" : pack.freshness, pack.openPrMonitor.summary)]
         : []),
@@ -741,6 +751,77 @@ function defaultRecommendationEvidence(actionType: AgentActionType): Recommendat
   };
 }
 
+const OUTCOME_QUALITY_MIN_SAMPLE = 5;
+const OUTCOME_QUALITY_STRONG_MERGE_RATE = 0.6;
+const OUTCOME_QUALITY_HIGH_RISK_RATE = 0.3;
+
+type AggregateOutcomeQuality = {
+  signal: "strong" | "weak" | "high_risk" | "sparse" | "absent";
+  mergeRate: number | null;
+  sampleSize: number;
+  warning: string | null;
+  assumption: string | null;
+  sourceSummary: string;
+  freshness: RecommendationFreshness;
+};
+
+function aggregateOutcomeQuality(patterns: RepoOutcomeSummary | undefined): AggregateOutcomeQuality {
+  if (!patterns) {
+    return {
+      signal: "absent",
+      mergeRate: null,
+      sampleSize: 0,
+      warning: null,
+      assumption: "No aggregate repo outcome quality data is available; heuristic signals carry more weight.",
+      sourceSummary: "No aggregate repo outcome quality data available.",
+      freshness: "missing",
+    };
+  }
+  const { outsideContributorMergeRate: mergeRate, sampleSize } = patterns;
+  if (sampleSize < OUTCOME_QUALITY_MIN_SAMPLE) {
+    return {
+      signal: "sparse",
+      mergeRate,
+      sampleSize,
+      warning: null,
+      assumption: `Aggregate repo outcome quality has limited sample size (${sampleSize} decided PR(s)); signals carry reduced weight.`,
+      sourceSummary: `Sparse aggregate outcome data (${sampleSize} decided PR(s)); confidence impact is limited.`,
+      freshness: "degraded",
+    };
+  }
+  if (mergeRate >= OUTCOME_QUALITY_STRONG_MERGE_RATE) {
+    return {
+      signal: "strong",
+      mergeRate,
+      sampleSize,
+      warning: null,
+      assumption: null,
+      sourceSummary: `Aggregate outside-contributor merge rate is strong across ${sampleSize} decided PR(s).`,
+      freshness: "fresh",
+    };
+  }
+  if (mergeRate <= OUTCOME_QUALITY_HIGH_RISK_RATE) {
+    return {
+      signal: "high_risk",
+      mergeRate,
+      sampleSize,
+      warning: `Aggregate repo outcome quality shows high closure risk across ${sampleSize} decided PR(s); review risk patterns before opening work.`,
+      assumption: null,
+      sourceSummary: `Aggregate outside-contributor merge rate is low across ${sampleSize} decided PR(s); high closure risk.`,
+      freshness: "fresh",
+    };
+  }
+  return {
+    signal: "weak",
+    mergeRate,
+    sampleSize,
+    warning: `Aggregate repo outcome quality shows moderate closure risk across ${sampleSize} decided PR(s).`,
+    assumption: null,
+    sourceSummary: `Aggregate outside-contributor merge rate is moderate across ${sampleSize} decided PR(s).`,
+    freshness: "fresh",
+  };
+}
+
 function confidenceForDecisionPack(
   pack: ContributorDecisionPack,
   decision: RepoDecision,
@@ -757,6 +838,9 @@ function confidenceForDecisionPack(
   if (!pack.profile.officialStats || pack.profile.source !== "gittensor_api") confidence = lowerConfidence(confidence, "medium");
   if (!decision.outcome && !decision.roleContext.maintainerLane) confidence = lowerConfidence(confidence, "medium");
   if (userSuppliedScenarioCount > 0) confidence = lowerConfidence(confidence, "medium");
+  const outcomeQuality = aggregateOutcomeQuality(decision.repoOutcomePatterns);
+  if (outcomeQuality.signal === "high_risk") confidence = lowerConfidence(confidence, "low");
+  else if (outcomeQuality.signal === "weak") confidence = lowerConfidence(confidence, "medium");
   return confidence;
 }
 
@@ -983,4 +1067,5 @@ export const __agentOrchestratorInternals = {
   sanitizePublicSummary,
   jsonPayload,
   sameRepo,
+  aggregateOutcomeQuality,
 };

test/unit/agent-orchestrator.test.ts

@@ -12,7 +12,7 @@ import {
   type AgentRunBundle,
 } from "../../src/services/agent-orchestrator";
 import { buildAgentActionExplanationCard } from "../../src/services/agent-action-explanation-card";
-import { CONTRIBUTOR_DECISION_PACK_SIGNAL, type ContributorDecisionPack } from "../../src/services/decision-pack";
+import { CONTRIBUTOR_DECISION_PACK_SIGNAL, type ContributorDecisionPack, type RepoOutcomeSummary } from "../../src/services/decision-pack";
 import { buildPublicAgentCommandComment, parseGittensoryMentionCommand } from "../../src/github/commands";
 import { normalizeRegistryPayload } from "../../src/registry/normalize";
 import { persistRegistrySnapshot } from "../../src/registry/sync";
@@ -1008,6 +1008,136 @@ describe("agent orchestrator", () => {
     expect(JSON.stringify(preflight.actions)).toContain("linked_issue_bounty_historical");
     expect(JSON.stringify(preflight.actions)).toContain("Source upload disabled");
   });
+
+  it("incorporates aggregate outcome quality into recommendation confidence and evidence", () => {
+    const run = __agentOrchestratorInternals.buildRunRecord({
+      objective: "outcome quality confidence",
+      actorLogin: "oktofeesh1",
+      surface: "mcp",
+      status: "running",
+      payload: {},
+    });
+
+    const strongPatterns: RepoOutcomeSummary = {
+      summary: "High merge rate.",
+      outsideContributorMergeRate: 0.75,
+      sampleSize: 10,
+      successPatterns: [{ title: "Focused file changes", detail: "Focused file changes merge well.", confidence: "high" }],
+      riskPatterns: [],
+    };
+    const highRiskPatterns: RepoOutcomeSummary = {
+      summary: "Low merge rate.",
+      outsideContributorMergeRate: 0.20,
+      sampleSize: 10,
+      successPatterns: [],
+      riskPatterns: [{ title: "High closure rate", detail: "Low-quality label mix correlates with closures.", confidence: "high" }],
+    };
+    const weakPatterns: RepoOutcomeSummary = {
+      summary: "Moderate merge rate.",
+      outsideContributorMergeRate: 0.45,
+      sampleSize: 10,
+      successPatterns: [],
+      riskPatterns: [],
+    };
+    const sparsePatterns: RepoOutcomeSummary = {
+      summary: "Sparse data.",
+      outsideContributorMergeRate: 0.20,
+      sampleSize: 3,
+      successPatterns: [],
+      riskPatterns: [],
+    };
+
+    // Fresh, complete-fidelity pack with official stats for a clean baseline
+    const goodPack = decisionPackFixture({
+      freshness: "fresh",
+      rebuildEnqueued: false,
+      dataQuality: {
+        signalFidelity: {
+          status: "complete",
+          repoCount: 1,
+          completeRepos: 1,
+          degradedRepos: 0,
+          blockedRepos: 0,
+          partialRepos: [],
+          cappedRepos: [],
+          staleRepos: [],
+          rateLimitedRepos: [],
+        },
+      } as unknown as ContributorDecisionPack["dataQuality"],
+    } as unknown as Partial<ContributorDecisionPack>);
+
+    const fakeOutcome = { repoFullName: "owner/repo" } as ContributorDecisionPack["repoDecisions"][number]["outcome"];
+
+    const strongDecision = repoDecision({ repoFullName: "owner/strong", outcome: fakeOutcome, repoOutcomePatterns: strongPatterns });
+    const highRiskDecision = repoDecision({ repoFullName: "owner/high-risk", outcome: fakeOutcome, repoOutcomePatterns: highRiskPatterns });
+    const weakDecision = repoDecision({ repoFullName: "owner/weak", outcome: fakeOutcome, repoOutcomePatterns: weakPatterns });
+    const sparseDecision = repoDecision({ repoFullName: "owner/sparse", outcome: fakeOutcome, repoOutcomePatterns: sparsePatterns });
+    const absentDecision = repoDecision({ repoFullName: "owner/absent", outcome: fakeOutcome });
+
+    const strongAction = __agentOrchestratorInternals.actionFromDecisionAction(run, action("open_new_direct_pr", "owner/strong", "pursue", 80), strongDecision, 0, goodPack);
+    const highRiskAction = __agentOrchestratorInternals.actionFromDecisionAction(run, action("open_new_direct_pr", "owner/high-risk", "pursue", 80), highRiskDecision, 1, goodPack);
+    const weakAction = __agentOrchestratorInternals.actionFromDecisionAction(run, action("open_new_direct_pr", "owner/weak", "pursue", 80), weakDecision, 2, goodPack);
+    const sparseAction = __agentOrchestratorInternals.actionFromDecisionAction(run, action("open_new_direct_pr", "owner/sparse", "pursue", 80), sparseDecision, 3, goodPack);
+    const absentAction = __agentOrchestratorInternals.actionFromDecisionAction(run, action("open_new_direct_pr", "owner/absent", "pursue", 80), absentDecision, 4, goodPack);
+
+    // Strong outcome quality (≥60% merge, adequate sample) → confidence stays high
+    expect(strongAction.payload.recommendationEvidence).toMatchObject({
+      confidence: "high",
+      sources: expect.arrayContaining([
+        expect.objectContaining({ name: "aggregate_outcome_quality", freshness: "fresh", source: "cached_repo_patterns" }),
+      ]),
+    });
+    expect((strongAction.payload.recommendationEvidence as { warnings?: string[] }).warnings ?? []).not.toContain(
+      expect.stringMatching(/closure risk/i),
+    );
+
+    // High-risk outcome quality (≤30% merge, adequate sample) → confidence lowered to "low"
+    expect(highRiskAction.payload.recommendationEvidence).toMatchObject({
+      confidence: "low",
+      warnings: expect.arrayContaining([expect.stringMatching(/high closure risk/i)]),
+      sources: expect.arrayContaining([
+        expect.objectContaining({ name: "aggregate_outcome_quality", freshness: "fresh", source: "cached_repo_patterns" }),
+      ]),
+    });
+
+    // Weak outcome quality (30–60% merge, adequate sample) → confidence lowered to "medium"
+    expect(weakAction.payload.recommendationEvidence).toMatchObject({
+      confidence: "medium",
+      warnings: expect.arrayContaining([expect.stringMatching(/moderate closure risk/i)]),
+      sources: expect.arrayContaining([
+        expect.objectContaining({ name: "aggregate_outcome_quality", freshness: "fresh", source: "cached_repo_patterns" }),
+      ]),
+    });
+
+    // Sparse outcome quality (< min sample) → confidence unchanged, assumption added, source degraded
+    expect(sparseAction.payload.recommendationEvidence).toMatchObject({
+      confidence: "high",
+      assumptions: expect.arrayContaining([expect.stringMatching(/limited sample size/i)]),
+      sources: expect.arrayContaining([
+        expect.objectContaining({ name: "aggregate_outcome_quality", freshness: "degraded", source: "cached_repo_patterns" }),
+      ]),
+    });
+
+    // Absent outcome patterns → assumption added, source is missing
+    expect(absentAction.payload.recommendationEvidence).toMatchObject({
+      confidence: "high",
+      assumptions: expect.arrayContaining([expect.stringMatching(/no aggregate repo outcome quality/i)]),
+      sources: expect.arrayContaining([
+        expect.objectContaining({ name: "aggregate_outcome_quality", freshness: "missing", source: null }),
+      ]),
+    });
+
+    // Public sanitizer: private aggregate quality must not appear in public-facing card text
+    expect(JSON.stringify(highRiskAction.explanationCard?.publicSafe)).not.toMatch(/merge rate|closure risk|aggregate outcome/i);
+    expect(JSON.stringify(highRiskAction.payload.recommendationEvidence)).not.toMatch(/wallet|hotkey|raw trust score/i);
+
+    // aggregateOutcomeQuality helper covers all signal branches directly
+    expect(__agentOrchestratorInternals.aggregateOutcomeQuality(undefined).signal).toBe("absent");
+    expect(__agentOrchestratorInternals.aggregateOutcomeQuality(sparsePatterns).signal).toBe("sparse");
+    expect(__agentOrchestratorInternals.aggregateOutcomeQuality(strongPatterns).signal).toBe("strong");
+    expect(__agentOrchestratorInternals.aggregateOutcomeQuality(highRiskPatterns).signal).toBe("high_risk");
+    expect(__agentOrchestratorInternals.aggregateOutcomeQuality(weakPatterns).signal).toBe("weak");
+  });
 });
 
 async function persistDecisionPack(env: Env, pack: ContributorDecisionPack): Promise<void> {