diff --git a/src/services/agent-orchestrator.ts b/src/services/agent-orchestrator.ts index 8ca66dc883..707c120098 100644 --- a/src/services/agent-orchestrator.ts +++ b/src/services/agent-orchestrator.ts @@ -22,7 +22,7 @@ import { import { contributorRepoStatsFromGittensor, fetchGittensorContributorSnapshot } from "../gittensor/api"; import { fetchPublicContributorProfile } from "../github/public"; import { getOrCreateScoringModelSnapshot } from "../scoring/model"; -import { loadContributorDecisionPackForServing, repoDecisionFromPack, type ActionPortfolio, type ActionPortfolioBucketName, type ContributorDecisionPack, type DecisionAction, type RepoDecision } from "./decision-pack"; +import { loadContributorDecisionPackForServing, repoDecisionFromPack, type ActionPortfolio, type ActionPortfolioBucketName, type ContributorDecisionPack, type DecisionAction, type RepoDecision, type RepoOutcomeSummary } from "./decision-pack"; import { loadOrComputeIssueQualityResponse } from "./issue-quality"; import { summarizeAgentBundleWithAi } from "./ai-summaries"; import { buildContributorFit, buildContributorOutcomeHistory, buildContributorProfile, buildContributorScoringProfile } from "../signals/engine"; @@ -630,6 +630,7 @@ function decisionPackEvidence(pack: ContributorDecisionPack, decision: RepoDecis const missingOfficialStats = !pack.profile.officialStats || pack.profile.source !== "gittensor_api"; const missingRepoOutcome = !decision.outcome && !decision.roleContext.maintainerLane; const freshness = pack.freshness !== "fresh" ? pack.freshness : repoQuality.freshness; + const outcomeQuality = aggregateOutcomeQuality(decision.repoOutcomePatterns); const warnings = uniqueStrings([ ...(pack.freshness === "rebuilding" ? ["Decision pack is stale; a background rebuild was enqueued."] : []), ...(pack.freshness === "stale" ? ["Decision pack is stale and no rebuild was enqueued."] : []), @@ -637,11 +638,13 @@ function decisionPackEvidence(pack: ContributorDecisionPack, decision: RepoDecis ...repoQuality.warnings, ...(missingOfficialStats ? ["Official Gittensor contributor stats were unavailable; confidence is reduced."] : []), ...(missingRepoOutcome ? ["No repo-specific official outcome row was available; confidence is reduced."] : []), + ...(outcomeQuality.warning ? [outcomeQuality.warning] : []), ]); const assumptions = uniqueStrings([ ...(missingOfficialStats ? ["Contributor-level official stats are missing, so cached GitHub and registry data carry more weight."] : []), ...(missingRepoOutcome ? ["Repo-specific prior outcomes are missing, so queue, lane, and role heuristics carry more weight."] : []), ...(userSuppliedScenarioCount > 0 ? ["Pending-PR scenario projections include user-supplied assumptions."] : []), + ...(outcomeQuality.assumption ? [outcomeQuality.assumption] : []), ]); return { confidence: confidenceForDecisionPack(pack, decision, repoQuality, userSuppliedScenarioCount), @@ -664,6 +667,13 @@ function decisionPackEvidence(pack: ContributorDecisionPack, decision: RepoDecis decision.outcome ? "fresh" : "missing", decision.outcome ? "Repo-specific contributor outcomes present." : "Repo-specific contributor outcomes missing.", ), + evidenceSource( + "aggregate_outcome_quality", + decision.repoOutcomePatterns ? "cached_repo_patterns" : null, + pack.generatedAt, + outcomeQuality.freshness, + outcomeQuality.sourceSummary, + ), ...(pack.openPrMonitor ? [evidenceSource("open_pr_monitor", "cached_github_data", pack.openPrMonitor.generatedAt, pack.freshness === "fresh" ? "fresh" : pack.freshness, pack.openPrMonitor.summary)] : []), @@ -741,6 +751,77 @@ function defaultRecommendationEvidence(actionType: AgentActionType): Recommendat }; } +const OUTCOME_QUALITY_MIN_SAMPLE = 5; +const OUTCOME_QUALITY_STRONG_MERGE_RATE = 0.6; +const OUTCOME_QUALITY_HIGH_RISK_RATE = 0.3; + +type AggregateOutcomeQuality = { + signal: "strong" | "weak" | "high_risk" | "sparse" | "absent"; + mergeRate: number | null; + sampleSize: number; + warning: string | null; + assumption: string | null; + sourceSummary: string; + freshness: RecommendationFreshness; +}; + +function aggregateOutcomeQuality(patterns: RepoOutcomeSummary | undefined): AggregateOutcomeQuality { + if (!patterns) { + return { + signal: "absent", + mergeRate: null, + sampleSize: 0, + warning: null, + assumption: "No aggregate repo outcome quality data is available; heuristic signals carry more weight.", + sourceSummary: "No aggregate repo outcome quality data available.", + freshness: "missing", + }; + } + const { outsideContributorMergeRate: mergeRate, sampleSize } = patterns; + if (sampleSize < OUTCOME_QUALITY_MIN_SAMPLE) { + return { + signal: "sparse", + mergeRate, + sampleSize, + warning: null, + assumption: `Aggregate repo outcome quality has limited sample size (${sampleSize} decided PR(s)); signals carry reduced weight.`, + sourceSummary: `Sparse aggregate outcome data (${sampleSize} decided PR(s)); confidence impact is limited.`, + freshness: "degraded", + }; + } + if (mergeRate >= OUTCOME_QUALITY_STRONG_MERGE_RATE) { + return { + signal: "strong", + mergeRate, + sampleSize, + warning: null, + assumption: null, + sourceSummary: `Aggregate outside-contributor merge rate is strong across ${sampleSize} decided PR(s).`, + freshness: "fresh", + }; + } + if (mergeRate <= OUTCOME_QUALITY_HIGH_RISK_RATE) { + return { + signal: "high_risk", + mergeRate, + sampleSize, + warning: `Aggregate repo outcome quality shows high closure risk across ${sampleSize} decided PR(s); review risk patterns before opening work.`, + assumption: null, + sourceSummary: `Aggregate outside-contributor merge rate is low across ${sampleSize} decided PR(s); high closure risk.`, + freshness: "fresh", + }; + } + return { + signal: "weak", + mergeRate, + sampleSize, + warning: `Aggregate repo outcome quality shows moderate closure risk across ${sampleSize} decided PR(s).`, + assumption: null, + sourceSummary: `Aggregate outside-contributor merge rate is moderate across ${sampleSize} decided PR(s).`, + freshness: "fresh", + }; +} + function confidenceForDecisionPack( pack: ContributorDecisionPack, decision: RepoDecision, @@ -757,6 +838,9 @@ function confidenceForDecisionPack( if (!pack.profile.officialStats || pack.profile.source !== "gittensor_api") confidence = lowerConfidence(confidence, "medium"); if (!decision.outcome && !decision.roleContext.maintainerLane) confidence = lowerConfidence(confidence, "medium"); if (userSuppliedScenarioCount > 0) confidence = lowerConfidence(confidence, "medium"); + const outcomeQuality = aggregateOutcomeQuality(decision.repoOutcomePatterns); + if (outcomeQuality.signal === "high_risk") confidence = lowerConfidence(confidence, "low"); + else if (outcomeQuality.signal === "weak") confidence = lowerConfidence(confidence, "medium"); return confidence; } @@ -983,4 +1067,5 @@ export const __agentOrchestratorInternals = { sanitizePublicSummary, jsonPayload, sameRepo, + aggregateOutcomeQuality, }; diff --git a/test/unit/agent-orchestrator.test.ts b/test/unit/agent-orchestrator.test.ts index 63bfbf91c0..aac9f8de58 100644 --- a/test/unit/agent-orchestrator.test.ts +++ b/test/unit/agent-orchestrator.test.ts @@ -12,7 +12,7 @@ import { type AgentRunBundle, } from "../../src/services/agent-orchestrator"; import { buildAgentActionExplanationCard } from "../../src/services/agent-action-explanation-card"; -import { CONTRIBUTOR_DECISION_PACK_SIGNAL, type ContributorDecisionPack } from "../../src/services/decision-pack"; +import { CONTRIBUTOR_DECISION_PACK_SIGNAL, type ContributorDecisionPack, type RepoOutcomeSummary } from "../../src/services/decision-pack"; import { buildPublicAgentCommandComment, parseGittensoryMentionCommand } from "../../src/github/commands"; import { normalizeRegistryPayload } from "../../src/registry/normalize"; import { persistRegistrySnapshot } from "../../src/registry/sync"; @@ -1008,6 +1008,136 @@ describe("agent orchestrator", () => { expect(JSON.stringify(preflight.actions)).toContain("linked_issue_bounty_historical"); expect(JSON.stringify(preflight.actions)).toContain("Source upload disabled"); }); + + it("incorporates aggregate outcome quality into recommendation confidence and evidence", () => { + const run = __agentOrchestratorInternals.buildRunRecord({ + objective: "outcome quality confidence", + actorLogin: "oktofeesh1", + surface: "mcp", + status: "running", + payload: {}, + }); + + const strongPatterns: RepoOutcomeSummary = { + summary: "High merge rate.", + outsideContributorMergeRate: 0.75, + sampleSize: 10, + successPatterns: [{ title: "Focused file changes", detail: "Focused file changes merge well.", confidence: "high" }], + riskPatterns: [], + }; + const highRiskPatterns: RepoOutcomeSummary = { + summary: "Low merge rate.", + outsideContributorMergeRate: 0.20, + sampleSize: 10, + successPatterns: [], + riskPatterns: [{ title: "High closure rate", detail: "Low-quality label mix correlates with closures.", confidence: "high" }], + }; + const weakPatterns: RepoOutcomeSummary = { + summary: "Moderate merge rate.", + outsideContributorMergeRate: 0.45, + sampleSize: 10, + successPatterns: [], + riskPatterns: [], + }; + const sparsePatterns: RepoOutcomeSummary = { + summary: "Sparse data.", + outsideContributorMergeRate: 0.20, + sampleSize: 3, + successPatterns: [], + riskPatterns: [], + }; + + // Fresh, complete-fidelity pack with official stats for a clean baseline + const goodPack = decisionPackFixture({ + freshness: "fresh", + rebuildEnqueued: false, + dataQuality: { + signalFidelity: { + status: "complete", + repoCount: 1, + completeRepos: 1, + degradedRepos: 0, + blockedRepos: 0, + partialRepos: [], + cappedRepos: [], + staleRepos: [], + rateLimitedRepos: [], + }, + } as unknown as ContributorDecisionPack["dataQuality"], + } as unknown as Partial); + + const fakeOutcome = { repoFullName: "owner/repo" } as ContributorDecisionPack["repoDecisions"][number]["outcome"]; + + const strongDecision = repoDecision({ repoFullName: "owner/strong", outcome: fakeOutcome, repoOutcomePatterns: strongPatterns }); + const highRiskDecision = repoDecision({ repoFullName: "owner/high-risk", outcome: fakeOutcome, repoOutcomePatterns: highRiskPatterns }); + const weakDecision = repoDecision({ repoFullName: "owner/weak", outcome: fakeOutcome, repoOutcomePatterns: weakPatterns }); + const sparseDecision = repoDecision({ repoFullName: "owner/sparse", outcome: fakeOutcome, repoOutcomePatterns: sparsePatterns }); + const absentDecision = repoDecision({ repoFullName: "owner/absent", outcome: fakeOutcome }); + + const strongAction = __agentOrchestratorInternals.actionFromDecisionAction(run, action("open_new_direct_pr", "owner/strong", "pursue", 80), strongDecision, 0, goodPack); + const highRiskAction = __agentOrchestratorInternals.actionFromDecisionAction(run, action("open_new_direct_pr", "owner/high-risk", "pursue", 80), highRiskDecision, 1, goodPack); + const weakAction = __agentOrchestratorInternals.actionFromDecisionAction(run, action("open_new_direct_pr", "owner/weak", "pursue", 80), weakDecision, 2, goodPack); + const sparseAction = __agentOrchestratorInternals.actionFromDecisionAction(run, action("open_new_direct_pr", "owner/sparse", "pursue", 80), sparseDecision, 3, goodPack); + const absentAction = __agentOrchestratorInternals.actionFromDecisionAction(run, action("open_new_direct_pr", "owner/absent", "pursue", 80), absentDecision, 4, goodPack); + + // Strong outcome quality (≥60% merge, adequate sample) → confidence stays high + expect(strongAction.payload.recommendationEvidence).toMatchObject({ + confidence: "high", + sources: expect.arrayContaining([ + expect.objectContaining({ name: "aggregate_outcome_quality", freshness: "fresh", source: "cached_repo_patterns" }), + ]), + }); + expect((strongAction.payload.recommendationEvidence as { warnings?: string[] }).warnings ?? []).not.toContain( + expect.stringMatching(/closure risk/i), + ); + + // High-risk outcome quality (≤30% merge, adequate sample) → confidence lowered to "low" + expect(highRiskAction.payload.recommendationEvidence).toMatchObject({ + confidence: "low", + warnings: expect.arrayContaining([expect.stringMatching(/high closure risk/i)]), + sources: expect.arrayContaining([ + expect.objectContaining({ name: "aggregate_outcome_quality", freshness: "fresh", source: "cached_repo_patterns" }), + ]), + }); + + // Weak outcome quality (30–60% merge, adequate sample) → confidence lowered to "medium" + expect(weakAction.payload.recommendationEvidence).toMatchObject({ + confidence: "medium", + warnings: expect.arrayContaining([expect.stringMatching(/moderate closure risk/i)]), + sources: expect.arrayContaining([ + expect.objectContaining({ name: "aggregate_outcome_quality", freshness: "fresh", source: "cached_repo_patterns" }), + ]), + }); + + // Sparse outcome quality (< min sample) → confidence unchanged, assumption added, source degraded + expect(sparseAction.payload.recommendationEvidence).toMatchObject({ + confidence: "high", + assumptions: expect.arrayContaining([expect.stringMatching(/limited sample size/i)]), + sources: expect.arrayContaining([ + expect.objectContaining({ name: "aggregate_outcome_quality", freshness: "degraded", source: "cached_repo_patterns" }), + ]), + }); + + // Absent outcome patterns → assumption added, source is missing + expect(absentAction.payload.recommendationEvidence).toMatchObject({ + confidence: "high", + assumptions: expect.arrayContaining([expect.stringMatching(/no aggregate repo outcome quality/i)]), + sources: expect.arrayContaining([ + expect.objectContaining({ name: "aggregate_outcome_quality", freshness: "missing", source: null }), + ]), + }); + + // Public sanitizer: private aggregate quality must not appear in public-facing card text + expect(JSON.stringify(highRiskAction.explanationCard?.publicSafe)).not.toMatch(/merge rate|closure risk|aggregate outcome/i); + expect(JSON.stringify(highRiskAction.payload.recommendationEvidence)).not.toMatch(/wallet|hotkey|raw trust score/i); + + // aggregateOutcomeQuality helper covers all signal branches directly + expect(__agentOrchestratorInternals.aggregateOutcomeQuality(undefined).signal).toBe("absent"); + expect(__agentOrchestratorInternals.aggregateOutcomeQuality(sparsePatterns).signal).toBe("sparse"); + expect(__agentOrchestratorInternals.aggregateOutcomeQuality(strongPatterns).signal).toBe("strong"); + expect(__agentOrchestratorInternals.aggregateOutcomeQuality(highRiskPatterns).signal).toBe("high_risk"); + expect(__agentOrchestratorInternals.aggregateOutcomeQuality(weakPatterns).signal).toBe("weak"); + }); }); async function persistDecisionPack(env: Env, pack: ContributorDecisionPack): Promise {