diff --git a/apps/gittensory-ui/public/openapi.json b/apps/gittensory-ui/public/openapi.json index 38be3f72a6..7412f68379 100644 --- a/apps/gittensory-ui/public/openapi.json +++ b/apps/gittensory-ui/public/openapi.json @@ -2250,6 +2250,9 @@ } } }, + "recommendationOutcomeFeedback": { + "$ref": "#/components/schemas/AgentRecommendationOutcomeSummary" + }, "evidenceGraph": { "type": "object", "additionalProperties": { @@ -2296,6 +2299,7 @@ "avoidRepos", "maintainerLaneRepos", "scoreBlockers", + "recommendationOutcomeFeedback", "dataQuality", "summary", "nextActions" @@ -2310,6 +2314,201 @@ "missing" ] }, + "AgentRecommendationOutcomeSummary": { + "type": "object", + "properties": { + "login": { + "type": "string" + }, + "generatedAt": { + "type": "string" + }, + "windowDays": { + "type": "number" + }, + "totals": { + "type": "object", + "properties": { + "total": { + "type": "number" + }, + "accepted": { + "type": "number" + }, + "ignored": { + "type": "number" + }, + "stale": { + "type": "number" + }, + "merged": { + "type": "number" + }, + "closed": { + "type": "number" + }, + "improved": { + "type": "number" + }, + "positive": { + "type": "number" + }, + "negative": { + "type": "number" + }, + "maintainerLaneTotal": { + "type": "number" + } + }, + "required": [ + "total", + "accepted", + "ignored", + "stale", + "merged", + "closed", + "improved", + "positive", + "negative", + "maintainerLaneTotal" + ] + }, + "states": { + "type": "array", + "items": { + "$ref": "#/components/schemas/AgentRecommendationOutcomeStateBucket" + } + }, + "repos": { + "type": "array", + "items": { + "$ref": "#/components/schemas/AgentRecommendationOutcomeRepoSummary" + } + }, + "maintainerLane": { + "type": "object", + "properties": { + "total": { + "type": "number" + }, + "states": { + "type": "array", + "items": { + "$ref": "#/components/schemas/AgentRecommendationOutcomeStateBucket" + } + } + }, + "required": [ + "total", + "states" + ] + }, + "privateSummary": { + "type": "string" + } + }, + "required": [ + "login", + "generatedAt", + "windowDays", + "totals", + "states", + "repos", + "maintainerLane", + "privateSummary" + ] + }, + "AgentRecommendationOutcomeStateBucket": { + "type": "object", + "properties": { + "state": { + "$ref": "#/components/schemas/AgentRecommendationOutcomeState" + }, + "count": { + "type": "number" + } + }, + "required": [ + "state", + "count" + ] + }, + "AgentRecommendationOutcomeState": { + "type": "string", + "enum": [ + "accepted", + "ignored", + "stale", + "merged", + "closed", + "improved" + ] + }, + "AgentRecommendationOutcomeRepoSummary": { + "type": "object", + "properties": { + "repoFullName": { + "type": "string" + }, + "total": { + "type": "number" + }, + "accepted": { + "type": "number" + }, + "ignored": { + "type": "number" + }, + "stale": { + "type": "number" + }, + "merged": { + "type": "number" + }, + "closed": { + "type": "number" + }, + "improved": { + "type": "number" + }, + "positive": { + "type": "number" + }, + "negative": { + "type": "number" + }, + "maintainerLaneTotal": { + "type": "number" + }, + "latestOutcomeAt": { + "type": "string", + "nullable": true + }, + "signal": { + "type": "string", + "enum": [ + "positive", + "negative", + "mixed", + "neutral" + ] + } + }, + "required": [ + "repoFullName", + "total", + "accepted", + "ignored", + "stale", + "merged", + "closed", + "improved", + "positive", + "negative", + "maintainerLaneTotal", + "signal" + ] + }, "ContributorOpenPrMonitor": { "type": "object", "properties": { diff --git a/migrations/0017_agent_recommendation_outcomes.sql b/migrations/0017_agent_recommendation_outcomes.sql new file mode 100644 index 0000000000..fb16e73ab1 --- /dev/null +++ b/migrations/0017_agent_recommendation_outcomes.sql @@ -0,0 +1,37 @@ +CREATE TABLE IF NOT EXISTS agent_recommendation_outcomes ( + id TEXT PRIMARY KEY NOT NULL, + action_id TEXT NOT NULL, + run_id TEXT NOT NULL, + actor_login TEXT NOT NULL, + action_type TEXT NOT NULL, + target_repo_full_name TEXT, + target_pull_number INTEGER, + target_issue_number INTEGER, + outcome_state TEXT NOT NULL, + outcome_target_type TEXT NOT NULL, + outcome_repo_full_name TEXT, + outcome_pull_number INTEGER, + outcome_issue_number INTEGER, + maintainer_lane INTEGER NOT NULL DEFAULT 0, + confidence TEXT NOT NULL, + reason TEXT NOT NULL, + source_updated_at TEXT, + detected_at TEXT NOT NULL DEFAULT CURRENT_TIMESTAMP, + metadata_json TEXT NOT NULL DEFAULT '{}', + created_at TEXT NOT NULL DEFAULT CURRENT_TIMESTAMP, + updated_at TEXT NOT NULL DEFAULT CURRENT_TIMESTAMP, + FOREIGN KEY(action_id) REFERENCES agent_actions(id), + FOREIGN KEY(run_id) REFERENCES agent_runs(id) +); + +CREATE UNIQUE INDEX IF NOT EXISTS agent_recommendation_outcomes_action_unique + ON agent_recommendation_outcomes(action_id); + +CREATE INDEX IF NOT EXISTS agent_recommendation_outcomes_actor_state_idx + ON agent_recommendation_outcomes(actor_login, outcome_state, updated_at); + +CREATE INDEX IF NOT EXISTS agent_recommendation_outcomes_target_idx + ON agent_recommendation_outcomes(target_repo_full_name, target_pull_number, target_issue_number); + +CREATE INDEX IF NOT EXISTS agent_recommendation_outcomes_maintainer_idx + ON agent_recommendation_outcomes(actor_login, maintainer_lane, updated_at); diff --git a/src/db/repositories.ts b/src/db/repositories.ts index 94807c0592..caf7ed76f8 100644 --- a/src/db/repositories.ts +++ b/src/db/repositories.ts @@ -5,6 +5,7 @@ import { aiUsageEvents, agentActions, agentContextSnapshots, + agentRecommendationOutcomes, agentRuns, auditEvents, authSessions, @@ -57,6 +58,11 @@ import type { AgentCommandAnswerRecord, AgentCommandFeedbackRecord, AgentContextSnapshotRecord, + AgentRecommendationOutcomeConfidence, + AgentRecommendationOutcomeRecord, + AgentRecommendationOutcomeState, + AgentRecommendationOutcomeSummary, + AgentRecommendationOutcomeTargetType, AgentMode, AgentRunRecord, AgentRunStatus, @@ -1469,6 +1475,73 @@ function maxIso(left: string | null | undefined, right: string | null | undefine return right > left ? right : left; } +function outcomeStateBuckets(outcomes: AgentRecommendationOutcomeRecord[]): AgentRecommendationOutcomeSummary["states"] { + const states: AgentRecommendationOutcomeState[] = ["accepted", "merged", "improved", "closed", "stale", "ignored"]; + return states.flatMap((state) => { + const count = outcomes.filter((outcome) => outcome.outcomeState === state).length; + return count > 0 ? [{ state, count }] : []; + }); +} + +function recommendationOutcomeTotals( + outcomes: AgentRecommendationOutcomeRecord[], + maintainerLaneTotal: number, +): AgentRecommendationOutcomeSummary["totals"] { + const accepted = outcomes.filter((outcome) => outcome.outcomeState === "accepted").length; + const merged = outcomes.filter((outcome) => outcome.outcomeState === "merged").length; + const improved = outcomes.filter((outcome) => outcome.outcomeState === "improved").length; + const closed = outcomes.filter((outcome) => outcome.outcomeState === "closed").length; + const stale = outcomes.filter((outcome) => outcome.outcomeState === "stale").length; + const ignored = outcomes.filter((outcome) => outcome.outcomeState === "ignored").length; + return { + total: outcomes.length, + accepted, + ignored, + stale, + merged, + closed, + improved, + positive: accepted + merged + improved, + negative: closed + stale + ignored, + maintainerLaneTotal, + }; +} + +function summarizeRecommendationOutcomeRepos(outcomes: AgentRecommendationOutcomeRecord[]): AgentRecommendationOutcomeSummary["repos"] { + const byRepo = new Map(); + for (const outcome of outcomes) { + const repoFullName = outcome.outcomeRepoFullName ?? outcome.targetRepoFullName; + if (!repoFullName) continue; + const key = repoFullName.toLowerCase(); + byRepo.set(key, [...(byRepo.get(key) ?? []), outcome]); + } + return [...byRepo.values()] + .map((repoOutcomes) => { + const firstRepo = repoOutcomes[0]!; + const nonMaintainer = repoOutcomes.filter((outcome) => !outcome.maintainerLane); + const totals = recommendationOutcomeTotals(nonMaintainer, repoOutcomes.length - nonMaintainer.length); + const signal: AgentRecommendationOutcomeSummary["repos"][number]["signal"] = + totals.positive > totals.negative ? "positive" : totals.negative > totals.positive ? "negative" : totals.total > 0 ? "mixed" : "neutral"; + return { + repoFullName: firstRepo.outcomeRepoFullName ?? firstRepo.targetRepoFullName ?? "unknown/repo", + total: totals.total, + accepted: totals.accepted, + ignored: totals.ignored, + stale: totals.stale, + merged: totals.merged, + closed: totals.closed, + improved: totals.improved, + positive: totals.positive, + negative: totals.negative, + maintainerLaneTotal: totals.maintainerLaneTotal, + latestOutcomeAt: repoOutcomes.reduce((latest, outcome) => maxIso(latest, outcome.updatedAt ?? outcome.detectedAt), null as string | null), + signal, + }; + }) + .sort((left, right) => right.total - left.total || left.repoFullName.localeCompare(right.repoFullName)) + .slice(0, 20); +} + function finiteNumber(value: unknown): number { return typeof value === "number" && Number.isFinite(value) ? value : 0; } @@ -2340,6 +2413,117 @@ export async function listAgentContextSnapshots(env: Env, runId: string): Promis return rows.map(toAgentContextSnapshotRecord); } +export async function upsertAgentRecommendationOutcome(env: Env, outcome: AgentRecommendationOutcomeRecord): Promise { + const now = outcome.updatedAt ?? nowIso(); + const values = { + id: outcome.id ?? `outcome:${outcome.actionId}`, + actionId: outcome.actionId, + runId: outcome.runId, + actorLogin: boundedString(outcome.actorLogin, 100), + actionType: outcome.actionType, + targetRepoFullName: outcome.targetRepoFullName ? boundedString(outcome.targetRepoFullName, 200) : null, + targetPullNumber: outcome.targetPullNumber ?? null, + targetIssueNumber: outcome.targetIssueNumber ?? null, + outcomeState: outcome.outcomeState, + outcomeTargetType: outcome.outcomeTargetType, + outcomeRepoFullName: outcome.outcomeRepoFullName ? boundedString(outcome.outcomeRepoFullName, 200) : null, + outcomePullNumber: outcome.outcomePullNumber ?? null, + outcomeIssueNumber: outcome.outcomeIssueNumber ?? null, + maintainerLane: outcome.maintainerLane, + confidence: outcome.confidence, + reason: boundedString(outcome.reason, 500), + sourceUpdatedAt: outcome.sourceUpdatedAt ?? null, + detectedAt: outcome.detectedAt ?? now, + metadataJson: jsonString(outcome.metadata ?? {}), + createdAt: outcome.createdAt ?? now, + updatedAt: now, + }; + await getDb(env.DB) + .insert(agentRecommendationOutcomes) + .values(values) + .onConflictDoUpdate({ + target: agentRecommendationOutcomes.actionId, + set: { + actorLogin: values.actorLogin, + actionType: values.actionType, + targetRepoFullName: values.targetRepoFullName, + targetPullNumber: values.targetPullNumber, + targetIssueNumber: values.targetIssueNumber, + outcomeState: values.outcomeState, + outcomeTargetType: values.outcomeTargetType, + outcomeRepoFullName: values.outcomeRepoFullName, + outcomePullNumber: values.outcomePullNumber, + outcomeIssueNumber: values.outcomeIssueNumber, + maintainerLane: values.maintainerLane, + confidence: values.confidence, + reason: values.reason, + sourceUpdatedAt: values.sourceUpdatedAt, + detectedAt: values.detectedAt, + metadataJson: values.metadataJson, + updatedAt: values.updatedAt, + }, + }); + return (await getAgentRecommendationOutcome(env, outcome.actionId))!; +} + +export async function getAgentRecommendationOutcome(env: Env, actionId: string): Promise { + const [row] = await getDb(env.DB).select().from(agentRecommendationOutcomes).where(eq(agentRecommendationOutcomes.actionId, actionId)).limit(1); + return row ? toAgentRecommendationOutcomeRecord(row) : null; +} + +export async function listAgentRecommendationOutcomes( + env: Env, + options: { actorLogin?: string; windowDays?: number; now?: string; limit?: number } = {}, +): Promise { + const limit = clampInteger(options.limit ?? 500, 1, 5000); + const conditions = []; + if (options.actorLogin) conditions.push(eq(agentRecommendationOutcomes.actorLogin, options.actorLogin)); + if (options.windowDays !== undefined) { + const windowDays = clampInteger(options.windowDays, 1, 365); + const now = options.now ?? nowIso(); + conditions.push(gte(agentRecommendationOutcomes.updatedAt, new Date(Date.parse(now) - windowDays * 24 * 60 * 60 * 1000).toISOString())); + } + const rows = await getDb(env.DB) + .select() + .from(agentRecommendationOutcomes) + .where(conditions.length === 0 ? undefined : and(...conditions)) + .orderBy(desc(agentRecommendationOutcomes.updatedAt), agentRecommendationOutcomes.actionId) + .limit(limit); + return rows.map(toAgentRecommendationOutcomeRecord); +} + +export async function getAgentRecommendationOutcomeSummary( + env: Env, + actorLogin: string, + options: { windowDays?: number; now?: string } = {}, +): Promise { + const windowDays = clampInteger(options.windowDays ?? 90, 1, 365); + const generatedAt = options.now ?? nowIso(); + const outcomes = await listAgentRecommendationOutcomes(env, { actorLogin, windowDays, now: generatedAt }); + const nonMaintainer = outcomes.filter((outcome) => !outcome.maintainerLane); + const maintainer = outcomes.filter((outcome) => outcome.maintainerLane); + const states = outcomeStateBuckets(nonMaintainer); + const maintainerStates = outcomeStateBuckets(maintainer); + const repos = summarizeRecommendationOutcomeRepos(outcomes); + const totals = recommendationOutcomeTotals(nonMaintainer, maintainer.length); + return { + login: actorLogin, + generatedAt, + windowDays, + totals, + states, + repos, + maintainerLane: { + total: maintainer.length, + states: maintainerStates, + }, + privateSummary: + outcomes.length === 0 + ? `${actorLogin} has no evaluated recommendation outcomes in the last ${windowDays} day(s).` + : `${actorLogin} has ${nonMaintainer.length} contributor-lane recommendation outcome(s), ${totals.positive} positive and ${totals.negative} negative, plus ${maintainer.length} maintainer-lane outcome(s) kept separate.`, + }; +} + export async function upsertInstallationHealth(env: Env, health: InstallationHealthRecord): Promise { const db = getDb(env.DB); await db @@ -3005,6 +3189,32 @@ function toAgentContextSnapshotRecord(row: typeof agentContextSnapshots.$inferSe }; } +function toAgentRecommendationOutcomeRecord(row: typeof agentRecommendationOutcomes.$inferSelect): AgentRecommendationOutcomeRecord { + return { + id: row.id, + actionId: row.actionId, + runId: row.runId, + actorLogin: row.actorLogin, + actionType: parseAgentActionType(row.actionType), + targetRepoFullName: row.targetRepoFullName, + targetPullNumber: row.targetPullNumber, + targetIssueNumber: row.targetIssueNumber, + outcomeState: parseAgentRecommendationOutcomeState(row.outcomeState), + outcomeTargetType: parseAgentRecommendationOutcomeTargetType(row.outcomeTargetType), + outcomeRepoFullName: row.outcomeRepoFullName, + outcomePullNumber: row.outcomePullNumber, + outcomeIssueNumber: row.outcomeIssueNumber, + maintainerLane: row.maintainerLane, + confidence: parseAgentRecommendationOutcomeConfidence(row.confidence), + reason: row.reason, + sourceUpdatedAt: row.sourceUpdatedAt, + detectedAt: row.detectedAt, + metadata: parseJson>(row.metadataJson, {}), + createdAt: row.createdAt, + updatedAt: row.updatedAt, + }; +} + function toInstallationHealthRecord(row: typeof installationHealth.$inferSelect): InstallationHealthRecord { return { installationId: row.installationId, @@ -3779,6 +3989,21 @@ function parseAgentSafetyClass(value: string): AgentSafetyClass { return "private"; } +function parseAgentRecommendationOutcomeState(value: string): AgentRecommendationOutcomeState { + if (value === "accepted" || value === "ignored" || value === "stale" || value === "merged" || value === "closed" || value === "improved") return value; + return "ignored"; +} + +function parseAgentRecommendationOutcomeTargetType(value: string): AgentRecommendationOutcomeTargetType { + if (value === "pull_request" || value === "issue" || value === "repository") return value; + return "none"; +} + +function parseAgentRecommendationOutcomeConfidence(value: string): AgentRecommendationOutcomeConfidence { + if (value === "high" || value === "low") return value; + return "medium"; +} + function parseCommentMode(value: string): RepositorySettings["commentMode"] { if (value === "detected_contributors_only" || value === "all_prs") return value; return "off"; diff --git a/src/db/schema.ts b/src/db/schema.ts index 3b041dce63..f1502e5bd6 100644 --- a/src/db/schema.ts +++ b/src/db/schema.ts @@ -473,6 +473,39 @@ export const agentContextSnapshots = sqliteTable( }), ); +export const agentRecommendationOutcomes = sqliteTable( + "agent_recommendation_outcomes", + { + id: text("id").primaryKey(), + actionId: text("action_id").notNull(), + runId: text("run_id").notNull(), + actorLogin: text("actor_login").notNull(), + actionType: text("action_type").notNull(), + targetRepoFullName: text("target_repo_full_name"), + targetPullNumber: integer("target_pull_number"), + targetIssueNumber: integer("target_issue_number"), + outcomeState: text("outcome_state").notNull(), + outcomeTargetType: text("outcome_target_type").notNull(), + outcomeRepoFullName: text("outcome_repo_full_name"), + outcomePullNumber: integer("outcome_pull_number"), + outcomeIssueNumber: integer("outcome_issue_number"), + maintainerLane: integer("maintainer_lane", { mode: "boolean" }).notNull().default(false), + confidence: text("confidence").notNull(), + reason: text("reason").notNull(), + sourceUpdatedAt: text("source_updated_at"), + detectedAt: text("detected_at").notNull().default("CURRENT_TIMESTAMP"), + metadataJson: text("metadata_json").notNull().default("{}"), + createdAt: text("created_at").notNull().default("CURRENT_TIMESTAMP"), + updatedAt: text("updated_at").notNull().default("CURRENT_TIMESTAMP"), + }, + (table) => ({ + action: uniqueIndex("agent_recommendation_outcomes_action_unique").on(table.actionId), + actorState: index("agent_recommendation_outcomes_actor_state_idx").on(table.actorLogin, table.outcomeState, table.updatedAt), + target: index("agent_recommendation_outcomes_target_idx").on(table.targetRepoFullName, table.targetPullNumber, table.targetIssueNumber), + maintainer: index("agent_recommendation_outcomes_maintainer_idx").on(table.actorLogin, table.maintainerLane, table.updatedAt), + }), +); + export const installationHealth = sqliteTable("installation_health", { installationId: integer("installation_id").primaryKey(), accountLogin: text("account_login").notNull(), diff --git a/src/openapi/schemas.ts b/src/openapi/schemas.ts index ba6a736291..f9695ff9cd 100644 --- a/src/openapi/schemas.ts +++ b/src/openapi/schemas.ts @@ -1470,6 +1470,60 @@ export const ContributorStrategySchema = z export const DecisionPackFreshnessSchema = z.enum(["fresh", "stale", "rebuilding", "missing"]).openapi("DecisionPackFreshness"); +export const AgentRecommendationOutcomeStateSchema = z.enum(["accepted", "ignored", "stale", "merged", "closed", "improved"]).openapi("AgentRecommendationOutcomeState"); + +export const AgentRecommendationOutcomeStateBucketSchema = z + .object({ + state: AgentRecommendationOutcomeStateSchema, + count: z.number(), + }) + .openapi("AgentRecommendationOutcomeStateBucket"); + +export const AgentRecommendationOutcomeRepoSummarySchema = z + .object({ + repoFullName: z.string(), + total: z.number(), + accepted: z.number(), + ignored: z.number(), + stale: z.number(), + merged: z.number(), + closed: z.number(), + improved: z.number(), + positive: z.number(), + negative: z.number(), + maintainerLaneTotal: z.number(), + latestOutcomeAt: z.string().nullable().optional(), + signal: z.enum(["positive", "negative", "mixed", "neutral"]), + }) + .openapi("AgentRecommendationOutcomeRepoSummary"); + +export const AgentRecommendationOutcomeSummarySchema = z + .object({ + login: z.string(), + generatedAt: z.string(), + windowDays: z.number(), + totals: z.object({ + total: z.number(), + accepted: z.number(), + ignored: z.number(), + stale: z.number(), + merged: z.number(), + closed: z.number(), + improved: z.number(), + positive: z.number(), + negative: z.number(), + maintainerLaneTotal: z.number(), + }), + states: z.array(AgentRecommendationOutcomeStateBucketSchema), + repos: z.array(AgentRecommendationOutcomeRepoSummarySchema), + maintainerLane: z.object({ + total: z.number(), + states: z.array(AgentRecommendationOutcomeStateBucketSchema), + }), + privateSummary: z.string(), + }) + .openapi("AgentRecommendationOutcomeSummary"); + export const DecisionRecommendationSchema = z.enum(["pursue", "cleanup_first", "maintainer_lane", "avoid_for_now", "watch"]).openapi("DecisionRecommendation"); export const DecisionActionKindSchema = z @@ -1550,6 +1604,7 @@ export const ContributorDecisionPackSchema = z avoidRepos: z.array(z.record(z.string(), z.unknown())), maintainerLaneRepos: z.array(z.record(z.string(), z.unknown())), scoreBlockers: z.array(z.record(z.string(), z.unknown())), + recommendationOutcomeFeedback: AgentRecommendationOutcomeSummarySchema, evidenceGraph: z.record(z.string(), z.unknown()).optional(), dataQuality: z.record(z.string(), z.unknown()), summary: z.string(), diff --git a/src/services/contributor-evidence-graph.ts b/src/services/contributor-evidence-graph.ts index 21c002158c..75346f992b 100644 --- a/src/services/contributor-evidence-graph.ts +++ b/src/services/contributor-evidence-graph.ts @@ -209,7 +209,7 @@ export function buildContributorEvidenceGraph(args: ContributorEvidenceGraphInpu const key = repoFullName.toLowerCase(); if (!repoNamesByKey.has(key)) repoNamesByKey.set(key, repoFullName); }; - for (const repoFullName of args.profile.registeredRepoActivity.reposTouched) addRepo(repoFullName); + for (const repoFullName of args.profile.registeredRepoActivity?.reposTouched ?? []) addRepo(repoFullName); for (const stat of repoStatsByRepo.values()) addRepo(stat.repoFullName); for (const pr of contributorPullRequests) addRepo(pr.repoFullName); for (const issue of contributorIssues) addRepo(issue.repoFullName); @@ -310,7 +310,7 @@ export function evidenceGraphTouchedRepoFullNames(args: { const key = repoFullName.toLowerCase(); if (!namesByKey.has(key)) namesByKey.set(key, repoFullName); }; - for (const repoFullName of args.profile?.registeredRepoActivity.reposTouched ?? []) add(repoFullName); + for (const repoFullName of args.profile?.registeredRepoActivity?.reposTouched ?? []) add(repoFullName); for (const repo of args.profile?.gittensor?.repositories ?? []) add(repo.repoFullName); for (const stat of args.repoStats ?? []) if (sameLogin(stat.login, args.login)) add(stat.repoFullName); for (const pr of args.pullRequests ?? []) if (sameLogin(pr.authorLogin, args.login)) add(pr.repoFullName); @@ -370,8 +370,7 @@ function preferredLabelEdges(buckets: LabelBucket[], generatedAt: string): Contr (left, right) => right.pullRequests + right.issues - (left.pullRequests + left.issues) || left.repoFullName.localeCompare(right.repoFullName) || - left.label.localeCompare(right.label) || - SOURCE_PRIORITY[left.source] - SOURCE_PRIORITY[right.source], + left.label.localeCompare(right.label), ); } diff --git a/src/services/decision-pack.ts b/src/services/decision-pack.ts index 515c1e20b0..4456fdc0ee 100644 --- a/src/services/decision-pack.ts +++ b/src/services/decision-pack.ts @@ -17,6 +17,7 @@ import { listSignalSnapshots, persistSignalSnapshot, recordAuditEvent, + getAgentRecommendationOutcomeSummary, upsertContributorEvidence, upsertContributorScoringProfile, } from "../db/repositories"; @@ -48,8 +49,11 @@ import { } from "./contributor-evidence-graph"; import { loadIssueQualityReportMap } from "./issue-quality"; import { loadRepoOutcomePatternsMap } from "./repo-outcome-patterns"; +import { evaluateRecommendationOutcomes } from "./recommendation-outcomes"; import type { BountyRecord, + AgentRecommendationOutcomeRepoSummary, + AgentRecommendationOutcomeSummary, ContributorRepoStatRecord, IssueRecord, JsonValue, @@ -103,6 +107,7 @@ export type ContributorDecisionPack = { avoidRepos: RepoDecision[]; maintainerLaneRepos: RepoDecision[]; scoreBlockers: ScoreBlocker[]; + recommendationOutcomeFeedback: AgentRecommendationOutcomeSummary; evidenceGraph?: ContributorEvidenceGraph | undefined; dataQuality: { signalFidelity: ReturnType; @@ -172,6 +177,7 @@ export type RepoDecision = { labelFit: string[]; scoreBlockers: ScoreBlocker[]; repoOutcomePatterns?: RepoOutcomeSummary | undefined; + recommendationOutcomeFeedback?: RepoRecommendationOutcomeFeedback | undefined; riskReasons: string[]; whyThisHelps: string[]; nextActions: string[]; @@ -200,6 +206,20 @@ export type RepoOutcomeSummary = { riskPatterns: OutcomePattern[]; }; +export type RepoRecommendationOutcomeFeedback = { + signal: AgentRecommendationOutcomeRepoSummary["signal"]; + total: number; + positive: number; + negative: number; + merged: number; + closed: number; + stale: number; + ignored: number; + improved: number; + maintainerLaneTotal: number; + latestOutcomeAt?: string | null | undefined; +}; + export type DecisionAction = { actionKind: DecisionActionKind; repoFullName: string; @@ -391,9 +411,11 @@ export async function buildAndPersistContributorDecisionPack(env: Env, login: st fetchGittensorContributorSnapshot(login), ]); const repoStats = authoritativeContributorRepoStats(gittensorSnapshot, cachedRepoStats); - const [issueQualityByRepo, repoOutcomePatternsByRepo] = await Promise.all([ + await evaluateRecommendationOutcomes(env, login); + const [issueQualityByRepo, repoOutcomePatternsByRepo, recommendationOutcomeFeedback] = await Promise.all([ loadIssueQualityReportMap(env, repositories), loadRepoOutcomePatternsMap(env, repositories), + getAgentRecommendationOutcomeSummary(env, login), ]); const focusManifests = await loadRepoFocusManifests( env, @@ -443,6 +465,7 @@ export async function buildAndPersistContributorDecisionPack(env: Env, login: st openPrMonitor, focusManifests, repoOutcomePatternsByRepo, + recommendationOutcomeFeedback, }); await upsertContributorEvidence(env, { @@ -509,7 +532,9 @@ function buildContributorDecisionPack(args: { openPrMonitor: ContributorOpenPrMonitor; focusManifests?: Map | undefined; repoOutcomePatternsByRepo?: Map | undefined; + recommendationOutcomeFeedback?: AgentRecommendationOutcomeSummary | undefined; }): ContributorDecisionPack { + const recommendationOutcomeFeedback = args.recommendationOutcomeFeedback ?? emptyRecommendationOutcomeFeedback(args.login); const registeredRepositories = args.repositories.filter((repo) => repo.isRegistered); const syncByRepo = new Map(args.syncStates.map((state) => [state.repoFullName.toLowerCase(), state])); const totalsByRepo = new Map(args.totals.map((total) => [total.repoFullName.toLowerCase(), total])); @@ -530,6 +555,7 @@ function buildContributorDecisionPack(args: { }), ); const roleByRepo = new Map(roleContexts.map((role) => [role.repoFullName.toLowerCase(), role])); + const recommendationFeedbackByRepo = new Map(recommendationOutcomeFeedback.repos.map((repo) => [repo.repoFullName.toLowerCase(), repo])); const repoDecisions = registeredRepositories .map((repo) => { const key = repo.fullName.toLowerCase(); @@ -544,6 +570,7 @@ function buildContributorDecisionPack(args: { issueQuality: issueQualityByRepo.get(key), focusManifest: args.focusManifests?.get(key), repoOutcomePatterns: args.repoOutcomePatternsByRepo?.get(key), + recommendationOutcomeFeedback: recommendationFeedbackByRepo.get(key), }); }) .sort((left, right) => right.priorityScore - left.priorityScore || left.repoFullName.localeCompare(right.repoFullName)); @@ -604,9 +631,10 @@ function buildContributorDecisionPack(args: { avoidRepos: repoDecisions.filter((decision) => decision.recommendation === "avoid_for_now").slice(0, 8), maintainerLaneRepos: repoDecisions.filter((decision) => decision.recommendation === "maintainer_lane").slice(0, 8), scoreBlockers, + recommendationOutcomeFeedback, evidenceGraph, dataQuality, - summary: `${args.login} has ${topActions.length} ranked action(s), ${scoreBlockers.length} scoreability blocker(s), and ${repoDecisions.length} registered repo decision(s).${monitorSummary}`, + summary: `${args.login} has ${topActions.length} ranked action(s), ${scoreBlockers.length} scoreability blocker(s), and ${repoDecisions.length} registered repo decision(s).${monitorSummary}${recommendationFeedbackSummary(recommendationOutcomeFeedback)}`, nextActions: packNextActions, openPrMonitor: monitor, }; @@ -623,6 +651,7 @@ function buildRepoDecision(args: { issueQuality?: IssueQualityReport | undefined; focusManifest?: FocusManifest | undefined; repoOutcomePatterns?: RepoOutcomePatterns | undefined; + recommendationOutcomeFeedback?: AgentRecommendationOutcomeRepoSummary | undefined; }): RepoDecision { const lane = buildLaneAdvice(args.repo, args.repo.fullName); const config = args.repo.registryConfig; @@ -642,6 +671,7 @@ function buildRepoDecision(args: { }; const blockers = scoreBlockersFor(args.repo.fullName, lane.lane, args.roleContext, args.outcome); const issueQuality = summarizeIssueQuality(args.issueQuality); + const recommendationFeedback = summarizeRecommendationOutcomeFeedback(args.recommendationOutcomeFeedback); const riskReasons = [ ...(queue.openPullRequests >= 25 ? [`Repo queue is busy with ${queue.openPullRequests} open PR(s).`] : []), ...(queue.openIssues >= 100 ? [`Repo issue queue is large with ${queue.openIssues} open issue(s).`] : []), @@ -652,7 +682,7 @@ function buildRepoDecision(args: { ...(issueQuality && issueQuality.readyCount === 0 && (lane.lane === "issue_discovery" || lane.lane === "split") ? ["No ready issue-quality candidate is cached for this repo."] : []), ]; const recommendation = recommendationFor(lane.lane, args.roleContext, args.outcome, blockers); - const priorityScore = clamp(priorityFor(recommendation, rewardUpside, args.outcome, queue, blockers) + issueQualityPriorityAdjustment(lane.lane, issueQuality), 0, 100); + const priorityScore = clamp(priorityFor(recommendation, rewardUpside, args.outcome, queue, blockers) + issueQualityPriorityAdjustment(lane.lane, issueQuality) + recommendationOutcomePriorityAdjustment(recommendationFeedback), 0, 100); const syncLanguage = args.syncState?.primaryLanguage ?? null; const languageMatch: LanguageMatch = { language: syncLanguage, @@ -678,6 +708,8 @@ function buildRepoDecision(args: { const repoOutcomePatterns = summarizeRepoOutcomePatterns(args.repoOutcomePatterns); const outcomeRiskLines = args.roleContext.maintainerLane ? [] : (repoOutcomePatterns?.riskPatterns ?? []).slice(0, 2).map((pattern) => pattern.detail); const outcomeSuccessLines = recommendation === "pursue" ? (repoOutcomePatterns?.successPatterns ?? []).slice(0, 1).map((pattern) => pattern.detail) : []; + const recommendationFeedbackRiskLines = args.roleContext.maintainerLane ? [] : recommendationFeedbackRiskReasons(recommendationFeedback); + const recommendationFeedbackSuccessLines = recommendationFeedbackWhyThisHelps(recommendationFeedback); const tradeoffSummary = buildRepoDecisionTradeoffSummary({ repoFullName: args.repo.fullName, lane: lane.lane, @@ -701,8 +733,9 @@ function buildRepoDecision(args: { labelFit, scoreBlockers: blockers, repoOutcomePatterns, - riskReasons: [...new Set([...riskReasons, ...manifestReasons.riskReasons, ...outcomeRiskLines])], - whyThisHelps: [...new Set([...whyThisHelpsFor(recommendation, copyContext), ...manifestReasons.whyThisHelps, ...outcomeSuccessLines])], + recommendationOutcomeFeedback: recommendationFeedback, + riskReasons: [...new Set([...riskReasons, ...manifestReasons.riskReasons, ...outcomeRiskLines, ...recommendationFeedbackRiskLines])], + whyThisHelps: [...new Set([...whyThisHelpsFor(recommendation, copyContext), ...manifestReasons.whyThisHelps, ...outcomeSuccessLines, ...recommendationFeedbackSuccessLines])], nextActions: [...new Set([...nextActionsFor(recommendation, copyContext), ...manifestReasons.nextActions])], publicNextActions: [...new Set([...publicNextActionsFor(recommendation, copyContext), ...manifestReasons.publicNextActions])], issueQuality, @@ -942,6 +975,70 @@ function summarizeRepoOutcomePatterns(patterns: RepoOutcomePatterns | undefined) }; } +function summarizeRecommendationOutcomeFeedback(feedback: AgentRecommendationOutcomeRepoSummary | undefined): RepoRecommendationOutcomeFeedback | undefined { + if (!feedback || feedback.total === 0) return undefined; + return { + signal: feedback.signal, + total: feedback.total, + positive: feedback.positive, + negative: feedback.negative, + merged: feedback.merged, + closed: feedback.closed, + stale: feedback.stale, + ignored: feedback.ignored, + improved: feedback.improved, + maintainerLaneTotal: feedback.maintainerLaneTotal, + latestOutcomeAt: feedback.latestOutcomeAt, + }; +} + +function recommendationFeedbackWhyThisHelps(feedback: RepoRecommendationOutcomeFeedback | undefined): string[] { + if (!feedback || feedback.positive === 0) return []; + return [`Private recommendation feedback has ${feedback.positive} positive contributor-lane outcome(s) for this repo (${feedback.merged} merged, ${feedback.improved} improved, ${feedback.total - feedback.negative - feedback.merged - feedback.improved} accepted).`]; +} + +function recommendationFeedbackRiskReasons(feedback: RepoRecommendationOutcomeFeedback | undefined): string[] { + if (!feedback || feedback.negative === 0) return []; + return [`Private recommendation feedback has ${feedback.negative} unresolved or negative contributor-lane outcome(s) for this repo (${feedback.closed} closed, ${feedback.stale} stale, ${feedback.ignored} ignored).`]; +} + +function recommendationOutcomePriorityAdjustment(feedback: RepoRecommendationOutcomeFeedback | undefined): number { + if (!feedback) return 0; + if (feedback.signal === "positive") return Math.min(8, feedback.positive * 2); + if (feedback.signal === "negative") return -Math.min(12, feedback.negative * 2); + if (feedback.signal === "mixed") return -Math.min(4, feedback.negative); + return 0; +} + +function recommendationFeedbackSummary(feedback: AgentRecommendationOutcomeSummary): string { + if (feedback.totals.total === 0 && feedback.totals.maintainerLaneTotal === 0) return ""; + return ` Recommendation feedback: ${feedback.totals.positive} positive, ${feedback.totals.negative} negative, ${feedback.totals.maintainerLaneTotal} maintainer-lane separated.`; +} + +function emptyRecommendationOutcomeFeedback(login: string): AgentRecommendationOutcomeSummary { + return { + login, + generatedAt: nowIso(), + windowDays: 90, + totals: { + total: 0, + accepted: 0, + ignored: 0, + stale: 0, + merged: 0, + closed: 0, + improved: 0, + positive: 0, + negative: 0, + maintainerLaneTotal: 0, + }, + states: [], + repos: [], + maintainerLane: { total: 0, states: [] }, + privateSummary: `${login} has no evaluated recommendation outcomes in the last 90 day(s).`, + }; +} + function scoreBlockersFor(repoFullName: string, lane: string, roleContext: RoleContext, outcome: ContributorOutcomeHistory["repoOutcomes"][number] | undefined): ScoreBlocker[] { const blockers: ScoreBlocker[] = []; const openPullRequests = outcome?.openPullRequests ?? 0; @@ -1382,6 +1479,7 @@ function sanitizeOfficialStats(profile: ContributorProfile): ContributorDecision function withSnapshotMetadata(snapshot: SignalSnapshotRecord): ContributorDecisionPack { const payload = snapshot.payload as unknown as ContributorDecisionPack; + const login = payload.login ?? snapshot.targetKey; const generatedAt = snapshot.generatedAt ?? payload.generatedAt ?? nowIso(); const ageSeconds = Math.max(0, Math.floor(snapshotAgeMs(generatedAt) / 1000)); const stale = snapshotAgeMs(generatedAt) > DECISION_PACK_MAX_AGE_MS; @@ -1397,6 +1495,7 @@ function withSnapshotMetadata(snapshot: SignalSnapshotRecord): ContributorDecisi ...payload, status: "ready", source: "snapshot", + login, generatedAt, snapshotAgeSeconds: ageSeconds, stale, @@ -1404,6 +1503,7 @@ function withSnapshotMetadata(snapshot: SignalSnapshotRecord): ContributorDecisi rebuildEnqueued: false, opportunities: payload.opportunities ?? [], actionPortfolio, + recommendationOutcomeFeedback: payload.recommendationOutcomeFeedback ?? emptyRecommendationOutcomeFeedback(login), }; } diff --git a/src/services/recommendation-outcomes.ts b/src/services/recommendation-outcomes.ts new file mode 100644 index 0000000000..c31e15b844 --- /dev/null +++ b/src/services/recommendation-outcomes.ts @@ -0,0 +1,345 @@ +import { + listAgentActions, + listAgentRunsForActor, + listContributorIssues, + listContributorPullRequests, + upsertAgentRecommendationOutcome, +} from "../db/repositories"; +import type { + AgentActionRecord, + AgentRecommendationOutcomeRecord, + AgentRecommendationOutcomeState, + AgentRecommendationOutcomeTargetType, + AgentRunRecord, + IssueRecord, + PullRequestRecord, +} from "../types"; +import { nowIso } from "../utils/json"; + +export const DEFAULT_RECOMMENDATION_OUTCOME_STALE_DAYS = 14; +export const DEFAULT_RECOMMENDATION_OUTCOME_IGNORED_DAYS = 7; + +export type RecommendationOutcomeEvaluationResult = { + login: string; + evaluatedAt: string; + outcomes: AgentRecommendationOutcomeRecord[]; + skippedFreshActions: number; +}; + +export async function evaluateRecommendationOutcomes( + env: Env, + login: string, + options: { now?: string; runLimit?: number; staleAfterDays?: number; ignoredAfterDays?: number } = {}, +): Promise { + const evaluatedAt = options.now ?? nowIso(); + const staleAfterMs = daysToMs(options.staleAfterDays ?? DEFAULT_RECOMMENDATION_OUTCOME_STALE_DAYS); + const ignoredAfterMs = daysToMs(options.ignoredAfterDays ?? DEFAULT_RECOMMENDATION_OUTCOME_IGNORED_DAYS); + const [runs, pullRequests, issues] = await Promise.all([ + listAgentRunsForActor(env, login, options.runLimit ?? 50), + listContributorPullRequests(env, login), + listContributorIssues(env, login), + ]); + const completedRuns = runs.filter((run) => run.status === "completed"); + const actionGroups = await Promise.all(completedRuns.map(async (run) => ({ run, actions: await listAgentActions(env, run.id) }))); + const classifications = actionGroups.flatMap(({ run, actions }) => + actions.map((action) => classifyRecommendationOutcome({ run, action, pullRequests, issues, evaluatedAt, staleAfterMs, ignoredAfterMs })), + ); + const classified = classifications.filter((outcome): outcome is AgentRecommendationOutcomeRecord => outcome !== null); + const outcomes = []; + for (const outcome of classified) outcomes.push(await upsertAgentRecommendationOutcome(env, outcome)); + return { + login, + evaluatedAt, + outcomes, + skippedFreshActions: classifications.length - classified.length, + }; +} + +export function classifyRecommendationOutcome(args: { + run: AgentRunRecord; + action: AgentActionRecord; + pullRequests: PullRequestRecord[]; + issues: IssueRecord[]; + evaluatedAt: string; + staleAfterMs: number; + ignoredAfterMs: number; +}): AgentRecommendationOutcomeRecord | null { + const actionAt = timestamp(args.action.createdAt ?? args.run.updatedAt ?? args.run.createdAt); + const evaluatedAt = timestamp(args.evaluatedAt); + if (!Number.isFinite(actionAt) || !Number.isFinite(evaluatedAt)) return null; + const actionAgeMs = evaluatedAt - actionAt; + const targetRepoFullName = args.action.targetRepoFullName ?? repoFromPayload(args.action); + const exactPr = targetRepoFullName && args.action.targetPullNumber + ? args.pullRequests.find((pr) => sameRepo(pr.repoFullName, targetRepoFullName) && pr.number === args.action.targetPullNumber) + : undefined; + if (exactPr) { + return outcomeFromPullRequest({ + run: args.run, + action: args.action, + pr: exactPr, + matchedBy: "target_pull_request", + evaluatedAt: args.evaluatedAt, + actionAt, + actionAgeMs, + staleAfterMs: args.staleAfterMs, + }); + } + + const exactIssue = targetRepoFullName && args.action.targetIssueNumber + ? args.issues.find((issue) => sameRepo(issue.repoFullName, targetRepoFullName) && issue.number === args.action.targetIssueNumber) + : undefined; + if (exactIssue) { + return outcomeFromIssue({ + run: args.run, + action: args.action, + issue: exactIssue, + matchedBy: "target_issue", + evaluatedAt: args.evaluatedAt, + actionAt, + actionAgeMs, + staleAfterMs: args.staleAfterMs, + }); + } + + const laterPr = targetRepoFullName ? firstLaterPullRequest(args.pullRequests, args.run.actorLogin, targetRepoFullName, args.action.targetIssueNumber, actionAt) : undefined; + if (laterPr) { + return outcomeFromPullRequest({ + run: args.run, + action: args.action, + pr: laterPr, + matchedBy: args.action.targetIssueNumber ? "linked_issue_pull_request" : "later_repo_pull_request", + evaluatedAt: args.evaluatedAt, + actionAt, + actionAgeMs, + staleAfterMs: args.staleAfterMs, + }); + } + + const laterIssue = targetRepoFullName ? firstLaterIssue(args.issues, args.run.actorLogin, targetRepoFullName, actionAt) : undefined; + if (laterIssue) { + return outcomeFromIssue({ + run: args.run, + action: args.action, + issue: laterIssue, + matchedBy: "later_repo_issue", + evaluatedAt: args.evaluatedAt, + actionAt, + actionAgeMs, + staleAfterMs: args.staleAfterMs, + }); + } + + if (actionAgeMs < args.ignoredAfterMs) return null; + return baseOutcome(args.run, args.action, { + outcomeState: "ignored", + outcomeTargetType: targetRepoFullName ? "repository" : "none", + outcomeRepoFullName: targetRepoFullName ?? null, + maintainerLane: targetRepoFullName ? isMaintainerLane(args.run.actorLogin, targetRepoFullName) : false, + confidence: targetRepoFullName ? "medium" : "low", + reason: targetRepoFullName + ? "No later cached PR or issue activity matched this recommendation after the ignored-outcome window." + : "Recommendation did not include a target repo, PR, or issue that can be deterministically matched.", + detectedAt: args.evaluatedAt, + metadata: { matchedBy: "no_cached_activity", actionAgeDays: Math.floor(actionAgeMs / daysToMs(1)) }, + }); +} + +function outcomeFromPullRequest(args: { + run: AgentRunRecord; + action: AgentActionRecord; + pr: PullRequestRecord; + matchedBy: string; + evaluatedAt: string; + actionAt: number; + actionAgeMs: number; + staleAfterMs: number; +}): AgentRecommendationOutcomeRecord { + const state = pullRequestOutcomeState(args.pr, args.action, args.actionAt, args.actionAgeMs, args.staleAfterMs); + const maintainerLane = isMaintainerLane(args.run.actorLogin, args.pr.repoFullName, args.pr.authorAssociation); + return baseOutcome(args.run, args.action, { + outcomeState: state, + outcomeTargetType: "pull_request", + outcomeRepoFullName: args.pr.repoFullName, + outcomePullNumber: args.pr.number, + maintainerLane, + confidence: state === "ignored" ? "medium" : "high", + reason: pullRequestOutcomeReason(state, args.pr), + sourceUpdatedAt: args.pr.updatedAt ?? args.pr.createdAt, + detectedAt: args.evaluatedAt, + metadata: { + matchedBy: args.matchedBy, + pullRequestState: args.pr.state, + reviewDecision: args.pr.reviewDecision ?? null, + mergeableState: args.pr.mergeableState ?? null, + }, + }); +} + +function outcomeFromIssue(args: { + run: AgentRunRecord; + action: AgentActionRecord; + issue: IssueRecord; + matchedBy: string; + evaluatedAt: string; + actionAt: number; + actionAgeMs: number; + staleAfterMs: number; +}): AgentRecommendationOutcomeRecord { + const state = issueOutcomeState(args.issue, args.actionAt, args.actionAgeMs, args.staleAfterMs); + const maintainerLane = isMaintainerLane(args.run.actorLogin, args.issue.repoFullName, args.issue.authorAssociation); + return baseOutcome(args.run, args.action, { + outcomeState: state, + outcomeTargetType: "issue", + outcomeRepoFullName: args.issue.repoFullName, + outcomeIssueNumber: args.issue.number, + maintainerLane, + confidence: state === "ignored" ? "medium" : "high", + reason: issueOutcomeReason(state, args.issue), + sourceUpdatedAt: args.issue.updatedAt ?? args.issue.createdAt, + detectedAt: args.evaluatedAt, + metadata: { + matchedBy: args.matchedBy, + issueState: args.issue.state, + }, + }); +} + +function baseOutcome( + run: AgentRunRecord, + action: AgentActionRecord, + outcome: { + outcomeState: AgentRecommendationOutcomeState; + outcomeTargetType: AgentRecommendationOutcomeTargetType; + outcomeRepoFullName?: string | null | undefined; + outcomePullNumber?: number | null | undefined; + outcomeIssueNumber?: number | null | undefined; + maintainerLane: boolean; + confidence: AgentRecommendationOutcomeRecord["confidence"]; + reason: string; + sourceUpdatedAt?: string | null | undefined; + detectedAt: string; + metadata: AgentRecommendationOutcomeRecord["metadata"]; + }, +): AgentRecommendationOutcomeRecord { + return { + actionId: action.id, + runId: run.id, + actorLogin: run.actorLogin, + actionType: action.actionType, + targetRepoFullName: action.targetRepoFullName, + targetPullNumber: action.targetPullNumber, + targetIssueNumber: action.targetIssueNumber, + outcomeState: outcome.outcomeState, + outcomeTargetType: outcome.outcomeTargetType, + outcomeRepoFullName: outcome.outcomeRepoFullName, + outcomePullNumber: outcome.outcomePullNumber, + outcomeIssueNumber: outcome.outcomeIssueNumber, + maintainerLane: outcome.maintainerLane, + confidence: outcome.confidence, + reason: outcome.reason, + sourceUpdatedAt: outcome.sourceUpdatedAt, + detectedAt: outcome.detectedAt, + metadata: { + actionStatus: action.status, + actionType: action.actionType, + safetyClass: action.safetyClass, + ...outcome.metadata, + }, + }; +} + +function pullRequestOutcomeState( + pr: PullRequestRecord, + action: AgentActionRecord, + actionAt: number, + actionAgeMs: number, + staleAfterMs: number, +): AgentRecommendationOutcomeState { + const updatedAt = timestamp(pr.updatedAt ?? pr.createdAt); + const createdAt = timestamp(pr.createdAt ?? pr.updatedAt); + const mergedAt = timestamp(pr.mergedAt); + if ((Number.isFinite(mergedAt) && mergedAt >= actionAt) || (!pr.mergedAt && pr.state === "merged" && updatedAt >= actionAt)) return "merged"; + if (pr.state === "closed" && updatedAt >= actionAt) return "closed"; + const positiveOpenSignal = pr.reviewDecision === "APPROVED" || pr.mergeableState === "clean"; + if (action.targetPullNumber && positiveOpenSignal && updatedAt >= actionAt) return "improved"; + if (createdAt >= actionAt || updatedAt > actionAt) return "accepted"; + if (actionAgeMs >= staleAfterMs) return "stale"; + return "ignored"; +} + +function issueOutcomeState(issue: IssueRecord, actionAt: number, actionAgeMs: number, staleAfterMs: number): AgentRecommendationOutcomeState { + const updatedAt = timestamp(issue.updatedAt ?? issue.createdAt); + const createdAt = timestamp(issue.createdAt ?? issue.updatedAt); + if (issue.state !== "open" && updatedAt >= actionAt) return "closed"; + if (createdAt >= actionAt || updatedAt > actionAt) return "accepted"; + if (actionAgeMs >= staleAfterMs) return "stale"; + return "ignored"; +} + +function pullRequestOutcomeReason(state: AgentRecommendationOutcomeState, pr: PullRequestRecord): string { + if (state === "merged") return `${pr.repoFullName}#${pr.number} merged after the recommendation snapshot.`; + if (state === "closed") return `${pr.repoFullName}#${pr.number} closed without a merge after the recommendation snapshot.`; + if (state === "improved") return `${pr.repoFullName}#${pr.number} remains open but now has approval or clean mergeability evidence.`; + if (state === "accepted") return `${pr.repoFullName}#${pr.number} shows later cached activity matching the recommendation.`; + if (state === "stale") return `${pr.repoFullName}#${pr.number} remains open with no later activity past the stale-outcome window.`; + return `${pr.repoFullName}#${pr.number} is visible but has no later positive or terminal outcome yet.`; +} + +function issueOutcomeReason(state: AgentRecommendationOutcomeState, issue: IssueRecord): string { + if (state === "closed") return `${issue.repoFullName}#${issue.number} closed after the recommendation snapshot.`; + if (state === "accepted") return `${issue.repoFullName}#${issue.number} shows later cached issue activity matching the recommendation.`; + if (state === "stale") return `${issue.repoFullName}#${issue.number} remains open with no later activity past the stale-outcome window.`; + return `${issue.repoFullName}#${issue.number} is visible but has no later terminal outcome yet.`; +} + +function firstLaterPullRequest( + pullRequests: PullRequestRecord[], + login: string, + repoFullName: string, + linkedIssueNumber: number | null | undefined, + actionAt: number, +): PullRequestRecord | undefined { + return pullRequests + .filter((pr) => sameRepo(pr.repoFullName, repoFullName) && sameLogin(pr.authorLogin, login)) + .filter((pr) => !linkedIssueNumber || pr.linkedIssues.includes(linkedIssueNumber)) + .filter((pr) => timestamp(pr.createdAt ?? pr.updatedAt) >= actionAt) + .sort((left, right) => timestamp(left.createdAt ?? left.updatedAt) - timestamp(right.createdAt ?? right.updatedAt) || left.number - right.number)[0]; +} + +function firstLaterIssue(issues: IssueRecord[], login: string, repoFullName: string, actionAt: number): IssueRecord | undefined { + return issues + .filter((issue) => sameRepo(issue.repoFullName, repoFullName) && sameLogin(issue.authorLogin, login)) + .filter((issue) => timestamp(issue.createdAt ?? issue.updatedAt) >= actionAt) + .sort((left, right) => timestamp(left.createdAt ?? left.updatedAt) - timestamp(right.createdAt ?? right.updatedAt) || left.number - right.number)[0]; +} + +function repoFromPayload(action: AgentActionRecord): string | null { + const payload = action.payload as Record; + const decision = payload.decision as { repoFullName?: unknown } | undefined; + const nestedAction = payload.action as { repoFullName?: unknown } | undefined; + if (typeof decision?.repoFullName === "string") return decision.repoFullName; + if (typeof nestedAction?.repoFullName === "string") return nestedAction.repoFullName; + return null; +} + +function isMaintainerLane(login: string, repoFullName: string, association?: string | null | undefined): boolean { + const owner = repoFullName.split("/")[0] ?? ""; + return sameLogin(owner, login) || association === "OWNER" || association === "MEMBER" || association === "COLLABORATOR"; +} + +function sameLogin(left: string | null | undefined, right: string | null | undefined): boolean { + return Boolean(left && right && left.toLowerCase() === right.toLowerCase()); +} + +function sameRepo(left: string | null | undefined, right: string | null | undefined): boolean { + return sameLogin(left, right); +} + +function timestamp(value: string | null | undefined): number { + const parsed = Date.parse(value ?? ""); + return Number.isFinite(parsed) ? parsed : Number.NEGATIVE_INFINITY; +} + +function daysToMs(days: number): number { + return Math.max(0, days) * 24 * 60 * 60 * 1000; +} diff --git a/src/types.ts b/src/types.ts index cd643da2da..651e83ce7d 100644 --- a/src/types.ts +++ b/src/types.ts @@ -684,6 +684,80 @@ export type AgentContextSnapshotRecord = { createdAt?: string | null | undefined; }; +export type AgentRecommendationOutcomeState = "accepted" | "ignored" | "stale" | "merged" | "closed" | "improved"; +export type AgentRecommendationOutcomeTargetType = "pull_request" | "issue" | "repository" | "none"; +export type AgentRecommendationOutcomeConfidence = "high" | "medium" | "low"; + +export type AgentRecommendationOutcomeRecord = { + id?: string | undefined; + actionId: string; + runId: string; + actorLogin: string; + actionType: AgentActionType; + targetRepoFullName?: string | null | undefined; + targetPullNumber?: number | null | undefined; + targetIssueNumber?: number | null | undefined; + outcomeState: AgentRecommendationOutcomeState; + outcomeTargetType: AgentRecommendationOutcomeTargetType; + outcomeRepoFullName?: string | null | undefined; + outcomePullNumber?: number | null | undefined; + outcomeIssueNumber?: number | null | undefined; + maintainerLane: boolean; + confidence: AgentRecommendationOutcomeConfidence; + reason: string; + sourceUpdatedAt?: string | null | undefined; + detectedAt?: string | null | undefined; + metadata: Record; + createdAt?: string | null | undefined; + updatedAt?: string | null | undefined; +}; + +export type AgentRecommendationOutcomeStateBucket = { + state: AgentRecommendationOutcomeState; + count: number; +}; + +export type AgentRecommendationOutcomeRepoSummary = { + repoFullName: string; + total: number; + accepted: number; + ignored: number; + stale: number; + merged: number; + closed: number; + improved: number; + positive: number; + negative: number; + maintainerLaneTotal: number; + latestOutcomeAt?: string | null | undefined; + signal: "positive" | "negative" | "mixed" | "neutral"; +}; + +export type AgentRecommendationOutcomeSummary = { + login: string; + generatedAt: string; + windowDays: number; + totals: { + total: number; + accepted: number; + ignored: number; + stale: number; + merged: number; + closed: number; + improved: number; + positive: number; + negative: number; + maintainerLaneTotal: number; + }; + states: AgentRecommendationOutcomeStateBucket[]; + repos: AgentRecommendationOutcomeRepoSummary[]; + maintainerLane: { + total: number; + states: AgentRecommendationOutcomeStateBucket[]; + }; + privateSummary: string; +}; + export type InstallationRecord = { id: number; accountLogin: string; diff --git a/test/unit/contributor-evidence-graph.test.ts b/test/unit/contributor-evidence-graph.test.ts index 44b801afd0..10c3f96abe 100644 --- a/test/unit/contributor-evidence-graph.test.ts +++ b/test/unit/contributor-evidence-graph.test.ts @@ -382,6 +382,43 @@ describe("contributor evidence graph", () => { expect(graph.paths).toEqual([expect.objectContaining({ repoFullName: issueOnlyRepo, path: "src/from-merge.ts", mergedPullRequests: 1 })]); }); + it("orders equal-weight cached labels and paths while using issue-only freshness", () => { + const leftRepo = "owner/a"; + const rightRepo = "owner/b"; + const issueOnlyRepo = "owner/c"; + const graph = buildContributorEvidenceGraph({ + login: "dev", + generatedAt: GENERATED_AT, + profile: profile({ + registeredRepoActivity: { pullRequests: 3, mergedPullRequests: 1, issues: 1, reposTouched: [leftRepo, rightRepo, issueOnlyRepo], dominantLabels: [] }, + }), + outcomeHistory: history([]), + roleContexts: [], + repositories: [repo(leftRepo), repo(rightRepo), repo(issueOnlyRepo)], + pullRequests: [ + pr(rightRepo, 2, { labels: ["tie"], authorAssociation: "COLLABORATOR" }), + pr(leftRepo, 1, { state: "MERGED", labels: ["tie", "zeta", "alpha"], updatedAt: undefined, createdAt: undefined, mergedAt: FRESH_AT }), + ], + issues: [{ repoFullName: issueOnlyRepo, number: 3, title: "Issue-only", state: "open", authorLogin: "dev", authorAssociation: "CONTRIBUTOR", labels: ["issue"], linkedPrs: [], createdAt: undefined, updatedAt: FRESH_AT }], + pullRequestFiles: [file(rightRepo, 2, "src/a.ts"), file(leftRepo, 1, "src/z.ts"), file(leftRepo, 1, "src/a.ts")], + }); + + expect(graph.labels.map((label) => `${label.repoFullName}:${label.label}`)).toEqual([ + "owner/a:alpha", + "owner/a:tie", + "owner/a:zeta", + "owner/b:tie", + "owner/c:issue", + ]); + expect(graph.paths.map((path) => `${path.repoFullName}:${path.path}:${path.mergedPullRequests}`)).toEqual([ + "owner/a:src/a.ts:1", + "owner/a:src/z.ts:1", + "owner/b:src/a.ts:0", + ]); + expect(graph.repos.find((entry) => entry.repoFullName === issueOnlyRepo)).toMatchObject({ source: "github_cache", freshness: "fresh", issues: 1 }); + expect(graph.repos.find((entry) => entry.repoFullName === rightRepo)).toMatchObject({ maintainerLane: true, normalContributorEvidenceAllowed: false }); + }); + it("orders graph relationships deterministically and applies worker-safe bounds", () => { const repoNames = Array.from({ length: CONTRIBUTOR_EVIDENCE_GRAPH_MAX_REPOS + 5 }, (_, index) => `owner/repo-${String(index).padStart(2, "0")}`); const omittedRepo = repoNames[CONTRIBUTOR_EVIDENCE_GRAPH_MAX_REPOS + 1]!; @@ -444,10 +481,61 @@ describe("contributor evidence graph", () => { login: "dev", profile: profile({ registeredRepoActivity: { pullRequests: 2, mergedPullRequests: 1, issues: 0, reposTouched: ["owner/registered", "owner/unregistered", ""], dominantLabels: [] } }), pullRequests: [pr("owner/registered", 1), pr("other/repo", 2, { authorLogin: "someone-else" })], - repoStats: [{ login: "dev", repoFullName: "owner/stats", pullRequests: 1, mergedPullRequests: 1, openPullRequests: 0, issues: 0, stalePullRequests: 0, unlinkedPullRequests: 0, dominantLabels: [] }], + issues: [{ repoFullName: "other/issue", number: 3, title: "Ignored issue", state: "open", authorLogin: "someone-else", authorAssociation: "CONTRIBUTOR", labels: [], linkedPrs: [], createdAt: FRESH_AT, updatedAt: FRESH_AT }], + repoStats: [ + { login: "dev", repoFullName: "owner/stats", pullRequests: 1, mergedPullRequests: 1, openPullRequests: 0, issues: 0, stalePullRequests: 0, unlinkedPullRequests: 0, dominantLabels: [] }, + { login: "someone-else", repoFullName: "owner/ignored-stats", pullRequests: 1, mergedPullRequests: 1, openPullRequests: 0, issues: 0, stalePullRequests: 0, unlinkedPullRequests: 0, dominantLabels: [] }, + ], repositories: [repo("owner/registered"), repo("owner/stats"), { ...repo("owner/unregistered"), isRegistered: false }], }), ).toEqual(["owner/registered", "owner/stats"]); expect(evidenceGraphTouchedRepoFullNames({ login: "dev" })).toEqual([]); }); + + it("keeps missing cache timestamps partial without optional cache fields", () => { + const repoFullName = "owner/missing-cache-dates"; + const graph = buildContributorEvidenceGraph({ + login: "dev", + generatedAt: GENERATED_AT, + profile: profile({ registeredRepoActivity: { pullRequests: 0, mergedPullRequests: 0, issues: 0, reposTouched: [repoFullName], dominantLabels: [] } }), + outcomeHistory: history([]), + roleContexts: [], + repositories: [repo(repoFullName)], + pullRequests: [pr(repoFullName, 4, { createdAt: undefined, updatedAt: undefined, mergedAt: undefined, authorAssociation: undefined, labels: ["cache"] })], + issues: [ + { + repoFullName, + number: 5, + title: "Issue missing optional cache fields", + state: "open", + authorLogin: "dev", + authorAssociation: undefined, + labels: undefined, + linkedPrs: [], + createdAt: undefined, + updatedAt: undefined, + } as any, + { + repoFullName, + number: 6, + title: "Issue with label but missing cache dates", + state: "open", + authorLogin: "dev", + authorAssociation: "CONTRIBUTOR", + labels: ["cache-issue"], + linkedPrs: [], + createdAt: undefined, + updatedAt: undefined, + }, + ], + pullRequestFiles: [file(repoFullName, 4, "src/undated.ts")], + }); + + expect(graph.repos).toEqual([expect.objectContaining({ repoFullName, source: "github_cache", freshness: "partial" })]); + expect(graph.labels).toEqual([ + expect.objectContaining({ repoFullName, label: "cache", freshness: "partial" }), + expect.objectContaining({ repoFullName, label: "cache-issue", freshness: "partial" }), + ]); + expect(graph.paths).toEqual([expect.objectContaining({ repoFullName, path: "src/undated.ts", freshness: "partial" })]); + }); }); diff --git a/test/unit/decision-pack.test.ts b/test/unit/decision-pack.test.ts index 74efb8eb79..1a0f052746 100644 --- a/test/unit/decision-pack.test.ts +++ b/test/unit/decision-pack.test.ts @@ -145,6 +145,195 @@ describe("decision-pack service", () => { expect(maintainer.riskReasons.some((line) => line.includes("high closure risk"))).toBe(false); }); + it("feeds private recommendation outcome feedback without changing public next actions", () => { + const baseline = __decisionPackInternals.buildRepoDecision({ + repo: repo("owner/direct", 0.03, 0), + roleContext: { maintainerLane: false } as any, + outcome: undefined, + }); + const decision = __decisionPackInternals.buildRepoDecision({ + repo: repo("owner/direct", 0.03, 0), + roleContext: { maintainerLane: false } as any, + outcome: undefined, + recommendationOutcomeFeedback: { + repoFullName: "owner/direct", + total: 4, + accepted: 1, + ignored: 1, + stale: 0, + merged: 1, + closed: 1, + improved: 1, + positive: 3, + negative: 1, + maintainerLaneTotal: 2, + latestOutcomeAt: "2026-05-30T00:00:00.000Z", + signal: "positive", + }, + }); + + expect(decision.recommendationOutcomeFeedback).toMatchObject({ signal: "positive", positive: 3, negative: 1, maintainerLaneTotal: 2 }); + expect(decision.priorityScore).toBeGreaterThan(baseline.priorityScore); + expect(decision.whyThisHelps.some((line) => line.includes("Private recommendation feedback"))).toBe(true); + expect(decision.riskReasons.some((line) => line.includes("Private recommendation feedback"))).toBe(true); + expect(decision.publicNextActions.join(" ")).not.toMatch(/Private recommendation feedback|wallet|hotkey|raw trust score|reward estimate|payout|farming/i); + }); + + it("penalizes negative and mixed private recommendation feedback without leaking it publicly", () => { + const baseline = __decisionPackInternals.buildRepoDecision({ + repo: repo("owner/direct", 0.03, 0), + roleContext: { maintainerLane: false } as any, + outcome: undefined, + }); + const negative = __decisionPackInternals.buildRepoDecision({ + repo: repo("owner/direct", 0.03, 0), + roleContext: { maintainerLane: false } as any, + outcome: undefined, + recommendationOutcomeFeedback: { + repoFullName: "owner/direct", + total: 6, + accepted: 0, + ignored: 2, + stale: 2, + merged: 0, + closed: 2, + improved: 0, + positive: 0, + negative: 6, + maintainerLaneTotal: 0, + latestOutcomeAt: "2026-05-30T00:00:00.000Z", + signal: "negative", + }, + }); + const mixed = __decisionPackInternals.buildRepoDecision({ + repo: repo("owner/direct", 0.03, 0), + roleContext: { maintainerLane: false } as any, + outcome: undefined, + recommendationOutcomeFeedback: { + repoFullName: "owner/direct", + total: 4, + accepted: 1, + ignored: 1, + stale: 0, + merged: 1, + closed: 1, + improved: 0, + positive: 2, + negative: 2, + maintainerLaneTotal: 1, + latestOutcomeAt: null, + signal: "mixed", + }, + }); + + expect(negative.priorityScore).toBeLessThan(baseline.priorityScore); + expect(negative.riskReasons.join(" ")).toMatch(/6 unresolved or negative/); + expect(negative.whyThisHelps.some((line) => line.includes("Private recommendation feedback"))).toBe(false); + expect(mixed.priorityScore).toBeLessThan(baseline.priorityScore); + expect(mixed.riskReasons.join(" ")).toMatch(/2 unresolved or negative/); + expect(mixed.whyThisHelps.join(" ")).toMatch(/2 positive/); + expect(`${negative.publicNextActions.join(" ")} ${mixed.publicNextActions.join(" ")}`).not.toMatch(/Private recommendation feedback/i); + }); + + it("keeps zero-count and maintainer-lane private feedback from changing repo decisions", () => { + const baseline = __decisionPackInternals.buildRepoDecision({ + repo: repo("owner/direct", 0.03, 0), + roleContext: { maintainerLane: false } as any, + outcome: undefined, + }); + const zero = __decisionPackInternals.buildRepoDecision({ + repo: repo("owner/direct", 0.03, 0), + roleContext: { maintainerLane: false } as any, + outcome: undefined, + recommendationOutcomeFeedback: { + repoFullName: "owner/direct", + total: 0, + accepted: 0, + ignored: 0, + stale: 0, + merged: 0, + closed: 0, + improved: 0, + positive: 0, + negative: 0, + maintainerLaneTotal: 3, + latestOutcomeAt: null, + signal: "neutral", + }, + }); + const maintainerLane = __decisionPackInternals.buildRepoDecision({ + repo: repo("owner/direct", 0.03, 0), + roleContext: { maintainerLane: true } as any, + outcome: undefined, + recommendationOutcomeFeedback: { + repoFullName: "owner/direct", + total: 2, + accepted: 0, + ignored: 1, + stale: 0, + merged: 0, + closed: 1, + improved: 0, + positive: 0, + negative: 2, + maintainerLaneTotal: 0, + latestOutcomeAt: null, + signal: "negative", + }, + }); + + expect(zero.recommendationOutcomeFeedback).toBeUndefined(); + expect(zero.priorityScore).toBe(baseline.priorityScore); + expect(maintainerLane.recommendation).toBe("maintainer_lane"); + expect(maintainerLane.riskReasons.join(" ")).not.toMatch(/Private recommendation feedback/); + }); + + it("summarizes non-empty recommendation feedback in contributor decision packs", () => { + const pack = __decisionPackInternals.buildContributorDecisionPack({ + login: "dev", + profile: { + login: "dev", + github: {}, + source: {}, + gittensor: null, + registeredRepoActivity: {}, + trustSignals: {}, + } as any, + outcomeHistory: { login: "dev", totals: {}, repoOutcomes: [], successPatterns: [], failurePatterns: [], summary: "" } as any, + repositories: [], + syncStates: [], + syncSegments: [], + totals: [], + scoringModelSnapshotId: "scoring-1", + contributorPullRequests: [], + contributorIssues: [], + openPrMonitor: emptyOpenPrMonitor("dev"), + recommendationOutcomeFeedback: { + login: "dev", + generatedAt: "2026-05-30T00:00:00.000Z", + windowDays: 90, + totals: { + total: 1, + accepted: 1, + ignored: 0, + stale: 0, + merged: 0, + closed: 0, + improved: 0, + positive: 1, + negative: 0, + maintainerLaneTotal: 1, + }, + states: [{ state: "accepted", count: 1 }], + repos: [], + maintainerLane: { total: 1, states: [{ state: "merged", count: 1 }] }, + privateSummary: "dev has feedback.", + }, + }); + + expect(pack.summary).toContain("Recommendation feedback: 1 positive, 0 negative, 1 maintainer-lane separated."); + }); + it("redacts official hotkeys, loads stale snapshots, and resolves repo decisions case-insensitively", async () => { const env = createTestEnv(); const pack = { @@ -180,6 +369,7 @@ describe("decision-pack service", () => { const loaded = await loadContributorDecisionPack(env, "jsonbored"); expect(loaded).toMatchObject({ source: "snapshot", snapshotAgeSeconds: expect.any(Number), stale: expect.any(Boolean), freshness: "stale", rebuildEnqueued: false }); + expect(loaded?.recommendationOutcomeFeedback).toMatchObject({ login: "jsonbored", totals: { total: 0, maintainerLaneTotal: 0 } }); expect(repoDecisionFromPack(loaded!, "jsonbored/AWESOME-CLAUDE")).toMatchObject({ recommendation: "maintainer_lane" }); expect(repoDecisionFromPack(loaded!, "missing/repo")).toBeNull(); diff --git a/test/unit/openapi.test.ts b/test/unit/openapi.test.ts index 03181ee8c7..e99c621846 100644 --- a/test/unit/openapi.test.ts +++ b/test/unit/openapi.test.ts @@ -107,6 +107,7 @@ describe("OpenAPI contract", () => { expect(JSON.stringify(spec.components?.schemas?.RepoIntelligence)).toContain("burdenForecastFreshness"); expect(JSON.stringify(spec.components?.schemas?.CommandPreviewResponse)).toContain("missing_permission"); expect(JSON.stringify(spec.components?.schemas?.ContributorOutcomeHistory)).toContain("reconciliation"); + expect(JSON.stringify(spec.components?.schemas?.ContributorDecisionPack)).toContain("recommendationOutcomeFeedback"); expect(JSON.stringify(spec.components?.schemas?.LocalBranchAnalysis)).toContain("baseFreshness"); expect(JSON.stringify(spec.components?.schemas?.LocalBranchAnalysis)).toContain("recommendedRerunCondition"); expect(JSON.stringify(spec.components?.schemas?.Health)).toContain("minMcpVersion"); diff --git a/test/unit/recommendation-outcomes.test.ts b/test/unit/recommendation-outcomes.test.ts new file mode 100644 index 0000000000..7e09c74ad1 --- /dev/null +++ b/test/unit/recommendation-outcomes.test.ts @@ -0,0 +1,495 @@ +import { describe, expect, it } from "vitest"; +import { + createAgentRun, + getAgentRecommendationOutcomeSummary, + listAgentRecommendationOutcomes, + replaceAgentActions, + upsertAgentRecommendationOutcome, + upsertIssueFromGitHub, + upsertPullRequestFromGitHub, +} from "../../src/db/repositories"; +import { classifyRecommendationOutcome, evaluateRecommendationOutcomes } from "../../src/services/recommendation-outcomes"; +import type { AgentActionRecord, AgentRunRecord, GitHubIssuePayload, GitHubPullRequestPayload, IssueRecord, PullRequestRecord } from "../../src/types"; +import { createTestEnv } from "../helpers/d1"; + +describe("recommendation outcome feedback", () => { + it("matches later PR and issue outcomes while separating maintainer-lane activity", async () => { + const env = createTestEnv(); + const run = runRecord("run-outcomes", "dev", "2026-05-01T00:00:00.000Z"); + await createAgentRun(env, run); + await replaceAgentActions(env, run.id, [ + action(run, 0, { targetRepoFullName: "owner/merged" }), + action(run, 1, { targetRepoFullName: "owner/closed", targetPullNumber: 12 }), + action(run, 2, { targetRepoFullName: "owner/stale", targetPullNumber: 13 }), + action(run, 3, { targetRepoFullName: "owner/ignored" }), + action(run, 4, { targetRepoFullName: "owner/issue", targetIssueNumber: 7 }), + action(run, 5, { targetRepoFullName: "owner/improved", targetPullNumber: 14 }), + action(run, 6, { targetRepoFullName: "dev/own" }), + ]); + + await upsertPullRequestFromGitHub(env, "owner/merged", pr(11, { state: "closed", merged_at: "2026-05-05T00:00:00.000Z", created_at: "2026-05-02T00:00:00.000Z", updated_at: "2026-05-05T00:00:00.000Z" })); + await upsertPullRequestFromGitHub(env, "owner/closed", pr(12, { state: "closed", created_at: "2026-04-20T00:00:00.000Z", updated_at: "2026-05-03T00:00:00.000Z" })); + await upsertPullRequestFromGitHub(env, "owner/stale", pr(13, { state: "open", created_at: "2026-04-01T00:00:00.000Z", updated_at: "2026-04-02T00:00:00.000Z" })); + await upsertPullRequestFromGitHub(env, "owner/improved", pr(14, { state: "open", created_at: "2026-04-20T00:00:00.000Z", updated_at: "2026-05-03T00:00:00.000Z", reviewDecision: "APPROVED" })); + await upsertIssueFromGitHub(env, "owner/issue", issue(7, { state: "closed", created_at: "2026-04-20T00:00:00.000Z", updated_at: "2026-05-04T00:00:00.000Z" })); + await upsertPullRequestFromGitHub(env, "dev/own", pr(21, { state: "closed", merged_at: "2026-05-06T00:00:00.000Z", created_at: "2026-05-02T00:00:00.000Z", updated_at: "2026-05-06T00:00:00.000Z", author_association: "OWNER" })); + + const result = await evaluateRecommendationOutcomes(env, "dev", { now: "2026-06-01T00:00:00.000Z", staleAfterDays: 14, ignoredAfterDays: 7 }); + expect(result.skippedFreshActions).toBe(0); + expect(result.outcomes.map((outcome) => [outcome.targetRepoFullName, outcome.outcomeState, outcome.maintainerLane])).toEqual( + expect.arrayContaining([ + ["owner/merged", "merged", false], + ["owner/closed", "closed", false], + ["owner/stale", "stale", false], + ["owner/ignored", "ignored", false], + ["owner/issue", "closed", false], + ["owner/improved", "improved", false], + ["dev/own", "merged", true], + ]), + ); + + const summary = await getAgentRecommendationOutcomeSummary(env, "dev", { now: "2026-06-01T00:00:00.000Z", windowDays: 90 }); + expect(summary.totals).toMatchObject({ total: 6, merged: 1, closed: 2, stale: 1, ignored: 1, improved: 1, positive: 2, negative: 4, maintainerLaneTotal: 1 }); + expect(summary.maintainerLane).toMatchObject({ total: 1, states: [{ state: "merged", count: 1 }] }); + expect(summary.repos.find((repo) => repo.repoFullName === "dev/own")).toMatchObject({ total: 0, maintainerLaneTotal: 1, signal: "neutral" }); + }); + + it("skips fresh unmatched actions and upserts later terminal outcomes idempotently", async () => { + const env = createTestEnv(); + const run = runRecord("run-idempotent", "dev", "2026-05-01T00:00:00.000Z"); + await createAgentRun(env, run); + await replaceAgentActions(env, run.id, [ + action(run, 0, { targetRepoFullName: "owner/later" }), + action(run, 1, { targetRepoFullName: "owner/fresh", createdAt: "2026-05-30T00:00:00.000Z" }), + ]); + await upsertPullRequestFromGitHub(env, "owner/later", pr(31, { state: "open", created_at: "2026-05-02T00:00:00.000Z", updated_at: "2026-05-02T00:00:00.000Z" })); + + const accepted = await evaluateRecommendationOutcomes(env, "dev", { now: "2026-06-01T00:00:00.000Z", ignoredAfterDays: 7 }); + expect(accepted.outcomes.map((outcome) => outcome.outcomeState)).toEqual(["accepted"]); + expect(accepted.skippedFreshActions).toBe(1); + + await upsertPullRequestFromGitHub(env, "owner/later", pr(31, { state: "closed", merged_at: "2026-06-02T00:00:00.000Z", created_at: "2026-05-02T00:00:00.000Z", updated_at: "2026-06-02T00:00:00.000Z" })); + const merged = await evaluateRecommendationOutcomes(env, "dev", { now: "2026-06-03T00:00:00.000Z", ignoredAfterDays: 7 }); + + expect(merged.outcomes.find((outcome) => outcome.targetRepoFullName === "owner/later")).toMatchObject({ outcomeState: "merged", outcomePullNumber: 31 }); + const rows = await listAgentRecommendationOutcomes(env, { actorLogin: "dev" }); + expect(rows.filter((row) => row.targetRepoFullName === "owner/later")).toHaveLength(1); + }); + + it("classifies linked PRs, later issues, payload targets, no-target, and defensive timestamp branches", () => { + const run = runRecord("run-classifier", "dev", "2026-05-01T00:00:00.000Z"); + const base = { + run, + evaluatedAt: "2026-06-01T00:00:00.000Z", + staleAfterMs: 14 * 24 * 60 * 60 * 1000, + ignoredAfterMs: 7 * 24 * 60 * 60 * 1000, + }; + + expect( + classifyRecommendationOutcome({ + ...base, + action: action(run, 0, { targetRepoFullName: "owner/linked", targetIssueNumber: 99 }), + pullRequests: [prRecord(40, "owner/linked", { linkedIssues: [99] })], + issues: [], + }), + ).toMatchObject({ outcomeState: "accepted", outcomeTargetType: "pull_request", metadata: { matchedBy: "linked_issue_pull_request" } }); + + expect( + classifyRecommendationOutcome({ + ...base, + action: action(run, 1, { targetRepoFullName: "owner/issues" }), + pullRequests: [], + issues: [issueRecord(41, "owner/issues")], + }), + ).toMatchObject({ outcomeState: "accepted", outcomeTargetType: "issue", metadata: { matchedBy: "later_repo_issue" } }); + + expect( + classifyRecommendationOutcome({ + ...base, + action: action(run, 2, { targetRepoFullName: undefined, payload: { decision: { repoFullName: "owner/payload" } } }), + pullRequests: [], + issues: [], + }), + ).toMatchObject({ outcomeState: "ignored", outcomeTargetType: "repository", outcomeRepoFullName: "owner/payload", confidence: "medium" }); + + expect( + classifyRecommendationOutcome({ + ...base, + action: action(run, 3, { targetRepoFullName: undefined, payload: { action: { repoFullName: "owner/nested" } } }), + pullRequests: [], + issues: [], + }), + ).toMatchObject({ outcomeState: "ignored", outcomeTargetType: "repository", outcomeRepoFullName: "owner/nested" }); + + expect( + classifyRecommendationOutcome({ + ...base, + action: action(run, 4, { targetRepoFullName: undefined, payload: {} }), + pullRequests: [], + issues: [], + }), + ).toMatchObject({ outcomeState: "ignored", outcomeTargetType: "none", confidence: "low" }); + + expect( + classifyRecommendationOutcome({ + ...base, + action: action(run, 5, { createdAt: "not-a-date" }), + pullRequests: [], + issues: [], + }), + ).toBeNull(); + }); + + it("classifies non-terminal open targets as stale or ignored from exact cached targets", () => { + const run = runRecord("run-open-targets", "dev", "2026-05-01T00:00:00.000Z"); + const staleAfterMs = 14 * 24 * 60 * 60 * 1000; + + expect( + classifyRecommendationOutcome({ + run, + action: action(run, 0, { targetRepoFullName: "owner/stale-issue", targetIssueNumber: 50 }), + pullRequests: [], + issues: [issueRecord(50, "owner/stale-issue", { state: "open", createdAt: "2026-04-01T00:00:00.000Z", updatedAt: "2026-04-01T00:00:00.000Z" })], + evaluatedAt: "2026-06-01T00:00:00.000Z", + staleAfterMs, + ignoredAfterMs: 7 * 24 * 60 * 60 * 1000, + }), + ).toMatchObject({ outcomeState: "stale", outcomeTargetType: "issue" }); + + expect( + classifyRecommendationOutcome({ + run, + action: action(run, 1, { targetRepoFullName: "owner/ignored-pr", targetPullNumber: 51 }), + pullRequests: [prRecord(51, "owner/ignored-pr", { state: "open", createdAt: "2026-04-01T00:00:00.000Z", updatedAt: "2026-04-01T00:00:00.000Z" })], + issues: [], + evaluatedAt: "2026-05-05T00:00:00.000Z", + staleAfterMs, + ignoredAfterMs: 1, + }), + ).toMatchObject({ outcomeState: "ignored", outcomeTargetType: "pull_request", confidence: "medium" }); + }); + + it("does not count terminal target state that predates the recommendation", () => { + const run = runRecord("run-predated-terminals", "dev", "2026-05-01T00:00:00.000Z"); + const staleAfterMs = 14 * 24 * 60 * 60 * 1000; + const base = { + run, + evaluatedAt: "2026-06-01T00:00:00.000Z", + staleAfterMs, + ignoredAfterMs: 7 * 24 * 60 * 60 * 1000, + }; + + expect( + classifyRecommendationOutcome({ + ...base, + action: action(run, 0, { targetRepoFullName: "owner/old-merged", targetPullNumber: 60 }), + pullRequests: [ + prRecord(60, "owner/old-merged", { + state: "closed", + mergedAt: "2026-04-01T00:00:00.000Z", + createdAt: "2026-03-01T00:00:00.000Z", + updatedAt: "2026-04-01T00:00:00.000Z", + }), + ], + issues: [], + }), + ).toMatchObject({ outcomeState: "stale", outcomeTargetType: "pull_request" }); + + expect( + classifyRecommendationOutcome({ + ...base, + action: action(run, 1, { targetRepoFullName: "owner/old-issue", targetIssueNumber: 61 }), + pullRequests: [], + issues: [ + issueRecord(61, "owner/old-issue", { + state: "closed", + createdAt: "2026-03-01T00:00:00.000Z", + updatedAt: "2026-04-01T00:00:00.000Z", + }), + ], + }), + ).toMatchObject({ outcomeState: "stale", outcomeTargetType: "issue" }); + }); + + it("uses timestamp fallbacks and deterministic earliest-match ordering for cached activity", () => { + const run = runRecord("run-fallbacks", "Dev", "2026-05-01T00:00:00.000Z"); + const runWithoutUpdate = { ...run, updatedAt: undefined as unknown as string }; + const base = { + evaluatedAt: "2026-06-01T00:00:00.000Z", + staleAfterMs: 14 * 24 * 60 * 60 * 1000, + ignoredAfterMs: 7 * 24 * 60 * 60 * 1000, + }; + + expect( + classifyRecommendationOutcome({ + ...base, + run, + action: action(run, 0, { targetRepoFullName: "owner/run-updated", createdAt: undefined }), + pullRequests: [], + issues: [], + }), + ).toMatchObject({ outcomeState: "ignored", outcomeRepoFullName: "owner/run-updated" }); + + expect( + classifyRecommendationOutcome({ + ...base, + run: runWithoutUpdate, + action: action(runWithoutUpdate, 1, { targetRepoFullName: "owner/run-created", createdAt: undefined }), + pullRequests: [], + issues: [], + }), + ).toMatchObject({ outcomeState: "ignored", outcomeRepoFullName: "owner/run-created" }); + + expect( + classifyRecommendationOutcome({ + ...base, + run: { ...runWithoutUpdate, createdAt: undefined as unknown as string }, + action: action(runWithoutUpdate, 2, { createdAt: undefined }), + pullRequests: [], + issues: [], + }), + ).toBeNull(); + + expect( + classifyRecommendationOutcome({ + ...base, + run, + action: action(run, 3, { targetRepoFullName: "owner/pr-sort" }), + pullRequests: [ + prRecord(72, "owner/pr-sort", { createdAt: undefined, updatedAt: "2026-05-03T00:00:00.000Z", authorLogin: "dev" }), + prRecord(71, "owner/pr-sort", { createdAt: undefined, updatedAt: "2026-05-03T00:00:00.000Z", authorLogin: "DEV" }), + ], + issues: [], + }), + ).toMatchObject({ outcomePullNumber: 71, metadata: { matchedBy: "later_repo_pull_request" } }); + + expect( + classifyRecommendationOutcome({ + ...base, + run, + action: action(run, 4, { targetRepoFullName: "owner/issue-sort" }), + pullRequests: [], + issues: [ + issueRecord(82, "owner/issue-sort", { createdAt: undefined, updatedAt: "2026-05-03T00:00:00.000Z", authorLogin: "dev" }), + issueRecord(81, "owner/issue-sort", { createdAt: undefined, updatedAt: "2026-05-03T00:00:00.000Z", authorLogin: "DEV" }), + ], + }), + ).toMatchObject({ outcomeIssueNumber: 81, metadata: { matchedBy: "later_repo_issue" } }); + }); + + it("classifies open fallback targets and maintainer associations without losing source timestamps", () => { + const run = runRecord("run-open-fallbacks", "dev", "2026-05-01T00:00:00.000Z"); + const staleAfterMs = 14 * 24 * 60 * 60 * 1000; + const base = { + run, + evaluatedAt: "2026-05-05T00:00:00.000Z", + staleAfterMs, + ignoredAfterMs: 1, + }; + + expect( + classifyRecommendationOutcome({ + ...base, + action: action(run, 0, { targetRepoFullName: "owner/pr-fallback", targetPullNumber: 90 }), + pullRequests: [prRecord(90, "owner/pr-fallback", { createdAt: undefined, updatedAt: "2026-05-03T00:00:00.000Z", mergeableState: "clean" })], + issues: [], + }), + ).toMatchObject({ outcomeState: "improved", sourceUpdatedAt: "2026-05-03T00:00:00.000Z", metadata: { mergeableState: "clean" } }); + + expect( + classifyRecommendationOutcome({ + ...base, + action: action(run, 1, { targetRepoFullName: "owner/pr-created-fallback", targetPullNumber: 91 }), + pullRequests: [prRecord(91, "owner/pr-created-fallback", { createdAt: "2026-05-03T00:00:00.000Z", updatedAt: undefined })], + issues: [], + }), + ).toMatchObject({ outcomeState: "accepted", sourceUpdatedAt: "2026-05-03T00:00:00.000Z" }); + + expect( + classifyRecommendationOutcome({ + ...base, + action: action(run, 2, { targetRepoFullName: "owner/issue-fallback", targetIssueNumber: 92 }), + pullRequests: [], + issues: [issueRecord(92, "owner/issue-fallback", { createdAt: "2026-04-01T00:00:00.000Z", updatedAt: undefined })], + }), + ).toMatchObject({ outcomeState: "ignored", confidence: "medium", sourceUpdatedAt: "2026-04-01T00:00:00.000Z" }); + + expect( + classifyRecommendationOutcome({ + ...base, + action: action(run, 3, { targetRepoFullName: "owner/issue-updated-fallback", targetIssueNumber: 93 }), + pullRequests: [], + issues: [issueRecord(93, "owner/issue-updated-fallback", { createdAt: undefined, updatedAt: "2026-05-03T00:00:00.000Z" })], + }), + ).toMatchObject({ outcomeState: "accepted", sourceUpdatedAt: "2026-05-03T00:00:00.000Z" }); + + expect( + classifyRecommendationOutcome({ + ...base, + action: action(run, 4, { targetRepoFullName: "owner/merged-state", targetPullNumber: 94 }), + pullRequests: [prRecord(94, "owner/merged-state", { state: "merged", mergedAt: undefined, updatedAt: "2026-05-03T00:00:00.000Z" })], + issues: [], + }), + ).toMatchObject({ outcomeState: "merged", sourceUpdatedAt: "2026-05-03T00:00:00.000Z" }); + + expect( + classifyRecommendationOutcome({ + ...base, + action: action(run, 5, { targetRepoFullName: "owner/member", targetPullNumber: 95 }), + pullRequests: [prRecord(95, "owner/member", { authorAssociation: "MEMBER" })], + issues: [], + }), + ).toMatchObject({ maintainerLane: true }); + + expect( + classifyRecommendationOutcome({ + ...base, + action: action(run, 6, { targetRepoFullName: "owner/collab", targetIssueNumber: 96 }), + pullRequests: [], + issues: [issueRecord(96, "owner/collab", { authorAssociation: "COLLABORATOR" })], + }), + ).toMatchObject({ maintainerLane: true }); + }); + + it("maps legacy recommendation outcome rows to safe enum defaults", async () => { + const env = createTestEnv(); + const run = runRecord("legacy-run", "dev", "2026-05-01T00:00:00.000Z"); + const legacyAction = action(run, 0, { targetRepoFullName: "owner/legacy" }); + await createAgentRun(env, run); + await replaceAgentActions(env, run.id, [legacyAction]); + await upsertAgentRecommendationOutcome(env, { + actionId: legacyAction.id, + runId: run.id, + actorLogin: "dev", + actionType: "choose_next_work", + targetRepoFullName: "owner/legacy", + targetPullNumber: null, + targetIssueNumber: null, + outcomeState: "accepted", + outcomeTargetType: "repository", + outcomeRepoFullName: "owner/legacy", + outcomePullNumber: null, + outcomeIssueNumber: null, + maintainerLane: false, + confidence: "high", + reason: "legacy row fixture", + detectedAt: "2026-05-01T00:00:00.000Z", + metadata: {}, + }); + await env.DB.prepare( + "update agent_recommendation_outcomes set action_type = ?, outcome_state = ?, outcome_target_type = ?, confidence = ? where action_id = ?", + ) + .bind("legacy_action", "legacy_state", "legacy_target", "legacy_confidence", legacyAction.id) + .run(); + + const [row] = await listAgentRecommendationOutcomes(env, { actorLogin: "dev" }); + expect(row).toMatchObject({ + actionType: "choose_next_work", + outcomeState: "ignored", + outcomeTargetType: "none", + confidence: "medium", + }); + }); +}); + +function runRecord(id: string, actorLogin: string, createdAt: string): AgentRunRecord { + return { + id, + objective: "Plan the next Gittensor OSS contribution action.", + actorLogin, + surface: "api", + mode: "copilot", + status: "completed", + dataQualityStatus: "complete", + payload: { kind: "plan_next_work", login: actorLogin }, + createdAt, + updatedAt: createdAt, + }; +} + +function action(run: AgentRunRecord, index: number, overrides: Partial = {}): AgentActionRecord { + return { + id: `${run.id}:${String(index).padStart(2, "0")}:choose_next_work`, + runId: run.id, + actionType: "choose_next_work", + targetRepoFullName: "owner/repo", + status: "recommended", + recommendation: "Pick narrow work and validate it.", + why: ["The repo has cached opportunity signals."], + blockedBy: [], + publicSafeSummary: "Use local branch preflight before posting.", + approvalRequired: true, + safetyClass: "private", + payload: {}, + createdAt: "2026-05-01T00:00:00.000Z", + ...overrides, + }; +} + +function pr(number: number, overrides: Partial = {}): GitHubPullRequestPayload { + return { + number, + title: `PR ${number}`, + state: "open", + user: { login: "dev" }, + author_association: "CONTRIBUTOR", + html_url: `https://github.com/owner/repo/pull/${number}`, + body: "", + labels: [], + head: { sha: `sha-${number}`, ref: `branch-${number}` }, + base: { ref: "main" }, + created_at: "2026-05-02T00:00:00.000Z", + updated_at: "2026-05-02T00:00:00.000Z", + ...overrides, + } as GitHubPullRequestPayload; +} + +function prRecord(number: number, repoFullName: string, overrides: Partial = {}): PullRequestRecord { + return { + repoFullName, + number, + title: `PR ${number}`, + state: "open", + authorLogin: "dev", + authorAssociation: "CONTRIBUTOR", + htmlUrl: `https://github.com/${repoFullName}/pull/${number}`, + createdAt: "2026-05-03T00:00:00.000Z", + updatedAt: "2026-05-03T00:00:00.000Z", + labels: [], + linkedIssues: [], + ...overrides, + }; +} + +function issue(number: number, overrides: Partial = {}): GitHubIssuePayload { + return { + number, + title: `Issue ${number}`, + state: "open", + user: { login: "dev" }, + author_association: "CONTRIBUTOR", + html_url: `https://github.com/owner/repo/issues/${number}`, + body: "", + labels: [], + created_at: "2026-05-02T00:00:00.000Z", + updated_at: "2026-05-02T00:00:00.000Z", + ...overrides, + } as GitHubIssuePayload; +} + +function issueRecord(number: number, repoFullName: string, overrides: Partial = {}): IssueRecord { + return { + repoFullName, + number, + title: `Issue ${number}`, + state: "open", + authorLogin: "dev", + authorAssociation: "CONTRIBUTOR", + htmlUrl: `https://github.com/${repoFullName}/issues/${number}`, + createdAt: "2026-05-03T00:00:00.000Z", + updatedAt: "2026-05-03T00:00:00.000Z", + labels: [], + linkedPrs: [], + ...overrides, + }; +}