From 4f2cd431d58bb50dbd34f82ec64c5542642057db Mon Sep 17 00:00:00 2001 From: YB0y Date: Wed, 17 Jun 2026 18:56:17 +0200 Subject: [PATCH 1/2] feat(scoring): wire TEST_FILE_CONTRIBUTION_WEIGHT and open-issue spam gate into scoring engine (#808) Co-Authored-By: YB0y --- src/scenarios/scenario-summary.ts | 1 + src/scoring/model.ts | 6 ++ src/scoring/preview.ts | 47 +++++++++++- test/unit/scoring.test.ts | 116 ++++++++++++++++++++++++++++++ 4 files changed, 167 insertions(+), 3 deletions(-) diff --git a/src/scenarios/scenario-summary.ts b/src/scenarios/scenario-summary.ts index 34cdbe4306..3e3408ccf6 100644 --- a/src/scenarios/scenario-summary.ts +++ b/src/scenarios/scenario-summary.ts @@ -78,6 +78,7 @@ const OPTION_NEXT_STEPS: Record = { const PUBLIC_BLOCKER_TEXT: Partial> = { base_token_gate: "The change size may be too small to meet the contribution threshold.", open_pr_threshold: "Too many concurrent open PRs exist; landing or closing some would help.", + open_issue_threshold: "Too many open issues exist; closing excess issues would help.", credibility_floor: "Contributor credibility evidence is below the expected floor.", review_penalty: "Review churn history may reduce the contribution quality signal.", metadata_only: "Only metadata signals are available; detailed analysis requires full context.", diff --git a/src/scoring/model.ts b/src/scoring/model.ts index 86f028c6ac..ba4491bacc 100644 --- a/src/scoring/model.ts +++ b/src/scoring/model.ts @@ -11,18 +11,24 @@ export const DEFAULT_SCORING_CONSTANTS: Record = { // Upstream name is ISSUES_TREASURY_EMISSION_SHARE (plural). The prior singular spelling never matched // upstream, freezing this at the local default and showing up as a false "unmodeled" drift warning (#806). ISSUES_TREASURY_EMISSION_SHARE: 0.1, + // Lookback window used upstream for PR history; stored so it syncs and does not surface as unmodeled drift. PR_LOOKBACK_DAYS: 30, MERGED_PR_BASE_SCORE: 25, // Upstream MAX_CONTRIBUTION_BONUS is 5. This local value is only the fetch-failure fallback; keeping it at // 25 silently 5x-inflated the contribution bonus whenever the upstream fetch failed (#807). MAX_CONTRIBUTION_BONUS: 5, CONTRIBUTION_SCORE_FOR_FULL_BONUS: 1500, + // Applied in preview.ts when computing totalTokenScore from components (#808). TEST_FILE_CONTRIBUTION_WEIGHT: 0.05, + // Upstream-enforced eligibility floors for PR and issue-discovery history (#808). + // These gate whether a validator counts a contributor's submissions, not the per-PR/issue score itself. + // Stored here so they sync from upstream and no longer appear as unmodeled drift warnings. MIN_VALID_MERGED_PRS: 3, MIN_CREDIBILITY: 0.8, MIN_VALID_SOLVED_ISSUES: 3, MIN_ISSUE_CREDIBILITY: 0.8, MIN_TOKEN_SCORE_FOR_VALID_ISSUE: 5, + // Open-issue spam gate constants — wired into the issue-discovery scoring lane in preview.ts (#808). OPEN_ISSUE_SPAM_BASE_THRESHOLD: 2, OPEN_ISSUE_SPAM_TOKEN_SCORE_PER_SLOT: 300, MAX_OPEN_ISSUE_THRESHOLD: 30, diff --git a/src/scoring/preview.ts b/src/scoring/preview.ts index d358cedb4b..cf1790105b 100644 --- a/src/scoring/preview.ts +++ b/src/scoring/preview.ts @@ -16,6 +16,8 @@ export type ScorePreviewInput = { nonCodeTokenScore?: number | undefined; existingContributorTokenScore?: number | undefined; openPrCount?: number | undefined; + /** Contributor's current open-issue count for the repo, used for the open-issue spam gate (#808). */ + openIssueCount?: number | undefined; credibility?: number | undefined; changesRequestedCount?: number | undefined; fixedBaseScore?: number | undefined; @@ -101,6 +103,7 @@ export type ScoreGateBlocker = { | "inactive_allocation" | "base_token_gate" | "open_pr_threshold" + | "open_issue_threshold" | "credibility_floor" | "review_penalty" | "metadata_only" @@ -115,7 +118,7 @@ export type ScoreGateBlocker = { }; export type ScoreGateDelta = { - gate: "open_pr_threshold" | "credibility_floor" | "linked_issue_multiplier"; + gate: "open_pr_threshold" | "open_issue_threshold" | "credibility_floor" | "linked_issue_multiplier"; current: string; projected: string; explanation: string; @@ -157,6 +160,7 @@ export type ScorePreviewResult = { credibilityMultiplier: number; reviewPenaltyMultiplier: number; openPrMultiplier: number; + openIssueMultiplier: number; /** Upstream sigmoid time-decay multiplier (#703). 1 = no decay (fresh PR, or feature off). */ timeDecayMultiplier: number; estimatedMergedScore: number; @@ -170,6 +174,8 @@ export type ScorePreviewResult = { collateralFraction: number; credibilityFloor: number; credibilityObserved: number; + openIssueThreshold: number; + openIssueCount: number; }; branchEligibility: BranchEligibilityResult; effectiveEstimatedScore: number; @@ -204,6 +210,7 @@ export function buildScorePreview(args: { const actions = [ ...(!current.gates.baseTokenGatePassed ? ["Increase meaningful source change size or scope clarity before relying on this preview."] : []), ...(current.scoreEstimate.openPrMultiplier === 0 ? ["Land or close existing open PRs before opening more concurrent work."] : []), + ...(current.scoreEstimate.openIssueMultiplier === 0 ? ["Close excess open issues to stay within the open-issue spam threshold."] : []), ...(current.scoreEstimate.credibilityMultiplier < 1 ? ["Build or wait for contributor credibility evidence before relying on this preview."] : []), ...(current.scoreEstimate.reviewPenaltyMultiplier < 1 ? ["Reduce review churn with tighter tests and clearer evidence."] : []), ...(current.scoreEstimate.labelMultiplier <= 1 && Object.keys(args.repo?.registryConfig?.labelMultipliers ?? {}).length > 0 @@ -288,7 +295,10 @@ function computeScoreCore( const directPrSlice = repoSlice * (1 - issueDiscoveryShare); const issueDiscoverySlice = repoSlice * issueDiscoveryShare; const sourceTokenScore = nonNegative(input.sourceTokenScore); - const totalTokenScore = nonNegative(input.totalTokenScore ?? sourceTokenScore + nonNegative(input.testTokenScore) + nonNegative(input.nonCodeTokenScore)); + // TEST_FILE_CONTRIBUTION_WEIGHT (#808): upstream weights test-file tokens at 0.05× relative to source tokens. + // Applied only when totalTokenScore is not explicitly provided — an explicit caller total is honoured as-is. + const testFileWeight = constant(constants, "TEST_FILE_CONTRIBUTION_WEIGHT", 0.05); + const totalTokenScore = nonNegative(input.totalTokenScore ?? sourceTokenScore + testFileWeight * nonNegative(input.testTokenScore) + nonNegative(input.nonCodeTokenScore)); const sourceLines = Math.max(1, nonNegative(input.sourceLines ?? sourceTokenScore)); const fixedBaseScore = input.fixedBaseScore ?? config?.fixedBaseScore ?? undefined; const rawDensity = sourceTokenScore / sourceLines; @@ -325,13 +335,22 @@ function computeScoreCore( Math.floor(nonNegative(input.existingContributorTokenScore) / constant(constants, "OPEN_PR_THRESHOLD_TOKEN_SCORE", 300)), ); const openPrMultiplier = openPrCount <= openPrThreshold ? 1 : 0; + // Open-issue spam gate (#808): mirrors the open-PR gate for the issue-discovery channel. + // A contributor earns extra open-issue slots from their existing merged-history token score. + const openIssueCount = nonNegative(input.openIssueCount); + const openIssueThreshold = Math.min( + constant(constants, "MAX_OPEN_ISSUE_THRESHOLD", 30), + constant(constants, "OPEN_ISSUE_SPAM_BASE_THRESHOLD", 2) + + Math.floor(nonNegative(input.existingContributorTokenScore) / constant(constants, "OPEN_ISSUE_SPAM_TOKEN_SCORE_PER_SLOT", 300)), + ); + const openIssueMultiplier = openIssueCount <= openIssueThreshold ? 1 : 0; // Upstream time-decay (#703): mirrors upstream's `scored.time_decay_multiplier` applied to a PR's score. // Opt-in + env-gated (default off). A fresh PR (prAgeHours below the grace period) yields 1.0, so a normal // new-PR preview is unchanged even when enabled — only an aged-PR projection decays. // Per-repo curve (#703): the repo's registry `scoring.time_decay` overrides overlay the snapshot defaults. const timeDecayMultiplier = input.applyTimeDecay ? calculateTimeDecay(nonNegative(input.prAgeHours), constants, config?.timeDecay) : 1; const estimatedMergedScore = roundScore( - baseScore * labelMultiplier * issueMultiplier * credibilityMultiplier * reviewPenaltyMultiplier * openPrMultiplier * timeDecayMultiplier, + baseScore * labelMultiplier * issueMultiplier * credibilityMultiplier * reviewPenaltyMultiplier * openPrMultiplier * openIssueMultiplier * timeDecayMultiplier, ); const pendingSaturationScore = roundScore(saturationBaseScore); return { @@ -352,6 +371,7 @@ function computeScoreCore( credibilityMultiplier: roundScore(credibilityMultiplier), reviewPenaltyMultiplier: roundScore(reviewPenaltyMultiplier), openPrMultiplier, + openIssueMultiplier, timeDecayMultiplier: roundScore(timeDecayMultiplier), estimatedMergedScore, pendingSaturationScore, @@ -364,6 +384,8 @@ function computeScoreCore( collateralFraction: constant(constants, "OPEN_PR_COLLATERAL_PERCENT", 0.2), credibilityFloor, credibilityObserved, + openIssueThreshold, + openIssueCount, }, }; } @@ -571,6 +593,15 @@ function blockedByFor(input: ScorePreviewInput, repo: RepositoryRecord | null, c }, ] : []), + ...(core.scoreEstimate.openIssueMultiplier === 0 + ? [ + { + code: "open_issue_threshold" as const, + severity: "blocker" as const, + detail: `Open issue count ${core.gates.openIssueCount} exceeds spam threshold ${core.gates.openIssueThreshold}.`, + }, + ] + : []), ...(core.gates.credibilityObserved < core.gates.credibilityFloor ? [ { @@ -643,6 +674,16 @@ function buildGateDeltas(current: ScoreCore, scenarios: ScoreScenarioPreview[]): }, ] : []), + ...(current.scoreEstimate.openIssueMultiplier !== best.scoreEstimate.openIssueMultiplier || current.gates.openIssueCount !== best.gates.openIssueCount + ? [ + { + gate: "open_issue_threshold" as const, + current: `${current.gates.openIssueCount}/${current.gates.openIssueThreshold} open issues, multiplier ${current.scoreEstimate.openIssueMultiplier}`, + projected: `${best.gates.openIssueCount}/${best.gates.openIssueThreshold} open issues, multiplier ${best.scoreEstimate.openIssueMultiplier}`, + explanation: `Open issue spam pressure changes estimated score ${current.scoreEstimate.estimatedMergedScore} -> ${best.scoreEstimate.estimatedMergedScore}.`, + }, + ] + : []), ...(current.gates.credibilityObserved !== best.gates.credibilityObserved || current.scoreEstimate.credibilityMultiplier !== best.scoreEstimate.credibilityMultiplier ? [ { diff --git a/test/unit/scoring.test.ts b/test/unit/scoring.test.ts index fc05c09756..59e339bbb1 100644 --- a/test/unit/scoring.test.ts +++ b/test/unit/scoring.test.ts @@ -740,6 +740,122 @@ MAX_CODE_DENSITY_MULTIPLIER = 1.15 expect(thrownFallback.activeModel).toBe("unknown"); }); + describe("issue-discovery scoring constants (#808)", () => { + it("TEST_FILE_CONTRIBUTION_WEIGHT weights test tokens at 0.05× when totalTokenScore is derived from components", () => { + const snapshotWith808 = { ...snapshot, constants: { ...snapshot.constants, TEST_FILE_CONTRIBUTION_WEIGHT: 0.05 } }; + const base = buildScorePreview({ + repo, + snapshot: snapshotWith808, + input: { repoFullName: repo.fullName, sourceTokenScore: 60, sourceLines: 50, openPrCount: 0, credibility: 1 }, + }); + // With testTokenScore=200, the derived total should be 60 + 0.05*200 = 70 (not 260). + const withTest = buildScorePreview({ + repo, + snapshot: snapshotWith808, + input: { repoFullName: repo.fullName, sourceTokenScore: 60, testTokenScore: 200, sourceLines: 50, openPrCount: 0, credibility: 1 }, + }); + // Contribution bonus ramp is based on totalTokenScore; 70 vs 60 produces a slightly higher bonus. + expect(withTest.scoreEstimate.contributionBonus).toBeGreaterThan(base.scoreEstimate.contributionBonus); + // But an explicit totalTokenScore overrides the weight completely — caller-supplied value is honoured as-is. + const explicit = buildScorePreview({ + repo, + snapshot: snapshotWith808, + input: { repoFullName: repo.fullName, sourceTokenScore: 60, testTokenScore: 200, totalTokenScore: 260, sourceLines: 50, openPrCount: 0, credibility: 1 }, + }); + // explicit 260 is LARGER than weighted 70; contribution bonus must be greater. + expect(explicit.scoreEstimate.contributionBonus).toBeGreaterThan(withTest.scoreEstimate.contributionBonus); + }); + + it("open-issue spam gate blocks scoring when openIssueCount exceeds the threshold", () => { + const snapshotWith808 = { + ...snapshot, + constants: { + ...snapshot.constants, + OPEN_ISSUE_SPAM_BASE_THRESHOLD: 2, + OPEN_ISSUE_SPAM_TOKEN_SCORE_PER_SLOT: 300, + MAX_OPEN_ISSUE_THRESHOLD: 30, + }, + }; + const baseInput = { repoFullName: repo.fullName, sourceTokenScore: 60, totalTokenScore: 90, sourceLines: 50, openPrCount: 0, credibility: 1, existingContributorTokenScore: 0 }; + + // At the threshold (2) — gate passes. + const atThreshold = buildScorePreview({ repo, snapshot: snapshotWith808, input: { ...baseInput, openIssueCount: 2 } }); + expect(atThreshold.gates.openIssueThreshold).toBe(2); + expect(atThreshold.gates.openIssueCount).toBe(2); + expect(atThreshold.scoreEstimate.openIssueMultiplier).toBe(1); + expect(atThreshold.effectiveEstimatedScore).toBeGreaterThan(0); + + // One over the threshold — gate blocks. + const overThreshold = buildScorePreview({ repo, snapshot: snapshotWith808, input: { ...baseInput, openIssueCount: 3 } }); + expect(overThreshold.scoreEstimate.openIssueMultiplier).toBe(0); + expect(overThreshold.effectiveEstimatedScore).toBe(0); + expect(overThreshold.blockedBy.some((b) => b.code === "open_issue_threshold")).toBe(true); + expect(overThreshold.blockedBy.find((b) => b.code === "open_issue_threshold")?.severity).toBe("blocker"); + }); + + it("open-issue threshold scales with established merged-history token score", () => { + const snapshotWith808 = { + ...snapshot, + constants: { ...snapshot.constants, OPEN_ISSUE_SPAM_BASE_THRESHOLD: 2, OPEN_ISSUE_SPAM_TOKEN_SCORE_PER_SLOT: 300, MAX_OPEN_ISSUE_THRESHOLD: 30 }, + }; + // No history: base 2 + floor(0/300) = 2. + const noHistory = buildScorePreview({ + repo, snapshot: snapshotWith808, + input: { repoFullName: repo.fullName, sourceTokenScore: 60, totalTokenScore: 90, sourceLines: 50, openPrCount: 0, credibility: 1, existingContributorTokenScore: 0, openIssueCount: 3 }, + }); + expect(noHistory.gates.openIssueThreshold).toBe(2); + expect(noHistory.scoreEstimate.openIssueMultiplier).toBe(0); + + // With 900 tokens of history: 2 + floor(900/300) = 5. + const withHistory = buildScorePreview({ + repo, snapshot: snapshotWith808, + input: { repoFullName: repo.fullName, sourceTokenScore: 60, totalTokenScore: 90, sourceLines: 50, openPrCount: 0, credibility: 1, existingContributorTokenScore: 900, openIssueCount: 3 }, + }); + expect(withHistory.gates.openIssueThreshold).toBe(5); + expect(withHistory.scoreEstimate.openIssueMultiplier).toBe(1); // 3 <= 5 + }); + + it("open-issue gate defaults to 0 issues when openIssueCount is not supplied (never blocks)", () => { + const preview = buildScorePreview({ + repo, + snapshot, + input: { repoFullName: repo.fullName, sourceTokenScore: 60, totalTokenScore: 90, sourceLines: 50, openPrCount: 0, credibility: 1 }, + }); + expect(preview.gates.openIssueCount).toBe(0); + expect(preview.scoreEstimate.openIssueMultiplier).toBe(1); + }); + + it("MAX_OPEN_ISSUE_THRESHOLD caps the issue allowance even with a very large token history", () => { + const snapshotWith808 = { + ...snapshot, + constants: { ...snapshot.constants, OPEN_ISSUE_SPAM_BASE_THRESHOLD: 2, OPEN_ISSUE_SPAM_TOKEN_SCORE_PER_SLOT: 300, MAX_OPEN_ISSUE_THRESHOLD: 5 }, + }; + // Even with a huge token history, the threshold cannot exceed MAX_OPEN_ISSUE_THRESHOLD (5). + const preview = buildScorePreview({ + repo, snapshot: snapshotWith808, + input: { repoFullName: repo.fullName, sourceTokenScore: 60, totalTokenScore: 90, sourceLines: 50, openPrCount: 0, credibility: 1, existingContributorTokenScore: 90000, openIssueCount: 6 }, + }); + expect(preview.gates.openIssueThreshold).toBe(5); + expect(preview.scoreEstimate.openIssueMultiplier).toBe(0); // 6 > 5 + }); + + it("all nine issue-discovery constants are modeled and do not surface as upstream drift warnings (#808)", () => { + const upstreamSource = [ + "TEST_FILE_CONTRIBUTION_WEIGHT = 0.05", + "MIN_VALID_MERGED_PRS = 3", + "MIN_VALID_SOLVED_ISSUES = 3", + "MIN_ISSUE_CREDIBILITY = 0.8", + "MIN_TOKEN_SCORE_FOR_VALID_ISSUE = 5", + "OPEN_ISSUE_SPAM_BASE_THRESHOLD = 2", + "OPEN_ISSUE_SPAM_TOKEN_SCORE_PER_SLOT = 300", + "MAX_OPEN_ISSUE_THRESHOLD = 30", + "PR_LOOKBACK_DAYS = 30", + ].join("\n"); + const unmodeled = findUnmodeledUpstreamConstants(upstreamSource); + expect(unmodeled).toEqual([]); + }); + }); + describe("upstream time-decay (#703)", () => { it("calculateTimeDecay matches the upstream sigmoid (grace, 50%-at-midpoint, floor, monotonic)", () => { const c = DEFAULT_SCORING_CONSTANTS; From 2db1d3c287323996d083437664077eaf412fbf79 Mon Sep 17 00:00:00 2001 From: YB0y Date: Thu, 18 Jun 2026 22:05:10 +0200 Subject: [PATCH 2/2] fix: codecov test failing --- src/scoring/preview.ts | 5 ++++- test/unit/scoring.test.ts | 27 +++++++++++++++++++++++++++ 2 files changed, 31 insertions(+), 1 deletion(-) diff --git a/src/scoring/preview.ts b/src/scoring/preview.ts index cf1790105b..e1752f2911 100644 --- a/src/scoring/preview.ts +++ b/src/scoring/preview.ts @@ -446,6 +446,9 @@ function buildScenarioPreviews( input.expectedOpenPrCountAfterMerge !== undefined ? expectedOpenPrCountAfterMerge : Math.max(0, current.gates.openPrCount - combinedPendingCount), current.gates.openPrThreshold, ), + // Project open-issue spam cleanup (#808): mirror the open-PR projection so the + // "best reasonable case" can clear the open-issue gate just like it clears open-PR pressure. + openIssueCount: Math.min(current.gates.openIssueCount, current.gates.openIssueThreshold), credibility: Math.max(projectedCredibility, observedApprovalCredibility, current.gates.credibilityFloor), }; return [ @@ -510,7 +513,7 @@ function buildScenarioPreviews( : "Linked issue mode was already supplied; this scenario projects solved-by-PR validation where needed.", ], repo), scenario("bestReasonableCase", "gittensory_projection", bestReasonableInput, computeScoreCore(bestReasonableInput, repo, snapshot, contributorEvidence), [ - "Combines plausible near-term gate cleanup: open PR pressure at threshold or below, credibility at floor or above, and linked-issue context where applicable.", + "Combines plausible near-term gate cleanup: open PR pressure at threshold or below, open-issue spam pressure at threshold or below, credibility at floor or above, and linked-issue context where applicable.", ...(input.scenarioNotes ?? []), ...observedScenarioNotes(input), ], repo), diff --git a/test/unit/scoring.test.ts b/test/unit/scoring.test.ts index 59e339bbb1..2c22199b21 100644 --- a/test/unit/scoring.test.ts +++ b/test/unit/scoring.test.ts @@ -839,6 +839,33 @@ MAX_CODE_DENSITY_MULTIPLIER = 1.15 expect(preview.scoreEstimate.openIssueMultiplier).toBe(0); // 6 > 5 }); + it("bestReasonableCase clears the open-issue gate and surfaces an open_issue_threshold gate delta (#808)", () => { + const snapshotWith808 = { + ...snapshot, + constants: { ...snapshot.constants, OPEN_ISSUE_SPAM_BASE_THRESHOLD: 2, OPEN_ISSUE_SPAM_TOKEN_SCORE_PER_SLOT: 300, MAX_OPEN_ISSUE_THRESHOLD: 30 }, + }; + // Current state is over the threshold (3 > 2) so the gate blocks the live preview. + const preview = buildScorePreview({ + repo, snapshot: snapshotWith808, + input: { repoFullName: repo.fullName, sourceTokenScore: 60, totalTokenScore: 90, sourceLines: 50, openPrCount: 0, credibility: 1, existingContributorTokenScore: 0, openIssueCount: 3 }, + }); + expect(preview.scoreEstimate.openIssueMultiplier).toBe(0); + expect(preview.effectiveEstimatedScore).toBe(0); + // The best-reasonable-case scenario projects the open-issue count down to the threshold (2), + // clearing the gate so the underlying potential is visible there. + const bestReasonable = preview.scenarioPreviews.find((scenario) => scenario.name === "bestReasonableCase"); + expect(bestReasonable?.gates.openIssueCount).toBe(2); + expect(bestReasonable?.gates.openIssueThreshold).toBe(2); + expect(bestReasonable?.scoreEstimate.openIssueMultiplier).toBe(1); + expect(bestReasonable?.effectiveEstimatedScore).toBeGreaterThan(0); + // Because the multiplier differs between current and best-reasonable-case, an open_issue_threshold + // gate delta must be emitted — this is the branch previously missing coverage. + expect(preview.gateDeltas).toEqual(expect.arrayContaining([expect.objectContaining({ gate: "open_issue_threshold" })])); + const issueDelta = preview.gateDeltas.find((delta) => delta.gate === "open_issue_threshold"); + expect(issueDelta?.current).toContain("multiplier 0"); + expect(issueDelta?.projected).toContain("multiplier 1"); + }); + it("all nine issue-discovery constants are modeled and do not surface as upstream drift warnings (#808)", () => { const upstreamSource = [ "TEST_FILE_CONTRIBUTION_WEIGHT = 0.05",