diff --git a/src/queue/processors.ts b/src/queue/processors.ts index 1d019a26a0..4f301b5212 100644 --- a/src/queue/processors.ts +++ b/src/queue/processors.ts @@ -1821,39 +1821,53 @@ export async function regatePullRequest( ); return; } - // #orb-retry-storm: record the repair attempt NOW — after rate-limit admission — so the cap in - // surfaceRepairPriorityPullNumbers counts actual executions, not queued dispatches that may have - // been deferred or dropped before running (the old dispatch-time recording let rate-limit deferrals - // exhaust the 2-attempt budget without ever trying the repair). - if (repairHeadSha) { - await recordAuditEvent(env, { - eventType: REGATE_REPAIR_ATTEMPT_EVENT_TYPE, - actor: "gittensory", - targetKey: regateRepairTargetKey(repoFullName, prNumber, repairHeadSha), - outcome: "completed", - detail: `outage-repair re-review executing for ${repoFullName}#${prNumber}`, - metadata: { repoFullName, prNumber, headSha: repairHeadSha }, - }); - } const settings = await resolveRepositorySettings(env, repoFullName); - await reReviewStoredPullRequest( - env, - deliveryId, - installationId, - repoFullName, - prNumber, - undefined, - // Run the AI review on the sweep for BOTH advisory and block modes (#sweep-all-modes) — only skip when AI is - // OFF. The #1462 per-(repo,pr,headSha,mode) cache bounds the cost: an unchanged PR re-gates from cache with no - // re-spend, so an advisory PR gets a posted review without burning a token every sweep tick. `force` (#regate- - // churn req 8) bypasses that cache/cooldown reuse entirely for an explicit manual re-gate request. - { - skipAiReview: settings.aiReviewMode === "off", - ...(force ? { force: true } : {}), - }, - ).catch((error) => { + // #orb-retry-storm/#5385-sentry (GITTENSORY-1E): record the repair attempt only once + // reReviewStoredPullRequest confirms it actually got PAST the review pipeline's own readiness gate + // (prReadyForReview) -- not merely once the job cleared rate-limit admission above. A PR legitimately + // waiting on a still-missing branch-protection-required check defers UNCONDITIONALLY and INDEFINITELY + // (prReadyForReview's own deliberate #3947 design -- there is no finalize escape for that specific case), + // so recording the OLD way (before ever checking readiness) charged a full attempt to a healthy PR doing + // nothing wrong on every ~2-minute sweep tick, exhausting the 5-attempt budget in ~10 minutes -- an order + // of magnitude shorter than realistic required-CI latency -- and firing a false "repair exhausted" alert + // for a review that was never actually broken. Mirrors the same "count executions, not deferrals" + // reasoning #orb-retry-storm already applied one layer out (rate-limit admission, above). + // + // `reachedReadiness` is set via reReviewStoredPullRequest's own onReachedReadiness callback -- NOT inferred + // from whether the call below returns vs. throws. A retryable error (GitHub rate limit / actuation-lock + // contention) can surface from real post-readiness work, and that is still a genuinely executed attempt that + // must consume the repair budget before the queue retries the message, or a repair stuck behind repeated + // contention could reselect indefinitely without ever exhausting. Conversely, an error thrown BEFORE + // readiness (e.g. a DB read failing) must NOT charge the budget for a pass that never got a real chance to + // review -- the callback (fired exactly once, right as the gate passes) is the only way to tell these apart + // once the call has thrown, since the boolean return value alone is lost on a throw. + let reachedReadiness = false; + try { + await reReviewStoredPullRequest( + env, + deliveryId, + installationId, + repoFullName, + prNumber, + undefined, + // Run the AI review on the sweep for BOTH advisory and block modes (#sweep-all-modes) — only skip when AI is + // OFF. The #1462 per-(repo,pr,headSha,mode) cache bounds the cost: an unchanged PR re-gates from cache with no + // re-spend, so an advisory PR gets a posted review without burning a token every sweep tick. `force` (#regate- + // churn req 8) bypasses that cache/cooldown reuse entirely for an explicit manual re-gate request. + { + skipAiReview: settings.aiReviewMode === "off", + ...(force ? { force: true } : {}), + onReachedReadiness: () => { + reachedReadiness = true; + }, + }, + ); + } catch (error) { /* v8 ignore next -- retryable/rate-limit propagation is exercised by queue retry tests; this catch only preserves that contract. */ - if (isGitHubRateLimitedError(error) || isRetryableJobError(error)) throw error; + if (isGitHubRateLimitedError(error) || isRetryableJobError(error)) { + // The finally block below still records the attempt (if reached) before this rethrow completes. + throw error; + } console.error( JSON.stringify({ level: "warn", @@ -1864,7 +1878,21 @@ export async function regatePullRequest( error: errorMessage(error), }), ); - }); + } finally { + // Best-effort, same as every other recordAuditEvent call in this file (`.catch(() => undefined)`) -- a + // failure writing THIS audit row must never replace a pending rethrown retryable error (or a normal + // return) with its own, which `finally` would otherwise do per JS semantics. + if (repairHeadSha && reachedReadiness) { + await recordAuditEvent(env, { + eventType: REGATE_REPAIR_ATTEMPT_EVENT_TYPE, + actor: "gittensory", + targetKey: regateRepairTargetKey(repoFullName, prNumber, repairHeadSha), + outcome: "completed", + detail: `outage-repair re-review executing for ${repoFullName}#${prNumber}`, + metadata: { repoFullName, prNumber, headSha: repairHeadSha }, + }).catch(() => undefined); + } + } } export function changedPathsForGuardrail( @@ -3050,6 +3078,18 @@ async function runAgentMaintenancePlanAndExecute( * re-run auto-maintain. Shared by the CI-completion (check_suite/check_run) handler below, mirroring reviewbot's * "the CI event WAKES the existing row and re-runs the full review". The PR's persisted head SHA is used as-is * (never overwritten from the CI payload — reviewbot scope parity). Best-effort throughout. + * + * Returns `true` once the review pipeline's own readiness gate (`prReadyForReview`, below) has actually + * passed and this call is genuinely proceeding with a real review/gate attempt; `false` for every early + * decline before that point (PR missing/closed, terminal-state reconcile, automation-bot skip, or + * `prReadyForReview` itself deferring — e.g. CI/required-context still pending). regatePullRequest (#5385- + * sentry, GITTENSORY-1E) uses this to only charge its bounded repair-attempt budget for a pass that actually + * got a chance to review, not one `prReadyForReview` correctly, harmlessly declined. + * + * `options.onReachedReadiness` fires the instant the gate passes, BEFORE any further (throwable) work runs — + * a side channel so a caller can still know readiness was reached even when this call later THROWS instead of + * returning (e.g. a retryable GitHub-rate-limit/lock-contention error surfacing from the post-readiness public- + * surface publish below). The `true`/`false` return value alone cannot carry that signal across a throw. */ export async function reReviewStoredPullRequest( env: Env, @@ -3058,14 +3098,14 @@ export async function reReviewStoredPullRequest( repoFullName: string, prNumber: number, previewPollAttempt?: number, - options: { skipAiReview?: boolean; force?: boolean } = {}, -): Promise { + options: { skipAiReview?: boolean; force?: boolean; onReachedReadiness?: () => void } = {}, +): Promise { const [repo, settings] = await Promise.all([ getRepository(env, repoFullName), resolveRepositorySettings(env, repoFullName), ]); let pr = await getPullRequest(env, repoFullName, prNumber); - if (!pr || pr.state !== "open") return; + if (!pr || pr.state !== "open") return false; const automationBotSkipEnabled = resolveSkipAutomationBotPullRequests( isSkipAutomationBotPullRequestsEnabledGlobally(env), settings.skipAutomationBotAuthors, @@ -3105,7 +3145,7 @@ export async function reReviewStoredPullRequest( if (current?.state === "open" && current.updatedAt === pr.updatedAt) { await upsertPullRequestFromGitHub(env, repoFullName, live).catch(() => undefined); } - return; + return false; } if (live?.head?.sha && live.head.sha !== pr.headSha) { await upsertPullRequestFromGitHub(env, repoFullName, live).catch( @@ -3125,7 +3165,7 @@ export async function reReviewStoredPullRequest( isTrustedAutomationBotAuthor(pr.authorLogin) && live?.head?.sha === storedHeadShaBeforeResync ) - return; + return false; // Operator review flow: rebase-if-behind → wait for ALL CI to finish → only THEN review. Defers (returns) when // a rebase fired a synchronize, or CI is still running — the synchronize / CI-completion webhook re-triggers // once the head is current and CI has settled (the sweep backstops a missed event). REST-budget dedup @@ -3152,7 +3192,10 @@ export async function reReviewStoredPullRequest( ), )) ) - return; + return false; + // Fire BEFORE any further (throwable) work below -- this is the one instant readiness is confirmed, so a + // caller learns it even if this call goes on to THROW instead of returning (see the JSDoc above). + options.onReachedReadiness?.(); const [cachedOtherOpenPullRequests, { linkedIssueAuthorLogins, confirmedNoOpenLinkedIssue }] = await Promise.all([ listOtherOpenPullRequests(env, repoFullName, prNumber), @@ -3268,6 +3311,7 @@ export async function reReviewStoredPullRequest( }), ); }); + return true; } /** diff --git a/test/unit/queue-2.test.ts b/test/unit/queue-2.test.ts index 48b2935e52..ee5f70fe5d 100644 --- a/test/unit/queue-2.test.ts +++ b/test/unit/queue-2.test.ts @@ -1877,6 +1877,178 @@ describe("queue processors", () => { expect(exhausted?.n).toBe(1); }, 60_000); + it("REGRESSION (#5385-sentry, GITTENSORY-1E): a repair dispatch that prReadyForReview correctly defers (missing required CI context) records NO repair_attempt at all, so it can never falsely exhaust the budget", async () => { + const env = createTestEnv({ GITHUB_APP_PRIVATE_KEY: await generatePrivateKeyPem() }); + await upsertInstallation(env, { action: "created", installation: { id: 9409, account: { login: "owner", id: 1, type: "Organization" }, target_type: "Organization", repository_selection: "selected", permissions: { pull_requests: "write", checks: "write" }, events: [] } }); + await upsertRepositoryFromGitHub(env, { name: "agent-repo", full_name: "owner/agent-repo", private: false, owner: { login: "owner" } }, 9409); + await upsertRepositorySettings(env, { repoFullName: "owner/agent-repo", autonomy: { merge: "auto" }, aiReviewMode: "off", gatePack: "oss-anti-slop", reviewCheckMode: "required", checkRunMode: "off", commentMode: "off", publicSurface: "off" }); + await upsertPullRequestFromGitHub(env, "owner/agent-repo", { number: 3, title: "Healthy PR, still waiting on required CI", state: "open", user: { login: "contributor" }, head: { sha: "pending-sha" }, base: { ref: "main" }, labels: [], body: "" }); + const targetKey = "owner/agent-repo#3#pending-sha"; + vi.setSystemTime(new Date("2026-05-28T02:00:00.000Z")); + // Same fixture shape as "keeps deferring a missing-required-context PR" (queue.test.ts): a required + // status check has simply not posted yet -- prReadyForReview's own #3947 design defers this + // UNCONDITIONALLY and INDEFINITELY (no finalize escape), by design, for exactly this case. + const requiredContextsSpy = vi.spyOn(backfillModule, "fetchRequiredStatusContexts").mockResolvedValue(null); + const liveCiSpy = vi.spyOn(backfillModule, "fetchLiveCiAggregatePreferGraphQl").mockResolvedValue({ + ciState: "pending", + hasPending: true, + hasVisiblePending: false, + hasMissingRequiredContext: true, + failingDetails: [], + nonRequiredFailingDetails: [], + ciCompletenessWarning: null, + }); + vi.stubGlobal("fetch", async (input: RequestInfo | URL, init?: RequestInit) => { + const url = input.toString(); + const method = (init?.method ?? "GET").toUpperCase(); + if (url.includes("/access_tokens")) return Response.json({ token: "installation-token" }); + if (/\/pulls\/3(?:\?|$)/.test(url) && method === "GET") return Response.json({ number: 3, title: "Healthy PR, still waiting on required CI", state: "open", user: { login: "contributor" }, head: { sha: "pending-sha" }, mergeable_state: "clean", labels: [], body: "" }); + if (url.includes("/pulls/3/files")) return Response.json([{ filename: "src/a.ts", status: "modified", additions: 1, deletions: 0, changes: 1, patch: "@@\n+export const ok = true;" }]); + return Response.json({}); + }); + + try { + // Simulate the sweep re-selecting this PR as an outage-repair priority candidate and re-dispatching a + // repair job for its (still current, still-pending) head SHA on every ~2-minute tick, well past what + // used to be the ~10-minute false-exhaustion window (5 ticks here). + for (let tick = 0; tick < 6; tick += 1) { + await processJob(env, { type: "agent-regate-pr", deliveryId: `regate-repair:owner/agent-repo#3:tick${tick}`, repoFullName: "owner/agent-repo", prNumber: 3, installationId: 9409, repairHeadSha: "pending-sha" }); + } + + const attempts = await env.DB.prepare("select count(*) as n from audit_events where event_type = ? and target_key = ?") + .bind("agent.sweep.regate.repair_attempt", targetKey) + .first<{ n: number }>(); + expect(attempts?.n).toBe(0); // never charged -- prReadyForReview declined before any attempt was recorded + const exhausted = await env.DB.prepare("select count(*) as n from audit_events where event_type = ? and target_key = ?") + .bind("agent.sweep.regate.repair_exhausted", targetKey) + .first<{ n: number }>(); + expect(exhausted?.n).toBe(0); // so the false "repair exhausted" alert never fires for a healthy, still-pending PR + } finally { + liveCiSpy.mockRestore(); + requiredContextsSpy.mockRestore(); + } + }, 60_000); + + it("REGRESSION (#5385-sentry, GITTENSORY-1E gate-finding): a retryable GitHub rate-limit error surfacing from real post-readiness review work STILL records the repair attempt before propagating for the queue's own retry", async () => { + // Gittensory review finding on PR #5482: the original fix recorded the attempt AFTER reReviewStoredPullRequest + // returns, so a retryable error thrown from a genuinely-executed (post-readiness) pass never got charged -- + // it propagates straight out (correctly, for the queue's own retry), but the repair budget was never + // decremented, letting a PR stuck behind repeated rate-limiting/lock-contention reselect indefinitely. + const env = createTestEnv({ GITHUB_APP_PRIVATE_KEY: await generatePrivateKeyPem() }); + await upsertInstallation(env, { action: "created", installation: { id: 9411, account: { login: "owner", id: 1, type: "Organization" }, target_type: "Organization", repository_selection: "selected", permissions: { pull_requests: "write", checks: "write" }, events: [] } }); + await upsertRepositoryFromGitHub(env, { name: "agent-repo3", full_name: "owner/agent-repo3", private: false, owner: { login: "owner" } }, 9411); + await upsertRepositorySettings(env, { repoFullName: "owner/agent-repo3", autonomy: { merge: "auto" }, aiReviewMode: "off", checkRunMode: "off", commentMode: "all_prs", publicSurface: "comment_only" }); + await upsertPullRequestFromGitHub(env, "owner/agent-repo3", { number: 5, title: "Healthy PR ready to review", state: "open", user: { login: "contributor" }, head: { sha: "ready-sha" }, base: { ref: "main" }, labels: [], body: "" }); + const targetKey = "owner/agent-repo3#5#ready-sha"; + let finalCommentAttempted = false; + vi.stubGlobal("fetch", async (input: RequestInfo | URL, init?: RequestInit) => { + const url = input.toString(); + const method = (init?.method ?? "GET").toUpperCase(); + if (url.includes("/access_tokens")) return Response.json({ token: "installation-token" }); + if (/\/pulls\/5(?:\?|$)/.test(url) && method === "GET") return Response.json({ number: 5, title: "Healthy PR ready to review", state: "open", user: { login: "contributor" }, head: { sha: "ready-sha" }, mergeable_state: "clean", labels: [], body: "" }); + if (url.includes("/pulls/5/files")) return Response.json([{ filename: "src/a.ts", status: "modified", additions: 1, deletions: 0, changes: 1, patch: "@@\n+export const ok = true;" }]); + if (url.includes("/commits/ready-sha/check-runs")) return Response.json({ total_count: 0, check_runs: [] }); + if (url.includes("/commits/ready-sha/status")) return Response.json({ state: "success", statuses: [] }); + if (url.includes("/issues/5/comments") && method === "GET") return Response.json([]); + if (url.includes("/issues/5/comments") && method === "POST") { + finalCommentAttempted = true; + return new Response(JSON.stringify({ message: "API rate limit exceeded" }), { status: 403, headers: { "x-ratelimit-remaining": "0" } }); + } + return Response.json({}); + }); + + await expect( + processJob(env, { type: "agent-regate-pr", deliveryId: "regate-repair-ratelimit", repoFullName: "owner/agent-repo3", prNumber: 5, installationId: 9411, repairHeadSha: "ready-sha" }), + ).rejects.toThrow(/rate limit/i); // still propagates -- the queue must still retry this message + + expect(finalCommentAttempted).toBe(true); // confirms the failure genuinely happened past readiness, mid real work + const attempts = await env.DB.prepare("select count(*) as n from audit_events where event_type = ? and target_key = ?") + .bind("agent.sweep.regate.repair_attempt", targetKey) + .first<{ n: number }>(); + expect(attempts?.n).toBe(1); // charged BEFORE the retryable error propagated -- the reported blocker + }); + + it("REGRESSION: a failing repair_attempt audit write in the finally block does NOT mask the original retryable error the queue needs to see", async () => { + // The finally block's own recordAuditEvent(...).catch(() => undefined) exists so a hiccup writing THIS + // audit row can never replace the pending rethrown rate-limit error with its own -- a `finally` that + // itself threw would otherwise silently swap in a non-retryable error, breaking the queue's retry + // classification for a failure that IS genuinely retryable. + const env = createTestEnv({ GITHUB_APP_PRIVATE_KEY: await generatePrivateKeyPem() }); + await upsertInstallation(env, { action: "created", installation: { id: 9413, account: { login: "owner", id: 1, type: "Organization" }, target_type: "Organization", repository_selection: "selected", permissions: { pull_requests: "write", checks: "write" }, events: [] } }); + await upsertRepositoryFromGitHub(env, { name: "agent-repo5", full_name: "owner/agent-repo5", private: false, owner: { login: "owner" } }, 9413); + await upsertRepositorySettings(env, { repoFullName: "owner/agent-repo5", autonomy: { merge: "auto" }, aiReviewMode: "off", checkRunMode: "off", commentMode: "all_prs", publicSurface: "comment_only" }); + await upsertPullRequestFromGitHub(env, "owner/agent-repo5", { number: 5, title: "Healthy PR ready to review", state: "open", user: { login: "contributor" }, head: { sha: "ready-sha" }, base: { ref: "main" }, labels: [], body: "" }); + const originalRecordAuditEvent = repositoriesModule.recordAuditEvent; + const auditSpy = vi.spyOn(repositoriesModule, "recordAuditEvent").mockImplementation(async (auditEnv, event) => { + if (event.eventType === "agent.sweep.regate.repair_attempt") throw new Error("audit DB down"); + await originalRecordAuditEvent(auditEnv, event); + }); + vi.stubGlobal("fetch", async (input: RequestInfo | URL, init?: RequestInit) => { + const url = input.toString(); + const method = (init?.method ?? "GET").toUpperCase(); + if (url.includes("/access_tokens")) return Response.json({ token: "installation-token" }); + if (/\/pulls\/5(?:\?|$)/.test(url) && method === "GET") return Response.json({ number: 5, title: "Healthy PR ready to review", state: "open", user: { login: "contributor" }, head: { sha: "ready-sha" }, mergeable_state: "clean", labels: [], body: "" }); + if (url.includes("/pulls/5/files")) return Response.json([{ filename: "src/a.ts", status: "modified", additions: 1, deletions: 0, changes: 1, patch: "@@\n+export const ok = true;" }]); + if (url.includes("/commits/ready-sha/check-runs")) return Response.json({ total_count: 0, check_runs: [] }); + if (url.includes("/commits/ready-sha/status")) return Response.json({ state: "success", statuses: [] }); + if (url.includes("/issues/5/comments") && method === "GET") return Response.json([]); + if (url.includes("/issues/5/comments") && method === "POST") { + return new Response(JSON.stringify({ message: "API rate limit exceeded" }), { status: 403, headers: { "x-ratelimit-remaining": "0" } }); + } + return Response.json({}); + }); + + try { + await expect( + processJob(env, { type: "agent-regate-pr", deliveryId: "regate-repair-ratelimit-audit-fail", repoFullName: "owner/agent-repo5", prNumber: 5, installationId: 9413, repairHeadSha: "ready-sha" }), + ).rejects.toThrow(/rate limit/i); // the ORIGINAL rate-limit error still wins, not "audit DB down" + } finally { + auditSpy.mockRestore(); + } + }); + + it("REGRESSION (#5385-sentry, GITTENSORY-1E nit): a swallowed non-retryable failure AFTER readiness still records the repair attempt (unchanged contract, now driven by the onReachedReadiness callback rather than inferred from any error reaching the catch)", async () => { + // Gittensory review nit on PR #5482: confirms the catch's non-retryable branch can't ALSO fire for an error + // thrown BEFORE readiness was ever reached (which must NOT charge the budget) -- distinguishing the two no + // longer relies on "any swallowed error here = post-readiness", but on the callback actually having fired. + const env = createTestEnv({ GITHUB_APP_PRIVATE_KEY: await generatePrivateKeyPem() }); + await upsertInstallation(env, { action: "created", installation: { id: 9412, account: { login: "owner", id: 1, type: "Organization" }, target_type: "Organization", repository_selection: "selected", permissions: {}, events: [] } }); + await upsertRepositoryFromGitHub(env, { name: "agent-repo4", full_name: "owner/agent-repo4", private: false, owner: { login: "owner" } }, 9412); + await upsertRepositorySettings(env, { repoFullName: "owner/agent-repo4", autonomy: { merge: "auto" }, aiReviewMode: "off", checkRunMode: "off", commentMode: "off", publicSurface: "off" }); + await upsertPullRequestFromGitHub(env, "owner/agent-repo4", { number: 6, title: "Healthy PR ready to review", state: "open", user: { login: "contributor" }, head: { sha: "ready-sha-2" }, base: { ref: "main" }, labels: [], body: "" }); + const targetKey = "owner/agent-repo4#6#ready-sha-2"; + const errors = vi.spyOn(console, "error").mockImplementation(() => undefined); + const realPrepare = env.DB.prepare.bind(env.DB); + // Same poison as "agent re-gate sweep ... swallows a failing re-review" above: only the advisories INSERT + // (persistAdvisory, which runs immediately after readiness passes) fails; every other read/write -- including + // this fix's own repair_attempt insert -- keeps working. + env.DB.prepare = ((sql: string) => { + if (/insert\s+into\s+["'`]?advisories/i.test(sql)) throw new Error("advisory persist failed"); + return realPrepare(sql); + }) as typeof env.DB.prepare; + vi.stubGlobal("fetch", async (input: RequestInfo | URL, init?: RequestInit) => { + const url = input.toString(); + const method = (init?.method ?? "GET").toUpperCase(); + if (url.includes("/access_tokens")) return Response.json({ token: "installation-token" }); + if (/\/pulls\/6(?:\?|$)/.test(url) && method === "GET") return Response.json({ number: 6, title: "Healthy PR ready to review", state: "open", user: { login: "contributor" }, head: { sha: "ready-sha-2" }, mergeable_state: "clean", labels: [], body: "" }); + if (url.includes("/pulls/6/files")) return Response.json([{ filename: "src/a.ts", status: "modified", additions: 1, deletions: 0, changes: 1, patch: "@@\n+export const ok = true;" }]); + if (url.includes("/commits/ready-sha-2/check-runs")) return Response.json({ total_count: 0, check_runs: [] }); + if (url.includes("/commits/ready-sha-2/status")) return Response.json({ state: "success", statuses: [] }); + return Response.json({}); + }); + + await expect( + processJob(env, { type: "agent-regate-pr", deliveryId: "regate-repair-advisory-fail", repoFullName: "owner/agent-repo4", prNumber: 6, installationId: 9412, repairHeadSha: "ready-sha-2" }), + ).resolves.toBeUndefined(); // swallowed, not rethrown -- matches the pre-existing non-retryable contract + + expect(errors.mock.calls.some((call) => String(call[0]).includes("sweep_rereview_failed"))).toBe(true); + const attempts = await realPrepare("select count(*) as n from audit_events where event_type = ? and target_key = ?") + .bind("agent.sweep.regate.repair_attempt", targetKey) + .first<{ n: number }>(); + expect(attempts?.n).toBe(1); // still charged -- readiness genuinely passed before persistAdvisory threw + errors.mockRestore(); + }); + it("agent re-gate sweep fail-opens when current Gate check reads fail during repair priority selection", async () => { const sent: import("../../src/types").JobMessage[] = []; const env = createTestEnv({ JOBS: { async send(m: import("../../src/types").JobMessage) { sent.push(m); } } as unknown as Queue });