From 630e975a3bbac05d737de4bc99246f24c65115cd Mon Sep 17 00:00:00 2001 From: Lang-bt Date: Tue, 14 Jul 2026 10:15:59 -0700 Subject: [PATCH 1/2] feat(miner): honor kill-switch mid-attempt during iterate-loop (#5670) Probe shouldAbort before each driver iteration and re-resolve kill after handoff so a tripped switch stops the rented loop without opening a PR, and halt the outer loop immediately with a released queue claim. --- packages/loopover-engine/src/index.ts | 1 + .../loopover-engine/src/miner/iterate-loop.ts | 77 ++++++++++++++++++- .../src/miner/iterate-policy.ts | 10 ++- .../loopover-engine/test/iterate-loop.test.ts | 50 ++++++++++++ packages/loopover-miner/lib/attempt-cli.js | 44 ++++++++++- .../loopover-miner/lib/attempt-runner.d.ts | 4 + packages/loopover-miner/lib/attempt-runner.js | 25 ++++++ packages/loopover-miner/lib/loop-cli.js | 42 +++++++++- test/unit/miner-attempt-runner.test.ts | 32 ++++++++ test/unit/miner-loop-cli.test.ts | 46 +++++++++++ 10 files changed, 324 insertions(+), 7 deletions(-) diff --git a/packages/loopover-engine/src/index.ts b/packages/loopover-engine/src/index.ts index 1e7ebe0d83..da048188f9 100644 --- a/packages/loopover-engine/src/index.ts +++ b/packages/loopover-engine/src/index.ts @@ -263,6 +263,7 @@ export { export { runIterateLoop, type IterateLoopDeps, + type IterateLoopShouldAbort, type IterateLoopInput, type IterateLoopIterationRecord, type IterateLoopOutcome, diff --git a/packages/loopover-engine/src/miner/iterate-loop.ts b/packages/loopover-engine/src/miner/iterate-loop.ts index 1f4df3219c..70f5ad3b8d 100644 --- a/packages/loopover-engine/src/miner/iterate-loop.ts +++ b/packages/loopover-engine/src/miner/iterate-loop.ts @@ -86,6 +86,15 @@ export type IterateLoopInput = { rejectionSignaled: boolean; }; +/** Optional cooperative abort probed BEFORE every driver invocation (#5670). A bare `true` or + * `{ abort: true }` abandons with `kill_switch_engaged` without calling the driver for that iteration. */ +export type IterateLoopShouldAbort = + | boolean + | { + abort: boolean; + reason?: string | undefined; + }; + export type IterateLoopDeps = { driver: CodingAgentDriver; runSlopAssessment: SelfReviewAdapterDeps["runSlopAssessment"]; @@ -94,6 +103,8 @@ export type IterateLoopDeps = { * injected-dependency discipline elsewhere (never a hardcoded `Date.now()` a test can't control). Defaults * to the real `Date.now` when omitted. */ nowMs?: (() => number) | undefined; + /** Mid-iteration kill-switch / pause probe (#5670). Omitted = never abort mid-loop (pre-#5670 behavior). */ + shouldAbort?: (() => IterateLoopShouldAbort) | undefined; }; /** The terminal outcomes a full loop run can end in -- never `"continue"`, which is only ever a per-iteration, @@ -186,10 +197,41 @@ function attemptLogEventTypeForDecision(decision: IterateLoopDecision): AttemptL // reads as aborted; a genuine failure to converge (ceiling reached, or stuck with no progress) reads as // failed. Both are still `action: "abandon"` in the decision itself -- this is only a coarser attempt-log // classification layered on top, for the fixed six-value ATTEMPT_LOG_EVENT_TYPES vocabulary. - if (decision.abandonReason === "rejection_signaled" || decision.abandonReason === "self_review_ambiguous") return "attempt_aborted"; + if ( + decision.abandonReason === "rejection_signaled" || + decision.abandonReason === "self_review_ambiguous" || + decision.abandonReason === "kill_switch_engaged" + ) { + return "attempt_aborted"; + } return "attempt_failed"; } +function resolveShouldAbort(deps: IterateLoopDeps): { abort: boolean; reason: string } { + if (typeof deps.shouldAbort !== "function") { + return { abort: false, reason: "" }; + } + const raw = deps.shouldAbort(); + if (typeof raw === "boolean") { + return { + abort: raw, + reason: raw + ? "Kill-switch engaged mid-attempt; abandoning without starting another driver iteration." + : "", + }; + } + if (raw && typeof raw === "object" && raw.abort === true) { + return { + abort: true, + reason: + typeof raw.reason === "string" && raw.reason.trim() + ? raw.reason.trim() + : "Kill-switch engaged mid-attempt; abandoning without starting another driver iteration.", + }; + } + return { abort: false, reason: "" }; +} + /** A logging failure must never crash the loop or alter its decision -- mirrors the governor-ledger and * pretooluse-hook append-failure handling elsewhere in this package. */ function safeAppendAttemptLogEvent(deps: IterateLoopDeps, event: AttemptLogEvent): void { @@ -318,6 +360,39 @@ async function runIterateLoopCore(input: IterateLoopInput, deps: IterateLoopDeps let totalCostUsd = 0; for (let iterationNumber = 1; iterationNumber <= maxIterations; iterationNumber += 1) { + // Cooperative mid-iteration halt (#5670): probed BEFORE each driver call so a kill-switch that trips + // after iteration N prevents iteration N+1 (and prevents the first iteration when already tripped). + // Hard SIGKILL of an in-flight driver call is intentionally out of scope — matching #5437's budget + // abort, which also stops between iterations rather than interrupting a running LLM turn. + const abort = resolveShouldAbort(deps); + if (abort.abort) { + const decision: IterateLoopDecision = { + action: "abandon", + abandonReason: "kill_switch_engaged", + reason: abort.reason, + }; + safeAppendAttemptLogEvent(deps, { + eventType: "attempt_aborted", + attemptId: input.attemptId, + actionClass: "iterate_loop", + mode: input.mode, + reason: decision.reason, + payload: { + iterationNumber: iterationNumber - 1, + action: decision.action, + abandonReason: decision.abandonReason, + }, + }); + return { + outcome: "abandon", + finalDecision: decision, + iterationsUsed: iterationNumber - 1, + totalTurnsUsed, + totalCostUsd, + iterations, + }; + } + const iterationStartMs = nowMs(); const driverResult = await runDriverSafely(input, deps, { attemptId: input.attemptId, diff --git a/packages/loopover-engine/src/miner/iterate-policy.ts b/packages/loopover-engine/src/miner/iterate-policy.ts index b7e9a13ee7..4d1ee4dcbf 100644 --- a/packages/loopover-engine/src/miner/iterate-policy.ts +++ b/packages/loopover-engine/src/miner/iterate-policy.ts @@ -27,7 +27,15 @@ export type IterateLoopAction = "continue" | "handoff" | "abandon"; /** Every distinct reason `decideNextAction` can abandon for -- kept as a closed literal union so a caller * recording the decision (the attempt-log primitive, per #2333) has a stable, exhaustive vocabulary. */ -export type AbandonReason = "rejection_signaled" | "self_review_ambiguous" | "max_iterations_reached" | "cost_ceiling_reached" | "no_progress"; +export type AbandonReason = + | "rejection_signaled" + | "self_review_ambiguous" + | "max_iterations_reached" + | "cost_ceiling_reached" + | "no_progress" + /** Mid-attempt emergency stop (#5670): kill-switch (or operator pause acting as a stop signal) tripped + * between iterate-loop iterations — cooperative, not a hard SIGKILL of an in-flight driver call. */ + | "kill_switch_engaged"; /** * The self-review outcome as the policy needs it -- narrower than the full {@link SelfReviewVerdict} (self- diff --git a/packages/loopover-engine/test/iterate-loop.test.ts b/packages/loopover-engine/test/iterate-loop.test.ts index 28076271fa..1ba18a6e22 100644 --- a/packages/loopover-engine/test/iterate-loop.test.ts +++ b/packages/loopover-engine/test/iterate-loop.test.ts @@ -406,3 +406,53 @@ test("a logging failure never crashes the loop or alters its decision", async () assert.equal(result.outcome, "handoff", "the tool call is still decided correctly even though every audit write failed"); }); + +test("abandon (kill_switch_engaged): shouldAbort before the first driver call abandons with zero iterations (#5670)", async () => { + let driverCalled = false; + const { deps, events } = collectingDeps({ + driver: { + async run() { + driverCalled = true; + return okResult(); + }, + }, + shouldAbort: () => true, + }); + const result = await runIterateLoop(passingInput({ maxIterations: 3 }), deps); + + assert.equal(result.outcome, "abandon"); + assert.equal(result.finalDecision.abandonReason, "kill_switch_engaged"); + assert.equal(result.iterationsUsed, 0); + assert.equal(driverCalled, false); + assert.equal(events.some((event) => event.eventType === "attempt_aborted"), true); +}); + +test("abandon (kill_switch_engaged): shouldAbort after iteration 1 prevents iteration 2 (#5670 mid-iteration)", async () => { + let probes = 0; + let callCount = 0; + // Same duplicate-PR fixture as the no_progress test: iter 1 fails predicted-gate and continues. + const pullRequests: PullRequestRecord[] = [openPr(42, "Retry uploads on 5xx responses", [7])]; + const { deps } = collectingDeps({ + driver: { + async run() { + callCount += 1; + return okResult(["src/upload.ts"], 2); + }, + }, + shouldAbort: () => { + probes += 1; + return probes > 1 ? { abort: true, reason: "operator tripped kill mid-run" } : false; + }, + }); + const result = await runIterateLoop( + passingInput({ maxIterations: 5, reviewContext: baseReviewContext({ pullRequests }) }), + deps, + ); + + assert.equal(result.outcome, "abandon"); + assert.equal(result.finalDecision.abandonReason, "kill_switch_engaged"); + assert.match(result.finalDecision.reason, /operator tripped kill mid-run/); + assert.equal(callCount, 1); + assert.equal(result.iterationsUsed, 1); + assert.equal(probes, 2); +}); diff --git a/packages/loopover-miner/lib/attempt-cli.js b/packages/loopover-miner/lib/attempt-cli.js index 3aadab2d58..ac5fd2ea26 100644 --- a/packages/loopover-miner/lib/attempt-cli.js +++ b/packages/loopover-miner/lib/attempt-cli.js @@ -32,7 +32,7 @@ import { cleanupAttemptWorktree, prepareAttemptWorktree } from "./attempt-worktr import { fetchSelfReviewContext } from "./self-review-context.js"; import { buildCodingTaskSpec } from "./coding-task-spec.js"; import { resolveAmsPolicy } from "./ams-policy.js"; -import { checkMinerKillSwitch } from "./governor-kill-switch.js"; +import { checkMinerKillSwitch, recordMinerKillSwitchTransition } from "./governor-kill-switch.js"; import { buildAttemptGovernorContext, buildAttemptLoopInput } from "./attempt-input-builder.js"; import { getAttemptHistory } from "./portfolio-queue.js"; import { loadReputationHistory, recordOwnSubmission } from "./governor-state.js"; @@ -388,7 +388,38 @@ export async function runAttempt(args, options = {}) { const repoPaused = minerGoalSpec.spec.killSwitch.paused; const checkKillSwitch = options.checkMinerKillSwitch ?? checkMinerKillSwitch; - const killSwitchScope = checkKillSwitch({ env, repoPaused }).scope; + let killSwitchScope = checkKillSwitch({ env, repoPaused }).scope; + let previousKillSwitchScope = killSwitchScope; + + const resolveLiveKillSwitch = () => { + // Re-read the YAML flag each probe so an on-disk unpause/pause is reflected mid-attempt (#5670). + const liveRepoPaused = resolveGoalSpec(worktreeResult.repoPath).spec.killSwitch.paused; + const live = checkKillSwitch({ env, repoPaused: liveRepoPaused }); + if (live.scope !== previousKillSwitchScope) { + try { + recordMinerKillSwitchTransition({ + repoFullName: parsed.repoFullName, + actionClass: "attempt", + previousScope: previousKillSwitchScope, + scope: live.scope, + }); + } catch { + // Ledger append must never crash an aborting attempt. + } + previousKillSwitchScope = live.scope; + } + killSwitchScope = live.scope; + return live; + }; + + const shouldAbort = () => { + const live = resolveLiveKillSwitch(); + if (!live.active) return false; + return { + abort: true, + reason: `Kill-switch (${live.scope}) engaged mid-attempt; abandoning without starting another driver iteration.`, + }; + }; const loopInput = buildAttemptLoopInput({ codingTaskSpec, @@ -435,7 +466,11 @@ export async function runAttempt(args, options = {}) { submissionMode: amsPolicy.spec.submissionMode, governor, }, - deps, + { + ...deps, + shouldAbort, + resolveKillSwitchScope: () => resolveLiveKillSwitch().scope, + }, ); worktreeResult.attemptOk = result.outcome === "submitted"; @@ -510,6 +545,9 @@ export async function runAttempt(args, options = {}) { // on any iteration this attempt ran, never fabricated. totalTokensUsed: result.loopResult.finalMeterTotals.tokens, iterationsUsed: result.loopResult.iterationsUsed, + ...(result.outcome === "abandon" && result.loopResult.finalDecision?.abandonReason + ? { abandonReason: result.loopResult.finalDecision.abandonReason } + : {}), ...("reason" in result ? { reason: result.reason } : {}), ...("decision" in result ? { decision: result.decision } : {}), ...("spec" in result ? { spec: result.spec } : {}), diff --git a/packages/loopover-miner/lib/attempt-runner.d.ts b/packages/loopover-miner/lib/attempt-runner.d.ts index 1a6807dc45..0e966049f9 100644 --- a/packages/loopover-miner/lib/attempt-runner.d.ts +++ b/packages/loopover-miner/lib/attempt-runner.d.ts @@ -47,6 +47,10 @@ export type AttemptDeps = { sessionStartMs?: number; nowMs: number; executeLocalWrite: (spec: LocalWriteActionSpec) => Promise; + /** Mid-attempt kill-switch probe threaded into `runIterateLoop` (#5670). */ + shouldAbort?: () => import("@loopover/engine").IterateLoopShouldAbort; + /** Live kill-switch scope resolver after handoff (#5670); defaults to the frozen attempt-start scope. */ + resolveKillSwitchScope?: () => "global" | "repo" | "none"; }; export type AttemptResult = diff --git a/packages/loopover-miner/lib/attempt-runner.js b/packages/loopover-miner/lib/attempt-runner.js index 7134aead58..7efb5a2d02 100644 --- a/packages/loopover-miner/lib/attempt-runner.js +++ b/packages/loopover-miner/lib/attempt-runner.js @@ -99,6 +99,8 @@ function assertInput(input) { * sessionStartMs?: number, * nowMs: number, * executeLocalWrite: (spec: import("@loopover/engine").LocalWriteActionSpec) => Promise, + * shouldAbort?: () => import("@loopover/engine").IterateLoopShouldAbort, + * resolveKillSwitchScope?: () => "global"|"repo"|"none", * }} deps */ export async function runMinerAttempt(input, deps) { @@ -109,6 +111,7 @@ export async function runMinerAttempt(input, deps) { driver: deps.driver, runSlopAssessment: deps.runSlopAssessment, appendAttemptLogEvent: deps.appendAttemptLogEvent, + ...(typeof deps.shouldAbort === "function" ? { shouldAbort: deps.shouldAbort } : {}), }); if (loopResult.outcome === "abandon") { @@ -117,6 +120,28 @@ export async function runMinerAttempt(input, deps) { const handoffPacket = loopResult.handoffPacket; + // Re-check kill-switch AFTER handoff and BEFORE any write (#5670) when a live resolver is supplied. + // Without a live resolver, preserve pre-#5670 behavior: the frozen attempt-start scope is threaded into + // prepareOpenPrSubmission / the submission gate (which itself denies active kill scopes). + if (typeof deps.resolveKillSwitchScope === "function") { + const liveKillSwitchScope = deps.resolveKillSwitchScope(); + if (liveKillSwitchScope !== "none") { + return { + outcome: "abandon", + loopResult: { + ...loopResult, + outcome: "abandon", + finalDecision: { + action: "abandon", + abandonReason: "kill_switch_engaged", + reason: `Kill-switch (${liveKillSwitchScope}) engaged after handoff; refusing to open a PR.`, + }, + handoffPacket: undefined, + }, + }; + } + } + const freshness = await checkSubmissionFreshness( { repoFullName: input.loopInput.repoFullName, issueNumber: input.issueNumber, minerLogin: input.minerLogin }, { claimLedger: deps.claimLedger, fetchLiveIssueSnapshot: deps.fetchLiveIssueSnapshot, eventLedger: deps.eventLedger }, diff --git a/packages/loopover-miner/lib/loop-cli.js b/packages/loopover-miner/lib/loop-cli.js index ecb53e1f65..8666661348 100644 --- a/packages/loopover-miner/lib/loop-cli.js +++ b/packages/loopover-miner/lib/loop-cli.js @@ -311,14 +311,35 @@ export async function runLoop(args, options = {}) { const killSwitch = checkKillSwitchFn({ env }); if (killSwitch.active) { haltReason = `kill_switch_${killSwitch.scope}`; - cycles.push({ cycle: cycleIndex, outcome: "halted", reason: haltReason }); + // Release the in-flight claim so left state is defined (#5670 / mirrors run-halt's markFailed). + if (claimed) { + portfolioQueue.markFailed(claimed.repoFullName, claimed.identifier, claimed.apiBaseUrl); + } + cycles.push({ + cycle: cycleIndex, + outcome: "halted", + reason: haltReason, + ...(claimed + ? { repoFullName: claimed.repoFullName, identifier: claimed.identifier } + : {}), + }); break; } const pauseState = governorState.loadPauseState(); if (pauseState.paused) { haltReason = "paused"; - cycles.push({ cycle: cycleIndex, outcome: "halted", reason: haltReason }); + if (claimed) { + portfolioQueue.markFailed(claimed.repoFullName, claimed.identifier, claimed.apiBaseUrl); + } + cycles.push({ + cycle: cycleIndex, + outcome: "halted", + reason: haltReason, + ...(claimed + ? { repoFullName: claimed.repoFullName, identifier: claimed.identifier } + : {}), + }); break; } @@ -410,6 +431,9 @@ export async function runLoop(args, options = {}) { // different iteration budget) and is requeued -- a genuinely stuck item is caught by non-convergence // (reenqueues threshold) rather than silently retried forever. const permanentBlock = attemptOutcome === "blocked_rejection_signaled"; + // Mid-attempt kill-switch abandon (#5670): stop the outer loop immediately instead of waiting for the + // next between-cycle probe, and treat the item like any other re-queued abandon via markFailed below. + const killSwitchAbandon = lastResult?.abandonReason === "kill_switch_engaged"; if (submitted || permanentBlock) { // Both terminal -- a submitted PR is done, and a repo-wide AI-usage-policy ban never resolves on retry -- @@ -421,6 +445,20 @@ export async function runLoop(args, options = {}) { portfolioQueue.markFailed(claimed.repoFullName, claimed.identifier, claimed.apiBaseUrl); } + if (killSwitchAbandon) { + const liveKill = checkKillSwitchFn({ env }); + haltReason = liveKill.active ? `kill_switch_${liveKill.scope}` : "kill_switch_engaged"; + cycles.push({ + cycle: cycleIndex, + outcome: "halted", + reason: haltReason, + repoFullName: claimed.repoFullName, + identifier: claimed.identifier, + attemptOutcome, + }); + break; + } + let reentryOutcome = "other"; let prNumber = null; let prDisposition = null; diff --git a/test/unit/miner-attempt-runner.test.ts b/test/unit/miner-attempt-runner.test.ts index 8b53112c04..cc0d576cd2 100644 --- a/test/unit/miner-attempt-runner.test.ts +++ b/test/unit/miner-attempt-runner.test.ts @@ -429,4 +429,36 @@ describe("runMinerAttempt — real self-plagiarism wiring into the chokepoint (# const result = await runMinerAttempt(baseAttemptInput(), deps as never); expect(result.outcome).toBe("governed"); }); + + it("honors shouldAbort before the first driver iteration and refuses to open a PR (#5670)", async () => { + const executeLocalWrite = vi.fn(async () => ({ ranAt: 10_000 })); + const driver = vi.fn(async () => okDriverResult()); + const result = await runMinerAttempt( + baseAttemptInput(), + baseDeps({ + driver: { run: driver }, + shouldAbort: () => ({ abort: true, reason: "kill already engaged" }), + executeLocalWrite, + }), + ); + expect(result.outcome).toBe("abandon"); + expect(result.loopResult.finalDecision.abandonReason).toBe("kill_switch_engaged"); + expect(result.loopResult.iterationsUsed).toBe(0); + expect(driver).not.toHaveBeenCalled(); + expect(executeLocalWrite).not.toHaveBeenCalled(); + }); + + it("re-checks kill-switch after handoff and refuses open_pr (#5670)", async () => { + const executeLocalWrite = vi.fn(async () => ({ ranAt: 10_000 })); + const result = await runMinerAttempt( + baseAttemptInput({ killSwitchScope: "none" }), + baseDeps({ + resolveKillSwitchScope: () => "global", + executeLocalWrite, + }), + ); + expect(result.outcome).toBe("abandon"); + expect(result.loopResult.finalDecision.abandonReason).toBe("kill_switch_engaged"); + expect(executeLocalWrite).not.toHaveBeenCalled(); + }); }); diff --git a/test/unit/miner-loop-cli.test.ts b/test/unit/miner-loop-cli.test.ts index 298e5f6447..311a58076c 100644 --- a/test/unit/miner-loop-cli.test.ts +++ b/test/unit/miner-loop-cli.test.ts @@ -810,4 +810,50 @@ describe("runLoop (#5135)", () => { }); for (const spy of jsonCloseSpies) expect(spy).toHaveBeenCalledTimes(1); }); + + it("halts immediately when an attempt returns kill_switch_engaged mid-run (#5670)", async () => { + const { eventLedger, governorLedger, portfolioQueue, runState, governorState, paths } = tempStores(); + portfolioQueue.enqueue({ repoFullName: "acme/widgets", identifier: "issue:7" }); + const log = vi.spyOn(console, "log").mockImplementation(() => undefined); + let killActive = false; + const runAttemptSpy = vi.fn(async (_args: string[], options: { onResult?: (result: unknown) => void }) => { + killActive = true; + options.onResult?.({ + outcome: "attempt_abandon", + abandonReason: "kill_switch_engaged", + totalTurnsUsed: 1, + totalCostUsd: 0, + }); + return 0; + }); + + const exitCode = await runLoop(["acme/widgets", "--miner-login", "alice", "--json", "--max-cycles", "3"], { + openGovernorState: () => governorState, + initEventLedger: () => eventLedger, + initGovernorLedger: () => governorLedger, + initPortfolioQueue: () => portfolioQueue, + initRunStateStore: () => runState, + runDiscover: async () => 0, + runAttempt: runAttemptSpy, + ...readyLoopOptions({ + checkMinerKillSwitch: () => + killActive ? { scope: "global" as const, active: true } : { scope: "none" as const, active: false }, + }), + }); + + expect(exitCode).toBe(0); + expect(runAttemptSpy).toHaveBeenCalledTimes(1); + const printed = JSON.parse(String(log.mock.calls[0]?.[0])); + expect(printed.haltReason).toBe("kill_switch_global"); + expect(printed.cycles.at(-1)).toMatchObject({ + outcome: "halted", + reason: "kill_switch_global", + identifier: "issue:7", + attemptOutcome: "attempt_abandon", + }); + // Claim released back to queued via markFailed — reopen after runLoop closes its handles. + const reopened = initPortfolioQueueStore(paths.portfolioQueuePath); + expect(reopened.listQueue()[0]).toMatchObject({ identifier: "issue:7", status: "queued" }); + reopened.close(); + }); }); From e6c735c9fc90e795b22196e78165f88bcfc10afe Mon Sep 17 00:00:00 2001 From: jony376 Date: Tue, 14 Jul 2026 10:29:43 -0700 Subject: [PATCH 2/2] test(miner): cover mid-attempt kill-switch branches and fix typecheck Raise patch coverage for #5670 abandon/claim-release paths and satisfy validate-code typecheck on the loop-cli spy. --- .../loopover-engine/src/miner/iterate-loop.ts | 2 +- packages/loopover-miner/lib/attempt-cli.js | 3 +- test/unit/miner-attempt-cli.test.ts | 96 ++++++++ test/unit/miner-attempt-runner.test.ts | 80 +++++++ test/unit/miner-loop-cli.test.ts | 209 +++++++++++++++++- 5 files changed, 381 insertions(+), 9 deletions(-) diff --git a/packages/loopover-engine/src/miner/iterate-loop.ts b/packages/loopover-engine/src/miner/iterate-loop.ts index 70f5ad3b8d..c3febc1cfe 100644 --- a/packages/loopover-engine/src/miner/iterate-loop.ts +++ b/packages/loopover-engine/src/miner/iterate-loop.ts @@ -372,7 +372,7 @@ async function runIterateLoopCore(input: IterateLoopInput, deps: IterateLoopDeps reason: abort.reason, }; safeAppendAttemptLogEvent(deps, { - eventType: "attempt_aborted", + eventType: attemptLogEventTypeForDecision(decision), attemptId: input.attemptId, actionClass: "iterate_loop", mode: input.mode, diff --git a/packages/loopover-miner/lib/attempt-cli.js b/packages/loopover-miner/lib/attempt-cli.js index ac5fd2ea26..cfb503950e 100644 --- a/packages/loopover-miner/lib/attempt-cli.js +++ b/packages/loopover-miner/lib/attempt-cli.js @@ -388,6 +388,7 @@ export async function runAttempt(args, options = {}) { const repoPaused = minerGoalSpec.spec.killSwitch.paused; const checkKillSwitch = options.checkMinerKillSwitch ?? checkMinerKillSwitch; + const recordKillTransition = options.recordMinerKillSwitchTransition ?? recordMinerKillSwitchTransition; let killSwitchScope = checkKillSwitch({ env, repoPaused }).scope; let previousKillSwitchScope = killSwitchScope; @@ -397,7 +398,7 @@ export async function runAttempt(args, options = {}) { const live = checkKillSwitch({ env, repoPaused: liveRepoPaused }); if (live.scope !== previousKillSwitchScope) { try { - recordMinerKillSwitchTransition({ + recordKillTransition({ repoFullName: parsed.repoFullName, actionClass: "attempt", previousScope: previousKillSwitchScope, diff --git a/test/unit/miner-attempt-cli.test.ts b/test/unit/miner-attempt-cli.test.ts index 5577192944..a021130b2b 100644 --- a/test/unit/miner-attempt-cli.test.ts +++ b/test/unit/miner-attempt-cli.test.ts @@ -1450,4 +1450,100 @@ describe("runAttempt: real claim-ledger wiring (#5393)", () => { expect(claimIssueSpy).not.toHaveBeenCalled(); }); + + it("wires live shouldAbort + resolveKillSwitchScope and surfaces abandonReason (#5670)", async () => { + const { allocator, claimLedger, eventLedger, attemptLog, governorLedger } = tempLedgers(); + const log = vi.spyOn(console, "log").mockImplementation(() => undefined); + let killChecks = 0; + const checkMinerKillSwitchSpy = vi.fn(() => { + killChecks += 1; + // First resolve seeds previousScope=none; later live probes trip global. + if (killChecks === 1) return { scope: "none" as const, active: false }; + return { scope: "global" as const, active: true }; + }); + const recordTransitionSpy = vi.fn(); + const runMinerAttemptSpy = vi.fn(async (_input: unknown, deps: { + shouldAbort?: () => boolean | { abort: boolean; reason?: string }; + resolveKillSwitchScope?: () => string; + }) => { + expect(deps.shouldAbort?.()).toEqual({ + abort: true, + reason: expect.stringContaining("Kill-switch (global)"), + }); + expect(deps.resolveKillSwitchScope?.()).toBe("global"); + return { + outcome: "abandon", + loopResult: fakeLoopResult({ + outcome: "abandon", + finalDecision: { + action: "abandon", + abandonReason: "kill_switch_engaged", + reason: "Kill-switch (global) engaged mid-attempt; abandoning without starting another driver iteration.", + }, + }), + }; + }); + + const exitCode = await runAttempt(["acme/widgets", "7", "--miner-login", "alice", "--json"], { + env: { MINER_CODING_AGENT_PROVIDER: "noop" }, + openWorktreeAllocator: () => allocator, + openClaimLedger: () => claimLedger, + initEventLedger: () => eventLedger, + initAttemptLog: () => attemptLog, + initGovernorLedger: () => governorLedger, + ...readyPipelineOptions({ + checkMinerKillSwitch: checkMinerKillSwitchSpy, + recordMinerKillSwitchTransition: recordTransitionSpy, + runMinerAttempt: runMinerAttemptSpy, + }), + }); + + expect(exitCode).toBe(7); + expect(runMinerAttemptSpy).toHaveBeenCalledTimes(1); + expect(recordTransitionSpy).toHaveBeenCalledWith({ + repoFullName: "acme/widgets", + actionClass: "attempt", + previousScope: "none", + scope: "global", + }); + expect(JSON.parse(String(log.mock.calls[0]?.[0])).abandonReason).toBe("kill_switch_engaged"); + }); + + it("shouldAbort stays false while kill is inactive, and a broken transition never crashes (#5670)", async () => { + const { allocator, claimLedger, eventLedger, attemptLog, governorLedger } = tempLedgers(); + vi.spyOn(console, "log").mockImplementation(() => undefined); + let killChecks = 0; + const checkMinerKillSwitchSpy = vi.fn(() => { + killChecks += 1; + if (killChecks === 1) return { scope: "none" as const, active: false }; + // Scope changes (triggers transition) but stays inactive so shouldAbort returns false. + return { scope: "repo" as const, active: false }; + }); + const recordTransitionSpy = vi.fn(() => { + throw new Error("ledger unavailable"); + }); + const runMinerAttemptSpy = vi.fn(async (_input: unknown, deps: { + shouldAbort?: () => boolean | { abort: boolean; reason?: string }; + }) => { + expect(deps.shouldAbort?.()).toBe(false); + return { outcome: "abandon", loopResult: fakeLoopResult() }; + }); + + const exitCode = await runAttempt(["acme/widgets", "7", "--miner-login", "alice", "--json"], { + env: { MINER_CODING_AGENT_PROVIDER: "noop" }, + openWorktreeAllocator: () => allocator, + openClaimLedger: () => claimLedger, + initEventLedger: () => eventLedger, + initAttemptLog: () => attemptLog, + initGovernorLedger: () => governorLedger, + ...readyPipelineOptions({ + checkMinerKillSwitch: checkMinerKillSwitchSpy, + recordMinerKillSwitchTransition: recordTransitionSpy, + runMinerAttempt: runMinerAttemptSpy, + }), + }); + + expect(exitCode).toBe(7); + expect(recordTransitionSpy).toHaveBeenCalledTimes(1); + }); }); diff --git a/test/unit/miner-attempt-runner.test.ts b/test/unit/miner-attempt-runner.test.ts index cc0d576cd2..50b0d5d553 100644 --- a/test/unit/miner-attempt-runner.test.ts +++ b/test/unit/miner-attempt-runner.test.ts @@ -461,4 +461,84 @@ describe("runMinerAttempt — real self-plagiarism wiring into the chokepoint (# expect(result.loopResult.finalDecision.abandonReason).toBe("kill_switch_engaged"); expect(executeLocalWrite).not.toHaveBeenCalled(); }); + + it("treats boolean shouldAbort true as kill_switch_engaged with the default reason (#5670)", async () => { + const driver = vi.fn(async () => okDriverResult()); + const result = await runMinerAttempt( + baseAttemptInput(), + baseDeps({ + driver: { run: driver }, + shouldAbort: () => true, + }), + ); + expect(result.outcome).toBe("abandon"); + expect(result.loopResult.finalDecision.abandonReason).toBe("kill_switch_engaged"); + expect(result.loopResult.finalDecision.reason).toMatch(/Kill-switch engaged mid-attempt/); + expect(driver).not.toHaveBeenCalled(); + }); + + it("uses the default reason when shouldAbort returns { abort: true } without a reason (#5670)", async () => { + const driver = vi.fn(async () => okDriverResult()); + const result = await runMinerAttempt( + baseAttemptInput(), + baseDeps({ + driver: { run: driver }, + shouldAbort: () => ({ abort: true }), + }), + ); + expect(result.outcome).toBe("abandon"); + expect(result.loopResult.finalDecision.abandonReason).toBe("kill_switch_engaged"); + expect(result.loopResult.finalDecision.reason).toMatch(/Kill-switch engaged mid-attempt/); + expect(driver).not.toHaveBeenCalled(); + }); + + it("ignores a non-aborting shouldAbort object and continues the attempt (#5670)", async () => { + const executeLocalWrite = vi.fn(async () => ({ ranAt: 10_000 })); + const result = await runMinerAttempt( + baseAttemptInput(), + baseDeps({ + shouldAbort: () => ({ abort: false }), + executeLocalWrite, + }), + ); + // Handoff proceeds into the real post-loop gates (not abandoned for kill). + expect(result.outcome).not.toBe("abandon"); + }); + + it("treats boolean shouldAbort false as a no-op and continues (#5670)", async () => { + const result = await runMinerAttempt( + baseAttemptInput(), + baseDeps({ + shouldAbort: () => false, + }), + ); + expect(result.outcome).not.toBe("abandon"); + }); + + it("uses the default reason when shouldAbort supplies a whitespace-only reason (#5670)", async () => { + const driver = vi.fn(async () => okDriverResult()); + const result = await runMinerAttempt( + baseAttemptInput(), + baseDeps({ + driver: { run: driver }, + shouldAbort: () => ({ abort: true, reason: " " }), + }), + ); + expect(result.outcome).toBe("abandon"); + expect(result.loopResult.finalDecision.reason).toMatch(/Kill-switch engaged mid-attempt/); + expect(driver).not.toHaveBeenCalled(); + }); + + it("continues past the post-handoff kill re-check when live scope is none (#5670)", async () => { + const executeLocalWrite = vi.fn(async () => ({ ranAt: 10_000 })); + const result = await runMinerAttempt( + baseAttemptInput({ killSwitchScope: "none" }), + baseDeps({ + resolveKillSwitchScope: () => "none", + executeLocalWrite, + }), + ); + expect(result.outcome).not.toBe("abandon"); + expect(executeLocalWrite).toHaveBeenCalled(); + }); }); diff --git a/test/unit/miner-loop-cli.test.ts b/test/unit/miner-loop-cli.test.ts index 311a58076c..7ae78eb73a 100644 --- a/test/unit/miner-loop-cli.test.ts +++ b/test/unit/miner-loop-cli.test.ts @@ -816,14 +816,17 @@ describe("runLoop (#5135)", () => { portfolioQueue.enqueue({ repoFullName: "acme/widgets", identifier: "issue:7" }); const log = vi.spyOn(console, "log").mockImplementation(() => undefined); let killActive = false; - const runAttemptSpy = vi.fn(async (_args: string[], options: { onResult?: (result: unknown) => void }) => { + const runAttemptSpy = vi.fn(async (_args: string[], options?: Record) => { killActive = true; - options.onResult?.({ - outcome: "attempt_abandon", - abandonReason: "kill_switch_engaged", - totalTurnsUsed: 1, - totalCostUsd: 0, - }); + const onResult = options?.onResult; + if (typeof onResult === "function") { + onResult({ + outcome: "attempt_abandon", + abandonReason: "kill_switch_engaged", + totalTurnsUsed: 1, + totalCostUsd: 0, + }); + } return 0; }); @@ -856,4 +859,196 @@ describe("runLoop (#5135)", () => { expect(reopened.listQueue()[0]).toMatchObject({ identifier: "issue:7", status: "queued" }); reopened.close(); }); + + it("halts with kill_switch_engaged when mid-attempt abandon is reported but live kill cleared (#5670)", async () => { + const { eventLedger, governorLedger, portfolioQueue, runState, governorState } = tempStores(); + portfolioQueue.enqueue({ repoFullName: "acme/widgets", identifier: "issue:7" }); + const log = vi.spyOn(console, "log").mockImplementation(() => undefined); + const runAttemptSpy = vi.fn(async (_args: string[], options?: Record) => { + const onResult = options?.onResult; + if (typeof onResult === "function") { + onResult({ + outcome: "attempt_abandon", + abandonReason: "kill_switch_engaged", + totalTurnsUsed: 1, + totalCostUsd: 0, + }); + } + return 0; + }); + + const exitCode = await runLoop(["acme/widgets", "--miner-login", "alice", "--json", "--max-cycles", "3"], { + openGovernorState: () => governorState, + initEventLedger: () => eventLedger, + initGovernorLedger: () => governorLedger, + initPortfolioQueue: () => portfolioQueue, + initRunStateStore: () => runState, + runDiscover: async () => 0, + runAttempt: runAttemptSpy, + ...readyLoopOptions({ + checkMinerKillSwitch: () => ({ scope: "none" as const, active: false }), + }), + }); + + expect(exitCode).toBe(0); + const printed = JSON.parse(String(log.mock.calls[0]?.[0])); + expect(printed.haltReason).toBe("kill_switch_engaged"); + expect(printed.cycles.at(-1)).toMatchObject({ + outcome: "halted", + reason: "kill_switch_engaged", + identifier: "issue:7", + }); + }); + + it("releases an in-flight claim when kill trips at the top of a later cycle (#5670)", async () => { + const { eventLedger, governorLedger, portfolioQueue, runState, governorState, paths } = tempStores(); + portfolioQueue.enqueue({ repoFullName: "acme/widgets", identifier: "issue:7" }); + const log = vi.spyOn(console, "log").mockImplementation(() => undefined); + let killChecks = 0; + const runAttemptSpy = vi.fn(async (_args: string[], options?: Record) => { + const onResult = options?.onResult; + if (typeof onResult === "function") { + onResult({ + outcome: "attempt_abandon", + totalTurnsUsed: 1, + totalCostUsd: 0, + }); + } + return 0; + }); + + const exitCode = await runLoop(["acme/widgets", "--miner-login", "alice", "--json", "--max-cycles", "3"], { + openGovernorState: () => governorState, + initEventLedger: () => eventLedger, + initGovernorLedger: () => governorLedger, + initPortfolioQueue: () => portfolioQueue, + initRunStateStore: () => runState, + runDiscover: async () => 0, + runAttempt: runAttemptSpy, + ...readyLoopOptions({ + checkMinerKillSwitch: () => { + killChecks += 1; + // Initial check inactive (discovery + first attempt). Second cycle's top-of-loop probe trips. + // Sequence: initial, cycle1 top, (after attempt + requeue) cycle2 top. + return killChecks >= 3 + ? { scope: "repo" as const, active: true } + : { scope: "none" as const, active: false }; + }, + }), + }); + + expect(exitCode).toBe(0); + expect(runAttemptSpy).toHaveBeenCalledTimes(1); + const printed = JSON.parse(String(log.mock.calls[0]?.[0])); + expect(printed.haltReason).toBe("kill_switch_repo"); + expect(printed.cycles.at(-1)).toMatchObject({ + outcome: "halted", + reason: "kill_switch_repo", + identifier: "issue:7", + }); + const reopened = initPortfolioQueueStore(paths.portfolioQueuePath); + expect(reopened.listQueue()[0]).toMatchObject({ identifier: "issue:7", status: "queued" }); + reopened.close(); + }); + + it("releases an in-flight claim when pause trips at the top of a later cycle (#5670)", async () => { + const { eventLedger, governorLedger, portfolioQueue, runState, governorState, paths } = tempStores(); + portfolioQueue.enqueue({ repoFullName: "acme/widgets", identifier: "issue:7" }); + const log = vi.spyOn(console, "log").mockImplementation(() => undefined); + const loadPauseState = vi + .fn() + .mockReturnValueOnce({ paused: false, reason: null, pausedAt: null }) // initial pre-loop + .mockReturnValueOnce({ paused: false, reason: null, pausedAt: null }) // cycle 1 top + .mockReturnValue({ paused: true, reason: "stop after attempt", pausedAt: "2026-07-14T00:00:00.000Z" }); + const runAttemptSpy = vi.fn(async (_args: string[], options?: Record) => { + const onResult = options?.onResult; + if (typeof onResult === "function") { + onResult({ outcome: "attempt_abandon", totalTurnsUsed: 1, totalCostUsd: 0 }); + } + return 0; + }); + + const exitCode = await runLoop(["acme/widgets", "--miner-login", "alice", "--json", "--max-cycles", "3"], { + openGovernorState: () => ({ ...governorState, loadPauseState }), + initEventLedger: () => eventLedger, + initGovernorLedger: () => governorLedger, + initPortfolioQueue: () => portfolioQueue, + initRunStateStore: () => runState, + runDiscover: async () => 0, + runAttempt: runAttemptSpy, + ...readyLoopOptions(), + }); + + expect(exitCode).toBe(0); + expect(runAttemptSpy).toHaveBeenCalledTimes(1); + const printed = JSON.parse(String(log.mock.calls[0]?.[0])); + expect(printed.haltReason).toBe("paused"); + expect(printed.cycles.at(-1)).toMatchObject({ + outcome: "halted", + reason: "paused", + identifier: "issue:7", + }); + const reopened = initPortfolioQueueStore(paths.portfolioQueuePath); + expect(reopened.listQueue()[0]).toMatchObject({ identifier: "issue:7", status: "queued" }); + reopened.close(); + }); + + it("halts on kill mid-loop without a claim and omits claim fields (#5670)", async () => { + const { eventLedger, governorLedger, portfolioQueue, runState, governorState } = tempStores(); + const log = vi.spyOn(console, "log").mockImplementation(() => undefined); + let killChecks = 0; + + const exitCode = await runLoop(["acme/widgets", "--miner-login", "alice", "--json", "--max-cycles", "2"], { + openGovernorState: () => governorState, + initEventLedger: () => eventLedger, + initGovernorLedger: () => governorLedger, + initPortfolioQueue: () => portfolioQueue, + initRunStateStore: () => runState, + runDiscover: async () => 0, + runAttempt: vi.fn(), + ...readyLoopOptions({ + checkMinerKillSwitch: () => { + killChecks += 1; + // Initial inactive (discovery). Empty queue → claimed null → top-of-loop kill. + return killChecks === 1 + ? { scope: "none" as const, active: false } + : { scope: "global" as const, active: true }; + }, + }), + }); + + expect(exitCode).toBe(0); + const printed = JSON.parse(String(log.mock.calls[0]?.[0])); + expect(printed.haltReason).toBe("kill_switch_global"); + expect(printed.cycles.at(-1)).toEqual({ + cycle: 1, + outcome: "halted", + reason: "kill_switch_global", + }); + }); + + it("halts on pause mid-loop without a claim and omits claim fields (#5670)", async () => { + const { eventLedger, governorLedger, portfolioQueue, runState, governorState } = tempStores(); + const log = vi.spyOn(console, "log").mockImplementation(() => undefined); + const loadPauseState = vi + .fn() + .mockReturnValueOnce({ paused: false, reason: null, pausedAt: null }) + .mockReturnValue({ paused: true, reason: "empty-queue stop", pausedAt: "2026-07-14T00:00:00.000Z" }); + + const exitCode = await runLoop(["acme/widgets", "--miner-login", "alice", "--json", "--max-cycles", "2"], { + openGovernorState: () => ({ ...governorState, loadPauseState }), + initEventLedger: () => eventLedger, + initGovernorLedger: () => governorLedger, + initPortfolioQueue: () => portfolioQueue, + initRunStateStore: () => runState, + runDiscover: async () => 0, + runAttempt: vi.fn(), + ...readyLoopOptions(), + }); + + expect(exitCode).toBe(0); + const printed = JSON.parse(String(log.mock.calls[0]?.[0])); + expect(printed.haltReason).toBe("paused"); + expect(printed.cycles.at(-1)).toEqual({ cycle: 1, outcome: "halted", reason: "paused" }); + }); });