diff --git a/apps/loopover-ui/content/docs/ams-kill-switch-incident.mdx b/apps/loopover-ui/content/docs/ams-kill-switch-incident.mdx index 90a9bd9289..bf1c67773e 100644 --- a/apps/loopover-ui/content/docs/ams-kill-switch-incident.mdx +++ b/apps/loopover-ui/content/docs/ams-kill-switch-incident.mdx @@ -52,6 +52,10 @@ Flag a misbehaving loop from any of: 3. Fleet observability — error-rate spikes, abnormal claim/submission patterns, elevated `consecutive_failures` / rejection-reason clustering, or an explicit customer/support escalation (see the rented-loop escalation path from #4806). +4. **Automated PagerDuty page** (#7666) — when `LOOPOVER_ENABLE_PAGERDUTY` is on and a routing key + is configured, a kill-switch **trip** (engage) fires an Events API v2 page via the same path ORB + uses (`notify-pagerduty`). A resume does not page. Treat an `ams_kill_switch:*` incident as + "operator already needed on the page" and start the 15-minute response clock immediately. **Triage before acting:** decide one-repo vs fleet-wide. Prefer the narrowest scope that stops the harm so one bad tenant does not force an unnecessary global halt. diff --git a/packages/loopover-engine/src/governor/kill-switch.ts b/packages/loopover-engine/src/governor/kill-switch.ts index 1f49f119e9..fce296a243 100644 --- a/packages/loopover-engine/src/governor/kill-switch.ts +++ b/packages/loopover-engine/src/governor/kill-switch.ts @@ -7,7 +7,9 @@ // // DETECTOR ONLY — no IO, no persistence. Composing this with the other pure calculators into one fail-closed // allow/deny verdict (and recording every CHECK, not just a transition) is the Governor chokepoint's job -// (#2340), which consults this module first in its "safest wins" precedence. +// (#2340), which consults this module first in its "safest wins" precedence. Paging on a trip (#7666) is also +// an IO concern: this module only builds the pure PagerDuty alert payload; the miner IO seam / +// `src/services/notify-pagerduty.ts` fire the Events API call. import type { GovernorLedgerEvent } from "../governor-ledger.js"; @@ -73,3 +75,43 @@ export function buildMinerKillSwitchTransitionGovernorLedgerEvent(input: { payload: { previousScope: input.previousScope, scope: input.scope }, }; } + +/** + * Pure page payload for a kill-switch TRIP (#7666). Returns `null` when the transition is not a trip + * (no-op same-scope, or a resume) — paging wakes humans for engage, not for clear. `repoFullName` falls + * back to `ams/fleet` for a global halt with no single-repo context so routing still resolves against the + * operator's global PagerDuty key. Consumers (miner IO seam / hosted `notify-pagerduty`) own the actual + * Events API call — this module stays detector-only. + */ +export type MinerKillSwitchPagerDutyAlert = { + repoFullName: string; + summary: string; + severity: "critical"; + dedupKey: string; + customDetails: { + previousScope: MinerKillSwitchScope; + scope: MinerKillSwitchScope; + reason: string; + }; +}; + +export function buildMinerKillSwitchPagerDutyAlert(input: { + repoFullName?: string | null | undefined; + previousScope: MinerKillSwitchScope; + scope: MinerKillSwitchScope; +}): MinerKillSwitchPagerDutyAlert | null { + if (input.previousScope === input.scope) return null; + if (!isMinerKillSwitchActive(input.scope)) return null; + const repoFullName = (input.repoFullName ?? "").trim() || "ams/fleet"; + const reason = `${input.scope}_kill_switch_engaged`; + return { + repoFullName, + summary: + input.scope === "global" + ? `AMS miner kill-switch engaged (global / fleet-wide)` + : `AMS miner kill-switch engaged (repo) for ${repoFullName}`, + severity: "critical", + dedupKey: `ams_kill_switch:${input.scope}:${repoFullName.toLowerCase()}`, + customDetails: { previousScope: input.previousScope, scope: input.scope, reason }, + }; +} diff --git a/packages/loopover-engine/test/kill-switch.test.ts b/packages/loopover-engine/test/kill-switch.test.ts index b06e5a5c15..4c563a2718 100644 --- a/packages/loopover-engine/test/kill-switch.test.ts +++ b/packages/loopover-engine/test/kill-switch.test.ts @@ -3,6 +3,7 @@ import { test } from "node:test"; import { MINER_KILL_SWITCH_ENV_VAR, + buildMinerKillSwitchPagerDutyAlert, buildMinerKillSwitchTransitionGovernorLedgerEvent, isGlobalMinerKillSwitch, isMinerKillSwitchActive, @@ -14,6 +15,7 @@ test("barrel: the public entrypoint re-exports the kill-switch primitive (#2341) assert.equal(typeof resolveMinerKillSwitch, "function"); assert.equal(typeof isMinerKillSwitchActive, "function"); assert.equal(typeof buildMinerKillSwitchTransitionGovernorLedgerEvent, "function"); + assert.equal(typeof buildMinerKillSwitchPagerDutyAlert, "function"); assert.equal(MINER_KILL_SWITCH_ENV_VAR, "LOOPOVER_MINER_KILL_SWITCH"); }); @@ -105,3 +107,46 @@ test("buildMinerKillSwitchTransitionGovernorLedgerEvent: clearing the switch rec payload: { previousScope: "global", scope: "none" }, }); }); + +test("buildMinerKillSwitchPagerDutyAlert: trip builds a critical page payload (#7666)", () => { + const alert = buildMinerKillSwitchPagerDutyAlert({ + repoFullName: "acme/widgets", + previousScope: "none", + scope: "repo", + }); + assert.deepEqual(alert, { + repoFullName: "acme/widgets", + summary: "AMS miner kill-switch engaged (repo) for acme/widgets", + severity: "critical", + dedupKey: "ams_kill_switch:repo:acme/widgets", + customDetails: { previousScope: "none", scope: "repo", reason: "repo_kill_switch_engaged" }, + }); +}); + +test("buildMinerKillSwitchPagerDutyAlert: global trip without a repo uses ams/fleet (#7666)", () => { + const alert = buildMinerKillSwitchPagerDutyAlert({ + previousScope: "none", + scope: "global", + }); + assert.equal(alert?.repoFullName, "ams/fleet"); + assert.equal(alert?.dedupKey, "ams_kill_switch:global:ams/fleet"); + assert.match(alert?.summary ?? "", /fleet-wide/); + + const blankRepo = buildMinerKillSwitchPagerDutyAlert({ + repoFullName: " ", + previousScope: "none", + scope: "global", + }); + assert.equal(blankRepo?.repoFullName, "ams/fleet"); +}); + +test("buildMinerKillSwitchPagerDutyAlert: resume / same-scope are silent (#7666)", () => { + assert.equal( + buildMinerKillSwitchPagerDutyAlert({ repoFullName: "acme/widgets", previousScope: "repo", scope: "none" }), + null, + ); + assert.equal( + buildMinerKillSwitchPagerDutyAlert({ repoFullName: "acme/widgets", previousScope: "repo", scope: "repo" }), + null, + ); +}); diff --git a/packages/loopover-miner/lib/governor-kill-switch.d.ts b/packages/loopover-miner/lib/governor-kill-switch.d.ts index 191fa6e423..7f7a100962 100644 --- a/packages/loopover-miner/lib/governor-kill-switch.d.ts +++ b/packages/loopover-miner/lib/governor-kill-switch.d.ts @@ -1,3 +1,4 @@ +import { type MinerKillSwitchPagerDutyAlert } from "@loopover/engine"; import type { MinerKillSwitchScope } from "@loopover/engine"; import type { AppendGovernorEventInput, GovernorLedgerEntry } from "./governor-ledger.js"; export type CheckMinerKillSwitchInput = { @@ -19,11 +20,21 @@ export type RecordMinerKillSwitchTransitionInput = { previousScope: MinerKillSwitchScope; scope: MinerKillSwitchScope; }; +export type NotifyMinerKillSwitchTrip = (alert: MinerKillSwitchPagerDutyAlert, env: Record) => void | Promise; +/** + * Miner-side mirror of `triggerPagerDutyIncident` (#7666): same flag, same global routing key, same Events + * API v2 enqueue. No D1 audit/cooldown (miner has no Worker Env) -- PagerDuty's own `dedup_key` still + * coalesces duplicate incidents. Best-effort: never throws. + */ +export declare function notifyMinerKillSwitchPagerDuty(alert: MinerKillSwitchPagerDutyAlert, env?: Record): Promise; /** * Record a kill-switch state transition to the governor ledger. No-op (returns null, appends nothing) when the - * scope has not actually changed since the previous check — callers own tracking the previous scope (in-memory - * or persisted); this module holds no state of its own. + * scope has not actually changed since the previous check -- callers own tracking the previous scope (in-memory + * or persisted); this module holds no state of its own. On a trip, also fires the PagerDuty page (#7666) + * unless `notify` is overridden (tests) or the integration flag/key is unset. */ export declare function recordMinerKillSwitchTransition(input: RecordMinerKillSwitchTransitionInput, options?: { append?: (event: AppendGovernorEventInput) => GovernorLedgerEntry; + notify?: NotifyMinerKillSwitchTrip; + env?: Record; }): GovernorLedgerEntry | null; diff --git a/packages/loopover-miner/lib/governor-kill-switch.js b/packages/loopover-miner/lib/governor-kill-switch.js index 3c479144f3..847b60c0f9 100644 --- a/packages/loopover-miner/lib/governor-kill-switch.js +++ b/packages/loopover-miner/lib/governor-kill-switch.js @@ -2,7 +2,12 @@ // env, or for one repo, via its .loopover-miner.yml MinerGoalSpec) and records STATE TRANSITIONS to the // append-only governor ledger. Every-check allow/deny recording for a real write action is the fail-closed // Governor chokepoint's job (#2340), which consults this module first in its "safest wins" precedence. -import { buildMinerKillSwitchTransitionGovernorLedgerEvent, isGlobalMinerKillSwitch, isMinerKillSwitchActive, resolveMinerKillSwitch, } from "@loopover/engine"; +// +// #7666: a TRIP also pages via the same PagerDuty Events API v2 path ORB uses (`src/services/notify-pagerduty.ts` +// / LOOPOVER_ENABLE_PAGERDUTY + PAGERDUTY_ROUTING_KEY), so a kill-switch engage is not ledger-only. Resume +// stays silent -- clearing a halt must not wake anyone. The page is best-effort and never throws: a paging +// failure must never block the ledger write or the mid-attempt abandon that depends on it. +import { buildMinerKillSwitchPagerDutyAlert, buildMinerKillSwitchTransitionGovernorLedgerEvent, isGlobalMinerKillSwitch, isMinerKillSwitchActive, resolveMinerKillSwitch, } from "@loopover/engine"; import { appendGovernorEvent } from "./governor-ledger.js"; /** * Resolve the current kill-switch scope for a repo from process env plus a per-repo paused flag (typically @@ -14,16 +19,93 @@ export function checkMinerKillSwitch(input = {}) { const scope = resolveMinerKillSwitch({ global, repoPaused: input.repoPaused }); return { scope, active: isMinerKillSwitchActive(scope) }; } +const PAGERDUTY_EVENTS_URL = "https://events.pagerduty.com/v2/enqueue"; +const ROUTING_KEY_RE = /^[a-f0-9]{32}$/i; +const TRUTHY_ENV = /^(1|true|yes|on)$/i; +function envString(env, name) { + const value = env[name]; + return typeof value === "string" && value.trim().length > 0 ? value.trim() : undefined; +} +function pagerDutyFailMessage(error) { + // Prefer Error.message when present; otherwise coerce. Single helper so both sync and async + // failure paths share one branch surface for Codecov patch. + return (error instanceof Error ? error.message : String(error)).slice(0, 200); +} +function warnKillSwitchPagerDutyFailed(repo, error) { + console.warn(JSON.stringify({ event: "kill_switch_pagerduty_failed", repo, message: pagerDutyFailMessage(error) })); +} +/** + * Miner-side mirror of `triggerPagerDutyIncident` (#7666): same flag, same global routing key, same Events + * API v2 enqueue. No D1 audit/cooldown (miner has no Worker Env) -- PagerDuty's own `dedup_key` still + * coalesces duplicate incidents. Best-effort: never throws. + */ +export async function notifyMinerKillSwitchPagerDuty(alert, env = process.env) { + if (!TRUTHY_ENV.test((env.LOOPOVER_ENABLE_PAGERDUTY ?? "").trim())) + return; + const routingKey = envString(env, "PAGERDUTY_ROUTING_KEY"); + if (!routingKey || !ROUTING_KEY_RE.test(routingKey)) + return; + try { + const response = await fetch(PAGERDUTY_EVENTS_URL, { + method: "POST", + headers: { "content-type": "application/json" }, + body: JSON.stringify({ + routing_key: routingKey, + event_action: "trigger", + dedup_key: alert.dedupKey, + payload: { + summary: alert.summary.slice(0, 1024), + source: "loopover-miner", + severity: alert.severity, + timestamp: new Date().toISOString(), + component: alert.repoFullName, + custom_details: alert.customDetails, + }, + }), + signal: AbortSignal.timeout(5000), + }); + if (!response.ok) { + console.warn(JSON.stringify({ + event: "kill_switch_pagerduty_failed", + repo: alert.repoFullName, + status: response.status, + })); + } + } + catch (error) { + warnKillSwitchPagerDutyFailed(alert.repoFullName, error); + } +} /** * Record a kill-switch state transition to the governor ledger. No-op (returns null, appends nothing) when the - * scope has not actually changed since the previous check — callers own tracking the previous scope (in-memory - * or persisted); this module holds no state of its own. + * scope has not actually changed since the previous check -- callers own tracking the previous scope (in-memory + * or persisted); this module holds no state of its own. On a trip, also fires the PagerDuty page (#7666) + * unless `notify` is overridden (tests) or the integration flag/key is unset. */ export function recordMinerKillSwitchTransition(input, options = {}) { const event = buildMinerKillSwitchTransitionGovernorLedgerEvent(input); if (!event) return null; const append = options.append ?? appendGovernorEvent; - return append(event); + const recorded = append(event); + const alert = buildMinerKillSwitchPagerDutyAlert({ + repoFullName: input.repoFullName, + previousScope: input.previousScope, + scope: input.scope, + }); + if (alert) { + const notify = options.notify ?? notifyMinerKillSwitchPagerDuty; + const env = options.env ?? process.env; + try { + // Promise.resolve wraps sync returns so both sync throws and async rejects share one failure path. + void Promise.resolve(notify(alert, env)).catch((error) => { + warnKillSwitchPagerDutyFailed(alert.repoFullName, error); + }); + } + catch (error) { + warnKillSwitchPagerDutyFailed(alert.repoFullName, error); + } + } + return recorded; } -//# sourceMappingURL=data:application/json;base64,eyJ2ZXJzaW9uIjozLCJmaWxlIjoiZ292ZXJub3Ita2lsbC1zd2l0Y2guanMiLCJzb3VyY2VSb290IjoiIiwic291cmNlcyI6WyJnb3Zlcm5vci1raWxsLXN3aXRjaC50cyJdLCJuYW1lcyI6W10sIm1hcHBpbmdzIjoiQUFBQSw4R0FBOEc7QUFDOUcsd0dBQXdHO0FBQ3hHLDJHQUEyRztBQUMzRyx1R0FBdUc7QUFFdkcsT0FBTyxFQUNMLGlEQUFpRCxFQUNqRCx1QkFBdUIsRUFDdkIsdUJBQXVCLEVBQ3ZCLHNCQUFzQixHQUN2QixNQUFNLGtCQUFrQixDQUFDO0FBRTFCLE9BQU8sRUFBRSxtQkFBbUIsRUFBRSxNQUFNLHNCQUFzQixDQUFDO0FBYTNEOzs7R0FHRztBQUNILE1BQU0sVUFBVSxvQkFBb0IsQ0FBQyxRQUFtQyxFQUFFO0lBQ3hFLE1BQU0sR0FBRyxHQUFHLEtBQUssQ0FBQyxHQUFHLElBQUksT0FBTyxDQUFDLEdBQUcsQ0FBQztJQUNyQyxNQUFNLE1BQU0sR0FBRyx1QkFBdUIsQ0FBQyxHQUFHLENBQUMsQ0FBQztJQUM1QyxNQUFNLEtBQUssR0FBRyxzQkFBc0IsQ0FBQyxFQUFFLE1BQU0sRUFBRSxVQUFVLEVBQUUsS0FBSyxDQUFDLFVBQVUsRUFBRSxDQUFDLENBQUM7SUFDL0UsT0FBTyxFQUFFLEtBQUssRUFBRSxNQUFNLEVBQUUsdUJBQXVCLENBQUMsS0FBSyxDQUFDLEVBQUUsQ0FBQztBQUMzRCxDQUFDO0FBU0Q7Ozs7R0FJRztBQUNILE1BQU0sVUFBVSwrQkFBK0IsQ0FDN0MsS0FBMkMsRUFDM0MsVUFBaUYsRUFBRTtJQUVuRixNQUFNLEtBQUssR0FBRyxpREFBaUQsQ0FBQyxLQUFLLENBQUMsQ0FBQztJQUN2RSxJQUFJLENBQUMsS0FBSztRQUFFLE9BQU8sSUFBSSxDQUFDO0lBQ3hCLE1BQU0sTUFBTSxHQUFHLE9BQU8sQ0FBQyxNQUFNLElBQUksbUJBQW1CLENBQUM7SUFDckQsT0FBTyxNQUFNLENBQUMsS0FBaUMsQ0FBQyxDQUFDO0FBQ25ELENBQUMifQ== \ No newline at end of file +//# sourceMappingURL=data:application/json;base64,eyJ2ZXJzaW9uIjozLCJmaWxlIjoiZ292ZXJub3Ita2lsbC1zd2l0Y2guanMiLCJzb3VyY2VSb290IjoiIiwic291cmNlcyI6WyJnb3Zlcm5vci1raWxsLXN3aXRjaC50cyJdLCJuYW1lcyI6W10sIm1hcHBpbmdzIjoiQUFBQSw4R0FBOEc7QUFDOUcsd0dBQXdHO0FBQ3hHLDJHQUEyRztBQUMzRyx1R0FBdUc7QUFDdkcsRUFBRTtBQUNGLGtIQUFrSDtBQUNsSCwyR0FBMkc7QUFDM0csMkdBQTJHO0FBQzNHLDJGQUEyRjtBQUUzRixPQUFPLEVBQ0wsa0NBQWtDLEVBQ2xDLGlEQUFpRCxFQUNqRCx1QkFBdUIsRUFDdkIsdUJBQXVCLEVBQ3ZCLHNCQUFzQixHQUV2QixNQUFNLGtCQUFrQixDQUFDO0FBRTFCLE9BQU8sRUFBRSxtQkFBbUIsRUFBRSxNQUFNLHNCQUFzQixDQUFDO0FBYTNEOzs7R0FHRztBQUNILE1BQU0sVUFBVSxvQkFBb0IsQ0FBQyxRQUFtQyxFQUFFO0lBQ3hFLE1BQU0sR0FBRyxHQUFHLEtBQUssQ0FBQyxHQUFHLElBQUksT0FBTyxDQUFDLEdBQUcsQ0FBQztJQUNyQyxNQUFNLE1BQU0sR0FBRyx1QkFBdUIsQ0FBQyxHQUFHLENBQUMsQ0FBQztJQUM1QyxNQUFNLEtBQUssR0FBRyxzQkFBc0IsQ0FBQyxFQUFFLE1BQU0sRUFBRSxVQUFVLEVBQUUsS0FBSyxDQUFDLFVBQVUsRUFBRSxDQUFDLENBQUM7SUFDL0UsT0FBTyxFQUFFLEtBQUssRUFBRSxNQUFNLEVBQUUsdUJBQXVCLENBQUMsS0FBSyxDQUFDLEVBQUUsQ0FBQztBQUMzRCxDQUFDO0FBY0QsTUFBTSxvQkFBb0IsR0FBRyx5Q0FBeUMsQ0FBQztBQUN2RSxNQUFNLGNBQWMsR0FBRyxpQkFBaUIsQ0FBQztBQUN6QyxNQUFNLFVBQVUsR0FBRyxvQkFBb0IsQ0FBQztBQUV4QyxTQUFTLFNBQVMsQ0FBQyxHQUF1QyxFQUFFLElBQVk7SUFDdEUsTUFBTSxLQUFLLEdBQUcsR0FBRyxDQUFDLElBQUksQ0FBQyxDQUFDO0lBQ3hCLE9BQU8sT0FBTyxLQUFLLEtBQUssUUFBUSxJQUFJLEtBQUssQ0FBQyxJQUFJLEVBQUUsQ0FBQyxNQUFNLEdBQUcsQ0FBQyxDQUFDLENBQUMsQ0FBQyxLQUFLLENBQUMsSUFBSSxFQUFFLENBQUMsQ0FBQyxDQUFDLFNBQVMsQ0FBQztBQUN6RixDQUFDO0FBRUQsU0FBUyxvQkFBb0IsQ0FBQyxLQUFjO0lBQzFDLDRGQUE0RjtJQUM1Riw0REFBNEQ7SUFDNUQsT0FBTyxDQUFDLEtBQUssWUFBWSxLQUFLLENBQUMsQ0FBQyxDQUFDLEtBQUssQ0FBQyxPQUFPLENBQUMsQ0FBQyxDQUFDLE1BQU0sQ0FBQyxLQUFLLENBQUMsQ0FBQyxDQUFDLEtBQUssQ0FBQyxDQUFDLEVBQUUsR0FBRyxDQUFDLENBQUM7QUFDaEYsQ0FBQztBQUVELFNBQVMsNkJBQTZCLENBQUMsSUFBWSxFQUFFLEtBQWM7SUFDakUsT0FBTyxDQUFDLElBQUksQ0FBQyxJQUFJLENBQUMsU0FBUyxDQUFDLEVBQUUsS0FBSyxFQUFFLDhCQUE4QixFQUFFLElBQUksRUFBRSxPQUFPLEVBQUUsb0JBQW9CLENBQUMsS0FBSyxDQUFDLEVBQUUsQ0FBQyxDQUFDLENBQUM7QUFDdEgsQ0FBQztBQUVEOzs7O0dBSUc7QUFDSCxNQUFNLENBQUMsS0FBSyxVQUFVLDhCQUE4QixDQUNsRCxLQUFvQyxFQUNwQyxNQUEwQyxPQUFPLENBQUMsR0FBRztJQUVyRCxJQUFJLENBQUMsVUFBVSxDQUFDLElBQUksQ0FBQyxDQUFDLEdBQUcsQ0FBQyx5QkFBeUIsSUFBSSxFQUFFLENBQUMsQ0FBQyxJQUFJLEVBQUUsQ0FBQztRQUFFLE9BQU87SUFDM0UsTUFBTSxVQUFVLEdBQUcsU0FBUyxDQUFDLEdBQUcsRUFBRSx1QkFBdUIsQ0FBQyxDQUFDO0lBQzNELElBQUksQ0FBQyxVQUFVLElBQUksQ0FBQyxjQUFjLENBQUMsSUFBSSxDQUFDLFVBQVUsQ0FBQztRQUFFLE9BQU87SUFFNUQsSUFBSSxDQUFDO1FBQ0gsTUFBTSxRQUFRLEdBQUcsTUFBTSxLQUFLLENBQUMsb0JBQW9CLEVBQUU7WUFDakQsTUFBTSxFQUFFLE1BQU07WUFDZCxPQUFPLEVBQUUsRUFBRSxjQUFjLEVBQUUsa0JBQWtCLEVBQUU7WUFDL0MsSUFBSSxFQUFFLElBQUksQ0FBQyxTQUFTLENBQUM7Z0JBQ25CLFdBQVcsRUFBRSxVQUFVO2dCQUN2QixZQUFZLEVBQUUsU0FBUztnQkFDdkIsU0FBUyxFQUFFLEtBQUssQ0FBQyxRQUFRO2dCQUN6QixPQUFPLEVBQUU7b0JBQ1AsT0FBTyxFQUFFLEtBQUssQ0FBQyxPQUFPLENBQUMsS0FBSyxDQUFDLENBQUMsRUFBRSxJQUFJLENBQUM7b0JBQ3JDLE1BQU0sRUFBRSxnQkFBZ0I7b0JBQ3hCLFFBQVEsRUFBRSxLQUFLLENBQUMsUUFBUTtvQkFDeEIsU0FBUyxFQUFFLElBQUksSUFBSSxFQUFFLENBQUMsV0FBVyxFQUFFO29CQUNuQyxTQUFTLEVBQUUsS0FBSyxDQUFDLFlBQVk7b0JBQzdCLGNBQWMsRUFBRSxLQUFLLENBQUMsYUFBYTtpQkFDcEM7YUFDRixDQUFDO1lBQ0YsTUFBTSxFQUFFLFdBQVcsQ0FBQyxPQUFPLENBQUMsSUFBSSxDQUFDO1NBQ2xDLENBQUMsQ0FBQztRQUNILElBQUksQ0FBQyxRQUFRLENBQUMsRUFBRSxFQUFFLENBQUM7WUFDakIsT0FBTyxDQUFDLElBQUksQ0FDVixJQUFJLENBQUMsU0FBUyxDQUFDO2dCQUNiLEtBQUssRUFBRSw4QkFBOEI7Z0JBQ3JDLElBQUksRUFBRSxLQUFLLENBQUMsWUFBWTtnQkFDeEIsTUFBTSxFQUFFLFFBQVEsQ0FBQyxNQUFNO2FBQ3hCLENBQUMsQ0FDSCxDQUFDO1FBQ0osQ0FBQztJQUNILENBQUM7SUFBQyxPQUFPLEtBQUssRUFBRSxDQUFDO1FBQ2YsNkJBQTZCLENBQUMsS0FBSyxDQUFDLFlBQVksRUFBRSxLQUFLLENBQUMsQ0FBQztJQUMzRCxDQUFDO0FBQ0gsQ0FBQztBQUVEOzs7OztHQUtHO0FBQ0gsTUFBTSxVQUFVLCtCQUErQixDQUM3QyxLQUEyQyxFQUMzQyxVQUlJLEVBQUU7SUFFTixNQUFNLEtBQUssR0FBRyxpREFBaUQsQ0FBQyxLQUFLLENBQUMsQ0FBQztJQUN2RSxJQUFJLENBQUMsS0FBSztRQUFFLE9BQU8sSUFBSSxDQUFDO0lBQ3hCLE1BQU0sTUFBTSxHQUFHLE9BQU8sQ0FBQyxNQUFNLElBQUksbUJBQW1CLENBQUM7SUFDckQsTUFBTSxRQUFRLEdBQUcsTUFBTSxDQUFDLEtBQWlDLENBQUMsQ0FBQztJQUUzRCxNQUFNLEtBQUssR0FBRyxrQ0FBa0MsQ0FBQztRQUMvQyxZQUFZLEVBQUUsS0FBSyxDQUFDLFlBQVk7UUFDaEMsYUFBYSxFQUFFLEtBQUssQ0FBQyxhQUFhO1FBQ2xDLEtBQUssRUFBRSxLQUFLLENBQUMsS0FBSztLQUNuQixDQUFDLENBQUM7SUFDSCxJQUFJLEtBQUssRUFBRSxDQUFDO1FBQ1YsTUFBTSxNQUFNLEdBQUcsT0FBTyxDQUFDLE1BQU0sSUFBSSw4QkFBOEIsQ0FBQztRQUNoRSxNQUFNLEdBQUcsR0FBRyxPQUFPLENBQUMsR0FBRyxJQUFJLE9BQU8sQ0FBQyxHQUFHLENBQUM7UUFDdkMsSUFBSSxDQUFDO1lBQ0gsbUdBQW1HO1lBQ25HLEtBQUssT0FBTyxDQUFDLE9BQU8sQ0FBQyxNQUFNLENBQUMsS0FBSyxFQUFFLEdBQUcsQ0FBQyxDQUFDLENBQUMsS0FBSyxDQUFDLENBQUMsS0FBYyxFQUFFLEVBQUU7Z0JBQ2hFLDZCQUE2QixDQUFDLEtBQUssQ0FBQyxZQUFZLEVBQUUsS0FBSyxDQUFDLENBQUM7WUFDM0QsQ0FBQyxDQUFDLENBQUM7UUFDTCxDQUFDO1FBQUMsT0FBTyxLQUFLLEVBQUUsQ0FBQztZQUNmLDZCQUE2QixDQUFDLEtBQUssQ0FBQyxZQUFZLEVBQUUsS0FBSyxDQUFDLENBQUM7UUFDM0QsQ0FBQztJQUNILENBQUM7SUFFRCxPQUFPLFFBQVEsQ0FBQztBQUNsQixDQUFDIn0= \ No newline at end of file diff --git a/packages/loopover-miner/lib/governor-kill-switch.ts b/packages/loopover-miner/lib/governor-kill-switch.ts index 71a4de6773..676cadc761 100644 --- a/packages/loopover-miner/lib/governor-kill-switch.ts +++ b/packages/loopover-miner/lib/governor-kill-switch.ts @@ -2,12 +2,19 @@ // env, or for one repo, via its .loopover-miner.yml MinerGoalSpec) and records STATE TRANSITIONS to the // append-only governor ledger. Every-check allow/deny recording for a real write action is the fail-closed // Governor chokepoint's job (#2340), which consults this module first in its "safest wins" precedence. +// +// #7666: a TRIP also pages via the same PagerDuty Events API v2 path ORB uses (`src/services/notify-pagerduty.ts` +// / LOOPOVER_ENABLE_PAGERDUTY + PAGERDUTY_ROUTING_KEY), so a kill-switch engage is not ledger-only. Resume +// stays silent -- clearing a halt must not wake anyone. The page is best-effort and never throws: a paging +// failure must never block the ledger write or the mid-attempt abandon that depends on it. import { + buildMinerKillSwitchPagerDutyAlert, buildMinerKillSwitchTransitionGovernorLedgerEvent, isGlobalMinerKillSwitch, isMinerKillSwitchActive, resolveMinerKillSwitch, + type MinerKillSwitchPagerDutyAlert, } from "@loopover/engine"; import type { MinerKillSwitchScope } from "@loopover/engine"; import { appendGovernorEvent } from "./governor-ledger.js"; @@ -41,17 +48,112 @@ export type RecordMinerKillSwitchTransitionInput = { scope: MinerKillSwitchScope; }; +export type NotifyMinerKillSwitchTrip = ( + alert: MinerKillSwitchPagerDutyAlert, + env: Record, +) => void | Promise; + +const PAGERDUTY_EVENTS_URL = "https://events.pagerduty.com/v2/enqueue"; +const ROUTING_KEY_RE = /^[a-f0-9]{32}$/i; +const TRUTHY_ENV = /^(1|true|yes|on)$/i; + +function envString(env: Record, name: string): string | undefined { + const value = env[name]; + return typeof value === "string" && value.trim().length > 0 ? value.trim() : undefined; +} + +function pagerDutyFailMessage(error: unknown): string { + // Prefer Error.message when present; otherwise coerce. Single helper so both sync and async + // failure paths share one branch surface for Codecov patch. + return (error instanceof Error ? error.message : String(error)).slice(0, 200); +} + +function warnKillSwitchPagerDutyFailed(repo: string, error: unknown): void { + console.warn(JSON.stringify({ event: "kill_switch_pagerduty_failed", repo, message: pagerDutyFailMessage(error) })); +} + +/** + * Miner-side mirror of `triggerPagerDutyIncident` (#7666): same flag, same global routing key, same Events + * API v2 enqueue. No D1 audit/cooldown (miner has no Worker Env) -- PagerDuty's own `dedup_key` still + * coalesces duplicate incidents. Best-effort: never throws. + */ +export async function notifyMinerKillSwitchPagerDuty( + alert: MinerKillSwitchPagerDutyAlert, + env: Record = process.env, +): Promise { + if (!TRUTHY_ENV.test((env.LOOPOVER_ENABLE_PAGERDUTY ?? "").trim())) return; + const routingKey = envString(env, "PAGERDUTY_ROUTING_KEY"); + if (!routingKey || !ROUTING_KEY_RE.test(routingKey)) return; + + try { + const response = await fetch(PAGERDUTY_EVENTS_URL, { + method: "POST", + headers: { "content-type": "application/json" }, + body: JSON.stringify({ + routing_key: routingKey, + event_action: "trigger", + dedup_key: alert.dedupKey, + payload: { + summary: alert.summary.slice(0, 1024), + source: "loopover-miner", + severity: alert.severity, + timestamp: new Date().toISOString(), + component: alert.repoFullName, + custom_details: alert.customDetails, + }, + }), + signal: AbortSignal.timeout(5000), + }); + if (!response.ok) { + console.warn( + JSON.stringify({ + event: "kill_switch_pagerduty_failed", + repo: alert.repoFullName, + status: response.status, + }), + ); + } + } catch (error) { + warnKillSwitchPagerDutyFailed(alert.repoFullName, error); + } +} + /** * Record a kill-switch state transition to the governor ledger. No-op (returns null, appends nothing) when the - * scope has not actually changed since the previous check — callers own tracking the previous scope (in-memory - * or persisted); this module holds no state of its own. + * scope has not actually changed since the previous check -- callers own tracking the previous scope (in-memory + * or persisted); this module holds no state of its own. On a trip, also fires the PagerDuty page (#7666) + * unless `notify` is overridden (tests) or the integration flag/key is unset. */ export function recordMinerKillSwitchTransition( input: RecordMinerKillSwitchTransitionInput, - options: { append?: (event: AppendGovernorEventInput) => GovernorLedgerEntry } = {}, + options: { + append?: (event: AppendGovernorEventInput) => GovernorLedgerEntry; + notify?: NotifyMinerKillSwitchTrip; + env?: Record; + } = {}, ): GovernorLedgerEntry | null { const event = buildMinerKillSwitchTransitionGovernorLedgerEvent(input); if (!event) return null; const append = options.append ?? appendGovernorEvent; - return append(event as AppendGovernorEventInput); + const recorded = append(event as AppendGovernorEventInput); + + const alert = buildMinerKillSwitchPagerDutyAlert({ + repoFullName: input.repoFullName, + previousScope: input.previousScope, + scope: input.scope, + }); + if (alert) { + const notify = options.notify ?? notifyMinerKillSwitchPagerDuty; + const env = options.env ?? process.env; + try { + // Promise.resolve wraps sync returns so both sync throws and async rejects share one failure path. + void Promise.resolve(notify(alert, env)).catch((error: unknown) => { + warnKillSwitchPagerDutyFailed(alert.repoFullName, error); + }); + } catch (error) { + warnKillSwitchPagerDutyFailed(alert.repoFullName, error); + } + } + + return recorded; } diff --git a/src/services/notify-pagerduty.ts b/src/services/notify-pagerduty.ts index 073aeaf4a0..8ab33dfe1e 100644 --- a/src/services/notify-pagerduty.ts +++ b/src/services/notify-pagerduty.ts @@ -1,6 +1,8 @@ import { countRecentAuditEventsForActorAndTarget, recordAuditEvent } from "../db/repositories"; import { errorMessage } from "../utils/json"; import { meetsSeverityThreshold, resolveSeverityThreshold, type LoopoverSeverity } from "./severity-threshold"; +import { buildMinerKillSwitchPagerDutyAlert } from "../../packages/loopover-engine/src/governor/kill-switch.js"; +import type { MinerKillSwitchScope } from "../../packages/loopover-engine/src/governor/kill-switch.js"; // PagerDuty Events API v2 (https://developer.pagerduty.com/docs/events-api-v2/overview/). Experimental, // default-OFF (LOOPOVER_ENABLE_PAGERDUTY) — a self-host operator opts in per #4937's paging epic. @@ -218,3 +220,28 @@ export async function triggerPagerDutyIncident( await auditPagerDutyNotification(env, { repoFullName: params.repoFullName, dedupKey: params.dedupKey }, "error", message.slice(0, 280)); } } + +/** + * Page on an AMS miner kill-switch TRIP (#7666), reusing {@link triggerPagerDutyIncident} so hosted/ORB and + * the miner share one Events API path (flag, routing key, severity floor, cooldown). No-op on resume / + * same-scope — only an engage wakes a human. Pure payload comes from `@loopover/engine`'s + * `buildMinerKillSwitchPagerDutyAlert`. + */ +export async function notifyMinerKillSwitchPagerDuty( + env: Env, + input: { + repoFullName?: string | null | undefined; + previousScope: MinerKillSwitchScope; + scope: MinerKillSwitchScope; + }, +): Promise { + const alert = buildMinerKillSwitchPagerDutyAlert(input); + if (!alert) return; + await triggerPagerDutyIncident(env, { + repoFullName: alert.repoFullName, + summary: alert.summary, + severity: alert.severity, + dedupKey: alert.dedupKey, + customDetails: alert.customDetails, + }); +} diff --git a/test/unit/kill-switch-incident-runbook.test.ts b/test/unit/kill-switch-incident-runbook.test.ts index 582fc5e0fd..299bbb9c67 100644 --- a/test/unit/kill-switch-incident-runbook.test.ts +++ b/test/unit/kill-switch-incident-runbook.test.ts @@ -4,6 +4,7 @@ import { describe, expect, it } from "vitest"; import { MINER_KILL_SWITCH_ENV_VAR, + buildMinerKillSwitchPagerDutyAlert, buildMinerKillSwitchTransitionGovernorLedgerEvent, resolveMinerKillSwitch, } from "../../packages/loopover-engine/src/governor/kill-switch"; @@ -36,6 +37,8 @@ describe("kill-switch incident runbook (#4809)", () => { expect(runbook).toContain("15 minutes"); expect(runbook).toContain("2 minutes"); expect(runbook).toContain("#7180"); + expect(runbook).toContain("LOOPOVER_ENABLE_PAGERDUTY"); + expect(runbook).toContain("ams_kill_switch"); expect(resolveMinerKillSwitch({ global: true, repoPaused: true })).toBe("global"); expect(resolveMinerKillSwitch({ global: false, repoPaused: true })).toBe("repo"); @@ -78,3 +81,39 @@ describe("kill-switch incident runbook (#4809)", () => { expect(runbook).toMatch(/do not reach for those commands/i); }); }); + +describe("buildMinerKillSwitchPagerDutyAlert under vitest (#7666)", () => { + it("covers trip / resume / same-scope / fleet / blank-repo branches", () => { + expect( + buildMinerKillSwitchPagerDutyAlert({ repoFullName: "acme/widgets", previousScope: "none", scope: "repo" }), + ).toMatchObject({ + repoFullName: "acme/widgets", + severity: "critical", + dedupKey: "ams_kill_switch:repo:acme/widgets", + }); + + expect( + buildMinerKillSwitchPagerDutyAlert({ previousScope: "none", scope: "global" }), + ).toMatchObject({ + repoFullName: "ams/fleet", + dedupKey: "ams_kill_switch:global:ams/fleet", + summary: expect.stringContaining("fleet-wide"), + }); + + expect( + buildMinerKillSwitchPagerDutyAlert({ repoFullName: null, previousScope: "none", scope: "global" }), + ).toMatchObject({ repoFullName: "ams/fleet" }); + + expect( + buildMinerKillSwitchPagerDutyAlert({ repoFullName: " ", previousScope: "none", scope: "global" }), + ).toMatchObject({ repoFullName: "ams/fleet" }); + + expect( + buildMinerKillSwitchPagerDutyAlert({ repoFullName: "acme/widgets", previousScope: "repo", scope: "none" }), + ).toBeNull(); + + expect( + buildMinerKillSwitchPagerDutyAlert({ repoFullName: "acme/widgets", previousScope: "repo", scope: "repo" }), + ).toBeNull(); + }); +}); diff --git a/test/unit/miner-governor-kill-switch.test.ts b/test/unit/miner-governor-kill-switch.test.ts index 412defcbf8..8d87a8aa12 100644 --- a/test/unit/miner-governor-kill-switch.test.ts +++ b/test/unit/miner-governor-kill-switch.test.ts @@ -7,7 +7,7 @@ vi.mock("@loopover/engine", async () => { return import("../../packages/loopover-engine/src/index"); }); -import { checkMinerKillSwitch, recordMinerKillSwitchTransition } from "../../packages/loopover-miner/lib/governor-kill-switch.js"; +import { checkMinerKillSwitch, notifyMinerKillSwitchPagerDuty, recordMinerKillSwitchTransition } from "../../packages/loopover-miner/lib/governor-kill-switch.js"; import { initGovernorLedger } from "../../packages/loopover-miner/lib/governor-ledger.js"; const roots: string[] = []; @@ -119,7 +119,7 @@ describe("recordMinerKillSwitchTransition (#2341)", () => { actionClass: "open_pr", previousScope: "none", scope: "repo", - }); + }, { notify: () => undefined }); expect(tripped?.decision).toBe("tripped"); closeDefaultGovernorLedger(); @@ -133,4 +133,289 @@ describe("recordMinerKillSwitchTransition (#2341)", () => { else process.env.LOOPOVER_MINER_GOVERNOR_LEDGER_DB = previousDbPath; } }); + + it("pages on trip via the injectable notify hook and stays silent on resume (#7666)", () => { + const root = mkdtempSync(join(tmpdir(), "loopover-miner-governor-kill-switch-page-")); + roots.push(root); + const ledger = initGovernorLedger(join(root, "governor-ledger.sqlite3")); + ledgers.push(ledger); + const notify = vi.fn(); + + recordMinerKillSwitchTransition( + { repoFullName: "acme/widgets", actionClass: "open_pr", previousScope: "none", scope: "repo" }, + { append: (event) => ledger.appendGovernorEvent(event), notify }, + ); + expect(notify).toHaveBeenCalledTimes(1); + expect(notify.mock.calls[0]?.[0]).toMatchObject({ + repoFullName: "acme/widgets", + severity: "critical", + dedupKey: "ams_kill_switch:repo:acme/widgets", + }); + + notify.mockClear(); + recordMinerKillSwitchTransition( + { repoFullName: "acme/widgets", actionClass: "open_pr", previousScope: "repo", scope: "none" }, + { append: (event) => ledger.appendGovernorEvent(event), notify }, + ); + expect(notify).not.toHaveBeenCalled(); + }); + + it("swallows a rejected notify promise without failing the ledger write (#7666)", async () => { + const root = mkdtempSync(join(tmpdir(), "loopover-miner-governor-kill-switch-notify-reject-")); + roots.push(root); + const ledger = initGovernorLedger(join(root, "governor-ledger.sqlite3")); + ledgers.push(ledger); + const warn = vi.spyOn(console, "warn").mockImplementation(() => undefined); + + const tripped = recordMinerKillSwitchTransition( + { repoFullName: "acme/widgets", actionClass: "open_pr", previousScope: "none", scope: "repo" }, + { + append: (event) => ledger.appendGovernorEvent(event), + notify: async () => { + throw new Error("async pagerduty down"); + }, + }, + ); + expect(tripped?.decision).toBe("tripped"); + await vi.waitFor(() => { + expect(warn).toHaveBeenCalledWith(expect.stringContaining("kill_switch_pagerduty_failed")); + }); + warn.mockRestore(); + }); + + it("a sync void notify is accepted without treating it as a rejected promise (#7666)", () => { + const root = mkdtempSync(join(tmpdir(), "loopover-miner-governor-kill-switch-sync-void-")); + roots.push(root); + const ledger = initGovernorLedger(join(root, "governor-ledger.sqlite3")); + ledgers.push(ledger); + const warn = vi.spyOn(console, "warn").mockImplementation(() => undefined); + let called = false; + + const tripped = recordMinerKillSwitchTransition( + { repoFullName: "acme/widgets", actionClass: "open_pr", previousScope: "none", scope: "repo" }, + { + append: (event) => ledger.appendGovernorEvent(event), + notify: () => { + called = true; + }, + }, + ); + expect(tripped?.decision).toBe("tripped"); + expect(called).toBe(true); + expect(warn).not.toHaveBeenCalled(); + warn.mockRestore(); + }); + + it("swallows a sync Error throw from notify (#7666)", () => { + const root = mkdtempSync(join(tmpdir(), "loopover-miner-governor-kill-switch-notify-error-throw-")); + roots.push(root); + const ledger = initGovernorLedger(join(root, "governor-ledger.sqlite3")); + ledgers.push(ledger); + const warn = vi.spyOn(console, "warn").mockImplementation(() => undefined); + + const tripped = recordMinerKillSwitchTransition( + { repoFullName: "acme/widgets", actionClass: "open_pr", previousScope: "none", scope: "global" }, + { + append: (event) => ledger.appendGovernorEvent(event), + notify: () => { + throw new Error("pagerduty error fail"); + }, + }, + ); + expect(tripped?.decision).toBe("tripped"); + expect(warn).toHaveBeenCalledWith(expect.stringContaining("pagerduty error fail")); + warn.mockRestore(); + }); + + it("swallows a sync non-Error throw from notify (#7666)", () => { + const root = mkdtempSync(join(tmpdir(), "loopover-miner-governor-kill-switch-notify-string-throw-")); + roots.push(root); + const ledger = initGovernorLedger(join(root, "governor-ledger.sqlite3")); + ledgers.push(ledger); + const warn = vi.spyOn(console, "warn").mockImplementation(() => undefined); + + const tripped = recordMinerKillSwitchTransition( + { repoFullName: "acme/widgets", actionClass: "open_pr", previousScope: "none", scope: "global" }, + { + append: (event) => ledger.appendGovernorEvent(event), + notify: () => { + throw "pagerduty string fail"; + }, + }, + ); + expect(tripped?.decision).toBe("tripped"); + expect(warn).toHaveBeenCalledWith(expect.stringContaining("pagerduty string fail")); + warn.mockRestore(); + }); + + it("swallows a non-Error rejected notify promise (#7666)", async () => { + const root = mkdtempSync(join(tmpdir(), "loopover-miner-governor-kill-switch-notify-string-reject-")); + roots.push(root); + const ledger = initGovernorLedger(join(root, "governor-ledger.sqlite3")); + ledgers.push(ledger); + const warn = vi.spyOn(console, "warn").mockImplementation(() => undefined); + + recordMinerKillSwitchTransition( + { repoFullName: "acme/widgets", actionClass: "open_pr", previousScope: "none", scope: "repo" }, + { + append: (event) => ledger.appendGovernorEvent(event), + notify: async () => { + throw "async string fail"; + }, + }, + ); + await vi.waitFor(() => { + expect(warn).toHaveBeenCalledWith(expect.stringContaining("async string fail")); + }); + warn.mockRestore(); + }); + + it("uses the default notify + process.env when overrides are omitted (#7666)", async () => { + const root = mkdtempSync(join(tmpdir(), "loopover-miner-governor-kill-switch-default-notify-")); + roots.push(root); + const ledger = initGovernorLedger(join(root, "governor-ledger.sqlite3")); + ledgers.push(ledger); + const calls: unknown[] = []; + vi.stubGlobal("fetch", async (...args: unknown[]) => { + calls.push(args); + return new Response(null, { status: 202 }); + }); + const previousFlag = process.env.LOOPOVER_ENABLE_PAGERDUTY; + const previousKey = process.env.PAGERDUTY_ROUTING_KEY; + process.env.LOOPOVER_ENABLE_PAGERDUTY = "1"; + process.env.PAGERDUTY_ROUTING_KEY = "a".repeat(32); + try { + const tripped = recordMinerKillSwitchTransition( + { repoFullName: "acme/widgets", actionClass: "open_pr", previousScope: "none", scope: "repo" }, + { append: (event) => ledger.appendGovernorEvent(event) }, + ); + expect(tripped?.decision).toBe("tripped"); + await vi.waitFor(() => { + expect(calls.length).toBeGreaterThan(0); + }); + } finally { + if (previousFlag === undefined) delete process.env.LOOPOVER_ENABLE_PAGERDUTY; + else process.env.LOOPOVER_ENABLE_PAGERDUTY = previousFlag; + if (previousKey === undefined) delete process.env.PAGERDUTY_ROUTING_KEY; + else process.env.PAGERDUTY_ROUTING_KEY = previousKey; + vi.unstubAllGlobals(); + } + }); +}); + +describe("notifyMinerKillSwitchPagerDuty (#7666)", () => { + const VALID_KEY = "a".repeat(32); + const ALERT = { + repoFullName: "acme/widgets", + summary: "AMS miner kill-switch engaged (repo) for acme/widgets", + severity: "critical" as const, + dedupKey: "ams_kill_switch:repo:acme/widgets", + customDetails: { previousScope: "none" as const, scope: "repo" as const, reason: "repo_kill_switch_engaged" }, + }; + + it("no-ops when the PagerDuty flag is off or unset", async () => { + const calls: unknown[] = []; + vi.stubGlobal("fetch", async (...args: unknown[]) => { + calls.push(args); + return new Response(null, { status: 202 }); + }); + await notifyMinerKillSwitchPagerDuty(ALERT, { LOOPOVER_ENABLE_PAGERDUTY: "0", PAGERDUTY_ROUTING_KEY: VALID_KEY }); + await notifyMinerKillSwitchPagerDuty(ALERT, { PAGERDUTY_ROUTING_KEY: VALID_KEY }); + expect(calls).toHaveLength(0); + vi.unstubAllGlobals(); + }); + + it("posts Events API v2 when enabled with a valid routing key", async () => { + const calls: Array<{ url: string; body: Record }> = []; + vi.stubGlobal("fetch", async (url: RequestInfo | URL, init?: RequestInit) => { + calls.push({ url: String(url), body: init?.body ? (JSON.parse(String(init.body)) as Record) : {} }); + return new Response(null, { status: 202 }); + }); + await notifyMinerKillSwitchPagerDuty(ALERT, { LOOPOVER_ENABLE_PAGERDUTY: "1", PAGERDUTY_ROUTING_KEY: VALID_KEY }); + expect(calls).toHaveLength(1); + expect(calls[0]?.url).toBe("https://events.pagerduty.com/v2/enqueue"); + expect(calls[0]?.body).toMatchObject({ + routing_key: VALID_KEY, + event_action: "trigger", + dedup_key: "ams_kill_switch:repo:acme/widgets", + payload: { source: "loopover-miner", severity: "critical", component: "acme/widgets" }, + }); + vi.unstubAllGlobals(); + }); + + it("no-ops when the routing key is missing, blank, invalid, or non-string", async () => { + const calls: unknown[] = []; + vi.stubGlobal("fetch", async (...args: unknown[]) => { + calls.push(args); + return new Response(null, { status: 202 }); + }); + await notifyMinerKillSwitchPagerDuty(ALERT, { LOOPOVER_ENABLE_PAGERDUTY: "1" }); + await notifyMinerKillSwitchPagerDuty(ALERT, { LOOPOVER_ENABLE_PAGERDUTY: "1", PAGERDUTY_ROUTING_KEY: " " }); + await notifyMinerKillSwitchPagerDuty(ALERT, { LOOPOVER_ENABLE_PAGERDUTY: "1", PAGERDUTY_ROUTING_KEY: "not-a-key" }); + await notifyMinerKillSwitchPagerDuty(ALERT, { + LOOPOVER_ENABLE_PAGERDUTY: "1", + PAGERDUTY_ROUTING_KEY: 42 as unknown as string, + }); + expect(calls).toHaveLength(0); + vi.unstubAllGlobals(); + }); + + it("warns but does not throw when PagerDuty returns a non-ok status", async () => { + vi.stubGlobal("fetch", async () => new Response(null, { status: 500 })); + const warn = vi.spyOn(console, "warn").mockImplementation(() => undefined); + await expect( + notifyMinerKillSwitchPagerDuty(ALERT, { LOOPOVER_ENABLE_PAGERDUTY: "yes", PAGERDUTY_ROUTING_KEY: VALID_KEY }), + ).resolves.toBeUndefined(); + expect(warn).toHaveBeenCalledWith(expect.stringContaining("kill_switch_pagerduty_failed")); + warn.mockRestore(); + vi.unstubAllGlobals(); + }); + + it("never throws when fetch rejects with a non-Error value", async () => { + vi.stubGlobal("fetch", async () => { + throw "network string fail"; + }); + const warn = vi.spyOn(console, "warn").mockImplementation(() => undefined); + await expect( + notifyMinerKillSwitchPagerDuty(ALERT, { LOOPOVER_ENABLE_PAGERDUTY: "true", PAGERDUTY_ROUTING_KEY: VALID_KEY }), + ).resolves.toBeUndefined(); + expect(warn).toHaveBeenCalledWith(expect.stringContaining("network string fail")); + warn.mockRestore(); + vi.unstubAllGlobals(); + }); + + it("never throws when fetch rejects with an Error", async () => { + vi.stubGlobal("fetch", async () => { + throw new Error("network down"); + }); + const warn = vi.spyOn(console, "warn").mockImplementation(() => undefined); + await expect( + notifyMinerKillSwitchPagerDuty(ALERT, { LOOPOVER_ENABLE_PAGERDUTY: "true", PAGERDUTY_ROUTING_KEY: VALID_KEY }), + ).resolves.toBeUndefined(); + expect(warn).toHaveBeenCalledWith(expect.stringContaining("network down")); + warn.mockRestore(); + vi.unstubAllGlobals(); + }); + + it("defaults env to process.env when the second arg is omitted", async () => { + const calls: unknown[] = []; + vi.stubGlobal("fetch", async (...args: unknown[]) => { + calls.push(args); + return new Response(null, { status: 202 }); + }); + const previousFlag = process.env.LOOPOVER_ENABLE_PAGERDUTY; + const previousKey = process.env.PAGERDUTY_ROUTING_KEY; + process.env.LOOPOVER_ENABLE_PAGERDUTY = "on"; + process.env.PAGERDUTY_ROUTING_KEY = VALID_KEY; + try { + await notifyMinerKillSwitchPagerDuty(ALERT); + expect(calls).toHaveLength(1); + } finally { + if (previousFlag === undefined) delete process.env.LOOPOVER_ENABLE_PAGERDUTY; + else process.env.LOOPOVER_ENABLE_PAGERDUTY = previousFlag; + if (previousKey === undefined) delete process.env.PAGERDUTY_ROUTING_KEY; + else process.env.PAGERDUTY_ROUTING_KEY = previousKey; + vi.unstubAllGlobals(); + } + }); }); diff --git a/test/unit/notify-pagerduty.test.ts b/test/unit/notify-pagerduty.test.ts index 9b01efc361..c3484bd21c 100644 --- a/test/unit/notify-pagerduty.test.ts +++ b/test/unit/notify-pagerduty.test.ts @@ -1,6 +1,7 @@ import { afterEach, describe, expect, it, vi } from "vitest"; import { isPagerDutyEnabled, + notifyMinerKillSwitchPagerDuty, resolvePagerDutyCooldownMinutes, resolvePagerDutyMinSeverity, resolvePagerDutyRoutingKey, @@ -284,3 +285,43 @@ describe("triggerPagerDutyIncident — HTTP delivery", () => { warn.mockRestore(); }); }); + +describe("notifyMinerKillSwitchPagerDuty (#7666)", () => { + it("pages through triggerPagerDutyIncident on a trip", async () => { + const calls = stubFetch(202); + const env = enabledEnv(); + await notifyMinerKillSwitchPagerDuty(env, { + repoFullName: "acme/widgets", + previousScope: "none", + scope: "repo", + }); + expect(calls).toHaveLength(1); + expect(calls[0]?.body).toMatchObject({ + dedup_key: "ams_kill_switch:repo:acme/widgets", + payload: { severity: "critical", component: "acme/widgets", source: "loopover" }, + }); + expect(await pagerDutyAudit(env)).toEqual([expect.objectContaining({ outcome: "completed", detail: "triggered" })]); + }); + + it("is a no-op on resume so clearing a halt does not page", async () => { + const calls = stubFetch(202); + const env = enabledEnv(); + await notifyMinerKillSwitchPagerDuty(env, { + repoFullName: "acme/widgets", + previousScope: "repo", + scope: "none", + }); + expect(calls).toHaveLength(0); + expect(await pagerDutyAudit(env)).toHaveLength(0); + }); + + it("uses ams/fleet when a global trip has no repoFullName", async () => { + const calls = stubFetch(202); + const env = enabledEnv(); + await notifyMinerKillSwitchPagerDuty(env, { previousScope: "none", scope: "global" }); + expect(calls[0]?.body).toMatchObject({ + dedup_key: "ams_kill_switch:global:ams/fleet", + payload: { component: "ams/fleet" }, + }); + }); +});