diff --git a/apps/mobile/src/features/threads/thread-list-v2-items.tsx b/apps/mobile/src/features/threads/thread-list-v2-items.tsx index 69981775482e..b571c02bbc92 100644 --- a/apps/mobile/src/features/threads/thread-list-v2-items.tsx +++ b/apps/mobile/src/features/threads/thread-list-v2-items.tsx @@ -59,6 +59,8 @@ const STATUS_LABEL_BY_STATUS: Partial< approval: { label: "Approval", className: "text-amber-700 dark:text-amber-300" }, input: { label: "Input", className: "text-indigo-600 dark:text-indigo-300" }, working: { label: "Working", className: "text-sky-600 dark:text-sky-400" }, + // Colorless like the web sidebar: parked on background work, not "act now". + waiting: { label: "Waiting", className: "text-foreground-tertiary" }, failed: { label: "Failed", className: "text-red-700 dark:text-red-300" }, }; diff --git a/apps/mobile/src/features/threads/threadListV2.test.ts b/apps/mobile/src/features/threads/threadListV2.test.ts index 98a6f92e448f..adc9cf781156 100644 --- a/apps/mobile/src/features/threads/threadListV2.test.ts +++ b/apps/mobile/src/features/threads/threadListV2.test.ts @@ -123,6 +123,26 @@ describe("resolveThreadListV2Status", () => { expect(resolveThreadListV2Status(thread)).toBe("approval"); }); + it("reports waiting when presentation parks runtime idle for background tasks", () => { + expect( + resolveThreadListV2Status( + makeThread({ + id: ThreadId.make("t"), + title: "t", + pendingBackgroundTasks: [{ taskId: "bg-1", description: "Run Codex review" }], + runtime: { + status: "idle", + activeRunId: null, + providerInstanceId: ProviderInstanceId.make("codex"), + providerName: "Codex", + lastError: null, + updatedAt: NOW, + }, + }), + ), + ).toBe("waiting"); + }); + it("resolves ready for quiescent threads", () => { expect(resolveThreadListV2Status(makeThread({ id: ThreadId.make("t"), title: "t" }))).toBe( "ready", diff --git a/apps/mobile/src/features/threads/threadListV2.ts b/apps/mobile/src/features/threads/threadListV2.ts index 6ab7df66a923..1c3183001440 100644 --- a/apps/mobile/src/features/threads/threadListV2.ts +++ b/apps/mobile/src/features/threads/threadListV2.ts @@ -19,11 +19,13 @@ export { snoozeWakeLabel }; * Thread List v2 model, ported from the web sidebar v2 * (apps/web/src/components/Sidebar.logic.ts + SidebarV2.tsx). * - * Four visual states, three colors: color is reserved for "act now" - * (approval), "in motion" (working), and "broken" (failed). Ready is the - * unlabeled resting state. + * Six visual states. Color distinguishes approval, input, active work, and + * failures. Ready is the unlabeled resting state; waiting (runtime status "idle") is the agent + * parked on open background tasks, grey like working rather than a false Done. + * The orchestrator v2 presentation bridge parks runtime at idle when the + * post-settlement background roster is nonempty. */ -export type ThreadListV2Status = "approval" | "input" | "working" | "failed" | "ready"; +export type ThreadListV2Status = "approval" | "input" | "working" | "waiting" | "failed" | "ready"; export type ThreadListV2SwipeAction = "archive" | "settle" | "unsettle" | "snooze" | "unsnooze"; export function resolveThreadListV2SnoozeMenuSelection(input: { @@ -157,6 +159,9 @@ export function resolveThreadListV2Status( ) { return "working"; } + if (thread.runtime?.status === "idle") { + return "waiting"; + } if (thread.runtime?.status === "failed") { return "failed"; } diff --git a/apps/mobile/src/state/use-thread-composer-state.ts b/apps/mobile/src/state/use-thread-composer-state.ts index 4b199fbb6f2a..08718df03f17 100644 --- a/apps/mobile/src/state/use-thread-composer-state.ts +++ b/apps/mobile/src/state/use-thread-composer-state.ts @@ -3,6 +3,7 @@ import { threadRuntimeIsActive } from "@t3tools/client-runtime/state/shell"; import { deriveThreadActivityRun, deriveThreadRuntime, + threadRuntimeHasInterruptibleRun, } from "@t3tools/client-runtime/state/thread-execution"; import { useCallback, useEffect, useMemo } from "react"; @@ -150,7 +151,9 @@ export function useThreadComposerState() { }, [selectedThreadActivityRun, selectedThreadSessionActivity, selectedThreadShell]); const activeThreadBusy = threadRuntimeIsActive(selectedThreadRuntime); - const interruptibleRunId = selectedThreadRuntime?.activeRunId ?? null; + const interruptibleRunId = threadRuntimeHasInterruptibleRun(selectedThreadRuntime) + ? (selectedThreadRuntime?.activeRunId ?? null) + : null; const onSendMessage = useCallback(async () => { if (!selectedThreadShell) { diff --git a/apps/server/src/mcp/OrchestratorMcpService.activity.test.ts b/apps/server/src/mcp/OrchestratorMcpService.activity.test.ts new file mode 100644 index 000000000000..ccb4393684cf --- /dev/null +++ b/apps/server/src/mcp/OrchestratorMcpService.activity.test.ts @@ -0,0 +1,307 @@ +import { + EnvironmentId, + NodeId, + type OrchestrationV2ThreadProjection, + ProjectId, + ProviderDriverKind, + ProviderInstanceId, + RunId, + ThreadId, +} from "@t3tools/contracts"; +import * as DateTime from "effect/DateTime"; +import * as Effect from "effect/Effect"; +import * as Layer from "effect/Layer"; +import * as NodeCrypto from "@effect/platform-node/NodeCrypto"; +import { expect, it } from "vite-plus/test"; + +import { ProviderRegistry } from "../provider/Services/ProviderRegistry.ts"; +import { ScheduledTaskService } from "../scheduledTasks/ScheduledTaskService.ts"; +import { ThreadManagementService } from "../orchestration-v2/ThreadManagementService.ts"; +import type * as McpInvocationContext from "./McpInvocationContext.ts"; +import { + layer as orchestratorMcpServiceLayer, + OrchestratorMcpService, +} from "./OrchestratorMcpService.ts"; + +const environmentId = EnvironmentId.make("environment-mcp-orchestrator-detail"); +const projectId = ProjectId.make("project-mcp-orchestrator-detail"); +const parentThreadId = ThreadId.make("thread-mcp-orchestrator-parent"); +const childThreadId = ThreadId.make("thread-mcp-orchestrator-child"); +const activeRunId = RunId.make("run-mcp-active"); +const cancelledRunId = RunId.make("run-mcp-cancelled"); +const childRunId = RunId.make("run-mcp-child"); +const taskId = NodeId.make("node-mcp-task-1"); +const now = DateTime.makeUnsafe("2026-08-04T12:00:00.000Z"); +const codexDriver = ProviderDriverKind.make("codex"); +// Distinct from driver kind so a regression that re-derives from driver fails. +const customCodexInstanceId = ProviderInstanceId.make("codex-custom-workspace"); +const parentInstanceId = ProviderInstanceId.make("codex"); + +const makeScope = (): McpInvocationContext.McpInvocationScope => ({ + environmentId, + threadId: parentThreadId, + providerSessionId: "provider-session-mcp-orchestrator-detail", + providerInstanceId: parentInstanceId, + capabilities: new Set(["orchestration"]), + issuedAt: 1, +}); + +function baseThread(input: { + readonly threadId: ThreadId; + readonly title: string; + readonly instanceId: ProviderInstanceId; + readonly model: string; +}) { + return { + id: input.threadId, + projectId, + title: input.title, + createdBy: "user" as const, + creationSource: "mcp" as const, + modelSelection: { + instanceId: input.instanceId, + model: input.model, + }, + runtimeMode: "full-access" as const, + interactionMode: "default" as const, + branch: null, + worktreePath: null, + lineage: { + parentThreadId: null, + relationshipToParent: null, + rootThreadId: input.threadId, + }, + archivedAt: null, + deletedAt: null, + providerInstanceId: input.instanceId, + createdAt: now, + updatedAt: now, + }; +} + +function makeRun(input: { + readonly id: RunId; + readonly ordinal: number; + readonly status: "running" | "waiting" | "cancelled" | "queued" | "completed"; + readonly instanceId?: ProviderInstanceId; +}) { + return { + id: input.id, + ordinal: input.ordinal, + status: input.status, + modelSelection: { + instanceId: input.instanceId ?? parentInstanceId, + model: "gpt-5.4", + }, + providerInstanceId: input.instanceId ?? parentInstanceId, + requestedAt: now, + startedAt: input.status === "cancelled" || input.status === "queued" ? null : now, + completedAt: input.status === "cancelled" || input.status === "completed" ? now : null, + }; +} + +it("readThread prefers activity-run status over a newer cancelled queued run", async () => { + const projection = { + thread: baseThread({ + threadId: parentThreadId, + title: "Parent", + instanceId: parentInstanceId, + model: "gpt-5.4", + }), + runs: [ + makeRun({ id: activeRunId, ordinal: 1, status: "running" }), + makeRun({ id: cancelledRunId, ordinal: 2, status: "cancelled" }), + ], + visibleTurnItems: [], + runtimeRequests: [], + messages: [], + contextTransfers: [], + subagents: [], + updatedAt: now, + } as unknown as OrchestrationV2ThreadProjection; + + const layer = orchestratorMcpServiceLayer.pipe( + Layer.provide( + Layer.mergeAll( + Layer.mock(ThreadManagementService)({ + getThreadProjection: (threadId) => + threadId === parentThreadId + ? Effect.succeed(projection) + : Effect.die(`unexpected thread ${threadId}`), + } satisfies Partial), + Layer.mock(ProviderRegistry)({ + getProviders: Effect.succeed([]), + } satisfies Partial), + Layer.mock(ScheduledTaskService)({ + list: () => Effect.succeed({ tasks: [] }), + } satisfies Partial), + NodeCrypto.layer, + ), + ), + ); + + await Effect.gen(function* () { + const service = yield* OrchestratorMcpService; + const result = yield* service.readThread(makeScope(), { threadId: parentThreadId }); + expect(result.thread.status).toBe("running"); + expect(result.thread.latestRunId).toBe(cancelledRunId); + expect(result.thread.activeRunId).toBe(activeRunId); + }).pipe(Effect.provide(layer), Effect.runPromise); +}); + +it("readThread prefers waiting activity status over a newer cancelled queued run", async () => { + const projection = { + thread: baseThread({ + threadId: parentThreadId, + title: "Parent waiting", + instanceId: parentInstanceId, + model: "gpt-5.4", + }), + runs: [ + makeRun({ id: activeRunId, ordinal: 1, status: "waiting" }), + makeRun({ id: cancelledRunId, ordinal: 2, status: "cancelled" }), + ], + visibleTurnItems: [], + runtimeRequests: [], + messages: [], + contextTransfers: [], + subagents: [], + updatedAt: now, + } as unknown as OrchestrationV2ThreadProjection; + + const layer = orchestratorMcpServiceLayer.pipe( + Layer.provide( + Layer.mergeAll( + Layer.mock(ThreadManagementService)({ + getThreadProjection: (threadId) => + threadId === parentThreadId + ? Effect.succeed(projection) + : Effect.die(`unexpected thread ${threadId}`), + } satisfies Partial), + Layer.mock(ProviderRegistry)({ + getProviders: Effect.succeed([]), + } satisfies Partial), + Layer.mock(ScheduledTaskService)({ + list: () => Effect.succeed({ tasks: [] }), + } satisfies Partial), + NodeCrypto.layer, + ), + ), + ); + + await Effect.gen(function* () { + const service = yield* OrchestratorMcpService; + const result = yield* service.readThread(makeScope(), { threadId: parentThreadId }); + expect(result.thread.status).toBe("waiting"); + expect(result.thread.activeRunId).toBe(activeRunId); + }).pipe(Effect.provide(layer), Effect.runPromise); +}); + +it("taskStatus returns task.providerInstanceId rather than the driver kind", async () => { + const parentProjection = { + thread: baseThread({ + threadId: parentThreadId, + title: "Parent", + instanceId: parentInstanceId, + model: "gpt-5.4", + }), + runs: [makeRun({ id: activeRunId, ordinal: 1, status: "running" })], + visibleTurnItems: [], + runtimeRequests: [], + messages: [], + contextTransfers: [], + subagents: [ + { + id: taskId, + threadId: parentThreadId, + runId: activeRunId, + parentNodeId: NodeId.make("node-parent"), + origin: "app_owned", + createdBy: "agent", + driver: codexDriver, + providerInstanceId: customCodexInstanceId, + providerThreadId: null, + childThreadId, + nativeTaskRef: null, + prompt: "Inspect the custom instance.", + title: null, + model: "gpt-5.4", + status: "running", + result: null, + startedAt: now, + completedAt: null, + updatedAt: now, + }, + ], + updatedAt: now, + } as unknown as OrchestrationV2ThreadProjection; + + const childProjection = { + thread: { + ...baseThread({ + threadId: childThreadId, + title: "Child", + instanceId: customCodexInstanceId, + model: "gpt-5.4", + }), + lineage: { + parentThreadId, + relationshipToParent: "subagent", + rootThreadId: parentThreadId, + }, + createdBy: "agent", + }, + runs: [ + makeRun({ + id: childRunId, + ordinal: 1, + status: "running", + instanceId: customCodexInstanceId, + }), + ], + visibleTurnItems: [], + runtimeRequests: [], + messages: [], + contextTransfers: [ + { + type: "subagent_spawn", + sourceThreadId: parentThreadId, + targetThreadId: childThreadId, + targetRunId: childRunId, + }, + ], + subagents: [], + updatedAt: now, + } as unknown as OrchestrationV2ThreadProjection; + + const layer = orchestratorMcpServiceLayer.pipe( + Layer.provide( + Layer.mergeAll( + Layer.mock(ThreadManagementService)({ + getThreadProjection: (threadId) => { + if (threadId === parentThreadId) return Effect.succeed(parentProjection); + if (threadId === childThreadId) return Effect.succeed(childProjection); + return Effect.die(`unexpected thread ${threadId}`); + }, + } satisfies Partial), + Layer.mock(ProviderRegistry)({ + getProviders: Effect.succeed([]), + } satisfies Partial), + Layer.mock(ScheduledTaskService)({ + list: () => Effect.succeed({ tasks: [] }), + } satisfies Partial), + NodeCrypto.layer, + ), + ), + ); + + await Effect.gen(function* () { + const service = yield* OrchestratorMcpService; + const result = yield* service.taskStatus(makeScope(), taskId); + expect(result.providerInstanceId).toBe(customCodexInstanceId); + expect(result.providerInstanceId).not.toBe(ProviderInstanceId.make(String(codexDriver))); + expect(result.status).toBe("running"); + expect(result.taskId).toBe(taskId); + expect(result.childThreadId).toBe(childThreadId); + }).pipe(Effect.provide(layer), Effect.runPromise); +}); diff --git a/apps/server/src/mcp/OrchestratorMcpService.ts b/apps/server/src/mcp/OrchestratorMcpService.ts index 1fa02efd2d58..7d7bd24f02de 100644 --- a/apps/server/src/mcp/OrchestratorMcpService.ts +++ b/apps/server/src/mcp/OrchestratorMcpService.ts @@ -43,7 +43,6 @@ import { type OrchestratorMcpThreadWaitInput, type OrchestratorMcpThreadWaitResult, type ProviderInteractionMode, - ProviderInstanceId, type ProviderOptionDescriptor, type ProviderOptionSelection, type RuntimeMode, @@ -512,7 +511,7 @@ function listItemFromShell(shell: OrchestrationV2ThreadShell): OrchestratorMcpTh title: shell.title, createdBy: shell.createdBy, creationSource: shell.creationSource, - status: shell.status, + status: shell.activityRunStatus ?? shell.status, latestRunId: shell.latestRunId, providerInstanceId: shell.modelSelection.instanceId, model: shell.modelSelection.model, @@ -535,7 +534,7 @@ function threadDetail(projection: OrchestrationV2ThreadProjection): Orchestrator title: projection.thread.title, createdBy: projection.thread.createdBy, creationSource: projection.thread.creationSource, - status: latest?.status ?? "idle", + status: active?.status ?? latest?.status ?? "idle", latestRunId: latest?.id ?? null, activeRunId: active?.id ?? null, providerInstanceId: projection.thread.modelSelection.instanceId, @@ -876,7 +875,7 @@ const make = Effect.gen(function* () { childRunId: childRun?.id ?? null, childNodeId: task.id, status, - providerInstanceId: ProviderInstanceId.make(task.driver), + providerInstanceId: task.providerInstanceId, model: task.model, summary: derivedResult, resultContextTransferId: resultTransfer?.id ?? null, @@ -1488,7 +1487,10 @@ const make = Effect.gen(function* () { const statuses = input.statuses === undefined ? null : new Set(input.statuses); const titleContains = input.titleContains?.toLocaleLowerCase(); const filtered = projectThreads - .filter((thread) => statuses === null || statuses.has(thread.status)) + .filter( + (thread) => + statuses === null || statuses.has(thread.activityRunStatus ?? thread.status), + ) .filter( (thread) => titleContains === undefined || diff --git a/apps/server/src/orchestration-v2/Adapters/AcpAdapterV2.test.ts b/apps/server/src/orchestration-v2/Adapters/AcpAdapterV2.test.ts index 32e5f62cf969..6f856f1920a1 100644 --- a/apps/server/src/orchestration-v2/Adapters/AcpAdapterV2.test.ts +++ b/apps/server/src/orchestration-v2/Adapters/AcpAdapterV2.test.ts @@ -50,6 +50,8 @@ import { ServerConfig } from "../../config.ts"; import * as McpProviderSession from "../../mcp/McpProviderSession.ts"; import * as AcpSessionRuntime from "../../provider/acp/AcpSessionRuntime.ts"; import { + extractXAiAcpSubagentEndNotice, + extractXAiAcpSubagentUpdate, normalizeXAiAcpToolCallState, registerXAiBackgroundTaskTracking, } from "../../provider/acp/XAiAcpExtension.ts"; @@ -63,6 +65,7 @@ import { import type { ProviderContinuationRequest } from "../ProviderContinuationRequests.ts"; import { AcpProviderCapabilitiesV2, + acpCarryoverTerminalShouldClearContinuation, acpCanonicalJson, acpClaimNativeTransportRequest, acpPermissionDisposition, @@ -73,10 +76,12 @@ import { acpPostSettleWakeEvidence, acpPostSettleWakeShouldBuffer, acpProjectedCommandExitCode, + acpTurnStartShouldPreserveContinuation, makeAcpAdapterV2, type AcpAdapterV2ExtensionContext, type AcpAdapterV2Flavor, type AcpAdapterV2RuntimeInput, + type AcpAdapterV2SubagentUpdate, } from "./AcpAdapterV2.ts"; const serverConfigLayer = ServerConfig.layerTest(process.cwd(), { @@ -279,6 +284,53 @@ describe("acpProjectedCommandExitCode", () => { }); }); +describe("ACP continuation ownership", () => { + it("preserves a continuation offered during non-buffered carryover handling", () => { + assert.isFalse( + acpCarryoverTerminalShouldClearContinuation({ + continuationOffered: true, + wakeBufferLength: 0, + }), + ); + assert.isFalse( + acpCarryoverTerminalShouldClearContinuation({ + continuationOffered: false, + wakeBufferLength: 1, + }), + ); + assert.isTrue( + acpCarryoverTerminalShouldClearContinuation({ + continuationOffered: false, + wakeBufferLength: 0, + }), + ); + }); + + it("preserves only user-raced offers that still own buffered wake traffic", () => { + assert.isTrue( + acpTurnStartShouldPreserveContinuation({ + continuationRequested: true, + isContinuationTurn: false, + wakeBufferLength: 1, + }), + ); + assert.isFalse( + acpTurnStartShouldPreserveContinuation({ + continuationRequested: true, + isContinuationTurn: false, + wakeBufferLength: 0, + }), + ); + assert.isFalse( + acpTurnStartShouldPreserveContinuation({ + continuationRequested: true, + isContinuationTurn: true, + wakeBufferLength: 1, + }), + ); + }); +}); + const taskkillPlatformError = (method: string) => PlatformError.systemError({ _tag: "Unknown", module: "taskkill-test", method }); @@ -586,6 +638,10 @@ function makeTurnInput(input: { readonly now: DateTime.Utc; readonly ordinal?: number; readonly modelSelection?: ModelSelection; + /** agent+provider marks a post-settle continuation attach (drains wakeBuffer). */ + readonly messageCreatedBy?: "user" | "agent"; + readonly messageCreationSource?: "web" | "mobile" | "mcp" | "provider" | "server"; + readonly messageText?: string; }): ProviderAdapterV2TurnInput { const ordinal = input.ordinal ?? 1; const suffix = `${input.threadId}:${ordinal}`; @@ -627,10 +683,10 @@ function makeTurnInput(input: { rootNodeId: NodeId.make(`node:${suffix}`), providerThread: input.providerThread, message: { - createdBy: "user", - creationSource: "web", + createdBy: input.messageCreatedBy ?? "user", + creationSource: input.messageCreationSource ?? "web", messageId: MessageId.make(`message:${suffix}`), - text: "test prompt", + text: input.messageText ?? "test prompt", attachments: [], }, modelSelection, @@ -3186,7 +3242,7 @@ describe("AcpAdapterV2", () => { ); it.effect( - "preserveRuntimeOnSettledInterrupt keeps the process alive and carries subagents through a settled steering interrupt", + "pins hasPendingBackgroundWork while carryover holds a live subagent after root settle", () => Effect.gen(function* () { const childProcessSpawner = yield* ChildProcessSpawner.ChildProcessSpawner; @@ -3200,8 +3256,6 @@ describe("AcpAdapterV2", () => { const protocolEvents = yield* Queue.bounded(256); const instanceId = ProviderInstanceId.make("acp-test"); let subagentPhase: "spawn" | "complete" = "spawn"; - let cancelCalled = false; - let runtimeOrdinalSeen = 0; const adapter = makeAcpAdapterV2({ crypto: yield* Crypto.Crypto, instanceId, @@ -3209,14 +3263,7 @@ describe("AcpAdapterV2", () => { driver: ACP_TEST_DRIVER, capabilities: AcpProviderCapabilitiesV2, deferFinalizeForBackgroundWork: true, - // Hard interrupt flags (stricter than production Grok, which no - // longer sets restartRuntimeOnEveryInterrupt): every interrupt - // would hard-kill the process group without the settled-soft gate - // under test. - restartRuntimeAfterInterrupt: true, - restartRuntimeOnEveryInterrupt: true, - terminateRuntimeProcessGroupOnInterrupt: true, - preserveRuntimeOnSettledInterrupt: true, + enablePostSettleContinuation: true, extractSubagentUpdate: (toolCall) => toolCall.toolCallId !== "tool-call-generic-1" ? undefined @@ -3239,28 +3286,19 @@ describe("AcpAdapterV2", () => { childSessionId: null, result: "SUB_DONE", }, - // No ownDetachedProcessGroup: if the interrupt wrongly takes the - // hard path, terminateProcessGroup is missing and the interrupt - // fails loudly with a poisoned session. makeRuntime: makeMockRuntime({ childProcessSpawner, mockAgentPath, - environment: (runtimeOrdinal) => { - runtimeOrdinalSeen = Math.max(runtimeOrdinalSeen, runtimeOrdinal); - return { T3_ACP_EMIT_GENERIC_TOOL_PLACEHOLDERS: "1" }; - }, + environment: { T3_ACP_EMIT_GENERIC_TOOL_PLACEHOLDERS: "1" }, protocolEvents, - wrapCancel: (cancel) => - Effect.sync(() => { - cancelCalled = true; - }).pipe(Effect.andThen(cancel)), }), }, fileSystem, idAllocator, serverConfig, + continuationRequests: { offer: () => Effect.void }, }); - const threadId = ThreadId.make("thread-acp-settled-soft-steer"); + const threadId = ThreadId.make("thread-acp-carryover-pending-pin"); const runtimePolicy = ProviderAdapterV2RuntimePolicy.make({ runtimeMode: "full-access", interactionMode: "default", @@ -3269,10 +3307,16 @@ describe("AcpAdapterV2", () => { const modelSelection = { instanceId, model: "default" } as const; const runtime = yield* adapter.openSession({ threadId, - providerSessionId: ProviderSessionId.make("provider-session-acp-settled-soft-steer"), + providerSessionId: ProviderSessionId.make("provider-session-acp-carryover-pending-pin"), modelSelection, runtimePolicy, }); + if (runtime.hasPendingBackgroundWork === undefined) { + return yield* Effect.die( + "ACP runtime must expose hasPendingBackgroundWork when post-settle continuation is enabled.", + ); + } + const hasPendingBackgroundWork = runtime.hasPendingBackgroundWork; const events = yield* Queue.unbounded(); yield* runtime.events.pipe( Stream.runForEach((event) => Queue.offer(events, event)), @@ -3287,8 +3331,6 @@ describe("AcpAdapterV2", () => { yield* runtime.startTurn( makeTurnInput({ threadId, providerThread, instanceId, runtimePolicy, now }), ); - // The still-running subagent defers finalize after session/prompt returns, - // so the interrupt below hits a settled turn held open for background work. yield* Stream.fromQueue(protocolEvents).pipe( Stream.filter( (event) => @@ -3311,24 +3353,20 @@ describe("AcpAdapterV2", () => { .pipe(Effect.forkScoped); yield* TestClock.adjust("10 seconds"); yield* Fiber.join(interruptFiber); - assert.isFalse( - cancelCalled, - "settled soft steer must not send session/cancel (the real Grok CLI kills background subagents on cancel)", - ); - let subagentTurnItemId: string | null = null; let firstTerminalStatus: string | null = null; while (firstTerminalStatus === null) { const event = yield* Queue.take(events); - if (event.type === "turn_item.updated" && event.turnItem.type === "subagent") { - subagentTurnItemId = event.turnItem.id; - } if (event.type === "turn.terminal" && event.providerTurnId === firstProviderTurnId) { firstTerminalStatus = event.status; } } assert.equal(firstTerminalStatus, "interrupted"); - assert.notEqual(subagentTurnItemId, null); + // Root settled with a live projected subagent in carryover: pin idle release. + assert.isTrue( + yield* hasPendingBackgroundWork, + "carryover live subagent must pin hasPendingBackgroundWork after root settle", + ); subagentPhase = "complete"; const secondNow = yield* DateTime.now; @@ -3342,38 +3380,27 @@ describe("AcpAdapterV2", () => { ordinal: 2, }), ); - // Same runtime process (mock-session-1): a respawn would start - // mock-session-2 and drop the carryover on the session mismatch. const secondProviderTurnId = idAllocator.derive.providerTurn({ driver: ACP_TEST_DRIVER, nativeTurnId: "mock-session-1:turn:2", }); - let carriedItemStatus: string | null = null; let secondTerminalStatus: string | null = null; while (secondTerminalStatus === null) { const event = yield* Queue.take(events); - if ( - event.type === "turn_item.updated" && - event.turnItem.type === "subagent" && - event.turnItem.id === subagentTurnItemId - ) { - carriedItemStatus = event.turnItem.status; - } if (event.type === "turn.terminal" && event.providerTurnId === secondProviderTurnId) { secondTerminalStatus = event.status; } } - assert.equal(carriedItemStatus, "completed"); assert.equal(secondTerminalStatus, "completed"); - assert.equal( - runtimeOrdinalSeen, - 1, - "settled soft steer must not respawn the ACP runtime process", + // Carryover is consumed into the next turn and terminalized; pin clears. + assert.isFalse( + yield* hasPendingBackgroundWork, + "hasPendingBackgroundWork must clear after carryover subagent terminals", ); }).pipe(Effect.provide(testLayer), Effect.scoped), ); - it.live("preserveRuntimeOnSettledInterrupt does not soften a mid-prompt steering interrupt", () => + it.effect("handles a child terminal after carryover rehydrate", () => Effect.gen(function* () { const childProcessSpawner = yield* ChildProcessSpawner.ChildProcessSpawner; const fileSystem = yield* FileSystem.FileSystem; @@ -3384,35 +3411,74 @@ describe("AcpAdapterV2", () => { new URL("../../../scripts/acp-mock-agent.ts", import.meta.url), ); const protocolEvents = yield* Queue.bounded(256); + const secondPromptWireReturned = yield* Deferred.make(); + const releaseSecondPromptCompletion = yield* Deferred.make(); const instanceId = ProviderInstanceId.make("acp-test"); + const childSessionId = "mock-child-session-active-carryover"; + let promptCount = 0; + let subagentPhase: "spawn" | "complete" = "spawn"; + type RuntimeService = AcpSessionRuntime.AcpSessionRuntime["Service"]; + let sessionUpdateHandler: Parameters[0] | undefined; const adapter = makeAcpAdapterV2({ crypto: yield* Crypto.Crypto, instanceId, flavor: { driver: ACP_TEST_DRIVER, capabilities: AcpProviderCapabilitiesV2, - restartRuntimeAfterInterrupt: true, - // Local hard-flavor gate: production Grok no longer sets - // restartRuntimeOnEveryInterrupt, but when a flavor does, the - // settled-soft gate must not leak onto an unsettled prompt. - restartRuntimeOnEveryInterrupt: true, - terminateRuntimeProcessGroupOnInterrupt: true, - preserveRuntimeOnSettledInterrupt: true, - // No ownDetachedProcessGroup: the expected hard path fails loudly - // on the missing terminateProcessGroup, proving the settled-soft - // gate did not apply to an unsettled prompt. + deferFinalizeForBackgroundWork: true, + enablePostSettleContinuation: true, + extractSubagentUpdate: (toolCall) => + toolCall.toolCallId !== "tool-call-generic-1" + ? undefined + : subagentPhase === "spawn" + ? { + nativeTaskId: "task-generic-1", + prompt: "background subagent", + title: "background subagent", + model: null, + status: "running", + childSessionId, + result: null, + } + : { + nativeTaskId: "task-generic-1", + prompt: "", + title: null, + model: null, + status: "completed", + childSessionId, + result: "SUB_DONE", + }, makeRuntime: makeMockRuntime({ childProcessSpawner, mockAgentPath, - environment: { T3_ACP_HANG_PROMPT_FOREVER: "1" }, + environment: { T3_ACP_EMIT_GENERIC_TOOL_PLACEHOLDERS: "1" }, protocolEvents, + wrapRuntime: (runtime) => ({ + ...runtime, + handleSessionUpdate: (handler) => + Effect.sync(() => { + sessionUpdateHandler = handler; + }).pipe(Effect.andThen(runtime.handleSessionUpdate(handler))), + prompt: (payload) => + Effect.gen(function* () { + const currentPrompt = ++promptCount; + const result = yield* runtime.prompt(payload); + if (currentPrompt === 2) { + yield* Deferred.succeed(secondPromptWireReturned, undefined); + yield* Deferred.await(releaseSecondPromptCompletion); + } + return result; + }), + }), }), }, fileSystem, idAllocator, serverConfig, + continuationRequests: { offer: () => Effect.void }, }); - const threadId = ThreadId.make("thread-acp-unsettled-steer-stays-hard"); + const threadId = ThreadId.make("thread-acp-active-carryover-pending-pin"); const runtimePolicy = ProviderAdapterV2RuntimePolicy.make({ runtimeMode: "full-access", interactionMode: "default", @@ -3422,45 +3488,165 @@ describe("AcpAdapterV2", () => { const runtime = yield* adapter.openSession({ threadId, providerSessionId: ProviderSessionId.make( - "provider-session-acp-unsettled-steer-stays-hard", + "provider-session-acp-active-carryover-pending-pin", ), modelSelection, runtimePolicy, }); + if (runtime.hasPendingBackgroundWork === undefined) { + return yield* Effect.die( + "ACP runtime must expose hasPendingBackgroundWork when post-settle continuation is enabled.", + ); + } + const hasPendingBackgroundWork = runtime.hasPendingBackgroundWork; + const events = yield* Queue.unbounded(); + yield* runtime.events.pipe( + Stream.runForEach((event) => Queue.offer(events, event)), + Effect.forkScoped, + ); const providerThread = yield* runtime.ensureThread({ threadId, modelSelection, runtimePolicy, }); const now = yield* DateTime.now; - yield* runtime - .startTurn( - makeTurnInput({ threadId, providerThread, instanceId, runtimePolicy, now, ordinal: 1 }), - ) - .pipe(Effect.forkScoped); + yield* runtime.startTurn( + makeTurnInput({ threadId, providerThread, instanceId, runtimePolicy, now }), + ); yield* Stream.fromQueue(protocolEvents).pipe( Stream.filter( (event) => - event.direction === "outgoing" && rawProtocolMethod(event) === "session/prompt", + event.direction === "incoming" && + event.stage === "raw" && + typeof event.payload === "string" && + event.payload.includes('"stopReason"'), ), Stream.runHead, ); - const providerTurnId = idAllocator.derive.providerTurn({ + yield* Effect.yieldNow; + yield* Effect.yieldNow; + + const firstProviderTurnId = idAllocator.derive.providerTurn({ driver: ACP_TEST_DRIVER, nativeTurnId: "mock-session-1:turn:1", }); - const interruptExit = yield* runtime - .interruptTurn({ providerThread, providerTurnId }) - .pipe(Effect.exit); - if (Exit.isSuccess(interruptExit)) { - assert.fail("mid-prompt steering interrupt must still take the hard teardown path"); + const interruptFiber = yield* runtime + .interruptTurn({ providerThread, providerTurnId: firstProviderTurnId }) + .pipe(Effect.forkScoped); + yield* TestClock.adjust("10 seconds"); + yield* Fiber.join(interruptFiber); + + let firstTerminalStatus: string | null = null; + while (firstTerminalStatus === null) { + const event = yield* Queue.take(events); + if (event.type === "turn.terminal" && event.providerTurnId === firstProviderTurnId) { + firstTerminalStatus = event.status; + } } - assert.include(Cause.pretty(interruptExit.cause), "session is poisoned"); + assert.equal(firstTerminalStatus, "interrupted"); + assert.isTrue(yield* hasPendingBackgroundWork); + assert.isDefined(sessionUpdateHandler, "session update handler must be wired"); + + const secondNow = yield* DateTime.now; + const secondTurnFiber = yield* runtime + .startTurn( + makeTurnInput({ + threadId, + providerThread, + instanceId, + runtimePolicy, + now: secondNow, + ordinal: 2, + }), + ) + .pipe(Effect.forkScoped); + yield* Deferred.await(secondPromptWireReturned); + assert.isTrue( + yield* hasPendingBackgroundWork, + "rehydrated live subagent must pin from activeTurn", + ); + while (Option.isSome(yield* Queue.poll(events))) { + // Discard setup events so the assertions below cover only child-session + // tool traffic after rehydration. + } + + subagentPhase = "complete"; + yield* sessionUpdateHandler!({ + sessionId: childSessionId, + update: { + sessionUpdate: "tool_call_update", + toolCallId: "tool-call-generic-1", + title: "background subagent", + kind: "other", + status: "completed", + rawOutput: { content: "SUB_DONE" }, + }, + }); + // Duplicate terminal replay is harmless, and an older running frame cannot + // resurrect the completed lineage. + yield* sessionUpdateHandler!({ + sessionId: childSessionId, + update: { + sessionUpdate: "tool_call_update", + toolCallId: "tool-call-generic-1", + title: "background subagent", + kind: "other", + status: "completed", + rawOutput: { content: "SUB_DONE" }, + }, + }); + subagentPhase = "spawn"; + yield* sessionUpdateHandler!({ + sessionId: childSessionId, + update: { + sessionUpdate: "tool_call_update", + toolCallId: "tool-call-generic-1", + title: "background subagent", + kind: "other", + status: "in_progress", + rawOutput: {}, + }, + }); + yield* Effect.yieldNow; + yield* Effect.yieldNow; + assert.isFalse( + yield* hasPendingBackgroundWork, + "active-turn pin must clear after the child-session terminal is projected", + ); + let completedSubagentUpdates = 0; + let resurrectedSubagentUpdates = 0; + let normalChildToolUpdates = 0; + let polled = yield* Queue.poll(events); + while (Option.isSome(polled)) { + const event = polled.value; + if (event.type === "turn_item.updated") { + if (event.turnItem.type === "subagent") { + if (event.turnItem.status === "completed") { + completedSubagentUpdates += 1; + } + if ( + event.turnItem.status === "running" && + event.turnItem.nativeItemRef?.nativeId === "task-generic-1" + ) { + resurrectedSubagentUpdates += 1; + } + } else if (event.turnItem.nativeItemRef?.nativeId === "tool-call-generic-1") { + normalChildToolUpdates += 1; + } + } + polled = yield* Queue.poll(events); + } + assert.equal(completedSubagentUpdates, 1); + assert.equal(resurrectedSubagentUpdates, 0); + assert.equal(normalChildToolUpdates, 0); + + yield* Deferred.succeed(releaseSecondPromptCompletion, undefined); + yield* Fiber.join(secondTurnFiber); }).pipe(Effect.provide(testLayer), Effect.scoped), ); - it.live( - "soft mid-prompt interrupt cancels in place, reuses the runtime, and tracks cancel-backgrounded work", + it.effect( + "projects a child terminal in the completed-root finalization window exactly once", () => Effect.gen(function* () { const childProcessSpawner = yield* ChildProcessSpawner.ChildProcessSpawner; @@ -3472,40 +3658,2330 @@ describe("AcpAdapterV2", () => { new URL("../../../scripts/acp-mock-agent.ts", import.meta.url), ); const protocolEvents = yield* Queue.bounded(256); - const continuationRequests: Array = []; - const instanceId = ProviderInstanceId.make("acp-test"); - let cancelCalled = false; - let runtimeOrdinalSeen = 0; - const adapter = makeAcpAdapterV2({ - crypto: yield* Crypto.Crypto, - instanceId, - flavor: { - driver: ACP_TEST_DRIVER, - capabilities: AcpProviderCapabilitiesV2, - enablePostSettleContinuation: true, - // Production Grok interrupt flags: hard teardown only with - // requestRuntimeRestart (user Stop). Without - // restartRuntimeOnEveryInterrupt a mid-prompt steering interrupt - // stays soft: session/cancel, same process, session reuse. - restartRuntimeAfterInterrupt: true, - terminateRuntimeProcessGroupOnInterrupt: true, - preserveRuntimeOnSettledInterrupt: true, - registerExtensions: ({ runtime: extensionRuntime, applyBackgroundTaskMutation }) => - registerXAiBackgroundTaskTracking(extensionRuntime, applyBackgroundTaskMutation), - // No ownDetachedProcessGroup: if the interrupt wrongly takes the - // hard path, terminateProcessGroup is missing and the interrupt - // fails loudly with a poisoned session. - makeRuntime: makeMockRuntime({ - childProcessSpawner, - mockAgentPath, - environment: (runtimeOrdinal) => { - runtimeOrdinalSeen = Math.max(runtimeOrdinalSeen, runtimeOrdinal); - return { - T3_ACP_EMIT_RUNNING_COMMAND_THEN_HANG_FIRST_PROMPT: "1", - T3_ACP_EMIT_TASK_BACKGROUNDED_AFTER_CANCEL: "1", - }; - }, - protocolEvents, + const baseClock = yield* Clock.Clock; + const finalizationClockRead = yield* Deferred.make(); + const releaseFinalizationClockRead = yield* Deferred.make(); + let blockNextClockRead = false; + const blockingClock: Clock.Clock = { + ...baseClock, + currentTimeMillis: Effect.suspend(() => { + if (!blockNextClockRead) return baseClock.currentTimeMillis; + blockNextClockRead = false; + return Deferred.succeed(finalizationClockRead, undefined).pipe( + Effect.andThen(Deferred.await(releaseFinalizationClockRead)), + Effect.andThen(baseClock.currentTimeMillis), + ); + }), + }; + yield* Effect.gen(function* () { + const instanceId = ProviderInstanceId.make("acp-test"); + const firstChildSessionId = "mock-child-session-finalize-window-first"; + const secondChildSessionId = "mock-child-session-finalize-window-second"; + let firstSubagentStatus: "running" | "completed" = "running"; + // Keep the pending-work pin without blocking the deferred-finalize + // timer, so the test can stop inside finalizeTurn deterministically. + let secondSubagentStatus: "waiting" | "completed" = "waiting"; + type RuntimeService = AcpSessionRuntime.AcpSessionRuntime["Service"]; + let sessionUpdateHandler: + | Parameters[0] + | undefined; + const adapter = makeAcpAdapterV2({ + crypto: yield* Crypto.Crypto, + instanceId, + flavor: { + driver: ACP_TEST_DRIVER, + capabilities: AcpProviderCapabilitiesV2, + deferFinalizeForBackgroundWork: true, + enablePostSettleContinuation: true, + settleRootTurnWhenIdle: true, + extractSubagentUpdate: (toolCall) => { + if (toolCall.toolCallId === "tool-call-generic-1") { + return { + nativeTaskId: "task-finalize-window-first", + prompt: "first background subagent", + title: "first background subagent", + model: null, + status: firstSubagentStatus, + childSessionId: firstChildSessionId, + result: firstSubagentStatus === "completed" ? "FIRST_DONE" : null, + suppressNormalTool: true, + }; + } + if (toolCall.toolCallId === "tool-call-generic-2") { + return { + nativeTaskId: "task-finalize-window-second", + prompt: "second background subagent", + title: "second background subagent", + model: null, + status: secondSubagentStatus, + childSessionId: secondChildSessionId, + result: secondSubagentStatus === "completed" ? "SECOND_DONE" : null, + suppressNormalTool: true, + } as AcpAdapterV2SubagentUpdate; + } + return undefined; + }, + makeRuntime: makeMockRuntime({ + childProcessSpawner, + mockAgentPath, + environment: { T3_ACP_EMIT_GENERIC_TOOL_PLACEHOLDERS: "1" }, + protocolEvents, + wrapRuntime: (runtime) => ({ + ...runtime, + handleSessionUpdate: (handler) => + Effect.sync(() => { + sessionUpdateHandler = handler; + }).pipe(Effect.andThen(runtime.handleSessionUpdate(handler))), + }), + }), + }, + fileSystem, + idAllocator, + serverConfig, + continuationRequests: { offer: () => Effect.void }, + }); + const threadId = ThreadId.make("thread-acp-subagent-finalize-window"); + const runtimePolicy = ProviderAdapterV2RuntimePolicy.make({ + runtimeMode: "full-access", + interactionMode: "default", + cwd: process.cwd(), + }); + const modelSelection = { instanceId, model: "default" } as const; + const runtime = yield* adapter.openSession({ + threadId, + providerSessionId: ProviderSessionId.make( + "provider-session-acp-subagent-finalize-window", + ), + modelSelection, + runtimePolicy, + }); + if (runtime.hasPendingBackgroundWork === undefined) { + return yield* Effect.die( + "ACP runtime must expose hasPendingBackgroundWork when post-settle continuation is enabled.", + ); + } + const hasPendingBackgroundWork = runtime.hasPendingBackgroundWork; + const events = yield* Queue.unbounded(); + yield* runtime.events.pipe( + Stream.runForEach((event) => Queue.offer(events, event)), + Effect.forkScoped, + ); + const providerThread = yield* runtime.ensureThread({ + threadId, + modelSelection, + runtimePolicy, + }); + const now = yield* DateTime.now; + yield* runtime.startTurn( + makeTurnInput({ threadId, providerThread, instanceId, runtimePolicy, now }), + ); + yield* Stream.fromQueue(protocolEvents).pipe( + Stream.filter( + (event) => + event.direction === "incoming" && + event.stage === "raw" && + typeof event.payload === "string" && + event.payload.includes('"stopReason"'), + ), + Stream.runHead, + ); + yield* Effect.yieldNow; + yield* Effect.yieldNow; + yield* TestClock.adjust("1 second"); + assert.isDefined(sessionUpdateHandler, "session update handler must be wired"); + + firstSubagentStatus = "completed"; + yield* sessionUpdateHandler!({ + sessionId: "mock-session-1", + update: { + sessionUpdate: "tool_call_update", + toolCallId: "tool-call-generic-1", + title: "first background subagent", + kind: "other", + status: "completed", + rawOutput: { content: "FIRST_DONE" }, + }, + }).pipe(Effect.provideService(Clock.Clock, blockingClock)); + for (let attempt = 0; attempt < 10; attempt += 1) { + yield* Effect.yieldNow; + } + + yield* sessionUpdateHandler!({ + sessionId: "mock-session-1", + update: { + sessionUpdate: "tool_call_update", + toolCallId: "tool-call-generic-2", + title: "second background subagent", + kind: "other", + status: "in_progress", + rawOutput: {}, + }, + }); + assert.isTrue(yield* hasPendingBackgroundWork); + + while (Option.isSome(yield* Queue.poll(events))) { + // Discard setup and first-subagent events. + } + + blockNextClockRead = true; + const adjustFiber = yield* TestClock.adjust("3 seconds").pipe(Effect.forkScoped); + yield* Deferred.await(finalizationClockRead); + secondSubagentStatus = "completed"; + yield* sessionUpdateHandler!({ + sessionId: secondChildSessionId, + update: { + sessionUpdate: "tool_call_update", + toolCallId: "tool-call-generic-2", + title: "second background subagent", + kind: "other", + status: "completed", + rawOutput: { content: "SECOND_DONE" }, + }, + }); + yield* Deferred.succeed(releaseFinalizationClockRead, undefined); + yield* Fiber.join(adjustFiber); + yield* Effect.yieldNow; + yield* Effect.yieldNow; + + const providerTurnId = idAllocator.derive.providerTurn({ + driver: ACP_TEST_DRIVER, + nativeTurnId: "mock-session-1:turn:1", + }); + let completedSubagentUpdates = 0; + let rootTerminalStatus: string | null = null; + let polled = yield* Queue.poll(events); + while (Option.isSome(polled)) { + const event = polled.value; + if ( + event.type === "turn_item.updated" && + event.turnItem.type === "subagent" && + event.turnItem.nativeItemRef?.nativeId === "task-finalize-window-second" && + event.turnItem.status === "completed" + ) { + completedSubagentUpdates += 1; + } + if (event.type === "turn.terminal" && event.providerTurnId === providerTurnId) { + rootTerminalStatus = event.status; + } + polled = yield* Queue.poll(events); + } + assert.equal(rootTerminalStatus, "completed"); + assert.equal(completedSubagentUpdates, 1); + assert.isFalse( + yield* hasPendingBackgroundWork, + "finalize-window terminal must clear the carryover pin", + ); + }).pipe(Effect.provideService(Clock.Clock, blockingClock)); + }).pipe(Effect.provide(testLayer), Effect.scoped), + ); + + it.effect( + "defers an interrupted pending-spawn carryover terminal until the next observable attach", + () => + Effect.gen(function* () { + const childProcessSpawner = yield* ChildProcessSpawner.ChildProcessSpawner; + const fileSystem = yield* FileSystem.FileSystem; + const idAllocator = yield* IdAllocatorV2; + const path = yield* Path.Path; + const serverConfig = yield* ServerConfig; + const mockAgentPath = yield* path.fromFileUrl( + new URL("../../../scripts/acp-mock-agent.ts", import.meta.url), + ); + const protocolEvents = yield* Queue.bounded(256); + const continuationRequests: Array = []; + const instanceId = ProviderInstanceId.make("acp-test"); + const childSessionId = "mock-child-session-post-settle"; + let subagentPhase: "spawn" | "complete" = "spawn"; + type RuntimeService = AcpSessionRuntime.AcpSessionRuntime["Service"]; + let sessionUpdateHandler: Parameters[0] | undefined; + const adapter = makeAcpAdapterV2({ + crypto: yield* Crypto.Crypto, + instanceId, + flavor: { + driver: ACP_TEST_DRIVER, + capabilities: AcpProviderCapabilitiesV2, + deferFinalizeForBackgroundWork: true, + enablePostSettleContinuation: true, + extractSubagentUpdate: (toolCall) => + toolCall.toolCallId !== "tool-call-generic-1" + ? undefined + : subagentPhase === "spawn" + ? { + nativeTaskId: "task-generic-1", + prompt: "background subagent", + title: "background subagent", + model: null, + status: "pending", + childSessionId, + result: null, + } + : { + nativeTaskId: "task-generic-1", + prompt: "", + title: null, + model: null, + status: "completed", + childSessionId, + result: "SUB_DONE", + }, + makeRuntime: makeMockRuntime({ + childProcessSpawner, + mockAgentPath, + environment: { T3_ACP_EMIT_GENERIC_TOOL_PLACEHOLDERS: "1" }, + protocolEvents, + wrapRuntime: (runtime) => ({ + ...runtime, + handleSessionUpdate: (handler) => + Effect.sync(() => { + sessionUpdateHandler = handler; + }).pipe(Effect.andThen(runtime.handleSessionUpdate(handler))), + }), + }), + }, + fileSystem, + idAllocator, + serverConfig, + continuationRequests: { + offer: (request) => + Effect.sync(() => { + continuationRequests.push(request); + }), + }, + }); + const threadId = ThreadId.make("thread-acp-carryover-child-session-post-settle"); + const runtimePolicy = ProviderAdapterV2RuntimePolicy.make({ + runtimeMode: "full-access", + interactionMode: "default", + cwd: process.cwd(), + }); + const modelSelection = { instanceId, model: "default" } as const; + const runtime = yield* adapter.openSession({ + threadId, + providerSessionId: ProviderSessionId.make( + "provider-session-acp-carryover-child-session-post-settle", + ), + modelSelection, + runtimePolicy, + }); + if (runtime.hasPendingBackgroundWork === undefined) { + return yield* Effect.die( + "ACP runtime must expose hasPendingBackgroundWork when post-settle continuation is enabled.", + ); + } + const hasPendingBackgroundWork = runtime.hasPendingBackgroundWork; + const events = yield* Queue.unbounded(); + yield* runtime.events.pipe( + Stream.runForEach((event) => Queue.offer(events, event)), + Effect.forkScoped, + ); + const providerThread = yield* runtime.ensureThread({ + threadId, + modelSelection, + runtimePolicy, + }); + const now = yield* DateTime.now; + yield* runtime.startTurn( + makeTurnInput({ threadId, providerThread, instanceId, runtimePolicy, now }), + ); + yield* Stream.fromQueue(protocolEvents).pipe( + Stream.filter( + (event) => + event.direction === "incoming" && + event.stage === "raw" && + typeof event.payload === "string" && + event.payload.includes('"stopReason"'), + ), + Stream.runHead, + ); + yield* Effect.yieldNow; + yield* Effect.yieldNow; + + const firstProviderTurnId = idAllocator.derive.providerTurn({ + driver: ACP_TEST_DRIVER, + nativeTurnId: "mock-session-1:turn:1", + }); + const interruptFiber = yield* runtime + .interruptTurn({ providerThread, providerTurnId: firstProviderTurnId }) + .pipe(Effect.forkScoped); + yield* TestClock.adjust("10 seconds"); + yield* Fiber.join(interruptFiber); + + let firstTerminalStatus: string | null = null; + while (firstTerminalStatus === null) { + const event = yield* Queue.take(events); + if (event.type === "turn.terminal" && event.providerTurnId === firstProviderTurnId) { + firstTerminalStatus = event.status; + } + } + assert.equal(firstTerminalStatus, "interrupted"); + assert.isTrue( + yield* hasPendingBackgroundWork, + "carryover live subagent must pin hasPendingBackgroundWork after root settle", + ); + assert.isDefined(sessionUpdateHandler, "session update handler must be wired"); + + // The interrupted root's subscription is already closed in execution + // service, so the adapter must retain this terminal without claiming it + // reached the projection. + subagentPhase = "complete"; + yield* sessionUpdateHandler!({ + sessionId: childSessionId, + update: { + sessionUpdate: "tool_call_update", + toolCallId: "tool-call-generic-1", + title: "background subagent", + kind: "other", + status: "completed", + rawOutput: { content: "SUB_DONE" }, + }, + }); + yield* Effect.yieldNow; + yield* Effect.yieldNow; + let completedBeforeAttach = 0; + let polled = yield* Queue.poll(events); + while (Option.isSome(polled)) { + const event = polled.value; + if ( + event.type === "turn_item.updated" && + event.turnItem.type === "subagent" && + event.turnItem.status === "completed" + ) { + completedBeforeAttach += 1; + } + polled = yield* Queue.poll(events); + } + assert.equal( + completedBeforeAttach, + 0, + "closed interrupted subscription must not be treated as projected", + ); + assert.isTrue( + yield* hasPendingBackgroundWork, + "terminal carryover must stay pinned until an attach can project it", + ); + assert.lengthOf( + continuationRequests, + 0, + "child-session completion must not open a root continuation", + ); + + const attachNow = yield* DateTime.now; + yield* runtime.startTurn( + makeTurnInput({ + threadId, + providerThread, + instanceId, + runtimePolicy, + now: attachNow, + ordinal: 2, + messageCreatedBy: "agent", + messageCreationSource: "provider", + messageText: "Attach deferred completion.", + }), + ); + const attachProviderTurnId = idAllocator.derive.providerTurn({ + driver: ACP_TEST_DRIVER, + nativeTurnId: "mock-session-1:turn:2", + }); + let attachTerminal: string | null = null; + let completedAfterAttach = 0; + while (attachTerminal === null) { + const event = yield* Queue.take(events); + if ( + event.type === "turn_item.updated" && + event.turnItem.type === "subagent" && + event.turnItem.status === "completed" + ) { + completedAfterAttach += 1; + } + if (event.type === "turn.terminal" && event.providerTurnId === attachProviderTurnId) { + attachTerminal = event.status; + } + } + assert.equal(attachTerminal, "completed"); + assert.equal(completedAfterAttach, 1); + assert.isFalse(yield* hasPendingBackgroundWork); + }).pipe(Effect.provide(testLayer), Effect.scoped), + ); + + it.effect("projects completed-root carryover eagerly and drain cannot resurrect it", () => + Effect.gen(function* () { + const childProcessSpawner = yield* ChildProcessSpawner.ChildProcessSpawner; + const fileSystem = yield* FileSystem.FileSystem; + const idAllocator = yield* IdAllocatorV2; + const path = yield* Path.Path; + const serverConfig = yield* ServerConfig; + const mockAgentPath = yield* path.fromFileUrl( + new URL("../../../scripts/acp-mock-agent.ts", import.meta.url), + ); + const protocolEvents = yield* Queue.bounded(256); + const continuationRequests: Array = []; + const instanceId = ProviderInstanceId.make("acp-test"); + const childSessionId = "019f44a6-4820-7402-925d-bc862ee711dd"; + let subagentPhase: "spawn" | "complete" = "spawn"; + type RuntimeService = AcpSessionRuntime.AcpSessionRuntime["Service"]; + let sessionUpdateHandler: Parameters[0] | undefined; + const adapter = makeAcpAdapterV2({ + crypto: yield* Crypto.Crypto, + instanceId, + flavor: { + driver: ACP_TEST_DRIVER, + capabilities: AcpProviderCapabilitiesV2, + enablePostSettleContinuation: true, + normalizeToolCall: normalizeXAiAcpToolCallState, + extractSubagentUpdate: (toolCall) => + extractXAiAcpSubagentUpdate(toolCall) ?? + (toolCall.toolCallId !== "tool-call-generic-1" + ? undefined + : subagentPhase === "spawn" + ? { + nativeTaskId: "task-generic-1", + prompt: "background subagent", + title: "background subagent", + model: null, + status: "running", + childSessionId, + result: null, + } + : { + nativeTaskId: "task-generic-1", + prompt: "", + title: null, + model: null, + status: "completed", + childSessionId, + result: "SUB_DONE", + }), + makeRuntime: makeMockRuntime({ + childProcessSpawner, + mockAgentPath, + environment: { T3_ACP_EMIT_GENERIC_TOOL_PLACEHOLDERS: "1" }, + protocolEvents, + wrapRuntime: (runtime) => ({ + ...runtime, + handleSessionUpdate: (handler) => + Effect.sync(() => { + sessionUpdateHandler = handler; + }).pipe(Effect.andThen(runtime.handleSessionUpdate(handler))), + }), + }), + }, + fileSystem, + idAllocator, + serverConfig, + continuationRequests: { + offer: (request) => + Effect.sync(() => { + continuationRequests.push(request); + }), + }, + }); + const threadId = ThreadId.make("thread-acp-completed-root-eager-carryover"); + const runtimePolicy = ProviderAdapterV2RuntimePolicy.make({ + runtimeMode: "full-access", + interactionMode: "default", + cwd: process.cwd(), + }); + const modelSelection = { instanceId, model: "default" } as const; + const runtime = yield* adapter.openSession({ + threadId, + providerSessionId: ProviderSessionId.make( + "provider-session-acp-completed-root-eager-carryover", + ), + modelSelection, + runtimePolicy, + }); + if (runtime.hasPendingBackgroundWork === undefined) { + return yield* Effect.die( + "ACP runtime must expose hasPendingBackgroundWork when post-settle continuation is enabled.", + ); + } + const hasPendingBackgroundWork = runtime.hasPendingBackgroundWork; + const events = yield* Queue.unbounded(); + yield* runtime.events.pipe( + Stream.runForEach((event) => Queue.offer(events, event)), + Effect.forkScoped, + ); + const providerThread = yield* runtime.ensureThread({ + threadId, + modelSelection, + runtimePolicy, + }); + const now = yield* DateTime.now; + yield* runtime.startTurn( + makeTurnInput({ threadId, providerThread, instanceId, runtimePolicy, now }), + ); + yield* Stream.fromQueue(protocolEvents).pipe( + Stream.filter( + (event) => + event.direction === "incoming" && + event.stage === "raw" && + typeof event.payload === "string" && + event.payload.includes('"stopReason"'), + ), + Stream.runHead, + ); + yield* Effect.yieldNow; + yield* Effect.yieldNow; + yield* TestClock.adjust("1 second"); + + const firstProviderTurnId = idAllocator.derive.providerTurn({ + driver: ACP_TEST_DRIVER, + nativeTurnId: "mock-session-1:turn:1", + }); + let firstTerminalStatus: string | null = null; + while (firstTerminalStatus === null) { + const event = yield* Queue.take(events); + if (event.type === "turn.terminal" && event.providerTurnId === firstProviderTurnId) { + firstTerminalStatus = event.status; + } + } + assert.equal(firstTerminalStatus, "completed"); + assert.isTrue(yield* hasPendingBackgroundWork); + assert.isDefined(sessionUpdateHandler, "session update handler must be wired"); + + // A production Grok spawn ACK has raw tool status completed but extracts + // as a running subagent with its original non-empty prompt. It buffers and + // offers a continuation after root settlement. + yield* sessionUpdateHandler!({ + sessionId: "mock-session-1", + update: { + sessionUpdate: "tool_call", + toolCallId: "tool-call-generic-1", + title: "spawn_subagent", + kind: "other", + status: "completed", + rawInput: { + description: "background subagent", + prompt: "background subagent", + subagent_type: "general-purpose", + }, + rawOutput: { + type: "Text", + text: [ + "Subagent started in background.", + `subagent_id: ${childSessionId}`, + "type: general-purpose", + "description: background subagent", + "", + `Use get_command_or_subagent_output with task_ids=["${childSessionId}"] and timeout_ms to wait for results.`, + ].join("\n"), + }, + }, + }); + + // The child-session terminal bypasses the root wake buffer and projects + // eagerly through the still-observable completed-root subscriber. + subagentPhase = "complete"; + yield* sessionUpdateHandler!({ + sessionId: childSessionId, + update: { + sessionUpdate: "tool_call_update" as const, + toolCallId: "tool-call-generic-1", + title: "background subagent", + kind: "other" as const, + status: "completed" as const, + rawOutput: { content: "SUB_DONE" }, + }, + }); + yield* Effect.yieldNow; + yield* Effect.yieldNow; + + let eagerCompletedUpdates = 0; + let eagerRunningUpdates = 0; + let polled = yield* Queue.poll(events); + while (Option.isSome(polled)) { + const event = polled.value; + if (event.type === "turn_item.updated" && event.turnItem.type === "subagent") { + if (event.turnItem.status === "completed") eagerCompletedUpdates += 1; + if (event.turnItem.status === "running") eagerRunningUpdates += 1; + } + polled = yield* Queue.poll(events); + } + assert.equal(eagerCompletedUpdates, 1, "completed root must project before any attach"); + assert.equal(eagerRunningUpdates, 0); + assert.lengthOf(continuationRequests, 1); + assert.isTrue(yield* hasPendingBackgroundWork, "buffered spawn ACK still requires a drain"); + + const continuationNow = yield* DateTime.now; + const continuationInput = makeTurnInput({ + threadId, + providerThread, + instanceId, + runtimePolicy, + now: continuationNow, + ordinal: 2, + messageCreatedBy: "agent", + messageCreationSource: "provider", + messageText: "Background task completed.", + }); + yield* runtime.startTurn(continuationInput); + yield* Effect.yieldNow; + let replayedLineages = 0; + let replayedSubagentUpdates = 0; + let replayedTurnItems = 0; + polled = yield* Queue.poll(events); + while (Option.isSome(polled)) { + const event = polled.value; + if (event.type === "app_thread.created") { + replayedLineages += 1; + } + if (event.type === "subagent.updated" && event.subagent.runId === continuationInput.runId) { + replayedSubagentUpdates += 1; + } + if ( + event.type === "turn_item.updated" && + event.turnItem.runId === continuationInput.runId + ) { + replayedTurnItems += 1; + } + polled = yield* Queue.poll(events); + } + assert.equal(replayedLineages, 0, "buffered spawn ACK must not create a second lineage"); + assert.equal( + replayedSubagentUpdates, + 0, + "buffered spawn ACK must not re-open the terminal subagent", + ); + assert.equal( + replayedTurnItems, + 0, + "buffered spawn ACK must not create a continuation-owned turn item", + ); + assert.isFalse(yield* hasPendingBackgroundWork); + }).pipe(Effect.provide(testLayer), Effect.scoped), + ); + + it.effect( + "user attach flushes a deferred terminal but leaves wake traffic for its continuation", + () => + Effect.gen(function* () { + const childProcessSpawner = yield* ChildProcessSpawner.ChildProcessSpawner; + const fileSystem = yield* FileSystem.FileSystem; + const idAllocator = yield* IdAllocatorV2; + const path = yield* Path.Path; + const serverConfig = yield* ServerConfig; + const mockAgentPath = yield* path.fromFileUrl( + new URL("../../../scripts/acp-mock-agent.ts", import.meta.url), + ); + const protocolEvents = yield* Queue.bounded(256); + const continuationRequests: Array = []; + const bufferedAssistantText = "BUFFERED_WAKE_AFTER_USER_ATTACH"; + const instanceId = ProviderInstanceId.make("acp-test"); + let subagentPhase: "spawn" | "complete" = "spawn"; + type RuntimeService = AcpSessionRuntime.AcpSessionRuntime["Service"]; + let sessionUpdateHandler: Parameters[0] | undefined; + const adapter = makeAcpAdapterV2({ + crypto: yield* Crypto.Crypto, + instanceId, + flavor: { + driver: ACP_TEST_DRIVER, + capabilities: AcpProviderCapabilitiesV2, + deferFinalizeForBackgroundWork: true, + enablePostSettleContinuation: true, + extractSubagentUpdate: (toolCall) => + toolCall.toolCallId !== "tool-call-generic-1" + ? undefined + : subagentPhase === "spawn" + ? { + nativeTaskId: "task-generic-1", + prompt: "background subagent", + title: "background subagent", + model: null, + status: "running", + childSessionId: null, + result: null, + } + : { + nativeTaskId: "task-generic-1", + prompt: "", + title: null, + model: null, + status: "completed", + childSessionId: null, + result: "SUB_DONE", + }, + makeRuntime: makeMockRuntime({ + childProcessSpawner, + mockAgentPath, + environment: { T3_ACP_EMIT_GENERIC_TOOL_PLACEHOLDERS: "1" }, + protocolEvents, + wrapRuntime: (runtime) => ({ + ...runtime, + handleSessionUpdate: (handler) => + Effect.sync(() => { + sessionUpdateHandler = handler; + }).pipe(Effect.andThen(runtime.handleSessionUpdate(handler))), + }), + }), + }, + fileSystem, + idAllocator, + serverConfig, + continuationRequests: { + offer: (request) => + Effect.sync(() => { + continuationRequests.push(request); + }), + }, + }); + const threadId = ThreadId.make("thread-acp-carryover-root-session-continuation"); + const runtimePolicy = ProviderAdapterV2RuntimePolicy.make({ + runtimeMode: "full-access", + interactionMode: "default", + cwd: process.cwd(), + }); + const modelSelection = { instanceId, model: "default" } as const; + const runtime = yield* adapter.openSession({ + threadId, + providerSessionId: ProviderSessionId.make( + "provider-session-acp-carryover-root-session-continuation", + ), + modelSelection, + runtimePolicy, + }); + if (runtime.hasPendingBackgroundWork === undefined) { + return yield* Effect.die( + "ACP runtime must expose hasPendingBackgroundWork when post-settle continuation is enabled.", + ); + } + const hasPendingBackgroundWork = runtime.hasPendingBackgroundWork; + const events = yield* Queue.unbounded(); + yield* runtime.events.pipe( + Stream.runForEach((event) => Queue.offer(events, event)), + Effect.forkScoped, + ); + const providerThread = yield* runtime.ensureThread({ + threadId, + modelSelection, + runtimePolicy, + }); + const now = yield* DateTime.now; + yield* runtime.startTurn( + makeTurnInput({ threadId, providerThread, instanceId, runtimePolicy, now }), + ); + yield* Stream.fromQueue(protocolEvents).pipe( + Stream.filter( + (event) => + event.direction === "incoming" && + event.stage === "raw" && + typeof event.payload === "string" && + event.payload.includes('"stopReason"'), + ), + Stream.runHead, + ); + yield* Effect.yieldNow; + yield* Effect.yieldNow; + + const firstProviderTurnId = idAllocator.derive.providerTurn({ + driver: ACP_TEST_DRIVER, + nativeTurnId: "mock-session-1:turn:1", + }); + const interruptFiber = yield* runtime + .interruptTurn({ providerThread, providerTurnId: firstProviderTurnId }) + .pipe(Effect.forkScoped); + yield* TestClock.adjust("10 seconds"); + yield* Fiber.join(interruptFiber); + + let firstTerminalStatus: string | null = null; + while (firstTerminalStatus === null) { + const event = yield* Queue.take(events); + if (event.type === "turn.terminal" && event.providerTurnId === firstProviderTurnId) { + firstTerminalStatus = event.status; + } + } + assert.equal(firstTerminalStatus, "interrupted"); + assert.isTrue( + yield* hasPendingBackgroundWork, + "carryover live subagent must pin hasPendingBackgroundWork after root settle", + ); + assert.isDefined(sessionUpdateHandler, "session update handler must be wired"); + + // Root-session terminal tool is wake evidence and will buffer: in-memory + // only so the continuation drain projects exactly once. + subagentPhase = "complete"; + yield* sessionUpdateHandler!({ + sessionId: "mock-session-1", + update: { + sessionUpdate: "tool_call_update", + toolCallId: "tool-call-generic-1", + title: "background subagent", + kind: "other", + status: "completed", + rawOutput: { content: "SUB_DONE" }, + }, + }); + yield* sessionUpdateHandler!({ + sessionId: "mock-session-1", + update: { + sessionUpdate: "agent_message_chunk", + content: { type: "text", text: bufferedAssistantText }, + }, + }); + yield* Effect.yieldNow; + yield* Effect.yieldNow; + + let completedBeforeDrain = 0; + let polled = yield* Queue.poll(events); + while (Option.isSome(polled)) { + const event = polled.value; + if ( + event.type === "turn_item.updated" && + event.turnItem.type === "subagent" && + event.turnItem.status === "completed" + ) { + completedBeforeDrain += 1; + } + polled = yield* Queue.poll(events); + } + assert.equal( + completedBeforeDrain, + 0, + "interrupted root-session terminal must wait for an observable attach", + ); + assert.lengthOf( + continuationRequests, + 1, + "root-session post-settle subagent terminal must offer a continuation", + ); + assert.isTrue( + yield* hasPendingBackgroundWork, + "interrupted unprojected terminal must pin hasPendingBackgroundWork until attach", + ); + + const userTurnNow = yield* DateTime.now; + yield* runtime.startTurn( + makeTurnInput({ + threadId, + providerThread, + instanceId, + runtimePolicy, + now: userTurnNow, + ordinal: 2, + messageText: "What finished while the prior turn was settling?", + }), + ); + yield* TestClock.adjust("3 seconds"); + yield* Effect.yieldNow; + yield* Effect.yieldNow; + + const userProviderTurnId = idAllocator.derive.providerTurn({ + driver: ACP_TEST_DRIVER, + nativeTurnId: "mock-session-1:turn:2", + }); + let userTurnTerminal: string | null = null; + let completedSubagentTurnItems = 0; + let bufferedTextSeenInUserTurn = false; + while (userTurnTerminal === null) { + const event = yield* Queue.take(events); + if ( + event.type === "turn_item.updated" && + event.turnItem.type === "subagent" && + event.turnItem.status === "completed" + ) { + completedSubagentTurnItems += 1; + } + if ( + event.type === "message.updated" && + event.message.role === "assistant" && + event.message.text.includes(bufferedAssistantText) + ) { + bufferedTextSeenInUserTurn = true; + } + if (event.type === "turn.terminal" && event.providerTurnId === userProviderTurnId) { + userTurnTerminal = event.status; + } + } + assert.equal(userTurnTerminal, "completed"); + assert.equal( + completedSubagentTurnItems, + 1, + "user attach must project the deferred terminal subagent turn item once", + ); + assert.isFalse(bufferedTextSeenInUserTurn, "user attach must not drain wake traffic"); + assert.isTrue( + yield* hasPendingBackgroundWork, + "wake traffic must remain pinned for the already-dispatched continuation", + ); + + const continuationNow = yield* DateTime.now; + yield* runtime.startTurn( + makeTurnInput({ + threadId, + providerThread, + instanceId, + runtimePolicy, + now: continuationNow, + ordinal: 3, + messageCreatedBy: "agent", + messageCreationSource: "provider", + messageText: "Background task completed.", + }), + ); + yield* TestClock.adjust("3 seconds"); + const continuationProviderTurnId = idAllocator.derive.providerTurn({ + driver: ACP_TEST_DRIVER, + nativeTurnId: "mock-session-1:turn:3", + }); + let continuationTerminal: string | null = null; + let bufferedTextSeenInContinuation = false; + while (continuationTerminal === null) { + const event = yield* Queue.take(events); + if ( + event.type === "message.updated" && + event.message.role === "assistant" && + event.message.text.includes(bufferedAssistantText) + ) { + bufferedTextSeenInContinuation = true; + } + if ( + event.type === "turn.terminal" && + event.providerTurnId === continuationProviderTurnId + ) { + continuationTerminal = event.status; + } + } + assert.equal(continuationTerminal, "completed"); + assert.isTrue( + bufferedTextSeenInContinuation, + "queued continuation must drain the wake content after the user run", + ); + assert.isFalse(yield* hasPendingBackgroundWork); + }).pipe(Effect.provide(testLayer), Effect.scoped), + ); + + it.effect( + "projects a carryover terminal when an in-turn-handled tool is re-reported post-settle", + () => + Effect.gen(function* () { + const childProcessSpawner = yield* ChildProcessSpawner.ChildProcessSpawner; + const fileSystem = yield* FileSystem.FileSystem; + const idAllocator = yield* IdAllocatorV2; + const path = yield* Path.Path; + const serverConfig = yield* ServerConfig; + const mockAgentPath = yield* path.fromFileUrl( + new URL("../../../scripts/acp-mock-agent.ts", import.meta.url), + ); + const continuationRequests: Array = []; + const protocolEvents = yield* Queue.bounded(256); + const promptWireReturned = yield* Deferred.make(); + const releasePromptCompletion = yield* Deferred.make(); + const instanceId = ProviderInstanceId.make("acp-test"); + let subagentPhase: "spawn" | "complete" = "spawn"; + type RuntimeService = AcpSessionRuntime.AcpSessionRuntime["Service"]; + let sessionUpdateHandler: Parameters[0] | undefined; + const adapter = makeAcpAdapterV2({ + crypto: yield* Crypto.Crypto, + instanceId, + flavor: { + driver: ACP_TEST_DRIVER, + capabilities: AcpProviderCapabilitiesV2, + deferFinalizeForBackgroundWork: true, + enablePostSettleContinuation: true, + extractBackgroundTaskId: (toolCall) => + toolCall.toolCallId === "tool-call-generic-1" ? "task-generic-1" : undefined, + extractSubagentUpdate: (toolCall) => + toolCall.toolCallId !== "tool-call-generic-1" + ? undefined + : subagentPhase === "spawn" + ? { + nativeTaskId: "task-generic-1", + prompt: "background subagent", + title: "background subagent", + model: null, + status: "running", + childSessionId: null, + result: null, + } + : { + nativeTaskId: "task-generic-1", + prompt: "", + title: null, + model: null, + status: "completed", + childSessionId: null, + result: "SUB_DONE", + }, + makeRuntime: makeMockRuntime({ + childProcessSpawner, + mockAgentPath, + environment: { T3_ACP_EMIT_GENERIC_TOOL_PLACEHOLDERS: "1" }, + protocolEvents, + wrapRuntime: (runtime) => ({ + ...runtime, + handleSessionUpdate: (handler) => + Effect.sync(() => { + sessionUpdateHandler = handler; + }).pipe(Effect.andThen(runtime.handleSessionUpdate(handler))), + prompt: (payload) => + Effect.gen(function* () { + const result = yield* runtime.prompt(payload); + yield* Deferred.succeed(promptWireReturned, undefined); + yield* Deferred.await(releasePromptCompletion); + return result; + }), + }), + }), + }, + fileSystem, + idAllocator, + serverConfig, + continuationRequests: { + offer: (request) => + Effect.sync(() => { + continuationRequests.push(request); + }), + }, + }); + const threadId = ThreadId.make("thread-acp-carryover-already-handled-re-report"); + const runtimePolicy = ProviderAdapterV2RuntimePolicy.make({ + runtimeMode: "full-access", + interactionMode: "default", + cwd: process.cwd(), + }); + const modelSelection = { instanceId, model: "default" } as const; + const runtime = yield* adapter.openSession({ + threadId, + providerSessionId: ProviderSessionId.make( + "provider-session-acp-carryover-already-handled-re-report", + ), + modelSelection, + runtimePolicy, + }); + if (runtime.hasPendingBackgroundWork === undefined) { + return yield* Effect.die( + "ACP runtime must expose hasPendingBackgroundWork when post-settle continuation is enabled.", + ); + } + const hasPendingBackgroundWork = runtime.hasPendingBackgroundWork; + const events = yield* Queue.unbounded(); + yield* runtime.events.pipe( + Stream.runForEach((event) => Queue.offer(events, event)), + Effect.forkScoped, + ); + const providerThread = yield* runtime.ensureThread({ + threadId, + modelSelection, + runtimePolicy, + }); + const now = yield* DateTime.now; + yield* runtime.startTurn( + makeTurnInput({ threadId, providerThread, instanceId, runtimePolicy, now }), + ); + yield* Deferred.await(promptWireReturned); + assert.isDefined(sessionUpdateHandler, "session update handler must be wired"); + + // The root turn consumes a terminal re-report while the native prompt is + // still open. The subagent flavor keeps its carried lineage running. + yield* sessionUpdateHandler!({ + sessionId: "mock-session-1", + update: { + sessionUpdate: "tool_call_update", + toolCallId: "tool-call-generic-1", + title: "background subagent", + kind: "other", + status: "completed", + rawOutput: { content: "IN_TURN_RESULT" }, + }, + }); + yield* Deferred.succeed(releasePromptCompletion, undefined); + yield* Effect.yieldNow; + yield* Effect.yieldNow; + + const firstProviderTurnId = idAllocator.derive.providerTurn({ + driver: ACP_TEST_DRIVER, + nativeTurnId: "mock-session-1:turn:1", + }); + const interruptFiber = yield* runtime + .interruptTurn({ providerThread, providerTurnId: firstProviderTurnId }) + .pipe(Effect.forkScoped); + yield* TestClock.adjust("10 seconds"); + yield* Fiber.join(interruptFiber); + + let firstTerminalStatus: string | null = null; + while (firstTerminalStatus === null) { + const event = yield* Queue.take(events); + if (event.type === "turn.terminal" && event.providerTurnId === firstProviderTurnId) { + firstTerminalStatus = event.status; + } + } + assert.equal(firstTerminalStatus, "interrupted"); + assert.isTrue( + yield* hasPendingBackgroundWork, + "live carryover must remain pending after the superseded root settles", + ); + + // The same tool is now a terminal carryover update. Since it was handled + // in-turn, bufferPostSettleWake drops it instead of offering a drain. + subagentPhase = "complete"; + yield* sessionUpdateHandler!({ + sessionId: "mock-session-1", + update: { + sessionUpdate: "tool_call_update", + toolCallId: "tool-call-generic-1", + title: "background subagent", + kind: "other", + status: "completed", + rawOutput: { content: "SUB_DONE" }, + }, + }); + yield* Effect.yieldNow; + yield* Effect.yieldNow; + + let completedSubagentTurnItems = 0; + let polled = yield* Queue.poll(events); + while (Option.isSome(polled)) { + const event = polled.value; + if ( + event.type === "turn_item.updated" && + event.turnItem.type === "subagent" && + event.turnItem.status === "completed" + ) { + completedSubagentTurnItems += 1; + } + polled = yield* Queue.poll(events); + } + assert.equal( + completedSubagentTurnItems, + 0, + "an interrupted root's non-buffered re-report must wait for an observable attach", + ); + assert.lengthOf( + continuationRequests, + 0, + "an in-turn-handled re-report must not offer a continuation", + ); + assert.isTrue( + yield* hasPendingBackgroundWork, + "the unprojected terminal carryover must remain pinned", + ); + + const attachNow = yield* DateTime.now; + yield* runtime.startTurn( + makeTurnInput({ + threadId, + providerThread, + instanceId, + runtimePolicy, + now: attachNow, + ordinal: 2, + messageCreatedBy: "agent", + messageCreationSource: "provider", + messageText: "Attach deferred completion.", + }), + ); + const attachProviderTurnId = idAllocator.derive.providerTurn({ + driver: ACP_TEST_DRIVER, + nativeTurnId: "mock-session-1:turn:2", + }); + let attachTerminal: string | null = null; + while (attachTerminal === null) { + const event = yield* Queue.take(events); + if ( + event.type === "turn_item.updated" && + event.turnItem.type === "subagent" && + event.turnItem.status === "completed" + ) { + completedSubagentTurnItems += 1; + } + if (event.type === "turn.terminal" && event.providerTurnId === attachProviderTurnId) { + attachTerminal = event.status; + } + } + assert.equal(completedSubagentTurnItems, 1); + assert.isFalse(yield* hasPendingBackgroundWork); + }).pipe(Effect.provide(testLayer), Effect.scoped), + ); + + it.effect("projects an interrupted root-session end notice at the next attach", () => + Effect.gen(function* () { + const childProcessSpawner = yield* ChildProcessSpawner.ChildProcessSpawner; + const fileSystem = yield* FileSystem.FileSystem; + const idAllocator = yield* IdAllocatorV2; + const path = yield* Path.Path; + const serverConfig = yield* ServerConfig; + const mockAgentPath = yield* path.fromFileUrl( + new URL("../../../scripts/acp-mock-agent.ts", import.meta.url), + ); + const protocolEvents = yield* Queue.bounded(256); + const continuationRequests: Array = []; + const instanceId = ProviderInstanceId.make("acp-test"); + const childSessionId = "019f5470-bf92-7a90-afb3-5a6cea5b34a3"; + let subagentPhase: "spawn" | "complete" = "spawn"; + type RuntimeService = AcpSessionRuntime.AcpSessionRuntime["Service"]; + let sessionUpdateHandler: Parameters[0] | undefined; + const adapter = makeAcpAdapterV2({ + crypto: yield* Crypto.Crypto, + instanceId, + flavor: { + driver: ACP_TEST_DRIVER, + capabilities: AcpProviderCapabilitiesV2, + deferFinalizeForBackgroundWork: true, + enablePostSettleContinuation: true, + extractSubagentEndNotice: (text) => { + // Prefer the production parser; fall back only if the harness text + // is too short for its UUID + outcome rules. + const parsed = extractXAiAcpSubagentEndNotice(text); + if (parsed !== undefined) return parsed; + if (!text.includes(childSessionId)) return undefined; + if (/completed successfully/i.test(text)) { + return { childSessionId, status: "completed" as const }; + } + return undefined; + }, + extractSubagentUpdate: (toolCall) => + toolCall.toolCallId !== "tool-call-generic-1" + ? undefined + : subagentPhase === "spawn" + ? { + nativeTaskId: "task-generic-1", + prompt: "background subagent", + title: "background subagent", + model: null, + status: "running", + childSessionId, + result: null, + } + : { + nativeTaskId: "task-generic-1", + prompt: "", + title: null, + model: null, + status: "completed", + childSessionId, + result: "SUB_DONE", + }, + makeRuntime: makeMockRuntime({ + childProcessSpawner, + mockAgentPath, + environment: { T3_ACP_EMIT_GENERIC_TOOL_PLACEHOLDERS: "1" }, + protocolEvents, + wrapRuntime: (runtime) => ({ + ...runtime, + handleSessionUpdate: (handler) => + Effect.sync(() => { + sessionUpdateHandler = handler; + }).pipe(Effect.andThen(runtime.handleSessionUpdate(handler))), + }), + }), + }, + fileSystem, + idAllocator, + serverConfig, + continuationRequests: { + offer: (request) => + Effect.sync(() => { + continuationRequests.push(request); + }), + }, + }); + const threadId = ThreadId.make("thread-acp-carryover-root-end-notice"); + const runtimePolicy = ProviderAdapterV2RuntimePolicy.make({ + runtimeMode: "full-access", + interactionMode: "default", + cwd: process.cwd(), + }); + const modelSelection = { instanceId, model: "default" } as const; + const runtime = yield* adapter.openSession({ + threadId, + providerSessionId: ProviderSessionId.make("provider-session-acp-carryover-root-end-notice"), + modelSelection, + runtimePolicy, + }); + if (runtime.hasPendingBackgroundWork === undefined) { + return yield* Effect.die( + "ACP runtime must expose hasPendingBackgroundWork when post-settle continuation is enabled.", + ); + } + const hasPendingBackgroundWork = runtime.hasPendingBackgroundWork; + const events = yield* Queue.unbounded(); + yield* runtime.events.pipe( + Stream.runForEach((event) => Queue.offer(events, event)), + Effect.forkScoped, + ); + const providerThread = yield* runtime.ensureThread({ + threadId, + modelSelection, + runtimePolicy, + }); + const now = yield* DateTime.now; + yield* runtime.startTurn( + makeTurnInput({ threadId, providerThread, instanceId, runtimePolicy, now }), + ); + yield* Stream.fromQueue(protocolEvents).pipe( + Stream.filter( + (event) => + event.direction === "incoming" && + event.stage === "raw" && + typeof event.payload === "string" && + event.payload.includes('"stopReason"'), + ), + Stream.runHead, + ); + yield* Effect.yieldNow; + yield* Effect.yieldNow; + + const firstProviderTurnId = idAllocator.derive.providerTurn({ + driver: ACP_TEST_DRIVER, + nativeTurnId: "mock-session-1:turn:1", + }); + const interruptFiber = yield* runtime + .interruptTurn({ providerThread, providerTurnId: firstProviderTurnId }) + .pipe(Effect.forkScoped); + yield* TestClock.adjust("10 seconds"); + yield* Fiber.join(interruptFiber); + + let firstTerminalStatus: string | null = null; + while (firstTerminalStatus === null) { + const event = yield* Queue.take(events); + if (event.type === "turn.terminal" && event.providerTurnId === firstProviderTurnId) { + firstTerminalStatus = event.status; + } + } + assert.equal(firstTerminalStatus, "interrupted"); + assert.isTrue( + yield* hasPendingBackgroundWork, + "carryover live subagent must pin hasPendingBackgroundWork after root settle", + ); + assert.isDefined(sessionUpdateHandler, "session update handler must be wired"); + + // End notices are root user_message_chunk text and never buffer. The + // interrupted root still cannot project them until the next subscription + // attaches. + yield* sessionUpdateHandler!({ + sessionId: "mock-session-1", + update: { + sessionUpdate: "user_message_chunk", + content: { + type: "text", + text: `Background subagent "${childSessionId}" (general-purpose: "background subagent") completed successfully.`, + }, + }, + }); + // Do not use Effect.timeout under TestClock; poll after yielding so a + // missed projection fails the assertion instead of hanging the suite. + yield* Effect.yieldNow; + yield* Effect.yieldNow; + + let completedSubagentTurnItems = 0; + let polled = yield* Queue.poll(events); + while (Option.isSome(polled)) { + const event = polled.value; + if ( + event.type === "turn_item.updated" && + event.turnItem.type === "subagent" && + event.turnItem.status === "completed" + ) { + completedSubagentTurnItems += 1; + } + polled = yield* Queue.poll(events); + } + assert.equal( + completedSubagentTurnItems, + 0, + "root-session end notice must remain memory-only after interrupted settle", + ); + assert.lengthOf( + continuationRequests, + 0, + "root-session end notice must not open a continuation", + ); + assert.isTrue( + yield* hasPendingBackgroundWork, + "hasPendingBackgroundWork must retain the unprojected end notice", + ); + + const attachNow = yield* DateTime.now; + yield* runtime.startTurn( + makeTurnInput({ + threadId, + providerThread, + instanceId, + runtimePolicy, + now: attachNow, + ordinal: 2, + messageCreatedBy: "agent", + messageCreationSource: "provider", + messageText: "Attach deferred completion.", + }), + ); + const attachProviderTurnId = idAllocator.derive.providerTurn({ + driver: ACP_TEST_DRIVER, + nativeTurnId: "mock-session-1:turn:2", + }); + let attachTerminal: string | null = null; + while (attachTerminal === null) { + const event = yield* Queue.take(events); + if ( + event.type === "turn_item.updated" && + event.turnItem.type === "subagent" && + event.turnItem.status === "completed" + ) { + completedSubagentTurnItems += 1; + } + if (event.type === "turn.terminal" && event.providerTurnId === attachProviderTurnId) { + attachTerminal = event.status; + } + } + assert.equal(completedSubagentTurnItems, 1); + assert.isFalse(yield* hasPendingBackgroundWork); + }).pipe(Effect.provide(testLayer), Effect.scoped), + ); + + it.effect( + "preserves wakeBuffer when a child-session completes while a continuation is pending", + () => + Effect.gen(function* () { + const childProcessSpawner = yield* ChildProcessSpawner.ChildProcessSpawner; + const fileSystem = yield* FileSystem.FileSystem; + const idAllocator = yield* IdAllocatorV2; + const path = yield* Path.Path; + const serverConfig = yield* ServerConfig; + const mockAgentPath = yield* path.fromFileUrl( + new URL("../../../scripts/acp-mock-agent.ts", import.meta.url), + ); + const protocolEvents = yield* Queue.bounded(256); + const continuationRequests: Array = []; + const instanceId = ProviderInstanceId.make("acp-test"); + const childSessionId = "mock-child-session-pending-continuation"; + const bufferedAssistantText = "POST_SETTLE_BUFFERED_ASSISTANT_TEXT"; + let subagentPhase: "spawn" | "complete" = "spawn"; + type RuntimeService = AcpSessionRuntime.AcpSessionRuntime["Service"]; + let sessionUpdateHandler: Parameters[0] | undefined; + const adapter = makeAcpAdapterV2({ + crypto: yield* Crypto.Crypto, + instanceId, + flavor: { + driver: ACP_TEST_DRIVER, + capabilities: AcpProviderCapabilitiesV2, + deferFinalizeForBackgroundWork: true, + enablePostSettleContinuation: true, + extractSubagentUpdate: (toolCall) => + toolCall.toolCallId !== "tool-call-generic-1" + ? undefined + : subagentPhase === "spawn" + ? { + nativeTaskId: "task-generic-1", + prompt: "background subagent", + title: "background subagent", + model: null, + status: "running", + childSessionId, + result: null, + } + : { + nativeTaskId: "task-generic-1", + prompt: "", + title: null, + model: null, + status: "completed", + childSessionId, + result: "SUB_DONE", + }, + makeRuntime: makeMockRuntime({ + childProcessSpawner, + mockAgentPath, + environment: { T3_ACP_EMIT_GENERIC_TOOL_PLACEHOLDERS: "1" }, + protocolEvents, + wrapRuntime: (runtime) => ({ + ...runtime, + handleSessionUpdate: (handler) => + Effect.sync(() => { + sessionUpdateHandler = handler; + }).pipe(Effect.andThen(runtime.handleSessionUpdate(handler))), + }), + }), + }, + fileSystem, + idAllocator, + serverConfig, + continuationRequests: { + offer: (request) => + Effect.sync(() => { + continuationRequests.push(request); + }), + }, + }); + const threadId = ThreadId.make("thread-acp-carryover-child-pending-continuation"); + const runtimePolicy = ProviderAdapterV2RuntimePolicy.make({ + runtimeMode: "full-access", + interactionMode: "default", + cwd: process.cwd(), + }); + const modelSelection = { instanceId, model: "default" } as const; + const runtime = yield* adapter.openSession({ + threadId, + providerSessionId: ProviderSessionId.make( + "provider-session-acp-carryover-child-pending-continuation", + ), + modelSelection, + runtimePolicy, + }); + if (runtime.hasPendingBackgroundWork === undefined) { + return yield* Effect.die( + "ACP runtime must expose hasPendingBackgroundWork when post-settle continuation is enabled.", + ); + } + const hasPendingBackgroundWork = runtime.hasPendingBackgroundWork; + const events = yield* Queue.unbounded(); + yield* runtime.events.pipe( + Stream.runForEach((event) => Queue.offer(events, event)), + Effect.forkScoped, + ); + const providerThread = yield* runtime.ensureThread({ + threadId, + modelSelection, + runtimePolicy, + }); + const now = yield* DateTime.now; + yield* runtime.startTurn( + makeTurnInput({ threadId, providerThread, instanceId, runtimePolicy, now }), + ); + yield* Stream.fromQueue(protocolEvents).pipe( + Stream.filter( + (event) => + event.direction === "incoming" && + event.stage === "raw" && + typeof event.payload === "string" && + event.payload.includes('"stopReason"'), + ), + Stream.runHead, + ); + yield* Effect.yieldNow; + yield* Effect.yieldNow; + + const firstProviderTurnId = idAllocator.derive.providerTurn({ + driver: ACP_TEST_DRIVER, + nativeTurnId: "mock-session-1:turn:1", + }); + const interruptFiber = yield* runtime + .interruptTurn({ providerThread, providerTurnId: firstProviderTurnId }) + .pipe(Effect.forkScoped); + yield* TestClock.adjust("10 seconds"); + yield* Fiber.join(interruptFiber); + + let firstTerminalStatus: string | null = null; + while (firstTerminalStatus === null) { + const event = yield* Queue.take(events); + if (event.type === "turn.terminal" && event.providerTurnId === firstProviderTurnId) { + firstTerminalStatus = event.status; + } + } + assert.equal(firstTerminalStatus, "interrupted"); + assert.isTrue( + yield* hasPendingBackgroundWork, + "carryover live subagent must pin hasPendingBackgroundWork after root settle", + ); + assert.isDefined(sessionUpdateHandler, "session update handler must be wired"); + + // Root-session terminal + distinctive assistant text: both enter wakeBuffer + // and the terminal offers a continuation that will drain them. + subagentPhase = "complete"; + yield* sessionUpdateHandler!({ + sessionId: "mock-session-1", + update: { + sessionUpdate: "tool_call_update", + toolCallId: "tool-call-generic-1", + title: "background subagent", + kind: "other", + status: "completed", + rawOutput: { content: "SUB_DONE" }, + }, + }); + yield* sessionUpdateHandler!({ + sessionId: "mock-session-1", + update: { + sessionUpdate: "agent_message_chunk", + content: { type: "text", text: bufferedAssistantText }, + }, + }); + assert.lengthOf( + continuationRequests, + 1, + "root-session terminal must offer a continuation before child completion", + ); + assert.isTrue( + yield* hasPendingBackgroundWork, + "pending continuation must keep hasPendingBackgroundWork pinned", + ); + + // Child path must not wipe wakeBuffer or clear the sticky continuation pin. + yield* sessionUpdateHandler!({ + sessionId: childSessionId, + update: { + sessionUpdate: "tool_call_update", + toolCallId: "tool-call-generic-1", + title: "background subagent", + kind: "other", + status: "completed", + rawOutput: { content: "SUB_DONE" }, + }, + }); + yield* Effect.yieldNow; + yield* Effect.yieldNow; + + let completedBeforeDrain = 0; + let polled = yield* Queue.poll(events); + while (Option.isSome(polled)) { + const event = polled.value; + if ( + event.type === "turn_item.updated" && + event.turnItem.type === "subagent" && + event.turnItem.status === "completed" + ) { + completedBeforeDrain += 1; + } + polled = yield* Queue.poll(events); + } + assert.equal( + completedBeforeDrain, + 0, + "interrupted child-session terminal must wait for the continuation attach", + ); + assert.isTrue( + yield* hasPendingBackgroundWork, + "child-session must not clear the pin while wakeBuffer still has delivery", + ); + + // Continuation attach drains the preserved buffer and projects it. + const continuationNow = yield* DateTime.now; + yield* runtime.startTurn( + makeTurnInput({ + threadId, + providerThread, + instanceId, + runtimePolicy, + now: continuationNow, + ordinal: 2, + messageCreatedBy: "agent", + messageCreationSource: "provider", + messageText: "Background task completed.", + }), + ); + yield* TestClock.adjust("3 seconds"); + yield* Effect.yieldNow; + yield* Effect.yieldNow; + + const continuationProviderTurnId = idAllocator.derive.providerTurn({ + driver: ACP_TEST_DRIVER, + nativeTurnId: "mock-session-1:turn:2", + }); + let continuationTerminal: string | null = null; + let completedSubagentTurnItems = 0; + let bufferedTextSeen = false; + while (continuationTerminal === null) { + const event = yield* Queue.take(events); + if ( + event.type === "turn_item.updated" && + event.turnItem.type === "subagent" && + event.turnItem.status === "completed" + ) { + completedSubagentTurnItems += 1; + } + if ( + event.type === "message.updated" && + event.message.role === "assistant" && + event.message.text.includes(bufferedAssistantText) + ) { + bufferedTextSeen = true; + } + if ( + event.type === "turn.terminal" && + event.providerTurnId === continuationProviderTurnId + ) { + continuationTerminal = event.status; + } + } + assert.equal(continuationTerminal, "completed"); + assert.isTrue( + bufferedTextSeen, + "continuation drain must project buffered post-settle assistant text (buffer not wiped)", + ); + assert.equal( + completedSubagentTurnItems, + 1, + "continuation drain must project the terminal subagent turn item once", + ); + assert.isFalse( + yield* hasPendingBackgroundWork, + "pin must clear after the continuation drains, not when the child session completed", + ); + }).pipe(Effect.provide(testLayer), Effect.scoped), + ); + + it.effect( + "projects once when root-session then child-session complete the same carryover subagent", + () => + Effect.gen(function* () { + const childProcessSpawner = yield* ChildProcessSpawner.ChildProcessSpawner; + const fileSystem = yield* FileSystem.FileSystem; + const idAllocator = yield* IdAllocatorV2; + const path = yield* Path.Path; + const serverConfig = yield* ServerConfig; + const mockAgentPath = yield* path.fromFileUrl( + new URL("../../../scripts/acp-mock-agent.ts", import.meta.url), + ); + const protocolEvents = yield* Queue.bounded(256); + const continuationRequests: Array = []; + const instanceId = ProviderInstanceId.make("acp-test"); + const childSessionId = "mock-child-session-root-then-child"; + let subagentPhase: "spawn" | "complete" = "spawn"; + type RuntimeService = AcpSessionRuntime.AcpSessionRuntime["Service"]; + let sessionUpdateHandler: Parameters[0] | undefined; + const adapter = makeAcpAdapterV2({ + crypto: yield* Crypto.Crypto, + instanceId, + flavor: { + driver: ACP_TEST_DRIVER, + capabilities: AcpProviderCapabilitiesV2, + deferFinalizeForBackgroundWork: true, + enablePostSettleContinuation: true, + extractSubagentUpdate: (toolCall) => + toolCall.toolCallId !== "tool-call-generic-1" + ? undefined + : subagentPhase === "spawn" + ? { + nativeTaskId: "task-generic-1", + prompt: "background subagent", + title: "background subagent", + model: null, + status: "running", + childSessionId, + result: null, + } + : { + nativeTaskId: "task-generic-1", + prompt: "", + title: null, + model: null, + status: "completed", + childSessionId, + result: "SUB_DONE", + }, + makeRuntime: makeMockRuntime({ + childProcessSpawner, + mockAgentPath, + environment: { T3_ACP_EMIT_GENERIC_TOOL_PLACEHOLDERS: "1" }, + protocolEvents, + wrapRuntime: (runtime) => ({ + ...runtime, + handleSessionUpdate: (handler) => + Effect.sync(() => { + sessionUpdateHandler = handler; + }).pipe(Effect.andThen(runtime.handleSessionUpdate(handler))), + }), + }), + }, + fileSystem, + idAllocator, + serverConfig, + continuationRequests: { + offer: (request) => + Effect.sync(() => { + continuationRequests.push(request); + }), + }, + }); + const threadId = ThreadId.make("thread-acp-carryover-root-then-child"); + const runtimePolicy = ProviderAdapterV2RuntimePolicy.make({ + runtimeMode: "full-access", + interactionMode: "default", + cwd: process.cwd(), + }); + const modelSelection = { instanceId, model: "default" } as const; + const runtime = yield* adapter.openSession({ + threadId, + providerSessionId: ProviderSessionId.make( + "provider-session-acp-carryover-root-then-child", + ), + modelSelection, + runtimePolicy, + }); + if (runtime.hasPendingBackgroundWork === undefined) { + return yield* Effect.die( + "ACP runtime must expose hasPendingBackgroundWork when post-settle continuation is enabled.", + ); + } + const hasPendingBackgroundWork = runtime.hasPendingBackgroundWork; + const events = yield* Queue.unbounded(); + yield* runtime.events.pipe( + Stream.runForEach((event) => Queue.offer(events, event)), + Effect.forkScoped, + ); + const providerThread = yield* runtime.ensureThread({ + threadId, + modelSelection, + runtimePolicy, + }); + const now = yield* DateTime.now; + yield* runtime.startTurn( + makeTurnInput({ threadId, providerThread, instanceId, runtimePolicy, now }), + ); + yield* Stream.fromQueue(protocolEvents).pipe( + Stream.filter( + (event) => + event.direction === "incoming" && + event.stage === "raw" && + typeof event.payload === "string" && + event.payload.includes('"stopReason"'), + ), + Stream.runHead, + ); + yield* Effect.yieldNow; + yield* Effect.yieldNow; + + const firstProviderTurnId = idAllocator.derive.providerTurn({ + driver: ACP_TEST_DRIVER, + nativeTurnId: "mock-session-1:turn:1", + }); + const interruptFiber = yield* runtime + .interruptTurn({ providerThread, providerTurnId: firstProviderTurnId }) + .pipe(Effect.forkScoped); + yield* TestClock.adjust("10 seconds"); + yield* Fiber.join(interruptFiber); + + let firstTerminalStatus: string | null = null; + while (firstTerminalStatus === null) { + const event = yield* Queue.take(events); + if (event.type === "turn.terminal" && event.providerTurnId === firstProviderTurnId) { + firstTerminalStatus = event.status; + } + } + assert.equal(firstTerminalStatus, "interrupted"); + assert.isTrue( + yield* hasPendingBackgroundWork, + "carryover live subagent must pin hasPendingBackgroundWork after root settle", + ); + assert.isDefined(sessionUpdateHandler, "session update handler must be wired"); + + // The interrupted root-session completion advances in-memory carryover + // without projecting and offers a continuation. Child-session replay + // must not project a second copy before that observable attach. + subagentPhase = "complete"; + yield* sessionUpdateHandler!({ + sessionId: "mock-session-1", + update: { + sessionUpdate: "tool_call_update", + toolCallId: "tool-call-generic-1", + title: "background subagent", + kind: "other", + status: "completed", + rawOutput: { content: "SUB_DONE" }, + }, + }); + assert.lengthOf( + continuationRequests, + 1, + "root-session terminal must still offer a continuation", + ); + + yield* sessionUpdateHandler!({ + sessionId: childSessionId, + update: { + sessionUpdate: "tool_call_update", + toolCallId: "tool-call-generic-1", + title: "background subagent", + kind: "other", + status: "completed", + rawOutput: { content: "SUB_DONE" }, + }, + }); + yield* Effect.yieldNow; + yield* Effect.yieldNow; + + let completedBeforeDrain = 0; + let polled = yield* Queue.poll(events); + while (Option.isSome(polled)) { + const event = polled.value; + if ( + event.type === "turn_item.updated" && + event.turnItem.type === "subagent" && + event.turnItem.status === "completed" + ) { + completedBeforeDrain += 1; + } + polled = yield* Queue.poll(events); + } + assert.equal( + completedBeforeDrain, + 0, + "interrupted root-then-child completion must defer to the continuation attach", + ); + + const continuationNow = yield* DateTime.now; + yield* runtime.startTurn( + makeTurnInput({ + threadId, + providerThread, + instanceId, + runtimePolicy, + now: continuationNow, + ordinal: 2, + messageCreatedBy: "agent", + messageCreationSource: "provider", + messageText: "Background task completed.", + }), + ); + yield* TestClock.adjust("3 seconds"); + yield* Effect.yieldNow; + yield* Effect.yieldNow; + + const continuationProviderTurnId = idAllocator.derive.providerTurn({ + driver: ACP_TEST_DRIVER, + nativeTurnId: "mock-session-1:turn:2", + }); + let continuationTerminal: string | null = null; + let completedSubagentTurnItems = 0; + while (continuationTerminal === null) { + const event = yield* Queue.take(events); + if ( + event.type === "turn_item.updated" && + event.turnItem.type === "subagent" && + event.turnItem.status === "completed" + ) { + completedSubagentTurnItems += 1; + } + if ( + event.type === "turn.terminal" && + event.providerTurnId === continuationProviderTurnId + ) { + continuationTerminal = event.status; + } + } + assert.equal(continuationTerminal, "completed"); + assert.equal( + completedSubagentTurnItems, + 1, + "root-then-child completion must project the terminal subagent turn item exactly once", + ); + assert.isFalse( + yield* hasPendingBackgroundWork, + "hasPendingBackgroundWork must end false after the continuation drains", + ); + }).pipe(Effect.provide(testLayer), Effect.scoped), + ); + + it.effect( + "preserveRuntimeOnSettledInterrupt keeps the process alive and carries subagents through a settled steering interrupt", + () => + Effect.gen(function* () { + const childProcessSpawner = yield* ChildProcessSpawner.ChildProcessSpawner; + const fileSystem = yield* FileSystem.FileSystem; + const idAllocator = yield* IdAllocatorV2; + const path = yield* Path.Path; + const serverConfig = yield* ServerConfig; + const mockAgentPath = yield* path.fromFileUrl( + new URL("../../../scripts/acp-mock-agent.ts", import.meta.url), + ); + const protocolEvents = yield* Queue.bounded(256); + const instanceId = ProviderInstanceId.make("acp-test"); + let subagentPhase: "spawn" | "complete" = "spawn"; + let cancelCalled = false; + let runtimeOrdinalSeen = 0; + const adapter = makeAcpAdapterV2({ + crypto: yield* Crypto.Crypto, + instanceId, + flavor: { + driver: ACP_TEST_DRIVER, + capabilities: AcpProviderCapabilitiesV2, + deferFinalizeForBackgroundWork: true, + // Hard interrupt flags (stricter than production Grok, which no + // longer sets restartRuntimeOnEveryInterrupt): every interrupt + // would hard-kill the process group without the settled-soft gate + // under test. + restartRuntimeAfterInterrupt: true, + restartRuntimeOnEveryInterrupt: true, + terminateRuntimeProcessGroupOnInterrupt: true, + preserveRuntimeOnSettledInterrupt: true, + extractSubagentUpdate: (toolCall) => + toolCall.toolCallId !== "tool-call-generic-1" + ? undefined + : subagentPhase === "spawn" + ? { + nativeTaskId: "task-generic-1", + prompt: "background subagent", + title: "background subagent", + model: null, + status: "running", + childSessionId: null, + result: null, + } + : { + nativeTaskId: "task-generic-1", + prompt: "", + title: null, + model: null, + status: "completed", + childSessionId: null, + result: "SUB_DONE", + }, + // No ownDetachedProcessGroup: if the interrupt wrongly takes the + // hard path, terminateProcessGroup is missing and the interrupt + // fails loudly with a poisoned session. + makeRuntime: makeMockRuntime({ + childProcessSpawner, + mockAgentPath, + environment: (runtimeOrdinal) => { + runtimeOrdinalSeen = Math.max(runtimeOrdinalSeen, runtimeOrdinal); + return { T3_ACP_EMIT_GENERIC_TOOL_PLACEHOLDERS: "1" }; + }, + protocolEvents, + wrapCancel: (cancel) => + Effect.sync(() => { + cancelCalled = true; + }).pipe(Effect.andThen(cancel)), + }), + }, + fileSystem, + idAllocator, + serverConfig, + }); + const threadId = ThreadId.make("thread-acp-settled-soft-steer"); + const runtimePolicy = ProviderAdapterV2RuntimePolicy.make({ + runtimeMode: "full-access", + interactionMode: "default", + cwd: process.cwd(), + }); + const modelSelection = { instanceId, model: "default" } as const; + const runtime = yield* adapter.openSession({ + threadId, + providerSessionId: ProviderSessionId.make("provider-session-acp-settled-soft-steer"), + modelSelection, + runtimePolicy, + }); + const events = yield* Queue.unbounded(); + yield* runtime.events.pipe( + Stream.runForEach((event) => Queue.offer(events, event)), + Effect.forkScoped, + ); + const providerThread = yield* runtime.ensureThread({ + threadId, + modelSelection, + runtimePolicy, + }); + const now = yield* DateTime.now; + yield* runtime.startTurn( + makeTurnInput({ threadId, providerThread, instanceId, runtimePolicy, now }), + ); + // The still-running subagent defers finalize after session/prompt returns, + // so the interrupt below hits a settled turn held open for background work. + yield* Stream.fromQueue(protocolEvents).pipe( + Stream.filter( + (event) => + event.direction === "incoming" && + event.stage === "raw" && + typeof event.payload === "string" && + event.payload.includes('"stopReason"'), + ), + Stream.runHead, + ); + yield* Effect.yieldNow; + yield* Effect.yieldNow; + + const firstProviderTurnId = idAllocator.derive.providerTurn({ + driver: ACP_TEST_DRIVER, + nativeTurnId: "mock-session-1:turn:1", + }); + const interruptFiber = yield* runtime + .interruptTurn({ providerThread, providerTurnId: firstProviderTurnId }) + .pipe(Effect.forkScoped); + yield* TestClock.adjust("10 seconds"); + yield* Fiber.join(interruptFiber); + assert.isFalse( + cancelCalled, + "settled soft steer must not send session/cancel (the real Grok CLI kills background subagents on cancel)", + ); + + let subagentTurnItemId: string | null = null; + let firstTerminalStatus: string | null = null; + while (firstTerminalStatus === null) { + const event = yield* Queue.take(events); + if (event.type === "turn_item.updated" && event.turnItem.type === "subagent") { + subagentTurnItemId = event.turnItem.id; + } + if (event.type === "turn.terminal" && event.providerTurnId === firstProviderTurnId) { + firstTerminalStatus = event.status; + } + } + assert.equal(firstTerminalStatus, "interrupted"); + assert.notEqual(subagentTurnItemId, null); + + subagentPhase = "complete"; + const secondNow = yield* DateTime.now; + yield* runtime.startTurn( + makeTurnInput({ + threadId, + providerThread, + instanceId, + runtimePolicy, + now: secondNow, + ordinal: 2, + }), + ); + // Same runtime process (mock-session-1): a respawn would start + // mock-session-2 and drop the carryover on the session mismatch. + const secondProviderTurnId = idAllocator.derive.providerTurn({ + driver: ACP_TEST_DRIVER, + nativeTurnId: "mock-session-1:turn:2", + }); + let carriedItemStatus: string | null = null; + let secondTerminalStatus: string | null = null; + while (secondTerminalStatus === null) { + const event = yield* Queue.take(events); + if ( + event.type === "turn_item.updated" && + event.turnItem.type === "subagent" && + event.turnItem.id === subagentTurnItemId + ) { + carriedItemStatus = event.turnItem.status; + } + if (event.type === "turn.terminal" && event.providerTurnId === secondProviderTurnId) { + secondTerminalStatus = event.status; + } + } + assert.equal(carriedItemStatus, "completed"); + assert.equal(secondTerminalStatus, "completed"); + assert.equal( + runtimeOrdinalSeen, + 1, + "settled soft steer must not respawn the ACP runtime process", + ); + }).pipe(Effect.provide(testLayer), Effect.scoped), + ); + + it.live("preserveRuntimeOnSettledInterrupt does not soften a mid-prompt steering interrupt", () => + Effect.gen(function* () { + const childProcessSpawner = yield* ChildProcessSpawner.ChildProcessSpawner; + const fileSystem = yield* FileSystem.FileSystem; + const idAllocator = yield* IdAllocatorV2; + const path = yield* Path.Path; + const serverConfig = yield* ServerConfig; + const mockAgentPath = yield* path.fromFileUrl( + new URL("../../../scripts/acp-mock-agent.ts", import.meta.url), + ); + const protocolEvents = yield* Queue.bounded(256); + const instanceId = ProviderInstanceId.make("acp-test"); + const adapter = makeAcpAdapterV2({ + crypto: yield* Crypto.Crypto, + instanceId, + flavor: { + driver: ACP_TEST_DRIVER, + capabilities: AcpProviderCapabilitiesV2, + restartRuntimeAfterInterrupt: true, + // Local hard-flavor gate: production Grok no longer sets + // restartRuntimeOnEveryInterrupt, but when a flavor does, the + // settled-soft gate must not leak onto an unsettled prompt. + restartRuntimeOnEveryInterrupt: true, + terminateRuntimeProcessGroupOnInterrupt: true, + preserveRuntimeOnSettledInterrupt: true, + // No ownDetachedProcessGroup: the expected hard path fails loudly + // on the missing terminateProcessGroup, proving the settled-soft + // gate did not apply to an unsettled prompt. + makeRuntime: makeMockRuntime({ + childProcessSpawner, + mockAgentPath, + environment: { T3_ACP_HANG_PROMPT_FOREVER: "1" }, + protocolEvents, + }), + }, + fileSystem, + idAllocator, + serverConfig, + }); + const threadId = ThreadId.make("thread-acp-unsettled-steer-stays-hard"); + const runtimePolicy = ProviderAdapterV2RuntimePolicy.make({ + runtimeMode: "full-access", + interactionMode: "default", + cwd: process.cwd(), + }); + const modelSelection = { instanceId, model: "default" } as const; + const runtime = yield* adapter.openSession({ + threadId, + providerSessionId: ProviderSessionId.make( + "provider-session-acp-unsettled-steer-stays-hard", + ), + modelSelection, + runtimePolicy, + }); + const providerThread = yield* runtime.ensureThread({ + threadId, + modelSelection, + runtimePolicy, + }); + const now = yield* DateTime.now; + yield* runtime + .startTurn( + makeTurnInput({ threadId, providerThread, instanceId, runtimePolicy, now, ordinal: 1 }), + ) + .pipe(Effect.forkScoped); + yield* Stream.fromQueue(protocolEvents).pipe( + Stream.filter( + (event) => + event.direction === "outgoing" && rawProtocolMethod(event) === "session/prompt", + ), + Stream.runHead, + ); + const providerTurnId = idAllocator.derive.providerTurn({ + driver: ACP_TEST_DRIVER, + nativeTurnId: "mock-session-1:turn:1", + }); + const interruptExit = yield* runtime + .interruptTurn({ providerThread, providerTurnId }) + .pipe(Effect.exit); + if (Exit.isSuccess(interruptExit)) { + assert.fail("mid-prompt steering interrupt must still take the hard teardown path"); + } + assert.include(Cause.pretty(interruptExit.cause), "session is poisoned"); + }).pipe(Effect.provide(testLayer), Effect.scoped), + ); + + it.live( + "soft mid-prompt interrupt cancels in place, reuses the runtime, and tracks cancel-backgrounded work", + () => + Effect.gen(function* () { + const childProcessSpawner = yield* ChildProcessSpawner.ChildProcessSpawner; + const fileSystem = yield* FileSystem.FileSystem; + const idAllocator = yield* IdAllocatorV2; + const path = yield* Path.Path; + const serverConfig = yield* ServerConfig; + const mockAgentPath = yield* path.fromFileUrl( + new URL("../../../scripts/acp-mock-agent.ts", import.meta.url), + ); + const protocolEvents = yield* Queue.bounded(256); + const continuationRequests: Array = []; + const instanceId = ProviderInstanceId.make("acp-test"); + let cancelCalled = false; + let runtimeOrdinalSeen = 0; + const adapter = makeAcpAdapterV2({ + crypto: yield* Crypto.Crypto, + instanceId, + flavor: { + driver: ACP_TEST_DRIVER, + capabilities: AcpProviderCapabilitiesV2, + enablePostSettleContinuation: true, + // Production Grok interrupt flags: hard teardown only with + // requestRuntimeRestart (user Stop). Without + // restartRuntimeOnEveryInterrupt a mid-prompt steering interrupt + // stays soft: session/cancel, same process, session reuse. + restartRuntimeAfterInterrupt: true, + terminateRuntimeProcessGroupOnInterrupt: true, + preserveRuntimeOnSettledInterrupt: true, + registerExtensions: ({ runtime: extensionRuntime, applyBackgroundTaskMutation }) => + registerXAiBackgroundTaskTracking(extensionRuntime, applyBackgroundTaskMutation), + // No ownDetachedProcessGroup: if the interrupt wrongly takes the + // hard path, terminateProcessGroup is missing and the interrupt + // fails loudly with a poisoned session. + makeRuntime: makeMockRuntime({ + childProcessSpawner, + mockAgentPath, + environment: (runtimeOrdinal) => { + runtimeOrdinalSeen = Math.max(runtimeOrdinalSeen, runtimeOrdinal); + return { + T3_ACP_EMIT_RUNNING_COMMAND_THEN_HANG_FIRST_PROMPT: "1", + T3_ACP_EMIT_TASK_BACKGROUNDED_AFTER_CANCEL: "1", + }; + }, + protocolEvents, wrapCancel: (cancel) => Effect.sync(() => { cancelCalled = true; @@ -4150,87 +6626,280 @@ describe("AcpAdapterV2", () => { subagentUpdatedResult = event.subagent.result; } } - assert.equal( - subagentStopStatus, - "interrupted", - "orphan Stop must emit interrupted terminal for turn-1 carryover subagent", - ); - assert.equal( - subagentStopResult, - streamedSubagentText, - "orphan Stop must merge streamed assistantText into the interrupted result", - ); - assert.equal( - subagentStopProviderThreadId, - providerThread.id, - "orphan Stop parent-level events must use the spawn-time parent provider thread id", - ); - assert.equal( - subagentUpdatedResult, - streamedSubagentText, - "orphan Stop subagent.updated must also carry the streamed result", - ); + assert.equal( + subagentStopStatus, + "interrupted", + "orphan Stop must emit interrupted terminal for turn-1 carryover subagent", + ); + assert.equal( + subagentStopResult, + streamedSubagentText, + "orphan Stop must merge streamed assistantText into the interrupted result", + ); + assert.equal( + subagentStopProviderThreadId, + providerThread.id, + "orphan Stop parent-level events must use the spawn-time parent provider thread id", + ); + assert.equal( + subagentUpdatedResult, + streamedSubagentText, + "orphan Stop subagent.updated must also carry the streamed result", + ); + + yield* Queue.clear(protocolEvents); + const secondNow = yield* DateTime.now; + yield* runtime.startTurn( + makeTurnInput({ + threadId, + providerThread, + instanceId, + runtimePolicy, + now: secondNow, + ordinal: 2, + }), + ); + const loadAfterRestart = yield* Stream.fromQueue(protocolEvents).pipe( + Stream.filter( + (event) => + event.direction === "outgoing" && + (rawProtocolMethod(event) === "session/load" || + rawProtocolMethod(event) === "session/new"), + ), + Stream.runHead, + ); + assert.isTrue( + Option.isSome(loadAfterRestart), + "orphan containment must force a runtime respawn before the next turn", + ); + assert.equal( + runtimeOrdinalSeen, + 2, + "Stop after soft steer must replace the orphan ACP runtime process", + ); + + // nativeTurnId is `${sessionId}:turn:${ordinal}`. The mock always uses + // mock-session-1; ordinal 2 yields turn:2 on the replacement process. + // Carryover was quarantined by Stop, so turn-1's subagent must not re-attach. + subagentPhase = "complete"; + const secondProviderTurnId = idAllocator.derive.providerTurn({ + driver: ACP_TEST_DRIVER, + nativeTurnId: "mock-session-1:turn:2", + }); + let carriedItemStatus: string | null = null; + let secondTerminalStatus: string | null = null; + while (secondTerminalStatus === null) { + const event = yield* Queue.take(events); + if ( + event.type === "turn_item.updated" && + event.turnItem.type === "subagent" && + event.turnItem.id === subagentTurnItemId + ) { + carriedItemStatus = event.turnItem.status; + } + if (event.type === "turn.terminal" && event.providerTurnId === secondProviderTurnId) { + secondTerminalStatus = event.status; + } + } + assert.isNull( + carriedItemStatus, + "Stop quarantine must drop turn-1 subagent carryover on the respawned runtime", + ); + assert.equal(secondTerminalStatus, "completed"); + }).pipe(Effect.provide(testLayer), Effect.scoped), + ); + + it.live("Direct Stop projects an interrupt-deferred terminal exactly once", () => + Effect.gen(function* () { + const childProcessSpawner = yield* ChildProcessSpawner.ChildProcessSpawner; + const fileSystem = yield* FileSystem.FileSystem; + const idAllocator = yield* IdAllocatorV2; + const path = yield* Path.Path; + const serverConfig = yield* ServerConfig; + const mockAgentPath = yield* path.fromFileUrl( + new URL("../../../scripts/acp-mock-agent.ts", import.meta.url), + ); + const protocolEvents = yield* Queue.bounded(256); + const instanceId = ProviderInstanceId.make("acp-test"); + const childSessionId = "mock-child-session-direct-stop-deferred-terminal"; + let subagentPhase: "spawn" | "complete" = "spawn"; + type RuntimeService = AcpSessionRuntime.AcpSessionRuntime["Service"]; + let sessionUpdateHandler: Parameters[0] | undefined; + const adapter = makeAcpAdapterV2({ + crypto: yield* Crypto.Crypto, + instanceId, + flavor: { + driver: ACP_TEST_DRIVER, + capabilities: AcpProviderCapabilitiesV2, + deferFinalizeForBackgroundWork: true, + enablePostSettleContinuation: true, + restartRuntimeAfterInterrupt: true, + terminateRuntimeProcessGroupOnInterrupt: true, + preserveRuntimeOnSettledInterrupt: true, + extractSubagentUpdate: (toolCall) => + toolCall.toolCallId !== "tool-call-generic-1" + ? undefined + : subagentPhase === "spawn" + ? { + nativeTaskId: "task-generic-1", + prompt: "background subagent", + title: "background subagent", + model: null, + status: "running", + childSessionId, + result: null, + } + : { + nativeTaskId: "task-generic-1", + prompt: "", + title: null, + model: null, + status: "completed", + childSessionId, + result: "SUB_DONE", + }, + makeRuntime: makeMockRuntime({ + childProcessSpawner, + mockAgentPath, + environment: { T3_ACP_EMIT_GENERIC_TOOL_PLACEHOLDERS: "1" }, + ownDetachedProcessGroup: true, + protocolEvents, + wrapRuntime: (runtime) => ({ + ...runtime, + handleSessionUpdate: (handler) => + Effect.sync(() => { + sessionUpdateHandler = handler; + }).pipe(Effect.andThen(runtime.handleSessionUpdate(handler))), + }), + }), + }, + fileSystem, + idAllocator, + serverConfig, + continuationRequests: { offer: () => Effect.void }, + }); + const threadId = ThreadId.make("thread-acp-direct-stop-deferred-terminal"); + const runtimePolicy = ProviderAdapterV2RuntimePolicy.make({ + runtimeMode: "full-access", + interactionMode: "default", + cwd: process.cwd(), + }); + const modelSelection = { instanceId, model: "default" } as const; + const runtime = yield* adapter.openSession({ + threadId, + providerSessionId: ProviderSessionId.make( + "provider-session-acp-direct-stop-deferred-terminal", + ), + modelSelection, + runtimePolicy, + }); + if (runtime.hasPendingBackgroundWork === undefined) { + return yield* Effect.die( + "ACP runtime must expose hasPendingBackgroundWork when post-settle continuation is enabled.", + ); + } + const hasPendingBackgroundWork = runtime.hasPendingBackgroundWork; + const events = yield* Queue.unbounded(); + yield* runtime.events.pipe( + Stream.runForEach((event) => Queue.offer(events, event)), + Effect.forkScoped, + ); + const providerThread = yield* runtime.ensureThread({ + threadId, + modelSelection, + runtimePolicy, + }); + const now = yield* DateTime.now; + yield* runtime.startTurn( + makeTurnInput({ threadId, providerThread, instanceId, runtimePolicy, now }), + ); + yield* Stream.fromQueue(protocolEvents).pipe( + Stream.filter( + (event) => + event.direction === "incoming" && + event.stage === "raw" && + typeof event.payload === "string" && + event.payload.includes('"stopReason"'), + ), + Stream.runHead, + ); + yield* Effect.yieldNow; + yield* Effect.yieldNow; + + const providerTurnId = idAllocator.derive.providerTurn({ + driver: ACP_TEST_DRIVER, + nativeTurnId: "mock-session-1:turn:1", + }); + yield* runtime.interruptTurn({ providerThread, providerTurnId }); + let rootTerminal: string | null = null; + while (rootTerminal === null) { + const event = yield* Queue.take(events); + if (event.type === "turn.terminal" && event.providerTurnId === providerTurnId) { + rootTerminal = event.status; + } + } + assert.equal(rootTerminal, "interrupted"); + assert.isDefined(sessionUpdateHandler, "session update handler must be wired"); + + subagentPhase = "complete"; + yield* sessionUpdateHandler!({ + sessionId: childSessionId, + update: { + sessionUpdate: "tool_call_update", + toolCallId: "tool-call-generic-1", + title: "background subagent", + kind: "other", + status: "completed", + rawOutput: { content: "SUB_DONE" }, + }, + }); + yield* Effect.yieldNow; + let completedBeforeStop = 0; + let polled = yield* Queue.poll(events); + while (Option.isSome(polled)) { + const event = polled.value; + if ( + event.type === "turn_item.updated" && + event.turnItem.type === "subagent" && + event.turnItem.status === "completed" + ) { + completedBeforeStop += 1; + } + polled = yield* Queue.poll(events); + } + assert.equal(completedBeforeStop, 0); + assert.isTrue( + yield* hasPendingBackgroundWork, + "interrupt-deferred terminal must pin until hard-stop projection", + ); - yield* Queue.clear(protocolEvents); - const secondNow = yield* DateTime.now; - yield* runtime.startTurn( - makeTurnInput({ - threadId, - providerThread, - instanceId, - runtimePolicy, - now: secondNow, - ordinal: 2, - }), - ); - const loadAfterRestart = yield* Stream.fromQueue(protocolEvents).pipe( - Stream.filter( - (event) => - event.direction === "outgoing" && - (rawProtocolMethod(event) === "session/load" || - rawProtocolMethod(event) === "session/new"), - ), - Stream.runHead, - ); - assert.isTrue( - Option.isSome(loadAfterRestart), - "orphan containment must force a runtime respawn before the next turn", - ); - assert.equal( - runtimeOrdinalSeen, - 2, - "Stop after soft steer must replace the orphan ACP runtime process", - ); + const stopExit = yield* runtime + .interruptTurn({ + providerThread, + providerTurnId, + requestRuntimeRestart: true, + }) + .pipe(Effect.exit); + if (Exit.isFailure(stopExit)) { + assert.fail(`Direct Stop must contain the orphan runtime: ${Cause.pretty(stopExit.cause)}`); + } - // nativeTurnId is `${sessionId}:turn:${ordinal}`. The mock always uses - // mock-session-1; ordinal 2 yields turn:2 on the replacement process. - // Carryover was quarantined by Stop, so turn-1's subagent must not re-attach. - subagentPhase = "complete"; - const secondProviderTurnId = idAllocator.derive.providerTurn({ - driver: ACP_TEST_DRIVER, - nativeTurnId: "mock-session-1:turn:2", - }); - let carriedItemStatus: string | null = null; - let secondTerminalStatus: string | null = null; - while (secondTerminalStatus === null) { - const event = yield* Queue.take(events); - if ( - event.type === "turn_item.updated" && - event.turnItem.type === "subagent" && - event.turnItem.id === subagentTurnItemId - ) { - carriedItemStatus = event.turnItem.status; - } - if (event.type === "turn.terminal" && event.providerTurnId === secondProviderTurnId) { - secondTerminalStatus = event.status; - } + let completedAfterStop = 0; + for (let attempt = 0; attempt < 64; attempt += 1) { + const maybeEvent = yield* Queue.take(events).pipe(Effect.timeoutOption("50 millis")); + if (Option.isNone(maybeEvent)) break; + const event = maybeEvent.value; + if ( + event.type === "turn_item.updated" && + event.turnItem.type === "subagent" && + event.turnItem.status === "completed" + ) { + completedAfterStop += 1; } - assert.isNull( - carriedItemStatus, - "Stop quarantine must drop turn-1 subagent carryover on the respawned runtime", - ); - assert.equal(secondTerminalStatus, "completed"); - }).pipe(Effect.provide(testLayer), Effect.scoped), + } + assert.equal(completedAfterStop, 1); + assert.isFalse(yield* hasPendingBackgroundWork); + }).pipe(Effect.provide(testLayer), Effect.scoped), ); it.effect( @@ -4650,6 +7319,211 @@ describe("AcpAdapterV2", () => { }).pipe(Effect.provide(testLayer), Effect.scoped), ); + it.effect("keeps a buffered continuation current when a user turn starts before dispatch", () => + Effect.gen(function* () { + const childProcessSpawner = yield* ChildProcessSpawner.ChildProcessSpawner; + const fileSystem = yield* FileSystem.FileSystem; + const idAllocator = yield* IdAllocatorV2; + const path = yield* Path.Path; + const serverConfig = yield* ServerConfig; + const mockAgentPath = yield* path.fromFileUrl( + new URL("../../../scripts/acp-mock-agent.ts", import.meta.url), + ); + const continuationRequests: Array = []; + const userPromptStarted = yield* Deferred.make(); + const releaseUserPrompt = yield* Deferred.make(); + const bufferedAssistantText = "BUFFERED_WAKE_QUEUED_AFTER_USER"; + let promptOrdinal = 0; + type RuntimeService = AcpSessionRuntime.AcpSessionRuntime["Service"]; + let sessionUpdateHandler: Parameters[0] | undefined; + const instanceId = ProviderInstanceId.make("acp-test"); + const adapter = makeAcpAdapterV2({ + crypto: yield* Crypto.Crypto, + instanceId, + flavor: { + driver: ACP_TEST_DRIVER, + capabilities: AcpProviderCapabilitiesV2, + deferFinalizeForBackgroundWork: true, + enablePostSettleContinuation: true, + makeRuntime: makeMockRuntime({ + childProcessSpawner, + mockAgentPath, + wrapRuntime: (runtime) => ({ + ...runtime, + handleSessionUpdate: (handler) => + Effect.sync(() => { + sessionUpdateHandler = handler; + }).pipe(Effect.andThen(runtime.handleSessionUpdate(handler))), + prompt: (payload) => + Effect.gen(function* () { + promptOrdinal += 1; + const currentPromptOrdinal = promptOrdinal; + const result = yield* runtime.prompt(payload); + if (currentPromptOrdinal === 2) { + yield* Deferred.succeed(userPromptStarted, undefined); + yield* Deferred.await(releaseUserPrompt); + } + return result; + }), + }), + }), + }, + fileSystem, + idAllocator, + serverConfig, + continuationRequests: { + offer: (request) => + Effect.sync(() => { + continuationRequests.push(request); + }), + }, + }); + const threadId = ThreadId.make("thread-acp-buffered-continuation-user-race"); + const runtimePolicy = ProviderAdapterV2RuntimePolicy.make({ + runtimeMode: "full-access", + interactionMode: "default", + cwd: process.cwd(), + }); + const modelSelection = { instanceId, model: "default" } as const; + const runtime = yield* adapter.openSession({ + threadId, + providerSessionId: ProviderSessionId.make( + "provider-session-acp-buffered-continuation-user-race", + ), + modelSelection, + runtimePolicy, + }); + if (runtime.hasPendingBackgroundWork === undefined) { + return yield* Effect.die( + "ACP runtime must expose hasPendingBackgroundWork when post-settle continuation is enabled.", + ); + } + const hasPendingBackgroundWork = runtime.hasPendingBackgroundWork; + const events = yield* Queue.unbounded(); + yield* runtime.events.pipe( + Stream.runForEach((event) => Queue.offer(events, event)), + Effect.forkScoped, + ); + const providerThread = yield* runtime.ensureThread({ + threadId, + modelSelection, + runtimePolicy, + }); + yield* runtime.startTurn( + makeTurnInput({ + threadId, + providerThread, + instanceId, + runtimePolicy, + now: yield* DateTime.now, + }), + ); + const firstProviderTurnId = idAllocator.derive.providerTurn({ + driver: ACP_TEST_DRIVER, + nativeTurnId: "mock-session-1:turn:1", + }); + let firstTerminalStatus: string | null = null; + while (firstTerminalStatus === null) { + const event = yield* Queue.take(events); + if (event.type === "turn.terminal" && event.providerTurnId === firstProviderTurnId) { + firstTerminalStatus = event.status; + } + } + assert.equal(firstTerminalStatus, "completed"); + assert.isDefined(sessionUpdateHandler, "session update handler must be wired"); + + yield* sessionUpdateHandler!({ + sessionId: "mock-session-1", + update: { + sessionUpdate: "agent_message_chunk", + content: { type: "text", text: bufferedAssistantText }, + }, + }); + assert.lengthOf(continuationRequests, 1); + const continuationRequest = continuationRequests[0]!; + assert.isDefined(continuationRequest.dispatchIfCurrent); + + yield* runtime.startTurn( + makeTurnInput({ + threadId, + providerThread, + instanceId, + runtimePolicy, + now: yield* DateTime.now, + ordinal: 2, + messageText: "User turn won the continuation dispatch race.", + }), + ); + yield* Deferred.await(userPromptStarted); + let continuationDispatched = false; + const dispatchOutcome = yield* continuationRequest.dispatchIfCurrent!( + Effect.sync(() => { + continuationDispatched = true; + }), + ); + assert.isTrue( + Option.isSome(dispatchOutcome), + "queue_after_active dispatch must remain current while the user run is active", + ); + assert.isTrue(continuationDispatched); + yield* Deferred.succeed(releaseUserPrompt, undefined); + + const userProviderTurnId = idAllocator.derive.providerTurn({ + driver: ACP_TEST_DRIVER, + nativeTurnId: "mock-session-1:turn:2", + }); + let userTerminalStatus: string | null = null; + while (userTerminalStatus === null) { + const event = yield* Queue.take(events); + if (event.type === "turn.terminal" && event.providerTurnId === userProviderTurnId) { + userTerminalStatus = event.status; + } + } + assert.equal(userTerminalStatus, "completed"); + assert.isTrue( + yield* hasPendingBackgroundWork, + "the queued continuation must retain ownership of its buffered wake traffic", + ); + + yield* runtime.startTurn( + makeTurnInput({ + threadId, + providerThread, + instanceId, + runtimePolicy, + now: yield* DateTime.now, + ordinal: 3, + messageCreatedBy: "agent", + messageCreationSource: "provider", + messageText: "Background task completed.", + }), + ); + yield* TestClock.adjust("3 seconds"); + const continuationProviderTurnId = idAllocator.derive.providerTurn({ + driver: ACP_TEST_DRIVER, + nativeTurnId: "mock-session-1:turn:3", + }); + let continuationTerminalStatus: string | null = null; + let bufferedTextSeen = false; + while (continuationTerminalStatus === null) { + const event = yield* Queue.take(events); + if ( + event.type === "message.updated" && + event.message.role === "assistant" && + event.message.text.includes(bufferedAssistantText) + ) { + bufferedTextSeen = true; + } + if (event.type === "turn.terminal" && event.providerTurnId === continuationProviderTurnId) { + continuationTerminalStatus = event.status; + } + } + assert.equal(continuationTerminalStatus, "completed"); + assert.isTrue(bufferedTextSeen, "continuation must drain the wake buffer after the user run"); + assert.isFalse(yield* hasPendingBackgroundWork); + }).pipe(Effect.provide(testLayer), Effect.scoped), + ); + it.effect( "holds a settled turn until the injected monitor report streams instead of finalizing into it", () => diff --git a/apps/server/src/orchestration-v2/Adapters/AcpAdapterV2.ts b/apps/server/src/orchestration-v2/Adapters/AcpAdapterV2.ts index 6ae3c65078a1..4704e7489af3 100644 --- a/apps/server/src/orchestration-v2/Adapters/AcpAdapterV2.ts +++ b/apps/server/src/orchestration-v2/Adapters/AcpAdapterV2.ts @@ -151,7 +151,7 @@ export interface AcpRootTurnIdleSnapshot { readonly hasRunningTool: boolean; readonly hasPendingRuntimeRequest: boolean; readonly hasToolHistory: boolean; - readonly hasRunningSubagent: boolean; + readonly hasActiveSubagent: boolean; readonly hasOutput: boolean; } @@ -176,7 +176,7 @@ export function acpRootTurnIsIdle(snapshot: AcpRootTurnIdleSnapshot): boolean { if (snapshot.finalized || snapshot.interrupted) return false; if (snapshot.assistantStreamOpen || snapshot.reasoningStreamOpen) return false; if (snapshot.hasRunningTool || snapshot.hasPendingRuntimeRequest) return false; - if (snapshot.hasRunningSubagent) return false; + if (snapshot.hasActiveSubagent) return false; if (!snapshot.hasOutput) return false; // Structural gates above stay for unit tests / future re-enable. Speculative // idle completion is intentionally disabled. @@ -327,7 +327,7 @@ export interface AcpAdapterV2SubagentUpdate { readonly prompt: string; readonly title: string | null; readonly model: string | null; - readonly status: "running" | "completed" | "failed" | "interrupted" | "cancelled"; + readonly status: "pending" | "running" | "completed" | "failed" | "interrupted" | "cancelled"; readonly childSessionId: string | null; readonly result: string | null; /** @@ -1064,6 +1064,7 @@ interface ActiveAcpTurn { } | null; interrupted: boolean; finalized: boolean; + finalizedStatus: "completed" | "interrupted" | "failed" | "cancelled" | null; settleScheduleGeneration: number; /** session/prompt already returned; finalize deferred for background work. */ promptSettled: boolean; @@ -1203,6 +1204,21 @@ export function acpIsAppOwnedWakeTurn(message: { return message.createdBy === "agent" && message.creationSource === "server"; } +export function acpCarryoverTerminalShouldClearContinuation(input: { + readonly continuationOffered: boolean; + readonly wakeBufferLength: number; +}): boolean { + return !input.continuationOffered && input.wakeBufferLength === 0; +} + +export function acpTurnStartShouldPreserveContinuation(input: { + readonly continuationRequested: boolean; + readonly isContinuationTurn: boolean; + readonly wakeBufferLength: number; +}): boolean { + return !input.isContinuationTurn && input.continuationRequested && input.wakeBufferLength > 0; +} + export function acpPostSettleMonitorPromptShouldSuppress( mutation: | { @@ -1236,8 +1252,44 @@ interface ActiveAcpSubagent { childSessionId: string | null; assistantText: string; nextChildOrdinal: number; + /** + * Whether a terminal carryover status has been projected to events. + * Completed roots project post-settle terminals immediately while their + * subscriber remains observable. Non-completed roots retain terminals in + * memory until the next attach. A later project:true path for the same entry + * must still project once; this flag prevents double emission and pins + * hasPendingBackgroundWork until projection lands. + */ + terminalStatusProjected: boolean; +} + +function acpSubagentStatusIsTerminal(status: OrchestrationV2Subagent["status"]): boolean { + return ( + status === "completed" || + status === "failed" || + status === "interrupted" || + status === "cancelled" + ); +} + +export function acpSubagentStatusBlocksTurnSettlement( + status: OrchestrationV2Subagent["status"], +): boolean { + return status === "running" || status === "pending"; } +function acpSubagentHasPendingBackgroundWork(subagent: ActiveAcpSubagent): boolean { + return ( + acpSubagentStatusBlocksTurnSettlement(subagent.task.status) || !subagent.terminalStatusProjected + ); +} + +type AcpCarryoverSubagents = { + readonly sessionId: string; + readonly rootTerminalStatus: "completed" | "interrupted" | "failed" | "cancelled"; + readonly subagents: ReadonlyArray; +}; + function acpTurnHasPendingRuntimeRequest( providerTurnId: OrchestrationV2ProviderTurn["id"], pending: ReadonlyMap, @@ -1585,10 +1637,7 @@ export function makeAcpAdapterV2(options: AcpAdapterV2Options): ProviderAdapterV // lineages into the next turn on the same session so their terminal // signals can still flip the original turn items instead of leaving // them running forever. - const carryoverSubagents = yield* Ref.make<{ - readonly sessionId: string; - readonly subagents: ReadonlyArray; - } | null>(null); + const carryoverSubagents = yield* Ref.make(null); const handledBackgroundTaskIdsInActiveTurn = yield* Ref.make>( new Set(), ); @@ -2025,6 +2074,22 @@ export function makeAcpAdapterV2(options: AcpAdapterV2Options): ProviderAdapterV (update.childSessionId !== null ? context.subagentsBySessionId.get(update.childSessionId) : undefined); + const updateIsTerminal = acpSubagentStatusIsTerminal(update.status); + if ( + existing !== undefined && + acpSubagentStatusIsTerminal(existing.task.status) && + !updateIsTerminal + ) { + return; + } + if ( + existing !== undefined && + existing.task.status === update.status && + updateIsTerminal && + existing.terminalStatusProjected + ) { + return; + } // get_command_or_subagent_output may target monitors/bash tasks. Only // hydrate when we already have a matching subagent lineage. Spawn ACKs // (non-empty prompt) may create a new lineage; empty-prompt hydration @@ -2090,7 +2155,7 @@ export function makeAcpAdapterV2(options: AcpAdapterV2Options): ProviderAdapterV }), status: taskStatus, result: existing?.assistantText || update.result, - completedAt: taskStatus === "running" ? null : now, + completedAt: acpSubagentStatusIsTerminal(taskStatus) ? now : null, updatedAt: now, }; const subagent: ActiveAcpSubagent = existing ?? { @@ -2104,6 +2169,7 @@ export function makeAcpAdapterV2(options: AcpAdapterV2Options): ProviderAdapterV childSessionId: null, assistantText: "", nextChildOrdinal: 101, + terminalStatusProjected: false, }; subagent.task = task; context.subagents.set(nativeTaskId, subagent); @@ -2208,7 +2274,7 @@ export function makeAcpAdapterV2(options: AcpAdapterV2Options): ProviderAdapterV ...subagent.task, status: taskStatus, result, - completedAt: taskStatus === "running" ? null : now, + completedAt: acpSubagentStatusIsTerminal(taskStatus) ? now : null, updatedAt: now, }; const providerThreadId = subagent.task.providerThreadId; @@ -2283,6 +2349,9 @@ export function makeAcpAdapterV2(options: AcpAdapterV2Options): ProviderAdapterV result, }, }); + if (acpSubagentStatusIsTerminal(taskStatus)) { + subagent.terminalStatusProjected = true; + } }); const toolOutputText = (toolCall: AcpToolCallState): string => { @@ -2331,7 +2400,7 @@ export function makeAcpAdapterV2(options: AcpAdapterV2Options): ProviderAdapterV if (status === "pending" || status === "running") return true; } for (const subagent of context.subagents.values()) { - if (subagent.task.status === "running" || subagent.task.status === "pending") { + if (acpSubagentStatusBlocksTurnSettlement(subagent.task.status)) { return true; } } @@ -2855,7 +2924,7 @@ export function makeAcpAdapterV2(options: AcpAdapterV2Options): ProviderAdapterV const offerContinuationRun = Effect.fnUntraced(function* (_sessionId: string) { if (continuationRequests === undefined) { - return; + return false; } const pending = yield* continuationPermit.withPermit( Effect.gen(function* () { @@ -2872,7 +2941,7 @@ export function makeAcpAdapterV2(options: AcpAdapterV2Options): ProviderAdapterV return Option.some({ route, generation }); }), ); - if (Option.isNone(pending)) return; + if (Option.isNone(pending)) return false; const { route, generation } = pending.value; yield* Effect.logInfo("orchestration-v2.acp-wake-turn-detected", { driver, @@ -2924,6 +2993,7 @@ export function makeAcpAdapterV2(options: AcpAdapterV2Options): ProviderAdapterV }), ), }); + return true; }); const applyLateBackgroundMutation = Effect.fnUntraced(function* ( @@ -2998,19 +3068,35 @@ export function makeAcpAdapterV2(options: AcpAdapterV2Options): ProviderAdapterV notification: EffectAcpSchema.SessionNotification, ) { if (!postSettleContinuationEnabled || continuationRequests === undefined) { - return false; + return { + buffered: false, + offerContinuation: false, + stopProcessing: false, + }; } // Direct Stop quarantine: drop residual wake evidence instead of // buffering it for a later continuation or follow-up run. if (yield* Ref.get(stoppedRunQuarantine)) { - return true; + return { + buffered: false, + offerContinuation: false, + stopProcessing: true, + }; } const rootSessionId = yield* Ref.get(activeSessionId); if (rootSessionId === null || notification.sessionId !== rootSessionId) { - return false; + return { + buffered: false, + offerContinuation: false, + stopProcessing: false, + }; } if (!acpPostSettleWakeEvidence(notification, flavor)) { - return false; + return { + buffered: false, + offerContinuation: false, + stopProcessing: false, + }; } const update = notification.update; let alreadyHandledToolUpdate = false; @@ -3072,16 +3158,22 @@ export function makeAcpAdapterV2(options: AcpAdapterV2Options): ProviderAdapterV // (the already-handled gate below intentionally skips // `offerContinuationRun` to avoid synthetic "Background task // completed." spam). + let buffered = false; if ( !alreadyHandledToolUpdate && !isInTurnHandledAgentChatter && acpPostSettleWakeShouldBuffer(notification, backgroundWorkRunning) ) { yield* Ref.update(wakeBuffer, (current) => [...current, notification]); + buffered = true; } // Buffer progress without offering; only completion-like frames open a run. if (!acpPostSettleContinuationOfferEvidence(notification, flavor)) { - return true; + return { + buffered, + offerContinuation: false, + stopProcessing: true, + }; } // While a monitor is still streaming, tool re-reports buffer without // offering and per-event agent commentary is consumed without being @@ -3093,7 +3185,11 @@ export function makeAcpAdapterV2(options: AcpAdapterV2Options): ProviderAdapterV // actually ends (end-notice mutation below, or the first frame after // it). if (backgroundWorkRunning) { - return true; + return { + buffered, + offerContinuation: false, + stopProcessing: true, + }; } if (alreadyHandledToolUpdate) { // Drop leftover wake noise for in-turn-handled work so idle release @@ -3103,23 +3199,45 @@ export function makeAcpAdapterV2(options: AcpAdapterV2Options): ProviderAdapterV if (!(yield* Ref.get(continuationRequested))) { yield* Ref.set(wakeBuffer, []); } - return true; + return { + buffered, + offerContinuation: false, + stopProcessing: true, + }; } // Same in-turn-handled agent chatter: do not open a synthetic wake. // Do not clear wakeBuffer here: frames for other still-tracked tasks // must remain drainable when a real (tool) completion later offers. if (isInTurnHandledAgentChatter) { - return true; + return { + buffered, + offerContinuation: false, + stopProcessing: true, + }; } - yield* offerContinuationRun(notification.sessionId); - return true; + return { + buffered, + offerContinuation: true, + stopProcessing: true, + }; }); + let applyFinalizedActiveTurnSubagentTerminal: ( + context: ActiveAcpTurn, + notification: EffectAcpSchema.SessionNotification, + ) => Effect.Effect = () => Effect.succeed(false); + const handleSessionUpdate = Effect.fnUntraced(function* ( notification: EffectAcpSchema.SessionNotification, ) { const context = yield* Ref.get(activeTurn); const update = notification.update; + if ( + context?.finalized === true && + (yield* applyFinalizedActiveTurnSubagentTerminal(context, notification)) + ) { + return; + } // Only while a finalized turn is still the active context. When // activeTurn is null, post-settle agent frames must reach // bufferPostSettleWake so continuation can attach (context?.finalized @@ -3159,9 +3277,102 @@ export function makeAcpAdapterV2(options: AcpAdapterV2Options): ProviderAdapterV if (yield* Ref.get(stoppedRunQuarantine)) { return; } + const bufferOutcome = yield* bufferPostSettleWake(notification); + // Post-settle carryover sync: keep in-memory carryover accurate so + // hasPendingBackgroundWork reasons correctly after root settle. + // A completed root keeps its subscriber open while background items + // remain, so project its terminals immediately even when the wake + // frame is buffered for a continuation. Non-completed roots stay + // memory-only: durable ingest requires a subscriber that owns the + // original runId, which the interrupted path does not guarantee. + const carryover = yield* Ref.get(carryoverSubagents); + const rootTerminalCanStillProject = + carryover !== null && + carryover.sessionId === (yield* Ref.get(activeSessionId)) && + carryover.rootTerminalStatus === "completed"; + const projectCarryover = rootTerminalCanStillProject; + let carryoverTerminalized = false; + if ( + flavor.extractSubagentUpdate !== undefined && + (update.sessionUpdate === "tool_call" || update.sessionUpdate === "tool_call_update") + ) { + for (const event of parseSessionUpdateEvent(notification).events) { + if (event._tag !== "ToolCallUpdated") continue; + const toolCall = flavor.normalizeToolCall?.(event.toolCall) ?? event.toolCall; + const subagentUpdate = flavor.extractSubagentUpdate(toolCall); + if (subagentUpdate === undefined) continue; + if (!acpSubagentStatusIsTerminal(subagentUpdate.status)) { + continue; + } + if ( + yield* updateCarryoverSubagentStatus( + subagentUpdate.nativeTaskId, + subagentUpdate.status, + subagentUpdate.result, + { project: projectCarryover }, + ) + ) { + carryoverTerminalized = true; + } + if ( + subagentUpdate.childSessionId !== null && + (yield* updateCarryoverSubagentStatus( + subagentUpdate.childSessionId, + subagentUpdate.status, + subagentUpdate.result, + { project: projectCarryover }, + )) + ) { + carryoverTerminalized = true; + } + } + } + if ( + update.sessionUpdate === "user_message_chunk" && + update.content.type === "text" && + flavor.extractSubagentEndNotice !== undefined + ) { + const notice = flavor.extractSubagentEndNotice(update.content.text); + if ( + notice !== undefined && + (yield* updateCarryoverSubagentStatus( + notice.childSessionId, + notice.status, + undefined, + { + project: projectCarryover, + }, + )) + ) { + carryoverTerminalized = true; + } + } + // Synchronize carryover before an eager continuation can attach and + // drain the frame. + const continuationOffered = bufferOutcome.offerContinuation + ? yield* offerContinuationRun(notification.sessionId) + : false; + const wakeOutcome = { ...bufferOutcome, continuationOffered }; + // Frames that will not buffer never reach the continuation drain for + // this traffic. Once carryover is terminalized, skip history append / + // residual handling (and avoid double-projecting on child re-entry). + if (carryoverTerminalized && !wakeOutcome.buffered) { + // Projected above. Never clear a non-empty wakeBuffer. A sticky + // continuationRequested with an empty buffer is safe to drop so + // idle release is not wed on a pin with nothing to deliver. + if ( + acpCarryoverTerminalShouldClearContinuation({ + continuationOffered: wakeOutcome.continuationOffered, + wakeBufferLength: (yield* Ref.get(wakeBuffer)).length, + }) + ) { + yield* Ref.set(continuationRequested, false); + } + return; + } // Prefer continuation buffering over history append so the same // frames are not double-counted once a continuation run attaches. - if (yield* bufferPostSettleWake(notification)) { + if (wakeOutcome.stopProcessing) { return; } if ( @@ -3204,10 +3415,32 @@ export function makeAcpAdapterV2(options: AcpAdapterV2Options): ProviderAdapterV // Finalize may have completed during the activeSessionId yield. if (context.finalized) return; if (flavor.extractSubagentUpdate === undefined) return; + const subagent = context.subagentsBySessionId.get(notification.sessionId); + if ( + update.sessionUpdate === "tool_call" || + update.sessionUpdate === "tool_call_update" + ) { + if (subagent === undefined) return; + const nativeTaskId = + subagent.task.nativeTaskRef?.nativeId ?? String(subagent.task.id); + for (const event of parseSessionUpdateEvent(notification).events) { + if (event._tag !== "ToolCallUpdated") continue; + const toolCall = flavor.normalizeToolCall?.(event.toolCall) ?? event.toolCall; + const subagentUpdate = flavor.extractSubagentUpdate(toolCall); + if ( + subagentUpdate === undefined || + (subagentUpdate.nativeTaskId !== nativeTaskId && + subagentUpdate.childSessionId !== notification.sessionId) + ) { + continue; + } + yield* emitSubagent(context, subagentUpdate); + } + return; + } if (update.sessionUpdate !== "agent_message_chunk" || update.content.type !== "text") { return; } - const subagent = context.subagentsBySessionId.get(notification.sessionId); if (subagent !== undefined) { yield* projectSubagentNotification(subagent, notification); return; @@ -3716,113 +3949,305 @@ export function makeAcpAdapterV2(options: AcpAdapterV2Options): ProviderAdapterV ); }); + /** + * Mutate a carryover subagent's in-memory task status without projecting. + * Used after a non-completed root whose original run subscriber is no + * longer guaranteed to ingest the terminal event. + */ + const mutateCarryoverSubagentStatus = Effect.fnUntraced(function* ( + subagent: ActiveAcpSubagent, + status: OrchestrationV2Subagent["status"], + resultOverride?: string | null, + ) { + const now = yield* DateTime.now; + const result = + resultOverride !== undefined && resultOverride !== null && resultOverride.length > 0 + ? resultOverride + : subagent.assistantText || subagent.task.result; + const completedAt = acpSubagentStatusIsTerminal(status) ? now : null; + subagent.task = { + ...subagent.task, + status, + result, + completedAt, + updatedAt: now, + }; + }); + + /** + * Project a carryover subagent status change without an ActiveAcpTurn. + * Shared by completed-root post-settle completion, deferred attach, and + * Direct Stop terminalization. + */ + const projectCarryoverSubagentStatus = Effect.fnUntraced(function* ( + subagent: ActiveAcpSubagent, + status: OrchestrationV2Subagent["status"], + resultOverride?: string | null, + ) { + yield* mutateCarryoverSubagentStatus(subagent, status, resultOverride); + const now = subagent.task.updatedAt; + const nativeTaskId = subagent.task.nativeTaskRef?.nativeId ?? subagent.task.id; + const nativeItemRef = { + driver, + nativeId: nativeTaskId, + strength: "strong" as const, + }; + const parentProviderThreadId = subagent.parentProviderThreadId; + const result = subagent.task.result; + const completedAt = subagent.task.completedAt; + yield* emitProviderEvent({ + type: "node.updated", + driver, + node: { + id: subagent.task.id, + threadId: subagent.task.threadId, + runId: subagent.task.runId, + parentNodeId: subagent.task.parentNodeId, + rootNodeId: subagent.task.parentNodeId, + kind: "subagent", + status, + countsForRun: false, + providerThreadId: parentProviderThreadId, + providerTurnId: subagent.providerTurnId, + nativeItemRef, + runtimeRequestId: null, + checkpointScopeId: null, + startedAt: subagent.task.startedAt, + completedAt, + }, + }); + yield* emitProviderEvent({ + type: "node.updated", + driver, + node: { + id: subagent.childRootNodeId, + threadId: subagent.childThreadId, + runId: null, + parentNodeId: null, + rootNodeId: subagent.childRootNodeId, + kind: "root_turn", + status, + countsForRun: false, + providerThreadId: subagent.task.providerThreadId, + providerTurnId: null, + nativeItemRef, + runtimeRequestId: null, + checkpointScopeId: null, + startedAt: subagent.task.startedAt, + completedAt, + }, + }); + yield* emitProviderEvent({ + type: "subagent.updated", + driver, + subagent: subagent.task, + }); + yield* emitProviderEvent({ + type: "turn_item.updated", + driver, + turnItem: { + id: subagent.turnItemId, + threadId: subagent.task.threadId, + runId: subagent.task.runId, + nodeId: subagent.task.id, + providerThreadId: parentProviderThreadId, + providerTurnId: subagent.providerTurnId, + nativeItemRef, + parentItemId: null, + ordinal: subagent.turnItemOrdinal, + status, + title: subagent.task.title, + startedAt: subagent.task.startedAt, + completedAt, + updatedAt: now, + type: "subagent", + subagentId: subagent.task.id, + origin: "provider_native", + driver, + providerInstanceId: subagent.task.providerInstanceId, + childThreadId: subagent.childThreadId, + prompt: subagent.task.prompt, + result, + }, + }); + if (acpSubagentStatusIsTerminal(status)) { + subagent.terminalStatusProjected = true; + } + }); + + /** + * Update a carryover subagent matched by native task id or child session + * id. Post-settle completions must flip the pin in hasPendingBackgroundWork + * without waiting for a new user turn to consume carryover. + * When `project` is false, only the in-memory carryover status advances + * (the next attach projects the terminal state). + * If status already matches but projection was deferred, a later + * project:true caller still projects once via terminalStatusProjected. + */ + const updateCarryoverSubagentStatus = Effect.fnUntraced(function* ( + nativeIdOrChildSessionId: string, + status: OrchestrationV2Subagent["status"], + result?: string | null, + options?: { readonly project?: boolean }, + ) { + const carryover = yield* Ref.get(carryoverSubagents); + if (carryover === null) return false; + const match = carryover.subagents.find((subagent) => { + const nativeId = subagent.task.nativeTaskRef?.nativeId ?? String(subagent.task.id); + return ( + nativeId === nativeIdOrChildSessionId || + subagent.childSessionId === nativeIdOrChildSessionId + ); + }); + if (match === undefined) return false; + const shouldProject = options?.project !== false; + if (match.task.status === status) { + // Already at this status: only project if requested and not yet done. + if (!shouldProject || match.terminalStatusProjected) { + return true; + } + yield* projectCarryoverSubagentStatus(match, status, result); + return true; + } + // Only advance nonterminal entries; do not resurrect a terminal one. + if (match.task.status !== "running" && match.task.status !== "pending") { + return false; + } + if (!shouldProject) { + yield* mutateCarryoverSubagentStatus(match, status, result); + } else { + yield* projectCarryoverSubagentStatus(match, status, result); + } + return true; + }); + + applyFinalizedActiveTurnSubagentTerminal = Effect.fnUntraced(function* ( + context: ActiveAcpTurn, + notification: EffectAcpSchema.SessionNotification, + ) { + if (flavor.extractSubagentUpdate === undefined) return false; + + const applyTerminal = Effect.fnUntraced(function* ( + subagent: ActiveAcpSubagent, + update: AcpAdapterV2SubagentUpdate, + ) { + if (!acpSubagentStatusIsTerminal(update.status)) return false; + if (acpSubagentStatusIsTerminal(subagent.task.status)) { + if (subagent.terminalStatusProjected || context.finalizedStatus !== "completed") { + return true; + } + } + if (context.finalizedStatus === "completed") { + yield* emitSubagent(context, update); + } else { + yield* mutateCarryoverSubagentStatus(subagent, update.status, update.result); + } + return true; + }); + + const sessionUpdate = notification.update; + if ( + sessionUpdate.sessionUpdate === "tool_call" || + sessionUpdate.sessionUpdate === "tool_call_update" + ) { + for (const event of parseSessionUpdateEvent(notification).events) { + if (event._tag !== "ToolCallUpdated") continue; + const toolCall = flavor.normalizeToolCall?.(event.toolCall) ?? event.toolCall; + const subagentUpdate = flavor.extractSubagentUpdate(toolCall); + if ( + subagentUpdate === undefined || + !acpSubagentStatusIsTerminal(subagentUpdate.status) + ) { + continue; + } + const subagent = + context.subagents.get(subagentUpdate.nativeTaskId) ?? + (subagentUpdate.childSessionId === null + ? undefined + : context.subagentsBySessionId.get(subagentUpdate.childSessionId)) ?? + context.subagentsBySessionId.get(notification.sessionId); + if (subagent === undefined) continue; + const nativeTaskId = + subagent.task.nativeTaskRef?.nativeId ?? String(subagent.task.id); + if ( + subagentUpdate.nativeTaskId !== nativeTaskId && + (subagentUpdate.childSessionId === null || + (subagentUpdate.childSessionId !== subagent.childSessionId && + subagentUpdate.childSessionId !== notification.sessionId)) + ) { + continue; + } + if (yield* applyTerminal(subagent, subagentUpdate)) return true; + } + } + + if ( + sessionUpdate.sessionUpdate === "user_message_chunk" && + sessionUpdate.content.type === "text" && + flavor.extractSubagentEndNotice !== undefined + ) { + const notice = flavor.extractSubagentEndNotice(sessionUpdate.content.text); + const subagent = + notice === undefined + ? undefined + : context.subagentsBySessionId.get(notice.childSessionId); + if (notice !== undefined && subagent !== undefined) { + return yield* applyTerminal(subagent, { + nativeTaskId: subagent.task.nativeTaskRef?.nativeId ?? String(subagent.task.id), + prompt: subagent.task.prompt, + title: subagent.task.title, + model: subagent.task.model, + status: notice.status, + childSessionId: notice.childSessionId, + result: null, + suppressNormalTool: true, + }); + } + } + + return false; + }); + + const projectDeferredCarryoverTerminals = Effect.fnUntraced(function* ( + subagents: ReadonlyArray, + ) { + for (const subagent of subagents) { + if ( + subagent.task.status === "running" || + subagent.task.status === "pending" || + subagent.terminalStatusProjected + ) { + continue; + } + yield* projectCarryoverSubagentStatus( + subagent, + subagent.task.status, + subagent.task.result, + ); + } + }); + /** * Direct Stop after a soft steer clears carryover without an active turn. * Emit the same interrupted terminal events terminalizeOpenRunOwnedItems * would have, context-free (no ActiveAcpTurn). */ const terminalizeCarryoverSubagents = Effect.fnUntraced(function* ( - carryover: { - readonly sessionId: string; - readonly subagents: ReadonlyArray; - } | null, + carryover: AcpCarryoverSubagents | null, ) { if (carryover === null) return; - const now = yield* DateTime.now; for (const subagent of carryover.subagents) { - if (subagent.task.status !== "running" && subagent.task.status !== "pending") { + if (acpSubagentStatusIsTerminal(subagent.task.status)) { + if (!subagent.terminalStatusProjected) { + yield* projectCarryoverSubagentStatus( + subagent, + subagent.task.status, + subagent.task.result, + ); + } continue; } - const nativeTaskId = subagent.task.nativeTaskRef?.nativeId ?? subagent.task.id; - const nativeItemRef = { - driver, - nativeId: nativeTaskId, - strength: "strong" as const, - }; - const parentProviderThreadId = subagent.parentProviderThreadId; - const result = subagent.assistantText || subagent.task.result; - subagent.task = { - ...subagent.task, - status: "interrupted", - result, - completedAt: now, - updatedAt: now, - }; - yield* emitProviderEvent({ - type: "node.updated", - driver, - node: { - id: subagent.task.id, - threadId: subagent.task.threadId, - runId: subagent.task.runId, - parentNodeId: subagent.task.parentNodeId, - rootNodeId: subagent.task.parentNodeId, - kind: "subagent", - status: "interrupted", - countsForRun: false, - providerThreadId: parentProviderThreadId, - providerTurnId: subagent.providerTurnId, - nativeItemRef, - runtimeRequestId: null, - checkpointScopeId: null, - startedAt: subagent.task.startedAt, - completedAt: now, - }, - }); - yield* emitProviderEvent({ - type: "node.updated", - driver, - node: { - id: subagent.childRootNodeId, - threadId: subagent.childThreadId, - runId: null, - parentNodeId: null, - rootNodeId: subagent.childRootNodeId, - kind: "root_turn", - status: "interrupted", - countsForRun: false, - providerThreadId: subagent.task.providerThreadId, - providerTurnId: null, - nativeItemRef, - runtimeRequestId: null, - checkpointScopeId: null, - startedAt: subagent.task.startedAt, - completedAt: now, - }, - }); - yield* emitProviderEvent({ - type: "subagent.updated", - driver, - subagent: subagent.task, - }); - yield* emitProviderEvent({ - type: "turn_item.updated", - driver, - turnItem: { - id: subagent.turnItemId, - threadId: subagent.task.threadId, - runId: subagent.task.runId, - nodeId: subagent.task.id, - providerThreadId: parentProviderThreadId, - providerTurnId: subagent.providerTurnId, - nativeItemRef, - parentItemId: null, - ordinal: subagent.turnItemOrdinal, - status: "interrupted", - title: subagent.task.title, - startedAt: subagent.task.startedAt, - completedAt: now, - updatedAt: now, - type: "subagent", - subagentId: subagent.task.id, - origin: "provider_native", - driver, - providerInstanceId: subagent.task.providerInstanceId, - childThreadId: subagent.childThreadId, - prompt: subagent.task.prompt, - result, - }, - }); + yield* projectCarryoverSubagentStatus(subagent, "interrupted"); } }); @@ -4318,6 +4743,7 @@ export function makeAcpAdapterV2(options: AcpAdapterV2Options): ProviderAdapterV ) { if (context.finalized) return; const settledStatus = context.interrupted ? "interrupted" : status; + context.finalizedStatus = settledStatus; context.finalized = true; if (options?.drainTrailingChunks === true) { yield* drainTrailingRootTurnChunks(); @@ -4386,14 +4812,18 @@ export function makeAcpAdapterV2(options: AcpAdapterV2Options): ProviderAdapterV threadDisposition: "reusable", }, ); - const liveSubagents = [...context.subagents.values()].filter( - (subagent) => subagent.task.status === "running" || subagent.task.status === "pending", + const subagentsRequiringCarryover = [...context.subagents.values()].filter( + acpSubagentHasPendingBackgroundWork, ); // Direct Stop must not carry residual subagents into a later run. - if (liveSubagents.length > 0 && !directStopQuarantine) { + if (subagentsRequiringCarryover.length > 0 && !directStopQuarantine) { const sessionId = yield* Ref.get(activeSessionId); if (sessionId !== null) { - yield* Ref.set(carryoverSubagents, { sessionId, subagents: liveSubagents }); + yield* Ref.set(carryoverSubagents, { + sessionId, + rootTerminalStatus: settledStatus, + subagents: subagentsRequiringCarryover, + }); } } yield* Ref.set(activeTurn, null); @@ -4443,8 +4873,8 @@ export function makeAcpAdapterV2(options: AcpAdapterV2Options): ProviderAdapterV }); // Debounce already proved root-session quiescence; open segment handles // without an explicit close should not block settlement. - const hasRunningSubagent = [...context.subagents.values()].some( - (subagent) => subagent.task.status === "running", + const hasActiveSubagent = [...context.subagents.values()].some((subagent) => + acpSubagentStatusBlocksTurnSettlement(subagent.task.status), ); if ( !acpRootTurnIsIdle({ @@ -4455,7 +4885,7 @@ export function makeAcpAdapterV2(options: AcpAdapterV2Options): ProviderAdapterV hasRunningTool, hasPendingRuntimeRequest, hasToolHistory: context.tools.size > 0, - hasRunningSubagent, + hasActiveSubagent, hasOutput: context.assistant.nextSegment > 0, }) ) { @@ -4650,23 +5080,28 @@ export function makeAcpAdapterV2(options: AcpAdapterV2Options): ProviderAdapterV if (!isAppOwnedWakeTurn) { yield* Ref.set(midTurnUnreportedCompletedTaskIds, new Set()); } - // User turns must not inherit prior-turn wake residue. Stale injected- - // turn ack chatter buffered for in-turn-handled work can otherwise - // arm a mid-turn offer when a later monitor completes (multiturn - // live repro). Continuation turns keep the buffer so attach mode can - // drain it. Still-running tasks remain in runningBackgroundTaskIds - // and re-buffer evidence when they complete; a user message means - // this turn owns the conversation, so prior wake frames cannot be - // legitimate for a synthetic continuation of the previous turn. - if (!isContinuationTurn && !isAppOwnedWakeTurn) { - yield* Ref.set(wakeBuffer, []); - } - // Drop a sticky continuation offer when any new turn starts so idle - // pin and further offers cannot wed on a completed or failed dispatch. - yield* continuationPermit.withPermit( + // A user turn supersedes an empty offer, but not an offer that owns + // buffered wake traffic. ProviderContinuationService queues that + // continuation behind the user run so it can drain afterwards. + const continuationWasRequested = yield* continuationPermit.withPermit( Effect.gen(function* () { + const wasRequested = yield* Ref.get(continuationRequested); + const preserveBufferedContinuation = acpTurnStartShouldPreserveContinuation({ + continuationRequested: wasRequested, + isContinuationTurn, + wakeBufferLength: (yield* Ref.get(wakeBuffer)).length, + }); + // User turns must not inherit prior-turn wake residue. Stale + // injected-turn ack chatter can otherwise arm a later offer. + // Preserve only a queued continuation that owns buffered wake + // traffic, and exempt app-owned sibling wakes entirely. + if (!isContinuationTurn && !isAppOwnedWakeTurn && !preserveBufferedContinuation) { + yield* Ref.set(wakeBuffer, []); + } + if (preserveBufferedContinuation) return wasRequested; yield* Ref.update(continuationGeneration, (value) => value + 1); yield* Ref.set(continuationRequested, false); + return wasRequested; }), ); const prompt = isContinuationTurn ? null : yield* resolvePromptParts(turnInput); @@ -4697,6 +5132,7 @@ export function makeAcpAdapterV2(options: AcpAdapterV2Options): ProviderAdapterV plan: null, interrupted: false, finalized: false, + finalizedStatus: null, settleScheduleGeneration: 0, promptSettled: false, promptSettledStatus: null, @@ -4704,7 +5140,9 @@ export function makeAcpAdapterV2(options: AcpAdapterV2Options): ProviderAdapterV backgroundFinalizeGeneration: 0, }; const carryover = yield* Ref.getAndSet(carryoverSubagents, null); + let rehydratedCarryoverSubagents: ReadonlyArray = []; if (carryover !== null && carryover.sessionId === requestedSessionId) { + rehydratedCarryoverSubagents = carryover.subagents; for (const subagent of carryover.subagents) { const nativeId = subagent.task.nativeTaskRef?.nativeId ?? null; if (nativeId !== null) { @@ -4755,22 +5193,42 @@ export function makeAcpAdapterV2(options: AcpAdapterV2Options): ProviderAdapterV createdAt: startedAt, updatedAt: startedAt, }); + // Every attach ends the deferred-terminal contract, but only the + // queued provider continuation owns wake traffic. A user turn may + // start before that continuation and must leave its buffer intact. + yield* projectDeferredCarryoverTerminals(rehydratedCarryoverSubagents); + // Keep projected terminals addressable through the wake drain so + // emitSubagent's monotonic guard can reject replayed spawn frames. + // Finalize carries only entries with pending background work, so a + // terminal-and-projected lineage still expires with this turn. if (isContinuationTurn) { - const drained = yield* Ref.modify(wakeBuffer, (current) => { - const next: Array = []; - return [current.slice(), next] as const; - }); yield* Ref.set(continuationRequested, false); + const drainedWakeCount = yield* Ref.modify(wakeBuffer, (current) => { + const next: Array = []; + return [ + current.filter((notification) => notification.sessionId === requestedSessionId), + next, + ] as const; + }).pipe( + Effect.tap((drained) => + Effect.forEach(drained, handleSessionUpdate, { + concurrency: 1, + discard: true, + }), + ), + Effect.map((drained) => drained.length), + ); // Treat attach mode as prompt-settled so deferred finalize / quiet // windows can complete the continuation after wake traffic drains. context.promptSettled = true; context.promptSettledStatus = "completed"; - if (drained.length === 0) { - // Empty wakeBuffer (midTurn-only offer): wait the quiet window - // so late CLI frames can attach. scheduleDeferredFinalize - // no-ops without deferFinalizeForBackgroundWork; fall back to - // immediate finalize so the turn cannot wedge. - if (flavor.deferFinalizeForBackgroundWork === true) { + if (drainedWakeCount === 0) { + // A requested continuation with only mid-turn evidence has no + // buffered frame yet. Wait the quiet window so late CLI frames + // can attach. A provider-authored attach without a matching + // request only exists to project deferred carryover terminals, + // so it can finish immediately once those terminals are visible. + if (continuationWasRequested && flavor.deferFinalizeForBackgroundWork === true) { if (hasDeferredBackgroundWork(context)) { yield* rearmDeferredFinalize(context); } else { @@ -4783,9 +5241,6 @@ export function makeAcpAdapterV2(options: AcpAdapterV2Options): ProviderAdapterV } return; } - for (const notification of drained) { - yield* handleSessionUpdate(notification); - } if (!context.finalized) { if (hasDeferredBackgroundWork(context)) { yield* rearmDeferredFinalize(context); @@ -4952,6 +5407,27 @@ export function makeAcpAdapterV2(options: AcpAdapterV2Options): ProviderAdapterV if ((yield* Ref.get(wakeBuffer)).length > 0) return true; if (yield* Ref.get(continuationRequested)) return true; if ((yield* Ref.get(runningBackgroundTaskIds)).size > 0) return true; + // Projected post-settle Grok subagents can outlive the root + // turn via carryover; keep the ACP process pinned until they + // terminalize or teardown clears the carryover. + // Also pin while a terminal status is held only in memory + // (project:false) so idle release cannot drop the session + // before the continuation drain (or a later project:true path) + // delivers the turn_item terminal. + const active = yield* Ref.get(activeTurn); + if ( + active !== null && + [...active.subagents.values()].some(acpSubagentHasPendingBackgroundWork) + ) { + return true; + } + const carryover = yield* Ref.get(carryoverSubagents); + if ( + carryover !== null && + carryover.subagents.some(acpSubagentHasPendingBackgroundWork) + ) { + return true; + } return false; }), } diff --git a/apps/server/src/orchestration-v2/Adapters/ClaudeAdapterV2.test.ts b/apps/server/src/orchestration-v2/Adapters/ClaudeAdapterV2.test.ts index 6a37a684d43d..e73fba0298bc 100644 --- a/apps/server/src/orchestration-v2/Adapters/ClaudeAdapterV2.test.ts +++ b/apps/server/src/orchestration-v2/Adapters/ClaudeAdapterV2.test.ts @@ -55,6 +55,7 @@ import { CLAUDE_READ_ONLY_T3_MCP_ALLOWED_TOOLS, CLAUDE_T3_MCP_TOOL_WILDCARD, ClaudeProviderCapabilitiesV2, + ClaudeAgentSdkQueryRunnerError, claudeEffectiveQueryPolicyKey, claudeMcpQueryOverrides, claudeQueryMessages, @@ -124,6 +125,8 @@ function makeClaudeTestTurnInput(input: { readonly providerTurnOrdinal?: number; readonly messageCreatedBy?: ProviderAdapterV2TurnInput["message"]["createdBy"]; readonly messageCreationSource?: ProviderAdapterV2TurnInput["message"]["creationSource"]; + readonly modelSelection?: ModelSelection; + readonly runtimePolicy?: ProviderAdapterV2RuntimePolicy; }): ProviderAdapterV2TurnInput { return { appThread: makeClaudeTestAppThread(input), @@ -141,8 +144,8 @@ function makeClaudeTestTurnInput(input: { text: input.text, attachments: input.attachments, }, - modelSelection: CLAUDE_TEST_MODEL_SELECTION, - runtimePolicy: CLAUDE_TEST_RUNTIME_POLICY, + modelSelection: input.modelSelection ?? CLAUDE_TEST_MODEL_SELECTION, + runtimePolicy: input.runtimePolicy ?? CLAUDE_TEST_RUNTIME_POLICY, }; } @@ -1122,6 +1125,7 @@ describe("ClaudeAdapterV2 background wake turns", () => { const WAKE_NATIVE_SESSION = "native-thread-claude-wake"; const WAKE_TASK_ID = "task-wake-build"; const WAKE_SUMMARY = "Background build completed successfully"; + const WAKE_ASSISTANT_TEXT = "The background build has finished."; const WAKE_RESULT_TEXT = "The background build finished; everything passed."; function claudeSdkFrame(frame: unknown): SDKMessage { @@ -1214,6 +1218,16 @@ describe("ClaudeAdapterV2 background wake turns", () => { uuid: "00000000-0000-4000-8000-000000000103", session_id: WAKE_NATIVE_SESSION, }); + const wakeAssistant = claudeSdkFrame({ + type: "assistant", + message: { + role: "assistant", + content: [{ type: "text", text: WAKE_ASSISTANT_TEXT }], + }, + parent_tool_use_id: null, + uuid: "00000000-0000-4000-8000-000000000107", + session_id: WAKE_NATIVE_SESSION, + }); const wakeResult = makeResultFrame({ uuid: "00000000-0000-4000-8000-000000000104", result: WAKE_RESULT_TEXT, @@ -1445,6 +1459,528 @@ describe("ClaudeAdapterV2 background wake turns", () => { ), ); + const providerThreadRosterEvents = (events: ReadonlyArray) => + events.filter( + (event): event is Extract => + event.type === "provider_thread.updated", + ); + + it.effect( + "projects an authoritative background_tasks_changed roster on the provider thread", + () => + Effect.scoped( + Effect.gen(function* () { + const harness = yield* makeWakeHarness; + const now = yield* DateTime.now; + const rosterSnapshot = claudeSdkFrame({ + type: "system", + subtype: "background_tasks_changed", + tasks: [ + { + task_id: WAKE_TASK_ID, + description: "npm run build", + task_type: "local_bash", + }, + ], + uuid: "00000000-0000-4000-8000-000000000201", + session_id: WAKE_NATIVE_SESSION, + }); + + yield* harness.runtime.startTurn( + makeClaudeTestTurnInput({ + threadId: harness.threadId, + providerThread: harness.providerThread, + now, + attemptId: RunAttemptId.make("attempt-claude-roster-snapshot"), + text: "Run the build in the background.", + attachments: [], + }), + ); + yield* Queue.offer(harness.sdkMessages, rosterSnapshot); + yield* Queue.offer(harness.sdkMessages, turnOneResult); + yield* awaitUntil(() => harness.terminalEvents().length === 1, "first turn terminal"); + + const rosterEvents = providerThreadRosterEvents(harness.events).filter( + (event) => (event.providerThread.pendingBackgroundTasks?.length ?? 0) > 0, + ); + assert.isAtLeast(rosterEvents.length, 1); + assert.deepEqual(rosterEvents.at(-1)?.providerThread.pendingBackgroundTasks ?? [], [ + { + taskId: WAKE_TASK_ID, + description: "npm run build", + taskType: "local_bash", + }, + ]); + assert.isTrue(yield* harness.hasPendingBackgroundWork); + }).pipe(Effect.provide(Layer.merge(idAllocatorLayer, NodeServices.layer))), + ), + ); + + it.effect( + "uses task_started as an incremental roster fallback and clears on empty snapshot", + () => + Effect.scoped( + Effect.gen(function* () { + const harness = yield* makeWakeHarness; + const now = yield* DateTime.now; + const emptyRoster = claudeSdkFrame({ + type: "system", + subtype: "background_tasks_changed", + tasks: [], + uuid: "00000000-0000-4000-8000-000000000202", + session_id: WAKE_NATIVE_SESSION, + }); + + yield* harness.runtime.startTurn( + makeClaudeTestTurnInput({ + threadId: harness.threadId, + providerThread: harness.providerThread, + now, + attemptId: RunAttemptId.make("attempt-claude-roster-fallback"), + text: "Run the build in the background.", + attachments: [], + }), + ); + yield* Queue.offer(harness.sdkMessages, wakeTaskStarted); + yield* Queue.offer(harness.sdkMessages, turnOneResult); + yield* awaitUntil(() => harness.terminalEvents().length === 1, "first turn terminal"); + + const afterStart = providerThreadRosterEvents(harness.events).filter( + (event) => (event.providerThread.pendingBackgroundTasks?.length ?? 0) > 0, + ); + assert.isAtLeast(afterStart.length, 1); + assert.equal( + (afterStart.at(-1)?.providerThread.pendingBackgroundTasks ?? [])[0]?.taskId, + WAKE_TASK_ID, + ); + + yield* Queue.offer(harness.sdkMessages, emptyRoster); + yield* awaitUntil( + () => + providerThreadRosterEvents(harness.events).some( + (event) => + event.providerThread.status === "idle" && + (event.providerThread.pendingBackgroundTasks?.length ?? 0) === 0, + ), + "empty roster clear", + ); + assert.isFalse(yield* harness.hasPendingBackgroundWork); + }).pipe(Effect.provide(Layer.merge(idAllocatorLayer, NodeServices.layer))), + ), + ); + + it.effect("clears the roster when a turn fails", () => + Effect.scoped( + Effect.gen(function* () { + const harness = yield* makeWakeHarness; + const now = yield* DateTime.now; + const failedResult = claudeSdkFrame({ + type: "result", + subtype: "error_during_execution", + duration_ms: 10, + duration_api_ms: 10, + is_error: true, + num_turns: 1, + result: "boom", + stop_reason: "end_turn", + total_cost_usd: 0, + usage: { + input_tokens: 1, + output_tokens: 1, + cache_creation_input_tokens: 0, + cache_read_input_tokens: 0, + }, + modelUsage: {}, + permission_denials: [], + errors: ["boom"], + uuid: "00000000-0000-4000-8000-000000000203", + session_id: WAKE_NATIVE_SESSION, + }); + + yield* harness.runtime.startTurn( + makeClaudeTestTurnInput({ + threadId: harness.threadId, + providerThread: harness.providerThread, + now, + attemptId: RunAttemptId.make("attempt-claude-roster-fail"), + text: "Run the build in the background.", + attachments: [], + }), + ); + yield* Queue.offer(harness.sdkMessages, wakeTaskStarted); + yield* awaitUntil( + () => + providerThreadRosterEvents(harness.events).some( + (event) => (event.providerThread.pendingBackgroundTasks?.length ?? 0) > 0, + ), + "roster after task_started", + ); + yield* Queue.offer(harness.sdkMessages, failedResult); + yield* awaitUntil(() => harness.terminalEvents().length === 1, "failed terminal"); + assert.equal(harness.terminalEvents()[0]?.status, "failed"); + + const afterFailure = providerThreadRosterEvents(harness.events).at(-1); + assert.deepEqual(afterFailure?.providerThread.pendingBackgroundTasks ?? [], []); + assert.isFalse(yield* harness.hasPendingBackgroundWork); + }).pipe(Effect.provide(Layer.merge(idAllocatorLayer, NodeServices.layer))), + ), + ); + + it.effect("clears the native-thread roster when a turn is interrupted", () => + Effect.scoped( + Effect.gen(function* () { + const fileSystem = yield* FileSystem.FileSystem; + const idAllocator = yield* IdAllocatorV2; + const attachmentsDir = yield* fileSystem.makeTempDirectoryScoped({ + prefix: "t3-claude-v2-roster-interrupt-", + }); + const sdkMessages = yield* Queue.unbounded(); + const events: Array = []; + const adapter = makeClaudeAdapterV2({ + instanceId: CLAUDE_DEFAULT_INSTANCE_ID, + settings: DEFAULT_CLAUDE_SETTINGS, + environment: {}, + attachmentsDir, + fileSystem, + idAllocator, + continuationRequests: { offer: () => Effect.void }, + queryRunner: { + allocateSessionId: Effect.succeed(WAKE_NATIVE_SESSION), + open: () => + Effect.succeed({ + messages: Stream.fromQueue(sdkMessages), + offer: () => Effect.void, + setModel: () => Effect.void, + interrupt: Effect.void, + // End the message stream so interruptTurn's closed wait resolves + // via stream exit finalize (interrupted status clears roster). + close: Queue.shutdown(sdkMessages), + }), + forkSession: () => Effect.die("unused forkSession"), + assertComplete: Effect.void, + }, + }); + const threadId = ThreadId.make("thread-claude-roster-interrupt"); + const runtime = yield* adapter.openSession({ + threadId, + providerSessionId: ProviderSessionId.make("provider-session-claude-roster-interrupt"), + modelSelection: CLAUDE_TEST_MODEL_SELECTION, + runtimePolicy: CLAUDE_TEST_RUNTIME_POLICY, + }); + const providerThread = yield* runtime.ensureThread({ + threadId, + modelSelection: CLAUDE_TEST_MODEL_SELECTION, + runtimePolicy: CLAUDE_TEST_RUNTIME_POLICY, + }); + yield* runtime.events.pipe( + Stream.runForEach((event) => + Effect.sync(() => { + events.push(event); + }), + ), + Effect.forkScoped, + ); + if (runtime.hasPendingBackgroundWork === undefined) { + return yield* Effect.die("Claude adapter runtime must expose hasPendingBackgroundWork."); + } + const now = yield* DateTime.now; + + yield* runtime.startTurn( + makeClaudeTestTurnInput({ + threadId, + providerThread, + now, + attemptId: RunAttemptId.make("attempt-claude-roster-interrupt"), + text: "Run the build in the background.", + attachments: [], + }), + ); + yield* Queue.offer(sdkMessages, wakeTaskStarted); + yield* awaitUntil( + () => + providerThreadRosterEvents(events).some( + (event) => (event.providerThread.pendingBackgroundTasks?.length ?? 0) > 0, + ), + "roster after task_started", + ); + + const providerTurnId = events.find( + (event): event is Extract => + event.type === "provider_turn.updated", + )?.providerTurn.id; + assert.isDefined(providerTurnId); + yield* runtime.interruptTurn({ + providerThread, + providerTurnId: providerTurnId!, + }); + yield* awaitUntil( + () => + events.some( + (event) => event.type === "turn.terminal" && event.status === "interrupted", + ), + "interrupted terminal", + ); + + const afterInterrupt = providerThreadRosterEvents(events).at(-1); + assert.deepEqual(afterInterrupt?.providerThread.pendingBackgroundTasks ?? [], []); + assert.isFalse(yield* runtime.hasPendingBackgroundWork); + }).pipe(Effect.provide(Layer.merge(idAllocatorLayer, NodeServices.layer))), + ), + ); + + it.effect( + "clears the replaced sibling native thread roster when openQuery switches processes", + () => + Effect.scoped( + Effect.gen(function* () { + const fileSystem = yield* FileSystem.FileSystem; + const idAllocator = yield* IdAllocatorV2; + const attachmentsDir = yield* fileSystem.makeTempDirectoryScoped({ + prefix: "t3-claude-v2-sibling-replace-", + }); + const nativeIds = ["native-thread-roster-a", "native-thread-roster-b"] as const; + let allocateIndex = 0; + // Real two-process model: each openQuery owns its own message queue. + // A shared queue would mask sibling process death on replacement. + const processQueues: Array<{ + readonly nativeThreadId: string; + readonly queue: Queue.Queue; + }> = []; + const events: Array = []; + const adapter = makeClaudeAdapterV2({ + instanceId: CLAUDE_DEFAULT_INSTANCE_ID, + settings: DEFAULT_CLAUDE_SETTINGS, + environment: {}, + attachmentsDir, + fileSystem, + idAllocator, + continuationRequests: { + offer: () => Effect.void, + }, + queryRunner: { + allocateSessionId: Effect.sync(() => { + const next = + nativeIds[allocateIndex] ?? `native-thread-roster-extra-${allocateIndex}`; + allocateIndex += 1; + return next; + }), + open: (openInput) => + Effect.gen(function* () { + const nativeThreadId = openInput.options.sessionId ?? openInput.options.resume; + if (typeof nativeThreadId !== "string" || nativeThreadId.length === 0) { + return yield* Effect.die("openQuery must supply a native session id"); + } + const queue = yield* Queue.unbounded(); + processQueues.push({ nativeThreadId, queue }); + return { + messages: Stream.fromQueue(queue), + offer: () => Effect.void, + setModel: () => Effect.void, + interrupt: Effect.void, + close: Queue.shutdown(queue), + }; + }), + forkSession: () => Effect.die("unused forkSession"), + assertComplete: Effect.void, + }, + }); + const appThreadA = ThreadId.make("thread-claude-roster-a"); + const appThreadB = ThreadId.make("thread-claude-roster-b"); + const runtime = yield* adapter.openSession({ + threadId: appThreadA, + providerSessionId: ProviderSessionId.make("provider-session-claude-sibling-replace"), + modelSelection: CLAUDE_TEST_MODEL_SELECTION, + runtimePolicy: CLAUDE_TEST_RUNTIME_POLICY, + }); + const providerThreadA = yield* runtime.ensureThread({ + threadId: appThreadA, + modelSelection: CLAUDE_TEST_MODEL_SELECTION, + runtimePolicy: CLAUDE_TEST_RUNTIME_POLICY, + }); + const providerThreadB = yield* runtime.ensureThread({ + threadId: appThreadB, + modelSelection: CLAUDE_TEST_MODEL_SELECTION, + runtimePolicy: CLAUDE_TEST_RUNTIME_POLICY, + }); + assert.notEqual( + providerThreadA.nativeThreadRef?.nativeId, + providerThreadB.nativeThreadRef?.nativeId, + ); + yield* runtime.events.pipe( + Stream.runForEach((event) => + Effect.sync(() => { + events.push(event); + }), + ), + Effect.forkScoped, + ); + if (runtime.hasPendingBackgroundWork === undefined) { + return yield* Effect.die( + "Claude adapter runtime must expose hasPendingBackgroundWork.", + ); + } + if (runtime.hasPendingBackgroundWorkForThread === undefined) { + return yield* Effect.die( + "Claude adapter runtime must expose hasPendingBackgroundWorkForThread.", + ); + } + const hasPendingBackgroundWork = runtime.hasPendingBackgroundWork; + const hasPendingBackgroundWorkForThread = runtime.hasPendingBackgroundWorkForThread; + const now = yield* DateTime.now; + const taskA = "task-roster-a"; + const taskB = "task-roster-b"; + + yield* runtime.startTurn( + makeClaudeTestTurnInput({ + threadId: appThreadA, + providerThread: providerThreadA, + now, + attemptId: RunAttemptId.make("attempt-roster-iso-a"), + text: "Background work on A.", + attachments: [], + }), + ); + assert.equal(processQueues.length, 1); + const processA = processQueues[0]!; + yield* Queue.offer( + processA.queue, + claudeSdkFrame({ + type: "system", + subtype: "task_started", + task_id: taskA, + description: "work on A", + task_type: "local_bash", + uuid: "00000000-0000-4000-8000-000000000301", + session_id: nativeIds[0], + }), + ); + yield* Queue.offer( + processA.queue, + makeResultFrame({ + uuid: "00000000-0000-4000-8000-000000000302", + result: "A settled with background work.", + }), + ); + yield* awaitUntil( + () => + events.some( + (event) => + event.type === "turn.terminal" && + event.providerThreadId === providerThreadA.id && + event.status === "completed", + ), + "thread A terminal", + ); + const rosterAAfterSettle = providerThreadRosterEvents(events).findLast( + (event) => event.providerThread.id === providerThreadA.id, + )?.providerThread.pendingBackgroundTasks; + assert.deepEqual(rosterAAfterSettle ?? [], [ + { taskId: taskA, description: "work on A", taskType: "local_bash" }, + ]); + assert.isTrue(yield* hasPendingBackgroundWork); + assert.isTrue(yield* hasPendingBackgroundWorkForThread(providerThreadA)); + assert.isFalse(yield* hasPendingBackgroundWorkForThread(providerThreadB)); + + // Starting B closes A's only live query. A can never emit a roster + // clear from a dead process, so openQuery must idle-clear A. + yield* runtime.startTurn( + makeClaudeTestTurnInput({ + threadId: appThreadB, + providerThread: { ...providerThreadB, status: "active" }, + now, + attemptId: RunAttemptId.make("attempt-roster-iso-b"), + text: "Background work on B.", + attachments: [], + }), + ); + assert.equal(processQueues.length, 2); + yield* awaitUntil( + () => + providerThreadRosterEvents(events).some( + (event) => + event.providerThread.id === providerThreadA.id && + event.providerThread.status === "idle" && + (event.providerThread.pendingBackgroundTasks?.length ?? 0) === 0, + ), + "sibling A roster cleared idle on process replacement", + ); + assert.isFalse(yield* hasPendingBackgroundWorkForThread(providerThreadA)); + + const processB = processQueues[1]!; + yield* Queue.offer( + processB.queue, + claudeSdkFrame({ + type: "system", + subtype: "task_started", + task_id: taskB, + description: "work on B", + task_type: "local_bash", + uuid: "00000000-0000-4000-8000-000000000303", + session_id: nativeIds[1], + }), + ); + yield* awaitUntil( + () => + providerThreadRosterEvents(events).some( + (event) => + event.providerThread.id === providerThreadB.id && + (event.providerThread.pendingBackgroundTasks?.length ?? 0) > 0, + ), + "thread B roster populated", + ); + assert.isTrue(yield* hasPendingBackgroundWorkForThread(providerThreadB)); + assert.isTrue(yield* hasPendingBackgroundWork); + // Starting B's process clears only B's process-scoped level; A stays + // empty from the sibling replacement clear above. + assert.isFalse(yield* hasPendingBackgroundWorkForThread(providerThreadA)); + + yield* Queue.offer( + processB.queue, + claudeSdkFrame({ + type: "result", + subtype: "error_during_execution", + duration_ms: 10, + duration_api_ms: 10, + is_error: true, + num_turns: 1, + result: "B failed", + stop_reason: "end_turn", + total_cost_usd: 0, + usage: { + input_tokens: 1, + output_tokens: 1, + cache_creation_input_tokens: 0, + cache_read_input_tokens: 0, + }, + modelUsage: {}, + permission_denials: [], + errors: ["B failed"], + uuid: "00000000-0000-4000-8000-000000000304", + session_id: nativeIds[1], + }), + ); + yield* awaitUntil( + () => + events.some( + (event) => + event.type === "turn.terminal" && + event.providerThreadId === providerThreadB.id && + event.status === "failed", + ), + "thread B failed terminal", + ); + + const rosterBAfterFail = providerThreadRosterEvents(events).findLast( + (event) => event.providerThread.id === providerThreadB.id, + )?.providerThread.pendingBackgroundTasks; + assert.deepEqual(rosterBAfterFail ?? [], []); + assert.isFalse(yield* hasPendingBackgroundWorkForThread(providerThreadA)); + assert.isFalse(yield* hasPendingBackgroundWorkForThread(providerThreadB)); + assert.isFalse(yield* hasPendingBackgroundWork); + }).pipe(Effect.provide(Layer.merge(idAllocatorLayer, NodeServices.layer))), + ), + ); + it.effect("buffers wake output and requests a single continuation run", () => Effect.scoped( Effect.gen(function* () { @@ -1469,6 +2005,12 @@ describe("ClaudeAdapterV2 background wake turns", () => { assert.lengthOf(harness.continuationRequests, 0); yield* Queue.offer(harness.sdkMessages, wakeNotification); + let quietYields = 0; + yield* awaitUntil(() => quietYields++ >= 50, "notification-only quiet window"); + assert.lengthOf(harness.continuationRequests, 0); + assert.isFalse(yield* harness.hasPendingBackgroundWork); + + yield* Queue.offer(harness.sdkMessages, wakeAssistant); yield* awaitUntil(() => harness.continuationRequests.length === 1, "continuation request"); assert.equal(harness.continuationRequests[0]?.threadId, harness.threadId); assert.equal(harness.continuationRequests[0]?.providerThreadId, harness.providerThread.id); @@ -1485,6 +2027,57 @@ describe("ClaudeAdapterV2 background wake turns", () => { ), ); + it.effect("does not offer a continuation for notification-only opaque work", () => + Effect.scoped( + Effect.gen(function* () { + const harness = yield* makeWakeHarness; + const now = yield* DateTime.now; + + yield* harness.runtime.startTurn( + makeClaudeTestTurnInput({ + threadId: harness.threadId, + providerThread: harness.providerThread, + now, + attemptId: RunAttemptId.make("attempt-claude-notification-only"), + text: "Start opaque background work.", + attachments: [], + }), + ); + yield* Queue.offer(harness.sdkMessages, wakeTaskStarted); + yield* Queue.offer(harness.sdkMessages, turnOneResult); + yield* awaitUntil(() => harness.terminalEvents().length === 1, "first turn terminal"); + assert.isTrue(yield* harness.hasPendingBackgroundWork); + + yield* Queue.offer(harness.sdkMessages, wakeNotification); + let quietYields = 0; + yield* awaitUntil(() => quietYields++ >= 100, "notification-only quiet window"); + assert.lengthOf(harness.continuationRequests, 0); + assert.lengthOf(harness.terminalEvents(), 1); + assert.isFalse(yield* harness.hasPendingBackgroundWork); + + yield* harness.runtime.startTurn( + makeClaudeTestTurnInput({ + threadId: harness.threadId, + providerThread: harness.providerThread, + now, + attemptId: RunAttemptId.make("attempt-claude-notification-only-continuation"), + text: "Background task completed.", + attachments: [], + providerTurnOrdinal: 2, + messageCreatedBy: "agent", + messageCreationSource: "provider", + }), + ); + yield* awaitUntil( + () => harness.terminalEvents().length === 2, + "notification-only continuation terminal", + ); + assert.equal(harness.terminalEvents()[1]?.status, "completed"); + assert.lengthOf(harness.offeredMessages, 1); + }).pipe(Effect.provide(Layer.merge(idAllocatorLayer, NodeServices.layer))), + ), + ); + it.effect("drains buffered wake messages into a continuation turn", () => Effect.scoped( Effect.gen(function* () { @@ -1536,7 +2129,11 @@ describe("ClaudeAdapterV2 background wake turns", () => { ); // The background task never renders as a subagent node. assert.isFalse( - harness.events.some((event) => JSON.stringify(event).includes(WAKE_TASK_ID)), + harness.events.some( + (event) => + event.type !== "provider_thread.updated" && + JSON.stringify(event).includes(WAKE_TASK_ID), + ), ); assert.isFalse(yield* harness.hasPendingBackgroundWork); }).pipe(Effect.provide(Layer.merge(idAllocatorLayer, NodeServices.layer))), @@ -1623,8 +2220,12 @@ describe("ClaudeAdapterV2 background wake turns", () => { ), ); assert.isFalse( - harness.events.some((event) => JSON.stringify(event).includes(WAKE_TASK_ID)), - ); + harness.events.some( + (event) => + event.type !== "provider_thread.updated" && + JSON.stringify(event).includes(WAKE_TASK_ID), + ), + ); assert.isFalse(yield* harness.hasPendingBackgroundWork); }).pipe(Effect.provide(Layer.merge(idAllocatorLayer, NodeServices.layer))), ), @@ -2109,10 +2710,10 @@ describe("ClaudeAdapterV2 background wake turns", () => { session_id: WAKE_NATIVE_SESSION, }), ); + yield* Queue.offer(harness.sdkMessages, wakeResult); yield* awaitUntil(() => harness.continuationRequests.length === 1, "continuation request"); assert.isNull(harness.continuationRequests[0]?.detail); - yield* Queue.offer(harness.sdkMessages, wakeResult); yield* harness.runtime.startTurn( makeClaudeTestTurnInput({ threadId: harness.threadId, @@ -2986,6 +3587,1243 @@ describe("ClaudeAdapterV2 background wake turns", () => { }).pipe(Effect.provide(Layer.merge(idAllocatorLayer, NodeServices.layer))), ), ); + + it.effect( + "orders nonempty level, empty level, notification, and continuation drain without subagent projection", + () => + Effect.scoped( + Effect.gen(function* () { + const harness = yield* makeWakeHarness; + const now = yield* DateTime.now; + const nonemptyRoster = claudeSdkFrame({ + type: "system", + subtype: "background_tasks_changed", + tasks: [ + { + task_id: WAKE_TASK_ID, + description: "npm run build", + task_type: "local_bash", + }, + ], + uuid: "00000000-0000-4000-8000-000000000600", + session_id: WAKE_NATIVE_SESSION, + }); + const emptyRoster = claudeSdkFrame({ + type: "system", + subtype: "background_tasks_changed", + tasks: [], + uuid: "00000000-0000-4000-8000-000000000601", + session_id: WAKE_NATIVE_SESSION, + }); + const duplicateNotification = claudeSdkFrame({ + type: "system", + subtype: "task_notification", + task_id: WAKE_TASK_ID, + status: "completed", + output_file: "/tmp/task-wake-build-dup.log", + summary: "duplicate should not re-buffer", + uuid: "00000000-0000-4000-8000-000000000605", + session_id: WAKE_NATIVE_SESSION, + }); + + yield* harness.runtime.startTurn( + makeClaudeTestTurnInput({ + threadId: harness.threadId, + providerThread: harness.providerThread, + now, + attemptId: RunAttemptId.make("attempt-claude-level-before-edge-a"), + text: "Run the build in the background.", + attachments: [], + }), + ); + // 1) Nonempty authoritative level admits local_bash to Waiting + + // wake eligibility. + yield* Queue.offer(harness.sdkMessages, nonemptyRoster); + yield* awaitUntil( + () => + providerThreadRosterEvents(harness.events).some( + (event) => (event.providerThread.pendingBackgroundTasks?.length ?? 0) > 0, + ), + "nonempty level populated Waiting roster", + ); + assert.deepEqual( + providerThreadRosterEvents(harness.events).at(-1)?.providerThread + .pendingBackgroundTasks ?? [], + [ + { + taskId: WAKE_TASK_ID, + description: "npm run build", + taskType: "local_bash", + }, + ], + ); + yield* Queue.offer(harness.sdkMessages, turnOneResult); + yield* awaitUntil(() => harness.terminalEvents().length === 1, "first turn terminal"); + assert.isTrue(yield* harness.hasPendingBackgroundWork); + + // 2) Empty level clears Waiting but keeps wake eligibility so the + // later notification can still offer exactly one continuation. + yield* Queue.offer(harness.sdkMessages, emptyRoster); + yield* awaitUntil( + () => + providerThreadRosterEvents(harness.events).some( + (event) => + event.providerThread.status === "idle" && + (event.providerThread.pendingBackgroundTasks?.length ?? 0) === 0, + ), + "empty level cleared Waiting roster", + ); + assert.isFalse(yield* harness.hasPendingBackgroundWork); + assert.lengthOf(harness.continuationRequests, 0); + + // 3) First idle notification buffers and consumes eligibility. The + // following native assistant frame proves Claude began a wake turn. + yield* Queue.offer(harness.sdkMessages, wakeNotification); + yield* Queue.offer(harness.sdkMessages, wakeAssistant); + yield* awaitUntil( + () => harness.continuationRequests.length === 1, + "continuation after level-before-edge", + ); + assert.equal(harness.continuationRequests[0]?.detail, WAKE_SUMMARY); + + // A duplicate notification must not re-buffer or re-offer. + yield* Queue.offer(harness.sdkMessages, duplicateNotification); + let settleYields = 0; + yield* awaitUntil(() => settleYields++ >= 50, "duplicate notification settle"); + assert.lengthOf(harness.continuationRequests, 1); + + // 4) Continuation drain classifies the buffered notification as + // local_bash (replay tombstone) and never fabricates a subagent. + yield* Queue.offer(harness.sdkMessages, wakeResult); + yield* harness.runtime.startTurn( + makeClaudeTestTurnInput({ + threadId: harness.threadId, + providerThread: harness.providerThread, + now, + attemptId: RunAttemptId.make("attempt-claude-level-before-edge-b"), + text: "Background task completed.", + attachments: [], + providerTurnOrdinal: 2, + messageCreatedBy: "agent", + messageCreationSource: "provider", + }), + ); + yield* awaitUntil(() => harness.terminalEvents().length === 2, "continuation terminal"); + assert.equal(harness.terminalEvents()[1]?.status, "completed"); + assert.lengthOf(harness.continuationRequests, 1); + assert.isTrue( + harness.events.some( + (event) => + event.type === "message.updated" && event.message.text === WAKE_ASSISTANT_TEXT, + ), + ); + assert.isFalse( + harness.events.some( + (event) => + event.type === "subagent.updated" || + (event.type === "node.updated" && event.node.kind === "subagent"), + ), + ); + assert.isFalse( + harness.events.some( + (event) => + event.type !== "provider_thread.updated" && + JSON.stringify(event).includes(WAKE_TASK_ID), + ), + ); + assert.isFalse(yield* harness.hasPendingBackgroundWork); + }).pipe(Effect.provide(Layer.merge(idAllocatorLayer, NodeServices.layer))), + ), + ); + + it.effect("resets Waiting roster and wake eligibility when the CLI process is replaced", () => + Effect.scoped( + Effect.gen(function* () { + const fileSystem = yield* FileSystem.FileSystem; + const idAllocator = yield* IdAllocatorV2; + const attachmentsDir = yield* fileSystem.makeTempDirectoryScoped({ + prefix: "t3-claude-v2-process-reset-", + }); + const processQueues: Array> = []; + const events: Array = []; + const continuationRequests: Array = []; + const adapter = makeClaudeAdapterV2({ + instanceId: CLAUDE_DEFAULT_INSTANCE_ID, + settings: DEFAULT_CLAUDE_SETTINGS, + environment: {}, + attachmentsDir, + fileSystem, + idAllocator, + continuationRequests: { + offer: (request) => + Effect.sync(() => { + continuationRequests.push(request); + }), + }, + queryRunner: { + allocateSessionId: Effect.succeed(WAKE_NATIVE_SESSION), + open: () => + Effect.gen(function* () { + const sdkMessages = yield* Queue.unbounded(); + processQueues.push(sdkMessages); + return { + messages: Stream.fromQueue(sdkMessages), + offer: () => Effect.void, + setModel: () => Effect.void, + interrupt: Effect.void, + // End this process stream so openQuery can replace it. + close: Queue.shutdown(sdkMessages), + }; + }), + forkSession: () => Effect.die("unused forkSession"), + assertComplete: Effect.void, + }, + }); + const threadId = ThreadId.make("thread-claude-process-reset"); + const runtime = yield* adapter.openSession({ + threadId, + providerSessionId: ProviderSessionId.make("provider-session-claude-process-reset"), + modelSelection: CLAUDE_TEST_MODEL_SELECTION, + runtimePolicy: CLAUDE_TEST_RUNTIME_POLICY, + }); + const providerThread = yield* runtime.ensureThread({ + threadId, + modelSelection: CLAUDE_TEST_MODEL_SELECTION, + runtimePolicy: CLAUDE_TEST_RUNTIME_POLICY, + }); + yield* runtime.events.pipe( + Stream.runForEach((event) => + Effect.sync(() => { + events.push(event); + }), + ), + Effect.forkScoped, + ); + if (runtime.hasPendingBackgroundWork === undefined) { + return yield* Effect.die("Claude adapter runtime must expose hasPendingBackgroundWork."); + } + const hasPendingBackgroundWork = runtime.hasPendingBackgroundWork; + const now = yield* DateTime.now; + + yield* runtime.startTurn( + makeClaudeTestTurnInput({ + threadId, + providerThread, + now, + attemptId: RunAttemptId.make("attempt-claude-process-reset-a"), + text: "Run the build in the background.", + attachments: [], + }), + ); + assert.equal(processQueues.length, 1); + const firstProcess = processQueues[0]!; + yield* Queue.offer(firstProcess, wakeTaskStarted); + yield* Queue.offer(firstProcess, turnOneResult); + yield* awaitUntil( + () => events.some((event) => event.type === "turn.terminal"), + "first turn terminal", + ); + assert.isTrue(yield* hasPendingBackgroundWork); + + const alternateModel = { + ...CLAUDE_TEST_MODEL_SELECTION, + model: "claude-haiku-4-5-20251001", + } satisfies ModelSelection; + // ProviderTurnStartService marks the thread active before startTurn; + // the process-reset clear must preserve that status. + const activeProviderThread = { + ...providerThread, + status: "active" as const, + } satisfies OrchestrationV2ProviderThread; + yield* runtime.startTurn( + makeClaudeTestTurnInput({ + threadId, + providerThread: activeProviderThread, + now, + attemptId: RunAttemptId.make("attempt-claude-process-reset-b"), + text: "Continue after process restart.", + attachments: [], + providerTurnOrdinal: 2, + modelSelection: alternateModel, + }), + ); + assert.equal(processQueues.length, 2); + + // Process-scoped level resets to empty on CLI (re)start while the + // starting turn's provider thread remains active (not idle). + yield* awaitUntil( + () => + providerThreadRosterEvents(events).some( + (event) => + event.providerThread.status === "active" && + (event.providerThread.pendingBackgroundTasks?.length ?? 0) === 0 && + // Prefer the post-replace clear over the initial empty thread. + event.providerThread.updatedAt !== undefined, + ), + "roster cleared on process replace while remaining active", + ); + // After replace, the in-memory Waiting probe must be false even if a + // late empty-level event was already present before background work. + assert.isFalse(yield* hasPendingBackgroundWork); + const emptyActiveRosterEvents = providerThreadRosterEvents(events).filter( + (event) => + event.providerThread.status === "active" && + (event.providerThread.pendingBackgroundTasks?.length ?? 0) === 0, + ); + assert.isAtLeast(emptyActiveRosterEvents.length, 1); + assert.deepEqual( + emptyActiveRosterEvents.at(-1)?.providerThread.pendingBackgroundTasks ?? [], + [], + ); + assert.equal(emptyActiveRosterEvents.at(-1)?.providerThread.status, "active"); + + // A late notification from the previous process must not wake after + // eligibility was reset with the process. Offer on the new process + // stream (the old queue is shut down). + const secondProcess = processQueues[1]!; + yield* Queue.offer(secondProcess, wakeNotification); + let settleYields = 0; + yield* awaitUntil(() => settleYields++ >= 50, "stale notification settle"); + assert.lengthOf(continuationRequests, 0); + + yield* Queue.offer( + secondProcess, + makeResultFrame({ + uuid: "00000000-0000-4000-8000-000000000602", + result: "Process restart turn finished.", + }), + ); + yield* awaitUntil( + () => events.filter((event) => event.type === "turn.terminal").length === 2, + "second turn terminal", + ); + }).pipe(Effect.provide(Layer.merge(idAllocatorLayer, NodeServices.layer))), + ), + ); + + it.effect("admits only local_bash from a mixed background_tasks_changed snapshot", () => + Effect.scoped( + Effect.gen(function* () { + const harness = yield* makeWakeHarness; + const now = yield* DateTime.now; + const SUBAGENT_TASK_ID = "task-mixed-snapshot-subagent"; + const SUBAGENT_TOOL_USE_ID = "toolu-mixed-snapshot-subagent"; + const mixedSnapshot = claudeSdkFrame({ + type: "system", + subtype: "background_tasks_changed", + tasks: [ + { + task_id: WAKE_TASK_ID, + description: "npm run build", + task_type: "local_bash", + }, + { + task_id: SUBAGENT_TASK_ID, + description: "Agent review", + task_type: "local_agent", + }, + { + task_id: "task-mixed-foreground-agent", + description: "Backgrounded foreground agent", + task_type: "local_agent", + }, + ], + uuid: "00000000-0000-4000-8000-000000000603", + session_id: WAKE_NATIVE_SESSION, + }); + const subagentTaskStarted = claudeSdkFrame({ + type: "system", + subtype: "task_started", + task_id: SUBAGENT_TASK_ID, + tool_use_id: SUBAGENT_TOOL_USE_ID, + description: "Agent review", + subagent_type: "general-purpose", + task_type: "local_agent", + prompt: "Review the change.", + uuid: "00000000-0000-4000-8000-000000000604", + session_id: WAKE_NATIVE_SESSION, + }); + + yield* harness.runtime.startTurn( + makeClaudeTestTurnInput({ + threadId: harness.threadId, + providerThread: harness.providerThread, + now, + attemptId: RunAttemptId.make("attempt-claude-mixed-snapshot"), + text: "Background a bash task and a subagent.", + attachments: [], + }), + ); + yield* Queue.offer(harness.sdkMessages, subagentTaskStarted); + yield* awaitUntil( + () => + harness.events.some( + (event) => + event.type === "subagent.updated" && + event.subagent.nativeTaskRef?.nativeId === SUBAGENT_TASK_ID, + ), + "subagent projected normally", + ); + yield* Queue.offer(harness.sdkMessages, mixedSnapshot); + yield* awaitUntil( + () => + providerThreadRosterEvents(harness.events).some( + (event) => (event.providerThread.pendingBackgroundTasks?.length ?? 0) > 0, + ), + "roster after mixed snapshot", + ); + + const roster = providerThreadRosterEvents(harness.events).at(-1)?.providerThread + .pendingBackgroundTasks; + assert.deepEqual(roster ?? [], [ + { + taskId: WAKE_TASK_ID, + description: "npm run build", + taskType: "local_bash", + }, + ]); + // Subagent lifecycle stays on the subagent path, not the Waiting roster. + assert.isTrue( + harness.events.some( + (event) => + event.type === "subagent.updated" && + event.subagent.nativeTaskRef?.nativeId === SUBAGENT_TASK_ID && + event.subagent.status === "running", + ), + ); + assert.isFalse((roster ?? []).some((task) => task.taskId === SUBAGENT_TASK_ID)); + + yield* Queue.offer(harness.sdkMessages, turnOneResult); + yield* awaitUntil(() => harness.terminalEvents().length === 1, "turn terminal"); + assert.isTrue(yield* harness.hasPendingBackgroundWork); + }).pipe(Effect.provide(Layer.merge(idAllocatorLayer, NodeServices.layer))), + ), + ); + + it.effect( + "preserves buffered local_bash notification classification across model/policy query replacement", + () => + Effect.scoped( + Effect.gen(function* () { + const fileSystem = yield* FileSystem.FileSystem; + const idAllocator = yield* IdAllocatorV2; + const attachmentsDir = yield* fileSystem.makeTempDirectoryScoped({ + prefix: "t3-claude-v2-buffer-replace-", + }); + const processQueues: Array> = []; + const events: Array = []; + const continuationRequests: Array = []; + const adapter = makeClaudeAdapterV2({ + instanceId: CLAUDE_DEFAULT_INSTANCE_ID, + settings: DEFAULT_CLAUDE_SETTINGS, + environment: {}, + attachmentsDir, + fileSystem, + idAllocator, + continuationRequests: { + offer: (request) => + Effect.sync(() => { + continuationRequests.push(request); + }), + }, + queryRunner: { + allocateSessionId: Effect.succeed(WAKE_NATIVE_SESSION), + open: () => + Effect.gen(function* () { + const sdkMessages = yield* Queue.unbounded(); + processQueues.push(sdkMessages); + return { + messages: Stream.fromQueue(sdkMessages), + offer: () => Effect.void, + setModel: () => Effect.void, + interrupt: Effect.void, + close: Queue.shutdown(sdkMessages), + }; + }), + forkSession: () => Effect.die("unused forkSession"), + assertComplete: Effect.void, + }, + }); + const threadId = ThreadId.make("thread-claude-buffer-replace"); + const runtime = yield* adapter.openSession({ + threadId, + providerSessionId: ProviderSessionId.make("provider-session-claude-buffer-replace"), + modelSelection: CLAUDE_TEST_MODEL_SELECTION, + runtimePolicy: CLAUDE_TEST_RUNTIME_POLICY, + }); + const providerThread = yield* runtime.ensureThread({ + threadId, + modelSelection: CLAUDE_TEST_MODEL_SELECTION, + runtimePolicy: CLAUDE_TEST_RUNTIME_POLICY, + }); + yield* runtime.events.pipe( + Stream.runForEach((event) => + Effect.sync(() => { + events.push(event); + }), + ), + Effect.forkScoped, + ); + if (runtime.hasPendingBackgroundWork === undefined) { + return yield* Effect.die( + "Claude adapter runtime must expose hasPendingBackgroundWork.", + ); + } + const hasPendingBackgroundWork = runtime.hasPendingBackgroundWork; + const now = yield* DateTime.now; + + yield* runtime.startTurn( + makeClaudeTestTurnInput({ + threadId, + providerThread, + now, + attemptId: RunAttemptId.make("attempt-claude-buffer-replace-a"), + text: "Run the build in the background.", + attachments: [], + }), + ); + assert.equal(processQueues.length, 1); + const firstProcess = processQueues[0]!; + yield* Queue.offer(firstProcess, wakeTaskStarted); + yield* Queue.offer(firstProcess, turnOneResult); + yield* awaitUntil( + () => events.some((event) => event.type === "turn.terminal"), + "first turn terminal", + ); + assert.isTrue(yield* hasPendingBackgroundWork); + + // Idle completion notification buffers before any continuation runs. + yield* Queue.offer(firstProcess, wakeNotification); + let quietYields = 0; + yield* awaitUntil(() => quietYields++ >= 50, "notification-only quiet window"); + assert.lengthOf(continuationRequests, 0); + + // User turn changes model, replacing the query while the wake buffer + // stays queued for the later provider continuation. + const alternateModel = { + ...CLAUDE_TEST_MODEL_SELECTION, + model: "claude-haiku-4-5-20251001", + } satisfies ModelSelection; + yield* runtime.startTurn( + makeClaudeTestTurnInput({ + threadId, + providerThread: { ...providerThread, status: "active" }, + now, + attemptId: RunAttemptId.make("attempt-claude-buffer-replace-user"), + text: "Switch model while background work completes.", + attachments: [], + providerTurnOrdinal: 2, + modelSelection: alternateModel, + }), + ); + assert.equal(processQueues.length, 2); + const secondProcess = processQueues[1]!; + yield* Queue.offer( + secondProcess, + makeResultFrame({ + uuid: "00000000-0000-4000-8000-000000000701", + result: "User turn finished after model switch.", + }), + ); + yield* awaitUntil( + () => events.filter((event) => event.type === "turn.terminal").length === 2, + "user turn terminal after replace", + ); + // The terminal notification remains buffered for classification, but + // notification-only traffic no longer pins pending work. + assert.isFalse(yield* hasPendingBackgroundWork); + assert.lengthOf(continuationRequests, 0); + + // Continuation drains the buffered local_bash notification with no + // fabricated subagent/node and attributes the wake result text. + yield* Queue.offer(secondProcess, wakeResult); + yield* awaitUntil(() => continuationRequests.length === 1, "continuation after result"); + assert.equal(continuationRequests[0]?.detail, WAKE_SUMMARY); + yield* runtime.startTurn( + makeClaudeTestTurnInput({ + threadId, + providerThread: { ...providerThread, status: "active" }, + now, + attemptId: RunAttemptId.make("attempt-claude-buffer-replace-cont"), + text: "Background task completed.", + attachments: [], + providerTurnOrdinal: 3, + modelSelection: alternateModel, + messageCreatedBy: "agent", + messageCreationSource: "provider", + }), + ); + yield* awaitUntil( + () => events.filter((event) => event.type === "turn.terminal").length === 3, + "continuation terminal after buffered drain", + ); + assert.isTrue( + events.some( + (event) => + event.type === "message.updated" && event.message.text === WAKE_RESULT_TEXT, + ), + ); + assert.isFalse( + events.some( + (event) => + event.type === "subagent.updated" || + (event.type === "node.updated" && event.node.kind === "subagent"), + ), + ); + // Must not re-project the opaque task id as anything but roster history. + assert.isFalse( + events.some( + (event) => + event.type !== "provider_thread.updated" && + JSON.stringify(event).includes(WAKE_TASK_ID), + ), + ); + assert.isFalse(yield* hasPendingBackgroundWork); + }).pipe(Effect.provide(Layer.merge(idAllocatorLayer, NodeServices.layer))), + ), + ); + + it.effect( + "does not opaque-misclassify a buffered subagent notification across model/policy query replacement", + () => + Effect.scoped( + Effect.gen(function* () { + const SUBAGENT_TASK_ID = "task-buffer-replace-subagent"; + const SUBAGENT_TOOL_USE_ID = "toolu-buffer-replace-subagent"; + const SUBAGENT_SUMMARY = "SUB_BUFFER_REPLACE_DONE"; + const subagentTaskStarted = claudeSdkFrame({ + type: "system", + subtype: "task_started", + task_id: SUBAGENT_TASK_ID, + tool_use_id: SUBAGENT_TOOL_USE_ID, + description: "Background research", + subagent_type: "general-purpose", + task_type: "local_agent", + prompt: "Research then return SUB_BUFFER_REPLACE_DONE.", + uuid: "00000000-0000-4000-8000-000000000801", + session_id: WAKE_NATIVE_SESSION, + }); + const subagentNotification = claudeSdkFrame({ + type: "system", + subtype: "task_notification", + task_id: SUBAGENT_TASK_ID, + tool_use_id: SUBAGENT_TOOL_USE_ID, + status: "completed", + output_file: "/tmp/task-buffer-replace-subagent.output", + summary: SUBAGENT_SUMMARY, + uuid: "00000000-0000-4000-8000-000000000802", + session_id: WAKE_NATIVE_SESSION, + }); + const subagentAsyncAck = claudeSdkFrame({ + type: "user", + message: { + role: "user", + content: [ + { + type: "tool_result", + tool_use_id: SUBAGENT_TOOL_USE_ID, + content: [{ type: "text", text: "Async agent launched successfully." }], + }, + ], + }, + parent_tool_use_id: null, + uuid: "00000000-0000-4000-8000-000000000803", + session_id: WAKE_NATIVE_SESSION, + tool_use_result: { + isAsync: true, + status: "async_launched", + agentId: SUBAGENT_TASK_ID, + prompt: "Research then return SUB_BUFFER_REPLACE_DONE.", + }, + }); + + const fileSystem = yield* FileSystem.FileSystem; + const idAllocator = yield* IdAllocatorV2; + const attachmentsDir = yield* fileSystem.makeTempDirectoryScoped({ + prefix: "t3-claude-v2-subagent-buffer-replace-", + }); + const processQueues: Array> = []; + const events: Array = []; + const continuationRequests: Array = []; + const adapter = makeClaudeAdapterV2({ + instanceId: CLAUDE_DEFAULT_INSTANCE_ID, + settings: DEFAULT_CLAUDE_SETTINGS, + environment: {}, + attachmentsDir, + fileSystem, + idAllocator, + continuationRequests: { + offer: (request) => + Effect.sync(() => { + continuationRequests.push(request); + }), + }, + queryRunner: { + allocateSessionId: Effect.succeed(WAKE_NATIVE_SESSION), + open: () => + Effect.gen(function* () { + const sdkMessages = yield* Queue.unbounded(); + processQueues.push(sdkMessages); + return { + messages: Stream.fromQueue(sdkMessages), + offer: () => Effect.void, + setModel: () => Effect.void, + interrupt: Effect.void, + close: Queue.shutdown(sdkMessages), + }; + }), + forkSession: () => Effect.die("unused forkSession"), + assertComplete: Effect.void, + }, + }); + const threadId = ThreadId.make("thread-claude-subagent-buffer-replace"); + const runtime = yield* adapter.openSession({ + threadId, + providerSessionId: ProviderSessionId.make( + "provider-session-claude-subagent-buffer-replace", + ), + modelSelection: CLAUDE_TEST_MODEL_SELECTION, + runtimePolicy: CLAUDE_TEST_RUNTIME_POLICY, + }); + const providerThread = yield* runtime.ensureThread({ + threadId, + modelSelection: CLAUDE_TEST_MODEL_SELECTION, + runtimePolicy: CLAUDE_TEST_RUNTIME_POLICY, + }); + yield* runtime.events.pipe( + Stream.runForEach((event) => + Effect.sync(() => { + events.push(event); + }), + ), + Effect.forkScoped, + ); + if (runtime.hasPendingBackgroundWork === undefined) { + return yield* Effect.die( + "Claude adapter runtime must expose hasPendingBackgroundWork.", + ); + } + const hasPendingBackgroundWork = runtime.hasPendingBackgroundWork; + const subagentEvents = () => + events.filter( + (event): event is Extract => + event.type === "subagent.updated", + ); + const now = yield* DateTime.now; + + yield* runtime.startTurn( + makeClaudeTestTurnInput({ + threadId, + providerThread, + now, + attemptId: RunAttemptId.make("attempt-claude-subagent-buffer-replace-a"), + text: "Spawn a background subagent and stop.", + attachments: [], + }), + ); + assert.equal(processQueues.length, 1); + const firstProcess = processQueues[0]!; + yield* Queue.offer(firstProcess, subagentTaskStarted); + yield* awaitUntil(() => subagentEvents().length >= 1, "subagent node created"); + assert.equal(subagentEvents()[0]?.subagent.status, "running"); + yield* Queue.offer(firstProcess, subagentAsyncAck); + yield* Queue.offer( + firstProcess, + makeResultFrame({ + uuid: "00000000-0000-4000-8000-000000000804", + result: "Spawned the subagent in the background.", + }), + ); + yield* awaitUntil( + () => events.some((event) => event.type === "turn.terminal"), + "first turn terminal", + ); + assert.isTrue(yield* hasPendingBackgroundWork); + + // Session-registered subagent completion buffers; no opaque tombstone. + yield* Queue.offer(firstProcess, subagentNotification); + yield* awaitUntil(() => continuationRequests.length === 1, "continuation after notify"); + assert.equal(continuationRequests[0]?.detail, SUBAGENT_SUMMARY); + + // Model-changing user turn replaces the query while continuation stays + // queued. Process reset must not invent opaque classification for the + // buffered subagent notification. + const alternateModel = { + ...CLAUDE_TEST_MODEL_SELECTION, + model: "claude-haiku-4-5-20251001", + } satisfies ModelSelection; + yield* runtime.startTurn( + makeClaudeTestTurnInput({ + threadId, + providerThread: { ...providerThread, status: "active" }, + now, + attemptId: RunAttemptId.make("attempt-claude-subagent-buffer-replace-user"), + text: "Switch model while the subagent completes.", + attachments: [], + providerTurnOrdinal: 2, + modelSelection: alternateModel, + }), + ); + assert.equal(processQueues.length, 2); + const secondProcess = processQueues[1]!; + yield* Queue.offer( + secondProcess, + makeResultFrame({ + uuid: "00000000-0000-4000-8000-000000000805", + result: "User turn finished after model switch.", + }), + ); + yield* awaitUntil( + () => events.filter((event) => event.type === "turn.terminal").length === 2, + "user turn terminal after replace", + ); + assert.isTrue(yield* hasPendingBackgroundWork); + assert.lengthOf(continuationRequests, 1); + + yield* Queue.offer( + secondProcess, + makeResultFrame({ + uuid: "00000000-0000-4000-8000-000000000806", + result: "The subagent finished with SUB_BUFFER_REPLACE_DONE.", + }), + ); + yield* runtime.startTurn( + makeClaudeTestTurnInput({ + threadId, + providerThread: { ...providerThread, status: "active" }, + now, + attemptId: RunAttemptId.make("attempt-claude-subagent-buffer-replace-cont"), + text: "Background task completed.", + attachments: [], + providerTurnOrdinal: 3, + modelSelection: alternateModel, + messageCreatedBy: "agent", + messageCreationSource: "provider", + }), + ); + yield* awaitUntil( + () => events.filter((event) => event.type === "turn.terminal").length === 3, + "continuation terminal after buffered subagent drain", + ); + + const finalSubagent = subagentEvents().at(-1)?.subagent; + assert.equal(finalSubagent?.status, "completed"); + assert.equal(finalSubagent?.result, SUBAGENT_SUMMARY); + assert.equal(finalSubagent?.runId, subagentEvents()[0]?.subagent.runId); + const subagentNodeEvents = events.filter( + (event): event is Extract => + event.type === "node.updated" && + event.node.kind === "subagent" && + event.node.nativeItemRef?.nativeId === SUBAGENT_TASK_ID, + ); + assert.equal(subagentNodeEvents.at(-1)?.node.status, "completed"); + assert.isFalse(yield* hasPendingBackgroundWork); + }).pipe(Effect.provide(Layer.merge(idAllocatorLayer, NodeServices.layer))), + ), + ); + + it.effect( + "clears process-scoped roster when same-native-thread replacement open fails after close", + () => + Effect.scoped( + Effect.gen(function* () { + const fileSystem = yield* FileSystem.FileSystem; + const idAllocator = yield* IdAllocatorV2; + const attachmentsDir = yield* fileSystem.makeTempDirectoryScoped({ + prefix: "t3-claude-v2-replace-open-fail-", + }); + let openCount = 0; + const processQueues: Array> = []; + const events: Array = []; + const adapter = makeClaudeAdapterV2({ + instanceId: CLAUDE_DEFAULT_INSTANCE_ID, + settings: DEFAULT_CLAUDE_SETTINGS, + environment: {}, + attachmentsDir, + fileSystem, + idAllocator, + continuationRequests: { + offer: () => Effect.void, + }, + queryRunner: { + allocateSessionId: Effect.succeed(WAKE_NATIVE_SESSION), + open: () => { + openCount += 1; + if (openCount === 2) { + return Effect.fail( + new ClaudeAgentSdkQueryRunnerError({ + method: "open", + cause: "forced replacement open failure", + }), + ); + } + return Effect.gen(function* () { + const sdkMessages = yield* Queue.unbounded(); + processQueues.push(sdkMessages); + return { + messages: Stream.fromQueue(sdkMessages), + offer: () => Effect.void, + setModel: () => Effect.void, + interrupt: Effect.void, + close: Queue.shutdown(sdkMessages), + }; + }); + }, + forkSession: () => Effect.die("unused forkSession"), + assertComplete: Effect.void, + }, + }); + const threadId = ThreadId.make("thread-claude-replace-open-fail"); + const runtime = yield* adapter.openSession({ + threadId, + providerSessionId: ProviderSessionId.make("provider-session-claude-replace-open-fail"), + modelSelection: CLAUDE_TEST_MODEL_SELECTION, + runtimePolicy: CLAUDE_TEST_RUNTIME_POLICY, + }); + const providerThread = yield* runtime.ensureThread({ + threadId, + modelSelection: CLAUDE_TEST_MODEL_SELECTION, + runtimePolicy: CLAUDE_TEST_RUNTIME_POLICY, + }); + yield* runtime.events.pipe( + Stream.runForEach((event) => + Effect.sync(() => { + events.push(event); + }), + ), + Effect.forkScoped, + ); + if (runtime.hasPendingBackgroundWork === undefined) { + return yield* Effect.die( + "Claude adapter runtime must expose hasPendingBackgroundWork.", + ); + } + const hasPendingBackgroundWork = runtime.hasPendingBackgroundWork; + const now = yield* DateTime.now; + + yield* runtime.startTurn( + makeClaudeTestTurnInput({ + threadId, + providerThread, + now, + attemptId: RunAttemptId.make("attempt-claude-replace-open-fail-a"), + text: "Run the build in the background.", + attachments: [], + }), + ); + assert.equal(processQueues.length, 1); + yield* Queue.offer(processQueues[0]!, wakeTaskStarted); + yield* Queue.offer(processQueues[0]!, turnOneResult); + yield* awaitUntil( + () => events.some((event) => event.type === "turn.terminal"), + "first turn terminal", + ); + assert.isTrue(yield* hasPendingBackgroundWork); + + const alternateModel = { + ...CLAUDE_TEST_MODEL_SELECTION, + model: "claude-haiku-4-5-20251001", + } satisfies ModelSelection; + const failedStart = yield* runtime + .startTurn( + makeClaudeTestTurnInput({ + threadId, + providerThread: { ...providerThread, status: "active" }, + now, + attemptId: RunAttemptId.make("attempt-claude-replace-open-fail-b"), + text: "Replace process but fail open.", + attachments: [], + providerTurnOrdinal: 2, + modelSelection: alternateModel, + }), + ) + .pipe(Effect.exit); + assert.isTrue(Exit.isFailure(failedStart)); + // Old process was closed before the failed open: roster must not stick. + yield* awaitUntil( + () => + providerThreadRosterEvents(events).some( + (event) => + event.providerThread.status === "idle" && + (event.providerThread.pendingBackgroundTasks?.length ?? 0) === 0, + ), + "roster cleared after failed same-thread replacement open", + ); + assert.isFalse(yield* hasPendingBackgroundWork); + }).pipe(Effect.provide(Layer.merge(idAllocatorLayer, NodeServices.layer))), + ), + ); + + it.effect( + "clears buffered wake and continuation state when same-native-thread replacement open fails", + () => + Effect.scoped( + Effect.gen(function* () { + const fileSystem = yield* FileSystem.FileSystem; + const idAllocator = yield* IdAllocatorV2; + const attachmentsDir = yield* fileSystem.makeTempDirectoryScoped({ + prefix: "t3-claude-v2-replace-open-fail-wake-", + }); + let openCount = 0; + const processQueues: Array> = []; + const events: Array = []; + const continuationRequests: Array = []; + const adapter = makeClaudeAdapterV2({ + instanceId: CLAUDE_DEFAULT_INSTANCE_ID, + settings: DEFAULT_CLAUDE_SETTINGS, + environment: {}, + attachmentsDir, + fileSystem, + idAllocator, + continuationRequests: { + offer: (request) => + Effect.sync(() => { + continuationRequests.push(request); + }), + }, + queryRunner: { + allocateSessionId: Effect.succeed(WAKE_NATIVE_SESSION), + open: () => { + openCount += 1; + if (openCount === 2) { + return Effect.fail( + new ClaudeAgentSdkQueryRunnerError({ + method: "open", + cause: "forced replacement open failure", + }), + ); + } + return Effect.gen(function* () { + const sdkMessages = yield* Queue.unbounded(); + processQueues.push(sdkMessages); + return { + messages: Stream.fromQueue(sdkMessages), + offer: () => Effect.void, + setModel: () => Effect.void, + interrupt: Effect.void, + close: Queue.shutdown(sdkMessages), + }; + }); + }, + forkSession: () => Effect.die("unused forkSession"), + assertComplete: Effect.void, + }, + }); + const threadId = ThreadId.make("thread-claude-replace-open-fail-wake"); + const runtime = yield* adapter.openSession({ + threadId, + providerSessionId: ProviderSessionId.make( + "provider-session-claude-replace-open-fail-wake", + ), + modelSelection: CLAUDE_TEST_MODEL_SELECTION, + runtimePolicy: CLAUDE_TEST_RUNTIME_POLICY, + }); + const providerThread = yield* runtime.ensureThread({ + threadId, + modelSelection: CLAUDE_TEST_MODEL_SELECTION, + runtimePolicy: CLAUDE_TEST_RUNTIME_POLICY, + }); + yield* runtime.events.pipe( + Stream.runForEach((event) => + Effect.sync(() => { + events.push(event); + }), + ), + Effect.forkScoped, + ); + if (runtime.hasPendingBackgroundWork === undefined) { + return yield* Effect.die( + "Claude adapter runtime must expose hasPendingBackgroundWork.", + ); + } + const hasPendingBackgroundWork = runtime.hasPendingBackgroundWork; + const now = yield* DateTime.now; + + yield* runtime.startTurn( + makeClaudeTestTurnInput({ + threadId, + providerThread, + now, + attemptId: RunAttemptId.make("attempt-claude-replace-open-fail-wake-a"), + text: "Run the build in the background.", + attachments: [], + }), + ); + yield* Queue.offer(processQueues[0]!, wakeTaskStarted); + yield* Queue.offer(processQueues[0]!, turnOneResult); + yield* awaitUntil( + () => events.filter((event) => event.type === "turn.terminal").length === 1, + "first turn terminal", + ); + yield* Queue.offer(processQueues[0]!, wakeNotification); + yield* Queue.offer(processQueues[0]!, wakeAssistant); + yield* awaitUntil(() => continuationRequests.length === 1, "first continuation request"); + assert.isTrue(yield* hasPendingBackgroundWork); + + const alternateModel = { + ...CLAUDE_TEST_MODEL_SELECTION, + model: "claude-haiku-4-5-20251001", + } satisfies ModelSelection; + const failedStart = yield* runtime + .startTurn( + makeClaudeTestTurnInput({ + threadId, + providerThread: { ...providerThread, status: "active" }, + now, + attemptId: RunAttemptId.make("attempt-claude-replace-open-fail-wake-b"), + text: "Replace process but fail open.", + attachments: [], + providerTurnOrdinal: 2, + modelSelection: alternateModel, + }), + ) + .pipe(Effect.exit); + assert.isTrue(Exit.isFailure(failedStart)); + assert.isFalse(yield* hasPendingBackgroundWork); + + yield* runtime.startTurn( + makeClaudeTestTurnInput({ + threadId, + providerThread: { ...providerThread, status: "active" }, + now, + attemptId: RunAttemptId.make("attempt-claude-replace-open-fail-wake-c"), + text: "Retry after the failed replacement.", + attachments: [], + providerTurnOrdinal: 2, + modelSelection: alternateModel, + }), + ); + const retryProcess = processQueues[1]!; + const retryTaskId = "task-wake-build-after-retry"; + yield* Queue.offer( + retryProcess, + claudeSdkFrame({ + type: "system", + subtype: "task_started", + task_id: retryTaskId, + description: "npm run build after retry", + task_type: "local_bash", + uuid: "00000000-0000-4000-8000-000000000901", + session_id: WAKE_NATIVE_SESSION, + }), + ); + yield* Queue.offer( + retryProcess, + makeResultFrame({ + uuid: "00000000-0000-4000-8000-000000000902", + result: "Kicked off the retry build in the background.", + }), + ); + yield* awaitUntil( + () => events.filter((event) => event.type === "turn.terminal").length === 2, + "retry turn terminal", + ); + yield* Queue.offer( + retryProcess, + claudeSdkFrame({ + type: "system", + subtype: "task_notification", + task_id: retryTaskId, + status: "completed", + output_file: "/tmp/task-wake-build-after-retry.log", + summary: "Retry build completed successfully", + uuid: "00000000-0000-4000-8000-000000000903", + session_id: WAKE_NATIVE_SESSION, + }), + ); + yield* Queue.offer( + retryProcess, + claudeSdkFrame({ + type: "assistant", + message: { + role: "assistant", + content: [{ type: "text", text: "The retry build has finished." }], + }, + parent_tool_use_id: null, + uuid: "00000000-0000-4000-8000-000000000904", + session_id: WAKE_NATIVE_SESSION, + }), + ); + yield* awaitUntil( + () => continuationRequests.length === 2, + "continuation request after retry", + ); + assert.equal(continuationRequests[1]?.detail, "Retry build completed successfully"); + }).pipe(Effect.provide(Layer.merge(idAllocatorLayer, NodeServices.layer))), + ), + ); + + it.effect("does not invent process reset state on a first-ever failed open", () => + Effect.scoped( + Effect.gen(function* () { + const fileSystem = yield* FileSystem.FileSystem; + const idAllocator = yield* IdAllocatorV2; + const attachmentsDir = yield* fileSystem.makeTempDirectoryScoped({ + prefix: "t3-claude-v2-first-open-fail-", + }); + const events: Array = []; + const adapter = makeClaudeAdapterV2({ + instanceId: CLAUDE_DEFAULT_INSTANCE_ID, + settings: DEFAULT_CLAUDE_SETTINGS, + environment: {}, + attachmentsDir, + fileSystem, + idAllocator, + continuationRequests: { + offer: () => Effect.void, + }, + queryRunner: { + allocateSessionId: Effect.succeed(WAKE_NATIVE_SESSION), + open: () => + Effect.fail( + new ClaudeAgentSdkQueryRunnerError({ + method: "open", + cause: "forced first open failure", + }), + ), + forkSession: () => Effect.die("unused forkSession"), + assertComplete: Effect.void, + }, + }); + const threadId = ThreadId.make("thread-claude-first-open-fail"); + const runtime = yield* adapter.openSession({ + threadId, + providerSessionId: ProviderSessionId.make("provider-session-claude-first-open-fail"), + modelSelection: CLAUDE_TEST_MODEL_SELECTION, + runtimePolicy: CLAUDE_TEST_RUNTIME_POLICY, + }); + const providerThread = yield* runtime.ensureThread({ + threadId, + modelSelection: CLAUDE_TEST_MODEL_SELECTION, + runtimePolicy: CLAUDE_TEST_RUNTIME_POLICY, + }); + yield* runtime.events.pipe( + Stream.runForEach((event) => + Effect.sync(() => { + events.push(event); + }), + ), + Effect.forkScoped, + ); + const now = yield* DateTime.now; + const failedStart = yield* runtime + .startTurn( + makeClaudeTestTurnInput({ + threadId, + providerThread, + now, + attemptId: RunAttemptId.make("attempt-claude-first-open-fail"), + text: "First open fails.", + attachments: [], + }), + ) + .pipe(Effect.exit); + assert.isTrue(Exit.isFailure(failedStart)); + // No live process ever existed: do not emit a fabricated empty roster. + assert.lengthOf(providerThreadRosterEvents(events), 0); + }).pipe(Effect.provide(Layer.merge(idAllocatorLayer, NodeServices.layer))), + ), + ); }); describe("ClaudeAdapterV2 query message stream", () => { diff --git a/apps/server/src/orchestration-v2/Adapters/ClaudeAdapterV2.testkit.test.ts b/apps/server/src/orchestration-v2/Adapters/ClaudeAdapterV2.testkit.test.ts index cfe3009548db..a246a74b3ad5 100644 --- a/apps/server/src/orchestration-v2/Adapters/ClaudeAdapterV2.testkit.test.ts +++ b/apps/server/src/orchestration-v2/Adapters/ClaudeAdapterV2.testkit.test.ts @@ -8,6 +8,7 @@ import { import * as Cause from "effect/Cause"; import * as Effect from "effect/Effect"; import * as Exit from "effect/Exit"; +import * as Fiber from "effect/Fiber"; import * as Schema from "effect/Schema"; import * as Stream from "effect/Stream"; import { vi } from "vite-plus/test"; @@ -20,14 +21,19 @@ import { } from "./ClaudeAdapterV2.ts"; import { CLAUDE_AGENT_SDK_REPLAY_PROTOCOL, + ClaudeReplayIncompleteError, ClaudeReplayRuntimeExitError, + ClaudeReplayUnexpectedOutboundError, makeReplayQueryRunner, recordInterruptedClaudeQuery, recordMessagesUntilTurnResultAndFinalize, } from "./ClaudeAdapterV2.testkit.ts"; +import { makeProviderReplayGate } from "../testkit/ProviderReplayGate.testkit.ts"; const isClaudeAgentSdkQueryRunnerError = Schema.is(ClaudeAgentSdkQueryRunnerError); +const isClaudeReplayIncompleteError = Schema.is(ClaudeReplayIncompleteError); const isClaudeReplayRuntimeExitError = Schema.is(ClaudeReplayRuntimeExitError); +const isClaudeReplayUnexpectedOutboundError = Schema.is(ClaudeReplayUnexpectedOutboundError); const claudeSdkMock = vi.hoisted(() => { const close = vi.fn(); @@ -63,7 +69,284 @@ vi.mock("@anthropic-ai/claude-agent-sdk", () => ({ query: claudeSdkMock.query, })); +function makeGatedReplaySession() { + const options = { + model: "claude-sonnet-4-6", + tools: [], + permissionMode: "default", + sessionId: "session-replay-gate", + } satisfies ClaudeAgentSdkQueryOptions; + const label = "background_tasks_changed:empty"; + const replayGate = makeProviderReplayGate([label]); + const runner = makeReplayQueryRunner( + { + provider: CLAUDE_PROVIDER, + protocol: CLAUDE_AGENT_SDK_REPLAY_PROTOCOL, + version: "test", + scenario: "labeled-inbound-replay-gate", + entries: [ + { + type: "expect_outbound", + frame: { + type: "query.open", + options, + }, + }, + { + type: "emit_inbound", + label, + frame: { + type: "system", + subtype: "background_tasks_changed", + tasks: [], + uuid: "replay-gate-message", + session_id: options.sessionId, + }, + }, + { + type: "runtime_exit", + status: "success", + }, + ], + }, + { replayGate }, + ); + const session = runner.open({ + options, + threadId: ThreadId.make("thread-replay-gate"), + providerSessionId: ProviderSessionId.make("provider-session-replay-gate"), + }); + return { label, replayGate, runner, session }; +} + +const yieldToReplayStream = Effect.promise( + () => + new Promise((resolve) => { + setImmediate(resolve); + }), +); + describe("ClaudeAdapterV2 replay testkit", () => { + it.effect("holds a labeled inbound frame until its replay gate is released", () => + Effect.gen(function* () { + const { label, replayGate, runner, session } = makeGatedReplaySession(); + const streamFiber = yield* Stream.runDrain(session.messages).pipe(Effect.forkChild); + + yield* yieldToReplayStream; + assert.isTrue(replayGate.hasReached(label)); + assert.isUndefined(streamFiber.pollUnsafe()); + + assert.isTrue(replayGate.release(label)); + yield* Fiber.join(streamFiber); + runner.assertComplete(); + }), + ); + + it.effect("releases a replay gate when its stream consumer is interrupted", () => + Effect.gen(function* () { + const { label, replayGate, session } = makeGatedReplaySession(); + const streamFiber = yield* Stream.runDrain(session.messages).pipe(Effect.forkChild); + + yield* yieldToReplayStream; + assert.isTrue(replayGate.hasReached(label)); + + yield* Fiber.interrupt(streamFiber); + assert.isFalse(replayGate.release(label)); + }), + ); + + it.effect("interrupts a delayed inbound frame without emitting or advancing it", () => + Effect.gen(function* () { + const options = { + model: "claude-sonnet-4-6", + tools: [], + permissionMode: "default", + sessionId: "session-replay-delayed-interrupt", + } satisfies ClaudeAgentSdkQueryOptions; + const label = "delayed-inbound"; + const replayGate = makeProviderReplayGate([label]); + const runner = makeReplayQueryRunner( + { + provider: CLAUDE_PROVIDER, + protocol: CLAUDE_AGENT_SDK_REPLAY_PROTOCOL, + version: "test", + scenario: "delayed-inbound-interruption", + entries: [ + { + type: "expect_outbound", + frame: { + type: "query.open", + options, + }, + }, + { + type: "emit_inbound", + label, + afterMs: 30_000, + frame: { + type: "assistant", + message: { content: [] }, + parent_tool_use_id: null, + session_id: options.sessionId, + uuid: "delayed-inbound-message", + }, + }, + { + type: "runtime_exit", + status: "success", + }, + ], + }, + { replayGate }, + ); + const session = runner.open({ + options, + threadId: ThreadId.make("thread-replay-delayed-interrupt"), + providerSessionId: ProviderSessionId.make("provider-session-replay-delayed-interrupt"), + }); + const emittedMessages: Array = []; + const streamFiber = yield* session.messages.pipe( + Stream.tap((message) => Effect.sync(() => emittedMessages.push(message))), + Stream.runDrain, + Effect.forkChild, + ); + + yield* yieldToReplayStream; + assert.isTrue(replayGate.hasReached(label)); + assert.isTrue(replayGate.release(label)); + yield* yieldToReplayStream; + + yield* Fiber.interrupt(streamFiber); + + assert.deepEqual(emittedMessages, []); + const incomplete = assert.throws(() => runner.assertComplete()); + assert.isTrue(isClaudeReplayIncompleteError(incomplete)); + if (isClaudeReplayIncompleteError(incomplete)) { + assert.equal(incomplete.cursor, 1); + assert.equal(incomplete.remaining, 2); + } + }), + ); + + it.effect("wakes a gated replay stream when an outbound frame mismatches", () => + Effect.gen(function* () { + const { label, replayGate, session } = makeGatedReplaySession(); + const emittedMessages: Array = []; + const streamFiber = yield* session.messages.pipe( + Stream.tap((message) => Effect.sync(() => emittedMessages.push(message))), + Stream.runDrain, + Effect.exit, + Effect.forkChild, + ); + + yield* yieldToReplayStream; + assert.isTrue(replayGate.hasReached(label)); + + const offerExit = yield* Effect.exit( + session.offer(makeClaudeUserMessage({ text: "unexpected gated prompt" })), + ); + assert.isTrue(Exit.isFailure(offerExit)); + + const streamExit = yield* Fiber.join(streamFiber); + assert.isTrue(Exit.isFailure(streamExit)); + if (Exit.isFailure(streamExit)) { + const error = Cause.squash(streamExit.cause); + assert.isTrue(isClaudeAgentSdkQueryRunnerError(error)); + if (isClaudeAgentSdkQueryRunnerError(error)) { + assert.isTrue(isClaudeReplayUnexpectedOutboundError(error.cause)); + } + } + assert.deepEqual(emittedMessages, []); + }), + ); + + it.effect("surfaces an outbound mismatch during a delay without inbound callbacks", () => + Effect.gen(function* () { + const stableOptions = { + model: "claude-sonnet-4-6", + tools: [], + permissionMode: "default", + sessionId: "session-replay-gated-permission", + } satisfies ClaudeAgentSdkQueryOptions; + const canUseTool = vi.fn>( + async (_toolName, input, options) => ({ + behavior: "allow", + updatedInput: input, + toolUseID: options.toolUseID, + }), + ); + const label = "permission_request"; + const replayGate = makeProviderReplayGate([label]); + const runner = makeReplayQueryRunner( + { + provider: CLAUDE_PROVIDER, + protocol: CLAUDE_AGENT_SDK_REPLAY_PROTOCOL, + version: "test", + scenario: "gated-permission-stops-after-failure", + entries: [ + { + type: "expect_outbound", + frame: { + type: "query.open", + options: stableOptions, + }, + }, + { + type: "emit_inbound", + label, + afterMs: 30_000, + frame: { + type: "permission.request", + toolName: "Read", + input: { file_path: "/workspace/gated.ts" }, + options: { toolUseID: "tool-use-gated" }, + }, + }, + { + type: "runtime_exit", + status: "success", + }, + ], + }, + { replayGate }, + ); + const session = runner.open({ + options: { ...stableOptions, canUseTool }, + threadId: ThreadId.make("thread-replay-gated-permission"), + providerSessionId: ProviderSessionId.make("provider-session-replay-gated-permission"), + }); + const emittedMessages: Array = []; + const streamFiber = yield* session.messages.pipe( + Stream.tap((message) => Effect.sync(() => emittedMessages.push(message))), + Stream.runDrain, + Effect.exit, + Effect.forkChild, + ); + + yield* yieldToReplayStream; + assert.isTrue(replayGate.hasReached(label)); + assert.isTrue(replayGate.release(label)); + yield* yieldToReplayStream; + + const offerExit = yield* Effect.exit( + session.offer(makeClaudeUserMessage({ text: "unexpected gated prompt" })), + ); + assert.isTrue(Exit.isFailure(offerExit)); + + const streamExit = yield* Fiber.join(streamFiber); + assert.isTrue(Exit.isFailure(streamExit)); + if (Exit.isFailure(streamExit)) { + const error = Cause.squash(streamExit.cause); + assert.isTrue(isClaudeAgentSdkQueryRunnerError(error)); + if (isClaudeAgentSdkQueryRunnerError(error)) { + assert.isTrue(isClaudeReplayUnexpectedOutboundError(error.cause)); + } + } + assert.deepEqual(emittedMessages, []); + assert.equal(canUseTool.mock.calls.length, 0); + }), + ); + it.effect("wakes a replay stream waiting on an outbound frame when that frame mismatches", () => Effect.gen(function* () { const options = { @@ -118,6 +401,146 @@ describe("ClaudeAdapterV2 replay testkit", () => { }), ); + it.effect("interrupts a replay stream waiting on its next outbound frame", () => + Effect.gen(function* () { + const options = { + model: "claude-sonnet-4-6", + tools: [], + permissionMode: "default", + sessionId: "session-replay-outbound-interrupt", + } satisfies ClaudeAgentSdkQueryOptions; + const runner = makeReplayQueryRunner({ + provider: CLAUDE_PROVIDER, + protocol: CLAUDE_AGENT_SDK_REPLAY_PROTOCOL, + version: "test", + scenario: "outbound-wait-interruption", + entries: [ + { + type: "expect_outbound", + frame: { + type: "query.open", + options, + }, + }, + { + type: "expect_outbound", + frame: { + type: "prompt.offer", + message: makeClaudeUserMessage({ text: "next prompt" }), + }, + }, + ], + }); + const session = runner.open({ + options, + threadId: ThreadId.make("thread-replay-outbound-interrupt"), + providerSessionId: ProviderSessionId.make("provider-session-replay-outbound-interrupt"), + }); + const streamFiber = yield* Stream.runDrain(session.messages).pipe(Effect.forkChild); + + yield* yieldToReplayStream; + assert.isUndefined(streamFiber.pollUnsafe()); + + yield* Fiber.interrupt(streamFiber); + + const incomplete = assert.throws(() => runner.assertComplete()); + assert.isTrue(isClaudeReplayIncompleteError(incomplete)); + if (isClaudeReplayIncompleteError(incomplete)) { + assert.equal(incomplete.cursor, 1); + assert.equal(incomplete.remaining, 1); + } + }), + ); + + it.effect("aborts a pending permission callback when replay fails", () => + Effect.gen(function* () { + const stableOptions = { + model: "claude-sonnet-4-6", + tools: [], + permissionMode: "default", + sessionId: "session-replay-permission-abort", + } satisfies ClaudeAgentSdkQueryOptions; + let callbackSignal: AbortSignal | undefined; + let notifyCallbackStarted = () => {}; + const callbackStarted = new Promise((resolve) => { + notifyCallbackStarted = resolve; + }); + const canUseTool = vi.fn>( + async (_toolName, _input, options) => { + callbackSignal = options.signal; + notifyCallbackStarted(); + await new Promise((_resolve, reject) => { + const rejectOnAbort = () => reject(new Error("permission callback aborted")); + options.signal.addEventListener("abort", rejectOnAbort, { once: true }); + if (options.signal.aborted) { + rejectOnAbort(); + } + }); + return { + behavior: "allow", + updatedInput: _input, + toolUseID: options.toolUseID, + }; + }, + ); + const runner = makeReplayQueryRunner({ + provider: CLAUDE_PROVIDER, + protocol: CLAUDE_AGENT_SDK_REPLAY_PROTOCOL, + version: "test", + scenario: "pending-permission-aborts-on-replay-failure", + entries: [ + { + type: "expect_outbound", + frame: { + type: "query.open", + options: stableOptions, + }, + }, + { + type: "emit_inbound", + frame: { + type: "permission.request", + toolName: "Read", + input: { file_path: "/workspace/pending.ts" }, + options: { toolUseID: "tool-use-pending" }, + }, + }, + { + type: "runtime_exit", + status: "success", + }, + ], + }); + const session = runner.open({ + options: { ...stableOptions, canUseTool }, + threadId: ThreadId.make("thread-replay-permission-abort"), + providerSessionId: ProviderSessionId.make("provider-session-replay-permission-abort"), + }); + const streamFiber = yield* Stream.runDrain(session.messages).pipe( + Effect.exit, + Effect.forkChild, + ); + + yield* Effect.promise(() => callbackStarted); + const offerExit = yield* Effect.exit( + session.offer(makeClaudeUserMessage({ text: "unexpected prompt" })), + ); + + assert.isTrue(Exit.isFailure(offerExit)); + const streamExit = yield* Fiber.join(streamFiber); + assert.isTrue(Exit.isFailure(streamExit)); + if (Exit.isFailure(streamExit)) { + const error = Cause.squash(streamExit.cause); + assert.isTrue(isClaudeAgentSdkQueryRunnerError(error)); + if (isClaudeAgentSdkQueryRunnerError(error)) { + assert.isTrue(isClaudeReplayUnexpectedOutboundError(error.cause)); + } + } + assert.equal(canUseTool.mock.calls.length, 1); + assert.isTrue(callbackSignal?.aborted); + }), + ); + it.effect("keeps permission callbacks scoped to the query that opened the stream", () => Effect.gen(function* () { const firstCanUseTool = vi.fn>( diff --git a/apps/server/src/orchestration-v2/Adapters/ClaudeAdapterV2.testkit.ts b/apps/server/src/orchestration-v2/Adapters/ClaudeAdapterV2.testkit.ts index dd791248006c..76cb486d391f 100644 --- a/apps/server/src/orchestration-v2/Adapters/ClaudeAdapterV2.testkit.ts +++ b/apps/server/src/orchestration-v2/Adapters/ClaudeAdapterV2.testkit.ts @@ -14,8 +14,10 @@ import { type ProviderApprovalDecision, type ProviderReplayTranscript, } from "@t3tools/contracts"; +import * as Cause from "effect/Cause"; import * as Duration from "effect/Duration"; import * as Effect from "effect/Effect"; +import * as Exit from "effect/Exit"; import * as Layer from "effect/Layer"; import * as Schema from "effect/Schema"; import * as Stream from "effect/Stream"; @@ -44,6 +46,7 @@ import { makeReplayServerConfig, type OrchestratorV2ProviderReplayHarness, } from "../testkit/ProviderReplayHarness.ts"; +import type { ProviderReplayGate } from "../testkit/ProviderReplayGate.testkit.ts"; export const CLAUDE_AGENT_SDK_REPLAY_PROTOCOL = "claude-agent-sdk.query" as const; @@ -333,11 +336,11 @@ function makeClaudePermissionResponseFrame( function permissionRequestOptionsFromFrame( frame: ClaudePermissionRequestFrame, + signal: AbortSignal, ): Parameters[2] { - const abortController = new AbortController(); const options = frame.options; return { - signal: abortController.signal, + signal, ...(options.suggestions === undefined ? {} : { suggestions: options.suggestions }), ...(options.blockedPath === undefined ? {} : { blockedPath: options.blockedPath }), ...(options.decisionReason === undefined ? {} : { decisionReason: options.decisionReason }), @@ -363,6 +366,35 @@ function makeCursorSignal(): { return { promise, resolve }; } +function waitForCursorAdvance(promise: Promise, signal: AbortSignal): Promise { + if (signal.aborted) { + return Promise.resolve(); + } + return new Promise((resolve) => { + let waiting = true; + const finish = () => { + if (!waiting) { + return; + } + waiting = false; + signal.removeEventListener("abort", finish); + resolve(); + }; + signal.addEventListener("abort", finish, { once: true }); + void promise.then(finish); + if (signal.aborted) { + finish(); + } + }); +} + +async function waitForReplayDelay(afterMs: number, signal: AbortSignal): Promise { + const exit = await Effect.runPromiseExit(Effect.sleep(Duration.millis(afterMs)), { signal }); + if (Exit.isFailure(exit) && !Cause.hasInterruptsOnly(exit.cause)) { + throw Cause.squash(exit.cause); + } +} + function stableClaudeQueryOptions(options: ClaudeAgentSdkQueryOptions): ClaudeAgentSdkQueryOptions { const stable = { model: options.model, @@ -417,17 +449,33 @@ function makeClaudeSessionForkFrame( export function makeReplayQueryRunner( transcript: ClaudeAgentSdkReplayTranscript, + replayOptions: { readonly replayGate?: ProviderReplayGate } = {}, ): ClaudeQueryRunner { let cursor = 0; let failure: ClaudeAgentSdkReplayError | null = null; let cursorAdvanced = makeCursorSignal(); + const iteratorAbortControllers = new Set(); + + const abortReplayIterators = () => { + for (const abortController of iteratorAbortControllers) { + abortController.abort(); + } + }; const fail = (error: ClaudeAgentSdkReplayError): never => { failure = error; + abortReplayIterators(); + replayOptions.replayGate?.releaseAll(); cursorAdvanced.resolve(); throw error; }; + const throwIfFailed = () => { + if (failure !== null) { + throw failure; + } + }; + const advance = () => { cursor += 1; const signal = cursorAdvanced; @@ -437,11 +485,10 @@ export function makeReplayQueryRunner( async function* replayMessages( options: ClaudeAgentSdkQueryOptions, + signal: AbortSignal, ): AsyncGenerator { while (true) { - if (failure !== null) { - throw failure; - } + throwIfFailed(); const entry = transcript.entries[cursor]; if (entry === undefined) { @@ -449,6 +496,20 @@ export function makeReplayQueryRunner( } if (entry.type === "emit_inbound") { + if (replayOptions.replayGate !== undefined) { + await replayOptions.replayGate.beforeEmit(entry.label, signal); + throwIfFailed(); + if (signal.aborted) { + return; + } + } + if (entry.afterMs !== undefined && entry.afterMs > 0) { + await waitForReplayDelay(entry.afterMs, signal); + throwIfFailed(); + if (signal.aborted) { + return; + } + } if (isClaudePermissionRequestFrame(entry.frame)) { const request = entry.frame; const invokeCanUseTool = options.canUseTool; @@ -459,15 +520,27 @@ export function makeReplayQueryRunner( expectedType: "permission.request", actual: request, }); - failure = error; - throw error; + return fail(error); } advance(); - const result = await invokeCanUseTool( - request.toolName, - request.input, - permissionRequestOptionsFromFrame(request), - ); + let result: PermissionResult | null; + try { + result = await invokeCanUseTool( + request.toolName, + request.input, + permissionRequestOptionsFromFrame(request, signal), + ); + } catch (cause) { + throwIfFailed(); + if (signal.aborted) { + return; + } + throw cause; + } + throwIfFailed(); + if (signal.aborted) { + return; + } if (result === null) { const error = new ClaudeReplayUnexpectedOutboundError({ scenario: transcript.scenario, @@ -475,8 +548,7 @@ export function makeReplayQueryRunner( expectedType: "permission.response", actual: null, }); - failure = error; - throw error; + return fail(error); } assertNextOutboundFrame(makeClaudePermissionResponseFrame(result)); continue; @@ -502,13 +574,49 @@ export function makeReplayQueryRunner( } if (entry.type === "expect_outbound") { - const signal = cursorAdvanced; - await signal.promise; + await waitForCursorAdvance(cursorAdvanced.promise, signal); + throwIfFailed(); + if (signal.aborted) { + return; + } continue; } } } + const replayMessagesWithGateCleanup = ( + options: ClaudeAgentSdkQueryOptions, + ): AsyncIterable => ({ + [Symbol.asyncIterator]: () => { + const abortController = new AbortController(); + iteratorAbortControllers.add(abortController); + const iterator = replayMessages(options, abortController.signal); + return { + next: async () => { + try { + const result = await iterator.next(); + if (result.done) { + iteratorAbortControllers.delete(abortController); + } + return result; + } catch (cause) { + iteratorAbortControllers.delete(abortController); + throw cause; + } + }, + return: async () => { + abortController.abort(); + replayOptions.replayGate?.releaseAll(); + try { + return await iterator.return(); + } finally { + iteratorAbortControllers.delete(abortController); + } + }, + }; + }, + }); + const assertNextOutboundFrame = (actual: ClaudeOutboundFrame) => { if (failure !== null) { throw failure; @@ -602,7 +710,7 @@ export function makeReplayQueryRunner( open: (input) => { assertNextOutboundFrame(makeClaudeQueryOpenFrame(input)); return { - messages: Stream.fromAsyncIterable(replayMessages(input.options), (cause) => + messages: Stream.fromAsyncIterable(replayMessagesWithGateCleanup(input.options), (cause) => replayQueryRunnerError(transcript, cause), ), offer: (message) => @@ -631,6 +739,7 @@ export function makeReplayQueryRunner( throw failure; } if (cursor !== transcript.entries.length) { + replayOptions.replayGate?.releaseAll(); throw new ClaudeReplayIncompleteError({ scenario: transcript.scenario, cursor, @@ -680,8 +789,11 @@ function replayQueryRunnerError( } const makeClaudeAgentSdkReplayQueryRunner = Effect.fn("ClaudeAgentSdkReplayQueryRunner.layer")( - function* (transcript: ClaudeAgentSdkReplayTranscript) { - const queryRunner = makeReplayQueryRunner(transcript); + function* ( + transcript: ClaudeAgentSdkReplayTranscript, + options: { readonly replayGate?: ProviderReplayGate } = {}, + ) { + const queryRunner = makeReplayQueryRunner(transcript, options); yield* Effect.addFinalizer(() => Effect.sync(() => { queryRunner.assertComplete(); @@ -710,14 +822,19 @@ const makeClaudeAgentSdkReplayQueryRunner = Effect.fn("ClaudeAgentSdkReplayQuery export function makeClaudeAgentSdkReplayQueryRunnerLayer( transcript: ClaudeAgentSdkReplayTranscript, + options: { readonly replayGate?: ProviderReplayGate } = {}, ): Layer.Layer { - return Layer.effect(ClaudeAgentSdkQueryRunner, makeClaudeAgentSdkReplayQueryRunner(transcript)); + return Layer.effect( + ClaudeAgentSdkQueryRunner, + makeClaudeAgentSdkReplayQueryRunner(transcript, options), + ); } export function makeClaudeAgentSdkReplayLayer( transcript: ClaudeAgentSdkReplayTranscript, + options: { readonly replayGate?: ProviderReplayGate } = {}, ): Layer.Layer { - const queryRunner = makeReplayQueryRunner(transcript); + const queryRunner = makeReplayQueryRunner(transcript, options); return Layer.effect( ClaudeAgentSdkQueryRunner, Effect.gen(function* () { @@ -750,6 +867,7 @@ export function makeClaudeAgentSdkReplayLayer( export function makeClaudeProviderAdapterRegistryReplayLayer( transcript: ClaudeAgentSdkReplayTranscript, + options: { readonly replayGate?: ProviderReplayGate } = {}, ) { const serverConfigLayer = Layer.effect( ServerConfig, @@ -765,7 +883,7 @@ export function makeClaudeProviderAdapterRegistryReplayLayer( }).pipe( Layer.provide( Layer.mergeAll( - makeClaudeAgentSdkReplayLayer(transcript), + makeClaudeAgentSdkReplayLayer(transcript, options), idAllocatorLayer, NodeServices.layer, serverConfigLayer, @@ -2432,6 +2550,6 @@ export const ClaudeOrchestratorReplayHarness: OrchestratorV2ProviderReplayHarnes }), ), ), - makeProviderAdapterRegistryLayer: (transcript) => - makeClaudeProviderAdapterRegistryReplayLayer(transcript), + makeProviderAdapterRegistryLayer: (transcript, options) => + makeClaudeProviderAdapterRegistryReplayLayer(transcript, options), }; diff --git a/apps/server/src/orchestration-v2/Adapters/ClaudeAdapterV2.ts b/apps/server/src/orchestration-v2/Adapters/ClaudeAdapterV2.ts index 2524f6e4337f..416fda8b032e 100644 --- a/apps/server/src/orchestration-v2/Adapters/ClaudeAdapterV2.ts +++ b/apps/server/src/orchestration-v2/Adapters/ClaudeAdapterV2.ts @@ -28,6 +28,7 @@ import { type OrchestrationV2ExecutionNode, type OrchestrationV2ProviderCapabilities, type OrchestrationV2ProviderFailure, + type OrchestrationV2PendingBackgroundTask, type OrchestrationV2ProviderRetry, type OrchestrationV2ProviderSession, type OrchestrationV2ProviderThread, @@ -1399,6 +1400,15 @@ function commandInputFromClaudeTool(toolName: string, input: ClaudeNativeToolInp ); } +// Opaque non-subagent background work admitted onto the Waiting roster. +// Subagents project through the normal subagent lifecycle and must not be +// double-counted when background_tasks_changed includes them. +const CLAUDE_OPAQUE_BACKGROUND_TASK_TYPES = new Set(["local_bash"]); + +function isClaudeOpaqueBackgroundTaskType(taskType: string | null | undefined): boolean { + return typeof taskType === "string" && CLAUDE_OPAQUE_BACKGROUND_TASK_TYPES.has(taskType); +} + function claudeTaskTypeFromSdkMessage(message: SDKMessage): string | null { if (typeof message !== "object" || message === null) { return null; @@ -1408,7 +1418,45 @@ function claudeTaskTypeFromSdkMessage(message: SDKMessage): string | null { } function isClaudeNonSubagentTask(message: SDKMessage): boolean { - return claudeTaskTypeFromSdkMessage(message) === "local_bash"; + return isClaudeOpaqueBackgroundTaskType(claudeTaskTypeFromSdkMessage(message)); +} + +function isClaudeBackgroundTasksChangedMessage(message: SDKMessage): boolean { + return ( + message.type === "system" && + // Undeclared SDK subtype: full roster snapshot of live background tasks. + (message.subtype as string) === "background_tasks_changed" + ); +} + +function claudePendingBackgroundTasksFromRoster( + roster: ReadonlyMap, +): ReadonlyArray { + return Array.from(roster.values()); +} + +function parseClaudeBackgroundTaskEntry( + entry: unknown, +): OrchestrationV2PendingBackgroundTask | null { + if (entry === null || typeof entry !== "object") { + return null; + } + const taskId = Reflect.get(entry, "task_id"); + if (typeof taskId !== "string" || taskId.length === 0) { + return null; + } + const taskType = Reflect.get(entry, "task_type"); + // Mirror the incremental path: only opaque non-subagent types currently + // supported for Waiting. Subagent/agent entries stay on the subagent path. + if (!isClaudeOpaqueBackgroundTaskType(typeof taskType === "string" ? taskType : null)) { + return null; + } + const description = Reflect.get(entry, "description"); + return { + taskId, + ...(typeof description === "string" && description.trim().length > 0 ? { description } : {}), + taskType, + }; } function fileNameFromClaudeTool(toolName: string, input: ClaudeNativeToolInput): string { @@ -2039,7 +2087,32 @@ export function makeClaudeAdapterV2( { readonly threadId: ThreadId; readonly providerThreadId: ProviderThreadId } >(), ); - const pendingBackgroundTaskIds = yield* Ref.make(new Set()); + // Authoritative + incremental background-task roster for post-settle + // Waiting UI. Outer key is native Claude session id so concurrent + // provider threads on one runtime cannot share or clear each other. + const pendingBackgroundTasksByNativeThread = yield* Ref.make( + new Map>(), + ); + // Wake eligibility is separate from the Waiting roster. It survives + // empty background_tasks_changed levels (SDK: empty level can precede + // task_notification) and is consumed when the first idle notification + // is buffered/offered so duplicates cannot re-buffer. A short-lived + // replay tombstone then covers continuation drain classification so + // local_bash is never projected as a subagent; the tombstone is + // cleared after that drained notification is processed. Both sets + // clear on CLI process open/replacement and failed/interrupted turns. + const wakeEligibleBackgroundTasksByNativeThread = yield* Ref.make( + new Map>(), + ); + const opaqueBackgroundTaskReplayTombstonesByNativeThread = yield* Ref.make( + new Map>(), + ); + // Last known provider-thread payload per native session, used to emit + // roster-only provider_thread.updated events between turns without + // resurrecting an active status after root settlement. + const lastProviderThreadByNativeThread = yield* Ref.make( + new Map(), + ); // Subagent registry that survives turn settle: a background subagent // (Agent with run_in_background) can complete after the root turn // ended, and its task_notification must both count as wake evidence @@ -2059,6 +2132,359 @@ export function makeClaudeAdapterV2( const emitProviderEvent = (event: ProviderAdapterV2Event) => Queue.offer(events, event).pipe(Effect.asVoid); + const rememberProviderThread = (providerThread: OrchestrationV2ProviderThread) => + Effect.gen(function* () { + const nativeThreadId = providerThread.nativeThreadRef?.nativeId; + if (nativeThreadId === undefined || nativeThreadId === null) { + return; + } + yield* Ref.update(lastProviderThreadByNativeThread, (current) => + new Map(current).set(nativeThreadId, providerThread), + ); + }); + + const rosterForNativeThread = ( + all: ReadonlyMap>, + nativeThreadId: string, + ): Map => + all.get(nativeThreadId) ?? new Map(); + + const hasPendingBackgroundTaskOnNativeThread = (nativeThreadId: string, taskId: string) => + Ref.get(pendingBackgroundTasksByNativeThread).pipe( + Effect.map((all) => rosterForNativeThread(all, nativeThreadId).has(taskId)), + ); + + const taskIdSetForNativeThread = ( + all: ReadonlyMap>, + nativeThreadId: string, + ): Set => all.get(nativeThreadId) ?? new Set(); + + const addTaskIdsToNativeThreadSet = ( + ref: Ref.Ref>>, + nativeThreadId: string, + taskIds: ReadonlyArray, + ) => + Ref.update(ref, (current) => { + if (taskIds.length === 0) { + return current; + } + const next = new Set(taskIdSetForNativeThread(current, nativeThreadId)); + let changed = false; + for (const taskId of taskIds) { + if (!next.has(taskId)) { + next.add(taskId); + changed = true; + } + } + return changed ? new Map(current).set(nativeThreadId, next) : current; + }); + + const clearTaskIdFromNativeThreadSet = ( + ref: Ref.Ref>>, + nativeThreadId: string, + taskId: string, + ) => + Ref.update(ref, (current) => { + const existing = taskIdSetForNativeThread(current, nativeThreadId); + if (!existing.has(taskId)) { + return current; + } + const next = new Set(existing); + next.delete(taskId); + const updated = new Map(current); + if (next.size === 0) { + updated.delete(nativeThreadId); + } else { + updated.set(nativeThreadId, next); + } + return updated; + }); + + const clearNativeThreadTaskIdSet = ( + ref: Ref.Ref>>, + nativeThreadId: string, + ) => + Ref.update(ref, (current) => { + if (!current.has(nativeThreadId)) { + return current; + } + const updated = new Map(current); + updated.delete(nativeThreadId); + return updated; + }); + + // First-notification wake offering only: not the Waiting roster and + // not the post-buffer replay tombstone. + const isWakeEligibleOpaqueBackgroundTaskOnNativeThread = ( + nativeThreadId: string, + taskId: string, + ) => + Ref.get(wakeEligibleBackgroundTasksByNativeThread).pipe( + Effect.map((all) => taskIdSetForNativeThread(all, nativeThreadId).has(taskId)), + ); + + const hasOpaqueBackgroundTaskReplayTombstoneOnNativeThread = ( + nativeThreadId: string, + taskId: string, + ) => + Ref.get(opaqueBackgroundTaskReplayTombstonesByNativeThread).pipe( + Effect.map((all) => taskIdSetForNativeThread(all, nativeThreadId).has(taskId)), + ); + + // Classify a task_notification as opaque local_bash (not a subagent): + // live roster, still-eligible first notification, or short-lived + // replay tombstone left after the idle notification was buffered. + const isKnownOpaqueBackgroundTaskOnNativeThread = ( + nativeThreadId: string, + taskId: string, + ) => + Effect.gen(function* () { + if (yield* hasPendingBackgroundTaskOnNativeThread(nativeThreadId, taskId)) { + return true; + } + if (yield* isWakeEligibleOpaqueBackgroundTaskOnNativeThread(nativeThreadId, taskId)) { + return true; + } + return yield* hasOpaqueBackgroundTaskReplayTombstoneOnNativeThread( + nativeThreadId, + taskId, + ); + }); + + // Admit onto wake eligibility only. Replay tombstones are created when + // the first idle notification is buffered, not at task start. + const markWakeEligibleOpaqueBackgroundTasks = ( + nativeThreadId: string, + taskIds: ReadonlyArray, + ) => + addTaskIdsToNativeThreadSet( + wakeEligibleBackgroundTasksByNativeThread, + nativeThreadId, + taskIds, + ); + + // After the first idle opaque notification is buffered/offered: stop + // further wake buffering for this task id, but keep a replay tombstone + // until the continuation drain classifies the buffered notification. + const consumeWakeEligibilityForBufferedNotification = ( + nativeThreadId: string, + taskId: string, + ) => + Effect.gen(function* () { + yield* clearTaskIdFromNativeThreadSet( + wakeEligibleBackgroundTasksByNativeThread, + nativeThreadId, + taskId, + ); + yield* addTaskIdsToNativeThreadSet( + opaqueBackgroundTaskReplayTombstonesByNativeThread, + nativeThreadId, + [taskId], + ); + }); + + const clearOpaqueBackgroundTaskReplayTombstone = (nativeThreadId: string, taskId: string) => + clearTaskIdFromNativeThreadSet( + opaqueBackgroundTaskReplayTombstonesByNativeThread, + nativeThreadId, + taskId, + ); + + const emitProviderThreadRoster = Effect.fnUntraced(function* (input: { + readonly nativeThreadId: string; + readonly providerThread: OrchestrationV2ProviderThread; + readonly status?: OrchestrationV2ProviderThread["status"]; + }) { + const roster = rosterForNativeThread( + yield* Ref.get(pendingBackgroundTasksByNativeThread), + input.nativeThreadId, + ); + const now = yield* DateTime.now; + const providerThread: OrchestrationV2ProviderThread = { + ...input.providerThread, + providerSessionId: session.id, + ...(input.status === undefined ? {} : { status: input.status }), + pendingBackgroundTasks: claudePendingBackgroundTasksFromRoster(roster), + updatedAt: now, + }; + yield* rememberProviderThread(providerThread); + yield* emitProviderEvent({ + type: "provider_thread.updated", + driver: CLAUDE_PROVIDER, + providerThread, + }); + }); + + const replacePendingBackgroundTasks = ( + nativeThreadId: string, + tasks: ReadonlyArray, + ) => + Effect.gen(function* () { + yield* Ref.update(pendingBackgroundTasksByNativeThread, (current) => { + const updated = new Map(current); + if (tasks.length === 0) { + updated.delete(nativeThreadId); + } else { + updated.set( + nativeThreadId, + new Map(tasks.map((task) => [task.taskId, task] as const)), + ); + } + return updated; + }); + // Empty level must not drop wake eligibility: notification may + // still be in flight. Non-empty level admits new task ids to + // wake eligibility only (replay tombstones are edge-created). + if (tasks.length > 0) { + yield* markWakeEligibleOpaqueBackgroundTasks( + nativeThreadId, + tasks.map((task) => task.taskId), + ); + } + }); + + const upsertPendingBackgroundTask = ( + nativeThreadId: string, + task: OrchestrationV2PendingBackgroundTask, + ) => + Effect.gen(function* () { + yield* Ref.update(pendingBackgroundTasksByNativeThread, (current) => { + const roster = new Map(rosterForNativeThread(current, nativeThreadId)); + roster.set(task.taskId, task); + return new Map(current).set(nativeThreadId, roster); + }); + yield* markWakeEligibleOpaqueBackgroundTasks(nativeThreadId, [task.taskId]); + }); + + const clearPendingBackgroundTask = (nativeThreadId: string, taskId: string) => + Ref.modify(pendingBackgroundTasksByNativeThread, (current) => { + const roster = rosterForNativeThread(current, nativeThreadId); + if (!roster.has(taskId)) { + return [false, current] as const; + } + const nextRoster = new Map(roster); + nextRoster.delete(taskId); + const updated = new Map(current); + if (nextRoster.size === 0) { + updated.delete(nativeThreadId); + } else { + updated.set(nativeThreadId, nextRoster); + } + return [true, updated] as const; + }); + + const clearPendingBackgroundTasksForNativeThread = (nativeThreadId: string) => + Ref.update(pendingBackgroundTasksByNativeThread, (current) => { + if (!current.has(nativeThreadId)) { + return current; + } + const updated = new Map(current); + updated.delete(nativeThreadId); + return updated; + }); + + // Drop idle wake traffic for a dead native process so it cannot pin + // session-wide pending work after sibling query replacement. + const clearWakeStateForNativeThread = (nativeThreadId: string) => + Effect.gen(function* () { + yield* Ref.update(wakeBuffers, (current) => { + if (!current.has(nativeThreadId)) { + return current; + } + const updated = new Map(current); + updated.delete(nativeThreadId); + return updated; + }); + yield* Ref.update(requestedContinuations, (current) => { + if (!current.has(nativeThreadId)) { + return current; + } + const updated = new Set(current); + updated.delete(nativeThreadId); + return updated; + }); + }); + + // Process-scoped level: SDK emits nothing at CLI start, so both the + // Waiting roster and wake eligibility reset when a live query opens + // or is replaced for this native thread. Opaque replay tombstones that + // already covered buffered task_notification frames are restored so a + // model/policy query replacement still classifies those local_bash + // completions on continuation drain. Buffer membership alone must not + // invent opaque classification: session-registered subagent + // notifications share the same buffer. + const resetBackgroundTaskStateForNativeThreadProcess = Effect.fnUntraced(function* ( + nativeThreadId: string, + options?: { + // openQuery during startTurn: activeTurn is not installed yet, but + // ProviderTurnStartService already marked the provider thread active. + readonly status?: OrchestrationV2ProviderThread["status"]; + }, + ) { + const hadRoster = + rosterForNativeThread( + yield* Ref.get(pendingBackgroundTasksByNativeThread), + nativeThreadId, + ).size > 0; + const remembered = (yield* Ref.get(lastProviderThreadByNativeThread)).get(nativeThreadId); + const hadPersistedRoster = (remembered?.pendingBackgroundTasks?.length ?? 0) > 0; + const priorOpaqueTombstones = taskIdSetForNativeThread( + yield* Ref.get(opaqueBackgroundTaskReplayTombstonesByNativeThread), + nativeThreadId, + ); + const bufferedTaskNotificationIds = new Set(); + const buffered = (yield* Ref.get(wakeBuffers)).get(nativeThreadId); + if (buffered !== undefined) { + for (const message of buffered.messages) { + if (message.type === "system" && message.subtype === "task_notification") { + bufferedTaskNotificationIds.add(message.task_id); + } + } + } + const preservedOpaqueTombstones = [...priorOpaqueTombstones].filter((taskId) => + bufferedTaskNotificationIds.has(taskId), + ); + yield* clearPendingBackgroundTasksForNativeThread(nativeThreadId); + yield* clearNativeThreadTaskIdSet( + wakeEligibleBackgroundTasksByNativeThread, + nativeThreadId, + ); + yield* clearNativeThreadTaskIdSet( + opaqueBackgroundTaskReplayTombstonesByNativeThread, + nativeThreadId, + ); + if (preservedOpaqueTombstones.length > 0) { + yield* addTaskIdsToNativeThreadSet( + opaqueBackgroundTaskReplayTombstonesByNativeThread, + nativeThreadId, + preservedOpaqueTombstones, + ); + } + if (!hadRoster && !hadPersistedRoster) { + return; + } + if (remembered === undefined) { + return; + } + // Prefer an explicit starting-turn status so a successful openQuery + // replacement clear cannot emit idle over an already-active thread. + // Otherwise: between turns never resurrect active from process reset; + // with a live activeTurn context, upgrade idle → active. + const activeContext = yield* Ref.get(activeTurn); + const status = + options?.status ?? + (activeContext === null + ? ("idle" as const) + : remembered.status === "idle" + ? ("active" as const) + : remembered.status); + yield* emitProviderThreadRoster({ + nativeThreadId, + providerThread: remembered, + status, + }); + }); + const resolveItemOrdinal = Effect.fnUntraced(function* ( context: ActiveClaudeTurnContext, nativeItemId: string, @@ -2946,26 +3372,55 @@ export function makeClaudeAdapterV2( completedAt: input.completedAt, }), }), - ...(input.status === "completed" && - input.context.input.providerThread.nativeConversationHeadRef !== null - ? [ - emitProviderEvent({ - type: "provider_thread.updated" as const, - driver: CLAUDE_PROVIDER, - providerThread: { - ...input.context.input.providerThread, - providerSessionId: session.id, - nativeConversationHeadRef: null, - status: "active" as const, - firstRunOrdinal: - input.context.input.providerThread.firstRunOrdinal ?? - input.context.input.runOrdinal, - lastRunOrdinal: input.context.input.runOrdinal, - updatedAt: input.completedAt, - }, - }), - ] - : []), + // Surface this native thread's roster before the root turn + // terminals so writeFinalRunEvents preserves it. Failed or + // interrupted turns drop only this thread's roster so sibling + // native threads keep their Waiting state. + Effect.gen(function* () { + const nativeThreadId = + input.context.input.providerThread.nativeThreadRef?.nativeId ?? null; + if (nativeThreadId !== null) { + if (input.status !== "completed") { + yield* clearPendingBackgroundTasksForNativeThread(nativeThreadId); + yield* clearNativeThreadTaskIdSet( + wakeEligibleBackgroundTasksByNativeThread, + nativeThreadId, + ); + yield* clearNativeThreadTaskIdSet( + opaqueBackgroundTaskReplayTombstonesByNativeThread, + nativeThreadId, + ); + } + } + const roster = + nativeThreadId === null + ? new Map() + : rosterForNativeThread( + yield* Ref.get(pendingBackgroundTasksByNativeThread), + nativeThreadId, + ); + const clearConversationHead = + input.status === "completed" && + input.context.input.providerThread.nativeConversationHeadRef !== null; + const providerThread: OrchestrationV2ProviderThread = { + ...input.context.input.providerThread, + providerSessionId: session.id, + ...(clearConversationHead ? { nativeConversationHeadRef: null } : {}), + firstRunOrdinal: + input.context.input.providerThread.firstRunOrdinal ?? + input.context.input.runOrdinal, + lastRunOrdinal: input.context.input.runOrdinal, + pendingBackgroundTasks: claudePendingBackgroundTasksFromRoster(roster), + status: input.status === "completed" ? "active" : "idle", + updatedAt: input.completedAt, + }; + yield* rememberProviderThread(providerThread); + yield* emitProviderEvent({ + type: "provider_thread.updated" as const, + driver: CLAUDE_PROVIDER, + providerThread, + }); + }), emitProviderEvent(terminalEvent), ], { concurrency: 1 }, @@ -3060,16 +3515,6 @@ export function makeClaudeAdapterV2( } }); - const clearPendingBackgroundTask = (taskId: string) => - Ref.modify(pendingBackgroundTaskIds, (current) => { - if (!current.has(taskId)) { - return [false, current] as const; - } - const updated = new Set(current); - updated.delete(taskId); - return [true, updated] as const; - }); - const bufferWakeMessage = Effect.fnUntraced(function* (wakeInput: { readonly nativeThreadId: string; readonly message: SDKMessage; @@ -3078,13 +3523,17 @@ export function makeClaudeAdapterV2( const isNotification = message.type === "system" && message.subtype === "task_notification"; // Only notifications for tracked tasks count as wake evidence: a - // pending local_bash background task, or a session-registered - // subagent that is still running (Agent with run_in_background - // settling after the root turn). A stray notification for an - // unknown task is dropped as before instead of triggering a - // spurious continuation. + // wake-eligible local_bash task (eligibility set, not the Waiting + // roster), or a session-registered subagent that is still running + // (Agent with run_in_background settling after the root turn). A + // stray notification for an unknown task is dropped as before + // instead of triggering a spurious continuation. const isPendingTaskNotification = - isNotification && (yield* Ref.get(pendingBackgroundTaskIds)).has(message.task_id); + isNotification && + (yield* isWakeEligibleOpaqueBackgroundTaskOnNativeThread( + wakeInput.nativeThreadId, + message.task_id, + )); const isPendingSubagentNotification = isNotification && !isPendingTaskNotification && @@ -3148,12 +3597,31 @@ export function makeClaudeAdapterV2( }); return updated; }); - // Request a continuation run once per wake, when the wake turn has - // either announced the finished task or fully settled. Earlier - // messages only buffer; the continuation turn drains them. + // First idle opaque notification: consume wake eligibility so a + // duplicate cannot re-buffer, and leave a short-lived replay + // tombstone for continuation-drain classification. + if (isPendingTaskNotification) { + yield* consumeWakeEligibilityForBufferedNotification( + wakeInput.nativeThreadId, + message.task_id, + ); + } + // A terminal task notification can clear the Waiting roster without + // Claude dequeuing it into a native model turn. Buffer it for replay, + // but do not open an opaque-task continuation until native user, + // assistant, or result output proves that Claude actually began the + // wake turn. Subagent notifications retain their existing immediate + // offer because their projected lifecycle owns the continuation. + const buffered = (yield* Ref.get(wakeBuffers)).get(wakeInput.nativeThreadId); + const hasBufferedNotification = + buffered?.messages.some( + (entry) => entry.type === "system" && entry.subtype === "task_notification", + ) ?? false; + const isNativeOpaqueWakeFrame = + hasBufferedNotification && (message.type === "assistant" || message.type === "user"); if ( - !isPendingTaskNotification && !isPendingSubagentNotification && + !isNativeOpaqueWakeFrame && message.type !== "result" ) { return; @@ -3192,6 +3660,90 @@ export function makeClaudeAdapterV2( }); }); + const applyBackgroundTaskRosterMessage = Effect.fnUntraced(function* (input: { + readonly nativeThreadId: string; + readonly message: SDKMessage; + readonly activeContext: ActiveClaudeTurnContext | null; + }) { + const message = input.message; + let rosterChanged = false; + + if (isClaudeBackgroundTasksChangedMessage(message)) { + const roster = Reflect.get(message, "tasks"); + if (!Array.isArray(roster)) { + return false; + } + const nextTasks: OrchestrationV2PendingBackgroundTask[] = []; + for (const entry of roster) { + const task = parseClaudeBackgroundTaskEntry(entry); + if (task !== null) { + nextTasks.push(task); + } + } + yield* replacePendingBackgroundTasks(input.nativeThreadId, nextTasks); + rosterChanged = true; + } else if (message.type === "system" && message.subtype === "task_started") { + // Incremental fallback when background_tasks_changed is absent. + // Subagent tasks project as subagent turn items; only non-subagent + // background work (e.g. local_bash) lives on the provider-thread roster. + if (!isClaudeNonSubagentTask(message)) { + return false; + } + const description = + typeof message.description === "string" && message.description.trim().length > 0 + ? message.description + : undefined; + const taskType = claudeTaskTypeFromSdkMessage(message) ?? undefined; + yield* upsertPendingBackgroundTask(input.nativeThreadId, { + taskId: message.task_id, + ...(description === undefined ? {} : { description }), + ...(taskType === undefined ? {} : { taskType }), + }); + rosterChanged = true; + } else if (message.type === "system" && message.subtype === "task_notification") { + const removed = yield* clearPendingBackgroundTask( + input.nativeThreadId, + message.task_id, + ); + // Waiting roster clears on the notification edge. Wake eligibility + // is consumed when the first idle notification is buffered; clear + // here too for same-turn active notifications that never entered + // the idle buffer path. Replay tombstones are not cleared here. + yield* clearTaskIdFromNativeThreadSet( + wakeEligibleBackgroundTasksByNativeThread, + input.nativeThreadId, + message.task_id, + ); + rosterChanged = removed; + } + + if (!rosterChanged) { + return false; + } + + const baseThread = + input.activeContext?.input.providerThread ?? + (yield* Ref.get(lastProviderThreadByNativeThread)).get(input.nativeThreadId); + if (baseThread === undefined) { + return true; + } + + // Between turns, never resurrect active status from a late empty + // roster update. During an active turn, preserve the thread status. + const status = + input.activeContext === null + ? ("idle" as const) + : baseThread.status === "idle" + ? ("active" as const) + : baseThread.status; + yield* emitProviderThreadRoster({ + nativeThreadId: input.nativeThreadId, + providerThread: baseThread, + status, + }); + return true; + }); + const handleSdkMessage = Effect.fnUntraced(function* (input: { readonly query: ClaudeAgentSdkQuerySession; readonly message: SDKMessage; @@ -3204,7 +3756,23 @@ export function makeClaudeAdapterV2( const message = input.message; const context = yield* Ref.get(activeTurn); if (context === null) { - yield* bufferWakeMessage({ nativeThreadId: liveQuery.nativeThreadId, message }); + // task_notification must buffer wake evidence while still tracked + // on the roster; clearing first would drop the wake pin. + if (message.type === "system" && message.subtype === "task_notification") { + yield* bufferWakeMessage({ nativeThreadId: liveQuery.nativeThreadId, message }); + yield* applyBackgroundTaskRosterMessage({ + nativeThreadId: liveQuery.nativeThreadId, + message, + activeContext: null, + }); + } else { + yield* applyBackgroundTaskRosterMessage({ + nativeThreadId: liveQuery.nativeThreadId, + message, + activeContext: null, + }); + yield* bufferWakeMessage({ nativeThreadId: liveQuery.nativeThreadId, message }); + } return; } @@ -3257,12 +3825,23 @@ export function makeClaudeAdapterV2( return; } + if (isClaudeBackgroundTasksChangedMessage(message)) { + yield* applyBackgroundTaskRosterMessage({ + nativeThreadId: liveQuery.nativeThreadId, + message, + activeContext: context, + }); + return; + } + if (message.type === "system" && message.subtype === "task_started") { if (isClaudeNonSubagentTask(message)) { context.ignoredTaskIds.add(message.task_id); - yield* Ref.update(pendingBackgroundTaskIds, (current) => - new Set(current).add(message.task_id), - ); + yield* applyBackgroundTaskRosterMessage({ + nativeThreadId: liveQuery.nativeThreadId, + message, + activeContext: context, + }); } else { yield* updateClaudeSubagentNode({ context, @@ -3278,7 +3857,8 @@ export function makeClaudeAdapterV2( if (message.type === "system" && message.subtype === "task_progress") { const progress = message.description.trim(); - const isBackgroundTask = (yield* Ref.get(pendingBackgroundTaskIds)).has( + const isBackgroundTask = yield* hasPendingBackgroundTaskOnNativeThread( + liveQuery.nativeThreadId, message.task_id, ); if ( @@ -3297,9 +3877,19 @@ export function makeClaudeAdapterV2( } if (message.type === "system" && message.subtype === "task_notification") { - // A wake-replay turn has empty ignoredTaskIds, so the session-level - // background registry is the durable ignore signal across turns. - const wasBackgroundTask = yield* clearPendingBackgroundTask(message.task_id); + // A wake-replay turn has empty ignoredTaskIds, so opaque-task + // tracking (live roster, wake eligibility, or the short-lived + // post-buffer replay tombstone) classifies local_bash before any + // subagent handling. + const wasBackgroundTask = yield* isKnownOpaqueBackgroundTaskOnNativeThread( + liveQuery.nativeThreadId, + message.task_id, + ); + yield* applyBackgroundTaskRosterMessage({ + nativeThreadId: liveQuery.nativeThreadId, + message, + activeContext: context, + }); if (!wasBackgroundTask && !context.ignoredTaskIds.has(message.task_id)) { yield* updateClaudeSubagentNode({ context, @@ -3314,6 +3904,14 @@ export function makeClaudeAdapterV2( : "failed", }); } + // Replay tombstone only needs to outlive buffering until this + // drained/live classification runs; drop it so it cannot leak. + if (wasBackgroundTask) { + yield* clearOpaqueBackgroundTaskReplayTombstone( + liveQuery.nativeThreadId, + message.task_id, + ); + } } for (const toolUse of claudeToolUseBlocksFromAssistantMessage(message)) { @@ -3609,8 +4207,20 @@ export function makeClaudeAdapterV2( return existing; } + // openQuery owns one live process. Closing it for another native + // thread kills that sibling's CLI; it can never emit a roster clear, + // so drop its process-scoped Waiting/wake state immediately. Closing + // for the same native thread leaves a non-authoritative roster until + // the replacement open succeeds or fails below. + const closedExistingNativeThreadId = existing !== null ? existing.nativeThreadId : null; if (existing !== null) { yield* existing.query.close.pipe(Effect.ignore); + if (existing.nativeThreadId !== nativeThreadId) { + yield* clearWakeStateForNativeThread(existing.nativeThreadId); + yield* resetBackgroundTaskStateForNativeThreadProcess(existing.nativeThreadId, { + status: "idle", + }); + } } const openedWithResume = (yield* Ref.get(openedNativeThreads)).has(nativeThreadId); @@ -3622,26 +4232,45 @@ export function makeClaudeAdapterV2( const hasPersistedProviderTurn = turnInput.providerTurnOrdinal > 1; const shouldResume = resumeSessionAt !== undefined || openedWithResume || hasPersistedProviderTurn; - const querySession = yield* queryRunner.open({ - threadId: turnInput.threadId, - providerSessionId: input.providerSessionId, - options: makeClaudeQueryOptions({ - modelSelection: turnInput.modelSelection, - nativeThreadId, - resume: shouldResume, - ...(resumeSessionAt === undefined ? {} : { resumeSessionAt }), - cwd: turnInput.runtimePolicy.cwd, - settings: adapterOptions.settings, - environment: adapterOptions.environment, - tools: queryPolicy.tools ?? CLAUDE_CODE_PRESET_TOOLS, - ...mcpOverrides, - permissionMode: queryPolicy.permissionMode, - ...(queryPolicy.allowDangerouslySkipPermissions === undefined - ? {} - : { allowDangerouslySkipPermissions: queryPolicy.allowDangerouslySkipPermissions }), - ...(shouldInstallClaudePermissionCallback(queryPolicy) ? { canUseTool } : {}), - }), - }); + const querySession = yield* queryRunner + .open({ + threadId: turnInput.threadId, + providerSessionId: input.providerSessionId, + options: makeClaudeQueryOptions({ + modelSelection: turnInput.modelSelection, + nativeThreadId, + resume: shouldResume, + ...(resumeSessionAt === undefined ? {} : { resumeSessionAt }), + cwd: turnInput.runtimePolicy.cwd, + settings: adapterOptions.settings, + environment: adapterOptions.environment, + tools: queryPolicy.tools ?? CLAUDE_CODE_PRESET_TOOLS, + ...mcpOverrides, + permissionMode: queryPolicy.permissionMode, + ...(queryPolicy.allowDangerouslySkipPermissions === undefined + ? {} + : { + allowDangerouslySkipPermissions: queryPolicy.allowDangerouslySkipPermissions, + }), + ...(shouldInstallClaudePermissionCallback(queryPolicy) ? { canUseTool } : {}), + }), + }) + .pipe( + Effect.tapError(() => + // Same-native-thread replacement: the old process is already + // dead, so its process-scoped roster is not authoritative. + // First-ever failed open (no prior live query) must not invent + // native-session reset events. + closedExistingNativeThreadId === nativeThreadId + ? Effect.gen(function* () { + yield* clearWakeStateForNativeThread(nativeThreadId); + yield* resetBackgroundTaskStateForNativeThreadProcess(nativeThreadId, { + status: "idle", + }); + }) + : Effect.void, + ), + ); // Marked only after a successful open: a failed create must not // leave the runtime believing the native session exists, or the // retry would resume a session that was never created. @@ -3653,6 +4282,16 @@ export function makeClaudeAdapterV2( updated.add(nativeThreadId); return updated; }); + // Level is per CLI process: reset Waiting roster and wake + // eligibility whenever this native thread's process starts or is + // replaced. Membership repopulates on the next snapshot/edge. + // openQuery only runs from startTurn after ProviderTurnStartService + // marked the provider thread active, and before activeTurn is set. + // Buffered local_bash task_notification classification is preserved + // across this reset (see resetBackgroundTaskStateForNativeThreadProcess). + yield* resetBackgroundTaskStateForNativeThreadProcess(nativeThreadId, { + status: "active", + }); const closed = yield* Deferred.make(); const context: ClaudeLiveQueryContext = { nativeThreadId, @@ -3708,6 +4347,7 @@ export function makeClaudeAdapterV2( }); return updated; }); + yield* rememberProviderThread(turnInput.providerThread); const context: ActiveClaudeTurnContext = { input: turnInput, nativeTurnId, @@ -3786,6 +4426,16 @@ export function makeClaudeAdapterV2( // replaying it before the rest would drop them back into the wake // buffer and request another continuation. const resultMessages = drained.filter((entry) => entry.type === "result"); + const opaqueReplayTombstones = taskIdSetForNativeThread( + yield* Ref.get(opaqueBackgroundTaskReplayTombstonesByNativeThread), + nativeThreadId, + ); + const hasOpaqueTaskNotification = drained.some( + (entry) => + entry.type === "system" && + entry.subtype === "task_notification" && + opaqueReplayTombstones.has(entry.task_id), + ); for (const entry of drained) { if (entry.type !== "result") { yield* handleSdkMessage({ query: querySession.query, message: entry }); @@ -3794,6 +4444,14 @@ export function makeClaudeAdapterV2( const lastResult = resultMessages.at(-1); if (lastResult !== undefined) { yield* handleSdkMessage({ query: querySession.query, message: lastResult }); + return; + } + const hasNativeWakeFrame = drained.some( + (entry) => entry.type === "user" || entry.type === "assistant", + ); + if (hasOpaqueTaskNotification && !hasNativeWakeFrame) { + const completedAt = yield* DateTime.now; + yield* finalizeActiveTurn({ context, status: "completed", completedAt }); } }, (effect, turnInput) => @@ -3966,8 +4624,11 @@ export function makeClaudeAdapterV2( providerSession: session, events: Stream.fromEffectRepeat(Queue.take(events)), hasPendingBackgroundWork: Effect.gen(function* () { - if ((yield* Ref.get(pendingBackgroundTaskIds)).size > 0) { - return true; + // Session capability: any native thread with pending work pins idle. + for (const roster of (yield* Ref.get(pendingBackgroundTasksByNativeThread)).values()) { + if (roster.size > 0) { + return true; + } } for (const subagent of (yield* Ref.get(sessionSubagentsByTaskId)).values()) { if (subagent.task.status === "running") { @@ -3976,12 +4637,34 @@ export function makeClaudeAdapterV2( } const buffers = yield* Ref.get(wakeBuffers); for (const entry of buffers.values()) { - if (entry.messages.length > 0) { + if ( + entry.messages.some( + (message) => + message.type === "user" || + message.type === "assistant" || + message.type === "result", + ) + ) { return true; } } return false; }), + hasPendingBackgroundWorkForThread: (providerThread) => + Effect.gen(function* () { + const nativeThreadId = providerThread.nativeThreadRef?.nativeId; + if (nativeThreadId === undefined || nativeThreadId === null) { + return false; + } + // Root-run stop gate: only this native thread's roster. Session + // subagents and wake buffers stay on the session-wide probe. + return ( + rosterForNativeThread( + yield* Ref.get(pendingBackgroundTasksByNativeThread), + nativeThreadId, + ).size > 0 + ); + }), ensureThread: Effect.fn("ClaudeAdapterV2.ensureThread")( function* (threadInput: ProviderAdapterV2EnsureThreadInput) { const createdAt = yield* DateTime.now; diff --git a/apps/server/src/orchestration-v2/Adapters/CodexAdapterV2.testkit.ts b/apps/server/src/orchestration-v2/Adapters/CodexAdapterV2.testkit.ts index 8b2254e207a5..16c695fe780a 100644 --- a/apps/server/src/orchestration-v2/Adapters/CodexAdapterV2.testkit.ts +++ b/apps/server/src/orchestration-v2/Adapters/CodexAdapterV2.testkit.ts @@ -15,6 +15,7 @@ import { ProviderAdapterOpenSessionError } from "../ProviderAdapter.ts"; import { ProviderAdapterDriverCreateError } from "../ProviderAdapterDriver.ts"; import { makeDriverLayer as makeProviderAdapterRegistryDriverLayer } from "../ProviderAdapterRegistry.ts"; import type { OrchestratorV2ProviderReplayHarness } from "../testkit/ProviderReplayHarness.ts"; +import type { ProviderReplayGate } from "../testkit/ProviderReplayGate.testkit.ts"; import { CODEX_DEFAULT_INSTANCE_ID, CODEX_DRIVER_KIND, @@ -205,13 +206,29 @@ export const CodexOrchestratorReplayHarness: OrchestratorV2ProviderReplayHarness }), ), ), - makeProviderAdapterRegistryLayer: (transcript) => { + makeProviderAdapterRegistryLayer: ( + transcript, + options: { readonly replayGate?: ProviderReplayGate } = {}, + ) => { return Layer.effectContext( - CodexReplay.makeReplayDriver(transcript).pipe( - Effect.flatMap((driver) => - Layer.build(makeCodexProviderAdapterRegistryReplayLayer({ transcript, driver })), - ), - ), + Effect.gen(function* () { + const replayGate = options.replayGate; + if (replayGate !== undefined) { + yield* Effect.addFinalizer(() => Effect.sync(() => replayGate.releaseAll())); + } + const driver = yield* CodexReplay.makeReplayDriver( + transcript, + replayGate === undefined + ? {} + : { + beforeEmitInbound: (entry) => + Effect.promise((signal) => replayGate.beforeEmit(entry.label, signal)), + }, + ); + return yield* Layer.build( + makeCodexProviderAdapterRegistryReplayLayer({ transcript, driver }), + ); + }), ); }, }; diff --git a/apps/server/src/orchestration-v2/Adapters/GrokAdapterV2.test.ts b/apps/server/src/orchestration-v2/Adapters/GrokAdapterV2.test.ts index 27f8a6038013..e6d620500521 100644 --- a/apps/server/src/orchestration-v2/Adapters/GrokAdapterV2.test.ts +++ b/apps/server/src/orchestration-v2/Adapters/GrokAdapterV2.test.ts @@ -13,6 +13,7 @@ import { acpRootTurnIsIdle, acpRootTurnSettleDebounceMs, acpRootTurnShouldRearmRecoveryTimers, + acpSubagentStatusBlocksTurnSettlement, acpSupportsImagePrompts, } from "./AcpAdapterV2.ts"; import { @@ -159,6 +160,20 @@ describe("acpRootTurn recovery timer re-arm", () => { }); }); +describe("acpSubagentStatusBlocksTurnSettlement", () => { + it("blocks settlement for pending and running subagents", () => { + assert.isTrue(acpSubagentStatusBlocksTurnSettlement("pending")); + assert.isTrue(acpSubagentStatusBlocksTurnSettlement("running")); + }); + + it("does not block settlement for terminal subagents", () => { + assert.isFalse(acpSubagentStatusBlocksTurnSettlement("cancelled")); + assert.isFalse(acpSubagentStatusBlocksTurnSettlement("completed")); + assert.isFalse(acpSubagentStatusBlocksTurnSettlement("failed")); + assert.isFalse(acpSubagentStatusBlocksTurnSettlement("interrupted")); + }); +}); + describe("acpRootTurnIsIdle", () => { const quiet = { finalized: false, @@ -168,7 +183,7 @@ describe("acpRootTurnIsIdle", () => { hasRunningTool: false, hasPendingRuntimeRequest: false, hasToolHistory: false, - hasRunningSubagent: false, + hasActiveSubagent: false, hasOutput: true, } as const; @@ -185,7 +200,7 @@ describe("acpRootTurnIsIdle", () => { }); it("is false while a native subagent task is still running", () => { - assert.isFalse(acpRootTurnIsIdle({ ...quiet, hasRunningSubagent: true })); + assert.isFalse(acpRootTurnIsIdle({ ...quiet, hasActiveSubagent: true })); }); it("is false when only reasoning or tools have streamed", () => { diff --git a/apps/server/src/orchestration-v2/EventSink.ts b/apps/server/src/orchestration-v2/EventSink.ts index a6448d542127..a9f3db0062f1 100644 --- a/apps/server/src/orchestration-v2/EventSink.ts +++ b/apps/server/src/orchestration-v2/EventSink.ts @@ -3,6 +3,7 @@ import { type OrchestrationV2Run, OrchestrationV2DomainEvent, OrchestrationV2StoredEvent, + ProviderThreadId, RunAttemptId, RunId, ThreadId, @@ -99,6 +100,26 @@ export interface EventSinkV2Shape { }, EventSinkV2Error >; + /** + * Atomically commit only when the provider thread is still owned by the + * expected run attempt and ordinal. Used for late post-terminal + * provider_thread updates so a completed or superseded attempt cannot clobber + * a newer attempt that already claimed the thread. + */ + readonly writeIfProviderThreadOwner: (input: { + readonly commandId?: CommandId; + readonly providerThreadId: ProviderThreadId; + readonly runId: RunId; + readonly activeAttemptId: RunAttemptId; + readonly expectedLastRunOrdinal: number; + readonly events: ReadonlyArray; + }) => Effect.Effect< + { + readonly committed: boolean; + readonly storedEvents: ReadonlyArray; + }, + EventSinkV2Error + >; readonly commitCommand: (input: { readonly commandId: CommandId; readonly threadId: ThreadId; @@ -299,6 +320,62 @@ const baseLayer: Layer.Layer< }, ); + const writeIfProviderThreadOwnerEffect = Effect.fn( + "orchestrationV2.EventSink.writeIfProviderThreadOwner", + )(function* (input: Parameters[0]) { + yield* Effect.annotateCurrentSpan({ + "orchestration_v2.command_id": input.commandId ?? null, + "orchestration_v2.event_count": input.events.length, + "orchestration_v2.provider_thread_id": input.providerThreadId, + "orchestration_v2.run_id": input.runId, + "orchestration_v2.active_attempt_id": input.activeAttemptId, + "orchestration_v2.expected_last_run_ordinal": input.expectedLastRunOrdinal, + }); + + const result = yield* sql.withTransaction( + Effect.gen(function* () { + const rows = yield* sql<{ + readonly active_attempt_id: string | null; + readonly last_run_ordinal: number | null; + }>` + SELECT + json_extract(r.payload_json, '$.activeAttemptId') AS active_attempt_id, + p.last_run_ordinal + FROM orchestration_v2_projection_provider_threads p + JOIN orchestration_v2_projection_runs r + ON r.run_id = ${input.runId} + AND r.thread_id = p.thread_id + WHERE p.provider_thread_id = ${input.providerThreadId} + LIMIT 1 + `; + const current = rows[0]; + if ( + current === undefined || + current.active_attempt_id !== input.activeAttemptId || + current.last_run_ordinal !== input.expectedLastRunOrdinal + ) { + return { + committed: false as const, + storedEvents: [] as ReadonlyArray, + }; + } + + const normalized = yield* normalizeEvents(input.events); + const storedEvents = yield* eventStore.append({ + ...(input.commandId === undefined ? {} : { commandId: input.commandId }), + events: normalized, + }); + yield* applyStoredEvents(storedEvents); + return { committed: true as const, storedEvents }; + }), + ); + if (result.committed) { + yield* eventStore.publishCommitted(result.storedEvents); + yield* PubSub.publishAll(liveEvents, result.storedEvents); + } + return result; + }); + const existingCommandResult = (commandId: CommandId) => Effect.gen(function* () { const existing = yield* commandReceipts.getByCommandId(commandId); @@ -497,6 +574,17 @@ const baseLayer: Layer.Layer< }), ), ), + writeIfProviderThreadOwner: (input) => + writeIfProviderThreadOwnerEffect(input).pipe( + Effect.mapError( + (cause) => + new EventSinkWriteError({ + eventCount: input.events.length, + ...(input.commandId === undefined ? {} : { commandId: input.commandId }), + cause, + }), + ), + ), commitCommand: (input) => commitCommandEffect(input).pipe( Effect.mapError( diff --git a/apps/server/src/orchestration-v2/FoundationPersistence.test.ts b/apps/server/src/orchestration-v2/FoundationPersistence.test.ts index e6159ee0772b..fdb244174846 100644 --- a/apps/server/src/orchestration-v2/FoundationPersistence.test.ts +++ b/apps/server/src/orchestration-v2/FoundationPersistence.test.ts @@ -867,6 +867,229 @@ it.layer(TestLayer)("orchestration V2 foundation persistence", (it) => { }), ); + it.effect("guards post-terminal provider-thread writes by attempt and run ordinal", () => + Effect.gen(function* () { + const eventSink = yield* EventSinkV2; + const projectionStore = yield* ProjectionStoreV2; + const now = yield* DateTime.now; + const threadId = ThreadId.make("thread:foundation-provider-thread-owner"); + const runId = RunId.make("run:foundation-provider-thread-owner"); + const attemptId = RunAttemptId.make("attempt:foundation-provider-thread-owner"); + const replacementAttemptId = RunAttemptId.make( + "attempt:foundation-provider-thread-owner:replacement", + ); + const rootNodeId = NodeId.make("node:foundation-provider-thread-owner"); + const providerThreadId = ProviderThreadId.make( + "provider-thread:foundation-provider-thread-owner", + ); + const thread = makeThread(threadId, now); + const run: OrchestrationV2Run = { + id: runId, + threadId, + ordinal: 1, + providerInstanceId, + modelSelection, + providerThreadId, + userMessageId: MessageId.make("message:foundation-provider-thread-owner"), + rootNodeId, + activeAttemptId: attemptId, + status: "completed", + queuePosition: null, + requestedAt: now, + startedAt: now, + completedAt: now, + checkpointId: null, + contextHandoffId: null, + }; + const baseProviderThread = { + id: providerThreadId, + driver: providerDriver, + providerInstanceId, + providerSessionId: null, + appThreadId: threadId, + ownerNodeId: null, + nativeThreadRef: null, + nativeConversationHeadRef: null, + status: "idle" as const, + firstRunOrdinal: 1, + lastRunOrdinal: 1, + handoffIds: [] as const, + forkedFrom: null, + pendingBackgroundTasks: [ + { taskId: "bg-owner", description: "sleep 20", taskType: "local_bash" }, + ], + createdAt: now, + updatedAt: now, + }; + + yield* eventSink.write({ + events: [ + threadCreatedEvent({ + id: "event:foundation-provider-thread-owner:thread", + thread, + now, + }), + { + id: EventId.make("event:foundation-provider-thread-owner:run"), + type: "run.created", + threadId, + runId, + nodeId: rootNodeId, + providerInstanceId, + occurredAt: now, + payload: run, + }, + { + id: EventId.make("event:foundation-provider-thread-owner:provider-thread"), + type: "provider-thread.updated", + threadId, + driver: providerDriver, + providerInstanceId, + occurredAt: now, + payload: baseProviderThread, + }, + ], + }); + + const ownedClear = yield* eventSink.writeIfProviderThreadOwner({ + providerThreadId, + runId, + activeAttemptId: attemptId, + expectedLastRunOrdinal: 1, + events: [ + { + id: EventId.make("event:foundation-provider-thread-owner:clear"), + type: "provider-thread.updated", + threadId, + driver: providerDriver, + providerInstanceId, + occurredAt: now, + payload: { + ...baseProviderThread, + pendingBackgroundTasks: [], + updatedAt: now, + }, + }, + ], + }); + assert.isTrue(ownedClear.committed); + assert.equal(ownedClear.storedEvents.length, 1); + + const afterReplacement = yield* DateTime.now; + yield* eventSink.write({ + events: [ + { + id: EventId.make("event:foundation-provider-thread-owner:replacement-attempt"), + type: "run.updated", + threadId, + runId, + nodeId: rootNodeId, + providerInstanceId, + occurredAt: afterReplacement, + payload: { + ...run, + activeAttemptId: replacementAttemptId, + status: "running", + }, + }, + ], + }); + + const supersededAttemptWrite = yield* eventSink.writeIfProviderThreadOwner({ + providerThreadId, + runId, + activeAttemptId: attemptId, + expectedLastRunOrdinal: 1, + events: [ + { + id: EventId.make("event:foundation-provider-thread-owner:superseded-attempt"), + type: "provider-thread.updated", + threadId, + driver: providerDriver, + providerInstanceId, + occurredAt: afterReplacement, + payload: { + ...baseProviderThread, + status: "active", + pendingBackgroundTasks: [ + { + taskId: "bg-superseded", + description: "should not land", + taskType: "local_bash", + }, + ], + updatedAt: afterReplacement, + }, + }, + ], + }); + assert.isFalse(supersededAttemptWrite.committed); + assert.deepEqual(supersededAttemptWrite.storedEvents, []); + + yield* eventSink.write({ + events: [ + { + id: EventId.make("event:foundation-provider-thread-owner:replacement-completed"), + type: "run.updated", + threadId, + runId, + nodeId: rootNodeId, + providerInstanceId, + occurredAt: afterReplacement, + payload: { + ...run, + activeAttemptId: replacementAttemptId, + }, + }, + { + id: EventId.make("event:foundation-provider-thread-owner:newer-run"), + type: "provider-thread.updated", + threadId, + driver: providerDriver, + providerInstanceId, + occurredAt: afterReplacement, + payload: { + ...baseProviderThread, + lastRunOrdinal: 2, + status: "active", + pendingBackgroundTasks: [], + updatedAt: afterReplacement, + }, + }, + ], + }); + + const staleOrdinalWrite = yield* eventSink.writeIfProviderThreadOwner({ + providerThreadId, + runId, + activeAttemptId: replacementAttemptId, + expectedLastRunOrdinal: 1, + events: [ + { + id: EventId.make("event:foundation-provider-thread-owner:stale-ordinal"), + type: "provider-thread.updated", + threadId, + driver: providerDriver, + providerInstanceId, + occurredAt: afterReplacement, + payload: baseProviderThread, + }, + ], + }); + assert.isFalse(staleOrdinalWrite.committed); + assert.deepEqual(staleOrdinalWrite.storedEvents, []); + + const projection = yield* projectionStore.getThreadProjection(threadId); + const providerThread = projection.providerThreads.find( + (candidate) => candidate.id === providerThreadId, + ); + assert.isDefined(providerThread); + assert.equal(providerThread?.lastRunOrdinal, 2); + assert.equal(providerThread?.status, "active"); + assert.deepEqual(providerThread?.pendingBackgroundTasks ?? [], []); + }), + ); + it.effect("interrupts a running process-bound effect when it is cancelled", () => Effect.gen(function* () { const outbox = yield* EffectOutboxV2; diff --git a/apps/server/src/orchestration-v2/ProjectionStore.test.ts b/apps/server/src/orchestration-v2/ProjectionStore.test.ts index 41f9f768d8d8..98cec43d2bc1 100644 --- a/apps/server/src/orchestration-v2/ProjectionStore.test.ts +++ b/apps/server/src/orchestration-v2/ProjectionStore.test.ts @@ -818,6 +818,9 @@ it.layer(TestLayer)("ProjectionStoreV2", (it) => { const assistantMessageId = MessageId.make("message:projection-rollback-prune:assistant"); const userTurnItemId = TurnItemId.make("turn-item:projection-rollback-prune:user"); const assistantTurnItemId = TurnItemId.make("turn-item:projection-rollback-prune:assistant"); + const backgroundTurnItemId = TurnItemId.make( + "turn-item:projection-rollback-prune:background", + ); yield* projectionStore.apply({ id: EventId.make("event:projection-rollback-prune:thread-created"), @@ -1103,6 +1106,33 @@ it.layer(TestLayer)("ProjectionStoreV2", (it) => { streaming: false, }, }); + yield* projectionStore.apply({ + id: EventId.make("event:projection-rollback-prune:background-item"), + type: "turn-item.updated", + threadId, + runId, + nodeId: rootNodeId, + driver, + occurredAt: now, + payload: { + id: backgroundTurnItemId, + threadId, + runId, + nodeId: rootNodeId, + providerThreadId, + providerTurnId: null, + nativeItemRef: null, + parentItemId: null, + ordinal: 300, + status: "running", + title: "rolled back background command", + startedAt: now, + completedAt: null, + updatedAt: now, + type: "command_execution", + input: "sleep 60", + }, + }); yield* projectionStore.apply({ id: EventId.make("event:projection-rollback-prune:run-rolled-back"), type: "run.updated", @@ -1171,8 +1201,16 @@ it.layer(TestLayer)("ProjectionStoreV2", (it) => { ); assert.lengthOf(projection.providerTurns, 1); assert.lengthOf(projection.messages, 2); - assert.lengthOf(projection.turnItems, 2); + assert.lengthOf(projection.turnItems, 3); assert.lengthOf(projection.visibleTurnItems, 0); + + // A rolled-back run's background item is abandoned, not pending. The + // shell must not report it as Waiting, or the sidebar shows Waiting for + // work nothing will ever finish. + const shell = yield* projectionStore.getShellSnapshot(); + const rolledBackShellThread = shell.threads.find((entry) => entry.id === threadId); + assert.isDefined(rolledBackShellThread); + assert.deepEqual(rolledBackShellThread?.pendingBackgroundTasks ?? [], []); }), ); diff --git a/apps/server/src/orchestration-v2/ProjectionStore.ts b/apps/server/src/orchestration-v2/ProjectionStore.ts index 783f338ed78a..84fb7e7a8917 100644 --- a/apps/server/src/orchestration-v2/ProjectionStore.ts +++ b/apps/server/src/orchestration-v2/ProjectionStore.ts @@ -36,6 +36,7 @@ import { isOrchestrationV2SupersededInterrupt, isOrchestrationV2TurnItemVisible, } from "@t3tools/shared/orchestrationV2Timeline"; +import { derivePendingBackgroundWork } from "@t3tools/shared/orchestrationV2PendingBackgroundWork"; import * as Context from "effect/Context"; import * as DateTime from "effect/DateTime"; import * as Effect from "effect/Effect"; @@ -441,6 +442,7 @@ type ShellThreadRow = { readonly latest_run_started_at: string | null; readonly latest_run_completed_at: string | null; readonly active_run_id: string | null; + readonly activity_run_status: string | null; readonly last_error: string | null; readonly pending_request_payload_json: string | null; readonly latest_message_payload_json: string | null; @@ -817,6 +819,10 @@ export function threadShellFromProjection( projection.runs .filter(isInterruptibleRunForShell) .toSorted((left, right) => right.ordinal - left.ordinal)[0] ?? null; + const activityRun = + projection.runs + .filter(isActivityRunForShell) + .toSorted((left, right) => right.ordinal - left.ordinal)[0] ?? null; const pendingRuntimeRequest = projection.runtimeRequests .filter((request) => request.status === "pending") @@ -843,6 +849,13 @@ export function threadShellFromProjection( (left, right) => DateTime.toEpochMillis(right.updatedAt) - DateTime.toEpochMillis(left.updatedAt), )[0] ?? null; + const pendingBackgroundTasks = derivePendingBackgroundWork({ + latestRun, + providerThreads: projection.providerThreads, + turnItems: projection.turnItems, + activeProviderThreadId: projection.thread.activeProviderThreadId, + runs: projection.runs, + }); return { createdBy: projection.thread.createdBy, creationSource: projection.thread.creationSource, @@ -866,6 +879,7 @@ export function threadShellFromProjection( latestRunStartedAt: latestRun?.startedAt ?? null, latestRunCompletedAt: latestRun?.completedAt ?? null, activeRunId: activeRun?.id ?? null, + activityRunStatus: activityRun?.status ?? null, status: latestRun?.status ?? "idle", lastError: providerSession?.lastError ?? null, pendingRuntimeRequest: @@ -889,6 +903,7 @@ export function threadShellFromProjection( hasActionableProposedPlan: projection.plans.some( (plan) => plan.kind === "proposed_plan" && plan.status === "active", ), + pendingBackgroundTasks: [...pendingBackgroundTasks], itemCount: activeLocalTurnItems(projection).length, visibleItemCount: projection.visibleTurnItems.length, createdAt: projection.thread.createdAt, @@ -909,6 +924,16 @@ function isInterruptibleRunForShell(run: OrchestrationV2ThreadProjection["runs"] return run.status === "preparing" || run.status === "starting" || run.status === "running"; } +type ShellActivityRunStatus = "preparing" | "running" | "starting" | "waiting"; + +function isActivityRunForShell( + run: OrchestrationV2ThreadProjection["runs"][number], +): run is OrchestrationV2ThreadProjection["runs"][number] & { + readonly status: ShellActivityRunStatus; +} { + return isInterruptibleRunForShell(run) || run.status === "waiting"; +} + type ShellThreadState = { readonly thread: OrchestrationV2ThreadProjection["thread"]; readonly latestRunId: RunId | null; @@ -917,11 +942,13 @@ type ShellThreadState = { readonly latestRunStartedAt: DateTime.Utc | null; readonly latestRunCompletedAt: DateTime.Utc | null; readonly activeRunId: RunId | null; + readonly activityRunStatus: ShellActivityRunStatus | null; readonly lastError: string | null; readonly pendingRuntimeRequest: OrchestrationV2ThreadProjection["runtimeRequests"][number] | null; readonly latestVisibleMessage: OrchestrationV2ConversationMessage | null; readonly latestUserMessageAt: DateTime.Utc | null; readonly hasActionableProposedPlan: boolean; + readonly pendingBackgroundTasks: OrchestrationV2ThreadShell["pendingBackgroundTasks"]; readonly itemCount: number; readonly runlessItemCount: number; readonly updatedAt: OrchestrationV2ThreadProjection["updatedAt"]; @@ -1038,6 +1065,7 @@ function shellFromState(input: { latestRunStartedAt: input.state.latestRunStartedAt, latestRunCompletedAt: input.state.latestRunCompletedAt, activeRunId: input.state.activeRunId, + activityRunStatus: input.state.activityRunStatus, status: input.state.latestRunStatus, lastError: input.state.lastError, pendingRuntimeRequest: @@ -1059,6 +1087,7 @@ function shellFromState(input: { }, latestUserMessageAt: input.state.latestUserMessageAt, hasActionableProposedPlan: input.state.hasActionableProposedPlan, + pendingBackgroundTasks: input.state.pendingBackgroundTasks, itemCount: input.state.itemCount, visibleItemCount: input.visibleItemCount, createdAt: input.state.thread.createdAt, @@ -2242,6 +2271,14 @@ export const layer: Layer.Layer = ORDER BY r.ordinal DESC, r.run_id DESC LIMIT 1 ) AS active_run_id, + ( + SELECT r.status + FROM orchestration_v2_projection_runs r + WHERE r.thread_id = t.thread_id + AND r.status IN ('preparing', 'starting', 'running', 'waiting') + ORDER BY r.ordinal DESC, r.run_id DESC + LIMIT 1 + ) AS activity_run_status, ( SELECT json_extract(session.payload_json, '$.lastError') FROM orchestration_v2_projection_provider_sessions session @@ -2329,6 +2366,44 @@ export const layer: Layer.Layer = GROUP BY thread_id, run_id `; + const selectShellProviderThreadRows = (threadIds?: ReadonlyArray) => + threadIds === undefined + ? sql` + SELECT thread_id, payload_json + FROM orchestration_v2_projection_provider_threads + WHERE thread_id IS NOT NULL + ` + : sql` + SELECT thread_id, payload_json + FROM orchestration_v2_projection_provider_threads + WHERE thread_id IN ${sql.in(threadIds)} + `; + + const selectShellPendingTurnItemRows = (threadIds?: ReadonlyArray) => + threadIds === undefined + ? sql` + SELECT i.thread_id, i.payload_json + FROM orchestration_v2_projection_turn_items i + LEFT JOIN orchestration_v2_projection_runs r + ON r.run_id = i.run_id + WHERE i.type IN ('command_execution', 'dynamic_tool', 'subagent') + AND i.status NOT IN ('completed', 'interrupted', 'failed', 'cancelled') + -- A rolled-back run's items are abandoned, not pending. Without + -- this the shell reports Waiting for work no one will finish, + -- matching the item_count query's exclusion above. + AND (i.run_id IS NULL OR r.status <> 'rolled_back') + ` + : sql` + SELECT i.thread_id, i.payload_json + FROM orchestration_v2_projection_turn_items i + LEFT JOIN orchestration_v2_projection_runs r + ON r.run_id = i.run_id + WHERE i.type IN ('command_execution', 'dynamic_tool', 'subagent') + AND i.status NOT IN ('completed', 'interrupted', 'failed', 'cancelled') + AND (i.run_id IS NULL OR r.status <> 'rolled_back') + AND i.thread_id IN ${sql.in(threadIds)} + `; + const runMapsByThreadId = (input: { readonly runRows: ReadonlyArray; readonly itemCountRows: ReadonlyArray; @@ -2352,13 +2427,60 @@ export const layer: Layer.Layer = return { runOrdinalsByThreadId, itemCountsByThreadId }; }; + const pendingBackgroundDataByThreadId = (input: { + readonly providerThreadRows: ReadonlyArray; + readonly pendingTurnItemRows: ReadonlyArray; + }) => + Effect.gen(function* () { + const providerThreadsByThreadId = new Map< + ThreadId, + Array + >(); + for (const row of input.providerThreadRows) { + const providerThread = yield* decodeProviderThreadPayload(row.payload_json); + const threadId = + row.thread_id.length > 0 ? ThreadId.make(row.thread_id) : providerThread.appThreadId; + if (threadId === null) { + continue; + } + const existing = providerThreadsByThreadId.get(threadId) ?? []; + existing.push(providerThread); + providerThreadsByThreadId.set(threadId, existing); + } + + const pendingTurnItemsByThreadId = new Map>(); + for (const row of input.pendingTurnItemRows) { + const turnItem = yield* decodeTurnItemPayload(row.payload_json); + const threadId = ThreadId.make(row.thread_id); + const existing = pendingTurnItemsByThreadId.get(threadId) ?? []; + existing.push(turnItem); + pendingTurnItemsByThreadId.set(threadId, existing); + } + + return { providerThreadsByThreadId, pendingTurnItemsByThreadId }; + }); + const shellThreadStateFromRow = (input: { readonly row: ShellThreadRow; readonly runOrdinalsByThreadId: ReadonlyMap>; readonly itemCountsByThreadId: ReadonlyMap>; + readonly providerThreadsByThreadId: ReadonlyMap< + ThreadId, + ReadonlyArray + >; + readonly pendingTurnItemsByThreadId: ReadonlyMap< + ThreadId, + ReadonlyArray + >; }) => Effect.gen(function* () { - const { row, runOrdinalsByThreadId, itemCountsByThreadId } = input; + const { + row, + runOrdinalsByThreadId, + itemCountsByThreadId, + providerThreadsByThreadId, + pendingTurnItemsByThreadId, + } = input; const thread = yield* decodeThreadPayload(row.payload_json); const pendingRuntimeRequest = row.pending_request_payload_json === null @@ -2368,10 +2490,28 @@ export const layer: Layer.Layer = row.latest_message_payload_json === null ? null : yield* decodeMessagePayload(row.latest_message_payload_json); + const latestRunId = row.latest_run_id === null ? null : RunId.make(row.latest_run_id); + const latestRunStatus = shellStatusFromStoredRunStatus(row.latest_run_status); + const pendingBackgroundTasks = [ + ...derivePendingBackgroundWork({ + latestRun: + latestRunId === null || latestRunStatus === "idle" + ? null + : { + id: latestRunId, + ordinal: 0, + status: latestRunStatus, + }, + providerThreads: providerThreadsByThreadId.get(thread.id) ?? [], + turnItems: pendingTurnItemsByThreadId.get(thread.id) ?? [], + activeProviderThreadId: thread.activeProviderThreadId, + hasActiveRun: row.active_run_id !== null, + }), + ]; return { thread, - latestRunId: row.latest_run_id === null ? null : RunId.make(row.latest_run_id), - latestRunStatus: shellStatusFromStoredRunStatus(row.latest_run_status), + latestRunId, + latestRunStatus, latestRunRequestedAt: row.latest_run_requested_at === null ? null @@ -2385,6 +2525,13 @@ export const layer: Layer.Layer = ? null : DateTime.makeUnsafe(row.latest_run_completed_at), activeRunId: row.active_run_id === null ? null : RunId.make(row.active_run_id), + activityRunStatus: + row.activity_run_status === "preparing" || + row.activity_run_status === "starting" || + row.activity_run_status === "running" || + row.activity_run_status === "waiting" + ? row.activity_run_status + : null, lastError: row.last_error, pendingRuntimeRequest, latestVisibleMessage, @@ -2393,6 +2540,7 @@ export const layer: Layer.Layer = ? null : DateTime.makeUnsafe(row.latest_user_message_at), hasActionableProposedPlan: row.has_actionable_proposed_plan === 1, + pendingBackgroundTasks, itemCount: row.item_count, runlessItemCount: row.runless_item_count, updatedAt: thread.updatedAt, @@ -2405,7 +2553,14 @@ export const layer: Layer.Layer = sql .withTransaction( Effect.gen(function* () { - const [threadRows, runRows, itemCountRows, sequenceRows] = yield* Effect.all([ + const [ + threadRows, + runRows, + itemCountRows, + sequenceRows, + providerThreadRows, + pendingTurnItemRows, + ] = yield* Effect.all([ selectShellThreadRows(), selectShellRunRows(), selectShellRunItemCounts(), @@ -2415,15 +2570,24 @@ export const layer: Layer.Layer = WHERE application_event_version = 2 AND aggregate_kind = 'thread' `, + selectShellProviderThreadRows(), + selectShellPendingTurnItemRows(), ]); const { runOrdinalsByThreadId, itemCountsByThreadId } = runMapsByThreadId({ runRows, itemCountRows, }); - + const { providerThreadsByThreadId, pendingTurnItemsByThreadId } = + yield* pendingBackgroundDataByThreadId({ providerThreadRows, pendingTurnItemRows }); const states = yield* Effect.forEach(threadRows, (row) => - shellThreadStateFromRow({ row, runOrdinalsByThreadId, itemCountsByThreadId }), + shellThreadStateFromRow({ + row, + runOrdinalsByThreadId, + itemCountsByThreadId, + providerThreadsByThreadId, + pendingTurnItemsByThreadId, + }), ); const statesByThreadId = new Map(states.map((state) => [state.thread.id, state])); @@ -2484,17 +2648,28 @@ export const layer: Layer.Layer = } const threadIds = [...rowsByThreadId.keys()]; - const [runRows, itemCountRows] = yield* Effect.all([ - selectShellRunRows(threadIds), - selectShellRunItemCounts(threadIds), - ]); + const [runRows, itemCountRows, providerThreadRows, pendingTurnItemRows] = + yield* Effect.all([ + selectShellRunRows(threadIds), + selectShellRunItemCounts(threadIds), + selectShellProviderThreadRows(threadIds), + selectShellPendingTurnItemRows(threadIds), + ]); const { runOrdinalsByThreadId, itemCountsByThreadId } = runMapsByThreadId({ runRows, itemCountRows, }); + const { providerThreadsByThreadId, pendingTurnItemsByThreadId } = + yield* pendingBackgroundDataByThreadId({ providerThreadRows, pendingTurnItemRows }); const states = yield* Effect.forEach([...rowsByThreadId.values()], (row) => - shellThreadStateFromRow({ row, runOrdinalsByThreadId, itemCountsByThreadId }), + shellThreadStateFromRow({ + row, + runOrdinalsByThreadId, + itemCountsByThreadId, + providerThreadsByThreadId, + pendingTurnItemsByThreadId, + }), ); const statesByThreadId = new Map(states.map((state) => [state.thread.id, state])); const state = statesByThreadId.get(threadId); diff --git a/apps/server/src/orchestration-v2/ProviderAdapter.ts b/apps/server/src/orchestration-v2/ProviderAdapter.ts index a86fb0b48f68..ab163db12e0e 100644 --- a/apps/server/src/orchestration-v2/ProviderAdapter.ts +++ b/apps/server/src/orchestration-v2/ProviderAdapter.ts @@ -485,6 +485,15 @@ export interface ProviderAdapterV2SessionRuntime { * here so the session manager defers idle release while it is pending. */ readonly hasPendingBackgroundWork?: Effect.Effect; + /** + * Per-provider-thread pending work for root-run ingestion stop gates. When + * present, RunExecutionService uses only this probe (never the session-wide + * hasPendingBackgroundWork) so sibling native threads cannot pin an + * unrelated root subscription open. + */ + readonly hasPendingBackgroundWorkForThread?: ( + providerThread: OrchestrationV2ProviderThread, + ) => Effect.Effect; readonly ensureThread: ( input: ProviderAdapterV2EnsureThreadInput, ) => Effect.Effect; diff --git a/apps/server/src/orchestration-v2/ProviderContinuationService.test.ts b/apps/server/src/orchestration-v2/ProviderContinuationService.test.ts index e1274c8ffc1e..03d218289502 100644 --- a/apps/server/src/orchestration-v2/ProviderContinuationService.test.ts +++ b/apps/server/src/orchestration-v2/ProviderContinuationService.test.ts @@ -588,6 +588,25 @@ describe("ProviderContinuationService", () => { }); }); + it.effect("queues a continuation behind an active user run", () => { + return Effect.gen(function* () { + const dispatched = yield* Queue.unbounded(); + yield* Effect.gen(function* () { + const requests = yield* ProviderContinuationRequests; + yield* requests.offer(request()); + const command = yield* Queue.take(dispatched); + assert.deepEqual((command as { readonly dispatchMode?: unknown }).dispatchMode, { + type: "queue_after_active", + }); + }).pipe( + Effect.provide( + testLayer({ dispatched, getThreadProjection: () => Effect.succeed(projection) }), + ), + Effect.scoped, + ); + }); + }); + it.effect("drops a request invalidated before dispatch", () => { return Effect.gen(function* () { const dispatched = yield* Queue.unbounded(); diff --git a/apps/server/src/orchestration-v2/ProviderEventIngestor.test.ts b/apps/server/src/orchestration-v2/ProviderEventIngestor.test.ts index 5b77cb6e659f..463dd01d1d2a 100644 --- a/apps/server/src/orchestration-v2/ProviderEventIngestor.test.ts +++ b/apps/server/src/orchestration-v2/ProviderEventIngestor.test.ts @@ -6,8 +6,13 @@ import { type OrchestrationV2AppThread, type OrchestrationV2DomainEvent, type OrchestrationV2ProviderThread, + type OrchestrationV2Run, + type OrchestrationV2TurnItem, ProviderDriverKind, ProviderInstanceId, + RunAttemptId, + RunId, + TurnItemId, } from "@t3tools/contracts"; import * as DateTime from "effect/DateTime"; import * as Effect from "effect/Effect"; @@ -28,6 +33,12 @@ import { layer as providerEventIngestorLayer, } from "./ProviderEventIngestor.ts"; import { makeProviderFailure } from "./ProviderFailure.ts"; +import { + makeProviderEventRoutingState, + type ProviderEventRouteIdentity, + routeProviderEvent, + selectInheritedBackgroundTurnItems, +} from "./RunExecutionService.ts"; const TestDatabaseLayer = SqlitePersistenceMemory; const TestStoresLayer = Layer.merge(eventStoreLayer, projectionStoreLayer).pipe( @@ -233,6 +244,263 @@ layer("ProviderEventIngestorV2", (it) => { }), ); + it.effect("persists an interrupted run's inherited terminal through the live run router", () => + Effect.gen(function* () { + const now = yield* DateTime.now; + const eventSink = yield* EventSinkV2; + const projectionStore = yield* ProjectionStoreV2; + const ingestor = yield* ProviderEventIngestorV2; + const idAllocator = yield* IdAllocatorV2; + const threadEvent = yield* threadCreatedEvent(now); + const priorRunId = RunId.make("run:provider-event-inherited:prior"); + const currentRunId = RunId.make("run:provider-event-inherited:current"); + const itemId = TurnItemId.make("turn-item:provider-event-inherited"); + const providerSessionId = yield* idAllocator.allocate.providerSession({ + providerInstanceId: modelSelection.instanceId, + threadId: threadEvent.threadId, + }); + const providerThreadId = idAllocator.derive.providerThread({ + driver: CODEX_DRIVER, + nativeThreadId: "native-thread-inherited", + }); + const providerTurnId = idAllocator.derive.providerTurn({ + driver: CODEX_DRIVER, + nativeTurnId: "native-turn-inherited", + }); + const runningItem = { + id: itemId, + threadId: threadEvent.threadId, + runId: priorRunId, + nodeId: NodeId.make("node:provider-event-inherited"), + providerThreadId, + providerTurnId, + nativeItemRef: null, + parentItemId: null, + ordinal: 101, + status: "running", + title: "Inherited background command", + startedAt: now, + completedAt: null, + updatedAt: now, + type: "command_execution", + input: "sleep 60", + } satisfies OrchestrationV2TurnItem; + const terminalItem = { + ...runningItem, + status: "completed" as const, + completedAt: now, + updatedAt: now, + }; + + yield* eventSink.write({ events: [threadEvent] }); + yield* ingestor.ingestNormalized({ + providerSessionId, + providerInstanceId: modelSelection.instanceId, + threadId: threadEvent.threadId, + runId: priorRunId, + event: { type: "turn_item.updated", driver: CODEX_DRIVER, turnItem: runningItem }, + }); + + const identity: ProviderEventRouteIdentity = { + threadId: threadEvent.threadId, + runId: currentRunId, + attemptId: RunAttemptId.make("attempt:provider-event-inherited:current"), + providerThreadId, + }; + const inheritedBackgroundTurnItems = selectInheritedBackgroundTurnItems({ + threadId: threadEvent.threadId, + currentProviderThreadId: providerThreadId, + currentRunOrdinal: 2, + runs: [ + { + id: priorRunId, + threadId: threadEvent.threadId, + ordinal: 1, + status: "interrupted", + } as OrchestrationV2Run, + { + id: currentRunId, + threadId: threadEvent.threadId, + ordinal: 2, + status: "running", + } as OrchestrationV2Run, + ], + turnItems: [runningItem], + }); + const routeState = makeProviderEventRoutingState({ + identity, + inheritedBackgroundTurnItems, + providerTurnId: null, + }); + const terminalEvent = { + type: "turn_item.updated", + driver: CODEX_DRIVER, + turnItem: terminalItem, + } as const; + const [accepted] = routeProviderEvent(terminalEvent, identity, routeState); + assert.isTrue(accepted); + + const stored = yield* ingestor.ingestNormalized({ + providerSessionId, + providerInstanceId: modelSelection.instanceId, + threadId: threadEvent.threadId, + runId: currentRunId, + event: terminalEvent, + }); + const projection = yield* projectionStore.getThreadProjection(threadEvent.threadId); + const persisted = projection.turnItems.find((item) => item.id === itemId); + + assert.equal(stored.length, 1); + assert.equal(stored[0]?.event.type, "turn-item.updated"); + assert.equal(persisted?.runId, priorRunId); + assert.equal(persisted?.threadId, threadEvent.threadId); + assert.equal(persisted?.status, "completed"); + }), + ); + + it.effect("persists a completed run's late background terminal exactly once", () => + Effect.gen(function* () { + const now = yield* DateTime.now; + const eventSink = yield* EventSinkV2; + const eventStore = yield* EventStoreV2; + const ingestor = yield* ProviderEventIngestorV2; + const idAllocator = yield* IdAllocatorV2; + const threadEvent = yield* threadCreatedEvent(now); + const priorRunId = RunId.make("run:provider-event-completed:prior"); + const currentRunId = RunId.make("run:provider-event-completed:current"); + const itemId = TurnItemId.make("turn-item:provider-event-completed"); + const providerSessionId = yield* idAllocator.allocate.providerSession({ + providerInstanceId: modelSelection.instanceId, + threadId: threadEvent.threadId, + }); + const providerThreadId = idAllocator.derive.providerThread({ + driver: CODEX_DRIVER, + nativeThreadId: "native-thread-completed", + }); + const providerTurnId = idAllocator.derive.providerTurn({ + driver: CODEX_DRIVER, + nativeTurnId: "native-turn-completed", + }); + const runningItem = { + id: itemId, + threadId: threadEvent.threadId, + runId: priorRunId, + nodeId: NodeId.make("node:provider-event-completed"), + providerThreadId, + providerTurnId, + nativeItemRef: null, + parentItemId: null, + ordinal: 101, + status: "running", + title: "Completed run background command", + startedAt: now, + completedAt: null, + updatedAt: now, + type: "command_execution", + input: "sleep 60", + } satisfies OrchestrationV2TurnItem; + const terminalEvent = { + type: "turn_item.updated", + driver: CODEX_DRIVER, + turnItem: { + ...runningItem, + status: "completed" as const, + completedAt: now, + updatedAt: now, + }, + } as const; + + yield* eventSink.write({ events: [threadEvent] }); + yield* ingestor.ingestNormalized({ + providerSessionId, + providerInstanceId: modelSelection.instanceId, + threadId: threadEvent.threadId, + runId: priorRunId, + event: { type: "turn_item.updated", driver: CODEX_DRIVER, turnItem: runningItem }, + }); + + const priorIdentity: ProviderEventRouteIdentity = { + threadId: threadEvent.threadId, + runId: priorRunId, + attemptId: RunAttemptId.make("attempt:provider-event-completed:prior"), + providerThreadId, + }; + const currentIdentity: ProviderEventRouteIdentity = { + threadId: threadEvent.threadId, + runId: currentRunId, + attemptId: RunAttemptId.make("attempt:provider-event-completed:current"), + providerThreadId, + }; + const inheritedBackgroundTurnItems = selectInheritedBackgroundTurnItems({ + threadId: threadEvent.threadId, + currentProviderThreadId: providerThreadId, + currentRunOrdinal: 2, + runs: [ + { + id: priorRunId, + threadId: threadEvent.threadId, + ordinal: 1, + status: "completed", + } as OrchestrationV2Run, + { + id: currentRunId, + threadId: threadEvent.threadId, + ordinal: 2, + status: "running", + } as OrchestrationV2Run, + ], + turnItems: [runningItem], + }); + const routers = [ + { + identity: priorIdentity, + state: makeProviderEventRoutingState({ + identity: priorIdentity, + providerTurnId: providerTurnId, + }), + }, + { + identity: currentIdentity, + state: makeProviderEventRoutingState({ + identity: currentIdentity, + inheritedBackgroundTurnItems, + providerTurnId: null, + }), + }, + ]; + const acceptedRouters = routers.filter( + ({ identity, state }) => routeProviderEvent(terminalEvent, identity, state)[0], + ); + + yield* Effect.forEach( + acceptedRouters, + ({ identity }) => + ingestor.ingestNormalized({ + providerSessionId, + providerInstanceId: modelSelection.instanceId, + threadId: threadEvent.threadId, + runId: identity.runId, + event: terminalEvent, + }), + { concurrency: 1 }, + ); + + const storedEvents = yield* eventStore + .read({ threadId: threadEvent.threadId }) + .pipe(Stream.runCollect); + const storedTerminals = Array.from(storedEvents).filter( + (stored) => + stored.event.type === "turn-item.updated" && + stored.event.payload.id === itemId && + stored.event.payload.status === "completed", + ); + + assert.equal(storedTerminals.length, 1); + assert.equal(acceptedRouters.length, 1); + assert.equal(acceptedRouters[0]?.identity.runId, priorRunId); + }), + ); + it.effect("persists a failed provider terminal as one expected error item", () => Effect.gen(function* () { const now = yield* DateTime.now; diff --git a/apps/server/src/orchestration-v2/ProviderEventIngestor.ts b/apps/server/src/orchestration-v2/ProviderEventIngestor.ts index 233db488e85e..46bd6174af2a 100644 --- a/apps/server/src/orchestration-v2/ProviderEventIngestor.ts +++ b/apps/server/src/orchestration-v2/ProviderEventIngestor.ts @@ -6,6 +6,7 @@ import { type OrchestrationV2Run, ProviderInstanceId, ProviderSessionId, + ProviderThreadId, RawEventId, RunAttemptId, RunId, @@ -81,6 +82,16 @@ export interface ProviderEventIngestorV2Shape { readonly activeAttemptId: RunAttemptId; readonly expectedStatus: OrchestrationV2Run["status"]; }; + /** + * Atomically reject provider-thread snapshots from an attempt that no + * longer owns the run or from a run that no longer owns the thread. + */ + readonly writeIfProviderThreadOwner?: { + readonly providerThreadId: ProviderThreadId; + readonly runId: RunId; + readonly activeAttemptId: RunAttemptId; + readonly expectedLastRunOrdinal: number; + }; }, ) => Effect.Effect, ProviderEventIngestorV2Error>; } @@ -283,6 +294,16 @@ export const layer: Layer.Layer { + const threadId = ThreadId.make("thread_recovery_background_no_provider_threads"); + const runId = RunId.make("run_recovery_background_no_provider_threads"); + const itemId = TurnItemId.make("turn_item_recovery_background_no_provider_threads"); + const providerInstanceId = ProviderInstanceId.make("codex"); + let committedInput: Parameters[0] | null = + null; + const projection = { + thread: { id: threadId, providerInstanceId }, + runtimeRequests: [], + providerSessions: [], + providerThreads: [], + providerTurns: [], + runs: [{ id: runId, status: "completed", providerInstanceId }], + attempts: [], + nodes: [], + subagents: [], + messages: [], + turnItems: [ + { + id: itemId, + runId: null, + nodeId: null, + providerThreadId: null, + type: "command_execution", + status: "running", + }, + ], + } as unknown as OrchestrationV2ThreadProjection; + const layer = ProviderRuntimeRecovery.layer.pipe( + Layer.provide( + Layer.mergeAll( + Layer.mock(ProjectionStore.ProjectionStoreV2)({ + getShellSnapshot: () => + Effect.succeed({ + schemaVersion: 2, + snapshotSequence: 0, + threads: [{ id: threadId }], + archivedThreads: [], + } as never), + getThreadProjection: () => Effect.succeed(projection), + }), + Layer.mock(EventSink.EventSinkV2)({ + commitCommand: (input) => { + committedInput = input; + return Effect.succeed({ committed: true, cancelledEffectCount: 0 } as never); + }, + }), + IdAllocator.layer, + Layer.mock(EffectWorker.OrchestrationEffectWorkerV2)({ runOnce: Effect.succeed(false) }), + Layer.mock(EffectOutbox.EffectOutboxV2)({ + reconcileAfterProcessLoss: Effect.succeed({ requeued: 0, cancelled: 0 }), + }), + ), + ), + ); + + await Effect.gen(function* () { + yield* (yield* ProviderRuntimeRecovery.ProviderRuntimeRecoveryService).reconcile("startup"); + const event = committedInput?.events.find( + (candidate) => candidate.type === "turn-item.updated", + ); + expect(event?.providerInstanceId).toBe(providerInstanceId); + expect(event?.type === "turn-item.updated" ? event.payload.status : null).toBe("cancelled"); + }).pipe(Effect.provide(layer), Effect.runPromise); +}); diff --git a/apps/server/src/orchestration-v2/ProviderRuntimeRecoveryService.test.ts b/apps/server/src/orchestration-v2/ProviderRuntimeRecoveryService.test.ts index 932beec89958..8e8fe66253a9 100644 --- a/apps/server/src/orchestration-v2/ProviderRuntimeRecoveryService.test.ts +++ b/apps/server/src/orchestration-v2/ProviderRuntimeRecoveryService.test.ts @@ -192,57 +192,114 @@ it.effect("uses the same reconciliation path to cancel runtime requests during s }).pipe(Effect.provide(layer)); }); -it.effect("preserves a waiting run while its replay-safe checkpoint capture is unsettled", () => { - const threadId = ThreadId.make("thread_waiting_checkpoint"); - const runId = RunId.make("run_waiting_checkpoint"); - const committed = vi.fn(() => Effect.succeed({ committed: true } as never)); - const projection = { - thread: { id: threadId }, - runtimeRequests: [], - providerSessions: [], - providerThreads: [], - runs: [{ id: runId, status: "waiting" }], - } as unknown as OrchestrationV2ThreadProjection; - const layer = ProviderRuntimeRecovery.layer.pipe( - Layer.provide( - Layer.mergeAll( - Layer.mock(ProjectionStore.ProjectionStoreV2)({ - getShellSnapshot: () => - Effect.succeed({ - schemaVersion: 2, - snapshotSequence: 0, - threads: [{ id: threadId }], - archivedThreads: [], - } as never), - getThreadProjection: () => Effect.succeed(projection), - }), - Layer.mock(EventSink.EventSinkV2)({ commitCommand: committed }), - IdAllocator.layer, - Layer.mock(EffectWorker.OrchestrationEffectWorkerV2)({ runOnce: Effect.succeed(false) }), - Layer.mock(EffectOutbox.EffectOutboxV2)({ - listByCommandId: () => - Effect.succeed([ - { - request: { type: "checkpoint.capture", runId }, - status: "running", - }, - ] as never), - cancelUnsettled: () => Effect.succeed([]), - signalCancellations: () => Effect.void, - reconcileAfterProcessLoss: Effect.succeed({ requeued: 1, cancelled: 0 }), - }), +it.effect( + "preserves a replayable waiting run while cancelling its process-bound background work", + () => { + const threadId = ThreadId.make("thread_waiting_checkpoint"); + const runId = RunId.make("run_waiting_checkpoint"); + const providerThreadId = ProviderThreadId.make("provider_thread_waiting_checkpoint"); + const providerInstanceId = ProviderInstanceId.make("claude"); + const backgroundItemId = TurnItemId.make("turn_item_waiting_checkpoint"); + let committedInput: Parameters[0] | null = + null; + const projection = { + thread: { id: threadId }, + runtimeRequests: [], + providerSessions: [], + providerThreads: [ + { + id: providerThreadId, + driver: ProviderDriverKind.make("claude"), + providerInstanceId, + status: "idle", + pendingBackgroundTasks: [ + { taskId: String(backgroundItemId), description: "Finish background task" }, + ], + }, + ], + runs: [{ id: runId, status: "waiting", providerInstanceId }], + attempts: [], + nodes: [], + subagents: [], + messages: [], + turnItems: [ + { + id: backgroundItemId, + runId, + nodeId: null, + providerThreadId, + type: "command_execution", + status: "running", + }, + ], + } as unknown as OrchestrationV2ThreadProjection; + const layer = ProviderRuntimeRecovery.layer.pipe( + Layer.provide( + Layer.mergeAll( + Layer.mock(ProjectionStore.ProjectionStoreV2)({ + getShellSnapshot: () => + Effect.succeed({ + schemaVersion: 2, + snapshotSequence: 0, + threads: [{ id: threadId }], + archivedThreads: [], + } as never), + getThreadProjection: () => Effect.succeed(projection), + }), + Layer.mock(EventSink.EventSinkV2)({ + commitCommand: (input) => { + committedInput = input; + return Effect.succeed({ + committed: true, + cancelledEffectCount: 0, + } as never); + }, + }), + IdAllocator.layer, + Layer.mock(EffectWorker.OrchestrationEffectWorkerV2)({ runOnce: Effect.succeed(false) }), + Layer.mock(EffectOutbox.EffectOutboxV2)({ + listByCommandId: () => + Effect.succeed([ + { + request: { type: "checkpoint.capture", runId }, + status: "running", + }, + ] as never), + reconcileAfterProcessLoss: Effect.succeed({ requeued: 1, cancelled: 0 }), + }), + ), ), - ), - ); + ); - return Effect.gen(function* () { - const summary = - yield* (yield* ProviderRuntimeRecovery.ProviderRuntimeRecoveryService).reconcile("startup"); - assert.equal(summary.terminalizedRuns, 0); - assert.equal(summary.requeuedEffects, 1); - assert.equal(committed.mock.calls.length, 0); - }).pipe(Effect.provide(layer)); -}); + return Effect.gen(function* () { + const summary = + yield* (yield* ProviderRuntimeRecovery.ProviderRuntimeRecoveryService).reconcile("startup"); + assert.equal(summary.terminalizedRuns, 0); + assert.equal(summary.requeuedEffects, 1); + + const command = committedInput; + assert.isNotNull(command); + if (command === null) return; + assert.isFalse(command.events.some((event) => event.type === "run.updated")); + assert.isTrue( + command.events.some( + (event) => + event.type === "turn-item.updated" && + event.payload.id === backgroundItemId && + event.payload.status === "cancelled", + ), + ); + assert.isTrue( + command.events.some( + (event) => + event.type === "provider-thread.updated" && + event.payload.id === providerThreadId && + event.payload.pendingBackgroundTasks?.length === 0, + ), + ); + }).pipe(Effect.provide(layer)); + }, +); it.effect("cancels a stale waiting run when no checkpoint capture can finish it", () => { const threadId = ThreadId.make("thread_stale_waiting"); @@ -524,3 +581,368 @@ it.effect( }).pipe(Effect.provide(layer)); }, ); + +it.effect( + "clears persisted pendingBackgroundTasks and terminalizes stale background items on settled runs", + () => { + const threadId = ThreadId.make("thread_recovery_background"); + const settledRunId = RunId.make("run_recovery_background_settled"); + const activeRunId = RunId.make("run_recovery_background_active"); + const activeAttemptId = RunAttemptId.make("attempt_recovery_background_active"); + const activeRootNodeId = NodeId.make("node_recovery_background_active"); + const idleProviderThreadId = ProviderThreadId.make("provider_thread_recovery_background_idle"); + const activeProviderThreadId = ProviderThreadId.make( + "provider_thread_recovery_background_active", + ); + const secondaryProviderThreadId = ProviderThreadId.make( + "provider_thread_recovery_background_secondary", + ); + const providerSessionId = ProviderSessionId.make("provider_session_recovery_background"); + const settledStaleItemId = TurnItemId.make("turn_item_recovery_background_stale"); + const activeRunItemId = TurnItemId.make("turn_item_recovery_background_active"); + const nullRunCommandItemId = TurnItemId.make("turn_item_recovery_background_null_run"); + const nullRunSubagentItemId = TurnItemId.make("turn_item_recovery_background_null_subagent"); + const claudeInstanceId = ProviderInstanceId.make("claude"); + const secondaryInstanceId = ProviderInstanceId.make("claude-secondary"); + const subagentInstanceId = ProviderInstanceId.make("claude-subagent"); + let committedInput: Parameters[0] | null = + null; + const projection = { + thread: { id: threadId }, + runtimeRequests: [], + providerSessions: [ + { + id: providerSessionId, + driver: ProviderDriverKind.make("claude"), + providerInstanceId: claudeInstanceId, + status: "ready", + }, + ], + providerThreads: [ + { + id: idleProviderThreadId, + driver: ProviderDriverKind.make("claude"), + // Index-0 is intentionally a different instance so misattribution + // to providerThreads[0] fails the assertions below. + providerInstanceId: claudeInstanceId, + status: "idle", + pendingBackgroundTasks: [{ taskId: "bg-settled", description: "sleep 30" }], + }, + { + id: activeProviderThreadId, + driver: ProviderDriverKind.make("claude"), + providerInstanceId: claudeInstanceId, + status: "active", + pendingBackgroundTasks: [{ taskId: "bg-active", description: "npm test" }], + }, + { + id: secondaryProviderThreadId, + driver: ProviderDriverKind.make("claude"), + providerInstanceId: secondaryInstanceId, + status: "idle", + pendingBackgroundTasks: [], + }, + ], + providerTurns: [], + runs: [ + { + id: settledRunId, + status: "completed", + providerInstanceId: claudeInstanceId, + }, + { + id: activeRunId, + status: "running", + providerInstanceId: claudeInstanceId, + }, + ], + attempts: [ + { + id: activeAttemptId, + runId: activeRunId, + rootNodeId: activeRootNodeId, + status: "running", + }, + ], + nodes: [{ id: activeRootNodeId, runId: activeRunId, status: "running" }], + subagents: [], + messages: [], + turnItems: [ + { + id: settledStaleItemId, + runId: settledRunId, + nodeId: null, + providerThreadId: idleProviderThreadId, + type: "command_execution", + status: "running", + }, + { + id: activeRunItemId, + runId: activeRunId, + nodeId: activeRootNodeId, + providerThreadId: activeProviderThreadId, + type: "dynamic_tool", + status: "running", + }, + { + // Missing run: must attribute via providerThreadId, not index 0. + id: nullRunCommandItemId, + runId: null, + nodeId: null, + providerThreadId: secondaryProviderThreadId, + type: "command_execution", + status: "running", + }, + { + // Missing run with a real matching provider thread whose instance + // differs from the subagent's own: own providerInstanceId must win. + id: nullRunSubagentItemId, + runId: null, + nodeId: null, + providerThreadId: secondaryProviderThreadId, + type: "subagent", + status: "running", + providerInstanceId: subagentInstanceId, + }, + ], + } as unknown as OrchestrationV2ThreadProjection; + const layer = ProviderRuntimeRecovery.layer.pipe( + Layer.provide( + Layer.mergeAll( + Layer.mock(ProjectionStore.ProjectionStoreV2)({ + getShellSnapshot: () => + Effect.succeed({ + schemaVersion: 2, + snapshotSequence: 0, + threads: [{ id: threadId }], + archivedThreads: [], + } as never), + getThreadProjection: () => Effect.succeed(projection), + }), + Layer.mock(EventSink.EventSinkV2)({ + commitCommand: (input) => { + committedInput = input; + return Effect.succeed({ committed: true, cancelledEffectCount: 0 } as never); + }, + }), + IdAllocator.layer, + Layer.mock(EffectWorker.OrchestrationEffectWorkerV2)({ runOnce: Effect.succeed(false) }), + Layer.mock(EffectOutbox.EffectOutboxV2)({ + listByCommandId: () => Effect.succeed([]), + reconcileAfterProcessLoss: Effect.succeed({ requeued: 0, cancelled: 0 }), + }), + ), + ), + ); + + return Effect.gen(function* () { + const summary = + yield* (yield* ProviderRuntimeRecovery.ProviderRuntimeRecoveryService).reconcile("startup"); + assert.equal(summary.terminalizedRuns, 1); + const events = committedInput?.events ?? []; + + const turnItemCancels = events.filter( + (event) => event.type === "turn-item.updated" && event.payload.status === "cancelled", + ); + // Active-run item + settled-run stale + null-run command + null-run subagent. + assert.equal(turnItemCancels.length, 4); + assert.deepEqual( + turnItemCancels + .map((event) => event.type === "turn-item.updated" && event.payload.id) + .sort(), + [activeRunItemId, nullRunCommandItemId, nullRunSubagentItemId, settledStaleItemId].sort(), + ); + + const cancelById = (id: TurnItemId) => + turnItemCancels.find( + (event) => event.type === "turn-item.updated" && event.payload.id === id, + ); + assert.equal(cancelById(nullRunCommandItemId)?.providerInstanceId, secondaryInstanceId); + // Subagent own instance wins over the matching thread's secondary instance. + assert.notEqual(subagentInstanceId, secondaryInstanceId); + assert.equal(cancelById(nullRunSubagentItemId)?.providerInstanceId, subagentInstanceId); + // Settled-run item still prefers the run's provider instance when present. + assert.equal(cancelById(settledStaleItemId)?.providerInstanceId, claudeInstanceId); + + const providerThreadEvents = events.filter( + (event) => event.type === "provider-thread.updated", + ); + // Only threads with active status or nonempty rosters are rewritten. + assert.equal(providerThreadEvents.length, 2); + for (const event of providerThreadEvents) { + if (event.type !== "provider-thread.updated") continue; + assert.deepEqual(event.payload.pendingBackgroundTasks ?? [], []); + } + const idleThreadEvent = providerThreadEvents.find( + (event) => + event.type === "provider-thread.updated" && event.payload.id === idleProviderThreadId, + ); + assert.equal( + idleThreadEvent?.type === "provider-thread.updated" ? idleThreadEvent.payload.status : null, + "idle", + ); + const activeThreadEvent = providerThreadEvents.find( + (event) => + event.type === "provider-thread.updated" && event.payload.id === activeProviderThreadId, + ); + assert.equal( + activeThreadEvent?.type === "provider-thread.updated" + ? activeThreadEvent.payload.status + : null, + "idle", + ); + }).pipe(Effect.provide(layer)); + }, +); + +it.effect( + "terminalizes the linked subagent and node for a stale subagent item on a settled run", + () => { + const threadId = ThreadId.make("thread_recovery_subagent"); + const settledRunId = RunId.make("run_recovery_subagent_settled"); + const providerThreadId = ProviderThreadId.make("provider_thread_recovery_subagent"); + const staleSubagentNodeId = NodeId.make("node_recovery_subagent_stale"); + const doneSubagentNodeId = NodeId.make("node_recovery_subagent_done"); + const staleItemId = TurnItemId.make("turn_item_recovery_subagent_stale"); + const doneItemId = TurnItemId.make("turn_item_recovery_subagent_done"); + const claudeInstanceId = ProviderInstanceId.make("claude"); + let committedInput: Parameters[0] | null = + null; + const projection = { + thread: { id: threadId }, + runtimeRequests: [], + providerSessions: [], + providerThreads: [ + { + id: providerThreadId, + driver: ProviderDriverKind.make("claude"), + providerInstanceId: claudeInstanceId, + status: "idle", + pendingBackgroundTasks: [], + }, + ], + providerTurns: [], + // Settled run: the stale-item loop owns it, not the nonterminal loop. + runs: [{ id: settledRunId, status: "completed", providerInstanceId: claudeInstanceId }], + attempts: [], + nodes: [ + { id: staleSubagentNodeId, runId: settledRunId, status: "running" }, + { id: doneSubagentNodeId, runId: settledRunId, status: "completed" }, + ], + subagents: [ + { + id: staleSubagentNodeId, + runId: settledRunId, + driver: ProviderDriverKind.make("claude"), + providerInstanceId: claudeInstanceId, + status: "running", + }, + { + // Already finished with a real result: must never be overwritten. + id: doneSubagentNodeId, + runId: settledRunId, + driver: ProviderDriverKind.make("claude"), + providerInstanceId: claudeInstanceId, + status: "completed", + result: "done", + }, + ], + messages: [], + turnItems: [ + { + id: staleItemId, + runId: settledRunId, + nodeId: staleSubagentNodeId, + providerThreadId, + type: "subagent", + status: "running", + subagentId: staleSubagentNodeId, + providerInstanceId: claudeInstanceId, + }, + { + id: doneItemId, + runId: settledRunId, + nodeId: doneSubagentNodeId, + providerThreadId, + type: "subagent", + status: "completed", + subagentId: doneSubagentNodeId, + providerInstanceId: claudeInstanceId, + }, + ], + } as unknown as OrchestrationV2ThreadProjection; + const layer = ProviderRuntimeRecovery.layer.pipe( + Layer.provide( + Layer.mergeAll( + Layer.mock(ProjectionStore.ProjectionStoreV2)({ + getShellSnapshot: () => + Effect.succeed({ + schemaVersion: 2, + snapshotSequence: 0, + threads: [{ id: threadId }], + archivedThreads: [], + } as never), + getThreadProjection: () => Effect.succeed(projection), + }), + Layer.mock(EventSink.EventSinkV2)({ + commitCommand: (input) => { + committedInput = input; + return Effect.succeed({ committed: true, cancelledEffectCount: 0 } as never); + }, + }), + IdAllocator.layer, + Layer.mock(EffectWorker.OrchestrationEffectWorkerV2)({ runOnce: Effect.succeed(false) }), + Layer.mock(EffectOutbox.EffectOutboxV2)({ + listByCommandId: () => Effect.succeed([]), + reconcileAfterProcessLoss: Effect.succeed({ requeued: 0, cancelled: 0 }), + }), + ), + ), + ); + + return Effect.gen(function* () { + yield* (yield* ProviderRuntimeRecovery.ProviderRuntimeRecoveryService).reconcile("startup"); + const events = committedInput?.events ?? []; + + // Only the nonterminal subagent item is cancelled. + const turnItemCancels = events.filter( + (event) => event.type === "turn-item.updated" && event.payload.status === "cancelled", + ); + assert.equal(turnItemCancels.length, 1); + + // The linked subagent entity is terminalized alongside its turn item. + const subagentCancels = events.filter((event) => event.type === "subagent.updated"); + assert.equal(subagentCancels.length, 1); + const subagentCancel = subagentCancels[0]; + assert.equal( + subagentCancel?.type === "subagent.updated" ? subagentCancel.payload.id : null, + staleSubagentNodeId, + ); + assert.equal( + subagentCancel?.type === "subagent.updated" ? subagentCancel.payload.status : null, + "cancelled", + ); + + // So is its execution node, which no live process can terminalize. + const nodeCancels = events.filter((event) => event.type === "node.updated"); + assert.equal(nodeCancels.length, 1); + const nodeCancel = nodeCancels[0]; + assert.equal( + nodeCancel?.type === "node.updated" ? nodeCancel.payload.id : null, + staleSubagentNodeId, + ); + + // The already-completed subagent and node are left untouched. + assert.isFalse( + events.some( + (event) => event.type === "subagent.updated" && event.payload.id === doneSubagentNodeId, + ), + ); + assert.isFalse( + events.some( + (event) => event.type === "node.updated" && event.payload.id === doneSubagentNodeId, + ), + ); + }).pipe(Effect.provide(layer)); + }, +); diff --git a/apps/server/src/orchestration-v2/ProviderRuntimeRecoveryService.ts b/apps/server/src/orchestration-v2/ProviderRuntimeRecoveryService.ts index 5dadaa7b01e4..b9a851980568 100644 --- a/apps/server/src/orchestration-v2/ProviderRuntimeRecoveryService.ts +++ b/apps/server/src/orchestration-v2/ProviderRuntimeRecoveryService.ts @@ -69,6 +69,58 @@ function nonterminalRuns(projection: OrchestrationV2ThreadProjection) { }); } +function isBackgroundCapableTurnItemType(type: string): boolean { + return type === "command_execution" || type === "dynamic_tool" || type === "subagent"; +} + +function isNonterminalTurnItemStatus(status: string): boolean { + return status === "pending" || status === "running" || status === "waiting"; +} + +function isNonterminalSubagentStatus(status: string): boolean { + return status === "pending" || status === "running" || status === "waiting"; +} + +function isNonterminalNodeStatus(status: string): boolean { + return status === "pending" || status === "running" || status === "waiting"; +} + +function providerThreadHasPendingBackgroundTasks( + providerThread: OrchestrationV2ThreadProjection["providerThreads"][number], +): boolean { + return (providerThread.pendingBackgroundTasks?.length ?? 0) > 0; +} + +/** + * Resolve providerInstanceId for a stale background-capable turn item whose + * run is missing/null (or not found). Prefer an existing run, then a subagent + * item's own instance id, then the item's provider thread, then a last-resort + * first provider thread, then the thread's selected provider. + */ +function resolveStaleBackgroundItemProviderInstanceId( + item: OrchestrationV2ThreadProjection["turnItems"][number], + projection: OrchestrationV2ThreadProjection, +): OrchestrationV2ThreadProjection["thread"]["providerInstanceId"] { + if (item.runId !== null) { + const run = projection.runs.find((candidate) => candidate.id === item.runId); + if (run !== undefined) { + return run.providerInstanceId; + } + } + if (item.type === "subagent") { + return item.providerInstanceId; + } + if (item.providerThreadId !== null && item.providerThreadId !== undefined) { + const providerThread = projection.providerThreads.find( + (candidate) => candidate.id === item.providerThreadId, + ); + if (providerThread !== undefined) { + return providerThread.providerInstanceId; + } + } + return projection.providerThreads[0]?.providerInstanceId ?? projection.thread.providerInstanceId; +} + export const make = Effect.gen(function* () { const projections = yield* ProjectionStore.ProjectionStoreV2; const eventSink = yield* EventSink.EventSinkV2; @@ -253,9 +305,82 @@ export const make = Effect.gen(function* () { }); } } - for (const providerThread of projection.providerThreads.filter( - (candidate) => candidate.status === "active", - )) { + // Process loss also orphans background-capable turn items on already- + // settled runs (e.g. post-settle Waiting work). Skip items already + // cancelled above for recovered nonterminal runs to avoid duplicate + // cancellation events. + const recoveredNonterminalRunIds = new Set(runs.map((run) => run.id)); + for (const item of projection.turnItems ?? []) { + if (item.runId !== null && recoveredNonterminalRunIds.has(item.runId)) { + continue; + } + if (!isBackgroundCapableTurnItemType(item.type)) { + continue; + } + if (!isNonterminalTurnItemStatus(item.status)) { + continue; + } + const providerInstanceId = resolveStaleBackgroundItemProviderInstanceId(item, projection); + events.push({ + id: yield* allocateEventId(), + type: "turn-item.updated", + threadId: projection.thread.id, + ...(item.runId === null ? {} : { runId: item.runId }), + ...(item.nodeId === null || item.nodeId === undefined ? {} : { nodeId: item.nodeId }), + providerInstanceId, + occurredAt: now, + payload: { ...item, status: "cancelled", completedAt: now, updatedAt: now }, + }); + if (item.type !== "subagent") { + continue; + } + // Cancelling only the turn item would leave the linked subagent entity + // and its execution node non-terminal forever, since the dead provider + // process can no longer emit their terminal events. Match the exact + // linked ids so a subagent that already finished is never overwritten. + const staleSubagent = projection.subagents.find( + (candidate) => + candidate.id === item.subagentId && isNonterminalSubagentStatus(candidate.status), + ); + if (staleSubagent !== undefined) { + events.push({ + id: yield* allocateEventId(), + type: "subagent.updated", + threadId: projection.thread.id, + ...(item.runId === null ? {} : { runId: item.runId }), + nodeId: staleSubagent.id, + driver: staleSubagent.driver, + providerInstanceId: staleSubagent.providerInstanceId, + occurredAt: now, + payload: { ...staleSubagent, status: "cancelled", completedAt: now, updatedAt: now }, + }); + } + const staleSubagentNode = projection.nodes.find( + (candidate) => + candidate.id === item.subagentId && isNonterminalNodeStatus(candidate.status), + ); + if (staleSubagentNode !== undefined) { + events.push({ + id: yield* allocateEventId(), + type: "node.updated", + threadId: projection.thread.id, + ...(item.runId === null ? {} : { runId: item.runId }), + nodeId: staleSubagentNode.id, + providerInstanceId, + occurredAt: now, + payload: { ...staleSubagentNode, status: "cancelled", completedAt: now }, + }); + } + } + // All provider processes are gone on startup/shutdown: clear any + // persisted Waiting roster (including idle threads from settled roots) + // and idle active threads without resurrecting active status. + for (const providerThread of projection.providerThreads ?? []) { + const needsIdle = providerThread.status === "active"; + const needsRosterClear = providerThreadHasPendingBackgroundTasks(providerThread); + if (!needsIdle && !needsRosterClear) { + continue; + } events.push({ id: yield* allocateEventId(), type: "provider-thread.updated", @@ -263,7 +388,12 @@ export const make = Effect.gen(function* () { driver: providerThread.driver, providerInstanceId: providerThread.providerInstanceId, occurredAt: now, - payload: { ...providerThread, status: "idle", updatedAt: now }, + payload: { + ...providerThread, + status: needsIdle ? "idle" : providerThread.status, + pendingBackgroundTasks: [], + updatedAt: now, + }, }); } for (const session of projection.providerSessions.filter( diff --git a/apps/server/src/orchestration-v2/ProviderTurnStartService.test.ts b/apps/server/src/orchestration-v2/ProviderTurnStartService.test.ts new file mode 100644 index 000000000000..95c9a03187a4 --- /dev/null +++ b/apps/server/src/orchestration-v2/ProviderTurnStartService.test.ts @@ -0,0 +1,103 @@ +import { expect, it, vi } from "vite-plus/test"; +import { + CheckpointScopeId, + MessageId, + NodeId, + ProviderSessionId, + ProviderThreadId, + RunAttemptId, + RunId, + ThreadId, + type OrchestrationV2ThreadProjection, +} from "@t3tools/contracts"; +import * as Effect from "effect/Effect"; +import * as Layer from "effect/Layer"; + +import * as ContextHandoffService from "./ContextHandoffService.ts"; +import * as EventSink from "./EventSink.ts"; +import * as IdAllocator from "./IdAllocator.ts"; +import * as ProjectionStore from "./ProjectionStore.ts"; +import * as ProviderSessionManager from "./ProviderSessionManager.ts"; +import * as ProviderTurnStart from "./ProviderTurnStartService.ts"; +import * as RunExecutionService from "./RunExecutionService.ts"; +import * as RuntimePolicy from "./RuntimePolicy.ts"; + +it("does not commit running state when inherited background routing cannot be read", async () => { + const threadId = ThreadId.make("thread_provider_turn_start_projection_failure"); + const runId = RunId.make("run_provider_turn_start_projection_failure"); + const attemptId = RunAttemptId.make("attempt_provider_turn_start_projection_failure"); + const rootNodeId = NodeId.make("node_provider_turn_start_projection_failure"); + const providerThreadId = ProviderThreadId.make( + "provider_thread_provider_turn_start_projection_failure", + ); + const providerSessionId = ProviderSessionId.make( + "provider_session_provider_turn_start_projection_failure", + ); + const messageId = MessageId.make("message_provider_turn_start_projection_failure"); + const checkpointScopeId = CheckpointScopeId.make( + "checkpoint_scope_provider_turn_start_projection_failure", + ); + const projection = { + thread: { id: threadId }, + runs: [ + { + id: runId, + status: "starting", + rootNodeId, + activeAttemptId: attemptId, + providerThreadId, + userMessageId: messageId, + ordinal: 2, + }, + ], + nodes: [{ id: rootNodeId, checkpointScopeId }], + attempts: [{ id: attemptId }], + providerThreads: [{ id: providerThreadId, providerSessionId }], + messages: [{ id: messageId }], + checkpointScopes: [{ id: checkpointScopeId }], + contextHandoffs: [], + contextTransfers: [], + turnItems: [], + } as unknown as OrchestrationV2ThreadProjection; + let projectionReadCount = 0; + const writeIfRunCurrent = vi.fn(() => + Effect.succeed({ committed: true, storedEvents: [] } as never), + ); + const startRootRun = vi.fn(() => Effect.void); + const layer = ProviderTurnStart.layer.pipe( + Layer.provide( + Layer.mergeAll( + Layer.mock(ContextHandoffService.ContextHandoffServiceV2)({}), + Layer.mock(EventSink.EventSinkV2)({ writeIfRunCurrent }), + IdAllocator.layer, + Layer.mock(ProjectionStore.ProjectionStoreV2)({ + getThreadProjection: () => { + projectionReadCount += 1; + return projectionReadCount === 1 + ? Effect.succeed(projection) + : Effect.fail( + new ProjectionStore.ProjectionStoreReadError({ + threadId, + cause: "simulated inherited-background projection failure", + }), + ); + }, + }), + Layer.mock(ProviderSessionManager.ProviderSessionManagerV2)({}), + Layer.mock(RunExecutionService.RunExecutionServiceV2)({ startRootRun }), + Layer.mock(RuntimePolicy.RuntimePolicyV2)({}), + ), + ), + ); + + await Effect.gen(function* () { + const error = yield* (yield* ProviderTurnStart.ProviderTurnStartServiceV2) + .start({ threadId, runId }) + .pipe(Effect.flip); + + expect(error._tag).toBe("ProviderTurnStartError"); + expect(projectionReadCount).toBe(2); + expect(writeIfRunCurrent).not.toHaveBeenCalled(); + expect(startRootRun).not.toHaveBeenCalled(); + }).pipe(Effect.provide(layer), Effect.runPromise); +}); diff --git a/apps/server/src/orchestration-v2/ProviderTurnStartService.ts b/apps/server/src/orchestration-v2/ProviderTurnStartService.ts index 3b6f74bf5f75..15f7a5374367 100644 --- a/apps/server/src/orchestration-v2/ProviderTurnStartService.ts +++ b/apps/server/src/orchestration-v2/ProviderTurnStartService.ts @@ -22,7 +22,11 @@ import { import { IdAllocatorV2 } from "./IdAllocator.ts"; import { ProjectionStoreV2 } from "./ProjectionStore.ts"; import { ProviderSessionManagerV2 } from "./ProviderSessionManager.ts"; -import { canRouteRelatedSubagent, RunExecutionServiceV2 } from "./RunExecutionService.ts"; +import { + canRouteRelatedSubagent, + RunExecutionServiceV2, + selectInheritedBackgroundTurnItems, +} from "./RunExecutionService.ts"; import { RuntimePolicyV2 } from "./RuntimePolicy.ts"; export class ProviderTurnStartError extends Schema.TaggedErrorClass()( @@ -124,6 +128,19 @@ export const layer: Layer.Layer< cause: `Run ${runId} is missing its execution projection state.`, }); } + const selectInheritedBackgroundItems = ( + current: typeof projection, + ): ReturnType => + selectInheritedBackgroundTurnItems({ + threadId: current.thread.id, + currentProviderThreadId: providerThread.id, + currentRunOrdinal: run.ordinal, + runs: current.runs, + turnItems: current.turnItems, + }); + const inheritedBackgroundTurnItems = yield* projectionStore + .getThreadProjection(projection.thread.id) + .pipe(Effect.map(selectInheritedBackgroundItems)); const providerSessionId = providerThread.providerSessionId; const isCurrentAttemptInStatus = ( expectedStatus: OrchestrationV2Run["status"], @@ -417,6 +434,11 @@ export const layer: Layer.Layer< providerThread: runningProviderThread, attempt: runningAttempt, attemptId: attempt.id, + loadInheritedBackgroundTurnItems: () => + projectionStore.getThreadProjection(projection.thread.id).pipe( + Effect.map(selectInheritedBackgroundItems), + Effect.catchCause(() => Effect.succeed(inheritedBackgroundTurnItems)), + ), relatedThreadIds: routableSubagents.flatMap((subagent) => subagent.childThreadId === null ? [] : [subagent.childThreadId], ), diff --git a/apps/server/src/orchestration-v2/RunExecutionService.test.ts b/apps/server/src/orchestration-v2/RunExecutionService.test.ts index dc5982bffad5..56fb8ce1a039 100644 --- a/apps/server/src/orchestration-v2/RunExecutionService.test.ts +++ b/apps/server/src/orchestration-v2/RunExecutionService.test.ts @@ -48,6 +48,7 @@ import { type ProviderEventRouteIdentity, routeProviderEvent, RunExecutionServiceV2, + selectInheritedBackgroundTurnItems, } from "./RunExecutionService.ts"; const driver = ProviderDriverKind.make("codex"); @@ -188,6 +189,197 @@ it("does not route a superseded attempt through a reused provider thread", () => assert.isFalse(routeProviderEvent(oldTurnEvent, newAttempt, newState)[0]); }); +it("routes only exact same-thread background items inherited from settled runs", () => { + const threadId = ThreadId.make("thread:inherited-background-routing"); + const otherThreadId = ThreadId.make("thread:inherited-background-routing:other"); + const priorRunId = RunId.make("run:inherited-background-routing:prior"); + const currentRunId = RunId.make("run:inherited-background-routing:current"); + const itemId = TurnItemId.make("turn-item:inherited-background-routing"); + const identity: ProviderEventRouteIdentity = { + threadId, + runId: currentRunId, + attemptId: RunAttemptId.make("attempt:inherited-background-routing:current"), + providerThreadId: ProviderThreadId.make("provider-thread:inherited-background-routing:current"), + }; + const initial = makeProviderEventRoutingState({ + identity, + inheritedBackgroundTurnItems: [{ id: itemId, runId: priorRunId }], + providerTurnId: null, + }); + const inheritedRunning = { + type: "turn_item.updated", + driver, + turnItem: { + id: itemId, + threadId, + runId: priorRunId, + providerTurnId: null, + ordinal: 1, + type: "subagent", + status: "running", + }, + } as Extract; + const unrelatedRunItem = { + ...inheritedRunning, + turnItem: { + ...inheritedRunning.turnItem, + runId: RunId.make("run:inherited-background-routing:unrelated"), + }, + } as ProviderAdapterV2Event; + const unlistedPriorItem = { + ...inheritedRunning, + turnItem: { + ...inheritedRunning.turnItem, + id: TurnItemId.make("turn-item:inherited-background-routing:unrelated"), + }, + } as ProviderAdapterV2Event; + const unrelatedThreadItem = { + ...inheritedRunning, + turnItem: { ...inheritedRunning.turnItem, threadId: otherThreadId }, + } as ProviderAdapterV2Event; + const ordinaryItem = { + ...inheritedRunning, + turnItem: { ...inheritedRunning.turnItem, type: "reasoning" as const }, + } as ProviderAdapterV2Event; + const inheritedTerminal = { + ...inheritedRunning, + turnItem: { ...inheritedRunning.turnItem, status: "completed" as const }, + } as ProviderAdapterV2Event; + + const [runningAccepted, afterRunning] = routeProviderEvent(inheritedRunning, identity, initial); + assert.isTrue(runningAccepted); + assert.isFalse(routeProviderEvent(unrelatedRunItem, identity, afterRunning)[0]); + assert.isFalse(routeProviderEvent(unlistedPriorItem, identity, afterRunning)[0]); + assert.isFalse(routeProviderEvent(unrelatedThreadItem, identity, afterRunning)[0]); + assert.isFalse(routeProviderEvent(ordinaryItem, identity, afterRunning)[0]); + + const [terminalAccepted, afterTerminal] = routeProviderEvent( + inheritedTerminal, + identity, + afterRunning, + ); + assert.isTrue(terminalAccepted); + assert.isFalse( + routeProviderEvent(inheritedRunning, identity, afterTerminal)[0], + "a nonterminal replay must not resurrect an inherited terminal", + ); +}); + +it("selects only live background items from non-completed settled prior runs", () => { + const threadId = ThreadId.make("thread:inherited-background-selection"); + const otherThreadId = ThreadId.make("thread:inherited-background-selection:other"); + const currentProviderThreadId = ProviderThreadId.make( + "provider-thread:inherited-background-selection:current", + ); + const foreignProviderThreadId = ProviderThreadId.make( + "provider-thread:inherited-background-selection:foreign-session", + ); + const interruptedRunId = RunId.make("run:inherited-background-selection:interrupted"); + const failedRunId = RunId.make("run:inherited-background-selection:failed"); + const cancelledRunId = RunId.make("run:inherited-background-selection:cancelled"); + const completedRunId = RunId.make("run:inherited-background-selection:completed"); + const rolledBackRunId = RunId.make("run:inherited-background-selection:rolled-back"); + const currentRunId = RunId.make("run:inherited-background-selection:current"); + const inheritedItemId = TurnItemId.make("turn-item:inherited-background-selection:live"); + const failedItemId = TurnItemId.make("turn-item:inherited-background-selection:failed"); + const cancelledItemId = TurnItemId.make("turn-item:inherited-background-selection:cancelled"); + const makeRun = ( + id: RunId, + ordinal: number, + status: OrchestrationV2Run["status"], + runThreadId = threadId, + ) => + ({ + id, + threadId: runThreadId, + ordinal, + status, + }) as OrchestrationV2Run; + const makeItem = ( + id: TurnItemId, + runId: RunId, + status: OrchestrationV2TurnItem["status"], + type: OrchestrationV2TurnItem["type"] = "subagent", + itemThreadId = threadId, + providerThreadId = currentProviderThreadId, + ) => + ({ + id, + threadId: itemThreadId, + runId, + providerThreadId, + type, + status, + }) as OrchestrationV2TurnItem; + + const selected = selectInheritedBackgroundTurnItems({ + threadId, + currentProviderThreadId, + currentRunOrdinal: 6, + runs: [ + makeRun(interruptedRunId, 1, "interrupted"), + makeRun(failedRunId, 2, "failed"), + makeRun(cancelledRunId, 3, "cancelled"), + makeRun(completedRunId, 4, "completed"), + makeRun(rolledBackRunId, 5, "rolled_back"), + makeRun(currentRunId, 6, "running"), + makeRun( + RunId.make("run:inherited-background-selection:other"), + 1, + "interrupted", + otherThreadId, + ), + ], + turnItems: [ + makeItem(inheritedItemId, interruptedRunId, "running"), + makeItem(failedItemId, failedRunId, "running"), + makeItem(cancelledItemId, cancelledRunId, "running"), + makeItem( + TurnItemId.make("turn-item:inherited-background-selection:completed-run"), + completedRunId, + "running", + ), + makeItem( + TurnItemId.make("turn-item:inherited-background-selection:terminal"), + interruptedRunId, + "completed", + ), + makeItem( + TurnItemId.make("turn-item:inherited-background-selection:ordinary"), + interruptedRunId, + "running", + "reasoning", + ), + makeItem( + TurnItemId.make("turn-item:inherited-background-selection:rolled-back"), + rolledBackRunId, + "running", + ), + makeItem( + TurnItemId.make("turn-item:inherited-background-selection:other-thread"), + interruptedRunId, + "running", + "subagent", + otherThreadId, + ), + makeItem( + TurnItemId.make("turn-item:inherited-background-selection:foreign-provider"), + interruptedRunId, + "running", + "subagent", + threadId, + foreignProviderThreadId, + ), + ], + }); + + assert.deepEqual(selected, [ + { id: inheritedItemId, runId: interruptedRunId }, + { id: failedItemId, runId: failedRunId }, + { id: cancelledItemId, runId: cancelledRunId }, + ]); +}); + it("does not carry interrupted child ownership into later attempts", () => { assert.isFalse(canRouteRelatedSubagent("interrupted")); assert.isFalse(canRouteRelatedSubagent("failed")); @@ -615,55 +807,952 @@ it.effect("ingests the trailing subagent item completion after the subagent row }), ); -it.effect("keeps ingesting a child thread's late background item completion", () => - Effect.gen(function* () { - const observed = yield* runBackgroundItemScenario("bg-child-item", (ids) => [ - childThreadCreatedEvent(ids), - subagentEvent(ids, "running"), - childBackgroundTurnItemEvent(ids, "running", 1), - rootTerminalEvent(ids, "completed"), - subagentEvent(ids, "completed"), - childBackgroundTurnItemEvent(ids, "completed", 2), - ]); - assert.deepEqual(observed, [ - "subagent:running", - "turn_item:running", +it.effect("keeps ingesting a child thread's late background item completion", () => + Effect.gen(function* () { + const observed = yield* runBackgroundItemScenario("bg-child-item", (ids) => [ + childThreadCreatedEvent(ids), + subagentEvent(ids, "running"), + childBackgroundTurnItemEvent(ids, "running", 1), + rootTerminalEvent(ids, "completed"), + subagentEvent(ids, "completed"), + childBackgroundTurnItemEvent(ids, "completed", 2), + ]); + assert.deepEqual(observed, [ + "subagent:running", + "turn_item:running", + "root-finalized", + "subagent:completed", + "turn_item:completed", + ]); + }), +); + +it.effect("keeps ingesting until the last of several background items terminalizes", () => + Effect.gen(function* () { + const secondItemId = TurnItemId.make("turn-item:bg-multi:second"); + const observed = yield* runBackgroundItemScenario("bg-multi", (ids) => [ + backgroundTurnItemEvent(ids, "command_execution", "running", 1), + backgroundTurnItemEvent(ids, "dynamic_tool", "running", 2, secondItemId), + rootTerminalEvent(ids, "completed"), + backgroundTurnItemEvent(ids, "command_execution", "completed", 3), + backgroundTurnItemEvent(ids, "dynamic_tool", "completed", 4, secondItemId), + ]); + assert.deepEqual(observed, [ + "turn_item:running", + "turn_item:running", + "root-finalized", + "turn_item:completed", + "turn_item:completed", + ]); + }), +); + +it.effect("does not pin ingestion on background items when the root turn is interrupted", () => + Effect.gen(function* () { + const observed = yield* runBackgroundItemScenario("bg-interrupted", (ids) => [ + backgroundTurnItemEvent(ids, "command_execution", "running", 1), + rootTerminalEvent(ids, "interrupted"), + backgroundTurnItemEvent(ids, "command_execution", "completed", 2), + ]); + assert.deepEqual(observed, ["turn_item:running", "root-finalized"]); + }), +); + +it.effect("seeds inherited background items before their next update", () => + Effect.gen(function* () { + const key = "inherited-background-seeded"; + const priorRunId = RunId.make(`run:${key}:prior`); + const observed = yield* runBackgroundItemScenario( + key, + (ids) => [ + rootTerminalEvent(ids, "completed"), + backgroundTurnItemEventForRun(ids, priorRunId, "subagent", "completed", 1), + ], + { + loadInheritedBackgroundTurnItems: () => + Effect.succeed([{ id: TurnItemId.make(`turn-item:${key}`), runId: priorRunId }]), + }, + ); + + assert.deepEqual(observed, ["root-finalized", "turn_item:completed"]); + }), +); + +it.effect("releases the live run after an inherited background item terminalizes", () => + Effect.gen(function* () { + const key = "inherited-background-terminal"; + const priorRunId = RunId.make(`run:${key}:prior`); + const observed = yield* runBackgroundItemScenario( + key, + (ids) => [ + backgroundTurnItemEventForRun(ids, priorRunId, "subagent", "running", 1), + rootTerminalEvent(ids, "completed"), + backgroundTurnItemEventForRun(ids, priorRunId, "subagent", "completed", 2), + ], + { + loadInheritedBackgroundTurnItems: () => + Effect.succeed([{ id: TurnItemId.make(`turn-item:${key}`), runId: priorRunId }]), + }, + ); + + assert.deepEqual(observed, ["turn_item:running", "root-finalized", "turn_item:completed"]); + }), +); + +it.effect("does not hold the live stream open for a foreign provider's background item", () => + Effect.gen(function* () { + const key = "inherited-background-foreign-provider"; + const ids = backgroundScenarioIds(key); + const priorRunId = RunId.make(`run:${key}:prior`); + const foreignProviderThreadId = ProviderThreadId.make(`provider-thread:${key}:foreign-session`); + const foreignItem = { + id: ids.itemId, + threadId: ids.threadId, + runId: priorRunId, + providerThreadId: foreignProviderThreadId, + type: "subagent", + status: "running", + } as OrchestrationV2TurnItem; + const inherited = selectInheritedBackgroundTurnItems({ + threadId: ids.threadId, + currentProviderThreadId: ids.providerThreadId, + currentRunOrdinal: 2, + runs: [ + { + id: priorRunId, + threadId: ids.threadId, + ordinal: 1, + status: "interrupted", + } as OrchestrationV2Run, + ], + turnItems: [foreignItem], + }); + + const observed = yield* runBackgroundItemScenario( + key, + (scenarioIds) => [rootTerminalEvent(scenarioIds, "completed")], + { + keepEventStreamOpen: true, + loadInheritedBackgroundTurnItems: () => Effect.succeed(inherited), + }, + ); + + assert.deepEqual(inherited, []); + assert.deepEqual(observed, ["root-finalized"]); + }), +); + +it.effect("refreshes inherited background items after event subscription", () => + Effect.gen(function* () { + const key = "inherited-background-subscription-refresh"; + const ids = backgroundScenarioIds(key); + const priorRunId = RunId.make(`run:${key}:prior`); + const itemStatus = yield* Ref.make("running"); + const loadInheritedBackgroundTurnItems = () => + Ref.get(itemStatus).pipe( + Effect.map((status) => + selectInheritedBackgroundTurnItems({ + threadId: ids.threadId, + currentProviderThreadId: ids.providerThreadId, + currentRunOrdinal: 2, + runs: [ + { + id: priorRunId, + threadId: ids.threadId, + ordinal: 1, + status: "interrupted", + } as OrchestrationV2Run, + ], + turnItems: [ + { + id: ids.itemId, + threadId: ids.threadId, + runId: priorRunId, + providerThreadId: ids.providerThreadId, + type: "subagent", + status, + } as OrchestrationV2TurnItem, + ], + }), + ), + ); + + const observed = yield* runBackgroundItemScenario( + key, + (scenarioIds) => [rootTerminalEvent(scenarioIds, "completed")], + { + keepEventStreamOpen: true, + loadInheritedBackgroundTurnItems, + onSubscribe: Ref.set(itemStatus, "completed"), + }, + ); + + assert.deepEqual(yield* loadInheritedBackgroundTurnItems(), []); + assert.deepEqual(observed, ["root-finalized"]); + }), +); + +it.effect( + "keeps ingesting a late empty provider-thread roster while thread-scoped pending work is true", + () => + Effect.gen(function* () { + const key = "bg-roster-pending-work"; + const ids = backgroundScenarioIds(key); + const providerInstanceId = ProviderInstanceId.make("codex"); + const now = yield* DateTime.now; + const observed = yield* Ref.make>([]); + const pendingByProviderThreadId = yield* Ref.make(new Map([[ids.providerThreadId, true]])); + const scopedProbeArgs = yield* Ref.make>([]); + const ingestCalls = yield* Ref.make< + ReadonlyArray<{ + readonly activeAttemptId: RunAttemptId | null; + readonly eventType: string; + readonly hasWriteIfRunCurrent: boolean; + readonly hasWriteIfProviderThreadOwner: boolean; + readonly expectedLastRunOrdinal: number | null; + readonly runId: RunId | null; + readonly rosterLength: number | null; + }> + >([]); + const ingestionDone = yield* Deferred.make(); + const testLayer = runExecutionServiceLayer.pipe( + Layer.provide( + Layer.mergeAll( + Layer.mock(CheckpointServiceV2)({ captureBaseline: () => Effect.void }), + Layer.mock(EventSinkV2)({ + write: () => Effect.succeed([]), + writeWithEffects: (input) => + Effect.gen(function* () { + if ( + input.events.some( + (event) => event.type === "run.updated" && event.runId === ids.runId, + ) + ) { + yield* Ref.update(observed, (current) => [...current, "root-finalized"]); + } + return []; + }), + writeIfRunCurrent: () => Effect.succeed({ committed: true, storedEvents: [] }), + writeIfProviderThreadOwner: () => + Effect.succeed({ committed: true, storedEvents: [] }), + }), + idAllocatorLayer, + Layer.mock(ProviderEventIngestorV2)({ + ingestNormalized: (input) => + Effect.gen(function* () { + const event = input.event; + const rosterLength = + event.type === "provider_thread.updated" + ? (event.providerThread.pendingBackgroundTasks?.length ?? 0) + : null; + yield* Ref.update(ingestCalls, (current) => [ + ...current, + { + activeAttemptId: input.writeIfProviderThreadOwner?.activeAttemptId ?? null, + eventType: event.type, + hasWriteIfRunCurrent: input.writeIfRunCurrent !== undefined, + hasWriteIfProviderThreadOwner: input.writeIfProviderThreadOwner !== undefined, + expectedLastRunOrdinal: + input.writeIfProviderThreadOwner?.expectedLastRunOrdinal ?? null, + runId: input.writeIfProviderThreadOwner?.runId ?? null, + rosterLength, + }, + ]); + if (event.type === "provider_thread.updated" && rosterLength === 0) { + yield* Ref.update(pendingByProviderThreadId, (current) => { + const next = new Map(current); + next.set(event.providerThread.id, false); + return next; + }); + yield* Ref.update(observed, (current) => [...current, "roster-cleared"]); + } + if (event.type === "turn.terminal") { + yield* Ref.update(observed, (current) => [...current, "terminal"]); + } + return []; + }), + }), + ServerSettingsService.layerTest(), + ), + ), + ); + + const providerThreadBase = { + id: ids.providerThreadId, + driver, + providerInstanceId, + providerSessionId: ProviderSessionId.make(`session:${key}`), + appThreadId: ids.threadId, + ownerNodeId: null, + nativeThreadRef: null, + nativeConversationHeadRef: null, + status: "idle" as const, + firstRunOrdinal: 1, + lastRunOrdinal: 1, + handoffIds: [], + forkedFrom: null, + createdAt: now, + updatedAt: now, + }; + + yield* Effect.gen(function* () { + const runExecution = yield* RunExecutionServiceV2; + yield* runExecution.startRootRun({ + commandId: CommandId.make(`command:${key}`), + appThread: { id: ids.threadId } as OrchestrationV2AppThread, + providerSessionId: ProviderSessionId.make(`session:${key}`), + session: { + events: Stream.empty, + // Session-wide stays true forever; the root must consult the + // thread-scoped probe instead of being pinned by siblings. + hasPendingBackgroundWork: Effect.succeed(true), + hasPendingBackgroundWorkForThread: (providerThread: OrchestrationV2ProviderThread) => + Effect.gen(function* () { + yield* Ref.update(scopedProbeArgs, (current) => [...current, providerThread.id]); + return (yield* Ref.get(pendingByProviderThreadId)).get(providerThread.id) === true; + }), + subscribeEvents: Effect.succeed({ + events: Stream.fromIterable([ + { + type: "provider_thread.updated", + driver, + providerThread: { + ...providerThreadBase, + status: "active" as const, + pendingBackgroundTasks: [ + { taskId: "bg-1", description: "sleep 20", taskType: "local_bash" }, + ], + updatedAt: now, + }, + } as ProviderAdapterV2Event, + rootTerminalEvent(ids, "completed"), + { + type: "provider_thread.updated", + driver, + providerThread: { + ...providerThreadBase, + status: "idle" as const, + pendingBackgroundTasks: [], + updatedAt: now, + }, + } as ProviderAdapterV2Event, + ]), + close: Deferred.succeed(ingestionDone, undefined), + }), + startTurn: () => Effect.void, + } as unknown as ProviderAdapterV2SessionRuntime, + run: { + id: ids.runId, + threadId: ids.threadId, + ordinal: 1, + providerInstanceId, + } as OrchestrationV2Run, + rootNode: { id: ids.rootNodeId } as OrchestrationV2ExecutionNode, + checkpointScope: { + id: CheckpointScopeId.make(`checkpoint-scope:${key}`), + } as OrchestrationV2CheckpointScope, + providerThread: providerThreadBase as OrchestrationV2ProviderThread, + attempt: { + id: ids.attemptId, + providerTurnId: ids.rootProviderTurnId, + } as OrchestrationV2RunAttempt, + attemptId: ids.attemptId, + providerTurnOrdinal: 1, + message: { + messageId: MessageId.make(`message:${key}:user`), + text: "Start background work and settle.", + attachments: [], + createdBy: "user", + creationSource: "web", + }, + modelSelection: { instanceId: providerInstanceId, model: "gpt-5.4" }, + runtimePolicy: { + runtimeMode: "full-access", + interactionMode: "default", + cwd: process.cwd(), + approvalPolicy: "never", + sandboxPolicy: { + type: "readOnly", + access: { type: "fullAccess" }, + networkAccess: false, + }, + }, + }); + }).pipe(Effect.provide(testLayer)); + + const closed = yield* Deferred.await(ingestionDone).pipe(Effect.timeoutOption("2 seconds")); + assert.isTrue(Option.isSome(closed), "event subscription did not release"); + assert.deepEqual(yield* Ref.get(observed), ["terminal", "root-finalized", "roster-cleared"]); + assert.isTrue((yield* Ref.get(scopedProbeArgs)).includes(ids.providerThreadId)); + + const calls = yield* Ref.get(ingestCalls); + const preTerminalRoster = calls.find( + (call) => call.eventType === "provider_thread.updated" && call.rosterLength === 1, + ); + const lateClear = calls.find( + (call) => call.eventType === "provider_thread.updated" && call.rosterLength === 0, + ); + assert.isDefined(preTerminalRoster); + assert.isTrue(preTerminalRoster?.hasWriteIfRunCurrent); + assert.isFalse(preTerminalRoster?.hasWriteIfProviderThreadOwner); + assert.isDefined(lateClear); + assert.isFalse( + lateClear?.hasWriteIfRunCurrent, + "late empty roster must not use stale writeIfRunCurrent running-gate", + ); + assert.isTrue( + lateClear?.hasWriteIfProviderThreadOwner, + "late empty roster must gate on provider-thread ownership", + ); + assert.equal(lateClear?.expectedLastRunOrdinal, 1); + assert.equal(lateClear?.runId, ids.runId); + assert.equal(lateClear?.activeAttemptId, ids.attemptId); + }), +); + +it.effect("drops late root provider-thread writes from a superseded attempt", () => + Effect.gen(function* () { + const key = "bg-roster-attempt-owner-lost"; + const ids = backgroundScenarioIds(key); + const replacementAttemptId = RunAttemptId.make(`attempt:${key}:replacement`); + const providerInstanceId = ProviderInstanceId.make("codex"); + const now = yield* DateTime.now; + const observed = yield* Ref.make>([]); + // Probe stays true forever so only ownership-loss can release the stream. + const ingestionDone = yield* Deferred.make(); + const ingestCalls = yield* Ref.make< + ReadonlyArray<{ + readonly activeAttemptId: RunAttemptId | null; + readonly eventType: string; + readonly hasWriteIfProviderThreadOwner: boolean; + readonly expectedLastRunOrdinal: number | null; + readonly rosterLength: number | null; + readonly committed: boolean; + }> + >([]); + const testLayer = runExecutionServiceLayer.pipe( + Layer.provide( + Layer.mergeAll( + Layer.mock(CheckpointServiceV2)({ captureBaseline: () => Effect.void }), + Layer.mock(EventSinkV2)({ + write: () => Effect.succeed([]), + writeWithEffects: (input) => + Effect.gen(function* () { + if ( + input.events.some( + (event) => event.type === "run.updated" && event.runId === ids.runId, + ) + ) { + yield* Ref.update(observed, (current) => [...current, "root-finalized"]); + } + return []; + }), + writeIfRunCurrent: () => Effect.succeed({ committed: true, storedEvents: [] }), + writeIfProviderThreadOwner: () => + Effect.succeed({ committed: false, storedEvents: [] }), + }), + idAllocatorLayer, + Layer.mock(ProviderEventIngestorV2)({ + ingestNormalized: (input) => + Effect.gen(function* () { + const event = input.event; + const rosterLength = + event.type === "provider_thread.updated" + ? (event.providerThread.pendingBackgroundTasks?.length ?? 0) + : null; + const ownerGate = input.writeIfProviderThreadOwner; + // Simulate the EventSink reject after a same-run replacement + // changes activeAttemptId without advancing lastRunOrdinal. + const rejectAsStaleOwner = + ownerGate !== undefined && ownerGate.activeAttemptId !== replacementAttemptId; + yield* Ref.update(ingestCalls, (current) => [ + ...current, + { + activeAttemptId: ownerGate?.activeAttemptId ?? null, + eventType: event.type, + hasWriteIfProviderThreadOwner: ownerGate !== undefined, + expectedLastRunOrdinal: ownerGate?.expectedLastRunOrdinal ?? null, + rosterLength, + committed: !rejectAsStaleOwner, + }, + ]); + if (event.type === "turn.terminal") { + yield* Ref.update(observed, (current) => [...current, "terminal"]); + } + if (event.type === "provider_thread.updated" && rejectAsStaleOwner) { + yield* Ref.update(observed, (current) => [...current, "stale-owner-rejected"]); + return []; + } + if (event.type === "provider_thread.updated") { + yield* Ref.update(observed, (current) => [...current, "roster-written"]); + } + return []; + }), + }), + ServerSettingsService.layerTest(), + ), + ), + ); + + const providerThreadBase = { + id: ids.providerThreadId, + driver, + providerInstanceId, + providerSessionId: ProviderSessionId.make(`session:${key}`), + appThreadId: ids.threadId, + ownerNodeId: null, + nativeThreadRef: null, + nativeConversationHeadRef: null, + status: "idle" as const, + firstRunOrdinal: 1, + lastRunOrdinal: 1, + handoffIds: [], + forkedFrom: null, + createdAt: now, + updatedAt: now, + }; + + yield* Effect.gen(function* () { + const runExecution = yield* RunExecutionServiceV2; + yield* runExecution.startRootRun({ + commandId: CommandId.make(`command:${key}`), + appThread: { id: ids.threadId } as OrchestrationV2AppThread, + providerSessionId: ProviderSessionId.make(`session:${key}`), + session: { + events: Stream.empty, + hasPendingBackgroundWork: Effect.succeed(true), + hasPendingBackgroundWorkForThread: () => Effect.succeed(true), + subscribeEvents: Effect.succeed({ + events: Stream.fromIterable([ + { + type: "provider_thread.updated", + driver, + providerThread: { + ...providerThreadBase, + status: "active" as const, + pendingBackgroundTasks: [ + { taskId: "bg-stale", description: "sleep 20", taskType: "local_bash" }, + ], + updatedAt: now, + }, + } as ProviderAdapterV2Event, + rootTerminalEvent(ids, "completed"), + // Late snapshot after a replacement attempt claimed the same + // run ordinal. Without attempt gating this would clobber it. + { + type: "provider_thread.updated", + driver, + providerThread: { + ...providerThreadBase, + status: "idle" as const, + lastRunOrdinal: 1, + pendingBackgroundTasks: [ + { taskId: "bg-stale", description: "sleep 20", taskType: "local_bash" }, + ], + updatedAt: now, + }, + } as ProviderAdapterV2Event, + ]), + close: Deferred.succeed(ingestionDone, undefined), + }), + startTurn: () => Effect.void, + } as unknown as ProviderAdapterV2SessionRuntime, + run: { + id: ids.runId, + threadId: ids.threadId, + ordinal: 1, + providerInstanceId, + } as OrchestrationV2Run, + rootNode: { id: ids.rootNodeId } as OrchestrationV2ExecutionNode, + checkpointScope: { + id: CheckpointScopeId.make(`checkpoint-scope:${key}`), + } as OrchestrationV2CheckpointScope, + providerThread: providerThreadBase as OrchestrationV2ProviderThread, + attempt: { + id: ids.attemptId, + providerTurnId: ids.rootProviderTurnId, + } as OrchestrationV2RunAttempt, + attemptId: ids.attemptId, + providerTurnOrdinal: 1, + message: { + messageId: MessageId.make(`message:${key}:user`), + text: "Background work that loses ownership.", + attachments: [], + createdBy: "user", + creationSource: "web", + }, + modelSelection: { instanceId: providerInstanceId, model: "gpt-5.4" }, + runtimePolicy: { + runtimeMode: "full-access", + interactionMode: "default", + cwd: process.cwd(), + approvalPolicy: "never", + sandboxPolicy: { + type: "readOnly", + access: { type: "fullAccess" }, + networkAccess: false, + }, + }, + }); + }).pipe(Effect.provide(testLayer)); + + const closed = yield* Deferred.await(ingestionDone).pipe(Effect.timeoutOption("2 seconds")); + assert.isTrue( + Option.isSome(closed), + "subscription must release after ownership-loss reject even when probe stays true", + ); + assert.deepEqual(yield* Ref.get(observed), [ + "roster-written", + "terminal", "root-finalized", - "subagent:completed", - "turn_item:completed", + "stale-owner-rejected", ]); + + const calls = yield* Ref.get(ingestCalls); + const lateStale = calls.find( + (call) => + call.eventType === "provider_thread.updated" && + call.hasWriteIfProviderThreadOwner && + call.rosterLength === 1, + ); + assert.isDefined(lateStale); + assert.equal(lateStale?.expectedLastRunOrdinal, 1); + assert.equal(lateStale?.activeAttemptId, ids.attemptId); + assert.isFalse(lateStale?.committed); }), ); -it.effect("keeps ingesting until the last of several background items terminalizes", () => - Effect.gen(function* () { - const secondItemId = TurnItemId.make("turn-item:bg-multi:second"); - const observed = yield* runBackgroundItemScenario("bg-multi", (ids) => [ - backgroundTurnItemEvent(ids, "command_execution", "running", 1), - backgroundTurnItemEvent(ids, "dynamic_tool", "running", 2, secondItemId), - rootTerminalEvent(ids, "completed"), - backgroundTurnItemEvent(ids, "command_execution", "completed", 3), - backgroundTurnItemEvent(ids, "dynamic_tool", "completed", 4, secondItemId), - ]); - assert.deepEqual(observed, [ - "turn_item:running", - "turn_item:running", - "root-finalized", - "turn_item:completed", - "turn_item:completed", - ]); - }), +it.effect( + "keeps ingesting a late background turn-item completion after ownership-loss rejects roster writes", + () => + Effect.gen(function* () { + const key = "bg-item-after-owner-lost"; + const ids = backgroundScenarioIds(key); + const providerInstanceId = ProviderInstanceId.make("codex"); + const now = yield* DateTime.now; + const observed = yield* Ref.make>([]); + // Probe stays true forever; open background items must pin the stream + // past ownership-loss so late turn_item completions still land. + const ingestionDone = yield* Deferred.make(); + const testLayer = runExecutionServiceLayer.pipe( + Layer.provide( + Layer.mergeAll( + Layer.mock(CheckpointServiceV2)({ captureBaseline: () => Effect.void }), + Layer.mock(EventSinkV2)({ + write: () => Effect.succeed([]), + writeWithEffects: (input) => + Effect.gen(function* () { + if ( + input.events.some( + (event) => event.type === "run.updated" && event.runId === ids.runId, + ) + ) { + yield* Ref.update(observed, (current) => [...current, "root-finalized"]); + } + return []; + }), + writeIfRunCurrent: () => Effect.succeed({ committed: true, storedEvents: [] }), + writeIfProviderThreadOwner: () => + Effect.succeed({ committed: false, storedEvents: [] }), + }), + idAllocatorLayer, + Layer.mock(ProviderEventIngestorV2)({ + ingestNormalized: (input) => + Effect.gen(function* () { + const event = input.event; + if (event.type === "turn.terminal") { + yield* Ref.update(observed, (current) => [...current, "terminal"]); + } + if ( + event.type === "provider_thread.updated" && + input.writeIfProviderThreadOwner !== undefined + ) { + yield* Ref.update(observed, (current) => [...current, "stale-owner-rejected"]); + return []; + } + if (event.type === "turn_item.updated") { + yield* Ref.update(observed, (current) => [ + ...current, + `turn_item:${event.turnItem.status}`, + ]); + } + return []; + }), + }), + ServerSettingsService.layerTest(), + ), + ), + ); + + const providerThreadBase = { + id: ids.providerThreadId, + driver, + providerInstanceId, + providerSessionId: ProviderSessionId.make(`session:${key}`), + appThreadId: ids.threadId, + ownerNodeId: null, + nativeThreadRef: null, + nativeConversationHeadRef: null, + status: "idle" as const, + firstRunOrdinal: 1, + lastRunOrdinal: 1, + handoffIds: [], + forkedFrom: null, + createdAt: now, + updatedAt: now, + }; + + yield* Effect.gen(function* () { + const runExecution = yield* RunExecutionServiceV2; + yield* runExecution.startRootRun({ + commandId: CommandId.make(`command:${key}`), + appThread: { id: ids.threadId } as OrchestrationV2AppThread, + providerSessionId: ProviderSessionId.make(`session:${key}`), + session: { + events: Stream.empty, + hasPendingBackgroundWork: Effect.succeed(true), + hasPendingBackgroundWorkForThread: () => Effect.succeed(true), + subscribeEvents: Effect.succeed({ + events: Stream.fromIterable([ + backgroundTurnItemEvent(ids, "command_execution", "running", 1), + rootTerminalEvent(ids, "completed"), + // Ownership reject after a newer run claimed lastRunOrdinal. + { + type: "provider_thread.updated", + driver, + providerThread: { + ...providerThreadBase, + status: "idle" as const, + lastRunOrdinal: 1, + pendingBackgroundTasks: [ + { taskId: "bg-stale", description: "sleep 20", taskType: "local_bash" }, + ], + updatedAt: now, + }, + } as ProviderAdapterV2Event, + // Late completion still writable (turn_item writes are not + // ownership-gated); stream must stay open for it. + backgroundTurnItemEvent(ids, "command_execution", "completed", 2), + ]), + close: Deferred.succeed(ingestionDone, undefined), + }), + startTurn: () => Effect.void, + } as unknown as ProviderAdapterV2SessionRuntime, + run: { + id: ids.runId, + threadId: ids.threadId, + ordinal: 1, + providerInstanceId, + } as OrchestrationV2Run, + rootNode: { id: ids.rootNodeId } as OrchestrationV2ExecutionNode, + checkpointScope: { + id: CheckpointScopeId.make(`checkpoint-scope:${key}`), + } as OrchestrationV2CheckpointScope, + providerThread: providerThreadBase as OrchestrationV2ProviderThread, + attempt: { + id: ids.attemptId, + providerTurnId: ids.rootProviderTurnId, + } as OrchestrationV2RunAttempt, + attemptId: ids.attemptId, + providerTurnOrdinal: 1, + message: { + messageId: MessageId.make(`message:${key}:user`), + text: "Background item after ownership loss.", + attachments: [], + createdBy: "user", + creationSource: "web", + }, + modelSelection: { instanceId: providerInstanceId, model: "gpt-5.4" }, + runtimePolicy: { + runtimeMode: "full-access", + interactionMode: "default", + cwd: process.cwd(), + approvalPolicy: "never", + sandboxPolicy: { + type: "readOnly", + access: { type: "fullAccess" }, + networkAccess: false, + }, + }, + }); + }).pipe(Effect.provide(testLayer)); + + const closed = yield* Deferred.await(ingestionDone).pipe(Effect.timeoutOption("2 seconds")); + assert.isTrue( + Option.isSome(closed), + "subscription must release after background item completes past ownership loss", + ); + assert.deepEqual(yield* Ref.get(observed), [ + "turn_item:running", + "terminal", + "root-finalized", + "stale-owner-rejected", + "turn_item:completed", + ]); + }), ); -it.effect("does not pin ingestion on background items when the root turn is interrupted", () => - Effect.gen(function* () { - const observed = yield* runBackgroundItemScenario("bg-interrupted", (ids) => [ - backgroundTurnItemEvent(ids, "command_execution", "running", 1), - rootTerminalEvent(ids, "interrupted"), - backgroundTurnItemEvent(ids, "command_execution", "completed", 2), - ]); - assert.deepEqual(observed, ["turn_item:running", "root-finalized"]); - }), +it.effect( + "does not pin ingestion on a sibling session-wide pending state when this thread has no roster", + () => + Effect.gen(function* () { + const key = "bg-roster-sibling-not-pin"; + const ids = backgroundScenarioIds(key); + const providerInstanceId = ProviderInstanceId.make("codex"); + const now = yield* DateTime.now; + const observed = yield* Ref.make>([]); + const ingestionDone = yield* Deferred.make(); + const scopedProbeArgs = yield* Ref.make>([]); + const testLayer = runExecutionServiceLayer.pipe( + Layer.provide( + Layer.mergeAll( + Layer.mock(CheckpointServiceV2)({ captureBaseline: () => Effect.void }), + Layer.mock(EventSinkV2)({ + write: () => Effect.succeed([]), + writeWithEffects: (input) => + Effect.gen(function* () { + if ( + input.events.some( + (event) => event.type === "run.updated" && event.runId === ids.runId, + ) + ) { + yield* Ref.update(observed, (current) => [...current, "root-finalized"]); + } + return []; + }), + writeIfRunCurrent: () => Effect.succeed({ committed: true, storedEvents: [] }), + }), + idAllocatorLayer, + Layer.mock(ProviderEventIngestorV2)({ + ingestNormalized: (input) => + Effect.gen(function* () { + if (input.event.type === "turn.terminal") { + yield* Ref.update(observed, (current) => [...current, "terminal"]); + } + return []; + }), + }), + ServerSettingsService.layerTest(), + ), + ), + ); + + const providerThreadBase = { + id: ids.providerThreadId, + driver, + providerInstanceId, + providerSessionId: ProviderSessionId.make(`session:${key}`), + appThreadId: ids.threadId, + ownerNodeId: null, + nativeThreadRef: { + driver, + nativeId: "native-self", + strength: "strong" as const, + }, + nativeConversationHeadRef: null, + status: "idle" as const, + firstRunOrdinal: 1, + lastRunOrdinal: 1, + handoffIds: [], + forkedFrom: null, + createdAt: now, + updatedAt: now, + }; + const siblingProviderThreadId = ProviderThreadId.make(`provider-thread:${key}:sibling`); + + yield* Effect.gen(function* () { + const runExecution = yield* RunExecutionServiceV2; + yield* runExecution.startRootRun({ + commandId: CommandId.make(`command:${key}`), + appThread: { id: ids.threadId } as OrchestrationV2AppThread, + providerSessionId: ProviderSessionId.make(`session:${key}`), + session: { + events: Stream.empty, + // Session-wide stays true (sibling has work). Stop must use only + // the scoped probe for this root's provider thread. + hasPendingBackgroundWork: Effect.succeed(true), + hasPendingBackgroundWorkForThread: (providerThread: OrchestrationV2ProviderThread) => + Effect.gen(function* () { + yield* Ref.update(scopedProbeArgs, (current) => [...current, providerThread.id]); + // Own thread has no roster; sibling would report true if probed. + return providerThread.id !== ids.providerThreadId; + }), + subscribeEvents: Effect.succeed({ + events: Stream.fromIterable([ + rootTerminalEvent(ids, "completed"), + // Sibling thread update after terminal. Own-thread scoped + // pending is false, so this root must release without waiting + // for sibling-driven session-wide pending work. + { + type: "provider_thread.updated", + driver, + providerThread: { + ...providerThreadBase, + id: siblingProviderThreadId, + appThreadId: ThreadId.make(`thread:${key}:sibling`), + nativeThreadRef: { + driver, + nativeId: "native-sibling", + strength: "strong" as const, + }, + pendingBackgroundTasks: [{ taskId: "sibling-bg", description: "other thread" }], + updatedAt: now, + }, + } as ProviderAdapterV2Event, + ]), + close: Deferred.succeed(ingestionDone, undefined), + }), + startTurn: () => Effect.void, + } as unknown as ProviderAdapterV2SessionRuntime, + run: { + id: ids.runId, + threadId: ids.threadId, + ordinal: 1, + providerInstanceId, + } as OrchestrationV2Run, + rootNode: { id: ids.rootNodeId } as OrchestrationV2ExecutionNode, + checkpointScope: { + id: CheckpointScopeId.make(`checkpoint-scope:${key}`), + } as OrchestrationV2CheckpointScope, + providerThread: providerThreadBase as OrchestrationV2ProviderThread, + attempt: { + id: ids.attemptId, + providerTurnId: ids.rootProviderTurnId, + } as OrchestrationV2RunAttempt, + attemptId: ids.attemptId, + providerTurnOrdinal: 1, + message: { + messageId: MessageId.make(`message:${key}:user`), + text: "Settle without local pending work.", + attachments: [], + createdBy: "user", + creationSource: "web", + }, + modelSelection: { instanceId: providerInstanceId, model: "gpt-5.4" }, + runtimePolicy: { + runtimeMode: "full-access", + interactionMode: "default", + cwd: process.cwd(), + approvalPolicy: "never", + sandboxPolicy: { + type: "readOnly", + access: { type: "fullAccess" }, + networkAccess: false, + }, + }, + }); + }).pipe(Effect.provide(testLayer)); + + const closed = yield* Deferred.await(ingestionDone).pipe(Effect.timeoutOption("2 seconds")); + assert.isTrue(Option.isSome(closed), "event subscription did not release"); + // Critical: release while session-wide hasPendingBackgroundWork stays true + // and the scoped probe reports false for this root's provider thread. + const observedEvents = [...(yield* Ref.get(observed))]; + assert.includeMembers(observedEvents, ["terminal", "root-finalized"]); + const probedIds = yield* Ref.get(scopedProbeArgs); + assert.isTrue(probedIds.includes(ids.providerThreadId)); + assert.isFalse(probedIds.includes(siblingProviderThreadId)); + }), ); it.effect( @@ -1822,6 +2911,20 @@ function backgroundTurnItemEvent( } as ProviderAdapterV2Event; } +function backgroundTurnItemEventForRun( + ids: BackgroundScenarioIds, + runId: RunId, + type: "command_execution" | "dynamic_tool" | "subagent", + status: "running" | "completed", + ordinal: number, +): ProviderAdapterV2Event { + const event = backgroundTurnItemEvent(ids, type, status, ordinal); + if (event.type !== "turn_item.updated") { + return event; + } + return { ...event, turnItem: { ...event.turnItem, runId } }; +} + function subagentEvent( ids: BackgroundScenarioIds, status: "running" | "completed", @@ -2028,6 +3131,13 @@ function rootTerminalEvent( function runBackgroundItemScenario( key: string, makeEvents: (ids: BackgroundScenarioIds) => ReadonlyArray, + options?: { + readonly keepEventStreamOpen?: boolean; + readonly loadInheritedBackgroundTurnItems?: () => Effect.Effect< + ReadonlyArray<{ readonly id: TurnItemId; readonly runId: RunId }> + >; + readonly onSubscribe?: Effect.Effect; + }, ) { return Effect.gen(function* () { const ids = backgroundScenarioIds(key); @@ -2086,9 +3196,16 @@ function runBackgroundItemScenario( providerSessionId: ProviderSessionId.make(`session:${key}`), session: { events: Stream.empty, - subscribeEvents: Effect.succeed({ - events: Stream.fromIterable(makeEvents(ids)), - close: Deferred.succeed(ingestionDone, undefined), + subscribeEvents: Effect.gen(function* () { + yield* options?.onSubscribe ?? Effect.void; + const events = Stream.fromIterable(makeEvents(ids)); + return { + events: + options?.keepEventStreamOpen === true + ? events.pipe(Stream.concat(Stream.never)) + : events, + close: Deferred.succeed(ingestionDone, undefined), + }; }), startTurn: () => Effect.void, } as unknown as ProviderAdapterV2SessionRuntime, @@ -2111,6 +3228,11 @@ function runBackgroundItemScenario( providerTurnId: ids.rootProviderTurnId, } as OrchestrationV2RunAttempt, attemptId: ids.attemptId, + ...(options?.loadInheritedBackgroundTurnItems === undefined + ? {} + : { + loadInheritedBackgroundTurnItems: options.loadInheritedBackgroundTurnItems, + }), providerTurnOrdinal: 1, message: { messageId: MessageId.make(`message:${key}:user`), diff --git a/apps/server/src/orchestration-v2/RunExecutionService.ts b/apps/server/src/orchestration-v2/RunExecutionService.ts index 2f47e6488ddf..c6a8c24afe97 100644 --- a/apps/server/src/orchestration-v2/RunExecutionService.ts +++ b/apps/server/src/orchestration-v2/RunExecutionService.ts @@ -53,6 +53,7 @@ export interface ProviderEventRoutingState { readonly ownedThreadIds: ReadonlySet; readonly ownedProviderThreadIds: ReadonlySet; readonly ownedProviderTurnIds: ReadonlySet; + readonly inheritedBackgroundTurnItems: ReadonlyMap; readonly rootProviderTurnId: ProviderTurnId | null; } @@ -63,6 +64,11 @@ export interface ProviderEventRouteIdentity { readonly providerThreadId: ProviderThreadId; } +export interface InheritedBackgroundTurnItemRoute { + readonly id: TurnItemId; + readonly runId: OrchestrationV2Run["id"]; +} + type ProviderTerminalEvent = Extract; function isTerminalProviderTurnStatus(status: OrchestrationV2ProviderTurn["status"]): boolean { @@ -102,6 +108,46 @@ function isTerminalTurnItemStatus(status: OrchestrationV2TurnItem["status"]): bo ); } +function isSettledRunEligibleForInheritedBackground(status: OrchestrationV2Run["status"]): boolean { + return status === "interrupted" || status === "failed" || status === "cancelled"; +} + +/** + * Transfer delivery permission for exact live background items whose original + * run no longer has a subscriber. Provider sessions are runtime containers and + * can host multiple provider threads, so the durable provider-thread lineage is + * the discriminator. Completed, rolled-back, and already-terminal items remain + * excluded. + */ +export function selectInheritedBackgroundTurnItems(input: { + readonly threadId: ThreadId; + readonly currentProviderThreadId: ProviderThreadId; + readonly currentRunOrdinal: number; + readonly runs: ReadonlyArray; + readonly turnItems: ReadonlyArray; +}): ReadonlyArray { + const settledPriorRunIds = new Set( + input.runs + .filter( + (run) => + run.threadId === input.threadId && + run.ordinal < input.currentRunOrdinal && + isSettledRunEligibleForInheritedBackground(run.status), + ) + .map((run) => run.id), + ); + return input.turnItems.flatMap((turnItem) => + turnItem.threadId === input.threadId && + turnItem.providerThreadId === input.currentProviderThreadId && + turnItem.runId !== null && + settledPriorRunIds.has(turnItem.runId) && + backgroundCapableTurnItemTypes.has(turnItem.type) && + !isTerminalTurnItemStatus(turnItem.status) + ? [{ id: turnItem.id, runId: turnItem.runId }] + : [], + ); +} + type SubagentTurnItem = Extract; type OpenRunOwnedSubagentProjection = { @@ -274,6 +320,7 @@ export function finalProviderThreadStatus( export function makeProviderEventRoutingState(input: { readonly identity: ProviderEventRouteIdentity; + readonly inheritedBackgroundTurnItems?: ReadonlyArray; readonly providerTurnId: ProviderTurnId | null; readonly relatedThreadIds?: ReadonlyArray; readonly relatedProviderThreadIds?: ReadonlyArray; @@ -286,6 +333,9 @@ export function makeProviderEventRoutingState(input: { ]), ownedProviderTurnIds: input.providerTurnId === null ? new Set() : new Set([input.providerTurnId]), + inheritedBackgroundTurnItems: new Map( + (input.inheritedBackgroundTurnItems ?? []).map((item) => [item.id, item.runId]), + ), rootProviderTurnId: input.providerTurnId, }; } @@ -363,8 +413,28 @@ export function routeProviderEvent( return [ownsRun(event.subagent.runId) || ownsChildThread(event.subagent.threadId), state]; case "message.updated": return [ownsRun(event.message.runId) || ownsChildThread(event.message.threadId), state]; - case "turn_item.updated": - return [ownsRun(event.turnItem.runId) || ownsChildThread(event.turnItem.threadId), state]; + case "turn_item.updated": { + if (ownsRun(event.turnItem.runId) || ownsChildThread(event.turnItem.threadId)) { + return [true, state]; + } + const inheritedRunId = state.inheritedBackgroundTurnItems.get(event.turnItem.id); + // Preserve the item's original ownership while allowing the one live run + // to deliver an exact carryover identity selected from the projection. + const isInheritedBackgroundItem = + event.turnItem.threadId === input.threadId && + event.turnItem.runId !== null && + event.turnItem.runId === inheritedRunId && + backgroundCapableTurnItemTypes.has(event.turnItem.type); + if (!isInheritedBackgroundItem) { + return [false, state]; + } + if (!isTerminalTurnItemStatus(event.turnItem.status)) { + return [true, state]; + } + const inheritedBackgroundTurnItems = new Map(state.inheritedBackgroundTurnItems); + inheritedBackgroundTurnItems.delete(event.turnItem.id); + return [true, { ...state, inheritedBackgroundTurnItems }]; + } case "plan.updated": return [ownsRun(event.plan.runId) || ownsChildThread(event.plan.threadId), state]; case "runtime_request.updated": @@ -428,6 +498,10 @@ export interface RunExecutionServiceV2StartRootRunInput { readonly attempt: OrchestrationV2RunAttempt; readonly attemptId: RunAttemptId; readonly providerTurnOrdinal: number; + readonly loadInheritedBackgroundTurnItems?: () => Effect.Effect< + ReadonlyArray, + unknown + >; readonly relatedThreadIds?: ReadonlyArray; readonly relatedProviderThreadIds?: ReadonlyArray; readonly shouldStartProviderTurn?: () => Effect.Effect; @@ -765,9 +839,30 @@ export const layer: Layer.Layer< attemptId: input.attempt.id, providerThreadId: input.providerThread.id, }; + const eventSubscription = + input.session.subscribeEvents === undefined + ? { events: input.session.events, close: Effect.void } + : yield* input.session.subscribeEvents; + const inheritedBackgroundTurnItems = yield* ( + input.loadInheritedBackgroundTurnItems?.() ?? Effect.succeed([]) + ).pipe( + Effect.onError(() => eventSubscription.close), + Effect.mapError( + (cause) => + new RunExecutionStartError({ + commandId: input.commandId, + runId: input.run.id, + cause, + }), + ), + ); + const inheritedBackgroundTurnItemsById = new Map( + inheritedBackgroundTurnItems.map((item) => [item.id, item.runId]), + ); const eventRouting = yield* Ref.make( makeProviderEventRoutingState({ identity: routeIdentity, + inheritedBackgroundTurnItems, providerTurnId: input.attempt.providerTurnId, ...(input.relatedThreadIds === undefined ? {} @@ -779,11 +874,12 @@ export const layer: Layer.Layer< ); const rootTerminalSeen = yield* Ref.make(false); const rootRunFinalized = yield* Ref.make(false); + const providerThreadOwnerLost = yield* Ref.make(false); const activeChildProviderTurns = yield* Ref.make>(new Set()); const activeChildSubagents = yield* Ref.make>(new Set()); const activeBackgroundTurnItems = yield* Ref.make< ReadonlySet - >(new Set()); + >(new Set(inheritedBackgroundTurnItemsById.keys())); const openRunOwnedSubagents = yield* Ref.make(emptyOpenRunOwnedSubagentProjection()); const finalizeRootRun = (terminal: ProviderTerminalEvent) => Effect.gen(function* () { @@ -895,9 +991,15 @@ export const layer: Layer.Layer< const belongsToOwnedChildThread = event.turnItem.threadId !== input.run.threadId && routing.ownedThreadIds.has(event.turnItem.threadId); + const belongsToInheritedBackgroundItem = + event.turnItem.threadId === input.run.threadId && + event.turnItem.runId !== null && + inheritedBackgroundTurnItemsById.get(event.turnItem.id) === event.turnItem.runId; if ( backgroundCapableTurnItemTypes.has(event.turnItem.type) && - (belongsToRootRun || belongsToOwnedChildThread) + (belongsToRootRun || + belongsToOwnedChildThread || + belongsToInheritedBackgroundItem) ) { yield* Ref.update(activeBackgroundTurnItems, (current) => { const next = new Set(current); @@ -940,6 +1042,7 @@ export const layer: Layer.Layer< return false; } const terminal = yield* Ref.get(terminalEvent); + // Non-completed terminals drop background tracking immediately. if (terminal !== null && terminal.status !== "completed") { return true; } @@ -956,16 +1059,40 @@ export const layer: Layer.Layer< // non-terminal, so their late completion events reach the // projection (stuck-spinner fix). Only for completed runs: // interrupted/failed turns intentionally drop background tracking - // rather than pinning the stream open. Assumes adapters emit an - // item's non-terminal event before the root terminal; an item - // first seen after the terminal is not pinned. + // rather than pinning the stream open. Newly owned items depend on + // adapters emitting a non-terminal event before the root terminal. + // Exact inherited items are seeded from their selected durable rows. + // + // Owner loss (a newer run claimed lastRunOrdinal) must not close + // this stream while these sets are non-empty: turn_item.updated + // writes are not ownership-gated, so late completions still land. const backgroundItems = yield* Ref.get(activeBackgroundTurnItems); - return backgroundItems.size === 0; + if (backgroundItems.size > 0) { + return false; + } + // Owner loss means do not hold the stream open solely for the + // roster probe; once background sets are empty, release. + if (yield* Ref.get(providerThreadOwnerLost)) { + return true; + } + // Claude background Bash has no turn-item projection. Keep the + // stream open while this root's provider thread still reports + // pending roster work so late empty updates can clear Waiting. + // Use only the thread-scoped probe: session-wide pending work + // (siblings, wake buffers, session subagents) must not pin this + // root subscription. Session idle release still uses + // hasPendingBackgroundWork via ProviderSessionManager. + const latestProviderThreadSnapshot = yield* Ref.get(latestProviderThread); + if (input.session.hasPendingBackgroundWorkForThread !== undefined) { + const hasPendingWork = yield* input.session + .hasPendingBackgroundWorkForThread(latestProviderThreadSnapshot) + .pipe(Effect.catchCause(() => Effect.succeed(false))); + if (hasPendingWork) { + return false; + } + } + return true; }); - const eventSubscription = - input.session.subscribeEvents === undefined - ? { events: input.session.events, close: Effect.void } - : yield* input.session.subscribeEvents; const providerEventFiber = yield* eventSubscription.events.pipe( Stream.filterEffect((event) => Ref.modify(eventRouting, (state) => routeProviderEvent(event, routeIdentity, state)), @@ -975,6 +1102,15 @@ export const layer: Layer.Layer< let storedEventCount = 0; const shouldDeliver = shouldDeliverProviderEvent(event, assistantStreamingEnabled); if (shouldDeliver) { + // Root provider_thread.updated always uses an ownership gate: + // pre-terminal writeIfRunCurrent (attempt still running), or + // post-terminal writeIfProviderThreadOwner so late roster + // clears still land while this attempt owns the run and this + // run owns lastRunOrdinal. + const rootTerminalAlreadySeen = yield* Ref.get(rootTerminalSeen); + const isRootProviderThreadUpdate = + event.type === "provider_thread.updated" && + event.providerThread.id === input.providerThread.id; const storedEvents = yield* providerEventIngestor.ingestNormalized({ providerSessionId: input.providerSessionId, providerInstanceId: input.run.providerInstanceId, @@ -982,18 +1118,35 @@ export const layer: Layer.Layer< runId: input.run.id, nodeId: input.rootNode.id, event, - ...(event.type === "provider_thread.updated" && - event.providerThread.id === input.providerThread.id - ? { - writeIfRunCurrent: { - runId: input.run.id, - activeAttemptId: input.attempt.id, - expectedStatus: "running" as const, - }, - } + ...(isRootProviderThreadUpdate + ? rootTerminalAlreadySeen + ? { + writeIfProviderThreadOwner: { + providerThreadId: input.providerThread.id, + runId: input.run.id, + activeAttemptId: input.attempt.id, + expectedLastRunOrdinal: input.run.ordinal, + }, + } + : { + writeIfRunCurrent: { + runId: input.run.id, + activeAttemptId: input.attempt.id, + expectedStatus: "running" as const, + }, + } : {}), }); storedEventCount = storedEvents.length; + if ( + isRootProviderThreadUpdate && + rootTerminalAlreadySeen && + storedEventCount === 0 + ) { + // Ownership lost (or thread row missing). Stop pinning the + // stream on this run's background probe. + yield* Ref.set(providerThreadOwnerLost, true); + } } if (event.type === "provider_thread.updated") { if (event.providerThread.id === input.providerThread.id && storedEventCount > 0) { diff --git a/apps/server/src/orchestration-v2/testkit/CodexReplayFixtures.integration.test.ts b/apps/server/src/orchestration-v2/testkit/CodexReplayFixtures.integration.test.ts index 9c0b279332db..ecc7e14aaeda 100644 --- a/apps/server/src/orchestration-v2/testkit/CodexReplayFixtures.integration.test.ts +++ b/apps/server/src/orchestration-v2/testkit/CodexReplayFixtures.integration.test.ts @@ -29,44 +29,56 @@ type ProtocolReplayEntry = Extract< { readonly type: "expect_outbound" | "emit_inbound" } >; -const CODEX_REPLAY_FIXTURES = ORCHESTRATOR_REPLAY_FIXTURES.flatMap((fixture) => +interface CodexReplayFixtureRegistration { + readonly registrationScenario: string; + readonly recordedScenario: string; + readonly transcriptFile: URL; +} + +const CODEX_REPLAY_FIXTURE_REGISTRATIONS = ORCHESTRATOR_REPLAY_FIXTURES.flatMap((fixture) => fixture.providers .filter((provider) => provider.driver === "codex") .map((provider) => ({ - scenario: fixture.name, + registrationScenario: fixture.name, + recordedScenario: provider.recordedScenario ?? fixture.name, transcriptFile: provider.transcriptFile, })), ).concat([ { - scenario: "provider_thread_resume", + registrationScenario: "provider_thread_resume", + recordedScenario: "provider_thread_resume", transcriptFile: new URL( "./fixtures/provider_thread_resume/codex_transcript.ndjson", import.meta.url, ), }, { - scenario: "thread_fork_native_continue", + registrationScenario: "thread_fork_native_continue", + recordedScenario: "thread_fork_native_continue", transcriptFile: new URL( "./fixtures/thread_fork_native_continue/codex_transcript.ndjson", import.meta.url, ), }, { - scenario: "thread_fork_native_siblings", + registrationScenario: "thread_fork_native_siblings", + recordedScenario: "thread_fork_native_siblings", transcriptFile: new URL( "./fixtures/thread_fork_native_siblings/codex_transcript.ndjson", import.meta.url, ), }, { - scenario: "thread_merge_back_continue", + registrationScenario: "thread_merge_back_continue", + recordedScenario: "thread_merge_back_continue", transcriptFile: new URL( "./fixtures/thread_merge_back_continue/codex_transcript.ndjson", import.meta.url, ), }, { - scenario: "thread_merge_back_siblings", + registrationScenario: "thread_merge_back_siblings", + recordedScenario: "thread_merge_back_siblings", transcriptFile: new URL( "./fixtures/thread_merge_back_siblings/codex_transcript.ndjson", import.meta.url, @@ -74,6 +86,25 @@ const CODEX_REPLAY_FIXTURES = ORCHESTRATOR_REPLAY_FIXTURES.flatMap((fixture) => }, ]); +function uniqueCanonicalTranscripts( + registrations: ReadonlyArray, +): ReadonlyArray { + const canonicalTranscripts = new Map(); + for (const registration of registrations) { + const canonicalUrl = registration.transcriptFile.href; + const existing = canonicalTranscripts.get(canonicalUrl); + if (existing !== undefined && existing.recordedScenario !== registration.recordedScenario) { + throw new Error( + `Codex replay transcript ${canonicalUrl} has conflicting recorded scenarios ${existing.recordedScenario} (${existing.registrationScenario}) and ${registration.recordedScenario} (${registration.registrationScenario}).`, + ); + } + canonicalTranscripts.set(canonicalUrl, existing ?? registration); + } + return Array.from(canonicalTranscripts.values()); +} + +const CODEX_REPLAY_TRANSCRIPTS = uniqueCanonicalTranscripts(CODEX_REPLAY_FIXTURE_REGISTRATIONS); + const scenarioExpectations = { simple: { outgoing: ["initialize", "initialized", "thread/start", "turn/start"], @@ -641,23 +672,25 @@ function assertSiblingMergeBackSemantics(transcript: ProviderReplayTranscript) { } describe("Codex replay fixtures", () => { - it.effect("loads every current Codex fixture as a codex app-server replay transcript", () => + it.effect("loads each canonical Codex fixture as an app-server replay transcript", () => Effect.gen(function* () { - for (const fixture of CODEX_REPLAY_FIXTURES) { + for (const fixture of CODEX_REPLAY_TRANSCRIPTS) { const transcript = yield* readTranscript(fixture.transcriptFile); const codexTranscript = yield* decodeCodexTranscript(transcript); const first = transcript.entries[0]; assert.equal(codexTranscript.provider, "codex"); assert.equal(codexTranscript.protocol, "codex.app-server"); - assert.equal(codexTranscript.scenario, fixture.scenario); + assert.equal(codexTranscript.scenario, fixture.recordedScenario); assert.deepEqual(codexTranscript.entries.at(-1), { type: "runtime_exit", status: "success", }); assert.equal(first?.type, "expect_outbound"); if (first?.type !== "expect_outbound") { - throw new Error(`Expected ${fixture.scenario} to start with initialize outbound frame.`); + throw new Error( + `Expected ${fixture.recordedScenario} to start with initialize outbound frame.`, + ); } assert.equal(first.label, "initialize"); @@ -671,16 +704,32 @@ describe("Codex replay fixtures", () => { }), ); - it.effect("covers the expected replay suite exactly", () => - Effect.gen(function* () { - const transcripts = yield* Effect.forEach(CODEX_REPLAY_FIXTURES, (fixture) => - readTranscript(fixture.transcriptFile), - ); + it("covers the expected replay suite exactly", () => { + assert.deepEqual( + CODEX_REPLAY_TRANSCRIPTS.map((fixture) => fixture.recordedScenario).toSorted(), + Object.keys(scenarioExpectations).toSorted(), + ); + }); - assert.deepEqual( - transcripts.map((transcript) => transcript.scenario).toSorted(), - Object.keys(scenarioExpectations).toSorted(), - ); - }), - ); + it("rejects conflicting recorded scenarios for one canonical transcript", () => { + const transcriptFile = new URL( + "./fixtures/queued_turn/codex_transcript.ndjson", + import.meta.url, + ); + + assert.throws(() => + uniqueCanonicalTranscripts([ + { + registrationScenario: "queued_turn", + recordedScenario: "queued_turn", + transcriptFile, + }, + { + registrationScenario: "conflicting_alias", + recordedScenario: "different_recording", + transcriptFile, + }, + ]), + ); + }); }); diff --git a/apps/server/src/orchestration-v2/testkit/OrchestratorReplayFixtures.contract.test.ts b/apps/server/src/orchestration-v2/testkit/OrchestratorReplayFixtures.contract.test.ts index 2c09ef245add..6021642ee97f 100644 --- a/apps/server/src/orchestration-v2/testkit/OrchestratorReplayFixtures.contract.test.ts +++ b/apps/server/src/orchestration-v2/testkit/OrchestratorReplayFixtures.contract.test.ts @@ -51,6 +51,192 @@ describe("orchestrator replay fixture contract", () => { }).pipe(Effect.provide(idAllocatorLayer), provideDeterministicTestRuntime), ); + it.effect( + "keeps consecutive queue_message inputs queued without an intermediate idle barrier", + () => + Effect.gen(function* () { + const materialized = yield* materializeFixtureInput({ + scenario: "consecutive-queue-message-ordering", + fixtureInput: { + steps: [ + { type: "message", text: "active run" }, + { type: "queue_message", text: "queued run 1" }, + { type: "queue_message", text: "queued run 2" }, + ], + }, + driver: ProviderDriverKind.make("codex"), + modelSelection: CODEX_MODEL_SELECTION, + }); + const queueCommands = materialized.commands.filter( + (command) => + command.type === "message.dispatch" && + (command.text === "queued run 1" || command.text === "queued run 2"), + ); + assert.lengthOf(queueCommands, 2); + for (const command of queueCommands) { + assert.deepInclude(command, { + dispatchMode: { type: "queue_after_active" }, + }); + } + + const firstQueueDispatchIndex = materialized.steps.findIndex( + (step) => + step.type === "dispatch" && + step.command.type === "message.dispatch" && + step.command.text === "queued run 1", + ); + const secondQueueDispatchIndex = materialized.steps.findIndex( + (step) => + step.type === "dispatch" && + step.command.type === "message.dispatch" && + step.command.text === "queued run 2", + ); + assert.isAtLeast(firstQueueDispatchIndex, 0); + assert.isAbove(secondQueueDispatchIndex, firstQueueDispatchIndex); + + const betweenQueueSteps = materialized.steps.slice( + firstQueueDispatchIndex + 1, + secondQueueDispatchIndex, + ); + assert.isFalse( + betweenQueueSteps.some( + (step) => step.type === "await" || step.type === "await_thread_idle", + ), + "consecutive queue_message steps must not await the active run or thread idle between queues", + ); + + const afterSecondQueue = materialized.steps.slice(secondQueueDispatchIndex + 1); + const barrierAwaitIndex = afterSecondQueue.findIndex((step) => step.type === "await"); + const barrierIdleIndex = afterSecondQueue.findIndex( + (step) => step.type === "await_thread_idle", + ); + assert.isAtLeast( + barrierAwaitIndex, + 0, + "the final queue_message still inserts the post-queue await barrier", + ); + assert.deepEqual(afterSecondQueue[barrierAwaitIndex], { + type: "await", + key: "run:1", + }); + assert.isAbove( + barrierIdleIndex, + barrierAwaitIndex, + "the final queue_message still inserts await_thread_idle after its await", + ); + }).pipe(Effect.provide(idAllocatorLayer), provideDeterministicTestRuntime), + ); + + it.effect( + "does not await thread idle between steer/restart and answer_next_user_input_request", + () => + Effect.gen(function* () { + for (const steeringType of ["steer", "restart"] as const) { + const answers = { "question-0": "answer" }; + const materialized = yield* materializeFixtureInput({ + scenario: `${steeringType}-then-answer-user-input-ordering`, + fixtureInput: { + steps: [ + { type: "message", text: "active run" }, + { + type: steeringType, + text: `${steeringType} active run`, + targetRunIndex: 1, + }, + { + type: "answer_next_user_input_request", + answers, + }, + ], + }, + driver: ProviderDriverKind.make("codex"), + modelSelection: CODEX_MODEL_SELECTION, + }); + + const steeringDispatchIndex = materialized.steps.findIndex( + (step) => + step.type === "dispatch" && + step.command.type === "message.dispatch" && + step.command.text === `${steeringType} active run`, + ); + const answerStepIndex = materialized.steps.findIndex( + (step) => + step.type === "respond_to_next_runtime_request" && + step.answers !== undefined && + Object.keys(step.answers).includes("question-0"), + ); + assert.isAtLeast(steeringDispatchIndex, 0, `${steeringType} dispatch must materialize`); + assert.isAbove( + answerStepIndex, + steeringDispatchIndex, + `${steeringType} answer must follow the steering dispatch`, + ); + + const betweenSteps = materialized.steps.slice(steeringDispatchIndex + 1, answerStepIndex); + assert.isFalse( + betweenSteps.some((step) => step.type === "await" || step.type === "await_thread_idle"), + `${steeringType} followed by answer_next_user_input_request must not await idle before answering`, + ); + } + }).pipe(Effect.provide(idAllocatorLayer), provideDeterministicTestRuntime), + ); + + it.effect("keeps the active dispatch barrier through queued-run replacement", () => + Effect.gen(function* () { + const materialized = yield* materializeFixtureInput({ + scenario: "queued-run-replacement-ordering", + fixtureInput: { + steps: [ + { type: "message", text: "active run" }, + { type: "queue_message", text: "queued run" }, + { type: "cancel_queued_run", targetRunIndex: 2 }, + { type: "queue_message", text: "replacement queued run" }, + ], + }, + driver: ProviderDriverKind.make("codex"), + modelSelection: CODEX_MODEL_SELECTION, + }); + const replacementQueueDispatchIndex = materialized.steps.findIndex( + (step) => + step.type === "dispatch" && + step.command.type === "message.dispatch" && + step.command.text === "replacement queued run", + ); + assert.isAtLeast(replacementQueueDispatchIndex, 0); + + const afterReplacementQueue = materialized.steps.slice(replacementQueueDispatchIndex + 1); + const barrierAwait = afterReplacementQueue.find((step) => step.type === "await"); + assert.deepEqual(barrierAwait, { + type: "await", + key: "run:1", + }); + }).pipe(Effect.provide(idAllocatorLayer), provideDeterministicTestRuntime), + ); + + it.effect("materializes fixture run-status waits against derived run IDs", () => + Effect.gen(function* () { + const idAllocator = yield* IdAllocatorV2; + const materialized = yield* materializeFixtureInput({ + scenario: "await-fixture-run-status", + fixtureInput: { + steps: [{ type: "await_run_status", targetRunIndex: 1, status: "running" }], + }, + driver: ProviderDriverKind.make("codex"), + modelSelection: CODEX_MODEL_SELECTION, + }); + const threadId = materialized.projectionThreadIds[0]; + assert.isDefined(threadId); + const runStatusWait = materialized.steps.find((step) => step.type === "await_run_status"); + + assert.deepEqual(runStatusWait, { + type: "await_run_status", + threadId, + runId: idAllocator.derive.run({ threadId, ordinal: 1 }), + status: "running", + }); + }).pipe(Effect.provide(idAllocatorLayer), provideDeterministicTestRuntime), + ); + it.effect("keeps message ordinals separate from app run ordinals after steering", () => Effect.gen(function* () { const idAllocator = yield* IdAllocatorV2; @@ -112,6 +298,14 @@ describe("orchestrator replay fixture contract", () => { for (const provider of fixture.providers) { const transcript = yield* readTranscript(provider.transcriptFile); + const replayGateLabels = fixture + .buildInput() + .steps.flatMap((step) => + step.type === "release_replay_gate" || + step.type === "release_replay_gate_after_waiting" + ? [step.label] + : [], + ); const materialized = yield* materializeFixtureInput({ scenario: fixture.name, fixtureInput: fixture.buildInput(), @@ -120,7 +314,7 @@ describe("orchestrator replay fixture contract", () => { }).pipe(Effect.provide(idAllocatorLayer), provideDeterministicTestRuntime); const firstCommand = materialized.commands[0]; - assert.equal(transcript.scenario, fixture.name); + assert.equal(transcript.scenario, provider.recordedScenario ?? fixture.name); if (provider.driver === "acpRegistry") { assert.include( ["acpRegistry", "grok"], @@ -140,14 +334,39 @@ describe("orchestrator replay fixture contract", () => { throw new Error(`${fixture.name}/${provider.driver} must start with thread.create`); } assert.equal(firstCommand.threadId, materialized.projectionThreadIds[0]); - // advance_clock only moves the test clock; every other input step - // dispatches a command. + // These fixture steps coordinate the test runtime; every other + // input step dispatches a command. const commandProducingSteps = fixture .buildInput() - .steps.filter((step) => step.type !== "advance_clock"); + .steps.filter( + (step) => + step.type !== "advance_clock" && + step.type !== "await_run_status" && + step.type !== "capture_shell_snapshot" && + step.type !== "release_replay_gate" && + step.type !== "release_replay_gate_after_waiting", + ); assert.equal(materialized.commands.length, commandProducingSteps.length + 1); assert.isAtLeast(materialized.steps.length, materialized.commands.length); assert.equal(typeof provider.assertOutput, "function"); + for (const label of replayGateLabels) { + assert.isTrue( + transcript.entries.some( + (entry) => entry.type === "emit_inbound" && entry.label === label, + ), + `${fixture.name}/${provider.driver} replay gate ${label} must label an inbound frame`, + ); + } + if (provider.transcriptEntriesThroughLabel !== undefined) { + assert.isTrue( + transcript.entries.some( + (entry) => + entry.type === "emit_inbound" && + entry.label === provider.transcriptEntriesThroughLabel, + ), + `${fixture.name}/${provider.driver} transcript slice must name an inbound frame`, + ); + } assertUnique( materialized.commands.map((command) => command.commandId), diff --git a/apps/server/src/orchestration-v2/testkit/OrchestratorReplayFixtures.integration.test.ts b/apps/server/src/orchestration-v2/testkit/OrchestratorReplayFixtures.integration.test.ts index 09b756655dc4..b822220d76e9 100644 --- a/apps/server/src/orchestration-v2/testkit/OrchestratorReplayFixtures.integration.test.ts +++ b/apps/server/src/orchestration-v2/testkit/OrchestratorReplayFixtures.integration.test.ts @@ -39,6 +39,25 @@ function normalizeTestError(cause: unknown): Error { return cause instanceof Error ? cause : new Error(String(cause)); } +function transcriptEntriesThroughLabel( + transcript: ProviderReplayTranscript, + label: string | undefined, +): ProviderReplayTranscript { + if (label === undefined) { + return transcript; + } + const entryIndex = transcript.entries.findIndex( + (entry) => entry.type === "emit_inbound" && entry.label === label, + ); + if (entryIndex === -1) { + throw new Error(`${transcript.scenario} is missing inbound transcript label ${label}.`); + } + return { + ...transcript, + entries: transcript.entries.slice(0, entryIndex + 1), + }; +} + function isStreamingAssistantEvent(event: OrchestrationV2DomainEvent): boolean { switch (event.type) { case "node.updated": @@ -63,11 +82,15 @@ const runFixtureProvider = Effect.fn("runOrchestratorReplayFixture")(function* < readonly enableAssistantStreaming?: boolean; }) { const rawTranscript = yield* readTranscript(input.driver.transcriptFile); + const replayTranscript = transcriptEntriesThroughLabel( + rawTranscript, + input.driver.transcriptEntriesThroughLabel, + ); const workspace = yield* checkpointWorkspace(input.fixtureName); const transcript = yield* input.harness.decodeTranscript( input.driver.driver === "codex" - ? materializeReplayTranscriptWorkspace(rawTranscript, workspace) - : rawTranscript, + ? materializeReplayTranscriptWorkspace(replayTranscript, workspace) + : replayTranscript, ); const materialized = yield* materializeFixtureInput({ scenario: input.fixtureName, diff --git a/apps/server/src/orchestration-v2/testkit/OrchestratorScenario.ts b/apps/server/src/orchestration-v2/testkit/OrchestratorScenario.ts index 7d4c62982435..ba8a13b97408 100644 --- a/apps/server/src/orchestration-v2/testkit/OrchestratorScenario.ts +++ b/apps/server/src/orchestration-v2/testkit/OrchestratorScenario.ts @@ -21,6 +21,7 @@ import * as Stream from "effect/Stream"; import { TestClock } from "effect/testing"; import { OrchestratorV2, type OrchestratorV2Error } from "../Orchestrator.ts"; +import type { ProviderReplayGate } from "./ProviderReplayGate.testkit.ts"; export type OrchestratorV2ScenarioStep = | { @@ -61,6 +62,20 @@ export type OrchestratorV2ScenarioStep = readonly runId: OrchestrationV2Run["id"]; readonly itemType: OrchestrationV2TurnItem["type"]; } + | { + readonly type: "release_replay_gate_after_waiting"; + readonly label: string; + readonly threadId: ThreadId; + readonly runId: OrchestrationV2Run["id"]; + } + | { + readonly type: "release_replay_gate"; + readonly label: string; + } + | { + readonly type: "capture_shell_snapshot"; + readonly key: string; + } | { readonly type: "respond_to_next_runtime_request"; readonly threadId: ThreadId; @@ -81,6 +96,7 @@ export interface OrchestratorV2ScenarioResult { readonly domainEvents: ReadonlyArray; readonly projections: ReadonlyMap; readonly shellSnapshot: OrchestrationV2ThreadShellSnapshot; + readonly capturedShellSnapshots: ReadonlyMap; } export class OrchestratorV2ScenarioStepError extends Schema.TaggedErrorClass()( @@ -204,6 +220,9 @@ function collectProjectionThreadIds(scenario: OrchestratorV2Scenario): ReadonlyA export function runOrchestratorV2Scenario( scenario: OrchestratorV2Scenario, + options: { + readonly replayGate?: ProviderReplayGate; + } = {}, ): Effect.Effect< OrchestratorV2ScenarioResult, OrchestratorV2Error | OrchestratorV2ScenarioStepError, @@ -224,6 +243,7 @@ export function runOrchestratorV2Scenario( string, Fiber.Fiber, OrchestratorV2Error> >(); + const capturedShellSnapshots = new Map(); let anonymousBackgroundDispatchIndex = 0; const awaitDispatch = (key: string) => @@ -384,6 +404,100 @@ export function runOrchestratorV2Scenario( ); }); + const releaseReplayGateAfterWaiting = ( + label: string, + threadId: ThreadId, + runId: OrchestrationV2Run["id"], + attemptsRemaining = SCENARIO_WAIT_ATTEMPTS, + ): Effect.Effect => + Effect.gen(function* () { + const projection = yield* orchestrator.getThreadProjection(threadId); + const run = projection.runs.find((candidate) => candidate.id === runId); + const providerThread = projection.providerThreads.find( + (candidate) => candidate.id === run?.providerThreadId, + ); + const pendingTaskCount = providerThread?.pendingBackgroundTasks?.length ?? 0; + const gateReached = options.replayGate?.hasReached(label) ?? false; + if ( + gateReached && + run?.status === "completed" && + providerThread !== undefined && + pendingTaskCount > 0 + ) { + options.replayGate?.release(label); + yield* waitForProviderBackgroundTasksCleared(threadId, providerThread.id); + return; + } + if (attemptsRemaining <= 0) { + options.replayGate?.release(label); + return yield* new OrchestratorV2ScenarioStepError({ + scenario: scenario.name, + step: `release_replay_gate_after_waiting:${label}:reached=${gateReached}:run=${run?.status ?? "missing"}:providerThread=${run?.providerThreadId ?? "missing"}:pending=${pendingTaskCount}`, + }); + } + yield* yieldToRuntime; + return yield* releaseReplayGateAfterWaiting( + label, + threadId, + runId, + attemptsRemaining - 1, + ); + }); + + const waitForProviderBackgroundTasksCleared = ( + threadId: ThreadId, + providerThreadId: NonNullable, + attemptsRemaining = SCENARIO_WAIT_ATTEMPTS, + ): Effect.Effect => + Effect.gen(function* () { + const projection = yield* orchestrator.getThreadProjection(threadId); + const providerThread = projection.providerThreads.find( + (candidate) => candidate.id === providerThreadId, + ); + const hasPendingTasks = (providerThread?.pendingBackgroundTasks?.length ?? 0) > 0; + if (!hasPendingTasks && providerThread?.status === "idle") { + return; + } + if (attemptsRemaining <= 0) { + const providerState = projection.providerThreads + .map( + (candidate) => + `${candidate.id}:${candidate.status}:pending=${candidate.pendingBackgroundTasks?.length ?? 0}`, + ) + .join(","); + return yield* new OrchestratorV2ScenarioStepError({ + scenario: scenario.name, + step: `await_provider_background_tasks_cleared:${threadId}:target=${providerThreadId}:providers=${providerState}`, + }); + } + yield* yieldToRuntime; + return yield* waitForProviderBackgroundTasksCleared( + threadId, + providerThreadId, + attemptsRemaining - 1, + ); + }); + + const releaseReplayGate = ( + label: string, + attemptsRemaining = SCENARIO_WAIT_ATTEMPTS, + ): Effect.Effect => + Effect.gen(function* () { + if (options.replayGate?.hasReached(label) ?? false) { + options.replayGate?.release(label); + return; + } + if (attemptsRemaining <= 0) { + options.replayGate?.release(label); + return yield* new OrchestratorV2ScenarioStepError({ + scenario: scenario.name, + step: `release_replay_gate:${label}:reached=false`, + }); + } + yield* yieldToRuntime; + return yield* releaseReplayGate(label, attemptsRemaining - 1); + }); + for (const step of scenarioSteps(scenario)) { switch (step.type) { case "dispatch": { @@ -427,6 +541,15 @@ export function runOrchestratorV2Scenario( case "await_run_turn_item": yield* waitForRunTurnItem(step.threadId, step.runId, step.itemType); break; + case "release_replay_gate_after_waiting": + yield* releaseReplayGateAfterWaiting(step.label, step.threadId, step.runId); + break; + case "release_replay_gate": + yield* releaseReplayGate(step.label); + break; + case "capture_shell_snapshot": + capturedShellSnapshots.set(step.key, yield* orchestrator.getShellSnapshot()); + break; case "respond_to_next_runtime_request": { const request = yield* waitForPendingRuntimeRequest(step.threadId); const result = yield* orchestrator.dispatch({ @@ -468,6 +591,7 @@ export function runOrchestratorV2Scenario( domainEvents: storedEvents.map((stored) => stored.event), projections, shellSnapshot, + capturedShellSnapshots, }; }), ); diff --git a/apps/server/src/orchestration-v2/testkit/ProviderReplayGate.testkit.test.ts b/apps/server/src/orchestration-v2/testkit/ProviderReplayGate.testkit.test.ts new file mode 100644 index 000000000000..371badcec069 --- /dev/null +++ b/apps/server/src/orchestration-v2/testkit/ProviderReplayGate.testkit.test.ts @@ -0,0 +1,17 @@ +import { describe, expect, it } from "vite-plus/test"; + +import { makeProviderReplayGate } from "./ProviderReplayGate.testkit.ts"; + +describe("ProviderReplayGate", () => { + it("stops waiting when the replay consumer is interrupted", async () => { + const label = "held-frame"; + const gate = makeProviderReplayGate([label]); + const controller = new AbortController(); + const waiting = gate.beforeEmit(label, controller.signal); + + expect(gate.hasReached(label)).toBe(true); + controller.abort(); + await waiting; + expect(gate.release(label)).toBe(true); + }); +}); diff --git a/apps/server/src/orchestration-v2/testkit/ProviderReplayGate.testkit.ts b/apps/server/src/orchestration-v2/testkit/ProviderReplayGate.testkit.ts new file mode 100644 index 000000000000..a9bbb3b9befd --- /dev/null +++ b/apps/server/src/orchestration-v2/testkit/ProviderReplayGate.testkit.ts @@ -0,0 +1,78 @@ +export interface ProviderReplayGate { + readonly beforeEmit: (label: string | undefined, signal?: AbortSignal) => Promise; + readonly hasReached: (label: string) => boolean; + readonly release: (label: string) => boolean; + readonly releaseAll: () => void; +} + +interface GateState { + reached: boolean; + released: boolean; + readonly promise: Promise; + readonly resolve: () => void; +} + +export function makeProviderReplayGate(labels: ReadonlyArray): ProviderReplayGate { + const states = new Map(); + for (const label of labels) { + if (states.has(label)) { + throw new Error(`Duplicate provider replay gate label ${label}.`); + } + let resolve = () => {}; + const promise = new Promise((resume) => { + resolve = resume; + }); + states.set(label, { + reached: false, + released: false, + promise, + resolve, + }); + } + + return { + beforeEmit: (label, signal) => { + if (label === undefined) { + return Promise.resolve(); + } + const state = states.get(label); + if (state === undefined) { + return Promise.resolve(); + } + state.reached = true; + if (signal === undefined) { + return state.promise; + } + if (signal.aborted) { + return Promise.resolve(); + } + return new Promise((resolve) => { + const stopWaiting = () => resolve(); + signal.addEventListener("abort", stopWaiting, { once: true }); + void state.promise.then(() => { + signal.removeEventListener("abort", stopWaiting); + resolve(); + }); + }); + }, + hasReached: (label) => states.get(label)?.reached ?? false, + release: (label) => { + const state = states.get(label); + if (state === undefined || state.released) { + return false; + } + state.released = true; + state.resolve(); + return true; + }, + releaseAll: () => { + for (const state of states.values()) { + if (state.released) { + continue; + } + state.released = true; + state.resolve(); + } + }, + }; +} diff --git a/apps/server/src/orchestration-v2/testkit/ProviderReplayHarness.ts b/apps/server/src/orchestration-v2/testkit/ProviderReplayHarness.ts index 679316a47856..1b922dbecd28 100644 --- a/apps/server/src/orchestration-v2/testkit/ProviderReplayHarness.ts +++ b/apps/server/src/orchestration-v2/testkit/ProviderReplayHarness.ts @@ -57,6 +57,7 @@ import { type OrchestratorV2Scenario, type OrchestratorV2ScenarioResult, } from "./OrchestratorScenario.ts"; +import { makeProviderReplayGate, type ProviderReplayGate } from "./ProviderReplayGate.testkit.ts"; export function makeReplayServerConfig( scenario: string, @@ -156,6 +157,7 @@ export interface OrchestratorV2ProviderReplayHarness< ) => Effect.Effect; readonly makeProviderAdapterRegistryLayer: ( transcript: Transcript, + options?: { readonly replayGate?: ProviderReplayGate }, ) => Layer.Layer; } @@ -183,9 +185,19 @@ export function runOrchestratorV2ProviderReplayScenario< | SqlError, never > { - const layer = makeOrchestratorV2ProviderReplayLayer(scenario, harness, options); + const replayGate = makeProviderReplayGate( + scenario.steps?.flatMap((step) => + step.type === "release_replay_gate" || step.type === "release_replay_gate_after_waiting" + ? [step.label] + : [], + ) ?? [], + ); + const layer = makeOrchestratorV2ProviderReplayLayer(scenario, harness, { + ...options, + replayGate, + }); - return runOrchestratorV2Scenario(scenario).pipe(Effect.provide(layer)); + return runOrchestratorV2Scenario(scenario, { replayGate }).pipe(Effect.provide(layer)); } export function makeOrchestratorV2ProviderReplayLayer< @@ -201,9 +213,13 @@ export function makeOrchestratorV2ProviderReplayLayer< >; readonly enableAssistantStreaming?: boolean; readonly runEffectWorker?: boolean; + readonly replayGate?: ProviderReplayGate; } = {}, ): Layer.Layer { - const registryLayer = harness.makeProviderAdapterRegistryLayer(scenario.transcript); + const registryLayer = harness.makeProviderAdapterRegistryLayer( + scenario.transcript, + options.replayGate === undefined ? {} : { replayGate: options.replayGate }, + ); return makeOrchestratorV2ReplayLayerWithRegistry(scenario, registryLayer, options); } diff --git a/apps/server/src/orchestration-v2/testkit/fixtures/claude_background_task_after_root/claude_transcript.ndjson b/apps/server/src/orchestration-v2/testkit/fixtures/claude_background_task_after_root/claude_transcript.ndjson new file mode 100644 index 000000000000..2e22da2d3b9b --- /dev/null +++ b/apps/server/src/orchestration-v2/testkit/fixtures/claude_background_task_after_root/claude_transcript.ndjson @@ -0,0 +1,13 @@ +{"type":"transcript_start","provider":"claudeAgent","protocol":"claude-agent-sdk.query","version":"0.2.111","scenario":"claude_background_task_after_root","metadata":{"prompts":["Live-test post-settle background Bash wake. Do exactly this in order, with no extra steps.\n\n1) Run this exact command using the Bash tool with run_in_background set to true:\n sleep 25 && echo L2_BG_DONE\n2) Immediately after starting it, reply with a short message containing exactly L2_STARTED and stop.\n3) Do NOT poll TaskOutput. Do NOT wait for the task. Do not spawn subagents or monitors.\n4) If its completion is reported later, acknowledge it once by replying with exactly L2_WAKE: L2_BG_DONE and stop.\n\nThe point is that your first turn ends while the command is still running."],"model":"claude-sonnet-4-6","nativeSessionId":"ec24a006-617b-4324-b516-57a7afff15a4","queryMode":"streaming","tools":"claude_code","permissionMode":"bypassPermissions","timingCompression":"25-second background gap controlled by the replay scenario gate","generatedBy":"manual-replay-from-thread-ec24a006 (ctm-catchup-l2-20260729, captured 2026-07-30T00:35:14Z)"}} +{"type":"expect_outbound","label":"query.open","frame":{"type":"query.open","options":{"model":"claude-sonnet-4-6","tools":{"type":"preset","preset":"claude_code"},"permissionMode":"bypassPermissions","allowDangerouslySkipPermissions":true,"sessionId":"ec24a006-617b-4324-b516-57a7afff15a4"}}} +{"type":"expect_outbound","label":"prompt.offer:1","frame":{"type":"prompt.offer","message":{"type":"user","message":{"role":"user","content":"Live-test post-settle background Bash wake. Do exactly this in order, with no extra steps.\n\n1) Run this exact command using the Bash tool with run_in_background set to true:\n sleep 25 && echo L2_BG_DONE\n2) Immediately after starting it, reply with a short message containing exactly L2_STARTED and stop.\n3) Do NOT poll TaskOutput. Do NOT wait for the task. Do not spawn subagents or monitors.\n4) If its completion is reported later, acknowledge it once by replying with exactly L2_WAKE: L2_BG_DONE and stop.\n\nThe point is that your first turn ends while the command is still running."},"parent_tool_use_id":null}}} +{"type":"emit_inbound","label":"system:init","frame":{"type":"system","subtype":"init","agents":[],"apiKeySource":"none","claude_code_version":"2.1.219","cwd":"/tmp/claude-replay-claude_background_task_after_root","tools":[],"mcp_servers":[],"model":"claude-sonnet-4-6","permissionMode":"bypassPermissions","slash_commands":[],"output_style":"default","skills":[],"plugins":[],"fast_mode_state":"off","uuid":"fdd8c2b3-f443-405c-996d-1a5ac798e100","session_id":"ec24a006-617b-4324-b516-57a7afff15a4"}} +{"type":"emit_inbound","label":"assistant:background-bash","frame":{"type":"assistant","message":{"model":"claude-sonnet-4-6","id":"msg_011CdXJcN9jaK2KDf7LcqkDx","type":"message","role":"assistant","content":[{"type":"tool_use","id":"toolu_01DAnwwvVvLM1cTfzrm6kkor","name":"Bash","input":{"command":"sleep 25 && echo L2_BG_DONE","run_in_background":true},"caller":{"type":"direct"}}],"stop_reason":"tool_use","stop_sequence":null,"stop_details":null,"usage":{"input_tokens":2,"cache_creation_input_tokens":0,"cache_read_input_tokens":0,"output_tokens":89,"service_tier":"standard","inference_geo":"not_available"},"context_management":null},"parent_tool_use_id":null,"session_id":"ec24a006-617b-4324-b516-57a7afff15a4","uuid":"6e8e602b-da0f-409d-92a4-8a912b9c7ca9"}} +{"type":"emit_inbound","label":"task_started:local_bash","frame":{"type":"system","subtype":"task_started","task_id":"bc9gkn8ei","tool_use_id":"toolu_01DAnwwvVvLM1cTfzrm6kkor","description":"sleep 25 && echo L2_BG_DONE","task_type":"local_bash","uuid":"ed0f530b-93fe-4f94-a385-367b440a04ba","session_id":"ec24a006-617b-4324-b516-57a7afff15a4"}} +{"type":"emit_inbound","label":"background_tasks_changed:pending","frame":{"type":"system","subtype":"background_tasks_changed","tasks":[{"task_id":"bc9gkn8ei","description":"sleep 25 && echo L2_BG_DONE","task_type":"local_bash"}],"uuid":"7d3cb8ba-6736-4b30-a748-c511cfa61803","session_id":"ec24a006-617b-4324-b516-57a7afff15a4"}} +{"type":"emit_inbound","label":"user:background-bash-ack","frame":{"type":"user","message":{"role":"user","content":[{"tool_use_id":"toolu_01DAnwwvVvLM1cTfzrm6kkor","type":"tool_result","content":"Command running in background with ID: bc9gkn8ei. Output is being written to: /tmp/claude-replay-claude_background_task_after_root/tasks/bc9gkn8ei.output. You will be notified when it completes. To check interim output, use Read on that file path.","is_error":false}]},"parent_tool_use_id":null,"session_id":"ec24a006-617b-4324-b516-57a7afff15a4","uuid":"7e95842b-dec4-4be5-9b20-9738697edf62","timestamp":"2026-07-30T00:35:18.555Z","tool_use_result":{"stdout":"","stderr":"","interrupted":false,"isImage":false,"noOutputExpected":false,"backgroundTaskId":"bc9gkn8ei"}}} +{"type":"emit_inbound","label":"assistant:root-final","frame":{"type":"assistant","message":{"model":"claude-sonnet-4-6","id":"msg_011CdXJcdoKdTMfN4depYTKM","type":"message","role":"assistant","content":[{"type":"text","text":"L2_STARTED"}],"stop_reason":"end_turn","stop_sequence":null,"stop_details":null,"usage":{"input_tokens":2,"cache_creation_input_tokens":0,"cache_read_input_tokens":0,"output_tokens":9,"service_tier":"standard","inference_geo":"not_available"},"context_management":null},"parent_tool_use_id":null,"session_id":"ec24a006-617b-4324-b516-57a7afff15a4","uuid":"12ddebd6-6f2d-4223-be97-56ed454f7d15"}} +{"type":"emit_inbound","label":"result:root","frame":{"type":"result","subtype":"success","is_error":false,"api_error_status":null,"duration_ms":7125,"duration_api_ms":6500,"num_turns":2,"result":"L2_STARTED","stop_reason":"end_turn","session_id":"ec24a006-617b-4324-b516-57a7afff15a4","total_cost_usd":0.001,"usage":{"input_tokens":2,"cache_creation_input_tokens":0,"cache_read_input_tokens":0,"output_tokens":9,"server_tool_use":{"web_search_requests":0,"web_fetch_requests":0},"service_tier":"standard","cache_creation":{"ephemeral_1h_input_tokens":0,"ephemeral_5m_input_tokens":0},"inference_geo":"","iterations":[{"input_tokens":2,"output_tokens":9,"cache_read_input_tokens":0,"cache_creation_input_tokens":0,"cache_creation":{"ephemeral_5m_input_tokens":0,"ephemeral_1h_input_tokens":0},"type":"message"}],"speed":"standard"},"modelUsage":{"claude-sonnet-4-6":{"inputTokens":2,"outputTokens":9,"cacheReadInputTokens":0,"cacheCreationInputTokens":0,"webSearchRequests":0,"costUSD":0.001,"contextWindow":200000,"maxOutputTokens":32000}},"permission_denials":[],"terminal_reason":"completed","fast_mode_state":"off","uuid":"633b885b-240f-4f44-bfe1-d554088b2dfd"}} +{"type":"emit_inbound","label":"background_tasks_changed:empty","frame":{"type":"system","subtype":"background_tasks_changed","tasks":[],"uuid":"3464e6d2-1a25-4e37-82de-d305440ed44b","session_id":"ec24a006-617b-4324-b516-57a7afff15a4"}} +{"type":"emit_inbound","label":"task_notification:local_bash","frame":{"type":"system","subtype":"task_notification","task_id":"bc9gkn8ei","tool_use_id":"toolu_01DAnwwvVvLM1cTfzrm6kkor","status":"completed","output_file":"/tmp/claude-replay-claude_background_task_after_root/tasks/bc9gkn8ei.output","summary":"Background command \"sleep 25 && echo L2_BG_DONE\" completed (exit code 0)","uuid":"0a8bc186-10a1-421c-a319-e3c7db9cb4a6","session_id":"ec24a006-617b-4324-b516-57a7afff15a4"}} +{"type":"runtime_exit","status":"success"} diff --git a/apps/server/src/orchestration-v2/testkit/fixtures/claude_background_task_after_root/input.ts b/apps/server/src/orchestration-v2/testkit/fixtures/claude_background_task_after_root/input.ts new file mode 100644 index 000000000000..8ed37f284cce --- /dev/null +++ b/apps/server/src/orchestration-v2/testkit/fixtures/claude_background_task_after_root/input.ts @@ -0,0 +1,26 @@ +import type { OrchestratorFixtureInput } from "../shared.ts"; + +export const CLAUDE_BACKGROUND_TASK_AFTER_ROOT_PROMPT = [ + "Live-test post-settle background Bash wake. Do exactly this in order, with no extra steps.", + "", + "1) Run this exact command using the Bash tool with run_in_background set to true:", + " sleep 25 && echo L2_BG_DONE", + "2) Immediately after starting it, reply with a short message containing exactly L2_STARTED and stop.", + "3) Do NOT poll TaskOutput. Do NOT wait for the task. Do not spawn subagents or monitors.", + "4) If its completion is reported later, acknowledge it once by replying with exactly L2_WAKE: L2_BG_DONE and stop.", + "", + "The point is that your first turn ends while the command is still running.", +].join("\n"); + +export function claudeBackgroundTaskAfterRootInput(): OrchestratorFixtureInput { + return { + steps: [ + { type: "message", text: CLAUDE_BACKGROUND_TASK_AFTER_ROOT_PROMPT }, + { + type: "release_replay_gate_after_waiting", + label: "background_tasks_changed:empty", + targetRunIndex: 1, + }, + ], + }; +} diff --git a/apps/server/src/orchestration-v2/testkit/fixtures/claude_background_task_after_root/output.ts b/apps/server/src/orchestration-v2/testkit/fixtures/claude_background_task_after_root/output.ts new file mode 100644 index 000000000000..38cbeaf35dae --- /dev/null +++ b/apps/server/src/orchestration-v2/testkit/fixtures/claude_background_task_after_root/output.ts @@ -0,0 +1,96 @@ +import { assert } from "@effect/vitest"; +import type { ProviderReplayTranscript } from "@t3tools/contracts"; + +import type { OrchestratorV2ScenarioResult } from "../../OrchestratorScenario.ts"; +import { + assertBaseProjection, + assertSemanticProjectionIntegrity, + assertUserMessagesInclude, + projectionFor, +} from "../shared.ts"; +import { CLAUDE_BACKGROUND_TASK_AFTER_ROOT_PROMPT } from "./input.ts"; + +const BACKGROUND_TASK_ID = "bc9gkn8ei"; + +export function assertClaudeBackgroundTaskAfterRootOutput( + result: OrchestratorV2ScenarioResult, + transcript: ProviderReplayTranscript, +) { + assertBaseProjection({ + result, + transcript, + runCount: 1, + runStatuses: ["completed"], + }); + + const projection = projectionFor(result, transcript.scenario); + assertSemanticProjectionIntegrity(projection); + assertUserMessagesInclude(projection, [CLAUDE_BACKGROUND_TASK_AFTER_ROOT_PROMPT]); + + const rootRun = projection.runs[0]; + assert.isDefined(rootRun); + const pendingRosterIndex = result.domainEvents.findIndex( + (event) => + event.type === "provider-thread.updated" && + event.payload.pendingBackgroundTasks?.some((task) => task.taskId === BACKGROUND_TASK_ID), + ); + const waitingRunIndex = result.domainEvents.findIndex( + (event) => + event.type === "run.updated" && + event.runId === rootRun.id && + event.payload.status === "waiting", + ); + const waitingRootNodeIndex = result.domainEvents.findIndex( + (event) => + event.type === "node.updated" && + event.payload.runId === rootRun.id && + event.payload.kind === "root_turn" && + event.payload.status === "waiting", + ); + const idleRosterIndex = result.domainEvents.findIndex( + (event, index) => + index > waitingRunIndex && + event.type === "provider-thread.updated" && + event.payload.status === "idle" && + (event.payload.pendingBackgroundTasks?.length ?? 0) === 0, + ); + + assert.isAtLeast(pendingRosterIndex, 0, "replay must project the live background task roster"); + assert.isAbove( + waitingRunIndex, + pendingRosterIndex, + "checkpoint-waiting root run must retain the background roster", + ); + assert.isAbove( + waitingRootNodeIndex, + pendingRosterIndex, + "checkpoint-waiting root node must retain the background roster", + ); + assert.isAbove( + idleRosterIndex, + waitingRunIndex, + "late background completion must clear the roster and return the provider thread to idle", + ); + assert.equal(rootRun.status, "completed"); + const rootNode = projection.nodes.find( + (node) => node.runId === rootRun.id && node.kind === "root_turn", + ); + assert.isDefined(rootNode); + assert.equal(rootNode.status, "completed"); + + assert.lengthOf(projection.providerThreads, 1); + assert.equal(projection.providerThreads[0]?.status, "idle"); + assert.deepEqual(projection.providerThreads[0]?.pendingBackgroundTasks ?? [], []); + assert.lengthOf(projection.subagents, 0); + + const assistantTexts = projection.turnItems.flatMap((item) => + item.type === "assistant_message" ? [item.text] : [], + ); + assert.deepEqual(assistantTexts, ["L2_STARTED"]); + + const shell = result.shellSnapshot.threads.find((thread) => thread.id === projection.thread.id); + assert.isDefined(shell); + assert.equal(shell.activeRunId, null); + assert.equal(shell.status, "completed"); + assert.deepEqual(shell.pendingBackgroundTasks ?? [], []); +} diff --git a/apps/server/src/orchestration-v2/testkit/fixtures/index.ts b/apps/server/src/orchestration-v2/testkit/fixtures/index.ts index 510b85c4c46a..b77993624462 100644 --- a/apps/server/src/orchestration-v2/testkit/fixtures/index.ts +++ b/apps/server/src/orchestration-v2/testkit/fixtures/index.ts @@ -1,5 +1,7 @@ import { ProviderDriverKind } from "@t3tools/contracts"; +import { claudeBackgroundTaskAfterRootInput } from "./claude_background_task_after_root/input.ts"; +import { assertClaudeBackgroundTaskAfterRootOutput } from "./claude_background_task_after_root/output.ts"; import { claudeIdleResumeInput } from "./claude_idle_resume/input.ts"; import { assertClaudeIdleResumeOutput } from "./claude_idle_resume/output.ts"; import { claudeLocalBashTaskInput } from "./claude_local_bash_task/input.ts"; @@ -24,6 +26,8 @@ import { planQuestionsInput } from "./plan_questions/input.ts"; import { assertProposedPlanOutput } from "./proposed_plan/codex_output.ts"; import { assertProposedPlanCursorOutput } from "./proposed_plan/cursor_output.ts"; import { proposedPlanInput } from "./proposed_plan/input.ts"; +import { assertQueuedCancelledWhileActiveOutput } from "./queued_cancelled_while_active/codex_output.ts"; +import { queuedCancelledWhileActiveInput } from "./queued_cancelled_while_active/input.ts"; import { assertQueuedTurnOutput } from "./queued_turn/codex_output.ts"; import { queuedTurnInput } from "./queued_turn/input.ts"; import { assertSimpleClaudeOutput } from "./simple/claude_output.ts"; @@ -83,7 +87,22 @@ import { WORKSPACE_NEVER_POLICY, } from "./shared.ts"; -export const ORCHESTRATOR_REPLAY_FIXTURES = [ +export const ORCHESTRATOR_REPLAY_FIXTURES: ReadonlyArray = [ + { + name: "claude_background_task_after_root", + buildInput: claudeBackgroundTaskAfterRootInput, + providers: [ + { + driver: ProviderDriverKind.make("claudeAgent"), + transcriptFile: new URL( + "./claude_background_task_after_root/claude_transcript.ndjson", + import.meta.url, + ), + modelSelection: CLAUDE_MODEL_SELECTION, + assertOutput: assertClaudeBackgroundTaskAfterRootOutput, + }, + ], + }, { name: "claude_local_bash_task", buildInput: claudeLocalBashTaskInput, @@ -457,6 +476,20 @@ export const ORCHESTRATOR_REPLAY_FIXTURES = [ }, ], }, + { + name: "queued_cancelled_while_active", + buildInput: queuedCancelledWhileActiveInput, + providers: [ + { + driver: ProviderDriverKind.make("codex"), + transcriptFile: new URL("./queued_turn/codex_transcript.ndjson", import.meta.url), + recordedScenario: "queued_turn", + transcriptEntriesThroughLabel: "turn/completed", + modelSelection: CODEX_MODEL_SELECTION, + assertOutput: assertQueuedCancelledWhileActiveOutput, + }, + ], + }, { name: "queued_turn", buildInput: queuedTurnInput, @@ -737,7 +770,7 @@ export const ORCHESTRATOR_REPLAY_FIXTURES = [ }, ], }, -] satisfies ReadonlyArray; +]; // TODO(claude-v2/approvals-denied): add denied write fixtures after the live query runner records // Claude denial callback responses. Cross-reference diff --git a/apps/server/src/orchestration-v2/testkit/fixtures/queued_cancelled_while_active/codex_output.ts b/apps/server/src/orchestration-v2/testkit/fixtures/queued_cancelled_while_active/codex_output.ts new file mode 100644 index 000000000000..9f20f064257e --- /dev/null +++ b/apps/server/src/orchestration-v2/testkit/fixtures/queued_cancelled_while_active/codex_output.ts @@ -0,0 +1,42 @@ +import { assert } from "@effect/vitest"; +import type { ProviderReplayTranscript } from "@t3tools/contracts"; + +import type { OrchestratorV2ScenarioResult } from "../../OrchestratorScenario.ts"; +import { assertBaseProjection, projectionFor } from "../shared.ts"; +import { ACTIVE_RUN_SHELL_SNAPSHOT_KEY } from "./input.ts"; + +export function assertQueuedCancelledWhileActiveOutput( + result: OrchestratorV2ScenarioResult, + transcript: ProviderReplayTranscript, +) { + assertBaseProjection({ + result, + transcript, + runCount: 2, + runStatuses: ["completed", "cancelled"], + providerTurnCountAtLeast: 1, + }); + + const projection = projectionFor(result, transcript.scenario); + const activeRun = projection.runs[0]; + const cancelledRun = projection.runs[1]; + assert.isDefined(activeRun); + assert.isDefined(cancelledRun); + assert.isNull(cancelledRun.startedAt); + const cancelledMessage = projection.messages.find( + (message) => message.id === cancelledRun.userMessageId, + ); + assert.isDefined(cancelledMessage); + assert.equal(cancelledMessage.createdBy, "agent"); + assert.equal(cancelledMessage.creationSource, "server"); + + const capturedShell = result.capturedShellSnapshots + .get(ACTIVE_RUN_SHELL_SNAPSHOT_KEY) + ?.threads.find((thread) => thread.id === projection.thread.id); + assert.isDefined(capturedShell); + assert.equal(capturedShell.latestRunId, cancelledRun.id); + assert.equal(capturedShell.status, "cancelled"); + assert.equal(capturedShell.activeRunId, activeRun.id); + assert.equal(capturedShell.activityRunStatus, "running"); + assert.deepEqual(capturedShell.pendingBackgroundTasks, []); +} diff --git a/apps/server/src/orchestration-v2/testkit/fixtures/queued_cancelled_while_active/input.ts b/apps/server/src/orchestration-v2/testkit/fixtures/queued_cancelled_while_active/input.ts new file mode 100644 index 000000000000..a8e0ba7651cc --- /dev/null +++ b/apps/server/src/orchestration-v2/testkit/fixtures/queued_cancelled_while_active/input.ts @@ -0,0 +1,25 @@ +import { + MULTI_TURN_FIRST_PROMPT, + MULTI_TURN_SECOND_PROMPT, + type OrchestratorFixtureInput, +} from "../shared.ts"; + +export const ACTIVE_RUN_SHELL_SNAPSHOT_KEY = "active-run-over-cancelled-latest"; + +export function queuedCancelledWhileActiveInput(): OrchestratorFixtureInput { + return { + steps: [ + { type: "message", text: MULTI_TURN_FIRST_PROMPT }, + { + type: "queue_message", + text: MULTI_TURN_SECOND_PROMPT, + createdBy: "agent", + creationSource: "server", + }, + { type: "cancel_queued_run", targetRunIndex: 2 }, + { type: "await_run_status", targetRunIndex: 1, status: "running" }, + { type: "capture_shell_snapshot", key: ACTIVE_RUN_SHELL_SNAPSHOT_KEY }, + { type: "release_replay_gate", label: "turn/completed" }, + ], + }; +} diff --git a/apps/server/src/orchestration-v2/testkit/fixtures/shared.ts b/apps/server/src/orchestration-v2/testkit/fixtures/shared.ts index 0a50aca4e5dd..685565c63fd9 100644 --- a/apps/server/src/orchestration-v2/testkit/fixtures/shared.ts +++ b/apps/server/src/orchestration-v2/testkit/fixtures/shared.ts @@ -162,6 +162,31 @@ export type OrchestratorFixtureInputStep = readonly type: "queue_message"; readonly text: string; readonly attachments?: ReadonlyArray; + readonly createdBy?: Extract< + OrchestrationV2Command, + { readonly type: "message.dispatch" } + >["createdBy"]; + readonly creationSource?: Extract< + OrchestrationV2Command, + { readonly type: "message.dispatch" } + >["creationSource"]; + } + | { + readonly type: "cancel_queued_run"; + readonly targetRunIndex: number; + } + | { + readonly type: "await_run_status"; + readonly targetRunIndex: number; + readonly status: OrchestrationV2RunStatus; + } + | { + readonly type: "capture_shell_snapshot"; + readonly key: string; + } + | { + readonly type: "release_replay_gate"; + readonly label: string; } | { readonly type: "steer"; @@ -180,6 +205,11 @@ export type OrchestratorFixtureInputStep = readonly targetRunIndex: number; readonly waitForTurnItemType?: OrchestrationV2TurnItem["type"]; } + | { + readonly type: "release_replay_gate_after_waiting"; + readonly label: string; + readonly targetRunIndex: number; + } | { readonly type: "approve_next_runtime_request"; readonly decision?: Extract< @@ -213,6 +243,8 @@ export interface OrchestratorFixtureInput { export interface ProviderOrchestratorReplayVariant { readonly driver: ProviderDriverKind; readonly transcriptFile: URL; + readonly recordedScenario?: string; + readonly transcriptEntriesThroughLabel?: string; readonly modelSelection: ModelSelection; readonly runtimePolicyOverride?: RuntimePolicyV2Override; readonly assertOutput: ( @@ -348,6 +380,14 @@ export function dispatchMessageCommand(input: { readonly messageId: MessageId; readonly text: string; readonly attachments?: ReadonlyArray; + readonly createdBy?: Extract< + OrchestrationV2Command, + { readonly type: "message.dispatch" } + >["createdBy"]; + readonly creationSource?: Extract< + OrchestrationV2Command, + { readonly type: "message.dispatch" } + >["creationSource"]; readonly dispatchMode?: Extract< OrchestrationV2Command, { readonly type: "message.dispatch" } @@ -355,8 +395,8 @@ export function dispatchMessageCommand(input: { }): OrchestrationV2Command { return { type: "message.dispatch", - createdBy: "user", - creationSource: "web", + createdBy: input.createdBy ?? "user", + creationSource: input.creationSource ?? "web", commandId: input.commandId, threadId: input.ids.threadId, messageId: input.messageId, @@ -436,7 +476,9 @@ export function materializeFixtureInput(input: { (nextStep !== undefined && ((nextStep.type === "interrupt" && nextStep.targetRunIndex === runIndex) || nextStep.type === "queue_message" || - (nextStep.type === "restart" && nextStep.targetRunIndex === runIndex))) || + (nextStep.type === "restart" && nextStep.targetRunIndex === runIndex) || + (nextStep.type === "release_replay_gate_after_waiting" && + nextStep.targetRunIndex === runIndex))) || nextStep?.type === "approve_next_runtime_request" || nextStep?.type === "answer_next_user_input_request"; const key = `run:${runIndex}`; @@ -470,9 +512,10 @@ export function materializeFixtureInput(input: { } } break; - case "queue_message": + case "queue_message": { messageIndex += 1; runIndex += 1; + const nextStep = input.fixtureInput.steps[stepIndex + 1]; pushDispatch( dispatchMessageCommand({ commandId: yield* idAllocator.allocate.command({ @@ -486,12 +529,48 @@ export function materializeFixtureInput(input: { ordinal: messageIndex, }), text: step.text, + ...(step.createdBy === undefined ? {} : { createdBy: step.createdBy }), + ...(step.creationSource === undefined ? {} : { creationSource: step.creationSource }), ...(step.attachments === undefined ? {} : { attachments: step.attachments }), dispatchMode: { type: "queue_after_active" }, }), ); - steps.push({ type: "await", key: `run:${runIndex - 1}` }); - steps.push({ type: "await_thread_idle", threadId: ids.threadId }); + const shouldSkipQueueBarrier = + nextStep?.type === "queue_message" || + (nextStep?.type === "cancel_queued_run" && nextStep.targetRunIndex === runIndex); + if (!shouldSkipQueueBarrier) { + const queueBarrierKey = + Array.from(activeRunDispatchKeys).at(-1) ?? `run:${runIndex - 1}`; + activeRunDispatchKeys.delete(queueBarrierKey); + steps.push({ type: "await", key: queueBarrierKey }); + steps.push({ type: "await_thread_idle", threadId: ids.threadId }); + } + break; + } + case "cancel_queued_run": + pushDispatch({ + type: "queued-run.cancel", + commandId: yield* idAllocator.allocate.command({ + fixtureName: input.scenario, + commandName: `cancel-queued-run-${step.targetRunIndex}`, + }), + threadId: ids.threadId, + runId: runIdFor(step.targetRunIndex), + }); + break; + case "await_run_status": + steps.push({ + type: "await_run_status", + threadId: ids.threadId, + runId: runIdFor(step.targetRunIndex), + status: step.status, + }); + break; + case "capture_shell_snapshot": + steps.push({ type: "capture_shell_snapshot", key: step.key }); + break; + case "release_replay_gate": + steps.push({ type: "release_replay_gate", label: step.label }); break; case "answer_next_user_input_request": pushDispatch( @@ -572,11 +651,17 @@ export function materializeFixtureInput(input: { }, }), ); - if (input.fixtureInput.steps[stepIndex + 1]?.type !== "approve_next_runtime_request") { - if (activeRunDispatchKeys.delete(`run:${step.targetRunIndex}`)) { - steps.push({ type: "await", key: `run:${step.targetRunIndex}` }); + { + const nextStepType = input.fixtureInput.steps[stepIndex + 1]?.type; + if ( + nextStepType !== "approve_next_runtime_request" && + nextStepType !== "answer_next_user_input_request" + ) { + if (activeRunDispatchKeys.delete(`run:${step.targetRunIndex}`)) { + steps.push({ type: "await", key: `run:${step.targetRunIndex}` }); + } + steps.push({ type: "await_thread_idle", threadId: ids.threadId }); } - steps.push({ type: "await_thread_idle", threadId: ids.threadId }); } break; case "restart": @@ -606,11 +691,17 @@ export function materializeFixtureInput(input: { }, }), ); - if (input.fixtureInput.steps[stepIndex + 1]?.type !== "approve_next_runtime_request") { - if (activeRunDispatchKeys.delete(`run:${step.targetRunIndex}`)) { - steps.push({ type: "await", key: `run:${step.targetRunIndex}` }); + { + const nextStepType = input.fixtureInput.steps[stepIndex + 1]?.type; + if ( + nextStepType !== "approve_next_runtime_request" && + nextStepType !== "answer_next_user_input_request" + ) { + if (activeRunDispatchKeys.delete(`run:${step.targetRunIndex}`)) { + steps.push({ type: "await", key: `run:${step.targetRunIndex}` }); + } + steps.push({ type: "await_thread_idle", threadId: ids.threadId }); } - steps.push({ type: "await_thread_idle", threadId: ids.threadId }); } break; case "interrupt": @@ -645,6 +736,14 @@ export function materializeFixtureInput(input: { steps.push({ type: "advance_clock", duration: "1 millis" }); steps.push({ type: "await_thread_idle", threadId: ids.threadId }); break; + case "release_replay_gate_after_waiting": + steps.push({ + type: "release_replay_gate_after_waiting", + label: step.label, + threadId: ids.threadId, + runId: runIdFor(step.targetRunIndex), + }); + break; case "advance_clock": steps.push({ type: "advance_clock", duration: step.duration }); break; diff --git a/apps/server/src/relay/AgentAwarenessRelay.ts b/apps/server/src/relay/AgentAwarenessRelay.ts index 1f392923b1c5..ccf8eea93f7b 100644 --- a/apps/server/src/relay/AgentAwarenessRelay.ts +++ b/apps/server/src/relay/AgentAwarenessRelay.ts @@ -193,6 +193,7 @@ export function describeThreadShellForAwareness( return { found: true, status: shell.status, + activityRunStatus: shell.activityRunStatus ?? null, activeRunId: shell.activeRunId ?? null, latestRunId: shell.latestRunId ?? null, pendingRuntimeRequestKind: shell.pendingRuntimeRequest?.kind ?? null, diff --git a/apps/web/src/components/ChatView.tsx b/apps/web/src/components/ChatView.tsx index 808e3272da14..14d9a30d7f8f 100644 --- a/apps/web/src/components/ChatView.tsx +++ b/apps/web/src/components/ChatView.tsx @@ -48,6 +48,7 @@ import { resolvePromptInjectedEffort, } from "@t3tools/shared/model"; import { CHAT_LIST_ANCHOR_OFFSET } from "@t3tools/shared/chatList"; +import { derivePendingBackgroundWork } from "@t3tools/shared/orchestrationV2PendingBackgroundWork"; import { projectScriptCwd, projectScriptRuntimeEnv } from "@t3tools/shared/projectScripts"; import { truncate } from "@t3tools/shared/String"; import { nextTerminalId, resolveTerminalSessionLabel } from "@t3tools/shared/terminalLabels"; @@ -2325,6 +2326,26 @@ function ChatViewContent(props: ChatViewProps) { threadError, }); const isWorking = phase === "running" || isSendBusy || isConnecting || isRevertingCheckpoint; + const pendingBackgroundTasks = useMemo(() => { + if (serverProjection === null || serverProjection === undefined) { + return []; + } + const latestRun = + serverProjection.runs.length === 0 + ? null + : serverProjection.runs.reduce((latest, candidate) => + candidate.ordinal > latest.ordinal ? candidate : latest, + ); + return [ + ...derivePendingBackgroundWork({ + latestRun, + providerThreads: serverProjection.providerThreads, + turnItems: serverProjection.turnItems, + activeProviderThreadId: serverProjection.thread.activeProviderThreadId, + runs: serverProjection.runs, + }), + ]; + }, [serverProjection]); const activeWorkStartedAt = deriveActiveWorkStartedAt( activeActivityRun, activeRuntime, @@ -6297,6 +6318,7 @@ function ChatViewContent(props: ChatViewProps) { isWorking={isWorking} activeTurnInProgress={isWorking || !latestRunSettled} activeTurnStartedAt={activeWorkStartedAt} + pendingBackgroundTasks={pendingBackgroundTasks} listRef={legendListRef} timelineEntries={timelineEntries} latestRun={activeActivityRun} diff --git a/apps/web/src/components/Sidebar.logic.test.ts b/apps/web/src/components/Sidebar.logic.test.ts index 2d51c3dd87e7..028df2c9c738 100644 --- a/apps/web/src/components/Sidebar.logic.test.ts +++ b/apps/web/src/components/Sidebar.logic.test.ts @@ -21,10 +21,12 @@ import { resolveSidebarStageBadgeLabel, resolveThreadRowClassName, resolveSidebarV2Status, + resolveSidebarV2TopStatus, resolveThreadStatusPill, resolveWorkingStartedAt, searchSidebarThreadsByTitle, formatWorkingDurationLabel, + shouldShowSidebarV2Duration, shouldNavigateAfterProjectRemoval, shouldClearThreadSelectionOnMouseDown, sortLogicalProjectsForSidebar, @@ -770,12 +772,23 @@ describe("resolveSidebarV2Status", () => { ...idle, runtime: { ...runtime, status: "idle" as const, lastError: "persisted" }, }), - ).toBe("ready"); + ).toBe("waiting"); }); it("defaults to ready with no runtime", () => { expect(resolveSidebarV2Status(idle)).toBe("ready"); }); + + it("keeps a waiting runtime visible ahead of unread and woke presentation", () => { + expect(resolveSidebarV2TopStatus({ status: "waiting", isUnread: true, isWoke: true })).toBe( + "waiting", + ); + }); + + it("keeps Waiting static while Working shows elapsed duration", () => { + expect(shouldShowSidebarV2Duration("waiting")).toBe(false); + expect(shouldShowSidebarV2Duration("working")).toBe(true); + }); }); describe("searchSidebarThreadsByTitle", () => { @@ -1006,6 +1019,54 @@ describe("resolveThreadStatusPill", () => { ).toMatchObject({ label: "Working", pulse: true }); }); + it("shows waiting for an idle thread with pending background tasks", () => { + expect( + resolveThreadStatusPill({ + thread: { + ...baseThread, + pendingBackgroundTasks: [{ taskId: "bg-1", description: "sleep 20" }], + runtime: { + ...baseThread.runtime, + status: "idle", + activeRunId: null, + }, + }, + }), + ).toMatchObject({ + label: "Waiting", + colorClass: "text-sidebar-muted-foreground", + dotClass: "bg-sidebar-muted-foreground", + pulse: false, + }); + }); + + it("keeps an active turn working when background tasks are also present", () => { + expect( + resolveThreadStatusPill({ + thread: { + ...baseThread, + pendingBackgroundTasks: [{ taskId: "bg-1", description: "sleep 20" }], + }, + }), + ).toMatchObject({ label: "Working", pulse: true }); + }); + + it("does not show waiting after the background task roster clears", () => { + expect( + resolveThreadStatusPill({ + thread: { + ...baseThread, + pendingBackgroundTasks: [], + runtime: { + ...baseThread.runtime, + status: "idle", + activeRunId: null, + }, + }, + }), + ).toBeNull(); + }); + it("shows plan ready when a settled plan turn has a proposed plan ready for follow-up", () => { expect( resolveThreadStatusPill({ @@ -1128,6 +1189,38 @@ describe("resolveProjectStatusIndicator", () => { ]), ).toMatchObject({ label: "Plan Ready", dotClass: "bg-violet-500" }); }); + + it("ranks waiting below active work and above plan-ready", () => { + const waiting = { + label: "Waiting" as const, + colorClass: "text-sidebar-muted-foreground", + dotClass: "bg-sidebar-muted-foreground", + pulse: false, + }; + + expect( + resolveProjectStatusIndicator([ + waiting, + { + label: "Working", + colorClass: "text-sky-600", + dotClass: "bg-sky-500", + pulse: true, + }, + ]), + ).toMatchObject({ label: "Working" }); + expect( + resolveProjectStatusIndicator([ + { + label: "Plan Ready", + colorClass: "text-violet-600", + dotClass: "bg-violet-500", + pulse: false, + }, + waiting, + ]), + ).toMatchObject({ label: "Waiting" }); + }); }); describe("getVisibleThreadsForProject", () => { diff --git a/apps/web/src/components/Sidebar.logic.ts b/apps/web/src/components/Sidebar.logic.ts index a5499bd8c39a..fcd594e67e0a 100644 --- a/apps/web/src/components/Sidebar.logic.ts +++ b/apps/web/src/components/Sidebar.logic.ts @@ -152,6 +152,7 @@ export interface ThreadStatusPill { | "Completed" | "Pending Approval" | "Awaiting Input" + | "Waiting" | "Plan Ready"; colorClass: string; dotClass: string; @@ -163,6 +164,7 @@ const THREAD_STATUS_PRIORITY: Record = { "Awaiting Input": 4, Working: 3, Connecting: 3, + Waiting: 2.5, "Plan Ready": 2, Completed: 1, }; @@ -177,6 +179,7 @@ type ThreadStatusInput = Pick< | "runtime" > & { lastVisitedAt?: string | null | undefined; + pendingBackgroundTasks?: SidebarThreadSummary["pendingBackgroundTasks"] | undefined; }; export interface ThreadJumpHintVisibilityController { @@ -464,13 +467,16 @@ export function resolveThreadRowClassName(input: { } // ── Sidebar v2 status model ───────────────────────────────────────── -// Five visual states, three colors: color is reserved for "act now" +// Six visual states, three colors: color is reserved for "act now" // (approval), "in motion" (working), and "broken" (failed). Ready is the // unlabeled resting state — the agent stopped and is waiting on the user, -// whether it finished, asked a question, or proposed a plan. +// whether it finished, asked a question, or proposed a plan. Waiting +// (runtime status "idle") is the agent stopped with background tasks still +// open: not the user's turn yet, so it renders grey like working, not as a +// false Done. // Unread completion is tracked separately: it describes whether a ready // thread needs attention, not what the thread is currently doing. -export type SidebarV2Status = "approval" | "input" | "working" | "failed" | "ready"; +export type SidebarV2Status = "approval" | "input" | "working" | "waiting" | "failed" | "ready"; type SidebarV2StatusInput = Pick< SidebarThreadSummary, @@ -490,12 +496,54 @@ export function resolveSidebarV2Status(thread: SidebarV2StatusInput): SidebarV2S ) { return "working"; } + if (thread.runtime?.status === "idle") { + return "waiting"; + } if (thread.runtime?.status === "failed") { return "failed"; } return "ready"; } +export type SidebarV2TopStatusKind = + | "approval" + | "done" + | "failed" + | "input" + | "waiting" + | "woke" + | "working"; + +export function resolveSidebarV2TopStatus(input: { + readonly status: SidebarV2Status; + readonly isUnread: boolean; + readonly isWoke: boolean; +}): SidebarV2TopStatusKind | null { + if (input.status === "working") { + return "working"; + } + if (input.status === "waiting") { + return "waiting"; + } + if (input.status === "approval") { + return "approval"; + } + if (input.status === "input") { + return "input"; + } + if (input.status === "failed") { + return "failed"; + } + if (input.isWoke) { + return "woke"; + } + return input.isUnread ? "done" : null; +} + +export function shouldShowSidebarV2Duration(status: SidebarV2Status): boolean { + return status === "working"; +} + /** NaN-safe Date.parse for sort comparators: a malformed timestamp must not poison the whole ordering, so it sinks to the epoch instead. */ export function parseTimestampMs(isoDate: string): number { @@ -669,6 +717,15 @@ export function resolveThreadStatusPill(input: { }; } + if ((thread.pendingBackgroundTasks?.length ?? 0) > 0) { + return { + label: "Waiting", + colorClass: "text-sidebar-muted-foreground", + dotClass: "bg-sidebar-muted-foreground", + pulse: false, + }; + } + const hasPlanReadyPrompt = !thread.hasPendingUserInput && thread.interactionMode === "plan" && diff --git a/apps/web/src/components/SidebarV2.tsx b/apps/web/src/components/SidebarV2.tsx index 06015a7a0304..ed4ddf3dca43 100644 --- a/apps/web/src/components/SidebarV2.tsx +++ b/apps/web/src/components/SidebarV2.tsx @@ -118,9 +118,11 @@ import { resolveAdjacentThreadId, resolveSettledTimestamp, resolveSidebarV2Status, + resolveSidebarV2TopStatus, resolveThreadLastVisitedAt, resolveWorkingStartedAt, searchSidebarThreadsByTitle, + shouldShowSidebarV2Duration, shouldNavigateAfterProjectRemoval, sortSidebarV2ProjectGroups, sortSettledThreadsForSidebarV2, @@ -175,6 +177,44 @@ const PROJECT_GROUPING_MODE_LABELS: Record = repository_path: "Group by repository path", separate: "Keep separate", }; +const SIDEBAR_V2_TOP_STATUS = { + approval: { + label: "Approval", + icon: null, + className: "text-amber-700 dark:text-amber-300", + }, + done: { + label: "Done", + icon: "done", + className: "text-emerald-700 dark:text-emerald-300", + }, + failed: { + label: "Failed", + icon: null, + className: "text-red-700 dark:text-red-300", + }, + input: { + label: "Input", + icon: null, + className: "text-indigo-600 dark:text-indigo-300", + }, + waiting: { + label: "Waiting", + icon: null, + className: "text-muted-foreground", + }, + woke: { + label: "Woke", + icon: "woke", + className: "text-amber-700 dark:text-amber-300", + }, + working: { + label: "Working", + icon: "working", + className: + "animate-sidebar-working-text text-sky-600 motion-reduce:animate-none dark:text-sky-400", + }, +} as const; function compactSidebarTimeLabel(label: string): string { if (label === "just now") return "now"; @@ -491,45 +531,8 @@ const SidebarV2Row = memo(function SidebarV2Row(props: { // Status hues follow the system-wide convention set by sidebar v1 and the // mobile Live Activity/widgets (amber approval, indigo input, sky working) // so a thread reads the same color everywhere it surfaces. - const topStatus = - status === "working" - ? { - label: "Working", - icon: "working" as const, - className: - "animate-sidebar-working-text text-sky-600 motion-reduce:animate-none dark:text-sky-400", - } - : status === "approval" - ? { - label: "Approval", - icon: null, - className: "text-amber-700 dark:text-amber-300", - } - : status === "input" - ? { - label: "Input", - icon: null, - className: "text-indigo-600 dark:text-indigo-300", - } - : status === "failed" - ? { - label: "Failed", - icon: null, - className: "text-red-700 dark:text-red-300", - } - : isWoke - ? { - label: "Woke", - icon: "woke" as const, - className: "text-amber-700 dark:text-amber-300", - } - : isUnread - ? { - label: "Done", - icon: "done" as const, - className: "text-emerald-700 dark:text-emerald-300", - } - : null; + const topStatusKind = resolveSidebarV2TopStatus({ status, isUnread, isWoke }); + const topStatus = topStatusKind === null ? null : SIDEBAR_V2_TOP_STATUS[topStatusKind]; const gitCwd = thread.worktreePath ?? props.projectCwd; const gitStatus = useEnvironmentQuery( @@ -987,7 +990,7 @@ const SidebarV2Row = memo(function SidebarV2Row(props: { wrapper around the ticking duration would make screen readers announce every second. */} {topStatus.label} - {status === "working" ? ( + {shouldShowSidebarV2Duration(status) ? ( diff --git a/apps/web/src/components/chat/MessagesTimeline.logic.test.ts b/apps/web/src/components/chat/MessagesTimeline.logic.test.ts index ce3afe9e3c9c..daff99ac4c47 100644 --- a/apps/web/src/components/chat/MessagesTimeline.logic.test.ts +++ b/apps/web/src/components/chat/MessagesTimeline.logic.test.ts @@ -1432,3 +1432,66 @@ describe("computeStableMessagesTimelineRows", () => { expect(reordered.result).toEqual([initial.result[1], initial.result[0]]); }); }); + +describe("deriveMessagesTimelineRows waiting-background", () => { + it("renders the waiting-background row instead of Working for a parked waiting runtime", () => { + const rows = deriveMessagesTimelineRows({ + timelineEntries: [], + isWorking: false, + activeTurnStartedAt: null, + pendingBackgroundTasks: [{ taskId: "bg-1", description: "Run Codex review" }], + turnDiffSummaryByAssistantMessageId: new Map(), + revertTurnCountByUserMessageId: new Map(), + }); + + expect(rows).toEqual([ + { + kind: "waiting-background", + id: "waiting-background-row", + createdAt: null, + description: "Run Codex review", + taskCount: 1, + label: "Waiting on background task: Run Codex review", + }, + ]); + }); + + it("suppresses a stale waiting roster while an active running turn is Working", () => { + const rows = deriveMessagesTimelineRows({ + timelineEntries: [], + isWorking: true, + activeTurnStartedAt: "2026-01-01T00:00:00Z", + pendingBackgroundTasks: [{ taskId: "bg-1", description: "Run Codex review" }], + turnDiffSummaryByAssistantMessageId: new Map(), + revertTurnCountByUserMessageId: new Map(), + }); + + expect(rows.some((row) => row.kind === "waiting-background")).toBe(false); + expect(rows.some((row) => row.kind === "working")).toBe(true); + }); + + it("includes task count for multiple background tasks", () => { + const rows = deriveMessagesTimelineRows({ + timelineEntries: [], + isWorking: false, + activeTurnStartedAt: null, + pendingBackgroundTasks: [ + { taskId: "bg-1", description: "first" }, + { taskId: "bg-2", description: "second" }, + ], + turnDiffSummaryByAssistantMessageId: new Map(), + revertTurnCountByUserMessageId: new Map(), + }); + + expect(rows).toEqual([ + { + kind: "waiting-background", + id: "waiting-background-row", + createdAt: null, + description: "first", + taskCount: 2, + label: "Waiting on 2 background tasks: first, …", + }, + ]); + }); +}); diff --git a/apps/web/src/components/chat/MessagesTimeline.logic.ts b/apps/web/src/components/chat/MessagesTimeline.logic.ts index 3377cacfa511..304417bb2d02 100644 --- a/apps/web/src/components/chat/MessagesTimeline.logic.ts +++ b/apps/web/src/components/chat/MessagesTimeline.logic.ts @@ -13,6 +13,7 @@ import { type RunId, } from "@t3tools/contracts"; import type { ThreadRunSummary } from "@t3tools/client-runtime/state/shell"; +import { formatPendingBackgroundWorkLabel } from "@t3tools/shared/orchestrationV2PendingBackgroundWork"; import { resolveT3McpToolPresentation, type T3McpToolPresentation, @@ -191,7 +192,15 @@ export type MessagesTimelineRow = createdAt: string; proposedPlan: ProposedPlan; } - | { kind: "working"; id: string; createdAt: string | null }; + | { kind: "working"; id: string; createdAt: string | null } + | { + kind: "waiting-background"; + id: string; + createdAt: string | null; + description: string | null; + taskCount: number; + label: string; + }; export interface StableMessagesTimelineRowsState { byId: Map; @@ -489,6 +498,10 @@ export function deriveMessagesTimelineRows(input: { expandedAttemptIds?: ReadonlySet; isWorking: boolean; activeTurnStartedAt: string | null; + pendingBackgroundTasks?: ReadonlyArray<{ + readonly taskId: string; + readonly description?: string | undefined; + }> | null; turnDiffSummaryByAssistantMessageId: ReadonlyMap; revertTurnCountByUserMessageId: ReadonlyMap; }): MessagesTimelineRow[] { @@ -664,6 +677,20 @@ export function deriveMessagesTimelineRows(input: { id: "working-indicator-row", createdAt: input.activeTurnStartedAt, }); + } else if (input.pendingBackgroundTasks && input.pendingBackgroundTasks.length > 0) { + // Root run settled but finite provider background work remains. Show Waiting + // instead of looking fully idle until the roster drains. + const firstTask = input.pendingBackgroundTasks[0]; + nextRows.push({ + kind: "waiting-background", + id: "waiting-background-row", + createdAt: input.activeTurnStartedAt, + description: firstTask?.description ?? null, + taskCount: input.pendingBackgroundTasks.length, + label: + formatPendingBackgroundWorkLabel(input.pendingBackgroundTasks) ?? + "Waiting on a background task", + }); } return nextRows; @@ -697,6 +724,16 @@ function isRowUnchanged(a: MessagesTimelineRow, b: MessagesTimelineRow): boolean case "working": return a.createdAt === (b as typeof a).createdAt; + case "waiting-background": { + const bw = b as typeof a; + return ( + a.createdAt === bw.createdAt && + a.description === bw.description && + a.taskCount === bw.taskCount && + a.label === bw.label + ); + } + case "turn-fold": { const bf = b as typeof a; return a.createdAt === bf.createdAt && a.label === bf.label && a.expanded === bf.expanded; diff --git a/apps/web/src/components/chat/MessagesTimeline.tsx b/apps/web/src/components/chat/MessagesTimeline.tsx index 78c858be6d83..b06576921cfa 100644 --- a/apps/web/src/components/chat/MessagesTimeline.tsx +++ b/apps/web/src/components/chat/MessagesTimeline.tsx @@ -189,6 +189,10 @@ interface MessagesTimelineProps { isWorking: boolean; activeTurnInProgress: boolean; activeTurnStartedAt: string | null; + pendingBackgroundTasks?: ReadonlyArray<{ + readonly taskId: string; + readonly description?: string | undefined; + }> | null; listRef: React.RefObject; timelineEntries: ReadonlyArray; latestRun: TimelineLatestRun | null; @@ -238,6 +242,7 @@ export const MessagesTimeline = memo(function MessagesTimeline({ isWorking, activeTurnInProgress, activeTurnStartedAt, + pendingBackgroundTasks = null, listRef, timelineEntries, latestRun, @@ -336,6 +341,7 @@ export const MessagesTimeline = memo(function MessagesTimeline({ expandedAttemptIds, isWorking, activeTurnStartedAt, + pendingBackgroundTasks, turnDiffSummaryByAssistantMessageId, revertTurnCountByUserMessageId, }), @@ -346,6 +352,7 @@ export const MessagesTimeline = memo(function MessagesTimeline({ expandedAttemptIds, isWorking, activeTurnStartedAt, + pendingBackgroundTasks, turnDiffSummaryByAssistantMessageId, revertTurnCountByUserMessageId, ], @@ -944,6 +951,7 @@ const TimelineRowContent = memo(function TimelineRowContent({ row }: { row: Time {row.kind === "proposed-plan" ? : null} {row.kind === "event" ? : null} {row.kind === "working" ? : null} + {row.kind === "waiting-background" ? : null} ); }); @@ -1599,6 +1607,25 @@ function WorkingTimelineRow({ row }: { row: Extract; +}) { + return ( +
+
+ + + + + + {row.label} +
+
+ ); +} + // --------------------------------------------------------------------------- // Self-ticking labels — update their own text nodes so elapsed-time display // does not create a React commit every second while a response is streaming. diff --git a/packages/client-runtime/src/state/entities.test.ts b/packages/client-runtime/src/state/entities.test.ts index 4d84e85bac3a..7bc9b4ff966f 100644 --- a/packages/client-runtime/src/state/entities.test.ts +++ b/packages/client-runtime/src/state/entities.test.ts @@ -107,6 +107,147 @@ describe("V2 client presentation", () => { }); }); + it("parks presented runtime at idle when a settled shell has pending background tasks", () => { + const runId = RunId.make("run-completed"); + const shell = presentThreadShell(environmentId, { + ...v2ThreadShell, + latestRunId: runId, + activeRunId: null, + status: "completed", + pendingBackgroundTasks: [{ taskId: "bg-1", description: "sleep 20" }], + }); + + expect(shell.latestRun).toMatchObject({ runId, status: "completed" }); + expect(shell.runtime).toMatchObject({ + status: "idle", + activeRunId: null, + }); + expect(shell.pendingBackgroundTasks).toEqual([{ taskId: "bg-1", description: "sleep 20" }]); + }); + + it("keeps terminal runtime completed when there are no pending background tasks", () => { + const runId = RunId.make("run-completed"); + const shell = presentThreadShell(environmentId, { + ...v2ThreadShell, + latestRunId: runId, + activeRunId: null, + status: "completed", + pendingBackgroundTasks: [], + }); + + expect(shell.latestRun).toMatchObject({ runId, status: "completed" }); + expect(shell.runtime).toMatchObject({ + status: "completed", + activeRunId: null, + }); + }); + + it("keeps runtime running when there is no background roster", () => { + const runId = RunId.make("run-running"); + const shell = presentThreadShell(environmentId, { + ...v2ThreadShell, + latestRunId: runId, + activeRunId: runId, + status: "running", + pendingBackgroundTasks: [], + }); + + expect(shell.latestRun).toMatchObject({ runId, status: "running" }); + expect(shell.runtime).toMatchObject({ + status: "running", + activeRunId: runId, + }); + }); + + it("keeps an older active run visible over a newer cancelled run", () => { + const activeRunId = RunId.make("run-active"); + const cancelledRunId = RunId.make("run-cancelled"); + const shell = presentThreadShell(environmentId, { + ...v2ThreadShell, + latestRunId: cancelledRunId, + latestRunStartedAt: null, + latestRunCompletedAt: DateTime.makeUnsafe("2026-06-20T01:05:00.000Z"), + activeRunId, + activityRunStatus: "running", + status: "cancelled", + pendingBackgroundTasks: [], + }); + + expect(shell.latestRun).toMatchObject({ runId: cancelledRunId, status: "cancelled" }); + expect(shell.runtime).toMatchObject({ + status: "running", + activeRunId, + }); + }); + + it("keeps an older waiting run visible over a newer cancelled run", () => { + const shell = presentThreadShell(environmentId, { + ...v2ThreadShell, + latestRunId: RunId.make("run-cancelled"), + activeRunId: null, + activityRunStatus: "waiting", + status: "cancelled", + pendingBackgroundTasks: [], + }); + + expect(shell.runtime).toMatchObject({ status: "waiting", activeRunId: null }); + }); + + it("keeps a post-settlement background roster ahead of activity status", () => { + const activeRunId = RunId.make("run-active"); + const shell = presentThreadShell(environmentId, { + ...v2ThreadShell, + latestRunId: RunId.make("run-cancelled"), + activeRunId, + activityRunStatus: "running", + status: "cancelled", + pendingBackgroundTasks: [{ taskId: "bg-activity", description: "background work" }], + }); + + expect(shell.runtime).toMatchObject({ status: "idle", activeRunId }); + }); + + it("parks runtime idle over stale shell running when the roster is nonempty", () => { + const runId = RunId.make("run-stale-running"); + const shell = presentThreadShell(environmentId, { + ...v2ThreadShell, + latestRunId: runId, + activeRunId: runId, + // Stale: server already projected a post-settlement roster, but shell + // status still says running (packaged orchestrator-v2 bug). + status: "running", + pendingBackgroundTasks: [{ taskId: "bg-1", description: "sleep 20" }], + }); + + expect(shell.latestRun).toMatchObject({ runId, status: "running" }); + expect(shell.runtime).toMatchObject({ + status: "idle", + activeRunId: runId, + }); + expect(shell.pendingBackgroundTasks).toEqual([{ taskId: "bg-1", description: "sleep 20" }]); + }); + + it("parks runtime idle over stale checkpoint waiting when the roster is nonempty", () => { + const runId = RunId.make("run-stale-waiting"); + const shell = presentThreadShell(environmentId, { + ...v2ThreadShell, + latestRunId: runId, + activeRunId: runId, + // Stale: checkpoint-oriented waiting masks post-settlement background work. + status: "waiting", + pendingBackgroundTasks: [{ taskId: "bg-2", description: "background bash" }], + }); + + expect(shell.latestRun).toMatchObject({ runId, status: "waiting" }); + expect(shell.runtime).toMatchObject({ + status: "idle", + activeRunId: runId, + }); + expect(shell.pendingBackgroundTasks).toEqual([ + { taskId: "bg-2", description: "background bash" }, + ]); + }); + it("derives execution summaries without wrapping or copying the projection", () => { const runId = RunId.make("run-1"); const now = DateTime.makeUnsafe("2026-06-20T01:00:00.000Z"); @@ -163,6 +304,67 @@ describe("V2 client presentation", () => { }); }); + it("parks waiting runtime for a post-settlement roster without hiding active running work", () => { + const runId = RunId.make("run-background-presentation"); + const now = DateTime.makeUnsafe("2026-06-20T01:00:00.000Z"); + const run = { + id: runId, + threadId: v2Projection.thread.id, + ordinal: 1, + providerInstanceId: v2Projection.thread.providerInstanceId, + modelSelection: v2Projection.thread.modelSelection, + providerThreadId: null, + userMessageId: MessageId.make("message-background-presentation"), + rootNodeId: null, + activeAttemptId: null, + status: "waiting" as const, + requestedAt: now, + startedAt: now, + completedAt: null, + checkpointId: null, + contextHandoffId: null, + }; + const backgroundItem = { + id: TurnItemId.make("item-background-command"), + threadId: v2Projection.thread.id, + runId, + nodeId: null, + providerThreadId: null, + providerTurnId: null, + nativeItemRef: null, + parentItemId: null, + ordinal: 0, + status: "running" as const, + title: "Background command", + startedAt: now, + completedAt: null, + updatedAt: now, + type: "command_execution" as const, + input: "sleep 20", + }; + + expect( + deriveThreadRuntime({ + ...v2Projection, + runs: [run], + turnItems: [backgroundItem], + }), + ).toMatchObject({ + status: "idle", + activeRunId: null, + }); + expect( + deriveThreadRuntime({ + ...v2Projection, + runs: [{ ...run, status: "running" as const }], + turnItems: [backgroundItem], + }), + ).toMatchObject({ + status: "running", + activeRunId: runId, + }); + }); + it("joins pending request entities to their native turn-item display data", () => { const now = DateTime.makeUnsafe("2026-06-20T01:00:00.000Z"); const requestId = RuntimeRequestId.make("request-approval"); diff --git a/packages/client-runtime/src/state/models.ts b/packages/client-runtime/src/state/models.ts index 606081156887..9ac77cda10af 100644 --- a/packages/client-runtime/src/state/models.ts +++ b/packages/client-runtime/src/state/models.ts @@ -86,6 +86,9 @@ export interface EnvironmentThreadShell { readonly hasPendingApprovals: boolean; readonly hasPendingUserInput: boolean; readonly hasActionableProposedPlan: boolean; + readonly pendingBackgroundTasks: ReadonlyArray< + NonNullable[number] + >; readonly itemCount: number; readonly visibleItemCount: number; readonly createdAt: string; @@ -126,10 +129,17 @@ function terminalRunStatus(status: OrchestrationV2RunStatus): boolean { ); } +// Park runtime at idle when the post-settlement background roster is nonempty +// so #4415 waiting-presentation Waiting (session.idle) can consume CTM runtime. +// The server suppresses the roster while an interruptible activity run exists, +// so a remaining roster is stronger than checkpoint-oriented waiting. +// latestRun keeps the latest run's status for history presentation. function shellRuntime(thread: OrchestrationV2ThreadShell): ThreadRuntimeSummary | null { if (thread.latestRunId === null && thread.activeProviderThreadId === null) return null; + const hasPendingBackgroundTasks = (thread.pendingBackgroundTasks?.length ?? 0) > 0; + const status = hasPendingBackgroundTasks ? "idle" : (thread.activityRunStatus ?? thread.status); return { - status: thread.status, + status, activeRunId: thread.activeRunId, providerInstanceId: thread.providerInstanceId, providerName: null, @@ -189,6 +199,7 @@ export function presentThreadShell( thread.pendingRuntimeRequest.kind !== "auth_refresh", hasPendingUserInput: thread.pendingRuntimeRequest?.kind === "user_input", hasActionableProposedPlan: thread.hasActionableProposedPlan, + pendingBackgroundTasks: thread.pendingBackgroundTasks ?? [], itemCount: thread.itemCount, visibleItemCount: thread.visibleItemCount, createdAt: iso(thread.createdAt), diff --git a/packages/client-runtime/src/state/threadExecution.test.ts b/packages/client-runtime/src/state/threadExecution.test.ts index 707e7ad0804d..ff11855bc6af 100644 --- a/packages/client-runtime/src/state/threadExecution.test.ts +++ b/packages/client-runtime/src/state/threadExecution.test.ts @@ -85,4 +85,30 @@ describe("thread execution presentation", () => { }); expect(threadRuntimeHasInterruptibleRun(runtime)).toBe(false); }); + + it("does not expose a stale active run after the runtime parks at idle", () => { + const runtime = { + status: "idle" as const, + activeRunId: RunId.make("run-stale"), + providerInstanceId: v2Projection.thread.providerInstanceId, + providerName: null, + lastError: null, + updatedAt: DateTime.formatIso(now), + }; + + expect(threadRuntimeHasInterruptibleRun(runtime)).toBe(false); + }); + + it.each(["preparing", "starting"] as const)("keeps an active %s run interruptible", (status) => { + const runtime = { + status, + activeRunId: RunId.make(`run-${status}`), + providerInstanceId: v2Projection.thread.providerInstanceId, + providerName: null, + lastError: null, + updatedAt: DateTime.formatIso(now), + }; + + expect(threadRuntimeHasInterruptibleRun(runtime)).toBe(true); + }); }); diff --git a/packages/client-runtime/src/state/threadExecution.ts b/packages/client-runtime/src/state/threadExecution.ts index 68c846a9ed35..62ad3a0ea52d 100644 --- a/packages/client-runtime/src/state/threadExecution.ts +++ b/packages/client-runtime/src/state/threadExecution.ts @@ -1,7 +1,12 @@ import type { OrchestrationV2ThreadProjection } from "@t3tools/contracts"; +import { derivePendingBackgroundWork } from "@t3tools/shared/orchestrationV2PendingBackgroundWork"; import * as DateTime from "effect/DateTime"; -import type { ThreadRunSummary, ThreadRuntimeSummary } from "./models.ts"; +import { + threadRuntimeIsActive, + type ThreadRunSummary, + type ThreadRuntimeSummary, +} from "./models.ts"; const ACTIVITY_RUN_STATUSES = new Set(["preparing", "starting", "running", "waiting"]); const INTERRUPTIBLE_RUN_STATUSES = new Set(["preparing", "starting", "running"]); @@ -62,6 +67,7 @@ export function deriveThreadRuntime( projection: OrchestrationV2ThreadProjection, ): ThreadRuntimeSummary | null { const latestRun = deriveLatestThreadRun(projection); + const latestRunProjection = latestMatchingRun(projection, () => true); const activityRun = deriveThreadActivityRun(projection); const providerSession = projection.providerSessions.findLast( (session) => session.providerInstanceId === projection.thread.providerInstanceId, @@ -69,11 +75,16 @@ export function deriveThreadRuntime( if (latestRun === null && projection.thread.activeProviderThreadId === null) return null; const activeRunId = latestMatchingRun(projection, (run) => INTERRUPTIBLE_RUN_STATUSES.has(run.status))?.id ?? null; + const hasPendingBackgroundTasks = + derivePendingBackgroundWork({ + latestRun: latestRunProjection, + providerThreads: projection.providerThreads, + turnItems: projection.turnItems, + activeProviderThreadId: projection.thread.activeProviderThreadId, + runs: projection.runs, + }).length > 0; return { - // Queueing creates a newer run, but does not replace the provider work - // already in flight. Present the executing run's status until it reaches - // a terminal state so clients do not flash from "running" to "queued". - status: activityRun?.status ?? "idle", + status: hasPendingBackgroundTasks ? "idle" : (activityRun?.status ?? "idle"), activeRunId, providerInstanceId: projection.thread.providerInstanceId, providerName: providerSession?.driver ?? null, @@ -85,5 +96,9 @@ export function deriveThreadRuntime( export function threadRuntimeHasInterruptibleRun( runtime: ThreadRuntimeSummary | null | undefined, ): boolean { - return runtime?.activeRunId !== null && runtime?.activeRunId !== undefined; + return ( + threadRuntimeIsActive(runtime) && + runtime?.activeRunId !== null && + runtime?.activeRunId !== undefined + ); } diff --git a/packages/client-runtime/src/state/threadRelationships.test.ts b/packages/client-runtime/src/state/threadRelationships.test.ts index c267dd30ce56..e24c3c572d5b 100644 --- a/packages/client-runtime/src/state/threadRelationships.test.ts +++ b/packages/client-runtime/src/state/threadRelationships.test.ts @@ -12,6 +12,32 @@ import { } from "./threadRelationships.ts"; describe("thread relationships", () => { + it("keeps an older activity run visible over a newer cancelled run", () => { + const parent = ThreadId.make("thread-parent"); + const child = ThreadId.make("thread-child"); + const graph = deriveThreadRelationshipGraph({ + threads: [ + { + id: child, + title: "Active child", + activityRunStatus: "running", + status: "cancelled", + forkedFrom: { type: "run", threadId: parent, runId: "run-parent" }, + lineage: { + rootThreadId: parent, + parentThreadId: parent, + relationshipToParent: "subagent", + }, + }, + ] as never, + projection: null, + }); + + expect(graph.edges).toEqual([ + expect.objectContaining({ targetThreadId: child, status: "running" }), + ]); + }); + it("keeps missing parents and cycles navigable without recursive traversal", () => { const root = ThreadId.make("thread-root"); const child = ThreadId.make("thread-child"); diff --git a/packages/client-runtime/src/state/threadRelationships.ts b/packages/client-runtime/src/state/threadRelationships.ts index cc8578d7b16a..c7ed61d709f4 100644 --- a/packages/client-runtime/src/state/threadRelationships.ts +++ b/packages/client-runtime/src/state/threadRelationships.ts @@ -84,7 +84,7 @@ export function deriveThreadRelationshipGraph(input: { sourceThreadId: parentThreadId, targetThreadId: thread.id, kind: thread.lineage.relationshipToParent === "subagent" ? "subagent" : "fork", - status: thread.status, + status: thread.activityRunStatus ?? thread.status, }); } diff --git a/packages/contracts/src/orchestrationV2.test.ts b/packages/contracts/src/orchestrationV2.test.ts index 020461bf0fa0..21ec7e75e214 100644 --- a/packages/contracts/src/orchestrationV2.test.ts +++ b/packages/contracts/src/orchestrationV2.test.ts @@ -24,9 +24,12 @@ import { OrchestrationV2CheckpointScope, OrchestrationV2Command, OrchestrationV2DomainEvent, + OrchestrationV2ProviderThread, + OrchestrationV2ProviderThreadJson, OrchestrationV2ShellSnapshot, OrchestrationV2Subagent, OrchestrationV2ThreadProjection, + OrchestrationV2ThreadShell, OrchestrationV2TurnItem, } from "./orchestrationV2.ts"; @@ -51,6 +54,11 @@ const decodeOrchestrationV2Subagent = Schema.decodeUnknownSync(OrchestrationV2Su const decodeOrchestrationV2ThreadProjection = Schema.decodeUnknownSync( OrchestrationV2ThreadProjection, ); +const decodeOrchestrationV2ProviderThreadJson = Schema.decodeUnknownSync( + OrchestrationV2ProviderThreadJson, +); +const decodeOrchestrationV2ProviderThread = Schema.decodeUnknownSync(OrchestrationV2ProviderThread); +const decodeOrchestrationV2ThreadShell = Schema.decodeUnknownSync(OrchestrationV2ThreadShell); describe("orchestration V2 contracts", () => { it("lets legacy snapshot decoders ignore enrichment metadata", () => { @@ -673,4 +681,92 @@ describe("orchestration V2 contracts", () => { expect(CheckpointRef.make("git-ref-1")).toBe("git-ref-1"); expect(ContextTransferId.make("context-transfer-1")).toBe("context-transfer-1"); }); + + it("decodes historical provider-thread JSON without pendingBackgroundTasks as empty roster", () => { + const providerThread = decodeOrchestrationV2ProviderThreadJson({ + id: "provider-thread-1", + driver: "claude", + providerInstanceId: "claudeAgent", + providerSessionId: "provider-session-1", + appThreadId: "thread-1", + ownerNodeId: null, + nativeThreadRef: { + driver: "claude", + nativeId: "native-session-1", + strength: "strong", + }, + nativeConversationHeadRef: null, + status: "idle", + firstRunOrdinal: 1, + lastRunOrdinal: 1, + handoffIds: [], + forkedFrom: null, + createdAt: "2026-04-20T00:00:00.000Z", + updatedAt: "2026-04-20T00:00:00.000Z", + }); + + expect(providerThread.pendingBackgroundTasks).toEqual([]); + + const runtimeThread = decodeOrchestrationV2ProviderThread({ + id: "provider-thread-2", + driver: "claude", + providerInstanceId: "claudeAgent", + providerSessionId: null, + appThreadId: "thread-2", + ownerNodeId: null, + nativeThreadRef: null, + nativeConversationHeadRef: null, + status: "idle", + firstRunOrdinal: null, + lastRunOrdinal: null, + handoffIds: [], + forkedFrom: null, + createdAt: now, + updatedAt: now, + }); + expect(runtimeThread.pendingBackgroundTasks).toEqual([]); + }); + + it("decodes historical thread shell JSON without pendingBackgroundTasks as empty roster", () => { + const shell = decodeOrchestrationV2ThreadShell({ + createdBy: "user", + creationSource: "web", + id: "thread-1", + projectId: "project-1", + title: "Thread", + providerInstanceId: "claudeAgent", + modelSelection: { + instanceId: ProviderInstanceId.make("claudeAgent"), + model: "claude-sonnet", + }, + runtimeMode: "full-access", + interactionMode: "default", + branch: null, + worktreePath: null, + lineage: { + parentThreadId: null, + relationshipToParent: null, + rootThreadId: "thread-1", + }, + forkedFrom: null, + activeProviderThreadId: "provider-thread-1", + latestRunId: "run-1", + activeRunId: null, + status: "completed", + pendingRuntimeRequest: null, + latestVisibleMessage: null, + latestUserMessageAt: null, + hasActionableProposedPlan: false, + itemCount: 0, + visibleItemCount: 0, + createdAt: now, + updatedAt: now, + archivedAt: null, + settledOverride: null, + settledAt: null, + deletedAt: null, + }); + + expect(shell.pendingBackgroundTasks).toEqual([]); + }); }); diff --git a/packages/contracts/src/orchestrationV2.ts b/packages/contracts/src/orchestrationV2.ts index dba88aee2df6..be052e0f26eb 100644 --- a/packages/contracts/src/orchestrationV2.ts +++ b/packages/contracts/src/orchestrationV2.ts @@ -557,6 +557,18 @@ export const OrchestrationV2ProviderSessionDetached = Schema.Struct({ export type OrchestrationV2ProviderSessionDetached = typeof OrchestrationV2ProviderSessionDetached.Type; +/** + * Provider-owned background work that can outlive the root turn (for example a + * Claude background Bash task). Associated with the provider thread so shared + * runtimes cannot make an unrelated app thread look busy. + */ +export const OrchestrationV2PendingBackgroundTask = Schema.Struct({ + taskId: TrimmedNonEmptyString, + description: Schema.optional(TrimmedNonEmptyString), + taskType: Schema.optional(TrimmedNonEmptyString), +}); +export type OrchestrationV2PendingBackgroundTask = typeof OrchestrationV2PendingBackgroundTask.Type; + export const OrchestrationV2ProviderThread = Schema.Struct({ id: ProviderThreadId, driver: ProviderDriverKind, @@ -577,6 +589,10 @@ export const OrchestrationV2ProviderThread = Schema.Struct({ checkpointId: Schema.optional(CheckpointId), }), ), + // Optional Type so adapters can omit empty rosters; historical JSON decodes to []. + pendingBackgroundTasks: Schema.optional(Schema.Array(OrchestrationV2PendingBackgroundTask)).pipe( + Schema.withDecodingDefault(Effect.succeed([])), + ), createdAt: Schema.DateTimeUtc, updatedAt: Schema.DateTimeUtc, }); @@ -1260,12 +1276,20 @@ export const OrchestrationV2ThreadShell = Schema.Struct({ latestRunStartedAt: Schema.optional(Schema.NullOr(Schema.DateTimeUtc)), latestRunCompletedAt: Schema.optional(Schema.NullOr(Schema.DateTimeUtc)), activeRunId: Schema.NullOr(RunId), + activityRunStatus: Schema.optional( + Schema.NullOr(Schema.Literals(["preparing", "starting", "running", "waiting"])), + ), status: OrchestrationV2ShellThreadStatus, lastError: Schema.optional(Schema.NullOr(Schema.String)), pendingRuntimeRequest: Schema.NullOr(OrchestrationV2PendingRuntimeRequestSummary), latestVisibleMessage: Schema.NullOr(OrchestrationV2LatestVisibleMessageSummary), latestUserMessageAt: Schema.NullOr(Schema.DateTimeUtc), hasActionableProposedPlan: Schema.Boolean, + // Normalized post-settlement background work for sidebar Waiting pills. + // Empty when the latest root run is still active or no pending work remains. + pendingBackgroundTasks: Schema.optional(Schema.Array(OrchestrationV2PendingBackgroundTask)).pipe( + Schema.withDecodingDefault(Effect.succeed([])), + ), itemCount: NonNegativeInt, visibleItemCount: NonNegativeInt, createdAt: Schema.DateTimeUtc, diff --git a/packages/effect-codex-app-server/src/replay.ts b/packages/effect-codex-app-server/src/replay.ts index ea37c1af5b47..dd03f6fc6cd7 100644 --- a/packages/effect-codex-app-server/src/replay.ts +++ b/packages/effect-codex-app-server/src/replay.ts @@ -144,10 +144,12 @@ export interface CodexAppServerReplayState { export interface CodexAppServerReplayDriver { readonly transcript: CodexAppServerReplayTranscript; readonly state: Ref.Ref; + readonly beforeEmitInbound?: ( + entry: Extract, + ) => Effect.Effect; } const encoder = new TextEncoder(); -const decoder = new TextDecoder(); function stableStringify(value: unknown): string { if (Array.isArray(value)) { @@ -334,10 +336,14 @@ export function layerReplay( } export const makeReplayDriver = Effect.fn("effect-codex-app-server/replay.makeReplayDriver")( - function* (transcript: CodexAppServerReplayTranscript) { + function* ( + transcript: CodexAppServerReplayTranscript, + options: Pick = {}, + ) { return { transcript, state: yield* Ref.make({ cursor: 0, failure: null }), + ...options, } satisfies CodexAppServerReplayDriver; }, ); @@ -368,6 +374,7 @@ const makeReplayClientWithState = Effect.fn( const input = yield* Queue.unbounded>(); const state = driver.state; const outboundRemainder = yield* Ref.make(""); + const decoder = new TextDecoder(); const failReplay = (error: CodexAppServerReplayError) => Ref.update(state, (current) => ({ @@ -388,6 +395,9 @@ const makeReplayClientWithState = Effect.fn( } if (entry.type === "emit_inbound") { + if (driver.beforeEmitInbound !== undefined) { + yield* driver.beforeEmitInbound(entry); + } if (entry.afterMs !== undefined && entry.afterMs > 0) { yield* Effect.sleep(Duration.millis(entry.afterMs)); } diff --git a/packages/shared/package.json b/packages/shared/package.json index 1df49ab57490..74ad011874ba 100644 --- a/packages/shared/package.json +++ b/packages/shared/package.json @@ -111,6 +111,10 @@ "types": "./src/orchestrationV2Timeline.ts", "import": "./src/orchestrationV2Timeline.ts" }, + "./orchestrationV2PendingBackgroundWork": { + "types": "./src/orchestrationV2PendingBackgroundWork.ts", + "import": "./src/orchestrationV2PendingBackgroundWork.ts" + }, "./remote": { "types": "./src/remote.ts", "import": "./src/remote.ts" diff --git a/packages/shared/src/agentAwareness.test.ts b/packages/shared/src/agentAwareness.test.ts index 8578cf493910..05f35fda7146 100644 --- a/packages/shared/src/agentAwareness.test.ts +++ b/packages/shared/src/agentAwareness.test.ts @@ -20,7 +20,9 @@ const project = { describe("projectThreadAwarenessV2", () => { const updatedAt = DateTime.makeUnsafe(NOW); const v2Thread = ( - overrides: Partial> = {}, + overrides: Partial< + Pick + > = {}, ) => ({ id: "thread-2" as ThreadId, title: "Integrate orchestration", @@ -41,6 +43,16 @@ describe("projectThreadAwarenessV2", () => { ).toMatchObject({ phase: "running", headline: "Agent is working" }); }); + it("keeps an older activity run visible over a newer cancelled run", () => { + expect( + projectThreadAwarenessV2({ + environmentId: "env-1" as EnvironmentId, + project, + thread: v2Thread({ status: "cancelled", activityRunStatus: "running" }), + }), + ).toMatchObject({ phase: "running", headline: "Agent is working" }); + }); + it("prioritizes V2 user-input requests", () => { expect( projectThreadAwarenessV2({ diff --git a/packages/shared/src/agentAwareness.ts b/packages/shared/src/agentAwareness.ts index 38143783d874..386f6c6e9cb0 100644 --- a/packages/shared/src/agentAwareness.ts +++ b/packages/shared/src/agentAwareness.ts @@ -40,7 +40,13 @@ export interface ProjectThreadAwarenessV2Input { readonly project: Pick; readonly thread: Pick< OrchestrationV2ThreadShell, - "id" | "title" | "modelSelection" | "status" | "pendingRuntimeRequest" | "updatedAt" + | "activityRunStatus" + | "id" + | "modelSelection" + | "pendingRuntimeRequest" + | "status" + | "title" + | "updatedAt" >; } @@ -85,7 +91,7 @@ function resolveThreadAwarenessPhaseV2( ) { return "waiting_for_approval"; } - switch (thread.status) { + switch (thread.activityRunStatus ?? thread.status) { case "preparing": case "starting": return "starting"; diff --git a/packages/shared/src/orchestrationV2PendingBackgroundWork.test.ts b/packages/shared/src/orchestrationV2PendingBackgroundWork.test.ts new file mode 100644 index 000000000000..1c9970bb8cea --- /dev/null +++ b/packages/shared/src/orchestrationV2PendingBackgroundWork.test.ts @@ -0,0 +1,432 @@ +import { describe, expect, it } from "vite-plus/test"; +import { + derivePendingBackgroundWork, + formatPendingBackgroundWorkLabel, +} from "./orchestrationV2PendingBackgroundWork.ts"; + +describe("derivePendingBackgroundWork", () => { + it("returns empty while the latest run is not settled", () => { + const tasks = derivePendingBackgroundWork({ + latestRun: { id: "run-1" as never, ordinal: 1, status: "running" }, + providerThreads: [ + { + id: "pt-1" as never, + pendingBackgroundTasks: [{ taskId: "bg-1", description: "sleep 20" }], + }, + ], + turnItems: [ + { + id: "item-1" as never, + type: "command_execution", + status: "running", + title: "npm test", + nativeItemRef: null, + input: "npm test", + }, + ], + }); + expect(tasks).toEqual([]); + }); + + it("returns pending work when the latest run is waiting (post-success, pre-checkpoint)", () => { + const tasks = derivePendingBackgroundWork({ + latestRun: { id: "run-1" as never, ordinal: 1, status: "waiting" }, + providerThreads: [{ id: "pt-1" as never }], + turnItems: [ + { + id: "item-1" as never, + type: "command_execution", + status: "running", + title: "npm test", + nativeItemRef: { nativeId: "cmd-1" }, + input: "npm test", + }, + ], + }); + expect(tasks).toEqual([ + { taskId: "cmd-1", description: "npm test", taskType: "command_execution" }, + ]); + }); + + it("still returns empty when the latest run is running even with background items", () => { + const tasks = derivePendingBackgroundWork({ + latestRun: { id: "run-1" as never, ordinal: 1, status: "running" }, + providerThreads: [ + { + id: "pt-1" as never, + pendingBackgroundTasks: [{ taskId: "bg-1", description: "sleep 20" }], + }, + ], + turnItems: [ + { + id: "item-1" as never, + type: "subagent", + status: "running", + title: "review", + nativeItemRef: { nativeId: "sub-1" }, + prompt: "review", + }, + ], + }); + expect(tasks).toEqual([]); + }); + + it("excludes rolled_back items when the latest run is waiting", () => { + const tasks = derivePendingBackgroundWork({ + latestRun: { id: "run-2" as never, ordinal: 2, status: "waiting" }, + providerThreads: [{ id: "pt-1" as never }], + runs: [ + { id: "run-1" as never, ordinal: 1, status: "rolled_back" }, + { id: "run-2" as never, ordinal: 2, status: "waiting" }, + ], + turnItems: [ + { + id: "item-old" as never, + type: "command_execution", + status: "running", + title: "from rolled-back run", + runId: "run-1", + nativeItemRef: { nativeId: "cmd-old" }, + input: "sleep 99", + }, + { + id: "item-new" as never, + type: "command_execution", + status: "running", + title: "still pending", + runId: "run-2", + nativeItemRef: { nativeId: "cmd-new" }, + input: "npm test", + }, + ], + }); + expect(tasks).toEqual([ + { taskId: "cmd-new", description: "still pending", taskType: "command_execution" }, + ]); + }); + + it("returns the provider-thread roster after settlement", () => { + const tasks = derivePendingBackgroundWork({ + latestRun: { id: "run-1" as never, ordinal: 1, status: "completed" }, + providerThreads: [ + { + id: "pt-1" as never, + pendingBackgroundTasks: [ + { taskId: "bg-1", description: "Run Codex review", taskType: "local_bash" }, + ], + }, + ], + turnItems: [], + activeProviderThreadId: "pt-1", + }); + expect(tasks).toEqual([ + { taskId: "bg-1", description: "Run Codex review", taskType: "local_bash" }, + ]); + }); + + it("includes nonterminal turn items and excludes completed ones", () => { + const tasks = derivePendingBackgroundWork({ + latestRun: { id: "run-1" as never, ordinal: 1, status: "completed" }, + providerThreads: [{ id: "pt-1" as never }], + turnItems: [ + { + id: "item-1" as never, + type: "command_execution", + status: "running", + title: "npm test", + nativeItemRef: { nativeId: "cmd-1" }, + input: "npm test", + }, + { + id: "item-2" as never, + type: "command_execution", + status: "completed", + title: "done", + nativeItemRef: { nativeId: "cmd-2" }, + input: "echo done", + }, + ], + }); + expect(tasks).toEqual([ + { taskId: "cmd-1", description: "npm test", taskType: "command_execution" }, + ]); + }); + + it("trims normalized background-work descriptions", () => { + const base = { + latestRun: { id: "run-1" as never, ordinal: 1, status: "completed" as const }, + providerThreads: [ + { + id: "pt-1" as never, + pendingBackgroundTasks: [ + { + taskId: "native-task", + description: " native background work ", + }, + ], + }, + ], + }; + expect( + derivePendingBackgroundWork({ + ...base, + turnItems: [ + { + id: "item-command" as never, + type: "command_execution", + status: "running", + title: " npm test ", + nativeItemRef: null, + }, + { + id: "item-tool" as never, + type: "dynamic_tool", + status: "running", + title: null, + nativeItemRef: null, + toolName: " browser.search ", + } as never, + ], + }), + ).toEqual([ + { + taskId: "native-task", + description: "native background work", + }, + { + taskId: "item-command", + description: "npm test", + taskType: "command_execution", + }, + { + taskId: "item-tool", + description: "browser.search", + taskType: "dynamic_tool", + }, + ]); + }); + + it("dedupes roster entries against turn items by native task id", () => { + const tasks = derivePendingBackgroundWork({ + latestRun: { id: "run-1" as never, ordinal: 1, status: "completed" }, + providerThreads: [ + { + id: "pt-1" as never, + pendingBackgroundTasks: [{ taskId: "task-9", description: "Agent review" }], + }, + ], + turnItems: [ + { + id: "item-sub" as never, + type: "subagent", + status: "running", + title: "Agent review", + nativeItemRef: { nativeId: "task-9" }, + prompt: "review the plan", + }, + ], + }); + expect(tasks).toEqual([{ taskId: "task-9", description: "Agent review" }]); + }); + + it("excludes Grok persistent monitors", () => { + const tasks = derivePendingBackgroundWork({ + latestRun: { id: "run-1" as never, ordinal: 1, status: "completed" }, + providerThreads: [{ id: "pt-1" as never }], + turnItems: [ + { + id: "item-1" as never, + type: "dynamic_tool", + status: "running", + title: "monitor logs", + nativeItemRef: { nativeId: "mon-1" }, + input: { persistent: true, command: "tail -f" }, + }, + { + id: "item-2" as never, + type: "dynamic_tool", + status: "running", + title: "finite monitor", + nativeItemRef: { nativeId: "mon-2" }, + input: { persistent: false, command: "sleep 5" }, + }, + ], + }); + expect(tasks).toEqual([ + { taskId: "mon-2", description: "finite monitor", taskType: "dynamic_tool" }, + ]); + }); + + it("returns multiple tasks with stable ordering from insertion", () => { + const tasks = derivePendingBackgroundWork({ + latestRun: { id: "run-1" as never, ordinal: 1, status: "completed" }, + providerThreads: [ + { + id: "pt-1" as never, + pendingBackgroundTasks: [ + { taskId: "bg-1", description: "first" }, + { taskId: "bg-2", description: "second" }, + ], + }, + ], + turnItems: [ + { + id: "item-1" as never, + type: "command_execution", + status: "running", + title: "third", + nativeItemRef: { nativeId: "cmd-3" }, + input: "third", + }, + ], + }); + expect(tasks.map((task) => task.taskId)).toEqual(["bg-1", "bg-2", "cmd-3"]); + }); + + it("excludes turn items owned by a rolled_back run", () => { + const tasks = derivePendingBackgroundWork({ + latestRun: { id: "run-1" as never, ordinal: 1, status: "rolled_back" }, + providerThreads: [{ id: "pt-1" as never }], + runs: [{ id: "run-1" as never, ordinal: 1, status: "rolled_back" }], + turnItems: [ + { + id: "item-1" as never, + type: "command_execution", + status: "running", + title: "abandoned", + runId: "run-1", + nativeItemRef: { nativeId: "cmd-1" }, + input: "sleep 99", + }, + ], + }); + expect(tasks).toEqual([]); + }); + + it("returns empty when the latest run is rolled_back even with a nonempty roster", () => { + const tasks = derivePendingBackgroundWork({ + latestRun: { id: "run-1" as never, ordinal: 1, status: "rolled_back" }, + providerThreads: [ + { + id: "pt-1" as never, + pendingBackgroundTasks: [{ taskId: "bg-1", description: "sleep 20" }], + }, + ], + turnItems: [ + { + id: "item-1" as never, + type: "command_execution", + status: "running", + title: "abandoned", + runId: "run-1", + nativeItemRef: { nativeId: "cmd-1" }, + input: "sleep 99", + }, + ], + }); + expect(tasks).toEqual([]); + }); + + it("excludes an older rolled_back nonterminal item when the latest run is completed", () => { + const tasks = derivePendingBackgroundWork({ + latestRun: { id: "run-2" as never, ordinal: 2, status: "completed" }, + providerThreads: [{ id: "pt-1" as never }], + runs: [ + { id: "run-1" as never, ordinal: 1, status: "rolled_back" }, + { id: "run-2" as never, ordinal: 2, status: "completed" }, + ], + turnItems: [ + { + id: "item-old" as never, + type: "command_execution", + status: "running", + title: "from rolled-back run", + runId: "run-1", + nativeItemRef: { nativeId: "cmd-old" }, + input: "sleep 99", + }, + { + id: "item-new" as never, + type: "command_execution", + status: "running", + title: "still pending", + runId: "run-2", + nativeItemRef: { nativeId: "cmd-new" }, + input: "npm test", + }, + ], + }); + expect(tasks).toEqual([ + { taskId: "cmd-new", description: "still pending", taskType: "command_execution" }, + ]); + }); + + it("does not reclassify an older active run as background work", () => { + const tasks = derivePendingBackgroundWork({ + latestRun: { id: "run-2" as never, ordinal: 2, status: "cancelled" }, + providerThreads: [ + { + id: "pt-1" as never, + pendingBackgroundTasks: [{ taskId: "provider-task" }], + }, + ], + runs: [ + { id: "run-1" as never, ordinal: 1, status: "running" }, + { id: "run-2" as never, ordinal: 2, status: "cancelled" }, + ], + turnItems: [ + { + id: "item-active" as never, + type: "command_execution", + status: "running", + title: "foreground work", + runId: "run-1", + nativeItemRef: { nativeId: "cmd-active" }, + input: "vp check", + }, + ], + }); + + expect(tasks).toEqual([]); + }); + + it("includes items with a null run id even when runs list has rolled_back rows", () => { + const tasks = derivePendingBackgroundWork({ + latestRun: { id: "run-1" as never, ordinal: 1, status: "completed" }, + providerThreads: [{ id: "pt-1" as never }], + runs: [{ id: "run-1" as never, ordinal: 1, status: "rolled_back" }], + turnItems: [ + { + id: "item-null-run" as never, + type: "command_execution", + status: "running", + title: "orphan item", + runId: null, + nativeItemRef: { nativeId: "cmd-null" }, + input: "echo orphan", + }, + ], + }); + expect(tasks).toEqual([ + { taskId: "cmd-null", description: "orphan item", taskType: "command_execution" }, + ]); + }); +}); + +describe("formatPendingBackgroundWorkLabel", () => { + it("formats single and multi-task labels", () => { + expect(formatPendingBackgroundWorkLabel([])).toBeNull(); + expect(formatPendingBackgroundWorkLabel([{ taskId: "a" }])).toBe( + "Waiting on a background task", + ); + expect( + formatPendingBackgroundWorkLabel([{ taskId: "a", description: "Run Codex review" }]), + ).toBe("Waiting on background task: Run Codex review"); + expect( + formatPendingBackgroundWorkLabel([ + { taskId: "a", description: "first" }, + { taskId: "b", description: "second" }, + ]), + ).toBe("Waiting on 2 background tasks: first, …"); + }); +}); diff --git a/packages/shared/src/orchestrationV2PendingBackgroundWork.ts b/packages/shared/src/orchestrationV2PendingBackgroundWork.ts new file mode 100644 index 000000000000..779fc8857ec6 --- /dev/null +++ b/packages/shared/src/orchestrationV2PendingBackgroundWork.ts @@ -0,0 +1,238 @@ +import type { + OrchestrationV2PendingBackgroundTask, + OrchestrationV2ProviderThread, + OrchestrationV2Run, + OrchestrationV2TurnItem, +} from "@t3tools/contracts"; + +const BACKGROUND_TURN_ITEM_TYPES = new Set([ + "command_execution", + "dynamic_tool", + "subagent", +]); + +const TERMINAL_TURN_ITEM_STATUSES = new Set([ + "completed", + "interrupted", + "failed", + "cancelled", +]); + +/** + * True terminal run statuses (includes `rolled_back`). Retained as the + * canonical terminal set for this module; do not widen or reuse it as the + * background-wait gate — that gate intentionally excludes `rolled_back`. + */ +const TERMINAL_RUN_STATUSES = new Set([ + "cancelled", + "completed", + "failed", + "interrupted", + "rolled_back", +]); + +/** + * Run statuses that allow a pending-background roster to surface. + * Includes `waiting`: a successful turn persists as waiting until checkpoint + * capture flips it to completed, and waiting is only set from completed. + * Excludes `rolled_back`: the provider-thread roster is not run-tied, so a + * rolled-back latest run must fail the gate entirely (not only item filter). + * Built explicitly rather than spreading or deleting from TERMINAL_RUN_STATUSES. + */ +const SETTLED_FOR_BACKGROUND_WAIT_RUN_STATUSES = new Set([ + "cancelled", + "completed", + "failed", + "interrupted", + "waiting", +]); + +// Keep TERMINAL_RUN_STATUSES referenced so the true-terminal set stays defined +// next to the background-wait subset (rolled_back is terminal, not wait-settled). +void TERMINAL_RUN_STATUSES; + +export type PendingBackgroundWorkTask = OrchestrationV2PendingBackgroundTask; + +type PendingBackgroundWorkRun = Pick; + +type PendingBackgroundWorkProviderThread = Pick< + OrchestrationV2ProviderThread, + "id" | "pendingBackgroundTasks" +>; + +type PendingBackgroundWorkTurnItem = { + readonly id: OrchestrationV2TurnItem["id"] | string; + readonly type: OrchestrationV2TurnItem["type"]; + readonly status: OrchestrationV2TurnItem["status"]; + readonly title: string | null; + /** When present and the run is rolled_back, the item is abandoned, not pending. */ + readonly runId?: OrchestrationV2Run["id"] | string | null; + readonly nativeItemRef?: { + readonly nativeId: string | null; + } | null; + readonly input?: unknown; + readonly prompt?: string | undefined; +}; + +function isTerminalTurnItemStatus(status: OrchestrationV2TurnItem["status"]): boolean { + return TERMINAL_TURN_ITEM_STATUSES.has(status); +} + +function isLatestRunSettledForBackgroundWait( + latestRun: PendingBackgroundWorkRun | null | undefined, +): boolean { + if (latestRun === undefined || latestRun === null) { + return false; + } + return SETTLED_FOR_BACKGROUND_WAIT_RUN_STATUSES.has(latestRun.status); +} + +function isPersistentDynamicToolInput(input: unknown): boolean { + if (input === null || typeof input !== "object" || Array.isArray(input)) { + return false; + } + return Reflect.get(input, "persistent") === true; +} + +function descriptionFromTurnItem(item: PendingBackgroundWorkTurnItem): string | undefined { + if (typeof item.title === "string" && item.title.trim().length > 0) { + return item.title.trim(); + } + if (item.type === "command_execution" && typeof item.input === "string") { + const command = item.input.trim(); + return command.length > 0 ? command : undefined; + } + if (item.type === "dynamic_tool") { + const toolName = Reflect.get(item, "toolName"); + if (typeof toolName === "string" && toolName.trim().length > 0) { + return toolName.trim(); + } + } + if (item.type === "subagent" && typeof item.prompt === "string") { + const prompt = item.prompt.trim(); + return prompt.length > 0 ? prompt : undefined; + } + return undefined; +} + +function nativeTaskIdFromTurnItem(item: PendingBackgroundWorkTurnItem): string { + const nativeId = item.nativeItemRef?.nativeId; + if (typeof nativeId === "string" && nativeId.length > 0) { + return nativeId; + } + return String(item.id); +} + +/** + * Derive one normalized pending-background-work list for post-settlement UI. + * + * Sources: + * - Provider-thread roster (Claude SDK background tasks) + * - Nonterminal command_execution / dynamic_tool / subagent turn items + * + * Gated on latest root run settlement. Dedupes by native task ID. Excludes + * the roster while any interruptible foreground run remains active. Excludes + * Grok persistent monitors (`dynamic_tool` input with `persistent: true`). + * Excludes turn items whose run resolves to `rolled_back` (abandoned work); + * items with a null or absent run id stay eligible (matches SQL shell path). + * Does not consult subagent entities (those double-count turn items). + */ +export function derivePendingBackgroundWork(input: { + readonly latestRun: PendingBackgroundWorkRun | null | undefined; + readonly providerThreads: ReadonlyArray; + readonly turnItems: ReadonlyArray; + readonly activeProviderThreadId?: string | null; + readonly hasActiveRun?: boolean; + /** + * Run rows used to exclude items owned by rolled_back runs. Optional for + * callers that already filtered (SQL shell path); in-memory callers should + * pass projection runs so policy cannot drift. + */ + readonly runs?: ReadonlyArray; +}): ReadonlyArray { + const hasActiveRun = + input.hasActiveRun ?? + input.runs?.some( + (run) => run.status === "preparing" || run.status === "starting" || run.status === "running", + ) ?? + false; + if (hasActiveRun) { + return []; + } + if (!isLatestRunSettledForBackgroundWait(input.latestRun)) { + return []; + } + + const byTaskId = new Map(); + const rolledBackRunIds = new Set( + (input.runs ?? []).filter((run) => run.status === "rolled_back").map((run) => String(run.id)), + ); + + const providerThreads = + input.activeProviderThreadId === undefined || input.activeProviderThreadId === null + ? input.providerThreads + : input.providerThreads.filter((thread) => thread.id === input.activeProviderThreadId); + + for (const providerThread of providerThreads) { + for (const task of providerThread.pendingBackgroundTasks ?? []) { + if (task.taskId.length === 0 || byTaskId.has(task.taskId)) { + continue; + } + const description = task.description?.trim(); + byTaskId.set(task.taskId, { + taskId: task.taskId, + ...(description === undefined || description.length === 0 ? {} : { description }), + ...(task.taskType === undefined ? {} : { taskType: task.taskType }), + }); + } + } + + for (const item of input.turnItems) { + if (!BACKGROUND_TURN_ITEM_TYPES.has(item.type)) { + continue; + } + if (isTerminalTurnItemStatus(item.status)) { + continue; + } + if (item.type === "dynamic_tool" && isPersistentDynamicToolInput(item.input)) { + continue; + } + // Null/absent run id stays eligible; only known rolled_back runs drop. + const itemRunId = item.runId; + if (itemRunId !== undefined && itemRunId !== null && rolledBackRunIds.has(String(itemRunId))) { + continue; + } + + const taskId = nativeTaskIdFromTurnItem(item); + if (byTaskId.has(taskId)) { + continue; + } + + const description = descriptionFromTurnItem(item); + byTaskId.set(taskId, { + taskId, + ...(description === undefined ? {} : { description }), + taskType: item.type, + }); + } + + return Array.from(byTaskId.values()); +} + +export function formatPendingBackgroundWorkLabel( + tasks: ReadonlyArray, +): string | null { + if (tasks.length === 0) { + return null; + } + const firstDescription = tasks[0]?.description?.trim(); + if (tasks.length === 1) { + return firstDescription && firstDescription.length > 0 + ? `Waiting on background task: ${firstDescription}` + : "Waiting on a background task"; + } + if (firstDescription && firstDescription.length > 0) { + return `Waiting on ${tasks.length} background tasks: ${firstDescription}, …`; + } + return `Waiting on ${tasks.length} background tasks`; +}