fix(server): repeated capture timeouts no longer delay turn starts

apps/server/src/orchestration-v2/CheckpointService.test.ts

@@ -96,3 +96,85 @@ it.effect(
       }
     }),
 );
+
+it.effect("backs off only baseline retries after repeated workspace capture timeouts", () =>
+  Effect.gen(function* () {
+    const scope: OrchestrationV2CheckpointScope = {
+      id: CheckpointScopeId.make("checkpoint-scope:slow-capture"),
+      threadId: ThreadId.make("thread:slow-capture"),
+      runId: RunId.make("run:slow-capture:1"),
+      nodeId: NodeId.make("node:slow-capture:1"),
+      parentScopeId: null,
+      providerThreadId: ProviderThreadId.make("provider-thread:slow-capture"),
+      kind: "root_run",
+      ordinalWithinParent: 0,
+      advancesAppRunCount: true,
+      cwd: "/repo/slow",
+      createdAt: DateTime.makeUnsafe("2026-07-28T00:00:00.000Z"),
+    };
+    const timeout = new VcsProcessTimeoutError({
+      operation: "GitVcsDriver.checkpoints.captureCheckpoint",
+      command: "git",
+      cwd: scope.cwd,
+      timeoutMs: 30_000,
+    });
+    let attempts = 0;
+    let healthy = false;
+    const layerTest = CheckpointService.layer.pipe(
+      Layer.provide(
+        Layer.mergeAll(
+          IdAllocator.layer,
+          Layer.mock(CheckpointStore.CheckpointStore)({
+            isGitRepository: () => Effect.succeed(true),
+            hasCheckpointRef: () => Effect.succeed(false),
+            captureCheckpoint: () =>
+              Effect.suspend(() => {
+                attempts++;
+                return healthy ? Effect.void : Effect.fail(timeout);
+              }),
+          }),
+        ),
+      ),
+      Layer.provideMerge(NodeCrypto.layer),
+    );
+    yield* Effect.gen(function* () {
+      const checkpoints = yield* CheckpointService.CheckpointServiceV2;
+      const baseline = { scope, ordinalWithinScope: 0 };
+      const turn = {
+        scope,
+        ordinalWithinScope: 1,
+        runId: scope.runId,
+        nodeId: scope.nodeId,
+        appRunOrdinal: 1,
+        capturedAt: scope.createdAt,
+      };
+      yield* Effect.result(checkpoints.captureBaseline(baseline));
+      assert.strictEqual((yield* checkpoints.capture(turn)).status, "error");
+      assert.strictEqual(attempts, 2);
+
+      const otherThread = { ...scope, id: CheckpointScopeId.make("checkpoint-scope:other") };
+      yield* checkpoints.captureBaseline({ scope: otherThread, ordinalWithinScope: 0 });
+      assert.strictEqual(attempts, 2, "baseline retried a repeatedly timing-out workspace");
+
+      yield* Effect.result(
+        checkpoints.captureBaseline({
+          scope: { ...otherThread, cwd: "/repo/other" },
+          ordinalWithinScope: 0,
+        }),
+      );
+      assert.strictEqual(attempts, 3, "another workspace must still attempt capture");
+
+      const failedTurn = yield* checkpoints.capture(turn);
+      assert.strictEqual(failedTurn.status, "error");
+      assert.strictEqual(attempts, 4, "post-turn capture must not back off");
+
+      yield* checkpoints.captureBaseline(baseline);
+      assert.strictEqual(attempts, 4, "baseline retries must wait for post-turn recovery");
+      healthy = true;
+      assert.strictEqual((yield* checkpoints.capture(turn)).status, "ready");
+      assert.strictEqual(attempts, 5);
+      yield* checkpoints.captureBaseline(baseline);
+      assert.strictEqual(attempts, 6, "a successful post-turn capture must clear backoff");
+    }).pipe(Effect.provide(layerTest));
+  }),
+);

apps/server/src/orchestration-v2/CheckpointService.ts

@@ -9,6 +9,7 @@ import {
   RunId,
   ThreadId,
 } from "@t3tools/contracts";
+import * as Cache from "effect/Cache";
 import * as Context from "effect/Context";
 import * as Digest from "effect/crypto/Digest";
 import * as DateTime from "effect/DateTime";
@@ -17,6 +18,7 @@ import * as Base64Url from "effect/encoding/Base64Url";
 import * as Hex from "effect/encoding/Hex";
 import * as KeyedLock from "@t3tools/shared/KeyedLock";
 import * as Layer from "effect/Layer";
+import * as Option from "effect/Option";
 import * as Schema from "effect/Schema";
 
 import { isGitImport, parseTurnDiffFilesFromNumstat } from "../checkpointing/Diffs.ts";
@@ -247,6 +249,27 @@ export const layer: Layer.Layer<
     const withWorkspaceLock = <A, E, R>(cwd: string, effect: Effect.Effect<A, E, R>) =>
       workspaceLocks.withLock(cwd, effect);
 
+    const captureTimeouts = yield* Cache.make<string, number>({
+      capacity: 128,
+      lookup: () => Effect.succeed(0),
+    });
+    // Both callers hold the workspace lock; post-turn captures always probe for recovery.
+    const captureWorkspaceCheckpoint = Effect.fn("CheckpointService.captureWorkspaceCheckpoint")(
+      function* (input: CheckpointStore.CaptureCheckpointInput) {
+        yield* checkpointStore.captureCheckpoint(input).pipe(
+          Effect.tap(() => Cache.invalidate(captureTimeouts, input.cwd)),
+          Effect.tapError((error) =>
+            error._tag === "VcsProcessTimeoutError"
+              ? Effect.gen(function* () {
+                  const previous = yield* Cache.get(captureTimeouts, input.cwd);
+                  yield* Cache.set(captureTimeouts, input.cwd, previous + 1);
+                })
+              : Cache.invalidate(captureTimeouts, input.cwd),
+          ),
+        );
+      },
+    );
+
     const isGitCheckpointable = (cwd: string) =>
       checkpointStore.isGitRepository(cwd).pipe(Effect.orElseSucceed(() => false));
 
@@ -270,16 +293,22 @@ export const layer: Layer.Layer<
             return;
           }
 
-          yield* checkpointStore
-            .captureCheckpoint({
-              cwd: input.scope.cwd,
-              checkpointRef,
-            })
-            .pipe(
-              Effect.catchTags({
-                VcsUnsupportedOperationError: () => Effect.void,
-              }),
+          const timeouts = yield* Cache.getOption(captureTimeouts, input.scope.cwd);
+          if (Option.isSome(timeouts) && timeouts.value >= 2) {
+            return yield* Effect.logWarning(
+              "orchestration V2 checkpoint baseline skipped after repeated capture timeouts",
+              { scopeId: input.scope.id, ordinalWithinScope: input.ordinalWithinScope },
             );
+          }
+
+          yield* captureWorkspaceCheckpoint({
+            cwd: input.scope.cwd,
+            checkpointRef,
+          }).pipe(
+            Effect.catchTags({
+              VcsUnsupportedOperationError: () => Effect.void,
+            }),
+          );
         }),
       ).pipe(
         Effect.mapError(
@@ -387,24 +416,22 @@ export const layer: Layer.Layer<
             });
           }
 
-          const captureStatus = yield* checkpointStore
-            .captureCheckpoint({
-              cwd: input.scope.cwd,
-              checkpointRef,
-            })
-            .pipe(
-              Effect.as("ready" as const),
-              Effect.catchTags({
-                VcsUnsupportedOperationError: () => Effect.succeed("missing" as const),
-              }),
-              Effect.catch((cause) =>
-                Effect.logWarning("orchestration V2 checkpoint capture failed", {
-                  scopeId: input.scope.id,
-                  checkpointRef,
-                  cause: String(cause),
-                }).pipe(Effect.as("error" as const)),
-              ),
-            );
+          const captureStatus = yield* captureWorkspaceCheckpoint({
+            cwd: input.scope.cwd,
+            checkpointRef,
+          }).pipe(
+            Effect.as("ready" as const),
+            Effect.catchTags({
+              VcsUnsupportedOperationError: () => Effect.succeed("missing" as const),
+            }),
+            Effect.catch((cause) =>
+              Effect.logWarning("orchestration V2 checkpoint capture failed", {
+                scopeId: input.scope.id,
+                checkpointRef,
+                cause: String(cause),
+              }).pipe(Effect.as("error" as const)),
+            ),
+          );
 
           if (captureStatus !== "ready") {
             return makeCheckpoint({