fix(relay): tunnel cleanup keeps up with new idle tunnels (#26135)

Co-authored-by: Claude Opus 5.5 <noreply@anthropic.com>

Author: juliusmarminge

docs/operations/relay-observability.md

@@ -147,7 +147,8 @@ error messages. An expired APNs key or FCM service account fails every job.
 
 **Cloudflare API rate limited / near its limit.** The warning fires on sustained 429s; the page fires
 when they are frequent enough to stall provisioning. Compare the call volume on the dashboard with the
-account limit.
+account limit, which Cloudflare raised to 20,000 calls per 5 minutes in October 2026 (peak use then
+was about 2,100).
 
 **Traffic dropped.** The Worker is down, a deploy broke it, or spans stopped reaching Axiom. Check
 `https://relay.t3.codes/health` first.

infra/relay/src/environments/ManagedEndpointReaper.test.ts

@@ -677,7 +677,8 @@ describe("ManagedEndpointReaper", () => {
   });
 
   it.effect("rotates the deletion order so persistent failures do not starve later tunnels", () => {
-    const entries = Array.from({ length: 200 }, (_, index) =>
+    const limit = ManagedEndpointReaper.MANAGED_ENDPOINT_SWEEP_ATTEMPT_LIMIT;
+    const entries = Array.from({ length: limit * 2 }, (_, index) =>
       tunnel({
         id: `tunnel-${index}`,
         suffix: index.toString(16).padStart(16, "0"),
@@ -688,7 +689,7 @@ describe("ManagedEndpointReaper", () => {
     const state = harness({
       tunnels: entries,
       allocations: recoverableOwners(entries),
-      failDeleteWhen: (id) => Number(id.split("-")[1]) < 100,
+      failDeleteWhen: (id) => Number(id.split("-")[1]) < limit,
     });
 
     return Effect.gen(function* () {
@@ -697,14 +698,15 @@ describe("ManagedEndpointReaper", () => {
       yield* reaper.sweep;
       yield* TestClock.setTime(NOW_MILLIS + 5 * 60 * 1_000);
       yield* reaper.sweep;
-      const later = state.deleted.filter((id) => Number(id.split("-")[1]) >= 100);
+      const later = state.deleted.filter((id) => Number(id.split("-")[1]) >= limit);
       expect(later.length).toBeGreaterThan(0);
     }).pipe(Effect.provide(state.layer));
   });
 
   it.effect("rotates the attempt budget across both statuses over consecutive sweeps", () => {
+    const limit = ManagedEndpointReaper.MANAGED_ENDPOINT_SWEEP_ATTEMPT_LIMIT;
     const entries = (["down", "inactive"] as const).flatMap((status) =>
-      Array.from({ length: 100 }, (_, index) =>
+      Array.from({ length: limit }, (_, index) =>
         tunnel({
           id: `${status}-${index}`,
           suffix: `${status === "down" ? "a" : "b"}${index.toString(16).padStart(15, "0")}`,
@@ -720,13 +722,13 @@ describe("ManagedEndpointReaper", () => {
       yield* TestClock.setTime(NOW_MILLIS);
       yield* reaper.sweep;
       const firstStatus = state.deleted[0]!.split("-")[0];
-      expect(state.deleted).toHaveLength(100);
+      expect(state.deleted).toHaveLength(limit);
       expect(state.deleted.every((id) => id.startsWith(`${firstStatus}-`))).toBe(true);
 
       state.deleted.length = 0;
       yield* TestClock.setTime(NOW_MILLIS + 5 * 60 * 1_000);
       yield* reaper.sweep;
-      expect(state.deleted).toHaveLength(100);
+      expect(state.deleted).toHaveLength(limit);
       expect(state.deleted.every((id) => !id.startsWith(`${firstStatus}-`))).toBe(true);
     }).pipe(Effect.provide(state.layer));
   });

infra/relay/src/environments/ManagedEndpointReaper.ts

@@ -17,13 +17,13 @@ export const MANAGED_ENDPOINT_GRACE_PERIOD_MINUTES = 5;
 // Never-connected tunnels may still be pairing; allow an hour.
 export const MANAGED_ENDPOINT_INACTIVE_GRACE_PERIOD_MINUTES = 60;
 export const MANAGED_ENDPOINT_SWEEP_PAGE_SIZE = 100;
-export const MANAGED_ENDPOINT_SWEEP_ATTEMPT_LIMIT = 100;
+export const MANAGED_ENDPOINT_SWEEP_ATTEMPT_LIMIT = 250;
 export const MANAGED_ENDPOINT_SWEEP_LIST_REQUEST_LIMIT = 10;
 const MANAGED_ENDPOINT_LEGACY_AGE_BUCKET_DAYS = [7, 30, 90] as const;
 // Hosts without recovery cannot replace deleted tunnels; allow a longer absence.
 const MANAGED_ENDPOINT_LEGACY_GRACE_PERIOD_DAYS = 7;
-// Parallelize to fit the cron budget, but stay below Hyperdrive's shared 20-connection limit.
-export const MANAGED_ENDPOINT_SWEEP_DELETE_CONCURRENCY = 4;
+// Each delete holds a Hyperdrive connection for ~2s; stay well below the 40 request handlers share.
+export const MANAGED_ENDPOINT_SWEEP_DELETE_CONCURRENCY = 8;
 // Leave time under the cron's two-minute limit to finish deletions and record counters.
 const MANAGED_ENDPOINT_SWEEP_DELETE_BUDGET_MS = 90_000;