fix(orb): ack the GitHub webhook before the relay forward (#1623)

src/orb/webhook.ts

@@ -78,18 +78,46 @@ export async function handleOrbWebhook(c: Context<{ Bindings: Env }>): Promise<R
   }
 
   await recordOrbWebhookEvent(c.env, { ...eventMeta, status: "received" });
-  // Forward the event to a brokered self-host registered for this installation (best-effort, fail-safe — a down
-  // container never fails the 202; a non-forwardable event / no registered relay is a fast no-op).
-  const installId = payload.installation?.id;
-  const relayResult = await forwardOrbEvent(c.env, { eventName, installationId: installId, deliveryId, rawBody });
-  // Persist failed deliveries so the retry-orb-relay cron can re-attempt them (containers that are temporarily
-  // down recover without losing events; max 5 attempts within a 1-hour TTL).
-  if (relayResult === "failed" && installId !== undefined) {
-    await storeRelayFailure(c.env, { deliveryId, eventName, installationId: installId, rawBody });
-  }
+  // Forward to a brokered self-host registered for this installation — but NEVER block the 202 we owe GitHub on it.
+  // A push-mode forward POSTs to the container's relay URL with a 10s timeout; a slow (e.g. tailnet) container would
+  // otherwise delay our response past GitHub's ~10s delivery deadline, so GitHub marks the delivery FAILED even
+  // though we received + queued it. Run the forward (+ its failure-persistence for the retry cron) AFTER the
+  // response via waitUntil. (#orb-ack-fast)
+  scheduleAfterResponse(c, relayForward(c.env, { eventName, installationId: payload.installation?.id, deliveryId, rawBody }));
   return c.json({ ok: true, deliveryId, eventName, status: "received" }, 202);
 }
 
+/** Forward an Orb webhook to the brokered self-host registered for the installation, persisting a FAILED push for
+ *  the retry-orb-relay cron (a temporarily-down container recovers without losing events; max 5 attempts / 1h TTL).
+ *  Self-contained + fail-safe (never throws) so it can run AFTER the response via {@link scheduleAfterResponse}. */
+export async function relayForward(
+  env: Env,
+  args: { eventName: string; installationId: number | null | undefined; deliveryId: string; rawBody: string },
+  fetchImpl: typeof fetch = fetch,
+): Promise<void> {
+  try {
+    const relayResult = await forwardOrbEvent(env, args, fetchImpl);
+    // forwardOrbEvent returns "failed" only for an ENROLLED install (a null/absent id "skips"), so installationId is
+    // non-null here — persist the failed push so the retry-orb-relay cron re-attempts it.
+    if (relayResult === "failed") {
+      await storeRelayFailure(env, { deliveryId: args.deliveryId, eventName: args.eventName, installationId: args.installationId!, rawBody: args.rawBody });
+    }
+  } catch {
+    /* v8 ignore next -- fail-safe: a forward/persist error must never surface from the deferred task */
+  }
+}
+
+/** Run `task` AFTER the response is sent (Cloudflare Workers `waitUntil`), so a slow downstream relay forward can't
+ *  delay the webhook ACK past GitHub's ~10s delivery deadline. Falls back to fire-and-forget where there is no
+ *  execution context (e.g. a unit-test harness); the self-host server provides its own waitUntil shim. */
+function scheduleAfterResponse(c: Context<{ Bindings: Env }>, task: Promise<unknown>): void {
+  try {
+    (c.executionCtx as unknown as { waitUntil(p: Promise<unknown>): void }).waitUntil(task);
+  } catch {
+    void task;
+  }
+}
+
 async function getOrbWebhookEvent(env: Env, deliveryId: string): Promise<{ payloadHash: string; status: string } | null> {
   const row = await env.DB.prepare("SELECT payload_hash AS payloadHash, status FROM orb_webhook_events WHERE delivery_id = ?")
     .bind(deliveryId)

test/integration/orb-relay.test.ts

@@ -1,6 +1,7 @@
 import { describe, expect, it } from "vitest";
 import { createApp } from "../../src/api/routes";
 import { issueOrbEnrollment } from "../../src/orb/broker";
+import { relayForward } from "../../src/orb/webhook";
 import { enqueueRelayPending, forwardOrbEvent, MAX_ORB_RELAY_REGISTER_BODY_BYTES, pullRelayPending, readOrbRelayRegisterBody, registerOrbRelay, relaySignature, relayVerify, retryFailedRelays, storeRelayFailure } from "../../src/orb/relay";
 import { createTestEnv, type TestD1Database } from "../helpers/d1";
 
@@ -197,6 +198,24 @@ describe("forwardOrbEvent", () => {
   });
 });
 
+describe("relayForward (deferred forward + failure persistence, #orb-ack-fast)", () => {
+  it("persists a FAILED push to orb_relay_failures so the retry cron re-attempts it", async () => {
+    const e = brokeredEnv();
+    const secret = await enroll(e, 820);
+    await registerOrbRelay(e, secret, "https://c.example/v1/orb/relay");
+    await relayForward(e, { eventName: "pull_request", installationId: 820, deliveryId: "rf-fail", rawBody: "{}" }, (() => Promise.resolve(new Response("no", { status: 503 }))) as typeof fetch);
+    const row = await db(e).prepare("SELECT installation_id, event_name FROM orb_relay_failures WHERE delivery_id='rf-fail'").first<{ installation_id: number; event_name: string }>();
+    expect(row).toMatchObject({ installation_id: 820, event_name: "pull_request" });
+  });
+
+  it("does NOT persist when the forward is skipped (enrolled but no relay) and never throws", async () => {
+    const e = brokeredEnv();
+    await enroll(e, 821); // enrolled, but no relay registered → forwardOrbEvent skips before any fetch
+    await expect(relayForward(e, { eventName: "pull_request", installationId: 821, deliveryId: "rf-skip", rawBody: "{}" })).resolves.toBeUndefined();
+    expect(await db(e).prepare("SELECT delivery_id FROM orb_relay_failures WHERE delivery_id='rf-skip'").first()).toBeFalsy();
+  });
+});
+
 describe("relayVerify", () => {
   it("accepts a valid signature (sha256= or bare hex) and rejects wrong-secret / malformed / missing", async () => {
     const body = '{"x":1}';

test/integration/orb-webhook.test.ts

@@ -13,11 +13,12 @@ async function sign(body: string, secret: string): Promise<string> {
   return `sha256=${[...new Uint8Array(signed)].map((b) => b.toString(16).padStart(2, "0")).join("")}`;
 }
 
-function ctx(e: Env, headers: Record<string, string | null>, request: Request): Context<{ Bindings: Env }> {
+function ctx(e: Env, headers: Record<string, string | null>, request: Request, executionCtx?: { waitUntil(p: Promise<unknown>): void }): Context<{ Bindings: Env }> {
   return {
     req: { raw: request, header: (n: string) => headers[n.toLowerCase()] ?? null },
     env: e,
     json: (payload: unknown, status?: number) => Response.json(payload, status === undefined ? undefined : { status }),
+    ...(executionCtx ? { executionCtx } : {}),
   } as unknown as Context<{ Bindings: Env }>;
 }
 
@@ -97,6 +98,19 @@ describe("handleOrbWebhook (POST /v1/orb/webhook)", () => {
     expect(await row(e, "ok-1")).toMatchObject({ action: "created", installation_id: 42, repository_full_name: "JSONbored/gittensory", status: "received" });
   });
 
+  it("ACKs 202 immediately and SCHEDULES the relay forward via waitUntil (never blocks the ACK on a slow container, #orb-ack-fast)", async () => {
+    const e = env();
+    const scheduled: Promise<unknown>[] = [];
+    const PR = JSON.stringify({ action: "opened", installation: { id: 99 }, repository: { full_name: "JSONbored/gittensory" }, number: 7 });
+    const request = new Request("https://collector/v1/orb/webhook", { method: "POST", body: PR });
+    const headers = { "x-github-delivery": "fwd-1", "x-github-event": "pull_request", "x-hub-signature-256": await sign(PR, SECRET) };
+    const res = await handleOrbWebhook(ctx(e, headers, request, { waitUntil: (p) => scheduled.push(p) }));
+    expect(res.status).toBe(202);
+    await expect(res.json()).resolves.toMatchObject({ status: "received", eventName: "pull_request" });
+    expect(scheduled).toHaveLength(1); // forward DEFERRED past the response, not awaited inline
+    await Promise.all(scheduled); // drain — install 99 has no enrollment → forward skips cleanly
+  });
+
   it("stores null fields for a payload with no action/installation/repository (e.g. ping)", async () => {
     const e = env();
     await post(e, JSON.stringify({ zen: "keep it logically awesome" }), { delivery: "ping-1", event: "ping" });