From 545bea119a85b264f469a179c1507a36ba79479a Mon Sep 17 00:00:00 2001 From: Adamulek123 Date: Wed, 16 Sep 2026 07:28:38 +0200 Subject: [PATCH 1/2] fix(relay): keep mint budget below the relay request deadline --- infra/relay/src/environments/EnvironmentConnector.ts | 5 ++++- 1 file changed, 4 insertions(+), 1 deletion(-) diff --git a/infra/relay/src/environments/EnvironmentConnector.ts b/infra/relay/src/environments/EnvironmentConnector.ts index 9201f814661e..13c26f6785fa 100644 --- a/infra/relay/src/environments/EnvironmentConnector.ts +++ b/infra/relay/src/environments/EnvironmentConnector.ts @@ -126,7 +126,10 @@ export type EnvironmentConnectorError = | EnvironmentLinks.EnvironmentLinkLookupPersistenceError | ManagedEndpointAllocations.ManagedEndpointAllocationPersistenceError; -export const ENVIRONMENT_MINT_REQUEST_TIMEOUT_MS = 10_000; +// Must stay below the relay request deadline (RELAY_REQUEST_DEADLINE_MS in +// http/Api.ts): a slower downstream budget can never surface its own typed +// timeout — the deadline 504s first and the extra downstream work is waste. +export const ENVIRONMENT_MINT_REQUEST_TIMEOUT_MS = 7_000; const ENVIRONMENT_HEALTH_CLOCK_SKEW_MILLIS = 60 * 1_000; export class EnvironmentConnector extends Context.Service< From e7dc492e0c57e26c07de2dc904a5ff0ae9b69721 Mon Sep 17 00:00:00 2001 From: Adamulek123 Date: Fri, 18 Sep 2026 23:45:33 +0200 Subject: [PATCH 2/2] fix(relay): apply mint timeout across whole connect operation --- .../environments/EnvironmentConnector.test.ts | 108 ++++++++- .../src/environments/EnvironmentConnector.ts | 225 +++++++++--------- infra/relay/src/http/Api.ts | 2 + 3 files changed, 225 insertions(+), 110 deletions(-) diff --git a/infra/relay/src/environments/EnvironmentConnector.test.ts b/infra/relay/src/environments/EnvironmentConnector.test.ts index da6f4acc192e..1d74dd15cb3f 100644 --- a/infra/relay/src/environments/EnvironmentConnector.test.ts +++ b/infra/relay/src/environments/EnvironmentConnector.test.ts @@ -2,6 +2,10 @@ import * as NodeCrypto from "node:crypto"; import * as NodeCryptoLayer from "@effect/platform-node/NodeCrypto"; import { + RelayApi, + RelayClientPrincipal, + RelayDpopClientAuth, + RelayEnvironmentConnectRequest, RelayCloudEnvironmentHealthRequest, RelayCloudMintCredentialRequest, RelayCloudEnvironmentHealthProofPayload, @@ -25,12 +29,23 @@ import * as Result from "effect/Result"; import * as Schema from "effect/Schema"; import * as TestClock from "effect/testing/TestClock"; import * as Tracer from "effect/Tracer"; -import { HttpClient, HttpClientRequest, HttpClientResponse } from "effect/unstable/http"; +import { + HttpClient, + HttpClientRequest, + HttpClientResponse, + HttpRouter, + HttpServer, + HttpServerRequest, + HttpServerResponse, +} from "effect/unstable/http"; +import { HttpApi, HttpApiBuilder } from "effect/unstable/httpapi"; import * as EnvironmentLinks from "./EnvironmentLinks.ts"; import * as RelayConfiguration from "../Config.ts"; import * as EnvironmentConnector from "./EnvironmentConnector.ts"; import * as ManagedEndpointAllocations from "./ManagedEndpointAllocations.ts"; +import * as DpopProofs from "../auth/DpopProofs.ts"; +import { dpopClientApi, RELAY_REQUEST_DEADLINE_MS, traceRelayHttpRequest } from "../http/Api.ts"; const cloudKeyPair = NodeCrypto.generateKeyPairSync("ed25519", { privateKeyEncoding: { format: "pem", type: "pkcs8" }, @@ -53,6 +68,9 @@ const decodeHealthRequestBody = Schema.decodeUnknownSync( const decodeMintRequestBody = Schema.decodeUnknownSync( Schema.fromJsonString(RelayCloudMintCredentialRequest), ); +const encodeConnectRequestBody = Schema.encodeSync( + Schema.fromJsonString(RelayEnvironmentConnectRequest), +); const isEnvironmentConnectNotAuthorized = Schema.is( EnvironmentConnector.EnvironmentConnectNotAuthorized, ); @@ -783,6 +801,94 @@ describe("EnvironmentConnector", () => { }).pipe(Effect.provide(connectorTestLayer(execute))); }); + for (const stalledStep of ["proof", "link", "allocation", "mint"] as const) { + it.effect(`preserves the endpoint timeout under the relay deadline during ${stalledStep}`, () => + Effect.gen(function* () { + const proofStarted = yield* Deferred.make(); + const releaseProof = yield* Deferred.make(); + const stepStarted = yield* Deferred.make(); + const interrupted = yield* Deferred.make(); + const stall = Deferred.succeed(stepStarted, undefined).pipe( + Effect.andThen(Effect.never), + Effect.onInterrupt(() => Deferred.succeed(interrupted, undefined)), + ); + const links = makeLinks(); + const allocations = makeAllocations(); + const handlers = dpopClientApi.pipe( + Layer.provide( + connectorTestLayer(() => stall, { + links: { + ...links, + getForUser: (input) => (stalledStep === "link" ? stall : links.getForUser(input)), + }, + allocations: { + ...allocations, + get: (input) => (stalledStep === "allocation" ? stall : allocations.get(input)), + }, + }), + ), + Layer.provide( + Layer.mock(DpopProofs.DpopProofReplay, { + verifyAndConsume: () => + Deferred.succeed(proofStarted, undefined).pipe( + Effect.andThen(stalledStep === "proof" ? stall : Deferred.await(releaseProof)), + Effect.as("client-proof-key-thumbprint"), + ), + }), + ), + Layer.provideMerge( + Layer.succeed(RelayDpopClientAuth, { + relayDpop: (effect) => + Effect.provideService(effect, RelayClientPrincipal, { + userId: "user_123", + token: "test-token", + proofKeyThumbprint: "client-proof-key-thumbprint", + dpopScopes: ["environment:connect"], + }), + }), + ), + ); + const httpEffect = yield* HttpRouter.toHttpEffect( + HttpApiBuilder.layer(HttpApi.make("RelayApi").add(RelayApi.groups.dpopClient)).pipe( + Layer.provide(handlers), + ), + ); + const fiber = yield* traceRelayHttpRequest(httpEffect).pipe( + Effect.provideService( + HttpServerRequest.HttpServerRequest, + HttpServerRequest.fromWeb( + new Request("https://relay.example.test/v1/environments/env-connector-test/connect", { + method: "POST", + headers: { + authorization: "DPoP test-token", + dpop: "test-proof", + "content-type": "application/json", + }, + body: encodeConnectRequestBody({ + clientProofKeyThumbprint: "client-proof-key-thumbprint", + }), + }), + ), + ), + Effect.forkScoped, + ); + yield* Deferred.await(proofStarted); + yield* TestClock.adjust(Duration.seconds(3)); + if (stalledStep !== "proof") { + yield* Deferred.succeed(releaseProof, undefined); + } + yield* Deferred.await(stepStarted); + yield* TestClock.adjust(Duration.millis(RELAY_REQUEST_DEADLINE_MS - 3_000)); + const response = yield* Fiber.join(fiber); + const body = yield* Effect.promise(() => HttpServerResponse.toWeb(response).json()); + + expect(response.status).toBe(504); + expect(body).toMatchObject({ code: "environment_endpoint_timed_out" }); + expect(yield* Deferred.isDone(interrupted)).toBe(true); + }).pipe(Effect.provide(HttpServer.layerServices), Effect.scoped), + ); + } + it.effect("times out hung managed endpoint mint requests", () => { let resolveRequestStarted: (() => void) | undefined; const requestStarted = new Promise((resolve) => { diff --git a/infra/relay/src/environments/EnvironmentConnector.ts b/infra/relay/src/environments/EnvironmentConnector.ts index 13c26f6785fa..630608a89609 100644 --- a/infra/relay/src/environments/EnvironmentConnector.ts +++ b/infra/relay/src/environments/EnvironmentConnector.ts @@ -132,6 +132,23 @@ export type EnvironmentConnectorError = export const ENVIRONMENT_MINT_REQUEST_TIMEOUT_MS = 7_000; const ENVIRONMENT_HEALTH_CLOCK_SKEW_MILLIS = 60 * 1_000; +export const withEnvironmentMintTimeout = ( + effect: Effect.Effect, + environmentId: string, +) => + effect.pipe( + Effect.timeoutOrElse({ + duration: Duration.millis(ENVIRONMENT_MINT_REQUEST_TIMEOUT_MS), + orElse: () => + Effect.fail( + new EnvironmentMintRequestTimedOut({ + environmentId, + timeoutMs: ENVIRONMENT_MINT_REQUEST_TIMEOUT_MS, + }), + ), + }), + ); + export class EnvironmentConnector extends Context.Service< EnvironmentConnector, { @@ -541,56 +558,43 @@ const make = Effect.gen(function* () { descriptor: decoded.descriptor, }; }), - connect: Effect.fn("relay.environment_connector.connect")(function* (input) { - yield* Effect.annotateCurrentSpan({ - "relay.environment_id": input.environmentId, - "relay.operation": "connect", - "relay.connect.has_device_id": input.deviceId !== undefined, - ...(input.deviceId ? { "relay.mobile.device_id": input.deviceId } : {}), - }); - if (input.clientProofKeyThumbprint.trim().length === 0) { - return yield* new EnvironmentConnectNotAuthorized({ - environmentId: input.environmentId, - operation: "connect", - reason: "client_proof_key_thumbprint_missing", + connect: Effect.fn("relay.environment_connector.connect")( + function* (input) { + yield* Effect.annotateCurrentSpan({ + "relay.environment_id": input.environmentId, + "relay.operation": "connect", + "relay.connect.has_device_id": input.deviceId !== undefined, + ...(input.deviceId ? { "relay.mobile.device_id": input.deviceId } : {}), }); - } - const { link, allocation } = yield* Effect.all( - { - link: links.getForUser(input), - allocation: allocations.get(input), - }, - { concurrency: 2 }, - ); - if (!link) { - return yield* new EnvironmentConnectNotAuthorized({ - environmentId: input.environmentId, + if (input.clientProofKeyThumbprint.trim().length === 0) { + return yield* new EnvironmentConnectNotAuthorized({ + environmentId: input.environmentId, + operation: "connect", + reason: "client_proof_key_thumbprint_missing", + }); + } + const { link, allocation } = yield* Effect.all( + { + link: links.getForUser(input), + allocation: allocations.get(input), + }, + { concurrency: 2 }, + ); + if (!link) { + return yield* new EnvironmentConnectNotAuthorized({ + environmentId: input.environmentId, + operation: "connect", + reason: "environment_link_not_found", + }); + } + const endpoint = yield* resolveManagedEndpoint({ operation: "connect", - reason: "environment_link_not_found", + link, + allocation, }); - } - const endpoint = yield* resolveManagedEndpoint({ - operation: "connect", - link, - allocation, - }); - const now = yield* DateTime.now; - const expiresAt = DateTime.add(now, { minutes: 2 }); - const nonce = yield* crypto.randomUUIDv4.pipe( - Effect.mapError( - (cause) => - new EnvironmentMintRequestFailed({ - environmentId: input.environmentId, - operation: "connect", - cause, - }), - ), - ); - const payload = { - iss: relayIssuer, - aud: `t3-env:${link.environmentId}`, - sub: input.userId, - jti: yield* crypto.randomUUIDv4.pipe( + const now = yield* DateTime.now; + const expiresAt = DateTime.add(now, { minutes: 2 }); + const nonce = yield* crypto.randomUUIDv4.pipe( Effect.mapError( (cause) => new EnvironmentMintRequestFailed({ @@ -599,35 +603,35 @@ const make = Effect.gen(function* () { cause, }), ), - ), - iat: Math.floor(now.epochMilliseconds / 1_000), - exp: Math.floor(expiresAt.epochMilliseconds / 1_000), - environmentId: link.environmentId, - clientProofKeyThumbprint: input.clientProofKeyThumbprint, - cnf: { jkt: input.clientProofKeyThumbprint }, - ...(input.deviceId ? { deviceId: input.deviceId } : {}), - nonce, - scope: ["environment:connect"], - } satisfies RelayCloudMintCredentialProofPayload; - const proof = yield* signRelayJwt({ - privateKey: Redacted.value(settings.cloudMintPrivateKey), - typ: RELAY_MINT_REQUEST_TYP, - payload, - }).pipe( - Effect.mapError( - (cause) => - new EnvironmentMintRequestFailed({ - environmentId: input.environmentId, - operation: "connect", - cause, - }), - ), - ); - const environmentClient = yield* makeEnvironmentClient(endpoint.httpBaseUrl); - const decoded = yield* environmentClient.connect - .t3MintCredential({ payload: { proof } }) - .pipe( - withoutRedirects, + ); + const payload = { + iss: relayIssuer, + aud: `t3-env:${link.environmentId}`, + sub: input.userId, + jti: yield* crypto.randomUUIDv4.pipe( + Effect.mapError( + (cause) => + new EnvironmentMintRequestFailed({ + environmentId: input.environmentId, + operation: "connect", + cause, + }), + ), + ), + iat: Math.floor(now.epochMilliseconds / 1_000), + exp: Math.floor(expiresAt.epochMilliseconds / 1_000), + environmentId: link.environmentId, + clientProofKeyThumbprint: input.clientProofKeyThumbprint, + cnf: { jkt: input.clientProofKeyThumbprint }, + ...(input.deviceId ? { deviceId: input.deviceId } : {}), + nonce, + scope: ["environment:connect"], + } satisfies RelayCloudMintCredentialProofPayload; + const proof = yield* signRelayJwt({ + privateKey: Redacted.value(settings.cloudMintPrivateKey), + typ: RELAY_MINT_REQUEST_TYP, + payload, + }).pipe( Effect.mapError( (cause) => new EnvironmentMintRequestFailed({ @@ -636,42 +640,45 @@ const make = Effect.gen(function* () { cause, }), ), - Effect.timeoutOption(Duration.millis(ENVIRONMENT_MINT_REQUEST_TIMEOUT_MS)), - Effect.flatMap( - Option.match({ - onNone: () => - Effect.fail( - new EnvironmentMintRequestTimedOut({ - environmentId: input.environmentId, - timeoutMs: ENVIRONMENT_MINT_REQUEST_TIMEOUT_MS, - }), - ), - onSome: Effect.succeed, - }), - ), ); - const verified = yield* verifyEnvironmentResponse({ - response: decoded, - environmentId: input.environmentId, - requestNonce: nonce, - clientProofKeyThumbprint: input.clientProofKeyThumbprint, - environmentPublicKeys: [link.environmentPublicKey], - relayIssuer, - nowEpochSeconds: Math.floor(now.epochMilliseconds / 1_000), - }); - if (!verified) { - return yield* new EnvironmentMintResponseInvalid({ + const environmentClient = yield* makeEnvironmentClient(endpoint.httpBaseUrl); + const decoded = yield* environmentClient.connect + .t3MintCredential({ payload: { proof } }) + .pipe( + withoutRedirects, + Effect.mapError( + (cause) => + new EnvironmentMintRequestFailed({ + environmentId: input.environmentId, + operation: "connect", + cause, + }), + ), + ); + const verified = yield* verifyEnvironmentResponse({ + response: decoded, environmentId: input.environmentId, - operation: "connect", + requestNonce: nonce, + clientProofKeyThumbprint: input.clientProofKeyThumbprint, + environmentPublicKeys: [link.environmentPublicKey], + relayIssuer, + nowEpochSeconds: Math.floor(now.epochMilliseconds / 1_000), }); - } - return { - environmentId: link.environmentId, - endpoint, - credential: decoded.credential, - expiresAt: decoded.expiresAt, - }; - }), + if (!verified) { + return yield* new EnvironmentMintResponseInvalid({ + environmentId: input.environmentId, + operation: "connect", + }); + } + return { + environmentId: link.environmentId, + endpoint, + credential: decoded.credential, + expiresAt: decoded.expiresAt, + }; + }, + (effect, input) => withEnvironmentMintTimeout(effect, input.environmentId), + ), }); }); diff --git a/infra/relay/src/http/Api.ts b/infra/relay/src/http/Api.ts index fbe867b7792a..70f40a952ebf 100644 --- a/infra/relay/src/http/Api.ts +++ b/infra/relay/src/http/Api.ts @@ -799,6 +799,8 @@ export const dpopClientApi = HttpApiBuilder.group( ...(payload.deviceId ? { deviceId: payload.deviceId } : {}), }); }, + (effect, args) => + EnvironmentConnector.withEnvironmentMintTimeout(effect, args.params.environmentId), mapRelayCommonApiErrors("invalid_dpop"), mapErrorTags({ EnvironmentConnectNotAuthorized: (error, traceId) =>