diff --git a/.server-changes/worker-deployment-lookup-ordering.md b/.server-changes/worker-deployment-lookup-ordering.md new file mode 100644 index 00000000000..1dbfbdd253a --- /dev/null +++ b/.server-changes/worker-deployment-lookup-ordering.md @@ -0,0 +1,6 @@ +--- +area: webapp +type: improvement +--- + +Speeds up resolving the latest worker version and deployment for an environment, removing an occasional stall when triggering runs in projects that have accumulated many deployed versions. diff --git a/apps/webapp/app/services/runsRepository/clickhouseRunsRepository.server.ts b/apps/webapp/app/services/runsRepository/clickhouseRunsRepository.server.ts index 2aa9d5f3a68..c9fefd1da10 100644 --- a/apps/webapp/app/services/runsRepository/clickhouseRunsRepository.server.ts +++ b/apps/webapp/app/services/runsRepository/clickhouseRunsRepository.server.ts @@ -271,9 +271,6 @@ export class ClickHouseRunsRepository implements IRunsRepository { in: ids, }, }, - orderBy: { - id: "desc", - }, select: { id: true, friendlyId: true, diff --git a/apps/webapp/app/v3/models/workerDeployment.server.ts b/apps/webapp/app/v3/models/workerDeployment.server.ts index 5934cd07620..48995bf2b29 100644 --- a/apps/webapp/app/v3/models/workerDeployment.server.ts +++ b/apps/webapp/app/v3/models/workerDeployment.server.ts @@ -128,14 +128,12 @@ export async function findCurrentWorkerDeployment({ } // We need to get the latest deployment of the given type - const latestDeployment = await prisma.workerDeployment.findFirst({ + const latestDeployment = await $prisma.workerDeployment.findFirst({ where: { environmentId, type, }, - orderBy: { - id: "desc", - }, + orderBy: [{ createdAt: "desc" }, { id: "desc" }], select: { id: true, imageReference: true, diff --git a/apps/webapp/app/v3/runOpsMigration/controlPlaneResolver.server.ts b/apps/webapp/app/v3/runOpsMigration/controlPlaneResolver.server.ts index d6a706c9665..eb19a7fb6c1 100644 --- a/apps/webapp/app/v3/runOpsMigration/controlPlaneResolver.server.ts +++ b/apps/webapp/app/v3/runOpsMigration/controlPlaneResolver.server.ts @@ -432,7 +432,7 @@ export class ControlPlaneResolver { // MANAGED deployment. const latestV2Deployment = await client.workerDeployment.findFirst({ where: { environmentId, type: "MANAGED" }, - orderBy: { id: "desc" }, + orderBy: [{ createdAt: "desc" }, { id: "desc" }], include: { worker: { include: { tasks: true, queues: true } } }, }); @@ -455,7 +455,6 @@ export class ControlPlaneResolver { const worker = await client.backgroundWorker.findFirst({ where: { id: workerId }, include: { deployment: true, tasks: true, queues: true }, - orderBy: { id: "desc" }, }); if (!worker) { @@ -472,7 +471,7 @@ export class ControlPlaneResolver { const worker = await client.backgroundWorker.findFirst({ where: { runtimeEnvironmentId: environmentId }, include: { tasks: true, queues: true }, - orderBy: { id: "desc" }, + orderBy: [{ createdAt: "desc" }, { id: "desc" }], }); if (!worker) { diff --git a/apps/webapp/test/v3/runOpsMigration/controlPlaneResolver.server.test.ts b/apps/webapp/test/v3/runOpsMigration/controlPlaneResolver.server.test.ts index e85843114db..af734ca1f75 100644 --- a/apps/webapp/test/v3/runOpsMigration/controlPlaneResolver.server.test.ts +++ b/apps/webapp/test/v3/runOpsMigration/controlPlaneResolver.server.test.ts @@ -62,7 +62,12 @@ async function seedControlPlane(prisma: PrismaClient) { async function seedWorker( prisma: PrismaClient, ctx: { projectId: string; environmentId: string }, - opts?: { promote?: boolean } + opts?: { + promote?: boolean; + createdAt?: Date; + deploymentType?: "MANAGED" | "UNMANAGED" | "V1"; + deploymentCreatedAt?: Date; + } ) { const n = seedCounter++; const worker = await prisma.backgroundWorker.create({ @@ -74,6 +79,7 @@ async function seedWorker( version: `2024.1.${n}`, metadata: {}, engine: "V2", + ...(opts?.createdAt ? { createdAt: opts.createdAt } : {}), }, }); const task = await prisma.backgroundWorkerTask.create({ @@ -104,11 +110,12 @@ async function seedWorker( contentHash: `hash_${n}`, version: worker.version, shortCode: `dep_${n}`, - type: "MANAGED", + type: opts?.deploymentType ?? "MANAGED", status: "DEPLOYED", projectId: ctx.projectId, environmentId: ctx.environmentId, workerId: worker.id, + ...(opts?.deploymentCreatedAt ? { createdAt: opts.deploymentCreatedAt } : {}), }, }); await prisma.workerDeploymentPromotion.create({ @@ -748,3 +755,103 @@ heteroPostgresTest( expect(reads()).toBe(readsAfterFirst * 2); } ); + +heteroPostgresTest( + "resolveWorkerVersion (DEVELOPMENT) resolves the newest worker by createdAt, not by id", + async ({ prisma14 }) => { + const { environment, project } = await seedControlPlane(prisma14); + const ctx = { projectId: project.id, environmentId: environment.id }; + + const newest = await seedWorker(prisma14, ctx, { + createdAt: new Date("2026-07-31T12:00:00.000Z"), + }); + const oldest = await seedWorker(prisma14, ctx, { + createdAt: new Date("2026-07-30T12:00:00.000Z"), + }); + + expect(oldest.worker.id > newest.worker.id).toBe(true); + expect(oldest.worker.createdAt < newest.worker.createdAt).toBe(true); + + const resolver = new ControlPlaneResolver({ + controlPlaneReplica: prisma14, + controlPlanePrimary: prisma14, + cache: new ControlPlaneCache(), + splitEnabled: () => true, + }); + + const resolved = await resolver.resolveWorkerVersion({ + environmentId: environment.id, + type: "DEVELOPMENT", + }); + + expect(resolved).not.toBeNull(); + expect(resolved!.worker.id).toBe(newest.worker.id); + } +); + +heteroPostgresTest( + "resolveWorkerVersion latest-MANAGED fallback resolves by createdAt, not by id", + async ({ prisma14 }) => { + const { environment, project } = await seedControlPlane(prisma14); + const ctx = { projectId: project.id, environmentId: environment.id }; + + await seedWorker(prisma14, ctx, { promote: true, deploymentType: "V1" }); + + const newest = await seedWorker(prisma14, ctx, { + promote: false, + deploymentCreatedAt: new Date("2026-07-31T12:00:00.000Z"), + }); + const oldest = await seedWorker(prisma14, ctx, { + promote: false, + deploymentCreatedAt: new Date("2026-07-30T12:00:00.000Z"), + }); + + const newestDeployment = await prisma14.workerDeployment.create({ + data: { + friendlyId: `deployment_newest_${environment.id}`, + contentHash: "hash_newest", + version: newest.worker.version, + shortCode: "dep_newest", + type: "MANAGED", + status: "DEPLOYED", + projectId: project.id, + environmentId: environment.id, + workerId: newest.worker.id, + createdAt: new Date("2026-07-31T12:00:00.000Z"), + }, + }); + const oldestDeployment = await prisma14.workerDeployment.create({ + data: { + friendlyId: `deployment_oldest_${environment.id}`, + contentHash: "hash_oldest", + version: oldest.worker.version, + shortCode: "dep_oldest", + type: "MANAGED", + status: "DEPLOYED", + projectId: project.id, + environmentId: environment.id, + workerId: oldest.worker.id, + createdAt: new Date("2026-07-30T12:00:00.000Z"), + }, + }); + + expect(oldestDeployment.id > newestDeployment.id).toBe(true); + expect(oldestDeployment.createdAt < newestDeployment.createdAt).toBe(true); + + const resolver = new ControlPlaneResolver({ + controlPlaneReplica: prisma14, + controlPlanePrimary: prisma14, + cache: new ControlPlaneCache(), + splitEnabled: () => true, + }); + + const resolved = await resolver.resolveWorkerVersion({ + environmentId: environment.id, + type: "PRODUCTION", + }); + + expect(resolved).not.toBeNull(); + expect(resolved!.deployment!.id).toBe(newestDeployment.id); + expect(resolved!.worker.id).toBe(newest.worker.id); + } +); diff --git a/internal-packages/database/prisma/migrations/20260731160000_add_worker_deployment_environment_id_created_at_index/migration.sql b/internal-packages/database/prisma/migrations/20260731160000_add_worker_deployment_environment_id_created_at_index/migration.sql new file mode 100644 index 00000000000..57d9eec96c4 --- /dev/null +++ b/internal-packages/database/prisma/migrations/20260731160000_add_worker_deployment_environment_id_created_at_index/migration.sql @@ -0,0 +1 @@ +CREATE INDEX CONCURRENTLY IF NOT EXISTS "WorkerDeployment_environmentId_createdAt_idx" ON "public"."WorkerDeployment"("environmentId", "createdAt"); diff --git a/internal-packages/database/prisma/schema.prisma b/internal-packages/database/prisma/schema.prisma index 74c6d72a9c5..ca1d868ab04 100644 --- a/internal-packages/database/prisma/schema.prisma +++ b/internal-packages/database/prisma/schema.prisma @@ -2208,6 +2208,7 @@ model WorkerDeployment { @@unique([projectId, shortCode]) @@unique([environmentId, version]) @@index([commitSHA]) + @@index([environmentId, createdAt]) } enum WorkerDeploymentStatus { diff --git a/internal-packages/run-engine/src/engine/controlPlaneResolver.ts b/internal-packages/run-engine/src/engine/controlPlaneResolver.ts index 89b0e15c48c..da5f7b00603 100644 --- a/internal-packages/run-engine/src/engine/controlPlaneResolver.ts +++ b/internal-packages/run-engine/src/engine/controlPlaneResolver.ts @@ -234,9 +234,7 @@ export class PassthroughControlPlaneResolver implements ControlPlaneResolver { tasks: true, queues: true, }, - orderBy: { - id: "desc", - }, + orderBy: [{ createdAt: "desc" }, { id: "desc" }], }); if (!worker) { @@ -256,9 +254,6 @@ export class PassthroughControlPlaneResolver implements ControlPlaneResolver { tasks: true, queues: true, }, - orderBy: { - id: "desc", - }, }); if (!worker) { @@ -315,9 +310,7 @@ export class PassthroughControlPlaneResolver implements ControlPlaneResolver { environmentId, type: "MANAGED", }, - orderBy: { - id: "desc", - }, + orderBy: [{ createdAt: "desc" }, { id: "desc" }], include: { worker: { include: { diff --git a/internal-packages/run-engine/src/engine/tests/dequeueSystem.controlPlaneResolver.test.ts b/internal-packages/run-engine/src/engine/tests/dequeueSystem.controlPlaneResolver.test.ts index 5ede5c89db2..3ea56bde8a6 100644 --- a/internal-packages/run-engine/src/engine/tests/dequeueSystem.controlPlaneResolver.test.ts +++ b/internal-packages/run-engine/src/engine/tests/dequeueSystem.controlPlaneResolver.test.ts @@ -538,3 +538,142 @@ describe("DequeueSystem controlPlaneResolver (single-DB passthrough)", () => { } ); }); + +heteroPostgresTest( + "resolveWorkerVersion (DEVELOPMENT) resolves the newest worker by createdAt, not by id", + async ({ prisma14 }) => { + const cp = await seedControlPlane( + prisma14 as unknown as PrismaClient, + "cpord", + "ordering-task" + ); + + await prisma14.backgroundWorker.update({ + where: { id: cp.worker.id }, + data: { createdAt: new Date("2026-07-01T12:00:00.000Z") }, + }); + + const newest = await prisma14.backgroundWorker.create({ + data: { + friendlyId: generateFriendlyId("worker"), + contentHash: "hash_newest", + projectId: cp.project.id, + runtimeEnvironmentId: cp.environment.id, + version: "20260731.1", + metadata: {}, + engine: "V2", + createdAt: new Date("2026-07-31T12:00:00.000Z"), + }, + }); + const oldest = await prisma14.backgroundWorker.create({ + data: { + friendlyId: generateFriendlyId("worker"), + contentHash: "hash_oldest", + projectId: cp.project.id, + runtimeEnvironmentId: cp.environment.id, + version: "20260730.1", + metadata: {}, + engine: "V2", + createdAt: new Date("2026-07-30T12:00:00.000Z"), + }, + }); + + expect(oldest.id > newest.id).toBe(true); + expect(oldest.createdAt < newest.createdAt).toBe(true); + + const resolver = new PassthroughControlPlaneResolver({ + prisma: prisma14 as unknown as PrismaClient, + }); + + const resolved = await resolver.resolveWorkerVersion({ + environmentId: cp.environment.id, + type: "DEVELOPMENT", + }); + + assertNonNullable(resolved); + expect(resolved.worker.id).toBe(newest.id); + } +); + +heteroPostgresTest( + "resolveWorkerVersion latest-MANAGED fallback resolves by createdAt, not by id", + async ({ prisma14 }) => { + const cp = await seedControlPlane( + prisma14 as unknown as PrismaClient, + "cpfall", + "fallback-task" + ); + + await prisma14.workerDeployment.update({ + where: { id: cp.deployment.id }, + data: { type: "V1" }, + }); + + const newestWorker = await prisma14.backgroundWorker.create({ + data: { + friendlyId: generateFriendlyId("worker"), + contentHash: "hash_newest", + projectId: cp.project.id, + runtimeEnvironmentId: cp.environment.id, + version: "20260731.1", + metadata: {}, + engine: "V2", + }, + }); + const oldestWorker = await prisma14.backgroundWorker.create({ + data: { + friendlyId: generateFriendlyId("worker"), + contentHash: "hash_oldest", + projectId: cp.project.id, + runtimeEnvironmentId: cp.environment.id, + version: "20260730.1", + metadata: {}, + engine: "V2", + }, + }); + + const newestDeployment = await prisma14.workerDeployment.create({ + data: { + friendlyId: generateFriendlyId("deployment"), + contentHash: "hash_newest", + version: "20260731.1", + shortCode: "short_code_newest", + status: "DEPLOYED", + projectId: cp.project.id, + environmentId: cp.environment.id, + workerId: newestWorker.id, + type: "MANAGED", + createdAt: new Date("2026-07-31T12:00:00.000Z"), + }, + }); + const oldestDeployment = await prisma14.workerDeployment.create({ + data: { + friendlyId: generateFriendlyId("deployment"), + contentHash: "hash_oldest", + version: "20260730.1", + shortCode: "short_code_oldest", + status: "DEPLOYED", + projectId: cp.project.id, + environmentId: cp.environment.id, + workerId: oldestWorker.id, + type: "MANAGED", + createdAt: new Date("2026-07-30T12:00:00.000Z"), + }, + }); + + expect(oldestDeployment.id > newestDeployment.id).toBe(true); + expect(oldestDeployment.createdAt < newestDeployment.createdAt).toBe(true); + + const resolver = new PassthroughControlPlaneResolver({ + prisma: prisma14 as unknown as PrismaClient, + }); + + const resolved = await resolver.resolveWorkerVersion({ + environmentId: cp.environment.id, + type: "PRODUCTION", + }); + + assertNonNullable(resolved); + expect(resolved.deployment?.id).toBe(newestDeployment.id); + } +);