diff --git a/src/selfhost/queue-common.ts b/src/selfhost/queue-common.ts index 63fdd9a33a..3739a57263 100644 --- a/src/selfhost/queue-common.ts +++ b/src/selfhost/queue-common.ts @@ -1,3 +1,5 @@ +import { createHash } from "node:crypto"; + import { retryableJobDelayMs } from "../queue/retryable"; import { LOW_REST_RATE_LIMIT_REMAINING, @@ -650,7 +652,8 @@ function normalizedPathScope(value: unknown): string | null { .map((entry) => entry.trim()), ), ].sort(); - return paths.length > 0 ? JSON.stringify(paths) : null; + if (paths.length === 0) return null; + return `sha256:${createHash("sha256").update(JSON.stringify(paths)).digest("hex")}`; } function boolFlag(value: unknown): string { diff --git a/test/unit/selfhost-queue-common.test.ts b/test/unit/selfhost-queue-common.test.ts index 8623b652cf..d5b267d21d 100644 --- a/test/unit/selfhost-queue-common.test.ts +++ b/test/unit/selfhost-queue-common.test.ts @@ -560,7 +560,7 @@ describe("self-host queue common helpers", () => { paths: ["README.md", "src/a.ts", "README.md"], }), ), - ).toBe('rag-index-repo:jsonbored/gittensory:["README.md","src/a.ts"]'); + ).toBe("rag-index-repo:jsonbored/gittensory:sha256:8812e979fc698c98d98665ad4ccd8630e396dabdce08ebf87b41600c94bb1df5"); expect( jobCoalesceKey( payload({ @@ -570,7 +570,7 @@ describe("self-host queue common helpers", () => { paths: ["a,b", "c"], }), ), - ).toBe('rag-index-repo:jsonbored/gittensory:["a,b","c"]'); + ).toBe("rag-index-repo:jsonbored/gittensory:sha256:569c363fb7e855f85eeae4e4dc032d1f8d262191b68ade7297566486982b5183"); expect( jobCoalesceKey( payload({ @@ -580,7 +580,27 @@ describe("self-host queue common helpers", () => { paths: ["a", "b,c"], }), ), - ).toBe('rag-index-repo:jsonbored/gittensory:["a","b,c"]'); + ).toBe("rag-index-repo:jsonbored/gittensory:sha256:472ecd0a16762a33c3090345032fcadcfe6b34ee43a5cce5385fef8e72169c92"); + const longPathKey = jobCoalesceKey( + payload({ + type: "rag-index-repo", + requestedBy: "webhook", + repoFullName: "JSONbored/Gittensory", + paths: Array.from({ length: 100 }, (_, index) => `src/${index}-${"a".repeat(220)}.ts`), + }), + ); + expect(longPathKey).toMatch(/^rag-index-repo:jsonbored\/gittensory:sha256:[a-f0-9]{64}$/); + expect(longPathKey?.length).toBe("rag-index-repo:jsonbored/gittensory:sha256:".length + 64); + expect( + jobCoalesceKey( + payload({ + type: "rag-index-repo", + requestedBy: "webhook", + repoFullName: "JSONbored/Gittensory", + paths: [" ", null], + }), + ), + ).toBe("rag-index-repo:jsonbored/gittensory:full"); expect(jobCoalesceKey(payload({ type: "prune-retention", requestedBy: "schedule", dryRun: true }))).toBe( "prune-retention:1", ); diff --git a/test/unit/selfhost-sqlite-queue.test.ts b/test/unit/selfhost-sqlite-queue.test.ts index f111b93194..2e06d528b3 100644 --- a/test/unit/selfhost-sqlite-queue.test.ts +++ b/test/unit/selfhost-sqlite-queue.test.ts @@ -892,8 +892,8 @@ describe("createSqliteQueue (durable #980)", () => { "backfill-registered-repos:jsonbored/gittensory:light:1", "generate-weekly-value-report:operator:7", "generate-weekly-value-report:public:7", - 'rag-index-repo:jsonbored/gittensory:["src/a.ts","src/b.ts"]', - 'rag-index-repo:jsonbored/gittensory:["src/c.ts"]', + "rag-index-repo:jsonbored/gittensory:sha256:170cb2cfb288ab59ba4d35b2633120223c9acc6893fd5baec3465c434ad5bedf", + "rag-index-repo:jsonbored/gittensory:sha256:f4f9970f7a842b1b7b619cbd49f05da577a7d725ff1616ba2de8beed1ae5616f", ]); expect(rows.map((row) => JSON.parse(row.payload).requestedBy)).toEqual([ "api",