From c6fed2064490abb2d8d21e94f8268926c09db4e6 Mon Sep 17 00:00:00 2001 From: Alejandro Akbal <37181533+AlejandroAkbal@users.noreply.github.com> Date: Fri, 12 Jun 2026 00:11:24 +0200 Subject: [PATCH 1/3] Add booru outbound proxy rotation --- .env.example | 3 + src/booru/booru.service.spec.ts | 224 +++++++++++++++++++++++++++++++- src/booru/booru.service.ts | 163 +++++++++++++++++++++++ 3 files changed, 388 insertions(+), 2 deletions(-) diff --git a/.env.example b/.env.example index 522860e..213bb12 100644 --- a/.env.example +++ b/.env.example @@ -14,3 +14,6 @@ SENTRY_DSN= GITHUB_TOKEN= BOORU_AUTH_CONFIG='{"gelbooru.com":[{"user":"001","password":"gelbooru-api-key","rateLimit":{"requests":10,"windowSeconds":1}}],"rule34.xxx":[{"user":"002","password":"rule34-api-key","rateLimit":{"requests":60,"windowSeconds":60}}]}' + +# Optional URL-rewriter proxy per provider. The proxy must accept the final upstream URL in targetParam. +BOORU_OUTBOUND_PROXY_CONFIG='{"gelbooru.com":[{"baseUrl":"https://cors-proxy2.rule34.workers.dev/","targetParam":"q"},{"baseUrl":"https://cors-proxy.refinedsoftware00.workers.dev/","targetParam":"q"}]}' diff --git a/src/booru/booru.service.spec.ts b/src/booru/booru.service.spec.ts index 8d9fec9..ae912be 100644 --- a/src/booru/booru.service.spec.ts +++ b/src/booru/booru.service.spec.ts @@ -30,6 +30,31 @@ function getApiAuth(api: unknown): ApiAuth | undefined { return (api as ApiAuthOptions).options?.auth } +function buildPostUrl(api: unknown): URL { + const internalApi = api as { + generateEndpointUrl(endpoint: string): URL + addPostQueries(url: URL, queries: { limit: number; pageID: number; tags: string[] }): URL + } + + return internalApi.addPostQueries(internalApi.generateEndpointUrl('/index.php?page=dapi&s=post&q=index'), { + limit: 1, + pageID: 1, + tags: ['diana'] + }) +} + +function buildTagUrl(api: unknown): URL { + const internalApi = api as { + generateEndpointUrl(endpoint: string): URL + addTagsQueries(url: URL, queries: { tag: string; limit: number }): URL + } + + return internalApi.addTagsQueries(internalApi.generateEndpointUrl('/index.php?page=dapi&s=tag&q=index'), { + tag: 'dian', + limit: 1 + }) +} + describe('BooruService', () => { let service: BooruService let mockAuthManager: MockAuthManager @@ -47,6 +72,8 @@ describe('BooruService', () => { } beforeEach(async () => { + mockConfigService.get.mockReset() + mockAuthManager = { reserveAvailableCredential: jest.fn() as jest.MockedFunction< BooruAuthManagerService['reserveAvailableCredential'] @@ -72,8 +99,6 @@ describe('BooruService', () => { service = module.get(BooruService) - jest.clearAllMocks() - mockAuthManager.getDomainStats.mockReturnValue({ domain: 'gelbooru.com', total: 1, @@ -181,6 +206,201 @@ describe('BooruService', () => { }) }) + describe('Outbound Proxy Resolution', () => { + it('should proxy configured provider URLs after query and auth parameters are applied', () => { + mockConfigService.get.mockImplementation((key: string) => { + if (key === 'BOORU_OUTBOUND_PROXY_CONFIG') { + return JSON.stringify({ + 'gelbooru.com': { + baseUrl: 'https://cors-proxy2.rule34.workers.dev/', + targetParam: 'q' + } + }) + } + + return undefined + }) + + const queries = { + ...baseQueries, + baseEndpoint: 'gelbooru.com', + auth_user: 'managed_1', + auth_pass: 'pass_1' + } as booruQueriesDTO + + const api = service.buildApiClass(mockParams, queries) + const proxiedUrl = buildPostUrl(api) + const upstreamUrl = new URL(proxiedUrl.searchParams.get('q') ?? '') + + expect(proxiedUrl.origin).toBe('https://cors-proxy2.rule34.workers.dev') + expect(upstreamUrl.origin).toBe('https://gelbooru.com') + expect(upstreamUrl.searchParams.get('limit')).toBe('1') + expect(upstreamUrl.searchParams.get('pid')).toBe('1') + expect(upstreamUrl.searchParams.get('tags')).toBe('diana') + expect(upstreamUrl.searchParams.get('user_id')).toBe('managed_1') + expect(upstreamUrl.searchParams.get('api_key')).toBe('pass_1') + }) + + it('should proxy configured provider tag URLs with the same policy', () => { + mockConfigService.get.mockImplementation((key: string) => { + if (key === 'BOORU_OUTBOUND_PROXY_CONFIG') { + return JSON.stringify({ + 'gelbooru.com': { + baseUrl: 'https://r34.app/api/cors-proxy/', + targetParam: 'q' + } + }) + } + + return undefined + }) + + const queries = { + ...baseQueries, + baseEndpoint: 'gelbooru.com', + auth_user: 'managed_1', + auth_pass: 'pass_1' + } as booruQueriesDTO + + const api = service.buildApiClass(mockParams, queries) + const proxiedUrl = buildTagUrl(api) + const upstreamUrl = new URL(proxiedUrl.searchParams.get('q') ?? '') + + expect(proxiedUrl.origin).toBe('https://r34.app') + expect(proxiedUrl.pathname).toBe('/api/cors-proxy/') + expect(upstreamUrl.origin).toBe('https://gelbooru.com') + expect(upstreamUrl.searchParams.get('name_pattern')).toBe('dian%') + expect(upstreamUrl.searchParams.get('limit')).toBe('1') + expect(upstreamUrl.searchParams.get('user_id')).toBe('managed_1') + expect(upstreamUrl.searchParams.get('api_key')).toBe('pass_1') + }) + + it('should leave unconfigured provider URLs direct', () => { + mockConfigService.get.mockImplementation((key: string) => { + if (key === 'BOORU_OUTBOUND_PROXY_CONFIG') { + return JSON.stringify({ + 'rule34.xxx': { + baseUrl: 'https://cors-proxy2.rule34.workers.dev/', + targetParam: 'q' + } + }) + } + + return undefined + }) + + const queries = { + ...baseQueries, + baseEndpoint: 'gelbooru.com', + auth_user: 'managed_1', + auth_pass: 'pass_1' + } as booruQueriesDTO + + const api = service.buildApiClass(mockParams, queries) + const directUrl = buildPostUrl(api) + + expect(directUrl.origin).toBe('https://gelbooru.com') + expect(directUrl.searchParams.get('q')).toBe('index') + expect(directUrl.searchParams.get('user_id')).toBe('managed_1') + expect(directUrl.searchParams.get('api_key')).toBe('pass_1') + }) + + it('should rotate through multiple configured provider proxies', () => { + mockConfigService.get.mockImplementation((key: string) => { + if (key === 'BOORU_OUTBOUND_PROXY_CONFIG') { + return JSON.stringify({ + 'gelbooru.com': [ + { + baseUrl: 'https://cors-proxy2.rule34.workers.dev/', + targetParam: 'q' + }, + { + baseUrl: 'https://cors-proxy.refinedsoftware00.workers.dev/', + targetParam: 'q' + } + ] + }) + } + + return undefined + }) + + const queries = { + ...baseQueries, + baseEndpoint: 'gelbooru.com', + auth_user: 'managed_1', + auth_pass: 'pass_1' + } as booruQueriesDTO + + const firstUrl = buildPostUrl(service.buildApiClass(mockParams, queries)) + const secondUrl = buildPostUrl(service.buildApiClass(mockParams, queries)) + const thirdUrl = buildPostUrl(service.buildApiClass(mockParams, queries)) + + expect(firstUrl.origin).toBe('https://cors-proxy2.rule34.workers.dev') + expect(secondUrl.origin).toBe('https://cors-proxy.refinedsoftware00.workers.dev') + expect(thirdUrl.origin).toBe('https://cors-proxy2.rule34.workers.dev') + }) + + it('should not rotate proxies while building an API that never fetches', () => { + mockConfigService.get.mockImplementation((key: string) => { + if (key === 'BOORU_OUTBOUND_PROXY_CONFIG') { + return JSON.stringify({ + 'gelbooru.com': [ + { + baseUrl: 'https://cors-proxy2.rule34.workers.dev/', + targetParam: 'q' + }, + { + baseUrl: 'https://cors-proxy.refinedsoftware00.workers.dev/', + targetParam: 'q' + } + ] + }) + } + + return undefined + }) + + const queries = { + ...baseQueries, + baseEndpoint: 'gelbooru.com', + auth_user: 'managed_1', + auth_pass: 'pass_1' + } as booruQueriesDTO + + service.buildApiClass(mockParams, queries) + const firstFetchedUrl = buildPostUrl(service.buildApiClass(mockParams, queries)) + const secondFetchedUrl = buildPostUrl(service.buildApiClass(mockParams, queries)) + + expect(firstFetchedUrl.origin).toBe('https://cors-proxy2.rule34.workers.dev') + expect(secondFetchedUrl.origin).toBe('https://cors-proxy.refinedsoftware00.workers.dev') + }) + + it('should reject invalid outbound proxy config shapes', () => { + mockConfigService.get.mockImplementation((key: string) => { + if (key === 'BOORU_OUTBOUND_PROXY_CONFIG') { + return JSON.stringify({ + 'gelbooru.com': { + baseUrl: 'not-a-url', + targetParam: 'q' + } + }) + } + + return undefined + }) + + const queries = { + ...baseQueries, + baseEndpoint: 'gelbooru.com' + } as booruQueriesDTO + + expect(() => service.buildApiClass(mockParams, queries)).toThrow( + 'Invalid BOORU_OUTBOUND_PROXY_CONFIG baseUrl for gelbooru.com' + ) + }) + }) + describe('Managed Strategy Execution', () => { it('should not fallback when explicit auth is provided', async () => { const queries = { diff --git a/src/booru/booru.service.ts b/src/booru/booru.service.ts index 095dbd4..1b2a1ed 100644 --- a/src/booru/booru.service.ts +++ b/src/booru/booru.service.ts @@ -41,6 +41,13 @@ interface BooruQueryIdentifierDefaults { tags?: Partial> } +interface BooruOutboundProxyPolicy { + baseUrl: string + targetParam: string +} + +type NormalizedBooruOutboundProxyConfig = Record + export class ManagedCredentialPoolUnavailableError extends Error { constructor( public readonly domain: string, @@ -55,6 +62,8 @@ export class ManagedCredentialPoolUnavailableError extends Error { @Injectable() export class BooruService { private readonly sensitiveAuthParams = new Set(SENSITIVE_AUTH_PARAMS) + private outboundProxyConfig: NormalizedBooruOutboundProxyConfig | null | undefined + private readonly outboundProxyCursors = new Map() constructor( private readonly configService: ConfigService, @@ -181,6 +190,7 @@ export class BooruService { undefined, options ) + this.applyOutboundProxy(Api, queries.baseEndpoint) return { api: Api, @@ -392,6 +402,159 @@ export class BooruService { return undefined } + private applyOutboundProxy(api: BooruTypes, domain: string): void { + if (!this.hasOutboundProxyPolicy(domain)) { + return + } + + const apiWithInternals = api as unknown as Record + const queryMethodNames = ['addPostQueries', 'addRandomPostQueries', 'addSinglePostQueries', 'addTagsQueries'] + + for (const methodName of queryMethodNames) { + const originalMethod = apiWithInternals[methodName] + + if (typeof originalMethod !== 'function') { + continue + } + + apiWithInternals[methodName] = (...args: unknown[]) => { + const upstreamUrl = originalMethod.apply(api, args) as URL + const proxyPolicy = this.getOutboundProxyPolicy(domain) + + if (proxyPolicy === undefined) { + return upstreamUrl + } + + return this.createProxiedOutboundUrl(upstreamUrl, proxyPolicy) + } + } + } + + private createProxiedOutboundUrl(upstreamUrl: URL, proxyPolicy: BooruOutboundProxyPolicy): URL { + const proxiedUrl = new URL(proxyPolicy.baseUrl) + proxiedUrl.searchParams.set(proxyPolicy.targetParam, upstreamUrl.toString()) + return proxiedUrl + } + + private hasOutboundProxyPolicy(domain: string): boolean { + const config = this.getOutboundProxyConfig() + + if (config === null) { + return false + } + + const policies = config[this.normalizeOutboundProxyDomain(domain)] + return policies !== undefined && policies.length > 0 + } + + private getOutboundProxyPolicy(domain: string): BooruOutboundProxyPolicy | undefined { + const config = this.getOutboundProxyConfig() + + if (config === null) { + return undefined + } + + const normalizedDomain = this.normalizeOutboundProxyDomain(domain) + const policies = config[normalizedDomain] + + if (policies === undefined || policies.length === 0) { + return undefined + } + + const cursor = this.outboundProxyCursors.get(normalizedDomain) ?? 0 + const policy = policies[cursor % policies.length] + this.outboundProxyCursors.set(normalizedDomain, cursor + 1) + + return policy + } + + private getOutboundProxyConfig(): NormalizedBooruOutboundProxyConfig | null { + if (this.outboundProxyConfig !== undefined) { + return this.outboundProxyConfig + } + + const configJson = this.configService.get('BOORU_OUTBOUND_PROXY_CONFIG') + + if (configJson === undefined || configJson.length === 0) { + this.outboundProxyConfig = null + return this.outboundProxyConfig + } + + try { + const parsedConfig = JSON.parse(configJson) as unknown + this.outboundProxyConfig = this.validateOutboundProxyConfig(parsedConfig) + return this.outboundProxyConfig + } catch (error) { + if (error instanceof SyntaxError) { + throw new Error('Failed to parse BOORU_OUTBOUND_PROXY_CONFIG', { cause: error }) + } + + throw error + } + } + + private validateOutboundProxyConfig(config: unknown): NormalizedBooruOutboundProxyConfig { + if (!this.isPlainObject(config)) { + throw new Error('Invalid BOORU_OUTBOUND_PROXY_CONFIG') + } + + const outboundProxyConfig: NormalizedBooruOutboundProxyConfig = {} + + for (const [domain, policyOrPolicies] of Object.entries(config)) { + const policies = Array.isArray(policyOrPolicies) ? policyOrPolicies : [policyOrPolicies] + + if (policies.length === 0) { + throw new Error(`Invalid BOORU_OUTBOUND_PROXY_CONFIG policy for ${domain}`) + } + + outboundProxyConfig[this.normalizeOutboundProxyDomain(domain)] = policies.map((policy) => + this.validateOutboundProxyPolicy(domain, policy) + ) + } + + return outboundProxyConfig + } + + private validateOutboundProxyPolicy(domain: string, policy: unknown): BooruOutboundProxyPolicy { + if (!this.isPlainObject(policy)) { + throw new Error(`Invalid BOORU_OUTBOUND_PROXY_CONFIG policy for ${domain}`) + } + + const baseUrl = policy['baseUrl'] + const targetParam = policy['targetParam'] ?? 'q' + + if (typeof baseUrl !== 'string' || !URL.canParse(baseUrl)) { + throw new Error(`Invalid BOORU_OUTBOUND_PROXY_CONFIG baseUrl for ${domain}`) + } + + const parsedBaseUrl = new URL(baseUrl) + + if (parsedBaseUrl.protocol !== 'https:' && parsedBaseUrl.protocol !== 'http:') { + throw new Error(`Invalid BOORU_OUTBOUND_PROXY_CONFIG baseUrl for ${domain}`) + } + + if (typeof targetParam !== 'string' || targetParam.length === 0) { + throw new Error(`Invalid BOORU_OUTBOUND_PROXY_CONFIG targetParam for ${domain}`) + } + + return { + baseUrl: parsedBaseUrl.toString(), + targetParam + } + } + + private normalizeOutboundProxyDomain(domain: string): string { + if (URL.canParse(domain)) { + return new URL(domain).hostname.toLowerCase() + } + + return domain.toLowerCase() + } + + private isPlainObject(value: unknown): value is Record { + return value !== null && typeof value === 'object' && !Array.isArray(value) + } + private getApiClassByType(booruType: BooruTypesStringEnum) { switch (booruType) { case BooruTypesStringEnum.DANBOORU: From 263397d53fbd1a1150280e69192d2125c053bd45 Mon Sep 17 00:00:00 2001 From: Alejandro Akbal <37181533+AlejandroAkbal@users.noreply.github.com> Date: Fri, 12 Jun 2026 00:21:51 +0200 Subject: [PATCH 2/3] Address outbound proxy review feedback --- src/booru/booru-auth.live.spec.ts | 39 +++++++++++++++++++++++-------- src/cluster.service.ts | 20 +++++++++++++++- 2 files changed, 48 insertions(+), 11 deletions(-) diff --git a/src/booru/booru-auth.live.spec.ts b/src/booru/booru-auth.live.spec.ts index c70c83b..37b3673 100644 --- a/src/booru/booru-auth.live.spec.ts +++ b/src/booru/booru-auth.live.spec.ts @@ -39,6 +39,21 @@ function sanitizeLiveSmokeErrorMessage(message: string): string { return sanitizedMessage } +function createSanitizedLiveSmokeErrorCause(error: unknown): Error { + if (!(error instanceof Error)) { + return new Error(sanitizeLiveSmokeErrorMessage(String(error))) + } + + const cause = new Error(sanitizeLiveSmokeErrorMessage(error.message)) + cause.name = error.name + + if (error.stack !== undefined) { + cause.stack = sanitizeLiveSmokeErrorMessage(error.stack) + } + + return cause +} + describe('live smoke error sanitization', () => { it('redacts auth query params before rethrowing non-quota live errors', () => { const message = @@ -54,6 +69,17 @@ describe('live smoke error sanitization', () => { expect(sanitizedMessage).not.toContain('user_id=123') expect(sanitizedMessage).not.toContain('auth_pass=pass') }) + + it('uses a sanitized cause when rethrowing non-quota live errors', () => { + const originalError = new Error('HTTP 403 https://gelbooru.com/index.php?page=dapi&api_key=secret&user_id=123') + originalError.stack = 'Error: api_key=secret user_id=123' + + const cause = createSanitizedLiveSmokeErrorCause(originalError) + + expect(cause.name).toBe('Error') + expect(cause.message).toBe('HTTP 403 https://gelbooru.com/index.php?page=dapi&api_key=REDACTED&user_id=REDACTED') + expect(cause.stack).toBe('Error: api_key=REDACTED user_id=REDACTED') + }) }) describeLive('authenticated booru live smoke tests', () => { @@ -110,16 +136,9 @@ describeLive('authenticated booru live smoke tests', () => { const message = error instanceof Error ? error.message : String(error) const sanitizedMessage = sanitizeLiveSmokeErrorMessage(message) - - if (error instanceof Error) { - error.message = sanitizedMessage - - if (error.stack !== undefined) { - error.stack = sanitizeLiveSmokeErrorMessage(error.stack) - } - } - - throw new Error(sanitizedMessage, { cause: error }) + // Do not preserve the original cause here; live upstream errors can carry credential-bearing request objects. + // eslint-disable-next-line preserve-caught-error + throw new Error(sanitizedMessage, { cause: createSanitizedLiveSmokeErrorCause(error) }) } expect(posts.length).toBeGreaterThan(0) diff --git a/src/cluster.service.ts b/src/cluster.service.ts index 44acafd..9a07417 100644 --- a/src/cluster.service.ts +++ b/src/cluster.service.ts @@ -67,7 +67,25 @@ export class AppClusterService { if (message.type === 'RESERVE_CREDENTIAL') { const payload = message.payload - const reservation = this.getPrimaryAuthManager().reserveAvailableCredentialLocally(payload.domain) + let reservation: ReturnType + + try { + reservation = this.getPrimaryAuthManager().reserveAvailableCredentialLocally(payload.domain) + } catch (error) { + console.error( + `Failed to reserve credential in primary process for ${payload.domain} request ${payload.requestId}`, + error + ) + + worker.send({ + type: 'RESERVE_CREDENTIAL_RESPONSE', + payload: { + requestId: payload.requestId, + credential: null + } + } satisfies IpcAuthMessage) + return + } worker.send({ type: 'RESERVE_CREDENTIAL_RESPONSE', From b51cd33e37fc5a7c828f2ffc43c6aba213f9f92c Mon Sep 17 00:00:00 2001 From: Alejandro Akbal <37181533+AlejandroAkbal@users.noreply.github.com> Date: Fri, 12 Jun 2026 09:36:12 +0200 Subject: [PATCH 3/3] Address proxy security review feedback --- src/booru/booru.service.spec.ts | 24 ++++++++++++++++++++++++ src/booru/booru.service.ts | 2 +- src/cluster.service.ts | 4 +++- 3 files changed, 28 insertions(+), 2 deletions(-) diff --git a/src/booru/booru.service.spec.ts b/src/booru/booru.service.spec.ts index ae912be..22257df 100644 --- a/src/booru/booru.service.spec.ts +++ b/src/booru/booru.service.spec.ts @@ -399,6 +399,30 @@ describe('BooruService', () => { 'Invalid BOORU_OUTBOUND_PROXY_CONFIG baseUrl for gelbooru.com' ) }) + + it('should reject plaintext outbound proxy URLs', () => { + mockConfigService.get.mockImplementation((key: string) => { + if (key === 'BOORU_OUTBOUND_PROXY_CONFIG') { + return JSON.stringify({ + 'gelbooru.com': { + baseUrl: 'http://cors-proxy.example.test/', + targetParam: 'q' + } + }) + } + + return undefined + }) + + const queries = { + ...baseQueries, + baseEndpoint: 'gelbooru.com' + } as booruQueriesDTO + + expect(() => service.buildApiClass(mockParams, queries)).toThrow( + 'Invalid BOORU_OUTBOUND_PROXY_CONFIG baseUrl for gelbooru.com' + ) + }) }) describe('Managed Strategy Execution', () => { diff --git a/src/booru/booru.service.ts b/src/booru/booru.service.ts index 1b2a1ed..67e035e 100644 --- a/src/booru/booru.service.ts +++ b/src/booru/booru.service.ts @@ -529,7 +529,7 @@ export class BooruService { const parsedBaseUrl = new URL(baseUrl) - if (parsedBaseUrl.protocol !== 'https:' && parsedBaseUrl.protocol !== 'http:') { + if (parsedBaseUrl.protocol !== 'https:') { throw new Error(`Invalid BOORU_OUTBOUND_PROXY_CONFIG baseUrl for ${domain}`) } diff --git a/src/cluster.service.ts b/src/cluster.service.ts index 9a07417..46bfaf2 100644 --- a/src/cluster.service.ts +++ b/src/cluster.service.ts @@ -72,9 +72,11 @@ export class AppClusterService { try { reservation = this.getPrimaryAuthManager().reserveAvailableCredentialLocally(payload.domain) } catch (error) { + const errorName = error instanceof Error ? error.name : 'NonErrorThrown' + console.error( `Failed to reserve credential in primary process for ${payload.domain} request ${payload.requestId}`, - error + { errorName } ) worker.send({