diff --git a/src/booru/constants/sensitive-auth-params.ts b/src/booru/constants/sensitive-auth-params.ts new file mode 100644 index 0000000..e78177e --- /dev/null +++ b/src/booru/constants/sensitive-auth-params.ts @@ -0,0 +1,20 @@ +export const SENSITIVE_AUTH_PARAMS = [ + 'user_id', + 'api_key', + 'password', + 'password_hash', + 'pass_hash', + 'auth_user', + 'auth_pass', + 'token', + 'secret', + 'key', + 'access_token', + 'auth_token', + 'session_id', + 'session', + 'login', + 'username', + 'user', + 'hash' +] as const diff --git a/src/booru/dto/booru-queries.dto.spec.ts b/src/booru/dto/booru-queries.dto.spec.ts index 9722383..8cc6df8 100644 --- a/src/booru/dto/booru-queries.dto.spec.ts +++ b/src/booru/dto/booru-queries.dto.spec.ts @@ -1,20 +1,19 @@ -import { BadRequestException } from '@nestjs/common' import { plainToInstance } from 'class-transformer' import { booruQueryValuesPostsDTO } from './booru-queries.dto' describe('booruQueryValuesPostsDTO', () => { describe('tags transform', () => { - it('should decode URL-encoded ampersands in tags', () => { + it('should normalize a single tag string into an array', () => { const dto = plainToInstance(booruQueryValuesPostsDTO, { - tags: 'panty_%26_stocking_with_garterbelt' + tags: 'panty_&_stocking_with_garterbelt' }) expect(dto.tags).toEqual(['panty_&_stocking_with_garterbelt']) }) - it('should split pipe-separated tags and decode each one', () => { + it('should split pipe-separated tags', () => { const dto = plainToInstance(booruQueryValuesPostsDTO, { - tags: 'panty_%26_stocking_with_garterbelt|rating%3Asafe' + tags: 'panty_&_stocking_with_garterbelt|rating:safe' }) expect(dto.tags).toEqual(['panty_&_stocking_with_garterbelt', 'rating:safe']) @@ -22,14 +21,10 @@ describe('booruQueryValuesPostsDTO', () => { it('should normalize array tag inputs and keep tag array shape', () => { const dto = plainToInstance(booruQueryValuesPostsDTO, { - tags: ['panty_%26_stocking_with_garterbelt|rating%3Asafe', 'score%3A%3E100'] + tags: ['panty_&_stocking_with_garterbelt|rating:safe', 'score:>100'] }) - expect(dto.tags).toEqual([ - 'panty_&_stocking_with_garterbelt', - 'rating:safe', - 'score:>100' - ]) + expect(dto.tags).toEqual(['panty_&_stocking_with_garterbelt', 'rating:safe', 'score:>100']) }) it('should normalize non-string tag input without throwing', () => { @@ -48,18 +43,12 @@ describe('booruQueryValuesPostsDTO', () => { expect(dto.tags).toEqual(['100%_real']) }) - it('should throw BadRequestException when encoded tag decoding fails', () => { - expect(() => - plainToInstance(booruQueryValuesPostsDTO, { - tags: 'bad%25%' - }) - ).toThrow(BadRequestException) - - expect(() => - plainToInstance(booruQueryValuesPostsDTO, { - tags: 'bad%25%' - }) - ).toThrow('Invalid tag encoding') + it('should keep malformed percent tag values unchanged', () => { + const dto = plainToInstance(booruQueryValuesPostsDTO, { + tags: 'bad%%' + }) + + expect(dto.tags).toEqual(['bad%%']) }) it('should return undefined when tags is undefined', () => { @@ -76,4 +65,4 @@ describe('booruQueryValuesPostsDTO', () => { expect(dto.tags).toBeNull() }) }) -}) \ No newline at end of file +}) diff --git a/src/booru/dto/booru-queries.dto.ts b/src/booru/dto/booru-queries.dto.ts index 3acc88e..4eaf77b 100644 --- a/src/booru/dto/booru-queries.dto.ts +++ b/src/booru/dto/booru-queries.dto.ts @@ -18,7 +18,6 @@ import { Min } from 'class-validator' import { Transform } from 'class-transformer' -import { BadRequestException } from '@nestjs/common' abstract class booruEndpointsDTO { @IsFQDN() @@ -189,17 +188,6 @@ export class booruQueryValuesPostsDTO extends booruQueriesDTO { return (Array.isArray(value) ? value : [value]) .map((tag) => (typeof tag === 'string' ? tag : String(tag))) .flatMap((tag) => tag.trim().split('|')) - .map((tag) => { - if (!/%[0-9A-Fa-f]{2}/.test(tag)) { - return tag - } - - try { - return decodeURIComponent(tag) - } catch { - throw new BadRequestException('Invalid tag encoding') - } - }) }) @IsOptional() readonly tags: IBooruQueryValues['posts']['tags'] diff --git a/src/booru/interceptors/booru-exception.interceptor.spec.ts b/src/booru/interceptors/booru-exception.interceptor.spec.ts index ea05b62..850b0ce 100644 --- a/src/booru/interceptors/booru-exception.interceptor.spec.ts +++ b/src/booru/interceptors/booru-exception.interceptor.spec.ts @@ -1,234 +1,133 @@ import { Test, TestingModule } from '@nestjs/testing' -import { CallHandler, ExecutionContext } from '@nestjs/common' -import { throwError } from 'rxjs' +import { ConfigModule } from '@nestjs/config' +import { Controller, Get, UseInterceptors } from '@nestjs/common' +import { FastifyAdapter, NestFastifyApplication } from '@nestjs/platform-fastify' +import request from 'supertest' +import { EmptyDataError, HttpError } from '@alejandroakbal/universal-booru-wrapper' import { BooruErrorsInterceptor } from './booru-exception.interceptor' -import { EmptyDataError, EndpointError, HttpError } from '@alejandroakbal/universal-booru-wrapper' -import { NoContentException } from '../../common/exceptions/no-content.exception' import { BooruAuthManagerService } from '../services/booru-auth-manager.service' +@Controller('test-booru-errors') +@UseInterceptors(BooruErrorsInterceptor) +class TestBooruErrorsController { + @Get('empty') + getEmpty() { + throw new EmptyDataError( + 'Request failed for https://gelbooru.com/index.php?page=dapi&user_id=12345&api_key=secret123&limit=10' + ) + } + + @Get('auth-failure') + getAuthFailure() { + const error = new HttpError( + 'Forbidden for https://www.gelbooru.com/index.php?page=dapi&auth_user=www-gel-user&auth_pass=secret123' + ) + + ;(error as any).statusCode = 403 + + throw error + } + + @Get('malformed-url') + getMalformedUrl() { + throw new EmptyDataError( + 'Request failed for https://%zz?page=dapi&auth_user=www-gel-user&auth_pass=secret123&limit=10' + ) + } +} + describe('BooruErrorsInterceptor', () => { - let interceptor: BooruErrorsInterceptor - let mockExecutionContext: ExecutionContext - let mockCallHandler: CallHandler + let app: NestFastifyApplication + let authManager: BooruAuthManagerService + + const originalAuthConfig = process.env.BOORU_AUTH_CONFIG beforeEach(async () => { - const mockAuthManager = { - reportAuthFailure: jest.fn() - } + process.env.BOORU_AUTH_CONFIG = JSON.stringify({ + 'www.gelbooru.com': [{ user: 'www-gel-user', password: 'www-gel-pass' }] + }) const module: TestingModule = await Test.createTestingModule({ - providers: [ - BooruErrorsInterceptor, - { - provide: BooruAuthManagerService, - useValue: mockAuthManager - } - ] + imports: [ConfigModule.forRoot({ isGlobal: true, cache: false, ignoreEnvFile: true })], + controllers: [TestBooruErrorsController], + providers: [BooruErrorsInterceptor, BooruAuthManagerService] }).compile() - interceptor = module.get(BooruErrorsInterceptor) - mockExecutionContext = {} as ExecutionContext - mockCallHandler = { - handle: jest.fn() - } as CallHandler + app = module.createNestApplication(new FastifyAdapter()) + await app.init() + await app.getHttpAdapter().getInstance().ready() + + authManager = app.get(BooruAuthManagerService) }) - // Helper function to test URL sanitization - const testUrlSanitization = ( - errorMessage: string, - expectedRedactedParams: string[], - forbiddenValues: string[], - preservedParams: string[] = [] - ): Promise => { - return new Promise((resolve, reject) => { - const originalError = new Error(errorMessage) - mockCallHandler.handle = jest.fn().mockReturnValue(throwError(() => originalError)) - - interceptor.intercept(mockExecutionContext, mockCallHandler).subscribe({ - error: (error) => { - try { - // Check that sensitive params are redacted - expectedRedactedParams.forEach((param) => { - expect(error.message).toContain(`${param}=REDACTED`) - }) - - // Check that forbidden values are not present - forbiddenValues.forEach((value) => { - expect(error.message).not.toContain(value) - }) - - // Check that preserved params remain unchanged - preservedParams.forEach((param) => { - expect(error.message).toContain(param) - }) - - resolve() - } catch (err) { - reject(err) - } - } - }) - }) - } + afterEach(async () => { + await app.close() + }) - describe('Error Type Handling', () => { - const errorTypeTests = [ - { - name: 'EmptyDataError to NoContentException', - errorClass: EmptyDataError, - expectedClass: NoContentException, - url: 'https://gelbooru.com/index.php?page=dapi&user_id=12345&api_key=secret123' - }, - { - name: 'HttpError to ServiceUnavailableException', - errorClass: HttpError, - expectedClass: Error, // ServiceUnavailableException extends Error - url: 'https://gelbooru.com/index.php?user_id=98765&api_key=topsecret&tags=test' - }, - { - name: 'EndpointError to MethodNotAllowedException', - errorClass: EndpointError, - expectedClass: Error, // MethodNotAllowedException extends Error - url: 'https://danbooru.donmai.us/posts.json?auth_user=testuser&auth_pass=password123' - } - ] - - errorTypeTests.forEach(({ name, errorClass, expectedClass, url }) => { - it(`should convert ${name} with sanitized message`, async () => { - const originalError = new errorClass(`Request failed for ${url}`) - mockCallHandler.handle = jest.fn().mockReturnValue(throwError(() => originalError)) - - return new Promise((resolve, reject) => { - interceptor.intercept(mockExecutionContext, mockCallHandler).subscribe({ - error: (error) => { - try { - expect(error).toBeInstanceOf(expectedClass) - expect(error.message).toContain('=REDACTED') - resolve() - } catch (err) { - reject(err) - } - } - }) - }) - }) - }) + afterAll(() => { + if (originalAuthConfig === undefined) { + delete process.env.BOORU_AUTH_CONFIG + return + } - it('should sanitize unknown error types', async () => { - await testUrlSanitization( - 'Custom error with URL: https://example.com/api?token=abc123&secret=xyz789', - ['token', 'secret'], - ['abc123', 'xyz789'] - ) - }) + process.env.BOORU_AUTH_CONFIG = originalAuthConfig }) - describe('URL Sanitization', () => { - it('should sanitize multiple sensitive parameters in a single URL', async () => { - await testUrlSanitization( - 'Failed: https://site.com/api?user_id=123&api_key=secret&password=password123&limit=10', - ['user_id', 'api_key', 'password'], - ['123', 'secret', 'password123'], - ['limit=10'] - ) - }) + it('should sanitize EmptyDataError responses from a real request', async () => { + const response = await request(app.getHttpServer()).get('/test-booru-errors/empty') + const body = JSON.stringify(response.body) - it('should sanitize multiple URLs in a single error message', async () => { - await testUrlSanitization( - 'Failed to connect to https://site1.com/api?user_id=111&api_key=key1 and https://site2.com/posts?auth_user=user2&auth_pass=pass2', - ['user_id', 'api_key', 'auth_user', 'auth_pass'], - ['111', 'key1', 'user2', 'pass2'] - ) - }) + expect(response.status).toBe(404) + expect(body).toContain('user_id=REDACTED') + expect(body).toContain('api_key=REDACTED') + expect(body).toContain('limit=10') + expect(body).not.toContain('12345') + expect(body).not.toContain('secret123') + }) - it('should handle URLs with case-insensitive parameter matching', async () => { - await testUrlSanitization( - 'Error with https://api.com/data?USER_ID=123&Api_Key=secret&AUTH_USER=test', - ['USER_ID', 'Api_Key', 'AUTH_USER'], - ['123', 'secret', 'test'] - ) + it('should report auth failures with preserved www subdomains from a real request', async () => { + const response = await request(app.getHttpServer()).get('/test-booru-errors/auth-failure').query({ + baseEndpoint: 'https://www.gelbooru.com/index.php?page=dapi', + auth_user: 'www-gel-user' }) - it('should preserve non-sensitive parameters', async () => { - await testUrlSanitization( - 'Request failed: https://booru.com/posts?limit=50&tags=safe&user_id=123&page=2&api_key=secret', - ['user_id', 'api_key'], - ['123', 'secret'], - ['limit=50', 'tags=safe', 'page=2'] - ) - }) + const disabledCredentials = authManager.getDisabledCredentials() + const body = JSON.stringify(response.body) - it('should leave malformed URLs unchanged', async () => { - const malformedUrl = 'not-a-valid-url-but-contains-user_id=123&api_key=secret' - await testUrlSanitization( - `Error with ${malformedUrl}`, - [], // No params should be redacted since it's not a valid URL - [], - [malformedUrl] // Should preserve the malformed URL as-is + expect(response.status).toBe(401) + expect( + disabledCredentials.some( + (credential) => credential.domain === 'www.gelbooru.com' && credential.user === 'www-gel-user' ) - }) + ).toBe(true) + expect(body).toContain('auth_user=REDACTED') + expect(body).toContain('auth_pass=REDACTED') + expect(body).not.toContain('www-gel-user') + expect(body).not.toContain('secret123') + }) - it('should handle errors with empty/default messages gracefully', async () => { - const testCases = [ - { name: 'explicit empty string', error: new Error('') }, - { name: 'default Error constructor', error: new Error() } - ] - - for (const { name, error: originalError } of testCases) { - mockCallHandler.handle = jest.fn().mockReturnValue(throwError(() => originalError)) - - await new Promise((resolve, reject) => { - interceptor.intercept(mockExecutionContext, mockCallHandler).subscribe({ - error: (error) => { - try { - expect(error.message).toBe('') - resolve() - } catch (err) { - reject(new Error(`Failed for ${name}: ${err.message}`)) - } - } - }) - }) - } + it('should report auth failures when baseEndpoint protocol casing is uppercase', async () => { + const response = await request(app.getHttpServer()).get('/test-booru-errors/auth-failure').query({ + baseEndpoint: 'HTTPS://WWW.GELBOORU.COM/index.php?page=dapi', + auth_user: 'www-gel-user' }) - it('should sanitize stack traces containing sensitive URLs', async () => { - const sensitiveUrl = 'https://api.com/endpoint?user_id=123&api_key=secret' - const originalError = new Error('Test error') - originalError.stack = `Error: Test error - at someFunction (${sensitiveUrl}:10:5) - at anotherFunction (file.js:20:10)` - - mockCallHandler.handle = jest.fn().mockReturnValue(throwError(() => originalError)) - - return new Promise((resolve, reject) => { - interceptor.intercept(mockExecutionContext, mockCallHandler).subscribe({ - error: (error) => { - try { - expect(error.stack).toContain('user_id=REDACTED') - expect(error.stack).toContain('api_key=REDACTED') - expect(error.stack).not.toContain('123') - expect(error.stack).not.toContain('secret') - resolve() - } catch (err) { - reject(err) - } - } - }) - }) - }) + const disabledCredentials = authManager.getDisabledCredentials() + + expect(response.status).toBe(401) + expect( + disabledCredentials.some( + (credential) => credential.domain === 'www.gelbooru.com' && credential.user === 'www-gel-user' + ) + ).toBe(true) }) - describe('Sensitive Parameter Detection', () => { - it('should detect all configured sensitive parameters', async () => { - const sensitiveParams = ['user_id', 'api_key', 'password', 'auth_user', 'auth_pass', 'token', 'secret', 'key'] - const paramString = sensitiveParams.map((param, index) => `${param}=${index + 1}`).join('&') + it('should not throw when sanitizing malformed URLs in error messages', async () => { + const response = await request(app.getHttpServer()).get('/test-booru-errors/malformed-url') + const body = JSON.stringify(response.body) - await testUrlSanitization( - `All params: https://api.com/test?${paramString}`, - sensitiveParams, - Array.from({ length: 8 }, (_, i) => `=${i + 1}`) // ['=1', '=2', '=3', ...] - ) - }) + expect(response.status).toBe(404) + expect(body).toContain('https://%zz?page=dapi&auth_user=REDACTED&auth_pass=REDACTED&limit=10') + expect(body).not.toContain('secret123') }) }) diff --git a/src/booru/interceptors/booru-exception.interceptor.ts b/src/booru/interceptors/booru-exception.interceptor.ts index e6e7163..dfa40fb 100644 --- a/src/booru/interceptors/booru-exception.interceptor.ts +++ b/src/booru/interceptors/booru-exception.interceptor.ts @@ -13,32 +13,14 @@ import { EmptyDataError, EndpointError, HttpError } from '@alejandroakbal/univer import { NoContentException } from '../../common/exceptions/no-content.exception' import { BooruAuthManagerService } from '../services/booru-auth-manager.service' import { AuthFailureEvent } from '../interfaces/auth-manager.interface' +import { SENSITIVE_AUTH_PARAMS } from '../constants/sensitive-auth-params' @Injectable() export class BooruErrorsInterceptor implements NestInterceptor { constructor(private readonly authManager: BooruAuthManagerService) {} // Common booru authentication parameters that should be redacted from error messages - private readonly sensitiveParams = [ - 'user_id', - 'api_key', - 'password', - 'password_hash', - 'pass_hash', - 'auth_user', - 'auth_pass', - 'token', - 'secret', - 'key', - 'access_token', - 'auth_token', - 'session_id', - 'session', - 'login', - 'username', - 'user', - 'hash' - ] + private readonly sensitiveParams = SENSITIVE_AUTH_PARAMS intercept(context: ExecutionContext, next: CallHandler): Observable { return next.handle().pipe( @@ -65,7 +47,7 @@ export class BooruErrorsInterceptor implements NestInterceptor { } return throwError(() => new ServiceUnavailableException(undefined, sanitizedMessage)) - default: + default: { // For unknown errors, also sanitize the message const sanitizedError = new Error(sanitizedMessage) @@ -76,6 +58,7 @@ export class BooruErrorsInterceptor implements NestInterceptor { } return throwError(() => sanitizedError) + } } }) ) @@ -89,7 +72,7 @@ export class BooruErrorsInterceptor implements NestInterceptor { return message } - const urlPattern = /https?:\/\/[^\s]+/g + const urlPattern = /https?:\/\/[^\s]+/gi return message.replace(urlPattern, (url) => this.sanitizeUrl(url)) } @@ -98,16 +81,32 @@ export class BooruErrorsInterceptor implements NestInterceptor { * Sanitizes a single URL by removing sensitive query parameters using native URL API */ private sanitizeUrl(url: string): string { - const urlObj = new URL(url) + try { + const urlObj = new URL(url) - // Check each query parameter and redact sensitive ones - for (const [key] of urlObj.searchParams.entries()) { - if (this.sensitiveParams.some((param) => param.toLowerCase() === key.toLowerCase())) { - urlObj.searchParams.set(key, 'REDACTED') + // Check each query parameter and redact sensitive ones + for (const [key] of urlObj.searchParams.entries()) { + if (this.sensitiveParams.some((param) => param.toLowerCase() === key.toLowerCase())) { + urlObj.searchParams.set(key, 'REDACTED') + } } + + return urlObj.toString() + } catch (error) { + return this.sanitizeRawUrl(url) + } + } + + private sanitizeRawUrl(url: string): string { + let sanitizedUrl = url + + for (const key of this.sensitiveParams) { + const escapedKey = key.replace(/[.*+?^${}()|[\]\\]/g, '\\$&') + const pattern = new RegExp(`([?&]${escapedKey}=)[^&#\\s]*`, 'gi') + sanitizedUrl = sanitizedUrl.replace(pattern, '$1REDACTED') } - return urlObj.toString() + return sanitizedUrl } private checkForAuthFailure(error: any, context: ExecutionContext): void { @@ -118,6 +117,7 @@ export class BooruErrorsInterceptor implements NestInterceptor { const request = context.switchToHttp().getRequest() const baseEndpoint = request.query?.baseEndpoint || request.body?.baseEndpoint const authUser = request.query?.auth_user || request.body?.auth_user + const authPass = request.query?.auth_pass || request.body?.auth_pass if (!baseEndpoint || !authUser) { return @@ -127,6 +127,7 @@ export class BooruErrorsInterceptor implements NestInterceptor { const authFailure: AuthFailureEvent = { domain, user: authUser, + password: authPass, error: this.getAuthErrorMessage(error), timestamp: new Date() } @@ -172,11 +173,16 @@ export class BooruErrorsInterceptor implements NestInterceptor { private extractDomainFromUrl(url: string): string { try { - const normalizedUrl = url.startsWith('http') ? url : `https://${url}` + const hasProtocol = /^https?:\/\//i.test(url) + const normalizedUrl = hasProtocol ? url : `https://${url}` const urlObj = new URL(normalizedUrl) - return urlObj.hostname.replace(/^www\./, '') + return urlObj.hostname.toLowerCase() } catch (error) { - return url.replace(/^(https?:\/\/)?(www\.)?/, '').split('/')[0] + return url + .replace(/^(https?:\/\/)?/i, '') + .split(/[?#]/)[0] + .split('/')[0] + .toLowerCase() } } } diff --git a/src/booru/interfaces/auth-manager.interface.ts b/src/booru/interfaces/auth-manager.interface.ts index 59b62f1..8b20c94 100644 --- a/src/booru/interfaces/auth-manager.interface.ts +++ b/src/booru/interfaces/auth-manager.interface.ts @@ -10,6 +10,7 @@ export interface BooruAuthConfig { export interface DisabledCredential { domain: string user: string + password?: string disabledAt: Date reason?: string } @@ -24,6 +25,7 @@ export interface AuthCredentialStats { export interface AuthFailureEvent { domain: string user: string + password?: string error: string timestamp: Date } diff --git a/src/booru/services/booru-auth-manager.service.spec.ts b/src/booru/services/booru-auth-manager.service.spec.ts new file mode 100644 index 0000000..671ca22 --- /dev/null +++ b/src/booru/services/booru-auth-manager.service.spec.ts @@ -0,0 +1,249 @@ +import { Test, TestingModule } from '@nestjs/testing' +import { ConfigModule } from '@nestjs/config' +import { BooruAuthManagerService } from './booru-auth-manager.service' + +describe('BooruAuthManagerService', () => { + let service: BooruAuthManagerService + + const originalAuthConfig = process.env.BOORU_AUTH_CONFIG + + beforeEach(async () => { + process.env.BOORU_AUTH_CONFIG = JSON.stringify({ + 'rule34.xxx': [{ user: 'canonical-user', password: 'canonical-pass' }], + 'api.rule34.xxx': [ + { user: 'canonical-user', password: 'canonical-pass' }, + { user: 'api-user', password: 'api-pass' } + ], + 'gelbooru.com': [{ user: 'gel-user', password: 'gel-pass' }], + 'www.gelbooru.com': [{ user: 'www-gel-user', password: 'www-gel-pass' }], + 'same-user.test': [ + { user: 'shared-user', password: 'first-pass' }, + { user: 'shared-user', password: 'second-pass' } + ], + 'colon-user.test': [ + { user: 'name:one', password: 'pass' }, + { user: 'name', password: 'one:pass' } + ] + }) + + const module: TestingModule = await Test.createTestingModule({ + imports: [ConfigModule.forRoot({ isGlobal: true, cache: false, ignoreEnvFile: true })], + providers: [BooruAuthManagerService] + }).compile() + + service = module.get(BooruAuthManagerService) + service.onModuleInit() + }) + + afterEach(() => { + if (originalAuthConfig === undefined) { + delete process.env.BOORU_AUTH_CONFIG + return + } + + process.env.BOORU_AUTH_CONFIG = originalAuthConfig + }) + + it('should normalize rule34 aliases into canonical deduplicated domain config', () => { + const stats = service.getCredentialStats() + const rule34Stats = stats.find((stat) => stat.domain === 'rule34.xxx') + + expect(rule34Stats).toEqual({ + domain: 'rule34.xxx', + total: 2, + available: 2, + disabled: 0 + }) + }) + + it('should keep non-aliased www domains separate from root domains', () => { + const rootCredential = service.getAvailableCredential('https://gelbooru.com/index.php?page=dapi') + const wwwCredential = service.getAvailableCredential('https://www.gelbooru.com/index.php?page=dapi') + + expect(rootCredential).toEqual({ user: 'gel-user', password: 'gel-pass' }) + expect(wwwCredential).toEqual({ user: 'www-gel-user', password: 'www-gel-pass' }) + }) + + it('should resolve credentials for api.rule34.xxx using rule34.xxx auth pool', () => { + const credential = service.getAvailableCredential('https://api.rule34.xxx/index.php?page=dapi') + + expect(credential).not.toBeNull() + expect(['canonical-user', 'api-user']).toContain(credential!.user) + }) + + it('should resolve credentials when base endpoint uses uppercase protocol', () => { + const credential = service.getAvailableCredential('HTTPS://API.RULE34.XXX/index.php?page=dapi') + + expect(credential).not.toBeNull() + expect(['canonical-user', 'api-user']).toContain(credential!.user) + }) + + it('should normalize reported auth failures to canonical rule34 domain', () => { + const selectedCredential = service.getAvailableCredential('https://rule34.xxx/index.php?page=dapi') + + expect(selectedCredential).not.toBeNull() + + service.reportAuthFailure({ + domain: 'https://api.rule34.xxx/index.php?page=dapi', + user: selectedCredential!.user, + error: 'HTTP 403', + timestamp: new Date() + }) + + const disabledCredentials = service.getDisabledCredentials() + + expect( + disabledCredentials.some((cred) => cred.domain === 'rule34.xxx' && cred.user === selectedCredential!.user) + ).toBe(true) + }) + + it('should redact sensitive auth params in auth failure logs', () => { + const errorSpy = jest.spyOn(console, 'error').mockImplementation(() => undefined) + const warnSpy = jest.spyOn(console, 'warn').mockImplementation(() => undefined) + + service.reportAuthFailure({ + domain: 'https://www.gelbooru.com/index.php?page=dapi', + user: 'www-gel-user', + password: 'www-gel-pass', + error: + 'HTTP 403: Forbidden for https://www.gelbooru.com/index.php?page=dapi&auth_user=www-gel-user&auth_pass=secret123', + timestamp: new Date() + }) + + const loggedMessage = errorSpy.mock.calls[0][0] + + expect(loggedMessage).toContain('auth_user=REDACTED') + expect(loggedMessage).toContain('auth_pass=REDACTED') + expect(loggedMessage).not.toContain('auth_pass=secret123') + expect(loggedMessage).not.toContain('www-gel-user') + + errorSpy.mockRestore() + warnSpy.mockRestore() + }) + + it('should redact sensitive key=value pairs outside of URLs in auth failure logs', () => { + const errorSpy = jest.spyOn(console, 'error').mockImplementation(() => undefined) + const warnSpy = jest.spyOn(console, 'warn').mockImplementation(() => undefined) + + service.reportAuthFailure({ + domain: 'https://www.gelbooru.com/index.php?page=dapi', + user: 'www-gel-user', + password: 'www-gel-pass', + error: + 'HTTP 403: Forbidden auth_user=www-gel-user auth_pass=secret123 token=abc123 api_key=xyz789 user_id=42 key=plain-key limit=10', + timestamp: new Date() + }) + + const loggedMessage = errorSpy.mock.calls[0][0] + + expect(loggedMessage).toContain('auth_user=REDACTED') + expect(loggedMessage).toContain('auth_pass=REDACTED') + expect(loggedMessage).toContain('token=REDACTED') + expect(loggedMessage).toContain('api_key=REDACTED') + expect(loggedMessage).toContain('user_id=REDACTED') + expect(loggedMessage).toContain('key=REDACTED') + expect(loggedMessage).toContain('limit=10') + expect(loggedMessage).not.toContain('www-gel-user') + expect(loggedMessage).not.toContain('secret123') + expect(loggedMessage).not.toContain('abc123') + expect(loggedMessage).not.toContain('xyz789') + expect(loggedMessage).not.toContain('plain-key') + + errorSpy.mockRestore() + warnSpy.mockRestore() + }) + + it('should redact malformed uppercase-protocol URLs in auth failure logs', () => { + const errorSpy = jest.spyOn(console, 'error').mockImplementation(() => undefined) + const warnSpy = jest.spyOn(console, 'warn').mockImplementation(() => undefined) + + service.reportAuthFailure({ + domain: 'https://www.gelbooru.com/index.php?page=dapi', + user: 'www-gel-user', + password: 'www-gel-pass', + error: 'HTTP 403: Forbidden for HTTPS://%ZZ?page=dapi&AUTH_USER=www-gel-user&AUTH_PASS=secret123&limit=10', + timestamp: new Date() + }) + + const loggedMessage = errorSpy.mock.calls[0][0] + + expect(loggedMessage).toContain('AUTH_USER=REDACTED') + expect(loggedMessage).toContain('AUTH_PASS=REDACTED') + expect(loggedMessage).not.toContain('AUTH_PASS=secret123') + + errorSpy.mockRestore() + warnSpy.mockRestore() + }) + + it('should disable only matching same-user credential when password is provided', () => { + service.reportAuthFailure({ + domain: 'same-user.test', + user: 'shared-user', + password: 'first-pass', + error: 'HTTP 403', + timestamp: new Date() + }) + + const stats = service.getCredentialStats() + const sameUserStats = stats.find((stat) => stat.domain === 'same-user.test') + + expect(sameUserStats).toEqual({ + domain: 'same-user.test', + total: 2, + available: 1, + disabled: 1 + }) + }) + + it('should disable all same-user credentials when password is missing', () => { + service.reportAuthFailure({ + domain: 'same-user.test', + user: 'shared-user', + error: 'HTTP 403', + timestamp: new Date() + }) + + const stats = service.getCredentialStats() + const sameUserStats = stats.find((stat) => stat.domain === 'same-user.test') + + expect(sameUserStats).toEqual({ + domain: 'same-user.test', + total: 2, + available: 0, + disabled: 2 + }) + }) + + it('should not collapse distinct credentials when user/password contain colons', () => { + const stats = service.getCredentialStats() + const colonStats = stats.find((stat) => stat.domain === 'colon-user.test') + + expect(colonStats).toEqual({ + domain: 'colon-user.test', + total: 2, + available: 2, + disabled: 0 + }) + }) + + it('should parse disabled credentials when domain contains a colon', () => { + service.reportAuthFailure({ + domain: 'invalid-domain:test', + user: 'domain-user', + password: 'domain-pass', + error: 'HTTP 403', + timestamp: new Date() + }) + + const disabledCredentials = service.getDisabledCredentials() + + expect( + disabledCredentials.some( + (credential) => + credential.domain === 'invalid-domain:test' && + credential.user === 'domain-user' && + credential.password === 'domain-pass' + ) + ).toBe(true) + }) +}) diff --git a/src/booru/services/booru-auth-manager.service.ts b/src/booru/services/booru-auth-manager.service.ts index a7076af..1c1e72d 100644 --- a/src/booru/services/booru-auth-manager.service.ts +++ b/src/booru/services/booru-auth-manager.service.ts @@ -9,11 +9,18 @@ import { AuthFailureEvent, IpcAuthMessage } from '../interfaces/auth-manager.interface' +import { SENSITIVE_AUTH_PARAMS } from '../constants/sensitive-auth-params' +import { createCredentialKey, parseCredentialKey } from './credential-key.util' @Injectable() export class BooruAuthManagerService implements OnModuleInit { private disabledCredentials = new Set() private authConfig: BooruAuthConfig = {} + private readonly domainAliases: Record = { + 'www.rule34.xxx': 'rule34.xxx', + 'api.rule34.xxx': 'rule34.xxx' + } + private readonly sensitiveParams = new Set(SENSITIVE_AUTH_PARAMS) constructor(private readonly configService: ConfigService) {} @@ -31,7 +38,8 @@ export class BooruAuthManagerService implements OnModuleInit { } try { - this.authConfig = JSON.parse(authConfigJson) + const parsedAuthConfig = JSON.parse(authConfigJson) as BooruAuthConfig + this.authConfig = this.normalizeAuthConfig(parsedAuthConfig) const stats = this.getCredentialStats() console.log( '🔐 Auth manager initialized with credentials for:', @@ -54,7 +62,7 @@ export class BooruAuthManagerService implements OnModuleInit { } public getAvailableCredential(domain: string): BooruAuthCredential | null { - const normalizedDomain = this.extractDomainFromUrl(domain) + const normalizedDomain = this.normalizeDomain(domain) const credentialsArray = this.authConfig[normalizedDomain] if (!credentialsArray || credentialsArray.length === 0) { @@ -62,7 +70,7 @@ export class BooruAuthManagerService implements OnModuleInit { } const availableCredentials = credentialsArray.filter( - (credential) => !this.isCredentialDisabled(normalizedDomain, credential.user) + (credential) => !this.isCredentialDisabled(normalizedDomain, credential.user, credential.password) ) if (availableCredentials.length === 0) { @@ -83,31 +91,35 @@ export class BooruAuthManagerService implements OnModuleInit { } public reportAuthFailure(authFailure: AuthFailureEvent): void { - const credentialKey = `${authFailure.domain}:${authFailure.user}` + const normalizedDomain = this.normalizeDomain(authFailure.domain) + const sanitizedError = this.sanitizeErrorMessage(authFailure.error) + const sanitizedUser = this.sanitizeUserIdentifier(authFailure.user) - if (this.disabledCredentials.has(credentialKey)) { + if (this.isCredentialDisabled(normalizedDomain, authFailure.user, authFailure.password)) { return } const disabledCredential: DisabledCredential = { - domain: authFailure.domain, + domain: normalizedDomain, user: authFailure.user, + password: authFailure.password, disabledAt: authFailure.timestamp, - reason: authFailure.error + reason: sanitizedError } this.disableCredentialLocally(disabledCredential) this.broadcastDisabledCredential(disabledCredential) - const stats = this.getDomainStats(authFailure.domain) - console.error(`❌ Auth failure for ${authFailure.domain}:${authFailure.user} - ${authFailure.error}`) + const stats = this.getDomainStats(normalizedDomain) + console.error(`❌ Auth failure for ${normalizedDomain}:${sanitizedUser} - ${sanitizedError}`) console.warn( - `📊 ${authFailure.domain} credentials: ${stats.available}/${stats.total} available, ${stats.disabled} disabled` + `📊 ${normalizedDomain} credentials: ${stats.available}/${stats.total} available, ${stats.disabled} disabled` ) } private disableCredentialLocally(credential: DisabledCredential): void { - const credentialKey = `${credential.domain}:${credential.user}` + const normalizedDomain = this.normalizeDomain(credential.domain) + const credentialKey = createCredentialKey(normalizedDomain, credential.user, credential.password) this.disabledCredentials.add(credentialKey) } @@ -121,9 +133,20 @@ export class BooruAuthManagerService implements OnModuleInit { } } - private isCredentialDisabled(domain: string, user: string): boolean { - const credentialKey = `${domain}:${user}` - return this.disabledCredentials.has(credentialKey) + private isCredentialDisabled(domain: string, user: string, password?: string): boolean { + const normalizedDomain = this.normalizeDomain(domain) + const userScopedCredentialKey = createCredentialKey(normalizedDomain, user) + + if (this.disabledCredentials.has(userScopedCredentialKey)) { + return true + } + + if (password === undefined) { + return false + } + + const passwordScopedCredentialKey = createCredentialKey(normalizedDomain, user, password) + return this.disabledCredentials.has(passwordScopedCredentialKey) } public getCredentialStats(): AuthCredentialStats[] { @@ -133,33 +156,133 @@ export class BooruAuthManagerService implements OnModuleInit { } private getDomainStats(domain: string): AuthCredentialStats { - const credentials = this.authConfig[domain] || [] - const disabled = credentials.filter((cred) => this.isCredentialDisabled(domain, cred.user)).length + const normalizedDomain = this.normalizeDomain(domain) + const credentials = this.authConfig[normalizedDomain] || [] + const disabled = credentials.filter((cred) => + this.isCredentialDisabled(normalizedDomain, cred.user, cred.password) + ).length return { - domain, + domain: normalizedDomain, total: credentials.length, available: credentials.length - disabled, disabled } } + private normalizeAuthConfig(authConfig: BooruAuthConfig): BooruAuthConfig { + const normalizedAuthConfig: BooruAuthConfig = {} + + for (const [domain, credentials] of Object.entries(authConfig)) { + const normalizedDomain = this.normalizeDomain(domain) + const mergedCredentials = [...(normalizedAuthConfig[normalizedDomain] || []), ...credentials] + + normalizedAuthConfig[normalizedDomain] = this.dedupeCredentials(mergedCredentials) + } + + return normalizedAuthConfig + } + + private dedupeCredentials(credentials: BooruAuthCredential[]): BooruAuthCredential[] { + const uniqueCredentials = new Map() + + for (const credential of credentials) { + const credentialKey = JSON.stringify([credential.user, credential.password]) + + if (!uniqueCredentials.has(credentialKey)) { + uniqueCredentials.set(credentialKey, credential) + } + } + + return Array.from(uniqueCredentials.values()) + } + + private normalizeDomain(domain: string): string { + const extractedDomain = this.extractDomainFromUrl(domain) + return this.domainAliases[extractedDomain] || extractedDomain + } + private extractDomainFromUrl(url: string): string { try { - const normalizedUrl = url.startsWith('http') ? url : `https://${url}` + const hasProtocol = /^https?:\/\//i.test(url) + const normalizedUrl = hasProtocol ? url : `https://${url}` const urlObj = new URL(normalizedUrl) - return urlObj.hostname.replace(/^www\./, '') + return urlObj.hostname.toLowerCase() } catch (error) { - return url.replace(/^(https?:\/\/)?(www\.)?/, '').split('/')[0] + return url + .replace(/^(https?:\/\/)?/i, '') + .split(/[?#]/)[0] + .split('/')[0] + .toLowerCase() + } + } + + private sanitizeErrorMessage(message: string): string { + if (!message) { + return message } + + const urlPattern = /https?:\/\/[^\s]+/gi + const sanitizedUrlMessage = message.replace(urlPattern, (url) => this.sanitizeUrl(url)) + return this.sanitizeKeyValueTokens(sanitizedUrlMessage) + } + + private sanitizeUserIdentifier(user: string): string { + if (!user) { + return 'REDACTED' + } + + return `REDACTED(${user.length})` + } + + private sanitizeKeyValueTokens(message: string): string { + let sanitizedMessage = message + + for (const key of this.sensitiveParams) { + const escapedKey = key.replace(/[.*+?^${}()|[\]\\]/g, '\\$&') + const pattern = new RegExp(`\\b(${escapedKey})(\\s*=\\s*)([^\\s&#,;\\]\\)\\}]+)`, 'gi') + sanitizedMessage = sanitizedMessage.replace(pattern, '$1$2REDACTED') + } + + return sanitizedMessage + } + + private sanitizeUrl(url: string): string { + try { + const urlObj = new URL(url) + + for (const [key] of urlObj.searchParams.entries()) { + if (this.sensitiveParams.has(key.toLowerCase())) { + urlObj.searchParams.set(key, 'REDACTED') + } + } + + return urlObj.toString() + } catch (error) { + return this.sanitizeRawUrl(url) + } + } + + private sanitizeRawUrl(url: string): string { + let sanitizedUrl = url + + for (const key of this.sensitiveParams) { + const escapedKey = key.replace(/[.*+?^${}()|[\]\\]/g, '\\$&') + const pattern = new RegExp(`([?&]${escapedKey}=)[^&#\\s]*`, 'gi') + sanitizedUrl = sanitizedUrl.replace(pattern, '$1REDACTED') + } + + return sanitizedUrl } public getDisabledCredentials(): DisabledCredential[] { return Array.from(this.disabledCredentials).map((key) => { - const [domain, user] = key.split(':') + const { domain, user, password } = parseCredentialKey(key) + return { domain, user, + password, disabledAt: new Date(), reason: 'Authentication failure' } diff --git a/src/booru/services/credential-key.util.ts b/src/booru/services/credential-key.util.ts new file mode 100644 index 0000000..315c863 --- /dev/null +++ b/src/booru/services/credential-key.util.ts @@ -0,0 +1,27 @@ +export interface ParsedCredentialKey { + domain: string + user: string + password?: string +} + +export function createCredentialKey(domain: string, user: string, password?: string): string { + const encodedDomain = encodeURIComponent(domain) + const encodedUser = encodeURIComponent(user) + + if (password === undefined) { + return `${encodedDomain}:${encodedUser}` + } + + const encodedPassword = encodeURIComponent(password) + return `${encodedDomain}:${encodedUser}:${encodedPassword}` +} + +export function parseCredentialKey(key: string): ParsedCredentialKey { + const [encodedDomain = '', encodedUser = '', ...encodedPasswordParts] = key.split(':') + + return { + domain: decodeURIComponent(encodedDomain), + user: decodeURIComponent(encodedUser), + password: encodedPasswordParts.length > 0 ? decodeURIComponent(encodedPasswordParts.join(':')) : undefined + } +} diff --git a/src/cluster.service.ts b/src/cluster.service.ts index e09f4ab..c0208d5 100644 --- a/src/cluster.service.ts +++ b/src/cluster.service.ts @@ -2,6 +2,7 @@ import { Injectable } from '@nestjs/common' import { availableParallelism } from 'os' import cluster from 'cluster' import { IpcAuthMessage, DisabledCredential } from './booru/interfaces/auth-manager.interface' +import { createCredentialKey } from './booru/services/credential-key.util' const numCPUs = process.env.NODE_ENV === 'development' ? 1 : availableParallelism() @@ -20,7 +21,7 @@ export class AppClusterService { cluster.fork() } - cluster.on('exit', (worker, code, signal) => { + cluster.on('exit', (worker) => { console.log(`Worker ${worker.process.pid} died. Restarting...`) cluster.fork() }) @@ -34,7 +35,7 @@ export class AppClusterService { cluster.on('message', (worker, message: IpcAuthMessage) => { if (message.type === 'DISABLE_CREDENTIAL') { const credential = message.payload as DisabledCredential - const credentialKey = `${credential.domain}:${credential.user}` + const credentialKey = createCredentialKey(credential.domain, credential.user, credential.password) // Store in primary process this.disabledCredentials.add(credentialKey) @@ -46,8 +47,10 @@ export class AppClusterService { } }) + const scope = credential.password === undefined ? 'user-scoped' : 'password-scoped' + console.log( - `🔄 Broadcasting disabled credential ${credentialKey} to ${Object.keys(cluster.workers || {}).length - 1} other workers` + `🔄 Broadcasting disabled ${scope} credential for ${credential.domain} to ${Object.keys(cluster.workers || {}).length - 1} other workers` ) } })