diff --git a/scripts/migrate-selfhost-sqlite-to-postgres.ts b/scripts/migrate-selfhost-sqlite-to-postgres.ts index 1a17606c38..13a400fc59 100644 --- a/scripts/migrate-selfhost-sqlite-to-postgres.ts +++ b/scripts/migrate-selfhost-sqlite-to-postgres.ts @@ -1,6 +1,7 @@ #!/usr/bin/env tsx import { existsSync } from "node:fs"; import { DatabaseSync } from "node:sqlite"; +import { pathToFileURL } from "node:url"; import pg, { type PoolClient } from "pg"; import { createPgAdapter } from "../src/selfhost/pg-adapter"; import { createPgQueue } from "../src/selfhost/pg-queue"; @@ -32,6 +33,7 @@ interface SkipResult { const INTERNAL_SQLITE_TABLES = new Set(["d1_migrations", "_cf_KV", "__drizzle_migrations", "_selfhost_migrations"]); const TABLES_ALLOWED_AFTER_SCHEMA_INIT = new Set(["global_agent_controls", "global_contributor_blacklist"]); +const POSTGRES_TEXT_NUL_REPLACEMENT = "\uFFFD"; function usage(): string { return `Usage: npm run selfhost:postgres:migrate -- --sqlite --postgres-url [--execute] @@ -242,6 +244,16 @@ function valuePlaceholder(index: number, table: string, column: string): string return base; } +export function normalizePostgresValue(value: unknown): unknown { + if (typeof value !== "string") return value; + // SQLite text can contain NUL bytes from arbitrary repo files; Postgres text/json inputs cannot. + return value.includes("\0") ? value.replaceAll("\0", POSTGRES_TEXT_NUL_REPLACEMENT) : value; +} + +function sqliteCellForPostgres(row: Record, column: string): unknown { + return normalizePostgresValue(row[column] ?? null); +} + function insertSql(table: string, columns: string[], primaryKey: string[], rowCount: number): string { const columnSql = columns.map(quoteIdent).join(", "); const valuesSql = Array.from({ length: rowCount }, (_, rowIndex) => { @@ -260,7 +272,7 @@ async function copyTable(db: DatabaseSync, client: PoolClient, table: string, co for (let offset = 0; offset < total; offset += batchSize) { const rows = sqliteRows(db, table, columns, batchSize, offset); if (rows.length === 0) continue; - const values = rows.flatMap((row) => columns.map((column) => row[column] ?? null)); + const values = rows.flatMap((row) => columns.map((column) => sqliteCellForPostgres(row, column))); await client.query(insertSql(table, columns, primaryKey, rows.length), values); } return total; @@ -280,7 +292,7 @@ async function countTargetRowsMatchingSourceRows( if (rows.length === 0) continue; const values: unknown[] = []; for (const row of rows) { - for (const column of columns) values.push(row[column] ?? null); + for (const column of columns) values.push(sqliteCellForPostgres(row, column)); } const condition = rows .map((_, rowIndex) => { @@ -313,7 +325,7 @@ async function countConflictingTargetRowsForSourceKeys( .map((row) => { const parameterByColumn = new Map(); for (const column of compareColumns) { - values.push(row[column] ?? null); + values.push(sqliteCellForPostgres(row, column)); parameterByColumn.set(column, values.length); } const keyPredicates = keyColumns.map((column) => `${quoteIdent(column)} IS NOT DISTINCT FROM $${parameterByColumn.get(column)}`); @@ -460,7 +472,9 @@ async function main(): Promise { } } -main().catch((error) => { - console.error(error instanceof Error ? error.message : String(error)); - process.exit(1); -}); +if (process.argv[1] && import.meta.url === pathToFileURL(process.argv[1]).href) { + main().catch((error) => { + console.error(error instanceof Error ? error.message : String(error)); + process.exit(1); + }); +} diff --git a/test/unit/selfhost-migrate.test.ts b/test/unit/selfhost-migrate.test.ts index e45e3d3125..9910713b14 100644 --- a/test/unit/selfhost-migrate.test.ts +++ b/test/unit/selfhost-migrate.test.ts @@ -5,6 +5,7 @@ import { DatabaseSync } from "node:sqlite"; import { describe, expect, it } from "vitest"; import { createD1Adapter, nodeSqliteDriver } from "../../src/selfhost/d1-adapter"; import { runSelfHostMigrations } from "../../src/selfhost/migrate"; +import { normalizePostgresValue } from "../../scripts/migrate-selfhost-sqlite-to-postgres"; describe("runSelfHostMigrations (#980)", () => { it("applies un-applied migrations in order, idempotently", async () => { @@ -100,3 +101,12 @@ INSERT INTO notes (body) VALUES ('triggered');`, }); }); + +describe("SQLite-to-Postgres migrator helpers", () => { + it("normalizes embedded NUL bytes in SQLite text before Postgres copy", () => { + expect(normalizePostgresValue("repo\0chunk")).toBe("repo\uFFFDchunk"); + expect(normalizePostgresValue("plain text")).toBe("plain text"); + expect(normalizePostgresValue(null)).toBeNull(); + expect(normalizePostgresValue(42)).toBe(42); + }); +});