From 637712e4fc3e0295b3f0606ddbee5789ffcc659e Mon Sep 17 00:00:00 2001 From: gauthier-th Date: Wed, 18 Feb 2026 14:50:40 +0100 Subject: [PATCH] feat: add script for SQLite to PostgreSQL migration --- package.json | 3 +- server/scripts/sqliteToPostgres.ts | 182 +++++++++++++++++++++++++++++ 2 files changed, 184 insertions(+), 1 deletion(-) create mode 100644 server/scripts/sqliteToPostgres.ts diff --git a/package.json b/package.json index d38b779d..a229d347 100644 --- a/package.json +++ b/package.json @@ -24,7 +24,8 @@ "prepare": "node bin/prepare.js", "cypress:open": "cypress open", "cypress:prepare": "ts-node -r tsconfig-paths/register --files --project server/tsconfig.json server/scripts/prepareTestDb.ts", - "cypress:build": "pnpm build && pnpm cypress:prepare" + "cypress:build": "pnpm build && pnpm cypress:prepare", + "db:migratetopostgres": "pnpm build:server && node dist/scripts/sqliteToPostgres.js" }, "repository": { "type": "git", diff --git a/server/scripts/sqliteToPostgres.ts b/server/scripts/sqliteToPostgres.ts new file mode 100644 index 00000000..01ecc836 --- /dev/null +++ b/server/scripts/sqliteToPostgres.ts @@ -0,0 +1,182 @@ +/* eslint-disable no-console */ + +import fs from 'fs'; +import path from 'path'; +import type { TlsOptions } from 'tls'; +import { + DataSource, + type DataSourceOptions, + type ObjectLiteral, +} from 'typeorm'; + +const DB_SSL_PREFIX = 'DB_SSL_'; + +function boolFromEnv(envVar: string, defaultVal = false) { + if (process.env[envVar]) { + return process.env[envVar]?.toLowerCase() === 'true'; + } + return defaultVal; +} + +function stringOrReadFileFromEnv(envVar: string): Buffer | string | undefined { + if (process.env[envVar]) { + return process.env[envVar]; + } + const filePath = process.env[`${envVar}_FILE`]; + if (filePath) { + return fs.readFileSync(filePath); + } + return undefined; +} + +function buildSslConfig(): TlsOptions | undefined { + if (process.env.DB_USE_SSL?.toLowerCase() !== 'true') { + return undefined; + } + return { + rejectUnauthorized: boolFromEnv( + `${DB_SSL_PREFIX}REJECT_UNAUTHORIZED`, + true + ), + ca: stringOrReadFileFromEnv(`${DB_SSL_PREFIX}CA`), + key: stringOrReadFileFromEnv(`${DB_SSL_PREFIX}KEY`), + cert: stringOrReadFileFromEnv(`${DB_SSL_PREFIX}CERT`), + }; +} + +const prodConfig: DataSourceOptions = { + type: 'sqlite', + database: process.env.CONFIG_DIRECTORY + ? `${process.env.CONFIG_DIRECTORY}/db/db.sqlite3` + : 'config/db/db.sqlite3', + synchronize: false, + migrationsRun: false, + logging: boolFromEnv('DB_LOG_QUERIES'), + enableWAL: true, + // entities: ['dist/entity/**/*.js'], + migrations: ['dist/migration/sqlite/**/*.js'], + subscribers: ['dist/subscriber/**/*.js'], +}; + +const postgresProdConfig: DataSourceOptions = { + type: 'postgres', + host: process.env.DB_SOCKET_PATH || process.env.DB_HOST, + port: process.env.DB_SOCKET_PATH + ? undefined + : parseInt(process.env.DB_PORT ?? '5432'), + username: process.env.DB_USER, + password: process.env.DB_PASS, + database: process.env.DB_NAME ?? 'seerr', + ssl: buildSslConfig(), + synchronize: false, + migrationsRun: true, + logging: boolFromEnv('DB_LOG_QUERIES'), + // entities: ['dist/entity/**/*.js'], + migrations: ['dist/migration/postgres/**/*.js'], + subscribers: ['dist/subscriber/**/*.js'], +}; + +async function loadEntities(type: 'sqlite' | 'postgres') { + process.env.DB_TYPE = type; + Object.keys(require.cache).forEach((key) => { + if (key.includes(path.join(__dirname, '../../dist'))) { + delete require.cache[key]; + } + }); + const entities = await Promise.all( + fs + .readdirSync(path.join(__dirname, '../../dist/entity')) + .filter((file) => file.endsWith('.js')) + .map((file) => { + /* eslint @typescript-eslint/no-var-requires: "off" */ + const entityModule = require( + path.join(__dirname, '../../dist/entity', file) + ); + return entityModule.default || entityModule[file.replace('.js', '')]; + }) + ); + return entities; +} + +async function migrate() { + const sqliteEntities = await loadEntities('sqlite'); + const sqliteDataSource = new DataSource({ + entities: sqliteEntities, + ...prodConfig, + }); + await sqliteDataSource.initialize(); + console.log('SQLite DataSource initialized.'); + + const postgresEntities = await loadEntities('postgres'); + const postgresDataSource = new DataSource({ + entities: postgresEntities, + ...postgresProdConfig, + }); + await postgresDataSource.initialize(); + console.log('Postgres DataSource initialized.'); + + // create query runner and disable foreign key constraints for Postgres + const queryRunner = postgresDataSource.createQueryRunner(); + await queryRunner.connect(); + console.log('Disabling foreign key constraints...'); + await queryRunner.query(`SET session_replication_role = 'replica';`); + + try { + const entities = sqliteDataSource.entityMetadatas; + + for (const entity of entities) { + const entityName = entity.name; + const tableName = entity.tableName; + + console.log(`Migrating table: ${tableName} (${entityName})...`); + + const sourceRepo = sqliteDataSource.getRepository(entityName); + // const targetRepo = postgresDataSource.getRepository(entityName); + const targetRepo = queryRunner.manager.getRepository(entityName); + + const BATCH_SIZE = 1000; + let skip = 0; + let totalCount = 0; + + let rows: ObjectLiteral[]; + do { + rows = await sourceRepo.find({ + take: BATCH_SIZE, + skip: skip, + loadEagerRelations: false, + loadRelationIds: true, + }); + + for (const row of rows) { + // set postgres ID seq to because TypeORM ignores the ID field when saving + if (row.id && typeof row.id === 'number' && row.id > 1) { + await queryRunner.query(` + SELECT setval(pg_get_serial_sequence('${tableName}', 'id'), ${row.id - 1}, true); + `); + } + await targetRepo.save(row, { + transaction: false, + listeners: false, + reload: false, + }); + } + + skip += BATCH_SIZE; + totalCount += rows.length; + } while (rows.length !== 0); + console.log(` -> Copied ${totalCount} rows.`); + } + } catch (err) { + console.error('Migration failed:', err); + } finally { + console.log('Re-enabling foreign key constraints...'); + await queryRunner.query(`SET session_replication_role = 'origin';`); + await queryRunner.release(); + + await sqliteDataSource.destroy(); + await postgresDataSource.destroy(); + console.log('Migration complete.'); + } +} + +migrate();