From aacb021493b9f0ae1f8430a37ca78535ee0fe6cc Mon Sep 17 00:00:00 2001 From: Xeu Date: Fri, 12 Jul 2024 22:07:45 +0800 Subject: [PATCH] fix: new database migration strategy (#219) * formula overflow * fix: new database migration strategy * ci: add more error message * fix: change typ to 'local' | 'remote' --------- Co-authored-by: hzy <1448424184@qq.com> --- package.json | 2 +- scripts/dev-migrator.ts | 29 +++++++++++++++++++--- scripts/fix-top-field.ts | 53 ++++++++++++++++++++++++++++++++++++++++ scripts/migrator.ts | 34 +++++++++++++++++++++++--- server/sql/0002.sql | 28 ++++----------------- server/src/db/schema.ts | 5 ++++ 6 files changed, 119 insertions(+), 32 deletions(-) create mode 100644 scripts/fix-top-field.ts diff --git a/package.json b/package.json index c084a5dc..72b1c9c3 100644 --- a/package.json +++ b/package.json @@ -11,7 +11,7 @@ "dev:client": "bun --filter './client' dev", "dev:server": "bun wrangler dev --port 11498", "check": "turbo check", - "cf-deploy": "bun server/migrator.ts", + "cf-deploy": "bun scripts/migrator.ts", "b": "turbo build", "t": "turbo t", "g": "turbo run g", diff --git a/scripts/dev-migrator.ts b/scripts/dev-migrator.ts index 129cd341..aadaa563 100644 --- a/scripts/dev-migrator.ts +++ b/scripts/dev-migrator.ts @@ -1,26 +1,35 @@ import * as fs from 'fs'; import * as path from 'path'; -import {execSync} from 'child_process'; +import { execSync } from 'child_process'; +import { fixTopField, getMigrationVersion, isInfoExist, updateMigrationVersion } from './fix-top-field'; const DB_NAME = "rin"; const SQL_DIR = path.join(__dirname, '..', 'server', 'sql'); // Change to the server/sql directory process.chdir(SQL_DIR); - +const typ = 'local'; +const migrationVersion = await getMigrationVersion(typ, DB_NAME); +const isInfoExistResult = await isInfoExist(typ, DB_NAME); // List all SQL files and sort them const sqlFiles = fs - .readdirSync(SQL_DIR, {withFileTypes: true}) + .readdirSync(SQL_DIR, { withFileTypes: true }) .filter(dirent => dirent.isFile() && dirent.name.endsWith('.sql')) .map(dirent => dirent.name) + .filter(file => { + const version = parseInt(file.split('-')[0]); + return version > migrationVersion; + }) .sort(); +console.log("migration_version:", migrationVersion, "Migration SQL List: ", sqlFiles) + // For each file in the sorted list for (const file of sqlFiles) { const filePath = path.join(SQL_DIR, file); // Run the migration try { - execSync(`bunx wrangler d1 execute ${DB_NAME} --local --file "${filePath}"`, {stdio: 'inherit'}); + execSync(`bunx wrangler d1 execute ${DB_NAME} --local --file "${filePath}"`, { stdio: 'inherit' }); console.log(`Executed ${file}`); } catch (error) { console.error(`Failed to execute ${file}: ${error}`); @@ -28,5 +37,17 @@ for (const file of sqlFiles) { } } +if (sqlFiles.length === 0) { + console.log("No migration needed.") +} else { + const lastVersion = parseInt(sqlFiles[sqlFiles.length - 1].split('-')[0]); + if (lastVersion > migrationVersion) { + // Update the migration version + await updateMigrationVersion(typ, DB_NAME, lastVersion); + } +} + +await fixTopField(typ, DB_NAME, isInfoExistResult); + // Back to the root directory (optional, as the script ends) process.chdir(__dirname); \ No newline at end of file diff --git a/scripts/fix-top-field.ts b/scripts/fix-top-field.ts new file mode 100644 index 00000000..a4162327 --- /dev/null +++ b/scripts/fix-top-field.ts @@ -0,0 +1,53 @@ +import { $ } from "bun" + +export async function fixTopField(typ: 'local' | 'remote', db: string, isInfoExistResult: boolean) { + if (!isInfoExistResult) { + console.log("Legacy database, check top field") + const result = await $`bunx wrangler d1 execute ${db} --${typ} --json --command "SELECT name FROM pragma_table_info('feeds') WHERE name='top'"`.quiet().json() + if (result[0].results.length === 0) { + console.log("Adding top field to feeds table") + await $`bunx wrangler d1 execute ${db} --${typ} --json --command "ALTER TABLE feeds ADD COLUMN top INTEGER DEFAULT 0"`.quiet() + } else { + console.log("Top field already exists in feeds table") + } + } else { + console.log("New database, skip top field check") + } +} + +export async function isInfoExist(typ: 'local' | 'remote', db: string) { + const result = await $`bunx wrangler d1 execute ${db} --${typ} --json --command "SELECT name FROM sqlite_master WHERE type='table' AND name='info'"`.quiet().json() + if (result[0].results.length === 0) { + console.log("info table not exists") + return false + } else { + console.log("info table already exists") + return true + } +} + +export async function getMigrationVersion(typ: 'local' | 'remote', db: string) { + const isInfoExistResult = await isInfoExist(typ, db) + if (!isInfoExistResult) { + console.log("Legacy database, migration_version not exists") + return -1 + } + const result = await $`bunx wrangler d1 execute ${db} --${typ} --json --command "SELECT value FROM info WHERE key='migration_version'"`.quiet().json() + if (result[0].results.length === 0) { + console.log("migration_version not exists") + return -1 + } else { + console.log("migration_version:", result[0].results[0].value) + return parseInt(result[0].results[0].value) + } +} + +export async function updateMigrationVersion(typ: 'local' | 'remote', db: string, version: number) { + const exists = await isInfoExist(typ, db) + if (!exists) { + console.log("info table not exists, skip update migration_version") + throw new Error("info table not exists") + } + await $`bunx wrangler d1 execute ${db} --${typ} --json --command "UPDATE info SET value='${version}' WHERE key='migration_version'"`.quiet() + console.log("Updated migration_version to", version) +} \ No newline at end of file diff --git a/scripts/migrator.ts b/scripts/migrator.ts index 7d928702..153d0c48 100644 --- a/scripts/migrator.ts +++ b/scripts/migrator.ts @@ -1,6 +1,7 @@ import { $ } from "bun" import { readdir } from "node:fs/promises" import stripIndent from 'strip-indent' +import { fixTopField, getMigrationVersion, isInfoExist, updateMigrationVersion } from "./fix-top-field" function env(name: string, defaultValue?: string, required = false) { const env = process.env @@ -12,7 +13,7 @@ function env(name: string, defaultValue?: string, required = false) { } // must be defined -const renv = (name: string, defaultValue?: string) => env(name, defaultValue, true) +const renv = (name: string, defaultValue?: string) => env(name, defaultValue, true)! const DB_NAME = renv("DB_NAME", 'rin') const WORKER_NAME = renv("WORKER_NAME", 'rin-server') @@ -102,21 +103,46 @@ if (existing) { } console.log(`----------------------------`) - console.log(`Migrating D1 "${DB_NAME}"`) +const typ = 'remote'; +const migrationVersion = await getMigrationVersion(typ, DB_NAME); +const isInfoExistResult = await isInfoExist(typ, DB_NAME); + try { const files = await readdir("./server/sql", { recursive: false }) - for (const file of files) { + const sqlFiles = files + .filter(name => name.endsWith('.sql')) + .filter(name => { + const version = parseInt(name.split('-')[0]); + return version > migrationVersion; + }) + .sort(); + console.log("migration_version:", migrationVersion, "Migration SQL List: ", sqlFiles) + for (const file of sqlFiles) { await $`bunx wrangler d1 execute ${DB_NAME} --remote --file ./server/sql/${file} -y` console.log(`Migrated ${file}`) } + if (sqlFiles.length === 0) { + console.log("No migration needed.") + } else { + const lastVersion = parseInt(sqlFiles[sqlFiles.length - 1].split('-')[0]); + if (lastVersion > migrationVersion) { + // Update the migration version + await updateMigrationVersion(typ, DB_NAME, lastVersion); + } + } } catch (e: any) { - console.error(e.stderr.toString()) + console.error(e.stdio?.toString()) + console.error(e.stdout?.toString()) + console.error(e.stderr?.toString()) process.exit(1) } console.log(`Migrated D1 "${DB_NAME}"`) console.log(`----------------------------`) +console.log(`Patch D1`) +await fixTopField(typ, DB_NAME, isInfoExistResult); +console.log(`----------------------------`) console.log(`Put secrets`) async function putSecret(name: string, value?: string) { diff --git a/server/sql/0002.sql b/server/sql/0002.sql index ddcc88da..c132b6bb 100644 --- a/server/sql/0002.sql +++ b/server/sql/0002.sql @@ -1,26 +1,8 @@ -PRAGMA foreign_keys=off; ---> statement-breakpoint -CREATE TABLE IF NOT EXISTS `feeds_new` ( - `id` integer PRIMARY KEY NOT NULL, - `alias` text, - `title` text, - `content` text NOT NULL, - `summary` text DEFAULT '' NOT NULL, - `listed` integer DEFAULT 1 NOT NULL, - `draft` integer DEFAULT 1 NOT NULL, - `uid` integer NOT NULL, - `top` integer DEFAULT 0 NOT NULL, - `created_at` integer DEFAULT (unixepoch()) NOT NULL, - `updated_at` integer DEFAULT (unixepoch()) NOT NULL, - FOREIGN KEY (`uid`) REFERENCES `users`(`id`) ON UPDATE no action ON DELETE no action +CREATE TABLE IF NOT EXISTS `info` ( + `key` text NOT NULL, + `value` text NOT NULL ); --> statement-breakpoint -INSERT INTO `feeds_new` (`id`, `alias`, `title`, `content`, `summary`, `listed`, `draft`, `uid`, `created_at`, `updated_at`) -SELECT `id`, `alias`, `title`, `content`, `summary`, `listed`, `draft`, `uid`, `created_at`, `updated_at` -FROM `feeds`; ---> statement-breakpoint -DROP TABLE `feeds`; ---> statement-breakpoint -ALTER TABLE `feeds_new` RENAME TO `feeds`; +CREATE UNIQUE INDEX `info_key_unique` ON `info` (`key`); --> statement-breakpoint -PRAGMA foreign_keys=on; \ No newline at end of file +INSERT INTO `info` (`key`, `value`) VALUES ('migration_version', '2'); \ No newline at end of file diff --git a/server/src/db/schema.ts b/server/src/db/schema.ts index 24fe34d8..76a7f768 100644 --- a/server/src/db/schema.ts +++ b/server/src/db/schema.ts @@ -25,6 +25,11 @@ export const visits = sqliteTable("visits", { createdAt: created_at, }); +export const info = sqliteTable("info", { + key: text("key").notNull().unique(), + value: text("value").notNull(), +}); + export const friends = sqliteTable("friends", { id: integer("id").primaryKey(), name: text("name").notNull(),