fix: new database migration strategy (#219)

* formula overflow * fix: new database migration strategy * ci: add more error message * fix: change typ to 'local' | 'remote' --------- Co-authored-by: hzy <[email protected]>
openRin · Jul 12, 2024 · aacb021 · aacb021
1 parent c827475
commit aacb021
Show file tree

Hide file tree

Showing 6 changed files with 119 additions and 32 deletions.
diff --git a/package.json b/package.json
@@ -11,7 +11,7 @@
     "dev:client": "bun --filter './client' dev",
     "dev:server": "bun wrangler dev --port 11498",
     "check": "turbo check",
-    "cf-deploy": "bun server/migrator.ts",
+    "cf-deploy": "bun scripts/migrator.ts",
     "b": "turbo build",
     "t": "turbo t",
     "g": "turbo run g",

diff --git a/scripts/dev-migrator.ts b/scripts/dev-migrator.ts
@@ -1,32 +1,53 @@
 import * as fs from 'fs';
 import * as path from 'path';
-import {execSync} from 'child_process';
+import { execSync } from 'child_process';
+import { fixTopField, getMigrationVersion, isInfoExist, updateMigrationVersion } from './fix-top-field';
 
 const DB_NAME = "rin";
 const SQL_DIR = path.join(__dirname, '..', 'server', 'sql');
 
 // Change to the server/sql directory
 process.chdir(SQL_DIR);
-
+const typ = 'local';
+const migrationVersion = await getMigrationVersion(typ, DB_NAME);
+const isInfoExistResult = await isInfoExist(typ, DB_NAME);
 // List all SQL files and sort them
 const sqlFiles = fs
-    .readdirSync(SQL_DIR, {withFileTypes: true})
+    .readdirSync(SQL_DIR, { withFileTypes: true })
     .filter(dirent => dirent.isFile() && dirent.name.endsWith('.sql'))
     .map(dirent => dirent.name)
+    .filter(file => {
+        const version = parseInt(file.split('-')[0]);
+        return version > migrationVersion;
+    })
     .sort();
 
+console.log("migration_version:", migrationVersion, "Migration SQL List: ", sqlFiles)
+
 // For each file in the sorted list
 for (const file of sqlFiles) {
     const filePath = path.join(SQL_DIR, file);
     // Run the migration
     try {
-        execSync(`bunx wrangler d1 execute ${DB_NAME} --local --file "${filePath}"`, {stdio: 'inherit'});
+        execSync(`bunx wrangler d1 execute ${DB_NAME} --local --file "${filePath}"`, { stdio: 'inherit' });
         console.log(`Executed ${file}`);
     } catch (error) {
         console.error(`Failed to execute ${file}: ${error}`);
         process.exit(1);
     }
 }
 
+if (sqlFiles.length === 0) {
+    console.log("No migration needed.")
+} else {
+    const lastVersion = parseInt(sqlFiles[sqlFiles.length - 1].split('-')[0]);
+    if (lastVersion > migrationVersion) {
+        // Update the migration version
+        await updateMigrationVersion(typ, DB_NAME, lastVersion);
+    }
+}
+
+await fixTopField(typ, DB_NAME, isInfoExistResult);
+
 // Back to the root directory (optional, as the script ends)
 process.chdir(__dirname);
diff --git a/scripts/fix-top-field.ts b/scripts/fix-top-field.ts
@@ -0,0 +1,53 @@
+import { $ } from "bun"
+
+export async function fixTopField(typ: 'local' | 'remote', db: string, isInfoExistResult: boolean) {
+    if (!isInfoExistResult) {
+        console.log("Legacy database, check top field")
+        const result = await $`bunx wrangler d1 execute ${db}  --${typ} --json --command "SELECT name FROM pragma_table_info('feeds') WHERE name='top'"`.quiet().json()
+        if (result[0].results.length === 0) {
+            console.log("Adding top field to feeds table")
+            await $`bunx wrangler d1 execute ${db}  --${typ} --json --command "ALTER TABLE feeds ADD COLUMN top INTEGER DEFAULT 0"`.quiet()
+        } else {
+            console.log("Top field already exists in feeds table")
+        }
+    } else {
+        console.log("New database, skip top field check")
+    }
+}
+
+export async function isInfoExist(typ: 'local' | 'remote', db: string) {
+    const result = await $`bunx wrangler d1 execute ${db}  --${typ} --json --command "SELECT name FROM sqlite_master WHERE type='table' AND name='info'"`.quiet().json()
+    if (result[0].results.length === 0) {
+        console.log("info table not exists")
+        return false
+    } else {
+        console.log("info table already exists")
+        return true
+    }
+}
+
+export async function getMigrationVersion(typ: 'local' | 'remote', db: string) {
+    const isInfoExistResult = await isInfoExist(typ, db)
+    if (!isInfoExistResult) {
+        console.log("Legacy database, migration_version not exists")
+        return -1
+    }
+    const result = await $`bunx wrangler d1 execute ${db}  --${typ} --json --command "SELECT value FROM info WHERE key='migration_version'"`.quiet().json()
+    if (result[0].results.length === 0) {
+        console.log("migration_version not exists")
+        return -1
+    } else {
+        console.log("migration_version:", result[0].results[0].value)
+        return parseInt(result[0].results[0].value)
+    }
+}
+
+export async function updateMigrationVersion(typ: 'local' | 'remote', db: string, version: number) {
+    const exists = await isInfoExist(typ, db)
+    if (!exists) {
+        console.log("info table not exists, skip update migration_version")
+        throw new Error("info table not exists")
+    }
+    await $`bunx wrangler d1 execute ${db}  --${typ} --json --command "UPDATE info SET value='${version}' WHERE key='migration_version'"`.quiet()
+    console.log("Updated migration_version to", version)
+}
diff --git a/scripts/migrator.ts b/scripts/migrator.ts
@@ -1,6 +1,7 @@
 import { $ } from "bun"
 import { readdir } from "node:fs/promises"
 import stripIndent from 'strip-indent'
+import { fixTopField, getMigrationVersion, isInfoExist, updateMigrationVersion } from "./fix-top-field"
 
 function env(name: string, defaultValue?: string, required = false) {
     const env = process.env
@@ -12,7 +13,7 @@ function env(name: string, defaultValue?: string, required = false) {
 }
 
 // must be defined
-const renv = (name: string, defaultValue?: string) => env(name, defaultValue, true)
+const renv = (name: string, defaultValue?: string) => env(name, defaultValue, true)!
 
 const DB_NAME = renv("DB_NAME", 'rin')
 const WORKER_NAME = renv("WORKER_NAME", 'rin-server')
@@ -102,21 +103,46 @@ if (existing) {
 }
 
 console.log(`----------------------------`)
-
 console.log(`Migrating D1 "${DB_NAME}"`)
+const typ = 'remote';
+const migrationVersion = await getMigrationVersion(typ, DB_NAME);
+const isInfoExistResult = await isInfoExist(typ, DB_NAME);
+
 try {
     const files = await readdir("./server/sql", { recursive: false })
-    for (const file of files) {
+    const sqlFiles = files
+        .filter(name => name.endsWith('.sql'))
+        .filter(name => {
+            const version = parseInt(name.split('-')[0]);
+            return version > migrationVersion;
+        })
+        .sort();
+    console.log("migration_version:", migrationVersion, "Migration SQL List: ", sqlFiles)
+    for (const file of sqlFiles) {
         await $`bunx wrangler d1 execute ${DB_NAME} --remote --file ./server/sql/${file} -y`
         console.log(`Migrated ${file}`)
     }
+    if (sqlFiles.length === 0) {
+        console.log("No migration needed.")
+    } else {
+        const lastVersion = parseInt(sqlFiles[sqlFiles.length - 1].split('-')[0]);
+        if (lastVersion > migrationVersion) {
+            // Update the migration version
+            await updateMigrationVersion(typ, DB_NAME, lastVersion);
+        }
+    }
 } catch (e: any) {
-    console.error(e.stderr.toString())
+    console.error(e.stdio?.toString())
+    console.error(e.stdout?.toString())
+    console.error(e.stderr?.toString())
     process.exit(1)
 }
 
 console.log(`Migrated D1 "${DB_NAME}"`)
 console.log(`----------------------------`)
+console.log(`Patch D1`)
+await fixTopField(typ, DB_NAME, isInfoExistResult);
+console.log(`----------------------------`)
 console.log(`Put secrets`)
 
 async function putSecret(name: string, value?: string) {

diff --git a/server/sql/0002.sql b/server/sql/0002.sql
@@ -1,26 +1,8 @@
-PRAGMA foreign_keys=off;
---> statement-breakpoint
-CREATE TABLE IF NOT EXISTS `feeds_new` (
-	`id` integer PRIMARY KEY NOT NULL,
-	`alias` text,
-	`title` text,
-	`content` text NOT NULL,
-    `summary` text DEFAULT '' NOT NULL,
-	`listed` integer DEFAULT 1 NOT NULL,
-	`draft` integer DEFAULT 1 NOT NULL,
-	`uid` integer NOT NULL,
-    `top` integer DEFAULT 0 NOT NULL,
-	`created_at` integer DEFAULT (unixepoch()) NOT NULL,
-	`updated_at` integer DEFAULT (unixepoch()) NOT NULL,
-	FOREIGN KEY (`uid`) REFERENCES `users`(`id`) ON UPDATE no action ON DELETE no action
+CREATE TABLE IF NOT EXISTS `info` (
+	`key` text NOT NULL,
+	`value` text NOT NULL
 );
 --> statement-breakpoint
-INSERT INTO `feeds_new` (`id`, `alias`, `title`, `content`, `summary`, `listed`, `draft`, `uid`, `created_at`, `updated_at`)
-SELECT `id`, `alias`, `title`, `content`, `summary`, `listed`, `draft`, `uid`, `created_at`, `updated_at`
-FROM `feeds`;
---> statement-breakpoint
-DROP TABLE `feeds`;
---> statement-breakpoint
-ALTER TABLE `feeds_new` RENAME TO `feeds`;
+CREATE UNIQUE INDEX `info_key_unique` ON `info` (`key`);
 --> statement-breakpoint
-PRAGMA foreign_keys=on;
+INSERT INTO `info` (`key`, `value`) VALUES ('migration_version', '2');
diff --git a/server/src/db/schema.ts b/server/src/db/schema.ts
@@ -25,6 +25,11 @@ export const visits = sqliteTable("visits", {
     createdAt: created_at,
 });
 
+export const info = sqliteTable("info", {
+    key: text("key").notNull().unique(),
+    value: text("value").notNull(),
+});
+
 export const friends = sqliteTable("friends", {
     id: integer("id").primaryKey(),
     name: text("name").notNull(),