From 08de0090dc147f98434ce4790a15566274bc985d Mon Sep 17 00:00:00 2001 From: LouisLam Date: Fri, 10 Sep 2021 17:23:45 +0800 Subject: [PATCH] add a better approach for patching db, change setting.value from varchar to TEXT, restore Database.close() to 1.2.0 --- ...ch11.sql => patch-improve-performance.sql} | 0 db/patch-setting-value-type.sql | 22 ++ server/database.js | 221 ++++++++++++++++-- 3 files changed, 221 insertions(+), 22 deletions(-) rename db/{patch11.sql => patch-improve-performance.sql} (100%) create mode 100644 db/patch-setting-value-type.sql diff --git a/db/patch11.sql b/db/patch-improve-performance.sql similarity index 100% rename from db/patch11.sql rename to db/patch-improve-performance.sql diff --git a/db/patch-setting-value-type.sql b/db/patch-setting-value-type.sql new file mode 100644 index 000000000..18d6390c3 --- /dev/null +++ b/db/patch-setting-value-type.sql @@ -0,0 +1,22 @@ +-- You should not modify if this have pushed to Github, unless it does serious wrong with the db. +BEGIN TRANSACTION; + +-- Generated by Intellij IDEA +create table setting_dg_tmp +( + id INTEGER + primary key autoincrement, + key VARCHAR(200) not null + unique, + value TEXT, + type VARCHAR(20) +); + +insert into setting_dg_tmp(id, key, value, type) select id, key, value, type from setting; + +drop table setting; + +alter table setting_dg_tmp rename to setting; + + +COMMIT; diff --git a/server/database.js b/server/database.js index b76b38713..e0bb0c9b8 100644 --- a/server/database.js +++ b/server/database.js @@ -1,15 +1,44 @@ const fs = require("fs"); const { R } = require("redbean-node"); const { setSetting, setting } = require("./util-server"); +const { debug, sleep } = require("../src/util"); +const dayjs = require("dayjs"); class Database { - static templatePath = "./db/kuma.db" + static templatePath = "./db/kuma.db"; static dataDir; static path; + + /** + * @type {boolean} + */ + static patched = false; + + /** + * For Backup only + */ + static backupPath = null; + + /** + * Add patch filename in key + * Values: + * true: Add it regardless of order + * false: Do nothing + * { parents: []}: Need parents before add it + */ + static patchList = { + "patch-setting-value-type.sql": true, + "patch-improve-performance.sql": true, + } + + /** + * The finally version should be 10 after merged tag feature + * @deprecated Use patchList for any new feature + */ static latestVersion = 9; + static noReject = true; - static sqliteInstance = null; static async connect() { const acquireConnectionTimeout = 120 * 1000; @@ -60,19 +89,7 @@ class Database { } else { console.info("Database patch is needed") - console.info("Backup the db") - const backupPath = this.dataDir + "kuma.db.bak" + version; - fs.copyFileSync(Database.path, backupPath); - - const shmPath = Database.path + "-shm"; - if (fs.existsSync(shmPath)) { - fs.copyFileSync(shmPath, shmPath + ".bak" + version); - } - - const walPath = Database.path + "-wal"; - if (fs.existsSync(walPath)) { - fs.copyFileSync(walPath, walPath + ".bak" + version); - } + this.backup(version); // Try catch anything here, if gone wrong, restore the backup try { @@ -83,18 +100,92 @@ class Database { console.info(`Patched ${sqlFile}`); await setSetting("database_version", i); } - console.log("Database Patched Successfully"); } catch (ex) { await Database.close(); - console.error("Patch db failed!!! Restoring the backup") - fs.copyFileSync(backupPath, Database.path); - console.error(ex) + this.restore(); + console.error(ex) console.error("Start Uptime-Kuma failed due to patch db failed") console.error("Please submit the bug report if you still encounter the problem after restart: https://github.com/louislam/uptime-kuma/issues") process.exit(1); } } + + await this.patch2(); + } + + /** + * Call it from patch() only + * @returns {Promise} + */ + static async patch2() { + console.log("Database Patch 2.0 Process"); + let databasePatchedFiles = await setting("databasePatchedFiles"); + + if (! databasePatchedFiles) { + databasePatchedFiles = {}; + } + + debug("Patched files:"); + debug(databasePatchedFiles); + + try { + for (let sqlFilename in this.patchList) { + await this.patch2Recursion(sqlFilename, databasePatchedFiles) + } + + if (this.patched) { + console.log("Database Patched Successfully"); + } + + } catch (ex) { + await Database.close(); + this.restore(); + + console.error(ex) + console.error("Start Uptime-Kuma failed due to patch db failed"); + console.error("Please submit the bug report if you still encounter the problem after restart: https://github.com/louislam/uptime-kuma/issues"); + process.exit(1); + } + + await setSetting("databasePatchedFiles", databasePatchedFiles); + } + + /** + * Used it patch2() only + * @param sqlFilename + * @param databasePatchedFiles + */ + static async patch2Recursion(sqlFilename, databasePatchedFiles) { + let value = this.patchList[sqlFilename]; + + if (! value) { + console.log(sqlFilename + " skip"); + return; + } + + // Check if patched + if (! databasePatchedFiles[sqlFilename]) { + console.log(sqlFilename + " is not patched"); + + if (value.parents) { + console.log(sqlFilename + " need parents"); + for (let parentSQLFilename of value.parents) { + await this.patch2Recursion(parentSQLFilename, databasePatchedFiles); + } + } + + this.backup(dayjs().format("YYYYMMDDHHmmss")); + + console.log(sqlFilename + " is patching"); + this.patched = true; + await this.importSQLFile("./db/" + sqlFilename); + databasePatchedFiles[sqlFilename] = true; + console.log(sqlFilename + " is patched successfully"); + + } else { + console.log(sqlFilename + " is already patched, skip"); + } } /** @@ -140,10 +231,96 @@ class Database { * @returns {Promise} */ static async close() { - if (this.sqliteInstance) { - this.sqliteInstance.close(); + const listener = (reason, p) => { + Database.noReject = false; + }; + process.addListener("unhandledRejection", listener); + + console.log("Closing DB"); + + while (true) { + Database.noReject = true; + await R.close(); + await sleep(2000); + + if (Database.noReject) { + break; + } else { + console.log("Waiting to close the db"); + } + } + console.log("SQLite closed"); + + process.removeListener("unhandledRejection", listener); + } + + /** + * One backup one time in this process. + * Reset this.backupPath if you want to backup again + * @param version + */ + static backup(version) { + if (! this.backupPath) { + console.info("Backup the db") + this.backupPath = this.dataDir + "kuma.db.bak" + version; + fs.copyFileSync(Database.path, this.backupPath); + + const shmPath = Database.path + "-shm"; + if (fs.existsSync(shmPath)) { + this.backupShmPath = shmPath + ".bak" + version; + fs.copyFileSync(shmPath, this.backupShmPath); + } + + const walPath = Database.path + "-wal"; + if (fs.existsSync(walPath)) { + this.backupWalPath = walPath + ".bak" + version; + fs.copyFileSync(walPath, this.backupWalPath); + } + } + } + + /** + * + */ + static restore() { + if (this.backupPath) { + console.error("Patch db failed!!! Restoring the backup"); + + const shmPath = Database.path + "-shm"; + const walPath = Database.path + "-wal"; + + // Delete patch failed db + try { + if (fs.existsSync(Database.path)) { + fs.unlinkSync(Database.path); + } + + if (fs.existsSync(shmPath)) { + fs.unlinkSync(shmPath); + } + + if (fs.existsSync(walPath)) { + fs.unlinkSync(walPath); + } + } catch (e) { + console.log("Restore failed, you may need to restore the backup manually"); + process.exit(1); + } + + // Restore backup + fs.copyFileSync(this.backupPath, Database.path); + + if (this.backupShmPath) { + fs.copyFileSync(this.backupShmPath, shmPath); + } + + if (this.backupWalPath) { + fs.copyFileSync(this.backupWalPath, walPath); + } + + } else { + console.log("Nothing to restore"); } - console.log("Stopped database"); } }