From 2dfa6886b44fe642039d9896057f33528a2146a5 Mon Sep 17 00:00:00 2001 From: Louis Lam Date: Sun, 1 Sep 2024 17:19:18 +0800 Subject: [PATCH 1/5] Preparing --- server/database.js | 9 +++++++++ server/jobs/clear-old-data.js | 11 +++++++++++ 2 files changed, 20 insertions(+) diff --git a/server/database.js b/server/database.js index 3374aff9e..76d1139ee 100644 --- a/server/database.js +++ b/server/database.js @@ -711,6 +711,15 @@ class Database { } } + /** + * TODO: Migrate the old data in the heartbeat table to the new format (stat_daily, stat_hourly, stat_minutely) + * It should be run once while upgrading V1 to V2 + * @returns {Promise} + */ + static async migrateAggregateTable() { + + } + } module.exports = Database; diff --git a/server/jobs/clear-old-data.js b/server/jobs/clear-old-data.js index 248a4d409..d74f34845 100644 --- a/server/jobs/clear-old-data.js +++ b/server/jobs/clear-old-data.js @@ -11,6 +11,17 @@ const DEFAULT_KEEP_PERIOD = 180; */ const clearOldData = async () => { + + /* + * TODO: + * Since we have aggregated table now, we don't need so much data in heartbeat table. + * But we still need to keep the important rows, because they contain the message. + * + * In the heartbeat table: + * - important rows: keep according to the setting (keepDataPeriodDays) (default 180 days) + * - not important rows: keep 2 days + */ + let period = await setting("keepDataPeriodDays"); // Set Default Period From 124effb55211fa1fbbfc0319f98f99a442cf065e Mon Sep 17 00:00:00 2001 From: Louis Lam Date: Sun, 22 Sep 2024 16:01:45 +0800 Subject: [PATCH 2/5] wip --- server/jobs/clear-old-data.js | 3 +++ 1 file changed, 3 insertions(+) diff --git a/server/jobs/clear-old-data.js b/server/jobs/clear-old-data.js index d74f34845..5bbd48a92 100644 --- a/server/jobs/clear-old-data.js +++ b/server/jobs/clear-old-data.js @@ -20,6 +20,9 @@ const clearOldData = async () => { * In the heartbeat table: * - important rows: keep according to the setting (keepDataPeriodDays) (default 180 days) * - not important rows: keep 2 days + * + * stat_* tables: + * - keep according to the setting (keepDataPeriodDays) (default 180 days) */ let period = await setting("keepDataPeriodDays"); From 0f3c727aa4669f05fe1237abc4b7978a1d189c01 Mon Sep 17 00:00:00 2001 From: Louis Lam Date: Sun, 22 Sep 2024 18:39:46 +0800 Subject: [PATCH 3/5] wip and fix sqlite migration because of foreign_keys --- server/database.js | 45 +++++++++++++++++++++++++++++++++++++++++++++ 1 file changed, 45 insertions(+) diff --git a/server/database.js b/server/database.js index 76d1139ee..317c21662 100644 --- a/server/database.js +++ b/server/database.js @@ -6,6 +6,7 @@ const knex = require("knex"); const path = require("path"); const { EmbeddedMariaDB } = require("./embedded-mariadb"); const mysql = require("mysql2/promise"); +const { Settings } = require("./settings"); /** * Database & App Data Folder @@ -391,9 +392,23 @@ class Database { // https://knexjs.org/guide/migrations.html // https://gist.github.com/NigelEarle/70db130cc040cc2868555b29a0278261 try { + // Disable foreign key check for SQLite + // Known issue of knex: https://github.com/drizzle-team/drizzle-orm/issues/1813 + if (Database.dbConfig.type === "sqlite") { + await R.exec("PRAGMA foreign_keys = OFF"); + } + await R.knex.migrate.latest({ directory: Database.knexMigrationsPath, }); + + // Enable foreign key check for SQLite + if (Database.dbConfig.type === "sqlite") { + await R.exec("PRAGMA foreign_keys = ON"); + } + + await this.migrateAggregateTable(); + } catch (e) { // Allow missing patch files for downgrade or testing pr. if (e.message.includes("the following files are missing:")) { @@ -717,7 +732,37 @@ class Database { * @returns {Promise} */ static async migrateAggregateTable() { + log.debug("db", "Enter Migrate Aggregate Table function"); + // + let migrated = false; + + if (migrated) { + log.debug("db", "Migrated, skip migration"); + return; + } + + log.info("db", "Migrating Aggregate Table"); + + // Migrate heartbeat to stat_minutely, using knex transaction + const trx = await R.knex.transaction(); + + // Get a list of unique dates from the heartbeat table, using raw sql + let dates = await trx.raw(` + SELECT DISTINCT DATE(time) AS date + FROM heartbeat + `); + + // Get a list of unique monitors from the heartbeat table, using raw sql + let monitors = await trx.raw(` + SELECT DISTINCT monitor_id + FROM heartbeat + `); + + console.log("Dates", dates); + console.log("Monitors", monitors); + + trx.commit(); } } From 59e7607e1adab0e5dc7396a9b1734273c81c2299 Mon Sep 17 00:00:00 2001 From: Louis Lam Date: Fri, 27 Sep 2024 21:42:20 +0800 Subject: [PATCH 4/5] wip --- server/database.js | 12 +++++++++++- 1 file changed, 11 insertions(+), 1 deletion(-) diff --git a/server/database.js b/server/database.js index 317c21662..79ef091b0 100644 --- a/server/database.js +++ b/server/database.js @@ -735,7 +735,7 @@ class Database { log.debug("db", "Enter Migrate Aggregate Table function"); // - let migrated = false; + let migrated = Settings.get("migratedAggregateTable"); if (migrated) { log.debug("db", "Migrated, skip migration"); @@ -759,10 +759,20 @@ class Database { FROM heartbeat `); + // Show warning if stat_* tables are not empty + for (let table of [ "stat_minutely", "stat_hourly", "stat_daily" ]) { + let count = await trx(table).count("*").first(); + if (count.count > 0) { + log.warn("db", `Table ${table} is not empty, migration may cause data loss (Maybe you were using 2.0.0-dev?)`); + } + } + console.log("Dates", dates); console.log("Monitors", monitors); trx.commit(); + + //await Settings.set("migratedAggregateTable", true); } } From 67ad0f79b3c995bc1f07469ab4758727b0cd33aa Mon Sep 17 00:00:00 2001 From: Louis Lam Date: Sat, 28 Sep 2024 01:22:45 +0800 Subject: [PATCH 5/5] wip --- server/database.js | 37 ++++++++++++++++++++++++++++++++----- server/uptime-calculator.js | 7 +++++-- 2 files changed, 37 insertions(+), 7 deletions(-) diff --git a/server/database.js b/server/database.js index 4a6ac7433..55dce6a47 100644 --- a/server/database.js +++ b/server/database.js @@ -6,6 +6,8 @@ const path = require("path"); const { EmbeddedMariaDB } = require("./embedded-mariadb"); const mysql = require("mysql2/promise"); const { Settings } = require("./settings"); +const { UptimeCalculator } = require("./uptime-calculator"); +const dayjs = require("dayjs"); /** * Database & App Data Folder @@ -734,7 +736,7 @@ class Database { log.debug("db", "Enter Migrate Aggregate Table function"); // - let migrated = Settings.get("migratedAggregateTable"); + let migrated = await Settings.get("migratedAggregateTable"); if (migrated) { log.debug("db", "Migrated, skip migration"); @@ -750,6 +752,7 @@ class Database { let dates = await trx.raw(` SELECT DISTINCT DATE(time) AS date FROM heartbeat + ORDER BY date ASC `); // Get a list of unique monitors from the heartbeat table, using raw sql @@ -758,17 +761,41 @@ class Database { FROM heartbeat `); - // Show warning if stat_* tables are not empty + // Stop if stat_* tables are not empty for (let table of [ "stat_minutely", "stat_hourly", "stat_daily" ]) { - let count = await trx(table).count("*").first(); - if (count.count > 0) { - log.warn("db", `Table ${table} is not empty, migration may cause data loss (Maybe you were using 2.0.0-dev?)`); + let countResult = await trx.raw(`SELECT COUNT(*) AS count FROM ${table}`); + let count = countResult[0].count; + if (count > 0) { + log.warn("db", `Aggregate table ${table} is not empty, migration will not be started (Maybe you were using 2.0.0-dev?)`); + return; } } console.log("Dates", dates); console.log("Monitors", monitors); + for (let monitor of monitors) { + for (let date of dates) { + log.info("db", `Migrating monitor ${monitor.monitor_id} on date ${date.date}`); + + // New Uptime Calculator + let calculator = new UptimeCalculator(); + + // TODO: Pass transaction to the calculator + // calculator.setTransaction(trx); + + // Get all the heartbeats for this monitor and date + let heartbeats = await trx("heartbeat") + .where("monitor_id", monitor.monitor_id) + .whereRaw("DATE(time) = ?", [ date.date ]) + .orderBy("time", "asc"); + + for (let heartbeat of heartbeats) { + calculator.update(heartbeat.status, heartbeat.ping, dayjs(heartbeat.time)); + } + } + } + trx.commit(); //await Settings.set("migratedAggregateTable", true); diff --git a/server/uptime-calculator.js b/server/uptime-calculator.js index f2738b96a..e9ade777e 100644 --- a/server/uptime-calculator.js +++ b/server/uptime-calculator.js @@ -189,11 +189,14 @@ class UptimeCalculator { /** * @param {number} status status * @param {number} ping Ping + * @param {dayjs.Dayjs} date Date (Only for migration) * @returns {dayjs.Dayjs} date * @throws {Error} Invalid status */ - async update(status, ping = 0) { - let date = this.getCurrentDate(); + async update(status, ping = 0, date) { + if (!date) { + date = this.getCurrentDate(); + } let flatStatus = this.flatStatus(status);