From 301b2007a0666258dd68355ff1b550a9a42df56c Mon Sep 17 00:00:00 2001 From: Louis Lam Date: Tue, 19 Jul 2022 20:53:19 +0800 Subject: [PATCH 01/31] Drop Alpine support --- docker/alpine-base.dockerfile | 8 -------- docker/dockerfile-alpine | 25 ------------------------- package.json | 6 +----- 3 files changed, 1 insertion(+), 38 deletions(-) delete mode 100644 docker/alpine-base.dockerfile delete mode 100644 docker/dockerfile-alpine diff --git a/docker/alpine-base.dockerfile b/docker/alpine-base.dockerfile deleted file mode 100644 index cde65bb6..00000000 --- a/docker/alpine-base.dockerfile +++ /dev/null @@ -1,8 +0,0 @@ -# DON'T UPDATE TO alpine3.13, 1.14, see #41. -FROM node:16-alpine3.12 -WORKDIR /app - -# Install apprise, iputils for non-root ping, setpriv -RUN apk add --no-cache iputils setpriv dumb-init python3 py3-cryptography py3-pip py3-six py3-yaml py3-click py3-markdown py3-requests py3-requests-oauthlib && \ - pip3 --no-cache-dir install apprise==0.9.9 && \ - rm -rf /root/.cache diff --git a/docker/dockerfile-alpine b/docker/dockerfile-alpine deleted file mode 100644 index ab9255f9..00000000 --- a/docker/dockerfile-alpine +++ /dev/null @@ -1,25 +0,0 @@ -FROM louislam/uptime-kuma:base-alpine AS build -WORKDIR /app - -ENV PUPPETEER_SKIP_CHROMIUM_DOWNLOAD=1 - -COPY . . -RUN npm ci --production && \ - chmod +x /app/extra/entrypoint.sh - - -FROM louislam/uptime-kuma:base-alpine AS release -WORKDIR /app - -# Copy app files from build layer -COPY --from=build /app /app - -EXPOSE 3001 -VOLUME ["/app/data"] -HEALTHCHECK --interval=60s --timeout=30s --start-period=180s --retries=5 CMD node extra/healthcheck.js -ENTRYPOINT ["/usr/bin/dumb-init", "--", "extra/entrypoint.sh"] -CMD ["node", "server/server.js"] - - -FROM release AS nightly -RUN npm run mark-as-nightly diff --git a/package.json b/package.json index 7a18dbdf..676a0558 100644 --- a/package.json +++ b/package.json @@ -30,13 +30,10 @@ "jest-backend": "cross-env TEST_BACKEND=1 jest --config=./config/jest-backend.config.js", "tsc": "tsc", "vite-preview-dist": "vite preview --host --config ./config/vite.config.js", - "build-docker": "npm run build && npm run build-docker-debian && npm run build-docker-alpine", - "build-docker-alpine-base": "docker buildx build -f docker/alpine-base.dockerfile --platform linux/amd64,linux/arm64,linux/arm/v7 -t louislam/uptime-kuma:base-alpine . --push", + "build-docker": "npm run build && npm run build-docker-debian", "build-docker-debian-base": "docker buildx build -f docker/debian-base.dockerfile --platform linux/amd64,linux/arm64,linux/arm/v7 -t louislam/uptime-kuma:base-debian . --push", - "build-docker-alpine": "node ./extra/env2arg.js docker buildx build -f docker/dockerfile-alpine --platform linux/amd64,linux/arm64,linux/arm/v7 -t louislam/uptime-kuma:alpine -t louislam/uptime-kuma:1-alpine -t louislam/uptime-kuma:$VERSION-alpine --target release . --push", "build-docker-debian": "node ./extra/env2arg.js docker buildx build -f docker/dockerfile --platform linux/amd64,linux/arm64,linux/arm/v7 -t louislam/uptime-kuma -t louislam/uptime-kuma:1 -t louislam/uptime-kuma:$VERSION -t louislam/uptime-kuma:debian -t louislam/uptime-kuma:1-debian -t louislam/uptime-kuma:$VERSION-debian --target release . --push", "build-docker-nightly": "npm run build && docker buildx build -f docker/dockerfile --platform linux/amd64,linux/arm64,linux/arm/v7 -t louislam/uptime-kuma:nightly --target nightly . --push", - "build-docker-nightly-alpine": "docker buildx build -f docker/dockerfile-alpine --platform linux/amd64,linux/arm64,linux/arm/v7 -t louislam/uptime-kuma:nightly-alpine --target nightly . --push", "build-docker-nightly-amd64": "docker buildx build -f docker/dockerfile --platform linux/amd64 -t louislam/uptime-kuma:nightly-amd64 --target nightly . --push --progress plain", "upload-artifacts": "docker buildx build -f docker/dockerfile --platform linux/amd64 -t louislam/uptime-kuma:upload-artifact --build-arg VERSION --build-arg GITHUB_TOKEN --target upload-artifact . --progress plain", "setup": "git checkout 1.17.1 && npm ci --production && npm run download-dist", @@ -46,7 +43,6 @@ "remove-2fa": "node extra/remove-2fa.js", "compile-install-script": "@powershell -NoProfile -ExecutionPolicy Unrestricted -Command ./extra/compile-install-script.ps1", "test-install-script-centos7": "npm run compile-install-script && docker build --progress plain -f test/test_install_script/centos7.dockerfile .", - "test-install-script-alpine3": "npm run compile-install-script && docker build --progress plain -f test/test_install_script/alpine3.dockerfile .", "test-install-script-ubuntu": "npm run compile-install-script && docker build --progress plain -f test/test_install_script/ubuntu.dockerfile .", "test-install-script-ubuntu1604": "npm run compile-install-script && docker build --progress plain -f test/test_install_script/ubuntu1604.dockerfile .", "test-nodejs16": "docker build --progress plain -f test/ubuntu-nodejs16.dockerfile .", From b0d39b44ce8eedc976fd13111c5b83f0ac73bb79 Mon Sep 17 00:00:00 2001 From: Louis Lam Date: Fri, 22 Jul 2022 23:15:55 +0800 Subject: [PATCH 02/31] Testing --- docker/debian-base.dockerfile | 2 +- docker/dockerfile | 9 +++++++-- package-lock.json | 2 +- package.json | 3 ++- 4 files changed, 11 insertions(+), 5 deletions(-) diff --git a/docker/debian-base.dockerfile b/docker/debian-base.dockerfile index f90968a8..19d996db 100644 --- a/docker/debian-base.dockerfile +++ b/docker/debian-base.dockerfile @@ -7,7 +7,7 @@ WORKDIR /app # Install Curl # Install Apprise, add sqlite3 cli for debugging in the future, iputils-ping for ping, util-linux for setpriv -# Stupid python3 and python3-pip actually install a lot of useless things into Debian, specify --no-install-recommends to skip them, make the base even smaller than alpine! +# Stupid python3 and python3-pip actually install a lot of useless things into Debian, specify --no-install-recommends to skip them. RUN apt update && \ apt --yes --no-install-recommends install python3 python3-pip python3-cryptography python3-six python3-yaml python3-click python3-markdown python3-requests python3-requests-oauthlib \ sqlite3 iputils-ping util-linux dumb-init && \ diff --git a/docker/dockerfile b/docker/dockerfile index a9984351..174775a5 100644 --- a/docker/dockerfile +++ b/docker/dockerfile @@ -20,11 +20,16 @@ HEALTHCHECK --interval=60s --timeout=30s --start-period=180s --retries=5 CMD nod ENTRYPOINT ["/usr/bin/dumb-init", "--", "extra/entrypoint.sh"] CMD ["node", "server/server.js"] +FROM release AS mariadb +# Install MariaDB +RUN apt update && \ + apt --yes --no-install-recommends install mariadb-server && \ + rm -rf /var/lib/apt/lists/* && \ + apt --yes autoremove -FROM release AS nightly +FROM mariadb AS nightly RUN npm run mark-as-nightly - # Upload the artifact to Github FROM louislam/uptime-kuma:base-debian AS upload-artifact WORKDIR / diff --git a/package-lock.json b/package-lock.json index e64f9fa7..d76f5a94 100644 --- a/package-lock.json +++ b/package-lock.json @@ -16,7 +16,7 @@ "badge-maker": "^3.3.1", "bcryptjs": "~2.4.3", "bree": "~7.1.5", - "cacheable-lookup": "^6.0.4", + "cacheable-lookup": "~6.0.4", "chardet": "^1.3.0", "check-password-strength": "^2.0.5", "cheerio": "^1.0.0-rc.10", diff --git a/package.json b/package.json index 676a0558..1fc45546 100644 --- a/package.json +++ b/package.json @@ -30,9 +30,10 @@ "jest-backend": "cross-env TEST_BACKEND=1 jest --config=./config/jest-backend.config.js", "tsc": "tsc", "vite-preview-dist": "vite preview --host --config ./config/vite.config.js", - "build-docker": "npm run build && npm run build-docker-debian", + "build-docker": "npm run build && npm run build-docker-debian && npm run build-docker-debian-mariadb", "build-docker-debian-base": "docker buildx build -f docker/debian-base.dockerfile --platform linux/amd64,linux/arm64,linux/arm/v7 -t louislam/uptime-kuma:base-debian . --push", "build-docker-debian": "node ./extra/env2arg.js docker buildx build -f docker/dockerfile --platform linux/amd64,linux/arm64,linux/arm/v7 -t louislam/uptime-kuma -t louislam/uptime-kuma:1 -t louislam/uptime-kuma:$VERSION -t louislam/uptime-kuma:debian -t louislam/uptime-kuma:1-debian -t louislam/uptime-kuma:$VERSION-debian --target release . --push", + "build-docker-debian-mariadb": "node ./extra/env2arg.js docker buildx build -f docker/dockerfile --platform linux/amd64,linux/arm64,linux/arm/v7 -t louislam/uptime-kuma:mariadb -t louislam/uptime-kuma:1-mariadb -t louislam/uptime-kuma:$VERSION-mariadb --target mariadb . --push", "build-docker-nightly": "npm run build && docker buildx build -f docker/dockerfile --platform linux/amd64,linux/arm64,linux/arm/v7 -t louislam/uptime-kuma:nightly --target nightly . --push", "build-docker-nightly-amd64": "docker buildx build -f docker/dockerfile --platform linux/amd64 -t louislam/uptime-kuma:nightly-amd64 --target nightly . --push --progress plain", "upload-artifacts": "docker buildx build -f docker/dockerfile --platform linux/amd64 -t louislam/uptime-kuma:upload-artifact --build-arg VERSION --build-arg GITHUB_TOKEN --target upload-artifact . --progress plain", From 0039f1f52191b57f786e80810654c5beb18b3a1b Mon Sep 17 00:00:00 2001 From: Louis Lam Date: Wed, 14 Sep 2022 17:36:55 +0800 Subject: [PATCH 03/31] Drop support for Alpine docker image --- docker/alpine-base.dockerfile | 8 -------- docker/dockerfile-alpine | 25 ------------------------- package.json | 6 +----- 3 files changed, 1 insertion(+), 38 deletions(-) delete mode 100644 docker/alpine-base.dockerfile delete mode 100644 docker/dockerfile-alpine diff --git a/docker/alpine-base.dockerfile b/docker/alpine-base.dockerfile deleted file mode 100644 index 1d74de05..00000000 --- a/docker/alpine-base.dockerfile +++ /dev/null @@ -1,8 +0,0 @@ -# DON'T UPDATE TO alpine3.13, 1.14, see #41. -FROM node:16-alpine3.12 -WORKDIR /app - -# Install apprise, iputils for non-root ping, setpriv -RUN apk add --no-cache iputils setpriv dumb-init python3 py3-cryptography py3-pip py3-six py3-yaml py3-click py3-markdown py3-requests py3-requests-oauthlib && \ - pip3 --no-cache-dir install apprise==1.0.0 && \ - rm -rf /root/.cache diff --git a/docker/dockerfile-alpine b/docker/dockerfile-alpine deleted file mode 100644 index ab9255f9..00000000 --- a/docker/dockerfile-alpine +++ /dev/null @@ -1,25 +0,0 @@ -FROM louislam/uptime-kuma:base-alpine AS build -WORKDIR /app - -ENV PUPPETEER_SKIP_CHROMIUM_DOWNLOAD=1 - -COPY . . -RUN npm ci --production && \ - chmod +x /app/extra/entrypoint.sh - - -FROM louislam/uptime-kuma:base-alpine AS release -WORKDIR /app - -# Copy app files from build layer -COPY --from=build /app /app - -EXPOSE 3001 -VOLUME ["/app/data"] -HEALTHCHECK --interval=60s --timeout=30s --start-period=180s --retries=5 CMD node extra/healthcheck.js -ENTRYPOINT ["/usr/bin/dumb-init", "--", "extra/entrypoint.sh"] -CMD ["node", "server/server.js"] - - -FROM release AS nightly -RUN npm run mark-as-nightly diff --git a/package.json b/package.json index 219042aa..d200aa91 100644 --- a/package.json +++ b/package.json @@ -30,13 +30,10 @@ "jest-backend": "cross-env TEST_BACKEND=1 jest --config=./config/jest-backend.config.js", "tsc": "tsc", "vite-preview-dist": "vite preview --host --config ./config/vite.config.js", - "build-docker": "npm run build && npm run build-docker-debian && npm run build-docker-alpine", - "build-docker-alpine-base": "docker buildx build -f docker/alpine-base.dockerfile --platform linux/amd64,linux/arm64,linux/arm/v7 -t louislam/uptime-kuma:base-alpine . --push", + "build-docker": "npm run build && npm run build-docker-debian", "build-docker-debian-base": "docker buildx build -f docker/debian-base.dockerfile --platform linux/amd64,linux/arm64,linux/arm/v7 -t louislam/uptime-kuma:base-debian . --push", - "build-docker-alpine": "node ./extra/env2arg.js docker buildx build -f docker/dockerfile-alpine --platform linux/amd64,linux/arm64,linux/arm/v7 -t louislam/uptime-kuma:alpine -t louislam/uptime-kuma:1-alpine -t louislam/uptime-kuma:$VERSION-alpine --target release . --push", "build-docker-debian": "node ./extra/env2arg.js docker buildx build -f docker/dockerfile --platform linux/amd64,linux/arm64,linux/arm/v7 -t louislam/uptime-kuma -t louislam/uptime-kuma:1 -t louislam/uptime-kuma:$VERSION -t louislam/uptime-kuma:debian -t louislam/uptime-kuma:1-debian -t louislam/uptime-kuma:$VERSION-debian --target release . --push", "build-docker-nightly": "npm run build && docker buildx build -f docker/dockerfile --platform linux/amd64,linux/arm64,linux/arm/v7 -t louislam/uptime-kuma:nightly --target nightly . --push", - "build-docker-nightly-alpine": "docker buildx build -f docker/dockerfile-alpine --platform linux/amd64,linux/arm64,linux/arm/v7 -t louislam/uptime-kuma:nightly-alpine --target nightly . --push", "build-docker-nightly-amd64": "docker buildx build -f docker/dockerfile --platform linux/amd64 -t louislam/uptime-kuma:nightly-amd64 --target nightly . --push --progress plain", "build-docker-pr-test": "docker buildx build -f docker/dockerfile --platform linux/amd64,linux/arm64 -t louislam/uptime-kuma:pr-test --target pr-test . --push", "upload-artifacts": "docker buildx build -f docker/dockerfile --platform linux/amd64 -t louislam/uptime-kuma:upload-artifact --build-arg VERSION --build-arg GITHUB_TOKEN --target upload-artifact . --progress plain", @@ -47,7 +44,6 @@ "remove-2fa": "node extra/remove-2fa.js", "compile-install-script": "@powershell -NoProfile -ExecutionPolicy Unrestricted -Command ./extra/compile-install-script.ps1", "test-install-script-centos7": "npm run compile-install-script && docker build --progress plain -f test/test_install_script/centos7.dockerfile .", - "test-install-script-alpine3": "npm run compile-install-script && docker build --progress plain -f test/test_install_script/alpine3.dockerfile .", "test-install-script-ubuntu": "npm run compile-install-script && docker build --progress plain -f test/test_install_script/ubuntu.dockerfile .", "test-install-script-ubuntu1604": "npm run compile-install-script && docker build --progress plain -f test/test_install_script/ubuntu1604.dockerfile .", "test-nodejs16": "docker build --progress plain -f test/ubuntu-nodejs16.dockerfile .", From 73f7fbabd36f2741ba3282184de727af27be491b Mon Sep 17 00:00:00 2001 From: Louis Lam Date: Wed, 14 Sep 2022 18:05:02 +0800 Subject: [PATCH 04/31] True rootless image --- docker/debian-base.dockerfile | 2 -- docker/dockerfile | 14 +++++++------- extra/entrypoint.sh | 21 --------------------- 3 files changed, 7 insertions(+), 30 deletions(-) delete mode 100644 extra/entrypoint.sh diff --git a/docker/debian-base.dockerfile b/docker/debian-base.dockerfile index 20bef3dd..ceb2cac1 100644 --- a/docker/debian-base.dockerfile +++ b/docker/debian-base.dockerfile @@ -3,8 +3,6 @@ FROM node:16-buster-slim ARG TARGETPLATFORM -WORKDIR /app - # Install Curl # Install Apprise, add sqlite3 cli for debugging in the future, iputils-ping for ping, util-linux for setpriv # Stupid python3 and python3-pip actually install a lot of useless things into Debian, specify --no-install-recommends to skip them, make the base even smaller than alpine! diff --git a/docker/dockerfile b/docker/dockerfile index eea6ba33..95f79f81 100644 --- a/docker/dockerfile +++ b/docker/dockerfile @@ -1,27 +1,27 @@ FROM louislam/uptime-kuma:base-debian AS build +USER node WORKDIR /app - ENV PUPPETEER_SKIP_CHROMIUM_DOWNLOAD=1 - -COPY . . -RUN npm ci --production && \ - chmod +x /app/extra/entrypoint.sh +COPY --chown=node:node . . +RUN npm ci --production FROM louislam/uptime-kuma:base-debian AS release +USER node WORKDIR /app # Copy app files from build layer -COPY --from=build /app /app +COPY --chown=node:node --from=build /app /app EXPOSE 3001 VOLUME ["/app/data"] HEALTHCHECK --interval=60s --timeout=30s --start-period=180s --retries=5 CMD node extra/healthcheck.js -ENTRYPOINT ["/usr/bin/dumb-init", "--", "extra/entrypoint.sh"] +ENTRYPOINT ["/usr/bin/dumb-init", "--"] CMD ["node", "server/server.js"] FROM release AS nightly +USER node RUN npm run mark-as-nightly # Build an image for testing pr diff --git a/extra/entrypoint.sh b/extra/entrypoint.sh deleted file mode 100644 index 23c4f017..00000000 --- a/extra/entrypoint.sh +++ /dev/null @@ -1,21 +0,0 @@ -#!/usr/bin/env sh - -# set -e Exit the script if an error happens -set -e -PUID=${PUID=0} -PGID=${PGID=0} - -files_ownership () { - # -h Changes the ownership of an encountered symbolic link and not that of the file or directory pointed to by the symbolic link. - # -R Recursively descends the specified directories - # -c Like verbose but report only when a change is made - chown -hRc "$PUID":"$PGID" /app/data -} - -echo "==> Performing startup jobs and maintenance tasks" -files_ownership - -echo "==> Starting application with user $PUID group $PGID" - -# --clear-groups Clear supplementary groups. -exec setpriv --reuid "$PUID" --regid "$PGID" --clear-groups "$@" From a4de93f9763603dcb3e375e15db421e1bd9d3957 Mon Sep 17 00:00:00 2001 From: Louis Lam Date: Fri, 23 Dec 2022 22:43:56 +0800 Subject: [PATCH 05/31] WIP --- docker/dockerfile | 3 ++ docker/my.cnf | 0 server/database.js | 64 ++++++++++++++++++++++++++++---------- server/embedded-mariadb.js | 51 ++++++++++++++++++++++++++++++ 4 files changed, 101 insertions(+), 17 deletions(-) create mode 100644 docker/my.cnf create mode 100644 server/embedded-mariadb.js diff --git a/docker/dockerfile b/docker/dockerfile index 3cb27b50..acba709e 100644 --- a/docker/dockerfile +++ b/docker/dockerfile @@ -48,7 +48,10 @@ CMD ["node", "server/server.js"] ############################################ FROM release-slim AS release RUN apt update && \ + apt --yes --no-install-recommends install curl && \ + curl -LsS https://r.mariadb.com/downloads/mariadb_repo_setup | bash -s -- --mariadb-server-version="mariadb-10.11" && \ apt --yes --no-install-recommends install mariadb-server && \ + apt remove curl && \ rm -rf /var/lib/apt/lists/* && \ apt --yes autoremove diff --git a/docker/my.cnf b/docker/my.cnf new file mode 100644 index 00000000..e69de29b diff --git a/server/database.js b/server/database.js index 2544f197..9bf94f9b 100644 --- a/server/database.js +++ b/server/database.js @@ -4,6 +4,7 @@ const { setSetting, setting } = require("./util-server"); const { log, sleep } = require("../src/util"); const dayjs = require("dayjs"); const knex = require("knex"); +const path = require("path"); /** * Database & App Data Folder @@ -109,24 +110,53 @@ class Database { static async connect(testMode = false, autoloadModels = true, noLog = false) { const acquireConnectionTimeout = 120 * 1000; - const Dialect = require("knex/lib/dialects/sqlite3/index.js"); - Dialect.prototype._driver = () => require("@louislam/sqlite3"); + let dbConfig; - const knexInstance = knex({ - client: Dialect, - connection: { - filename: Database.path, - acquireConnectionTimeout: acquireConnectionTimeout, - }, - useNullAsDefault: true, - pool: { - min: 1, - max: 1, - idleTimeoutMillis: 120 * 1000, - propagateCreateError: false, - acquireTimeoutMillis: acquireConnectionTimeout, - } - }); + try { + dbConfig = fs.readFileSync(path.join(Database.dataDir, "db-config.json")); + } catch (_) { + dbConfig = { + type: "sqlite", + }; + } + + let config = {}; + + if (dbConfig.type === "sqlite") { + const Dialect = require("knex/lib/dialects/sqlite3/index.js"); + Dialect.prototype._driver = () => require("@louislam/sqlite3"); + + config = { + client: Dialect, + connection: { + filename: Database.path, + acquireConnectionTimeout: acquireConnectionTimeout, + }, + useNullAsDefault: true, + pool: { + min: 1, + max: 1, + idleTimeoutMillis: 120 * 1000, + propagateCreateError: false, + acquireTimeoutMillis: acquireConnectionTimeout, + } + }; + } else if (dbConfig === "embedded-mariadb") { + config = { + client: "mysql", + connection: { + host: "127.0.0.1", + port: 3306, + user: "your_database_user", + password: "your_database_password", + database: "kuma" + } + }; + } else { + throw new Error("Unknown Database type"); + } + + const knexInstance = knex(config); R.setup(knexInstance); diff --git a/server/embedded-mariadb.js b/server/embedded-mariadb.js new file mode 100644 index 00000000..eef78920 --- /dev/null +++ b/server/embedded-mariadb.js @@ -0,0 +1,51 @@ +const { log } = require("../src/util"); +const childProcess = require("child_process"); + +class EmbeddedMariaDB { + + static childProcess = null; + static running = false; + + static init() { + + } + + static start() { + if (this.childProcess) { + log.log("mariadb", "Already started"); + return; + } + + this.running = true; + this.emitChange("Starting cloudflared"); + this.childProcess = childProcess.spawn(this.cloudflaredPath, args); + this.childProcess.stdout.pipe(process.stdout); + this.childProcess.stderr.pipe(process.stderr); + + this.childProcess.on("close", (code) => { + this.running = false; + this.childProcess = null; + this.emitChange("Stopped cloudflared", code); + }); + + this.childProcess.on("error", (err) => { + if (err.code === "ENOENT") { + this.emitError(`Cloudflared error: ${this.cloudflaredPath} is not found`); + } else { + this.emitError(err); + } + }); + + this.childProcess.stderr.on("data", (data) => { + this.emitError(data.toString()); + }); + } + + static stop() { + if (this.childProcess) { + this.childProcess.kill("SIGINT"); + this.childProcess = null; + } + } + +} From 27eddb7253d539c020035110811e669ed6f2d98b Mon Sep 17 00:00:00 2001 From: Louis Lam Date: Sat, 4 Feb 2023 18:37:12 +0800 Subject: [PATCH 06/31] Update dockerfile --- docker/debian-base.dockerfile | 2 +- docker/dockerfile | 8 ++++---- package.json | 11 +++++------ 3 files changed, 10 insertions(+), 11 deletions(-) diff --git a/docker/debian-base.dockerfile b/docker/debian-base.dockerfile index 8b5df5be..e4430100 100644 --- a/docker/debian-base.dockerfile +++ b/docker/debian-base.dockerfile @@ -1,6 +1,6 @@ # DON'T UPDATE TO node:14-bullseye-slim, see #372. # If the image changed, the second stage image should be changed too -FROM node:16-buster-slim +FROM node:18-buster-slim ARG TARGETPLATFORM WORKDIR /app diff --git a/docker/dockerfile b/docker/dockerfile index 328733cb..537b16d1 100644 --- a/docker/dockerfile +++ b/docker/dockerfile @@ -8,7 +8,7 @@ FROM louislam/uptime-kuma:builder-go AS build_healthcheck ############################################ # Build in Node.js ############################################ -FROM louislam/uptime-kuma:base-debian AS build +FROM louislam/uptime-kuma:base2 AS build WORKDIR /app ENV PUPPETEER_SKIP_CHROMIUM_DOWNLOAD=1 @@ -23,7 +23,7 @@ RUN chmod +x /app/extra/entrypoint.sh ############################################ # ⭐ Main Image (Slim) ############################################ -FROM louislam/uptime-kuma:base-debian AS release-slim +FROM louislam/uptime-kuma:base2 AS release-slim WORKDIR /app # Copy app files from build layer @@ -57,7 +57,7 @@ RUN npm run mark-as-nightly ############################################ # Build an image for testing pr ############################################ -FROM louislam/uptime-kuma:base-debian AS pr-test +FROM louislam/uptime-kuma:base2 AS pr-test WORKDIR /app @@ -89,7 +89,7 @@ CMD ["npm", "run", "start-pr-test"] ############################################ # Upload the artifact to Github ############################################ -FROM louislam/uptime-kuma:base-debian AS upload-artifact +FROM louislam/uptime-kuma:base2 AS upload-artifact WORKDIR / RUN apt update && \ apt --yes install curl file diff --git a/package.json b/package.json index 8f8daa96..b578145a 100644 --- a/package.json +++ b/package.json @@ -28,13 +28,12 @@ "jest-backend": "cross-env TEST_BACKEND=1 jest --runInBand --detectOpenHandles --forceExit --config=./config/jest-backend.config.js", "tsc": "tsc", "vite-preview-dist": "vite preview --host --config ./config/vite.config.js", - "build-docker": "npm run build && npm run build-docker-debian && npm run build-docker-debian-mariadb", - "build-docker-debian-base": "docker buildx build -f docker/debian-base.dockerfile --platform linux/amd64,linux/arm64,linux/arm/v7 -t louislam/uptime-kuma:base-debian . --push", + "build-docker": "npm run build && npm run build-docker-full && npm run build-docker-slim", + "build-docker-base": "docker buildx build -f docker/debian-base.dockerfile --platform linux/amd64,linux/arm64,linux/arm/v7 -t louislam/uptime-kuma:base2 . --push", "build-docker-builder-go": "docker buildx build -f docker/builder-go.dockerfile --platform linux/amd64,linux/arm64,linux/arm/v7 -t louislam/uptime-kuma:builder-go . --push", - "build-docker-debian": "node ./extra/env2arg.js docker buildx build -f docker/dockerfile --platform linux/amd64,linux/arm64,linux/arm/v7 -t louislam/uptime-kuma -t louislam/uptime-kuma:1 -t louislam/uptime-kuma:$VERSION -t louislam/uptime-kuma:debian -t louislam/uptime-kuma:1-debian -t louislam/uptime-kuma:$VERSION-debian --target release . --push", - "build-docker-debian-mariadb": "node ./extra/env2arg.js docker buildx build -f docker/dockerfile --platform linux/amd64,linux/arm64,linux/arm/v7 -t louislam/uptime-kuma:mariadb -t louislam/uptime-kuma:1-mariadb -t louislam/uptime-kuma:$VERSION-mariadb --target mariadb . --push", - "build-docker-nightly": "npm run build && docker buildx build -f docker/dockerfile --platform linux/amd64,linux/arm64,linux/arm/v7 -t louislam/uptime-kuma:nightly --target nightly . --push", - "build-docker-nightly-amd64": "docker buildx build -f docker/dockerfile --platform linux/amd64 -t louislam/uptime-kuma:nightly-amd64 --target nightly . --push --progress plain", + "build-docker-slim": "node ./extra/env2arg.js docker buildx build -f docker/dockerfile --platform linux/amd64,linux/arm64,linux/arm/v7 -t louislam/uptime-kuma:2-slim -t louislam/uptime-kuma:$VERSION-slim --target release-slim . --push", + "build-docker-full": "node ./extra/env2arg.js docker buildx build -f docker/dockerfile --platform linux/amd64,linux/arm64,linux/arm/v7 -t louislam/uptime-kuma:2 -t louislam/uptime-kuma:$VERSION --target release . --push", + "build-docker-nightly": "npm run build && docker buildx build -f docker/dockerfile --platform linux/amd64,linux/arm64,linux/arm/v7 -t louislam/uptime-kuma:nightly2 --target nightly . --push", "build-docker-pr-test": "docker buildx build -f docker/dockerfile --platform linux/amd64,linux/arm64 -t louislam/uptime-kuma:pr-test --target pr-test . --push", "upload-artifacts": "docker buildx build -f docker/dockerfile --platform linux/amd64 -t louislam/uptime-kuma:upload-artifact --build-arg VERSION --build-arg GITHUB_TOKEN --target upload-artifact . --progress plain", "setup": "git checkout 1.19.6 && npm ci --production && npm run download-dist", From dc4d2a77bb71550c51bb1021118c93999cf64a63 Mon Sep 17 00:00:00 2001 From: Louis Lam Date: Sun, 5 Feb 2023 17:45:36 +0800 Subject: [PATCH 07/31] WIP --- docker/debian-base.dockerfile | 12 +++- docker/docker-compose-dev.yml | 13 ++++ docker/docker-compose.yml | 15 ++-- docker/dockerfile | 28 ++------ docker/my.cnf | 0 package.json | 14 ++-- server/database.js | 30 +++++--- server/embedded-mariadb.js | 118 ++++++++++++++++++++++++++++---- test/ubuntu-nodejs16.dockerfile | 10 --- 9 files changed, 171 insertions(+), 69 deletions(-) create mode 100644 docker/docker-compose-dev.yml delete mode 100644 docker/my.cnf delete mode 100644 test/ubuntu-nodejs16.dockerfile diff --git a/docker/debian-base.dockerfile b/docker/debian-base.dockerfile index 543f52bc..c2b8bfb4 100644 --- a/docker/debian-base.dockerfile +++ b/docker/debian-base.dockerfile @@ -1,6 +1,6 @@ # DON'T UPDATE TO node:14-bullseye-slim, see #372. # If the image changed, the second stage image should be changed too -FROM node:18-buster-slim +FROM node:18-buster-slim AS base2-slim ARG TARGETPLATFORM # Install Curl @@ -24,3 +24,13 @@ RUN node ./extra/download-cloudflared.js $TARGETPLATFORM && \ rm -f cloudflared.deb && \ apt --yes autoremove +FROM base2-slim AS base2 +RUN apt update && \ + apt --yes --no-install-recommends install curl && \ + curl -LsS https://r.mariadb.com/downloads/mariadb_repo_setup | bash -s -- --mariadb-server-version="mariadb-10.11" && \ + apt --yes --no-install-recommends install mariadb-server && \ + apt --yes remove curl && \ + rm -rf /var/lib/apt/lists/* && \ + apt --yes autoremove +RUN chown -R node:node /var/lib/mysql + diff --git a/docker/docker-compose-dev.yml b/docker/docker-compose-dev.yml new file mode 100644 index 00000000..5510b0d8 --- /dev/null +++ b/docker/docker-compose-dev.yml @@ -0,0 +1,13 @@ +version: '3.8' + +services: + uptime-kuma: + container_name: uptime-kuma-dev + image: louislam/uptime-kuma:nightly2 + volumes: + - ./data:/app/data + - ../server:/app/server + ports: + - "3001:3001" # : + - "3307:3306" + diff --git a/docker/docker-compose.yml b/docker/docker-compose.yml index f5c8f366..20e37329 100644 --- a/docker/docker-compose.yml +++ b/docker/docker-compose.yml @@ -1,14 +1,15 @@ -# Simple docker-compose.yml -# You can change your port or volume location - -version: '3.3' +version: '3.8' services: uptime-kuma: - image: louislam/uptime-kuma:1 + image: louislam/uptime-kuma:2 container_name: uptime-kuma volumes: - - ./uptime-kuma-data:/app/data + - uptime-kuma:/app/data ports: - - 3001:3001 # : + - "3001:3001" # : restart: always + +volumes: + uptime-kuma: + diff --git a/docker/dockerfile b/docker/dockerfile index a5f4ed8a..a62f2340 100644 --- a/docker/dockerfile +++ b/docker/dockerfile @@ -1,6 +1,8 @@ +ARG BASE_IMAGE=louislam/uptime-kuma:base2 + ############################################ # Build in Golang -# Run npm run build-healthcheck-armv7 in the host first, another it will be super slow where it is building the armv7 healthcheck +# Run npm run build-healthcheck-armv7 in the host first, otherwise it will be super slow where it is building the armv7 healthcheck # Check file: builder-go.dockerfile ############################################ FROM louislam/uptime-kuma:builder-go AS build_healthcheck @@ -9,6 +11,7 @@ FROM louislam/uptime-kuma:builder-go AS build_healthcheck # Build in Node.js ############################################ FROM louislam/uptime-kuma:base2 AS build +USER node WORKDIR /app ENV PUPPETEER_SKIP_CHROMIUM_DOWNLOAD=1 @@ -20,34 +23,20 @@ COPY . . COPY --chown=node:node --from=build_healthcheck /app/extra/healthcheck /app/extra/healthcheck ############################################ -# ⭐ Main Image (Slim) +# ⭐ Main Image ############################################ -FROM louislam/uptime-kuma:base2 AS release-slim +FROM $BASE_IMAGE AS release USER node WORKDIR /app # Copy app files from build layer COPY --chown=node:node --from=build /app /app - EXPOSE 3001 -VOLUME ["/app/data"] HEALTHCHECK --interval=60s --timeout=30s --start-period=180s --retries=5 CMD extra/healthcheck ENTRYPOINT ["/usr/bin/dumb-init", "--"] CMD ["node", "server/server.js"] -############################################ -# ⭐ Main Image (With MariaDB) -############################################ -FROM release-slim AS release -RUN apt update && \ - apt --yes --no-install-recommends install curl && \ - curl -LsS https://r.mariadb.com/downloads/mariadb_repo_setup | bash -s -- --mariadb-server-version="mariadb-10.11" && \ - apt --yes --no-install-recommends install mariadb-server && \ - apt remove curl && \ - rm -rf /var/lib/apt/lists/* && \ - apt --yes autoremove - ############################################ # Mark as Nightly ############################################ @@ -58,10 +47,8 @@ RUN npm run mark-as-nightly ############################################ # Build an image for testing pr ############################################ -FROM louislam/uptime-kuma:base2 AS pr-test - +FROM louislam/uptime-kuma:base2 AS pr-test2 WORKDIR /app - ENV PUPPETEER_SKIP_CHROMIUM_DOWNLOAD=1 ## Install Git @@ -83,7 +70,6 @@ RUN git clone https://github.com/louislam/uptime-kuma.git . RUN npm ci EXPOSE 3000 3001 -VOLUME ["/app/data"] HEALTHCHECK --interval=60s --timeout=30s --start-period=180s --retries=5 CMD node extra/healthcheck.js CMD ["npm", "run", "start-pr-test"] diff --git a/docker/my.cnf b/docker/my.cnf deleted file mode 100644 index e69de29b..00000000 diff --git a/package.json b/package.json index b578145a..48e61003 100644 --- a/package.json +++ b/package.json @@ -29,11 +29,13 @@ "tsc": "tsc", "vite-preview-dist": "vite preview --host --config ./config/vite.config.js", "build-docker": "npm run build && npm run build-docker-full && npm run build-docker-slim", - "build-docker-base": "docker buildx build -f docker/debian-base.dockerfile --platform linux/amd64,linux/arm64,linux/arm/v7 -t louislam/uptime-kuma:base2 . --push", + "build-docker-base": "docker buildx build -f docker/debian-base.dockerfile --platform linux/amd64,linux/arm64,linux/arm/v7 -t louislam/uptime-kuma:base2 --target base2 . --push", + "build-docker-base-slim": "docker buildx build -f docker/debian-base.dockerfile --platform linux/amd64,linux/arm64,linux/arm/v7 -t louislam/uptime-kuma:base2-slim --target base2-slim . --push", "build-docker-builder-go": "docker buildx build -f docker/builder-go.dockerfile --platform linux/amd64,linux/arm64,linux/arm/v7 -t louislam/uptime-kuma:builder-go . --push", - "build-docker-slim": "node ./extra/env2arg.js docker buildx build -f docker/dockerfile --platform linux/amd64,linux/arm64,linux/arm/v7 -t louislam/uptime-kuma:2-slim -t louislam/uptime-kuma:$VERSION-slim --target release-slim . --push", + "build-docker-slim": "node ./extra/env2arg.js docker buildx build -f docker/dockerfile --platform linux/amd64,linux/arm64,linux/arm/v7 -t louislam/uptime-kuma:2-slim -t louislam/uptime-kuma:$VERSION-slim --target release --build-arg BASE_IMAGE=louislam/uptime-kuma:base2-slim . --push", "build-docker-full": "node ./extra/env2arg.js docker buildx build -f docker/dockerfile --platform linux/amd64,linux/arm64,linux/arm/v7 -t louislam/uptime-kuma:2 -t louislam/uptime-kuma:$VERSION --target release . --push", - "build-docker-nightly": "npm run build && docker buildx build -f docker/dockerfile --platform linux/amd64,linux/arm64,linux/arm/v7 -t louislam/uptime-kuma:nightly2 --target nightly . --push", + "build-docker-nightly": "npm run build && docker buildx build -f docker/dockerfile --platform linux/amd64,linux/arm64,linux/arm/v7 -t louislam/uptime-kuma:nightly2 --target nightly --build-arg . --push", + "build-docker-nightly-local": "docker build -f docker/dockerfile -t louislam/uptime-kuma:nightly2 --target nightly .", "build-docker-pr-test": "docker buildx build -f docker/dockerfile --platform linux/amd64,linux/arm64 -t louislam/uptime-kuma:pr-test --target pr-test . --push", "upload-artifacts": "docker buildx build -f docker/dockerfile --platform linux/amd64 -t louislam/uptime-kuma:upload-artifact --build-arg VERSION --build-arg GITHUB_TOKEN --target upload-artifact . --progress plain", "setup": "git checkout 1.19.6 && npm ci --production && npm run download-dist", @@ -45,11 +47,9 @@ "test-install-script-centos7": "npm run compile-install-script && docker build --progress plain -f test/test_install_script/centos7.dockerfile .", "test-install-script-ubuntu": "npm run compile-install-script && docker build --progress plain -f test/test_install_script/ubuntu.dockerfile .", "test-install-script-ubuntu1604": "npm run compile-install-script && docker build --progress plain -f test/test_install_script/ubuntu1604.dockerfile .", - "test-nodejs16": "docker build --progress plain -f test/ubuntu-nodejs16.dockerfile .", "simple-dns-server": "node extra/simple-dns-server.js", "simple-mqtt-server": "node extra/simple-mqtt-server.js", "update-language-files": "cd extra/update-language-files && node index.js && cross-env-shell eslint ../../src/languages/$npm_config_language.js --fix", - "ncu-patch": "npm-check-updates -u -t patch", "release-final": "node extra/update-version.js && npm run build-docker && node ./extra/press-any-key.js && npm run upload-artifacts && node ./extra/update-wiki-version.js", "release-beta": "node extra/beta/update-version.js && npm run build && node ./extra/env2arg.js docker buildx build -f docker/dockerfile --platform linux/amd64,linux/arm64,linux/arm/v7 -t louislam/uptime-kuma:$VERSION -t louislam/uptime-kuma:beta . --target release --push && node ./extra/press-any-key.js && npm run upload-artifacts", "git-remove-tag": "git tag -d", @@ -59,7 +59,9 @@ "cy:run": "npx cypress run --browser chrome --headless --config-file ./config/cypress.config.js", "cy:run:unit": "npx cypress run --browser chrome --headless --config-file ./config/cypress.frontend.config.js", "cypress-open": "concurrently -k -r \"node test/prepare-test-server.js && node server/server.js --port=3002 --data-dir=./data/test/\" \"cypress open --config-file ./config/cypress.config.js\"", - "build-healthcheck-armv7": "cross-env GOOS=linux GOARCH=arm GOARM=7 go build -x -o ./extra/healthcheck-armv7 ./extra/healthcheck.go" + "build-healthcheck-armv7": "cross-env GOOS=linux GOARCH=arm GOARM=7 go build -x -o ./extra/healthcheck-armv7 ./extra/healthcheck.go", + "quick-run-nightly": "docker run --rm -p 3001:3001 louislam/uptime-kuma:nightly2", + "start-dev-container": "cd docker && docker-compose -f docker-compose-dev.yml up" }, "dependencies": { "@grpc/grpc-js": "~1.7.3", diff --git a/server/database.js b/server/database.js index 974e7cc4..bb1151cf 100644 --- a/server/database.js +++ b/server/database.js @@ -6,6 +6,7 @@ const dayjs = require("dayjs"); const knex = require("knex"); const { PluginsManager } = require("./plugins-manager"); const path = require("path"); +const { EmbeddedMariaDB } = require("./embedded-mariadb"); /** * Database & App Data Folder @@ -123,10 +124,20 @@ class Database { let dbConfig; try { - dbConfig = fs.readFileSync(path.join(Database.dataDir, "db-config.json")); + let dbConfigString = fs.readFileSync(path.join(Database.dataDir, "db-config.json")).toString("utf-8"); + dbConfig = JSON.parse(dbConfigString); + + if (typeof dbConfig !== "object") { + throw new Error("Invalid db-config.json, it must be an object"); + } + + if (typeof dbConfig.type !== "string") { + throw new Error("Invalid db-config.json, type must be a string"); + } } catch (_) { dbConfig = { - type: "sqlite", + //type: "sqlite", + type: "embedded-mariadb", }; } @@ -151,19 +162,20 @@ class Database { acquireTimeoutMillis: acquireConnectionTimeout, } }; - } else if (dbConfig === "embedded-mariadb") { + } else if (dbConfig.type === "embedded-mariadb") { + let embeddedMariaDB = EmbeddedMariaDB.getInstance(); + await embeddedMariaDB.start(); + log.info("mariadb", "Embedded MariaDB started"); config = { - client: "mysql", + client: "mysql2", connection: { - host: "127.0.0.1", - port: 3306, - user: "your_database_user", - password: "your_database_password", + socketPath: embeddedMariaDB.socketPath, + user: "node", database: "kuma" } }; } else { - throw new Error("Unknown Database type"); + throw new Error("Unknown Database type: " + dbConfig.type); } const knexInstance = knex(config); diff --git a/server/embedded-mariadb.js b/server/embedded-mariadb.js index eef78920..50fbfb94 100644 --- a/server/embedded-mariadb.js +++ b/server/embedded-mariadb.js @@ -1,47 +1,131 @@ const { log } = require("../src/util"); const childProcess = require("child_process"); +const fs = require("fs"); +/** + * It is only used inside the docker container + */ class EmbeddedMariaDB { - static childProcess = null; - static running = false; + static instance = null; - static init() { + exec = "mariadbd"; + mariadbDataDir = "/app/data/mariadb"; + + runDir = "/app/data/run/mariadb"; + + socketPath = this.runDir + "/mysqld.sock"; + + childProcess = null; + running = false; + + started = false; + + /** + * + * @returns {EmbeddedMariaDB} + */ + static getInstance() { + if (!EmbeddedMariaDB.instance) { + EmbeddedMariaDB.instance = new EmbeddedMariaDB(); + } + return EmbeddedMariaDB.instance; } - static start() { + static hasInstance() { + return !!EmbeddedMariaDB.instance; + } + + /** + * + */ + start() { if (this.childProcess) { - log.log("mariadb", "Already started"); + log.info("mariadb", "Already started"); return; } + if (!fs.existsSync(this.mariadbDataDir)) { + log.info("mariadb", `Embedded MariaDB: ${this.mariadbDataDir} is not found, create one now.`); + fs.mkdirSync(this.mariadbDataDir, { + recursive: true, + }); + + let result = childProcess.spawnSync("mysql_install_db", [ + "--user=node", + "--ldata=" + this.mariadbDataDir, + ]); + + if (result.status !== 0) { + let error = result.stderr.toString("utf-8"); + log.error("mariadb", error); + return; + } else { + log.info("mariadb", "Embedded MariaDB: mysql_install_db done:" + result.stdout.toString("utf-8")); + } + } + + if (!fs.existsSync(this.runDir)) { + log.info("mariadb", `Embedded MariaDB: ${this.runDir} is not found, create one now.`); + fs.mkdirSync(this.runDir, { + recursive: true, + }); + } + this.running = true; - this.emitChange("Starting cloudflared"); - this.childProcess = childProcess.spawn(this.cloudflaredPath, args); - this.childProcess.stdout.pipe(process.stdout); - this.childProcess.stderr.pipe(process.stderr); + log.info("mariadb", "Starting Embedded MariaDB"); + this.childProcess = childProcess.spawn(this.exec, [ + "--user=node", + "--datadir=" + this.mariadbDataDir, + `--socket=${this.socketPath}`, + `--pid-file=${this.runDir}/mysqld.pid`, + ]); this.childProcess.on("close", (code) => { this.running = false; this.childProcess = null; - this.emitChange("Stopped cloudflared", code); + this.started = false; + log.info("mariadb", "Stopped Embedded MariaDB: " + code); + + if (code !== 0) { + log.info("mariadb", "Try to restart Embedded MariaDB as it is not stopped by user"); + this.start(); + } }); this.childProcess.on("error", (err) => { if (err.code === "ENOENT") { - this.emitError(`Cloudflared error: ${this.cloudflaredPath} is not found`); + log.error("mariadb", `Embedded MariaDB: ${this.exec} is not found`); } else { - this.emitError(err); + log.error("mariadb", err); } }); - this.childProcess.stderr.on("data", (data) => { - this.emitError(data.toString()); + let handler = (data) => { + log.debug("mariadb", data.toString("utf-8")); + if (data.toString("utf-8").includes("ready for connections")) { + log.info("mariadb", "Embedded MariaDB is ready for connections"); + this.started = true; + } + }; + + this.childProcess.stdout.on("data", handler); + this.childProcess.stderr.on("data", handler); + + return new Promise((resolve) => { + let interval = setInterval(() => { + if (this.started) { + clearInterval(interval); + resolve(); + } else { + log.info("mariadb", "Waiting for Embedded MariaDB to start..."); + } + }, 1000); }); } - static stop() { + stop() { if (this.childProcess) { this.childProcess.kill("SIGINT"); this.childProcess = null; @@ -49,3 +133,7 @@ class EmbeddedMariaDB { } } + +module.exports = { + EmbeddedMariaDB, +}; diff --git a/test/ubuntu-nodejs16.dockerfile b/test/ubuntu-nodejs16.dockerfile deleted file mode 100644 index a2dd2ec8..00000000 --- a/test/ubuntu-nodejs16.dockerfile +++ /dev/null @@ -1,10 +0,0 @@ -FROM ubuntu -WORKDIR /app -RUN apt update && apt --yes install git curl -RUN curl -sL https://deb.nodesource.com/setup_16.x | bash - -RUN apt --yes install nodejs -RUN git clone https://github.com/louislam/uptime-kuma.git . -RUN npm run setup - -# Option 1. Try it -RUN node server/server.js From d4752b65dead00e0b6f4b2933ca2b46459c0d4ef Mon Sep 17 00:00:00 2001 From: Louis Lam Date: Sun, 5 Feb 2023 18:01:54 +0800 Subject: [PATCH 08/31] WIP --- package.json | 2 +- server/embedded-mariadb.js | 74 +++++++++++++++++++++++--------------- server/server.js | 5 +++ 3 files changed, 52 insertions(+), 29 deletions(-) diff --git a/package.json b/package.json index 48e61003..31276c43 100644 --- a/package.json +++ b/package.json @@ -60,7 +60,7 @@ "cy:run:unit": "npx cypress run --browser chrome --headless --config-file ./config/cypress.frontend.config.js", "cypress-open": "concurrently -k -r \"node test/prepare-test-server.js && node server/server.js --port=3002 --data-dir=./data/test/\" \"cypress open --config-file ./config/cypress.config.js\"", "build-healthcheck-armv7": "cross-env GOOS=linux GOARCH=arm GOARM=7 go build -x -o ./extra/healthcheck-armv7 ./extra/healthcheck.go", - "quick-run-nightly": "docker run --rm -p 3001:3001 louislam/uptime-kuma:nightly2", + "quick-run-nightly": "docker run --rm --env NODE_ENV=development -p 3001:3001 louislam/uptime-kuma:nightly2", "start-dev-container": "cd docker && docker-compose -f docker-compose-dev.yml up" }, "dependencies": { diff --git a/server/embedded-mariadb.js b/server/embedded-mariadb.js index 50fbfb94..8f82e64d 100644 --- a/server/embedded-mariadb.js +++ b/server/embedded-mariadb.js @@ -1,6 +1,7 @@ const { log } = require("../src/util"); const childProcess = require("child_process"); const fs = require("fs"); +const mysql = require("mysql2"); /** * It is only used inside the docker container @@ -46,32 +47,7 @@ class EmbeddedMariaDB { return; } - if (!fs.existsSync(this.mariadbDataDir)) { - log.info("mariadb", `Embedded MariaDB: ${this.mariadbDataDir} is not found, create one now.`); - fs.mkdirSync(this.mariadbDataDir, { - recursive: true, - }); - - let result = childProcess.spawnSync("mysql_install_db", [ - "--user=node", - "--ldata=" + this.mariadbDataDir, - ]); - - if (result.status !== 0) { - let error = result.stderr.toString("utf-8"); - log.error("mariadb", error); - return; - } else { - log.info("mariadb", "Embedded MariaDB: mysql_install_db done:" + result.stdout.toString("utf-8")); - } - } - - if (!fs.existsSync(this.runDir)) { - log.info("mariadb", `Embedded MariaDB: ${this.runDir} is not found, create one now.`); - fs.mkdirSync(this.runDir, { - recursive: true, - }); - } + this.initDB(); this.running = true; log.info("mariadb", "Starting Embedded MariaDB"); @@ -105,8 +81,7 @@ class EmbeddedMariaDB { let handler = (data) => { log.debug("mariadb", data.toString("utf-8")); if (data.toString("utf-8").includes("ready for connections")) { - log.info("mariadb", "Embedded MariaDB is ready for connections"); - this.started = true; + this.initDBAfterStarted(); } }; @@ -132,6 +107,49 @@ class EmbeddedMariaDB { } } + initDB() { + if (!fs.existsSync(this.mariadbDataDir)) { + log.info("mariadb", `Embedded MariaDB: ${this.mariadbDataDir} is not found, create one now.`); + fs.mkdirSync(this.mariadbDataDir, { + recursive: true, + }); + + let result = childProcess.spawnSync("mysql_install_db", [ + "--user=node", + "--ldata=" + this.mariadbDataDir, + ]); + + if (result.status !== 0) { + let error = result.stderr.toString("utf-8"); + log.error("mariadb", error); + return; + } else { + log.info("mariadb", "Embedded MariaDB: mysql_install_db done:" + result.stdout.toString("utf-8")); + } + } + + if (!fs.existsSync(this.runDir)) { + log.info("mariadb", `Embedded MariaDB: ${this.runDir} is not found, create one now.`); + fs.mkdirSync(this.runDir, { + recursive: true, + }); + } + + } + + async initDBAfterStarted() { + const connection = mysql.createConnection({ + socketPath: this.socketPath, + user: "node", + }); + + let result = await connection.execute("CREATE DATABASE IF NOT EXISTS `kuma`"); + log.debug("mariadb", "CREATE DATABASE: " + JSON.stringify(result)); + + log.info("mariadb", "Embedded MariaDB is ready for connections"); + this.started = true; + } + } module.exports = { diff --git a/server/server.js b/server/server.js index 1073f3be..3870b09d 100644 --- a/server/server.js +++ b/server/server.js @@ -142,6 +142,7 @@ const { generalSocketHandler } = require("./socket-handlers/general-socket-handl const { Settings } = require("./settings"); const { CacheableDnsHttpAgent } = require("./cacheable-dns-http-agent"); const { pluginsHandler } = require("./socket-handlers/plugins-handler"); +const { EmbeddedMariaDB } = require("./embedded-mariadb"); app.use(express.json()); @@ -1770,6 +1771,10 @@ async function shutdownFunction(signal) { await sleep(2000); await Database.close(); + if (EmbeddedMariaDB.hasInstance()) { + EmbeddedMariaDB.getInstance().stop(); + } + stopBackgroundJobs(); await cloudflaredStop(); Settings.stopCacheCleaner(); From 68ead3414d6846159ca62fe54c30f72247f7f852 Mon Sep 17 00:00:00 2001 From: Louis Lam Date: Mon, 6 Feb 2023 22:26:13 +0800 Subject: [PATCH 09/31] WIP --- server/database.js | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/server/database.js b/server/database.js index bb1151cf..67d2b754 100644 --- a/server/database.js +++ b/server/database.js @@ -136,8 +136,8 @@ class Database { } } catch (_) { dbConfig = { - //type: "sqlite", - type: "embedded-mariadb", + type: "sqlite", + //type: "embedded-mariadb", }; } From e4183ee2b7d615d58659c076d4172b83f2cd64f6 Mon Sep 17 00:00:00 2001 From: Louis Lam Date: Sat, 11 Feb 2023 14:41:02 +0800 Subject: [PATCH 10/31] Database Setup Page (#2738) * WIP * WIP: Database setup process * Add database setup page --- .dockerignore | 2 +- .gitignore | 1 + docker/debian-base.dockerfile | 2 +- extra/remove-2fa.js | 2 +- extra/reset-password.js | 2 +- server/database.js | 64 ++++++----- server/jobs/util-worker.js | 2 +- server/server.js | 40 ++++--- server/setup-database.js | 194 +++++++++++++++++++++++++++++++ server/util-server.js | 1 + src/lang/en.json | 5 + src/mixins/socket.js | 5 + src/pages/Entry.vue | 30 +++-- src/pages/SetupDatabase.vue | 211 ++++++++++++++++++++++++++++++++++ src/router.js | 5 + test/backend.spec.js | 6 +- 16 files changed, 513 insertions(+), 59 deletions(-) create mode 100644 server/setup-database.js create mode 100644 src/pages/SetupDatabase.vue diff --git a/.dockerignore b/.dockerignore index 47e82a10..3c9fdd6e 100644 --- a/.dockerignore +++ b/.dockerignore @@ -1,6 +1,6 @@ /.idea /node_modules -/data +/data* /cypress /out /test diff --git a/.gitignore b/.gitignore index 06dca04b..7ef7c711 100644 --- a/.gitignore +++ b/.gitignore @@ -7,6 +7,7 @@ dist-ssr /data !/data/.gitkeep +/data* .vscode /private diff --git a/docker/debian-base.dockerfile b/docker/debian-base.dockerfile index c2b8bfb4..6701511f 100644 --- a/docker/debian-base.dockerfile +++ b/docker/debian-base.dockerfile @@ -33,4 +33,4 @@ RUN apt update && \ rm -rf /var/lib/apt/lists/* && \ apt --yes autoremove RUN chown -R node:node /var/lib/mysql - +ENV UPTIME_KUMA_ENABLE_EMBEDDED_MARIADB=1 diff --git a/extra/remove-2fa.js b/extra/remove-2fa.js index f88c43fc..e6a8b97c 100644 --- a/extra/remove-2fa.js +++ b/extra/remove-2fa.js @@ -12,7 +12,7 @@ const rl = readline.createInterface({ }); const main = async () => { - Database.init(args); + Database.initDataDir(args); await Database.connect(); try { diff --git a/extra/reset-password.js b/extra/reset-password.js index 16898331..3f6f79c1 100644 --- a/extra/reset-password.js +++ b/extra/reset-password.js @@ -13,7 +13,7 @@ const rl = readline.createInterface({ const main = async () => { console.log("Connecting the database"); - Database.init(args); + Database.initDataDir(args); await Database.connect(false, false, true); try { diff --git a/server/database.js b/server/database.js index ffc88ca2..954882a6 100644 --- a/server/database.js +++ b/server/database.js @@ -25,7 +25,7 @@ class Database { */ static uploadDir; - static path; + static sqlitePath; /** * @type {boolean} @@ -83,10 +83,10 @@ class Database { static noReject = true; /** - * Initialize the database + * Initialize the data directory * @param {Object} args Arguments to initialize DB with */ - static init(args) { + static initDataDir(args) { // Data Directory (must be end with "/") Database.dataDir = process.env.DATA_DIR || args["data-dir"] || "./data/"; @@ -96,7 +96,7 @@ class Database { PluginsManager.disable = true; } - Database.path = Database.dataDir + "kuma.db"; + Database.sqlitePath = Database.dataDir + "kuma.db"; if (! fs.existsSync(Database.dataDir)) { fs.mkdirSync(Database.dataDir, { recursive: true }); } @@ -110,6 +110,26 @@ class Database { log.info("db", `Data Dir: ${Database.dataDir}`); } + static readDBConfig() { + let dbConfig; + + let dbConfigString = fs.readFileSync(path.join(Database.dataDir, "db-config.json")).toString("utf-8"); + dbConfig = JSON.parse(dbConfigString); + + if (typeof dbConfig !== "object") { + throw new Error("Invalid db-config.json, it must be an object"); + } + + if (typeof dbConfig.type !== "string") { + throw new Error("Invalid db-config.json, type must be a string"); + } + return dbConfig; + } + + static writeDBConfig(dbConfig) { + fs.writeFileSync(path.join(Database.dataDir, "db-config.json"), JSON.stringify(dbConfig, null, 4)); + } + /** * Connect to the database * @param {boolean} [testMode=false] Should the connection be @@ -121,21 +141,11 @@ class Database { */ static async connect(testMode = false, autoloadModels = true, noLog = false) { const acquireConnectionTimeout = 120 * 1000; - let dbConfig; - try { - let dbConfigString = fs.readFileSync(path.join(Database.dataDir, "db-config.json")).toString("utf-8"); - dbConfig = JSON.parse(dbConfigString); - - if (typeof dbConfig !== "object") { - throw new Error("Invalid db-config.json, it must be an object"); - } - - if (typeof dbConfig.type !== "string") { - throw new Error("Invalid db-config.json, type must be a string"); - } - } catch (_) { + dbConfig = this.readDBConfig(); + } catch (err) { + log.warn("db", err.message); dbConfig = { type: "sqlite", //type: "embedded-mariadb", @@ -151,7 +161,7 @@ class Database { config = { client: Dialect, connection: { - filename: Database.path, + filename: Database.sqlitePath, acquireConnectionTimeout: acquireConnectionTimeout, }, useNullAsDefault: true, @@ -497,15 +507,15 @@ class Database { if (! this.backupPath) { log.info("db", "Backing up the database"); this.backupPath = this.dataDir + "kuma.db.bak" + version; - fs.copyFileSync(Database.path, this.backupPath); + fs.copyFileSync(Database.sqlitePath, this.backupPath); - const shmPath = Database.path + "-shm"; + const shmPath = Database.sqlitePath + "-shm"; if (fs.existsSync(shmPath)) { this.backupShmPath = shmPath + ".bak" + version; fs.copyFileSync(shmPath, this.backupShmPath); } - const walPath = Database.path + "-wal"; + const walPath = Database.sqlitePath + "-wal"; if (fs.existsSync(walPath)) { this.backupWalPath = walPath + ".bak" + version; fs.copyFileSync(walPath, this.backupWalPath); @@ -535,13 +545,13 @@ class Database { if (this.backupPath) { log.error("db", "Patching the database failed!!! Restoring the backup"); - const shmPath = Database.path + "-shm"; - const walPath = Database.path + "-wal"; + const shmPath = Database.sqlitePath + "-shm"; + const walPath = Database.sqlitePath + "-wal"; // Delete patch failed db try { - if (fs.existsSync(Database.path)) { - fs.unlinkSync(Database.path); + if (fs.existsSync(Database.sqlitePath)) { + fs.unlinkSync(Database.sqlitePath); } if (fs.existsSync(shmPath)) { @@ -557,7 +567,7 @@ class Database { } // Restore backup - fs.copyFileSync(this.backupPath, Database.path); + fs.copyFileSync(this.backupPath, Database.sqlitePath); if (this.backupShmPath) { fs.copyFileSync(this.backupShmPath, shmPath); @@ -575,7 +585,7 @@ class Database { /** Get the size of the database */ static getSize() { log.debug("db", "Database.getSize()"); - let stats = fs.statSync(Database.path); + let stats = fs.statSync(Database.sqlitePath); log.debug("db", stats); return stats.size; } diff --git a/server/jobs/util-worker.js b/server/jobs/util-worker.js index 1aeec794..76131203 100644 --- a/server/jobs/util-worker.js +++ b/server/jobs/util-worker.js @@ -36,7 +36,7 @@ const connectDb = async function () { process.env.DATA_DIR || workerData["data-dir"] || "./data/" ); - Database.init({ + Database.initDataDir({ "data-dir": dbPath, }); diff --git a/server/server.js b/server/server.js index 3870b09d..627c586b 100644 --- a/server/server.js +++ b/server/server.js @@ -143,6 +143,7 @@ const { Settings } = require("./settings"); const { CacheableDnsHttpAgent } = require("./cacheable-dns-http-agent"); const { pluginsHandler } = require("./socket-handlers/plugins-handler"); const { EmbeddedMariaDB } = require("./embedded-mariadb"); +const { SetupDatabase } = require("./setup-database"); app.use(express.json()); @@ -168,8 +169,20 @@ let jwtSecret = null; let needSetup = false; (async () => { - Database.init(args); + // Create a data directory + Database.initDataDir(args); + + // Check if is chosen a database type + let setupDatabase = new SetupDatabase(args, server); + if (setupDatabase.isNeedSetup()) { + // Hold here and start a special setup page until user choose a database type + await setupDatabase.start(hostname, port); + } + + // Connect to database await initDatabase(testMode); + + // Database should be ready now await server.initAfterDatabaseReady(); server.loadPlugins(); server.entryPage = await Settings.get("entryPage"); @@ -334,7 +347,7 @@ let needSetup = false; } // Login Rate Limit - if (! await loginRateLimiter.pass(callback)) { + if (!await loginRateLimiter.pass(callback)) { log.info("auth", `Too many failed requests for user ${data.username}. IP=${clientIP}`); return; } @@ -407,7 +420,7 @@ let needSetup = false; socket.on("logout", async (callback) => { // Rate Limit - if (! await loginRateLimiter.pass(callback)) { + if (!await loginRateLimiter.pass(callback)) { return; } @@ -421,7 +434,7 @@ let needSetup = false; socket.on("prepare2FA", async (currentPassword, callback) => { try { - if (! await twoFaRateLimiter.pass(callback)) { + if (!await twoFaRateLimiter.pass(callback)) { return; } @@ -470,7 +483,7 @@ let needSetup = false; const clientIP = await server.getClientIP(socket); try { - if (! await twoFaRateLimiter.pass(callback)) { + if (!await twoFaRateLimiter.pass(callback)) { return; } @@ -502,7 +515,7 @@ let needSetup = false; const clientIP = await server.getClientIP(socket); try { - if (! await twoFaRateLimiter.pass(callback)) { + if (!await twoFaRateLimiter.pass(callback)) { return; } @@ -809,9 +822,10 @@ let needSetup = false; } let list = await R.getAll(` - SELECT * FROM heartbeat - WHERE monitor_id = ? AND - time > DATETIME('now', '-' || ? || ' hours') + SELECT * + FROM heartbeat + WHERE monitor_id = ? + AND time > DATETIME('now', '-' || ? || ' hours') ORDER BY time ASC `, [ monitorID, @@ -1068,7 +1082,7 @@ let needSetup = false; try { checkLogin(socket); - if (! password.newPassword) { + if (!password.newPassword) { throw new Error("Invalid new password"); } @@ -1375,7 +1389,7 @@ let needSetup = false; ]); let tagId; - if (! tag) { + if (!tag) { // -> If it doesn't exist, create new tag from backup file let beanTag = R.dispense("tag"); beanTag.name = oldTag.name; @@ -1644,9 +1658,9 @@ async function afterLogin(socket, user) { * @returns {Promise} */ async function initDatabase(testMode = false) { - if (! fs.existsSync(Database.path)) { + if (! fs.existsSync(Database.sqlitePath)) { log.info("server", "Copying Database"); - fs.copyFileSync(Database.templatePath, Database.path); + fs.copyFileSync(Database.templatePath, Database.sqlitePath); } log.info("server", "Connecting to the Database"); diff --git a/server/setup-database.js b/server/setup-database.js new file mode 100644 index 00000000..53d68fd0 --- /dev/null +++ b/server/setup-database.js @@ -0,0 +1,194 @@ +const express = require("express"); +const { log } = require("../src/util"); +const expressStaticGzip = require("express-static-gzip"); +const fs = require("fs"); +const path = require("path"); +const Database = require("./database"); +const { allowDevAllOrigin } = require("./util-server"); + +/** + * A standalone express app that is used to setup database + * It is used when db-config.json and kuma.db are not found or invalid + * Once it is configured, it will shutdown and start the main server + */ +class SetupDatabase { + + /** + * Show Setup Page + * @type {boolean} + */ + needSetup = true; + + server; + + constructor(args, server) { + this.server = server; + + // Priority: env > db-config.json + // If env is provided, write it to db-config.json + // If db-config.json is found, check if it is valid + // If db-config.json is not found or invalid, check if kuma.db is found + // If kuma.db is not found, show setup page + + let dbConfig; + + try { + dbConfig = Database.readDBConfig(); + } catch (e) { + log.info("setup-database", "db-config.json is not found or invalid: " + e.message); + + // Check if kuma.db is found (1.X.X users) + if (fs.existsSync(path.join(Database.dataDir, "kuma.db"))) { + this.needSetup = false; + } else { + this.needSetup = true; + } + dbConfig = {}; + } + + if (process.env.UPTIME_KUMA_DB_TYPE) { + this.needSetup = false; + log.info("setup-database", "UPTIME_KUMA_DB_TYPE is provided by env, try to override db-config.json"); + dbConfig.type = process.env.UPTIME_KUMA_DB_TYPE; + dbConfig.hostname = process.env.UPTIME_KUMA_DB_HOSTNAME; + dbConfig.port = process.env.UPTIME_KUMA_DB_PORT; + dbConfig.database = process.env.UPTIME_KUMA_DB_NAME; + dbConfig.username = process.env.UPTIME_KUMA_DB_USERNAME; + dbConfig.password = process.env.UPTIME_KUMA_DB_PASSWORD; + Database.writeDBConfig(dbConfig); + } + + } + + /** + * Show Setup Page + */ + isNeedSetup() { + return this.needSetup; + } + + isEnabledEmbeddedMariaDB() { + return process.env.UPTIME_KUMA_ENABLE_EMBEDDED_MARIADB === "1"; + } + + start(hostname, port) { + return new Promise((resolve) => { + const app = express(); + let tempServer; + app.use(express.json()); + + app.get("/", async (request, response) => { + response.redirect("/setup-database"); + }); + + app.get("/api/entry-page", async (request, response) => { + allowDevAllOrigin(response); + response.json({ + type: "setup-database", + }); + }); + + app.get("/info", (request, response) => { + allowDevAllOrigin(response); + response.json({ + isEnabledEmbeddedMariaDB: this.isEnabledEmbeddedMariaDB(), + }); + }); + + app.post("/setup-database", async (request, response) => { + allowDevAllOrigin(response); + + console.log(request); + + let dbConfig = request.body.dbConfig; + + let supportedDBTypes = [ "mariadb", "sqlite" ]; + + if (this.isEnabledEmbeddedMariaDB()) { + supportedDBTypes.push("embedded-mariadb"); + } + + // Validate input + if (typeof dbConfig !== "object") { + response.status(400).json("Invalid dbConfig"); + return; + } + + if (!dbConfig.type) { + response.status(400).json("Database Type is required"); + return; + } + + if (!supportedDBTypes.includes(dbConfig.type)) { + response.status(400).json("Unsupported Database Type"); + return; + } + + if (dbConfig.type === "mariadb") { + if (!dbConfig.hostname) { + response.status(400).json("Hostname is required"); + return; + } + + if (!dbConfig.port) { + response.status(400).json("Port is required"); + return; + } + + if (!dbConfig.dbName) { + response.status(400).json("Database name is required"); + return; + } + + if (!dbConfig.username) { + response.status(400).json("Username is required"); + return; + } + + if (!dbConfig.password) { + response.status(400).json("Password is required"); + return; + } + } + + // Write db-config.json + Database.writeDBConfig(dbConfig); + + response.json({ + ok: true, + }); + + // Shutdown down this express and start the main server + log.info("setup-database", "Database is configured, close setup-database server and start the main server now."); + if (tempServer) { + tempServer.close(); + } + resolve(); + }); + + app.use("/", expressStaticGzip("dist", { + enableBrotli: true, + })); + + app.get("*", async (_request, response) => { + response.send(this.server.indexHTML); + }); + + app.options("*", async (_request, response) => { + allowDevAllOrigin(response); + response.end(); + }); + + tempServer = app.listen(port, hostname, () => { + log.info("setup-database", `Starting Setup Database on ${port}`); + let domain = (hostname) ? hostname : "localhost"; + log.info("setup-database", `Open http://${domain}:${port} in your browser`); + log.info("setup-database", "Waiting for user action..."); + }); + }); + } +} + +module.exports = { + SetupDatabase, +}; diff --git a/server/util-server.js b/server/util-server.js index edce2890..615edcbc 100644 --- a/server/util-server.js +++ b/server/util-server.js @@ -613,6 +613,7 @@ exports.allowDevAllOrigin = (res) => { */ exports.allowAllOrigin = (res) => { res.header("Access-Control-Allow-Origin", "*"); + res.header("Access-Control-Allow-Methods", "GET, PUT, POST, DELETE, OPTIONS"); res.header("Access-Control-Allow-Headers", "Origin, X-Requested-With, Content-Type, Accept"); }; diff --git a/src/lang/en.json b/src/lang/en.json index d907f4e0..39628cda 100644 --- a/src/lang/en.json +++ b/src/lang/en.json @@ -1,5 +1,10 @@ { "languageName": "English", + "setupDatabaseChooseDatabase": "Which database do you want to use?", + "setupDatabaseEmbeddedMariaDB": "You don't need to set anything. This docker image have embedded and configured a MariaDB for you automatically. Uptime Kuma will connect to this database via unix socket.", + "setupDatabaseMariaDB": "Connect to an external MariaDB database. You need to set the database connection information.", + "setupDatabaseSQLite": "A simple database file. It is recommended for small scale deployment. Before 2.0.0, Uptime Kuma used SQLite by default.", + "dbName": "Database Name", "Settings": "Settings", "Dashboard": "Dashboard", "Help": "Help", diff --git a/src/mixins/socket.js b/src/mixins/socket.js index 6bd0aafc..120b1162 100644 --- a/src/mixins/socket.js +++ b/src/mixins/socket.js @@ -86,6 +86,11 @@ export default { } } + // Also don't need to connect to the socket.io for setup database page + if (location.pathname === "/setup-database") { + return; + } + this.socket.initedSocketIO = true; let protocol = (location.protocol === "https:") ? "wss://" : "ws://"; diff --git a/src/pages/Entry.vue b/src/pages/Entry.vue index 30e314b2..b87c4d99 100644 --- a/src/pages/Entry.vue +++ b/src/pages/Entry.vue @@ -19,25 +19,33 @@ export default { }, async mounted() { - // There are only 2 cases that could come in here. + // There are only 3 cases that could come in here. // 1. Matched status Page domain name // 2. Vue Frontend Dev - let res = (await axios.get("/api/entry-page")).data; + // 3. Vue Frontend Dev (not setup database yet) + let res; + try { + res = (await axios.get("/api/entry-page")).data; - if (res.type === "statusPageMatchedDomain") { - this.statusPageSlug = res.statusPageSlug; - this.$root.forceStatusPageTheme = true; + if (res.type === "statusPageMatchedDomain") { + this.statusPageSlug = res.statusPageSlug; + this.$root.forceStatusPageTheme = true; - } else if (res.type === "entryPage") { // Dev only. For production, the logic is in the server side - const entryPage = res.entryPage; + } else if (res.type === "entryPage") { // Dev only. For production, the logic is in the server side + const entryPage = res.entryPage; - if (entryPage === "statusPage") { - this.$router.push("/status"); + if (entryPage === "statusPage") { + this.$router.push("/status"); + } else { + this.$router.push("/dashboard"); + } + } else if (res.type === "setup-database") { + this.$router.push("/setup-database"); } else { this.$router.push("/dashboard"); } - } else { - this.$router.push("/dashboard"); + } catch (e) { + alert("Cannot connect to the backend server. Did you start the backend server? (npm run start-server-dev)"); } }, diff --git a/src/pages/SetupDatabase.vue b/src/pages/SetupDatabase.vue new file mode 100644 index 00000000..122b548d --- /dev/null +++ b/src/pages/SetupDatabase.vue @@ -0,0 +1,211 @@ + + + + + diff --git a/src/router.js b/src/router.js index 35647511..41814bc8 100644 --- a/src/router.js +++ b/src/router.js @@ -19,6 +19,7 @@ import DockerHosts from "./components/settings/Docker.vue"; import MaintenanceDetails from "./pages/MaintenanceDetails.vue"; import ManageMaintenance from "./pages/ManageMaintenance.vue"; import Plugins from "./components/settings/Plugins.vue"; +import SetupDatabase from "./pages/SetupDatabase.vue"; // Settings - Sub Pages import Appearance from "./components/settings/Appearance.vue"; @@ -163,6 +164,10 @@ const routes = [ path: "/setup", component: Setup, }, + { + path: "/setup-database", + component: SetupDatabase, + }, { path: "/status-page", component: StatusPage, diff --git a/test/backend.spec.js b/test/backend.spec.js index 644a0fd0..66a8aac3 100644 --- a/test/backend.spec.js +++ b/test/backend.spec.js @@ -235,13 +235,13 @@ describe("The function filterAndJoin", () => { describe("Test uptimeKumaServer.getClientIP()", () => { it("should able to get a correct client IP", async () => { - Database.init({ + Database.initDataDir({ "data-dir": "./data/test" }); - if (! fs.existsSync(Database.path)) { + if (! fs.existsSync(Database.sqlitePath)) { log.info("server", "Copying Database"); - fs.copyFileSync(Database.templatePath, Database.path); + fs.copyFileSync(Database.templatePath, Database.sqlitePath); } await Database.connect(true); From 40569519155e563357b2d9c0a0d4ce34fd5c9bb2 Mon Sep 17 00:00:00 2001 From: Louis Lam Date: Sat, 11 Feb 2023 22:21:06 +0800 Subject: [PATCH 11/31] WIP: building database in knex.js --- db/knex_migrations/README.md | 46 ++++++++ db/kuma.js | 213 +++++++++++++++++++++++++++++++++++ package-lock.json | 1 + package.json | 3 +- server/database.js | 80 ++++++++++++- server/model/maintenance.js | 6 +- server/server.js | 12 +- server/setup-database.js | 3 + server/uptime-kuma-server.js | 2 +- 9 files changed, 350 insertions(+), 16 deletions(-) create mode 100644 db/knex_migrations/README.md create mode 100644 db/kuma.js diff --git a/db/knex_migrations/README.md b/db/knex_migrations/README.md new file mode 100644 index 00000000..bcad0468 --- /dev/null +++ b/db/knex_migrations/README.md @@ -0,0 +1,46 @@ +## Info + +https://knexjs.org/guide/migrations.html#knexfile-in-other-languages + + +## Template + +Filename: YYYYMMDDHHMMSS_name.js + +```js +exports.up = function(knex) { + +}; + +exports.down = function(knex) { + +}; + +// exports.config = { transaction: false }; +``` + +## Example + +20230211120000_create_users_products.js + +```js +exports.up = function(knex) { + return knex.schema + .createTable('users', function (table) { + table.increments('id'); + table.string('first_name', 255).notNullable(); + table.string('last_name', 255).notNullable(); + }) + .createTable('products', function (table) { + table.increments('id'); + table.decimal('price').notNullable(); + table.string('name', 1000).notNullable(); + }); +}; + +exports.down = function(knex) { + return knex.schema + .dropTable("products") + .dropTable("users"); +}; +``` diff --git a/db/kuma.js b/db/kuma.js new file mode 100644 index 00000000..8d1f3309 --- /dev/null +++ b/db/kuma.js @@ -0,0 +1,213 @@ +const { R } = require("redbean-node"); +const { log, sleep } = require("../src/util"); + +/** + * DO NOT ADD ANYTHING HERE! + * IF YOU NEED TO ADD SOMETHING, ADD IT TO ./db/knex_migrations + * @returns {Promise} + */ +async function createTables() { + log.info("mariadb", "Creating basic tables for MariaDB"); + const knex = R.knex; + + // Up to `patch-add-google-analytics-status-page-tag.sql` + + // docker_host + await knex.schema.createTable("docker_host", (table) => { + table.increments("id"); + table.integer("user_id").unsigned().notNullable(); + table.string("docker_daemon", 255); + table.string("docker_type", 255); + table.string("name", 255); + }); + + // group + await knex.schema.createTable("group", (table) => { + table.increments("id"); + table.string("name", 255).notNullable(); + table.datetime("created_date").notNullable().defaultTo(knex.fn.now()); + table.boolean("public").notNullable().defaultTo(false); + table.boolean("active").notNullable().defaultTo(true); + table.integer("weight").notNullable().defaultTo(1000); + }); + + // proxy + await knex.schema.createTable("proxy", (table) => { + table.increments("id"); + table.integer("user_id").unsigned().notNullable(); + table.string("protocol", 10).notNullable(); + table.string("host", 255).notNullable(); + table.smallint("port").notNullable(); // Maybe a issue with MariaDB, need migration to int + table.boolean("auth").notNullable(); + table.string("username", 255).nullable(); + table.string("password", 255).nullable(); + table.boolean("active").notNullable().defaultTo(true); + table.boolean("default").notNullable().defaultTo(false); + table.datetime("created_date").notNullable().defaultTo(knex.fn.now()); + + table.index("user_id", "proxy_user_id"); + }); + + // user + await knex.schema.createTable("user", (table) => { + table.increments("id"); + table.string("username", 255).notNullable().unique().collate("utf8_general_ci"); + table.string("password", 255); + table.boolean("active").notNullable().defaultTo(true); + table.string("timezone", 150); + table.string("twofa_secret", 64); + table.boolean("twofa_status").notNullable().defaultTo(false); + table.string("twofa_last_token", 6); + }); + + // monitor + await knex.schema.createTable("monitor", (table) => { + table.increments("id"); + table.string("name", 150); + table.boolean("active").notNullable().defaultTo(true); + table.integer("user_id").unsigned() + .references("id").inTable("user") + .onDelete("SET NULL") + .onUpdate("CASCADE"); + table.integer("interval").notNullable().defaultTo(20); + table.text("url"); + table.string("type", 20); + table.integer("weight").defaultTo(2000); + table.string("hostname", 255); + table.integer("port"); + table.datetime("created_date").notNullable().defaultTo(knex.fn.now()); + table.string("keyword", 255); + table.integer("maxretries").notNullable().defaultTo(0); + table.boolean("ignore_tls").notNullable().defaultTo(false); + table.boolean("upside_down").notNullable().defaultTo(false); + table.integer("maxredirects").notNullable().defaultTo(10); + table.text("accepted_statuscodes_json").notNullable().defaultTo("[\"200-299\"]"); + table.string("dns_resolve_type", 5); + table.string("dns_resolve_server", 255); + table.string("dns_last_result", 255); + table.integer("retry_interval").notNullable().defaultTo(0); + table.string("push_token", 20).defaultTo(null); + table.text("method").notNullable().defaultTo("GET"); + table.text("body").defaultTo(null); + table.text("headers").defaultTo(null); + table.text("basic_auth_user").defaultTo(null); + table.text("basic_auth_pass").defaultTo(null); + table.integer("docker_host").unsigned() + .references("id").inTable("docker_host"); + table.string("docker_container", 255); + table.integer("proxy_id").unsigned() + .references("id").inTable("proxy"); + table.boolean("expiry_notification").defaultTo(true); + table.text("mqtt_topic"); + table.string("mqtt_success_message", 255); + table.string("mqtt_username", 255); + table.string("mqtt_password", 255); + table.string("database_connection_string", 2000); + table.text("database_query"); + table.string("auth_method", 250); + table.text("auth_domain"); + table.text("auth_workstation"); + table.string("grpc_url", 255).defaultTo(null); + table.text("grpc_protobuf").defaultTo(null); + table.text("grpc_body").defaultTo(null); + table.text("grpc_metadata").defaultTo(null); + table.text("grpc_method").defaultTo(null); + table.text("grpc_service_name").defaultTo(null); + table.boolean("grpc_enable_tls").notNullable().defaultTo(false); + table.string("radius_username", 255); + table.string("radius_password", 255); + table.string("radius_calling_station_id", 50); + table.string("radius_called_station_id", 50); + table.string("radius_secret", 255); + table.integer("resend_interval").notNullable().defaultTo(0); + table.integer("packet_size").notNullable().defaultTo(56); + table.string("game", 255); + }); + + // heartbeat + await knex.schema.createTable("heartbeat", (table) => { + table.increments("id"); + table.boolean("important").notNullable().defaultTo(false); + table.integer("monitor_id").unsigned().notNullable() + .references("id").inTable("monitor") + .onDelete("CASCADE") + .onUpdate("CASCADE"); + table.smallint("status").notNullable(); + + table.text("msg"); + table.datetime("time").notNullable(); + table.integer("ping"); + table.integer("duration").notNullable().defaultTo(0); + table.integer("down_count").notNullable().defaultTo(0); + + table.index("important"); + table.index([ "monitor_id", "time" ], "monitor_time_index"); + table.index("monitor_id"); + table.index([ "monitor_id", "important", "time" ], "monitor_important_time_index"); + }); + + // incident + await knex.schema.createTable("incident", (table) => { + table.increments("id"); + table.string("title", 255).notNullable(); + table.text("content", 255).notNullable(); + table.string("style", 30).notNullable().defaultTo("warning"); + table.datetime("created_date").notNullable().defaultTo(knex.fn.now()); + table.datetime("last_updated_date"); + table.boolean("pin").notNullable().defaultTo(true); + table.boolean("active").notNullable().defaultTo(true); + table.integer("status_page_id").unsigned(); + }); + + // maintenance + await knex.schema.createTable("maintenance", (table) => { + table.increments("id"); + table.string("title", 150).notNullable(); + table.text("description").notNullable(); + table.integer("user_id").unsigned() + .references("id").inTable("user") + .onDelete("SET NULL") + .onUpdate("CASCADE"); + table.boolean("active").notNullable().defaultTo(true); + table.string("strategy", 50).notNullable().defaultTo("single"); + table.datetime("start_date"); + table.datetime("end_date"); + table.time("start_time"); + table.time("end_time"); + table.string("weekdays", 250).defaultTo("[]"); + table.text("days_of_month").defaultTo("[]"); + table.integer("interval_day"); + + table.index("active"); + table.index([ "strategy", "active" ], "manual_active"); + table.index("user_id", "maintenance_user_id"); + }); + + // maintenance_status_page + // maintenance_timeslot + // monitor_group + // monitor_maintenance + // monitor_notification + // monitor_tag + // monitor_tls_info + // notification + // notification_sent_history + // setting + await knex.schema.createTable("setting", (table) => { + table.increments("id"); + table.string("key", 200).notNullable().unique().collate("utf8_general_ci"); + table.text("value"); + table.string("type", 20); + }); + + // status_page + // status_page_cname + // tag + // user + + log.info("mariadb", "Created basic tables for MariaDB"); +} + +module.exports = { + createTables, +}; diff --git a/package-lock.json b/package-lock.json index 5d172288..ed3a8527 100644 --- a/package-lock.json +++ b/package-lock.json @@ -40,6 +40,7 @@ "jsesc": "~3.0.2", "jsonwebtoken": "~9.0.0", "jwt-decode": "~3.1.2", + "knex": "^2.4.2", "limiter": "~2.1.0", "mongodb": "~4.13.0", "mqtt": "~4.3.7", diff --git a/package.json b/package.json index 31276c43..c19577e7 100644 --- a/package.json +++ b/package.json @@ -95,6 +95,7 @@ "jsesc": "~3.0.2", "jsonwebtoken": "~9.0.0", "jwt-decode": "~3.1.2", + "knex": "^2.4.2", "limiter": "~2.1.0", "mongodb": "~4.13.0", "mqtt": "~4.3.7", @@ -148,8 +149,8 @@ "eslint": "~8.14.0", "eslint-plugin-vue": "~8.7.1", "favico.js": "~0.3.10", - "marked": "~4.2.5", "jest": "~27.2.5", + "marked": "~4.2.5", "postcss-html": "~1.5.0", "postcss-rtlcss": "~3.7.2", "postcss-scss": "~4.0.4", diff --git a/server/database.js b/server/database.js index 954882a6..b28bdfd0 100644 --- a/server/database.js +++ b/server/database.js @@ -38,11 +38,13 @@ class Database { static backupPath = null; /** + * SQLite only * Add patch filename in key * Values: * true: Add it regardless of order * false: Do nothing * { parents: []}: Need parents before add it + * @deprecated */ static patchList = { "patch-setting-value-type.sql": true, @@ -82,6 +84,10 @@ class Database { static noReject = true; + static dbConfig = {}; + + static knexMigrationsPath = "./db/knex_migrations"; + /** * Initialize the data directory * @param {Object} args Arguments to initialize DB with @@ -144,17 +150,23 @@ class Database { let dbConfig; try { dbConfig = this.readDBConfig(); + Database.dbConfig = dbConfig; } catch (err) { log.warn("db", err.message); dbConfig = { type: "sqlite", - //type: "embedded-mariadb", }; } let config = {}; if (dbConfig.type === "sqlite") { + + if (! fs.existsSync(Database.sqlitePath)) { + log.info("server", "Copying Database"); + fs.copyFileSync(Database.templatePath, Database.sqlitePath); + } + const Dialect = require("knex/lib/dialects/sqlite3/index.js"); Dialect.prototype._driver = () => require("@louislam/sqlite3"); @@ -173,6 +185,17 @@ class Database { acquireTimeoutMillis: acquireConnectionTimeout, } }; + } else if (dbConfig.type === "mariadb") { + config = { + client: "mysql2", + connection: { + host: dbConfig.hostname, + port: dbConfig.port, + user: dbConfig.username, + password: dbConfig.password, + database: dbConfig.dbName, + } + }; } else if (dbConfig.type === "embedded-mariadb") { let embeddedMariaDB = EmbeddedMariaDB.getInstance(); await embeddedMariaDB.start(); @@ -182,13 +205,22 @@ class Database { connection: { socketPath: embeddedMariaDB.socketPath, user: "node", - database: "kuma" + database: "kuma", } }; } else { throw new Error("Unknown Database type: " + dbConfig.type); } + // Set to utf8mb4 for MariaDB + if (dbConfig.type.endsWith("mariadb")) { + config.pool = { + afterCreate(conn, done) { + conn.query("SET CHARACTER SET utf8mb4;", (err) => done(err, conn)); + }, + }; + } + const knexInstance = knex(config); R.setup(knexInstance); @@ -204,6 +236,14 @@ class Database { await R.autoloadModels("./server/model"); } + if (dbConfig.type === "sqlite") { + await this.initSQLite(testMode, noLog); + } else if (dbConfig.type.endsWith("mariadb")) { + await this.initMariaDB(); + } + } + + static async initSQLite(testMode, noLog) { await R.exec("PRAGMA foreign_keys = ON"); if (testMode) { // Change to MEMORY @@ -228,8 +268,36 @@ class Database { } } - /** Patch the database */ + static async initMariaDB() { + log.debug("db", "Checking if MariaDB database exists..."); + + let hasTable = await R.hasTable("docker_host"); + if (!hasTable) { + const { createTables } = require("../db/kuma"); + await createTables(); + } else { + log.debug("db", "MariaDB database already exists"); + } + } + static async patch() { + if (Database.dbConfig.type === "sqlite") { + await this.patchSqlite(); + } + + // TODO: Using knex migrations + // https://knexjs.org/guide/migrations.html + // https://gist.github.com/NigelEarle/70db130cc040cc2868555b29a0278261 + await R.knex.migrate.latest({ + directory: Database.knexMigrationsPath, + }); + } + + /** + * Patch the database for SQLite + * @deprecated + */ + static async patchSqlite() { let version = parseInt(await setting("database_version")); if (! version) { @@ -275,17 +343,18 @@ class Database { } } - await this.patch2(); + await this.patchSqlite2(); await this.migrateNewStatusPage(); } /** * Patch DB using new process * Call it from patch() only + * @deprecated * @private * @returns {Promise} */ - static async patch2() { + static async patchSqlite2() { log.info("db", "Database Patch 2.0 Process"); let databasePatchedFiles = await setting("databasePatchedFiles"); @@ -321,6 +390,7 @@ class Database { } /** + * SQlite only * Migrate status page value in setting to "status_page" table * @returns {Promise} */ diff --git a/server/model/maintenance.js b/server/model/maintenance.js index 45db63d1..6c371fd2 100644 --- a/server/model/maintenance.js +++ b/server/model/maintenance.js @@ -212,8 +212,8 @@ class Maintenance extends BeanModel { static getActiveMaintenanceSQLCondition() { return ` ( - (maintenance_timeslot.start_date <= DATETIME('now') - AND maintenance_timeslot.end_date >= DATETIME('now') + (maintenance_timeslot.start_date <= CURRENT_TIMESTAMP + AND maintenance_timeslot.end_date >= CURRENT_TIMESTAMP AND maintenance.active = 1) OR (maintenance.strategy = 'manual' AND active = 1) @@ -228,7 +228,7 @@ class Maintenance extends BeanModel { static getActiveAndFutureMaintenanceSQLCondition() { return ` ( - ((maintenance_timeslot.end_date >= DATETIME('now') + ((maintenance_timeslot.end_date >= CURRENT_TIMESTAMP AND maintenance.active = 1) OR (maintenance.strategy = 'manual' AND active = 1)) diff --git a/server/server.js b/server/server.js index 627c586b..8d9ea76d 100644 --- a/server/server.js +++ b/server/server.js @@ -180,7 +180,12 @@ let needSetup = false; } // Connect to database - await initDatabase(testMode); + try { + await initDatabase(testMode); + } catch (e) { + log.error("server", "Failed to prepare your database: " + e.message); + process.exit(1); + } // Database should be ready now await server.initAfterDatabaseReady(); @@ -1658,11 +1663,6 @@ async function afterLogin(socket, user) { * @returns {Promise} */ async function initDatabase(testMode = false) { - if (! fs.existsSync(Database.sqlitePath)) { - log.info("server", "Copying Database"); - fs.copyFileSync(Database.templatePath, Database.sqlitePath); - } - log.info("server", "Connecting to the Database"); await Database.connect(testMode); log.info("server", "Connected"); diff --git a/server/setup-database.js b/server/setup-database.js index 53d68fd0..e40649fa 100644 --- a/server/setup-database.js +++ b/server/setup-database.js @@ -34,6 +34,9 @@ class SetupDatabase { try { dbConfig = Database.readDBConfig(); + log.info("setup-database", "db-config.json is found and is valid"); + this.needSetup = false; + } catch (e) { log.info("setup-database", "db-config.json is not found or invalid: " + e.message); diff --git a/server/uptime-kuma-server.js b/server/uptime-kuma-server.js index 0573f0d8..6ba11f59 100644 --- a/server/uptime-kuma-server.js +++ b/server/uptime-kuma-server.js @@ -272,7 +272,7 @@ class UptimeKumaServer { /** Load the timeslots for maintenance */ async generateMaintenanceTimeslots() { - let list = await R.find("maintenance_timeslot", " generated_next = 0 AND start_date <= DATETIME('now') "); + let list = await R.find("maintenance_timeslot", " generated_next = 0 AND start_date <= CURRENT_TIMESTAMP "); for (let maintenanceTimeslot of list) { let maintenance = await maintenanceTimeslot.maintenance; From f2633a5d0163b898b8214bb1308e76e359a425c6 Mon Sep 17 00:00:00 2001 From: Louis Lam Date: Sun, 12 Feb 2023 03:44:15 +0800 Subject: [PATCH 12/31] Finished knex_init_db.js --- db/{kuma.js => knex_init_db.js} | 157 ++++++++++++++++++++++++++++++-- server/database.js | 4 +- 2 files changed, 153 insertions(+), 8 deletions(-) rename db/{kuma.js => knex_init_db.js} (57%) diff --git a/db/kuma.js b/db/knex_init_db.js similarity index 57% rename from db/kuma.js rename to db/knex_init_db.js index 8d1f3309..a909ed02 100644 --- a/db/kuma.js +++ b/db/knex_init_db.js @@ -37,7 +37,7 @@ async function createTables() { table.integer("user_id").unsigned().notNullable(); table.string("protocol", 10).notNullable(); table.string("host", 255).notNullable(); - table.smallint("port").notNullable(); // Maybe a issue with MariaDB, need migration to int + table.smallint("port").notNullable(); // TODO: Maybe a issue with MariaDB, need migration to int table.boolean("auth").notNullable(); table.string("username", 255).nullable(); table.string("password", 255).nullable(); @@ -183,15 +183,153 @@ async function createTables() { table.index("user_id", "maintenance_user_id"); }); + // status_page + await knex.schema.createTable("status_page", (table) => { + table.increments("id"); + table.string("slug", 255).notNullable().unique().collate("utf8_general_ci"); + table.string("title", 255).notNullable(); + table.text("description"); + table.string("icon", 255).notNullable(); + table.string("theme", 30).notNullable(); + table.boolean("published").notNullable().defaultTo(true); + table.boolean("search_engine_index").notNullable().defaultTo(true); + table.boolean("show_tags").notNullable().defaultTo(false); + table.string("password"); + table.datetime("created_date").notNullable().defaultTo(knex.fn.now()); + table.datetime("modified_date").notNullable().defaultTo(knex.fn.now()); + table.text("footer_text"); + table.text("custom_css"); + table.boolean("show_powered_by").notNullable().defaultTo(true); + table.string("google_analytics_tag_id"); + }); + // maintenance_status_page + await knex.schema.createTable("maintenance_status_page", (table) => { + table.increments("id"); + + table.integer("status_page_id").unsigned().notNullable() + .references("id").inTable("status_page") + .onDelete("CASCADE") + .onUpdate("CASCADE"); + + table.integer("maintenance_id").unsigned().notNullable() + .references("id").inTable("maintenance") + .onDelete("CASCADE") + .onUpdate("CASCADE"); + }); + // maintenance_timeslot + await knex.schema.createTable("maintenance_timeslot", (table) => { + table.increments("id"); + table.integer("maintenance_id").unsigned().notNullable() + .references("id").inTable("maintenance") + .onDelete("CASCADE") + .onUpdate("CASCADE"); + table.datetime("start_date").notNullable(); + table.datetime("end_date"); + table.boolean("generated_next").defaultTo(false); + + table.index("maintenance_id"); + table.index([ "maintenance_id", "start_date", "end_date" ], "active_timeslot_index"); + table.index("generated_next", "generated_next_index"); + }); + // monitor_group + await knex.schema.createTable("monitor_group", (table) => { + table.increments("id"); + table.integer("monitor_id").unsigned().notNullable() + .references("id").inTable("monitor") + .onDelete("CASCADE") + .onUpdate("CASCADE"); + table.integer("group_id").unsigned().notNullable() + .references("id").inTable("group") + .onDelete("CASCADE") + .onUpdate("CASCADE"); + table.integer("weight").notNullable().defaultTo(1000); + table.boolean("send_url").notNullable().defaultTo(false); + + table.index([ "monitor_id", "group_id" ], "fk"); + }); // monitor_maintenance - // monitor_notification - // monitor_tag - // monitor_tls_info + await knex.schema.createTable("monitor_maintenance", (table) => { + table.increments("id"); + table.integer("monitor_id").unsigned().notNullable() + .references("id").inTable("monitor") + .onDelete("CASCADE") + .onUpdate("CASCADE"); + table.integer("maintenance_id").unsigned().notNullable() + .references("id").inTable("maintenance") + .onDelete("CASCADE") + .onUpdate("CASCADE"); + + table.index("maintenance_id", "maintenance_id_index2"); + table.index("monitor_id", "monitor_id_index"); + }); + // notification + await knex.schema.createTable("notification", (table) => { + table.increments("id"); + table.string("name", 255); + table.string("config", 255); // TODO: should use TEXT! + table.boolean("active").notNullable().defaultTo(true); + table.integer("user_id").unsigned(); + table.boolean("is_default").notNullable().defaultTo(false); + }); + + // monitor_notification + await knex.schema.createTable("monitor_notification", (table) => { + table.increments("id").unsigned(); // TODO: no auto increment???? + table.integer("monitor_id").unsigned().notNullable() + .references("id").inTable("monitor") + .onDelete("CASCADE") + .onUpdate("CASCADE"); + table.integer("notification_id").unsigned().notNullable() + .references("id").inTable("notification") + .onDelete("CASCADE") + .onUpdate("CASCADE"); + + table.index([ "monitor_id", "notification_id" ], "monitor_notification_index"); + }); + + // tag + await knex.schema.createTable("tag", (table) => { + table.increments("id"); + table.string("name", 255).notNullable(); + table.string("color", 255).notNullable(); + table.datetime("created_date").notNullable().defaultTo(knex.fn.now()); + }); + + // monitor_tag + await knex.schema.createTable("monitor_tag", (table) => { + table.increments("id"); + table.integer("monitor_id").unsigned().notNullable() + .references("id").inTable("monitor") + .onDelete("CASCADE") + .onUpdate("CASCADE"); + table.integer("tag_id").unsigned().notNullable() + .references("id").inTable("tag") + .onDelete("CASCADE") + .onUpdate("CASCADE"); + table.text("value"); + }); + + // monitor_tls_info + await knex.schema.createTable("monitor_tls_info", (table) => { + table.increments("id"); + table.integer("monitor_id").unsigned().notNullable(); //TODO: no fk ? + table.text("info_json"); + }); + // notification_sent_history + await knex.schema.createTable("notification_sent_history", (table) => { + table.increments("id"); + table.string("type", 50).notNullable(); + table.integer("monitor_id").unsigned().notNullable(); + table.integer("days").notNullable(); + table.unique([ "type", "monitor_id", "days" ]); + table.index([ "type", "monitor_id", "days" ], "good_index"); + }); + // setting await knex.schema.createTable("setting", (table) => { table.increments("id"); @@ -200,10 +338,15 @@ async function createTables() { table.string("type", 20); }); - // status_page // status_page_cname - // tag - // user + await knex.schema.createTable("status_page_cname", (table) => { + table.increments("id"); + table.integer("status_page_id").unsigned() + .references("id").inTable("status_page") + .onDelete("CASCADE") + .onUpdate("CASCADE"); + table.string("domain").notNullable().unique().collate("utf8_general_ci"); + }); log.info("mariadb", "Created basic tables for MariaDB"); } diff --git a/server/database.js b/server/database.js index b28bdfd0..6c435b43 100644 --- a/server/database.js +++ b/server/database.js @@ -160,6 +160,8 @@ class Database { let config = {}; + log.info("db", `Database Type: ${dbConfig.type}`); + if (dbConfig.type === "sqlite") { if (! fs.existsSync(Database.sqlitePath)) { @@ -273,7 +275,7 @@ class Database { let hasTable = await R.hasTable("docker_host"); if (!hasTable) { - const { createTables } = require("../db/kuma"); + const { createTables } = require("../db/knex_init_db"); await createTables(); } else { log.debug("db", "MariaDB database already exists"); From 8d5679a8ab365886d09cc4b15768395a8fa43d17 Mon Sep 17 00:00:00 2001 From: Nelson Chan Date: Mon, 3 Apr 2023 19:35:31 +0800 Subject: [PATCH 13/31] Fix: Create database before connect --- server/database.js | 14 ++++++++++++++ 1 file changed, 14 insertions(+) diff --git a/server/database.js b/server/database.js index 6c435b43..f0403566 100644 --- a/server/database.js +++ b/server/database.js @@ -7,6 +7,7 @@ const knex = require("knex"); const { PluginsManager } = require("./plugins-manager"); const path = require("path"); const { EmbeddedMariaDB } = require("./embedded-mariadb"); +const mysql = require("mysql2/promise"); /** * Database & App Data Folder @@ -188,6 +189,19 @@ class Database { } }; } else if (dbConfig.type === "mariadb") { + if (!/^\w+$/.test(dbConfig.dbName)) { + throw Error("Invalid Database name"); + } + + const connection = await mysql.createConnection({ + host: dbConfig.hostname, + port: dbConfig.port, + user: dbConfig.username, + password: dbConfig.password, + }); + + await connection.execute("CREATE DATABASE IF NOT EXISTS " + dbConfig.dbName + " CHARACTER SET utf8mb4"); + config = { client: "mysql2", connection: { From 38fab198bb3240ea17b5965e07fa08424706a63a Mon Sep 17 00:00:00 2001 From: Nelson Chan Date: Mon, 3 Apr 2023 19:36:07 +0800 Subject: [PATCH 14/31] Fix: Fix user count check --- server/server.js | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/server/server.js b/server/server.js index 8d9ea76d..021965a0 100644 --- a/server/server.js +++ b/server/server.js @@ -614,7 +614,7 @@ let needSetup = false; throw new Error("Password is too weak. It should contain alphabetic and numeric characters. It must be at least 6 characters in length."); } - if ((await R.count("user")) !== 0) { + if ((await R.knex("user").count("id as count").first()).count !== 0) { throw new Error("Uptime Kuma has been initialized. If you want to run setup again, please delete the database."); } @@ -1683,7 +1683,7 @@ async function initDatabase(testMode = false) { } // If there is no record in user table, it is a new Uptime Kuma instance, need to setup - if ((await R.count("user")) === 0) { + if ((await R.knex("user").count("id as count").first()).count === 0) { log.info("server", "No user, need setup"); needSetup = true; } From f70b97181068932e16e581121011757450dc29a9 Mon Sep 17 00:00:00 2001 From: Nelson Chan <3271800+chakflying@users.noreply.github.com> Date: Sun, 21 May 2023 15:42:13 +0800 Subject: [PATCH 15/31] Fix: Improve error message Co-authored-by: Frank Elsinga --- server/database.js | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/server/database.js b/server/database.js index f0403566..dbf5a6d6 100644 --- a/server/database.js +++ b/server/database.js @@ -190,7 +190,7 @@ class Database { }; } else if (dbConfig.type === "mariadb") { if (!/^\w+$/.test(dbConfig.dbName)) { - throw Error("Invalid Database name"); + throw Error("Invalid database name. A database name can only consist of letters, numbers and underscores"); } const connection = await mysql.createConnection({ From b2a1bd52143a8c4996f972e173d360454bea52b5 Mon Sep 17 00:00:00 2001 From: Louis Lam Date: Sun, 12 Feb 2023 14:56:43 +0800 Subject: [PATCH 16/31] WIP --- db/knex_init_db.js | 10 ++++++---- 1 file changed, 6 insertions(+), 4 deletions(-) diff --git a/db/knex_init_db.js b/db/knex_init_db.js index a909ed02..e838345a 100644 --- a/db/knex_init_db.js +++ b/db/knex_init_db.js @@ -1,16 +1,18 @@ const { R } = require("redbean-node"); -const { log, sleep } = require("../src/util"); +const { log } = require("../src/util"); /** - * DO NOT ADD ANYTHING HERE! - * IF YOU NEED TO ADD SOMETHING, ADD IT TO ./db/knex_migrations + * ⚠️⚠️⚠️⚠️⚠️⚠️ DO NOT ADD ANYTHING HERE! + * IF YOU NEED TO ADD FIELDS, ADD IT TO ./db/knex_migrations + * See ./db/knex_migrations/README.md for more information * @returns {Promise} */ async function createTables() { log.info("mariadb", "Creating basic tables for MariaDB"); const knex = R.knex; - // Up to `patch-add-google-analytics-status-page-tag.sql` + // Up to `patch-add-google-analytics-status-page-tag.sql` for now + // TODO: Should check later if it is really the final patch sql file. // docker_host await knex.schema.createTable("docker_host", (table) => { From 5388a37a2612c1fbd5b1bdd63674cb25ac572425 Mon Sep 17 00:00:00 2001 From: Louis Lam Date: Sun, 12 Feb 2023 15:14:41 +0800 Subject: [PATCH 17/31] Fix port NaN not working in MariaDB --- server/server.js | 5 +++++ 1 file changed, 5 insertions(+) diff --git a/server/server.js b/server/server.js index 021965a0..613b98f5 100644 --- a/server/server.js +++ b/server/server.js @@ -714,6 +714,11 @@ let needSetup = false; bean.game = monitor.game; bean.maxretries = monitor.maxretries; bean.port = parseInt(monitor.port); + + if (isNaN(bean.port)) { + bean.port = null; + } + bean.keyword = monitor.keyword; bean.ignoreTls = monitor.ignoreTls; bean.expiryNotification = monitor.expiryNotification; From 2e2747fb52069eb2b675ce37e15b1403457ffac0 Mon Sep 17 00:00:00 2001 From: Louis Lam Date: Sun, 12 Feb 2023 16:59:07 +0800 Subject: [PATCH 18/31] Handling DATE_ADD --- db/knex_init_db.js | 1 + server/database.js | 9 +++++++++ server/jobs/clear-old-data.js | 7 +++++-- server/model/monitor.js | 4 +++- server/routers/api-router.js | 9 +++++++-- server/server.js | 6 ++++-- server/util-server.js | 5 +++++ 7 files changed, 34 insertions(+), 7 deletions(-) diff --git a/db/knex_init_db.js b/db/knex_init_db.js index e838345a..fbf8d2b9 100644 --- a/db/knex_init_db.js +++ b/db/knex_init_db.js @@ -31,6 +31,7 @@ async function createTables() { table.boolean("public").notNullable().defaultTo(false); table.boolean("active").notNullable().defaultTo(true); table.integer("weight").notNullable().defaultTo(1000); + table.integer("status_page_id").unsigned(); }); // proxy diff --git a/server/database.js b/server/database.js index dbf5a6d6..98536ca4 100644 --- a/server/database.js +++ b/server/database.js @@ -683,6 +683,15 @@ class Database { static async shrink() { await R.exec("VACUUM"); } + + static sqlHourOffset() { + if (this.dbConfig.client === "sqlite3") { + return "DATETIME('now', ? || ' hours')"; + } else { + return "DATE_ADD(NOW(), INTERVAL ? HOUR)"; + } + } + } module.exports = Database; diff --git a/server/jobs/clear-old-data.js b/server/jobs/clear-old-data.js index ed80b0f7..516add2c 100644 --- a/server/jobs/clear-old-data.js +++ b/server/jobs/clear-old-data.js @@ -1,6 +1,7 @@ const { log, exit, connectDb } = require("./util-worker"); const { R } = require("redbean-node"); const { setSetting, setting } = require("../util-server"); +const Database = require("../database"); const DEFAULT_KEEP_PERIOD = 180; @@ -31,10 +32,12 @@ const DEFAULT_KEEP_PERIOD = 180; log(`Clearing Data older than ${parsedPeriod} days...`); + const sqlHourOffset = Database.sqlHourOffset(); + try { await R.exec( - "DELETE FROM heartbeat WHERE time < DATETIME('now', '-' || ? || ' days') ", - [ parsedPeriod ] + "DELETE FROM heartbeat WHERE time < " + sqlHourOffset, + [ parsedPeriod * -24 ] ); } catch (e) { log(`Failed to clear old data: ${e.message}`); diff --git a/server/model/monitor.js b/server/model/monitor.js index 4cbb56e1..6a73091c 100644 --- a/server/model/monitor.js +++ b/server/model/monitor.js @@ -19,6 +19,7 @@ const { DockerHost } = require("../docker"); const Maintenance = require("./maintenance"); const { UptimeCacheList } = require("../uptime-cache-list"); const Gamedig = require("gamedig"); +const Database = require("../database"); /** * status: @@ -935,11 +936,12 @@ class Monitor extends BeanModel { */ static async sendAvgPing(duration, io, monitorID, userID) { const timeLogger = new TimeLogger(); + const sqlHourOffset = Database.sqlHourOffset(); let avgPing = parseInt(await R.getCell(` SELECT AVG(ping) FROM heartbeat - WHERE time > DATETIME('now', ? || ' hours') + WHERE time > ${sqlHourOffset} AND ping IS NOT NULL AND monitor_id = ? `, [ -duration, diff --git a/server/routers/api-router.js b/server/routers/api-router.js index 665163ae..7dafaa3a 100644 --- a/server/routers/api-router.js +++ b/server/routers/api-router.js @@ -9,6 +9,7 @@ const StatusPage = require("../model/status_page"); const { UptimeKumaServer } = require("../uptime-kuma-server"); const { makeBadge } = require("badge-maker"); const { badgeConstants } = require("../config"); +const Database = require("../database"); let router = express.Router(); @@ -268,10 +269,12 @@ router.get("/api/badge/:id/ping/:duration?", cache("5 minutes"), async (request, const requestedDuration = Math.min(request.params.duration ? parseInt(request.params.duration, 10) : 24, 720); const overrideValue = value && parseFloat(value); + const sqlHourOffset = Database.sqlHourOffset(); + const publicAvgPing = parseInt(await R.getCell(` SELECT AVG(ping) FROM monitor_group, \`group\`, heartbeat WHERE monitor_group.group_id = \`group\`.id - AND heartbeat.time > DATETIME('now', ? || ' hours') + AND heartbeat.time > ${sqlHourOffset} AND heartbeat.ping IS NOT NULL AND public = 1 AND heartbeat.monitor_id = ? @@ -334,10 +337,12 @@ router.get("/api/badge/:id/avg-response/:duration?", cache("5 minutes"), async ( ); const overrideValue = value && parseFloat(value); + const sqlHourOffset = Database.sqlHourOffset(); + const publicAvgPing = parseInt(await R.getCell(` SELECT AVG(ping) FROM monitor_group, \`group\`, heartbeat WHERE monitor_group.group_id = \`group\`.id - AND heartbeat.time > DATETIME('now', ? || ' hours') + AND heartbeat.time > ${sqlHourOffset} AND heartbeat.ping IS NOT NULL AND public = 1 AND heartbeat.monitor_id = ? diff --git a/server/server.js b/server/server.js index 613b98f5..e93245de 100644 --- a/server/server.js +++ b/server/server.js @@ -831,15 +831,17 @@ let needSetup = false; throw new Error("Invalid period."); } + const sqlHourOffset = Database.sqlHourOffset(); + let list = await R.getAll(` SELECT * FROM heartbeat WHERE monitor_id = ? - AND time > DATETIME('now', '-' || ? || ' hours') + AND time > ${sqlHourOffset} ORDER BY time ASC `, [ monitorID, - period, + -period, ]); callback({ diff --git a/server/util-server.js b/server/util-server.js index 615edcbc..198ada51 100644 --- a/server/util-server.js +++ b/server/util-server.js @@ -27,6 +27,9 @@ const { }, } = require("node-radius-utils"); const dayjs = require("dayjs"); +const readline = require("readline"); +const rl = readline.createInterface({ input: process.stdin, + output: process.stdout }); const isWindows = process.platform === /^win/.test(process.platform); @@ -859,3 +862,5 @@ module.exports.grpcQuery = async (options) => { }); }; + +module.exports.prompt = (query) => new Promise((resolve) => rl.question(query, resolve)); From 7975caf29e51c4e434d026a45b4fd0e27574aff1 Mon Sep 17 00:00:00 2001 From: Louis Lam Date: Fri, 30 Jun 2023 17:26:37 +0800 Subject: [PATCH 19/31] Update db migration and dockerfile --- db/knex_init_db.js | 1 - .../2023-06-30-1348-http-body-encoding.js | 22 +++++++++++++ ...2023-06-30-1354-add-description-monitor.js | 12 +++++++ .../2023-06-30-1357-api-key-table.js | 30 +++++++++++++++++ .../2023-06-30-1400-monitor-tls.js | 25 ++++++++++++++ .../2023-06-30-1401-maintenance-cron.js | 25 ++++++++++++++ .../2023-06-30-1413-add-parent-monitor.js | 18 ++++++++++ db/knex_migrations/README.md | 6 +++- docker/debian-base.dockerfile | 33 +++++++++---------- server/database.js | 24 +++++++++++--- 10 files changed, 171 insertions(+), 25 deletions(-) create mode 100644 db/knex_migrations/2023-06-30-1348-http-body-encoding.js create mode 100644 db/knex_migrations/2023-06-30-1354-add-description-monitor.js create mode 100644 db/knex_migrations/2023-06-30-1357-api-key-table.js create mode 100644 db/knex_migrations/2023-06-30-1400-monitor-tls.js create mode 100644 db/knex_migrations/2023-06-30-1401-maintenance-cron.js create mode 100644 db/knex_migrations/2023-06-30-1413-add-parent-monitor.js diff --git a/db/knex_init_db.js b/db/knex_init_db.js index fbf8d2b9..c5ea0397 100644 --- a/db/knex_init_db.js +++ b/db/knex_init_db.js @@ -11,7 +11,6 @@ async function createTables() { log.info("mariadb", "Creating basic tables for MariaDB"); const knex = R.knex; - // Up to `patch-add-google-analytics-status-page-tag.sql` for now // TODO: Should check later if it is really the final patch sql file. // docker_host diff --git a/db/knex_migrations/2023-06-30-1348-http-body-encoding.js b/db/knex_migrations/2023-06-30-1348-http-body-encoding.js new file mode 100644 index 00000000..c4cc7d94 --- /dev/null +++ b/db/knex_migrations/2023-06-30-1348-http-body-encoding.js @@ -0,0 +1,22 @@ +// ALTER TABLE monitor ADD http_body_encoding VARCHAR(25); +// UPDATE monitor SET http_body_encoding = 'json' WHERE (type = 'http' or type = 'keyword') AND http_body_encoding IS NULL; +exports.up = function (knex) { + return knex.schema.table("monitor", function (table) { + table.string("http_body_encoding", 25); + }).then(function () { + knex("monitor") + .where(function () { + this.where("type", "http").orWhere("type", "keyword"); + }) + .whereNull("http_body_encoding") + .update({ + http_body_encoding: "json", + }); + }); +}; + +exports.down = function (knex) { + return knex.schema.table("monitor", function (table) { + table.dropColumn("http_body_encoding"); + }); +}; diff --git a/db/knex_migrations/2023-06-30-1354-add-description-monitor.js b/db/knex_migrations/2023-06-30-1354-add-description-monitor.js new file mode 100644 index 00000000..4b291777 --- /dev/null +++ b/db/knex_migrations/2023-06-30-1354-add-description-monitor.js @@ -0,0 +1,12 @@ +// ALTER TABLE monitor ADD description TEXT default null; +exports.up = function (knex) { + return knex.schema.table("monitor", function (table) { + table.text("description").defaultTo(null); + }); +}; + +exports.down = function (knex) { + return knex.schema.table("monitor", function (table) { + table.dropColumn("description"); + }); +}; diff --git a/db/knex_migrations/2023-06-30-1357-api-key-table.js b/db/knex_migrations/2023-06-30-1357-api-key-table.js new file mode 100644 index 00000000..d22721ed --- /dev/null +++ b/db/knex_migrations/2023-06-30-1357-api-key-table.js @@ -0,0 +1,30 @@ +/* +CREATE TABLE [api_key] ( + [id] INTEGER NOT NULL PRIMARY KEY AUTOINCREMENT, + [key] VARCHAR(255) NOT NULL, + [name] VARCHAR(255) NOT NULL, + [user_id] INTEGER NOT NULL, + [created_date] DATETIME DEFAULT (DATETIME('now')) NOT NULL, + [active] BOOLEAN DEFAULT 1 NOT NULL, + [expires] DATETIME DEFAULT NULL, + CONSTRAINT FK_user FOREIGN KEY ([user_id]) REFERENCES [user]([id]) ON DELETE CASCADE ON UPDATE CASCADE +); + */ +exports.up = function (knex) { + return knex.schema.createTable("api_key", function (table) { + table.increments("id").primary(); + table.string("key", 255).notNullable(); + table.string("name", 255).notNullable(); + table.integer("user_id").unsigned().notNullable() + .references("id").inTable("user") + .onDelete("CASCADE") + .onUpdate("CASCADE"); + table.dateTime("created_date").defaultTo(knex.fn.now()).notNullable(); + table.boolean("active").defaultTo(1).notNullable(); + table.dateTime("expires").defaultTo(null); + }); +}; + +exports.down = function (knex) { + return knex.schema.dropTable("api_key"); +}; diff --git a/db/knex_migrations/2023-06-30-1400-monitor-tls.js b/db/knex_migrations/2023-06-30-1400-monitor-tls.js new file mode 100644 index 00000000..95d66bab --- /dev/null +++ b/db/knex_migrations/2023-06-30-1400-monitor-tls.js @@ -0,0 +1,25 @@ +/* +ALTER TABLE monitor + ADD tls_ca TEXT default null; + +ALTER TABLE monitor + ADD tls_cert TEXT default null; + +ALTER TABLE monitor + ADD tls_key TEXT default null; + */ +exports.up = function (knex) { + return knex.schema.table("monitor", function (table) { + table.text("tls_ca").defaultTo(null); + table.text("tls_cert").defaultTo(null); + table.text("tls_key").defaultTo(null); + }); +}; + +exports.down = function (knex) { + return knex.schema.table("monitor", function (table) { + table.dropColumn("tls_ca"); + table.dropColumn("tls_cert"); + table.dropColumn("tls_key"); + }); +}; diff --git a/db/knex_migrations/2023-06-30-1401-maintenance-cron.js b/db/knex_migrations/2023-06-30-1401-maintenance-cron.js new file mode 100644 index 00000000..51ae7a9b --- /dev/null +++ b/db/knex_migrations/2023-06-30-1401-maintenance-cron.js @@ -0,0 +1,25 @@ +/* +-- 999 characters. https://stackoverflow.com/questions/46134830/maximum-length-for-cron-job +DROP TABLE maintenance_timeslot; +ALTER TABLE maintenance ADD cron TEXT; +ALTER TABLE maintenance ADD timezone VARCHAR(255); +ALTER TABLE maintenance ADD duration INTEGER; + */ +exports.up = function (knex) { + return knex.schema + .dropTableIfExists("maintenance_timeslot") + .table("maintenance", function (table) { + table.text("cron"); + table.string("timezone", 255); + table.integer("duration"); + }); +}; + +exports.down = function (knex) { + return knex.schema + .table("maintenance", function (table) { + table.dropColumn("cron"); + table.dropColumn("timezone"); + table.dropColumn("duration"); + }); +}; diff --git a/db/knex_migrations/2023-06-30-1413-add-parent-monitor.js b/db/knex_migrations/2023-06-30-1413-add-parent-monitor.js new file mode 100644 index 00000000..2d417b8c --- /dev/null +++ b/db/knex_migrations/2023-06-30-1413-add-parent-monitor.js @@ -0,0 +1,18 @@ +/* +ALTER TABLE monitor + ADD parent INTEGER REFERENCES [monitor] ([id]) ON DELETE SET NULL ON UPDATE CASCADE; + */ +exports.up = function (knex) { + return knex.schema.table("monitor", function (table) { + table.integer("parent").unsigned() + .references("id").inTable("monitor") + .onDelete("SET NULL") + .onUpdate("CASCADE"); + }); +}; + +exports.down = function (knex) { + return knex.schema.table("monitor", function (table) { + table.dropColumn("parent"); + }); +}; diff --git a/db/knex_migrations/README.md b/db/knex_migrations/README.md index bcad0468..5789307b 100644 --- a/db/knex_migrations/README.md +++ b/db/knex_migrations/README.md @@ -21,7 +21,9 @@ exports.down = function(knex) { ## Example -20230211120000_create_users_products.js +YYYY-MM-DD-HHMM-create-users-products.js + +2023-06-30-1348-create-users-products.js ```js exports.up = function(knex) { @@ -44,3 +46,5 @@ exports.down = function(knex) { .dropTable("users"); }; ``` + +https://knexjs.org/guide/migrations.html#transactions-in-migrations diff --git a/docker/debian-base.dockerfile b/docker/debian-base.dockerfile index 003e3f9e..cde5cd2f 100644 --- a/docker/debian-base.dockerfile +++ b/docker/debian-base.dockerfile @@ -1,36 +1,33 @@ -# DON'T UPDATE TO node:14-bullseye-slim, see #372. # If the image changed, the second stage image should be changed too -FROM node:18-buster-slim AS base2-slim +FROM node:18-bullseye-slim AS base2-slim ARG TARGETPLATFORM -# Install Curl -# Install Apprise, add sqlite3 cli for debugging in the future, iputils-ping for ping, util-linux for setpriv -# Stupid python3 and python3-pip actually install a lot of useless things into Debian, specify --no-install-recommends to skip them, make the base even smaller than alpine! -RUN apt-get update && \ - apt-get --yes --no-install-recommends install python3 python3-pip python3-cryptography python3-six python3-yaml python3-click python3-markdown python3-requests python3-requests-oauthlib \ +RUN apt update && \ + apt --yes --no-install-recommends install python3 python3-pip python3-cryptography python3-six python3-yaml python3-click python3-markdown python3-requests python3-requests-oauthlib \ sqlite3 iputils-ping util-linux dumb-init git curl ca-certificates && \ pip3 --no-cache-dir install apprise==1.4.0 && \ rm -rf /var/lib/apt/lists/* && \ apt --yes autoremove # Install cloudflared -RUN set -eux && \ - mkdir -p --mode=0755 /usr/share/keyrings && \ - curl --fail --show-error --silent --location --insecure https://pkg.cloudflare.com/cloudflare-main.gpg --output /usr/share/keyrings/cloudflare-main.gpg && \ - echo 'deb [signed-by=/usr/share/keyrings/cloudflare-main.gpg] https://pkg.cloudflare.com/cloudflared buster main' | tee /etc/apt/sources.list.d/cloudflared.list && \ - apt-get update && \ - apt-get install --yes --no-install-recommends cloudflared && \ +RUN curl https://pkg.cloudflare.com/cloudflare-main.gpg --output /usr/share/keyrings/cloudflare-main.gpg && \ + echo 'deb [signed-by=/usr/share/keyrings/cloudflare-main.gpg] https://pkg.cloudflare.com/cloudflared bullseye main' | tee /etc/apt/sources.list.d/cloudflared.list && \ + apt update && \ + apt install --yes --no-install-recommends cloudflared && \ cloudflared version && \ rm -rf /var/lib/apt/lists/* && \ apt --yes autoremove +# Full Base Image +# MariaDB, Chromium and fonts +# Not working for armv7, so use the older version (10.5) of MariaDB from the debian repo +# curl -LsS https://r.mariadb.com/downloads/mariadb_repo_setup | bash -s -- --mariadb-server-version="mariadb-11.1" && \ FROM base2-slim AS base2 RUN apt update && \ - apt --yes --no-install-recommends install curl && \ - curl -LsS https://r.mariadb.com/downloads/mariadb_repo_setup | bash -s -- --mariadb-server-version="mariadb-10.11" && \ - apt --yes --no-install-recommends install mariadb-server && \ + apt --yes --no-install-recommends install chromium fonts-indic fonts-noto fonts-noto-cjk mariadb-server && \ apt --yes remove curl && \ rm -rf /var/lib/apt/lists/* && \ - apt --yes autoremove -RUN chown -R node:node /var/lib/mysql + apt --yes autoremove && \ + chown -R node:node /var/lib/mysql + ENV UPTIME_KUMA_ENABLE_EMBEDDED_MARIADB=1 diff --git a/server/database.js b/server/database.js index e362128f..6c5d71ee 100644 --- a/server/database.js +++ b/server/database.js @@ -76,7 +76,7 @@ class Database { "patch-api-key-table.sql": true, "patch-monitor-tls.sql": true, "patch-maintenance-cron.sql": true, - "patch-add-parent-monitor.sql": true, + "patch-add-parent-monitor.sql": true, // The last file so far converted to a knex migration file }; /** @@ -305,16 +305,30 @@ class Database { } static async patch() { + // Still need to keep this for old versions of Uptime Kuma if (Database.dbConfig.type === "sqlite") { await this.patchSqlite(); } - // TODO: Using knex migrations + // Using knex migrations // https://knexjs.org/guide/migrations.html // https://gist.github.com/NigelEarle/70db130cc040cc2868555b29a0278261 - await R.knex.migrate.latest({ - directory: Database.knexMigrationsPath, - }); + try { + await R.knex.migrate.latest({ + directory: Database.knexMigrationsPath, + }); + } catch (e) { + log.error("db", "Database migration failed"); + throw e; + } + } + + /** + * + * @returns {Promise} + */ + static async rollbackLatestPatch() { + } /** From d286c534bd0d3cf07135d82b227e223ccc777eea Mon Sep 17 00:00:00 2001 From: Louis Lam Date: Fri, 30 Jun 2023 22:17:07 +0800 Subject: [PATCH 20/31] Improve the setup database for embedded MariaDB --- docker/debian-base.dockerfile | 3 +- docker/docker-compose-dev.yml | 2 +- package.json | 6 +- server/server.js | 12 ++- server/setup-database.js | 38 ++++++- src/pages/Setup.vue | 2 + src/pages/SetupDatabase.vue | 188 +++++++++++++++++++--------------- 7 files changed, 160 insertions(+), 91 deletions(-) diff --git a/docker/debian-base.dockerfile b/docker/debian-base.dockerfile index cde5cd2f..f84aa32a 100644 --- a/docker/debian-base.dockerfile +++ b/docker/debian-base.dockerfile @@ -23,6 +23,7 @@ RUN curl https://pkg.cloudflare.com/cloudflare-main.gpg --output /usr/share/keyr # Not working for armv7, so use the older version (10.5) of MariaDB from the debian repo # curl -LsS https://r.mariadb.com/downloads/mariadb_repo_setup | bash -s -- --mariadb-server-version="mariadb-11.1" && \ FROM base2-slim AS base2 +ENV UPTIME_KUMA_ENABLE_EMBEDDED_MARIADB=1 RUN apt update && \ apt --yes --no-install-recommends install chromium fonts-indic fonts-noto fonts-noto-cjk mariadb-server && \ apt --yes remove curl && \ @@ -30,4 +31,4 @@ RUN apt update && \ apt --yes autoremove && \ chown -R node:node /var/lib/mysql -ENV UPTIME_KUMA_ENABLE_EMBEDDED_MARIADB=1 + diff --git a/docker/docker-compose-dev.yml b/docker/docker-compose-dev.yml index 5510b0d8..6eeb91e6 100644 --- a/docker/docker-compose-dev.yml +++ b/docker/docker-compose-dev.yml @@ -5,7 +5,7 @@ services: container_name: uptime-kuma-dev image: louislam/uptime-kuma:nightly2 volumes: - - ./data:/app/data + #- ./data:/app/data - ../server:/app/server ports: - "3001:3001" # : diff --git a/package.json b/package.json index 3ebb6250..9913a480 100644 --- a/package.json +++ b/package.json @@ -34,8 +34,8 @@ "build-docker-builder-go": "docker buildx build -f docker/builder-go.dockerfile --platform linux/amd64,linux/arm64,linux/arm/v7 -t louislam/uptime-kuma:builder-go . --push", "build-docker-slim": "node ./extra/env2arg.js docker buildx build -f docker/dockerfile --platform linux/amd64,linux/arm64,linux/arm/v7 -t louislam/uptime-kuma:2-slim -t louislam/uptime-kuma:$VERSION-slim --target release --build-arg BASE_IMAGE=louislam/uptime-kuma:base2-slim . --push", "build-docker-full": "node ./extra/env2arg.js docker buildx build -f docker/dockerfile --platform linux/amd64,linux/arm64,linux/arm/v7 -t louislam/uptime-kuma:2 -t louislam/uptime-kuma:$VERSION --target release . --push", - "build-docker-nightly": "npm run build && docker buildx build -f docker/dockerfile --platform linux/amd64,linux/arm64,linux/arm/v7 -t louislam/uptime-kuma:nightly2 --target nightly --build-arg . --push", - "build-docker-nightly-local": "docker build -f docker/dockerfile -t louislam/uptime-kuma:nightly2 --target nightly .", + "build-docker-nightly": "npm run build && docker buildx build -f docker/dockerfile --platform linux/amd64,linux/arm64,linux/arm/v7 -t louislam/uptime-kuma:nightly2 --target nightly . --push", + "build-docker-nightly-local": "npm run build && docker build -f docker/dockerfile -t louislam/uptime-kuma:nightly2 --target nightly .", "build-docker-pr-test": "docker buildx build -f docker/dockerfile --platform linux/amd64,linux/arm64 -t louislam/uptime-kuma:pr-test --target pr-test . --push", "upload-artifacts": "docker buildx build -f docker/dockerfile --platform linux/amd64 -t louislam/uptime-kuma:upload-artifact --build-arg VERSION --build-arg GITHUB_TOKEN --target upload-artifact . --progress plain", "setup": "git checkout 1.22.0 && npm ci --production && npm run download-dist", @@ -63,7 +63,7 @@ "deploy-demo-server": "node extra/deploy-demo-server.js", "sort-contributors": "node extra/sort-contributors.js", "quick-run-nightly": "docker run --rm --env NODE_ENV=development -p 3001:3001 louislam/uptime-kuma:nightly2", - "start-dev-container": "cd docker && docker-compose -f docker-compose-dev.yml up" + "start-dev-container": "cd docker && docker-compose -f docker-compose-dev.yml up --force-recreate" }, "dependencies": { "@grpc/grpc-js": "~1.7.3", diff --git a/server/server.js b/server/server.js index b9ecdd21..26dfe1c1 100644 --- a/server/server.js +++ b/server/server.js @@ -76,7 +76,9 @@ log.info("server", "Importing this project modules"); log.debug("server", "Importing Monitor"); const Monitor = require("./model/monitor"); log.debug("server", "Importing Settings"); -const { getSettings, setSettings, setting, initJWTSecret, checkLogin, startUnitTest, FBSD, doubleCheckPassword, startE2eTests } = require("./util-server"); +const { getSettings, setSettings, setting, initJWTSecret, checkLogin, startUnitTest, FBSD, doubleCheckPassword, startE2eTests, + allowDevAllOrigin +} = require("./util-server"); log.debug("server", "Importing Notification"); const { Notification } = require("./notification"); @@ -228,6 +230,14 @@ let needSetup = false; } }); + app.get("/setup-database-info", (request, response) => { + allowDevAllOrigin(response); + response.json({ + runningSetup: false, + needSetup: false, + }); + }); + if (isDev) { app.post("/test-webhook", async (request, response) => { log.debug("test", request.headers); diff --git a/server/setup-database.js b/server/setup-database.js index e40649fa..26a55ab3 100644 --- a/server/setup-database.js +++ b/server/setup-database.js @@ -18,6 +18,7 @@ class SetupDatabase { * @type {boolean} */ needSetup = true; + runningSetup = false; server; @@ -80,6 +81,12 @@ class SetupDatabase { let tempServer; app.use(express.json()); + // Disable Keep Alive, otherwise the server will not shutdown, as the client will keep the connection alive + app.use(function (req, res, next) { + res.setHeader("Connection", "close"); + next(); + }); + app.get("/", async (request, response) => { response.redirect("/setup-database"); }); @@ -91,9 +98,12 @@ class SetupDatabase { }); }); - app.get("/info", (request, response) => { + app.get("/setup-database-info", (request, response) => { allowDevAllOrigin(response); + console.log("Request /setup-database-info"); response.json({ + runningSetup: this.runningSetup, + needSetup: this.needSetup, isEnabledEmbeddedMariaDB: this.isEnabledEmbeddedMariaDB(), }); }); @@ -101,7 +111,12 @@ class SetupDatabase { app.post("/setup-database", async (request, response) => { allowDevAllOrigin(response); - console.log(request); + if (this.runningSetup) { + response.status(400).json("Setup is already running"); + return; + } + + this.runningSetup = true; let dbConfig = request.body.dbConfig; @@ -114,42 +129,50 @@ class SetupDatabase { // Validate input if (typeof dbConfig !== "object") { response.status(400).json("Invalid dbConfig"); + this.runningSetup = false; return; } if (!dbConfig.type) { response.status(400).json("Database Type is required"); + this.runningSetup = false; return; } if (!supportedDBTypes.includes(dbConfig.type)) { response.status(400).json("Unsupported Database Type"); + this.runningSetup = false; return; } if (dbConfig.type === "mariadb") { if (!dbConfig.hostname) { response.status(400).json("Hostname is required"); + this.runningSetup = false; return; } if (!dbConfig.port) { response.status(400).json("Port is required"); + this.runningSetup = false; return; } if (!dbConfig.dbName) { response.status(400).json("Database name is required"); + this.runningSetup = false; return; } if (!dbConfig.username) { response.status(400).json("Username is required"); + this.runningSetup = false; return; } if (!dbConfig.password) { response.status(400).json("Password is required"); + this.runningSetup = false; return; } } @@ -162,11 +185,16 @@ class SetupDatabase { }); // Shutdown down this express and start the main server - log.info("setup-database", "Database is configured, close setup-database server and start the main server now."); + log.info("setup-database", "Database is configured, close the setup-database server and start the main server now."); if (tempServer) { - tempServer.close(); + tempServer.close(() => { + log.info("setup-database", "The setup-database server is closed"); + resolve(); + }); + } else { + resolve(); } - resolve(); + }); app.use("/", expressStaticGzip("dist", { diff --git a/src/pages/Setup.vue b/src/pages/Setup.vue index cd2d149c..b5eafcc6 100644 --- a/src/pages/Setup.vue +++ b/src/pages/Setup.vue @@ -62,6 +62,8 @@ export default { }, mounted() { + // TODO: Check if it is a database setup + this.$root.getSocket().emit("needSetup", (needSetup) => { if (! needSetup) { this.$router.push("/"); diff --git a/src/pages/SetupDatabase.vue b/src/pages/SetupDatabase.vue index 122b548d..74cf88cf 100644 --- a/src/pages/SetupDatabase.vue +++ b/src/pages/SetupDatabase.vue @@ -1,5 +1,5 @@