diff --git a/config.sample.yml b/config.sample.yml index 64f56696..5ceda6ad 100644 --- a/config.sample.yml +++ b/config.sample.yml @@ -125,6 +125,15 @@ uploads: offline: false +# --------------------------------------------------------------------- +# High-Availability +# --------------------------------------------------------------------- +# Set to true if you have multiple concurrent instances running off the +# same DB (e.g. Kubernetes pods / load balanced instances). Leave false +# otherwise. + +ha: false + # --------------------------------------------------------------------- # Data Path # --------------------------------------------------------------------- diff --git a/dev/build/config.yml b/dev/build/config.yml index 93395f1a..3a7467e4 100644 --- a/dev/build/config.yml +++ b/dev/build/config.yml @@ -16,3 +16,4 @@ ssl: domain: $(LETSENCRYPT_DOMAIN) subscriberEmail: $(LETSENCRYPT_EMAIL) logLevel: info +ha: $(HA_ACTIVE) diff --git a/dev/index.js b/dev/index.js index 8f8d759b..f3b719db 100644 --- a/dev/index.js +++ b/dev/index.js @@ -5,7 +5,6 @@ // Licensed under AGPLv3 // =========================================== -const Promise = require('bluebird') const _ = require('lodash') const chalk = require('chalk') @@ -60,16 +59,9 @@ const init = { }) }, async reload() { - console.warn(chalk.yellow('--- Stopping scheduled jobs...')) - if (global.WIKI.scheduler) { - global.WIKI.scheduler.stop() - } - console.warn(chalk.yellow('--- Closing DB connections...')) - await global.WIKI.models.knex.destroy() - console.warn(chalk.yellow('--- Closing Server connections...')) - if (global.WIKI.servers) { - await global.WIKI.servers.stopServers() - } + console.warn(chalk.yellow('--- Gracefully stopping server...')) + await global.WIKI.kernel.shutdown() + console.warn(chalk.yellow('--- Purging node modules cache...')) global.WIKI = {} diff --git a/package.json b/package.json index 52888c6e..4c08b1fe 100644 --- a/package.json +++ b/package.json @@ -69,6 +69,7 @@ "dotize": "0.3.0", "elasticsearch6": "npm:@elastic/elasticsearch@6", "elasticsearch7": "npm:@elastic/elasticsearch@7", + "eventemitter2": "6.0.0", "emoji-regex": "9.0.0", "express": "4.17.1", "express-brute": "1.0.1", @@ -146,6 +147,7 @@ "pg": "8.0.2", "pg-hstore": "2.3.3", "pg-query-stream": "3.0.6", + "pg-pubsub": "0.5.0", "pg-tsquery": "8.1.0", "pug": "2.0.4", "punycode": "2.1.1", diff --git a/server/app/data.yml b/server/app/data.yml index a04b1d99..310e4a90 100644 --- a/server/app/data.yml +++ b/server/app/data.yml @@ -28,6 +28,7 @@ defaults: maxFileSize: 5242880 maxFiles: 10 offline: false + ha: false # DB defaults api: isEnabled: false diff --git a/server/core/db.js b/server/core/db.js index 0d44229a..d16fc253 100644 --- a/server/core/db.js +++ b/server/core/db.js @@ -17,6 +17,7 @@ const migrateFromBeta = require('../db/beta') module.exports = { Objection, knex: null, + listener: null, /** * Initialize DB * @@ -182,5 +183,55 @@ module.exports = { ...this, ...models } + }, + /** + * Subscribe to database LISTEN / NOTIFY for multi-instances events + */ + async subscribeToNotifications () { + const useHA = (WIKI.config.ha === true || WIKI.config.ha === 'true' || WIKI.config.ha === 1 || WIKI.config.ha === '1') + if (!useHA) { + return + } else if (WIKI.config.db.type !== 'postgres') { + WIKI.logger.warn(`Database engine doesn't support pub/sub. Will not handle concurrent instances: [ DISABLED ]`) + return + } + + const PGPubSub = require('pg-pubsub') + + this.listener = new PGPubSub(this.knex.client.connectionSettings, { + log (ev) { + WIKI.logger.debug(ev) + } + }) + this.listener.addChannel('wiki', payload => { + if (_.has(payload.event) && payload.source !== WIKI.INSTANCE_ID) { + WIKI.events.emit(payload.event, payload.value) + } + }) + WIKI.events.onAny(this.notifyViaDB) + + WIKI.logger.info(`High-Availability Listener initialized successfully: [ OK ]`) + }, + /** + * Unsubscribe from database LISTEN / NOTIFY + */ + async unsubscribeToNotifications () { + if (this.listener) { + WIKI.events.offAny(this.notifyViaDB) + this.listener.close() + } + }, + /** + * Publish event via database NOTIFY + * + * @param {string} event Event fired + * @param {object} value Payload of the event + */ + notifyViaDB (event, value) { + this.listener.publish('wiki', { + source: WIKI.INSTANCE_ID, + event, + value + }) } } diff --git a/server/core/kernel.js b/server/core/kernel.js index 050769ed..fe8272c0 100644 --- a/server/core/kernel.js +++ b/server/core/kernel.js @@ -1,5 +1,5 @@ const _ = require('lodash') -const EventEmitter = require('events') +const EventEmitter = require('eventemitter2').EventEmitter2 /* global WIKI */ @@ -37,6 +37,7 @@ module.exports = { WIKI.servers = require('./servers') WIKI.sideloader = require('./sideloader').init() WIKI.events = new EventEmitter() + await WIKI.models.subscribeToNotifications() } catch (err) { WIKI.logger.error(err) process.exit(1) @@ -91,5 +92,21 @@ module.exports = { WIKI.logger.warn(err) WIKI.telemetry.sendError(err) }) + }, + /** + * Graceful shutdown + */ + async shutdown () { + if (WIKI.models) { + await WIKI.models.unsubscribeToNotifications() + await WIKI.models.knex.client.pool.destroy() + await WIKI.models.knex.destroy() + } + if (WIKI.scheduler) { + WIKI.scheduler.stop() + } + if (WIKI.servers) { + await WIKI.servers.stopServers() + } } } diff --git a/server/index.js b/server/index.js index ea20a33b..24f2cbbf 100644 --- a/server/index.js +++ b/server/index.js @@ -4,11 +4,13 @@ // =========================================== const path = require('path') +const nanoid = require('nanoid') let WIKI = { IS_DEBUG: process.env.NODE_ENV === 'development', IS_MASTER: true, ROOTPATH: process.cwd(), + INSTANCE_ID: nanoid(10), SERVERPATH: path.join(process.cwd(), 'server'), Error: require('./helpers/error'), configSvc: require('./core/config'), diff --git a/yarn.lock b/yarn.lock index 51f9b06a..7c8c9a4c 100644 --- a/yarn.lock +++ b/yarn.lock @@ -2401,6 +2401,19 @@ resolved "https://registry.yarnpkg.com/@types/prettier/-/prettier-1.19.1.tgz#33509849f8e679e4add158959fdb086440e9553f" integrity sha512-5qOlnZscTn4xxM5MeGXAMOsIOIKIbh9e85zJWfBRVPlRMEVawzoPhINYbRGkBZCI8LxvBe7tJCdWiarA99OZfQ== +"@types/pg-types@*": + version "1.11.5" + resolved "https://registry.yarnpkg.com/@types/pg-types/-/pg-types-1.11.5.tgz#1eebbe62b6772fcc75c18957a90f933d155e005b" + integrity sha512-L8ogeT6vDzT1vxlW3KITTCt+BVXXVkLXfZ/XNm6UqbcJgxf+KPO7yjWx7dQQE8RW07KopL10x2gNMs41+IkMGQ== + +"@types/pg@^7.4.14": + version "7.14.1" + resolved "https://registry.yarnpkg.com/@types/pg/-/pg-7.14.1.tgz#40358b57c34970f750f6a26e2a5463c9f4758136" + integrity sha512-gQgg4bLuykokypx4O1fwEzl5e6UjjyaBtN3znn5zhm0YB9BnKyHDw+e4cQY9rAPzpdM2qpJbn9TNzUazbmTsdw== + dependencies: + "@types/node" "*" + "@types/pg-types" "*" + "@types/q@^1.5.1": version "1.5.2" resolved "https://registry.yarnpkg.com/@types/q/-/q-1.5.2.tgz#690a1475b84f2a884fd07cd797c00f5f31356ea8" @@ -6862,6 +6875,11 @@ etag@~1.8.1: resolved "https://registry.yarnpkg.com/etag/-/etag-1.8.1.tgz#41ae2eeb65efa62268aebfea83ac7d79299b0887" integrity sha1-Qa4u62XvpiJorr/qg6x9eSmbCIc= +eventemitter2@6.0.0: + version "6.0.0" + resolved "https://registry.yarnpkg.com/eventemitter2/-/eventemitter2-6.0.0.tgz#218eb512c3603c5341724b6af7b686a1aa5ab8f5" + integrity sha512-ZuNWHD7S7IoikyEmx35vPU8H1W0L+oi644+4mSTg7nwXvBQpIwQL7DPjYUF0VMB0jPkNMo3MqD07E7MYrkFmjQ== + eventemitter3@^3.1.0: version "3.1.2" resolved "https://registry.yarnpkg.com/eventemitter3/-/eventemitter3-3.1.2.tgz#2d3d48f9c346698fce83a85d7d664e98535df6e7" @@ -11709,6 +11727,11 @@ pg-cursor@^2.1.9: resolved "https://registry.yarnpkg.com/pg-cursor/-/pg-cursor-2.1.9.tgz#ad73bd7b119db98d9da6bb148162d831d8cddc88" integrity sha512-ReAyfc0fiSzO268DZWtOoeUmMdMG368e7qu3zfwa7TpCjEmabk2XfZn+tdE23XzVlJ4OsdsFcxjASOWC0SS7jA== +pg-format@^1.0.2: + version "1.0.4" + resolved "https://registry.yarnpkg.com/pg-format/-/pg-format-1.0.4.tgz#27734236c2ad3f4e5064915a59334e20040a828e" + integrity sha1-J3NCNsKtP05QZJFaWTNOIAQKgo4= + pg-hstore@2.3.3: version "2.3.3" resolved "https://registry.yarnpkg.com/pg-hstore/-/pg-hstore-2.3.3.tgz#d1978c12a85359830b1388d3b0ff233b88928e96" @@ -11735,6 +11758,21 @@ pg-query-stream@3.0.6: version "3.0.6" resolved "https://registry.yarnpkg.com/pg-query-stream/-/pg-query-stream-3.0.6.tgz#12f405c2c8c9723d8d9f1616cf3d58ecd1d87c83" integrity sha512-/caOI36GVCz1pY35SkftzGowwym4p39e3Ku+sx8MZKNNf+G9WgE0h+Ui9FHTVV9HWf6WWu1GYt5aYfw5ZMeJsQ== +pg-pubsub@0.5.0: + version "0.5.0" + resolved "https://registry.yarnpkg.com/pg-pubsub/-/pg-pubsub-0.5.0.tgz#5469737af32ac6d13fc3153dc3944f55da3d8840" + integrity sha512-lfNMbiVt6d8yomYOWV9+smMdrdUUj3Q8Uq9SiNL+3q2G+jjjjaT9+FKDudw9VEN94pYmCtJzhXz/BlaD4LgyZg== + dependencies: + "@types/pg" "^7.4.14" + pg "^7.4.3" + pg-format "^1.0.2" + promised-retry "^0.3.0" + verror "^1.10.0" + +pg-query-stream@3.0.0: + version "3.0.0" + resolved "https://registry.yarnpkg.com/pg-query-stream/-/pg-query-stream-3.0.0.tgz#b2b4c3d5eb105df6cf75d3f95c4e157dd527a2ab" + integrity sha512-yxeFKwVCW0vmFYSkygV7hd4KVlGCMHGljSUQvYcIZUtfUaAITIl8QOq9oXfCMmk0L6JOuHesZKpurxyuOU8gKg== dependencies: pg-cursor "^2.1.9" @@ -11768,6 +11806,20 @@ pg@8.0.2: pgpass "1.x" semver "4.3.2" +pg@^7.4.3: + version "7.18.1" + resolved "https://registry.yarnpkg.com/pg/-/pg-7.18.1.tgz#67f59c47a99456fcb34f9fe53662b79d4a992f6d" + integrity sha512-1KtKBKg/zWrjEEv//klBbVOPGucuc7HHeJf6OEMueVcUeyF3yueHf+DvhVwBjIAe9/97RAydO/lWjkcMwssuEw== + dependencies: + buffer-writer "2.0.0" + packet-reader "1.0.0" + pg-connection-string "0.1.3" + pg-packet-stream "^1.1.0" + pg-pool "^2.0.10" + pg-types "^2.1.0" + pgpass "1.x" + semver "4.3.2" + pgpass@1.x: version "1.0.2" resolved "https://registry.yarnpkg.com/pgpass/-/pgpass-1.0.2.tgz#2a7bb41b6065b67907e91da1b07c1847c877b306" @@ -12944,6 +12996,11 @@ promise@^7.0.1: dependencies: asap "~2.0.3" +promised-retry@^0.3.0: + version "0.3.0" + resolved "https://registry.yarnpkg.com/promised-retry/-/promised-retry-0.3.0.tgz#fc3930ca1f76c4d1e559b7368b1f7e953ae0d6ad" + integrity sha512-oj7GS7q3g381QuIR8mMJeJe5GSKm16xXL7Q+7/+5MH2mlWBHXn7dUYmS/BBjGkoBm9M8h2KcVifj3vRZ62AB5A== + prompts@^2.0.1: version "2.2.1" resolved "https://registry.yarnpkg.com/prompts/-/prompts-2.2.1.tgz#f901dd2a2dfee080359c0e20059b24188d75ad35" @@ -15788,7 +15845,7 @@ vendors@^1.0.0: resolved "https://registry.yarnpkg.com/vendors/-/vendors-1.0.3.tgz#a6467781abd366217c050f8202e7e50cc9eef8c0" integrity sha512-fOi47nsJP5Wqefa43kyWSg80qF+Q3XA6MUkgi7Hp1HQaKDQW4cQrK2D0P7mmbFtsV1N89am55Yru/nyEwRubcw== -verror@1.10.0, verror@^1.8.1: +verror@1.10.0, verror@^1.10.0, verror@^1.8.1: version "1.10.0" resolved "https://registry.yarnpkg.com/verror/-/verror-1.10.0.tgz#3a105ca17053af55d6e270c1f8288682e18da400" integrity sha1-OhBcoXBTr1XW4nDB+CiGguGNpAA=