diff --git a/config.sample.yml b/config.sample.yml index a42a67ba..5421d5a8 100644 --- a/config.sample.yml +++ b/config.sample.yml @@ -42,6 +42,15 @@ redis: # --------------------------------------------------------------------- # Background Workers # --------------------------------------------------------------------- - # Leave 0 for auto based on CPU cores + workers: 0 + +# --------------------------------------------------------------------- +# High Availability +# --------------------------------------------------------------------- +# Read the docs BEFORE changing these settings! + +ha: + nodeuid: primary + readonly: false diff --git a/server/app/data.yml b/server/app/data.yml index 8ab67828..ded7b58a 100644 --- a/server/app/data.yml +++ b/server/app/data.yml @@ -21,6 +21,12 @@ defaults: db: 0 password: null workers: 0 + ha: + nodeuid: primary + readonly: false +queues: + - gitSync + - uplClearTemp authProviders: - local - microsoft diff --git a/server/index.js b/server/index.js index 24f65480..3f0007af 100644 --- a/server/index.js +++ b/server/index.js @@ -48,18 +48,27 @@ if (numWorkers > numCPUs) { } if (cluster.isMaster) { + wiki.logger.info('--------------------------') wiki.logger.info('Wiki.js is initializing...') + wiki.logger.info('--------------------------') - require('./master') + require('./master').then(() => { + // -> Create background workers + for (let i = 0; i < numWorkers; i++) { + cluster.fork() + } - for (let i = 0; i < numWorkers; i++) { - cluster.fork() - } + // -> Queue post-init tasks + + wiki.queue.uplClearTemp.add({}, { + repeat: { cron: '*/15 * * * *' } + }) + }) cluster.on('exit', (worker, code, signal) => { - wiki.logger.info(`Worker #${worker.id} died.`) + wiki.logger.info(`Background Worker #${worker.id} was terminated.`) }) } else { - wiki.logger.info(`Background Worker #${cluster.worker.id} is starting...`) + wiki.logger.info(`Background Worker #${cluster.worker.id} is initializing...`) require('./worker') } diff --git a/server/master.js b/server/master.js index 0ce0bf19..85754011 100644 --- a/server/master.js +++ b/server/master.js @@ -4,20 +4,23 @@ const Promise = require('bluebird') +wiki.redis = require('./modules/redis').init() +wiki.queue = require('./modules/queue').init() + module.exports = Promise.join( wiki.db.onReady, - wiki.configSvc.loadFromDb() + wiki.configSvc.loadFromDb(), + wiki.queue.clean() ).then(() => { // ---------------------------------------- // Load global modules // ---------------------------------------- wiki.disk = require('./modules/disk').init() - wiki.entries = require('./modules/entries').init() + wiki.docs = require('./modules/documents').init() wiki.git = require('./modules/git').init(false) wiki.lang = require('i18next') wiki.mark = require('./modules/markdown') - wiki.redis = require('./modules/redis').init() wiki.search = require('./modules/search').init() wiki.upl = require('./modules/uploads').init() @@ -75,7 +78,7 @@ module.exports = Promise.join( // Passport Authentication // ---------------------------------------- - require('./modules/auth')(passport) + require('./modules/auth').init(passport) wiki.rights = require('./modules/rights') // wiki.rights.init() diff --git a/server/modules/auth.js b/server/modules/auth.js index 45ee1b9e..2fd96206 100644 --- a/server/modules/auth.js +++ b/server/modules/auth.js @@ -4,87 +4,89 @@ const _ = require('lodash') -module.exports = (passport) => { +module.exports = { + init(passport) { // Serialization user methods - passport.serializeUser(function (user, done) { - done(null, user._id) - }) - - passport.deserializeUser(function (id, done) { - wiki.db.User.findById(id).then((user) => { - if (user) { - done(null, user) - } else { - done(new Error(wiki.lang.t('auth:errors:usernotfound')), null) - } - return true - }).catch((err) => { - done(err, null) + passport.serializeUser(function (user, done) { + done(null, user._id) }) - }) - // Load authentication strategies - - wiki.config.authStrategies = { - list: _.pickBy(wiki.config.auth, strategy => strategy.enabled), - socialEnabled: (_.chain(wiki.config.auth).omit('local').filter(['enabled', true]).value().length > 0) - } - - _.forOwn(wiki.config.authStrategies.list, (strategyConfig, strategyName) => { - strategyConfig.callbackURL = `${wiki.config.site.host}/login/${strategyName}/callback` - require(`../authentication/${strategyName}`)(passport, strategyConfig) - wiki.logger.info(`Authentication Provider ${_.upperFirst(strategyName)}: OK`) - }) - - // Create Guest account for first-time - - return wiki.db.User.findOne({ - where: { - provider: 'local', - email: 'guest@example.com' - } - }).then((c) => { - if (c < 1) { - return wiki.db.User.create({ - provider: 'local', - email: 'guest@example.com', - name: 'Guest', - password: '', - role: 'guest' - }).then(() => { - wiki.logger.info('[AUTH] Guest account created successfully!') + passport.deserializeUser(function (id, done) { + wiki.db.User.findById(id).then((user) => { + if (user) { + done(null, user) + } else { + done(new Error(wiki.lang.t('auth:errors:usernotfound')), null) + } return true }).catch((err) => { - wiki.logger.error('[AUTH] An error occured while creating guest account:') - wiki.logger.error(err) - return err + done(err, null) }) + }) + + // Load authentication strategies + + wiki.config.authStrategies = { + list: _.pickBy(wiki.config.auth, strategy => strategy.enabled), + socialEnabled: (_.chain(wiki.config.auth).omit('local').filter(['enabled', true]).value().length > 0) } - }) - // .then(() => { - // if (process.env.WIKI_JS_HEROKU) { - // return wiki.db.User.findOne({ provider: 'local', email: process.env.WIKI_ADMIN_EMAIL }).then((c) => { - // if (c < 1) { - // // Create root admin account (HEROKU ONLY) + _.forOwn(wiki.config.authStrategies.list, (strategyConfig, strategyName) => { + strategyConfig.callbackURL = `${wiki.config.site.host}/login/${strategyName}/callback` + require(`../authentication/${strategyName}`)(passport, strategyConfig) + wiki.logger.info(`Authentication Provider ${_.upperFirst(strategyName)}: OK`) + }) - // return wiki.db.User.create({ - // provider: 'local', - // email: process.env.WIKI_ADMIN_EMAIL, - // name: 'Administrator', - // password: '$2a$04$MAHRw785Xe/Jd5kcKzr3D.VRZDeomFZu2lius4gGpZZ9cJw7B7Mna', // admin123 (default) - // role: 'admin' - // }).then(() => { - // wiki.logger.info('[AUTH] Root admin account created successfully!') - // return true - // }).catch((err) => { - // wiki.logger.error('[AUTH] An error occured while creating root admin account:') - // wiki.logger.error(err) - // return err - // }) - // } else { return true } - // }) - // } else { return true } - // }) + // Create Guest account for first-time + + return wiki.db.User.findOne({ + where: { + provider: 'local', + email: 'guest@example.com' + } + }).then((c) => { + if (c < 1) { + return wiki.db.User.create({ + provider: 'local', + email: 'guest@example.com', + name: 'Guest', + password: '', + role: 'guest' + }).then(() => { + wiki.logger.info('[AUTH] Guest account created successfully!') + return true + }).catch((err) => { + wiki.logger.error('[AUTH] An error occured while creating guest account:') + wiki.logger.error(err) + return err + }) + } + }) + + // .then(() => { + // if (process.env.WIKI_JS_HEROKU) { + // return wiki.db.User.findOne({ provider: 'local', email: process.env.WIKI_ADMIN_EMAIL }).then((c) => { + // if (c < 1) { + // // Create root admin account (HEROKU ONLY) + + // return wiki.db.User.create({ + // provider: 'local', + // email: process.env.WIKI_ADMIN_EMAIL, + // name: 'Administrator', + // password: '$2a$04$MAHRw785Xe/Jd5kcKzr3D.VRZDeomFZu2lius4gGpZZ9cJw7B7Mna', // admin123 (default) + // role: 'admin' + // }).then(() => { + // wiki.logger.info('[AUTH] Root admin account created successfully!') + // return true + // }).catch((err) => { + // wiki.logger.error('[AUTH] An error occured while creating root admin account:') + // wiki.logger.error(err) + // return err + // }) + // } else { return true } + // }) + // } else { return true } + // }) + } } diff --git a/server/modules/config.js b/server/modules/config.js index da67d29f..334fba3c 100644 --- a/server/modules/config.js +++ b/server/modules/config.js @@ -13,9 +13,6 @@ module.exports = { /** * Load root config from disk - * - * @param {any} confPaths - * @returns */ init() { let confPaths = { diff --git a/server/modules/db.js b/server/modules/db.js index 3654afd2..f459039e 100644 --- a/server/modules/db.js +++ b/server/modules/db.js @@ -63,12 +63,32 @@ module.exports = { require(path.join(dbModelsPath, '_relations.js'))(self) - // Sync DB + // Set init tasks - self.onReady = (wiki.IS_MASTER) ? self.inst.sync({ - force: false, - logging: false - }) : Promise.resolve() + let initTasks = { + // -> Sync DB Schemas + syncSchemas() { + return self.inst.sync({ + force: false, + logging: false + }) + }, + // -> Set Connection App Name + setAppName() { + return self.inst.query(`set application_name = 'Wiki.js'`, { raw: true }) + } + } + + let initTasksQueue = (wiki.IS_MASTER) ? [ + initTasks.syncSchemas, + initTasks.setAppName + ] : [ + initTasks.setAppName + ] + + // Perform init tasks + + self.onReady = Promise.each(initTasksQueue, t => t()) return self } diff --git a/server/modules/entries.js b/server/modules/documents.js similarity index 99% rename from server/modules/entries.js rename to server/modules/documents.js index 802d0d6d..f89d6fab 100644 --- a/server/modules/entries.js +++ b/server/modules/documents.js @@ -10,7 +10,7 @@ const _ = require('lodash') const entryHelper = require('../helpers/entry') /** - * Entries Model + * Documents Model */ module.exports = { diff --git a/server/modules/queue.js b/server/modules/queue.js new file mode 100644 index 00000000..e7612ee3 --- /dev/null +++ b/server/modules/queue.js @@ -0,0 +1,37 @@ +'use strict' + +/* global wiki */ + +const Bull = require('bull') +const Promise = require('bluebird') + +module.exports = { + init() { + wiki.data.queues.forEach(queueName => { + this[queueName] = new Bull(queueName, { + prefix: `q-${wiki.config.ha.nodeuid}`, + redis: wiki.config.redis + }) + }) + return this + }, + clean() { + return Promise.each(wiki.data.queues, queueName => { + return new Promise((resolve, reject) => { + let keyStream = wiki.redis.scanStream({ + match: `q-${wiki.config.ha.nodeuid}:${queueName}:*` + }) + keyStream.on('data', resultKeys => { + if (resultKeys.length > 0) { + wiki.redis.del(resultKeys) + } + }) + keyStream.on('end', resolve) + }) + }).then(() => { + wiki.logger.info('Purging old queue jobs: OK') + }).catch(err => { + wiki.logger.error(err) + }) + } +} diff --git a/server/modules/redis.js b/server/modules/redis.js index 284c93d3..6eb7999b 100644 --- a/server/modules/redis.js +++ b/server/modules/redis.js @@ -19,7 +19,11 @@ module.exports = { */ init() { if (isPlainObject(wiki.config.redis)) { - return new Redis(wiki.config.redis) + let red = new Redis(wiki.config.redis) + red.on('ready', () => { + wiki.logger.info('Redis connection: OK') + }) + return red } else { wiki.logger.error('Invalid Redis configuration!') process.exit(1) diff --git a/server/queues/git-sync.js b/server/queues/git-sync.js index 9f9b1f57..ab4fefae 100644 --- a/server/queues/git-sync.js +++ b/server/queues/git-sync.js @@ -61,5 +61,8 @@ module.exports = (job, done) => { }) return jobCbStreamDocs + }).then(() => { + wiki.logger.info('Git remote repository sync: DONE') + return true }) } diff --git a/server/queues/upl-clear-temp.js b/server/queues/upl-clear-temp.js index 2162aae4..00e2bf1e 100644 --- a/server/queues/upl-clear-temp.js +++ b/server/queues/upl-clear-temp.js @@ -22,5 +22,8 @@ module.exports = (job, done) => { } }) }) + }).then(() => { + wiki.logger.info('Purging temporary upload files: DONE') + return true }) } diff --git a/server/worker.js b/server/worker.js index f64deca5..0028f755 100644 --- a/server/worker.js +++ b/server/worker.js @@ -2,69 +2,68 @@ /* global wiki */ -const path = require('path') - -wiki.REPOPATH = path.resolve(wiki.ROOTPATH, wiki.config.paths.repo) -wiki.DATAPATH = path.resolve(wiki.ROOTPATH, wiki.config.paths.data) -wiki.UPLTEMPPATH = path.join(wiki.DATAPATH, 'temp-upload') - -// ---------------------------------------- -// Load global modules -// ---------------------------------------- - -// wiki.upl = require('./modules/uploads-agent').init() -// wiki.git = require('./modules/git').init() -// wiki.entries = require('./modules/entries').init() -wiki.lang = require('i18next') -wiki.mark = require('./modules/markdown') - -// ---------------------------------------- -// Load local modules -// ---------------------------------------- - const Promise = require('bluebird') -const i18nBackend = require('i18next-node-fs-backend') -// ---------------------------------------- -// Localization Engine -// ---------------------------------------- - -wiki.lang.use(i18nBackend).init({ - load: 'languageOnly', - ns: ['common', 'admin', 'auth', 'errors', 'git'], - defaultNS: 'common', - saveMissing: false, - preload: [wiki.config.lang], - lng: wiki.config.lang, - fallbackLng: 'en', - backend: { - loadPath: path.join(wiki.SERVERPATH, 'locales/{{lng}}/{{ns}}.json') - } -}) - -// ---------------------------------------- -// Start Queues -// ---------------------------------------- - -const Bull = require('bull') -const autoload = require('auto-load') - -let queues = autoload(path.join(wiki.SERVERPATH, 'queues')) - -Promise.join( - wiki.db.onReady - // wiki.upl.initialScan() +module.exports = Promise.join( + wiki.db.onReady, + wiki.configSvc.loadFromDb(['features', 'git', 'logging', 'site', 'uploads']) ).then(() => { + const path = require('path') + + wiki.REPOPATH = path.resolve(wiki.ROOTPATH, wiki.config.paths.repo) + wiki.DATAPATH = path.resolve(wiki.ROOTPATH, wiki.config.paths.data) + wiki.UPLTEMPPATH = path.join(wiki.DATAPATH, 'temp-upload') + + // ---------------------------------------- + // Load global modules + // ---------------------------------------- + + // wiki.upl = require('./modules/uploads-agent').init() + // wiki.git = require('./modules/git').init() + // wiki.entries = require('./modules/entries').init() + wiki.lang = require('i18next') + wiki.mark = require('./modules/markdown') + + // ---------------------------------------- + // Localization Engine + // ---------------------------------------- + + const i18nBackend = require('i18next-node-fs-backend') + wiki.lang.use(i18nBackend).init({ + load: 'languageOnly', + ns: ['common', 'admin', 'auth', 'errors', 'git'], + defaultNS: 'common', + saveMissing: false, + preload: [wiki.config.lang], + lng: wiki.config.lang, + fallbackLng: 'en', + backend: { + loadPath: path.join(wiki.SERVERPATH, 'locales/{{lng}}/{{ns}}.json') + } + }) + + // ---------------------------------------- + // Start Queues + // ---------------------------------------- + + const Bull = require('bull') + const autoload = require('auto-load') + + let queues = autoload(path.join(wiki.SERVERPATH, 'queues')) + for (let queueName in queues) { - new Bull(queueName, { redis: wiki.config.redis }).process(queues[queueName]) + new Bull(queueName, { + prefix: `q-${wiki.config.ha.nodeuid}`, + redis: wiki.config.redis + }).process(queues[queueName]) } -}) -// ---------------------------------------- -// Shutdown gracefully -// ---------------------------------------- + // ---------------------------------------- + // Shutdown gracefully + // ---------------------------------------- -process.on('disconnect', () => { - wiki.logger.warn('Lost connection to Master. Exiting...') - process.exit() + process.on('disconnect', () => { + wiki.logger.warn('Lost connection to Master. Exiting...') + process.exit() + }) })