feat: queue handling

config.sample.yml

@@ -42,6 +42,15 @@ redis:
 # ---------------------------------------------------------------------
 # Background Workers
 # ---------------------------------------------------------------------
-
 # Leave 0 for auto based on CPU cores
+
 workers: 0
+
+# ---------------------------------------------------------------------
+# High Availability
+# ---------------------------------------------------------------------
+# Read the docs BEFORE changing these settings!
+
+ha:
+  nodeuid: primary
+  readonly: false

server/app/data.yml

@@ -21,6 +21,12 @@ defaults:
       db: 0
       password: null
     workers: 0
+    ha:
+      nodeuid: primary
+      readonly: false
+queues:
+  - gitSync
+  - uplClearTemp
 authProviders:
   - local
   - microsoft

server/index.js

@@ -48,18 +48,27 @@ if (numWorkers > numCPUs) {
 }
 
 if (cluster.isMaster) {
+  wiki.logger.info('--------------------------')
   wiki.logger.info('Wiki.js is initializing...')
+  wiki.logger.info('--------------------------')
 
-  require('./master')
+  require('./master').then(() => {
-
+    // -> Create background workers
     for (let i = 0; i < numWorkers; i++) {
       cluster.fork()
     }
 
+    // -> Queue post-init tasks
+
+    wiki.queue.uplClearTemp.add({}, {
+      repeat: { cron: '*/15 * * * *' }
+    })
+  })
+
   cluster.on('exit', (worker, code, signal) => {
-    wiki.logger.info(`Worker #${worker.id} died.`)
+    wiki.logger.info(`Background Worker #${worker.id} was terminated.`)
   })
 } else {
-  wiki.logger.info(`Background Worker #${cluster.worker.id} is starting...`)
+  wiki.logger.info(`Background Worker #${cluster.worker.id} is initializing...`)
   require('./worker')
 }

server/master.js

@@ -4,20 +4,23 @@
 
 const Promise = require('bluebird')
 
+wiki.redis = require('./modules/redis').init()
+wiki.queue = require('./modules/queue').init()
+
 module.exports = Promise.join(
   wiki.db.onReady,
-  wiki.configSvc.loadFromDb()
+  wiki.configSvc.loadFromDb(),
+  wiki.queue.clean()
 ).then(() => {
   // ----------------------------------------
   // Load global modules
   // ----------------------------------------
 
   wiki.disk = require('./modules/disk').init()
-  wiki.entries = require('./modules/entries').init()
+  wiki.docs = require('./modules/documents').init()
   wiki.git = require('./modules/git').init(false)
   wiki.lang = require('i18next')
   wiki.mark = require('./modules/markdown')
-  wiki.redis = require('./modules/redis').init()
   wiki.search = require('./modules/search').init()
   wiki.upl = require('./modules/uploads').init()
 
@@ -75,7 +78,7 @@ module.exports = Promise.join(
   // Passport Authentication
   // ----------------------------------------
 
-  require('./modules/auth')(passport)
+  require('./modules/auth').init(passport)
   wiki.rights = require('./modules/rights')
   // wiki.rights.init()
 

server/modules/auth.js

@@ -4,7 +4,8 @@
 
 const _ = require('lodash')
 
-module.exports = (passport) => {
+module.exports = {
+  init(passport) {
   // Serialization user methods
 
     passport.serializeUser(function (user, done) {
@@ -88,3 +89,4 @@ module.exports = (passport) => {
     //   } else { return true }
     // })
   }
+}

server/modules/config.js

@@ -13,9 +13,6 @@ module.exports = {
 
   /**
    * Load root config from disk
-   *
-   * @param {any} confPaths
-   * @returns
    */
   init() {
     let confPaths = {

server/modules/db.js

@@ -63,12 +63,32 @@ module.exports = {
 
     require(path.join(dbModelsPath, '_relations.js'))(self)
 
-    // Sync DB
+    // Set init tasks
 
-    self.onReady = (wiki.IS_MASTER) ? self.inst.sync({
+    let initTasks = {
+      // -> Sync DB Schemas
+      syncSchemas() {
+        return self.inst.sync({
           force: false,
           logging: false
-    }) : Promise.resolve()
+        })
+      },
+      // -> Set Connection App Name
+      setAppName() {
+        return self.inst.query(`set application_name = 'Wiki.js'`, { raw: true })
+      }
+    }
+
+    let initTasksQueue = (wiki.IS_MASTER) ? [
+      initTasks.syncSchemas,
+      initTasks.setAppName
+    ] : [
+      initTasks.setAppName
+    ]
+
+    // Perform init tasks
+
+    self.onReady = Promise.each(initTasksQueue, t => t())
 
     return self
   }

server/modules/documents.js

@@ -10,7 +10,7 @@ const _ = require('lodash')
 const entryHelper = require('../helpers/entry')
 
 /**
- * Entries Model
+ * Documents Model
  */
 module.exports = {
 

server/modules/queue.js

@@ -0,0 +1,37 @@
+'use strict'
+
+/* global wiki */
+
+const Bull = require('bull')
+const Promise = require('bluebird')
+
+module.exports = {
+  init() {
+    wiki.data.queues.forEach(queueName => {
+      this[queueName] = new Bull(queueName, {
+        prefix: `q-${wiki.config.ha.nodeuid}`,
+        redis: wiki.config.redis
+      })
+    })
+    return this
+  },
+  clean() {
+    return Promise.each(wiki.data.queues, queueName => {
+      return new Promise((resolve, reject) => {
+        let keyStream = wiki.redis.scanStream({
+          match: `q-${wiki.config.ha.nodeuid}:${queueName}:*`
+        })
+        keyStream.on('data', resultKeys => {
+          if (resultKeys.length > 0) {
+            wiki.redis.del(resultKeys)
+          }
+        })
+        keyStream.on('end', resolve)
+      })
+    }).then(() => {
+      wiki.logger.info('Purging old queue jobs: OK')
+    }).catch(err => {
+      wiki.logger.error(err)
+    })
+  }
+}

server/modules/redis.js

@@ -19,7 +19,11 @@ module.exports = {
    */
   init() {
     if (isPlainObject(wiki.config.redis)) {
-      return new Redis(wiki.config.redis)
+      let red = new Redis(wiki.config.redis)
+      red.on('ready', () => {
+        wiki.logger.info('Redis connection: OK')
+      })
+      return red
     } else {
       wiki.logger.error('Invalid Redis configuration!')
       process.exit(1)

server/queues/git-sync.js

@@ -61,5 +61,8 @@ module.exports = (job, done) => {
     })
 
     return jobCbStreamDocs
+  }).then(() => {
+    wiki.logger.info('Git remote repository sync: DONE')
+    return true
   })
 }

server/queues/upl-clear-temp.js

@@ -22,5 +22,8 @@ module.exports = (job, done) => {
         }
       })
     })
+  }).then(() => {
+    wiki.logger.info('Purging temporary upload files: DONE')
+    return true
   })
 }

server/worker.js

@@ -2,6 +2,12 @@
 
 /* global wiki */
 
+const Promise = require('bluebird')
+
+module.exports = Promise.join(
+  wiki.db.onReady,
+  wiki.configSvc.loadFromDb(['features', 'git', 'logging', 'site', 'uploads'])
+).then(() => {
   const path = require('path')
 
   wiki.REPOPATH = path.resolve(wiki.ROOTPATH, wiki.config.paths.repo)
@@ -18,17 +24,11 @@ wiki.UPLTEMPPATH = path.join(wiki.DATAPATH, 'temp-upload')
   wiki.lang = require('i18next')
   wiki.mark = require('./modules/markdown')
 
-// ----------------------------------------
-// Load local modules
-// ----------------------------------------
-
-const Promise = require('bluebird')
-const i18nBackend = require('i18next-node-fs-backend')
-
   // ----------------------------------------
   // Localization Engine
   // ----------------------------------------
 
+  const i18nBackend = require('i18next-node-fs-backend')
   wiki.lang.use(i18nBackend).init({
     load: 'languageOnly',
     ns: ['common', 'admin', 'auth', 'errors', 'git'],
@@ -51,14 +51,12 @@ const autoload = require('auto-load')
 
   let queues = autoload(path.join(wiki.SERVERPATH, 'queues'))
 
-Promise.join(
-  wiki.db.onReady
-  // wiki.upl.initialScan()
-).then(() => {
   for (let queueName in queues) {
-    new Bull(queueName, { redis: wiki.config.redis }).process(queues[queueName])
+    new Bull(queueName, {
+      prefix: `q-${wiki.config.ha.nodeuid}`,
+      redis: wiki.config.redis
+    }).process(queues[queueName])
   }
-})
 
   // ----------------------------------------
   // Shutdown gracefully
@@ -68,3 +66,4 @@ process.on('disconnect', () => {
     wiki.logger.warn('Lost connection to Master. Exiting...')
     process.exit()
   })
+})