feat: locales remote fetch+ deps update + fixes

server/app/data.yml

@@ -27,6 +27,7 @@ defaults:
     workers: 0
     ha:
       node: primary
+      uid: master
       readonly: false
     # DB defaults
     auth:
@@ -62,88 +63,16 @@ configNamespaces:
 localeNamespaces:
   - auth
   - common
-queues:
-  - gitSync
-  - uplClearTemp
+jobs:
+  purgeUploads:
+    onInit: true
+    cron: '*/15 * * * *'
+  syncGraphLocales:
+    onInit: true
+    cron: '0 0 * * *'
 telemetry:
   BUGSNAG_ID: 'bb4b324d0675bcbba10025617fd2cec8'
   BUGSNAG_REMOTE: 'https://notify.bugsnag.com'
   GA_ID: 'UA-9094100-7'
   GA_REMOTE: 'https://www.google-analytics.com/collect'
-authProviders:
-  - local
-  - microsoft
-  - google
-  - facebook
-  - github
-  - slack
-  - ldap
-  - azure
-colors:
-  - red
-  - pink
-  - purple
-  - deep-purple
-  - indigo
-  - blue
-  - light-blue
-  - cyan
-  - teal
-  - green
-  - light-green
-  - lime
-  - yellow
-  - amber
-  - orange
-  - deep-orange
-  - brown
-  - grey
-  - blue-grey
-langs:
-  -
-    id: en
-    name: English
-    original: English
-  -
-    id: zh
-    name: Chinese
-    original: 中文
-  -
-    id: nl
-    name: Dutch
-    original: Nederlands
-  -
-    id: fr
-    name: French
-    original: Français
-  -
-    id: de
-    name: German
-    original: Deutsch
-  -
-    id: ja
-    name: Japanese
-    original: 日本語
-  -
-    id: ko
-    name: Korean
-    original: 한국어
-  -
-    id: fa
-    name: Persian (Fārsi)
-    original: فارسی
-  -
-    id: pt
-    name: Portuguese
-    original: Português
-  -
-    id: ru
-    name: Russian
-    original: Русский
-  -
-    id: es
-    name: Spanish
-    original: Español
-rtlLangs:
-  - fa
 # ---------------------------------

server/core/kernel.js

@@ -10,7 +10,7 @@ module.exports = {
   init() {
     if (cluster.isMaster) {
       WIKI.logger.info('=======================================')
-      WIKI.logger.info('= WIKI.js =============================')
+      WIKI.logger.info('= Wiki.js =============================')
       WIKI.logger.info('=======================================')
 
       WIKI.redis = require('./redis').init()
@@ -55,13 +55,7 @@ module.exports = {
   async postBootMaster() {
     await require('../master')()
 
-    _.times(this.numWorkers, () => {
-      this.spawnWorker()
-    })
-
-    WIKI.queue.uplClearTemp.add({}, {
-      repeat: { cron: '*/15 * * * *' }
-    })
+    WIKI.queue.start()
 
     cluster.on('exit', (worker, code, signal) => {
       if (!global.DEV) {

server/core/localization.js

@@ -1,9 +1,7 @@
 const _ = require('lodash')
 const dotize = require('dotize')
-const i18nBackend = require('i18next-node-fs-backend')
 const i18nMW = require('i18next-express-middleware')
 const i18next = require('i18next')
-const path = require('path')
 const Promise = require('bluebird')
 
 /* global WIKI */
@@ -14,17 +12,14 @@ module.exports = {
   init() {
     this.namespaces = WIKI.data.localeNamespaces
     this.engine = i18next
-    this.engine.use(i18nBackend).init({
+    this.engine.init({
       load: 'languageOnly',
       ns: this.namespaces,
       defaultNS: 'common',
       saveMissing: false,
       preload: [WIKI.config.site.lang],
       lng: WIKI.config.site.lang,
-      fallbackLng: 'en',
-      backend: {
-        loadPath: path.join(WIKI.SERVERPATH, 'locales/{{lng}}/{{ns}}.yml')
-      }
+      fallbackLng: 'en'
     })
     return this
   },

server/core/logger.js

@@ -1,17 +1,16 @@
 const _ = require('lodash')
-const cluster = require('cluster')
 const winston = require('winston')
 
 /* global WIKI */
 
 module.exports = {
   loggers: {},
-  init() {
+  init(uid) {
     let logger = winston.createLogger({
       level: WIKI.config.logLevel,
       format: winston.format.combine(
         winston.format.colorize(),
-        winston.format.label({ label: (cluster.isMaster) ? 'MASTER' : `WORKER-${cluster.worker.id}` }),
+        winston.format.label({ label: uid }),
         winston.format.timestamp(),
         winston.format.printf(info => `${info.timestamp} [${info.label}] ${info.level}: ${info.message}`)
       )

server/core/queue.js

@@ -1,23 +1,40 @@
+const path = require('path')
 const Bull = require('bull')
 const Promise = require('bluebird')
+const _ = require('lodash')
 
 /* global WIKI */
 
 module.exports = {
+  job: {},
   init() {
-    WIKI.data.queues.forEach(queueName => {
-      this[queueName] = new Bull(queueName, {
-        prefix: `q-${WIKI.config.ha.nodeuid}`,
+    _.forOwn(WIKI.data.jobs, (queueParams, queueName) => {
+      this.job[queueName] = new Bull(queueName, {
+        prefix: `q-${WIKI.config.ha.uid}`,
         redis: WIKI.config.redis
       })
+      this.job[queueName].process(path.join(WIKI.SERVERPATH, `jobs/${_.kebabCase(queueName)}.js`))
     })
     return this
   },
-  clean() {
-    return Promise.each(WIKI.data.queues, queueName => {
+  start() {
+    _.forOwn(WIKI.data.jobs, (queueParams, queueName) => {
+      if (queueParams.onInit) {
+        this.job[queueName].add({}, {
+          removeOnComplete: true
+        })
+      }
+      this.job[queueName].add({}, {
+        repeat: { cron: queueParams.cron },
+        removeOnComplete: true
+      })
+    })
+  },
+  async clean() {
+    return Promise.each(_.keys(WIKI.data.jobs), queueName => {
       return new Promise((resolve, reject) => {
         let keyStream = WIKI.redis.scanStream({
-          match: `q-${WIKI.config.ha.nodeuid}:${queueName}:*`
+          match: `q-${WIKI.config.ha.uid}:${queueName}:*`
         })
         keyStream.on('data', resultKeys => {
           if (resultKeys.length > 0) {

server/core/worker.js

@@ -0,0 +1,18 @@
+const path = require('path')
+
+let WIKI = {
+  IS_DEBUG: process.env.NODE_ENV === 'development',
+  ROOTPATH: process.cwd(),
+  SERVERPATH: path.join(process.cwd(), 'server'),
+  Error: require('../helpers/error'),
+  configSvc: require('./config')
+}
+global.WIKI = WIKI
+
+WIKI.configSvc.init()
+
+// ----------------------------------------
+// Init Logger
+// ----------------------------------------
+
+WIKI.logger = require('./logger').init('JOB')

server/graph/resolvers/localization.js

@@ -0,0 +1,46 @@
+const graphHelper = require('../../helpers/graph')
+const _ = require('lodash')
+
+/* global WIKI */
+
+module.exports = {
+  Query: {
+    async localization() { return {} }
+  },
+  Mutation: {
+    async localization() { return {} }
+  },
+  LocalizationQuery: {
+    async locales(obj, args, context, info) {
+      let remoteLocales = await WIKI.redis.get('locales')
+      let localLocales = await WIKI.db.Locale.findAll({
+        attributes: {
+          exclude: ['strings']
+        },
+        raw: true
+      })
+      remoteLocales = (remoteLocales) ? JSON.parse(remoteLocales) : localLocales
+      return _.map(remoteLocales, rl => {
+        let isInstalled = _.some(localLocales, ['code', rl.code])
+        return {
+          ...rl,
+          isInstalled,
+          installDate: isInstalled ? _.find(localLocales, ['code', rl.code]).updatedAt : null
+        }
+      })
+    }
+  },
+  LocalizationMutation: {
+    async updateLocale(obj, args, context) {
+      try {
+        let authResult = await WIKI.db.User.login(args, context)
+        return {
+          ...authResult,
+          responseResult: graphHelper.generateSuccess('Login success')
+        }
+      } catch (err) {
+        return graphHelper.generateError(err)
+      }
+    }
+  }
+}

server/graph/schemas/localization.graphql

@@ -0,0 +1,45 @@
+# ===============================================
+# LOCALIZATION
+# ===============================================
+
+extend type Query {
+  localization: LocalizationQuery
+}
+
+extend type Mutation {
+  localization: LocalizationMutation
+}
+
+# -----------------------------------------------
+# QUERIES
+# -----------------------------------------------
+
+type LocalizationQuery {
+  locales: [LocalizationLocale]
+}
+
+# -----------------------------------------------
+# MUTATIONS
+# -----------------------------------------------
+
+type LocalizationMutation {
+  updateLocale(
+    localeId: String!
+    autoUpdate: Boolean!
+  ): DefaultResponse
+}
+
+# -----------------------------------------------
+# TYPES
+# -----------------------------------------------
+
+type LocalizationLocale {
+  code: String!
+  createdAt: Date!
+  installDate: Date
+  isInstalled: Boolean!
+  isRTL: Boolean!
+  name: String!
+  nativeName: String!
+  updatedAt: Date!
+}

server/index.js

@@ -23,7 +23,7 @@ WIKI.configSvc.init()
 // Init Logger
 // ----------------------------------------
 
-WIKI.logger = require('./core/logger').init()
+WIKI.logger = require('./core/logger').init('MASTER')
 
 // ----------------------------------------
 // Init Telemetry

server/jobs/purge-uploads.js

@@ -0,0 +1,33 @@
+require('../core/worker')
+
+/* global WIKI */
+
+const Promise = require('bluebird')
+const fs = Promise.promisifyAll(require('fs-extra'))
+const moment = require('moment')
+const path = require('path')
+
+module.exports = async (job) => {
+  WIKI.logger.info('Purging orphaned upload files...')
+
+  try {
+    const uplTempPath = path.resolve(process.cwd(), WIKI.config.paths.data, 'temp-upload')
+    const ls = await fs.readdirAsync(uplTempPath)
+    const fifteenAgo = moment().subtract(15, 'minutes')
+
+    await Promise.map(ls, (f) => {
+      return fs.statAsync(path.join(uplTempPath, f)).then((s) => { return { filename: f, stat: s } })
+    }).filter((s) => { return s.stat.isFile() }).then((arrFiles) => {
+      return Promise.map(arrFiles, (f) => {
+        if (moment(f.stat.ctime).isBefore(fifteenAgo, 'minute')) {
+          return fs.unlinkAsync(path.join(uplTempPath, f.filename))
+        }
+      })
+    })
+
+    WIKI.logger.info('Purging orphaned upload files: [ COMPLETED ]')
+  } catch (err) {
+    WIKI.logger.error('Purging orphaned upload files: [ FAILED ]')
+    WIKI.logger.error(err.message)
+  }
+}

server/jobs/sync-git.js

@@ -0,0 +1,69 @@
+'use strict'
+
+// /* global WIKI */
+
+// const Promise = require('bluebird')
+// const fs = Promise.promisifyAll(require('fs-extra'))
+// const klaw = require('klaw')
+// const moment = require('moment')
+// const path = require('path')
+// const entryHelper = require('../helpers/entry')
+
+module.exports = (job) => {
+  return true
+  // return WIKI.git.resync().then(() => {
+  //   // -> Stream all documents
+
+  //   let cacheJobs = []
+  //   let jobCbStreamDocsResolve = null
+  //   let jobCbStreamDocs = new Promise((resolve, reject) => {
+  //     jobCbStreamDocsResolve = resolve
+  //   })
+
+  //   klaw(WIKI.REPOPATH).on('data', function (item) {
+  //     if (path.extname(item.path) === '.md' && path.basename(item.path) !== 'README.md') {
+  //       let entryPath = entryHelper.parsePath(entryHelper.getEntryPathFromFullPath(item.path))
+  //       let cachePath = entryHelper.getCachePath(entryPath)
+
+  //       // -> Purge outdated cache
+
+  //       cacheJobs.push(
+  //         fs.statAsync(cachePath).then((st) => {
+  //           return moment(st.mtime).isBefore(item.stats.mtime) ? 'expired' : 'active'
+  //         }).catch((err) => {
+  //           return (err.code !== 'EEXIST') ? err : 'new'
+  //         }).then((fileStatus) => {
+  //           // -> Delete expired cache file
+
+  //           if (fileStatus === 'expired') {
+  //             return fs.unlinkAsync(cachePath).return(fileStatus)
+  //           }
+
+  //           return fileStatus
+  //         }).then((fileStatus) => {
+  //           // -> Update cache and search index
+
+  //           if (fileStatus !== 'active') {
+  //             return global.entries.updateCache(entryPath).then(entry => {
+  //               process.send({
+  //                 action: 'searchAdd',
+  //                 content: entry
+  //               })
+  //               return true
+  //             })
+  //           }
+
+  //           return true
+  //         })
+  //       )
+  //     }
+  //   }).on('end', () => {
+  //     jobCbStreamDocsResolve(Promise.all(cacheJobs))
+  //   })
+
+  //   return jobCbStreamDocs
+  // }).then(() => {
+  //   WIKI.logger.info('Git remote repository sync: DONE')
+  //   return true
+  // })
+}

server/jobs/sync-graph-locales.js

@@ -0,0 +1,38 @@
+require('../core/worker')
+const _ = require('lodash')
+const { createApolloFetch } = require('apollo-fetch')
+
+/* global WIKI */
+
+WIKI.redis = require('../core/redis').init()
+const apollo = createApolloFetch({
+  uri: 'https://graph.requarks.io'
+})
+
+module.exports = async (job) => {
+  WIKI.logger.info('Syncing locales with Graph endpoint...')
+
+  try {
+    const resp = await apollo({
+      query: `{
+        localization {
+          locales {
+            code
+            name
+            nativeName
+            isRTL
+            createdAt
+            updatedAt
+          }
+        }
+      }`
+    })
+    const locales = _.sortBy(_.get(resp, 'data.localization.locales', []), 'name').map(lc => ({...lc, isInstalled: (lc.code === 'en')}))
+    WIKI.redis.set('locales', JSON.stringify(locales))
+
+    WIKI.logger.info('Syncing locales with Graph endpoint: [ COMPLETED ]')
+  } catch (err) {
+    WIKI.logger.error('Syncing locales with Graph endpoint: [ FAILED ]')
+    WIKI.logger.error(err.message)
+  }
+}

server/locales/default.json

@@ -0,0 +1,38 @@
+{
+  "auth": {
+    "actions": {
+      "login": "Log In"
+    },
+    "errors": {
+      "invalidLogin": "Invalid Login",
+      "invalidLoginMsg": "The email or password is invalid.",
+      "invalidUserEmail": "Invalid User Email",
+      "loginError": "Login error",
+      "notYetAuthorized": "You have not been authorized to login to this site yet.",
+      "tooManyAttempts": "Too many attempts!",
+      "tooManyAttemptsMsg": "You've made too many failed attempts in a short period of time, please try again {{time}}.",
+      "userNotFound": "User not found"
+    },
+    "fields": {
+      "emailUser": "Email / Username",
+      "password": "Password"
+    },
+    "loginRequired": "Login required",
+    "providers": {
+      "azure": "Azure Active Directory",
+      "facebook": "Facebook",
+      "github": "GitHub",
+      "google": "Google ID",
+      "ldap": "LDAP / Active Directory",
+      "local": "Local",
+      "slack": "Slack",
+      "windowslive": "Microsoft Account"
+    },
+    "tfa": {
+      "placeholder": "XXXXXX",
+      "subtitle": "Security code required:",
+      "title": "Two Factor Authentication",
+      "verifyToken": "Verify"
+    }
+  }
+}

server/models/locale.js

@@ -0,0 +1,39 @@
+/**
+ * Locale schema
+ */
+module.exports = (sequelize, DataTypes) => {
+  let localeSchema = sequelize.define('locale', {
+    code: {
+      type: DataTypes.STRING,
+      allowNull: false
+    },
+    strings: {
+      type: DataTypes.JSON,
+      allowNull: true
+    },
+    isRTL: {
+      type: DataTypes.BOOLEAN,
+      allowNull: false,
+      defaultValue: false
+    },
+    name: {
+      type: DataTypes.STRING,
+      allowNull: false
+    },
+    nativeName: {
+      type: DataTypes.STRING,
+      allowNull: false
+    }
+  }, {
+    timestamps: true,
+    version: true,
+    indexes: [
+      {
+        unique: true,
+        fields: ['code']
+      }
+    ]
+  })
+
+  return localeSchema
+}

server/queues/git-sync.js

@@ -1,68 +0,0 @@
-'use strict'
-
-/* global WIKI */
-
-const Promise = require('bluebird')
-const fs = Promise.promisifyAll(require('fs-extra'))
-const klaw = require('klaw')
-const moment = require('moment')
-const path = require('path')
-const entryHelper = require('../helpers/entry')
-
-module.exports = (job) => {
-  return WIKI.git.resync().then(() => {
-    // -> Stream all documents
-
-    let cacheJobs = []
-    let jobCbStreamDocsResolve = null
-    let jobCbStreamDocs = new Promise((resolve, reject) => {
-      jobCbStreamDocsResolve = resolve
-    })
-
-    klaw(WIKI.REPOPATH).on('data', function (item) {
-      if (path.extname(item.path) === '.md' && path.basename(item.path) !== 'README.md') {
-        let entryPath = entryHelper.parsePath(entryHelper.getEntryPathFromFullPath(item.path))
-        let cachePath = entryHelper.getCachePath(entryPath)
-
-        // -> Purge outdated cache
-
-        cacheJobs.push(
-          fs.statAsync(cachePath).then((st) => {
-            return moment(st.mtime).isBefore(item.stats.mtime) ? 'expired' : 'active'
-          }).catch((err) => {
-            return (err.code !== 'EEXIST') ? err : 'new'
-          }).then((fileStatus) => {
-            // -> Delete expired cache file
-
-            if (fileStatus === 'expired') {
-              return fs.unlinkAsync(cachePath).return(fileStatus)
-            }
-
-            return fileStatus
-          }).then((fileStatus) => {
-            // -> Update cache and search index
-
-            if (fileStatus !== 'active') {
-              return global.entries.updateCache(entryPath).then(entry => {
-                process.send({
-                  action: 'searchAdd',
-                  content: entry
-                })
-                return true
-              })
-            }
-
-            return true
-          })
-        )
-      }
-    }).on('end', () => {
-      jobCbStreamDocsResolve(Promise.all(cacheJobs))
-    })
-
-    return jobCbStreamDocs
-  }).then(() => {
-    WIKI.logger.info('Git remote repository sync: DONE')
-    return true
-  })
-}

server/queues/upl-clear-temp.js

@@ -1,29 +0,0 @@
-'use strict'
-
-/* global WIKI */
-
-const Promise = require('bluebird')
-const fs = Promise.promisifyAll(require('fs-extra'))
-const moment = require('moment')
-const path = require('path')
-
-module.exports = (job) => {
-  return fs.readdirAsync(WIKI.UPLTEMPPATH).then((ls) => {
-    let fifteenAgo = moment().subtract(15, 'minutes')
-
-    return Promise.map(ls, (f) => {
-      return fs.statAsync(path.join(WIKI.UPLTEMPPATH, f)).then((s) => { return { filename: f, stat: s } })
-    }).filter((s) => { return s.stat.isFile() }).then((arrFiles) => {
-      return Promise.map(arrFiles, (f) => {
-        if (moment(f.stat.ctime).isBefore(fifteenAgo, 'minute')) {
-          return fs.unlinkAsync(path.join(WIKI.UPLTEMPPATH, f.filename))
-        } else {
-          return true
-        }
-      })
-    })
-  }).then(() => {
-    WIKI.logger.info('Purging temporary upload files: DONE')
-    return true
-  })
-}

server/setup.js

@@ -283,7 +283,7 @@ module.exports = () => {
 
       // Site namespace
       _.set(WIKI.config.site, 'title', req.body.title)
-      _.set(WIKI.config.site, 'lang', req.body.lang)
+      _.set(WIKI.config.site, 'lang', 'en')
       _.set(WIKI.config.site, 'rtl', _.includes(WIKI.data.rtlLangs, req.body.lang))
       _.set(WIKI.config.site, 'sessionSecret', (await crypto.randomBytesAsync(32)).toString('hex'))
 
@@ -310,6 +310,16 @@ module.exports = () => {
         tfaIsActive: false
       })
 
+      // Create default locale
+      WIKI.logger.info('Installing default locale...')
+      await WIKI.db.Locale.upsert({
+        code: 'en',
+        strings: require('./locales/default.json'),
+        isRTL: false,
+        name: 'English',
+        nativeName: 'English'
+      })
+
       WIKI.logger.info('Setup is complete!')
       res.json({
         ok: true,

server/views/main/setup.pug

@@ -3,4 +3,4 @@ extends ../master.pug
 block body
   body
     #app
-      setup(telemetry-id=telemetryClientID, wiki-version=packageObj.version, :langs!=JSON.stringify(data.langs).replace(/"/g, "'"))
+      setup(telemetry-id=telemetryClientID, wiki-version=packageObj.version)

server/worker.js

@@ -1,62 +0,0 @@
-const Promise = require('bluebird')
-
-/* global WIKI */
-
-module.exports = Promise.join(
-  WIKI.db.onReady,
-  WIKI.configSvc.loadFromDb(['features', 'logging', 'site', 'uploads'])
-).then(() => {
-  const path = require('path')
-
-  WIKI.REPOPATH = path.resolve(WIKI.ROOTPATH, WIKI.config.paths.repo)
-  WIKI.DATAPATH = path.resolve(WIKI.ROOTPATH, WIKI.config.paths.data)
-  WIKI.UPLTEMPPATH = path.join(WIKI.DATAPATH, 'temp-upload')
-
-  // ----------------------------------------
-  // Load global modules
-  // ----------------------------------------
-
-  WIKI.lang = require('i18next')
-
-  // ----------------------------------------
-  // Localization Engine
-  // ----------------------------------------
-
-  const i18nBackend = require('i18next-node-fs-backend')
-  WIKI.lang.use(i18nBackend).init({
-    load: 'languageOnly',
-    ns: ['common', 'admin', 'auth', 'errors'],
-    defaultNS: 'common',
-    saveMissing: false,
-    preload: [WIKI.config.lang],
-    lng: WIKI.config.lang,
-    fallbackLng: 'en',
-    backend: {
-      loadPath: path.join(WIKI.SERVERPATH, 'locales/{{lng}}/{{ns}}.yml')
-    }
-  })
-
-  // ----------------------------------------
-  // Start Queues
-  // ----------------------------------------
-
-  const Bull = require('bull')
-  const autoload = require('auto-load')
-
-  let queues = autoload(path.join(WIKI.SERVERPATH, 'queues'))
-
-  for (let queueName in queues) {
-    new Bull(queueName, {
-      prefix: `q-${WIKI.config.ha.nodeuid}`,
-      redis: WIKI.config.redis
-    }).process(queues[queueName])
-  }
-
-  // ----------------------------------------
-  // Shutdown gracefully
-  // ----------------------------------------
-
-  process.on('disconnect', () => {
-    process.exit()
-  })
-})