liz/wikijs-fork
1
0
Fork 0
Code Issues Pull Requests Packages Projects Releases Wiki Activity

refactor: removed redis + new scheduler engine

Browse Source
This commit is contained in:
Nick
2019-02-13 17:20:46 -05:00
parent e90873e13d
commit cd3f88bad0
51 changed files with 274 additions and 735 deletions
Download Patch File Download Diff File Expand all files Collapse all files

25
server/jobs/fetch-graph-locale.js
View File

@@ -1,17 +1,12 @@
require('../core/worker')
const _ = require('lodash')
const { createApolloFetch } = require('apollo-fetch')
/* global WIKI */
WIKI.redis = require('../core/redis').init()
WIKI.models = require('../core/db').init()
module.exports = async (job) => {
WIKI.logger.info(`Fetching locale ${job.data.locale} from Graph endpoint...`)
module.exports = async (localeCode) => {
WIKI.logger.info(`Fetching locale ${localeCode} from Graph endpoint...`)
try {
await WIKI.configSvc.loadFromDb()
const apollo = createApolloFetch({
uri: WIKI.config.graphEndpoint
})
@@ -26,7 +21,7 @@ module.exports = async (job) => {
}
}`,
variables: {
code: job.data.locale
code: localeCode
}
})
const strings = _.get(respStrings, 'data.localization.strings', [])
@@ -36,12 +31,12 @@ module.exports = async (job) => {
_.set(lcObj, row.key.replace(':', '.'), row.value)
})
const locales = await WIKI.redis.get('locales')
const locales = await WIKI.cache.get('locales')
if (locales) {
const currentLocale = _.find(JSON.parse(locales), ['code', job.data.locale]) || {}
await WIKI.models.locales.query().delete().where('code', job.data.locale)
const currentLocale = _.find(locales, ['code', localeCode]) || {}
await WIKI.models.locales.query().delete().where('code', localeCode)
await WIKI.models.locales.query().insert({
code: job.data.locale,
code: localeCode,
strings: lcObj,
isRTL: currentLocale.isRTL,
name: currentLocale.name,
@@ -51,11 +46,11 @@ module.exports = async (job) => {
throw new Error('Failed to fetch cached locales list! Restart server to resolve this issue.')
}
await WIKI.redis.publish('localization', 'reload')
await WIKI.lang.refreshNamespaces()
WIKI.logger.info(`Fetching locale ${job.data.locale} from Graph endpoint: [ COMPLETED ]`)
WIKI.logger.info(`Fetching locale ${localeCode} from Graph endpoint: [ COMPLETED ]`)
} catch (err) {
WIKI.logger.error(`Fetching locale ${job.data.locale} from Graph endpoint: [ FAILED ]`)
WIKI.logger.error(`Fetching locale ${localeCode} from Graph endpoint: [ FAILED ]`)
WIKI.logger.error(err.message)
}
}

12
server/jobs/purge-uploads.js
View File

@@ -1,26 +1,24 @@
require('../core/worker')
/* global WIKI */
const Promise = require('bluebird')
const fs = Promise.promisifyAll(require('fs-extra'))
const fs = require('fs-extra')
const moment = require('moment')
const path = require('path')
module.exports = async (job) => {
module.exports = async () => {
WIKI.logger.info('Purging orphaned upload files...')
try {
const uplTempPath = path.resolve(process.cwd(), WIKI.config.paths.data, 'uploads')
const ls = await fs.readdirAsync(uplTempPath)
const ls = await fs.readdir(uplTempPath)
const fifteenAgo = moment().subtract(15, 'minutes')
await Promise.map(ls, (f) => {
return fs.statAsync(path.join(uplTempPath, f)).then((s) => { return { filename: f, stat: s } })
return fs.stat(path.join(uplTempPath, f)).then((s) => { return { filename: f, stat: s } })
}).filter((s) => { return s.stat.isFile() }).then((arrFiles) => {
return Promise.map(arrFiles, (f) => {
if (moment(f.stat.ctime).isBefore(fifteenAgo, 'minute')) {
return fs.unlinkAsync(path.join(uplTempPath, f.filename))
return fs.unlink(path.join(uplTempPath, f.filename))
}
})
})

34
server/jobs/render-page.js
View File

@@ -1,23 +1,29 @@
require('../core/worker')
const _ = require('lodash')
const cheerio = require('cheerio')
/* global WIKI */
WIKI.models = require('../core/db').init()
module.exports = async (job) => {
WIKI.logger.info(`Rendering page ${job.data.page.path}...`)
module.exports = async (pageId) => {
WIKI.logger.info(`Rendering page ID ${pageId}...`)
try {
let output = job.data.page.content
for (let core of job.data.pipeline) {
WIKI.models = require('../core/db').init()
const page = await WIKI.models.pages.getPageFromDb(pageId)
if (!page) {
throw new Error('Invalid Page Id')
}
await WIKI.models.renderers.fetchDefinitions()
const pipeline = await WIKI.models.renderers.getRenderingPipeline(page.contentType)
let output = page.content
for (let core of pipeline) {
const renderer = require(`../modules/rendering/${_.kebabCase(core.key)}/renderer.js`)
output = await renderer.render.call({
config: core.config,
children: core.children,
page: job.data.page,
page: page,
input: output
})
}
@@ -61,18 +67,20 @@ module.exports = async (job) => {
render: output,
toc: JSON.stringify(toc.root)
})
.where('id', job.data.page.id)
.where('id', pageId)
// Save to cache
await WIKI.models.pages.savePageToCache({
...job.data.page,
...page,
render: output,
toc: JSON.stringify(toc.root)
})
WIKI.logger.info(`Rendering page ${job.data.page.path}: [ COMPLETED ]`)
await WIKI.models.knex.destroy()
WIKI.logger.info(`Rendering page ID ${pageId}: [ COMPLETED ]`)
} catch (err) {
WIKI.logger.error(`Rendering page ${job.data.page.path}: [ FAILED ]`)
WIKI.logger.error(`Rendering page ID ${pageId}: [ FAILED ]`)
WIKI.logger.error(err.message)
}
}

11
server/jobs/sync-graph-locales.js
View File

@@ -1,17 +1,12 @@
require('../core/worker')
const _ = require('lodash')
const { createApolloFetch } = require('apollo-fetch')
/* global WIKI */
WIKI.redis = require('../core/redis').init()
WIKI.models = require('../core/db').init()
module.exports = async (job) => {
module.exports = async () => {
WIKI.logger.info('Syncing locales with Graph endpoint...')
try {
await WIKI.configSvc.loadFromDb()
const apollo = createApolloFetch({
uri: WIKI.config.graphEndpoint
})
@@ -33,7 +28,7 @@ module.exports = async (job) => {
}`
})
const locales = _.sortBy(_.get(respList, 'data.localization.locales', []), 'name').map(lc => ({...lc, isInstalled: (lc.code === 'en')}))
WIKI.redis.set('locales', JSON.stringify(locales))
WIKI.cache.set('locales', locales)
const currentLocale = _.find(locales, ['code', WIKI.config.lang.code])
// -> Download locale strings
@@ -68,7 +63,7 @@ module.exports = async (job) => {
}).where('code', WIKI.config.lang.code)
}
await WIKI.redis.publish('localization', 'reload')
await WIKI.lang.refreshNamespaces()
WIKI.logger.info('Syncing locales with Graph endpoint: [ COMPLETED ]')
} catch (err) {

14
server/jobs/sync-graph-updates.js
View File

@@ -1,17 +1,12 @@
require('../core/worker')
const _ = require('lodash')
const { createApolloFetch } = require('apollo-fetch')
/* global WIKI */
WIKI.redis = require('../core/redis').init()
WIKI.models = require('../core/db').init()
module.exports = async (job) => {
module.exports = async () => {
WIKI.logger.info(`Fetching latest updates from Graph endpoint...`)
try {
await WIKI.configSvc.loadFromDb()
const apollo = createApolloFetch({
uri: WIKI.config.graphEndpoint
})
@@ -33,9 +28,10 @@ module.exports = async (job) => {
version: WIKI.version
}
})
const info = _.get(resp, 'data.releases.checkForUpdates', {})
await WIKI.redis.publish('updates', JSON.stringify(info))
const info = _.get(resp, 'data.releases.checkForUpdates', false)
if (info) {
WIKI.system.updates = info
}
WIKI.logger.info(`Fetching latest updates from Graph endpoint: [ COMPLETED ]`)
} catch (err) {