liz/wikijs-fork
liz/wikijs-fork
1
0
Fork 0
Code Issues Pull Requests Packages Projects Releases Wiki Activity

feat: scheduler storage sync

Browse Source
This commit is contained in:
Nick 2019-02-17 21:48:48 -05:00
parent aa27554bc7
commit 7e458f98b4
9 changed files with 159 additions and 127 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

24
client/components/admin/admin-storage.vue
View File

@ -39,11 +39,13 @@
.pa-3.grey.radius-7(:class='$vuetify.dark ? "darken-4" : "lighten-5"')
v-layout.pa-2(row, justify-space-between)
.body-2.grey--text.text--darken-1 Status
looping-rhombuses-spinner.mt-1(
:animation-duration='5000'
:rhombus-size='10'
color='#BBB'
)
.d-flex
looping-rhombuses-spinner.mt-1(
:animation-duration='5000'
:rhombus-size='10'
color='#BBB'
)
.caption.ml-3.grey--text This panel refreshes automatically.
v-divider
v-toolbar.mt-2.radius-7(
v-for='(tgt, n) in status'
@ -62,7 +64,7 @@
v-alert.mt-3.radius-7(v-if='status.length < 1', outline, :value='true', color='indigo') You don't have any active storage target.
v-tab-item(v-for='(tgt, n) in activeTargets', :key='tgt.key', :transition='false', :reverse-transition='false')
v-card.pa-3(flat, tile)
v-card.wiki-form.pa-3(flat, tile)
v-form
.targetlogo
img(:src='tgt.logo', :alt='tgt.title')
@ -145,12 +147,15 @@
.body-1.ml-3 For performance reasons, this storage target synchronize changes on an interval-based schedule, instead of on every change. Define at which interval should the synchronization occur.
.pa-3
duration-picker(v-model='tgt.syncInterval')
.caption.mt-3 The default is every #[strong 5 minutes].
.caption.mt-3 Currently set to every #[strong {{getDefaultSchedule(tgt.syncInterval)}}].
.caption The default is every #[strong {{getDefaultSchedule(tgt.syncIntervalDefault)}}].
</template>
<script>
import _ from 'lodash'
import moment from 'moment'
import momentDurationFormatSetup from 'moment-duration-format'
import DurationPicker from '../common/duration-picker.vue'
import { LoopingRhombusesSpinner } from 'epic-spinners'
@ -159,6 +164,8 @@ import statusQuery from 'gql/admin/storage/storage-query-status.gql'
import targetsQuery from 'gql/admin/storage/storage-query-targets.gql'
import targetsSaveMutation from 'gql/admin/storage/storage-mutation-save-targets.gql'
momentDurationFormatSetup(moment)
export default {
components: {
DurationPicker,
@ -221,6 +228,9 @@ export default {
default:
return 'grey darken-2'
}
},
getDefaultSchedule(val) {
return moment.duration(val).format('y [years], M [months], d [days], h [hours], m [minutes]')
}
},
apollo: {

2
client/components/common/duration-picker.vue
View File

@ -1,5 +1,5 @@
<template lang='pug'>
v-toolbar(flat, :color='$vuetify.dark ? "grey darken-4-l3" : "grey lighten-3"')
v-toolbar.radius-7(flat, :color='$vuetify.dark ? "grey darken-4-l3" : "grey lighten-3"')
.body-2.mr-3 Every
v-text-field(
solo

1
package.json
View File

@ -221,6 +221,7 @@
"ignore-loader": "0.1.2",
"js-cookie": "2.2.0",
"mini-css-extract-plugin": "0.5.0",
"moment-duration-format": "2.2.2",
"node-sass": "4.11.0",
"offline-plugin": "5.0.6",
"optimize-css-assets-webpack-plugin": "5.0.1",

3
server/app/data.yml
View File

@ -57,9 +57,6 @@ jobs:
syncGraphUpdates:
onInit: true
schedule: P1D
syncStorage:
onInit: true
schedule: storage.syncInterval
groups:
defaultPermissions:
- 'manage:pages'

84
server/core/job.js
View File

@ -1,84 +0,0 @@
const moment = require('moment')
const childProcess = require('child_process')
module.exports = class Job {
constructor({
name,
immediate = false,
schedule = 'P1D',
repeat = false,
worker = false
}) {
this.finished = Promise.resolve()
this.name = name
this.immediate = immediate
this.schedule = moment.duration(schedule)
this.repeat = repeat
this.worker = worker
}
/**
* Start Job
*
* @param {Object} data Job Data
*/
start(data) {
if (this.immediate) {
this.invoke(data)
} else {
this.queue(data)
}
}
/**
* Queue the next job run according to the wait duration
*
* @param {Object} data Job Data
*/
queue(data) {
this.timeout = setTimeout(this.invoke.bind(this), this.schedule.asMilliseconds(), data)
}
/**
* Run the actual job
*
* @param {Object} data Job Data
*/
async invoke(data) {
try {
if (this.worker) {
const proc = childProcess.fork(`server/core/worker.js`, [
`--job=${this.name}`,
`--data=${data}`
], {
cwd: WIKI.ROOTPATH
})
this.finished = new Promise((resolve, reject) => {
proc.on('exit', (code, signal) => {
if (code === 0) {
resolve()
} else {
reject(signal)
}
proc.kill()
})
})
} else {
this.finished = require(`../jobs/${this.name}`)(data)
}
await this.finished
} catch (err) {
WIKI.logger.warn(err)
}
if (this.repeat) {
this.queue(data)
}
}
/**
* Stop any future job invocation from occuring
*/
stop() {
clearTimeout(this.timeout)
}
}

100
server/core/scheduler.js
View File

@ -1,9 +1,93 @@
const Job = require('./job')
const moment = require('moment')
const childProcess = require('child_process')
const _ = require('lodash')
const configHelper = require('../helpers/config')
/* global WIKI */
class Job {
constructor({
name,
immediate = false,
schedule = 'P1D',
repeat = false,
worker = false
}) {
this.finished = Promise.resolve()
this.name = name
this.immediate = immediate
this.schedule = moment.duration(schedule)
this.repeat = repeat
this.worker = worker
}
/**
* Start Job
*
* @param {Object} data Job Data
*/
start(data) {
if (this.immediate) {
this.invoke(data)
} else {
this.queue(data)
}
}
/**
* Queue the next job run according to the wait duration
*
* @param {Object} data Job Data
*/
queue(data) {
this.timeout = setTimeout(this.invoke.bind(this), this.schedule.asMilliseconds(), data)
}
/**
* Run the actual job
*
* @param {Object} data Job Data
*/
async invoke(data) {
try {
if (this.worker) {
const proc = childProcess.fork(`server/core/worker.js`, [
`--job=${this.name}`,
`--data=${data}`
], {
cwd: WIKI.ROOTPATH
})
this.finished = new Promise((resolve, reject) => {
proc.on('exit', (code, signal) => {
if (code === 0) {
resolve()
} else {
reject(signal)
}
proc.kill()
})
})
} else {
this.finished = require(`../jobs/${this.name}`)(data)
}
await this.finished
} catch (err) {
WIKI.logger.warn(err)
}
if (this.repeat) {
this.queue(data)
}
}
/**
* Stop any future job invocation from occuring
*/
stop() {
clearTimeout(this.timeout)
}
}
module.exports = {
jobs: [],
init() {
@ -11,13 +95,13 @@ module.exports = {
},
start() {
_.forOwn(WIKI.data.jobs, (queueParams, queueName) => {
const schedule = (configHelper.isValidDurationString(queueParams.schedule)) ? queueParams : _.get(WIKI.config, queueParams.schedule)
// this.registerJob({
// name: _.kebabCase(queueName),
// immediate: queueParams.onInit,
// schedule: schedule,
// repeat: true
// })
const schedule = (configHelper.isValidDurationString(queueParams.schedule)) ? queueParams.schedule : _.get(WIKI.config, queueParams.schedule)
this.registerJob({
name: _.kebabCase(queueName),
immediate: queueParams.onInit,
schedule: schedule,
repeat: true
})
})
},
registerJob(opts, data) {

30
server/jobs/sync-storage.js
View File

@ -1,18 +1,20 @@
const _ = require('lodash')
/* global WIKI */
module.exports = async ({ target }) => {
WIKI.logger.info(`Syncing with storage provider ${job.data.target.title}...`)
module.exports = async (targetKey) => {
WIKI.logger.info(`Syncing with storage target ${targetKey}...`)
// try {
// const target = require(`../modules/storage/${job.data.target.key}/storage.js`)
// target[job.data.event].call({
// config: job.data.target.config,
// mode: job.data.target.mode,
// page: job.data.page
// })
// WIKI.logger.info(`Syncing with storage provider ${job.data.target.title}: [ COMPLETED ]`)
// } catch (err) {
// WIKI.logger.error(`Syncing with storage provider ${job.data.target.title}: [ FAILED ]`)
// WIKI.logger.error(err.message)
// }
try {
const target = _.find(WIKI.models.storage.targets, ['key', targetKey])
if (target) {
await target.fn.sync()
WIKI.logger.info(`Syncing with storage target ${targetKey}: [ COMPLETED ]`)
} else {
throw new Error('Invalid storage target. Unable to perform sync.')
}
} catch (err) {
WIKI.logger.error(`Syncing with storage target ${targetKey}: [ FAILED ]`)
WIKI.logger.error(err.message)
}
}

37
server/models/storage.js
View File

@ -7,8 +7,6 @@ const commonHelper = require('../helpers/common')
/* global WIKI */
let targets = []
/**
* Storage model
*/
@ -104,22 +102,46 @@ module.exports = class Storage extends Model {
}
}
/**
* Initialize active storage targets
*/
static async initTargets() {
targets = await WIKI.models.storage.query().where('isEnabled', true).orderBy('key')
this.targets = await WIKI.models.storage.query().where('isEnabled', true).orderBy('key')
try {
for(let target of targets) {
// -> Stop and delete existing jobs
const prevjobs = _.remove(WIKI.scheduler.jobs, job => job.name === `sync-storage`)
if (prevjobs.length > 0) {
prevjobs.forEach(job => job.stop())
}
// -> Initialize targets
for(let target of this.targets) {
const targetDef = _.find(WIKI.data.storage, ['key', target.key])
target.fn = require(`../modules/storage/${target.key}/storage`)
target.fn.config = target.config
target.fn.mode = target.mode
try {
await target.fn.init()
// -> Save succeeded init state
await WIKI.models.storage.query().patch({
state: {
status: 'operational',
message: ''
}
}).where('key', target.key)
// -> Set recurring sync job
if (targetDef.schedule && target.syncInterval !== `P0D`) {
WIKI.scheduler.registerJob({
name: `sync-storage`,
immediate: false,
schedule: target.syncInterval,
repeat: true
}, target.key)
}
} catch (err) {
// -> Save initialization error
await WIKI.models.storage.query().patch({
state: {
status: 'error',
@ -127,11 +149,6 @@ module.exports = class Storage extends Model {
}
}).where('key', target.key)
}
// if (target.schedule) {
// WIKI.scheduler.registerJob({
// name:
// }, target.fn.sync)
// }
}
} catch (err) {
WIKI.logger.warn(err)
@ -141,7 +158,7 @@ module.exports = class Storage extends Model {
static async pageEvent({ event, page }) {
try {
for(let target of targets) {
for(let target of this.targets) {
await target.fn[event](page)
}
} catch (err) {

5
yarn.lock
View File

@ -7840,6 +7840,11 @@ mkdirp@0.5.1, mkdirp@0.5.x, "mkdirp@>=0.5 0", mkdirp@^0.5.0, mkdirp@^0.5.1, mkdi
dependencies:
minimist "0.0.8"
moment-duration-format@2.2.2:
version "2.2.2"
resolved "https://registry.yarnpkg.com/moment-duration-format/-/moment-duration-format-2.2.2.tgz#b957612de26016c9ad9eb6087c054573e5127779"
integrity sha1-uVdhLeJgFsmtnrYIfAVFc+USd3k=
moment-timezone@0.5.23, moment-timezone@^0.5.x:
version "0.5.23"
resolved "https://registry.yarnpkg.com/moment-timezone/-/moment-timezone-0.5.23.tgz#7cbb00db2c14c71b19303cb47b0fb0a6d8651463"