liz/wikijs-fork
liz/wikijs-fork
1
0
Fork 0
Code Issues Pull Requests Packages Projects Releases Wiki Activity

feat: storage schedule + status

Browse Source
This commit is contained in:
Nick 2019-02-17 01:32:35 -05:00
parent 040f840807
commit aa27554bc7
19 changed files with 313 additions and 69 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

1
BACKERS.md
View File

@ -29,6 +29,7 @@ Funds donated via Patreon go directly to support lead developer [Nicolas Giard](
<!--5 start-->
- Brandon Curtis
- Loïc CRAMPON
- 张白驹
<!--5 end-->
<a href="https://www.patreon.com/requarks">

2
client/client-app.js
View File

@ -3,7 +3,6 @@
import Vue from 'vue'
import VueRouter from 'vue-router'
import VueClipboards from 'vue-clipboards'
import VueSimpleBreakpoints from 'vue-simple-breakpoints'
import VeeValidate from 'vee-validate'
import { ApolloClient } from 'apollo-client'
import { createPersistedQueryLink } from 'apollo-link-persisted-queries'
@ -144,7 +143,6 @@ Vue.config.productionTip = false
Vue.use(VueRouter)
Vue.use(VueApollo)
Vue.use(VueClipboards)
Vue.use(VueSimpleBreakpoints)
Vue.use(localization.VueI18Next)
Vue.use(helpers)
Vue.use(VeeValidate, { events: '' })

92
client/components/admin/admin-storage.vue
View File

@ -15,7 +15,7 @@
span {{$t('common:actions.apply')}}
v-card.mt-3
v-tabs(color='grey darken-2', fixed-tabs, slider-color='white', show-arrows, dark)
v-tabs(color='grey darken-2', fixed-tabs, slider-color='white', show-arrows, dark, v-model='currentTab')
v-tab(key='settings'): v-icon settings
v-tab(v-for='tgt in activeTargets', :key='tgt.key') {{ tgt.title }}
@ -37,16 +37,29 @@
)
v-flex(xs12, md6)
.pa-3.grey.radius-7(:class='$vuetify.dark ? "darken-4" : "lighten-5"')
.body-2.grey--text.text--darken-1 Advanced Settings
v-text-field.mt-3.md2(
v-model='syncInterval'
outline
background-color='grey lighten-2'
prepend-icon='schedule'
label='Synchronization Interval'
hint='For performance reasons, some storage targets synchronize changes on an interval-based schedule, instead of on every change. Define at which interval should the synchronization occur for all storage targets.'
persistent-hint
)
v-layout.pa-2(row, justify-space-between)
.body-2.grey--text.text--darken-1 Status
looping-rhombuses-spinner.mt-1(
:animation-duration='5000'
:rhombus-size='10'
color='#BBB'
)
v-divider
v-toolbar.mt-2.radius-7(
v-for='(tgt, n) in status'
:key='tgt.key'
dense
:color='getStatusColor(tgt.status)'
dark
flat
:extended='tgt.status === `error`',
extension-height='100'
)
.pa-3.red.darken-2.radius-7(v-if='tgt.status === `error`', slot='extension') {{tgt.message}}
.body-2 {{tgt.title}}
v-spacer
.body-1 {{tgt.status}}
v-alert.mt-3.radius-7(v-if='status.length < 1', outline, :value='true', color='indigo') You don't have any active storage target.
v-tab-item(v-for='(tgt, n) in activeTargets', :key='tgt.key', :transition='false', :reverse-transition='false')
v-card.pa-3(flat, tile)
@ -125,22 +138,40 @@
.pb-3 Content is always pushed to the storage target, overwriting any existing content. This is safest choice for backup scenarios.
strong Pull from target #[em.red--text.text--lighten-2(v-if='tgt.supportedModes.indexOf(`pull`) < 0') Unsupported]
.pb-3 Content is always pulled from the storage target, overwriting any local content which already exists. This choice is usually reserved for single-use content import. Caution with this option as any local content will always be overwritten!
template(v-if='tgt.hasSchedule')
v-divider.mt-3
v-subheader.pl-0 Sync Schedule
.body-1.ml-3 For performance reasons, this storage target synchronize changes on an interval-based schedule, instead of on every change. Define at which interval should the synchronization occur.
.pa-3
duration-picker(v-model='tgt.syncInterval')
.caption.mt-3 The default is every #[strong 5 minutes].
</template>
<script>
import _ from 'lodash'
import DurationPicker from '../common/duration-picker.vue'
import { LoopingRhombusesSpinner } from 'epic-spinners'
import statusQuery from 'gql/admin/storage/storage-query-status.gql'
import targetsQuery from 'gql/admin/storage/storage-query-targets.gql'
import targetsSaveMutation from 'gql/admin/storage/storage-mutation-save-targets.gql'
export default {
components: {
DurationPicker,
LoopingRhombusesSpinner
},
filters: {
startCase(val) { return _.startCase(val) }
},
data() {
return {
syncInterval: '5m',
targets: []
currentTab: 0,
targets: [],
status: []
}
},
computed: {
@ -163,19 +194,33 @@ export default {
mutation: targetsSaveMutation,
variables: {
targets: this.targets.map(tgt => _.pick(tgt, [
'isEnabled',
'key',
'config',
'mode'
])).map(str => ({...str, config: str.config.map(cfg => ({...cfg, value: JSON.stringify({ v: cfg.value.value })}))}))
'isEnabled',
'key',
'config',
'mode',
'syncInterval'
])).map(str => ({...str, config: str.config.map(cfg => ({...cfg, value: JSON.stringify({ v: cfg.value.value })}))}))
}
})
this.currentTab = 0
this.$store.commit('showNotification', {
message: 'Storage configuration saved successfully.',
style: 'success',
icon: 'check'
})
this.$store.commit(`loadingStop`, 'admin-storage-savetargets')
},
getStatusColor(state) {
switch (state) {
case 'pending':
return 'purple lighten-2'
case 'operational':
return 'green'
case 'error':
return 'red'
default:
return 'grey darken-2'
}
}
},
apollo: {
@ -190,8 +235,17 @@ export default {
})), [t => t.value.order])
})),
watchLoading (isLoading) {
this.$store.commit(`loading${isLoading ? 'Start' : 'Stop'}`, 'admin-storage-refresh')
this.$store.commit(`loading${isLoading ? 'Start' : 'Stop'}`, 'admin-storage-targets-refresh')
}
},
status: {
query: statusQuery,
fetchPolicy: 'network-only',
update: (data) => data.storage.status,
watchLoading (isLoading) {
this.$store.commit(`loading${isLoading ? 'Start' : 'Stop'}`, 'admin-storage-status-refresh')
},
pollInterval: 3000
}
}
}

15
client/components/admin/admin-system.vue
View File

@ -82,19 +82,6 @@
v-divider.mt-3
v-subheader Redis
v-list-tile(avatar)
v-list-tile-avatar
v-avatar.red(size='40')
icon-cube(fillColor='#FFFFFF')
v-list-tile-content
v-list-tile-title {{ info.redisVersion }}
v-list-tile-sub-title {{ info.redisHost }}
v-list-tile-action
v-list-tile-action-text {{ $t('admin:system.ramUsage', { used: info.redisUsedRAM, total: info.redisTotalRAM }) }}
v-divider.mt-3
v-subheader {{ info.dbType }}
v-list-tile(avatar)
v-list-tile-avatar
@ -108,7 +95,6 @@
<script>
import _ from 'lodash'
import IconCube from 'mdi/Cube'
import IconDatabase from 'mdi/Database'
import IconNodeJs from 'mdi/Nodejs'
@ -116,7 +102,6 @@ import systemInfoQuery from 'gql/admin/system/system-query-info.gql'
export default {
components: {
IconCube,
IconDatabase,
IconNodeJs
},

114
client/components/common/duration-picker.vue Normal file
View File

@ -0,0 +1,114 @@
<template lang='pug'>
v-toolbar(flat, :color='$vuetify.dark ? "grey darken-4-l3" : "grey lighten-3"')
.body-2.mr-3 Every
v-text-field(
solo
hide-details
flat
reverse
v-model='minutes'
)
.body-2.mx-3 Minute(s)
v-divider.mr-3()
v-text-field(
solo
hide-details
flat
reverse
v-model='hours'
)
.body-2.mx-3 Hour(s)
v-divider.mr-3()
v-text-field(
solo
hide-details
flat
reverse
v-model='days'
)
.body-2.mx-3 Day(s)
v-divider.mr-3()
v-text-field(
solo
hide-details
flat
reverse
v-model='months'
)
.body-2.mx-3 Month(s)
v-divider.mr-3()
v-text-field(
solo
hide-details
flat
reverse
v-model='years'
)
.body-2.mx-3 Year(s)
</template>
<script>
import _ from 'lodash'
import moment from 'moment'
export default {
props: {
value: {
type: String,
default: 'PT5M'
}
},
data() {
return {
duration: moment.duration(0)
}
},
computed: {
years: {
get() { return this.duration.years() || 0 },
set(val) { this.rebuild(_.toNumber(val), 'years') }
},
months: {
get() { return this.duration.months() || 0 },
set(val) { this.rebuild(_.toNumber(val), 'months') }
},
days: {
get() { return this.duration.days() || 0 },
set(val) { this.rebuild(_.toNumber(val), 'days') }
},
hours: {
get() { return this.duration.hours() || 0 },
set(val) { this.rebuild(_.toNumber(val), 'hours') }
},
minutes: {
get() { return this.duration.minutes() || 0 },
set(val) { this.rebuild(_.toNumber(val), 'minutes') }
}
},
watch: {
value(newValue, oldValue) {
this.duration = moment.duration(newValue)
}
},
methods: {
rebuild(val, unit) {
if (!_.isFinite(val) || val < 0) {
val = 0
}
const newDuration = {
minutes: this.duration.minutes(),
hours: this.duration.hours(),
days: this.duration.days(),
months: this.duration.months(),
years: this.duration.years()
}
_.set(newDuration, unit, val)
this.duration = moment.duration(newDuration)
this.$emit('input', this.duration.toISOString())
}
},
mounted() {
this.duration = moment.duration(this.value)
}
}
</script>

2
client/graph/admin/storage/storage-mutation-save-targets.gql
View File

@ -1,4 +1,4 @@
mutation($targets: [StorageTargetInput]) {
mutation($targets: [StorageTargetInput]!) {
storage {
updateTargets(targets: $targets) {
responseResult {

10
client/graph/admin/storage/storage-query-status.gql Normal file
View File

@ -0,0 +1,10 @@
query {
storage {
status {
key
title
status
message
}
}
}

3
client/graph/admin/storage/storage-query-targets.gql
View File

@ -10,6 +10,9 @@ query {
website
supportedModes
mode
hasSchedule
syncInterval
syncIntervalDefault
config {
key
value

2
client/themes/default/components/page.vue
View File

@ -28,7 +28,7 @@
)
v-icon menu
v-content
v-content(ref='content')
template(v-if='path !== `home`')
v-toolbar(:color='darkMode ? `grey darken-4-d3` : `grey lighten-3`', flat, dense)
v-btn.pl-0(v-if='$vuetify.breakpoint.xsOnly', flat, @click='toggleNavigation')

3
server/app/data.yml
View File

@ -57,6 +57,9 @@ jobs:
syncGraphUpdates:
onInit: true
schedule: P1D
syncStorage:
onInit: true
schedule: storage.syncInterval
groups:
defaultPermissions:
- 'manage:pages'

14
server/core/scheduler.js
View File

@ -1,5 +1,6 @@
const Job = require('./job')
const _ = require('lodash')
const configHelper = require('../helpers/config')
/* global WIKI */
@ -10,12 +11,13 @@ module.exports = {
},
start() {
_.forOwn(WIKI.data.jobs, (queueParams, queueName) => {
this.registerJob({
name: _.kebabCase(queueName),
immediate: queueParams.onInit,
schedule: queueParams.schedule,
repeat: true
})
const schedule = (configHelper.isValidDurationString(queueParams.schedule)) ? queueParams : _.get(WIKI.config, queueParams.schedule)
// this.registerJob({
// name: _.kebabCase(queueName),
// immediate: queueParams.onInit,
// schedule: schedule,
// repeat: true
// })
})
},
registerJob(opts, data) {

15
server/db/migrations/2.0.0-beta.38.js Normal file
View File

@ -0,0 +1,15 @@
exports.up = knex => {
return knex.schema
.table('storage', table => {
table.string('syncInterval')
table.json('state')
})
}
exports.down = knex => {
return knex.schema
.table('storage', table => {
table.dropColumn('syncInterval')
table.dropColumn('state')
})
}

5
server/graph/directives/rate-limit.js Normal file
View File

@ -0,0 +1,5 @@
const { createRateLimitDirective } = require('graphql-rate-limit-directive')
module.exports = createRateLimitDirective({
keyGenerator: (directiveArgs, source, args, context, info) => `${context.req.ip}:${info.parentType}.${info.fieldName}`
})

7
server/graph/index.js
View File

@ -6,7 +6,7 @@ const autoload = require('auto-load')
const PubSub = require('graphql-subscriptions').PubSub
const { LEVEL, MESSAGE } = require('triple-beam')
const Transport = require('winston-transport')
const { createRateLimitTypeDef, createRateLimitDirective } = require('graphql-rate-limit-directive')
const { createRateLimitTypeDef } = require('graphql-rate-limit-directive')
/* global WIKI */
@ -35,10 +35,7 @@ resolversObj.forEach(resolver => {
// Directives
let schemaDirectives = {
...autoload(path.join(WIKI.SERVERPATH, 'graph/directives')),
rateLimit: createRateLimitDirective({
keyGenerator: (directiveArgs, source, args, context, info) => `${context.req.ip}:${info.parentType}.${info.fieldName}`
})
...autoload(path.join(WIKI.SERVERPATH, 'graph/directives'))
}
// Live Trail Logger (admin)

22
server/graph/resolvers/storage.js
View File

@ -18,6 +18,9 @@ module.exports = {
return {
...targetInfo,
...tgt,
hasSchedule: (targetInfo.schedule !== false),
syncInterval: targetInfo.syncInterval || targetInfo.schedule || 'P0D',
syncIntervalDefault: targetInfo.schedule,
config: _.sortBy(_.transform(tgt.config, (res, value, key) => {
const configData = _.get(targetInfo.props, key, {})
res.push({
@ -33,6 +36,18 @@ module.exports = {
if (args.filter) { targets = graphHelper.filter(targets, args.filter) }
if (args.orderBy) { targets = graphHelper.orderBy(targets, args.orderBy) }
return targets
},
async status(obj, args, context, info) {
let activeTargets = await WIKI.models.storage.query().where('isEnabled', true)
return activeTargets.map(tgt => {
const targetInfo = _.find(WIKI.data.storage, ['key', tgt.key]) || {}
return {
key: tgt.key,
title: targetInfo.title,
status: _.get(tgt, 'state.status', 'pending'),
message: _.get(tgt, 'state.message', 'Initializing...')
}
})
}
},
StorageMutation: {
@ -42,10 +57,15 @@ module.exports = {
await WIKI.models.storage.query().patch({
isEnabled: tgt.isEnabled,
mode: tgt.mode,
syncInterval: tgt.syncInterval,
config: _.reduce(tgt.config, (result, value, key) => {
_.set(result, `${value.key}`, _.get(JSON.parse(value.value), 'v', null))
return result
}, {})
}, {}),
state: {
status: 'pending',
message: 'Initializing...'
}
}).where('key', tgt.key)
}
await WIKI.models.storage.initTargets()

15
server/graph/schemas/storage.graphql
View File

@ -19,6 +19,8 @@ type StorageQuery {
filter: String
orderBy: String
): [StorageTarget] @auth(requires: ["manage:system"])
status: [StorageStatus] @auth(requires: ["manage:system"])
}
# -----------------------------------------------
@ -27,7 +29,7 @@ type StorageQuery {
type StorageMutation {
updateTargets(
targets: [StorageTargetInput]
targets: [StorageTargetInput]!
): DefaultResponse @auth(requires: ["manage:system"])
}
@ -45,6 +47,9 @@ type StorageTarget {
website: String
supportedModes: [String]
mode: String
hasSchedule: Boolean!
syncInterval: String
syncIntervalDefault: String
config: [KeyValuePair]
}
@ -52,5 +57,13 @@ input StorageTargetInput {
isEnabled: Boolean!
key: String!
mode: String!
syncInterval: String
config: [KeyValuePairInput]
}
type StorageStatus {
key: String!
title: String!
status: String!
message: String
}

6
server/helpers/config.js
View File

@ -2,6 +2,8 @@
const _ = require('lodash')
const isoDurationReg = /^(-|\+)?P(?:([-+]?[0-9,.]*)Y)?(?:([-+]?[0-9,.]*)M)?(?:([-+]?[0-9,.]*)W)?(?:([-+]?[0-9,.]*)D)?(?:T(?:([-+]?[0-9,.]*)H)?(?:([-+]?[0-9,.]*)M)?(?:([-+]?[0-9,.]*)S)?)?$/
module.exports = {
/**
* Parse configuration value for environment vars
@ -15,5 +17,9 @@ module.exports = {
/\$\(([A-Z0-9_]+)\)/g,
(fm, m) => { return process.env[m] }
)
},
isValidDurationString (val) {
return isoDurationReg.test(val)
}
}

28
server/jobs/sync-storage.js
View File

@ -1,20 +1,18 @@
require('../core/worker')
/* global WIKI */
module.exports = async (job) => {
module.exports = async ({ target }) => {
WIKI.logger.info(`Syncing with storage provider ${job.data.target.title}...`)
try {
const target = require(`../modules/storage/${job.data.target.key}/storage.js`)
target[job.data.event].call({
config: job.data.target.config,
mode: job.data.target.mode,
page: job.data.page
})
WIKI.logger.info(`Syncing with storage provider ${job.data.target.title}: [ COMPLETED ]`)
} catch (err) {
WIKI.logger.error(`Syncing with storage provider ${job.data.target.title}: [ FAILED ]`)
WIKI.logger.error(err.message)
}
// try {
// const target = require(`../modules/storage/${job.data.target.key}/storage.js`)
// target[job.data.event].call({
// config: job.data.target.config,
// mode: job.data.target.mode,
// page: job.data.page
// })
// WIKI.logger.info(`Syncing with storage provider ${job.data.target.title}: [ COMPLETED ]`)
// } catch (err) {
// WIKI.logger.error(`Syncing with storage provider ${job.data.target.title}: [ FAILED ]`)
// WIKI.logger.error(err.message)
// }
}

26
server/models/storage.js
View File

@ -63,10 +63,15 @@ module.exports = class Storage extends Model {
key: target.key,
isEnabled: false,
mode: target.defaultMode || 'push',
syncInterval: target.schedule || 'P0D',
config: _.transform(target.props, (result, value, key) => {
_.set(result, key, value.default)
return result
}, {})
}, {}),
state: {
status: 'pending',
message: ''
}
})
} else {
const targetConfig = _.get(_.find(dbTargets, ['key', target.key]), 'config', {})
@ -100,13 +105,28 @@ module.exports = class Storage extends Model {
}
static async initTargets() {
targets = await WIKI.models.storage.query().where('isEnabled', true)
targets = await WIKI.models.storage.query().where('isEnabled', true).orderBy('key')
try {
for(let target of targets) {
target.fn = require(`../modules/storage/${target.key}/storage`)
target.fn.config = target.config
target.fn.mode = target.mode
await target.fn.init()
try {
await target.fn.init()
await WIKI.models.storage.query().patch({
state: {
status: 'operational',
message: ''
}
}).where('key', target.key)
} catch (err) {
await WIKI.models.storage.query().patch({
state: {
status: 'error',
message: err.message
}
}).where('key', target.key)
}
// if (target.schedule) {
// WIKI.scheduler.registerJob({
// name: