From aa27554bc765ea131cd268279fb142854b7372f0 Mon Sep 17 00:00:00 2001 From: Nick Date: Sun, 17 Feb 2019 01:32:35 -0500 Subject: [PATCH] feat: storage schedule + status --- BACKERS.md | 1 + client/client-app.js | 2 - client/components/admin/admin-storage.vue | 92 +++++++++++--- client/components/admin/admin-system.vue | 15 --- client/components/common/duration-picker.vue | 114 ++++++++++++++++++ .../storage/storage-mutation-save-targets.gql | 2 +- .../admin/storage/storage-query-status.gql | 10 ++ .../admin/storage/storage-query-targets.gql | 3 + client/themes/default/components/page.vue | 2 +- server/app/data.yml | 3 + server/core/scheduler.js | 14 ++- server/db/migrations/2.0.0-beta.38.js | 15 +++ server/graph/directives/rate-limit.js | 5 + server/graph/index.js | 7 +- server/graph/resolvers/storage.js | 22 +++- server/graph/schemas/storage.graphql | 15 ++- server/helpers/config.js | 6 + server/jobs/sync-storage.js | 28 ++--- server/models/storage.js | 26 +++- 19 files changed, 313 insertions(+), 69 deletions(-) create mode 100644 client/components/common/duration-picker.vue create mode 100644 client/graph/admin/storage/storage-query-status.gql create mode 100644 server/db/migrations/2.0.0-beta.38.js create mode 100644 server/graph/directives/rate-limit.js diff --git a/BACKERS.md b/BACKERS.md index ed104f01..b2ed7d8b 100644 --- a/BACKERS.md +++ b/BACKERS.md @@ -29,6 +29,7 @@ Funds donated via Patreon go directly to support lead developer [Nicolas Giard]( - Brandon Curtis - Loïc CRAMPON +- 张白驹 diff --git a/client/client-app.js b/client/client-app.js index f41981cb..d2b59038 100644 --- a/client/client-app.js +++ b/client/client-app.js @@ -3,7 +3,6 @@ import Vue from 'vue' import VueRouter from 'vue-router' import VueClipboards from 'vue-clipboards' -import VueSimpleBreakpoints from 'vue-simple-breakpoints' import VeeValidate from 'vee-validate' import { ApolloClient } from 'apollo-client' import { createPersistedQueryLink } from 'apollo-link-persisted-queries' @@ -144,7 +143,6 @@ Vue.config.productionTip = false Vue.use(VueRouter) Vue.use(VueApollo) Vue.use(VueClipboards) -Vue.use(VueSimpleBreakpoints) Vue.use(localization.VueI18Next) Vue.use(helpers) Vue.use(VeeValidate, { events: '' }) diff --git a/client/components/admin/admin-storage.vue b/client/components/admin/admin-storage.vue index 87de0154..67adc2db 100644 --- a/client/components/admin/admin-storage.vue +++ b/client/components/admin/admin-storage.vue @@ -15,7 +15,7 @@ span {{$t('common:actions.apply')}} v-card.mt-3 - v-tabs(color='grey darken-2', fixed-tabs, slider-color='white', show-arrows, dark) + v-tabs(color='grey darken-2', fixed-tabs, slider-color='white', show-arrows, dark, v-model='currentTab') v-tab(key='settings'): v-icon settings v-tab(v-for='tgt in activeTargets', :key='tgt.key') {{ tgt.title }} @@ -37,16 +37,29 @@ ) v-flex(xs12, md6) .pa-3.grey.radius-7(:class='$vuetify.dark ? "darken-4" : "lighten-5"') - .body-2.grey--text.text--darken-1 Advanced Settings - v-text-field.mt-3.md2( - v-model='syncInterval' - outline - background-color='grey lighten-2' - prepend-icon='schedule' - label='Synchronization Interval' - hint='For performance reasons, some storage targets synchronize changes on an interval-based schedule, instead of on every change. Define at which interval should the synchronization occur for all storage targets.' - persistent-hint - ) + v-layout.pa-2(row, justify-space-between) + .body-2.grey--text.text--darken-1 Status + looping-rhombuses-spinner.mt-1( + :animation-duration='5000' + :rhombus-size='10' + color='#BBB' + ) + v-divider + v-toolbar.mt-2.radius-7( + v-for='(tgt, n) in status' + :key='tgt.key' + dense + :color='getStatusColor(tgt.status)' + dark + flat + :extended='tgt.status === `error`', + extension-height='100' + ) + .pa-3.red.darken-2.radius-7(v-if='tgt.status === `error`', slot='extension') {{tgt.message}} + .body-2 {{tgt.title}} + v-spacer + .body-1 {{tgt.status}} + v-alert.mt-3.radius-7(v-if='status.length < 1', outline, :value='true', color='indigo') You don't have any active storage target. v-tab-item(v-for='(tgt, n) in activeTargets', :key='tgt.key', :transition='false', :reverse-transition='false') v-card.pa-3(flat, tile) @@ -125,22 +138,40 @@ .pb-3 Content is always pushed to the storage target, overwriting any existing content. This is safest choice for backup scenarios. strong Pull from target #[em.red--text.text--lighten-2(v-if='tgt.supportedModes.indexOf(`pull`) < 0') Unsupported] .pb-3 Content is always pulled from the storage target, overwriting any local content which already exists. This choice is usually reserved for single-use content import. Caution with this option as any local content will always be overwritten! + + template(v-if='tgt.hasSchedule') + v-divider.mt-3 + v-subheader.pl-0 Sync Schedule + .body-1.ml-3 For performance reasons, this storage target synchronize changes on an interval-based schedule, instead of on every change. Define at which interval should the synchronization occur. + .pa-3 + duration-picker(v-model='tgt.syncInterval') + .caption.mt-3 The default is every #[strong 5 minutes]. + diff --git a/client/graph/admin/storage/storage-mutation-save-targets.gql b/client/graph/admin/storage/storage-mutation-save-targets.gql index b8f95250..153ec199 100644 --- a/client/graph/admin/storage/storage-mutation-save-targets.gql +++ b/client/graph/admin/storage/storage-mutation-save-targets.gql @@ -1,4 +1,4 @@ -mutation($targets: [StorageTargetInput]) { +mutation($targets: [StorageTargetInput]!) { storage { updateTargets(targets: $targets) { responseResult { diff --git a/client/graph/admin/storage/storage-query-status.gql b/client/graph/admin/storage/storage-query-status.gql new file mode 100644 index 00000000..a5a81497 --- /dev/null +++ b/client/graph/admin/storage/storage-query-status.gql @@ -0,0 +1,10 @@ +query { + storage { + status { + key + title + status + message + } + } +} diff --git a/client/graph/admin/storage/storage-query-targets.gql b/client/graph/admin/storage/storage-query-targets.gql index 219dcdcb..ff7ff81a 100644 --- a/client/graph/admin/storage/storage-query-targets.gql +++ b/client/graph/admin/storage/storage-query-targets.gql @@ -10,6 +10,9 @@ query { website supportedModes mode + hasSchedule + syncInterval + syncIntervalDefault config { key value diff --git a/client/themes/default/components/page.vue b/client/themes/default/components/page.vue index 364204eb..ab78fc25 100644 --- a/client/themes/default/components/page.vue +++ b/client/themes/default/components/page.vue @@ -28,7 +28,7 @@ ) v-icon menu - v-content + v-content(ref='content') template(v-if='path !== `home`') v-toolbar(:color='darkMode ? `grey darken-4-d3` : `grey lighten-3`', flat, dense) v-btn.pl-0(v-if='$vuetify.breakpoint.xsOnly', flat, @click='toggleNavigation') diff --git a/server/app/data.yml b/server/app/data.yml index cce70b99..757ff4b5 100644 --- a/server/app/data.yml +++ b/server/app/data.yml @@ -57,6 +57,9 @@ jobs: syncGraphUpdates: onInit: true schedule: P1D + syncStorage: + onInit: true + schedule: storage.syncInterval groups: defaultPermissions: - 'manage:pages' diff --git a/server/core/scheduler.js b/server/core/scheduler.js index 0a2d01cc..bf11a302 100644 --- a/server/core/scheduler.js +++ b/server/core/scheduler.js @@ -1,5 +1,6 @@ const Job = require('./job') const _ = require('lodash') +const configHelper = require('../helpers/config') /* global WIKI */ @@ -10,12 +11,13 @@ module.exports = { }, start() { _.forOwn(WIKI.data.jobs, (queueParams, queueName) => { - this.registerJob({ - name: _.kebabCase(queueName), - immediate: queueParams.onInit, - schedule: queueParams.schedule, - repeat: true - }) + const schedule = (configHelper.isValidDurationString(queueParams.schedule)) ? queueParams : _.get(WIKI.config, queueParams.schedule) + // this.registerJob({ + // name: _.kebabCase(queueName), + // immediate: queueParams.onInit, + // schedule: schedule, + // repeat: true + // }) }) }, registerJob(opts, data) { diff --git a/server/db/migrations/2.0.0-beta.38.js b/server/db/migrations/2.0.0-beta.38.js new file mode 100644 index 00000000..1c7f5343 --- /dev/null +++ b/server/db/migrations/2.0.0-beta.38.js @@ -0,0 +1,15 @@ +exports.up = knex => { + return knex.schema + .table('storage', table => { + table.string('syncInterval') + table.json('state') + }) +} + +exports.down = knex => { + return knex.schema + .table('storage', table => { + table.dropColumn('syncInterval') + table.dropColumn('state') + }) +} diff --git a/server/graph/directives/rate-limit.js b/server/graph/directives/rate-limit.js new file mode 100644 index 00000000..325c8f4d --- /dev/null +++ b/server/graph/directives/rate-limit.js @@ -0,0 +1,5 @@ +const { createRateLimitDirective } = require('graphql-rate-limit-directive') + +module.exports = createRateLimitDirective({ + keyGenerator: (directiveArgs, source, args, context, info) => `${context.req.ip}:${info.parentType}.${info.fieldName}` +}) diff --git a/server/graph/index.js b/server/graph/index.js index c8856b57..005719e8 100644 --- a/server/graph/index.js +++ b/server/graph/index.js @@ -6,7 +6,7 @@ const autoload = require('auto-load') const PubSub = require('graphql-subscriptions').PubSub const { LEVEL, MESSAGE } = require('triple-beam') const Transport = require('winston-transport') -const { createRateLimitTypeDef, createRateLimitDirective } = require('graphql-rate-limit-directive') +const { createRateLimitTypeDef } = require('graphql-rate-limit-directive') /* global WIKI */ @@ -35,10 +35,7 @@ resolversObj.forEach(resolver => { // Directives let schemaDirectives = { - ...autoload(path.join(WIKI.SERVERPATH, 'graph/directives')), - rateLimit: createRateLimitDirective({ - keyGenerator: (directiveArgs, source, args, context, info) => `${context.req.ip}:${info.parentType}.${info.fieldName}` - }) + ...autoload(path.join(WIKI.SERVERPATH, 'graph/directives')) } // Live Trail Logger (admin) diff --git a/server/graph/resolvers/storage.js b/server/graph/resolvers/storage.js index 4b5beba3..4899af5a 100644 --- a/server/graph/resolvers/storage.js +++ b/server/graph/resolvers/storage.js @@ -18,6 +18,9 @@ module.exports = { return { ...targetInfo, ...tgt, + hasSchedule: (targetInfo.schedule !== false), + syncInterval: targetInfo.syncInterval || targetInfo.schedule || 'P0D', + syncIntervalDefault: targetInfo.schedule, config: _.sortBy(_.transform(tgt.config, (res, value, key) => { const configData = _.get(targetInfo.props, key, {}) res.push({ @@ -33,6 +36,18 @@ module.exports = { if (args.filter) { targets = graphHelper.filter(targets, args.filter) } if (args.orderBy) { targets = graphHelper.orderBy(targets, args.orderBy) } return targets + }, + async status(obj, args, context, info) { + let activeTargets = await WIKI.models.storage.query().where('isEnabled', true) + return activeTargets.map(tgt => { + const targetInfo = _.find(WIKI.data.storage, ['key', tgt.key]) || {} + return { + key: tgt.key, + title: targetInfo.title, + status: _.get(tgt, 'state.status', 'pending'), + message: _.get(tgt, 'state.message', 'Initializing...') + } + }) } }, StorageMutation: { @@ -42,10 +57,15 @@ module.exports = { await WIKI.models.storage.query().patch({ isEnabled: tgt.isEnabled, mode: tgt.mode, + syncInterval: tgt.syncInterval, config: _.reduce(tgt.config, (result, value, key) => { _.set(result, `${value.key}`, _.get(JSON.parse(value.value), 'v', null)) return result - }, {}) + }, {}), + state: { + status: 'pending', + message: 'Initializing...' + } }).where('key', tgt.key) } await WIKI.models.storage.initTargets() diff --git a/server/graph/schemas/storage.graphql b/server/graph/schemas/storage.graphql index 4a333aff..783929f1 100644 --- a/server/graph/schemas/storage.graphql +++ b/server/graph/schemas/storage.graphql @@ -19,6 +19,8 @@ type StorageQuery { filter: String orderBy: String ): [StorageTarget] @auth(requires: ["manage:system"]) + + status: [StorageStatus] @auth(requires: ["manage:system"]) } # ----------------------------------------------- @@ -27,7 +29,7 @@ type StorageQuery { type StorageMutation { updateTargets( - targets: [StorageTargetInput] + targets: [StorageTargetInput]! ): DefaultResponse @auth(requires: ["manage:system"]) } @@ -45,6 +47,9 @@ type StorageTarget { website: String supportedModes: [String] mode: String + hasSchedule: Boolean! + syncInterval: String + syncIntervalDefault: String config: [KeyValuePair] } @@ -52,5 +57,13 @@ input StorageTargetInput { isEnabled: Boolean! key: String! mode: String! + syncInterval: String config: [KeyValuePairInput] } + +type StorageStatus { + key: String! + title: String! + status: String! + message: String +} diff --git a/server/helpers/config.js b/server/helpers/config.js index e5dde5bf..0e6ab563 100644 --- a/server/helpers/config.js +++ b/server/helpers/config.js @@ -2,6 +2,8 @@ const _ = require('lodash') +const isoDurationReg = /^(-|\+)?P(?:([-+]?[0-9,.]*)Y)?(?:([-+]?[0-9,.]*)M)?(?:([-+]?[0-9,.]*)W)?(?:([-+]?[0-9,.]*)D)?(?:T(?:([-+]?[0-9,.]*)H)?(?:([-+]?[0-9,.]*)M)?(?:([-+]?[0-9,.]*)S)?)?$/ + module.exports = { /** * Parse configuration value for environment vars @@ -15,5 +17,9 @@ module.exports = { /\$\(([A-Z0-9_]+)\)/g, (fm, m) => { return process.env[m] } ) + }, + + isValidDurationString (val) { + return isoDurationReg.test(val) } } diff --git a/server/jobs/sync-storage.js b/server/jobs/sync-storage.js index cce94681..a1cb7b17 100644 --- a/server/jobs/sync-storage.js +++ b/server/jobs/sync-storage.js @@ -1,20 +1,18 @@ -require('../core/worker') - /* global WIKI */ -module.exports = async (job) => { +module.exports = async ({ target }) => { WIKI.logger.info(`Syncing with storage provider ${job.data.target.title}...`) - try { - const target = require(`../modules/storage/${job.data.target.key}/storage.js`) - target[job.data.event].call({ - config: job.data.target.config, - mode: job.data.target.mode, - page: job.data.page - }) - WIKI.logger.info(`Syncing with storage provider ${job.data.target.title}: [ COMPLETED ]`) - } catch (err) { - WIKI.logger.error(`Syncing with storage provider ${job.data.target.title}: [ FAILED ]`) - WIKI.logger.error(err.message) - } + // try { + // const target = require(`../modules/storage/${job.data.target.key}/storage.js`) + // target[job.data.event].call({ + // config: job.data.target.config, + // mode: job.data.target.mode, + // page: job.data.page + // }) + // WIKI.logger.info(`Syncing with storage provider ${job.data.target.title}: [ COMPLETED ]`) + // } catch (err) { + // WIKI.logger.error(`Syncing with storage provider ${job.data.target.title}: [ FAILED ]`) + // WIKI.logger.error(err.message) + // } } diff --git a/server/models/storage.js b/server/models/storage.js index 2113ce21..ddcf6927 100644 --- a/server/models/storage.js +++ b/server/models/storage.js @@ -63,10 +63,15 @@ module.exports = class Storage extends Model { key: target.key, isEnabled: false, mode: target.defaultMode || 'push', + syncInterval: target.schedule || 'P0D', config: _.transform(target.props, (result, value, key) => { _.set(result, key, value.default) return result - }, {}) + }, {}), + state: { + status: 'pending', + message: '' + } }) } else { const targetConfig = _.get(_.find(dbTargets, ['key', target.key]), 'config', {}) @@ -100,13 +105,28 @@ module.exports = class Storage extends Model { } static async initTargets() { - targets = await WIKI.models.storage.query().where('isEnabled', true) + targets = await WIKI.models.storage.query().where('isEnabled', true).orderBy('key') try { for(let target of targets) { target.fn = require(`../modules/storage/${target.key}/storage`) target.fn.config = target.config target.fn.mode = target.mode - await target.fn.init() + try { + await target.fn.init() + await WIKI.models.storage.query().patch({ + state: { + status: 'operational', + message: '' + } + }).where('key', target.key) + } catch (err) { + await WIKI.models.storage.query().patch({ + state: { + status: 'error', + message: err.message + } + }).where('key', target.key) + } // if (target.schedule) { // WIKI.scheduler.registerJob({ // name: