feat: storage schedule + status

BACKERS.md

@@ -29,6 +29,7 @@ Funds donated via Patreon go directly to support lead developer [Nicolas Giard](
 <!--5 start-->
 - Brandon Curtis
 - Loïc CRAMPON
+- 张白驹
 <!--5 end-->
 
 <a href="https://www.patreon.com/requarks">

client/client-app.js

@@ -3,7 +3,6 @@
 import Vue from 'vue'
 import VueRouter from 'vue-router'
 import VueClipboards from 'vue-clipboards'
-import VueSimpleBreakpoints from 'vue-simple-breakpoints'
 import VeeValidate from 'vee-validate'
 import { ApolloClient } from 'apollo-client'
 import { createPersistedQueryLink } from 'apollo-link-persisted-queries'
@@ -144,7 +143,6 @@ Vue.config.productionTip = false
 Vue.use(VueRouter)
 Vue.use(VueApollo)
 Vue.use(VueClipboards)
-Vue.use(VueSimpleBreakpoints)
 Vue.use(localization.VueI18Next)
 Vue.use(helpers)
 Vue.use(VeeValidate, { events: '' })

client/components/admin/admin-storage.vue

@@ -15,7 +15,7 @@
             span {{$t('common:actions.apply')}}
 
         v-card.mt-3
-          v-tabs(color='grey darken-2', fixed-tabs, slider-color='white', show-arrows, dark)
+          v-tabs(color='grey darken-2', fixed-tabs, slider-color='white', show-arrows, dark, v-model='currentTab')
             v-tab(key='settings'): v-icon settings
             v-tab(v-for='tgt in activeTargets', :key='tgt.key') {{ tgt.title }}
 
@@ -37,16 +37,29 @@
                       )
                   v-flex(xs12, md6)
                     .pa-3.grey.radius-7(:class='$vuetify.dark ? "darken-4" : "lighten-5"')
-                      .body-2.grey--text.text--darken-1 Advanced Settings
+                      v-layout.pa-2(row, justify-space-between)
-                      v-text-field.mt-3.md2(
+                        .body-2.grey--text.text--darken-1 Status
-                        v-model='syncInterval'
+                        looping-rhombuses-spinner.mt-1(
-                        outline
+                          :animation-duration='5000'
-                        background-color='grey lighten-2'
+                          :rhombus-size='10'
-                        prepend-icon='schedule'
+                          color='#BBB'
-                        label='Synchronization Interval'
+                        )
-                        hint='For performance reasons, some storage targets synchronize changes on an interval-based schedule, instead of on every change. Define at which interval should the synchronization occur for all storage targets.'
+                      v-divider
-                        persistent-hint
+                      v-toolbar.mt-2.radius-7(
-                      )
+                        v-for='(tgt, n) in status'
+                        :key='tgt.key'
+                        dense
+                        :color='getStatusColor(tgt.status)'
+                        dark
+                        flat
+                        :extended='tgt.status === `error`',
+                        extension-height='100'
+                        )
+                        .pa-3.red.darken-2.radius-7(v-if='tgt.status === `error`', slot='extension') {{tgt.message}}
+                        .body-2 {{tgt.title}}
+                        v-spacer
+                        .body-1 {{tgt.status}}
+                      v-alert.mt-3.radius-7(v-if='status.length < 1', outline, :value='true', color='indigo') You don't have any active storage target.
 
             v-tab-item(v-for='(tgt, n) in activeTargets', :key='tgt.key', :transition='false', :reverse-transition='false')
               v-card.pa-3(flat, tile)
@@ -125,22 +138,40 @@
                     .pb-3 Content is always pushed to the storage target, overwriting any existing content. This is safest choice for backup scenarios.
                     strong Pull from target #[em.red--text.text--lighten-2(v-if='tgt.supportedModes.indexOf(`pull`) < 0') Unsupported]
                     .pb-3 Content is always pulled from the storage target, overwriting any local content which already exists. This choice is usually reserved for single-use content import. Caution with this option as any local content will always be overwritten!
+
+                  template(v-if='tgt.hasSchedule')
+                    v-divider.mt-3
+                    v-subheader.pl-0 Sync Schedule
+                    .body-1.ml-3 For performance reasons, this storage target synchronize changes on an interval-based schedule, instead of on every change. Define at which interval should the synchronization occur.
+                    .pa-3
+                      duration-picker(v-model='tgt.syncInterval')
+                      .caption.mt-3 The default is every #[strong 5 minutes].
+
 </template>
 
 <script>
 import _ from 'lodash'
 
+import DurationPicker from '../common/duration-picker.vue'
+import { LoopingRhombusesSpinner } from 'epic-spinners'
+
+import statusQuery from 'gql/admin/storage/storage-query-status.gql'
 import targetsQuery from 'gql/admin/storage/storage-query-targets.gql'
 import targetsSaveMutation from 'gql/admin/storage/storage-mutation-save-targets.gql'
 
 export default {
+  components: {
+    DurationPicker,
+    LoopingRhombusesSpinner
+  },
   filters: {
     startCase(val) { return _.startCase(val) }
   },
   data() {
     return {
-      syncInterval: '5m',
+      currentTab: 0,
-      targets: []
+      targets: [],
+      status: []
     }
   },
   computed: {
@@ -163,19 +194,33 @@ export default {
         mutation: targetsSaveMutation,
         variables: {
           targets: this.targets.map(tgt => _.pick(tgt, [
-            'isEnabled',
+              'isEnabled',
-            'key',
+              'key',
-            'config',
+              'config',
-            'mode'
+              'mode',
-          ])).map(str => ({...str, config: str.config.map(cfg => ({...cfg, value: JSON.stringify({ v: cfg.value.value })}))}))
+              'syncInterval'
+            ])).map(str => ({...str, config: str.config.map(cfg => ({...cfg, value: JSON.stringify({ v: cfg.value.value })}))}))
         }
       })
+      this.currentTab = 0
       this.$store.commit('showNotification', {
         message: 'Storage configuration saved successfully.',
         style: 'success',
         icon: 'check'
       })
       this.$store.commit(`loadingStop`, 'admin-storage-savetargets')
+    },
+    getStatusColor(state) {
+      switch (state) {
+        case 'pending':
+          return 'purple lighten-2'
+        case 'operational':
+          return 'green'
+        case 'error':
+          return 'red'
+        default:
+          return 'grey darken-2'
+      }
     }
   },
   apollo: {
@@ -190,8 +235,17 @@ export default {
         })), [t => t.value.order])
       })),
       watchLoading (isLoading) {
-        this.$store.commit(`loading${isLoading ? 'Start' : 'Stop'}`, 'admin-storage-refresh')
+        this.$store.commit(`loading${isLoading ? 'Start' : 'Stop'}`, 'admin-storage-targets-refresh')
       }
+    },
+    status: {
+      query: statusQuery,
+      fetchPolicy: 'network-only',
+      update: (data) => data.storage.status,
+      watchLoading (isLoading) {
+        this.$store.commit(`loading${isLoading ? 'Start' : 'Stop'}`, 'admin-storage-status-refresh')
+      },
+      pollInterval: 3000
     }
   }
 }

client/components/admin/admin-system.vue

@@ -82,19 +82,6 @@
 
                 v-divider.mt-3
 
-                v-subheader Redis
-                v-list-tile(avatar)
-                  v-list-tile-avatar
-                    v-avatar.red(size='40')
-                      icon-cube(fillColor='#FFFFFF')
-                  v-list-tile-content
-                    v-list-tile-title {{ info.redisVersion }}
-                    v-list-tile-sub-title {{ info.redisHost }}
-                  v-list-tile-action
-                    v-list-tile-action-text {{ $t('admin:system.ramUsage', { used: info.redisUsedRAM, total: info.redisTotalRAM }) }}
-
-                v-divider.mt-3
-
                 v-subheader {{ info.dbType }}
                 v-list-tile(avatar)
                   v-list-tile-avatar
@@ -108,7 +95,6 @@
 <script>
 import _ from 'lodash'
 
-import IconCube from 'mdi/Cube'
 import IconDatabase from 'mdi/Database'
 import IconNodeJs from 'mdi/Nodejs'
 
@@ -116,7 +102,6 @@ import systemInfoQuery from 'gql/admin/system/system-query-info.gql'
 
 export default {
   components: {
-    IconCube,
     IconDatabase,
     IconNodeJs
   },

client/components/common/duration-picker.vue

@@ -0,0 +1,114 @@
+<template lang='pug'>
+  v-toolbar(flat, :color='$vuetify.dark ? "grey darken-4-l3" : "grey lighten-3"')
+    .body-2.mr-3 Every
+    v-text-field(
+      solo
+      hide-details
+      flat
+      reverse
+      v-model='minutes'
+    )
+    .body-2.mx-3 Minute(s)
+    v-divider.mr-3()
+    v-text-field(
+      solo
+      hide-details
+      flat
+      reverse
+      v-model='hours'
+    )
+    .body-2.mx-3 Hour(s)
+    v-divider.mr-3()
+    v-text-field(
+      solo
+      hide-details
+      flat
+      reverse
+      v-model='days'
+    )
+    .body-2.mx-3 Day(s)
+    v-divider.mr-3()
+    v-text-field(
+      solo
+      hide-details
+      flat
+      reverse
+      v-model='months'
+    )
+    .body-2.mx-3 Month(s)
+    v-divider.mr-3()
+    v-text-field(
+      solo
+      hide-details
+      flat
+      reverse
+      v-model='years'
+    )
+    .body-2.mx-3 Year(s)
+</template>
+
+<script>
+import _ from 'lodash'
+import moment from 'moment'
+
+export default {
+  props: {
+    value: {
+      type: String,
+      default: 'PT5M'
+    }
+  },
+  data() {
+    return {
+      duration: moment.duration(0)
+    }
+  },
+  computed: {
+    years: {
+      get() { return this.duration.years() || 0 },
+      set(val) { this.rebuild(_.toNumber(val), 'years') }
+    },
+    months: {
+      get() { return this.duration.months() || 0 },
+      set(val) { this.rebuild(_.toNumber(val), 'months') }
+    },
+    days: {
+      get() { return this.duration.days() || 0 },
+      set(val) { this.rebuild(_.toNumber(val), 'days') }
+    },
+    hours: {
+      get() { return this.duration.hours() || 0 },
+      set(val) { this.rebuild(_.toNumber(val), 'hours') }
+    },
+    minutes: {
+      get() { return this.duration.minutes() || 0 },
+      set(val) { this.rebuild(_.toNumber(val), 'minutes') }
+    }
+  },
+  watch: {
+    value(newValue, oldValue) {
+      this.duration = moment.duration(newValue)
+    }
+  },
+  methods: {
+    rebuild(val, unit) {
+      if (!_.isFinite(val) || val < 0) {
+        val = 0
+      }
+      const newDuration = {
+        minutes: this.duration.minutes(),
+        hours: this.duration.hours(),
+        days: this.duration.days(),
+        months: this.duration.months(),
+        years: this.duration.years()
+      }
+      _.set(newDuration, unit, val)
+      this.duration = moment.duration(newDuration)
+      this.$emit('input', this.duration.toISOString())
+    }
+  },
+  mounted() {
+    this.duration = moment.duration(this.value)
+  }
+}
+</script>

client/graph/admin/storage/storage-mutation-save-targets.gql

@@ -1,4 +1,4 @@
-mutation($targets: [StorageTargetInput]) {
+mutation($targets: [StorageTargetInput]!) {
   storage {
     updateTargets(targets: $targets) {
       responseResult {

client/graph/admin/storage/storage-query-status.gql

@@ -0,0 +1,10 @@
+query {
+  storage {
+    status {
+      key
+      title
+      status
+      message
+    }
+  }
+}

client/graph/admin/storage/storage-query-targets.gql

@@ -10,6 +10,9 @@ query {
       website
       supportedModes
       mode
+      hasSchedule
+      syncInterval
+      syncIntervalDefault
       config {
         key
         value

client/themes/default/components/page.vue

@@ -28,7 +28,7 @@
         )
         v-icon menu
 
-    v-content
+    v-content(ref='content')
       template(v-if='path !== `home`')
         v-toolbar(:color='darkMode ? `grey darken-4-d3` : `grey lighten-3`', flat, dense)
           v-btn.pl-0(v-if='$vuetify.breakpoint.xsOnly', flat, @click='toggleNavigation')

server/app/data.yml

@@ -57,6 +57,9 @@ jobs:
   syncGraphUpdates:
     onInit: true
     schedule: P1D
+  syncStorage:
+    onInit: true
+    schedule: storage.syncInterval
 groups:
   defaultPermissions:
     - 'manage:pages'

server/core/scheduler.js

@@ -1,5 +1,6 @@
 const Job = require('./job')
 const _ = require('lodash')
+const configHelper = require('../helpers/config')
 
 /* global WIKI */
 
@@ -10,12 +11,13 @@ module.exports = {
   },
   start() {
     _.forOwn(WIKI.data.jobs, (queueParams, queueName) => {
-      this.registerJob({
+      const schedule = (configHelper.isValidDurationString(queueParams.schedule)) ? queueParams : _.get(WIKI.config, queueParams.schedule)
-        name: _.kebabCase(queueName),
+      // this.registerJob({
-        immediate: queueParams.onInit,
+      //   name: _.kebabCase(queueName),
-        schedule: queueParams.schedule,
+      //   immediate: queueParams.onInit,
-        repeat: true
+      //   schedule: schedule,
-      })
+      //   repeat: true
+      // })
     })
   },
   registerJob(opts, data) {

server/db/migrations/2.0.0-beta.38.js

@@ -0,0 +1,15 @@
+exports.up = knex => {
+  return knex.schema
+    .table('storage', table => {
+      table.string('syncInterval')
+      table.json('state')
+    })
+}
+
+exports.down = knex => {
+  return knex.schema
+    .table('storage', table => {
+      table.dropColumn('syncInterval')
+      table.dropColumn('state')
+    })
+}

server/graph/directives/rate-limit.js

@@ -0,0 +1,5 @@
+const { createRateLimitDirective } = require('graphql-rate-limit-directive')
+
+module.exports = createRateLimitDirective({
+  keyGenerator: (directiveArgs, source, args, context, info) => `${context.req.ip}:${info.parentType}.${info.fieldName}`
+})

server/graph/index.js

@@ -6,7 +6,7 @@ const autoload = require('auto-load')
 const PubSub = require('graphql-subscriptions').PubSub
 const { LEVEL, MESSAGE } = require('triple-beam')
 const Transport = require('winston-transport')
-const { createRateLimitTypeDef, createRateLimitDirective } = require('graphql-rate-limit-directive')
+const { createRateLimitTypeDef } = require('graphql-rate-limit-directive')
 
 /* global WIKI */
 
@@ -35,10 +35,7 @@ resolversObj.forEach(resolver => {
 // Directives
 
 let schemaDirectives = {
-  ...autoload(path.join(WIKI.SERVERPATH, 'graph/directives')),
+  ...autoload(path.join(WIKI.SERVERPATH, 'graph/directives'))
-  rateLimit: createRateLimitDirective({
-    keyGenerator: (directiveArgs, source, args, context, info) => `${context.req.ip}:${info.parentType}.${info.fieldName}`
-  })
 }
 
 // Live Trail Logger (admin)

server/graph/resolvers/storage.js

@@ -18,6 +18,9 @@ module.exports = {
         return {
           ...targetInfo,
           ...tgt,
+          hasSchedule: (targetInfo.schedule !== false),
+          syncInterval: targetInfo.syncInterval || targetInfo.schedule || 'P0D',
+          syncIntervalDefault: targetInfo.schedule,
           config: _.sortBy(_.transform(tgt.config, (res, value, key) => {
             const configData = _.get(targetInfo.props, key, {})
             res.push({
@@ -33,6 +36,18 @@ module.exports = {
       if (args.filter) { targets = graphHelper.filter(targets, args.filter) }
       if (args.orderBy) { targets = graphHelper.orderBy(targets, args.orderBy) }
       return targets
+    },
+    async status(obj, args, context, info) {
+      let activeTargets = await WIKI.models.storage.query().where('isEnabled', true)
+      return activeTargets.map(tgt => {
+        const targetInfo = _.find(WIKI.data.storage, ['key', tgt.key]) || {}
+        return {
+          key: tgt.key,
+          title: targetInfo.title,
+          status: _.get(tgt, 'state.status', 'pending'),
+          message: _.get(tgt, 'state.message', 'Initializing...')
+        }
+      })
     }
   },
   StorageMutation: {
@@ -42,10 +57,15 @@ module.exports = {
           await WIKI.models.storage.query().patch({
             isEnabled: tgt.isEnabled,
             mode: tgt.mode,
+            syncInterval: tgt.syncInterval,
             config: _.reduce(tgt.config, (result, value, key) => {
               _.set(result, `${value.key}`, _.get(JSON.parse(value.value), 'v', null))
               return result
-            }, {})
+            }, {}),
+            state: {
+              status: 'pending',
+              message: 'Initializing...'
+            }
           }).where('key', tgt.key)
         }
         await WIKI.models.storage.initTargets()

server/graph/schemas/storage.graphql

@@ -19,6 +19,8 @@ type StorageQuery {
     filter: String
     orderBy: String
   ): [StorageTarget] @auth(requires: ["manage:system"])
+
+  status: [StorageStatus] @auth(requires: ["manage:system"])
 }
 
 # -----------------------------------------------
@@ -27,7 +29,7 @@ type StorageQuery {
 
 type StorageMutation {
   updateTargets(
-    targets: [StorageTargetInput]
+    targets: [StorageTargetInput]!
   ): DefaultResponse @auth(requires: ["manage:system"])
 }
 
@@ -45,6 +47,9 @@ type StorageTarget {
   website: String
   supportedModes: [String]
   mode: String
+  hasSchedule: Boolean!
+  syncInterval: String
+  syncIntervalDefault: String
   config: [KeyValuePair]
 }
 
@@ -52,5 +57,13 @@ input StorageTargetInput {
   isEnabled: Boolean!
   key: String!
   mode: String!
+  syncInterval: String
   config: [KeyValuePairInput]
 }
+
+type StorageStatus {
+  key: String!
+  title: String!
+  status: String!
+  message: String
+}

server/helpers/config.js

@@ -2,6 +2,8 @@
 
 const _ = require('lodash')
 
+const isoDurationReg = /^(-|\+)?P(?:([-+]?[0-9,.]*)Y)?(?:([-+]?[0-9,.]*)M)?(?:([-+]?[0-9,.]*)W)?(?:([-+]?[0-9,.]*)D)?(?:T(?:([-+]?[0-9,.]*)H)?(?:([-+]?[0-9,.]*)M)?(?:([-+]?[0-9,.]*)S)?)?$/
+
 module.exports = {
   /**
    * Parse configuration value for environment vars
@@ -15,5 +17,9 @@ module.exports = {
       /\$\(([A-Z0-9_]+)\)/g,
       (fm, m) => { return process.env[m] }
     )
+  },
+
+  isValidDurationString (val) {
+    return isoDurationReg.test(val)
   }
 }

server/jobs/sync-storage.js

@@ -1,20 +1,18 @@
-require('../core/worker')
-
 /* global WIKI */
 
-module.exports = async (job) => {
+module.exports = async ({ target }) => {
   WIKI.logger.info(`Syncing with storage provider ${job.data.target.title}...`)
 
-  try {
+  // try {
-    const target = require(`../modules/storage/${job.data.target.key}/storage.js`)
+  //   const target = require(`../modules/storage/${job.data.target.key}/storage.js`)
-    target[job.data.event].call({
+  //   target[job.data.event].call({
-      config: job.data.target.config,
+  //     config: job.data.target.config,
-      mode: job.data.target.mode,
+  //     mode: job.data.target.mode,
-      page: job.data.page
+  //     page: job.data.page
-    })
+  //   })
-    WIKI.logger.info(`Syncing with storage provider ${job.data.target.title}: [ COMPLETED ]`)
+  //   WIKI.logger.info(`Syncing with storage provider ${job.data.target.title}: [ COMPLETED ]`)
-  } catch (err) {
+  // } catch (err) {
-    WIKI.logger.error(`Syncing with storage provider ${job.data.target.title}: [ FAILED ]`)
+  //   WIKI.logger.error(`Syncing with storage provider ${job.data.target.title}: [ FAILED ]`)
-    WIKI.logger.error(err.message)
+  //   WIKI.logger.error(err.message)
-  }
+  // }
 }

server/models/storage.js

@@ -63,10 +63,15 @@ module.exports = class Storage extends Model {
             key: target.key,
             isEnabled: false,
             mode: target.defaultMode || 'push',
+            syncInterval: target.schedule || 'P0D',
             config: _.transform(target.props, (result, value, key) => {
               _.set(result, key, value.default)
               return result
-            }, {})
+            }, {}),
+            state: {
+              status: 'pending',
+              message: ''
+            }
           })
         } else {
           const targetConfig = _.get(_.find(dbTargets, ['key', target.key]), 'config', {})
@@ -100,13 +105,28 @@ module.exports = class Storage extends Model {
   }
 
   static async initTargets() {
-    targets = await WIKI.models.storage.query().where('isEnabled', true)
+    targets = await WIKI.models.storage.query().where('isEnabled', true).orderBy('key')
     try {
       for(let target of targets) {
         target.fn = require(`../modules/storage/${target.key}/storage`)
         target.fn.config = target.config
         target.fn.mode = target.mode
-        await target.fn.init()
+        try {
+          await target.fn.init()
+          await WIKI.models.storage.query().patch({
+            state: {
+              status: 'operational',
+              message: ''
+            }
+          }).where('key', target.key)
+        } catch (err) {
+          await WIKI.models.storage.query().patch({
+            state: {
+              status: 'error',
+              message: err.message
+            }
+          }).where('key', target.key)
+        }
         // if (target.schedule) {
         //   WIKI.scheduler.registerJob({
         //     name: