From 10df1b4b0cc0328bbd88d010e10a9f86a8f91c27 Mon Sep 17 00:00:00 2001
From: Nick <github@ngpixel.com>
Date: Sat, 6 Apr 2019 19:05:47 -0400
Subject: [PATCH] feat: storage actions + git module actions

---
 client/components/admin/admin-storage.vue     |  57 +++++-
 .../storage-mutation-executeaction.gql        |  12 ++
 .../admin/storage/storage-query-targets.gql   |   5 +
 server/graph/resolvers/storage.js             |  10 +
 server/graph/schemas/storage.graphql          |  12 ++
 server/helpers/page.js                        |  20 ++
 server/models/pages.js                        |  16 +-
 server/models/storage.js                      |  24 ++-
 server/modules/storage/git/definition.yml     |   8 +-
 server/modules/storage/git/storage.js         | 180 ++++++++++++------
 10 files changed, 261 insertions(+), 83 deletions(-)
 create mode 100644 client/graph/admin/storage/storage-mutation-executeaction.gql

diff --git a/client/components/admin/admin-storage.vue b/client/components/admin/admin-storage.vue
index ddc7d939..c6588194 100644
--- a/client/components/admin/admin-storage.vue
+++ b/client/components/admin/admin-storage.vue
@@ -158,6 +158,24 @@
                       .caption.mt-3 Currently set to every #[strong {{getDefaultSchedule(tgt.syncInterval)}}].
                       .caption The default is every #[strong {{getDefaultSchedule(tgt.syncIntervalDefault)}}].
 
+                  template(v-if='tgt.actions && tgt.actions.length > 0')
+                    v-divider.mt-3
+                    v-subheader.pl-0 Actions
+                    v-container.pt-0(grid-list-xl, fluid)
+                      v-layout(row, wrap, fill-height)
+                        v-flex(xs12, lg6, xl4, v-for='act of tgt.actions')
+                          v-card.radius-7.grey(flat, :class='$vuetify.dark ? `darken-3-d5` : `lighten-3`', height='100%')
+                            v-card-text
+                              .subheading(v-html='act.label')
+                              .body-1.mt-2(v-html='act.hint')
+                              v-btn.mx-0.mt-3(
+                                @click='executeAction(tgt.key, act.handler)'
+                                outline
+                                :color='$vuetify.dark ? `blue` : `primary`'
+                                :disabled='runningAction'
+                                :loading='runningActionHandler === act.handler'
+                                ) Run
+
 </template>
 
 <script>
@@ -170,6 +188,7 @@ import { LoopingRhombusesSpinner } from 'epic-spinners'
 
 import statusQuery from 'gql/admin/storage/storage-query-status.gql'
 import targetsQuery from 'gql/admin/storage/storage-query-targets.gql'
+import targetExecuteActionMutation from 'gql/admin/storage/storage-mutation-executeaction.gql'
 import targetsSaveMutation from 'gql/admin/storage/storage-mutation-save-targets.gql'
 
 momentDurationFormatSetup(moment)
@@ -184,6 +203,8 @@ export default {
   },
   data() {
     return {
+      runningAction: false,
+      runningActionHandler: '',
       currentTab: 0,
       targets: [],
       status: []
@@ -209,12 +230,12 @@ export default {
         mutation: targetsSaveMutation,
         variables: {
           targets: this.targets.map(tgt => _.pick(tgt, [
-              'isEnabled',
-              'key',
-              'config',
-              'mode',
-              'syncInterval'
-            ])).map(str => ({...str, config: str.config.map(cfg => ({...cfg, value: JSON.stringify({ v: cfg.value.value })}))}))
+            'isEnabled',
+            'key',
+            'config',
+            'mode',
+            'syncInterval'
+          ])).map(str => ({...str, config: str.config.map(cfg => ({...cfg, value: JSON.stringify({ v: cfg.value.value })}))}))
         }
       })
       this.currentTab = 0
@@ -239,6 +260,30 @@ export default {
     },
     getDefaultSchedule(val) {
       return moment.duration(val).format('y [years], M [months], d [days], h [hours], m [minutes]')
+    },
+    async executeAction(targetKey, handler) {
+      this.$store.commit(`loadingStart`, 'admin-storage-executeaction')
+      this.runningAction = true
+      this.runningActionHandler = handler
+      try {
+        await this.$apollo.mutate({
+          mutation: targetExecuteActionMutation,
+          variables: {
+            targetKey,
+            handler
+          }
+        })
+        this.$store.commit('showNotification', {
+          message: 'Action completed.',
+          style: 'success',
+          icon: 'check'
+        })
+      } catch (err) {
+        console.warn(err)
+      }
+      this.runningAction = false
+      this.runningActionHandler = ''
+      this.$store.commit(`loadingStop`, 'admin-storage-executeaction')
     }
   },
   apollo: {
diff --git a/client/graph/admin/storage/storage-mutation-executeaction.gql b/client/graph/admin/storage/storage-mutation-executeaction.gql
new file mode 100644
index 00000000..aa0fb07c
--- /dev/null
+++ b/client/graph/admin/storage/storage-mutation-executeaction.gql
@@ -0,0 +1,12 @@
+mutation($targetKey: String!, $handler: String!) {
+  storage {
+    executeAction(targetKey: $targetKey, handler: $handler) {
+      responseResult {
+        succeeded
+        errorCode
+        slug
+        message
+      }
+    }
+  }
+}
diff --git a/client/graph/admin/storage/storage-query-targets.gql b/client/graph/admin/storage/storage-query-targets.gql
index ff7ff81a..c9c66b23 100644
--- a/client/graph/admin/storage/storage-query-targets.gql
+++ b/client/graph/admin/storage/storage-query-targets.gql
@@ -17,6 +17,11 @@ query {
         key
         value
       }
+      actions {
+        handler
+        label
+        hint
+      }
     }
   }
 }
diff --git a/server/graph/resolvers/storage.js b/server/graph/resolvers/storage.js
index 632c8398..295fba1c 100644
--- a/server/graph/resolvers/storage.js
+++ b/server/graph/resolvers/storage.js
@@ -77,6 +77,16 @@ module.exports = {
       } catch (err) {
         return graphHelper.generateError(err)
       }
+    },
+    async executeAction(obj, args, context) {
+      try {
+        await WIKI.models.storage.executeAction(args.targetKey, args.handler)
+        return {
+          responseResult: graphHelper.generateSuccess('Action completed.')
+        }
+      } catch (err) {
+        return graphHelper.generateError(err)
+      }
     }
   }
 }
diff --git a/server/graph/schemas/storage.graphql b/server/graph/schemas/storage.graphql
index b82b1b1e..53a42b00 100644
--- a/server/graph/schemas/storage.graphql
+++ b/server/graph/schemas/storage.graphql
@@ -31,6 +31,11 @@ type StorageMutation {
   updateTargets(
     targets: [StorageTargetInput]!
   ): DefaultResponse @auth(requires: ["manage:system"])
+
+  executeAction(
+    targetKey: String!
+    handler: String!
+  ): DefaultResponse @auth(requires: ["manage:system"])
 }
 
 # -----------------------------------------------
@@ -51,6 +56,7 @@ type StorageTarget {
   syncInterval: String
   syncIntervalDefault: String
   config: [KeyValuePair]
+  actions: [StorageTargetAction]
 }
 
 input StorageTargetInput {
@@ -68,3 +74,9 @@ type StorageStatus {
   message: String!
   lastAttempt: String!
 }
+
+type StorageTargetAction {
+  handler: String!
+  label: String!
+  hint: String!
+}
diff --git a/server/helpers/page.js b/server/helpers/page.js
index 8d3adb1e..9893fe56 100644
--- a/server/helpers/page.js
+++ b/server/helpers/page.js
@@ -36,5 +36,25 @@ module.exports = {
    */
   generateHash(opts) {
     return crypto.createHash('sha1').update(`${opts.locale}|${opts.path}|${opts.privateNS}`).digest('hex')
+  },
+  /**
+   * Inject Page Metadata
+   */
+  injectPageMetadata(page) {
+    let meta = [
+      ['title', page.title],
+      ['description', page.description],
+      ['published', page.isPublished.toString()],
+      ['date', page.updatedAt],
+      ['tags', '']
+    ]
+    switch (page.contentType) {
+      case 'markdown':
+        return '---\n' + meta.map(mt => `${mt[0]}: ${mt[1]}`).join('\n') + '\n---\n\n' + page.content
+      case 'html':
+        return '<!--\n' + meta.map(mt => `${mt[0]}: ${mt[1]}`).join('\n') + '\n-->\n\n' + page.content
+      default:
+        return page.content
+    }
   }
 }
diff --git a/server/models/pages.js b/server/models/pages.js
index ba7981d5..a4b60e0a 100644
--- a/server/models/pages.js
+++ b/server/models/pages.js
@@ -130,21 +130,7 @@ module.exports = class Page extends Model {
    * Inject page metadata into contents
    */
   injectMetadata () {
-    let meta = [
-      ['title', this.title],
-      ['description', this.description],
-      ['published', this.isPublished.toString()],
-      ['date', this.updatedAt],
-      ['tags', '']
-    ]
-    switch (this.contentType) {
-      case 'markdown':
-        return '---\n' + meta.map(mt => `${mt[0]}: ${mt[1]}`).join('\n') + '\n---\n\n' + this.content
-      case 'html':
-        return '<!--\n' + meta.map(mt => `${mt[0]}: ${mt[1]}`).join('\n') + '\n-->\n\n' + this.content
-      default:
-        return this.content
-    }
+    return pageHelper.injectPageMetadata(this)
   }
 
   /**
diff --git a/server/models/storage.js b/server/models/storage.js
index aa601555..91cfd91e 100644
--- a/server/models/storage.js
+++ b/server/models/storage.js
@@ -60,7 +60,7 @@ module.exports = class Storage extends Model {
           newTargets.push({
             key: target.key,
             isEnabled: false,
-            mode: target.defaultMode || 'push',
+            mode: target.defaultMode || 'push',
             syncInterval: target.schedule || 'P0D',
             config: _.transform(target.props, (result, value, key) => {
               _.set(result, key, value.default)
@@ -116,7 +116,7 @@ module.exports = class Storage extends Model {
       }
 
       // -> Initialize targets
-      for(let target of this.targets) {
+      for (let target of this.targets) {
         const targetDef = _.find(WIKI.data.storage, ['key', target.key])
         target.fn = require(`../modules/storage/${target.key}/storage`)
         target.fn.config = target.config
@@ -161,7 +161,7 @@ module.exports = class Storage extends Model {
 
   static async pageEvent({ event, page }) {
     try {
-      for(let target of this.targets) {
+      for (let target of this.targets) {
         await target.fn[event](page)
       }
     } catch (err) {
@@ -169,4 +169,22 @@ module.exports = class Storage extends Model {
       throw err
     }
   }
+
+  static async executeAction(targetKey, handler) {
+    try {
+      const target = _.find(this.targets, ['key', targetKey])
+      if (target) {
+        if (_.has(target.fn, handler)) {
+          await target.fn[handler]()
+        } else {
+          throw new Error('Invalid Handler for Storage Target')
+        }
+      } else {
+        throw new Error('Invalid or Inactive Storage Target')
+      }
+    } catch (err) {
+      WIKI.logger.warn(err)
+      throw err
+    }
+  }
 }
diff --git a/server/modules/storage/git/definition.yml b/server/modules/storage/git/definition.yml
index 8c61176d..35ccf754 100644
--- a/server/modules/storage/git/definition.yml
+++ b/server/modules/storage/git/definition.yml
@@ -77,4 +77,10 @@ props:
 actions:
   - handler: syncUntracked
     label: Add Untracked Changes
-    hint: Output all content from the DB to the Git repo to ensure all untracked content is saved. If you enabled Git after content was created or you temporarily disabled Git, you'll want to execute this action to add the missing untracked changes.
+    hint: Output all content from the DB to the local Git repository to ensure all untracked content is saved. If you enabled Git after content was created or you temporarily disabled Git, you'll want to execute this action to add the missing untracked changes.
+  - handler: sync
+    label: Force Sync
+    hint: Will trigger an immediate sync operation, regardless of the current sync schedule. The sync direction is respected.
+  - handler: importAll
+    label: Import Everything
+    hint: Will import all content currently in the local Git repository, regardless of the latest commit state. Useful for importing content from the remote repository created before git was enabled.
diff --git a/server/modules/storage/git/storage.js b/server/modules/storage/git/storage.js
index 449e1718..bcc6195f 100644
--- a/server/modules/storage/git/storage.js
+++ b/server/modules/storage/git/storage.js
@@ -2,6 +2,11 @@ const path = require('path')
 const sgit = require('simple-git/promise')
 const fs = require('fs-extra')
 const _ = require('lodash')
+const stream = require('stream')
+const Promise = require('bluebird')
+const pipeline = Promise.promisify(stream.pipeline)
+const klaw = require('klaw')
+const pageHelper = require('../../../helpers/page.js')
 
 const localeFolderRegex = /^([a-z]{2}(?:-[a-z]{2})?\/)?(.*)/i
 
@@ -154,66 +159,74 @@ module.exports = {
 
       const diff = await this.git.diffSummary(['-M', currentCommitLog.hash, latestCommitLog.hash])
       if (_.get(diff, 'files', []).length > 0) {
-        for (const item of diff.files) {
-          const contentType = getContenType(item.file)
-          if (!contentType) {
-            continue
-          }
-          const contentPath = getPagePath(item.file)
+        await this.processFiles(diff.files)
+      }
+    }
+  },
+  /**
+   * Process Files
+   *
+   * @param {Array<String>} files Array of files to process
+   */
+  async processFiles(files) {
+    for (const item of files) {
+      const contentType = getContenType(item.file)
+      if (!contentType) {
+        continue
+      }
+      const contentPath = getPagePath(item.file)
 
-          let itemContents = ''
-          try {
-            itemContents = await fs.readFile(path.join(this.repoPath, item.file), 'utf8')
-            const pageData = WIKI.models.pages.parseMetadata(itemContents, contentType)
-            const currentPage = await WIKI.models.pages.query().findOne({
-              path: contentPath.path,
-              localeCode: contentPath.locale
-            })
-            if (currentPage) {
-              // Already in the DB, can mark as modified
-              WIKI.logger.info(`(STORAGE/GIT) Page marked as modified: ${item.file}`)
-              await WIKI.models.pages.updatePage({
-                id: currentPage.id,
-                title: _.get(pageData, 'title', currentPage.title),
-                description: _.get(pageData, 'description', currentPage.description),
-                isPublished: _.get(pageData, 'isPublished', currentPage.isPublished),
-                isPrivate: false,
-                content: pageData.content,
-                authorId: 1,
-                skipStorage: true
-              })
-            } else {
-              // Not in the DB, can mark as new
-              WIKI.logger.info(`(STORAGE/GIT) Page marked as new: ${item.file}`)
-              const pageEditor = await WIKI.models.editors.getDefaultEditor(contentType)
-              await WIKI.models.pages.createPage({
-                path: contentPath.path,
-                locale: contentPath.locale,
-                title: _.get(pageData, 'title', _.last(contentPath.path.split('/'))),
-                description: _.get(pageData, 'description', ''),
-                isPublished: _.get(pageData, 'isPublished', true),
-                isPrivate: false,
-                content: pageData.content,
-                authorId: 1,
-                editor: pageEditor,
-                skipStorage: true
-              })
-            }
-          } catch (err) {
-            if (err.code === 'ENOENT' && item.deletions > 0 && item.insertions === 0) {
-              // File was deleted by git, can safely mark as deleted in DB
-              WIKI.logger.info(`(STORAGE/GIT) Page marked as deleted: ${item.file}`)
+      let itemContents = ''
+      try {
+        itemContents = await fs.readFile(path.join(this.repoPath, item.file), 'utf8')
+        const pageData = WIKI.models.pages.parseMetadata(itemContents, contentType)
+        const currentPage = await WIKI.models.pages.query().findOne({
+          path: contentPath.path,
+          localeCode: contentPath.locale
+        })
+        if (currentPage) {
+          // Already in the DB, can mark as modified
+          WIKI.logger.info(`(STORAGE/GIT) Page marked as modified: ${item.file}`)
+          await WIKI.models.pages.updatePage({
+            id: currentPage.id,
+            title: _.get(pageData, 'title', currentPage.title),
+            description: _.get(pageData, 'description', currentPage.description),
+            isPublished: _.get(pageData, 'isPublished', currentPage.isPublished),
+            isPrivate: false,
+            content: pageData.content,
+            authorId: 1,
+            skipStorage: true
+          })
+        } else {
+          // Not in the DB, can mark as new
+          WIKI.logger.info(`(STORAGE/GIT) Page marked as new: ${item.file}`)
+          const pageEditor = await WIKI.models.editors.getDefaultEditor(contentType)
+          await WIKI.models.pages.createPage({
+            path: contentPath.path,
+            locale: contentPath.locale,
+            title: _.get(pageData, 'title', _.last(contentPath.path.split('/'))),
+            description: _.get(pageData, 'description', ''),
+            isPublished: _.get(pageData, 'isPublished', true),
+            isPrivate: false,
+            content: pageData.content,
+            authorId: 1,
+            editor: pageEditor,
+            skipStorage: true
+          })
+        }
+      } catch (err) {
+        if (err.code === 'ENOENT' && item.deletions > 0 && item.insertions === 0) {
+          // File was deleted by git, can safely mark as deleted in DB
+          WIKI.logger.info(`(STORAGE/GIT) Page marked as deleted: ${item.file}`)
 
-              await WIKI.models.pages.deletePage({
-                path: contentPath.path,
-                locale: contentPath.locale,
-                skipStorage: true
-              })
-            } else {
-              WIKI.logger.warn(`(STORAGE/GIT) Failed to open ${item.file}`)
-              WIKI.logger.warn(err)
-            }
-          }
+          await WIKI.models.pages.deletePage({
+            path: contentPath.path,
+            locale: contentPath.locale,
+            skipStorage: true
+          })
+        } else {
+          WIKI.logger.warn(`(STORAGE/GIT) Failed to open ${item.file}`)
+          WIKI.logger.warn(err)
         }
       }
     }
@@ -278,5 +291,56 @@ module.exports = {
     await this.git.commit(`docs: rename ${page.sourcePath} to ${destinationFilePath}`, destinationFilePath, {
       '--author': `"${page.authorName} <${page.authorEmail}>"`
     })
+  },
+
+  /**
+   * HANDLERS
+   */
+  async importAll() {
+    WIKI.logger.info(`(STORAGE/GIT) Importing all content from local Git repo to the DB...`)
+    await pipeline(
+      klaw(this.repoPath, {
+        filter: (f) => {
+          return !_.includes(f, '.git')
+        }
+      }),
+      new stream.Transform({
+        objectMode: true,
+        transform: async (file, enc, cb) => {
+          const relPath = file.path.substr(this.repoPath.length + 1)
+          if (relPath && relPath.length > 3) {
+            WIKI.logger.info(`(STORAGE/GIT) Processing ${relPath}...`)
+            await this.processFiles([{
+              file: relPath,
+              deletions: 0,
+              insertions: 0
+            }])
+          }
+          cb()
+        }
+      })
+    )
+    WIKI.logger.info('(STORAGE/GIT) Import completed.')
+  },
+  async syncUntracked() {
+    WIKI.logger.info(`(STORAGE/GIT) Adding all untracked content...`)
+    await pipeline(
+      WIKI.models.knex.column('path', 'localeCode', 'title', 'description', 'contentType', 'content', 'isPublished', 'updatedAt').select().from('pages').where({
+        isPrivate: false
+      }).stream(),
+      new stream.Transform({
+        objectMode: true,
+        transform: async (page, enc, cb) => {
+          const fileName = `${page.path}.${getFileExtension(page.contentType)}`
+          WIKI.logger.info(`(STORAGE/GIT) Adding ${fileName}...`)
+          const filePath = path.join(this.repoPath, fileName)
+          await fs.outputFile(filePath, pageHelper.injectPageMetadata(page), 'utf8')
+          await this.git.add(`./${fileName}`)
+          cb()
+        }
+      })
+    )
+    await this.git.commit(`docs: add all untracked content`)
+    WIKI.logger.info('(STORAGE/GIT) All content is now tracked.')
   }
 }