From e6f1f3add42065a0ad8fa6bae733188809af6034 Mon Sep 17 00:00:00 2001
From: Nick <github@ngpixel.com>
Date: Sun, 24 Feb 2019 23:48:28 -0500
Subject: [PATCH] feat: git changes processing

---
 CHANGELOG.md                                  |  13 ++
 client/components/admin/admin-storage.vue     |  12 +-
 .../admin/storage/storage-query-status.gql    |   1 +
 server/db/migrations-sqlite/2.0.0-beta.11.js  |   3 +-
 server/graph/resolvers/storage.js             |   6 +-
 server/graph/schemas/storage.graphql          |   3 +-
 server/jobs/sync-storage.js                   |  15 ++
 server/models/editors.js                      |  10 ++
 server/models/pageHistory.js                  |   4 +-
 server/models/pages.js                        |  92 +++++++++++--
 server/models/storage.js                      |   9 +-
 server/modules/storage/git/definition.yml     |   4 +
 server/modules/storage/git/storage.js         | 128 ++++++++++++++++++
 13 files changed, 275 insertions(+), 25 deletions(-)

diff --git a/CHANGELOG.md b/CHANGELOG.md
index 7b645870..42f2ce09 100644
--- a/CHANGELOG.md
+++ b/CHANGELOG.md
@@ -2,6 +2,19 @@
 All notable changes to this project will be documented in this file.
 This project adheres to [Semantic Versioning](http://semver.org/).
 
+## [2.0.0-beta.XX] - 2018-XX-XX
+### Added
+- Added Git changes processing (add/modify/delete)
+- Added Storage last sync date in status panel
+- Added Dev Flags
+- Added HTTP to HTTPS redirect server option
+
+### Fixed
+- Fixed SQLite migrations
+
+### Changed
+- Split admin dev section into separate pages
+
 ## [2.0.0-beta.42] - 2018-02-17
 ### Added
 - Added Patreon link in Contribute admin page
diff --git a/client/components/admin/admin-storage.vue b/client/components/admin/admin-storage.vue
index 55409338..ddc7d939 100644
--- a/client/components/admin/admin-storage.vue
+++ b/client/components/admin/admin-storage.vue
@@ -54,10 +54,18 @@
                         :color='getStatusColor(tgt.status)'
                         dark
                         flat
-                        :extended='tgt.status === `error`',
-                        extension-height='100'
+                        :extended='tgt.status !== `pending`',
+                        :extension-height='tgt.status === `error` ? 100 : 70'
                         )
                         .pa-3.red.darken-2.radius-7(v-if='tgt.status === `error`', slot='extension') {{tgt.message}}
+                        v-toolbar.radius-7(
+                          color='green darken-2'
+                          v-else-if='tgt.status !== `pending`'
+                          slot='extension'
+                          flat
+                          dense
+                          )
+                          span Last synchronization {{tgt.lastAttempt | moment('from') }}
                         .body-2 {{tgt.title}}
                         v-spacer
                         .body-1 {{tgt.status}}
diff --git a/client/graph/admin/storage/storage-query-status.gql b/client/graph/admin/storage/storage-query-status.gql
index a5a81497..9a9e1f44 100644
--- a/client/graph/admin/storage/storage-query-status.gql
+++ b/client/graph/admin/storage/storage-query-status.gql
@@ -5,6 +5,7 @@ query {
       title
       status
       message
+      lastAttempt
     }
   }
 }
diff --git a/server/db/migrations-sqlite/2.0.0-beta.11.js b/server/db/migrations-sqlite/2.0.0-beta.11.js
index 998730fa..ef4580ac 100644
--- a/server/db/migrations-sqlite/2.0.0-beta.11.js
+++ b/server/db/migrations-sqlite/2.0.0-beta.11.js
@@ -16,11 +16,12 @@ exports.up = knex => {
       table.string('createdAt').notNullable()
       table.string('action').defaultTo('updated')
 
+      table.integer('pageId').unsigned()
       table.string('editorKey').references('key').inTable('editors')
       table.string('localeCode', 2).references('code').inTable('locales')
       table.integer('authorId').unsigned().references('id').inTable('users')
     })
-    .raw(`INSERT INTO pageHistory SELECT id,path,hash,title,description,isPrivate,isPublished,publishStartDate,publishEndDate,content,contentType,createdAt,'updated' AS action,editorKey,localeCode,authorId FROM pageHistory_old;`)
+    .raw(`INSERT INTO pageHistory SELECT id,path,hash,title,description,isPrivate,isPublished,publishStartDate,publishEndDate,content,contentType,createdAt,'updated' AS action,pageId,editorKey,localeCode,authorId FROM pageHistory_old;`)
     .dropTable('pageHistory_old')
 }
 
diff --git a/server/graph/resolvers/storage.js b/server/graph/resolvers/storage.js
index 4899af5a..632c8398 100644
--- a/server/graph/resolvers/storage.js
+++ b/server/graph/resolvers/storage.js
@@ -45,7 +45,8 @@ module.exports = {
           key: tgt.key,
           title: targetInfo.title,
           status: _.get(tgt, 'state.status', 'pending'),
-          message: _.get(tgt, 'state.message', 'Initializing...')
+          message: _.get(tgt, 'state.message', 'Initializing...'),
+          lastAttempt: _.get(tgt, 'state.lastAttempt', null)
         }
       })
     }
@@ -64,7 +65,8 @@ module.exports = {
             }, {}),
             state: {
               status: 'pending',
-              message: 'Initializing...'
+              message: 'Initializing...',
+              lastAttempt: null
             }
           }).where('key', tgt.key)
         }
diff --git a/server/graph/schemas/storage.graphql b/server/graph/schemas/storage.graphql
index 783929f1..b82b1b1e 100644
--- a/server/graph/schemas/storage.graphql
+++ b/server/graph/schemas/storage.graphql
@@ -65,5 +65,6 @@ type StorageStatus {
   key: String!
   title: String!
   status: String!
-  message: String
+  message: String!
+  lastAttempt: String!
 }
diff --git a/server/jobs/sync-storage.js b/server/jobs/sync-storage.js
index dc0bb8d6..9208f2c4 100644
--- a/server/jobs/sync-storage.js
+++ b/server/jobs/sync-storage.js
@@ -10,11 +10,26 @@ module.exports = async (targetKey) => {
     if (target) {
       await target.fn.sync()
       WIKI.logger.info(`Syncing with storage target ${targetKey}: [ COMPLETED ]`)
+
+      await WIKI.models.storage.query().patch({
+        state: {
+          status: 'operational',
+          message: '',
+          lastAttempt: new Date().toISOString()
+        }
+      }).where('key', targetKey)
     } else {
       throw new Error('Invalid storage target. Unable to perform sync.')
     }
   } catch (err) {
     WIKI.logger.error(`Syncing with storage target ${targetKey}: [ FAILED ]`)
     WIKI.logger.error(err.message)
+    await WIKI.models.storage.query().patch({
+      state: {
+        status: 'error',
+        message: err.message,
+        lastAttempt: new Date().toISOString()
+      }
+    }).where('key', targetKey)
   }
 }
diff --git a/server/models/editors.js b/server/models/editors.js
index 29e3db6c..f7ee494d 100644
--- a/server/models/editors.js
+++ b/server/models/editors.js
@@ -90,4 +90,14 @@ module.exports = class Editor extends Model {
       WIKI.logger.error(err)
     }
   }
+
+  static async getDefaultEditor(contentType) {
+    // TODO - hardcoded for now
+    switch (contentType) {
+      case 'markdown':
+        return 'markdown'
+      default:
+        return 'code'
+    }
+  }
 }
diff --git a/server/models/pageHistory.js b/server/models/pageHistory.js
index be7370a8..145e9eeb 100644
--- a/server/models/pageHistory.js
+++ b/server/models/pageHistory.js
@@ -93,8 +93,8 @@ module.exports = class PageHistory extends Model {
       description: opts.description,
       editorKey: opts.editorKey,
       hash: opts.hash,
-      isPrivate: opts.isPrivate,
-      isPublished: opts.isPublished,
+      isPrivate: (opts.isPrivate === true || opts.isPrivate === 1),
+      isPublished: (opts.isPublished === true || opts.isPublished === 1),
       localeCode: opts.localeCode,
       path: opts.path,
       publishEndDate: opts.publishEndDate || '',
diff --git a/server/models/pages.js b/server/models/pages.js
index 7e922999..c3ff8acf 100644
--- a/server/models/pages.js
+++ b/server/models/pages.js
@@ -4,9 +4,16 @@ const JSBinType = require('js-binary').Type
 const pageHelper = require('../helpers/page')
 const path = require('path')
 const fs = require('fs-extra')
+const yaml = require('js-yaml')
 
 /* global WIKI */
 
+const frontmatterRegex = {
+  html: /^(<!-{2}(?:\n|\r)([\w\W]+?)(?:\n|\r)-{2}>)?(?:\n|\r)*([\w\W]*)*/,
+  legacy: /^(<!-- TITLE: ?([\w\W]+?) -{2}>)?(?:\n|\r)?(<!-- SUBTITLE: ?([\w\W]+?) -{2}>)?(?:\n|\r)*([\w\W]*)*/i,
+  markdown: /^(-{3}(?:\n|\r)([\w\W]+?)(?:\n|\r)-{3})?(?:\n|\r)*([\w\W]*)*/
+}
+
 /**
  * Pages model
  */
@@ -135,6 +142,49 @@ module.exports = class Page extends Model {
     }
   }
 
+  /**
+   * Parse injected page metadata from raw content
+   *
+   * @param {String} raw Raw file contents
+   * @param {String} contentType Content Type
+   */
+  static parseMetadata (raw, contentType) {
+    let result
+    switch (contentType) {
+      case 'markdown':
+        result = frontmatterRegex.markdown.exec(raw)
+        if (result[2]) {
+          return {
+            ...yaml.safeLoad(result[2]),
+            content: result[3]
+          }
+        } else {
+          // Attempt legacy v1 format
+          result = frontmatterRegex.legacy.exec(raw)
+          if (result[2]) {
+            return {
+              title: result[2],
+              description: result[4],
+              content: result[5]
+            }
+          }
+        }
+        break
+      case 'html':
+        result = frontmatterRegex.html.exec(raw)
+        if (result[2]) {
+          return {
+            ...yaml.safeLoad(result[2]),
+            content: result[3]
+          }
+        }
+        break
+    }
+    return {
+      content: raw
+    }
+  }
+
   static async createPage(opts) {
     await WIKI.models.pages.query().insert({
       authorId: opts.authorId,
@@ -160,10 +210,12 @@ module.exports = class Page extends Model {
       isPrivate: opts.isPrivate
     })
     await WIKI.models.pages.renderPage(page)
-    await WIKI.models.storage.pageEvent({
-      event: 'created',
-      page
-    })
+    if (!opts.skipStorage) {
+      await WIKI.models.storage.pageEvent({
+        event: 'created',
+        page
+      })
+    }
     return page
   }
 
@@ -181,7 +233,7 @@ module.exports = class Page extends Model {
       authorId: opts.authorId,
       content: opts.content,
       description: opts.description,
-      isPublished: opts.isPublished,
+      isPublished: opts.isPublished === true || opts.isPublished === 1,
       publishEndDate: opts.publishEndDate || '',
       publishStartDate: opts.publishStartDate || '',
       title: opts.title
@@ -193,15 +245,25 @@ module.exports = class Page extends Model {
       isPrivate: ogPage.isPrivate
     })
     await WIKI.models.pages.renderPage(page)
-    await WIKI.models.storage.pageEvent({
-      event: 'updated',
-      page
-    })
+    if (!opts.skipStorage) {
+      await WIKI.models.storage.pageEvent({
+        event: 'updated',
+        page
+      })
+    }
     return page
   }
 
   static async deletePage(opts) {
-    const page = await WIKI.models.pages.query().findById(opts.id)
+    let page
+    if (_.has(opts, 'id')) {
+      page = await WIKI.models.pages.query().findById(opts.id)
+    } else {
+      page = await await WIKI.models.pages.query().findOne({
+        path: opts.path,
+        localeCode: opts.locale
+      })
+    }
     if (!page) {
       throw new Error('Invalid Page Id')
     }
@@ -211,10 +273,12 @@ module.exports = class Page extends Model {
     })
     await WIKI.models.pages.query().delete().where('id', page.id)
     await WIKI.models.pages.deletePageFromCache(page)
-    await WIKI.models.storage.pageEvent({
-      event: 'deleted',
-      page
-    })
+    if (!opts.skipStorage) {
+      await WIKI.models.storage.pageEvent({
+        event: 'deleted',
+        page
+      })
+    }
   }
 
   static async renderPage(page) {
diff --git a/server/models/storage.js b/server/models/storage.js
index a500bbb6..aa601555 100644
--- a/server/models/storage.js
+++ b/server/models/storage.js
@@ -68,7 +68,8 @@ module.exports = class Storage extends Model {
             }, {}),
             state: {
               status: 'pending',
-              message: ''
+              message: '',
+              lastAttempt: null
             }
           })
         } else {
@@ -127,7 +128,8 @@ module.exports = class Storage extends Model {
           await WIKI.models.storage.query().patch({
             state: {
               status: 'operational',
-              message: ''
+              message: '',
+              lastAttempt: new Date().toISOString()
             }
           }).where('key', target.key)
 
@@ -145,7 +147,8 @@ module.exports = class Storage extends Model {
           await WIKI.models.storage.query().patch({
             state: {
               status: 'error',
-              message: err.message
+              message: err.message,
+              lastAttempt: new Date().toISOString()
             }
           }).where('key', target.key)
         }
diff --git a/server/modules/storage/git/definition.yml b/server/modules/storage/git/definition.yml
index 3b5f938b..5af24a93 100644
--- a/server/modules/storage/git/definition.yml
+++ b/server/modules/storage/git/definition.yml
@@ -69,3 +69,7 @@ props:
     default: 'John Smith'
     hint: 'Used as fallback in case the author of the change is not present.'
     order: 21
+actions:
+  - handler: syncUntracked
+    label: Add Untracked Changes
+    hint: Output all content from the DB to the Git repo to ensure all untracked content is saved. If you enabled Git after content was created or you temporarily disabled Git, you'll want to execute this action to add the missing untracked changes.
diff --git a/server/modules/storage/git/storage.js b/server/modules/storage/git/storage.js
index 9bb888d8..b49b838a 100644
--- a/server/modules/storage/git/storage.js
+++ b/server/modules/storage/git/storage.js
@@ -3,6 +3,8 @@ const sgit = require('simple-git/promise')
 const fs = require('fs-extra')
 const _ = require('lodash')
 
+const localeFolderRegex = /^([a-z]{2}(?:-[a-z]{2})?\/)?(.*)/i
+
 /**
  * Get file extension based on content type
  */
@@ -17,6 +19,33 @@ const getFileExtension = (contentType) => {
   }
 }
 
+const getContenType = (filePath) => {
+  const ext = _.last(filePath.split('.'))
+  switch (ext) {
+    case 'md':
+      return 'markdown'
+    case 'html':
+      return 'html'
+    default:
+      return false
+  }
+}
+
+const getPagePath = (filePath) => {
+  let meta = {
+    locale: 'en',
+    path: _.initial(filePath.split('.')).join('')
+  }
+  const result = localeFolderRegex.exec(meta.path)
+  if (result[1]) {
+    meta = {
+      locale: result[1],
+      path: result[2]
+    }
+  }
+  return meta
+}
+
 module.exports = {
   git: null,
   repoPath: path.join(process.cwd(), 'data/repo'),
@@ -26,6 +55,9 @@ module.exports = {
   async deactivated() {
     // not used
   },
+  /**
+   * INIT
+   */
   async init() {
     WIKI.logger.info('(STORAGE/GIT) Initializing...')
     this.repoPath = path.resolve(WIKI.ROOTPATH, this.config.localRepoPath)
@@ -87,7 +119,12 @@ module.exports = {
 
     WIKI.logger.info('(STORAGE/GIT) Initialization completed.')
   },
+  /**
+   * SYNC
+   */
   async sync() {
+    const currentCommitLog = _.get(await this.git.log(['-n', '1', this.config.branch]), 'latest', {})
+
     // Pull rebase
     if (_.includes(['sync', 'pull'], this.mode)) {
       WIKI.logger.info(`(STORAGE/GIT) Performing pull rebase from origin on branch ${this.config.branch}...`)
@@ -103,7 +140,83 @@ module.exports = {
       }
       await this.git.push('origin', this.config.branch, pushOpts)
     }
+
+    // Process Changes
+    if (_.includes(['sync', 'pull'], this.mode)) {
+      const latestCommitLog = _.get(await this.git.log(['-n', '1', this.config.branch]), 'latest', {})
+
+      const diff = await this.git.diffSummary([currentCommitLog.hash, latestCommitLog.hash])
+      if (_.get(diff, 'files', []).length > 0) {
+        for(const item of diff.files) {
+          const contentType = getContenType(item.file)
+          if (!contentType) {
+            continue
+          }
+          const contentPath = getPagePath(item.file)
+
+          let itemContents = ''
+          try {
+            itemContents = await fs.readFile(path.join(this.repoPath, item.file), 'utf8')
+            const pageData = WIKI.models.pages.parseMetadata(itemContents, contentType)
+            const currentPage = await WIKI.models.pages.query().findOne({
+              path: contentPath.path,
+              localeCode: contentPath.locale
+            })
+            if (currentPage) {
+              // Already in the DB, can mark as modified
+              WIKI.logger.info(`(STORAGE/GIT) Page marked as modified: ${item.file}`)
+              await WIKI.models.pages.updatePage({
+                id: currentPage.id,
+                title: _.get(pageData, 'title', currentPage.title),
+                description: _.get(pageData, 'description', currentPage.description),
+                isPublished: _.get(pageData, 'isPublished', currentPage.isPublished),
+                isPrivate: false,
+                content: pageData.content,
+                authorId: 1,
+                skipStorage: true
+              })
+            } else {
+              // Not in the DB, can mark as new
+              WIKI.logger.info(`(STORAGE/GIT) Page marked as new: ${item.file}`)
+              const pageEditor = await WIKI.models.editors.getDefaultEditor(contentType)
+              await WIKI.models.pages.createPage({
+                path: contentPath.path,
+                locale: contentPath.locale,
+                title: _.get(pageData, 'title', _.last(contentPath.path.split('/'))),
+                description: _.get(pageData, 'description', ''),
+                isPublished: _.get(pageData, 'isPublished', true),
+                isPrivate: false,
+                content: pageData.content,
+                authorId: 1,
+                editor: pageEditor,
+                skipStorage: true
+              })
+            }
+          } catch (err) {
+            if (err.code === 'ENOENT' && item.deletions > 0 && item.insertions === 0) {
+              // File was deleted by git, can safely mark as deleted in DB
+              WIKI.logger.info(`(STORAGE/GIT) Page marked as deleted: ${item.file}`)
+
+              await WIKI.models.pages.deletePage({
+                path: contentPath.path,
+                locale: contentPath.locale,
+                skipStorage: true
+              })
+
+            } else {
+              WIKI.logger.warn(`(STORAGE/GIT) Failed to open ${item.file}`)
+              WIKI.logger.warn(err)
+            }
+          }
+        }
+      }
+    }
   },
+  /**
+   * CREATE
+   *
+   * @param {Object} page Page to create
+   */
   async created(page) {
     WIKI.logger.info(`(STORAGE/GIT) Committing new file ${page.path}...`)
     const fileName = `${page.path}.${getFileExtension(page.contentType)}`
@@ -115,6 +228,11 @@ module.exports = {
       '--author': `"${page.authorName} <${page.authorEmail}>"`
     })
   },
+  /**
+   * UPDATE
+   *
+   * @param {Object} page Page to update
+   */
   async updated(page) {
     WIKI.logger.info(`(STORAGE/GIT) Committing updated file ${page.path}...`)
     const fileName = `${page.path}.${getFileExtension(page.contentType)}`
@@ -126,6 +244,11 @@ module.exports = {
       '--author': `"${page.authorName} <${page.authorEmail}>"`
     })
   },
+  /**
+   * DELETE
+   *
+   * @param {Object} page Page to delete
+   */
   async deleted(page) {
     WIKI.logger.info(`(STORAGE/GIT) Committing removed file ${page.path}...`)
     const fileName = `${page.path}.${getFileExtension(page.contentType)}`
@@ -135,6 +258,11 @@ module.exports = {
       '--author': `"${page.authorName} <${page.authorEmail}>"`
     })
   },
+  /**
+   * RENAME
+   *
+   * @param {Object} page Page to rename
+   */
   async renamed(page) {
     WIKI.logger.info(`(STORAGE/GIT) Committing file move from ${page.sourcePath} to ${page.destinationPath}...`)
     const sourceFilePath = `${page.sourcePath}.${getFileExtension(page.contentType)}`