feat: git changes processing

CHANGELOG.md

@@ -2,6 +2,19 @@
 All notable changes to this project will be documented in this file.
 This project adheres to [Semantic Versioning](http://semver.org/).
 
+## [2.0.0-beta.XX] - 2018-XX-XX
+### Added
+- Added Git changes processing (add/modify/delete)
+- Added Storage last sync date in status panel
+- Added Dev Flags
+- Added HTTP to HTTPS redirect server option
+
+### Fixed
+- Fixed SQLite migrations
+
+### Changed
+- Split admin dev section into separate pages
+
 ## [2.0.0-beta.42] - 2018-02-17
 ### Added
 - Added Patreon link in Contribute admin page

client/components/admin/admin-storage.vue

@@ -54,10 +54,18 @@
                         :color='getStatusColor(tgt.status)'
                         dark
                         flat
-                        :extended='tgt.status === `error`',
-                        extension-height='100'
+                        :extended='tgt.status !== `pending`',
+                        :extension-height='tgt.status === `error` ? 100 : 70'
                         )
                         .pa-3.red.darken-2.radius-7(v-if='tgt.status === `error`', slot='extension') {{tgt.message}}
+                        v-toolbar.radius-7(
+                          color='green darken-2'
+                          v-else-if='tgt.status !== `pending`'
+                          slot='extension'
+                          flat
+                          dense
+                          )
+                          span Last synchronization {{tgt.lastAttempt | moment('from') }}
                         .body-2 {{tgt.title}}
                         v-spacer
                         .body-1 {{tgt.status}}

client/graph/admin/storage/storage-query-status.gql

@@ -5,6 +5,7 @@ query {
       title
       status
       message
+      lastAttempt
     }
   }
 }

server/db/migrations-sqlite/2.0.0-beta.11.js

@@ -16,11 +16,12 @@ exports.up = knex => {
       table.string('createdAt').notNullable()
       table.string('action').defaultTo('updated')
 
+      table.integer('pageId').unsigned()
       table.string('editorKey').references('key').inTable('editors')
       table.string('localeCode', 2).references('code').inTable('locales')
       table.integer('authorId').unsigned().references('id').inTable('users')
     })
-    .raw(`INSERT INTO pageHistory SELECT id,path,hash,title,description,isPrivate,isPublished,publishStartDate,publishEndDate,content,contentType,createdAt,'updated' AS action,editorKey,localeCode,authorId FROM pageHistory_old;`)
+    .raw(`INSERT INTO pageHistory SELECT id,path,hash,title,description,isPrivate,isPublished,publishStartDate,publishEndDate,content,contentType,createdAt,'updated' AS action,pageId,editorKey,localeCode,authorId FROM pageHistory_old;`)
     .dropTable('pageHistory_old')
 }
 

server/graph/resolvers/storage.js

@@ -45,7 +45,8 @@ module.exports = {
           key: tgt.key,
           title: targetInfo.title,
           status: _.get(tgt, 'state.status', 'pending'),
-          message: _.get(tgt, 'state.message', 'Initializing...')
+          message: _.get(tgt, 'state.message', 'Initializing...'),
+          lastAttempt: _.get(tgt, 'state.lastAttempt', null)
         }
       })
     }
@@ -64,7 +65,8 @@ module.exports = {
             }, {}),
             state: {
               status: 'pending',
-              message: 'Initializing...'
+              message: 'Initializing...',
+              lastAttempt: null
             }
           }).where('key', tgt.key)
         }

server/graph/schemas/storage.graphql

@@ -65,5 +65,6 @@ type StorageStatus {
   key: String!
   title: String!
   status: String!
-  message: String
+  message: String!
+  lastAttempt: String!
 }

server/jobs/sync-storage.js

@@ -10,11 +10,26 @@ module.exports = async (targetKey) => {
     if (target) {
       await target.fn.sync()
       WIKI.logger.info(`Syncing with storage target ${targetKey}: [ COMPLETED ]`)
+
+      await WIKI.models.storage.query().patch({
+        state: {
+          status: 'operational',
+          message: '',
+          lastAttempt: new Date().toISOString()
+        }
+      }).where('key', targetKey)
     } else {
       throw new Error('Invalid storage target. Unable to perform sync.')
     }
   } catch (err) {
     WIKI.logger.error(`Syncing with storage target ${targetKey}: [ FAILED ]`)
     WIKI.logger.error(err.message)
+    await WIKI.models.storage.query().patch({
+      state: {
+        status: 'error',
+        message: err.message,
+        lastAttempt: new Date().toISOString()
+      }
+    }).where('key', targetKey)
   }
 }

server/models/editors.js

@@ -90,4 +90,14 @@ module.exports = class Editor extends Model {
       WIKI.logger.error(err)
     }
   }
+
+  static async getDefaultEditor(contentType) {
+    // TODO - hardcoded for now
+    switch (contentType) {
+      case 'markdown':
+        return 'markdown'
+      default:
+        return 'code'
+    }
+  }
 }

server/models/pageHistory.js

@@ -93,8 +93,8 @@ module.exports = class PageHistory extends Model {
       description: opts.description,
       editorKey: opts.editorKey,
       hash: opts.hash,
-      isPrivate: opts.isPrivate,
-      isPublished: opts.isPublished,
+      isPrivate: (opts.isPrivate === true || opts.isPrivate === 1),
+      isPublished: (opts.isPublished === true || opts.isPublished === 1),
       localeCode: opts.localeCode,
       path: opts.path,
       publishEndDate: opts.publishEndDate || '',

server/models/pages.js

@@ -4,9 +4,16 @@ const JSBinType = require('js-binary').Type
 const pageHelper = require('../helpers/page')
 const path = require('path')
 const fs = require('fs-extra')
+const yaml = require('js-yaml')
 
 /* global WIKI */
 
+const frontmatterRegex = {
+  html: /^(<!-{2}(?:\n|\r)([\w\W]+?)(?:\n|\r)-{2}>)?(?:\n|\r)*([\w\W]*)*/,
+  legacy: /^(<!-- TITLE: ?([\w\W]+?) -{2}>)?(?:\n|\r)?(<!-- SUBTITLE: ?([\w\W]+?) -{2}>)?(?:\n|\r)*([\w\W]*)*/i,
+  markdown: /^(-{3}(?:\n|\r)([\w\W]+?)(?:\n|\r)-{3})?(?:\n|\r)*([\w\W]*)*/
+}
+
 /**
  * Pages model
  */
@@ -135,6 +142,49 @@ module.exports = class Page extends Model {
     }
   }
 
+  /**
+   * Parse injected page metadata from raw content
+   *
+   * @param {String} raw Raw file contents
+   * @param {String} contentType Content Type
+   */
+  static parseMetadata (raw, contentType) {
+    let result
+    switch (contentType) {
+      case 'markdown':
+        result = frontmatterRegex.markdown.exec(raw)
+        if (result[2]) {
+          return {
+            ...yaml.safeLoad(result[2]),
+            content: result[3]
+          }
+        } else {
+          // Attempt legacy v1 format
+          result = frontmatterRegex.legacy.exec(raw)
+          if (result[2]) {
+            return {
+              title: result[2],
+              description: result[4],
+              content: result[5]
+            }
+          }
+        }
+        break
+      case 'html':
+        result = frontmatterRegex.html.exec(raw)
+        if (result[2]) {
+          return {
+            ...yaml.safeLoad(result[2]),
+            content: result[3]
+          }
+        }
+        break
+    }
+    return {
+      content: raw
+    }
+  }
+
   static async createPage(opts) {
     await WIKI.models.pages.query().insert({
       authorId: opts.authorId,
@@ -160,10 +210,12 @@ module.exports = class Page extends Model {
       isPrivate: opts.isPrivate
     })
     await WIKI.models.pages.renderPage(page)
+    if (!opts.skipStorage) {
       await WIKI.models.storage.pageEvent({
         event: 'created',
         page
       })
+    }
     return page
   }
 
@@ -181,7 +233,7 @@ module.exports = class Page extends Model {
       authorId: opts.authorId,
       content: opts.content,
       description: opts.description,
-      isPublished: opts.isPublished,
+      isPublished: opts.isPublished === true || opts.isPublished === 1,
       publishEndDate: opts.publishEndDate || '',
       publishStartDate: opts.publishStartDate || '',
       title: opts.title
@@ -193,15 +245,25 @@ module.exports = class Page extends Model {
       isPrivate: ogPage.isPrivate
     })
     await WIKI.models.pages.renderPage(page)
+    if (!opts.skipStorage) {
       await WIKI.models.storage.pageEvent({
         event: 'updated',
         page
       })
+    }
     return page
   }
 
   static async deletePage(opts) {
-    const page = await WIKI.models.pages.query().findById(opts.id)
+    let page
+    if (_.has(opts, 'id')) {
+      page = await WIKI.models.pages.query().findById(opts.id)
+    } else {
+      page = await await WIKI.models.pages.query().findOne({
+        path: opts.path,
+        localeCode: opts.locale
+      })
+    }
     if (!page) {
       throw new Error('Invalid Page Id')
     }
@@ -211,11 +273,13 @@ module.exports = class Page extends Model {
     })
     await WIKI.models.pages.query().delete().where('id', page.id)
     await WIKI.models.pages.deletePageFromCache(page)
+    if (!opts.skipStorage) {
       await WIKI.models.storage.pageEvent({
         event: 'deleted',
         page
       })
     }
+  }
 
   static async renderPage(page) {
     const renderJob = await WIKI.scheduler.registerJob({

server/models/storage.js

@@ -68,7 +68,8 @@ module.exports = class Storage extends Model {
             }, {}),
             state: {
               status: 'pending',
-              message: ''
+              message: '',
+              lastAttempt: null
             }
           })
         } else {
@@ -127,7 +128,8 @@ module.exports = class Storage extends Model {
           await WIKI.models.storage.query().patch({
             state: {
               status: 'operational',
-              message: ''
+              message: '',
+              lastAttempt: new Date().toISOString()
             }
           }).where('key', target.key)
 
@@ -145,7 +147,8 @@ module.exports = class Storage extends Model {
           await WIKI.models.storage.query().patch({
             state: {
               status: 'error',
-              message: err.message
+              message: err.message,
+              lastAttempt: new Date().toISOString()
             }
           }).where('key', target.key)
         }

server/modules/storage/git/definition.yml

@@ -69,3 +69,7 @@ props:
     default: 'John Smith'
     hint: 'Used as fallback in case the author of the change is not present.'
     order: 21
+actions:
+  - handler: syncUntracked
+    label: Add Untracked Changes
+    hint: Output all content from the DB to the Git repo to ensure all untracked content is saved. If you enabled Git after content was created or you temporarily disabled Git, you'll want to execute this action to add the missing untracked changes.

server/modules/storage/git/storage.js

@@ -3,6 +3,8 @@ const sgit = require('simple-git/promise')
 const fs = require('fs-extra')
 const _ = require('lodash')
 
+const localeFolderRegex = /^([a-z]{2}(?:-[a-z]{2})?\/)?(.*)/i
+
 /**
  * Get file extension based on content type
  */
@@ -17,6 +19,33 @@ const getFileExtension = (contentType) => {
   }
 }
 
+const getContenType = (filePath) => {
+  const ext = _.last(filePath.split('.'))
+  switch (ext) {
+    case 'md':
+      return 'markdown'
+    case 'html':
+      return 'html'
+    default:
+      return false
+  }
+}
+
+const getPagePath = (filePath) => {
+  let meta = {
+    locale: 'en',
+    path: _.initial(filePath.split('.')).join('')
+  }
+  const result = localeFolderRegex.exec(meta.path)
+  if (result[1]) {
+    meta = {
+      locale: result[1],
+      path: result[2]
+    }
+  }
+  return meta
+}
+
 module.exports = {
   git: null,
   repoPath: path.join(process.cwd(), 'data/repo'),
@@ -26,6 +55,9 @@ module.exports = {
   async deactivated() {
     // not used
   },
+  /**
+   * INIT
+   */
   async init() {
     WIKI.logger.info('(STORAGE/GIT) Initializing...')
     this.repoPath = path.resolve(WIKI.ROOTPATH, this.config.localRepoPath)
@@ -87,7 +119,12 @@ module.exports = {
 
     WIKI.logger.info('(STORAGE/GIT) Initialization completed.')
   },
+  /**
+   * SYNC
+   */
   async sync() {
+    const currentCommitLog = _.get(await this.git.log(['-n', '1', this.config.branch]), 'latest', {})
+
     // Pull rebase
     if (_.includes(['sync', 'pull'], this.mode)) {
       WIKI.logger.info(`(STORAGE/GIT) Performing pull rebase from origin on branch ${this.config.branch}...`)
@@ -103,7 +140,83 @@ module.exports = {
       }
       await this.git.push('origin', this.config.branch, pushOpts)
     }
+
+    // Process Changes
+    if (_.includes(['sync', 'pull'], this.mode)) {
+      const latestCommitLog = _.get(await this.git.log(['-n', '1', this.config.branch]), 'latest', {})
+
+      const diff = await this.git.diffSummary([currentCommitLog.hash, latestCommitLog.hash])
+      if (_.get(diff, 'files', []).length > 0) {
+        for(const item of diff.files) {
+          const contentType = getContenType(item.file)
+          if (!contentType) {
+            continue
+          }
+          const contentPath = getPagePath(item.file)
+
+          let itemContents = ''
+          try {
+            itemContents = await fs.readFile(path.join(this.repoPath, item.file), 'utf8')
+            const pageData = WIKI.models.pages.parseMetadata(itemContents, contentType)
+            const currentPage = await WIKI.models.pages.query().findOne({
+              path: contentPath.path,
+              localeCode: contentPath.locale
+            })
+            if (currentPage) {
+              // Already in the DB, can mark as modified
+              WIKI.logger.info(`(STORAGE/GIT) Page marked as modified: ${item.file}`)
+              await WIKI.models.pages.updatePage({
+                id: currentPage.id,
+                title: _.get(pageData, 'title', currentPage.title),
+                description: _.get(pageData, 'description', currentPage.description),
+                isPublished: _.get(pageData, 'isPublished', currentPage.isPublished),
+                isPrivate: false,
+                content: pageData.content,
+                authorId: 1,
+                skipStorage: true
+              })
+            } else {
+              // Not in the DB, can mark as new
+              WIKI.logger.info(`(STORAGE/GIT) Page marked as new: ${item.file}`)
+              const pageEditor = await WIKI.models.editors.getDefaultEditor(contentType)
+              await WIKI.models.pages.createPage({
+                path: contentPath.path,
+                locale: contentPath.locale,
+                title: _.get(pageData, 'title', _.last(contentPath.path.split('/'))),
+                description: _.get(pageData, 'description', ''),
+                isPublished: _.get(pageData, 'isPublished', true),
+                isPrivate: false,
+                content: pageData.content,
+                authorId: 1,
+                editor: pageEditor,
+                skipStorage: true
+              })
+            }
+          } catch (err) {
+            if (err.code === 'ENOENT' && item.deletions > 0 && item.insertions === 0) {
+              // File was deleted by git, can safely mark as deleted in DB
+              WIKI.logger.info(`(STORAGE/GIT) Page marked as deleted: ${item.file}`)
+
+              await WIKI.models.pages.deletePage({
+                path: contentPath.path,
+                locale: contentPath.locale,
+                skipStorage: true
+              })
+
+            } else {
+              WIKI.logger.warn(`(STORAGE/GIT) Failed to open ${item.file}`)
+              WIKI.logger.warn(err)
+            }
+          }
+        }
+      }
+    }
   },
+  /**
+   * CREATE
+   *
+   * @param {Object} page Page to create
+   */
   async created(page) {
     WIKI.logger.info(`(STORAGE/GIT) Committing new file ${page.path}...`)
     const fileName = `${page.path}.${getFileExtension(page.contentType)}`
@@ -115,6 +228,11 @@ module.exports = {
       '--author': `"${page.authorName} <${page.authorEmail}>"`
     })
   },
+  /**
+   * UPDATE
+   *
+   * @param {Object} page Page to update
+   */
   async updated(page) {
     WIKI.logger.info(`(STORAGE/GIT) Committing updated file ${page.path}...`)
     const fileName = `${page.path}.${getFileExtension(page.contentType)}`
@@ -126,6 +244,11 @@ module.exports = {
       '--author': `"${page.authorName} <${page.authorEmail}>"`
     })
   },
+  /**
+   * DELETE
+   *
+   * @param {Object} page Page to delete
+   */
   async deleted(page) {
     WIKI.logger.info(`(STORAGE/GIT) Committing removed file ${page.path}...`)
     const fileName = `${page.path}.${getFileExtension(page.contentType)}`
@@ -135,6 +258,11 @@ module.exports = {
       '--author': `"${page.authorName} <${page.authorEmail}>"`
     })
   },
+  /**
+   * RENAME
+   *
+   * @param {Object} page Page to rename
+   */
   async renamed(page) {
     WIKI.logger.info(`(STORAGE/GIT) Committing file move from ${page.sourcePath} to ${page.destinationPath}...`)
     const sourceFilePath = `${page.sourcePath}.${getFileExtension(page.contentType)}`