feat: AWS S3 + Digitalocean Spaces storage modules (#1015)

* Provide basic implementation of AWS S3 storage module

* Abstract S3 Compatible Storage Module logic

* Refactor `getFileExtension()` into the `page` object

* Add implementation for Digitalocean storage module

* Remove accidental `async`/`await` in S3 Storage Module

* Remove argument from the call to `page.getFileExtension()`

https://github.com/Requarks/wiki/pull/1015#discussion_r321990073

server/models/pages.js

@@ -147,6 +147,20 @@ module.exports = class Page extends Model {
     return pageHelper.injectPageMetadata(this)
   }
 
+  /**
+   * Get the page's file extension based on content type
+   */
+  getFileExtension() {
+    switch (this.contentType) {
+      case 'markdown':
+        return 'md'
+      case 'html':
+        return 'html'
+      default:
+        return 'txt'
+    }
+  }
+
   /**
    * Parse injected page metadata from raw content
    *

server/modules/storage/digitalocean/definition.yml

@@ -1,28 +1,28 @@
 key: digitalocean
 title: DigitalOcean Spaces
 description: DigitalOcean provides developers and businesses a reliable, easy-to-use cloud computing platform of virtual servers (Droplets), object storage (Spaces) and more.
-author: requarks.io
+author: andrewsim
 logo: https://static.requarks.io/logo/digitalocean.svg
 website: https://www.digitalocean.com/products/spaces/
-isAvailable: false
+isAvailable: true
 supportedModes:
   - push
 defaultMode: push
 schedule: false
 props:
-  region:
+  endpoint:
     type: String
-    title: Region
+    title: Endpoint
-    hint: The DigitalOcean datacenter region where the Space will be created.
+    hint: The DigitalOcean spaces endpoint that has the form ${REGION}.digitaloceanspaces.com
-    default: nyc3
+    default: nyc3.digitaloceanspaces.com
     enum:
-      - ams3
+      - ams3.digitaloceanspaces.com
-      - fra1
+      - fra1.digitaloceanspaces.com
-      - nyc3
+      - nyc3.digitaloceanspaces.com
-      - sfo2
+      - sfo2.digitaloceanspaces.com
-      - sgp1
+      - sgp1.digitaloceanspaces.com
     order: 1
-  spaceId:
+  bucket:
     type: String
     title: Space Unique Name
     hint: The unique space name to create (e.g. wiki-johndoe)

server/modules/storage/digitalocean/storage.js

@@ -1,23 +1,3 @@
-module.exports = {
+const S3CompatibleStorage = require('../s3/common')
-  async activated() {
 
-  },
+module.exports = new S3CompatibleStorage('Digitalocean')
-  async deactivated() {
-
-  },
-  async init() {
-
-  },
-  async created() {
-
-  },
-  async updated() {
-
-  },
-  async deleted() {
-
-  },
-  async renamed() {
-
-  }
-}

server/modules/storage/disk/storage.js

@@ -10,20 +10,6 @@ const moment = require('moment')
 
 /* global WIKI */
 
-/**
- * Get file extension based on content type
- */
-const getFileExtension = (contentType) => {
-  switch (contentType) {
-    case 'markdown':
-      return 'md'
-    case 'html':
-      return 'html'
-    default:
-      return 'txt'
-  }
-}
-
 module.exports = {
   async activated() {
     // not used
@@ -58,7 +44,7 @@ module.exports = {
   },
   async created(page) {
     WIKI.logger.info(`(STORAGE/DISK) Creating file ${page.path}...`)
-    let fileName = `${page.path}.${getFileExtension(page.contentType)}`
+    let fileName = `${page.path}.${page.getFileExtension()}`
     if (WIKI.config.lang.code !== page.localeCode) {
       fileName = `${page.localeCode}/${fileName}`
     }
@@ -67,7 +53,7 @@ module.exports = {
   },
   async updated(page) {
     WIKI.logger.info(`(STORAGE/DISK) Updating file ${page.path}...`)
-    let fileName = `${page.path}.${getFileExtension(page.contentType)}`
+    let fileName = `${page.path}.${page.getFileExtension()}`
     if (WIKI.config.lang.code !== page.localeCode) {
       fileName = `${page.localeCode}/${fileName}`
     }
@@ -76,7 +62,7 @@ module.exports = {
   },
   async deleted(page) {
     WIKI.logger.info(`(STORAGE/DISK) Deleting file ${page.path}...`)
-    let fileName = `${page.path}.${getFileExtension(page.contentType)}`
+    let fileName = `${page.path}.${page.getFileExtension()}`
     if (WIKI.config.lang.code !== page.localeCode) {
       fileName = `${page.localeCode}/${fileName}`
     }
@@ -85,8 +71,8 @@ module.exports = {
   },
   async renamed(page) {
     WIKI.logger.info(`(STORAGE/DISK) Renaming file ${page.sourcePath} to ${page.destinationPath}...`)
-    let sourceFilePath = `${page.sourcePath}.${getFileExtension(page.contentType)}`
+    let sourceFilePath = `${page.sourcePath}.${page.getFileExtension()}`
-    let destinationFilePath = `${page.destinationPath}.${getFileExtension(page.contentType)}`
+    let destinationFilePath = `${page.destinationPath}.${page.getFileExtension()}`
 
     if (WIKI.config.lang.code !== page.localeCode) {
       sourceFilePath = `${page.localeCode}/${sourceFilePath}`
@@ -107,7 +93,7 @@ module.exports = {
       new stream.Transform({
         objectMode: true,
         transform: async (page, enc, cb) => {
-          let fileName = `${page.path}.${getFileExtension(page.contentType)}`
+          let fileName = `${page.path}.${page.getFileExtension()}`
           if (WIKI.config.lang.code !== page.localeCode) {
             fileName = `${page.localeCode}/${fileName}`
           }

server/modules/storage/git/storage.js

@@ -12,20 +12,6 @@ const localeFolderRegex = /^([a-z]{2}(?:-[a-z]{2})?\/)?(.*)/i
 
 /* global WIKI */
 
-/**
- * Get file extension based on content type
- */
-const getFileExtension = (contentType) => {
-  switch (contentType) {
-    case 'markdown':
-      return 'md'
-    case 'html':
-      return 'html'
-    default:
-      return 'txt'
-  }
-}
-
 const getContenType = (filePath) => {
   const ext = _.last(filePath.split('.'))
   switch (ext) {
@@ -256,7 +242,7 @@ module.exports = {
    */
   async created(page) {
     WIKI.logger.info(`(STORAGE/GIT) Committing new file ${page.path}...`)
-    let fileName = `${page.path}.${getFileExtension(page.contentType)}`
+    let fileName = `${page.path}.${page.getFileExtension()}`
     if (WIKI.config.lang.namespacing && WIKI.config.lang.code !== page.localeCode) {
       fileName = `${page.localeCode}/${fileName}`
     }
@@ -275,7 +261,7 @@ module.exports = {
    */
   async updated(page) {
     WIKI.logger.info(`(STORAGE/GIT) Committing updated file ${page.path}...`)
-    let fileName = `${page.path}.${getFileExtension(page.contentType)}`
+    let fileName = `${page.path}.${page.getFileExtension()}`
     if (WIKI.config.lang.namespacing && WIKI.config.lang.code !== page.localeCode) {
       fileName = `${page.localeCode}/${fileName}`
     }
@@ -294,7 +280,7 @@ module.exports = {
    */
   async deleted(page) {
     WIKI.logger.info(`(STORAGE/GIT) Committing removed file ${page.path}...`)
-    let fileName = `${page.path}.${getFileExtension(page.contentType)}`
+    let fileName = `${page.path}.${page.getFileExtension()}`
     if (WIKI.config.lang.namespacing && WIKI.config.lang.code !== page.localeCode) {
       fileName = `${page.localeCode}/${fileName}`
     }
@@ -311,8 +297,8 @@ module.exports = {
    */
   async renamed(page) {
     WIKI.logger.info(`(STORAGE/GIT) Committing file move from ${page.sourcePath} to ${page.destinationPath}...`)
-    let sourceFilePath = `${page.sourcePath}.${getFileExtension(page.contentType)}`
+    let sourceFilePath = `${page.sourcePath}.${page.getFileExtension()}`
-    let destinationFilePath = `${page.destinationPath}.${getFileExtension(page.contentType)}`
+    let destinationFilePath = `${page.destinationPath}.${page.getFileExtension()}`
 
     if (WIKI.config.lang.namespacing && WIKI.config.lang.code !== page.localeCode) {
       sourceFilePath = `${page.localeCode}/${sourceFilePath}`
@@ -363,7 +349,7 @@ module.exports = {
       new stream.Transform({
         objectMode: true,
         transform: async (page, enc, cb) => {
-          let fileName = `${page.path}.${getFileExtension(page.contentType)}`
+          let fileName = `${page.path}.${page.getFileExtension()}`
           if (WIKI.config.lang.namespacing && WIKI.config.lang.code !== page.localeCode) {
             fileName = `${page.localeCode}/${fileName}`
           }

server/modules/storage/s3/common.js

@@ -0,0 +1,64 @@
+const S3 = require('aws-sdk/clients/s3')
+
+/* global WIKI */
+
+/**
+ * Deduce the file path given the `page` object and the object's key to the page's path.
+ */
+const getFilePath = (page, pathKey) => {
+  const fileName = `${page[pathKey]}.${page.getFileExtension()}`
+  const withLocaleCode = WIKI.config.lang.namespacing && WIKI.config.lang.code !== page.localeCode
+  return withLocaleCode ? `${page.localeCode}/${fileName}` : fileName
+}
+
+/**
+ * Can be used with S3 compatible storage.
+ */
+module.exports = class S3CompatibleStorage {
+  constructor(storageName) {
+    this.storageName = storageName
+  }
+  async activated() {
+    // not used
+  }
+  async deactivated() {
+    // not used
+  }
+  async init() {
+    WIKI.logger.info(`(STORAGE/${this.storageName}) Initializing...`)
+    const { accessKeyId, secretAccessKey, region, bucket, endpoint } = this.config
+    this.s3 = new S3({
+      accessKeyId,
+      secretAccessKey,
+      region,
+      endpoint,
+      params: { Bucket: bucket },
+      apiVersions: '2006-03-01'
+    })
+    // determine if a bucket exists and you have permission to access it
+    await this.s3.headBucket().promise()
+    WIKI.logger.info(`(STORAGE/${this.storageName}) Initialization completed.`)
+  }
+  async created(page) {
+    WIKI.logger.info(`(STORAGE/${this.storageName}) Creating file ${page.path}...`)
+    const filePath = getFilePath(page, 'path')
+    await this.s3.putObject({ Key: filePath, Body: page.injectMetadata() }).promise()
+  }
+  async updated(page) {
+    WIKI.logger.info(`(STORAGE/${this.storageName}) Updating file ${page.path}...`)
+    const filePath = getFilePath(page, 'path')
+    await this.s3.putObject({ Key: filePath, Body: page.injectMetadata() }).promise()
+  }
+  async deleted(page) {
+    WIKI.logger.info(`(STORAGE/${this.storageName}) Deleting file ${page.path}...`)
+    const filePath = getFilePath(page, 'path')
+    await this.s3.deleteObject({ Key: filePath }).promise()
+  }
+  async renamed(page) {
+    WIKI.logger.info(`(STORAGE/${this.storageName}) Renaming file ${page.sourcePath} to ${page.destinationPath}...`)
+    const sourceFilePath = getFilePath(page, 'sourcePath')
+    const destinationFilePath = getFilePath(page, 'destinationPath')
+    await this.s3.copyObject({ CopySource: sourceFilePath, Key: destinationFilePath }).promise()
+    await this.s3.deleteObject({ Key: sourceFilePath }).promise()
+  }
+}

server/modules/storage/s3/definition.yml

@@ -1,11 +1,32 @@
 key: s3
 title: Amazon S3
 description: Amazon S3 is a cloud computing web service offered by Amazon Web Services which provides object storage.
-author: requarks.io
+author: andrewsim
 logo: https://static.requarks.io/logo/aws-s3.svg
 website: https://aws.amazon.com/s3/
+isAvailable: true
+supportedModes:
+  - push
+defaultMode: push
+schedule: false
 props:
-  accessKeyId: String
+  region:
-  accessSecret: String
+    type: String
-  region: String
+    title: Region
-  bucket: String
+    hint: The AWS datacenter region where the bucket will be created.
+    order: 1
+  bucket:
+    type: String
+    title: Unique bucket name
+    hint: The unique bucket name to create (e.g. wiki-johndoe).
+    order: 2
+  accessKeyId:
+    type: String
+    title: Access Key ID
+    hint: The Access Key.
+    order: 3
+  secretAccessKey:
+    type: String
+    title: Secret Access Key
+    hint: The Secret Access Key for the Access Key ID you created above.
+    order: 4

server/modules/storage/s3/storage.js

@@ -1,23 +1,3 @@
-module.exports = {
+const S3CompatibleStorage = require('./common')
-  async activated() {
 
-  },
+module.exports = new S3CompatibleStorage('S3')
-  async deactivated() {
-
-  },
-  async init() {
-
-  },
-  async created() {
-
-  },
-  async updated() {
-
-  },
-  async deleted() {
-
-  },
-  async renamed() {
-
-  }
-}