liz/wikijs-fork
liz/wikijs-fork
1
0
Fork 0
Code Issues Pull Requests Packages Projects Releases Wiki Activity

feat: AWS S3 + Digitalocean Spaces storage modules (#1015)

Browse Source 
* Provide basic implementation of AWS S3 storage module

* Abstract S3 Compatible Storage Module logic

* Refactor `getFileExtension()` into the `page` object

* Add implementation for Digitalocean storage module

* Remove accidental `async`/`await` in S3 Storage Module

* Remove argument from the call to `page.getFileExtension()`

https://github.com/Requarks/wiki/pull/1015#discussion_r321990073
This commit is contained in:
Andrew Sim 2019-09-09 02:11:25 +01:00 committed by Nicolas Giard
parent 3ab7bcf8ea
commit 5202eadebb
8 changed files with 132 additions and 101 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

14
server/models/pages.js
View File

@ -147,6 +147,20 @@ module.exports = class Page extends Model {
return pageHelper.injectPageMetadata(this)
}
/**
* Get the page's file extension based on content type
*/
getFileExtension() {
switch (this.contentType) {
case 'markdown':
return 'md'
case 'html':
return 'html'
default:
return 'txt'
}
}
/**
* Parse injected page metadata from raw content
*

24
server/modules/storage/digitalocean/definition.yml
View File

@ -1,28 +1,28 @@
key: digitalocean
title: DigitalOcean Spaces
description: DigitalOcean provides developers and businesses a reliable, easy-to-use cloud computing platform of virtual servers (Droplets), object storage (Spaces) and more.
author: requarks.io
author: andrewsim
logo: https://static.requarks.io/logo/digitalocean.svg
website: https://www.digitalocean.com/products/spaces/
isAvailable: false
isAvailable: true
supportedModes:
- push
defaultMode: push
schedule: false
props:
region:
endpoint:
type: String
title: Region
hint: The DigitalOcean datacenter region where the Space will be created.
default: nyc3
title: Endpoint
hint: The DigitalOcean spaces endpoint that has the form ${REGION}.digitaloceanspaces.com
default: nyc3.digitaloceanspaces.com
enum:
- ams3
- fra1
- nyc3
- sfo2
- sgp1
- ams3.digitaloceanspaces.com
- fra1.digitaloceanspaces.com
- nyc3.digitaloceanspaces.com
- sfo2.digitaloceanspaces.com
- sgp1.digitaloceanspaces.com
order: 1
spaceId:
bucket:
type: String
title: Space Unique Name
hint: The unique space name to create (e.g. wiki-johndoe)

24
server/modules/storage/digitalocean/storage.js
View File

@ -1,23 +1,3 @@
module.exports = {
async activated() {
const S3CompatibleStorage = require('../s3/common')
},
async deactivated() {
},
async init() {
},
async created() {
},
async updated() {
},
async deleted() {
},
async renamed() {
}
}
module.exports = new S3CompatibleStorage('Digitalocean')

26
server/modules/storage/disk/storage.js
View File

@ -10,20 +10,6 @@ const moment = require('moment')
/* global WIKI */
/**
* Get file extension based on content type
*/
const getFileExtension = (contentType) => {
switch (contentType) {
case 'markdown':
return 'md'
case 'html':
return 'html'
default:
return 'txt'
}
}
module.exports = {
async activated() {
// not used
@ -58,7 +44,7 @@ module.exports = {
},
async created(page) {
WIKI.logger.info(`(STORAGE/DISK) Creating file ${page.path}...`)
let fileName = `${page.path}.${getFileExtension(page.contentType)}`
let fileName = `${page.path}.${page.getFileExtension()}`
if (WIKI.config.lang.code !== page.localeCode) {
fileName = `${page.localeCode}/${fileName}`
}
@ -67,7 +53,7 @@ module.exports = {
},
async updated(page) {
WIKI.logger.info(`(STORAGE/DISK) Updating file ${page.path}...`)
let fileName = `${page.path}.${getFileExtension(page.contentType)}`
let fileName = `${page.path}.${page.getFileExtension()}`
if (WIKI.config.lang.code !== page.localeCode) {
fileName = `${page.localeCode}/${fileName}`
}
@ -76,7 +62,7 @@ module.exports = {
},
async deleted(page) {
WIKI.logger.info(`(STORAGE/DISK) Deleting file ${page.path}...`)
let fileName = `${page.path}.${getFileExtension(page.contentType)}`
let fileName = `${page.path}.${page.getFileExtension()}`
if (WIKI.config.lang.code !== page.localeCode) {
fileName = `${page.localeCode}/${fileName}`
}
@ -85,8 +71,8 @@ module.exports = {
},
async renamed(page) {
WIKI.logger.info(`(STORAGE/DISK) Renaming file ${page.sourcePath} to ${page.destinationPath}...`)
let sourceFilePath = `${page.sourcePath}.${getFileExtension(page.contentType)}`
let destinationFilePath = `${page.destinationPath}.${getFileExtension(page.contentType)}`
let sourceFilePath = `${page.sourcePath}.${page.getFileExtension()}`
let destinationFilePath = `${page.destinationPath}.${page.getFileExtension()}`
if (WIKI.config.lang.code !== page.localeCode) {
sourceFilePath = `${page.localeCode}/${sourceFilePath}`
@ -107,7 +93,7 @@ module.exports = {
new stream.Transform({
objectMode: true,
transform: async (page, enc, cb) => {
let fileName = `${page.path}.${getFileExtension(page.contentType)}`
let fileName = `${page.path}.${page.getFileExtension()}`
if (WIKI.config.lang.code !== page.localeCode) {
fileName = `${page.localeCode}/${fileName}`
}

26
server/modules/storage/git/storage.js
View File

@ -12,20 +12,6 @@ const localeFolderRegex = /^([a-z]{2}(?:-[a-z]{2})?\/)?(.*)/i
/* global WIKI */
/**
* Get file extension based on content type
*/
const getFileExtension = (contentType) => {
switch (contentType) {
case 'markdown':
return 'md'
case 'html':
return 'html'
default:
return 'txt'
}
}
const getContenType = (filePath) => {
const ext = _.last(filePath.split('.'))
switch (ext) {
@ -256,7 +242,7 @@ module.exports = {
*/
async created(page) {
WIKI.logger.info(`(STORAGE/GIT) Committing new file ${page.path}...`)
let fileName = `${page.path}.${getFileExtension(page.contentType)}`
let fileName = `${page.path}.${page.getFileExtension()}`
if (WIKI.config.lang.namespacing && WIKI.config.lang.code !== page.localeCode) {
fileName = `${page.localeCode}/${fileName}`
}
@ -275,7 +261,7 @@ module.exports = {
*/
async updated(page) {
WIKI.logger.info(`(STORAGE/GIT) Committing updated file ${page.path}...`)
let fileName = `${page.path}.${getFileExtension(page.contentType)}`
let fileName = `${page.path}.${page.getFileExtension()}`
if (WIKI.config.lang.namespacing && WIKI.config.lang.code !== page.localeCode) {
fileName = `${page.localeCode}/${fileName}`
}
@ -294,7 +280,7 @@ module.exports = {
*/
async deleted(page) {
WIKI.logger.info(`(STORAGE/GIT) Committing removed file ${page.path}...`)
let fileName = `${page.path}.${getFileExtension(page.contentType)}`
let fileName = `${page.path}.${page.getFileExtension()}`
if (WIKI.config.lang.namespacing && WIKI.config.lang.code !== page.localeCode) {
fileName = `${page.localeCode}/${fileName}`
}
@ -311,8 +297,8 @@ module.exports = {
*/
async renamed(page) {
WIKI.logger.info(`(STORAGE/GIT) Committing file move from ${page.sourcePath} to ${page.destinationPath}...`)
let sourceFilePath = `${page.sourcePath}.${getFileExtension(page.contentType)}`
let destinationFilePath = `${page.destinationPath}.${getFileExtension(page.contentType)}`
let sourceFilePath = `${page.sourcePath}.${page.getFileExtension()}`
let destinationFilePath = `${page.destinationPath}.${page.getFileExtension()}`
if (WIKI.config.lang.namespacing && WIKI.config.lang.code !== page.localeCode) {
sourceFilePath = `${page.localeCode}/${sourceFilePath}`
@ -363,7 +349,7 @@ module.exports = {
new stream.Transform({
objectMode: true,
transform: async (page, enc, cb) => {
let fileName = `${page.path}.${getFileExtension(page.contentType)}`
let fileName = `${page.path}.${page.getFileExtension()}`
if (WIKI.config.lang.namespacing && WIKI.config.lang.code !== page.localeCode) {
fileName = `${page.localeCode}/${fileName}`
}

64
server/modules/storage/s3/common.js Normal file
View File

@ -0,0 +1,64 @@
const S3 = require('aws-sdk/clients/s3')
/* global WIKI */
/**
* Deduce the file path given the `page` object and the object's key to the page's path.
*/
const getFilePath = (page, pathKey) => {
const fileName = `${page[pathKey]}.${page.getFileExtension()}`
const withLocaleCode = WIKI.config.lang.namespacing && WIKI.config.lang.code !== page.localeCode
return withLocaleCode ? `${page.localeCode}/${fileName}` : fileName
}
/**
* Can be used with S3 compatible storage.
*/
module.exports = class S3CompatibleStorage {
constructor(storageName) {
this.storageName = storageName
}
async activated() {
// not used
}
async deactivated() {
// not used
}
async init() {
WIKI.logger.info(`(STORAGE/${this.storageName}) Initializing...`)
const { accessKeyId, secretAccessKey, region, bucket, endpoint } = this.config
this.s3 = new S3({
accessKeyId,
secretAccessKey,
region,
endpoint,
params: { Bucket: bucket },
apiVersions: '2006-03-01'
})
// determine if a bucket exists and you have permission to access it
await this.s3.headBucket().promise()
WIKI.logger.info(`(STORAGE/${this.storageName}) Initialization completed.`)
}
async created(page) {
WIKI.logger.info(`(STORAGE/${this.storageName}) Creating file ${page.path}...`)
const filePath = getFilePath(page, 'path')
await this.s3.putObject({ Key: filePath, Body: page.injectMetadata() }).promise()
}
async updated(page) {
WIKI.logger.info(`(STORAGE/${this.storageName}) Updating file ${page.path}...`)
const filePath = getFilePath(page, 'path')
await this.s3.putObject({ Key: filePath, Body: page.injectMetadata() }).promise()
}
async deleted(page) {
WIKI.logger.info(`(STORAGE/${this.storageName}) Deleting file ${page.path}...`)
const filePath = getFilePath(page, 'path')
await this.s3.deleteObject({ Key: filePath }).promise()
}
async renamed(page) {
WIKI.logger.info(`(STORAGE/${this.storageName}) Renaming file ${page.sourcePath} to ${page.destinationPath}...`)
const sourceFilePath = getFilePath(page, 'sourcePath')
const destinationFilePath = getFilePath(page, 'destinationPath')
await this.s3.copyObject({ CopySource: sourceFilePath, Key: destinationFilePath }).promise()
await this.s3.deleteObject({ Key: sourceFilePath }).promise()
}
}

31
server/modules/storage/s3/definition.yml
View File

@ -1,11 +1,32 @@
key: s3
title: Amazon S3
description: Amazon S3 is a cloud computing web service offered by Amazon Web Services which provides object storage.
author: requarks.io
author: andrewsim
logo: https://static.requarks.io/logo/aws-s3.svg
website: https://aws.amazon.com/s3/
isAvailable: true
supportedModes:
- push
defaultMode: push
schedule: false
props:
accessKeyId: String
accessSecret: String
region: String
bucket: String
region:
type: String
title: Region
hint: The AWS datacenter region where the bucket will be created.
order: 1
bucket:
type: String
title: Unique bucket name
hint: The unique bucket name to create (e.g. wiki-johndoe).
order: 2
accessKeyId:
type: String
title: Access Key ID
hint: The Access Key.
order: 3
secretAccessKey:
type: String
title: Secret Access Key
hint: The Secret Access Key for the Access Key ID you created above.
order: 4

24
server/modules/storage/s3/storage.js
View File

@ -1,23 +1,3 @@
module.exports = {
async activated() {
const S3CompatibleStorage = require('./common')
},
async deactivated() {
},
async init() {
},
async created() {
},
async updated() {
},
async deleted() {
},
async renamed() {
}
}
module.exports = new S3CompatibleStorage('S3')