feat: SFTP storage module + sensitive field option
This commit is contained in:
parent
4a2f1d045b
commit
0d6676c19b
@ -1,6 +1,6 @@
|
|||||||
query {
|
query {
|
||||||
storage {
|
storage {
|
||||||
targets(orderBy: "title ASC") {
|
targets {
|
||||||
isAvailable
|
isAvailable
|
||||||
isEnabled
|
isEnabled
|
||||||
key
|
key
|
||||||
|
@ -156,6 +156,8 @@
|
|||||||
"simple-git": "1.129.0",
|
"simple-git": "1.129.0",
|
||||||
"solr-node": "1.2.1",
|
"solr-node": "1.2.1",
|
||||||
"sqlite3": "4.1.1",
|
"sqlite3": "4.1.1",
|
||||||
|
"ssh2": "0.8.7",
|
||||||
|
"ssh2-promise": "0.1.6",
|
||||||
"striptags": "3.1.1",
|
"striptags": "3.1.1",
|
||||||
"subscriptions-transport-ws": "0.9.16",
|
"subscriptions-transport-ws": "0.9.16",
|
||||||
"tar-fs": "2.0.0",
|
"tar-fs": "2.0.0",
|
||||||
|
@ -13,7 +13,7 @@ module.exports = {
|
|||||||
StorageQuery: {
|
StorageQuery: {
|
||||||
async targets(obj, args, context, info) {
|
async targets(obj, args, context, info) {
|
||||||
let targets = await WIKI.models.storage.getTargets()
|
let targets = await WIKI.models.storage.getTargets()
|
||||||
targets = targets.map(tgt => {
|
targets = _.sortBy(targets.map(tgt => {
|
||||||
const targetInfo = _.find(WIKI.data.storage, ['key', tgt.key]) || {}
|
const targetInfo = _.find(WIKI.data.storage, ['key', tgt.key]) || {}
|
||||||
return {
|
return {
|
||||||
...targetInfo,
|
...targetInfo,
|
||||||
@ -28,15 +28,13 @@ module.exports = {
|
|||||||
key,
|
key,
|
||||||
value: JSON.stringify({
|
value: JSON.stringify({
|
||||||
...configData,
|
...configData,
|
||||||
value
|
value: (configData.sensitive && value.length > 0) ? '********' : value
|
||||||
})
|
})
|
||||||
})
|
})
|
||||||
}
|
}
|
||||||
}, []), 'key')
|
}, []), 'key')
|
||||||
}
|
}
|
||||||
})
|
}), ['title', 'key'])
|
||||||
// if (args.filter) { targets = graphHelper.filter(targets, args.filter) }
|
|
||||||
if (args.orderBy) { targets = _.sortBy(targets, [args.orderBy]) }
|
|
||||||
return targets
|
return targets
|
||||||
},
|
},
|
||||||
async status(obj, args, context, info) {
|
async status(obj, args, context, info) {
|
||||||
@ -56,13 +54,22 @@ module.exports = {
|
|||||||
StorageMutation: {
|
StorageMutation: {
|
||||||
async updateTargets(obj, args, context) {
|
async updateTargets(obj, args, context) {
|
||||||
try {
|
try {
|
||||||
|
let dbTargets = await WIKI.models.storage.getTargets()
|
||||||
for (let tgt of args.targets) {
|
for (let tgt of args.targets) {
|
||||||
|
const currentDbTarget = _.find(dbTargets, ['key', tgt.key])
|
||||||
|
if (!currentDbTarget) {
|
||||||
|
continue
|
||||||
|
}
|
||||||
await WIKI.models.storage.query().patch({
|
await WIKI.models.storage.query().patch({
|
||||||
isEnabled: tgt.isEnabled,
|
isEnabled: tgt.isEnabled,
|
||||||
mode: tgt.mode,
|
mode: tgt.mode,
|
||||||
syncInterval: tgt.syncInterval,
|
syncInterval: tgt.syncInterval,
|
||||||
config: _.reduce(tgt.config, (result, value, key) => {
|
config: _.reduce(tgt.config, (result, value, key) => {
|
||||||
_.set(result, `${value.key}`, _.get(JSON.parse(value.value), 'v', null))
|
let configValue = _.get(JSON.parse(value.value), 'v', null)
|
||||||
|
if (configValue === '********') {
|
||||||
|
configValue = _.get(currentDbTarget.config, value.key, '')
|
||||||
|
}
|
||||||
|
_.set(result, `${value.key}`, configValue)
|
||||||
return result
|
return result
|
||||||
}, {}),
|
}, {}),
|
||||||
state: {
|
state: {
|
||||||
|
@ -15,11 +15,7 @@ extend type Mutation {
|
|||||||
# -----------------------------------------------
|
# -----------------------------------------------
|
||||||
|
|
||||||
type StorageQuery {
|
type StorageQuery {
|
||||||
targets(
|
targets: [StorageTarget] @auth(requires: ["manage:system"])
|
||||||
filter: String
|
|
||||||
orderBy: String
|
|
||||||
): [StorageTarget] @auth(requires: ["manage:system"])
|
|
||||||
|
|
||||||
status: [StorageStatus] @auth(requires: ["manage:system"])
|
status: [StorageStatus] @auth(requires: ["manage:system"])
|
||||||
}
|
}
|
||||||
|
|
||||||
|
@ -32,6 +32,7 @@ module.exports = {
|
|||||||
hint: value.hint || false,
|
hint: value.hint || false,
|
||||||
enum: value.enum || false,
|
enum: value.enum || false,
|
||||||
multiline: value.multiline || false,
|
multiline: value.multiline || false,
|
||||||
|
sensitive: value.sensitive || false,
|
||||||
order: value.order || 100
|
order: value.order || 100
|
||||||
})
|
})
|
||||||
return result
|
return result
|
||||||
|
@ -94,6 +94,14 @@ module.exports = class Storage extends Model {
|
|||||||
} else {
|
} else {
|
||||||
WIKI.logger.info(`No new storage targets found: [ SKIPPED ]`)
|
WIKI.logger.info(`No new storage targets found: [ SKIPPED ]`)
|
||||||
}
|
}
|
||||||
|
|
||||||
|
// -> Delete removed targets
|
||||||
|
for (const target of dbTargets) {
|
||||||
|
if (!_.some(WIKI.data.storage, ['key', target.key])) {
|
||||||
|
await WIKI.models.storage.query().where('key', target.key).del()
|
||||||
|
WIKI.logger.info(`Removed target ${target.key} because it is no longer present in the modules folder: [ OK ]`)
|
||||||
|
}
|
||||||
|
}
|
||||||
} catch (err) {
|
} catch (err) {
|
||||||
WIKI.logger.error(`Failed to scan or load new storage providers: [ FAILED ]`)
|
WIKI.logger.error(`Failed to scan or load new storage providers: [ FAILED ]`)
|
||||||
WIKI.logger.error(err)
|
WIKI.logger.error(err)
|
||||||
|
@ -21,6 +21,7 @@ props:
|
|||||||
title: Account Access Key
|
title: Account Access Key
|
||||||
default: ''
|
default: ''
|
||||||
hint: Either key 1 or key 2.
|
hint: Either key 1 or key 2.
|
||||||
|
sensitive: true
|
||||||
order: 2
|
order: 2
|
||||||
containerName:
|
containerName:
|
||||||
type: String
|
type: String
|
||||||
@ -40,7 +41,4 @@ props:
|
|||||||
actions:
|
actions:
|
||||||
- handler: exportAll
|
- handler: exportAll
|
||||||
label: Export All
|
label: Export All
|
||||||
hint: Output all content from the DB to Azure Blog Storage, overwriting any existing data. If you enabled Azure Blog Storage after content was created or you temporarily disabled Git, you'll want to execute this action to add the missing content.
|
hint: Output all content from the DB to Azure Blog Storage, overwriting any existing data. If you enabled Azure Blog Storage after content was created or you temporarily disabled it, you'll want to execute this action to add the missing content.
|
||||||
- handler: importAll
|
|
||||||
label: Import Everything
|
|
||||||
hint: Will import all content currently in Azure Blog Storage. Useful for importing or restoring content from a previously backed up state.
|
|
||||||
|
@ -1,5 +1,9 @@
|
|||||||
const { BlobServiceClient, StorageSharedKeyCredential } = require('@azure/storage-blob')
|
const { BlobServiceClient, StorageSharedKeyCredential } = require('@azure/storage-blob')
|
||||||
|
const stream = require('stream')
|
||||||
|
const Promise = require('bluebird')
|
||||||
|
const pipeline = Promise.promisify(stream.pipeline)
|
||||||
const pageHelper = require('../../../helpers/page.js')
|
const pageHelper = require('../../../helpers/page.js')
|
||||||
|
const _ = require('lodash')
|
||||||
|
|
||||||
/* global WIKI */
|
/* global WIKI */
|
||||||
|
|
||||||
@ -110,5 +114,48 @@ module.exports = {
|
|||||||
await sourceBlockBlobClient.delete({
|
await sourceBlockBlobClient.delete({
|
||||||
deleteSnapshots: 'include'
|
deleteSnapshots: 'include'
|
||||||
})
|
})
|
||||||
|
},
|
||||||
|
/**
|
||||||
|
* HANDLERS
|
||||||
|
*/
|
||||||
|
async exportAll() {
|
||||||
|
WIKI.logger.info(`(STORAGE/AZURE) Exporting all content to Azure Blob Storage...`)
|
||||||
|
|
||||||
|
// -> Pages
|
||||||
|
await pipeline(
|
||||||
|
WIKI.models.knex.column('path', 'localeCode', 'title', 'description', 'contentType', 'content', 'isPublished', 'updatedAt').select().from('pages').where({
|
||||||
|
isPrivate: false
|
||||||
|
}).stream(),
|
||||||
|
new stream.Transform({
|
||||||
|
objectMode: true,
|
||||||
|
transform: async (page, enc, cb) => {
|
||||||
|
const filePath = getFilePath(page, 'path')
|
||||||
|
WIKI.logger.info(`(STORAGE/AZURE) Adding page ${filePath}...`)
|
||||||
|
const pageContent = pageHelper.injectPageMetadata(page)
|
||||||
|
const blockBlobClient = this.container.getBlockBlobClient(filePath)
|
||||||
|
await blockBlobClient.upload(pageContent, pageContent.length, { tier: this.config.storageTier })
|
||||||
|
cb()
|
||||||
|
}
|
||||||
|
})
|
||||||
|
)
|
||||||
|
|
||||||
|
// -> Assets
|
||||||
|
const assetFolders = await WIKI.models.assetFolders.getAllPaths()
|
||||||
|
|
||||||
|
await pipeline(
|
||||||
|
WIKI.models.knex.column('filename', 'folderId', 'data').select().from('assets').join('assetData', 'assets.id', '=', 'assetData.id').stream(),
|
||||||
|
new stream.Transform({
|
||||||
|
objectMode: true,
|
||||||
|
transform: async (asset, enc, cb) => {
|
||||||
|
const filename = (asset.folderId && asset.folderId > 0) ? `${_.get(assetFolders, asset.folderId)}/${asset.filename}` : asset.filename
|
||||||
|
WIKI.logger.info(`(STORAGE/AZURE) Adding asset ${filename}...`)
|
||||||
|
const blockBlobClient = this.container.getBlockBlobClient(filename)
|
||||||
|
await blockBlobClient.upload(asset.data, asset.data.length, { tier: this.config.storageTier })
|
||||||
|
cb()
|
||||||
|
}
|
||||||
|
})
|
||||||
|
)
|
||||||
|
|
||||||
|
WIKI.logger.info('(STORAGE/AZURE) All content has been pushed to Azure Blob Storage.')
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
@ -36,4 +36,10 @@ props:
|
|||||||
type: String
|
type: String
|
||||||
title: Access Key Secret
|
title: Access Key Secret
|
||||||
hint: The Access Key Secret for the Access Key ID you created above.
|
hint: The Access Key Secret for the Access Key ID you created above.
|
||||||
|
sensitive: true
|
||||||
order: 4
|
order: 4
|
||||||
|
actions:
|
||||||
|
- handler: exportAll
|
||||||
|
label: Export All
|
||||||
|
hint: Output all content from the DB to DigitalOcean Spaces, overwriting any existing data. If you enabled DigitalOcean Spaces after content was created or you temporarily disabled it, you'll want to execute this action to add the missing content.
|
||||||
|
|
||||||
|
@ -50,6 +50,7 @@ props:
|
|||||||
title: B - SSH Private Key Contents
|
title: B - SSH Private Key Contents
|
||||||
hint: SSH Authentication Only - Paste the contents of the private key. The key must NOT be passphrase-protected. Mode must be set to contents to use this option.
|
hint: SSH Authentication Only - Paste the contents of the private key. The key must NOT be passphrase-protected. Mode must be set to contents to use this option.
|
||||||
multiline: true
|
multiline: true
|
||||||
|
sensitive: true
|
||||||
order: 13
|
order: 13
|
||||||
verifySSL:
|
verifySSL:
|
||||||
type: Boolean
|
type: Boolean
|
||||||
@ -66,6 +67,7 @@ props:
|
|||||||
type: String
|
type: String
|
||||||
title: Password / PAT
|
title: Password / PAT
|
||||||
hint: Basic Authentication Only
|
hint: Basic Authentication Only
|
||||||
|
sensitive: true
|
||||||
order: 21
|
order: 21
|
||||||
defaultEmail:
|
defaultEmail:
|
||||||
type: String
|
type: String
|
||||||
|
@ -1,4 +1,8 @@
|
|||||||
const S3 = require('aws-sdk/clients/s3')
|
const S3 = require('aws-sdk/clients/s3')
|
||||||
|
const stream = require('stream')
|
||||||
|
const Promise = require('bluebird')
|
||||||
|
const pipeline = Promise.promisify(stream.pipeline)
|
||||||
|
const _ = require('lodash')
|
||||||
const pageHelper = require('../../../helpers/page.js')
|
const pageHelper = require('../../../helpers/page.js')
|
||||||
|
|
||||||
/* global WIKI */
|
/* global WIKI */
|
||||||
@ -98,4 +102,44 @@ module.exports = class S3CompatibleStorage {
|
|||||||
await this.s3.copyObject({ CopySource: asset.path, Key: asset.destinationPath }).promise()
|
await this.s3.copyObject({ CopySource: asset.path, Key: asset.destinationPath }).promise()
|
||||||
await this.s3.deleteObject({ Key: asset.path }).promise()
|
await this.s3.deleteObject({ Key: asset.path }).promise()
|
||||||
}
|
}
|
||||||
|
/**
|
||||||
|
* HANDLERS
|
||||||
|
*/
|
||||||
|
async exportAll() {
|
||||||
|
WIKI.logger.info(`(STORAGE/${this.storageName}) Exporting all content to the cloud provider...`)
|
||||||
|
|
||||||
|
// -> Pages
|
||||||
|
await pipeline(
|
||||||
|
WIKI.models.knex.column('path', 'localeCode', 'title', 'description', 'contentType', 'content', 'isPublished', 'updatedAt').select().from('pages').where({
|
||||||
|
isPrivate: false
|
||||||
|
}).stream(),
|
||||||
|
new stream.Transform({
|
||||||
|
objectMode: true,
|
||||||
|
transform: async (page, enc, cb) => {
|
||||||
|
const filePath = getFilePath(page, 'path')
|
||||||
|
WIKI.logger.info(`(STORAGE/${this.storageName}) Adding page ${filePath}...`)
|
||||||
|
await this.s3.putObject({ Key: filePath, Body: pageHelper.injectPageMetadata(page) }).promise()
|
||||||
|
cb()
|
||||||
|
}
|
||||||
|
})
|
||||||
|
)
|
||||||
|
|
||||||
|
// -> Assets
|
||||||
|
const assetFolders = await WIKI.models.assetFolders.getAllPaths()
|
||||||
|
|
||||||
|
await pipeline(
|
||||||
|
WIKI.models.knex.column('filename', 'folderId', 'data').select().from('assets').join('assetData', 'assets.id', '=', 'assetData.id').stream(),
|
||||||
|
new stream.Transform({
|
||||||
|
objectMode: true,
|
||||||
|
transform: async (asset, enc, cb) => {
|
||||||
|
const filename = (asset.folderId && asset.folderId > 0) ? `${_.get(assetFolders, asset.folderId)}/${asset.filename}` : asset.filename
|
||||||
|
WIKI.logger.info(`(STORAGE/${this.storageName}) Adding asset ${filename}...`)
|
||||||
|
await this.s3.putObject({ Key: filename, Body: asset.data }).promise()
|
||||||
|
cb()
|
||||||
|
}
|
||||||
|
})
|
||||||
|
)
|
||||||
|
|
||||||
|
WIKI.logger.info(`(STORAGE/${this.storageName}) All content has been pushed to the cloud provider.`)
|
||||||
|
}
|
||||||
}
|
}
|
||||||
|
@ -29,4 +29,9 @@ props:
|
|||||||
type: String
|
type: String
|
||||||
title: Secret Access Key
|
title: Secret Access Key
|
||||||
hint: The Secret Access Key for the Access Key ID you created above.
|
hint: The Secret Access Key for the Access Key ID you created above.
|
||||||
|
sensitive: true
|
||||||
order: 4
|
order: 4
|
||||||
|
actions:
|
||||||
|
- handler: exportAll
|
||||||
|
label: Export All
|
||||||
|
hint: Output all content from the DB to S3, overwriting any existing data. If you enabled S3 after content was created or you temporarily disabled it, you'll want to execute this action to add the missing content.
|
||||||
|
@ -1,16 +0,0 @@
|
|||||||
key: scp
|
|
||||||
title: SCP (SSH)
|
|
||||||
description: SSH is a software package that enables secure system administration and file transfers over insecure networks.
|
|
||||||
author: requarks.io
|
|
||||||
logo: https://static.requarks.io/logo/ssh.svg
|
|
||||||
website: https://www.ssh.com/ssh/
|
|
||||||
props:
|
|
||||||
host: String
|
|
||||||
port:
|
|
||||||
type: Number
|
|
||||||
default: 22
|
|
||||||
username: String
|
|
||||||
privateKeyPath: String
|
|
||||||
basePath:
|
|
||||||
type: String
|
|
||||||
default: '~'
|
|
@ -1,23 +0,0 @@
|
|||||||
module.exports = {
|
|
||||||
async activated() {
|
|
||||||
|
|
||||||
},
|
|
||||||
async deactivated() {
|
|
||||||
|
|
||||||
},
|
|
||||||
async init() {
|
|
||||||
|
|
||||||
},
|
|
||||||
async created() {
|
|
||||||
|
|
||||||
},
|
|
||||||
async updated() {
|
|
||||||
|
|
||||||
},
|
|
||||||
async deleted() {
|
|
||||||
|
|
||||||
},
|
|
||||||
async renamed() {
|
|
||||||
|
|
||||||
}
|
|
||||||
}
|
|
71
server/modules/storage/sftp/definition.yml
Normal file
71
server/modules/storage/sftp/definition.yml
Normal file
@ -0,0 +1,71 @@
|
|||||||
|
key: sftp
|
||||||
|
title: SFTP
|
||||||
|
description: SFTP (SSH File Transfer Protocol) is a secure file transfer protocol. It runs over the SSH protocol. It supports the full security and authentication functionality of SSH.
|
||||||
|
author: requarks.io
|
||||||
|
logo: https://static.requarks.io/logo/ssh.svg
|
||||||
|
website: https://www.ssh.com/ssh/sftp
|
||||||
|
isAvailable: true
|
||||||
|
supportedModes:
|
||||||
|
- push
|
||||||
|
defaultMode: push
|
||||||
|
schedule: false
|
||||||
|
props:
|
||||||
|
host:
|
||||||
|
type: String
|
||||||
|
title: Host
|
||||||
|
default: ''
|
||||||
|
hint: Hostname or IP of the remote SSH server.
|
||||||
|
order: 1
|
||||||
|
port:
|
||||||
|
type: Number
|
||||||
|
title: Port
|
||||||
|
default: 22
|
||||||
|
hint: SSH port of the remote server.
|
||||||
|
order: 2
|
||||||
|
authMode:
|
||||||
|
type: String
|
||||||
|
title: Authentication Method
|
||||||
|
default: 'privateKey'
|
||||||
|
hint: Whether to use Private Key or Password-based authentication. A private key is highly recommended for best security.
|
||||||
|
enum:
|
||||||
|
- privateKey
|
||||||
|
- password
|
||||||
|
order: 3
|
||||||
|
username:
|
||||||
|
type: String
|
||||||
|
title: Username
|
||||||
|
default: ''
|
||||||
|
hint: Username for authentication.
|
||||||
|
order: 4
|
||||||
|
privateKey:
|
||||||
|
type: String
|
||||||
|
title: Private Key Contents
|
||||||
|
default: ''
|
||||||
|
hint: (Private Key Authentication Only) - Contents of the private key
|
||||||
|
multiline: true
|
||||||
|
sensitive: true
|
||||||
|
order: 5
|
||||||
|
passphrase:
|
||||||
|
type: String
|
||||||
|
title: Private Key Passphrase
|
||||||
|
default: ''
|
||||||
|
hint: (Private Key Authentication Only) - Passphrase if the private key is encrypted, leave empty otherwise
|
||||||
|
sensitive: true
|
||||||
|
order: 6
|
||||||
|
password:
|
||||||
|
type: String
|
||||||
|
title: Password
|
||||||
|
default: ''
|
||||||
|
hint: (Password-based Authentication Only) - Password for authentication
|
||||||
|
sensitive: true
|
||||||
|
order: 6
|
||||||
|
basePath:
|
||||||
|
type: String
|
||||||
|
title: Base Directory Path
|
||||||
|
default: '/root/wiki'
|
||||||
|
hint: Base directory where files will be transferred to. The path must already exists and be writable by the user.
|
||||||
|
actions:
|
||||||
|
- handler: exportAll
|
||||||
|
label: Export All
|
||||||
|
hint: Output all content from the DB to the remote SSH server, overwriting any existing data. If you enabled SFTP after content was created or you temporarily disabled it, you'll want to execute this action to add the missing content.
|
||||||
|
|
160
server/modules/storage/sftp/storage.js
Normal file
160
server/modules/storage/sftp/storage.js
Normal file
@ -0,0 +1,160 @@
|
|||||||
|
const SSH2Promise = require('ssh2-promise')
|
||||||
|
const _ = require('lodash')
|
||||||
|
const path = require('path')
|
||||||
|
const stream = require('stream')
|
||||||
|
const Promise = require('bluebird')
|
||||||
|
const pipeline = Promise.promisify(stream.pipeline)
|
||||||
|
const pageHelper = require('../../../helpers/page.js')
|
||||||
|
|
||||||
|
/* global WIKI */
|
||||||
|
|
||||||
|
const getFilePath = (page, pathKey) => {
|
||||||
|
const fileName = `${page[pathKey]}.${pageHelper.getFileExtension(page.contentType)}`
|
||||||
|
const withLocaleCode = WIKI.config.lang.namespacing && WIKI.config.lang.code !== page.localeCode
|
||||||
|
return withLocaleCode ? `${page.localeCode}/${fileName}` : fileName
|
||||||
|
}
|
||||||
|
|
||||||
|
module.exports = {
|
||||||
|
client: null,
|
||||||
|
sftp: null,
|
||||||
|
async activated() {
|
||||||
|
|
||||||
|
},
|
||||||
|
async deactivated() {
|
||||||
|
|
||||||
|
},
|
||||||
|
async init() {
|
||||||
|
WIKI.logger.info(`(STORAGE/SFTP) Initializing...`)
|
||||||
|
this.client = new SSH2Promise({
|
||||||
|
host: this.config.host,
|
||||||
|
port: this.config.port || 22,
|
||||||
|
username: this.config.username,
|
||||||
|
password: (this.config.authMode === 'password') ? this.config.password : null,
|
||||||
|
privateKey: (this.config.authMode === 'privateKey') ? this.config.privateKey : null,
|
||||||
|
passphrase: (this.config.authMode === 'privateKey') ? this.config.passphrase : null
|
||||||
|
})
|
||||||
|
await this.client.connect()
|
||||||
|
this.sftp = this.client.sftp()
|
||||||
|
try {
|
||||||
|
await this.sftp.readdir(this.config.basePath)
|
||||||
|
} catch (err) {
|
||||||
|
WIKI.logger.warn(`(STORAGE/SFTP) ${err.message}`)
|
||||||
|
throw new Error(`Unable to read specified base directory: ${err.message}`)
|
||||||
|
}
|
||||||
|
WIKI.logger.info(`(STORAGE/SFTP) Initialization completed.`)
|
||||||
|
},
|
||||||
|
async created(page) {
|
||||||
|
WIKI.logger.info(`(STORAGE/SFTP) Creating file ${page.path}...`)
|
||||||
|
const filePath = getFilePath(page, 'path')
|
||||||
|
await this.ensureDirectory(filePath)
|
||||||
|
await this.sftp.writeFile(path.posix.join(this.config.basePath, filePath), page.injectMetadata())
|
||||||
|
},
|
||||||
|
async updated(page) {
|
||||||
|
WIKI.logger.info(`(STORAGE/SFTP) Updating file ${page.path}...`)
|
||||||
|
const filePath = getFilePath(page, 'path')
|
||||||
|
await this.ensureDirectory(filePath)
|
||||||
|
await this.sftp.writeFile(path.posix.join(this.config.basePath, filePath), page.injectMetadata())
|
||||||
|
},
|
||||||
|
async deleted(page) {
|
||||||
|
WIKI.logger.info(`(STORAGE/SFTP) Deleting file ${page.path}...`)
|
||||||
|
const filePath = getFilePath(page, 'path')
|
||||||
|
await this.sftp.unlink(path.posix.join(this.config.basePath, filePath))
|
||||||
|
},
|
||||||
|
async renamed(page) {
|
||||||
|
WIKI.logger.info(`(STORAGE/SFTP) Renaming file ${page.path} to ${page.destinationPath}...`)
|
||||||
|
let sourceFilePath = getFilePath(page, 'path')
|
||||||
|
let destinationFilePath = getFilePath(page, 'destinationPath')
|
||||||
|
if (WIKI.config.lang.namespacing) {
|
||||||
|
if (WIKI.config.lang.code !== page.localeCode) {
|
||||||
|
sourceFilePath = `${page.localeCode}/${sourceFilePath}`
|
||||||
|
}
|
||||||
|
if (WIKI.config.lang.code !== page.destinationLocaleCode) {
|
||||||
|
destinationFilePath = `${page.destinationLocaleCode}/${destinationFilePath}`
|
||||||
|
}
|
||||||
|
}
|
||||||
|
await this.ensureDirectory(destinationFilePath)
|
||||||
|
await this.sftp.rename(path.posix.join(this.config.basePath, sourceFilePath), path.posix.join(this.config.basePath, destinationFilePath))
|
||||||
|
},
|
||||||
|
/**
|
||||||
|
* ASSET UPLOAD
|
||||||
|
*
|
||||||
|
* @param {Object} asset Asset to upload
|
||||||
|
*/
|
||||||
|
async assetUploaded (asset) {
|
||||||
|
WIKI.logger.info(`(STORAGE/SFTP) Creating new file ${asset.path}...`)
|
||||||
|
await this.ensureDirectory(asset.path)
|
||||||
|
await this.sftp.writeFile(path.posix.join(this.config.basePath, asset.path), asset.data)
|
||||||
|
},
|
||||||
|
/**
|
||||||
|
* ASSET DELETE
|
||||||
|
*
|
||||||
|
* @param {Object} asset Asset to delete
|
||||||
|
*/
|
||||||
|
async assetDeleted (asset) {
|
||||||
|
WIKI.logger.info(`(STORAGE/SFTP) Deleting file ${asset.path}...`)
|
||||||
|
await this.sftp.unlink(path.posix.join(this.config.basePath, asset.path))
|
||||||
|
},
|
||||||
|
/**
|
||||||
|
* ASSET RENAME
|
||||||
|
*
|
||||||
|
* @param {Object} asset Asset to rename
|
||||||
|
*/
|
||||||
|
async assetRenamed (asset) {
|
||||||
|
WIKI.logger.info(`(STORAGE/SFTP) Renaming file from ${asset.path} to ${asset.destinationPath}...`)
|
||||||
|
await this.ensureDirectory(asset.destinationPath)
|
||||||
|
await this.sftp.rename(path.posix.join(this.config.basePath, asset.path), path.posix.join(this.config.basePath, asset.destinationPath))
|
||||||
|
},
|
||||||
|
/**
|
||||||
|
* HANDLERS
|
||||||
|
*/
|
||||||
|
async exportAll() {
|
||||||
|
WIKI.logger.info(`(STORAGE/SFTP) Exporting all content to the remote server...`)
|
||||||
|
|
||||||
|
// -> Pages
|
||||||
|
await pipeline(
|
||||||
|
WIKI.models.knex.column('path', 'localeCode', 'title', 'description', 'contentType', 'content', 'isPublished', 'updatedAt').select().from('pages').where({
|
||||||
|
isPrivate: false
|
||||||
|
}).stream(),
|
||||||
|
new stream.Transform({
|
||||||
|
objectMode: true,
|
||||||
|
transform: async (page, enc, cb) => {
|
||||||
|
const filePath = getFilePath(page, 'path')
|
||||||
|
WIKI.logger.info(`(STORAGE/SFTP) Adding page ${filePath}...`)
|
||||||
|
await this.ensureDirectory(filePath)
|
||||||
|
await this.sftp.writeFile(path.posix.join(this.config.basePath, filePath), pageHelper.injectPageMetadata(page))
|
||||||
|
cb()
|
||||||
|
}
|
||||||
|
})
|
||||||
|
)
|
||||||
|
|
||||||
|
// -> Assets
|
||||||
|
const assetFolders = await WIKI.models.assetFolders.getAllPaths()
|
||||||
|
|
||||||
|
await pipeline(
|
||||||
|
WIKI.models.knex.column('filename', 'folderId', 'data').select().from('assets').join('assetData', 'assets.id', '=', 'assetData.id').stream(),
|
||||||
|
new stream.Transform({
|
||||||
|
objectMode: true,
|
||||||
|
transform: async (asset, enc, cb) => {
|
||||||
|
const filename = (asset.folderId && asset.folderId > 0) ? `${_.get(assetFolders, asset.folderId)}/${asset.filename}` : asset.filename
|
||||||
|
WIKI.logger.info(`(STORAGE/SFTP) Adding asset ${filename}...`)
|
||||||
|
await this.ensureDirectory(filename)
|
||||||
|
await this.sftp.writeFile(path.posix.join(this.config.basePath, filename), asset.data)
|
||||||
|
cb()
|
||||||
|
}
|
||||||
|
})
|
||||||
|
)
|
||||||
|
|
||||||
|
WIKI.logger.info('(STORAGE/SFTP) All content has been pushed to the remote server.')
|
||||||
|
},
|
||||||
|
async ensureDirectory(filePath) {
|
||||||
|
if (filePath.indexOf('/') >= 0) {
|
||||||
|
try {
|
||||||
|
const folderPaths = _.dropRight(filePath.split('/'))
|
||||||
|
for (let i = 1; i <= folderPaths.length; i++) {
|
||||||
|
const folderSection = _.take(folderPaths, i).join('/')
|
||||||
|
await this.sftp.mkdir(path.posix.join(this.config.basePath, folderSection))
|
||||||
|
}
|
||||||
|
} catch (err) {}
|
||||||
|
}
|
||||||
|
}
|
||||||
|
}
|
53
yarn.lock
53
yarn.lock
@ -1270,6 +1270,13 @@
|
|||||||
dependencies:
|
dependencies:
|
||||||
passport-oauth2 "^1.4.0"
|
passport-oauth2 "^1.4.0"
|
||||||
|
|
||||||
|
"@heroku/socksv5@^0.0.9":
|
||||||
|
version "0.0.9"
|
||||||
|
resolved "https://registry.yarnpkg.com/@heroku/socksv5/-/socksv5-0.0.9.tgz#7a3905921136b2666979a0f86bb4f062f657f793"
|
||||||
|
integrity sha1-ejkFkhE2smZpeaD4a7TwYvZX95M=
|
||||||
|
dependencies:
|
||||||
|
ip-address "^5.8.8"
|
||||||
|
|
||||||
"@jest/console@^24.7.1", "@jest/console@^24.9.0":
|
"@jest/console@^24.7.1", "@jest/console@^24.9.0":
|
||||||
version "24.9.0"
|
version "24.9.0"
|
||||||
resolved "https://registry.yarnpkg.com/@jest/console/-/console-24.9.0.tgz#79b1bc06fb74a8cfb01cbdedf945584b1b9707f0"
|
resolved "https://registry.yarnpkg.com/@jest/console/-/console-24.9.0.tgz#79b1bc06fb74a8cfb01cbdedf945584b1b9707f0"
|
||||||
@ -2894,7 +2901,7 @@ asn1@0.2.3:
|
|||||||
resolved "https://registry.yarnpkg.com/asn1/-/asn1-0.2.3.tgz#dac8787713c9966849fc8180777ebe9c1ddf3b86"
|
resolved "https://registry.yarnpkg.com/asn1/-/asn1-0.2.3.tgz#dac8787713c9966849fc8180777ebe9c1ddf3b86"
|
||||||
integrity sha1-2sh4dxPJlmhJ/IGAd36+nB3fO4Y=
|
integrity sha1-2sh4dxPJlmhJ/IGAd36+nB3fO4Y=
|
||||||
|
|
||||||
asn1@~0.2.3:
|
asn1@~0.2.0, asn1@~0.2.3:
|
||||||
version "0.2.4"
|
version "0.2.4"
|
||||||
resolved "https://registry.yarnpkg.com/asn1/-/asn1-0.2.4.tgz#8d2475dfab553bb33e77b54e59e880bb8ce23136"
|
resolved "https://registry.yarnpkg.com/asn1/-/asn1-0.2.4.tgz#8d2475dfab553bb33e77b54e59e880bb8ce23136"
|
||||||
integrity sha512-jxwzQpLQjSmWXgwaCZE9Nz+glAG01yF1QnWgbhGwHI5A6FRIEY6IVqtHhIepHqI7/kyEyQEagBC5mBEFlIYvdg==
|
integrity sha512-jxwzQpLQjSmWXgwaCZE9Nz+glAG01yF1QnWgbhGwHI5A6FRIEY6IVqtHhIepHqI7/kyEyQEagBC5mBEFlIYvdg==
|
||||||
@ -3270,7 +3277,7 @@ base@^0.11.1:
|
|||||||
mixin-deep "^1.2.0"
|
mixin-deep "^1.2.0"
|
||||||
pascalcase "^0.1.1"
|
pascalcase "^0.1.1"
|
||||||
|
|
||||||
bcrypt-pbkdf@^1.0.0:
|
bcrypt-pbkdf@^1.0.0, bcrypt-pbkdf@^1.0.2:
|
||||||
version "1.0.2"
|
version "1.0.2"
|
||||||
resolved "https://registry.yarnpkg.com/bcrypt-pbkdf/-/bcrypt-pbkdf-1.0.2.tgz#a4301d389b6a43f9b67ff3ca11a3f6637e360e9e"
|
resolved "https://registry.yarnpkg.com/bcrypt-pbkdf/-/bcrypt-pbkdf-1.0.2.tgz#a4301d389b6a43f9b67ff3ca11a3f6637e360e9e"
|
||||||
integrity sha1-pDAdOJtqQ/m2f/PKEaP2Y342Dp4=
|
integrity sha1-pDAdOJtqQ/m2f/PKEaP2Y342Dp4=
|
||||||
@ -7195,6 +7202,15 @@ invert-kv@^2.0.0:
|
|||||||
resolved "https://registry.yarnpkg.com/invert-kv/-/invert-kv-2.0.0.tgz#7393f5afa59ec9ff5f67a27620d11c226e3eec02"
|
resolved "https://registry.yarnpkg.com/invert-kv/-/invert-kv-2.0.0.tgz#7393f5afa59ec9ff5f67a27620d11c226e3eec02"
|
||||||
integrity sha512-wPVv/y/QQ/Uiirj/vh3oP+1Ww+AWehmi1g5fFWGPF6IpCBCDVrhgHRMvrLfdYcwDh3QJbGXDW4JAuzxElLSqKA==
|
integrity sha512-wPVv/y/QQ/Uiirj/vh3oP+1Ww+AWehmi1g5fFWGPF6IpCBCDVrhgHRMvrLfdYcwDh3QJbGXDW4JAuzxElLSqKA==
|
||||||
|
|
||||||
|
ip-address@^5.8.8:
|
||||||
|
version "5.9.4"
|
||||||
|
resolved "https://registry.yarnpkg.com/ip-address/-/ip-address-5.9.4.tgz#4660ac261ad61bd397a860a007f7e98e4eaee386"
|
||||||
|
integrity sha512-dHkI3/YNJq4b/qQaz+c8LuarD3pY24JqZWfjB8aZx1gtpc2MDILu9L9jpZe1sHpzo/yWFweQVn+U//FhazUxmw==
|
||||||
|
dependencies:
|
||||||
|
jsbn "1.1.0"
|
||||||
|
lodash "^4.17.15"
|
||||||
|
sprintf-js "1.1.2"
|
||||||
|
|
||||||
ip-regex@^2.1.0:
|
ip-regex@^2.1.0:
|
||||||
version "2.1.0"
|
version "2.1.0"
|
||||||
resolved "https://registry.yarnpkg.com/ip-regex/-/ip-regex-2.1.0.tgz#fa78bf5d2e6913c911ce9f819ee5146bb6d844e9"
|
resolved "https://registry.yarnpkg.com/ip-regex/-/ip-regex-2.1.0.tgz#fa78bf5d2e6913c911ce9f819ee5146bb6d844e9"
|
||||||
@ -8041,6 +8057,11 @@ jsbi@^3.1.1:
|
|||||||
resolved "https://registry.yarnpkg.com/jsbi/-/jsbi-3.1.1.tgz#8ea18b3e08d102c6cc09acaa9a099921d775f4fa"
|
resolved "https://registry.yarnpkg.com/jsbi/-/jsbi-3.1.1.tgz#8ea18b3e08d102c6cc09acaa9a099921d775f4fa"
|
||||||
integrity sha512-+HQESPaV0mRiH614z4JPVPAftcRC2p53x92lySPzUzFwJbJTMpzHz8OYUkcXPN3fOcHUe0NdVcHnCtX/1+eCrA==
|
integrity sha512-+HQESPaV0mRiH614z4JPVPAftcRC2p53x92lySPzUzFwJbJTMpzHz8OYUkcXPN3fOcHUe0NdVcHnCtX/1+eCrA==
|
||||||
|
|
||||||
|
jsbn@1.1.0:
|
||||||
|
version "1.1.0"
|
||||||
|
resolved "https://registry.yarnpkg.com/jsbn/-/jsbn-1.1.0.tgz#b01307cb29b618a1ed26ec79e911f803c4da0040"
|
||||||
|
integrity sha1-sBMHyym2GKHtJux56RH4A8TaAEA=
|
||||||
|
|
||||||
jsbn@~0.1.0:
|
jsbn@~0.1.0:
|
||||||
version "0.1.1"
|
version "0.1.1"
|
||||||
resolved "https://registry.yarnpkg.com/jsbn/-/jsbn-0.1.1.tgz#a5e654c2e5a2deb5f201d96cefbca80c0ef2f513"
|
resolved "https://registry.yarnpkg.com/jsbn/-/jsbn-0.1.1.tgz#a5e654c2e5a2deb5f201d96cefbca80c0ef2f513"
|
||||||
@ -12956,7 +12977,7 @@ split@^1.0.0:
|
|||||||
dependencies:
|
dependencies:
|
||||||
through "2"
|
through "2"
|
||||||
|
|
||||||
sprintf-js@^1.1.2:
|
sprintf-js@1.1.2, sprintf-js@^1.1.2:
|
||||||
version "1.1.2"
|
version "1.1.2"
|
||||||
resolved "https://registry.yarnpkg.com/sprintf-js/-/sprintf-js-1.1.2.tgz#da1765262bf8c0f571749f2ad6c26300207ae673"
|
resolved "https://registry.yarnpkg.com/sprintf-js/-/sprintf-js-1.1.2.tgz#da1765262bf8c0f571749f2ad6c26300207ae673"
|
||||||
integrity sha512-VE0SOVEHCk7Qc8ulkWw3ntAzXuqf7S2lvwQaDLRnUeIEaKNQJzV6BwmLKhOqT61aGhfUMrXeaBk+oDGCzvhcug==
|
integrity sha512-VE0SOVEHCk7Qc8ulkWw3ntAzXuqf7S2lvwQaDLRnUeIEaKNQJzV6BwmLKhOqT61aGhfUMrXeaBk+oDGCzvhcug==
|
||||||
@ -12980,6 +13001,30 @@ sqlstring@^2.3.1:
|
|||||||
resolved "https://registry.yarnpkg.com/sqlstring/-/sqlstring-2.3.1.tgz#475393ff9e91479aea62dcaf0ca3d14983a7fb40"
|
resolved "https://registry.yarnpkg.com/sqlstring/-/sqlstring-2.3.1.tgz#475393ff9e91479aea62dcaf0ca3d14983a7fb40"
|
||||||
integrity sha1-R1OT/56RR5rqYtyvDKPRSYOn+0A=
|
integrity sha1-R1OT/56RR5rqYtyvDKPRSYOn+0A=
|
||||||
|
|
||||||
|
ssh2-promise@0.1.6:
|
||||||
|
version "0.1.6"
|
||||||
|
resolved "https://registry.yarnpkg.com/ssh2-promise/-/ssh2-promise-0.1.6.tgz#5758456591dedcfa621007ce588caf49b9ea02e9"
|
||||||
|
integrity sha512-t1tBCGeqelyhVqHFnaxoKbfh3D6Utvy1iMO4+ijwssbM3KDSreDC0RsFaUVIhefBpbrfWsemTu+fAB5kx2w3bQ==
|
||||||
|
dependencies:
|
||||||
|
"@heroku/socksv5" "^0.0.9"
|
||||||
|
ssh2 "^0.8.6"
|
||||||
|
|
||||||
|
ssh2-streams@~0.4.8:
|
||||||
|
version "0.4.8"
|
||||||
|
resolved "https://registry.yarnpkg.com/ssh2-streams/-/ssh2-streams-0.4.8.tgz#2ff92df2e0063fef86cf934eaea197967deda715"
|
||||||
|
integrity sha512-auxXfgYySz2vYw7TMU7PK7vFI7EPvhvTH8/tZPgGaWocK4p/vwCMiV3icz9AEkb0R40kOKZtFtqYIxDJyJiytw==
|
||||||
|
dependencies:
|
||||||
|
asn1 "~0.2.0"
|
||||||
|
bcrypt-pbkdf "^1.0.2"
|
||||||
|
streamsearch "~0.1.2"
|
||||||
|
|
||||||
|
ssh2@0.8.7, ssh2@^0.8.6:
|
||||||
|
version "0.8.7"
|
||||||
|
resolved "https://registry.yarnpkg.com/ssh2/-/ssh2-0.8.7.tgz#2dc15206f493010b98027201cf399b90bab79c89"
|
||||||
|
integrity sha512-/u1BO12kb0lDVxJXejWB9pxyF3/ncgRqI9vPCZuPzo05pdNDzqUeQRavScwSPsfMGK+5H/VRqp1IierIx0Bcxw==
|
||||||
|
dependencies:
|
||||||
|
ssh2-streams "~0.4.8"
|
||||||
|
|
||||||
sshpk@^1.7.0:
|
sshpk@^1.7.0:
|
||||||
version "1.16.1"
|
version "1.16.1"
|
||||||
resolved "https://registry.yarnpkg.com/sshpk/-/sshpk-1.16.1.tgz#fb661c0bef29b39db40769ee39fa70093d6f6877"
|
resolved "https://registry.yarnpkg.com/sshpk/-/sshpk-1.16.1.tgz#fb661c0bef29b39db40769ee39fa70093d6f6877"
|
||||||
@ -13091,7 +13136,7 @@ stream-shift@^1.0.0:
|
|||||||
resolved "https://registry.yarnpkg.com/stream-shift/-/stream-shift-1.0.0.tgz#d5c752825e5367e786f78e18e445ea223a155952"
|
resolved "https://registry.yarnpkg.com/stream-shift/-/stream-shift-1.0.0.tgz#d5c752825e5367e786f78e18e445ea223a155952"
|
||||||
integrity sha1-1cdSgl5TZ+eG944Y5EXqIjoVWVI=
|
integrity sha1-1cdSgl5TZ+eG944Y5EXqIjoVWVI=
|
||||||
|
|
||||||
streamsearch@0.1.2:
|
streamsearch@0.1.2, streamsearch@~0.1.2:
|
||||||
version "0.1.2"
|
version "0.1.2"
|
||||||
resolved "https://registry.yarnpkg.com/streamsearch/-/streamsearch-0.1.2.tgz#808b9d0e56fc273d809ba57338e929919a1a9f1a"
|
resolved "https://registry.yarnpkg.com/streamsearch/-/streamsearch-0.1.2.tgz#808b9d0e56fc273d809ba57338e929919a1a9f1a"
|
||||||
integrity sha1-gIudDlb8Jz2Am6VzOOkpkZoanxo=
|
integrity sha1-gIudDlb8Jz2Am6VzOOkpkZoanxo=
|
||||||
|
Loading…
Reference in New Issue
Block a user