wikijs-fork/server/modules/storage/disk/storage.js

193 lines
7.5 KiB
JavaScript

const fs = require('fs-extra')
const path = require('path')
const tar = require('tar-fs')
const zlib = require('zlib')
const stream = require('stream')
const _ = require('lodash')
const Promise = require('bluebird')
const pipeline = Promise.promisify(stream.pipeline)
const klaw = require('klaw')
const pageHelper = require('../../../helpers/page.js')
const moment = require('moment')
/* global WIKI */
module.exports = {
async activated() {
// not used
},
async deactivated() {
// not used
},
async init() {
WIKI.logger.info('(STORAGE/DISK) Initializing...')
await fs.ensureDir(this.config.path)
WIKI.logger.info('(STORAGE/DISK) Initialization completed.')
},
async sync({ manual } = { manual: false }) {
if (this.config.createDailyBackups || manual) {
const dirPath = path.join(this.config.path, manual ? '_manual' : '_daily')
await fs.ensureDir(dirPath)
const dateFilename = moment().format(manual ? 'YYYYMMDD-HHmmss' : 'DD')
WIKI.logger.info(`(STORAGE/DISK) Creating backup archive...`)
await pipeline(
tar.pack(this.config.path, {
ignore: (filePath) => {
return filePath.indexOf('_daily') >= 0 || filePath.indexOf('_manual') >= 0
}
}),
zlib.createGzip(),
fs.createWriteStream(path.join(dirPath, `wiki-${dateFilename}.tar.gz`))
)
WIKI.logger.info('(STORAGE/DISK) Backup archive created successfully.')
}
},
async created(page) {
WIKI.logger.info(`(STORAGE/DISK) Creating file [${page.localeCode}] ${page.path}...`)
let fileName = `${page.path}.${pageHelper.getFileExtension(page.contentType)}`
if (WIKI.config.lang.code !== page.localeCode) {
fileName = `${page.localeCode}/${fileName}`
}
const filePath = path.join(this.config.path, fileName)
await fs.outputFile(filePath, page.injectMetadata(), 'utf8')
},
async updated(page) {
WIKI.logger.info(`(STORAGE/DISK) Updating file [${page.localeCode}] ${page.path}...`)
let fileName = `${page.path}.${pageHelper.getFileExtension(page.contentType)}`
if (WIKI.config.lang.code !== page.localeCode) {
fileName = `${page.localeCode}/${fileName}`
}
const filePath = path.join(this.config.path, fileName)
await fs.outputFile(filePath, page.injectMetadata(), 'utf8')
},
async deleted(page) {
WIKI.logger.info(`(STORAGE/DISK) Deleting file [${page.localeCode}] ${page.path}...`)
let fileName = `${page.path}.${pageHelper.getFileExtension(page.contentType)}`
if (WIKI.config.lang.code !== page.localeCode) {
fileName = `${page.localeCode}/${fileName}`
}
const filePath = path.join(this.config.path, fileName)
await fs.unlink(filePath)
},
async renamed(page) {
WIKI.logger.info(`(STORAGE/DISK) Renaming file [${page.localeCode}] ${page.path} to [${page.destinationLocaleCode}] ${page.destinationPath}...`)
let sourceFilePath = `${page.path}.${pageHelper.getFileExtension(page.contentType)}`
let destinationFilePath = `${page.destinationPath}.${pageHelper.getFileExtension(page.contentType)}`
if (WIKI.config.lang.namespacing) {
if (WIKI.config.lang.code !== page.localeCode) {
sourceFilePath = `${page.localeCode}/${sourceFilePath}`
}
if (WIKI.config.lang.code !== page.destinationLocaleCode) {
destinationFilePath = `${page.destinationLocaleCode}/${destinationFilePath}`
}
}
await fs.move(path.join(this.config.path, sourceFilePath), path.join(this.config.path, destinationFilePath), { overwrite: true })
},
/**
* HANDLERS
*/
async dump() {
WIKI.logger.info(`(STORAGE/DISK) Dumping all content to disk...`)
await pipeline(
WIKI.models.knex.column('path', 'localeCode', 'title', 'description', 'contentType', 'content', 'isPublished', 'updatedAt').select().from('pages').where({
isPrivate: false
}).stream(),
new stream.Transform({
objectMode: true,
transform: async (page, enc, cb) => {
let fileName = `${page.path}.${pageHelper.getFileExtension(page.contentType)}`
if (WIKI.config.lang.code !== page.localeCode) {
fileName = `${page.localeCode}/${fileName}`
}
WIKI.logger.info(`(STORAGE/DISK) Dumping ${fileName}...`)
const filePath = path.join(this.config.path, fileName)
await fs.outputFile(filePath, pageHelper.injectPageMetadata(page), 'utf8')
cb()
}
})
)
WIKI.logger.info('(STORAGE/DISK) All content was dumped to disk successfully.')
},
async backup() {
return this.sync({ manual: true })
},
async importAll() {
WIKI.logger.info(`(STORAGE/DISK) Importing all content from local disk folder to the DB...`)
const rootUser = await WIKI.models.users.getRootUser()
await pipeline(
klaw(this.config.path, {
filter: (f) => {
return !_.includes(f, '.git')
}
}),
new stream.Transform({
objectMode: true,
transform: async (file, enc, cb) => {
const relPath = file.path.substr(this.config.path.length + 1)
if (relPath && relPath.length > 3) {
WIKI.logger.info(`(STORAGE/DISK) Processing ${relPath}...`)
const contentType = pageHelper.getContentType(relPath)
if (!contentType) {
return cb()
}
const contentPath = pageHelper.getPagePath(relPath)
let itemContents = ''
try {
itemContents = await fs.readFile(path.join(this.config.path, relPath), 'utf8')
const pageData = WIKI.models.pages.parseMetadata(itemContents, contentType)
const currentPage = await WIKI.models.pages.query().findOne({
path: contentPath.path,
localeCode: contentPath.locale
})
if (currentPage) {
// Already in the DB, can mark as modified
WIKI.logger.info(`(STORAGE/DISK) Page marked as modified: ${relPath}`)
await WIKI.models.pages.updatePage({
id: currentPage.id,
title: _.get(pageData, 'title', currentPage.title),
description: _.get(pageData, 'description', currentPage.description) || '',
isPublished: _.get(pageData, 'isPublished', currentPage.isPublished),
isPrivate: false,
content: pageData.content,
user: rootUser,
skipStorage: true
})
} else {
// Not in the DB, can mark as new
WIKI.logger.info(`(STORAGE/DISK) Page marked as new: ${relPath}`)
const pageEditor = await WIKI.models.editors.getDefaultEditor(contentType)
await WIKI.models.pages.createPage({
path: contentPath.path,
locale: contentPath.locale,
title: _.get(pageData, 'title', _.last(contentPath.path.split('/'))),
description: _.get(pageData, 'description', '') || '',
isPublished: _.get(pageData, 'isPublished', true),
isPrivate: false,
content: pageData.content,
user: rootUser,
editor: pageEditor,
skipStorage: true
})
}
} catch (err) {
WIKI.logger.warn(`(STORAGE/DISK) Failed to process ${relPath}`)
WIKI.logger.warn(err)
}
}
cb()
}
})
)
WIKI.logger.info('(STORAGE/DISK) Import completed.')
}
}