wikijs-fork/server/modules/storage/disk/storage.js

const fs = require('fs-extra')
const path = require('path')
const tar = require('tar-fs')
const zlib = require('zlib')
const stream = require('stream')
const _ = require('lodash')
const Promise = require('bluebird')
const pipeline = Promise.promisify(stream.pipeline)
const klaw = require('klaw')
const pageHelper = require('../../../helpers/page.js')
const moment = require('moment')
const mime = require('mime-types').lookup

/* global WIKI */

module.exports = {
  async activated() {
    // not used
  },
  async deactivated() {
    // not used
  },
  async init() {
    WIKI.logger.info('(STORAGE/DISK) Initializing...')
    await fs.ensureDir(this.config.path)
    WIKI.logger.info('(STORAGE/DISK) Initialization completed.')
  },
  async sync({ manual } = { manual: false }) {
    if (this.config.createDailyBackups || manual) {
      const dirPath = path.join(this.config.path, manual ? '_manual' : '_daily')
      await fs.ensureDir(dirPath)

      const dateFilename = moment().format(manual ? 'YYYYMMDD-HHmmss' : 'DD')

      WIKI.logger.info(`(STORAGE/DISK) Creating backup archive...`)
      await pipeline(
        tar.pack(this.config.path, {
          ignore: (filePath) => {
            return filePath.indexOf('_daily') >= 0 || filePath.indexOf('_manual') >= 0
          }
        }),
        zlib.createGzip(),
        fs.createWriteStream(path.join(dirPath, `wiki-${dateFilename}.tar.gz`))
      )
      WIKI.logger.info('(STORAGE/DISK) Backup archive created successfully.')
    }
  },
  async created(page) {
    WIKI.logger.info(`(STORAGE/DISK) Creating file [${page.localeCode}] ${page.path}...`)
    let fileName = `${page.path}.${pageHelper.getFileExtension(page.contentType)}`
    if (WIKI.config.lang.code !== page.localeCode) {
      fileName = `${page.localeCode}/${fileName}`
    }
    const filePath = path.join(this.config.path, fileName)
    await fs.outputFile(filePath, page.injectMetadata(), 'utf8')
  },
  async updated(page) {
    WIKI.logger.info(`(STORAGE/DISK) Updating file [${page.localeCode}] ${page.path}...`)
    let fileName = `${page.path}.${pageHelper.getFileExtension(page.contentType)}`
    if (WIKI.config.lang.code !== page.localeCode) {
      fileName = `${page.localeCode}/${fileName}`
    }
    const filePath = path.join(this.config.path, fileName)
    await fs.outputFile(filePath, page.injectMetadata(), 'utf8')
  },
  async deleted(page) {
    WIKI.logger.info(`(STORAGE/DISK) Deleting file [${page.localeCode}] ${page.path}...`)
    let fileName = `${page.path}.${pageHelper.getFileExtension(page.contentType)}`
    if (WIKI.config.lang.code !== page.localeCode) {
      fileName = `${page.localeCode}/${fileName}`
    }
    const filePath = path.join(this.config.path, fileName)
    await fs.unlink(filePath)
  },
  async renamed(page) {
    WIKI.logger.info(`(STORAGE/DISK) Renaming file [${page.localeCode}] ${page.path} to [${page.destinationLocaleCode}] ${page.destinationPath}...`)

    let sourceFilePath = `${page.path}.${pageHelper.getFileExtension(page.contentType)}`
    let destinationFilePath = `${page.destinationPath}.${pageHelper.getFileExtension(page.contentType)}`

    if (WIKI.config.lang.namespacing) {
      if (WIKI.config.lang.code !== page.localeCode) {
        sourceFilePath = `${page.localeCode}/${sourceFilePath}`
      }
      if (WIKI.config.lang.code !== page.destinationLocaleCode) {
        destinationFilePath = `${page.destinationLocaleCode}/${destinationFilePath}`
      }
    }

    await fs.move(path.join(this.config.path, sourceFilePath), path.join(this.config.path, destinationFilePath), { overwrite: true })
  },
  /**
   * ASSET UPLOAD
   *
   * @param {Object} asset Asset to upload
   */
  async assetUploaded (asset) {
    WIKI.logger.info(`(STORAGE/DISK) Creating new file ${asset.path}...`)
    await fs.outputFile(path.join(this.config.path, asset.path), asset.data)
  },
  /**
   * ASSET DELETE
   *
   * @param {Object} asset Asset to delete
   */
  async assetDeleted (asset) {
    WIKI.logger.info(`(STORAGE/DISK) Deleting file ${asset.path}...`)
    await fs.remove(path.join(this.config.path, asset.path))
  },
  /**
   * ASSET RENAME
   *
   * @param {Object} asset Asset to rename
   */
  async assetRenamed (asset) {
    WIKI.logger.info(`(STORAGE/DISK) Renaming file from ${asset.path} to ${asset.destinationPath}...`)
    await fs.move(path.join(this.config.path, asset.path), path.join(this.config.path, asset.destinationPath), { overwrite: true })
  },

  /**
   * HANDLERS
   */
  async dump() {
    WIKI.logger.info(`(STORAGE/DISK) Dumping all content to disk...`)

    // -> Pages
    await pipeline(
      WIKI.models.knex.column('path', 'localeCode', 'title', 'description', 'contentType', 'content', 'isPublished', 'updatedAt').select().from('pages').where({
        isPrivate: false
      }).stream(),
      new stream.Transform({
        objectMode: true,
        transform: async (page, enc, cb) => {
          let fileName = `${page.path}.${pageHelper.getFileExtension(page.contentType)}`
          if (WIKI.config.lang.code !== page.localeCode) {
            fileName = `${page.localeCode}/${fileName}`
          }
          WIKI.logger.info(`(STORAGE/DISK) Dumping page ${fileName}...`)
          const filePath = path.join(this.config.path, fileName)
          await fs.outputFile(filePath, pageHelper.injectPageMetadata(page), 'utf8')
          cb()
        }
      })
    )

    // -> Assets
    const assetFolders = await WIKI.models.assetFolders.getAllPaths()

    await pipeline(
      WIKI.models.knex.column('filename', 'folderId', 'data').select().from('assets').join('assetData', 'assets.id', '=', 'assetData.id').stream(),
      new stream.Transform({
        objectMode: true,
        transform: async (asset, enc, cb) => {
          const filename = (asset.folderId && asset.folderId > 0) ? `${_.get(assetFolders, asset.folderId)}/${asset.filename}` : asset.filename
          WIKI.logger.info(`(STORAGE/DISK) Dumping asset ${filename}...`)
          await fs.outputFile(path.join(this.config.path, filename), asset.data)
          cb()
        }
      })
    )

    WIKI.logger.info('(STORAGE/DISK) All content was dumped to disk successfully.')
  },
  async backup() {
    return this.sync({ manual: true })
  },
  async importAll() {
    WIKI.logger.info(`(STORAGE/DISK) Importing all content from local disk folder to the DB...`)

    const rootUser = await WIKI.models.users.getRootUser()
    let assetFolders = await WIKI.models.assetFolders.getAllPaths()

    await pipeline(
      klaw(this.config.path, {
        filter: (f) => {
          return !_.includes(f, '.git')
        }
      }),
      new stream.Transform({
        objectMode: true,
        transform: async (file, enc, cb) => {
          const relPath = file.path.substr(this.config.path.length + 1)
          if (file.stats.size < 1) {
            // Skip directories and zero-byte files
            return cb()
          } else if (relPath && relPath.length > 3) {
            WIKI.logger.info(`(STORAGE/DISK) Processing ${relPath}...`)
            const contentType = pageHelper.getContentType(relPath)
            if (contentType) {
              // -> Page
              const contentPath = pageHelper.getPagePath(relPath)

              let itemContents = ''
              try {
                itemContents = await fs.readFile(path.join(this.config.path, relPath), 'utf8')
                const pageData = WIKI.models.pages.parseMetadata(itemContents, contentType)
                const currentPage = await WIKI.models.pages.query().findOne({
                  path: contentPath.path,
                  localeCode: contentPath.locale
                })
                if (currentPage) {
                  // Already in the DB, can mark as modified
                  WIKI.logger.info(`(STORAGE/DISK) Page marked as modified: ${relPath}`)
                  await WIKI.models.pages.updatePage({
                    id: currentPage.id,
                    title: _.get(pageData, 'title', currentPage.title),
                    description: _.get(pageData, 'description', currentPage.description) || '',
                    isPublished: _.get(pageData, 'isPublished', currentPage.isPublished),
                    isPrivate: false,
                    content: pageData.content,
                    user: rootUser,
                    skipStorage: true
                  })
                } else {
                  // Not in the DB, can mark as new
                  WIKI.logger.info(`(STORAGE/DISK) Page marked as new: ${relPath}`)
                  const pageEditor = await WIKI.models.editors.getDefaultEditor(contentType)
                  await WIKI.models.pages.createPage({
                    path: contentPath.path,
                    locale: contentPath.locale,
                    title: _.get(pageData, 'title', _.last(contentPath.path.split('/'))),
                    description: _.get(pageData, 'description', '') || '',
                    isPublished: _.get(pageData, 'isPublished', true),
                    isPrivate: false,
                    content: pageData.content,
                    user: rootUser,
                    editor: pageEditor,
                    skipStorage: true
                  })
                }
              } catch (err) {
                WIKI.logger.warn(`(STORAGE/DISK) Failed to process ${relPath}`)
                WIKI.logger.warn(err)
              }
            } else {
              // -> Asset

              // -> Find existing folder
              const filePathInfo = path.parse(file.path)
              const folderPath = path.dirname(relPath).replace(/\\/g, '/')
              let folderId = _.toInteger(_.findKey(assetFolders, fld => { return fld === folderPath })) || null

              // -> Create missing folder structure
              if (!folderId && folderPath !== '.') {
                const folderParts = folderPath.split('/')
                let currentFolderPath = []
                let currentFolderParentId = null
                for (const folderPart of folderParts) {
                  currentFolderPath.push(folderPart)
                  const existingFolderId = _.findKey(assetFolders, fld => { return fld === currentFolderPath.join('/') })
                  if (!existingFolderId) {
                    const newFolderObj = await WIKI.models.assetFolders.query().insert({
                      slug: folderPart,
                      name: folderPart,
                      parentId: currentFolderParentId
                    })
                    _.set(assetFolders, newFolderObj.id, currentFolderPath.join('/'))
                    currentFolderParentId = newFolderObj.id
                  } else {
                    currentFolderParentId = _.toInteger(existingFolderId)
                  }
                }
                folderId = currentFolderParentId
              }

              // -> Import asset
              await WIKI.models.assets.upload({
                mode: 'import',
                originalname: filePathInfo.base,
                ext: filePathInfo.ext,
                mimetype: mime(filePathInfo.base) || 'application/octet-stream',
                size: file.stats.size,
                folderId: folderId,
                path: file.path,
                assetPath: relPath,
                user: rootUser,
                skipStorage: true
              })
            }
          }
          cb()
        }
      })
    )
    WIKI.logger.info('(STORAGE/DISK) Import completed.')
  }
}