wikijs-fork/server/modules/storage/git/storage.js

417 lines
14 KiB
JavaScript
Raw Normal View History

2019-01-28 06:27:04 +00:00
const path = require('path')
const sgit = require('simple-git/promise')
const fs = require('fs-extra')
const _ = require('lodash')
const stream = require('stream')
const Promise = require('bluebird')
const pipeline = Promise.promisify(stream.pipeline)
const klaw = require('klaw')
const pageHelper = require('../../../helpers/page.js')
2019-01-28 06:27:04 +00:00
2019-02-25 04:48:28 +00:00
const localeFolderRegex = /^([a-z]{2}(?:-[a-z]{2})?\/)?(.*)/i
/* global WIKI */
2019-02-25 04:48:28 +00:00
const getContenType = (filePath) => {
const ext = _.last(filePath.split('.'))
switch (ext) {
case 'md':
return 'markdown'
case 'html':
return 'html'
default:
return false
}
}
const getPagePath = (filePath) => {
2019-10-11 20:39:23 +00:00
let fpath = filePath
2019-10-07 04:06:47 +00:00
if (process.platform === 'win32') {
2019-10-11 20:39:23 +00:00
fpath = filePath.replace(/\\/g, '/')
2019-10-07 04:06:47 +00:00
}
2019-02-25 04:48:28 +00:00
let meta = {
locale: 'en',
2019-10-11 20:39:23 +00:00
path: _.initial(fpath.split('.')).join('')
2019-02-25 04:48:28 +00:00
}
const result = localeFolderRegex.exec(meta.path)
if (result[1]) {
meta = {
locale: result[1],
path: result[2]
}
}
return meta
}
module.exports = {
2019-02-03 22:08:06 +00:00
git: null,
repoPath: path.join(process.cwd(), 'data/repo'),
async activated() {
2019-02-03 22:08:06 +00:00
// not used
},
async deactivated() {
2019-02-03 22:08:06 +00:00
// not used
},
2019-02-25 04:48:28 +00:00
/**
* INIT
*/
async init() {
2019-02-03 07:48:30 +00:00
WIKI.logger.info('(STORAGE/GIT) Initializing...')
2019-02-03 22:08:06 +00:00
this.repoPath = path.resolve(WIKI.ROOTPATH, this.config.localRepoPath)
await fs.ensureDir(this.repoPath)
this.git = sgit(this.repoPath)
2019-01-28 06:27:04 +00:00
// Set custom binary path
if (!_.isEmpty(this.config.gitBinaryPath)) {
this.git.customBinary(this.config.gitBinaryPath)
}
2019-01-28 06:27:04 +00:00
// Initialize repo (if needed)
2019-02-03 07:48:30 +00:00
WIKI.logger.info('(STORAGE/GIT) Checking repository state...')
2019-02-03 22:08:06 +00:00
const isRepo = await this.git.checkIsRepo()
2019-01-28 06:27:04 +00:00
if (!isRepo) {
2019-02-03 07:48:30 +00:00
WIKI.logger.info('(STORAGE/GIT) Initializing local repository...')
2019-02-03 22:08:06 +00:00
await this.git.init()
2019-01-28 06:27:04 +00:00
}
2019-02-03 07:48:30 +00:00
// Set default author
2019-02-03 22:08:06 +00:00
await this.git.raw(['config', '--local', 'user.email', this.config.defaultEmail])
await this.git.raw(['config', '--local', 'user.name', this.config.defaultName])
2019-02-03 07:48:30 +00:00
// Purge existing remotes
WIKI.logger.info('(STORAGE/GIT) Listing existing remotes...')
2019-02-03 22:08:06 +00:00
const remotes = await this.git.getRemotes()
2019-02-03 07:48:30 +00:00
if (remotes.length > 0) {
WIKI.logger.info('(STORAGE/GIT) Purging existing remotes...')
for (let remote of remotes) {
2019-02-03 22:08:06 +00:00
await this.git.removeRemote(remote.name)
2019-02-03 07:48:30 +00:00
}
}
2019-01-28 06:27:04 +00:00
// Add remote
2019-02-03 07:48:30 +00:00
WIKI.logger.info('(STORAGE/GIT) Setting SSL Verification config...')
2019-02-03 22:08:06 +00:00
await this.git.raw(['config', '--local', '--bool', 'http.sslVerify', _.toString(this.config.verifySSL)])
2019-01-28 06:27:04 +00:00
switch (this.config.authType) {
case 'ssh':
2019-02-03 07:48:30 +00:00
WIKI.logger.info('(STORAGE/GIT) Setting SSH Command config...')
2019-07-14 22:55:12 +00:00
if (this.config.sshPrivateKeyMode === 'contents') {
try {
this.config.sshPrivateKeyPath = path.join(WIKI.ROOTPATH, 'data/secure/git-ssh.pem')
await fs.outputFile(this.config.sshPrivateKeyPath, this.config.sshPrivateKeyContent, {
encoding: 'utf8',
mode: 0o600
})
} catch (err) {
console.error(err)
throw err
}
}
2019-02-03 22:08:06 +00:00
await this.git.addConfig('core.sshCommand', `ssh -i "${this.config.sshPrivateKeyPath}" -o StrictHostKeyChecking=no`)
2019-02-03 07:48:30 +00:00
WIKI.logger.info('(STORAGE/GIT) Adding origin remote via SSH...')
2019-02-03 22:08:06 +00:00
await this.git.addRemote('origin', this.config.repoUrl)
2019-01-28 06:27:04 +00:00
break
default:
WIKI.logger.info('(STORAGE/GIT) Adding origin remote via HTTP/S...')
let originUrl = ''
if (_.startsWith(this.config.repoUrl, 'http')) {
originUrl = this.config.repoUrl.replace('://', `://${this.config.basicUsername}:${this.config.basicPassword}@`)
} else {
originUrl = `https://${this.config.basicUsername}:${this.config.basicPassword}@${this.config.repoUrl}`
}
2019-04-28 01:42:04 +00:00
await this.git.addRemote('origin', originUrl)
2019-01-28 06:27:04 +00:00
break
}
2019-02-03 07:48:30 +00:00
// Fetch updates for remote
WIKI.logger.info('(STORAGE/GIT) Fetch updates from remote...')
2019-02-03 22:08:06 +00:00
await this.git.raw(['remote', 'update', 'origin'])
2019-02-03 07:48:30 +00:00
// Checkout branch
2019-02-03 22:08:06 +00:00
const branches = await this.git.branch()
2019-02-03 07:48:30 +00:00
if (!_.includes(branches.all, this.config.branch) && !_.includes(branches.all, `remotes/origin/${this.config.branch}`)) {
throw new Error('Invalid branch! Make sure it exists on the remote first.')
}
WIKI.logger.info(`(STORAGE/GIT) Checking out branch ${this.config.branch}...`)
2019-02-03 22:08:06 +00:00
await this.git.checkout(this.config.branch)
// Perform initial sync
await this.sync()
2019-02-03 07:48:30 +00:00
2019-02-03 22:08:06 +00:00
WIKI.logger.info('(STORAGE/GIT) Initialization completed.')
},
2019-02-25 04:48:28 +00:00
/**
* SYNC
*/
2019-02-03 22:08:06 +00:00
async sync() {
2019-02-25 04:48:28 +00:00
const currentCommitLog = _.get(await this.git.log(['-n', '1', this.config.branch]), 'latest', {})
2019-02-03 07:48:30 +00:00
// Pull rebase
if (_.includes(['sync', 'pull'], this.mode)) {
WIKI.logger.info(`(STORAGE/GIT) Performing pull rebase from origin on branch ${this.config.branch}...`)
2019-02-03 22:08:06 +00:00
await this.git.pull('origin', this.config.branch, ['--rebase'])
2019-02-03 07:48:30 +00:00
}
// Push
if (_.includes(['sync', 'push'], this.mode)) {
WIKI.logger.info(`(STORAGE/GIT) Performing push to origin on branch ${this.config.branch}...`)
let pushOpts = ['--signed=if-asked']
if (this.mode === 'push') {
pushOpts.push('--force')
}
2019-02-03 22:08:06 +00:00
await this.git.push('origin', this.config.branch, pushOpts)
2019-02-03 07:48:30 +00:00
}
2019-02-25 04:48:28 +00:00
// Process Changes
if (_.includes(['sync', 'pull'], this.mode)) {
const latestCommitLog = _.get(await this.git.log(['-n', '1', this.config.branch]), 'latest', {})
const diff = await this.git.diffSummary(['-M', currentCommitLog.hash, latestCommitLog.hash])
2019-02-25 04:48:28 +00:00
if (_.get(diff, 'files', []).length > 0) {
await this.processFiles(diff.files)
}
}
},
/**
* Process Files
*
* @param {Array<String>} files Array of files to process
*/
async processFiles(files) {
for (const item of files) {
const contentType = getContenType(item.file)
if (!contentType) {
continue
}
const contentPath = getPagePath(item.file)
2019-02-25 04:48:28 +00:00
let itemContents = ''
try {
itemContents = await fs.readFile(path.join(this.repoPath, item.file), 'utf8')
const pageData = WIKI.models.pages.parseMetadata(itemContents, contentType)
const currentPage = await WIKI.models.pages.query().findOne({
path: contentPath.path,
localeCode: contentPath.locale
})
if (currentPage) {
// Already in the DB, can mark as modified
WIKI.logger.info(`(STORAGE/GIT) Page marked as modified: ${item.file}`)
await WIKI.models.pages.updatePage({
id: currentPage.id,
title: _.get(pageData, 'title', currentPage.title),
2019-10-07 04:06:47 +00:00
description: _.get(pageData, 'description', currentPage.description) || '',
isPublished: _.get(pageData, 'isPublished', currentPage.isPublished),
isPrivate: false,
content: pageData.content,
authorId: 1,
skipStorage: true
})
} else {
// Not in the DB, can mark as new
WIKI.logger.info(`(STORAGE/GIT) Page marked as new: ${item.file}`)
const pageEditor = await WIKI.models.editors.getDefaultEditor(contentType)
await WIKI.models.pages.createPage({
path: contentPath.path,
locale: contentPath.locale,
title: _.get(pageData, 'title', _.last(contentPath.path.split('/'))),
2019-10-07 04:06:47 +00:00
description: _.get(pageData, 'description', '') || '',
isPublished: _.get(pageData, 'isPublished', true),
isPrivate: false,
content: pageData.content,
authorId: 1,
editor: pageEditor,
skipStorage: true
})
}
} catch (err) {
if (err.code === 'ENOENT' && item.deletions > 0 && item.insertions === 0) {
// File was deleted by git, can safely mark as deleted in DB
WIKI.logger.info(`(STORAGE/GIT) Page marked as deleted: ${item.file}`)
2019-02-25 04:48:28 +00:00
await WIKI.models.pages.deletePage({
path: contentPath.path,
locale: contentPath.locale,
skipStorage: true
})
} else {
WIKI.logger.warn(`(STORAGE/GIT) Failed to open ${item.file}`)
2019-10-07 04:06:47 +00:00
console.error(err)
WIKI.logger.warn(err)
2019-02-25 04:48:28 +00:00
}
}
}
},
2019-02-25 04:48:28 +00:00
/**
* CREATE
*
* @param {Object} page Page to create
*/
2019-02-03 22:08:06 +00:00
async created(page) {
WIKI.logger.info(`(STORAGE/GIT) Committing new file ${page.path}...`)
2019-10-13 23:59:50 +00:00
let fileName = `${page.path}.${pageHelper.getFileExtension(page.contentType)}`
if (WIKI.config.lang.namespacing && WIKI.config.lang.code !== page.localeCode) {
fileName = `${page.localeCode}/${fileName}`
}
2019-02-03 22:08:06 +00:00
const filePath = path.join(this.repoPath, fileName)
await fs.outputFile(filePath, page.injectMetadata(), 'utf8')
2019-02-03 22:08:06 +00:00
await this.git.add(`./${fileName}`)
await this.git.commit(`docs: create ${page.path}`, fileName, {
'--author': `"${page.authorName} <${page.authorEmail}>"`
2019-01-28 06:27:04 +00:00
})
},
2019-02-25 04:48:28 +00:00
/**
* UPDATE
*
* @param {Object} page Page to update
*/
2019-02-03 22:08:06 +00:00
async updated(page) {
WIKI.logger.info(`(STORAGE/GIT) Committing updated file ${page.path}...`)
2019-10-13 23:59:50 +00:00
let fileName = `${page.path}.${pageHelper.getFileExtension(page.contentType)}`
if (WIKI.config.lang.namespacing && WIKI.config.lang.code !== page.localeCode) {
fileName = `${page.localeCode}/${fileName}`
}
2019-02-03 22:08:06 +00:00
const filePath = path.join(this.repoPath, fileName)
await fs.outputFile(filePath, page.injectMetadata(), 'utf8')
2019-02-03 22:08:06 +00:00
await this.git.add(`./${fileName}`)
await this.git.commit(`docs: update ${page.path}`, fileName, {
'--author': `"${page.authorName} <${page.authorEmail}>"`
2019-01-28 06:27:04 +00:00
})
},
2019-02-25 04:48:28 +00:00
/**
* DELETE
*
* @param {Object} page Page to delete
*/
2019-02-03 22:08:06 +00:00
async deleted(page) {
WIKI.logger.info(`(STORAGE/GIT) Committing removed file ${page.path}...`)
2019-10-13 23:59:50 +00:00
let fileName = `${page.path}.${pageHelper.getFileExtension(page.contentType)}`
if (WIKI.config.lang.namespacing && WIKI.config.lang.code !== page.localeCode) {
fileName = `${page.localeCode}/${fileName}`
}
2019-02-03 22:08:06 +00:00
await this.git.rm(`./${fileName}`)
await this.git.commit(`docs: delete ${page.path}`, fileName, {
'--author': `"${page.authorName} <${page.authorEmail}>"`
2019-01-28 06:27:04 +00:00
})
},
2019-02-25 04:48:28 +00:00
/**
* RENAME
*
* @param {Object} page Page to rename
*/
2019-02-03 22:08:06 +00:00
async renamed(page) {
2019-10-13 23:59:50 +00:00
WIKI.logger.info(`(STORAGE/GIT) Committing file move from ${page.path} to ${page.destinationPath}...`)
let sourceFilePath = `${page.path}.${pageHelper.getFileExtension(page.contentType)}`
let destinationFilePath = `${page.destinationPath}.${pageHelper.getFileExtension(page.contentType)}`
2019-10-13 23:59:50 +00:00
if (WIKI.config.lang.namespacing) {
if (WIKI.config.lang.code !== page.localeCode) {
sourceFilePath = `${page.localeCode}/${sourceFilePath}`
}
if (WIKI.config.lang.code !== page.destinationLocaleCode) {
destinationFilePath = `${page.destinationLocaleCode}/${destinationFilePath}`
}
}
2019-02-03 22:08:06 +00:00
await this.git.mv(`./${sourceFilePath}`, `./${destinationFilePath}`)
2019-10-13 23:59:50 +00:00
await this.git.commit(`docs: rename ${page.path} to ${page.destinationPath}`, destinationFilePath, {
'--author': `"${page.moveAuthorName} <${page.moveAuthorEmail}>"`
2019-01-28 06:27:04 +00:00
})
},
2019-10-07 04:06:47 +00:00
/**
* ASSET UPLOAD
*
* @param {Object} asset Asset to upload
*/
async assetUploaded (asset) {
WIKI.logger.info(`(STORAGE/GIT) Committing new file ${asset.path}...`)
const filePath = path.join(this.repoPath, asset.path)
await fs.outputFile(filePath, asset, 'utf8')
await this.git.add(`./${asset.path}`)
await this.git.commit(`docs: upload ${asset.path}`, asset.path, {
'--author': `"${asset.authorName} <${asset.authorEmail}>"`
})
},
/**
* ASSET DELETE
*
* @param {Object} asset Asset to upload
*/
async assetDeleted (asset) {
WIKI.logger.info(`(STORAGE/GIT) Committing removed file ${asset.path}...`)
await this.git.rm(`./${asset.path}`)
await this.git.commit(`docs: delete ${asset.path}`, asset.path, {
'--author': `"${asset.authorName} <${asset.authorEmail}>"`
})
},
/**
* ASSET RENAME
*
* @param {Object} asset Asset to upload
*/
async assetRenamed (asset) {
WIKI.logger.info(`(STORAGE/GIT) Committing file move from ${asset.sourcePath} to ${asset.destinationPath}...`)
2019-10-07 04:06:47 +00:00
await this.git.mv(`./${asset.sourcePath}`, `./${asset.destinationPath}`)
await this.git.commit(`docs: rename ${asset.sourcePath} to ${asset.destinationPath}`, asset.destinationPath, {
'--author': `"${asset.authorName} <${asset.authorEmail}>"`
})
},
/**
* HANDLERS
*/
async importAll() {
WIKI.logger.info(`(STORAGE/GIT) Importing all content from local Git repo to the DB...`)
await pipeline(
klaw(this.repoPath, {
filter: (f) => {
return !_.includes(f, '.git')
}
}),
new stream.Transform({
objectMode: true,
transform: async (file, enc, cb) => {
const relPath = file.path.substr(this.repoPath.length + 1)
if (relPath && relPath.length > 3) {
WIKI.logger.info(`(STORAGE/GIT) Processing ${relPath}...`)
await this.processFiles([{
file: relPath,
deletions: 0,
insertions: 0
}])
}
cb()
}
})
)
WIKI.logger.info('(STORAGE/GIT) Import completed.')
},
async syncUntracked() {
WIKI.logger.info(`(STORAGE/GIT) Adding all untracked content...`)
await pipeline(
WIKI.models.knex.column('path', 'localeCode', 'title', 'description', 'contentType', 'content', 'isPublished', 'updatedAt').select().from('pages').where({
isPrivate: false
}).stream(),
new stream.Transform({
objectMode: true,
transform: async (page, enc, cb) => {
2019-09-27 18:17:12 +00:00
let fileName = `${page.path}.${pageHelper.getFileExtension(page.contentType)}`
if (WIKI.config.lang.namespacing && WIKI.config.lang.code !== page.localeCode) {
fileName = `${page.localeCode}/${fileName}`
}
WIKI.logger.info(`(STORAGE/GIT) Adding ${fileName}...`)
const filePath = path.join(this.repoPath, fileName)
await fs.outputFile(filePath, pageHelper.injectPageMetadata(page), 'utf8')
await this.git.add(`./${fileName}`)
cb()
}
})
)
await this.git.commit(`docs: add all untracked content`)
WIKI.logger.info('(STORAGE/GIT) All content is now tracked.')
}
}