b3731dd26e
git has the quotopath option enabled by default, filepaths with unicode characters will be escaped, causing the wiki to not handle changes.
501 lines
18 KiB
JavaScript
501 lines
18 KiB
JavaScript
const path = require('path')
|
|
const sgit = require('simple-git/promise')
|
|
const fs = require('fs-extra')
|
|
const _ = require('lodash')
|
|
const stream = require('stream')
|
|
const Promise = require('bluebird')
|
|
const pipeline = Promise.promisify(stream.pipeline)
|
|
const klaw = require('klaw')
|
|
const os = require('os')
|
|
|
|
const pageHelper = require('../../../helpers/page')
|
|
const assetHelper = require('../../../helpers/asset')
|
|
const commonDisk = require('../disk/common')
|
|
|
|
/* global WIKI */
|
|
|
|
module.exports = {
|
|
git: null,
|
|
repoPath: path.resolve(WIKI.ROOTPATH, WIKI.config.dataPath, 'repo'),
|
|
async activated() {
|
|
// not used
|
|
},
|
|
async deactivated() {
|
|
// not used
|
|
},
|
|
/**
|
|
* INIT
|
|
*/
|
|
async init() {
|
|
WIKI.logger.info('(STORAGE/GIT) Initializing...')
|
|
this.repoPath = path.resolve(WIKI.ROOTPATH, this.config.localRepoPath)
|
|
await fs.ensureDir(this.repoPath)
|
|
this.git = sgit(this.repoPath)
|
|
|
|
// Set custom binary path
|
|
if (!_.isEmpty(this.config.gitBinaryPath)) {
|
|
this.git.customBinary(this.config.gitBinaryPath)
|
|
}
|
|
|
|
// Initialize repo (if needed)
|
|
WIKI.logger.info('(STORAGE/GIT) Checking repository state...')
|
|
const isRepo = await this.git.checkIsRepo()
|
|
if (!isRepo) {
|
|
WIKI.logger.info('(STORAGE/GIT) Initializing local repository...')
|
|
await this.git.init()
|
|
}
|
|
|
|
// Disable quotePath
|
|
// Link https://git-scm.com/docs/git-config#Documentation/git-config.txt-corequotePath
|
|
await this.git.raw(['config', '--local', 'core.quotepath', false])
|
|
|
|
// Set default author
|
|
await this.git.raw(['config', '--local', 'user.email', this.config.defaultEmail])
|
|
await this.git.raw(['config', '--local', 'user.name', this.config.defaultName])
|
|
|
|
// Purge existing remotes
|
|
WIKI.logger.info('(STORAGE/GIT) Listing existing remotes...')
|
|
const remotes = await this.git.getRemotes()
|
|
if (remotes.length > 0) {
|
|
WIKI.logger.info('(STORAGE/GIT) Purging existing remotes...')
|
|
for (let remote of remotes) {
|
|
await this.git.removeRemote(remote.name)
|
|
}
|
|
}
|
|
|
|
// Add remote
|
|
WIKI.logger.info('(STORAGE/GIT) Setting SSL Verification config...')
|
|
await this.git.raw(['config', '--local', '--bool', 'http.sslVerify', _.toString(this.config.verifySSL)])
|
|
switch (this.config.authType) {
|
|
case 'ssh':
|
|
WIKI.logger.info('(STORAGE/GIT) Setting SSH Command config...')
|
|
if (this.config.sshPrivateKeyMode === 'contents') {
|
|
try {
|
|
this.config.sshPrivateKeyPath = path.resolve(WIKI.ROOTPATH, WIKI.config.dataPath, 'secure/git-ssh.pem')
|
|
await fs.outputFile(this.config.sshPrivateKeyPath, this.config.sshPrivateKeyContent + os.EOL, {
|
|
encoding: 'utf8',
|
|
mode: 0o600
|
|
})
|
|
} catch (err) {
|
|
WIKI.logger.error(err)
|
|
throw err
|
|
}
|
|
}
|
|
await this.git.addConfig('core.sshCommand', `ssh -i "${this.config.sshPrivateKeyPath}" -o StrictHostKeyChecking=no`)
|
|
WIKI.logger.info('(STORAGE/GIT) Adding origin remote via SSH...')
|
|
await this.git.addRemote('origin', this.config.repoUrl)
|
|
break
|
|
default:
|
|
WIKI.logger.info('(STORAGE/GIT) Adding origin remote via HTTP/S...')
|
|
let originUrl = ''
|
|
if (_.startsWith(this.config.repoUrl, 'http')) {
|
|
originUrl = this.config.repoUrl.replace('://', `://${encodeURI(this.config.basicUsername)}:${encodeURI(this.config.basicPassword)}@`)
|
|
} else {
|
|
originUrl = `https://${encodeURI(this.config.basicUsername)}:${encodeURI(this.config.basicPassword)}@${this.config.repoUrl}`
|
|
}
|
|
await this.git.addRemote('origin', originUrl)
|
|
break
|
|
}
|
|
|
|
// Fetch updates for remote
|
|
WIKI.logger.info('(STORAGE/GIT) Fetch updates from remote...')
|
|
await this.git.raw(['remote', 'update', 'origin'])
|
|
|
|
// Checkout branch
|
|
const branches = await this.git.branch()
|
|
if (!_.includes(branches.all, this.config.branch) && !_.includes(branches.all, `remotes/origin/${this.config.branch}`)) {
|
|
throw new Error('Invalid branch! Make sure it exists on the remote first.')
|
|
}
|
|
WIKI.logger.info(`(STORAGE/GIT) Checking out branch ${this.config.branch}...`)
|
|
await this.git.checkout(this.config.branch)
|
|
|
|
// Perform initial sync
|
|
await this.sync()
|
|
|
|
WIKI.logger.info('(STORAGE/GIT) Initialization completed.')
|
|
},
|
|
/**
|
|
* SYNC
|
|
*/
|
|
async sync() {
|
|
const currentCommitLog = _.get(await this.git.log(['-n', '1', this.config.branch]), 'latest', {})
|
|
|
|
const rootUser = await WIKI.models.users.getRootUser()
|
|
|
|
// Pull rebase
|
|
if (_.includes(['sync', 'pull'], this.mode)) {
|
|
WIKI.logger.info(`(STORAGE/GIT) Performing pull rebase from origin on branch ${this.config.branch}...`)
|
|
await this.git.pull('origin', this.config.branch, ['--rebase'])
|
|
}
|
|
|
|
// Push
|
|
if (_.includes(['sync', 'push'], this.mode)) {
|
|
WIKI.logger.info(`(STORAGE/GIT) Performing push to origin on branch ${this.config.branch}...`)
|
|
let pushOpts = ['--signed=if-asked']
|
|
if (this.mode === 'push') {
|
|
pushOpts.push('--force')
|
|
}
|
|
await this.git.push('origin', this.config.branch, pushOpts)
|
|
}
|
|
|
|
// Process Changes
|
|
if (_.includes(['sync', 'pull'], this.mode)) {
|
|
const latestCommitLog = _.get(await this.git.log(['-n', '1', this.config.branch]), 'latest', {})
|
|
|
|
const diff = await this.git.diffSummary(['-M', currentCommitLog.hash, latestCommitLog.hash])
|
|
if (_.get(diff, 'files', []).length > 0) {
|
|
let filesToProcess = []
|
|
for (const f of diff.files) {
|
|
const fMoved = f.file.split(' => ')
|
|
const fName = fMoved.length === 2 ? fMoved[1] : fMoved[0]
|
|
const fPath = path.join(this.repoPath, fName)
|
|
let fStats = { size: 0 }
|
|
try {
|
|
fStats = await fs.stat(fPath)
|
|
} catch (err) {
|
|
if (err.code !== 'ENOENT') {
|
|
WIKI.logger.warn(`(STORAGE/GIT) Failed to access file ${f.file}! Skipping...`)
|
|
continue
|
|
}
|
|
}
|
|
|
|
filesToProcess.push({
|
|
...f,
|
|
file: {
|
|
path: fPath,
|
|
stats: fStats
|
|
},
|
|
oldPath: fMoved[0],
|
|
relPath: fName
|
|
})
|
|
}
|
|
await this.processFiles(filesToProcess, rootUser)
|
|
}
|
|
}
|
|
},
|
|
/**
|
|
* Process Files
|
|
*
|
|
* @param {Array<String>} files Array of files to process
|
|
*/
|
|
async processFiles(files, user) {
|
|
for (const item of files) {
|
|
const contentType = pageHelper.getContentType(item.relPath)
|
|
const fileExists = await fs.pathExists(item.file.path)
|
|
if (!item.binary && contentType) {
|
|
// -> Page
|
|
|
|
if (fileExists && !item.importAll && item.relPath !== item.oldPath) {
|
|
// Page was renamed by git, so rename in DB
|
|
WIKI.logger.info(`(STORAGE/GIT) Page marked as renamed: from ${item.oldPath} to ${item.relPath}`)
|
|
|
|
const contentPath = pageHelper.getPagePath(item.oldPath)
|
|
const contentDestinationPath = pageHelper.getPagePath(item.relPath)
|
|
await WIKI.models.pages.movePage({
|
|
user: user,
|
|
path: contentPath.path,
|
|
destinationPath: contentDestinationPath.path,
|
|
locale: contentPath.locale,
|
|
destinationLocale: contentPath.locale,
|
|
skipStorage: true
|
|
})
|
|
} else if (!fileExists && !item.importAll && item.deletions > 0 && item.insertions === 0) {
|
|
// Page was deleted by git, can safely mark as deleted in DB
|
|
WIKI.logger.info(`(STORAGE/GIT) Page marked as deleted: ${item.relPath}`)
|
|
|
|
const contentPath = pageHelper.getPagePath(item.relPath)
|
|
await WIKI.models.pages.deletePage({
|
|
user: user,
|
|
path: contentPath.path,
|
|
locale: contentPath.locale,
|
|
skipStorage: true
|
|
})
|
|
continue
|
|
}
|
|
|
|
try {
|
|
await commonDisk.processPage({
|
|
user,
|
|
relPath: item.relPath,
|
|
fullPath: this.repoPath,
|
|
contentType: contentType,
|
|
moduleName: 'GIT'
|
|
})
|
|
} catch (err) {
|
|
WIKI.logger.warn(`(STORAGE/GIT) Failed to process ${item.relPath}`)
|
|
WIKI.logger.warn(err)
|
|
}
|
|
} else {
|
|
// -> Asset
|
|
|
|
if (fileExists && !item.importAll && ((item.before === item.after) || (item.deletions === 0 && item.insertions === 0))) {
|
|
// Asset was renamed by git, so rename in DB
|
|
WIKI.logger.info(`(STORAGE/GIT) Asset marked as renamed: from ${item.oldPath} to ${item.relPath}`)
|
|
|
|
const fileHash = assetHelper.generateHash(item.relPath)
|
|
const assetToRename = await WIKI.models.assets.query().findOne({ hash: fileHash })
|
|
if (assetToRename) {
|
|
await WIKI.models.assets.query().patch({
|
|
filename: item.relPath,
|
|
hash: fileHash
|
|
}).findById(assetToRename.id)
|
|
await assetToRename.deleteAssetCache()
|
|
} else {
|
|
WIKI.logger.info(`(STORAGE/GIT) Asset was not found in the DB, nothing to rename: ${item.relPath}`)
|
|
}
|
|
continue
|
|
} else if (!fileExists && !item.importAll && ((item.before > 0 && item.after === 0) || (item.deletions > 0 && item.insertions === 0))) {
|
|
// Asset was deleted by git, can safely mark as deleted in DB
|
|
WIKI.logger.info(`(STORAGE/GIT) Asset marked as deleted: ${item.relPath}`)
|
|
|
|
const fileHash = assetHelper.generateHash(item.relPath)
|
|
const assetToDelete = await WIKI.models.assets.query().findOne({ hash: fileHash })
|
|
if (assetToDelete) {
|
|
await WIKI.models.knex('assetData').where('id', assetToDelete.id).del()
|
|
await WIKI.models.assets.query().deleteById(assetToDelete.id)
|
|
await assetToDelete.deleteAssetCache()
|
|
} else {
|
|
WIKI.logger.info(`(STORAGE/GIT) Asset was not found in the DB, nothing to delete: ${item.relPath}`)
|
|
}
|
|
continue
|
|
}
|
|
|
|
try {
|
|
await commonDisk.processAsset({
|
|
user,
|
|
relPath: item.relPath,
|
|
file: item.file,
|
|
contentType: contentType,
|
|
moduleName: 'GIT'
|
|
})
|
|
} catch (err) {
|
|
WIKI.logger.warn(`(STORAGE/GIT) Failed to process asset ${item.relPath}`)
|
|
WIKI.logger.warn(err)
|
|
}
|
|
}
|
|
}
|
|
},
|
|
/**
|
|
* CREATE
|
|
*
|
|
* @param {Object} page Page to create
|
|
*/
|
|
async created(page) {
|
|
WIKI.logger.info(`(STORAGE/GIT) Committing new file [${page.localeCode}] ${page.path}...`)
|
|
let fileName = `${page.path}.${pageHelper.getFileExtension(page.contentType)}`
|
|
if (WIKI.config.lang.namespacing && WIKI.config.lang.code !== page.localeCode) {
|
|
fileName = `${page.localeCode}/${fileName}`
|
|
}
|
|
const filePath = path.join(this.repoPath, fileName)
|
|
await fs.outputFile(filePath, page.injectMetadata(), 'utf8')
|
|
|
|
await this.git.add(`./${fileName}`)
|
|
await this.git.commit(`docs: create ${page.path}`, fileName, {
|
|
'--author': `"${page.authorName} <${page.authorEmail}>"`
|
|
})
|
|
},
|
|
/**
|
|
* UPDATE
|
|
*
|
|
* @param {Object} page Page to update
|
|
*/
|
|
async updated(page) {
|
|
WIKI.logger.info(`(STORAGE/GIT) Committing updated file [${page.localeCode}] ${page.path}...`)
|
|
let fileName = `${page.path}.${pageHelper.getFileExtension(page.contentType)}`
|
|
if (WIKI.config.lang.namespacing && WIKI.config.lang.code !== page.localeCode) {
|
|
fileName = `${page.localeCode}/${fileName}`
|
|
}
|
|
const filePath = path.join(this.repoPath, fileName)
|
|
await fs.outputFile(filePath, page.injectMetadata(), 'utf8')
|
|
|
|
await this.git.add(`./${fileName}`)
|
|
await this.git.commit(`docs: update ${page.path}`, fileName, {
|
|
'--author': `"${page.authorName} <${page.authorEmail}>"`
|
|
})
|
|
},
|
|
/**
|
|
* DELETE
|
|
*
|
|
* @param {Object} page Page to delete
|
|
*/
|
|
async deleted(page) {
|
|
WIKI.logger.info(`(STORAGE/GIT) Committing removed file [${page.localeCode}] ${page.path}...`)
|
|
let fileName = `${page.path}.${pageHelper.getFileExtension(page.contentType)}`
|
|
if (WIKI.config.lang.namespacing && WIKI.config.lang.code !== page.localeCode) {
|
|
fileName = `${page.localeCode}/${fileName}`
|
|
}
|
|
|
|
await this.git.rm(`./${fileName}`)
|
|
await this.git.commit(`docs: delete ${page.path}`, fileName, {
|
|
'--author': `"${page.authorName} <${page.authorEmail}>"`
|
|
})
|
|
},
|
|
/**
|
|
* RENAME
|
|
*
|
|
* @param {Object} page Page to rename
|
|
*/
|
|
async renamed(page) {
|
|
WIKI.logger.info(`(STORAGE/GIT) Committing file move from [${page.localeCode}] ${page.path} to [${page.destinationLocaleCode}] ${page.destinationPath}...`)
|
|
let sourceFileName = `${page.path}.${pageHelper.getFileExtension(page.contentType)}`
|
|
let destinationFileName = `${page.destinationPath}.${pageHelper.getFileExtension(page.contentType)}`
|
|
|
|
if (WIKI.config.lang.namespacing) {
|
|
if (WIKI.config.lang.code !== page.localeCode) {
|
|
sourceFileName = `${page.localeCode}/${sourceFileName}`
|
|
}
|
|
if (WIKI.config.lang.code !== page.destinationLocaleCode) {
|
|
destinationFileName = `${page.destinationLocaleCode}/${destinationFileName}`
|
|
}
|
|
}
|
|
|
|
const sourceFilePath = path.join(this.repoPath, sourceFileName)
|
|
const destinationFilePath = path.join(this.repoPath, destinationFileName)
|
|
await fs.move(sourceFilePath, destinationFilePath)
|
|
|
|
await this.git.rm(`./${sourceFileName}`)
|
|
await this.git.add(`./${destinationFileName}`)
|
|
await this.git.commit(`docs: rename ${page.path} to ${page.destinationPath}`, [sourceFilePath, destinationFilePath], {
|
|
'--author': `"${page.moveAuthorName} <${page.moveAuthorEmail}>"`
|
|
})
|
|
},
|
|
/**
|
|
* ASSET UPLOAD
|
|
*
|
|
* @param {Object} asset Asset to upload
|
|
*/
|
|
async assetUploaded (asset) {
|
|
WIKI.logger.info(`(STORAGE/GIT) Committing new file ${asset.path}...`)
|
|
const filePath = path.join(this.repoPath, asset.path)
|
|
await fs.outputFile(filePath, asset.data, 'utf8')
|
|
|
|
await this.git.add(`./${asset.path}`)
|
|
await this.git.commit(`docs: upload ${asset.path}`, asset.path, {
|
|
'--author': `"${asset.authorName} <${asset.authorEmail}>"`
|
|
})
|
|
},
|
|
/**
|
|
* ASSET DELETE
|
|
*
|
|
* @param {Object} asset Asset to upload
|
|
*/
|
|
async assetDeleted (asset) {
|
|
WIKI.logger.info(`(STORAGE/GIT) Committing removed file ${asset.path}...`)
|
|
|
|
await this.git.rm(`./${asset.path}`)
|
|
await this.git.commit(`docs: delete ${asset.path}`, asset.path, {
|
|
'--author': `"${asset.authorName} <${asset.authorEmail}>"`
|
|
})
|
|
},
|
|
/**
|
|
* ASSET RENAME
|
|
*
|
|
* @param {Object} asset Asset to upload
|
|
*/
|
|
async assetRenamed (asset) {
|
|
WIKI.logger.info(`(STORAGE/GIT) Committing file move from ${asset.path} to ${asset.destinationPath}...`)
|
|
|
|
await this.git.mv(`./${asset.path}`, `./${asset.destinationPath}`)
|
|
await this.git.commit(`docs: rename ${asset.path} to ${asset.destinationPath}`, [asset.path, asset.destinationPath], {
|
|
'--author': `"${asset.moveAuthorName} <${asset.moveAuthorEmail}>"`
|
|
})
|
|
},
|
|
async getLocalLocation (asset) {
|
|
return path.join(this.repoPath, asset.path)
|
|
},
|
|
/**
|
|
* HANDLERS
|
|
*/
|
|
async importAll() {
|
|
WIKI.logger.info(`(STORAGE/GIT) Importing all content from local Git repo to the DB...`)
|
|
|
|
const rootUser = await WIKI.models.users.getRootUser()
|
|
|
|
await pipeline(
|
|
klaw(this.repoPath, {
|
|
filter: (f) => {
|
|
return !_.includes(f, '.git')
|
|
}
|
|
}),
|
|
new stream.Transform({
|
|
objectMode: true,
|
|
transform: async (file, enc, cb) => {
|
|
const relPath = file.path.substr(this.repoPath.length + 1)
|
|
if (file.stats.size < 1) {
|
|
// Skip directories and zero-byte files
|
|
return cb()
|
|
} else if (relPath && relPath.length > 3) {
|
|
WIKI.logger.info(`(STORAGE/GIT) Processing ${relPath}...`)
|
|
await this.processFiles([{
|
|
user: rootUser,
|
|
relPath,
|
|
file,
|
|
deletions: 0,
|
|
insertions: 0,
|
|
importAll: true
|
|
}], rootUser)
|
|
}
|
|
cb()
|
|
}
|
|
})
|
|
)
|
|
|
|
commonDisk.clearFolderCache()
|
|
|
|
WIKI.logger.info('(STORAGE/GIT) Import completed.')
|
|
},
|
|
async syncUntracked() {
|
|
WIKI.logger.info(`(STORAGE/GIT) Adding all untracked content...`)
|
|
|
|
// -> Pages
|
|
await pipeline(
|
|
WIKI.models.knex.column('id', 'path', 'localeCode', 'title', 'description', 'contentType', 'content', 'isPublished', 'updatedAt', 'createdAt', 'editorKey').select().from('pages').where({
|
|
isPrivate: false
|
|
}).stream(),
|
|
new stream.Transform({
|
|
objectMode: true,
|
|
transform: async (page, enc, cb) => {
|
|
const pageObject = await WIKI.models.pages.query().findById(page.id)
|
|
page.tags = await pageObject.$relatedQuery('tags')
|
|
|
|
let fileName = `${page.path}.${pageHelper.getFileExtension(page.contentType)}`
|
|
if (WIKI.config.lang.namespacing && WIKI.config.lang.code !== page.localeCode) {
|
|
fileName = `${page.localeCode}/${fileName}`
|
|
}
|
|
WIKI.logger.info(`(STORAGE/GIT) Adding page ${fileName}...`)
|
|
const filePath = path.join(this.repoPath, fileName)
|
|
await fs.outputFile(filePath, pageHelper.injectPageMetadata(page), 'utf8')
|
|
await this.git.add(`./${fileName}`)
|
|
cb()
|
|
}
|
|
})
|
|
)
|
|
|
|
// -> Assets
|
|
const assetFolders = await WIKI.models.assetFolders.getAllPaths()
|
|
|
|
await pipeline(
|
|
WIKI.models.knex.column('filename', 'folderId', 'data').select().from('assets').join('assetData', 'assets.id', '=', 'assetData.id').stream(),
|
|
new stream.Transform({
|
|
objectMode: true,
|
|
transform: async (asset, enc, cb) => {
|
|
const filename = (asset.folderId && asset.folderId > 0) ? `${_.get(assetFolders, asset.folderId)}/${asset.filename}` : asset.filename
|
|
WIKI.logger.info(`(STORAGE/GIT) Adding asset ${filename}...`)
|
|
await fs.outputFile(path.join(this.repoPath, filename), asset.data)
|
|
await this.git.add(`./${filename}`)
|
|
cb()
|
|
}
|
|
})
|
|
)
|
|
|
|
await this.git.commit(`docs: add all untracked content`)
|
|
WIKI.logger.info('(STORAGE/GIT) All content is now tracked.')
|
|
},
|
|
async purge() {
|
|
WIKI.logger.info(`(STORAGE/GIT) Purging local repository...`)
|
|
await fs.emptyDir(this.repoPath)
|
|
WIKI.logger.info('(STORAGE/GIT) Local repository is now empty. Reinitializing...')
|
|
await this.init()
|
|
}
|
|
}
|