const path = require('path') const sgit = require('simple-git/promise') const fs = require('fs-extra') const _ = require('lodash') const stream = require('stream') const Promise = require('bluebird') const pipeline = Promise.promisify(stream.pipeline) const klaw = require('klaw') const pageHelper = require('../../../helpers/page.js') const localeFolderRegex = /^([a-z]{2}(?:-[a-z]{2})?\/)?(.*)/i /* global WIKI */ /** * Get file extension based on content type */ const getFileExtension = (contentType) => { switch (contentType) { case 'markdown': return 'md' case 'html': return 'html' default: return 'txt' } } const getContenType = (filePath) => { const ext = _.last(filePath.split('.')) switch (ext) { case 'md': return 'markdown' case 'html': return 'html' default: return false } } const getPagePath = (filePath) => { let meta = { locale: 'en', path: _.initial(filePath.split('.')).join('') } const result = localeFolderRegex.exec(meta.path) if (result[1]) { meta = { locale: result[1], path: result[2] } } return meta } module.exports = { git: null, repoPath: path.join(process.cwd(), 'data/repo'), async activated() { // not used }, async deactivated() { // not used }, /** * INIT */ async init() { WIKI.logger.info('(STORAGE/GIT) Initializing...') this.repoPath = path.resolve(WIKI.ROOTPATH, this.config.localRepoPath) await fs.ensureDir(this.repoPath) this.git = sgit(this.repoPath) // Set custom binary path if (!_.isEmpty(this.config.gitBinaryPath)) { this.git.customBinary(this.config.gitBinaryPath) } // Initialize repo (if needed) WIKI.logger.info('(STORAGE/GIT) Checking repository state...') const isRepo = await this.git.checkIsRepo() if (!isRepo) { WIKI.logger.info('(STORAGE/GIT) Initializing local repository...') await this.git.init() } // Set default author await this.git.raw(['config', '--local', 'user.email', this.config.defaultEmail]) await this.git.raw(['config', '--local', 'user.name', this.config.defaultName]) // Purge existing remotes WIKI.logger.info('(STORAGE/GIT) Listing existing remotes...') const remotes = await this.git.getRemotes() if (remotes.length > 0) { WIKI.logger.info('(STORAGE/GIT) Purging existing remotes...') for (let remote of remotes) { await this.git.removeRemote(remote.name) } } // Add remote WIKI.logger.info('(STORAGE/GIT) Setting SSL Verification config...') await this.git.raw(['config', '--local', '--bool', 'http.sslVerify', _.toString(this.config.verifySSL)]) switch (this.config.authType) { case 'ssh': WIKI.logger.info('(STORAGE/GIT) Setting SSH Command config...') await this.git.addConfig('core.sshCommand', `ssh -i "${this.config.sshPrivateKeyPath}" -o StrictHostKeyChecking=no`) WIKI.logger.info('(STORAGE/GIT) Adding origin remote via SSH...') await this.git.addRemote('origin', this.config.repoUrl) break default: WIKI.logger.info('(STORAGE/GIT) Adding origin remote via HTTP/S...') let originUrl = '' if (_.startsWith(this.config.repoUrl, 'http')) { originUrl = this.config.repoUrl.replace('://', `://${this.config.basicUsername}:${this.config.basicPassword}@`) } else { originUrl = `https://${this.config.basicUsername}:${this.config.basicPassword}@${this.config.repoUrl}` } await this.git.addRemote('origin', originUrl) break } // Fetch updates for remote WIKI.logger.info('(STORAGE/GIT) Fetch updates from remote...') await this.git.raw(['remote', 'update', 'origin']) // Checkout branch const branches = await this.git.branch() if (!_.includes(branches.all, this.config.branch) && !_.includes(branches.all, `remotes/origin/${this.config.branch}`)) { throw new Error('Invalid branch! Make sure it exists on the remote first.') } WIKI.logger.info(`(STORAGE/GIT) Checking out branch ${this.config.branch}...`) await this.git.checkout(this.config.branch) // Perform initial sync await this.sync() WIKI.logger.info('(STORAGE/GIT) Initialization completed.') }, /** * SYNC */ async sync() { const currentCommitLog = _.get(await this.git.log(['-n', '1', this.config.branch]), 'latest', {}) // Pull rebase if (_.includes(['sync', 'pull'], this.mode)) { WIKI.logger.info(`(STORAGE/GIT) Performing pull rebase from origin on branch ${this.config.branch}...`) await this.git.pull('origin', this.config.branch, ['--rebase']) } // Push if (_.includes(['sync', 'push'], this.mode)) { WIKI.logger.info(`(STORAGE/GIT) Performing push to origin on branch ${this.config.branch}...`) let pushOpts = ['--signed=if-asked'] if (this.mode === 'push') { pushOpts.push('--force') } await this.git.push('origin', this.config.branch, pushOpts) } // Process Changes if (_.includes(['sync', 'pull'], this.mode)) { const latestCommitLog = _.get(await this.git.log(['-n', '1', this.config.branch]), 'latest', {}) const diff = await this.git.diffSummary(['-M', currentCommitLog.hash, latestCommitLog.hash]) if (_.get(diff, 'files', []).length > 0) { await this.processFiles(diff.files) } } }, /** * Process Files * * @param {Array} files Array of files to process */ async processFiles(files) { for (const item of files) { const contentType = getContenType(item.file) if (!contentType) { continue } const contentPath = getPagePath(item.file) let itemContents = '' try { itemContents = await fs.readFile(path.join(this.repoPath, item.file), 'utf8') const pageData = WIKI.models.pages.parseMetadata(itemContents, contentType) const currentPage = await WIKI.models.pages.query().findOne({ path: contentPath.path, localeCode: contentPath.locale }) if (currentPage) { // Already in the DB, can mark as modified WIKI.logger.info(`(STORAGE/GIT) Page marked as modified: ${item.file}`) await WIKI.models.pages.updatePage({ id: currentPage.id, title: _.get(pageData, 'title', currentPage.title), description: _.get(pageData, 'description', currentPage.description), isPublished: _.get(pageData, 'isPublished', currentPage.isPublished), isPrivate: false, content: pageData.content, authorId: 1, skipStorage: true }) } else { // Not in the DB, can mark as new WIKI.logger.info(`(STORAGE/GIT) Page marked as new: ${item.file}`) const pageEditor = await WIKI.models.editors.getDefaultEditor(contentType) await WIKI.models.pages.createPage({ path: contentPath.path, locale: contentPath.locale, title: _.get(pageData, 'title', _.last(contentPath.path.split('/'))), description: _.get(pageData, 'description', ''), isPublished: _.get(pageData, 'isPublished', true), isPrivate: false, content: pageData.content, authorId: 1, editor: pageEditor, skipStorage: true }) } } catch (err) { if (err.code === 'ENOENT' && item.deletions > 0 && item.insertions === 0) { // File was deleted by git, can safely mark as deleted in DB WIKI.logger.info(`(STORAGE/GIT) Page marked as deleted: ${item.file}`) await WIKI.models.pages.deletePage({ path: contentPath.path, locale: contentPath.locale, skipStorage: true }) } else { WIKI.logger.warn(`(STORAGE/GIT) Failed to open ${item.file}`) WIKI.logger.warn(err) } } } }, /** * CREATE * * @param {Object} page Page to create */ async created(page) { WIKI.logger.info(`(STORAGE/GIT) Committing new file ${page.path}...`) let fileName = `${page.path}.${getFileExtension(page.contentType)}` if (WIKI.config.lang.namespacing && WIKI.config.lang.code !== page.localeCode) { fileName = `${page.localeCode}/${fileName}` } const filePath = path.join(this.repoPath, fileName) await fs.outputFile(filePath, page.injectMetadata(), 'utf8') await this.git.add(`./${fileName}`) await this.git.commit(`docs: create ${page.path}`, fileName, { '--author': `"${page.authorName} <${page.authorEmail}>"` }) }, /** * UPDATE * * @param {Object} page Page to update */ async updated(page) { WIKI.logger.info(`(STORAGE/GIT) Committing updated file ${page.path}...`) let fileName = `${page.path}.${getFileExtension(page.contentType)}` if (WIKI.config.lang.namespacing && WIKI.config.lang.code !== page.localeCode) { fileName = `${page.localeCode}/${fileName}` } const filePath = path.join(this.repoPath, fileName) await fs.outputFile(filePath, page.injectMetadata(), 'utf8') await this.git.add(`./${fileName}`) await this.git.commit(`docs: update ${page.path}`, fileName, { '--author': `"${page.authorName} <${page.authorEmail}>"` }) }, /** * DELETE * * @param {Object} page Page to delete */ async deleted(page) { WIKI.logger.info(`(STORAGE/GIT) Committing removed file ${page.path}...`) let fileName = `${page.path}.${getFileExtension(page.contentType)}` if (WIKI.config.lang.namespacing && WIKI.config.lang.code !== page.localeCode) { fileName = `${page.localeCode}/${fileName}` } await this.git.rm(`./${fileName}`) await this.git.commit(`docs: delete ${page.path}`, fileName, { '--author': `"${page.authorName} <${page.authorEmail}>"` }) }, /** * RENAME * * @param {Object} page Page to rename */ async renamed(page) { WIKI.logger.info(`(STORAGE/GIT) Committing file move from ${page.sourcePath} to ${page.destinationPath}...`) let sourceFilePath = `${page.sourcePath}.${getFileExtension(page.contentType)}` let destinationFilePath = `${page.destinationPath}.${getFileExtension(page.contentType)}` if (WIKI.config.lang.namespacing && WIKI.config.lang.code !== page.localeCode) { sourceFilePath = `${page.localeCode}/${sourceFilePath}` destinationFilePath = `${page.localeCode}/${destinationFilePath}` } await this.git.mv(`./${sourceFilePath}`, `./${destinationFilePath}`) await this.git.commit(`docs: rename ${page.sourcePath} to ${destinationFilePath}`, destinationFilePath, { '--author': `"${page.authorName} <${page.authorEmail}>"` }) }, /** * HANDLERS */ async importAll() { WIKI.logger.info(`(STORAGE/GIT) Importing all content from local Git repo to the DB...`) await pipeline( klaw(this.repoPath, { filter: (f) => { return !_.includes(f, '.git') } }), new stream.Transform({ objectMode: true, transform: async (file, enc, cb) => { const relPath = file.path.substr(this.repoPath.length + 1) if (relPath && relPath.length > 3) { WIKI.logger.info(`(STORAGE/GIT) Processing ${relPath}...`) await this.processFiles([{ file: relPath, deletions: 0, insertions: 0 }]) } cb() } }) ) WIKI.logger.info('(STORAGE/GIT) Import completed.') }, async syncUntracked() { WIKI.logger.info(`(STORAGE/GIT) Adding all untracked content...`) await pipeline( WIKI.models.knex.column('path', 'localeCode', 'title', 'description', 'contentType', 'content', 'isPublished', 'updatedAt').select().from('pages').where({ isPrivate: false }).stream(), new stream.Transform({ objectMode: true, transform: async (page, enc, cb) => { let fileName = `${page.path}.${getFileExtension(page.contentType)}` if (WIKI.config.lang.namespacing && WIKI.config.lang.code !== page.localeCode) { fileName = `${page.localeCode}/${fileName}` } WIKI.logger.info(`(STORAGE/GIT) Adding ${fileName}...`) const filePath = path.join(this.repoPath, fileName) await fs.outputFile(filePath, pageHelper.injectPageMetadata(page), 'utf8') await this.git.add(`./${fileName}`) cb() } }) ) await this.git.commit(`docs: add all untracked content`) WIKI.logger.info('(STORAGE/GIT) All content is now tracked.') } }