From cffd32dee0c8fd30a3f0c5e676146f6ba2ee392e Mon Sep 17 00:00:00 2001 From: Nicolas Giard Date: Mon, 14 Oct 2019 23:44:37 -0400 Subject: [PATCH] feat: local disk import all action + v1 import content (#1100) --- server/core/auth.js | 1 - server/helpers/page.js | 42 ++++++++++-- server/models/editors.js | 2 + server/models/pages.js | 2 +- server/models/users.js | 10 +++ server/modules/storage/disk/definition.yml | 3 + server/modules/storage/disk/storage.js | 74 ++++++++++++++++++++++ server/modules/storage/git/storage.js | 57 +++++------------ 8 files changed, 140 insertions(+), 51 deletions(-) diff --git a/server/core/auth.js b/server/core/auth.js index 3f4d3ab6..81263fe7 100644 --- a/server/core/auth.js +++ b/server/core/auth.js @@ -1,7 +1,6 @@ const passport = require('passport') const passportJWT = require('passport-jwt') const _ = require('lodash') -const path = require('path') const jwt = require('jsonwebtoken') const moment = require('moment') const Promise = require('bluebird') diff --git a/server/helpers/page.js b/server/helpers/page.js index 24d8d935..4471281e 100644 --- a/server/helpers/page.js +++ b/server/helpers/page.js @@ -4,6 +4,13 @@ const crypto = require('crypto') const path = require('path') const localeSegmentRegex = /^[A-Z]{2}(-[A-Z]{2})?$/i +const localeFolderRegex = /^([a-z]{2}(?:-[a-z]{2})?\/)?(.*)/i + +const contentToExt = { + markdown: 'md', + html: 'html' +} +const extToContent = _.invert(contentToExt) /* global WIKI */ @@ -94,13 +101,34 @@ module.exports = { * Get file extension from content type */ getFileExtension(contentType) { - switch (contentType) { - case 'markdown': - return 'md' - case 'html': - return 'html' - default: - return 'txt' + _.get(contentToExt, contentType, 'txt') + }, + /** + * Get content type from file extension + */ + getContentType (filePath) { + const ext = _.last(filePath.split('.')) + return _.get(extToContent, ext, false) + }, + /** + * Get Page Meta object from disk path + */ + getPagePath (filePath) { + let fpath = filePath + if (process.platform === 'win32') { + fpath = filePath.replace(/\\/g, '/') } + let meta = { + locale: WIKI.config.lang.code, + path: _.initial(fpath.split('.')).join('') + } + const result = localeFolderRegex.exec(meta.path) + if (result[1]) { + meta = { + locale: result[1], + path: result[2] + } + } + return meta } } diff --git a/server/models/editors.js b/server/models/editors.js index 34a688a9..8a05ba98 100644 --- a/server/models/editors.js +++ b/server/models/editors.js @@ -99,6 +99,8 @@ module.exports = class Editor extends Model { switch (contentType) { case 'markdown': return 'markdown' + case 'html': + return 'ckeditor' default: return 'code' } diff --git a/server/models/pages.js b/server/models/pages.js index 84b94d6b..1ed10d8f 100644 --- a/server/models/pages.js +++ b/server/models/pages.js @@ -366,7 +366,7 @@ module.exports = class Page extends Model { } // -> Perform move? - if (opts.locale !== page.localeCode || opts.path !== page.path) { + if ((opts.locale && opts.locale !== page.localeCode) || (opts.path && opts.path !== page.path)) { await WIKI.models.pages.movePage({ id: page.id, destinationLocale: opts.locale, diff --git a/server/models/users.js b/server/models/users.js index 5d59349c..9371709c 100644 --- a/server/models/users.js +++ b/server/models/users.js @@ -712,4 +712,14 @@ module.exports = class User extends Model { user.permissions = user.getGlobalPermissions() return user } + + static async getRootUser () { + let user = await WIKI.models.users.query().findById(1) + if (!user) { + WIKI.logger.error('CRITICAL ERROR: Root Administrator user is missing!') + process.exit(1) + } + user.permissions = ['manage:system'] + return user + } } diff --git a/server/modules/storage/disk/definition.yml b/server/modules/storage/disk/definition.yml index 1b326fdf..357ab5cd 100644 --- a/server/modules/storage/disk/definition.yml +++ b/server/modules/storage/disk/definition.yml @@ -29,3 +29,6 @@ actions: - handler: backup label: Create Backup hint: Will create a manual backup archive at this point in time, in a subfolder named _manual, from the contents currently on disk. + - handler: importAll + label: Import Everything + hint: Will import all content currently in the local disk folder. diff --git a/server/modules/storage/disk/storage.js b/server/modules/storage/disk/storage.js index ef4553ac..4784618c 100644 --- a/server/modules/storage/disk/storage.js +++ b/server/modules/storage/disk/storage.js @@ -3,8 +3,10 @@ const path = require('path') const tar = require('tar-fs') const zlib = require('zlib') const stream = require('stream') +const _ = require('lodash') const Promise = require('bluebird') const pipeline = Promise.promisify(stream.pipeline) +const klaw = require('klaw') const pageHelper = require('../../../helpers/page.js') const moment = require('moment') @@ -113,5 +115,77 @@ module.exports = { }, async backup() { return this.sync({ manual: true }) + }, + async importAll() { + WIKI.logger.info(`(STORAGE/DISK) Importing all content from local disk folder to the DB...`) + + const rootUser = await WIKI.models.users.getRootUser() + + await pipeline( + klaw(this.config.path, { + filter: (f) => { + return !_.includes(f, '.git') + } + }), + new stream.Transform({ + objectMode: true, + transform: async (file, enc, cb) => { + const relPath = file.path.substr(this.config.path.length + 1) + if (relPath && relPath.length > 3) { + WIKI.logger.info(`(STORAGE/DISK) Processing ${relPath}...`) + const contentType = pageHelper.getContentType(relPath) + if (!contentType) { + return cb() + } + const contentPath = pageHelper.getPagePath(relPath) + + let itemContents = '' + try { + itemContents = await fs.readFile(path.join(this.config.path, relPath), 'utf8') + const pageData = WIKI.models.pages.parseMetadata(itemContents, contentType) + const currentPage = await WIKI.models.pages.query().findOne({ + path: contentPath.path, + localeCode: contentPath.locale + }) + if (currentPage) { + // Already in the DB, can mark as modified + WIKI.logger.info(`(STORAGE/DISK) Page marked as modified: ${relPath}`) + await WIKI.models.pages.updatePage({ + id: currentPage.id, + title: _.get(pageData, 'title', currentPage.title), + description: _.get(pageData, 'description', currentPage.description) || '', + isPublished: _.get(pageData, 'isPublished', currentPage.isPublished), + isPrivate: false, + content: pageData.content, + user: rootUser, + skipStorage: true + }) + } else { + // Not in the DB, can mark as new + WIKI.logger.info(`(STORAGE/DISK) Page marked as new: ${relPath}`) + const pageEditor = await WIKI.models.editors.getDefaultEditor(contentType) + await WIKI.models.pages.createPage({ + path: contentPath.path, + locale: contentPath.locale, + title: _.get(pageData, 'title', _.last(contentPath.path.split('/'))), + description: _.get(pageData, 'description', '') || '', + isPublished: _.get(pageData, 'isPublished', true), + isPrivate: false, + content: pageData.content, + user: rootUser, + editor: pageEditor, + skipStorage: true + }) + } + } catch (err) { + WIKI.logger.warn(`(STORAGE/DISK) Failed to process ${relPath}`) + WIKI.logger.warn(err) + } + } + cb() + } + }) + ) + WIKI.logger.info('(STORAGE/DISK) Import completed.') } } diff --git a/server/modules/storage/git/storage.js b/server/modules/storage/git/storage.js index b71e00e6..41052b93 100644 --- a/server/modules/storage/git/storage.js +++ b/server/modules/storage/git/storage.js @@ -8,41 +8,8 @@ const pipeline = Promise.promisify(stream.pipeline) const klaw = require('klaw') const pageHelper = require('../../../helpers/page.js') -const localeFolderRegex = /^([a-z]{2}(?:-[a-z]{2})?\/)?(.*)/i - /* global WIKI */ -const getContenType = (filePath) => { - const ext = _.last(filePath.split('.')) - switch (ext) { - case 'md': - return 'markdown' - case 'html': - return 'html' - default: - return false - } -} - -const getPagePath = (filePath) => { - let fpath = filePath - if (process.platform === 'win32') { - fpath = filePath.replace(/\\/g, '/') - } - let meta = { - locale: 'en', - path: _.initial(fpath.split('.')).join('') - } - const result = localeFolderRegex.exec(meta.path) - if (result[1]) { - meta = { - locale: result[1], - path: result[2] - } - } - return meta -} - module.exports = { git: null, repoPath: path.join(process.cwd(), 'data/repo'), @@ -145,6 +112,8 @@ module.exports = { async sync() { const currentCommitLog = _.get(await this.git.log(['-n', '1', this.config.branch]), 'latest', {}) + const rootUser = await WIKI.models.users.getRootUser() + // Pull rebase if (_.includes(['sync', 'pull'], this.mode)) { WIKI.logger.info(`(STORAGE/GIT) Performing pull rebase from origin on branch ${this.config.branch}...`) @@ -167,7 +136,7 @@ module.exports = { const diff = await this.git.diffSummary(['-M', currentCommitLog.hash, latestCommitLog.hash]) if (_.get(diff, 'files', []).length > 0) { - await this.processFiles(diff.files) + await this.processFiles(diff.files, rootUser) } } }, @@ -176,13 +145,13 @@ module.exports = { * * @param {Array} files Array of files to process */ - async processFiles(files) { + async processFiles(files, user) { for (const item of files) { - const contentType = getContenType(item.file) + const contentType = pageHelper.getContentType(item.file) if (!contentType) { continue } - const contentPath = getPagePath(item.file) + const contentPath = pageHelper.getPagePath(item.file) let itemContents = '' try { @@ -202,7 +171,7 @@ module.exports = { isPublished: _.get(pageData, 'isPublished', currentPage.isPublished), isPrivate: false, content: pageData.content, - authorId: 1, + user: user, skipStorage: true }) } else { @@ -217,7 +186,7 @@ module.exports = { isPublished: _.get(pageData, 'isPublished', true), isPrivate: false, content: pageData.content, - authorId: 1, + user: user, editor: pageEditor, skipStorage: true }) @@ -233,8 +202,7 @@ module.exports = { skipStorage: true }) } else { - WIKI.logger.warn(`(STORAGE/GIT) Failed to open ${item.file}`) - console.error(err) + WIKI.logger.warn(`(STORAGE/GIT) Failed to process ${item.file}`) WIKI.logger.warn(err) } } @@ -365,6 +333,9 @@ module.exports = { */ async importAll() { WIKI.logger.info(`(STORAGE/GIT) Importing all content from local Git repo to the DB...`) + + const rootUser = await WIKI.models.users.getRootUser() + await pipeline( klaw(this.repoPath, { filter: (f) => { @@ -378,10 +349,11 @@ module.exports = { if (relPath && relPath.length > 3) { WIKI.logger.info(`(STORAGE/GIT) Processing ${relPath}...`) await this.processFiles([{ + user: rootUser, file: relPath, deletions: 0, insertions: 0 - }]) + }], rootUser) } cb() } @@ -391,6 +363,7 @@ module.exports = { }, async syncUntracked() { WIKI.logger.info(`(STORAGE/GIT) Adding all untracked content...`) + await pipeline( WIKI.models.knex.column('path', 'localeCode', 'title', 'description', 'contentType', 'content', 'isPublished', 'updatedAt').select().from('pages').where({ isPrivate: false