liz/wikijs-fork
liz/wikijs-fork
1
0
Fork 0
Code Issues Pull Requests Packages Projects Releases Wiki Activity

feat: local disk import all action + v1 import content (#1100)

Browse Source
This commit is contained in:
Nicolas Giard 2019-10-14 23:44:37 -04:00 committed by GitHub
parent 334d0a754e
commit cffd32dee0
No known key found for this signature in database
GPG Key ID: 4AEE18F83AFDEB23
8 changed files with 140 additions and 51 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

1
server/core/auth.js
View File

@ -1,7 +1,6 @@
const passport = require('passport')
const passportJWT = require('passport-jwt')
const _ = require('lodash')
const path = require('path')
const jwt = require('jsonwebtoken')
const moment = require('moment')
const Promise = require('bluebird')

42
server/helpers/page.js
View File

@ -4,6 +4,13 @@ const crypto = require('crypto')
const path = require('path')
const localeSegmentRegex = /^[A-Z]{2}(-[A-Z]{2})?$/i
const localeFolderRegex = /^([a-z]{2}(?:-[a-z]{2})?\/)?(.*)/i
const contentToExt = {
markdown: 'md',
html: 'html'
}
const extToContent = _.invert(contentToExt)
/* global WIKI */
@ -94,13 +101,34 @@ module.exports = {
* Get file extension from content type
*/
getFileExtension(contentType) {
switch (contentType) {
case 'markdown':
return 'md'
case 'html':
return 'html'
default:
return 'txt'
_.get(contentToExt, contentType, 'txt')
},
/**
* Get content type from file extension
*/
getContentType (filePath) {
const ext = _.last(filePath.split('.'))
return _.get(extToContent, ext, false)
},
/**
* Get Page Meta object from disk path
*/
getPagePath (filePath) {
let fpath = filePath
if (process.platform === 'win32') {
fpath = filePath.replace(/\\/g, '/')
}
let meta = {
locale: WIKI.config.lang.code,
path: _.initial(fpath.split('.')).join('')
}
const result = localeFolderRegex.exec(meta.path)
if (result[1]) {
meta = {
locale: result[1],
path: result[2]
}
}
return meta
}
}

2
server/models/editors.js
View File

@ -99,6 +99,8 @@ module.exports = class Editor extends Model {
switch (contentType) {
case 'markdown':
return 'markdown'
case 'html':
return 'ckeditor'
default:
return 'code'
}

2
server/models/pages.js
View File

@ -366,7 +366,7 @@ module.exports = class Page extends Model {
}
// -> Perform move?
if (opts.locale !== page.localeCode || opts.path !== page.path) {
if ((opts.locale && opts.locale !== page.localeCode) || (opts.path && opts.path !== page.path)) {
await WIKI.models.pages.movePage({
id: page.id,
destinationLocale: opts.locale,

10
server/models/users.js
View File

@ -712,4 +712,14 @@ module.exports = class User extends Model {
user.permissions = user.getGlobalPermissions()
return user
}
static async getRootUser () {
let user = await WIKI.models.users.query().findById(1)
if (!user) {
WIKI.logger.error('CRITICAL ERROR: Root Administrator user is missing!')
process.exit(1)
}
user.permissions = ['manage:system']
return user
}
}

3
server/modules/storage/disk/definition.yml
View File

@ -29,3 +29,6 @@ actions:
- handler: backup
label: Create Backup
hint: Will create a manual backup archive at this point in time, in a subfolder named _manual, from the contents currently on disk.
- handler: importAll
label: Import Everything
hint: Will import all content currently in the local disk folder.

74
server/modules/storage/disk/storage.js
View File

@ -3,8 +3,10 @@ const path = require('path')
const tar = require('tar-fs')
const zlib = require('zlib')
const stream = require('stream')
const _ = require('lodash')
const Promise = require('bluebird')
const pipeline = Promise.promisify(stream.pipeline)
const klaw = require('klaw')
const pageHelper = require('../../../helpers/page.js')
const moment = require('moment')
@ -113,5 +115,77 @@ module.exports = {
},
async backup() {
return this.sync({ manual: true })
},
async importAll() {
WIKI.logger.info(`(STORAGE/DISK) Importing all content from local disk folder to the DB...`)
const rootUser = await WIKI.models.users.getRootUser()
await pipeline(
klaw(this.config.path, {
filter: (f) => {
return !_.includes(f, '.git')
}
}),
new stream.Transform({
objectMode: true,
transform: async (file, enc, cb) => {
const relPath = file.path.substr(this.config.path.length + 1)
if (relPath && relPath.length > 3) {
WIKI.logger.info(`(STORAGE/DISK) Processing ${relPath}...`)
const contentType = pageHelper.getContentType(relPath)
if (!contentType) {
return cb()
}
const contentPath = pageHelper.getPagePath(relPath)
let itemContents = ''
try {
itemContents = await fs.readFile(path.join(this.config.path, relPath), 'utf8')
const pageData = WIKI.models.pages.parseMetadata(itemContents, contentType)
const currentPage = await WIKI.models.pages.query().findOne({
path: contentPath.path,
localeCode: contentPath.locale
})
if (currentPage) {
// Already in the DB, can mark as modified
WIKI.logger.info(`(STORAGE/DISK) Page marked as modified: ${relPath}`)
await WIKI.models.pages.updatePage({
id: currentPage.id,
title: _.get(pageData, 'title', currentPage.title),
description: _.get(pageData, 'description', currentPage.description) || '',
isPublished: _.get(pageData, 'isPublished', currentPage.isPublished),
isPrivate: false,
content: pageData.content,
user: rootUser,
skipStorage: true
})
} else {
// Not in the DB, can mark as new
WIKI.logger.info(`(STORAGE/DISK) Page marked as new: ${relPath}`)
const pageEditor = await WIKI.models.editors.getDefaultEditor(contentType)
await WIKI.models.pages.createPage({
path: contentPath.path,
locale: contentPath.locale,
title: _.get(pageData, 'title', _.last(contentPath.path.split('/'))),
description: _.get(pageData, 'description', '') || '',
isPublished: _.get(pageData, 'isPublished', true),
isPrivate: false,
content: pageData.content,
user: rootUser,
editor: pageEditor,
skipStorage: true
})
}
} catch (err) {
WIKI.logger.warn(`(STORAGE/DISK) Failed to process ${relPath}`)
WIKI.logger.warn(err)
}
}
cb()
}
})
)
WIKI.logger.info('(STORAGE/DISK) Import completed.')
}
}

57
server/modules/storage/git/storage.js
View File

@ -8,41 +8,8 @@ const pipeline = Promise.promisify(stream.pipeline)
const klaw = require('klaw')
const pageHelper = require('../../../helpers/page.js')
const localeFolderRegex = /^([a-z]{2}(?:-[a-z]{2})?\/)?(.*)/i
/* global WIKI */
const getContenType = (filePath) => {
const ext = _.last(filePath.split('.'))
switch (ext) {
case 'md':
return 'markdown'
case 'html':
return 'html'
default:
return false
}
}
const getPagePath = (filePath) => {
let fpath = filePath
if (process.platform === 'win32') {
fpath = filePath.replace(/\\/g, '/')
}
let meta = {
locale: 'en',
path: _.initial(fpath.split('.')).join('')
}
const result = localeFolderRegex.exec(meta.path)
if (result[1]) {
meta = {
locale: result[1],
path: result[2]
}
}
return meta
}
module.exports = {
git: null,
repoPath: path.join(process.cwd(), 'data/repo'),
@ -145,6 +112,8 @@ module.exports = {
async sync() {
const currentCommitLog = _.get(await this.git.log(['-n', '1', this.config.branch]), 'latest', {})
const rootUser = await WIKI.models.users.getRootUser()
// Pull rebase
if (_.includes(['sync', 'pull'], this.mode)) {
WIKI.logger.info(`(STORAGE/GIT) Performing pull rebase from origin on branch ${this.config.branch}...`)
@ -167,7 +136,7 @@ module.exports = {
const diff = await this.git.diffSummary(['-M', currentCommitLog.hash, latestCommitLog.hash])
if (_.get(diff, 'files', []).length > 0) {
await this.processFiles(diff.files)
await this.processFiles(diff.files, rootUser)
}
}
},
@ -176,13 +145,13 @@ module.exports = {
*
* @param {Array<String>} files Array of files to process
*/
async processFiles(files) {
async processFiles(files, user) {
for (const item of files) {
const contentType = getContenType(item.file)
const contentType = pageHelper.getContentType(item.file)
if (!contentType) {
continue
}
const contentPath = getPagePath(item.file)
const contentPath = pageHelper.getPagePath(item.file)
let itemContents = ''
try {
@ -202,7 +171,7 @@ module.exports = {
isPublished: _.get(pageData, 'isPublished', currentPage.isPublished),
isPrivate: false,
content: pageData.content,
authorId: 1,
user: user,
skipStorage: true
})
} else {
@ -217,7 +186,7 @@ module.exports = {
isPublished: _.get(pageData, 'isPublished', true),
isPrivate: false,
content: pageData.content,
authorId: 1,
user: user,
editor: pageEditor,
skipStorage: true
})
@ -233,8 +202,7 @@ module.exports = {
skipStorage: true
})
} else {
WIKI.logger.warn(`(STORAGE/GIT) Failed to open ${item.file}`)
console.error(err)
WIKI.logger.warn(`(STORAGE/GIT) Failed to process ${item.file}`)
WIKI.logger.warn(err)
}
}
@ -365,6 +333,9 @@ module.exports = {
*/
async importAll() {
WIKI.logger.info(`(STORAGE/GIT) Importing all content from local Git repo to the DB...`)
const rootUser = await WIKI.models.users.getRootUser()
await pipeline(
klaw(this.repoPath, {
filter: (f) => {
@ -378,10 +349,11 @@ module.exports = {
if (relPath && relPath.length > 3) {
WIKI.logger.info(`(STORAGE/GIT) Processing ${relPath}...`)
await this.processFiles([{
user: rootUser,
file: relPath,
deletions: 0,
insertions: 0
}])
}], rootUser)
}
cb()
}
@ -391,6 +363,7 @@ module.exports = {
},
async syncUntracked() {
WIKI.logger.info(`(STORAGE/GIT) Adding all untracked content...`)
await pipeline(
WIKI.models.knex.column('path', 'localeCode', 'title', 'description', 'contentType', 'content', 'isPublished', 'updatedAt').select().from('pages').where({
isPrivate: false