From f1668b9ac51eb19956b7580a48020aa4455f0442 Mon Sep 17 00:00:00 2001 From: NGPixel Date: Sun, 20 Oct 2019 14:42:10 -0400 Subject: [PATCH] feat: import assets from disk module --- client/components/admin.vue | 6 +- server/controllers/upload.js | 1 + server/models/assets.js | 13 ++- server/modules/storage/disk/storage.js | 135 +++++++++++++++++-------- 4 files changed, 106 insertions(+), 49 deletions(-) diff --git a/client/components/admin.vue b/client/components/admin.vue index d633e817..a0d680eb 100644 --- a/client/components/admin.vue +++ b/client/components/admin.vue @@ -26,7 +26,7 @@ v-list-item(to='/pages', v-if='hasPermission([`manage:system`, `write:pages`, `manage:pages`, `delete:pages`])') v-list-item-avatar(size='24'): v-icon mdi-file-document-outline v-list-item-title {{ $t('admin:pages.title') }} - v-list-item-action.pr-3 + v-list-item-action(style='min-width:auto;') v-chip(x-small, :color='darkMode ? `grey darken-3-d4` : `grey lighten-5`') .caption.grey--text {{ info.pagesTotal }} v-list-item(to='/theme', v-if='hasPermission([`manage:system`, `manage:theme`])') @@ -38,13 +38,13 @@ v-list-item(to='/groups', v-if='hasPermission([`manage:system`, `manage:groups`, `write:groups`])') v-list-item-avatar(size='24'): v-icon mdi-account-group v-list-item-title {{ $t('admin:groups.title') }} - v-list-item-action.pr-3 + v-list-item-action(style='min-width:auto;') v-chip(x-small, :color='darkMode ? `grey darken-3-d4` : `grey lighten-4`') .caption.grey--text {{ info.groupsTotal }} v-list-item(to='/users', v-if='hasPermission([`manage:system`, `manage:groups`, `write:groups`, `manage:users`, `write:users`])') v-list-item-avatar(size='24'): v-icon mdi-account-box v-list-item-title {{ $t('admin:users.title') }} - v-list-item-action.pr-3 + v-list-item-action(style='min-width:auto;') v-chip(x-small, :color='darkMode ? `grey darken-3-d4` : `grey lighten-4`') .caption.grey--text {{ info.usersTotal }} template(v-if='hasPermission(`manage:system`)') diff --git a/server/controllers/upload.js b/server/controllers/upload.js index 493f45be..9ee5fa5b 100644 --- a/server/controllers/upload.js +++ b/server/controllers/upload.js @@ -88,6 +88,7 @@ router.post('/u', multer({ // Process upload file await WIKI.models.assets.upload({ ...fileMeta, + mode: 'upload', folderId: folderId, assetPath, user: req.user diff --git a/server/models/assets.js b/server/models/assets.js index 8c6fb1e3..d0e0f3c3 100644 --- a/server/models/assets.js +++ b/server/models/assets.js @@ -95,8 +95,7 @@ module.exports = class Asset extends Model { kind: _.startsWith(opts.mimetype, 'image/') ? 'image' : 'binary', mime: opts.mimetype, fileSize: opts.size, - folderId: opts.folderId, - authorId: opts.user.id + folderId: opts.folderId } // Save asset data @@ -105,6 +104,9 @@ module.exports = class Asset extends Model { if (asset) { // Patch existing asset + if (opts.mode === 'upload') { + assetRow.authorId = opts.user.id + } await WIKI.models.assets.query().patch(assetRow).findById(asset.id) await WIKI.models.knex('assetData').where({ id: asset.id @@ -113,6 +115,7 @@ module.exports = class Asset extends Model { }) } else { // Create asset entry + assetRow.authorId = opts.user.id asset = await WIKI.models.assets.query().insert(assetRow) await WIKI.models.knex('assetData').insert({ id: asset.id, @@ -121,7 +124,11 @@ module.exports = class Asset extends Model { } // Move temp upload to cache - await fs.move(opts.path, path.join(process.cwd(), `data/cache/${fileHash}.dat`), { overwrite: true }) + if (opts.mode === 'upload') { + await fs.move(opts.path, path.join(process.cwd(), `data/cache/${fileHash}.dat`), { overwrite: true }) + } else { + await fs.copy(opts.path, path.join(process.cwd(), `data/cache/${fileHash}.dat`), { overwrite: true }) + } // Add to Storage if (!opts.skipStorage) { diff --git a/server/modules/storage/disk/storage.js b/server/modules/storage/disk/storage.js index f56105ca..e13e3c5c 100644 --- a/server/modules/storage/disk/storage.js +++ b/server/modules/storage/disk/storage.js @@ -9,6 +9,7 @@ const pipeline = Promise.promisify(stream.pipeline) const klaw = require('klaw') const pageHelper = require('../../../helpers/page.js') const moment = require('moment') +const mime = require('mime-types').lookup /* global WIKI */ @@ -167,6 +168,7 @@ module.exports = { WIKI.logger.info(`(STORAGE/DISK) Importing all content from local disk folder to the DB...`) const rootUser = await WIKI.models.users.getRootUser() + let assetFolders = await WIKI.models.assetFolders.getAllPaths() await pipeline( klaw(this.config.path, { @@ -178,55 +180,102 @@ module.exports = { objectMode: true, transform: async (file, enc, cb) => { const relPath = file.path.substr(this.config.path.length + 1) - if (relPath && relPath.length > 3) { + if (file.stats.size < 1) { + // Skip directories and zero-byte files + return cb() + } else if (relPath && relPath.length > 3) { WIKI.logger.info(`(STORAGE/DISK) Processing ${relPath}...`) const contentType = pageHelper.getContentType(relPath) - if (!contentType) { - return cb() - } - const contentPath = pageHelper.getPagePath(relPath) + if (contentType) { + // -> Page + const contentPath = pageHelper.getPagePath(relPath) - let itemContents = '' - try { - itemContents = await fs.readFile(path.join(this.config.path, relPath), 'utf8') - const pageData = WIKI.models.pages.parseMetadata(itemContents, contentType) - const currentPage = await WIKI.models.pages.query().findOne({ - path: contentPath.path, - localeCode: contentPath.locale - }) - if (currentPage) { - // Already in the DB, can mark as modified - WIKI.logger.info(`(STORAGE/DISK) Page marked as modified: ${relPath}`) - await WIKI.models.pages.updatePage({ - id: currentPage.id, - title: _.get(pageData, 'title', currentPage.title), - description: _.get(pageData, 'description', currentPage.description) || '', - isPublished: _.get(pageData, 'isPublished', currentPage.isPublished), - isPrivate: false, - content: pageData.content, - user: rootUser, - skipStorage: true - }) - } else { - // Not in the DB, can mark as new - WIKI.logger.info(`(STORAGE/DISK) Page marked as new: ${relPath}`) - const pageEditor = await WIKI.models.editors.getDefaultEditor(contentType) - await WIKI.models.pages.createPage({ + let itemContents = '' + try { + itemContents = await fs.readFile(path.join(this.config.path, relPath), 'utf8') + const pageData = WIKI.models.pages.parseMetadata(itemContents, contentType) + const currentPage = await WIKI.models.pages.query().findOne({ path: contentPath.path, - locale: contentPath.locale, - title: _.get(pageData, 'title', _.last(contentPath.path.split('/'))), - description: _.get(pageData, 'description', '') || '', - isPublished: _.get(pageData, 'isPublished', true), - isPrivate: false, - content: pageData.content, - user: rootUser, - editor: pageEditor, - skipStorage: true + localeCode: contentPath.locale }) + if (currentPage) { + // Already in the DB, can mark as modified + WIKI.logger.info(`(STORAGE/DISK) Page marked as modified: ${relPath}`) + await WIKI.models.pages.updatePage({ + id: currentPage.id, + title: _.get(pageData, 'title', currentPage.title), + description: _.get(pageData, 'description', currentPage.description) || '', + isPublished: _.get(pageData, 'isPublished', currentPage.isPublished), + isPrivate: false, + content: pageData.content, + user: rootUser, + skipStorage: true + }) + } else { + // Not in the DB, can mark as new + WIKI.logger.info(`(STORAGE/DISK) Page marked as new: ${relPath}`) + const pageEditor = await WIKI.models.editors.getDefaultEditor(contentType) + await WIKI.models.pages.createPage({ + path: contentPath.path, + locale: contentPath.locale, + title: _.get(pageData, 'title', _.last(contentPath.path.split('/'))), + description: _.get(pageData, 'description', '') || '', + isPublished: _.get(pageData, 'isPublished', true), + isPrivate: false, + content: pageData.content, + user: rootUser, + editor: pageEditor, + skipStorage: true + }) + } + } catch (err) { + WIKI.logger.warn(`(STORAGE/DISK) Failed to process ${relPath}`) + WIKI.logger.warn(err) } - } catch (err) { - WIKI.logger.warn(`(STORAGE/DISK) Failed to process ${relPath}`) - WIKI.logger.warn(err) + } else { + // -> Asset + + // -> Find existing folder + const filePathInfo = path.parse(file.path) + const folderPath = path.dirname(relPath).replace(/\\/g, '/') + let folderId = _.toInteger(_.findKey(assetFolders, fld => { return fld === folderPath })) || null + + // -> Create missing folder structure + if (!folderId && folderPath !== '.') { + const folderParts = folderPath.split('/') + let currentFolderPath = [] + let currentFolderParentId = null + for (const folderPart of folderParts) { + currentFolderPath.push(folderPart) + const existingFolderId = _.findKey(assetFolders, fld => { return fld === currentFolderPath.join('/') }) + if (!existingFolderId) { + const newFolderObj = await WIKI.models.assetFolders.query().insert({ + slug: folderPart, + name: folderPart, + parentId: currentFolderParentId + }) + _.set(assetFolders, newFolderObj.id, currentFolderPath.join('/')) + currentFolderParentId = newFolderObj.id + } else { + currentFolderParentId = _.toInteger(existingFolderId) + } + } + folderId = currentFolderParentId + } + + // -> Import asset + await WIKI.models.assets.upload({ + mode: 'import', + originalname: filePathInfo.base, + ext: filePathInfo.ext, + mimetype: mime(filePathInfo.base) || 'application/octet-stream', + size: file.stats.size, + folderId: folderId, + path: file.path, + assetPath: relPath, + user: rootUser, + skipStorage: true + }) } } cb()