From 38c33c58bbe671b08e658f4422b9268b4728e805 Mon Sep 17 00:00:00 2001 From: NGPixel Date: Mon, 7 Oct 2019 00:06:47 -0400 Subject: [PATCH] feat: rebuild page tree worker --- client/components/common/page-selector.vue | 81 +++++++++++----------- server/app/data.yml | 10 ++- server/core/scheduler.js | 7 +- server/graph/resolvers/asset.js | 18 ++++- server/jobs/rebuild-tree.js | 63 +++++++++++++++++ server/jobs/sync-git.js | 69 ------------------ server/models/assets.js | 8 +++ server/models/pages.js | 2 +- server/models/storage.js | 11 +++ server/modules/storage/git/storage.js | 48 ++++++++++++- 10 files changed, 198 insertions(+), 119 deletions(-) create mode 100644 server/jobs/rebuild-tree.js delete mode 100644 server/jobs/sync-git.js diff --git a/client/components/common/page-selector.vue b/client/components/common/page-selector.vue index 8ab5be7f..9a4c48e7 100644 --- a/client/components/common/page-selector.vue +++ b/client/components/common/page-selector.vue @@ -3,7 +3,7 @@ v-card.page-selector .dialog-header.is-dark v-icon.mr-3(color='white') mdi-page-next-outline - span Select Page Location + .body-1 Select Page Location v-spacer v-progress-circular( indeterminate @@ -12,44 +12,44 @@ :width='2' v-show='searchLoading' ) - //- .d-flex(style='min-height:400px;') - //- v-flex(xs4).grey(:class='darkMode ? `darken-4` : `lighten-3`') - //- v-toolbar(color='grey darken-3', dark, dense, flat) - //- .body-2 Folders - //- v-spacer - //- v-btn(icon): v-icon create_new_folder - //- v-treeview( - //- v-model='tree' - //- :items='treeFolders' - //- :load-children='fetchFolders' - //- activatable - //- open-on-click - //- hoverable - //- ) - //- template(slot='prepend', slot-scope='{ item, open, leaf }') - //- v-icon {{ open ? 'folder_open' : 'folder' }} - //- v-flex(xs8) - //- v-toolbar(color='grey darken-2', dark, dense, flat) - //- .body-2 Pages - //- v-spacer - //- v-btn(icon): v-icon forward - //- v-btn(icon): v-icon delete - //- v-list(dense) - //- v-list-item - //- v-list-item-avatar: v-icon insert_drive_file - //- v-list-item-title File A - //- v-divider - //- v-list-item - //- v-list-item-avatar: v-icon insert_drive_file - //- v-list-item-title File B - //- v-divider - //- v-list-item - //- v-list-item-avatar: v-icon insert_drive_file - //- v-list-item-title File C - //- v-divider - //- v-list-item - //- v-list-item-avatar: v-icon insert_drive_file - //- v-list-item-title File D + .d-flex(style='min-height:400px;') + v-flex.grey(xs4, :class='darkMode ? `darken-4` : `lighten-3`') + v-toolbar(color='grey darken-3', dark, dense, flat) + .body-2 Folders + //- v-spacer + //- v-btn(icon): v-icon create_new_folder + v-treeview( + v-model='tree' + :items='treeFolders' + :load-children='fetchFolders' + activatable + open-on-click + hoverable + ) + template(slot='prepend', slot-scope='{ item, open, leaf }') + v-icon mdi-{{ open ? 'folder-open' : 'folder' }} + v-flex(xs8) + v-toolbar(color='grey darken-2', dark, dense, flat) + .body-2 Pages + v-spacer + v-btn(icon): v-icon mdi-forward + v-btn(icon): v-icon mdi-delete + v-list(dense) + v-list-item + v-list-item-icon: v-icon mdi-file-document-box + v-list-item-title File A + v-divider + v-list-item + v-list-item-icon: v-icon mdi-file-document-box + v-list-item-title File B + v-divider + v-list-item + v-list-item-icon: v-icon mdi-file-document-box + v-list-item-title File C + v-divider + v-list-item + v-list-item-icon: v-icon mdi-file-document-box + v-list-item-title File D v-card-actions.grey.pa-2(:class='darkMode ? `darken-3-d5` : `lighten-1`') v-select( solo @@ -58,7 +58,7 @@ hide-details single-line :items='namespaces' - style='flex: 0 0 100px;' + style='flex: 0 0 100px; border-radius: 4px 0 0 4px;' v-model='currentLocale' ) v-text-field( @@ -68,6 +68,7 @@ v-model='currentPath' flat clearable + style='border-radius: 0 4px 4px 0;' ) v-card-chin v-spacer diff --git a/server/app/data.yml b/server/app/data.yml index 2447cd89..5c289da9 100644 --- a/server/app/data.yml +++ b/server/app/data.yml @@ -75,14 +75,18 @@ jobs: onInit: true schedule: PT15M offlineSkip: false + repeat: true syncGraphLocales: onInit: true schedule: P1D offlineSkip: true - syncGraphUpdates: + repeat: true + rebuildTree: onInit: true - schedule: P1D - offlineSkip: true + offlineSkip: false + repeat: false + immediate: true + worker: true groups: defaultPermissions: - 'read:pages' diff --git a/server/core/scheduler.js b/server/core/scheduler.js index cb2dc847..34c4e1f6 100644 --- a/server/core/scheduler.js +++ b/server/core/scheduler.js @@ -99,12 +99,13 @@ module.exports = { return } - const schedule = (configHelper.isValidDurationString(queueParams.schedule)) ? queueParams.schedule : _.get(WIKI.config, queueParams.schedule) + const schedule = (configHelper.isValidDurationString(queueParams.schedule)) ? queueParams.schedule : 'P1D' this.registerJob({ name: _.kebabCase(queueName), - immediate: queueParams.onInit, + immediate: _.get(queueParams, 'onInit', false), schedule: schedule, - repeat: true + repeat: _.get(queueParams, 'repeat', false), + worker: _.get(queueParams, 'worker', false) }) }) }, diff --git a/server/graph/resolvers/asset.js b/server/graph/resolvers/asset.js index 1f896f25..5e9c952d 100644 --- a/server/graph/resolvers/asset.js +++ b/server/graph/resolvers/asset.js @@ -97,7 +97,7 @@ module.exports = { } // Check source asset permissions - const assetSourcePath = (asset.folderId) ? hierarchy.map(h => h.slug).join('/') + `/${filename}` : filename + const assetSourcePath = (asset.folderId) ? hierarchy.map(h => h.slug).join('/') + `/${asset.filename}` : asset.filename if (!WIKI.auth.checkAccess(context.req.user, ['manage:assets'], { path: assetSourcePath })) { throw new WIKI.Error.AssetRenameForbidden() } @@ -118,6 +118,16 @@ module.exports = { // Delete old asset cache await asset.deleteAssetCache() + // Rename in Storage + await WIKI.models.storage.assetEvent({ + event: 'renamed', + asset: { + ...asset, + sourcePath: assetSourcePath, + destinationPath: assetTargetPath + } + }) + return { responseResult: graphHelper.generateSuccess('Asset has been renamed successfully.') } @@ -145,6 +155,12 @@ module.exports = { await WIKI.models.assets.query().deleteById(args.id) await asset.deleteAssetCache() + // Delete from Storage + await WIKI.models.storage.assetEvent({ + event: 'deleted', + asset + }) + return { responseResult: graphHelper.generateSuccess('Asset has been deleted successfully.') } diff --git a/server/jobs/rebuild-tree.js b/server/jobs/rebuild-tree.js new file mode 100644 index 00000000..7d731f0f --- /dev/null +++ b/server/jobs/rebuild-tree.js @@ -0,0 +1,63 @@ +const _ = require('lodash') + +/* global WIKI */ + +module.exports = async (pageId) => { + WIKI.logger.info(`Rebuilding page tree...`) + + try { + WIKI.models = require('../core/db').init() + await WIKI.configSvc.loadFromDb() + await WIKI.configSvc.applyFlags() + + await WIKI.models.knex.table('pageTree').truncate() + const pages = await WIKI.models.pages.query().select('id', 'path', 'localeCode', 'title', 'isPrivate', 'privateNS').orderBy(['localeCode', 'path']) + let tree = [] + let pik = 0 + + for (const page of pages) { + const pagePaths = page.path.split('/') + let currentPath = '' + let depth = 0 + let parentId = null + for (const part of pagePaths) { + depth++ + const isFolder = (depth < pagePaths.length) + currentPath = currentPath ? `${currentPath}/${part}` : part + const found = _.find(tree, { + localeCode: page.localeCode, + path: currentPath + }) + if (!found) { + pik++ + tree.push({ + id: pik, + localeCode: page.localeCode, + path: currentPath, + depth: depth, + title: isFolder ? part : page.title, + isFolder: isFolder, + isPrivate: !isFolder && page.isPrivate, + privateNS: !isFolder ? page.privateNS : null, + parent: parentId, + pageId: isFolder ? null : page.id + }) + parentId = pik + } else { + parentId = found.id + } + } + } + + if (tree.length > 0) { + await WIKI.models.knex.table('pageTree').insert(tree) + } + + await WIKI.models.knex.destroy() + + WIKI.logger.info(`Rebuilding page tree: [ COMPLETED ]`) + } catch (err) { + WIKI.logger.error(`Rebuilding page tree: [ FAILED ]`) + WIKI.logger.error(err.message) + } +} diff --git a/server/jobs/sync-git.js b/server/jobs/sync-git.js deleted file mode 100644 index e06cfae3..00000000 --- a/server/jobs/sync-git.js +++ /dev/null @@ -1,69 +0,0 @@ -'use strict' - -// /* global WIKI */ - -// const Promise = require('bluebird') -// const fs = Promise.promisifyAll(require('fs-extra')) -// const klaw = require('klaw') -// const moment = require('moment') -// const path = require('path') -// const entryHelper = require('../helpers/entry') - -module.exports = (job) => { - return true - // return WIKI.git.resync().then(() => { - // // -> Stream all documents - - // let cacheJobs = [] - // let jobCbStreamDocsResolve = null - // let jobCbStreamDocs = new Promise((resolve, reject) => { - // jobCbStreamDocsResolve = resolve - // }) - - // klaw(WIKI.REPOPATH).on('data', function (item) { - // if (path.extname(item.path) === '.md' && path.basename(item.path) !== 'README.md') { - // let entryPath = entryHelper.parsePath(entryHelper.getEntryPathFromFullPath(item.path)) - // let cachePath = entryHelper.getCachePath(entryPath) - - // // -> Purge outdated cache - - // cacheJobs.push( - // fs.statAsync(cachePath).then((st) => { - // return moment(st.mtime).isBefore(item.stats.mtime) ? 'expired' : 'active' - // }).catch((err) => { - // return (err.code !== 'EEXIST') ? err : 'new' - // }).then((fileStatus) => { - // // -> Delete expired cache file - - // if (fileStatus === 'expired') { - // return fs.unlinkAsync(cachePath).return(fileStatus) - // } - - // return fileStatus - // }).then((fileStatus) => { - // // -> Update cache and search index - - // if (fileStatus !== 'active') { - // return global.entries.updateCache(entryPath).then(entry => { - // process.send({ - // action: 'searchAdd', - // content: entry - // }) - // return true - // }) - // } - - // return true - // }) - // ) - // } - // }).on('end', () => { - // jobCbStreamDocsResolve(Promise.all(cacheJobs)) - // }) - - // return jobCbStreamDocs - // }).then(() => { - // WIKI.logger.info('Git remote repository sync: DONE') - // return true - // }) -} diff --git a/server/models/assets.js b/server/models/assets.js index 1033ed2f..ce1dd564 100644 --- a/server/models/assets.js +++ b/server/models/assets.js @@ -125,6 +125,14 @@ module.exports = class Asset extends Model { // Move temp upload to cache await fs.move(opts.path, path.join(process.cwd(), `data/cache/${fileHash}.dat`), { overwrite: true }) + + // Add to Storage + if (!opts.skipStorage) { + await WIKI.models.storage.assetEvent({ + event: 'uploaded', + asset + }) + } } static async getAsset(assetPath, res) { diff --git a/server/models/pages.js b/server/models/pages.js index 5e7f80e1..bdeac43b 100644 --- a/server/models/pages.js +++ b/server/models/pages.js @@ -251,7 +251,7 @@ module.exports = class Page extends Model { }) // -> Save Tags - if (opts.tags.length > 0) { + if (opts.tags && opts.tags.length > 0) { await WIKI.models.tags.associateTags({ tags: opts.tags, page }) } diff --git a/server/models/storage.js b/server/models/storage.js index 31e876bb..2a341d12 100644 --- a/server/models/storage.js +++ b/server/models/storage.js @@ -180,6 +180,17 @@ module.exports = class Storage extends Model { } } + static async assetEvent({ event, asset }) { + try { + for (let target of this.targets) { + await target.fn[`asset${_.capitalize(event)}`](asset) + } + } catch (err) { + WIKI.logger.warn(err) + throw err + } + } + static async executeAction(targetKey, handler) { try { const target = _.find(this.targets, ['key', targetKey]) diff --git a/server/modules/storage/git/storage.js b/server/modules/storage/git/storage.js index d175fdbf..b1e3412d 100644 --- a/server/modules/storage/git/storage.js +++ b/server/modules/storage/git/storage.js @@ -25,6 +25,9 @@ const getContenType = (filePath) => { } const getPagePath = (filePath) => { + if (process.platform === 'win32') { + filePath = filePath.replace(/\\/g, '/') + } let meta = { locale: 'en', path: _.initial(filePath.split('.')).join('') @@ -194,7 +197,7 @@ module.exports = { await WIKI.models.pages.updatePage({ id: currentPage.id, title: _.get(pageData, 'title', currentPage.title), - description: _.get(pageData, 'description', currentPage.description), + description: _.get(pageData, 'description', currentPage.description) || '', isPublished: _.get(pageData, 'isPublished', currentPage.isPublished), isPrivate: false, content: pageData.content, @@ -209,7 +212,7 @@ module.exports = { path: contentPath.path, locale: contentPath.locale, title: _.get(pageData, 'title', _.last(contentPath.path.split('/'))), - description: _.get(pageData, 'description', ''), + description: _.get(pageData, 'description', '') || '', isPublished: _.get(pageData, 'isPublished', true), isPrivate: false, content: pageData.content, @@ -230,6 +233,7 @@ module.exports = { }) } else { WIKI.logger.warn(`(STORAGE/GIT) Failed to open ${item.file}`) + console.error(err) WIKI.logger.warn(err) } } @@ -310,7 +314,47 @@ module.exports = { '--author': `"${page.authorName} <${page.authorEmail}>"` }) }, + /** + * ASSET UPLOAD + * + * @param {Object} asset Asset to upload + */ + async assetUploaded (asset) { + WIKI.logger.info(`(STORAGE/GIT) Committing new file ${asset.path}...`) + const filePath = path.join(this.repoPath, asset.path) + await fs.outputFile(filePath, asset, 'utf8') + await this.git.add(`./${asset.path}`) + await this.git.commit(`docs: upload ${asset.path}`, asset.path, { + '--author': `"${asset.authorName} <${asset.authorEmail}>"` + }) + }, + /** + * ASSET DELETE + * + * @param {Object} asset Asset to upload + */ + async assetDeleted (asset) { + WIKI.logger.info(`(STORAGE/GIT) Committing removed file ${asset.path}...`) + + await this.git.rm(`./${asset.path}`) + await this.git.commit(`docs: delete ${asset.path}`, asset.path, { + '--author': `"${asset.authorName} <${asset.authorEmail}>"` + }) + }, + /** + * ASSET RENAME + * + * @param {Object} asset Asset to upload + */ + async assetRenamed (asset) { + WIKI.logger.info(`(STORAGE/GIT) Committing file move from ${asset.sourcePath} to ${asset.destinationPath}...`) + + await this.git.mv(`./${asset.sourcePath}`, `./${asset.destinationPath}`) + await this.git.commit(`docs: rename ${asset.sourcePath} to ${asset.destinationPath}`, asset.destinationPath, { + '--author': `"${asset.authorName} <${asset.authorEmail}>"` + }) + }, /** * HANDLERS */