liz/wikijs-fork
liz/wikijs-fork
1
0
Fork 0
Code Issues Pull Requests Packages Projects Releases Wiki Activity

Uploads model + watcher

Browse Source
This commit is contained in:
NGPixel 2016-10-09 01:26:25 -04:00
parent 819d4ad346
commit 99a07d342c
12 changed files with 367 additions and 119 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

84
agent.js
View File

@ -31,7 +31,8 @@ global.WSInternalKey = process.argv[2];
winston.info('[AGENT] Background Agent is initializing...');
var appconfig = require('./models/config')('./config.yml');
let lcdata = require('./models/localdata').init(appconfig, 'agent');
global.lcdata = require('./models/localdata').init(appconfig, 'agent');
var upl = require('./models/uploads').init(appconfig);
global.git = require('./models/git').init(appconfig);
global.entries = require('./models/entries').init(appconfig);
@ -43,9 +44,6 @@ var Promise = require('bluebird');
var fs = Promise.promisifyAll(require("fs-extra"));
var path = require('path');
var cron = require('cron').CronJob;
var readChunk = require('read-chunk');
var fileType = require('file-type');
var farmhash = require('farmhash');
global.ws = require('socket.io-client')('http://localhost:' + appconfig.wsPort, { reconnectionAttempts: 10 });
@ -56,6 +54,8 @@ const mimeImgTypes = ['image/png', 'image/jpg']
// ----------------------------------------
var jobIsBusy = false;
var jobUplWatchStarted = false;
var job = new cron({
cronTime: '0 */5 * * * *',
onTick: () => {
@ -168,71 +168,11 @@ var job = new cron({
let fldPath = path.join(uploadsPath, fldName);
return fs.readdirAsync(fldPath).then((fList) => {
return Promise.map(fList, (f) => {
let fPath = path.join(fldPath, f);
let fPathObj = path.parse(fPath);
let fUid = farmhash.fingerprint32(fldName + '/' + f);
return fs.statAsync(fPath)
.then((s) => {
if(!s.isFile()) { return false; }
// Get MIME info
let mimeInfo = fileType(readChunk.sync(fPath, 0, 262));
// Images
if(s.size < 3145728) { // ignore files larger than 3MB
if(_.includes(['image/png', 'image/jpeg', 'image/gif', 'image/webp'], mimeInfo.mime)) {
return lcdata.getImageMetadata(fPath).then((mData) => {
let cacheThumbnailPath = path.parse(path.join(dataPath, 'thumbs', fUid + '.png'));
let cacheThumbnailPathStr = path.format(cacheThumbnailPath);
mData = _.pick(mData, ['format', 'width', 'height', 'density', 'hasAlpha', 'orientation']);
mData.uid = fUid;
mData.category = 'image';
mData.mime = mimeInfo.mime;
mData.folder = fldName;
mData.filename = f;
mData.basename = fPathObj.name;
mData.filesize = s.size;
mData.uploadedOn = moment().utc();
allFiles.push(mData);
// Generate thumbnail
return fs.statAsync(cacheThumbnailPathStr).then((st) => {
return st.isFile();
}).catch((err) => {
return false;
}).then((thumbExists) => {
return (thumbExists) ? true : fs.ensureDirAsync(cacheThumbnailPath.dir).then(() => {
return lcdata.generateThumbnail(fPath, cacheThumbnailPathStr);
});
});
})
}
}
// Other Files
allFiles.push({
uid: fUid,
category: 'file',
mime: mimeInfo.mime,
folder: fldName,
filename: f,
basename: fPathObj.name,
filesize: s.size,
uploadedOn: moment().utc()
});
});
return upl.processFile(fldName, f).then((mData) => {
if(mData) {
allFiles.push(mData);
}
});
}, {concurrency: 3});
});
}, {concurrency: 1}).finally(() => {
@ -255,6 +195,12 @@ var job = new cron({
Promise.all(jobs).then(() => {
winston.info('[AGENT] All jobs completed successfully! Going to sleep for now.');
if(!jobUplWatchStarted) {
jobUplWatchStarted = true;
upl.watch();
}
}).catch((err) => {
winston.error('[AGENT] One or more jobs have failed: ', err);
}).finally(() => {

2
assets/js/app.js
View File

File diff suppressed because one or more lines are too long

67
client/js/components/editor-image.js
View File

@ -13,7 +13,9 @@ let vueImage = new Vue({
currentFolder: '',
currentImage: '',
currentAlign: 'left',
images: []
images: [],
uploadSucceeded: false,
postUploadChecks: 0
},
methods: {
open: () => {
@ -126,13 +128,17 @@ let vueImage = new Vue({
*
* @return {Void} Void
*/
loadImages: () => {
vueImage.isLoading = true;
vueImage.isLoadingText = 'Fetching images...';
loadImages: (silent) => {
if(!silent) {
vueImage.isLoading = true;
vueImage.isLoadingText = 'Fetching images...';
}
Vue.nextTick(() => {
socket.emit('uploadsGetImages', { folder: vueImage.currentFolder }, (data) => {
vueImage.images = data;
vueImage.isLoading = false;
if(!silent) {
vueImage.isLoading = false;
}
vueImage.attachContextMenus();
});
});
@ -209,6 +215,31 @@ let vueImage = new Vue({
}
}
});
},
waitUploadComplete: () => {
vueImage.postUploadChecks++;
vueImage.isLoadingText = 'Processing uploads...';
let currentUplAmount = vueImage.images.length;
vueImage.loadImages(true);
Vue.nextTick(() => {
_.delay(() => {
if(currentUplAmount !== vueImage.images.length) {
vueImage.postUploadChecks = 0;
vueImage.isLoading = false;
} else if(vueImage.postUploadChecks > 5) {
vueImage.postUploadChecks = 0;
vueImage.isLoading = false;
alerts.pushError('Unable to fetch new uploads', 'Try again later');
} else {
vueImage.waitUploadComplete();
}
}, 2000);
});
}
}
@ -228,7 +259,8 @@ $('#btn-editor-uploadimage input').on('change', (ev) => {
allowedTypes: ['image/png', 'image/jpeg', 'image/gif', 'image/webp'],
maxFileSize: 3145728, // max 3 MB
init: () => {
init: (totalUploads) => {
vueImage.uploadSucceeded = false;
vueImage.isLoading = true;
vueImage.isLoadingText = 'Preparing to upload...';
},
@ -240,18 +272,37 @@ $('#btn-editor-uploadimage input').on('change', (ev) => {
success: (data) => {
if(data.ok) {
let failedUpls = _.filter(data.results, ['ok', false]);
if(failedUpls.length) {
_.forEach(failedUpls, (u) => {
alerts.pushError('Upload error', u.msg);
});
if(failedUpls.length < data.results.length) {
alerts.push({
title: 'Some uploads succeeded',
message: 'Files that are not mentionned in the errors above were uploaded successfully.'
});
vueImage.uploadSucceeded = true;
}
} else {
vueImage.uploadSucceeded = true;
}
} else {
alerts.pushError('Upload error', data.msg);
}
},
error: function(error) {
vueImage.isLoading = false;
alerts.pushError(error.message, this.upload.file.name);
},
finish: () => {
vueImage.isLoading = false;
if(vueImage.uploadSucceeded) {
vueImage.waitUploadComplete();
} else {
vueImage.isLoading = false;
}
}
});

10
controllers/pages.js
View File

@ -160,11 +160,15 @@ router.get('/*', (req, res, next) => {
if(pageData) {
return res.render('pages/view', { pageData });
} else {
res.render('error', {
message: err.message,
error: {}
res.render('error-notexist', {
newpath: safePath
});
}
}).error((err) => {
res.render('error-notexist', {
message: err.message,
newpath: safePath
});
}).catch((err) => {
res.render('error', {
message: err.message,

15
controllers/uploads.js
View File

@ -72,13 +72,24 @@ router.post('/img', lcdata.uploadImgHandler, (req, res, next) => {
}).then(() => {
return {
ok: true,
filename: destFilename,
filesize: f.size
};
});
}).reflect();
}, {concurrency: 3}).then((results) => {
res.json({ ok: true, results });
let uplResults = _.map(results, (r) => {
if(r.isFulfilled()) {
return r.value();
} else {
return {
ok: false,
msg: r.reason().message
}
}
});
res.json({ ok: true, results: uplResults });
}).catch((err) => {
res.json({ ok: false, msg: err.message });
});

2
models/entries.js
View File

@ -170,7 +170,7 @@ module.exports = {
return false;
}
}).catch((err) => {
return Promise.reject(new Error('Entry ' + entryPath + ' does not exist!'));
return Promise.reject(new Promise.OperationalError('Entry ' + entryPath + ' does not exist!'));
});
},

17
models/git.js
View File

@ -235,6 +235,23 @@ module.exports = {
return true;
})
},
/**
* Commits uploads changes.
*
* @return {Promise} Resolve on commit success
*/
commitUploads() {
let self = this;
return self._git.add('uploads').then(() => {
return self._git.commit("Uploads repository sync").catch((err) => {
if(_.includes(err.stdout, 'nothing to commit')) { return true; }
});
});
}
};

64
models/localdata.js
View File

@ -267,6 +267,22 @@ module.exports = {
},
/**
* Parse relative Uploads path
*
* @param {String} f Relative Uploads path
* @return {Object} Parsed path (folder and filename)
*/
parseUploadsRelPath(f) {
let fObj = path.parse(f);
return {
folder: fObj.dir,
filename: fObj.base
};
},
/**
* Sets the uploads files.
*
@ -288,6 +304,19 @@ module.exports = {
},
/**
* Adds one or more uploads files.
*
* @param {Array<Object>} arrFiles The uploads files
* @return {Void} Void
*/
addUploadsFiles(arrFiles) {
if(_.isArray(arrFiles) || _.isPlainObject(arrFiles)) {
this._uploadsDb.Files.insert(arrFiles);
}
return;
},
/**
* Gets the uploads files.
*
@ -301,41 +330,6 @@ module.exports = {
'$and': [{ 'category' : cat },{ 'folder' : fld }]
}).simplesort('filename').data();
},
/**
* Generate thumbnail of image
*
* @param {String} sourcePath The source path
* @return {Promise<Object>} Promise returning the resized image info
*/
generateThumbnail(sourcePath, destPath) {
let sharp = require('sharp');
return sharp(sourcePath)
.withoutEnlargement()
.resize(150,150)
.background('white')
.embed()
.flatten()
.toFormat('png')
.toFile(destPath);
},
/**
* Gets the image metadata.
*
* @param {String} sourcePath The source path
* @return {Object} The image metadata.
*/
getImageMetadata(sourcePath) {
let sharp = require('sharp');
return sharp(sourcePath).metadata();
}
};

176
models/uploads.js Normal file
View File

@ -0,0 +1,176 @@
"use strict";
var path = require('path'),
Promise = require('bluebird'),
fs = Promise.promisifyAll(require('fs-extra')),
readChunk = require('read-chunk'),
fileType = require('file-type'),
farmhash = require('farmhash'),
moment = require('moment'),
chokidar = require('chokidar'),
_ = require('lodash');
/**
* Uploads
*
* @param {Object} appconfig The application configuration
*/
module.exports = {
_uploadsPath: './repo/uploads',
_uploadsThumbsPath: './data/thumbs',
_watcher: null,
/**
* Initialize Uploads model
*
* @param {Object} appconfig The application config
* @return {Object} Uploads model instance
*/
init(appconfig) {
let self = this;
self._uploadsPath = path.resolve(ROOTPATH, appconfig.datadir.repo, 'uploads');
self._uploadsThumbsPath = path.resolve(ROOTPATH, appconfig.datadir.db, 'thumbs');
return self;
},
watch() {
let self = this;
self._watcher = chokidar.watch(self._uploadsPath, {
persistent: true,
ignoreInitial: true,
cwd: self._uploadsPath,
depth: 1,
awaitWriteFinish: true
});
self._watcher.on('add', (p) => {
let pInfo = lcdata.parseUploadsRelPath(p);
return self.processFile(pInfo.folder, pInfo.filename).then((mData) => {
ws.emit('uploadsAddFiles', {
auth: WSInternalKey,
content: mData
});
}).then(() => {
return git.commitUploads();
});
});
},
processFile(fldName, f) {
let self = this;
let fldPath = path.join(self._uploadsPath, fldName);
let fPath = path.join(fldPath, f);
let fPathObj = path.parse(fPath);
let fUid = farmhash.fingerprint32(fldName + '/' + f);
return fs.statAsync(fPath).then((s) => {
if(!s.isFile()) { return false; }
// Get MIME info
let mimeInfo = fileType(readChunk.sync(fPath, 0, 262));
// Images
if(s.size < 3145728) { // ignore files larger than 3MB
if(_.includes(['image/png', 'image/jpeg', 'image/gif', 'image/webp'], mimeInfo.mime)) {
return self.getImageMetadata(fPath).then((mData) => {
let cacheThumbnailPath = path.parse(path.join(self._uploadsThumbsPath, fUid + '.png'));
let cacheThumbnailPathStr = path.format(cacheThumbnailPath);
mData = _.pick(mData, ['format', 'width', 'height', 'density', 'hasAlpha', 'orientation']);
mData.uid = fUid;
mData.category = 'image';
mData.mime = mimeInfo.mime;
mData.folder = fldName;
mData.filename = f;
mData.basename = fPathObj.name;
mData.filesize = s.size;
mData.uploadedOn = moment().utc();
// Generate thumbnail
return fs.statAsync(cacheThumbnailPathStr).then((st) => {
return st.isFile();
}).catch((err) => {
return false;
}).then((thumbExists) => {
return (thumbExists) ? mData : fs.ensureDirAsync(cacheThumbnailPath.dir).then(() => {
return self.generateThumbnail(fPath, cacheThumbnailPathStr);
}).return(mData);
});
})
}
}
// Other Files
return {
uid: fUid,
category: 'file',
mime: mimeInfo.mime,
folder: fldName,
filename: f,
basename: fPathObj.name,
filesize: s.size,
uploadedOn: moment().utc()
};
});
},
/**
* Generate thumbnail of image
*
* @param {String} sourcePath The source path
* @return {Promise<Object>} Promise returning the resized image info
*/
generateThumbnail(sourcePath, destPath) {
let sharp = require('sharp');
return sharp(sourcePath)
.withoutEnlargement()
.resize(150,150)
.background('white')
.embed()
.flatten()
.toFormat('png')
.toFile(destPath);
},
/**
* Gets the image metadata.
*
* @param {String} sourcePath The source path
* @return {Object} The image metadata.
*/
getImageMetadata(sourcePath) {
let sharp = require('sharp');
return sharp(sourcePath).metadata();
}
};

1
package.json
View File

@ -39,6 +39,7 @@
"bson": "^0.5.5",
"cheerio": "^0.22.0",
"child-process-promise": "^2.1.3",
"chokidar": "^1.6.0",
"compression": "^1.6.2",
"connect-flash": "^0.1.1",
"connect-loki": "^1.0.6",

32
views/error-notexist.pug Normal file
View File

@ -0,0 +1,32 @@
doctype html
html
head
meta(http-equiv='X-UA-Compatible', content='IE=edge')
meta(charset='UTF-8')
meta(name='viewport', content='width=device-width, initial-scale=1')
meta(name='theme-color', content='#009688')
meta(name='msapplication-TileColor', content='#009688')
meta(name='msapplication-TileImage', content='/favicons/ms-icon-144x144.png')
title= appconfig.title
// Favicon
each favsize in [57, 60, 72, 76, 114, 120, 144, 152, 180]
link(rel='apple-touch-icon', sizes=favsize + 'x' + favsize, href='/favicons/apple-icon-' + favsize + 'x' + favsize + '.png')
link(rel='icon', type='image/png', sizes='192x192', href='/favicons/android-icon-192x192.png')
each favsize in [32, 96, 16]
link(rel='icon', type='image/png', sizes=favsize + 'x' + favsize, href='/favicons/favicon-' + favsize + 'x' + favsize + '.png')
link(rel='manifest', href='/manifest.json')
// CSS
link(type='text/css', rel='stylesheet', href='/css/libs.css')
link(type='text/css', rel='stylesheet', href='/css/app.css')
body(class='server-error')
section.hero.is-dark.is-fullheight
.hero-body
.container
a(href='/'): img(src='/favicons/android-icon-96x96.png')
h1.title(style={ 'margin-top': '30px'})= message
h2.subtitle(style={ 'margin-bottom': '50px'}) Would you like to create this entry?
a.button.is-dark.is-inverted(href='/create/' + newpath, style={'margin-right': '5px'}) Create
a.button.is-dark.is-inverted(href='/') Go Home

16
ws-server.js
View File

@ -108,12 +108,14 @@ io.on('connection', (socket) => {
});
socket.on('searchDel', (data, cb) => {
cb = cb || _.noop
if(internalAuth.validateKey(data.auth)) {
search.delete(data.entryPath);
}
});
socket.on('search', (data, cb) => {
cb = cb || _.noop
search.find(data.terms).then((results) => {
cb(results);
});
@ -124,28 +126,42 @@ io.on('connection', (socket) => {
//-----------------------------------------
socket.on('uploadsSetFolders', (data, cb) => {
cb = cb || _.noop
if(internalAuth.validateKey(data.auth)) {
lcdata.setUploadsFolders(data.content);
}
});
socket.on('uploadsGetFolders', (data, cb) => {
cb = cb || _.noop
cb(lcdata.getUploadsFolders());
});
socket.on('uploadsCreateFolder', (data, cb) => {
cb = cb || _.noop
lcdata.createUploadsFolder(data.foldername).then((fldList) => {
cb(fldList);
});
});
socket.on('uploadsSetFiles', (data, cb) => {
cb = cb || _.noop;
if(internalAuth.validateKey(data.auth)) {
lcdata.setUploadsFiles(data.content);
cb(true);
}
});
socket.on('uploadsAddFiles', (data, cb) => {
cb = cb || _.noop
if(internalAuth.validateKey(data.auth)) {
lcdata.addUploadsFiles(data.content);
cb(true);
}
});
socket.on('uploadsGetImages', (data, cb) => {
cb = cb || _.noop
cb(lcdata.getUploadsFiles('image', data.folder));
});