liz/wikijs-fork
1
0
Fork 0
Code Issues Pull Requests Packages Projects Releases Wiki Activity

Uploads model + watcher

Browse Source
This commit is contained in:
NGPixel
2016-10-09 01:26:25 -04:00
parent 819d4ad346
commit 99a07d342c
12 changed files with 367 additions and 119 deletions
Download Patch File Download Diff File Expand all files Collapse all files

84
agent.js
View File

@@ -31,7 +31,8 @@ global.WSInternalKey = process.argv[2];
winston.info('[AGENT] Background Agent is initializing...');
var appconfig = require('./models/config')('./config.yml');
let lcdata = require('./models/localdata').init(appconfig, 'agent');
global.lcdata = require('./models/localdata').init(appconfig, 'agent');
var upl = require('./models/uploads').init(appconfig);
global.git = require('./models/git').init(appconfig);
global.entries = require('./models/entries').init(appconfig);
@@ -43,9 +44,6 @@ var Promise = require('bluebird');
var fs = Promise.promisifyAll(require("fs-extra"));
var path = require('path');
var cron = require('cron').CronJob;
var readChunk = require('read-chunk');
var fileType = require('file-type');
var farmhash = require('farmhash');
global.ws = require('socket.io-client')('http://localhost:' + appconfig.wsPort, { reconnectionAttempts: 10 });
@@ -56,6 +54,8 @@ const mimeImgTypes = ['image/png', 'image/jpg']
// ----------------------------------------
var jobIsBusy = false;
var jobUplWatchStarted = false;
var job = new cron({
cronTime: '0 */5 * * * *',
onTick: () => {
@@ -168,71 +168,11 @@ var job = new cron({
let fldPath = path.join(uploadsPath, fldName);
return fs.readdirAsync(fldPath).then((fList) => {
return Promise.map(fList, (f) => {
let fPath = path.join(fldPath, f);
let fPathObj = path.parse(fPath);
let fUid = farmhash.fingerprint32(fldName + '/' + f);
return fs.statAsync(fPath)
.then((s) => {
if(!s.isFile()) { return false; }
// Get MIME info
let mimeInfo = fileType(readChunk.sync(fPath, 0, 262));
// Images
if(s.size < 3145728) { // ignore files larger than 3MB
if(_.includes(['image/png', 'image/jpeg', 'image/gif', 'image/webp'], mimeInfo.mime)) {
return lcdata.getImageMetadata(fPath).then((mData) => {
let cacheThumbnailPath = path.parse(path.join(dataPath, 'thumbs', fUid + '.png'));
let cacheThumbnailPathStr = path.format(cacheThumbnailPath);
mData = _.pick(mData, ['format', 'width', 'height', 'density', 'hasAlpha', 'orientation']);
mData.uid = fUid;
mData.category = 'image';
mData.mime = mimeInfo.mime;
mData.folder = fldName;
mData.filename = f;
mData.basename = fPathObj.name;
mData.filesize = s.size;
mData.uploadedOn = moment().utc();
allFiles.push(mData);
// Generate thumbnail
return fs.statAsync(cacheThumbnailPathStr).then((st) => {
return st.isFile();
}).catch((err) => {
return false;
}).then((thumbExists) => {
return (thumbExists) ? true : fs.ensureDirAsync(cacheThumbnailPath.dir).then(() => {
return lcdata.generateThumbnail(fPath, cacheThumbnailPathStr);
});
});
})
}
}
// Other Files
allFiles.push({
uid: fUid,
category: 'file',
mime: mimeInfo.mime,
folder: fldName,
filename: f,
basename: fPathObj.name,
filesize: s.size,
uploadedOn: moment().utc()
});
});
return upl.processFile(fldName, f).then((mData) => {
if(mData) {
allFiles.push(mData);
}
});
}, {concurrency: 3});
});
}, {concurrency: 1}).finally(() => {
@@ -255,6 +195,12 @@ var job = new cron({
Promise.all(jobs).then(() => {
winston.info('[AGENT] All jobs completed successfully! Going to sleep for now.');
if(!jobUplWatchStarted) {
jobUplWatchStarted = true;
upl.watch();
}
}).catch((err) => {
winston.error('[AGENT] One or more jobs have failed: ', err);
}).finally(() => {

2
assets/js/app.js
View File

File diff suppressed because one or more lines are too long

67
client/js/components/editor-image.js
View File

@@ -13,7 +13,9 @@ let vueImage = new Vue({
currentFolder: '',
currentImage: '',
currentAlign: 'left',
images: []
images: [],
uploadSucceeded: false,
postUploadChecks: 0
},
methods: {
open: () => {
@@ -126,13 +128,17 @@ let vueImage = new Vue({
*
* @return {Void} Void
*/
loadImages: () => {
vueImage.isLoading = true;
vueImage.isLoadingText = 'Fetching images...';
loadImages: (silent) => {
if(!silent) {
vueImage.isLoading = true;
vueImage.isLoadingText = 'Fetching images...';
}
Vue.nextTick(() => {
socket.emit('uploadsGetImages', { folder: vueImage.currentFolder }, (data) => {
vueImage.images = data;
vueImage.isLoading = false;
if(!silent) {
vueImage.isLoading = false;
}
vueImage.attachContextMenus();
});
});
@@ -209,6 +215,31 @@ let vueImage = new Vue({
}
}
});
},
waitUploadComplete: () => {
vueImage.postUploadChecks++;
vueImage.isLoadingText = 'Processing uploads...';
let currentUplAmount = vueImage.images.length;
vueImage.loadImages(true);
Vue.nextTick(() => {
_.delay(() => {
if(currentUplAmount !== vueImage.images.length) {
vueImage.postUploadChecks = 0;
vueImage.isLoading = false;
} else if(vueImage.postUploadChecks > 5) {
vueImage.postUploadChecks = 0;
vueImage.isLoading = false;
alerts.pushError('Unable to fetch new uploads', 'Try again later');
} else {
vueImage.waitUploadComplete();
}
}, 2000);
});
}
}
@@ -228,7 +259,8 @@ $('#btn-editor-uploadimage input').on('change', (ev) => {
allowedTypes: ['image/png', 'image/jpeg', 'image/gif', 'image/webp'],
maxFileSize: 3145728, // max 3 MB
init: () => {
init: (totalUploads) => {
vueImage.uploadSucceeded = false;
vueImage.isLoading = true;
vueImage.isLoadingText = 'Preparing to upload...';
},
@@ -240,18 +272,37 @@ $('#btn-editor-uploadimage input').on('change', (ev) => {
success: (data) => {
if(data.ok) {
let failedUpls = _.filter(data.results, ['ok', false]);
if(failedUpls.length) {
_.forEach(failedUpls, (u) => {
alerts.pushError('Upload error', u.msg);
});
if(failedUpls.length < data.results.length) {
alerts.push({
title: 'Some uploads succeeded',
message: 'Files that are not mentionned in the errors above were uploaded successfully.'
});
vueImage.uploadSucceeded = true;
}
} else {
vueImage.uploadSucceeded = true;
}
} else {
alerts.pushError('Upload error', data.msg);
}
},
error: function(error) {
vueImage.isLoading = false;
alerts.pushError(error.message, this.upload.file.name);
},
finish: () => {
vueImage.isLoading = false;
if(vueImage.uploadSucceeded) {
vueImage.waitUploadComplete();
} else {
vueImage.isLoading = false;
}
}
});

10
controllers/pages.js
View File

@@ -160,11 +160,15 @@ router.get('/*', (req, res, next) => {
if(pageData) {
return res.render('pages/view', { pageData });
} else {
res.render('error', {
message: err.message,
error: {}
res.render('error-notexist', {
newpath: safePath
});
}
}).error((err) => {
res.render('error-notexist', {
message: err.message,
newpath: safePath
});
}).catch((err) => {
res.render('error', {
message: err.message,

15
controllers/uploads.js
View File

@@ -72,13 +72,24 @@ router.post('/img', lcdata.uploadImgHandler, (req, res, next) => {
}).then(() => {
return {
ok: true,
filename: destFilename,
filesize: f.size
};
});
}).reflect();
}, {concurrency: 3}).then((results) => {
res.json({ ok: true, results });
let uplResults = _.map(results, (r) => {
if(r.isFulfilled()) {
return r.value();
} else {
return {
ok: false,
msg: r.reason().message
}
}
});
res.json({ ok: true, results: uplResults });
}).catch((err) => {
res.json({ ok: false, msg: err.message });
});

2
models/entries.js
View File

@@ -170,7 +170,7 @@ module.exports = {
return false;
}
}).catch((err) => {
return Promise.reject(new Error('Entry ' + entryPath + ' does not exist!'));
return Promise.reject(new Promise.OperationalError('Entry ' + entryPath + ' does not exist!'));
});
},

17
models/git.js
View File

@@ -235,6 +235,23 @@ module.exports = {
return true;
})
},
/**
* Commits uploads changes.
*
* @return {Promise} Resolve on commit success
*/
commitUploads() {
let self = this;
return self._git.add('uploads').then(() => {
return self._git.commit("Uploads repository sync").catch((err) => {
if(_.includes(err.stdout, 'nothing to commit')) { return true; }
});
});
}
};

64
models/localdata.js
View File

@@ -267,6 +267,22 @@ module.exports = {
},
/**
* Parse relative Uploads path
*
* @param {String} f Relative Uploads path
* @return {Object} Parsed path (folder and filename)
*/
parseUploadsRelPath(f) {
let fObj = path.parse(f);
return {
folder: fObj.dir,
filename: fObj.base
};
},
/**
* Sets the uploads files.
*
@@ -288,6 +304,19 @@ module.exports = {
},
/**
* Adds one or more uploads files.
*
* @param {Array<Object>} arrFiles The uploads files
* @return {Void} Void
*/
addUploadsFiles(arrFiles) {
if(_.isArray(arrFiles) || _.isPlainObject(arrFiles)) {
this._uploadsDb.Files.insert(arrFiles);
}
return;
},
/**
* Gets the uploads files.
*
@@ -301,41 +330,6 @@ module.exports = {
'$and': [{ 'category' : cat },{ 'folder' : fld }]
}).simplesort('filename').data();
},
/**
* Generate thumbnail of image
*
* @param {String} sourcePath The source path
* @return {Promise<Object>} Promise returning the resized image info
*/
generateThumbnail(sourcePath, destPath) {
let sharp = require('sharp');
return sharp(sourcePath)
.withoutEnlargement()
.resize(150,150)
.background('white')
.embed()
.flatten()
.toFormat('png')
.toFile(destPath);
},
/**
* Gets the image metadata.
*
* @param {String} sourcePath The source path
* @return {Object} The image metadata.
*/
getImageMetadata(sourcePath) {
let sharp = require('sharp');
return sharp(sourcePath).metadata();
}
};

176
models/uploads.js Normal file
View File

@@ -0,0 +1,176 @@
"use strict";
var path = require('path'),
Promise = require('bluebird'),
fs = Promise.promisifyAll(require('fs-extra')),
readChunk = require('read-chunk'),
fileType = require('file-type'),
farmhash = require('farmhash'),
moment = require('moment'),
chokidar = require('chokidar'),
_ = require('lodash');
/**
* Uploads
*
* @param {Object} appconfig The application configuration
*/
module.exports = {
_uploadsPath: './repo/uploads',
_uploadsThumbsPath: './data/thumbs',
_watcher: null,
/**
* Initialize Uploads model
*
* @param {Object} appconfig The application config
* @return {Object} Uploads model instance
*/
init(appconfig) {
let self = this;
self._uploadsPath = path.resolve(ROOTPATH, appconfig.datadir.repo, 'uploads');
self._uploadsThumbsPath = path.resolve(ROOTPATH, appconfig.datadir.db, 'thumbs');
return self;
},
watch() {
let self = this;
self._watcher = chokidar.watch(self._uploadsPath, {
persistent: true,
ignoreInitial: true,
cwd: self._uploadsPath,
depth: 1,
awaitWriteFinish: true
});
self._watcher.on('add', (p) => {
let pInfo = lcdata.parseUploadsRelPath(p);
return self.processFile(pInfo.folder, pInfo.filename).then((mData) => {
ws.emit('uploadsAddFiles', {
auth: WSInternalKey,
content: mData
});
}).then(() => {
return git.commitUploads();
});
});
},
processFile(fldName, f) {
let self = this;
let fldPath = path.join(self._uploadsPath, fldName);
let fPath = path.join(fldPath, f);
let fPathObj = path.parse(fPath);
let fUid = farmhash.fingerprint32(fldName + '/' + f);
return fs.statAsync(fPath).then((s) => {
if(!s.isFile()) { return false; }
// Get MIME info
let mimeInfo = fileType(readChunk.sync(fPath, 0, 262));
// Images
if(s.size < 3145728) { // ignore files larger than 3MB
if(_.includes(['image/png', 'image/jpeg', 'image/gif', 'image/webp'], mimeInfo.mime)) {
return self.getImageMetadata(fPath).then((mData) => {
let cacheThumbnailPath = path.parse(path.join(self._uploadsThumbsPath, fUid + '.png'));
let cacheThumbnailPathStr = path.format(cacheThumbnailPath);
mData = _.pick(mData, ['format', 'width', 'height', 'density', 'hasAlpha', 'orientation']);
mData.uid = fUid;
mData.category = 'image';
mData.mime = mimeInfo.mime;
mData.folder = fldName;
mData.filename = f;
mData.basename = fPathObj.name;
mData.filesize = s.size;
mData.uploadedOn = moment().utc();
// Generate thumbnail
return fs.statAsync(cacheThumbnailPathStr).then((st) => {
return st.isFile();
}).catch((err) => {
return false;
}).then((thumbExists) => {
return (thumbExists) ? mData : fs.ensureDirAsync(cacheThumbnailPath.dir).then(() => {
return self.generateThumbnail(fPath, cacheThumbnailPathStr);
}).return(mData);
});
})
}
}
// Other Files
return {
uid: fUid,
category: 'file',
mime: mimeInfo.mime,
folder: fldName,
filename: f,
basename: fPathObj.name,
filesize: s.size,
uploadedOn: moment().utc()
};
});
},
/**
* Generate thumbnail of image
*
* @param {String} sourcePath The source path
* @return {Promise<Object>} Promise returning the resized image info
*/
generateThumbnail(sourcePath, destPath) {
let sharp = require('sharp');
return sharp(sourcePath)
.withoutEnlargement()
.resize(150,150)
.background('white')
.embed()
.flatten()
.toFormat('png')
.toFile(destPath);
},
/**
* Gets the image metadata.
*
* @param {String} sourcePath The source path
* @return {Object} The image metadata.
*/
getImageMetadata(sourcePath) {
let sharp = require('sharp');
return sharp(sourcePath).metadata();
}
};

1
package.json
View File

@@ -39,6 +39,7 @@
"bson": "^0.5.5",
"cheerio": "^0.22.0",
"child-process-promise": "^2.1.3",
"chokidar": "^1.6.0",
"compression": "^1.6.2",
"connect-flash": "^0.1.1",
"connect-loki": "^1.0.6",

32
views/error-notexist.pug Normal file
View File

@@ -0,0 +1,32 @@
doctype html
html
head
meta(http-equiv='X-UA-Compatible', content='IE=edge')
meta(charset='UTF-8')
meta(name='viewport', content='width=device-width, initial-scale=1')
meta(name='theme-color', content='#009688')
meta(name='msapplication-TileColor', content='#009688')
meta(name='msapplication-TileImage', content='/favicons/ms-icon-144x144.png')
title= appconfig.title
// Favicon
each favsize in [57, 60, 72, 76, 114, 120, 144, 152, 180]
link(rel='apple-touch-icon', sizes=favsize + 'x' + favsize, href='/favicons/apple-icon-' + favsize + 'x' + favsize + '.png')
link(rel='icon', type='image/png', sizes='192x192', href='/favicons/android-icon-192x192.png')
each favsize in [32, 96, 16]
link(rel='icon', type='image/png', sizes=favsize + 'x' + favsize, href='/favicons/favicon-' + favsize + 'x' + favsize + '.png')
link(rel='manifest', href='/manifest.json')
// CSS
link(type='text/css', rel='stylesheet', href='/css/libs.css')
link(type='text/css', rel='stylesheet', href='/css/app.css')
body(class='server-error')
section.hero.is-dark.is-fullheight
.hero-body
.container
a(href='/'): img(src='/favicons/android-icon-96x96.png')
h1.title(style={ 'margin-top': '30px'})= message
h2.subtitle(style={ 'margin-bottom': '50px'}) Would you like to create this entry?
a.button.is-dark.is-inverted(href='/create/' + newpath, style={'margin-right': '5px'}) Create
a.button.is-dark.is-inverted(href='/') Go Home

16
ws-server.js
View File

@@ -108,12 +108,14 @@ io.on('connection', (socket) => {
});
socket.on('searchDel', (data, cb) => {
cb = cb || _.noop
if(internalAuth.validateKey(data.auth)) {
search.delete(data.entryPath);
}
});
socket.on('search', (data, cb) => {
cb = cb || _.noop
search.find(data.terms).then((results) => {
cb(results);
});
@@ -124,28 +126,42 @@ io.on('connection', (socket) => {
//-----------------------------------------
socket.on('uploadsSetFolders', (data, cb) => {
cb = cb || _.noop
if(internalAuth.validateKey(data.auth)) {
lcdata.setUploadsFolders(data.content);
}
});
socket.on('uploadsGetFolders', (data, cb) => {
cb = cb || _.noop
cb(lcdata.getUploadsFolders());
});
socket.on('uploadsCreateFolder', (data, cb) => {
cb = cb || _.noop
lcdata.createUploadsFolder(data.foldername).then((fldList) => {
cb(fldList);
});
});
socket.on('uploadsSetFiles', (data, cb) => {
cb = cb || _.noop;
if(internalAuth.validateKey(data.auth)) {
lcdata.setUploadsFiles(data.content);
cb(true);
}
});
socket.on('uploadsAddFiles', (data, cb) => {
cb = cb || _.noop
if(internalAuth.validateKey(data.auth)) {
lcdata.addUploadsFiles(data.content);
cb(true);
}
});
socket.on('uploadsGetImages', (data, cb) => {
cb = cb || _.noop
cb(lcdata.getUploadsFiles('image', data.folder));
});