Uploads model + watcher

agent.js

@@ -31,7 +31,8 @@ global.WSInternalKey = process.argv[2];
 winston.info('[AGENT] Background Agent is initializing...');
 
 var appconfig = require('./models/config')('./config.yml');
-let lcdata = require('./models/localdata').init(appconfig, 'agent');
+global.lcdata = require('./models/localdata').init(appconfig, 'agent');
+var upl = require('./models/uploads').init(appconfig);
 
 global.git = require('./models/git').init(appconfig);
 global.entries = require('./models/entries').init(appconfig);
@@ -43,9 +44,6 @@ var Promise = require('bluebird');
 var fs = Promise.promisifyAll(require("fs-extra"));
 var path = require('path');
 var cron = require('cron').CronJob;
-var readChunk = require('read-chunk');
-var fileType = require('file-type');
-var farmhash = require('farmhash');
 
 global.ws = require('socket.io-client')('http://localhost:' + appconfig.wsPort, { reconnectionAttempts: 10 });
 
@@ -56,6 +54,8 @@ const mimeImgTypes = ['image/png', 'image/jpg']
 // ----------------------------------------
 
 var jobIsBusy = false;
+var jobUplWatchStarted = false;
+
 var job = new cron({
 	cronTime: '0 */5 * * * *',
 	onTick: () => {
@@ -168,70 +168,10 @@ var job = new cron({
 					let fldPath = path.join(uploadsPath, fldName);
 					return fs.readdirAsync(fldPath).then((fList) => {
 						return Promise.map(fList, (f) => {
-							let fPath = path.join(fldPath, f);
+							return upl.processFile(fldName, f).then((mData) => {
-							let fPathObj = path.parse(fPath);
+								if(mData) {
-							let fUid = farmhash.fingerprint32(fldName + '/' + f);
-
-							return fs.statAsync(fPath)
-								.then((s) => {
-
-									if(!s.isFile()) { return false; }
-
-									// Get MIME info
-
-									let mimeInfo = fileType(readChunk.sync(fPath, 0, 262));
-
-									// Images
-
-									if(s.size < 3145728) { // ignore files larger than 3MB
-										if(_.includes(['image/png', 'image/jpeg', 'image/gif', 'image/webp'], mimeInfo.mime)) {
-											return lcdata.getImageMetadata(fPath).then((mData) => {
-
-												let cacheThumbnailPath = path.parse(path.join(dataPath, 'thumbs', fUid + '.png'));
-												let cacheThumbnailPathStr = path.format(cacheThumbnailPath);
-
-												mData = _.pick(mData, ['format', 'width', 'height', 'density', 'hasAlpha', 'orientation']);
-												mData.uid = fUid;
-												mData.category = 'image';
-												mData.mime = mimeInfo.mime;
-												mData.folder = fldName;
-												mData.filename = f;
-												mData.basename = fPathObj.name;
-												mData.filesize = s.size;
-												mData.uploadedOn = moment().utc();
 									allFiles.push(mData);
-
-												// Generate thumbnail
-
-												return fs.statAsync(cacheThumbnailPathStr).then((st) => {
-													return st.isFile();
-												}).catch((err) => {
-													return false;
-												}).then((thumbExists) => {
-
-													return (thumbExists) ? true : fs.ensureDirAsync(cacheThumbnailPath.dir).then(() => {
-														return lcdata.generateThumbnail(fPath, cacheThumbnailPathStr);
-													});
-
-												});
-
-											})
 								}
-									}
-
-									// Other Files
-									
-									allFiles.push({
-										uid: fUid,
-										category: 'file',
-										mime: mimeInfo.mime,
-										folder: fldName,
-										filename: f,
-										basename: fPathObj.name,
-										filesize: s.size,
-										uploadedOn: moment().utc()
-									});
-
 							});
 						}, {concurrency: 3});
 					});
@@ -255,6 +195,12 @@ var job = new cron({
 
 		Promise.all(jobs).then(() => {
 			winston.info('[AGENT] All jobs completed successfully! Going to sleep for now.');
+
+			if(!jobUplWatchStarted) {
+				jobUplWatchStarted = true;
+				upl.watch();
+			}
+
 		}).catch((err) => {
 			winston.error('[AGENT] One or more jobs have failed: ', err);
 		}).finally(() => {

client/js/components/editor-image.js

@@ -13,7 +13,9 @@ let vueImage = new Vue({
 		currentFolder: '',
 		currentImage: '',
 		currentAlign: 'left',
-		images: []
+		images: [],
+		uploadSucceeded: false,
+		postUploadChecks: 0
 	},
 	methods: {
 		open: () => {
@@ -126,13 +128,17 @@ let vueImage = new Vue({
 		 *
 		 * @return     {Void}  Void
 		 */
-		loadImages: () => {
+		loadImages: (silent) => {
+			if(!silent) {
 				vueImage.isLoading = true;
 				vueImage.isLoadingText = 'Fetching images...';
+			}
 			Vue.nextTick(() => {
 				socket.emit('uploadsGetImages', { folder: vueImage.currentFolder }, (data) => {
 					vueImage.images = data;
+					if(!silent) {
 						vueImage.isLoading = false;
+					}
 					vueImage.attachContextMenus();
 				});
 			});
@@ -209,6 +215,31 @@ let vueImage = new Vue({
 					}
 				}
 			});
+		},
+
+		waitUploadComplete: () => {
+
+			vueImage.postUploadChecks++;
+			vueImage.isLoadingText = 'Processing uploads...';
+
+			let currentUplAmount = vueImage.images.length;
+			vueImage.loadImages(true);
+
+			Vue.nextTick(() => {
+				_.delay(() => {
+					if(currentUplAmount !== vueImage.images.length) {
+						vueImage.postUploadChecks = 0;
+						vueImage.isLoading = false;
+					} else if(vueImage.postUploadChecks > 5) {
+						vueImage.postUploadChecks = 0;
+						vueImage.isLoading = false;
+						alerts.pushError('Unable to fetch new uploads', 'Try again later');
+					} else {
+						vueImage.waitUploadComplete();
+					}
+				}, 2000);
+			});
+
 		}
 
 	}
@@ -228,7 +259,8 @@ $('#btn-editor-uploadimage input').on('change', (ev) => {
 		allowedTypes: ['image/png', 'image/jpeg', 'image/gif', 'image/webp'],
 		maxFileSize: 3145728, // max 3 MB
 
-		init: () => {
+		init: (totalUploads) => {
+			vueImage.uploadSucceeded = false;
 			vueImage.isLoading = true;
 			vueImage.isLoadingText = 'Preparing to upload...';
 		},
@@ -240,19 +272,38 @@ $('#btn-editor-uploadimage input').on('change', (ev) => {
 		success: (data) => {
 			if(data.ok) {
 
+				let failedUpls = _.filter(data.results, ['ok', false]);
+				if(failedUpls.length) {
+					_.forEach(failedUpls, (u) => {
+						alerts.pushError('Upload error', u.msg);
+					});
+					if(failedUpls.length < data.results.length) {
+						alerts.push({
+							title: 'Some uploads succeeded',
+							message: 'Files that are not mentionned in the errors above were uploaded successfully.'
+						});
+						vueImage.uploadSucceeded = true;
+					} 
+				} else {
+					vueImage.uploadSucceeded = true;
+				}
+
 			} else {
 				alerts.pushError('Upload error', data.msg);
 			}
 		},
 
 		error: function(error) {
-			vueImage.isLoading = false;
 			alerts.pushError(error.message, this.upload.file.name);
 		},
 
 		finish: () => {
+			if(vueImage.uploadSucceeded) {
+				vueImage.waitUploadComplete();
+			} else {
 				vueImage.isLoading = false;
 			}
+		}
 
 	});
 

controllers/pages.js

@@ -160,11 +160,15 @@ router.get('/*', (req, res, next) => {
 		if(pageData) {
 			return res.render('pages/view', { pageData });
 		} else {
-			res.render('error', {
+			res.render('error-notexist', {
-				message: err.message,
+				newpath: safePath
-				error: {}
 			});
 		}
+	}).error((err) => {
+		res.render('error-notexist', {
+			message: err.message,
+			newpath: safePath
+		});
 	}).catch((err) => {
 		res.render('error', {
 			message: err.message,

controllers/uploads.js

@@ -72,13 +72,24 @@ router.post('/img', lcdata.uploadImgHandler, (req, res, next) => {
 
 		}).then(() => {
 			return {
+				ok: true,
 				filename: destFilename,
 				filesize: f.size
 			};
-		});
+		}).reflect();
 
 	}, {concurrency: 3}).then((results) => {
-		res.json({ ok: true, results });
+		let uplResults = _.map(results, (r) => {
+			if(r.isFulfilled()) {
+				return r.value();
+			} else {
+				return {
+					ok: false,
+					msg: r.reason().message
+				}
+			}
+		});
+		res.json({ ok: true, results: uplResults });
 	}).catch((err) => {
 		res.json({ ok: false, msg: err.message });
 	});

models/entries.js

@@ -170,7 +170,7 @@ module.exports = {
 				return false;
 			}
 		}).catch((err) => {
-			return Promise.reject(new Error('Entry ' + entryPath + ' does not exist!'));
+			return Promise.reject(new Promise.OperationalError('Entry ' + entryPath + ' does not exist!'));
 		});
 
 	},

models/git.js

@@ -235,6 +235,23 @@ module.exports = {
 			return true;
 		})
 
+	},
+
+	/**
+	 * Commits uploads changes.
+	 *
+	 * @return     {Promise}  Resolve on commit success
+	 */
+	commitUploads() {
+
+		let self = this;
+
+		return self._git.add('uploads').then(() => {
+			return self._git.commit("Uploads repository sync").catch((err) => {
+			  if(_.includes(err.stdout, 'nothing to commit')) { return true; }
+			});
+		});
+
 	}
 
 };

models/localdata.js

@@ -267,6 +267,22 @@ module.exports = {
 
 	},
 
+	/**
+	 * Parse relative Uploads path
+	 *
+	 * @param      {String}  f       Relative Uploads path
+	 * @return     {Object}  Parsed path (folder and filename)
+	 */
+	parseUploadsRelPath(f) {
+
+		let fObj = path.parse(f);
+		return {
+			folder: fObj.dir,
+			filename: fObj.base
+		};
+
+	},
+
 	/**
 	 * Sets the uploads files.
 	 *
@@ -288,6 +304,19 @@ module.exports = {
 
 	},
 
+	/**
+	 * Adds one or more uploads files.
+	 *
+	 * @param      {Array<Object>}  arrFiles  The uploads files
+	 * @return     {Void}  Void
+	 */
+	addUploadsFiles(arrFiles) {
+		if(_.isArray(arrFiles) || _.isPlainObject(arrFiles)) {
+			this._uploadsDb.Files.insert(arrFiles);
+		}
+		return;
+	},
+
 	/**
 	 * Gets the uploads files.
 	 *
@@ -301,41 +330,6 @@ module.exports = {
 			'$and': [{ 'category' : cat	},{ 'folder' : fld }]
 		}).simplesort('filename').data();
 
-	},
-
-	/**
-	 * Generate thumbnail of image
-	 *
-	 * @param      {String}           sourcePath  The source path
-	 * @return     {Promise<Object>}  Promise returning the resized image info
-	 */
-	generateThumbnail(sourcePath, destPath) {
-
-		let sharp = require('sharp');
-
-		return sharp(sourcePath)
-						.withoutEnlargement()
-						.resize(150,150)
-						.background('white')
-						.embed()
-						.flatten()
-						.toFormat('png')
-						.toFile(destPath);
-
-	},
-
-	/**
-	 * Gets the image metadata.
-	 *
-	 * @param      {String}  sourcePath  The source path
-	 * @return     {Object}  The image metadata.
-	 */
-	getImageMetadata(sourcePath) {
-
-		let sharp = require('sharp');
-
-		return sharp(sourcePath).metadata();
-
 	}
 
 };

models/uploads.js

@@ -0,0 +1,176 @@
+"use strict";
+
+var path = require('path'),
+	Promise = require('bluebird'),
+	fs = Promise.promisifyAll(require('fs-extra')),
+	readChunk = require('read-chunk'),
+	fileType = require('file-type'),
+	farmhash = require('farmhash'),
+	moment = require('moment'),
+	chokidar = require('chokidar'),
+	_ = require('lodash');
+
+/**
+ * Uploads
+ *
+ * @param      {Object}  appconfig  The application configuration
+ */
+module.exports = {
+
+	_uploadsPath: './repo/uploads',
+	_uploadsThumbsPath: './data/thumbs',
+
+	_watcher: null,
+
+	/**
+	 * Initialize Uploads model
+	 *
+	 * @param      {Object}  appconfig  The application config
+	 * @return     {Object}  Uploads model instance
+	 */
+	init(appconfig) {
+
+		let self = this;
+
+		self._uploadsPath = path.resolve(ROOTPATH, appconfig.datadir.repo, 'uploads');
+		self._uploadsThumbsPath = path.resolve(ROOTPATH, appconfig.datadir.db, 'thumbs');
+
+		return self;
+
+	},
+
+	watch() {
+
+		let self = this;
+
+		self._watcher = chokidar.watch(self._uploadsPath, {
+			persistent: true,
+			ignoreInitial: true,
+			cwd: self._uploadsPath,
+			depth: 1,
+			awaitWriteFinish: true
+		});
+
+		self._watcher.on('add', (p) => {
+
+			let pInfo = lcdata.parseUploadsRelPath(p);
+			return self.processFile(pInfo.folder, pInfo.filename).then((mData) => {
+				ws.emit('uploadsAddFiles', {
+					auth: WSInternalKey,
+					content: mData
+				});
+			}).then(() => {
+				return git.commitUploads();
+			});
+
+		});
+
+	},
+
+	processFile(fldName, f) {
+
+		let self = this;
+
+		let fldPath = path.join(self._uploadsPath, fldName);
+		let fPath = path.join(fldPath, f);
+		let fPathObj = path.parse(fPath);
+		let fUid = farmhash.fingerprint32(fldName + '/' + f);
+
+		return fs.statAsync(fPath).then((s) => {
+
+			if(!s.isFile()) { return false; }
+
+			// Get MIME info
+
+			let mimeInfo = fileType(readChunk.sync(fPath, 0, 262));
+
+			// Images
+
+			if(s.size < 3145728) { // ignore files larger than 3MB
+				if(_.includes(['image/png', 'image/jpeg', 'image/gif', 'image/webp'], mimeInfo.mime)) {
+					return self.getImageMetadata(fPath).then((mData) => {
+
+						let cacheThumbnailPath = path.parse(path.join(self._uploadsThumbsPath, fUid + '.png'));
+						let cacheThumbnailPathStr = path.format(cacheThumbnailPath);
+
+						mData = _.pick(mData, ['format', 'width', 'height', 'density', 'hasAlpha', 'orientation']);
+						mData.uid = fUid;
+						mData.category = 'image';
+						mData.mime = mimeInfo.mime;
+						mData.folder = fldName;
+						mData.filename = f;
+						mData.basename = fPathObj.name;
+						mData.filesize = s.size;
+						mData.uploadedOn = moment().utc();
+
+						// Generate thumbnail
+
+						return fs.statAsync(cacheThumbnailPathStr).then((st) => {
+							return st.isFile();
+						}).catch((err) => {
+							return false;
+						}).then((thumbExists) => {
+
+							return (thumbExists) ? mData : fs.ensureDirAsync(cacheThumbnailPath.dir).then(() => {
+								return self.generateThumbnail(fPath, cacheThumbnailPathStr);
+							}).return(mData);
+
+						});
+
+					})
+				}
+			}
+
+			// Other Files
+			
+			return {
+				uid: fUid,
+				category: 'file',
+				mime: mimeInfo.mime,
+				folder: fldName,
+				filename: f,
+				basename: fPathObj.name,
+				filesize: s.size,
+				uploadedOn: moment().utc()
+			};
+
+		});
+
+	},
+
+	/**
+	 * Generate thumbnail of image
+	 *
+	 * @param      {String}           sourcePath  The source path
+	 * @return     {Promise<Object>}  Promise returning the resized image info
+	 */
+	generateThumbnail(sourcePath, destPath) {
+
+		let sharp = require('sharp');
+
+		return sharp(sourcePath)
+						.withoutEnlargement()
+						.resize(150,150)
+						.background('white')
+						.embed()
+						.flatten()
+						.toFormat('png')
+						.toFile(destPath);
+
+	},
+
+	/**
+	 * Gets the image metadata.
+	 *
+	 * @param      {String}  sourcePath  The source path
+	 * @return     {Object}  The image metadata.
+	 */
+	getImageMetadata(sourcePath) {
+
+		let sharp = require('sharp');
+
+		return sharp(sourcePath).metadata();
+
+	}
+
+};

package.json

@@ -39,6 +39,7 @@
     "bson": "^0.5.5",
     "cheerio": "^0.22.0",
     "child-process-promise": "^2.1.3",
+    "chokidar": "^1.6.0",
     "compression": "^1.6.2",
     "connect-flash": "^0.1.1",
     "connect-loki": "^1.0.6",

views/error-notexist.pug

@@ -0,0 +1,32 @@
+doctype html
+html
+	head
+		meta(http-equiv='X-UA-Compatible', content='IE=edge')
+		meta(charset='UTF-8')
+		meta(name='viewport', content='width=device-width, initial-scale=1')
+		meta(name='theme-color', content='#009688')
+		meta(name='msapplication-TileColor', content='#009688')
+		meta(name='msapplication-TileImage', content='/favicons/ms-icon-144x144.png')
+		title= appconfig.title
+
+		// Favicon
+		each favsize in [57, 60, 72, 76, 114, 120, 144, 152, 180]
+			link(rel='apple-touch-icon', sizes=favsize + 'x' + favsize, href='/favicons/apple-icon-' + favsize + 'x' + favsize + '.png')
+		link(rel='icon', type='image/png', sizes='192x192', href='/favicons/android-icon-192x192.png')
+		each favsize in [32, 96, 16]
+			link(rel='icon', type='image/png', sizes=favsize + 'x' + favsize, href='/favicons/favicon-' + favsize + 'x' + favsize + '.png')
+		link(rel='manifest', href='/manifest.json')
+
+		// CSS
+		link(type='text/css', rel='stylesheet', href='/css/libs.css')
+		link(type='text/css', rel='stylesheet', href='/css/app.css')
+
+	body(class='server-error')
+		section.hero.is-dark.is-fullheight
+			.hero-body
+				.container
+					a(href='/'): img(src='/favicons/android-icon-96x96.png')
+					h1.title(style={ 'margin-top': '30px'})= message
+					h2.subtitle(style={ 'margin-bottom': '50px'}) Would you like to create this entry?
+					a.button.is-dark.is-inverted(href='/create/' + newpath, style={'margin-right': '5px'}) Create
+					a.button.is-dark.is-inverted(href='/') Go Home

ws-server.js

@@ -108,12 +108,14 @@ io.on('connection', (socket) => {
 	});
 
 	socket.on('searchDel', (data, cb) => {
+		cb = cb || _.noop
 		if(internalAuth.validateKey(data.auth)) {
 			search.delete(data.entryPath);
 		}
 	});
 
 	socket.on('search', (data, cb) => {
+		cb = cb || _.noop
 		search.find(data.terms).then((results) => {
 			cb(results);
 		});
@@ -124,28 +126,42 @@ io.on('connection', (socket) => {
 	//-----------------------------------------
 
 	socket.on('uploadsSetFolders', (data, cb) => {
+		cb = cb || _.noop
 		if(internalAuth.validateKey(data.auth)) {
 			lcdata.setUploadsFolders(data.content);
 		}
 	});
 
 	socket.on('uploadsGetFolders', (data, cb) => {
+		cb = cb || _.noop
 		cb(lcdata.getUploadsFolders());
 	});
 
 	socket.on('uploadsCreateFolder', (data, cb) => {
+		cb = cb || _.noop
 		lcdata.createUploadsFolder(data.foldername).then((fldList) => {
 			cb(fldList);
 		});
 	});
 
 	socket.on('uploadsSetFiles', (data, cb) => {
+		cb = cb || _.noop;
 		if(internalAuth.validateKey(data.auth)) {
 			lcdata.setUploadsFiles(data.content);
+			cb(true);
+		}
+	});
+
+	socket.on('uploadsAddFiles', (data, cb) => {
+		cb = cb || _.noop
+		if(internalAuth.validateKey(data.auth)) {
+			lcdata.addUploadsFiles(data.content);
+			cb(true);
 		}
 	});
 
 	socket.on('uploadsGetImages', (data, cb) => {
+		cb = cb || _.noop
 		cb(lcdata.getUploadsFiles('image', data.folder));
 	});