Added Background Agent (git resync + cache purge)

README.md

@@ -18,6 +18,7 @@
 ##### Milestones
 - [ ] Assets Management
 - [ ] Authentication
+- [x] Background Agent (git sync, cache purge, etc.)
 - [x] Caching
 - [x] Create Entry
 - [x] Edit Entry
@@ -28,3 +29,4 @@
 - [x] Parsing / Tree / Metadata
 - [ ] Search
 - [x] UI
+- [x] View Entry Source

agent.js

@@ -0,0 +1,91 @@
+// ===========================================
+// REQUARKS WIKI - Background Agent
+// 1.0.0
+// Licensed under AGPLv3
+// ===========================================
+
+global.ROOTPATH = __dirname;
+
+// ----------------------------------------
+// Load modules
+// ----------------------------------------
+
+global.winston = require('winston');
+winston.info('[AGENT] Requarks Wiki BgAgent is initializing...');
+
+var appconfig = require('./models/config')('./config.yml');
+
+global.git = require('./models/git').init(appconfig, true);
+global.entries = require('./models/entries').init(appconfig);
+global.mark = require('./models/markdown');
+
+var _ = require('lodash');
+var moment = require('moment');
+var Promise = require('bluebird');
+var cron = require('cron').CronJob;
+
+// ----------------------------------------
+// Start Cron
+// ----------------------------------------
+
+var jobIsBusy = false;
+var job = new cron({
+	cronTime: '0 */5 * * * *',
+	onTick: () => {
+
+		// Make sure we don't start two concurrent jobs
+
+		if(jobIsBusy) {
+			winston.warn('[AGENT] Previous job has not completed gracefully or is still running! Skipping for now. (This is not normal, you should investigate)');
+			return;
+		}
+		jobIsBusy = true;
+
+		// Prepare async job collector
+
+		let jobs = [];
+
+		// ----------------------------------------
+		// Compile Jobs
+		// ----------------------------------------
+
+		//-> Resync with Git remote
+
+		jobs.push(git.resync().then(() => {
+
+			//-> Purge outdated cache
+
+			return entries.purgeStaleCache();
+
+		}));
+
+		// ----------------------------------------
+		// Run
+		// ----------------------------------------
+
+		Promise.all(jobs).then(() => {
+			winston.info('[AGENT] All jobs completed successfully! Going to sleep for now... [' + moment().toISOString() + ']');
+		}).catch((err) => {
+			winston.error('[AGENT] One or more jobs have failed [' + moment().toISOString() + ']: ', err);
+		}).finally(() => {
+			jobIsBusy = false;
+		});
+
+	},
+	start: true,
+	timeZone: 'UTC'
+});
+
+// ----------------------------------------
+// Shutdown gracefully
+// ----------------------------------------
+
+process.on('disconnect', () => {
+	winston.warn('[AGENT] Lost connection to main server. Exiting... [' + moment().toISOString() + ']');
+	job.stop();
+	process.exit();
+});
+
+process.on('exit', () => {
+	job.stop();
+});

models/entries.js

@@ -6,7 +6,8 @@ var Promise = require('bluebird'),
 	_ = require('lodash'),
 	farmhash = require('farmhash'),
 	BSONModule = require('bson'),
-	BSON = new BSONModule.BSONPure.BSON();
+	BSON = new BSONModule.BSONPure.BSON(),
+	moment = require('moment');
 
 /**
  * Entries Model
@@ -259,6 +260,17 @@ module.exports = {
 		return path.join(this._cachePath, farmhash.fingerprint32(entryPath) + '.bson');
 	},
 
+	/**
+	 * Gets the entry path from full path.
+	 *
+	 * @param      {String}  fullPath  The full path
+	 * @return     {String}  The entry path
+	 */
+	getEntryPathFromFullPath(fullPath) {
+		let absRepoPath = path.resolve(ROOTPATH, this._repoPath);
+		return _.chain(fullPath).replace(absRepoPath, '').replace('.md', '').replace(new RegExp('\\\\', 'g'),'/').value();
+	},
+
 	/**
 	 * Update an existing document
 	 *
@@ -344,6 +356,35 @@ module.exports = {
 			return _.replace(contents, new RegExp('{TITLE}', 'g'), formattedTitle);
 		});
 
+	},
+
+	purgeStaleCache() {
+
+		let self = this;
+
+		let cacheJobs = [];
+
+		fs.walk(self._repoPath)
+		.on('data', function (item) {
+			if(path.extname(item.path) === '.md') {
+
+				let entryPath = self.parsePath(self.getEntryPathFromFullPath(item.path));
+				let cachePath = self.getCachePath(entryPath);
+
+				cacheJobs.push(fs.statAsync(cachePath).then((st) => {
+					if(moment(st.mtime).isBefore(item.stats.mtime)) {
+						return fs.unlinkAsync(cachePath);
+					} else {
+						return true;
+					}
+				}).catch((err) => {
+					return (err.code !== 'EEXIST') ? err : true;
+				}));
+			}
+		});
+
+		return Promise.all(cacheJobs);
+
 	}
 
 };

models/git.js

@@ -37,10 +37,12 @@ module.exports = {
 	 * @param      {Object}  appconfig  The application config
 	 * @return     {Object}  Git model instance
 	 */
-	init(appconfig) {
+	init(appconfig, sync) {
 
 		let self = this;
 
+		self._repo.sync = sync;
+
 		//-> Build repository path
 		
 		if(_.isEmpty(appconfig.datadir.repo)) {

package.json

@@ -42,6 +42,7 @@
     "connect-loki": "^1.0.6",
     "connect-redis": "^3.1.0",
     "cookie-parser": "^1.4.3",
+    "cron": "^1.1.0",
     "express": "^4.14.0",
     "express-brute": "^1.0.0",
     "express-brute-loki": "^1.0.0",

server.js

@@ -17,7 +17,7 @@ var appconfig = require('./models/config')('./config.yml');
 let lcdata = require('./models/localdata');
 
 global.db = require('./models/db')(appconfig);
-global.git = require('./models/git').init(appconfig);
+global.git = require('./models/git').init(appconfig, false);
 global.entries = require('./models/entries').init(appconfig);
 global.mark = require('./models/markdown');
 
@@ -193,3 +193,18 @@ server.on('error', (error) => {
 server.on('listening', () => {
   winston.info('[SERVER] HTTP server started successfully! [RUNNING]');
 });
+
+// ----------------------------------------
+// Start Background Agent
+// ----------------------------------------
+
+var fork = require('child_process').fork;
+var bgAgent = fork('agent.js');
+
+bgAgent.on('message', (m) => {
+
+});
+
+process.on('exit', (code) => {
+  bgAgent.disconnect();
+});