Added Background Agent (git resync + cache purge)
This commit is contained in:
parent
e8ebf9d231
commit
59feacd846
@ -18,6 +18,7 @@
|
||||
##### Milestones
|
||||
- [ ] Assets Management
|
||||
- [ ] Authentication
|
||||
- [x] Background Agent (git sync, cache purge, etc.)
|
||||
- [x] Caching
|
||||
- [x] Create Entry
|
||||
- [x] Edit Entry
|
||||
@ -27,4 +28,5 @@
|
||||
- [x] Navigation
|
||||
- [x] Parsing / Tree / Metadata
|
||||
- [ ] Search
|
||||
- [x] UI
|
||||
- [x] UI
|
||||
- [x] View Entry Source
|
91
agent.js
Normal file
91
agent.js
Normal file
@ -0,0 +1,91 @@
|
||||
// ===========================================
|
||||
// REQUARKS WIKI - Background Agent
|
||||
// 1.0.0
|
||||
// Licensed under AGPLv3
|
||||
// ===========================================
|
||||
|
||||
global.ROOTPATH = __dirname;
|
||||
|
||||
// ----------------------------------------
|
||||
// Load modules
|
||||
// ----------------------------------------
|
||||
|
||||
global.winston = require('winston');
|
||||
winston.info('[AGENT] Requarks Wiki BgAgent is initializing...');
|
||||
|
||||
var appconfig = require('./models/config')('./config.yml');
|
||||
|
||||
global.git = require('./models/git').init(appconfig, true);
|
||||
global.entries = require('./models/entries').init(appconfig);
|
||||
global.mark = require('./models/markdown');
|
||||
|
||||
var _ = require('lodash');
|
||||
var moment = require('moment');
|
||||
var Promise = require('bluebird');
|
||||
var cron = require('cron').CronJob;
|
||||
|
||||
// ----------------------------------------
|
||||
// Start Cron
|
||||
// ----------------------------------------
|
||||
|
||||
var jobIsBusy = false;
|
||||
var job = new cron({
|
||||
cronTime: '0 */5 * * * *',
|
||||
onTick: () => {
|
||||
|
||||
// Make sure we don't start two concurrent jobs
|
||||
|
||||
if(jobIsBusy) {
|
||||
winston.warn('[AGENT] Previous job has not completed gracefully or is still running! Skipping for now. (This is not normal, you should investigate)');
|
||||
return;
|
||||
}
|
||||
jobIsBusy = true;
|
||||
|
||||
// Prepare async job collector
|
||||
|
||||
let jobs = [];
|
||||
|
||||
// ----------------------------------------
|
||||
// Compile Jobs
|
||||
// ----------------------------------------
|
||||
|
||||
//-> Resync with Git remote
|
||||
|
||||
jobs.push(git.resync().then(() => {
|
||||
|
||||
//-> Purge outdated cache
|
||||
|
||||
return entries.purgeStaleCache();
|
||||
|
||||
}));
|
||||
|
||||
// ----------------------------------------
|
||||
// Run
|
||||
// ----------------------------------------
|
||||
|
||||
Promise.all(jobs).then(() => {
|
||||
winston.info('[AGENT] All jobs completed successfully! Going to sleep for now... [' + moment().toISOString() + ']');
|
||||
}).catch((err) => {
|
||||
winston.error('[AGENT] One or more jobs have failed [' + moment().toISOString() + ']: ', err);
|
||||
}).finally(() => {
|
||||
jobIsBusy = false;
|
||||
});
|
||||
|
||||
},
|
||||
start: true,
|
||||
timeZone: 'UTC'
|
||||
});
|
||||
|
||||
// ----------------------------------------
|
||||
// Shutdown gracefully
|
||||
// ----------------------------------------
|
||||
|
||||
process.on('disconnect', () => {
|
||||
winston.warn('[AGENT] Lost connection to main server. Exiting... [' + moment().toISOString() + ']');
|
||||
job.stop();
|
||||
process.exit();
|
||||
});
|
||||
|
||||
process.on('exit', () => {
|
||||
job.stop();
|
||||
});
|
@ -6,7 +6,8 @@ var Promise = require('bluebird'),
|
||||
_ = require('lodash'),
|
||||
farmhash = require('farmhash'),
|
||||
BSONModule = require('bson'),
|
||||
BSON = new BSONModule.BSONPure.BSON();
|
||||
BSON = new BSONModule.BSONPure.BSON(),
|
||||
moment = require('moment');
|
||||
|
||||
/**
|
||||
* Entries Model
|
||||
@ -259,6 +260,17 @@ module.exports = {
|
||||
return path.join(this._cachePath, farmhash.fingerprint32(entryPath) + '.bson');
|
||||
},
|
||||
|
||||
/**
|
||||
* Gets the entry path from full path.
|
||||
*
|
||||
* @param {String} fullPath The full path
|
||||
* @return {String} The entry path
|
||||
*/
|
||||
getEntryPathFromFullPath(fullPath) {
|
||||
let absRepoPath = path.resolve(ROOTPATH, this._repoPath);
|
||||
return _.chain(fullPath).replace(absRepoPath, '').replace('.md', '').replace(new RegExp('\\\\', 'g'),'/').value();
|
||||
},
|
||||
|
||||
/**
|
||||
* Update an existing document
|
||||
*
|
||||
@ -344,6 +356,35 @@ module.exports = {
|
||||
return _.replace(contents, new RegExp('{TITLE}', 'g'), formattedTitle);
|
||||
});
|
||||
|
||||
},
|
||||
|
||||
purgeStaleCache() {
|
||||
|
||||
let self = this;
|
||||
|
||||
let cacheJobs = [];
|
||||
|
||||
fs.walk(self._repoPath)
|
||||
.on('data', function (item) {
|
||||
if(path.extname(item.path) === '.md') {
|
||||
|
||||
let entryPath = self.parsePath(self.getEntryPathFromFullPath(item.path));
|
||||
let cachePath = self.getCachePath(entryPath);
|
||||
|
||||
cacheJobs.push(fs.statAsync(cachePath).then((st) => {
|
||||
if(moment(st.mtime).isBefore(item.stats.mtime)) {
|
||||
return fs.unlinkAsync(cachePath);
|
||||
} else {
|
||||
return true;
|
||||
}
|
||||
}).catch((err) => {
|
||||
return (err.code !== 'EEXIST') ? err : true;
|
||||
}));
|
||||
}
|
||||
});
|
||||
|
||||
return Promise.all(cacheJobs);
|
||||
|
||||
}
|
||||
|
||||
};
|
@ -37,10 +37,12 @@ module.exports = {
|
||||
* @param {Object} appconfig The application config
|
||||
* @return {Object} Git model instance
|
||||
*/
|
||||
init(appconfig) {
|
||||
init(appconfig, sync) {
|
||||
|
||||
let self = this;
|
||||
|
||||
self._repo.sync = sync;
|
||||
|
||||
//-> Build repository path
|
||||
|
||||
if(_.isEmpty(appconfig.datadir.repo)) {
|
||||
|
@ -42,6 +42,7 @@
|
||||
"connect-loki": "^1.0.6",
|
||||
"connect-redis": "^3.1.0",
|
||||
"cookie-parser": "^1.4.3",
|
||||
"cron": "^1.1.0",
|
||||
"express": "^4.14.0",
|
||||
"express-brute": "^1.0.0",
|
||||
"express-brute-loki": "^1.0.0",
|
||||
|
17
server.js
17
server.js
@ -17,7 +17,7 @@ var appconfig = require('./models/config')('./config.yml');
|
||||
let lcdata = require('./models/localdata');
|
||||
|
||||
global.db = require('./models/db')(appconfig);
|
||||
global.git = require('./models/git').init(appconfig);
|
||||
global.git = require('./models/git').init(appconfig, false);
|
||||
global.entries = require('./models/entries').init(appconfig);
|
||||
global.mark = require('./models/markdown');
|
||||
|
||||
@ -192,4 +192,19 @@ server.on('error', (error) => {
|
||||
|
||||
server.on('listening', () => {
|
||||
winston.info('[SERVER] HTTP server started successfully! [RUNNING]');
|
||||
});
|
||||
|
||||
// ----------------------------------------
|
||||
// Start Background Agent
|
||||
// ----------------------------------------
|
||||
|
||||
var fork = require('child_process').fork;
|
||||
var bgAgent = fork('agent.js');
|
||||
|
||||
bgAgent.on('message', (m) => {
|
||||
|
||||
});
|
||||
|
||||
process.on('exit', (code) => {
|
||||
bgAgent.disconnect();
|
||||
});
|
Loading…
Reference in New Issue
Block a user