Git pull & push functionnality + caching work

config.sample.yml

@@ -26,8 +26,8 @@ port: 80
 # -------------------------------------------------
 
 datadir:
+  repo: ./repo
   db: ./data
-  uploads: ./uploads
 
 # -------------------------------------------------
 # Git Connection Info
@@ -35,23 +35,22 @@ datadir:
 # Full explanation + examples in the documentation (https://requarks-wiki.readme.io/)
 
 git:
-  path: ./repo
-  remote: true
   url: https://github.com/Organization/Repo
   branch: master
   auth:
+
+    # Type: basic, oauth or ssh
     type: ssh
-    user: gitusername
+
-    publickey: /etc/requarkswiki/keys/git.pub
+    username: marty
-    privatekey: /etc/requarkswiki/keys/git.key
+
-    passphrase: SomeSshPassphrase
+    # Password, OAuth token or private key passphrase:
-  # auth:
+    password: MartyMcFly88
-  #   type: oauth
+
-  #   token: 1234567890abcdefghijklmnopqrstuvxyz
+    # Only for SSH authentication:
-  # auth:
+    publicKey: /etc/requarkswiki/keys/git.pub
-  #   type: basic
+    privateKey: /etc/requarkswiki/keys/git.key
-  #   user: johnsmith
+    sslVerify: true
-  #   pass: password123
 
 # -------------------------------------------------
 # Secret key to use when encrypting sessions

controllers/pages.js

@@ -3,20 +3,30 @@
 var express = require('express');
 var router = express.Router();
 
+router.get('/edit/*', (req, res, next) => {
+	res.send('EDIT MODE');
+});
+
+router.get('/new/*', (req, res, next) => {
+	res.send('CREATE MODE');
+});
+
 /**
  * Home
  */
-router.get('/', (req, res) => {
+router.get('/*', (req, res, next) => {
 
-	var Promise = require('bluebird');
+	let safePath = entries.parsePath(req.path);
-	var fs = Promise.promisifyAll(require("fs"));
 
-	fs.readFileAsync("repo/Storage/Redis.md", "utf8").then(function(contents) {
+	entries.fetch(safePath).then((pageData) => {
-		let pageData = mark.parse(contents);
+		console.log(pageData);
-		if(!pageData.meta.title) {
+		if(pageData) {
-			pageData.meta.title = 'Redis.md';
-		}
 			res.render('pages/view', { pageData });
+		} else {
+			next();
+		}
+	}).catch((err) => {
+		next();
 	});
 
 });

gulpfile.js

@@ -47,8 +47,7 @@ var paths = {
 	],
 	fonts: [
 		'./node_modules/font-awesome/fonts/*-webfont.*',
-		'.!/node_modules/font-awesome/fonts/*-webfont.svg',
+		'!./node_modules/font-awesome/fonts/*-webfont.svg'
-		'./node_modules/roboto-fontface/fonts/Roboto/*.woff'
 	],
 	deploypackage: [
 		'./**/*',
@@ -67,7 +66,7 @@ var paths = {
 gulp.task('server', ['scripts', 'css', 'fonts'], function() {
 	nodemon({
 		script: './server',
-		ignore: ['assets/', 'client/', 'tests/'],
+		ignore: ['assets/', 'client/', 'data/', 'repo/', 'tests/'],
 		ext: 'js json',
 		env: { 'NODE_ENV': 'development' }
 	});

models/db.js

@@ -6,6 +6,8 @@ var loki = require('lokijs'),
 	 Promise = require('bluebird'),
 	 _ = require('lodash');
 
+var cols = ['User','Entry'];
+
 /**
  * Loki.js module
  *
@@ -27,27 +29,17 @@ module.exports = function(appconfig) {
 			autosave: true,
 			autosaveInterval: 5000
 		}),
-		Models: {},
 		onReady: dbReady
 	};
 
 	// Load Models
 
-	let dbModelsPath = path.join(ROOTPATH, 'models/db');
-
 	dbModel.Store.loadDatabase({}, () => {
 
-		fs
+		_.forEach(cols, (col) => {
-		.readdirSync(dbModelsPath)
+			dbModel[col] = dbModel.Store.getCollection(col);
-		.filter(function(file) {
+			if(!dbModel[col]) {
-			return (file.indexOf(".") !== 0);
+				dbModel[col] = dbModel.Store.addCollection(col);
-		})
-		.forEach(function(file) {
-			let modelName = _.upperFirst(_.split(file,'.')[0]);
-			dbModel.Models[modelName] = require(path.join(dbModelsPath, file));
-			dbModel[modelName] = dbModel.Store.getCollection(modelName);
-			if(!dbModel[modelName]) {
-				dbModel[modelName] = dbModel.Store.addCollection(modelName);
 			}
 		});
 

models/db/user.js

@@ -1,9 +0,0 @@
-"use strict";
-
-var bcrypt = require('bcryptjs-then');
-var Promise = require('bluebird');
-var _ = require('lodash');
-
-module.exports = {
-
-};

models/entries.js

@@ -0,0 +1,110 @@
+"use strict";
+
+var Promise = require('bluebird'),
+	path = require('path'),
+	fs = Promise.promisifyAll(require("fs")),
+	_ = require('lodash'),
+	farmhash = require('farmhash'),
+	msgpack = require('msgpack5')();
+
+/**
+ * Entries Model
+ */
+module.exports = {
+
+	_repoPath: 'repo',
+	_cachePath: 'data/cache',
+
+	/**
+	 * Initialize Entries model
+	 *
+	 * @param      {Object}  appconfig  The application config
+	 * @return     {Object}  Entries model instance
+	 */
+	init(appconfig) {
+
+		let self = this;
+
+		self._repoPath = appconfig.datadir.repo;
+		self._cachePath = path.join(appconfig.datadir.db, 'cache');
+
+		return self;
+
+	},
+
+	fetch(entryPath) {
+
+		let self = this;
+
+		let fpath = path.join(self._repoPath, entryPath + '.md');
+		let cpath = path.join(self._cachePath, farmhash.fingerprint32(entryPath) + '.bin');
+
+		return fs.statAsync(cpath).then((st) => {
+			return st.isFile();
+		}).catch((err) => {
+			return false;
+		}).then((isCache) => {
+
+			if(isCache) {
+
+				console.log('from cache!');
+
+				return fs.readFileAsync(cpath, 'utf8').then((contents) => {
+					return msgpack.decode(contents);
+				}).catch((err) => {
+					winston.error('Corrupted cache file. Deleting it...');
+					fs.unlinkSync(cpath);
+					return false;
+				});
+
+			} else {
+
+				console.log('original!');
+
+				// Parse original and cache it
+
+				return fs.statAsync(fpath).then((st) => {
+					if(st.isFile()) {
+						return fs.readFileAsync(fpath, 'utf8').then((contents) => {
+							let pageData = mark.parse(contents);
+							if(!pageData.meta.title) {
+								pageData.meta.title = entryPath;
+							}
+							let cacheData = msgpack.encode(pageData);
+							return fs.writeFileAsync(cpath, cacheData, { encoding: 'utf8' }).then(() => {
+								return pageData;
+							}).catch((err) => {
+								winston.error('Unable to write to cache! Performance may be affected.');
+								return pageData;
+							});
+					 	});
+					} else {
+						return false;
+					}
+				});
+
+			}
+
+		});
+
+		
+
+	},
+
+	parsePath(urlPath) {
+
+		let wlist = new RegExp('[^a-z0-9/\-]','g');
+
+		urlPath = _.toLower(urlPath).replace(wlist, '');
+
+		if(urlPath === '/') {
+			urlPath = 'home';
+		}
+
+		let urlParts = _.filter(_.split(urlPath, '/'), (p) => { return !_.isEmpty(p); });
+
+		return _.join(urlParts, '/');
+
+	}
+
+};

models/git.js

@@ -1,12 +1,13 @@
 "use strict";
 
-var NodeGit = require("nodegit"),
+var Git = require("git-wrapper2-promise"),
 	Promise = require('bluebird'),
 	path = require('path'),
 	os = require('os'),
 	fs = Promise.promisifyAll(require("fs")),
 	moment = require('moment'),
-	_ = require('lodash');
+	_ = require('lodash'),
+	URL = require('url');
 
 /**
  * Git Model
@@ -14,11 +15,11 @@ var NodeGit = require("nodegit"),
 module.exports = {
 
 	_git: null,
+	_url: '',
 	_repo: {
 		path: '',
 		branch: 'master',
 		exists: false,
-		inst: null,
 		sync: true
 	},
 	_signature: {
@@ -42,16 +43,15 @@ module.exports = {
 
 		//-> Build repository path
 		
-		if(_.isEmpty(appconfig.git.path) || appconfig.git.path === 'auto') {
+		if(_.isEmpty(appconfig.datadir.repo)) {
 			self._repo.path = path.join(ROOTPATH, 'repo');
 		} else {
-			self._repo.path = appconfig.git.path;
+			self._repo.path = appconfig.datadir.repo;
 		}
 
 		//-> Initialize repository
 
 		self._initRepo(appconfig).then((repo) => {
-			self._repo.inst = repo;
 
 			if(self._repo.sync) {
 				self.resync();
@@ -61,8 +61,8 @@ module.exports = {
 
 		// Define signature
 
-		self._signature.name = appconfig.git.userinfo.name || 'Wiki';
+		self._signature.name = appconfig.git.signature.name || 'Wiki';
-		self._signature.email = appconfig.git.userinfo.email || 'user@example.com';
+		self._signature.email = appconfig.git.signature.email || 'user@example.com';
 
 		return self;
 
@@ -78,7 +78,7 @@ module.exports = {
 
 		let self = this;
 
-		winston.info('[GIT] Initializing Git repository...');
+		winston.info('[GIT] Checking Git repository...');
 
 		//-> Check if path is accessible
 
@@ -88,136 +88,46 @@ module.exports = {
 			}
 		}).then(() => {
 
+			self._git = new Git({ 'git-dir': self._repo.path });
+
 			//-> Check if path already contains a git working folder
 
-			return fs.statAsync(path.join(self._repo.path, '.git')).then((stat) => {
+			return self._git.isRepo().then((isRepo) => {
-				self._repo.exists = stat.isDirectory();
+				self._repo.exists = isRepo;
+				return (!isRepo) ? self._git.exec('init') : true;
 			}).catch((err) => {
 				self._repo.exists = false;
 			});
 
 		}).then(() => {
 
-			//-> Init repository
+			// Initialize remote
 
-			let repoInitOperation = null;
+			let urlObj = URL.parse(appconfig.git.url);
-			self._repo.branch = appconfig.git.branch;
+			urlObj.auth = appconfig.git.auth.username + ((appconfig.git.auth.type !== 'ssh') ? ':' + appconfig.git.auth.password : '');
-			self._repo.sync = appconfig.git.remote;
+			self._url = URL.format(urlObj);
-			self._opts.clone = self._generateCloneOptions(appconfig);
-			self._opts.push = self._generatePushOptions(appconfig);
-
-			if(self._repo.exists) {
-
-				winston.info('[GIT] Using existing repository...');
-				repoInitOperation = NodeGit.Repository.open(self._repo.path);
-
-			} else if(appconfig.git.remote) {
-
-				winston.info('[GIT] Cloning remote repository for first time...');
-				repoInitOperation = NodeGit.Clone(appconfig.git.url, self._repo.path, self._opts.clone);
 
+			return self._git.exec('remote', 'show').then((cProc) => {
+				let out = cProc.stdout.toString();
+				if(_.includes(out, 'origin')) {
+					return true;
 				} else {
-
+					return Promise.join(
-				winston.info('[GIT] Using offline local repository...');
+						self._git.exec('config', ['--local', 'user.name', self._signature.name]),
-				repoInitOperation = NodeGit.Repository.init(self._repo.path, 0);
+						self._git.exec('config', ['--local', 'user.email', self._signature.email])
-
+					).then(() => {
+						return self._git.exec('remote', ['add', 'origin', self._url]);
+					})
 				}
-
-			return repoInitOperation;
-
-		}).catch((err) => {
-			winston.error('Unable to open or clone Git repository!');
-			winston.error(err);
-		}).then((repo) => {
-
-			if(self._repo.sync) {
-				NodeGit.Remote.setPushurl(repo, 'origin', appconfig.git.url);
-			}
-
-			return repo;
-
-			winston.info('[GIT] Git repository is now ready.');
 			});
 
-	},
+		}).catch((err) => {
-
+			winston.error('Git remote error!');
-	/**
+			throw err;
-	 * Generate Clone Options object
+		}).then(() => {
-	 *
+			winston.info('[GIT] Git repository is now ready.');
-	 * @param      {Object}  appconfig  The application configuration
+			return true;
-	 * @return     {Object}  CloneOptions object
+		});
-	 */
-	_generateCloneOptions(appconfig) {
-
-		let cloneOptions = new NodeGit.CloneOptions();
-		cloneOptions.fetchOpts = this._generateFetchOptions(appconfig);
-		return cloneOptions;
-
-	},
-
-	_generateFetchOptions(appconfig) {
-
-		let fetchOptions = new NodeGit.FetchOptions();
-		fetchOptions.callbacks = this._generateRemoteCallbacks(appconfig);
-		return fetchOptions;
-
-	},
-
-	_generatePushOptions(appconfig) {
-
-		let pushOptions = new NodeGit.PushOptions();
-		pushOptions.callbacks = this._generateRemoteCallbacks(appconfig);
-		return pushOptions;
-
-	},
-
-	_generateRemoteCallbacks(appconfig) {
-
-		let remoteCallbacks = new NodeGit.RemoteCallbacks();
-		let credFunc = this._generateCredentials(appconfig);
-		remoteCallbacks.credentials = () => { return credFunc; };
-		remoteCallbacks.transferProgress = _.noop;
-
-		if(os.type() === 'Darwin') {
-			remoteCallbacks.certificateCheck = () => { return 1; }; // Bug in OS X, bypass certs check workaround
-		} else {
-			remoteCallbacks.certificateCheck = _.noop;
-		}
-
-		return remoteCallbacks;
-
-	},
-
-	_generateCredentials(appconfig) {
-
-		let cred = null;
-		switch(appconfig.git.auth.type) {
-			case 'basic':
-				cred = NodeGit.Cred.userpassPlaintextNew(
-					appconfig.git.auth.user,
-					appconfig.git.auth.pass
-				);
-			break;
-			case 'oauth':
-				cred = NodeGit.Cred.userpassPlaintextNew(
-					appconfig.git.auth.token,
-					"x-oauth-basic"
-				);
-			break;
-			case 'ssh':
-				cred = NodeGit.Cred.sshKeyNew(
-					appconfig.git.auth.user,
-					appconfig.git.auth.publickey,
-					appconfig.git.auth.privatekey,
-					appconfig.git.auth.passphrase
-				);
-			break;
-			default:
-				cred = NodeGit.Cred.defaultNew();
-			break;
-		}
-
-		return cred;
 
 	},
 
@@ -227,70 +137,45 @@ module.exports = {
 
 		// Fetch
 
-		return self._repo.inst.fetch('origin', self._opts.clone.fetchOpts)
+		winston.info('[GIT] Performing pull from remote repository...');
-		.catch((err) => {
+		return self._git.pull('origin', self._repo.branch).then((cProc) => {
-			winston.error('Unable to fetch from git origin!' + err);
+			winston.info('[GIT] Pull completed.');
-		})
-
-		// Merge
-
-		.then(() => {
-			return self._repo.inst.mergeBranches(self._repo.branch, 'origin/' + self._repo.branch);
 		})
 		.catch((err) => {
-			winston.error('Unable to merge from remote head!' + err);
+			winston.error('Unable to fetch from git origin!');
+			throw err;
 		})
-
-		// Push
-
 		.then(() => {
-			return self._repo.inst.getRemote('origin').then((remote) => {
 
-				// Get modified files
+			// Check for changes
 
-				return self._repo.inst.refreshIndex().then((index) => {
+			return self._git.exec('status').then((cProc) => {
-					return self._repo.inst.getStatus().then(function(arrayStatusFile) {
+				let out = cProc.stdout.toString();
+				if(!_.includes(out, 'nothing to commit')) {
 
-						let addOp = [];
+					// Add, commit and push
 
-						// Add to next commit
+					winston.info('[GIT] Performing push to remote repository...');
-
+					return self._git.add('-A').then(() => {
-						_.forEach(arrayStatusFile, (v) => {
+				    return self._git.commit("Resync");
-							addOp.push(arrayStatusFile[0].path());
+				  }).then(() => {
+				    return self._git.push('origin', self._repo.branch);
+				  }).then(() => {
+						return winston.info('[GIT] Push completed.');
 					});
 
-						console.log('DUDE1');
+				} else {
+					winston.info('[GIT] Repository is already up to date. Nothing to commit.');
+				}
 
-						// Create Commit
+				return true;
-
-						let sig = NodeGit.Signature.create(self._signature.name, self._signature.email, moment().utc().unix(),  0);
-						return self._repo.inst.createCommitOnHead(addOp, sig, sig, "Wiki Sync").then(() => {
-
-							console.log('DUDE2');
-
-							return remote.connect(NodeGit.Enums.DIRECTION.PUSH, self._opts.push.callbacks).then(() => {
-
-								console.log('DUDE3');
-
-								// Push to remote
-
-								return remote.push( ["refs/heads/master:refs/heads/master"], self._opts.push).then((errNum) => {
-									console.log('DUDE' + errNum);
-								}).catch((err) => {
-									console.log(err);
-								});
 
 			});
 
-						});
-
-					});
 		})
-
+		.catch((err) => {
-				/**/
+			winston.error('Unable to push changes to remote!');
-			});
+			throw err;
-		}).catch((err) => {
-			winston.error('Unable to push to git origin!' + err);
 		});
 
 	}

models/localdata.js

@@ -1,6 +1,7 @@
 "use strict";
 
 var fs = require('fs'),
+	path = require('path'),
 	_ = require('lodash');
 
 /**
@@ -10,7 +11,7 @@ var fs = require('fs'),
  */
 module.exports = (appconfig) => {
 
-	// Create DB folder
+	// Create data directories
 
 	try {
 		fs.mkdirSync(appconfig.datadir.db);
@@ -21,10 +22,8 @@ module.exports = (appconfig) => {
 		}
 	}
 
-	// Create Uploads folder
-
 	try {
-		fs.mkdirSync(appconfig.datadir.uploads);
+		fs.mkdirSync(path.join(appconfig.datadir.db, 'cache'));
 	} catch (err) {
 		if(err.code !== 'EEXIST') {
 			winston.error(err);

package.json

@@ -36,6 +36,7 @@
     "body-parser": "^1.15.2",
     "bulma": "^0.1.2",
     "cheerio": "^0.22.0",
+    "child-process-promise": "^2.1.3",
     "compression": "^1.6.2",
     "connect-flash": "^0.1.1",
     "connect-loki": "^1.0.6",
@@ -46,6 +47,8 @@
     "express-brute-loki": "^1.0.0",
     "express-session": "^1.14.0",
     "express-validator": "^2.20.8",
+    "farmhash": "^1.2.0",
+    "git-wrapper2-promise": "^0.2.9",
     "highlight.js": "^9.6.0",
     "i18next": "^3.4.1",
     "i18next-express-middleware": "^1.0.1",
@@ -66,11 +69,10 @@
     "markdown-it-toc-and-anchor": "^4.1.1",
     "moment": "^2.14.1",
     "moment-timezone": "^0.5.5",
-    "nodegit": "^0.14.1",
+    "msgpack5": "^3.4.0",
     "passport": "^0.3.2",
     "passport-local": "^1.0.0",
     "pug": "^2.0.0-beta5",
-    "roboto-fontface": "^0.6.0",
     "serve-favicon": "^2.3.0",
     "simplemde": "^1.11.2",
     "slug": "^0.9.1",

server.js

@@ -4,14 +4,6 @@
 // Licensed under AGPLv3
 // ===========================================
 
-process.on('uncaughtException', function (exception) {
-  console.log(exception);
-});
-process.on('unhandledRejection', (reason, p) => {
-    console.log("Unhandled Rejection at: Promise ", p, " reason: ", reason);
-    // application specific logging, throwing an error, or other logic here
-});
-
 global.ROOTPATH = __dirname;
 
 // ----------------------------------------
@@ -22,10 +14,11 @@ global.winston = require('winston');
 winston.info('[SERVER] Requarks Wiki is initializing...');
 
 var appconfig = require('./models/config')('./config.yml');
-var lcdata = require('./models/localdata')(appconfig);
+let lcdata = require('./models/localdata');
 
-global.db = require('./models/loki')(appconfig);
+global.db = require('./models/db')(appconfig);
 global.git = require('./models/git').init(appconfig);
+global.entries = require('./models/entries').init(appconfig);
 global.mark = require('./models/markdown');
 
 // ----------------------------------------