Git pull & push functionnality + caching work

This commit is contained in:
NGPixel 2016-08-27 01:04:08 -04:00
parent cc1e6b4432
commit 1d2893765c
10 changed files with 223 additions and 243 deletions

View File

@ -26,8 +26,8 @@ port: 80
# -------------------------------------------------
datadir:
repo: ./repo
db: ./data
uploads: ./uploads
# -------------------------------------------------
# Git Connection Info
@ -35,23 +35,22 @@ datadir:
# Full explanation + examples in the documentation (https://requarks-wiki.readme.io/)
git:
path: ./repo
remote: true
url: https://github.com/Organization/Repo
branch: master
auth:
# Type: basic, oauth or ssh
type: ssh
user: gitusername
publickey: /etc/requarkswiki/keys/git.pub
privatekey: /etc/requarkswiki/keys/git.key
passphrase: SomeSshPassphrase
# auth:
# type: oauth
# token: 1234567890abcdefghijklmnopqrstuvxyz
# auth:
# type: basic
# user: johnsmith
# pass: password123
username: marty
# Password, OAuth token or private key passphrase:
password: MartyMcFly88
# Only for SSH authentication:
publicKey: /etc/requarkswiki/keys/git.pub
privateKey: /etc/requarkswiki/keys/git.key
sslVerify: true
# -------------------------------------------------
# Secret key to use when encrypting sessions

View File

@ -3,21 +3,31 @@
var express = require('express');
var router = express.Router();
router.get('/edit/*', (req, res, next) => {
res.send('EDIT MODE');
});
router.get('/new/*', (req, res, next) => {
res.send('CREATE MODE');
});
/**
* Home
*/
router.get('/', (req, res) => {
router.get('/*', (req, res, next) => {
var Promise = require('bluebird');
var fs = Promise.promisifyAll(require("fs"));
let safePath = entries.parsePath(req.path);
fs.readFileAsync("repo/Storage/Redis.md", "utf8").then(function(contents) {
let pageData = mark.parse(contents);
if(!pageData.meta.title) {
pageData.meta.title = 'Redis.md';
entries.fetch(safePath).then((pageData) => {
console.log(pageData);
if(pageData) {
res.render('pages/view', { pageData });
} else {
next();
}
res.render('pages/view', { pageData });
});
}).catch((err) => {
next();
});
});

View File

@ -47,8 +47,7 @@ var paths = {
],
fonts: [
'./node_modules/font-awesome/fonts/*-webfont.*',
'.!/node_modules/font-awesome/fonts/*-webfont.svg',
'./node_modules/roboto-fontface/fonts/Roboto/*.woff'
'!./node_modules/font-awesome/fonts/*-webfont.svg'
],
deploypackage: [
'./**/*',
@ -67,7 +66,7 @@ var paths = {
gulp.task('server', ['scripts', 'css', 'fonts'], function() {
nodemon({
script: './server',
ignore: ['assets/', 'client/', 'tests/'],
ignore: ['assets/', 'client/', 'data/', 'repo/', 'tests/'],
ext: 'js json',
env: { 'NODE_ENV': 'development' }
});

View File

@ -6,6 +6,8 @@ var loki = require('lokijs'),
Promise = require('bluebird'),
_ = require('lodash');
var cols = ['User','Entry'];
/**
* Loki.js module
*
@ -27,27 +29,17 @@ module.exports = function(appconfig) {
autosave: true,
autosaveInterval: 5000
}),
Models: {},
onReady: dbReady
};
// Load Models
let dbModelsPath = path.join(ROOTPATH, 'models/db');
dbModel.Store.loadDatabase({}, () => {
fs
.readdirSync(dbModelsPath)
.filter(function(file) {
return (file.indexOf(".") !== 0);
})
.forEach(function(file) {
let modelName = _.upperFirst(_.split(file,'.')[0]);
dbModel.Models[modelName] = require(path.join(dbModelsPath, file));
dbModel[modelName] = dbModel.Store.getCollection(modelName);
if(!dbModel[modelName]) {
dbModel[modelName] = dbModel.Store.addCollection(modelName);
_.forEach(cols, (col) => {
dbModel[col] = dbModel.Store.getCollection(col);
if(!dbModel[col]) {
dbModel[col] = dbModel.Store.addCollection(col);
}
});

View File

@ -1,9 +0,0 @@
"use strict";
var bcrypt = require('bcryptjs-then');
var Promise = require('bluebird');
var _ = require('lodash');
module.exports = {
};

110
models/entries.js Normal file
View File

@ -0,0 +1,110 @@
"use strict";
var Promise = require('bluebird'),
path = require('path'),
fs = Promise.promisifyAll(require("fs")),
_ = require('lodash'),
farmhash = require('farmhash'),
msgpack = require('msgpack5')();
/**
* Entries Model
*/
module.exports = {
_repoPath: 'repo',
_cachePath: 'data/cache',
/**
* Initialize Entries model
*
* @param {Object} appconfig The application config
* @return {Object} Entries model instance
*/
init(appconfig) {
let self = this;
self._repoPath = appconfig.datadir.repo;
self._cachePath = path.join(appconfig.datadir.db, 'cache');
return self;
},
fetch(entryPath) {
let self = this;
let fpath = path.join(self._repoPath, entryPath + '.md');
let cpath = path.join(self._cachePath, farmhash.fingerprint32(entryPath) + '.bin');
return fs.statAsync(cpath).then((st) => {
return st.isFile();
}).catch((err) => {
return false;
}).then((isCache) => {
if(isCache) {
console.log('from cache!');
return fs.readFileAsync(cpath, 'utf8').then((contents) => {
return msgpack.decode(contents);
}).catch((err) => {
winston.error('Corrupted cache file. Deleting it...');
fs.unlinkSync(cpath);
return false;
});
} else {
console.log('original!');
// Parse original and cache it
return fs.statAsync(fpath).then((st) => {
if(st.isFile()) {
return fs.readFileAsync(fpath, 'utf8').then((contents) => {
let pageData = mark.parse(contents);
if(!pageData.meta.title) {
pageData.meta.title = entryPath;
}
let cacheData = msgpack.encode(pageData);
return fs.writeFileAsync(cpath, cacheData, { encoding: 'utf8' }).then(() => {
return pageData;
}).catch((err) => {
winston.error('Unable to write to cache! Performance may be affected.');
return pageData;
});
});
} else {
return false;
}
});
}
});
},
parsePath(urlPath) {
let wlist = new RegExp('[^a-z0-9/\-]','g');
urlPath = _.toLower(urlPath).replace(wlist, '');
if(urlPath === '/') {
urlPath = 'home';
}
let urlParts = _.filter(_.split(urlPath, '/'), (p) => { return !_.isEmpty(p); });
return _.join(urlParts, '/');
}
};

View File

@ -1,12 +1,13 @@
"use strict";
var NodeGit = require("nodegit"),
var Git = require("git-wrapper2-promise"),
Promise = require('bluebird'),
path = require('path'),
os = require('os'),
fs = Promise.promisifyAll(require("fs")),
moment = require('moment'),
_ = require('lodash');
_ = require('lodash'),
URL = require('url');
/**
* Git Model
@ -14,11 +15,11 @@ var NodeGit = require("nodegit"),
module.exports = {
_git: null,
_url: '',
_repo: {
path: '',
branch: 'master',
exists: false,
inst: null,
sync: true
},
_signature: {
@ -42,16 +43,15 @@ module.exports = {
//-> Build repository path
if(_.isEmpty(appconfig.git.path) || appconfig.git.path === 'auto') {
if(_.isEmpty(appconfig.datadir.repo)) {
self._repo.path = path.join(ROOTPATH, 'repo');
} else {
self._repo.path = appconfig.git.path;
self._repo.path = appconfig.datadir.repo;
}
//-> Initialize repository
self._initRepo(appconfig).then((repo) => {
self._repo.inst = repo;
if(self._repo.sync) {
self.resync();
@ -61,8 +61,8 @@ module.exports = {
// Define signature
self._signature.name = appconfig.git.userinfo.name || 'Wiki';
self._signature.email = appconfig.git.userinfo.email || 'user@example.com';
self._signature.name = appconfig.git.signature.name || 'Wiki';
self._signature.email = appconfig.git.signature.email || 'user@example.com';
return self;
@ -78,7 +78,7 @@ module.exports = {
let self = this;
winston.info('[GIT] Initializing Git repository...');
winston.info('[GIT] Checking Git repository...');
//-> Check if path is accessible
@ -88,209 +88,94 @@ module.exports = {
}
}).then(() => {
self._git = new Git({ 'git-dir': self._repo.path });
//-> Check if path already contains a git working folder
return fs.statAsync(path.join(self._repo.path, '.git')).then((stat) => {
self._repo.exists = stat.isDirectory();
return self._git.isRepo().then((isRepo) => {
self._repo.exists = isRepo;
return (!isRepo) ? self._git.exec('init') : true;
}).catch((err) => {
self._repo.exists = false;
});
}).then(() => {
//-> Init repository
// Initialize remote
let repoInitOperation = null;
self._repo.branch = appconfig.git.branch;
self._repo.sync = appconfig.git.remote;
self._opts.clone = self._generateCloneOptions(appconfig);
self._opts.push = self._generatePushOptions(appconfig);
let urlObj = URL.parse(appconfig.git.url);
urlObj.auth = appconfig.git.auth.username + ((appconfig.git.auth.type !== 'ssh') ? ':' + appconfig.git.auth.password : '');
self._url = URL.format(urlObj);
if(self._repo.exists) {
winston.info('[GIT] Using existing repository...');
repoInitOperation = NodeGit.Repository.open(self._repo.path);
} else if(appconfig.git.remote) {
winston.info('[GIT] Cloning remote repository for first time...');
repoInitOperation = NodeGit.Clone(appconfig.git.url, self._repo.path, self._opts.clone);
} else {
winston.info('[GIT] Using offline local repository...');
repoInitOperation = NodeGit.Repository.init(self._repo.path, 0);
}
return repoInitOperation;
return self._git.exec('remote', 'show').then((cProc) => {
let out = cProc.stdout.toString();
if(_.includes(out, 'origin')) {
return true;
} else {
return Promise.join(
self._git.exec('config', ['--local', 'user.name', self._signature.name]),
self._git.exec('config', ['--local', 'user.email', self._signature.email])
).then(() => {
return self._git.exec('remote', ['add', 'origin', self._url]);
})
}
});
}).catch((err) => {
winston.error('Unable to open or clone Git repository!');
winston.error(err);
}).then((repo) => {
if(self._repo.sync) {
NodeGit.Remote.setPushurl(repo, 'origin', appconfig.git.url);
}
return repo;
winston.error('Git remote error!');
throw err;
}).then(() => {
winston.info('[GIT] Git repository is now ready.');
return true;
});
},
/**
* Generate Clone Options object
*
* @param {Object} appconfig The application configuration
* @return {Object} CloneOptions object
*/
_generateCloneOptions(appconfig) {
let cloneOptions = new NodeGit.CloneOptions();
cloneOptions.fetchOpts = this._generateFetchOptions(appconfig);
return cloneOptions;
},
_generateFetchOptions(appconfig) {
let fetchOptions = new NodeGit.FetchOptions();
fetchOptions.callbacks = this._generateRemoteCallbacks(appconfig);
return fetchOptions;
},
_generatePushOptions(appconfig) {
let pushOptions = new NodeGit.PushOptions();
pushOptions.callbacks = this._generateRemoteCallbacks(appconfig);
return pushOptions;
},
_generateRemoteCallbacks(appconfig) {
let remoteCallbacks = new NodeGit.RemoteCallbacks();
let credFunc = this._generateCredentials(appconfig);
remoteCallbacks.credentials = () => { return credFunc; };
remoteCallbacks.transferProgress = _.noop;
if(os.type() === 'Darwin') {
remoteCallbacks.certificateCheck = () => { return 1; }; // Bug in OS X, bypass certs check workaround
} else {
remoteCallbacks.certificateCheck = _.noop;
}
return remoteCallbacks;
},
_generateCredentials(appconfig) {
let cred = null;
switch(appconfig.git.auth.type) {
case 'basic':
cred = NodeGit.Cred.userpassPlaintextNew(
appconfig.git.auth.user,
appconfig.git.auth.pass
);
break;
case 'oauth':
cred = NodeGit.Cred.userpassPlaintextNew(
appconfig.git.auth.token,
"x-oauth-basic"
);
break;
case 'ssh':
cred = NodeGit.Cred.sshKeyNew(
appconfig.git.auth.user,
appconfig.git.auth.publickey,
appconfig.git.auth.privatekey,
appconfig.git.auth.passphrase
);
break;
default:
cred = NodeGit.Cred.defaultNew();
break;
}
return cred;
},
resync() {
let self = this;
// Fetch
return self._repo.inst.fetch('origin', self._opts.clone.fetchOpts)
.catch((err) => {
winston.error('Unable to fetch from git origin!' + err);
})
// Merge
.then(() => {
return self._repo.inst.mergeBranches(self._repo.branch, 'origin/' + self._repo.branch);
winston.info('[GIT] Performing pull from remote repository...');
return self._git.pull('origin', self._repo.branch).then((cProc) => {
winston.info('[GIT] Pull completed.');
})
.catch((err) => {
winston.error('Unable to merge from remote head!' + err);
winston.error('Unable to fetch from git origin!');
throw err;
})
// Push
.then(() => {
return self._repo.inst.getRemote('origin').then((remote) => {
// Get modified files
// Check for changes
return self._repo.inst.refreshIndex().then((index) => {
return self._repo.inst.getStatus().then(function(arrayStatusFile) {
return self._git.exec('status').then((cProc) => {
let out = cProc.stdout.toString();
if(!_.includes(out, 'nothing to commit')) {
let addOp = [];
// Add to next commit
_.forEach(arrayStatusFile, (v) => {
addOp.push(arrayStatusFile[0].path());
});
console.log('DUDE1');
// Create Commit
let sig = NodeGit.Signature.create(self._signature.name, self._signature.email, moment().utc().unix(), 0);
return self._repo.inst.createCommitOnHead(addOp, sig, sig, "Wiki Sync").then(() => {
console.log('DUDE2');
return remote.connect(NodeGit.Enums.DIRECTION.PUSH, self._opts.push.callbacks).then(() => {
console.log('DUDE3');
// Push to remote
return remote.push( ["refs/heads/master:refs/heads/master"], self._opts.push).then((errNum) => {
console.log('DUDE' + errNum);
}).catch((err) => {
console.log(err);
});
});
});
// Add, commit and push
winston.info('[GIT] Performing push to remote repository...');
return self._git.add('-A').then(() => {
return self._git.commit("Resync");
}).then(() => {
return self._git.push('origin', self._repo.branch);
}).then(() => {
return winston.info('[GIT] Push completed.');
});
})
/**/
} else {
winston.info('[GIT] Repository is already up to date. Nothing to commit.');
}
return true;
});
}).catch((err) => {
winston.error('Unable to push to git origin!' + err);
})
.catch((err) => {
winston.error('Unable to push changes to remote!');
throw err;
});
}

View File

@ -1,6 +1,7 @@
"use strict";
var fs = require('fs'),
path = require('path'),
_ = require('lodash');
/**
@ -10,7 +11,7 @@ var fs = require('fs'),
*/
module.exports = (appconfig) => {
// Create DB folder
// Create data directories
try {
fs.mkdirSync(appconfig.datadir.db);
@ -21,10 +22,8 @@ module.exports = (appconfig) => {
}
}
// Create Uploads folder
try {
fs.mkdirSync(appconfig.datadir.uploads);
fs.mkdirSync(path.join(appconfig.datadir.db, 'cache'));
} catch (err) {
if(err.code !== 'EEXIST') {
winston.error(err);

View File

@ -36,6 +36,7 @@
"body-parser": "^1.15.2",
"bulma": "^0.1.2",
"cheerio": "^0.22.0",
"child-process-promise": "^2.1.3",
"compression": "^1.6.2",
"connect-flash": "^0.1.1",
"connect-loki": "^1.0.6",
@ -46,6 +47,8 @@
"express-brute-loki": "^1.0.0",
"express-session": "^1.14.0",
"express-validator": "^2.20.8",
"farmhash": "^1.2.0",
"git-wrapper2-promise": "^0.2.9",
"highlight.js": "^9.6.0",
"i18next": "^3.4.1",
"i18next-express-middleware": "^1.0.1",
@ -66,11 +69,10 @@
"markdown-it-toc-and-anchor": "^4.1.1",
"moment": "^2.14.1",
"moment-timezone": "^0.5.5",
"nodegit": "^0.14.1",
"msgpack5": "^3.4.0",
"passport": "^0.3.2",
"passport-local": "^1.0.0",
"pug": "^2.0.0-beta5",
"roboto-fontface": "^0.6.0",
"serve-favicon": "^2.3.0",
"simplemde": "^1.11.2",
"slug": "^0.9.1",

View File

@ -4,14 +4,6 @@
// Licensed under AGPLv3
// ===========================================
process.on('uncaughtException', function (exception) {
console.log(exception);
});
process.on('unhandledRejection', (reason, p) => {
console.log("Unhandled Rejection at: Promise ", p, " reason: ", reason);
// application specific logging, throwing an error, or other logic here
});
global.ROOTPATH = __dirname;
// ----------------------------------------
@ -22,10 +14,11 @@ global.winston = require('winston');
winston.info('[SERVER] Requarks Wiki is initializing...');
var appconfig = require('./models/config')('./config.yml');
var lcdata = require('./models/localdata')(appconfig);
let lcdata = require('./models/localdata');
global.db = require('./models/loki')(appconfig);
global.db = require('./models/db')(appconfig);
global.git = require('./models/git').init(appconfig);
global.entries = require('./models/entries').init(appconfig);
global.mark = require('./models/markdown');
// ----------------------------------------