From 1d2893765cad9d2d875a0d7e480279af42da7745 Mon Sep 17 00:00:00 2001 From: NGPixel Date: Sat, 27 Aug 2016 01:04:08 -0400 Subject: [PATCH] Git pull & push functionnality + caching work --- config.sample.yml | 27 ++--- controllers/pages.js | 28 +++-- gulpfile.js | 5 +- models/{loki.js => db.js} | 20 +--- models/db/user.js | 9 -- models/entries.js | 110 +++++++++++++++++ models/git.js | 241 ++++++++++---------------------------- models/localdata.js | 7 +- package.json | 6 +- server.js | 13 +- 10 files changed, 223 insertions(+), 243 deletions(-) rename models/{loki.js => db.js} (60%) delete mode 100644 models/db/user.js create mode 100644 models/entries.js diff --git a/config.sample.yml b/config.sample.yml index 46540363..b6aa879f 100644 --- a/config.sample.yml +++ b/config.sample.yml @@ -26,8 +26,8 @@ port: 80 # ------------------------------------------------- datadir: + repo: ./repo db: ./data - uploads: ./uploads # ------------------------------------------------- # Git Connection Info @@ -35,23 +35,22 @@ datadir: # Full explanation + examples in the documentation (https://requarks-wiki.readme.io/) git: - path: ./repo - remote: true url: https://github.com/Organization/Repo branch: master auth: + + # Type: basic, oauth or ssh type: ssh - user: gitusername - publickey: /etc/requarkswiki/keys/git.pub - privatekey: /etc/requarkswiki/keys/git.key - passphrase: SomeSshPassphrase - # auth: - # type: oauth - # token: 1234567890abcdefghijklmnopqrstuvxyz - # auth: - # type: basic - # user: johnsmith - # pass: password123 + + username: marty + + # Password, OAuth token or private key passphrase: + password: MartyMcFly88 + + # Only for SSH authentication: + publicKey: /etc/requarkswiki/keys/git.pub + privateKey: /etc/requarkswiki/keys/git.key + sslVerify: true # ------------------------------------------------- # Secret key to use when encrypting sessions diff --git a/controllers/pages.js b/controllers/pages.js index c66d2dba..43bd21bb 100644 --- a/controllers/pages.js +++ b/controllers/pages.js @@ -3,21 +3,31 @@ var express = require('express'); var router = express.Router(); +router.get('/edit/*', (req, res, next) => { + res.send('EDIT MODE'); +}); + +router.get('/new/*', (req, res, next) => { + res.send('CREATE MODE'); +}); + /** * Home */ -router.get('/', (req, res) => { +router.get('/*', (req, res, next) => { - var Promise = require('bluebird'); - var fs = Promise.promisifyAll(require("fs")); + let safePath = entries.parsePath(req.path); - fs.readFileAsync("repo/Storage/Redis.md", "utf8").then(function(contents) { - let pageData = mark.parse(contents); - if(!pageData.meta.title) { - pageData.meta.title = 'Redis.md'; + entries.fetch(safePath).then((pageData) => { + console.log(pageData); + if(pageData) { + res.render('pages/view', { pageData }); + } else { + next(); } - res.render('pages/view', { pageData }); - }); + }).catch((err) => { + next(); + }); }); diff --git a/gulpfile.js b/gulpfile.js index c69dc16b..06e6c3ff 100644 --- a/gulpfile.js +++ b/gulpfile.js @@ -47,8 +47,7 @@ var paths = { ], fonts: [ './node_modules/font-awesome/fonts/*-webfont.*', - '.!/node_modules/font-awesome/fonts/*-webfont.svg', - './node_modules/roboto-fontface/fonts/Roboto/*.woff' + '!./node_modules/font-awesome/fonts/*-webfont.svg' ], deploypackage: [ './**/*', @@ -67,7 +66,7 @@ var paths = { gulp.task('server', ['scripts', 'css', 'fonts'], function() { nodemon({ script: './server', - ignore: ['assets/', 'client/', 'tests/'], + ignore: ['assets/', 'client/', 'data/', 'repo/', 'tests/'], ext: 'js json', env: { 'NODE_ENV': 'development' } }); diff --git a/models/loki.js b/models/db.js similarity index 60% rename from models/loki.js rename to models/db.js index 9aee27b2..7093453a 100644 --- a/models/loki.js +++ b/models/db.js @@ -6,6 +6,8 @@ var loki = require('lokijs'), Promise = require('bluebird'), _ = require('lodash'); +var cols = ['User','Entry']; + /** * Loki.js module * @@ -27,27 +29,17 @@ module.exports = function(appconfig) { autosave: true, autosaveInterval: 5000 }), - Models: {}, onReady: dbReady }; // Load Models - let dbModelsPath = path.join(ROOTPATH, 'models/db'); - dbModel.Store.loadDatabase({}, () => { - fs - .readdirSync(dbModelsPath) - .filter(function(file) { - return (file.indexOf(".") !== 0); - }) - .forEach(function(file) { - let modelName = _.upperFirst(_.split(file,'.')[0]); - dbModel.Models[modelName] = require(path.join(dbModelsPath, file)); - dbModel[modelName] = dbModel.Store.getCollection(modelName); - if(!dbModel[modelName]) { - dbModel[modelName] = dbModel.Store.addCollection(modelName); + _.forEach(cols, (col) => { + dbModel[col] = dbModel.Store.getCollection(col); + if(!dbModel[col]) { + dbModel[col] = dbModel.Store.addCollection(col); } }); diff --git a/models/db/user.js b/models/db/user.js deleted file mode 100644 index a05e795f..00000000 --- a/models/db/user.js +++ /dev/null @@ -1,9 +0,0 @@ -"use strict"; - -var bcrypt = require('bcryptjs-then'); -var Promise = require('bluebird'); -var _ = require('lodash'); - -module.exports = { - -}; \ No newline at end of file diff --git a/models/entries.js b/models/entries.js new file mode 100644 index 00000000..489dfa27 --- /dev/null +++ b/models/entries.js @@ -0,0 +1,110 @@ +"use strict"; + +var Promise = require('bluebird'), + path = require('path'), + fs = Promise.promisifyAll(require("fs")), + _ = require('lodash'), + farmhash = require('farmhash'), + msgpack = require('msgpack5')(); + +/** + * Entries Model + */ +module.exports = { + + _repoPath: 'repo', + _cachePath: 'data/cache', + + /** + * Initialize Entries model + * + * @param {Object} appconfig The application config + * @return {Object} Entries model instance + */ + init(appconfig) { + + let self = this; + + self._repoPath = appconfig.datadir.repo; + self._cachePath = path.join(appconfig.datadir.db, 'cache'); + + return self; + + }, + + fetch(entryPath) { + + let self = this; + + let fpath = path.join(self._repoPath, entryPath + '.md'); + let cpath = path.join(self._cachePath, farmhash.fingerprint32(entryPath) + '.bin'); + + return fs.statAsync(cpath).then((st) => { + return st.isFile(); + }).catch((err) => { + return false; + }).then((isCache) => { + + if(isCache) { + + console.log('from cache!'); + + return fs.readFileAsync(cpath, 'utf8').then((contents) => { + return msgpack.decode(contents); + }).catch((err) => { + winston.error('Corrupted cache file. Deleting it...'); + fs.unlinkSync(cpath); + return false; + }); + + } else { + + console.log('original!'); + + // Parse original and cache it + + return fs.statAsync(fpath).then((st) => { + if(st.isFile()) { + return fs.readFileAsync(fpath, 'utf8').then((contents) => { + let pageData = mark.parse(contents); + if(!pageData.meta.title) { + pageData.meta.title = entryPath; + } + let cacheData = msgpack.encode(pageData); + return fs.writeFileAsync(cpath, cacheData, { encoding: 'utf8' }).then(() => { + return pageData; + }).catch((err) => { + winston.error('Unable to write to cache! Performance may be affected.'); + return pageData; + }); + }); + } else { + return false; + } + }); + + } + + }); + + + + }, + + parsePath(urlPath) { + + let wlist = new RegExp('[^a-z0-9/\-]','g'); + + urlPath = _.toLower(urlPath).replace(wlist, ''); + + if(urlPath === '/') { + urlPath = 'home'; + } + + let urlParts = _.filter(_.split(urlPath, '/'), (p) => { return !_.isEmpty(p); }); + + return _.join(urlParts, '/'); + + } + +}; \ No newline at end of file diff --git a/models/git.js b/models/git.js index b25e6878..f4fdf0ff 100644 --- a/models/git.js +++ b/models/git.js @@ -1,12 +1,13 @@ "use strict"; -var NodeGit = require("nodegit"), +var Git = require("git-wrapper2-promise"), Promise = require('bluebird'), path = require('path'), os = require('os'), fs = Promise.promisifyAll(require("fs")), moment = require('moment'), - _ = require('lodash'); + _ = require('lodash'), + URL = require('url'); /** * Git Model @@ -14,11 +15,11 @@ var NodeGit = require("nodegit"), module.exports = { _git: null, + _url: '', _repo: { path: '', branch: 'master', exists: false, - inst: null, sync: true }, _signature: { @@ -42,16 +43,15 @@ module.exports = { //-> Build repository path - if(_.isEmpty(appconfig.git.path) || appconfig.git.path === 'auto') { + if(_.isEmpty(appconfig.datadir.repo)) { self._repo.path = path.join(ROOTPATH, 'repo'); } else { - self._repo.path = appconfig.git.path; + self._repo.path = appconfig.datadir.repo; } //-> Initialize repository self._initRepo(appconfig).then((repo) => { - self._repo.inst = repo; if(self._repo.sync) { self.resync(); @@ -61,8 +61,8 @@ module.exports = { // Define signature - self._signature.name = appconfig.git.userinfo.name || 'Wiki'; - self._signature.email = appconfig.git.userinfo.email || 'user@example.com'; + self._signature.name = appconfig.git.signature.name || 'Wiki'; + self._signature.email = appconfig.git.signature.email || 'user@example.com'; return self; @@ -78,7 +78,7 @@ module.exports = { let self = this; - winston.info('[GIT] Initializing Git repository...'); + winston.info('[GIT] Checking Git repository...'); //-> Check if path is accessible @@ -88,209 +88,94 @@ module.exports = { } }).then(() => { + self._git = new Git({ 'git-dir': self._repo.path }); + //-> Check if path already contains a git working folder - return fs.statAsync(path.join(self._repo.path, '.git')).then((stat) => { - self._repo.exists = stat.isDirectory(); + return self._git.isRepo().then((isRepo) => { + self._repo.exists = isRepo; + return (!isRepo) ? self._git.exec('init') : true; }).catch((err) => { self._repo.exists = false; }); }).then(() => { - //-> Init repository + // Initialize remote - let repoInitOperation = null; - self._repo.branch = appconfig.git.branch; - self._repo.sync = appconfig.git.remote; - self._opts.clone = self._generateCloneOptions(appconfig); - self._opts.push = self._generatePushOptions(appconfig); + let urlObj = URL.parse(appconfig.git.url); + urlObj.auth = appconfig.git.auth.username + ((appconfig.git.auth.type !== 'ssh') ? ':' + appconfig.git.auth.password : ''); + self._url = URL.format(urlObj); - if(self._repo.exists) { - - winston.info('[GIT] Using existing repository...'); - repoInitOperation = NodeGit.Repository.open(self._repo.path); - - } else if(appconfig.git.remote) { - - winston.info('[GIT] Cloning remote repository for first time...'); - repoInitOperation = NodeGit.Clone(appconfig.git.url, self._repo.path, self._opts.clone); - - } else { - - winston.info('[GIT] Using offline local repository...'); - repoInitOperation = NodeGit.Repository.init(self._repo.path, 0); - - } - - return repoInitOperation; + return self._git.exec('remote', 'show').then((cProc) => { + let out = cProc.stdout.toString(); + if(_.includes(out, 'origin')) { + return true; + } else { + return Promise.join( + self._git.exec('config', ['--local', 'user.name', self._signature.name]), + self._git.exec('config', ['--local', 'user.email', self._signature.email]) + ).then(() => { + return self._git.exec('remote', ['add', 'origin', self._url]); + }) + } + }); }).catch((err) => { - winston.error('Unable to open or clone Git repository!'); - winston.error(err); - }).then((repo) => { - - if(self._repo.sync) { - NodeGit.Remote.setPushurl(repo, 'origin', appconfig.git.url); - } - - return repo; - + winston.error('Git remote error!'); + throw err; + }).then(() => { winston.info('[GIT] Git repository is now ready.'); + return true; }); }, - /** - * Generate Clone Options object - * - * @param {Object} appconfig The application configuration - * @return {Object} CloneOptions object - */ - _generateCloneOptions(appconfig) { - - let cloneOptions = new NodeGit.CloneOptions(); - cloneOptions.fetchOpts = this._generateFetchOptions(appconfig); - return cloneOptions; - - }, - - _generateFetchOptions(appconfig) { - - let fetchOptions = new NodeGit.FetchOptions(); - fetchOptions.callbacks = this._generateRemoteCallbacks(appconfig); - return fetchOptions; - - }, - - _generatePushOptions(appconfig) { - - let pushOptions = new NodeGit.PushOptions(); - pushOptions.callbacks = this._generateRemoteCallbacks(appconfig); - return pushOptions; - - }, - - _generateRemoteCallbacks(appconfig) { - - let remoteCallbacks = new NodeGit.RemoteCallbacks(); - let credFunc = this._generateCredentials(appconfig); - remoteCallbacks.credentials = () => { return credFunc; }; - remoteCallbacks.transferProgress = _.noop; - - if(os.type() === 'Darwin') { - remoteCallbacks.certificateCheck = () => { return 1; }; // Bug in OS X, bypass certs check workaround - } else { - remoteCallbacks.certificateCheck = _.noop; - } - - return remoteCallbacks; - - }, - - _generateCredentials(appconfig) { - - let cred = null; - switch(appconfig.git.auth.type) { - case 'basic': - cred = NodeGit.Cred.userpassPlaintextNew( - appconfig.git.auth.user, - appconfig.git.auth.pass - ); - break; - case 'oauth': - cred = NodeGit.Cred.userpassPlaintextNew( - appconfig.git.auth.token, - "x-oauth-basic" - ); - break; - case 'ssh': - cred = NodeGit.Cred.sshKeyNew( - appconfig.git.auth.user, - appconfig.git.auth.publickey, - appconfig.git.auth.privatekey, - appconfig.git.auth.passphrase - ); - break; - default: - cred = NodeGit.Cred.defaultNew(); - break; - } - - return cred; - - }, - resync() { let self = this; // Fetch - return self._repo.inst.fetch('origin', self._opts.clone.fetchOpts) - .catch((err) => { - winston.error('Unable to fetch from git origin!' + err); - }) - - // Merge - - .then(() => { - return self._repo.inst.mergeBranches(self._repo.branch, 'origin/' + self._repo.branch); + winston.info('[GIT] Performing pull from remote repository...'); + return self._git.pull('origin', self._repo.branch).then((cProc) => { + winston.info('[GIT] Pull completed.'); }) .catch((err) => { - winston.error('Unable to merge from remote head!' + err); + winston.error('Unable to fetch from git origin!'); + throw err; }) - - // Push - .then(() => { - return self._repo.inst.getRemote('origin').then((remote) => { - // Get modified files + // Check for changes - return self._repo.inst.refreshIndex().then((index) => { - return self._repo.inst.getStatus().then(function(arrayStatusFile) { + return self._git.exec('status').then((cProc) => { + let out = cProc.stdout.toString(); + if(!_.includes(out, 'nothing to commit')) { - let addOp = []; - - // Add to next commit - - _.forEach(arrayStatusFile, (v) => { - addOp.push(arrayStatusFile[0].path()); - }); - - console.log('DUDE1'); - - // Create Commit - - let sig = NodeGit.Signature.create(self._signature.name, self._signature.email, moment().utc().unix(), 0); - return self._repo.inst.createCommitOnHead(addOp, sig, sig, "Wiki Sync").then(() => { - - console.log('DUDE2'); - - return remote.connect(NodeGit.Enums.DIRECTION.PUSH, self._opts.push.callbacks).then(() => { - - console.log('DUDE3'); - - // Push to remote - - return remote.push( ["refs/heads/master:refs/heads/master"], self._opts.push).then((errNum) => { - console.log('DUDE' + errNum); - }).catch((err) => { - console.log(err); - }); - - }); - - }); + // Add, commit and push + winston.info('[GIT] Performing push to remote repository...'); + return self._git.add('-A').then(() => { + return self._git.commit("Resync"); + }).then(() => { + return self._git.push('origin', self._repo.branch); + }).then(() => { + return winston.info('[GIT] Push completed.'); }); - }) - /**/ + } else { + winston.info('[GIT] Repository is already up to date. Nothing to commit.'); + } + + return true; + }); - }).catch((err) => { - winston.error('Unable to push to git origin!' + err); + + }) + .catch((err) => { + winston.error('Unable to push changes to remote!'); + throw err; }); } diff --git a/models/localdata.js b/models/localdata.js index b70f3105..f677dbda 100644 --- a/models/localdata.js +++ b/models/localdata.js @@ -1,6 +1,7 @@ "use strict"; var fs = require('fs'), + path = require('path'), _ = require('lodash'); /** @@ -10,7 +11,7 @@ var fs = require('fs'), */ module.exports = (appconfig) => { - // Create DB folder + // Create data directories try { fs.mkdirSync(appconfig.datadir.db); @@ -21,10 +22,8 @@ module.exports = (appconfig) => { } } - // Create Uploads folder - try { - fs.mkdirSync(appconfig.datadir.uploads); + fs.mkdirSync(path.join(appconfig.datadir.db, 'cache')); } catch (err) { if(err.code !== 'EEXIST') { winston.error(err); diff --git a/package.json b/package.json index cd4734d2..ed3cf5d8 100644 --- a/package.json +++ b/package.json @@ -36,6 +36,7 @@ "body-parser": "^1.15.2", "bulma": "^0.1.2", "cheerio": "^0.22.0", + "child-process-promise": "^2.1.3", "compression": "^1.6.2", "connect-flash": "^0.1.1", "connect-loki": "^1.0.6", @@ -46,6 +47,8 @@ "express-brute-loki": "^1.0.0", "express-session": "^1.14.0", "express-validator": "^2.20.8", + "farmhash": "^1.2.0", + "git-wrapper2-promise": "^0.2.9", "highlight.js": "^9.6.0", "i18next": "^3.4.1", "i18next-express-middleware": "^1.0.1", @@ -66,11 +69,10 @@ "markdown-it-toc-and-anchor": "^4.1.1", "moment": "^2.14.1", "moment-timezone": "^0.5.5", - "nodegit": "^0.14.1", + "msgpack5": "^3.4.0", "passport": "^0.3.2", "passport-local": "^1.0.0", "pug": "^2.0.0-beta5", - "roboto-fontface": "^0.6.0", "serve-favicon": "^2.3.0", "simplemde": "^1.11.2", "slug": "^0.9.1", diff --git a/server.js b/server.js index 63d38a94..b18c7df4 100644 --- a/server.js +++ b/server.js @@ -4,14 +4,6 @@ // Licensed under AGPLv3 // =========================================== -process.on('uncaughtException', function (exception) { - console.log(exception); -}); -process.on('unhandledRejection', (reason, p) => { - console.log("Unhandled Rejection at: Promise ", p, " reason: ", reason); - // application specific logging, throwing an error, or other logic here -}); - global.ROOTPATH = __dirname; // ---------------------------------------- @@ -22,10 +14,11 @@ global.winston = require('winston'); winston.info('[SERVER] Requarks Wiki is initializing...'); var appconfig = require('./models/config')('./config.yml'); -var lcdata = require('./models/localdata')(appconfig); +let lcdata = require('./models/localdata'); -global.db = require('./models/loki')(appconfig); +global.db = require('./models/db')(appconfig); global.git = require('./models/git').init(appconfig); +global.entries = require('./models/entries').init(appconfig); global.mark = require('./models/markdown'); // ----------------------------------------