Early work on background agent for search

models/entries.js

@@ -27,8 +27,8 @@ module.exports = {
 
 		let self = this;
 
-		self._repoPath = appconfig.datadir.repo;
-		self._cachePath = path.join(appconfig.datadir.db, 'cache');
+		self._repoPath = path.resolve(ROOTPATH, appconfig.datadir.repo);
+		self._cachePath = path.resolve(ROOTPATH, appconfig.datadir.db, 'cache');
 
 		return self;
 
@@ -177,6 +177,32 @@ module.exports = {
 
 	},
 
+	/**
+	 * Fetches a text version of a Markdown-formatted document
+	 *
+	 * @param      {String}  entryPath  The entry path
+	 * @return     {String}  Text-only version
+	 */
+	fetchTextVersion(entryPath) {
+
+		let self = this;
+
+		return self.fetchOriginal(entryPath, {
+			parseMarkdown: false,
+			parseMeta: true,
+			parseTree: false,
+			includeMarkdown: true,
+			includeParentInfo: false,
+			cache: false
+		}).then((pageData) => {
+			return {
+				meta: pageData.meta,
+				text: mark.removeMarkdown(pageData.markdown)
+			};
+		});
+
+	},
+
 	/**
 	 * Parse raw url path and make it safe
 	 *
@@ -341,6 +367,8 @@ module.exports = {
 
 	},
 
+
+
 	/**
 	 * Generate a starter page content based on the entry path
 	 *
@@ -356,35 +384,6 @@ module.exports = {
 			return _.replace(contents, new RegExp('{TITLE}', 'g'), formattedTitle);
 		});
 
-	},
-
-	purgeStaleCache() {
-
-		let self = this;
-
-		let cacheJobs = [];
-
-		fs.walk(self._repoPath)
-		.on('data', function (item) {
-			if(path.extname(item.path) === '.md') {
-
-				let entryPath = self.parsePath(self.getEntryPathFromFullPath(item.path));
-				let cachePath = self.getCachePath(entryPath);
-
-				cacheJobs.push(fs.statAsync(cachePath).then((st) => {
-					if(moment(st.mtime).isBefore(item.stats.mtime)) {
-						return fs.unlinkAsync(cachePath);
-					} else {
-						return true;
-					}
-				}).catch((err) => {
-					return (err.code !== 'EEXIST') ? err : true;
-				}));
-			}
-		});
-
-		return Promise.all(cacheJobs);
-
 	}
 
 };

models/git.js

@@ -19,8 +19,7 @@ module.exports = {
 	_repo: {
 		path: '',
 		branch: 'master',
-		exists: false,
-		sync: true
+		exists: false
 	},
 	_signature: {
 		name: 'Wiki',
@@ -30,6 +29,7 @@ module.exports = {
 		clone: {},
 		push: {}
 	},
+	onReady: null,
 
 	/**
 	 * Initialize Git model
@@ -37,12 +37,10 @@ module.exports = {
 	 * @param      {Object}  appconfig  The application config
 	 * @return     {Object}  Git model instance
 	 */
-	init(appconfig, sync) {
+	init(appconfig) {
 
 		let self = this;
 
-		self._repo.sync = sync;
-
 		//-> Build repository path
 		
 		if(_.isEmpty(appconfig.datadir.repo)) {
@@ -53,13 +51,7 @@ module.exports = {
 
 		//-> Initialize repository
 
-		self._initRepo(appconfig).then((repo) => {
-
-			if(self._repo.sync) {
-				self.resync();
-			}
-
-		});
+		self.onReady = self._initRepo(appconfig);
 
 		// Define signature
 

models/markdown.js

@@ -12,7 +12,8 @@ var Promise = require('bluebird'),
 	mdAttrs = require('markdown-it-attrs'),
 	hljs = require('highlight.js'),
 	cheerio = require('cheerio'),
-	_ = require('lodash');
+	_ = require('lodash'),
+	mdRemove = require('remove-markdown');
 
 // Load plugins
 
@@ -157,6 +158,12 @@ const parseContent = (content)  => {
 
 };
 
+/**
+ * Parse meta-data tags from content
+ *
+ * @param      {String}  content  Markdown content
+ * @return     {Object}  Properties found in the content and their values
+ */
 const parseMeta = (content) => {
 
 	let commentMeta = new RegExp('<!-- ?([a-zA-Z]+):(.*)-->','g');
@@ -171,6 +178,12 @@ const parseMeta = (content) => {
 
 module.exports = {
 
+	/**
+	 * Parse content and return all data
+	 *
+	 * @param      {String}  content  Markdown-formatted content
+	 * @return     {Object}  Object containing meta, html and tree data
+	 */
 	parse(content) {
 		return {
 			meta: parseMeta(content),
@@ -181,6 +194,29 @@ module.exports = {
 
 	parseContent,
 	parseMeta,
-	parseTree
+	parseTree,
+
+	/**
+	 * Strips non-text elements from Markdown content
+	 *
+	 * @param      {String}  content  Markdown-formatted content
+	 * @return     {String}  Text-only version
+	 */
+	removeMarkdown(content) {
+		return mdRemove(_.chain(content)
+			.replace(/<!-- ?([a-zA-Z]+):(.*)-->/g, '')
+			.replace(/```[^`]+```/g, '')
+			.replace(/`[^`]+`/g, '')
+			.replace(new RegExp('(?!mailto:)(?:(?:http|https|ftp)://)(?:\\S+(?::\\S*)?@)?(?:(?:(?:[1-9]\\d?|1\\d\\d|2[01]\\d|22[0-3])(?:\\.(?:1?\\d{1,2}|2[0-4]\\d|25[0-5])){2}(?:\\.(?:[0-9]\\d?|1\\d\\d|2[0-4]\\d|25[0-4]))|(?:(?:[a-z\\u00a1-\\uffff0-9]+-?)*[a-z\\u00a1-\\uffff0-9]+)(?:\\.(?:[a-z\\u00a1-\\uffff0-9]+-?)*[a-z\\u00a1-\\uffff0-9]+)*(?:\\.(?:[a-z\\u00a1-\\uffff]{2,})))|localhost)(?::\\d{2,5})?(?:(/|\\?|#)[^\\s]*)?', 'g'), '')
+			.replace(/\r?\n|\r/g, ' ')
+			.deburr()
+			.toLower()
+			.replace(/(\b([^a-z]+)\b)/g, ' ')
+			.replace(/[^a-z]+/g, ' ')
+			.replace(/(\b(\w{1,2})\b(\W|$))/g, '')
+			.replace(/\s\s+/g, ' ')
+			.value()
+		);
+	}
 
 };

models/search.js

@@ -0,0 +1,42 @@
+"use strict";
+
+var Promise = require('bluebird'),
+	_ = require('lodash'),
+	path = require('path'),
+	searchIndex = Promise.promisifyAll(require('search-index')),
+	stopWord = require('stopword');
+
+/**
+ * Search Model
+ */
+module.exports = {
+
+	_si: null,
+
+	/**
+	 * Initialize Search model
+	 *
+	 * @param      {Object}  appconfig  The application config
+	 * @return     {Object}  Search model instance
+	 */
+	init(appconfig) {
+
+		let dbPath = path.resolve(ROOTPATH, appconfig.datadir.db, 'search-index');
+
+		this._si = searchIndex({
+			deletable: true,
+			fieldedSearch: true,
+			indexPath: dbPath,
+			logLevel: 'error',
+			stopwords: stopWord.getStopwords(appconfig.lang).sort()
+		}, (err, si) => {
+			if(err) {
+				winston.error('Failed to initialize search-index.', err);
+			}
+		});
+
+	}
+
+
+
+};