liz/wikijs-fork
liz/wikijs-fork
1
0
Fork 0
Code Issues Pull Requests Packages Projects Releases Wiki Activity

Search-Index integration + cache flush on start

Browse Source
This commit is contained in:
NGPixel 2017-02-12 15:40:43 -05:00
parent 8af9212837
commit 12ea967a84
13 changed files with 259 additions and 24 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

1
.eslintrc.json
View File

@ -32,6 +32,7 @@
"lcdata": true,
"mark": true,
"rights": true,
"search": true,
"upl": true,
"winston": true,
"ws": true,

4
CHANGELOG.md
View File

@ -5,6 +5,10 @@ This project adheres to [Semantic Versioning](http://semver.org/).
## [Unreleased]
### Added
- Offline mode (no remote git sync) can now be enabled by setting `git: false` in config.yml
- Improved search engine (Now using search-index engine instead of MongoDB text search)
### Changed
- Cache is now flushed when starting / restarting the server
## [v1.0-beta.4] - 2017-02-11
### Fixed

8
agent.js
View File

@ -113,7 +113,13 @@ var job = new Cron({
// -> Update cache and search index
if (fileStatus !== 'active') {
return entries.updateCache(entryPath)
return entries.updateCache(entryPath).then(entry => {
process.send({
action: 'searchAdd',
content: entry
})
return true
})
}
return true

4
assets/js/app.js
View File

File diff suppressed because one or more lines are too long

4
client/js/components/search.js
View File

@ -42,7 +42,7 @@ if ($('#search-input').length) {
searchmoveidx: (val, oldVal) => {
if (val > 0) {
vueHeader.searchmovekey = (vueHeader.searchmovearr[val - 1])
? 'res.' + vueHeader.searchmovearr[val - 1]._id
? 'res.' + vueHeader.searchmovearr[val - 1].entryPath
: 'sug.' + vueHeader.searchmovearr[val - 1]
} else {
vueHeader.searchmovekey = ''
@ -61,7 +61,7 @@ if ($('#search-input').length) {
let i = vueHeader.searchmoveidx - 1
if (vueHeader.searchmovearr[i]) {
window.location.assign('/' + vueHeader.searchmovearr[i]._id)
window.location.assign('/' + vueHeader.searchmovearr[i].entryPath)
} else {
vueHeader.searchq = vueHeader.searchmovearr[i]
}

2
controllers/ws.js
View File

@ -13,7 +13,7 @@ module.exports = (socket) => {
socket.on('search', (data, cb) => {
cb = cb || _.noop
entries.search(data.terms).then((results) => {
search.find(data.terms).then((results) => {
return cb(results) || true
})
})

4
libs/entries.js
View File

@ -256,7 +256,9 @@ module.exports = {
return fs.statAsync(fpath).then((st) => {
if (st.isFile()) {
return self.makePersistent(entryPath, contents).then(() => {
return self.updateCache(entryPath)
return self.updateCache(entryPath).then(entry => {
return search.add(entry)
})
})
} else {
return Promise.reject(new Error('Entry does not exist!'))

22
libs/git.js
View File

@ -68,13 +68,13 @@ module.exports = {
_initRepo (appconfig) {
let self = this
winston.info('[' + PROCNAME + '][GIT] Checking Git repository...')
winston.info('[' + PROCNAME + '.Git] Checking Git repository...')
// -> Check if path is accessible
return fs.mkdirAsync(self._repo.path).catch((err) => {
if (err.code !== 'EEXIST') {
winston.error('[' + PROCNAME + '][GIT] Invalid Git repository path or missing permissions.')
winston.error('[' + PROCNAME + '.Git] Invalid Git repository path or missing permissions.')
}
}).then(() => {
self._git = new Git({ 'git-dir': self._repo.path })
@ -89,7 +89,7 @@ module.exports = {
})
}).then(() => {
if (appconfig.git === false) {
winston.info('[' + PROCNAME + '][GIT] Remote syncing is disabled. Not recommended!')
winston.info('[' + PROCNAME + '.Git] Remote syncing is disabled. Not recommended!')
return Promise.resolve(true)
}
@ -126,10 +126,10 @@ module.exports = {
}
})
}).catch((err) => {
winston.error('[' + PROCNAME + '][GIT] Git remote error!')
winston.error('[' + PROCNAME + '.Git] Git remote error!')
throw err
}).then(() => {
winston.info('[' + PROCNAME + '][GIT] Git repository is OK.')
winston.info('[' + PROCNAME + '.Git] Git repository is OK.')
return true
})
},
@ -159,12 +159,12 @@ module.exports = {
// Fetch
winston.info('[' + PROCNAME + '][GIT] Performing pull from remote repository...')
winston.info('[' + PROCNAME + '.Git] Performing pull from remote repository...')
return self._git.pull('origin', self._repo.branch).then((cProc) => {
winston.info('[' + PROCNAME + '][GIT] Pull completed.')
winston.info('[' + PROCNAME + '.Git] Pull completed.')
})
.catch((err) => {
winston.error('[' + PROCNAME + '][GIT] Unable to fetch from git origin!')
winston.error('[' + PROCNAME + '.Git] Unable to fetch from git origin!')
throw err
})
.then(() => {
@ -174,12 +174,12 @@ module.exports = {
let out = cProc.stdout.toString()
if (_.includes(out, 'commit')) {
winston.info('[' + PROCNAME + '][GIT] Performing push to remote repository...')
winston.info('[' + PROCNAME + '.Git] Performing push to remote repository...')
return self._git.push('origin', self._repo.branch).then(() => {
return winston.info('[' + PROCNAME + '][GIT] Push completed.')
return winston.info('[' + PROCNAME + '.Git] Push completed.')
})
} else {
winston.info('[' + PROCNAME + '][GIT] Push skipped. Repository is already in sync.')
winston.info('[' + PROCNAME + '.Git] Push skipped. Repository is already in sync.')
}
return true

5
libs/local.js
View File

@ -98,11 +98,12 @@ module.exports = {
* @return {Void} Void
*/
createBaseDirectories (appconfig) {
winston.info('[SERVER] Checking data directories...')
winston.info('[SERVER.Local] Checking data directories...')
try {
fs.ensureDirSync(path.resolve(ROOTPATH, appconfig.paths.data))
fs.ensureDirSync(path.resolve(ROOTPATH, appconfig.paths.data, './cache'))
fs.emptyDirSync(path.resolve(ROOTPATH, appconfig.paths.data, './cache'))
fs.ensureDirSync(path.resolve(ROOTPATH, appconfig.paths.data, './thumbs'))
fs.ensureDirSync(path.resolve(ROOTPATH, appconfig.paths.data, './temp-upload'))
@ -120,7 +121,7 @@ module.exports = {
winston.error(err)
}
winston.info('[SERVER] Data and Repository directories are OK.')
winston.info('[SERVER.Local] Data and Repository directories are OK.')
return
},

206
libs/search.js Normal file
View File

@ -0,0 +1,206 @@
'use strict'
const Promise = require('bluebird')
const _ = require('lodash')
const path = require('path')
const searchIndex = require('search-index')
const stopWord = require('stopword')
const streamToPromise = require('stream-to-promise')
module.exports = {
_si: null,
_isReady: false,
/**
* Initialize search index
*
* @return {undefined} Void
*/
init () {
let self = this
let dbPath = path.resolve(ROOTPATH, appconfig.paths.data, 'search')
self._isReady = new Promise((resolve, reject) => {
searchIndex({
deletable: true,
fieldedSearch: true,
indexPath: dbPath,
logLevel: 'error',
stopwords: _.get(stopWord, appconfig.lang, [])
}, (err, si) => {
if (err) {
winston.error('[SERVER.Search] Failed to initialize search index.', err)
reject(err)
} else {
self._si = Promise.promisifyAll(si)
self._si.flushAsync().then(() => {
winston.info('[SERVER.Search] Search index flushed and ready.')
resolve(true)
})
}
})
})
return self
},
/**
* Add a document to the index
*
* @param {Object} content Document content
* @return {Promise} Promise of the add operation
*/
add (content) {
let self = this
return self._isReady.then(() => {
return self.delete(content._id).then(() => {
return self._si.concurrentAddAsync({
fieldOptions: [{
fieldName: 'entryPath',
searchable: true,
weight: 2
},
{
fieldName: 'title',
nGramLength: [1, 2],
searchable: true,
weight: 3
},
{
fieldName: 'subtitle',
searchable: true,
weight: 1,
storeable: false
},
{
fieldName: 'parent',
searchable: false
},
{
fieldName: 'content',
searchable: true,
weight: 0,
storeable: false
}]
}, [{
entryPath: content._id,
title: content.title,
subtitle: content.subtitle || '',
parent: content.parent || '',
content: content.content || ''
}]).then(() => {
winston.log('verbose', '[SERVER.Search] Entry ' + content._id + ' added/updated to index.')
return true
}).catch((err) => {
winston.error(err)
})
}).catch((err) => {
winston.error(err)
})
})
},
/**
* Delete an entry from the index
*
* @param {String} The entry path
* @return {Promise} Promise of the operation
*/
delete (entryPath) {
let self = this
return self._isReady.then(() => {
return streamToPromise(self._si.search({
query: [{
AND: { 'entryPath': [entryPath] }
}]
})).then((results) => {
if (results.totalHits > 0) {
let delIds = _.map(results.hits, 'id')
return self._si.delAsync(delIds)
} else {
return true
}
}).catch((err) => {
if (err.type === 'NotFoundError') {
return true
} else {
winston.error(err)
}
})
})
},
/**
* Flush the index
*
* @returns {Promise} Promise of the flush operation
*/
flush () {
let self = this
return self._isReady.then(() => {
return self._si.flushAsync()
})
},
/**
* Search the index
*
* @param {Array<String>} terms
* @returns {Promise<Object>} Hits and suggestions
*/
find (terms) {
let self = this
terms = _.chain(terms)
.deburr()
.toLower()
.trim()
.replace(/[^a-z0-9 ]/g, '')
.value()
let arrTerms = _.chain(terms)
.split(' ')
.filter((f) => { return !_.isEmpty(f) })
.value()
return streamToPromise(self._si.search({
query: [{
AND: { '*': arrTerms }
}],
pageSize: 10
})).then((hits) => {
if (hits.length > 0) {
hits = _.map(_.sortBy(hits, ['score']), h => {
return h.document
})
}
if (hits.length < 5) {
return streamToPromise(self._si.match({
beginsWith: terms,
threshold: 3,
limit: 5,
type: 'simple'
})).then((matches) => {
return {
match: hits,
suggest: matches
}
})
} else {
return {
match: hits,
suggest: []
}
}
}).catch((err) => {
if (err.type === 'NotFoundError') {
return {
match: [],
suggest: []
}
} else {
winston.error(err)
}
})
}
}

4
package.json
View File

@ -91,6 +91,8 @@
"simplemde": "^1.11.2",
"socket.io": "^1.7.2",
"sticky-js": "^1.0.7",
"stopword": "^0.1.1",
"stream-to-promise": "^2.2.0",
"validator": "^6.2.0",
"validator-as-promised": "^1.0.2",
"winston": "^2.3.0"
@ -139,7 +141,6 @@
"app",
"appconfig",
"appdata",
"bgAgent",
"db",
"entries",
"git",
@ -147,6 +148,7 @@
"lang",
"lcdata",
"rights",
"search",
"upl",
"winston",
"ws",

15
server.js
View File

@ -37,6 +37,7 @@ global.entries = require('./libs/entries').init()
global.git = require('./libs/git').init(false)
global.lang = require('i18next')
global.mark = require('./libs/markdown')
global.search = require('./libs/search').init()
global.upl = require('./libs/uploads').init()
// ----------------------------------------
@ -239,7 +240,19 @@ io.on('connection', ctrl.ws)
// Start child processes
// ----------------------------------------
global.bgAgent = fork('agent.js')
let bgAgent = fork('agent.js')
bgAgent.on('message', m => {
if (!m.action) {
return
}
switch (m.action) {
case 'searchAdd':
search.add(m.content)
break
}
})
process.on('exit', (code) => {
bgAgent.disconnect()

4
views/common/header.pug
View File

@ -26,8 +26,8 @@
ul.searchresults-list
li(v-if='searchres.length === 0')
a: em No results matching your query
li(v-for='sres in searchres', v-bind:class='{ "is-active": searchmovekey === "res." + sres._id }')
a(v-bind:href='"/" + sres._id') {{ sres.title }}
li(v-for='sres in searchres', v-bind:class='{ "is-active": searchmovekey === "res." + sres.entryPath }')
a(v-bind:href='"/" + sres.entryPath') {{ sres.title }}
p.searchresults-label(v-if='searchsuggest.length > 0') Did you mean...?
ul.searchresults-list(v-if='searchsuggest.length > 0')
li(v-for='sug in searchsuggest', v-bind:class='{ "is-active": searchmovekey === "sug." + sug }')