liz/wikijs-fork
1
0
Fork 0
Code Issues Pull Requests Packages Projects Releases Wiki Activity

Search-Index integration + cache flush on start

Browse Source
This commit is contained in:
NGPixel
2017-02-12 15:40:43 -05:00
parent 8af9212837
commit 12ea967a84
13 changed files with 259 additions and 24 deletions
Download Patch File Download Diff File Expand all files Collapse all files

4
libs/entries.js
View File

@@ -256,7 +256,9 @@ module.exports = {
return fs.statAsync(fpath).then((st) => {
if (st.isFile()) {
return self.makePersistent(entryPath, contents).then(() => {
return self.updateCache(entryPath)
return self.updateCache(entryPath).then(entry => {
return search.add(entry)
})
})
} else {
return Promise.reject(new Error('Entry does not exist!'))

22
libs/git.js
View File

@@ -68,13 +68,13 @@ module.exports = {
_initRepo (appconfig) {
let self = this
winston.info('[' + PROCNAME + '][GIT] Checking Git repository...')
winston.info('[' + PROCNAME + '.Git] Checking Git repository...')
// -> Check if path is accessible
return fs.mkdirAsync(self._repo.path).catch((err) => {
if (err.code !== 'EEXIST') {
winston.error('[' + PROCNAME + '][GIT] Invalid Git repository path or missing permissions.')
winston.error('[' + PROCNAME + '.Git] Invalid Git repository path or missing permissions.')
}
}).then(() => {
self._git = new Git({ 'git-dir': self._repo.path })
@@ -89,7 +89,7 @@ module.exports = {
})
}).then(() => {
if (appconfig.git === false) {
winston.info('[' + PROCNAME + '][GIT] Remote syncing is disabled. Not recommended!')
winston.info('[' + PROCNAME + '.Git] Remote syncing is disabled. Not recommended!')
return Promise.resolve(true)
}
@@ -126,10 +126,10 @@ module.exports = {
}
})
}).catch((err) => {
winston.error('[' + PROCNAME + '][GIT] Git remote error!')
winston.error('[' + PROCNAME + '.Git] Git remote error!')
throw err
}).then(() => {
winston.info('[' + PROCNAME + '][GIT] Git repository is OK.')
winston.info('[' + PROCNAME + '.Git] Git repository is OK.')
return true
})
},
@@ -159,12 +159,12 @@ module.exports = {
// Fetch
winston.info('[' + PROCNAME + '][GIT] Performing pull from remote repository...')
winston.info('[' + PROCNAME + '.Git] Performing pull from remote repository...')
return self._git.pull('origin', self._repo.branch).then((cProc) => {
winston.info('[' + PROCNAME + '][GIT] Pull completed.')
winston.info('[' + PROCNAME + '.Git] Pull completed.')
})
.catch((err) => {
winston.error('[' + PROCNAME + '][GIT] Unable to fetch from git origin!')
winston.error('[' + PROCNAME + '.Git] Unable to fetch from git origin!')
throw err
})
.then(() => {
@@ -174,12 +174,12 @@ module.exports = {
let out = cProc.stdout.toString()
if (_.includes(out, 'commit')) {
winston.info('[' + PROCNAME + '][GIT] Performing push to remote repository...')
winston.info('[' + PROCNAME + '.Git] Performing push to remote repository...')
return self._git.push('origin', self._repo.branch).then(() => {
return winston.info('[' + PROCNAME + '][GIT] Push completed.')
return winston.info('[' + PROCNAME + '.Git] Push completed.')
})
} else {
winston.info('[' + PROCNAME + '][GIT] Push skipped. Repository is already in sync.')
winston.info('[' + PROCNAME + '.Git] Push skipped. Repository is already in sync.')
}
return true

5
libs/local.js
View File

@@ -98,11 +98,12 @@ module.exports = {
* @return {Void} Void
*/
createBaseDirectories (appconfig) {
winston.info('[SERVER] Checking data directories...')
winston.info('[SERVER.Local] Checking data directories...')
try {
fs.ensureDirSync(path.resolve(ROOTPATH, appconfig.paths.data))
fs.ensureDirSync(path.resolve(ROOTPATH, appconfig.paths.data, './cache'))
fs.emptyDirSync(path.resolve(ROOTPATH, appconfig.paths.data, './cache'))
fs.ensureDirSync(path.resolve(ROOTPATH, appconfig.paths.data, './thumbs'))
fs.ensureDirSync(path.resolve(ROOTPATH, appconfig.paths.data, './temp-upload'))
@@ -120,7 +121,7 @@ module.exports = {
winston.error(err)
}
winston.info('[SERVER] Data and Repository directories are OK.')
winston.info('[SERVER.Local] Data and Repository directories are OK.')
return
},

206
libs/search.js Normal file
View File

@@ -0,0 +1,206 @@
'use strict'
const Promise = require('bluebird')
const _ = require('lodash')
const path = require('path')
const searchIndex = require('search-index')
const stopWord = require('stopword')
const streamToPromise = require('stream-to-promise')
module.exports = {
_si: null,
_isReady: false,
/**
* Initialize search index
*
* @return {undefined} Void
*/
init () {
let self = this
let dbPath = path.resolve(ROOTPATH, appconfig.paths.data, 'search')
self._isReady = new Promise((resolve, reject) => {
searchIndex({
deletable: true,
fieldedSearch: true,
indexPath: dbPath,
logLevel: 'error',
stopwords: _.get(stopWord, appconfig.lang, [])
}, (err, si) => {
if (err) {
winston.error('[SERVER.Search] Failed to initialize search index.', err)
reject(err)
} else {
self._si = Promise.promisifyAll(si)
self._si.flushAsync().then(() => {
winston.info('[SERVER.Search] Search index flushed and ready.')
resolve(true)
})
}
})
})
return self
},
/**
* Add a document to the index
*
* @param {Object} content Document content
* @return {Promise} Promise of the add operation
*/
add (content) {
let self = this
return self._isReady.then(() => {
return self.delete(content._id).then(() => {
return self._si.concurrentAddAsync({
fieldOptions: [{
fieldName: 'entryPath',
searchable: true,
weight: 2
},
{
fieldName: 'title',
nGramLength: [1, 2],
searchable: true,
weight: 3
},
{
fieldName: 'subtitle',
searchable: true,
weight: 1,
storeable: false
},
{
fieldName: 'parent',
searchable: false
},
{
fieldName: 'content',
searchable: true,
weight: 0,
storeable: false
}]
}, [{
entryPath: content._id,
title: content.title,
subtitle: content.subtitle || '',
parent: content.parent || '',
content: content.content || ''
}]).then(() => {
winston.log('verbose', '[SERVER.Search] Entry ' + content._id + ' added/updated to index.')
return true
}).catch((err) => {
winston.error(err)
})
}).catch((err) => {
winston.error(err)
})
})
},
/**
* Delete an entry from the index
*
* @param {String} The entry path
* @return {Promise} Promise of the operation
*/
delete (entryPath) {
let self = this
return self._isReady.then(() => {
return streamToPromise(self._si.search({
query: [{
AND: { 'entryPath': [entryPath] }
}]
})).then((results) => {
if (results.totalHits > 0) {
let delIds = _.map(results.hits, 'id')
return self._si.delAsync(delIds)
} else {
return true
}
}).catch((err) => {
if (err.type === 'NotFoundError') {
return true
} else {
winston.error(err)
}
})
})
},
/**
* Flush the index
*
* @returns {Promise} Promise of the flush operation
*/
flush () {
let self = this
return self._isReady.then(() => {
return self._si.flushAsync()
})
},
/**
* Search the index
*
* @param {Array<String>} terms
* @returns {Promise<Object>} Hits and suggestions
*/
find (terms) {
let self = this
terms = _.chain(terms)
.deburr()
.toLower()
.trim()
.replace(/[^a-z0-9 ]/g, '')
.value()
let arrTerms = _.chain(terms)
.split(' ')
.filter((f) => { return !_.isEmpty(f) })
.value()
return streamToPromise(self._si.search({
query: [{
AND: { '*': arrTerms }
}],
pageSize: 10
})).then((hits) => {
if (hits.length > 0) {
hits = _.map(_.sortBy(hits, ['score']), h => {
return h.document
})
}
if (hits.length < 5) {
return streamToPromise(self._si.match({
beginsWith: terms,
threshold: 3,
limit: 5,
type: 'simple'
})).then((matches) => {
return {
match: hits,
suggest: matches
}
})
} else {
return {
match: hits,
suggest: []
}
}
}).catch((err) => {
if (err.type === 'NotFoundError') {
return {
match: [],
suggest: []
}
} else {
winston.error(err)
}
})
}
}