feat: cluster implementation
This commit is contained in:
254
server/index.js
254
server/index.js
@@ -2,7 +2,6 @@
|
||||
|
||||
// ===========================================
|
||||
// Wiki.js
|
||||
// 1.0.1
|
||||
// Licensed under AGPLv3
|
||||
// ===========================================
|
||||
|
||||
@@ -28,251 +27,28 @@ wiki.data = appconf.data
|
||||
// Load Winston
|
||||
// ----------------------------------------
|
||||
|
||||
wiki.logger = require('./modules/logger')(wiki.IS_DEBUG, 'SERVER')
|
||||
wiki.logger.info('Wiki.js is initializing...')
|
||||
wiki.logger = require('./modules/logger')()
|
||||
|
||||
// ----------------------------------------
|
||||
// Load global modules
|
||||
// Start Cluster
|
||||
// ----------------------------------------
|
||||
|
||||
wiki.disk = require('./modules/disk').init()
|
||||
wiki.db = require('./modules/db').init()
|
||||
wiki.entries = require('./modules/entries').init()
|
||||
wiki.git = require('./modules/git').init(false)
|
||||
wiki.lang = require('i18next')
|
||||
wiki.mark = require('./modules/markdown')
|
||||
wiki.redis = require('./modules/redis').init()
|
||||
wiki.search = require('./modules/search').init()
|
||||
wiki.upl = require('./modules/uploads').init()
|
||||
const cluster = require('cluster')
|
||||
const numCPUs = require('os').cpus().length
|
||||
|
||||
// ----------------------------------------
|
||||
// Load modules
|
||||
// ----------------------------------------
|
||||
if (cluster.isMaster) {
|
||||
wiki.logger.info('Wiki.js is initializing...')
|
||||
|
||||
const autoload = require('auto-load')
|
||||
const bodyParser = require('body-parser')
|
||||
const compression = require('compression')
|
||||
const cookieParser = require('cookie-parser')
|
||||
const express = require('express')
|
||||
const favicon = require('serve-favicon')
|
||||
const flash = require('connect-flash')
|
||||
const fork = require('child_process').fork
|
||||
const http = require('http')
|
||||
const i18nBackend = require('i18next-node-fs-backend')
|
||||
const passport = require('passport')
|
||||
const passportSocketIo = require('passport.socketio')
|
||||
const session = require('express-session')
|
||||
const SessionRedisStore = require('connect-redis')(session)
|
||||
const graceful = require('node-graceful')
|
||||
const socketio = require('socket.io')
|
||||
const graphqlApollo = require('apollo-server-express')
|
||||
const graphqlSchema = require('./modules/graphql')
|
||||
require('./master')
|
||||
|
||||
var mw = autoload(path.join(wiki.SERVERPATH, '/middlewares'))
|
||||
var ctrl = autoload(path.join(wiki.SERVERPATH, '/controllers'))
|
||||
|
||||
// ----------------------------------------
|
||||
// Define Express App
|
||||
// ----------------------------------------
|
||||
|
||||
const app = express()
|
||||
wiki.app = app
|
||||
app.use(compression())
|
||||
|
||||
// ----------------------------------------
|
||||
// Security
|
||||
// ----------------------------------------
|
||||
|
||||
app.use(mw.security)
|
||||
|
||||
// ----------------------------------------
|
||||
// Public Assets
|
||||
// ----------------------------------------
|
||||
|
||||
app.use(favicon(path.join(wiki.ROOTPATH, 'assets', 'favicon.ico')))
|
||||
app.use(express.static(path.join(wiki.ROOTPATH, 'assets'), {
|
||||
index: false,
|
||||
maxAge: '7d'
|
||||
}))
|
||||
|
||||
// ----------------------------------------
|
||||
// Passport Authentication
|
||||
// ----------------------------------------
|
||||
|
||||
require('./modules/auth')(passport)
|
||||
wiki.rights = require('./modules/rights')
|
||||
wiki.rights.init()
|
||||
|
||||
let sessionStore = new SessionRedisStore({
|
||||
client: wiki.redis
|
||||
})
|
||||
|
||||
app.use(cookieParser())
|
||||
app.use(session({
|
||||
name: 'wikijs.sid',
|
||||
store: sessionStore,
|
||||
secret: wiki.config.sessionSecret,
|
||||
resave: false,
|
||||
saveUninitialized: false
|
||||
}))
|
||||
app.use(flash())
|
||||
app.use(passport.initialize())
|
||||
app.use(passport.session())
|
||||
|
||||
// ----------------------------------------
|
||||
// SEO
|
||||
// ----------------------------------------
|
||||
|
||||
app.use(mw.seo)
|
||||
|
||||
// ----------------------------------------
|
||||
// Localization Engine
|
||||
// ----------------------------------------
|
||||
|
||||
wiki.lang.use(i18nBackend).init({
|
||||
load: 'languageOnly',
|
||||
ns: ['common', 'admin', 'auth', 'errors', 'git'],
|
||||
defaultNS: 'common',
|
||||
saveMissing: false,
|
||||
preload: [wiki.config.lang],
|
||||
lng: wiki.config.lang,
|
||||
fallbackLng: 'en',
|
||||
backend: {
|
||||
loadPath: path.join(wiki.SERVERPATH, 'locales/{{lng}}/{{ns}}.json')
|
||||
for (let i = 0; i < numCPUs; i++) {
|
||||
cluster.fork()
|
||||
}
|
||||
})
|
||||
|
||||
// ----------------------------------------
|
||||
// View Engine Setup
|
||||
// ----------------------------------------
|
||||
|
||||
app.set('views', path.join(wiki.SERVERPATH, 'views'))
|
||||
app.set('view engine', 'pug')
|
||||
|
||||
app.use(bodyParser.json({ limit: '1mb' }))
|
||||
app.use(bodyParser.urlencoded({ extended: false, limit: '1mb' }))
|
||||
|
||||
// ----------------------------------------
|
||||
// View accessible data
|
||||
// ----------------------------------------
|
||||
|
||||
app.locals._ = require('lodash')
|
||||
app.locals.t = wiki.lang.t.bind(wiki.lang)
|
||||
app.locals.moment = require('moment')
|
||||
app.locals.moment.locale(wiki.config.lang)
|
||||
app.locals.appconfig = wiki.config
|
||||
app.use(mw.flash)
|
||||
|
||||
// ----------------------------------------
|
||||
// Controllers
|
||||
// ----------------------------------------
|
||||
|
||||
app.use('/', ctrl.auth)
|
||||
|
||||
app.use('/graphql', graphqlApollo.graphqlExpress({ schema: graphqlSchema }))
|
||||
app.use('/graphiql', graphqlApollo.graphiqlExpress({ endpointURL: '/graphql' }))
|
||||
app.use('/uploads', mw.auth, ctrl.uploads)
|
||||
app.use('/admin', mw.auth, ctrl.admin)
|
||||
app.use('/', mw.auth, ctrl.pages)
|
||||
|
||||
// ----------------------------------------
|
||||
// Error handling
|
||||
// ----------------------------------------
|
||||
|
||||
app.use(function (req, res, next) {
|
||||
var err = new Error('Not Found')
|
||||
err.status = 404
|
||||
next(err)
|
||||
})
|
||||
|
||||
app.use(function (err, req, res, next) {
|
||||
res.status(err.status || 500)
|
||||
res.render('error', {
|
||||
message: err.message,
|
||||
error: wiki.IS_DEBUG ? err : {}
|
||||
cluster.on('exit', (worker, code, signal) => {
|
||||
wiki.logger.info(`Worker #${worker.id} died.`)
|
||||
})
|
||||
})
|
||||
|
||||
// ----------------------------------------
|
||||
// Start HTTP server
|
||||
// ----------------------------------------
|
||||
|
||||
wiki.logger.info('Starting HTTP/WS server on port ' + wiki.config.port + '...')
|
||||
|
||||
app.set('port', wiki.config.port)
|
||||
var server = http.createServer(app)
|
||||
var io = socketio(server)
|
||||
|
||||
server.listen(wiki.config.port)
|
||||
server.on('error', (error) => {
|
||||
if (error.syscall !== 'listen') {
|
||||
throw error
|
||||
}
|
||||
|
||||
// handle specific listen errors with friendly messages
|
||||
switch (error.code) {
|
||||
case 'EACCES':
|
||||
wiki.logger.error('Listening on port ' + wiki.config.port + ' requires elevated privileges!')
|
||||
return process.exit(1)
|
||||
case 'EADDRINUSE':
|
||||
wiki.logger.error('Port ' + wiki.config.port + ' is already in use!')
|
||||
return process.exit(1)
|
||||
default:
|
||||
throw error
|
||||
}
|
||||
})
|
||||
|
||||
server.on('listening', () => {
|
||||
wiki.logger.info('HTTP/WS server started successfully! [RUNNING]')
|
||||
})
|
||||
|
||||
// ----------------------------------------
|
||||
// WebSocket
|
||||
// ----------------------------------------
|
||||
|
||||
io.use(passportSocketIo.authorize({
|
||||
key: 'wikijs.sid',
|
||||
store: sessionStore,
|
||||
secret: wiki.config.sessionSecret,
|
||||
cookieParser,
|
||||
success: (data, accept) => {
|
||||
accept()
|
||||
},
|
||||
fail: (data, message, error, accept) => {
|
||||
accept()
|
||||
}
|
||||
}))
|
||||
|
||||
io.on('connection', ctrl.ws)
|
||||
|
||||
// ----------------------------------------
|
||||
// Start child processes
|
||||
// ----------------------------------------
|
||||
|
||||
let bgAgent = fork(path.join(wiki.SERVERPATH, 'agent.js'))
|
||||
|
||||
bgAgent.on('message', m => {
|
||||
if (!m.action) {
|
||||
return
|
||||
}
|
||||
|
||||
switch (m.action) {
|
||||
case 'searchAdd':
|
||||
wiki.search.add(m.content)
|
||||
break
|
||||
}
|
||||
})
|
||||
|
||||
// ----------------------------------------
|
||||
// Graceful shutdown
|
||||
// ----------------------------------------
|
||||
|
||||
graceful.on('exit', () => {
|
||||
wiki.logger.info('- SHUTTING DOWN - Terminating Background Agent...')
|
||||
bgAgent.kill()
|
||||
wiki.logger.info('- SHUTTING DOWN - Performing git sync...')
|
||||
return global.git.resync().then(() => {
|
||||
wiki.logger.info('- SHUTTING DOWN - Git sync successful. Now safe to exit.')
|
||||
process.exit()
|
||||
})
|
||||
})
|
||||
} else {
|
||||
wiki.logger.info(`Background Worker #${cluster.worker.id} is starting...`)
|
||||
// require('./worker')
|
||||
}
|
||||
|
232
server/master.js
Normal file
232
server/master.js
Normal file
@@ -0,0 +1,232 @@
|
||||
'use strict'
|
||||
|
||||
/* global wiki */
|
||||
|
||||
const path = require('path')
|
||||
|
||||
// ----------------------------------------
|
||||
// Load global modules
|
||||
// ----------------------------------------
|
||||
|
||||
wiki.disk = require('./modules/disk').init()
|
||||
wiki.db = require('./modules/db').init()
|
||||
wiki.entries = require('./modules/entries').init()
|
||||
wiki.git = require('./modules/git').init(false)
|
||||
wiki.lang = require('i18next')
|
||||
wiki.mark = require('./modules/markdown')
|
||||
wiki.redis = require('./modules/redis').init()
|
||||
wiki.search = require('./modules/search').init()
|
||||
wiki.upl = require('./modules/uploads').init()
|
||||
|
||||
// ----------------------------------------
|
||||
// Load modules
|
||||
// ----------------------------------------
|
||||
|
||||
const autoload = require('auto-load')
|
||||
const bodyParser = require('body-parser')
|
||||
const compression = require('compression')
|
||||
const cookieParser = require('cookie-parser')
|
||||
const express = require('express')
|
||||
const favicon = require('serve-favicon')
|
||||
const flash = require('connect-flash')
|
||||
const http = require('http')
|
||||
const i18nBackend = require('i18next-node-fs-backend')
|
||||
const passport = require('passport')
|
||||
const passportSocketIo = require('passport.socketio')
|
||||
const session = require('express-session')
|
||||
const SessionRedisStore = require('connect-redis')(session)
|
||||
const graceful = require('node-graceful')
|
||||
const socketio = require('socket.io')
|
||||
const graphqlApollo = require('apollo-server-express')
|
||||
const graphqlSchema = require('./modules/graphql')
|
||||
|
||||
var mw = autoload(path.join(wiki.SERVERPATH, '/middlewares'))
|
||||
var ctrl = autoload(path.join(wiki.SERVERPATH, '/controllers'))
|
||||
|
||||
// ----------------------------------------
|
||||
// Define Express App
|
||||
// ----------------------------------------
|
||||
|
||||
const app = express()
|
||||
wiki.app = app
|
||||
app.use(compression())
|
||||
|
||||
// ----------------------------------------
|
||||
// Security
|
||||
// ----------------------------------------
|
||||
|
||||
app.use(mw.security)
|
||||
|
||||
// ----------------------------------------
|
||||
// Public Assets
|
||||
// ----------------------------------------
|
||||
|
||||
app.use(favicon(path.join(wiki.ROOTPATH, 'assets', 'favicon.ico')))
|
||||
app.use(express.static(path.join(wiki.ROOTPATH, 'assets'), {
|
||||
index: false,
|
||||
maxAge: '7d'
|
||||
}))
|
||||
|
||||
// ----------------------------------------
|
||||
// Passport Authentication
|
||||
// ----------------------------------------
|
||||
|
||||
require('./modules/auth')(passport)
|
||||
wiki.rights = require('./modules/rights')
|
||||
wiki.rights.init()
|
||||
|
||||
let sessionStore = new SessionRedisStore({
|
||||
client: wiki.redis
|
||||
})
|
||||
|
||||
app.use(cookieParser())
|
||||
app.use(session({
|
||||
name: 'wikijs.sid',
|
||||
store: sessionStore,
|
||||
secret: wiki.config.sessionSecret,
|
||||
resave: false,
|
||||
saveUninitialized: false
|
||||
}))
|
||||
app.use(flash())
|
||||
app.use(passport.initialize())
|
||||
app.use(passport.session())
|
||||
|
||||
// ----------------------------------------
|
||||
// SEO
|
||||
// ----------------------------------------
|
||||
|
||||
app.use(mw.seo)
|
||||
|
||||
// ----------------------------------------
|
||||
// Localization Engine
|
||||
// ----------------------------------------
|
||||
|
||||
wiki.lang.use(i18nBackend).init({
|
||||
load: 'languageOnly',
|
||||
ns: ['common', 'admin', 'auth', 'errors', 'git'],
|
||||
defaultNS: 'common',
|
||||
saveMissing: false,
|
||||
preload: [wiki.config.lang],
|
||||
lng: wiki.config.lang,
|
||||
fallbackLng: 'en',
|
||||
backend: {
|
||||
loadPath: path.join(wiki.SERVERPATH, 'locales/{{lng}}/{{ns}}.json')
|
||||
}
|
||||
})
|
||||
|
||||
// ----------------------------------------
|
||||
// View Engine Setup
|
||||
// ----------------------------------------
|
||||
|
||||
app.set('views', path.join(wiki.SERVERPATH, 'views'))
|
||||
app.set('view engine', 'pug')
|
||||
|
||||
app.use(bodyParser.json({ limit: '1mb' }))
|
||||
app.use(bodyParser.urlencoded({ extended: false, limit: '1mb' }))
|
||||
|
||||
// ----------------------------------------
|
||||
// View accessible data
|
||||
// ----------------------------------------
|
||||
|
||||
app.locals._ = require('lodash')
|
||||
app.locals.t = wiki.lang.t.bind(wiki.lang)
|
||||
app.locals.moment = require('moment')
|
||||
app.locals.moment.locale(wiki.config.lang)
|
||||
app.locals.appconfig = wiki.config
|
||||
app.use(mw.flash)
|
||||
|
||||
// ----------------------------------------
|
||||
// Controllers
|
||||
// ----------------------------------------
|
||||
|
||||
app.use('/', ctrl.auth)
|
||||
|
||||
app.use('/graphql', graphqlApollo.graphqlExpress({ schema: graphqlSchema }))
|
||||
app.use('/graphiql', graphqlApollo.graphiqlExpress({ endpointURL: '/graphql' }))
|
||||
app.use('/uploads', mw.auth, ctrl.uploads)
|
||||
app.use('/admin', mw.auth, ctrl.admin)
|
||||
app.use('/', mw.auth, ctrl.pages)
|
||||
|
||||
// ----------------------------------------
|
||||
// Error handling
|
||||
// ----------------------------------------
|
||||
|
||||
app.use(function (req, res, next) {
|
||||
var err = new Error('Not Found')
|
||||
err.status = 404
|
||||
next(err)
|
||||
})
|
||||
|
||||
app.use(function (err, req, res, next) {
|
||||
res.status(err.status || 500)
|
||||
res.render('error', {
|
||||
message: err.message,
|
||||
error: wiki.IS_DEBUG ? err : {}
|
||||
})
|
||||
})
|
||||
|
||||
// ----------------------------------------
|
||||
// Start HTTP server
|
||||
// ----------------------------------------
|
||||
|
||||
wiki.logger.info('Starting HTTP/WS server on port ' + wiki.config.port + '...')
|
||||
|
||||
app.set('port', wiki.config.port)
|
||||
var server = http.createServer(app)
|
||||
var io = socketio(server)
|
||||
|
||||
server.listen(wiki.config.port)
|
||||
server.on('error', (error) => {
|
||||
if (error.syscall !== 'listen') {
|
||||
throw error
|
||||
}
|
||||
|
||||
// handle specific listen errors with friendly messages
|
||||
switch (error.code) {
|
||||
case 'EACCES':
|
||||
wiki.logger.error('Listening on port ' + wiki.config.port + ' requires elevated privileges!')
|
||||
return process.exit(1)
|
||||
case 'EADDRINUSE':
|
||||
wiki.logger.error('Port ' + wiki.config.port + ' is already in use!')
|
||||
return process.exit(1)
|
||||
default:
|
||||
throw error
|
||||
}
|
||||
})
|
||||
|
||||
server.on('listening', () => {
|
||||
wiki.logger.info('HTTP/WS server started successfully! [RUNNING]')
|
||||
})
|
||||
|
||||
// ----------------------------------------
|
||||
// WebSocket
|
||||
// ----------------------------------------
|
||||
|
||||
io.use(passportSocketIo.authorize({
|
||||
key: 'wikijs.sid',
|
||||
store: sessionStore,
|
||||
secret: wiki.config.sessionSecret,
|
||||
cookieParser,
|
||||
success: (data, accept) => {
|
||||
accept()
|
||||
},
|
||||
fail: (data, message, error, accept) => {
|
||||
accept()
|
||||
}
|
||||
}))
|
||||
|
||||
io.on('connection', ctrl.ws)
|
||||
|
||||
// ----------------------------------------
|
||||
// Graceful shutdown
|
||||
// ----------------------------------------
|
||||
|
||||
graceful.on('exit', () => {
|
||||
// wiki.logger.info('- SHUTTING DOWN - Terminating Background Agent...')
|
||||
// bgAgent.kill()
|
||||
wiki.logger.info('- SHUTTING DOWN - Performing git sync...')
|
||||
return global.git.resync().then(() => {
|
||||
wiki.logger.info('- SHUTTING DOWN - Git sync successful. Now safe to exit.')
|
||||
process.exit()
|
||||
})
|
||||
})
|
@@ -4,7 +4,10 @@
|
||||
* Associate DB Model relations
|
||||
*/
|
||||
module.exports = db => {
|
||||
db.User.belongsToMany(db.Group, { through: 'UserGroups' })
|
||||
db.Group.hasMany(db.Right, { as: 'GroupRights' })
|
||||
db.User.belongsToMany(db.Group, { through: 'userGroups' })
|
||||
db.Group.hasMany(db.Right, { as: 'groupRights' })
|
||||
db.Document.hasMany(db.Tag, { as: 'documentTags' })
|
||||
db.File.belongsTo(db.Folder)
|
||||
db.Comment.belongsTo(db.Document)
|
||||
db.Comment.belongsTo(db.User, { as: 'author' })
|
||||
}
|
||||
|
18
server/models/comment.js
Normal file
18
server/models/comment.js
Normal file
@@ -0,0 +1,18 @@
|
||||
'use strict'
|
||||
|
||||
/**
|
||||
* Comment schema
|
||||
*/
|
||||
module.exports = (sequelize, DataTypes) => {
|
||||
let commentSchema = sequelize.define('comment', {
|
||||
content: {
|
||||
type: DataTypes.STRING,
|
||||
allowNull: false
|
||||
}
|
||||
}, {
|
||||
timestamps: true,
|
||||
version: true
|
||||
})
|
||||
|
||||
return commentSchema
|
||||
}
|
@@ -40,6 +40,11 @@ module.exports = (sequelize, DataTypes) => {
|
||||
type: DataTypes.BOOLEAN,
|
||||
allowNull: false,
|
||||
defaultValue: false
|
||||
},
|
||||
searchContent: {
|
||||
type: DataTypes.TEXT,
|
||||
allowNull: true,
|
||||
defaultValue: ''
|
||||
}
|
||||
}, {
|
||||
timestamps: true,
|
||||
|
24
server/models/tag.js
Normal file
24
server/models/tag.js
Normal file
@@ -0,0 +1,24 @@
|
||||
'use strict'
|
||||
|
||||
/**
|
||||
* Tags schema
|
||||
*/
|
||||
module.exports = (sequelize, DataTypes) => {
|
||||
let tagSchema = sequelize.define('tag', {
|
||||
key: {
|
||||
type: DataTypes.STRING,
|
||||
allowNull: false
|
||||
}
|
||||
}, {
|
||||
timestamps: true,
|
||||
version: true,
|
||||
indexes: [
|
||||
{
|
||||
unique: true,
|
||||
fields: ['key']
|
||||
}
|
||||
]
|
||||
})
|
||||
|
||||
return tagSchema
|
||||
}
|
@@ -64,7 +64,8 @@ module.exports = {
|
||||
// Sync DB
|
||||
|
||||
self.onReady = self.inst.sync({
|
||||
force: false
|
||||
force: false,
|
||||
logging: false
|
||||
})
|
||||
|
||||
return self
|
||||
|
@@ -2,12 +2,10 @@
|
||||
|
||||
/* global wiki */
|
||||
|
||||
module.exports = (processName) => {
|
||||
let winston = require('winston')
|
||||
const cluster = require('cluster')
|
||||
|
||||
if (typeof processName === 'undefined') {
|
||||
processName = 'SERVER'
|
||||
}
|
||||
module.exports = () => {
|
||||
let winston = require('winston')
|
||||
|
||||
// Console
|
||||
|
||||
@@ -25,6 +23,7 @@ module.exports = (processName) => {
|
||||
})
|
||||
|
||||
logger.filters.push((level, msg) => {
|
||||
let processName = (cluster.isMaster) ? 'MASTER' : `WORKER-${cluster.worker.id}`
|
||||
return '[' + processName + '] ' + msg
|
||||
})
|
||||
|
||||
|
@@ -1,81 +0,0 @@
|
||||
const bunyan = require('bunyan')
|
||||
const level = require('levelup')
|
||||
const down = require('memdown')
|
||||
const SearchIndexAdder = require('search-index-adder')
|
||||
const SearchIndexSearcher = require('search-index-searcher')
|
||||
|
||||
module.exports = function (givenOptions, moduleReady) {
|
||||
const optionsLoaded = function (err, SearchIndex) {
|
||||
const siUtil = require('./siUtil.js')(SearchIndex.options)
|
||||
if (err) return moduleReady(err)
|
||||
SearchIndex.close = siUtil.close
|
||||
SearchIndex.countDocs = siUtil.countDocs
|
||||
getAdder(SearchIndex, adderLoaded)
|
||||
}
|
||||
|
||||
const adderLoaded = function (err, SearchIndex) {
|
||||
if (err) return moduleReady(err)
|
||||
getSearcher(SearchIndex, searcherLoaded)
|
||||
}
|
||||
|
||||
const searcherLoaded = function (err, SearchIndex) {
|
||||
if (err) return moduleReady(err)
|
||||
return moduleReady(err, SearchIndex)
|
||||
}
|
||||
|
||||
getOptions(givenOptions, optionsLoaded)
|
||||
}
|
||||
|
||||
const getAdder = function (SearchIndex, done) {
|
||||
SearchIndexAdder(SearchIndex.options, function (err, searchIndexAdder) {
|
||||
SearchIndex.add = searchIndexAdder.add
|
||||
SearchIndex.callbackyAdd = searchIndexAdder.concurrentAdd // deprecated
|
||||
SearchIndex.concurrentAdd = searchIndexAdder.concurrentAdd
|
||||
SearchIndex.createWriteStream = searchIndexAdder.createWriteStream
|
||||
SearchIndex.dbWriteStream = searchIndexAdder.dbWriteStream
|
||||
SearchIndex.defaultPipeline = searchIndexAdder.defaultPipeline
|
||||
SearchIndex.del = searchIndexAdder.deleter
|
||||
SearchIndex.deleteStream = searchIndexAdder.deleteStream
|
||||
SearchIndex.flush = searchIndexAdder.flush
|
||||
done(err, SearchIndex)
|
||||
})
|
||||
}
|
||||
|
||||
const getSearcher = function (SearchIndex, done) {
|
||||
SearchIndexSearcher(SearchIndex.options, function (err, searchIndexSearcher) {
|
||||
SearchIndex.availableFields = searchIndexSearcher.availableFields
|
||||
SearchIndex.buckets = searchIndexSearcher.bucketStream
|
||||
SearchIndex.categorize = searchIndexSearcher.categoryStream
|
||||
SearchIndex.dbReadStream = searchIndexSearcher.dbReadStream
|
||||
SearchIndex.get = searchIndexSearcher.get
|
||||
SearchIndex.match = searchIndexSearcher.match
|
||||
SearchIndex.scan = searchIndexSearcher.scan
|
||||
SearchIndex.search = searchIndexSearcher.search
|
||||
SearchIndex.totalHits = searchIndexSearcher.totalHits
|
||||
done(err, SearchIndex)
|
||||
})
|
||||
}
|
||||
|
||||
const getOptions = function (options, done) {
|
||||
var SearchIndex = {}
|
||||
SearchIndex.options = Object.assign({}, {
|
||||
indexPath: 'si',
|
||||
keySeparator: '○',
|
||||
logLevel: 'error'
|
||||
}, options)
|
||||
options.log = bunyan.createLogger({
|
||||
name: 'search-index',
|
||||
level: options.logLevel
|
||||
})
|
||||
if (!options.indexes) {
|
||||
level(SearchIndex.options.indexPath || 'si', {
|
||||
valueEncoding: 'json',
|
||||
db: down
|
||||
}, function (err, db) {
|
||||
SearchIndex.options.indexes = db
|
||||
return done(err, SearchIndex)
|
||||
})
|
||||
} else {
|
||||
return done(null, SearchIndex)
|
||||
}
|
||||
}
|
@@ -1,36 +0,0 @@
|
||||
'use strict'
|
||||
|
||||
module.exports = function (siOptions) {
|
||||
var siUtil = {}
|
||||
|
||||
siUtil.countDocs = function (callback) {
|
||||
var count = 0
|
||||
const gte = 'DOCUMENT' + siOptions.keySeparator
|
||||
const lte = 'DOCUMENT' + siOptions.keySeparator + siOptions.keySeparator
|
||||
siOptions.indexes.createReadStream({gte: gte, lte: lte})
|
||||
.on('data', function (data) {
|
||||
count++
|
||||
})
|
||||
.on('error', function (err) {
|
||||
return callback(err, null)
|
||||
})
|
||||
.on('end', function () {
|
||||
return callback(null, count)
|
||||
})
|
||||
}
|
||||
|
||||
siUtil.close = function (callback) {
|
||||
siOptions.indexes.close(function (err) {
|
||||
while (!siOptions.indexes.isClosed()) {
|
||||
// log not always working here- investigate
|
||||
if (siOptions.log) siOptions.log.info('closing...')
|
||||
}
|
||||
if (siOptions.indexes.isClosed()) {
|
||||
if (siOptions.log) siOptions.log.info('closed...')
|
||||
callback(err)
|
||||
}
|
||||
})
|
||||
}
|
||||
|
||||
return siUtil
|
||||
}
|
@@ -4,7 +4,7 @@
|
||||
|
||||
const Promise = require('bluebird')
|
||||
const _ = require('lodash')
|
||||
const searchIndex = require('./search-index')
|
||||
// const searchIndex = require('./search-index')
|
||||
const stopWord = require('stopword')
|
||||
const streamToPromise = require('stream-to-promise')
|
||||
const searchAllowedChars = new RegExp('[^a-z0-9' + wiki.data.regex.cjk + wiki.data.regex.arabic + ' ]', 'g')
|
||||
@@ -22,7 +22,7 @@ module.exports = {
|
||||
init () {
|
||||
let self = this
|
||||
self._isReady = new Promise((resolve, reject) => {
|
||||
searchIndex({
|
||||
/*searchIndex({
|
||||
deletable: true,
|
||||
fieldedSearch: true,
|
||||
indexPath: 'wiki',
|
||||
@@ -39,7 +39,7 @@ module.exports = {
|
||||
resolve(true)
|
||||
})
|
||||
}
|
||||
})
|
||||
}) */
|
||||
})
|
||||
|
||||
return self
|
||||
|
@@ -1,39 +1,19 @@
|
||||
// ===========================================
|
||||
// Wiki.js - Background Agent
|
||||
// 1.0.1
|
||||
// Licensed under AGPLv3
|
||||
// ===========================================
|
||||
'use strict'
|
||||
|
||||
/* global wiki */
|
||||
|
||||
const path = require('path')
|
||||
const ROOTPATH = process.cwd()
|
||||
const SERVERPATH = path.join(ROOTPATH, 'server')
|
||||
|
||||
global.ROOTPATH = ROOTPATH
|
||||
global.SERVERPATH = SERVERPATH
|
||||
const IS_DEBUG = process.env.NODE_ENV === 'development'
|
||||
|
||||
let appconf = require('./modules/config')()
|
||||
global.appconfig = appconf.config
|
||||
global.appdata = appconf.data
|
||||
|
||||
// ----------------------------------------
|
||||
// Load Winston
|
||||
// ----------------------------------------
|
||||
|
||||
global.winston = require('./modules/logger')(IS_DEBUG, 'AGENT')
|
||||
|
||||
// ----------------------------------------
|
||||
// Load global modules
|
||||
// ----------------------------------------
|
||||
|
||||
global.winston.info('Background Agent is initializing...')
|
||||
|
||||
global.db = require('./modules/db').init()
|
||||
global.upl = require('./modules/uploads-agent').init()
|
||||
global.git = require('./modules/git').init()
|
||||
global.entries = require('./modules/entries').init()
|
||||
global.lang = require('i18next')
|
||||
global.mark = require('./modules/markdown')
|
||||
wiki.db = require('./modules/db').init()
|
||||
wiki.upl = require('./modules/uploads-agent').init()
|
||||
wiki.git = require('./modules/git').init()
|
||||
wiki.entries = require('./modules/entries').init()
|
||||
wiki.lang = require('i18next')
|
||||
wiki.mark = require('./modules/markdown')
|
||||
|
||||
// ----------------------------------------
|
||||
// Load modules
|
||||
@@ -52,20 +32,18 @@ const entryHelper = require('./helpers/entry')
|
||||
// Localization Engine
|
||||
// ----------------------------------------
|
||||
|
||||
global.lang
|
||||
.use(i18nBackend)
|
||||
.init({
|
||||
load: 'languageOnly',
|
||||
ns: ['common', 'admin', 'auth', 'errors', 'git'],
|
||||
defaultNS: 'common',
|
||||
saveMissing: false,
|
||||
preload: [appconfig.lang],
|
||||
lng: appconfig.lang,
|
||||
fallbackLng: 'en',
|
||||
backend: {
|
||||
loadPath: path.join(SERVERPATH, 'locales/{{lng}}/{{ns}}.json')
|
||||
}
|
||||
})
|
||||
wiki.lang.use(i18nBackend).init({
|
||||
load: 'languageOnly',
|
||||
ns: ['common', 'admin', 'auth', 'errors', 'git'],
|
||||
defaultNS: 'common',
|
||||
saveMissing: false,
|
||||
preload: [wiki.config.lang],
|
||||
lng: wiki.config.lang,
|
||||
fallbackLng: 'en',
|
||||
backend: {
|
||||
loadPath: path.join(wiki.SERVERPATH, 'locales/{{lng}}/{{ns}}.json')
|
||||
}
|
||||
})
|
||||
|
||||
// ----------------------------------------
|
||||
// Start Cron
|
||||
@@ -75,8 +53,8 @@ let job
|
||||
let jobIsBusy = false
|
||||
let jobUplWatchStarted = false
|
||||
|
||||
global.db.onReady.then(() => {
|
||||
return global.db.Entry.remove({})
|
||||
wiki.db.onReady.then(() => {
|
||||
return wiki.db.Entry.remove({})
|
||||
}).then(() => {
|
||||
job = new Cron({
|
||||
cronTime: '0 */5 * * * *',
|
||||
@@ -84,17 +62,17 @@ global.db.onReady.then(() => {
|
||||
// Make sure we don't start two concurrent jobs
|
||||
|
||||
if (jobIsBusy) {
|
||||
global.winston.warn('Previous job has not completed gracefully or is still running! Skipping for now. (This is not normal, you should investigate)')
|
||||
wiki.logger.warn('Previous job has not completed gracefully or is still running! Skipping for now. (This is not normal, you should investigate)')
|
||||
return
|
||||
}
|
||||
global.winston.info('Running all jobs...')
|
||||
wiki.logger.info('Running all jobs...')
|
||||
jobIsBusy = true
|
||||
|
||||
// Prepare async job collector
|
||||
|
||||
let jobs = []
|
||||
let repoPath = path.resolve(ROOTPATH, appconfig.paths.repo)
|
||||
let dataPath = path.resolve(ROOTPATH, appconfig.paths.data)
|
||||
let repoPath = path.resolve(wiki.ROOTPATH, wiki.config.paths.repo)
|
||||
let dataPath = path.resolve(wiki.ROOTPATH, wiki.config.paths.data)
|
||||
let uploadsTempPath = path.join(dataPath, 'temp-upload')
|
||||
|
||||
// ----------------------------------------
|
||||
@@ -105,7 +83,7 @@ global.db.onReady.then(() => {
|
||||
// -> Sync with Git remote
|
||||
//* ****************************************
|
||||
|
||||
jobs.push(global.git.resync().then(() => {
|
||||
jobs.push(wiki.git.resync().then(() => {
|
||||
// -> Stream all documents
|
||||
|
||||
let cacheJobs = []
|
||||
@@ -185,18 +163,18 @@ global.db.onReady.then(() => {
|
||||
// ----------------------------------------
|
||||
|
||||
Promise.all(jobs).then(() => {
|
||||
global.winston.info('All jobs completed successfully! Going to sleep for now.')
|
||||
wiki.logger.info('All jobs completed successfully! Going to sleep for now.')
|
||||
|
||||
if (!jobUplWatchStarted) {
|
||||
jobUplWatchStarted = true
|
||||
global.upl.initialScan().then(() => {
|
||||
wiki.upl.initialScan().then(() => {
|
||||
job.start()
|
||||
})
|
||||
}
|
||||
|
||||
return true
|
||||
}).catch((err) => {
|
||||
global.winston.error('One or more jobs have failed: ', err)
|
||||
wiki.logger.error('One or more jobs have failed: ', err)
|
||||
}).finally(() => {
|
||||
jobIsBusy = false
|
||||
})
|
||||
@@ -212,7 +190,7 @@ global.db.onReady.then(() => {
|
||||
// ----------------------------------------
|
||||
|
||||
process.on('disconnect', () => {
|
||||
global.winston.warn('Lost connection to main server. Exiting...')
|
||||
wiki.logger.warn('Lost connection to main server. Exiting...')
|
||||
job.stop()
|
||||
process.exit()
|
||||
})
|
Reference in New Issue
Block a user