liz/wikijs-fork
liz/wikijs-fork
1
0
Fork 0
Code Issues Pull Requests Packages Projects Releases Wiki Activity

feat: cluster implementation

Browse Source
This commit is contained in:
NGPixel 2017-07-29 00:11:22 -04:00
parent 60750eeed8
commit 9c112ab535
15 changed files with 414 additions and 564 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

153
config.sample.yml
View File

@ -4,23 +4,9 @@
# Full explanation + examples in the documentation:
# https://docs.requarks.io/wiki/install
# ---------------------------------------------------------------------
# Title of this site
# ---------------------------------------------------------------------
title: Wiki
# ---------------------------------------------------------------------
# Full public path to the site, without the trailing slash
# ---------------------------------------------------------------------
# INCLUDE CLIENT PORT IF NOT 80/443!
host: http://localhost
# ---------------------------------------------------------------------
# Port the main server should listen to (80 by default)
# ---------------------------------------------------------------------
# To use process.env.PORT, comment the line below:
port: 80
@ -33,136 +19,23 @@ paths:
data: ./data
# ---------------------------------------------------------------------
# Upload Limits
# ---------------------------------------------------------------------
# In megabytes (MB)
uploads:
maxImageFileSize: 3
maxOtherFileSize: 100
# ---------------------------------------------------------------------
# Site Language
# ---------------------------------------------------------------------
# Possible values: en, es, fr, ko, ru or zh
lang: en
# ---------------------------------------------------------------------
# Site Authentication
# Database
# ---------------------------------------------------------------------
public: false
auth:
defaultReadAccess: false
local:
enabled: true
google:
enabled: true
clientId: GOOGLE_CLIENT_ID
clientSecret: GOOGLE_CLIENT_SECRET
microsoft:
enabled: true
clientId: MS_APP_ID
clientSecret: MS_APP_SECRET
facebook:
enabled: false
clientId: FACEBOOK_APP_ID
clientSecret: FACEBOOK_APP_SECRET
github:
enabled: false
clientId: GITHUB_CLIENT_ID
clientSecret: GITHUB_CLIENT_SECRET
slack:
enabled: false
clientId: SLACK_CLIENT_ID
clientSecret: SLACK_CLIENT_SECRET
ldap:
enabled: false
url: ldap://serverhost:389
bindDn: cn='root'
bindCredentials: BIND_PASSWORD
searchBase: o=users,o=example.com
searchFilter: (uid={{username}})
tlsEnabled: false
tlsCertPath: C:\example\root_ca_cert.crt
azure:
enabled: false
clientID: APP_ID
clientSecret: APP_SECRET_KEY
resource: '00000002-0000-0000-c000-000000000000'
tenant: 'YOUR_TENANT.onmicrosoft.com'
db:
host: localhost
port: 5432
user: wikijs
pass: wikijsrocks
db: wiki
# ---------------------------------------------------------------------
# Secret key to use when encrypting sessions
# ---------------------------------------------------------------------
# Use a long and unique random string (256-bit keys are perfect!)
sessionSecret: 1234567890abcdefghijklmnopqrstuvxyz
# ---------------------------------------------------------------------
# Database Connection String
# ---------------------------------------------------------------------
# You can also use an ENV variable by using $ENV_VAR_NAME as the value
db: mongodb://localhost:27017/wiki
# ---------------------------------------------------------------------
# Git Connection Info
# Redis
# ---------------------------------------------------------------------
git:
url: https://github.com/Organization/Repo
branch: master
auth:
redis:
host: localhost
port: 6379
db: 0
password: null
# Type: basic or ssh
type: ssh
# Only for Basic authentication:
username: marty
password: MartyMcFly88
# Only for SSH authentication:
privateKey: /etc/wiki/keys/git.pem
sslVerify: true
# Default email to use as commit author
serverEmail: marty@example.com
# Whether to use user email as author in commits
showUserEmail: true
# ---------------------------------------------------------------------
# Features
# ---------------------------------------------------------------------
# You can enable / disable specific features below
features:
linebreaks: true
mathjax: true
# ---------------------------------------------------------------------
# External Logging
# ---------------------------------------------------------------------
externalLogging:
bugsnag: false
loggly: false
papertrail: false
rollbar: false
sentry: false
# ---------------------------------------------------------------------
# Color Theme
# ---------------------------------------------------------------------
theme:
primary: indigo
alt: blue-grey
footer: blue-grey
code:
dark: true
colorize: true

1
package.json
View File

@ -43,6 +43,7 @@
"bcryptjs-then": "~1.0.1",
"bluebird": "~3.5.0",
"body-parser": "~1.17.2",
"bull": "/home/nick3.0.0-rc.4",
"bunyan": "~1.8.10",
"cheerio": "~1.0.0-rc.2",
"child-process-promise": "~2.2.1",

254
server/index.js
View File

@ -2,7 +2,6 @@
// ===========================================
// Wiki.js
// 1.0.1
// Licensed under AGPLv3
// ===========================================
@ -28,251 +27,28 @@ wiki.data = appconf.data
// Load Winston
// ----------------------------------------
wiki.logger = require('./modules/logger')(wiki.IS_DEBUG, 'SERVER')
wiki.logger.info('Wiki.js is initializing...')
wiki.logger = require('./modules/logger')()
// ----------------------------------------
// Load global modules
// Start Cluster
// ----------------------------------------
wiki.disk = require('./modules/disk').init()
wiki.db = require('./modules/db').init()
wiki.entries = require('./modules/entries').init()
wiki.git = require('./modules/git').init(false)
wiki.lang = require('i18next')
wiki.mark = require('./modules/markdown')
wiki.redis = require('./modules/redis').init()
wiki.search = require('./modules/search').init()
wiki.upl = require('./modules/uploads').init()
const cluster = require('cluster')
const numCPUs = require('os').cpus().length
// ----------------------------------------
// Load modules
// ----------------------------------------
if (cluster.isMaster) {
wiki.logger.info('Wiki.js is initializing...')
const autoload = require('auto-load')
const bodyParser = require('body-parser')
const compression = require('compression')
const cookieParser = require('cookie-parser')
const express = require('express')
const favicon = require('serve-favicon')
const flash = require('connect-flash')
const fork = require('child_process').fork
const http = require('http')
const i18nBackend = require('i18next-node-fs-backend')
const passport = require('passport')
const passportSocketIo = require('passport.socketio')
const session = require('express-session')
const SessionRedisStore = require('connect-redis')(session)
const graceful = require('node-graceful')
const socketio = require('socket.io')
const graphqlApollo = require('apollo-server-express')
const graphqlSchema = require('./modules/graphql')
require('./master')
var mw = autoload(path.join(wiki.SERVERPATH, '/middlewares'))
var ctrl = autoload(path.join(wiki.SERVERPATH, '/controllers'))
// ----------------------------------------
// Define Express App
// ----------------------------------------
const app = express()
wiki.app = app
app.use(compression())
// ----------------------------------------
// Security
// ----------------------------------------
app.use(mw.security)
// ----------------------------------------
// Public Assets
// ----------------------------------------
app.use(favicon(path.join(wiki.ROOTPATH, 'assets', 'favicon.ico')))
app.use(express.static(path.join(wiki.ROOTPATH, 'assets'), {
index: false,
maxAge: '7d'
}))
// ----------------------------------------
// Passport Authentication
// ----------------------------------------
require('./modules/auth')(passport)
wiki.rights = require('./modules/rights')
wiki.rights.init()
let sessionStore = new SessionRedisStore({
client: wiki.redis
})
app.use(cookieParser())
app.use(session({
name: 'wikijs.sid',
store: sessionStore,
secret: wiki.config.sessionSecret,
resave: false,
saveUninitialized: false
}))
app.use(flash())
app.use(passport.initialize())
app.use(passport.session())
// ----------------------------------------
// SEO
// ----------------------------------------
app.use(mw.seo)
// ----------------------------------------
// Localization Engine
// ----------------------------------------
wiki.lang.use(i18nBackend).init({
load: 'languageOnly',
ns: ['common', 'admin', 'auth', 'errors', 'git'],
defaultNS: 'common',
saveMissing: false,
preload: [wiki.config.lang],
lng: wiki.config.lang,
fallbackLng: 'en',
backend: {
loadPath: path.join(wiki.SERVERPATH, 'locales/{{lng}}/{{ns}}.json')
for (let i = 0; i < numCPUs; i++) {
cluster.fork()
}
})
// ----------------------------------------
// View Engine Setup
// ----------------------------------------
app.set('views', path.join(wiki.SERVERPATH, 'views'))
app.set('view engine', 'pug')
app.use(bodyParser.json({ limit: '1mb' }))
app.use(bodyParser.urlencoded({ extended: false, limit: '1mb' }))
// ----------------------------------------
// View accessible data
// ----------------------------------------
app.locals._ = require('lodash')
app.locals.t = wiki.lang.t.bind(wiki.lang)
app.locals.moment = require('moment')
app.locals.moment.locale(wiki.config.lang)
app.locals.appconfig = wiki.config
app.use(mw.flash)
// ----------------------------------------
// Controllers
// ----------------------------------------
app.use('/', ctrl.auth)
app.use('/graphql', graphqlApollo.graphqlExpress({ schema: graphqlSchema }))
app.use('/graphiql', graphqlApollo.graphiqlExpress({ endpointURL: '/graphql' }))
app.use('/uploads', mw.auth, ctrl.uploads)
app.use('/admin', mw.auth, ctrl.admin)
app.use('/', mw.auth, ctrl.pages)
// ----------------------------------------
// Error handling
// ----------------------------------------
app.use(function (req, res, next) {
var err = new Error('Not Found')
err.status = 404
next(err)
})
app.use(function (err, req, res, next) {
res.status(err.status || 500)
res.render('error', {
message: err.message,
error: wiki.IS_DEBUG ? err : {}
cluster.on('exit', (worker, code, signal) => {
wiki.logger.info(`Worker #${worker.id} died.`)
})
})
// ----------------------------------------
// Start HTTP server
// ----------------------------------------
wiki.logger.info('Starting HTTP/WS server on port ' + wiki.config.port + '...')
app.set('port', wiki.config.port)
var server = http.createServer(app)
var io = socketio(server)
server.listen(wiki.config.port)
server.on('error', (error) => {
if (error.syscall !== 'listen') {
throw error
}
// handle specific listen errors with friendly messages
switch (error.code) {
case 'EACCES':
wiki.logger.error('Listening on port ' + wiki.config.port + ' requires elevated privileges!')
return process.exit(1)
case 'EADDRINUSE':
wiki.logger.error('Port ' + wiki.config.port + ' is already in use!')
return process.exit(1)
default:
throw error
}
})
server.on('listening', () => {
wiki.logger.info('HTTP/WS server started successfully! [RUNNING]')
})
// ----------------------------------------
// WebSocket
// ----------------------------------------
io.use(passportSocketIo.authorize({
key: 'wikijs.sid',
store: sessionStore,
secret: wiki.config.sessionSecret,
cookieParser,
success: (data, accept) => {
accept()
},
fail: (data, message, error, accept) => {
accept()
}
}))
io.on('connection', ctrl.ws)
// ----------------------------------------
// Start child processes
// ----------------------------------------
let bgAgent = fork(path.join(wiki.SERVERPATH, 'agent.js'))
bgAgent.on('message', m => {
if (!m.action) {
return
}
switch (m.action) {
case 'searchAdd':
wiki.search.add(m.content)
break
}
})
// ----------------------------------------
// Graceful shutdown
// ----------------------------------------
graceful.on('exit', () => {
wiki.logger.info('- SHUTTING DOWN - Terminating Background Agent...')
bgAgent.kill()
wiki.logger.info('- SHUTTING DOWN - Performing git sync...')
return global.git.resync().then(() => {
wiki.logger.info('- SHUTTING DOWN - Git sync successful. Now safe to exit.')
process.exit()
})
})
} else {
wiki.logger.info(`Background Worker #${cluster.worker.id} is starting...`)
// require('./worker')
}

232
server/master.js Normal file
View File

@ -0,0 +1,232 @@
'use strict'
/* global wiki */
const path = require('path')
// ----------------------------------------
// Load global modules
// ----------------------------------------
wiki.disk = require('./modules/disk').init()
wiki.db = require('./modules/db').init()
wiki.entries = require('./modules/entries').init()
wiki.git = require('./modules/git').init(false)
wiki.lang = require('i18next')
wiki.mark = require('./modules/markdown')
wiki.redis = require('./modules/redis').init()
wiki.search = require('./modules/search').init()
wiki.upl = require('./modules/uploads').init()
// ----------------------------------------
// Load modules
// ----------------------------------------
const autoload = require('auto-load')
const bodyParser = require('body-parser')
const compression = require('compression')
const cookieParser = require('cookie-parser')
const express = require('express')
const favicon = require('serve-favicon')
const flash = require('connect-flash')
const http = require('http')
const i18nBackend = require('i18next-node-fs-backend')
const passport = require('passport')
const passportSocketIo = require('passport.socketio')
const session = require('express-session')
const SessionRedisStore = require('connect-redis')(session)
const graceful = require('node-graceful')
const socketio = require('socket.io')
const graphqlApollo = require('apollo-server-express')
const graphqlSchema = require('./modules/graphql')
var mw = autoload(path.join(wiki.SERVERPATH, '/middlewares'))
var ctrl = autoload(path.join(wiki.SERVERPATH, '/controllers'))
// ----------------------------------------
// Define Express App
// ----------------------------------------
const app = express()
wiki.app = app
app.use(compression())
// ----------------------------------------
// Security
// ----------------------------------------
app.use(mw.security)
// ----------------------------------------
// Public Assets
// ----------------------------------------
app.use(favicon(path.join(wiki.ROOTPATH, 'assets', 'favicon.ico')))
app.use(express.static(path.join(wiki.ROOTPATH, 'assets'), {
index: false,
maxAge: '7d'
}))
// ----------------------------------------
// Passport Authentication
// ----------------------------------------
require('./modules/auth')(passport)
wiki.rights = require('./modules/rights')
wiki.rights.init()
let sessionStore = new SessionRedisStore({
client: wiki.redis
})
app.use(cookieParser())
app.use(session({
name: 'wikijs.sid',
store: sessionStore,
secret: wiki.config.sessionSecret,
resave: false,
saveUninitialized: false
}))
app.use(flash())
app.use(passport.initialize())
app.use(passport.session())
// ----------------------------------------
// SEO
// ----------------------------------------
app.use(mw.seo)
// ----------------------------------------
// Localization Engine
// ----------------------------------------
wiki.lang.use(i18nBackend).init({
load: 'languageOnly',
ns: ['common', 'admin', 'auth', 'errors', 'git'],
defaultNS: 'common',
saveMissing: false,
preload: [wiki.config.lang],
lng: wiki.config.lang,
fallbackLng: 'en',
backend: {
loadPath: path.join(wiki.SERVERPATH, 'locales/{{lng}}/{{ns}}.json')
}
})
// ----------------------------------------
// View Engine Setup
// ----------------------------------------
app.set('views', path.join(wiki.SERVERPATH, 'views'))
app.set('view engine', 'pug')
app.use(bodyParser.json({ limit: '1mb' }))
app.use(bodyParser.urlencoded({ extended: false, limit: '1mb' }))
// ----------------------------------------
// View accessible data
// ----------------------------------------
app.locals._ = require('lodash')
app.locals.t = wiki.lang.t.bind(wiki.lang)
app.locals.moment = require('moment')
app.locals.moment.locale(wiki.config.lang)
app.locals.appconfig = wiki.config
app.use(mw.flash)
// ----------------------------------------
// Controllers
// ----------------------------------------
app.use('/', ctrl.auth)
app.use('/graphql', graphqlApollo.graphqlExpress({ schema: graphqlSchema }))
app.use('/graphiql', graphqlApollo.graphiqlExpress({ endpointURL: '/graphql' }))
app.use('/uploads', mw.auth, ctrl.uploads)
app.use('/admin', mw.auth, ctrl.admin)
app.use('/', mw.auth, ctrl.pages)
// ----------------------------------------
// Error handling
// ----------------------------------------
app.use(function (req, res, next) {
var err = new Error('Not Found')
err.status = 404
next(err)
})
app.use(function (err, req, res, next) {
res.status(err.status || 500)
res.render('error', {
message: err.message,
error: wiki.IS_DEBUG ? err : {}
})
})
// ----------------------------------------
// Start HTTP server
// ----------------------------------------
wiki.logger.info('Starting HTTP/WS server on port ' + wiki.config.port + '...')
app.set('port', wiki.config.port)
var server = http.createServer(app)
var io = socketio(server)
server.listen(wiki.config.port)
server.on('error', (error) => {
if (error.syscall !== 'listen') {
throw error
}
// handle specific listen errors with friendly messages
switch (error.code) {
case 'EACCES':
wiki.logger.error('Listening on port ' + wiki.config.port + ' requires elevated privileges!')
return process.exit(1)
case 'EADDRINUSE':
wiki.logger.error('Port ' + wiki.config.port + ' is already in use!')
return process.exit(1)
default:
throw error
}
})
server.on('listening', () => {
wiki.logger.info('HTTP/WS server started successfully! [RUNNING]')
})
// ----------------------------------------
// WebSocket
// ----------------------------------------
io.use(passportSocketIo.authorize({
key: 'wikijs.sid',
store: sessionStore,
secret: wiki.config.sessionSecret,
cookieParser,
success: (data, accept) => {
accept()
},
fail: (data, message, error, accept) => {
accept()
}
}))
io.on('connection', ctrl.ws)
// ----------------------------------------
// Graceful shutdown
// ----------------------------------------
graceful.on('exit', () => {
// wiki.logger.info('- SHUTTING DOWN - Terminating Background Agent...')
// bgAgent.kill()
wiki.logger.info('- SHUTTING DOWN - Performing git sync...')
return global.git.resync().then(() => {
wiki.logger.info('- SHUTTING DOWN - Git sync successful. Now safe to exit.')
process.exit()
})
})

7
server/models/_relations.js
View File

@ -4,7 +4,10 @@
* Associate DB Model relations
*/
module.exports = db => {
db.User.belongsToMany(db.Group, { through: 'UserGroups' })
db.Group.hasMany(db.Right, { as: 'GroupRights' })
db.User.belongsToMany(db.Group, { through: 'userGroups' })
db.Group.hasMany(db.Right, { as: 'groupRights' })
db.Document.hasMany(db.Tag, { as: 'documentTags' })
db.File.belongsTo(db.Folder)
db.Comment.belongsTo(db.Document)
db.Comment.belongsTo(db.User, { as: 'author' })
}

18
server/models/comment.js Normal file
View File

@ -0,0 +1,18 @@
'use strict'
/**
* Comment schema
*/
module.exports = (sequelize, DataTypes) => {
let commentSchema = sequelize.define('comment', {
content: {
type: DataTypes.STRING,
allowNull: false
}
}, {
timestamps: true,
version: true
})
return commentSchema
}

5
server/models/document.js
View File

@ -40,6 +40,11 @@ module.exports = (sequelize, DataTypes) => {
type: DataTypes.BOOLEAN,
allowNull: false,
defaultValue: false
},
searchContent: {
type: DataTypes.TEXT,
allowNull: true,
defaultValue: ''
}
}, {
timestamps: true,

24
server/models/tag.js Normal file
View File

@ -0,0 +1,24 @@
'use strict'
/**
* Tags schema
*/
module.exports = (sequelize, DataTypes) => {
let tagSchema = sequelize.define('tag', {
key: {
type: DataTypes.STRING,
allowNull: false
}
}, {
timestamps: true,
version: true,
indexes: [
{
unique: true,
fields: ['key']
}
]
})
return tagSchema
}

3
server/modules/db.js
View File

@ -64,7 +64,8 @@ module.exports = {
// Sync DB
self.onReady = self.inst.sync({
force: false
force: false,
logging: false
})
return self

9
server/modules/logger.js
View File

@ -2,12 +2,10 @@
/* global wiki */
module.exports = (processName) => {
let winston = require('winston')
const cluster = require('cluster')
if (typeof processName === 'undefined') {
processName = 'SERVER'
}
module.exports = () => {
let winston = require('winston')
// Console
@ -25,6 +23,7 @@ module.exports = (processName) => {
})
logger.filters.push((level, msg) => {
let processName = (cluster.isMaster) ? 'MASTER' : `WORKER-${cluster.worker.id}`
return '[' + processName + '] ' + msg
})

81
server/modules/search-index/index.js
View File

@ -1,81 +0,0 @@
const bunyan = require('bunyan')
const level = require('levelup')
const down = require('memdown')
const SearchIndexAdder = require('search-index-adder')
const SearchIndexSearcher = require('search-index-searcher')
module.exports = function (givenOptions, moduleReady) {
const optionsLoaded = function (err, SearchIndex) {
const siUtil = require('./siUtil.js')(SearchIndex.options)
if (err) return moduleReady(err)
SearchIndex.close = siUtil.close
SearchIndex.countDocs = siUtil.countDocs
getAdder(SearchIndex, adderLoaded)
}
const adderLoaded = function (err, SearchIndex) {
if (err) return moduleReady(err)
getSearcher(SearchIndex, searcherLoaded)
}
const searcherLoaded = function (err, SearchIndex) {
if (err) return moduleReady(err)
return moduleReady(err, SearchIndex)
}
getOptions(givenOptions, optionsLoaded)
}
const getAdder = function (SearchIndex, done) {
SearchIndexAdder(SearchIndex.options, function (err, searchIndexAdder) {
SearchIndex.add = searchIndexAdder.add
SearchIndex.callbackyAdd = searchIndexAdder.concurrentAdd // deprecated
SearchIndex.concurrentAdd = searchIndexAdder.concurrentAdd
SearchIndex.createWriteStream = searchIndexAdder.createWriteStream
SearchIndex.dbWriteStream = searchIndexAdder.dbWriteStream
SearchIndex.defaultPipeline = searchIndexAdder.defaultPipeline
SearchIndex.del = searchIndexAdder.deleter
SearchIndex.deleteStream = searchIndexAdder.deleteStream
SearchIndex.flush = searchIndexAdder.flush
done(err, SearchIndex)
})
}
const getSearcher = function (SearchIndex, done) {
SearchIndexSearcher(SearchIndex.options, function (err, searchIndexSearcher) {
SearchIndex.availableFields = searchIndexSearcher.availableFields
SearchIndex.buckets = searchIndexSearcher.bucketStream
SearchIndex.categorize = searchIndexSearcher.categoryStream
SearchIndex.dbReadStream = searchIndexSearcher.dbReadStream
SearchIndex.get = searchIndexSearcher.get
SearchIndex.match = searchIndexSearcher.match
SearchIndex.scan = searchIndexSearcher.scan
SearchIndex.search = searchIndexSearcher.search
SearchIndex.totalHits = searchIndexSearcher.totalHits
done(err, SearchIndex)
})
}
const getOptions = function (options, done) {
var SearchIndex = {}
SearchIndex.options = Object.assign({}, {
indexPath: 'si',
keySeparator: '￮',
logLevel: 'error'
}, options)
options.log = bunyan.createLogger({
name: 'search-index',
level: options.logLevel
})
if (!options.indexes) {
level(SearchIndex.options.indexPath || 'si', {
valueEncoding: 'json',
db: down
}, function (err, db) {
SearchIndex.options.indexes = db
return done(err, SearchIndex)
})
} else {
return done(null, SearchIndex)
}
}

36
server/modules/search-index/siUtil.js
View File

@ -1,36 +0,0 @@
'use strict'
module.exports = function (siOptions) {
var siUtil = {}
siUtil.countDocs = function (callback) {
var count = 0
const gte = 'DOCUMENT' + siOptions.keySeparator
const lte = 'DOCUMENT' + siOptions.keySeparator + siOptions.keySeparator
siOptions.indexes.createReadStream({gte: gte, lte: lte})
.on('data', function (data) {
count++
})
.on('error', function (err) {
return callback(err, null)
})
.on('end', function () {
return callback(null, count)
})
}
siUtil.close = function (callback) {
siOptions.indexes.close(function (err) {
while (!siOptions.indexes.isClosed()) {
// log not always working here- investigate
if (siOptions.log) siOptions.log.info('closing...')
}
if (siOptions.indexes.isClosed()) {
if (siOptions.log) siOptions.log.info('closed...')
callback(err)
}
})
}
return siUtil
}

6
server/modules/search.js
View File

@ -4,7 +4,7 @@
const Promise = require('bluebird')
const _ = require('lodash')
const searchIndex = require('./search-index')
// const searchIndex = require('./search-index')
const stopWord = require('stopword')
const streamToPromise = require('stream-to-promise')
const searchAllowedChars = new RegExp('[^a-z0-9' + wiki.data.regex.cjk + wiki.data.regex.arabic + ' ]', 'g')
@ -22,7 +22,7 @@ module.exports = {
init () {
let self = this
self._isReady = new Promise((resolve, reject) => {
searchIndex({
/*searchIndex({
deletable: true,
fieldedSearch: true,
indexPath: 'wiki',
@ -39,7 +39,7 @@ module.exports = {
resolve(true)
})
}
})
}) */
})
return self

86
server/agent.js → server/worker.js
View File

@ -1,39 +1,19 @@
// ===========================================
// Wiki.js - Background Agent
// 1.0.1
// Licensed under AGPLv3
// ===========================================
'use strict'
/* global wiki */
const path = require('path')
const ROOTPATH = process.cwd()
const SERVERPATH = path.join(ROOTPATH, 'server')
global.ROOTPATH = ROOTPATH
global.SERVERPATH = SERVERPATH
const IS_DEBUG = process.env.NODE_ENV === 'development'
let appconf = require('./modules/config')()
global.appconfig = appconf.config
global.appdata = appconf.data
// ----------------------------------------
// Load Winston
// ----------------------------------------
global.winston = require('./modules/logger')(IS_DEBUG, 'AGENT')
// ----------------------------------------
// Load global modules
// ----------------------------------------
global.winston.info('Background Agent is initializing...')
global.db = require('./modules/db').init()
global.upl = require('./modules/uploads-agent').init()
global.git = require('./modules/git').init()
global.entries = require('./modules/entries').init()
global.lang = require('i18next')
global.mark = require('./modules/markdown')
wiki.db = require('./modules/db').init()
wiki.upl = require('./modules/uploads-agent').init()
wiki.git = require('./modules/git').init()
wiki.entries = require('./modules/entries').init()
wiki.lang = require('i18next')
wiki.mark = require('./modules/markdown')
// ----------------------------------------
// Load modules
@ -52,20 +32,18 @@ const entryHelper = require('./helpers/entry')
// Localization Engine
// ----------------------------------------
global.lang
.use(i18nBackend)
.init({
load: 'languageOnly',
ns: ['common', 'admin', 'auth', 'errors', 'git'],
defaultNS: 'common',
saveMissing: false,
preload: [appconfig.lang],
lng: appconfig.lang,
fallbackLng: 'en',
backend: {
loadPath: path.join(SERVERPATH, 'locales/{{lng}}/{{ns}}.json')
}
})
wiki.lang.use(i18nBackend).init({
load: 'languageOnly',
ns: ['common', 'admin', 'auth', 'errors', 'git'],
defaultNS: 'common',
saveMissing: false,
preload: [wiki.config.lang],
lng: wiki.config.lang,
fallbackLng: 'en',
backend: {
loadPath: path.join(wiki.SERVERPATH, 'locales/{{lng}}/{{ns}}.json')
}
})
// ----------------------------------------
// Start Cron
@ -75,8 +53,8 @@ let job
let jobIsBusy = false
let jobUplWatchStarted = false
global.db.onReady.then(() => {
return global.db.Entry.remove({})
wiki.db.onReady.then(() => {
return wiki.db.Entry.remove({})
}).then(() => {
job = new Cron({
cronTime: '0 */5 * * * *',
@ -84,17 +62,17 @@ global.db.onReady.then(() => {
// Make sure we don't start two concurrent jobs
if (jobIsBusy) {
global.winston.warn('Previous job has not completed gracefully or is still running! Skipping for now. (This is not normal, you should investigate)')
wiki.logger.warn('Previous job has not completed gracefully or is still running! Skipping for now. (This is not normal, you should investigate)')
return
}
global.winston.info('Running all jobs...')
wiki.logger.info('Running all jobs...')
jobIsBusy = true
// Prepare async job collector
let jobs = []
let repoPath = path.resolve(ROOTPATH, appconfig.paths.repo)
let dataPath = path.resolve(ROOTPATH, appconfig.paths.data)
let repoPath = path.resolve(wiki.ROOTPATH, wiki.config.paths.repo)
let dataPath = path.resolve(wiki.ROOTPATH, wiki.config.paths.data)
let uploadsTempPath = path.join(dataPath, 'temp-upload')
// ----------------------------------------
@ -105,7 +83,7 @@ global.db.onReady.then(() => {
// -> Sync with Git remote
//* ****************************************
jobs.push(global.git.resync().then(() => {
jobs.push(wiki.git.resync().then(() => {
// -> Stream all documents
let cacheJobs = []
@ -185,18 +163,18 @@ global.db.onReady.then(() => {
// ----------------------------------------
Promise.all(jobs).then(() => {
global.winston.info('All jobs completed successfully! Going to sleep for now.')
wiki.logger.info('All jobs completed successfully! Going to sleep for now.')
if (!jobUplWatchStarted) {
jobUplWatchStarted = true
global.upl.initialScan().then(() => {
wiki.upl.initialScan().then(() => {
job.start()
})
}
return true
}).catch((err) => {
global.winston.error('One or more jobs have failed: ', err)
wiki.logger.error('One or more jobs have failed: ', err)
}).finally(() => {
jobIsBusy = false
})
@ -212,7 +190,7 @@ global.db.onReady.then(() => {
// ----------------------------------------
process.on('disconnect', () => {
global.winston.warn('Lost connection to main server. Exiting...')
wiki.logger.warn('Lost connection to main server. Exiting...')
job.stop()
process.exit()
})

63
yarn.lock
View File

@ -1036,7 +1036,7 @@ block-stream@*:
dependencies:
inherits "~2.0.0"
bluebird@^3.1.1, bluebird@^3.3.4, bluebird@^3.4.1, bluebird@^3.4.6, bluebird@~3.5.0:
bluebird@^3.1.1, bluebird@^3.3.4, bluebird@^3.4.1, bluebird@^3.4.6, bluebird@^3.5.0, bluebird@~3.5.0:
version "3.5.0"
resolved "https://registry.yarnpkg.com/bluebird/-/bluebird-3.5.0.tgz#791420d7f551eea2897453a8a77653f96606d67c"
@ -1124,6 +1124,18 @@ builtin-modules@^1.0.0, builtin-modules@^1.1.1:
version "1.1.1"
resolved "https://registry.yarnpkg.com/builtin-modules/-/builtin-modules-1.1.1.tgz#270f076c5a72c02f5b65a47df94c5fe3a278892f"
bull@/home/nick3.0.0-rc.4:
version "3.0.0-rc.4"
resolved "https://registry.yarnpkg.com/bull/-/bull-3.0.0-rc.4.tgz#dea18e870787037183849fc0198982ed756589b7"
dependencies:
bluebird "^3.5.0"
cron-parser "^2.4.1"
debuglog "^1.0.0"
ioredis "^3.1.1"
lodash "^4.17.4"
semver "^5.3.0"
uuid "^3.1.0"
bunyan@^1.8.1, bunyan@^1.8.10, bunyan@^1.8.3, bunyan@~1.8.10:
version "1.8.10"
resolved "https://registry.yarnpkg.com/bunyan/-/bunyan-1.8.10.tgz#201fedd26c7080b632f416072f53a90b9a52981c"
@ -1590,6 +1602,13 @@ crc@3.4.4, crc@^3.4.0:
version "3.4.4"
resolved "https://registry.yarnpkg.com/crc/-/crc-3.4.4.tgz#9da1e980e3bd44fc5c93bf5ab3da3378d85e466b"
cron-parser@^2.4.1:
version "2.4.1"
resolved "https://registry.yarnpkg.com/cron-parser/-/cron-parser-2.4.1.tgz#022befce1af293e4d3144ff04c2cbd2edb491271"
dependencies:
is-nan "^1.2.1"
moment-timezone "^0.5.0"
cron@1.2.1, cron@~1.2.1:
version "1.2.1"
resolved "https://registry.yarnpkg.com/cron/-/cron-1.2.1.tgz#3a86c09b41b8f261ac863a7cc85ea4735857eab2"
@ -1702,6 +1721,10 @@ debug@~2.2.0:
dependencies:
ms "0.7.1"
debuglog@^1.0.0:
version "1.0.1"
resolved "https://registry.yarnpkg.com/debuglog/-/debuglog-1.0.1.tgz#aa24ffb9ac3df9a2351837cfb2d279360cd78492"
decamelize@^1.0.0, decamelize@^1.1.1, decamelize@^1.1.2:
version "1.2.0"
resolved "https://registry.yarnpkg.com/decamelize/-/decamelize-1.2.0.tgz#f6534d15148269b20352e7bee26f501f9a191290"
@ -1732,6 +1755,13 @@ deferred-leveldown@~1.2.1:
dependencies:
abstract-leveldown "~2.4.0"
define-properties@^1.1.1:
version "1.1.2"
resolved "https://registry.yarnpkg.com/define-properties/-/define-properties-1.1.2.tgz#83a73f2fea569898fb737193c8f873caf6d45c94"
dependencies:
foreach "^2.0.5"
object-keys "^1.0.8"
del@^2.0.2:
version "2.2.2"
resolved "https://registry.yarnpkg.com/del/-/del-2.2.2.tgz#c12c981d067846c84bcaf862cff930d907ffd1a8"
@ -2573,6 +2603,10 @@ for-own@^0.1.4:
dependencies:
for-in "^1.0.1"
foreach@^2.0.5:
version "2.0.5"
resolved "https://registry.yarnpkg.com/foreach/-/foreach-2.0.5.tgz#0bee005018aeb260d0a3af3ae658dd0136ec1b99"
forever-agent@~0.6.1:
version "0.6.1"
resolved "https://registry.yarnpkg.com/forever-agent/-/forever-agent-0.6.1.tgz#fbc71f0c41adeb37f96c577ad1ed42d8fdacca91"
@ -3207,6 +3241,19 @@ invert-kv@^1.0.0:
version "1.0.0"
resolved "https://registry.yarnpkg.com/invert-kv/-/invert-kv-1.0.0.tgz#104a8e4aaca6d3d8cd157a8ef8bfab2d7a3ffdb6"
ioredis@^3.1.1:
version "3.1.2"
resolved "https://registry.yarnpkg.com/ioredis/-/ioredis-3.1.2.tgz#2579e3eba6dc490f68f14c7b51346281332b467b"
dependencies:
bluebird "^3.3.4"
cluster-key-slot "^1.0.6"
debug "^2.2.0"
denque "^1.1.0"
flexbuffer "0.0.6"
lodash "^4.8.2"
redis-commands "^1.2.0"
redis-parser "^2.4.0"
ioredis@~3.1.1:
version "3.1.1"
resolved "https://registry.yarnpkg.com/ioredis/-/ioredis-3.1.1.tgz#cc2f1d3232b8c95cc153046bce168f2baa1186e8"
@ -3320,6 +3367,12 @@ is-glob@^2.0.0, is-glob@^2.0.1:
dependencies:
is-extglob "^1.0.0"
is-nan@^1.2.1:
version "1.2.1"
resolved "https://registry.yarnpkg.com/is-nan/-/is-nan-1.2.1.tgz#9faf65b6fb6db24b7f5c0628475ea71f988401e2"
dependencies:
define-properties "^1.1.1"
is-npm@^1.0.0:
version "1.0.0"
resolved "https://registry.yarnpkg.com/is-npm/-/is-npm-1.0.0.tgz#f2fb63a65e4905b406c86072765a1a4dc793b9f4"
@ -4574,7 +4627,7 @@ mkdirp@0.5.1, "mkdirp@>=0.5 0", mkdirp@^0.5.0, mkdirp@^0.5.1, mkdirp@~0.5.1:
dependencies:
minimist "0.0.8"
moment-timezone@^0.5.4, moment-timezone@^0.5.x, moment-timezone@~0.5.13:
moment-timezone@^0.5.0, moment-timezone@^0.5.4, moment-timezone@^0.5.x, moment-timezone@~0.5.13:
version "0.5.13"
resolved "https://registry.yarnpkg.com/moment-timezone/-/moment-timezone-0.5.13.tgz#99ce5c7d827262eb0f1f702044177f60745d7b90"
dependencies:
@ -4877,6 +4930,10 @@ object-component@0.0.3:
version "0.0.3"
resolved "https://registry.yarnpkg.com/object-component/-/object-component-0.0.3.tgz#f0c69aa50efc95b866c186f400a33769cb2f1291"
object-keys@^1.0.8:
version "1.0.11"
resolved "https://registry.yarnpkg.com/object-keys/-/object-keys-1.0.11.tgz#c54601778ad560f1142ce0e01bcca8b56d13426d"
object.omit@^2.0.0:
version "2.0.1"
resolved "https://registry.yarnpkg.com/object.omit/-/object.omit-2.0.1.tgz#1a9c744829f39dbb858c76ca3579ae2a54ebd1fa"
@ -7045,7 +7102,7 @@ uuid@^2.0.1, uuid@^2.0.2:
version "2.0.3"
resolved "https://registry.yarnpkg.com/uuid/-/uuid-2.0.3.tgz#67e2e863797215530dff318e5bf9dcebfd47b21a"
uuid@^3.0.0, uuid@^3.0.1:
uuid@^3.0.0, uuid@^3.0.1, uuid@^3.1.0:
version "3.1.0"
resolved "https://registry.yarnpkg.com/uuid/-/uuid-3.1.0.tgz#3dd3d3e790abc24d7b0d3a034ffababe28ebbc04"