liz/wikijs-fork
liz/wikijs-fork
1
0
Fork 0
Code Issues Pull Requests Packages Projects Releases Wiki Activity

feat: cluster implementation

Browse Source
This commit is contained in:
NGPixel 2017-07-29 00:11:22 -04:00
parent 60750eeed8
commit 9c112ab535
15 changed files with 414 additions and 564 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

153
config.sample.yml
View File

@ -4,23 +4,9 @@
# Full explanation + examples in the documentation: # Full explanation + examples in the documentation:
# https://docs.requarks.io/wiki/install # https://docs.requarks.io/wiki/install
# ---------------------------------------------------------------------
# Title of this site
# ---------------------------------------------------------------------
title: Wiki
# ---------------------------------------------------------------------
# Full public path to the site, without the trailing slash
# ---------------------------------------------------------------------
# INCLUDE CLIENT PORT IF NOT 80/443!
host: http://localhost
# --------------------------------------------------------------------- # ---------------------------------------------------------------------
# Port the main server should listen to (80 by default) # Port the main server should listen to (80 by default)
# --------------------------------------------------------------------- # ---------------------------------------------------------------------
# To use process.env.PORT, comment the line below:
port: 80 port: 80
@ -33,136 +19,23 @@ paths:
data: ./data data: ./data
# --------------------------------------------------------------------- # ---------------------------------------------------------------------
# Upload Limits # Database
# ---------------------------------------------------------------------
# In megabytes (MB)
uploads:
maxImageFileSize: 3
maxOtherFileSize: 100
# ---------------------------------------------------------------------
# Site Language
# ---------------------------------------------------------------------
# Possible values: en, es, fr, ko, ru or zh
lang: en
# ---------------------------------------------------------------------
# Site Authentication
# --------------------------------------------------------------------- # ---------------------------------------------------------------------
public: false db:
host: localhost
auth: port: 5432
defaultReadAccess: false user: wikijs
local: pass: wikijsrocks
enabled: true db: wiki
google:
enabled: true
clientId: GOOGLE_CLIENT_ID
clientSecret: GOOGLE_CLIENT_SECRET
microsoft:
enabled: true
clientId: MS_APP_ID
clientSecret: MS_APP_SECRET
facebook:
enabled: false
clientId: FACEBOOK_APP_ID
clientSecret: FACEBOOK_APP_SECRET
github:
enabled: false
clientId: GITHUB_CLIENT_ID
clientSecret: GITHUB_CLIENT_SECRET
slack:
enabled: false
clientId: SLACK_CLIENT_ID
clientSecret: SLACK_CLIENT_SECRET
ldap:
enabled: false
url: ldap://serverhost:389
bindDn: cn='root'
bindCredentials: BIND_PASSWORD
searchBase: o=users,o=example.com
searchFilter: (uid={{username}})
tlsEnabled: false
tlsCertPath: C:\example\root_ca_cert.crt
azure:
enabled: false
clientID: APP_ID
clientSecret: APP_SECRET_KEY
resource: '00000002-0000-0000-c000-000000000000'
tenant: 'YOUR_TENANT.onmicrosoft.com'
# --------------------------------------------------------------------- # ---------------------------------------------------------------------
# Secret key to use when encrypting sessions # Redis
# ---------------------------------------------------------------------
# Use a long and unique random string (256-bit keys are perfect!)
sessionSecret: 1234567890abcdefghijklmnopqrstuvxyz
# ---------------------------------------------------------------------
# Database Connection String
# ---------------------------------------------------------------------
# You can also use an ENV variable by using $ENV_VAR_NAME as the value
db: mongodb://localhost:27017/wiki
# ---------------------------------------------------------------------
# Git Connection Info
# --------------------------------------------------------------------- # ---------------------------------------------------------------------
git: redis:
url: https://github.com/Organization/Repo host: localhost
branch: master port: 6379
auth: db: 0
password: null
# Type: basic or ssh
type: ssh
# Only for Basic authentication:
username: marty
password: MartyMcFly88
# Only for SSH authentication:
privateKey: /etc/wiki/keys/git.pem
sslVerify: true
# Default email to use as commit author
serverEmail: marty@example.com
# Whether to use user email as author in commits
showUserEmail: true
# ---------------------------------------------------------------------
# Features
# ---------------------------------------------------------------------
# You can enable / disable specific features below
features:
linebreaks: true
mathjax: true
# ---------------------------------------------------------------------
# External Logging
# ---------------------------------------------------------------------
externalLogging:
bugsnag: false
loggly: false
papertrail: false
rollbar: false
sentry: false
# ---------------------------------------------------------------------
# Color Theme
# ---------------------------------------------------------------------
theme:
primary: indigo
alt: blue-grey
footer: blue-grey
code:
dark: true
colorize: true

1
package.json
View File

@ -43,6 +43,7 @@
"bcryptjs-then": "~1.0.1", "bcryptjs-then": "~1.0.1",
"bluebird": "~3.5.0", "bluebird": "~3.5.0",
"body-parser": "~1.17.2", "body-parser": "~1.17.2",
"bull": "/home/nick3.0.0-rc.4",
"bunyan": "~1.8.10", "bunyan": "~1.8.10",
"cheerio": "~1.0.0-rc.2", "cheerio": "~1.0.0-rc.2",
"child-process-promise": "~2.2.1", "child-process-promise": "~2.2.1",

254
server/index.js
View File

@ -2,7 +2,6 @@
// =========================================== // ===========================================
// Wiki.js // Wiki.js
// 1.0.1
// Licensed under AGPLv3 // Licensed under AGPLv3
// =========================================== // ===========================================
@ -28,251 +27,28 @@ wiki.data = appconf.data
// Load Winston // Load Winston
// ---------------------------------------- // ----------------------------------------
wiki.logger = require('./modules/logger')(wiki.IS_DEBUG, 'SERVER') wiki.logger = require('./modules/logger')()
wiki.logger.info('Wiki.js is initializing...')
// ---------------------------------------- // ----------------------------------------
// Load global modules // Start Cluster
// ---------------------------------------- // ----------------------------------------
wiki.disk = require('./modules/disk').init() const cluster = require('cluster')
wiki.db = require('./modules/db').init() const numCPUs = require('os').cpus().length
wiki.entries = require('./modules/entries').init()
wiki.git = require('./modules/git').init(false)
wiki.lang = require('i18next')
wiki.mark = require('./modules/markdown')
wiki.redis = require('./modules/redis').init()
wiki.search = require('./modules/search').init()
wiki.upl = require('./modules/uploads').init()
// ---------------------------------------- if (cluster.isMaster) {
// Load modules wiki.logger.info('Wiki.js is initializing...')
// ----------------------------------------
const autoload = require('auto-load') require('./master')
const bodyParser = require('body-parser')
const compression = require('compression')
const cookieParser = require('cookie-parser')
const express = require('express')
const favicon = require('serve-favicon')
const flash = require('connect-flash')
const fork = require('child_process').fork
const http = require('http')
const i18nBackend = require('i18next-node-fs-backend')
const passport = require('passport')
const passportSocketIo = require('passport.socketio')
const session = require('express-session')
const SessionRedisStore = require('connect-redis')(session)
const graceful = require('node-graceful')
const socketio = require('socket.io')
const graphqlApollo = require('apollo-server-express')
const graphqlSchema = require('./modules/graphql')
var mw = autoload(path.join(wiki.SERVERPATH, '/middlewares')) for (let i = 0; i < numCPUs; i++) {
var ctrl = autoload(path.join(wiki.SERVERPATH, '/controllers')) cluster.fork()
// ----------------------------------------
// Define Express App
// ----------------------------------------
const app = express()
wiki.app = app
app.use(compression())
// ----------------------------------------
// Security
// ----------------------------------------
app.use(mw.security)
// ----------------------------------------
// Public Assets
// ----------------------------------------
app.use(favicon(path.join(wiki.ROOTPATH, 'assets', 'favicon.ico')))
app.use(express.static(path.join(wiki.ROOTPATH, 'assets'), {
index: false,
maxAge: '7d'
}))
// ----------------------------------------
// Passport Authentication
// ----------------------------------------
require('./modules/auth')(passport)
wiki.rights = require('./modules/rights')
wiki.rights.init()
let sessionStore = new SessionRedisStore({
client: wiki.redis
})
app.use(cookieParser())
app.use(session({
name: 'wikijs.sid',
store: sessionStore,
secret: wiki.config.sessionSecret,
resave: false,
saveUninitialized: false
}))
app.use(flash())
app.use(passport.initialize())
app.use(passport.session())
// ----------------------------------------
// SEO
// ----------------------------------------
app.use(mw.seo)
// ----------------------------------------
// Localization Engine
// ----------------------------------------
wiki.lang.use(i18nBackend).init({
load: 'languageOnly',
ns: ['common', 'admin', 'auth', 'errors', 'git'],
defaultNS: 'common',
saveMissing: false,
preload: [wiki.config.lang],
lng: wiki.config.lang,
fallbackLng: 'en',
backend: {
loadPath: path.join(wiki.SERVERPATH, 'locales/{{lng}}/{{ns}}.json')
} }
})
// ---------------------------------------- cluster.on('exit', (worker, code, signal) => {
// View Engine Setup wiki.logger.info(`Worker #${worker.id} died.`)
// ----------------------------------------
app.set('views', path.join(wiki.SERVERPATH, 'views'))
app.set('view engine', 'pug')
app.use(bodyParser.json({ limit: '1mb' }))
app.use(bodyParser.urlencoded({ extended: false, limit: '1mb' }))
// ----------------------------------------
// View accessible data
// ----------------------------------------
app.locals._ = require('lodash')
app.locals.t = wiki.lang.t.bind(wiki.lang)
app.locals.moment = require('moment')
app.locals.moment.locale(wiki.config.lang)
app.locals.appconfig = wiki.config
app.use(mw.flash)
// ----------------------------------------
// Controllers
// ----------------------------------------
app.use('/', ctrl.auth)
app.use('/graphql', graphqlApollo.graphqlExpress({ schema: graphqlSchema }))
app.use('/graphiql', graphqlApollo.graphiqlExpress({ endpointURL: '/graphql' }))
app.use('/uploads', mw.auth, ctrl.uploads)
app.use('/admin', mw.auth, ctrl.admin)
app.use('/', mw.auth, ctrl.pages)
// ----------------------------------------
// Error handling
// ----------------------------------------
app.use(function (req, res, next) {
var err = new Error('Not Found')
err.status = 404
next(err)
})
app.use(function (err, req, res, next) {
res.status(err.status || 500)
res.render('error', {
message: err.message,
error: wiki.IS_DEBUG ? err : {}
}) })
}) } else {
wiki.logger.info(`Background Worker #${cluster.worker.id} is starting...`)
// ---------------------------------------- // require('./worker')
// Start HTTP server }
// ----------------------------------------
wiki.logger.info('Starting HTTP/WS server on port ' + wiki.config.port + '...')
app.set('port', wiki.config.port)
var server = http.createServer(app)
var io = socketio(server)
server.listen(wiki.config.port)
server.on('error', (error) => {
if (error.syscall !== 'listen') {
throw error
}
// handle specific listen errors with friendly messages
switch (error.code) {
case 'EACCES':
wiki.logger.error('Listening on port ' + wiki.config.port + ' requires elevated privileges!')
return process.exit(1)
case 'EADDRINUSE':
wiki.logger.error('Port ' + wiki.config.port + ' is already in use!')
return process.exit(1)
default:
throw error
}
})
server.on('listening', () => {
wiki.logger.info('HTTP/WS server started successfully! [RUNNING]')
})
// ----------------------------------------
// WebSocket
// ----------------------------------------
io.use(passportSocketIo.authorize({
key: 'wikijs.sid',
store: sessionStore,
secret: wiki.config.sessionSecret,
cookieParser,
success: (data, accept) => {
accept()
},
fail: (data, message, error, accept) => {
accept()
}
}))
io.on('connection', ctrl.ws)
// ----------------------------------------
// Start child processes
// ----------------------------------------
let bgAgent = fork(path.join(wiki.SERVERPATH, 'agent.js'))
bgAgent.on('message', m => {
if (!m.action) {
return
}
switch (m.action) {
case 'searchAdd':
wiki.search.add(m.content)
break
}
})
// ----------------------------------------
// Graceful shutdown
// ----------------------------------------
graceful.on('exit', () => {
wiki.logger.info('- SHUTTING DOWN - Terminating Background Agent...')
bgAgent.kill()
wiki.logger.info('- SHUTTING DOWN - Performing git sync...')
return global.git.resync().then(() => {
wiki.logger.info('- SHUTTING DOWN - Git sync successful. Now safe to exit.')
process.exit()
})
})

232
server/master.js Normal file
View File

@ -0,0 +1,232 @@
'use strict'
/* global wiki */
const path = require('path')
// ----------------------------------------
// Load global modules
// ----------------------------------------
wiki.disk = require('./modules/disk').init()
wiki.db = require('./modules/db').init()
wiki.entries = require('./modules/entries').init()
wiki.git = require('./modules/git').init(false)
wiki.lang = require('i18next')
wiki.mark = require('./modules/markdown')
wiki.redis = require('./modules/redis').init()
wiki.search = require('./modules/search').init()
wiki.upl = require('./modules/uploads').init()
// ----------------------------------------
// Load modules
// ----------------------------------------
const autoload = require('auto-load')
const bodyParser = require('body-parser')
const compression = require('compression')
const cookieParser = require('cookie-parser')
const express = require('express')
const favicon = require('serve-favicon')
const flash = require('connect-flash')
const http = require('http')
const i18nBackend = require('i18next-node-fs-backend')
const passport = require('passport')
const passportSocketIo = require('passport.socketio')
const session = require('express-session')
const SessionRedisStore = require('connect-redis')(session)
const graceful = require('node-graceful')
const socketio = require('socket.io')
const graphqlApollo = require('apollo-server-express')
const graphqlSchema = require('./modules/graphql')
var mw = autoload(path.join(wiki.SERVERPATH, '/middlewares'))
var ctrl = autoload(path.join(wiki.SERVERPATH, '/controllers'))
// ----------------------------------------
// Define Express App
// ----------------------------------------
const app = express()
wiki.app = app
app.use(compression())
// ----------------------------------------
// Security
// ----------------------------------------
app.use(mw.security)
// ----------------------------------------
// Public Assets
// ----------------------------------------
app.use(favicon(path.join(wiki.ROOTPATH, 'assets', 'favicon.ico')))
app.use(express.static(path.join(wiki.ROOTPATH, 'assets'), {
index: false,
maxAge: '7d'
}))
// ----------------------------------------
// Passport Authentication
// ----------------------------------------
require('./modules/auth')(passport)
wiki.rights = require('./modules/rights')
wiki.rights.init()
let sessionStore = new SessionRedisStore({
client: wiki.redis
})
app.use(cookieParser())
app.use(session({
name: 'wikijs.sid',
store: sessionStore,
secret: wiki.config.sessionSecret,
resave: false,
saveUninitialized: false
}))
app.use(flash())
app.use(passport.initialize())
app.use(passport.session())
// ----------------------------------------
// SEO
// ----------------------------------------
app.use(mw.seo)
// ----------------------------------------
// Localization Engine
// ----------------------------------------
wiki.lang.use(i18nBackend).init({
load: 'languageOnly',
ns: ['common', 'admin', 'auth', 'errors', 'git'],
defaultNS: 'common',
saveMissing: false,
preload: [wiki.config.lang],
lng: wiki.config.lang,
fallbackLng: 'en',
backend: {
loadPath: path.join(wiki.SERVERPATH, 'locales/{{lng}}/{{ns}}.json')
}
})
// ----------------------------------------
// View Engine Setup
// ----------------------------------------
app.set('views', path.join(wiki.SERVERPATH, 'views'))
app.set('view engine', 'pug')
app.use(bodyParser.json({ limit: '1mb' }))
app.use(bodyParser.urlencoded({ extended: false, limit: '1mb' }))
// ----------------------------------------
// View accessible data
// ----------------------------------------
app.locals._ = require('lodash')
app.locals.t = wiki.lang.t.bind(wiki.lang)
app.locals.moment = require('moment')
app.locals.moment.locale(wiki.config.lang)
app.locals.appconfig = wiki.config
app.use(mw.flash)
// ----------------------------------------
// Controllers
// ----------------------------------------
app.use('/', ctrl.auth)
app.use('/graphql', graphqlApollo.graphqlExpress({ schema: graphqlSchema }))
app.use('/graphiql', graphqlApollo.graphiqlExpress({ endpointURL: '/graphql' }))
app.use('/uploads', mw.auth, ctrl.uploads)
app.use('/admin', mw.auth, ctrl.admin)
app.use('/', mw.auth, ctrl.pages)
// ----------------------------------------
// Error handling
// ----------------------------------------
app.use(function (req, res, next) {
var err = new Error('Not Found')
err.status = 404
next(err)
})
app.use(function (err, req, res, next) {
res.status(err.status || 500)
res.render('error', {
message: err.message,
error: wiki.IS_DEBUG ? err : {}
})
})
// ----------------------------------------
// Start HTTP server
// ----------------------------------------
wiki.logger.info('Starting HTTP/WS server on port ' + wiki.config.port + '...')
app.set('port', wiki.config.port)
var server = http.createServer(app)
var io = socketio(server)
server.listen(wiki.config.port)
server.on('error', (error) => {
if (error.syscall !== 'listen') {
throw error
}
// handle specific listen errors with friendly messages
switch (error.code) {
case 'EACCES':
wiki.logger.error('Listening on port ' + wiki.config.port + ' requires elevated privileges!')
return process.exit(1)
case 'EADDRINUSE':
wiki.logger.error('Port ' + wiki.config.port + ' is already in use!')
return process.exit(1)
default:
throw error
}
})
server.on('listening', () => {
wiki.logger.info('HTTP/WS server started successfully! [RUNNING]')
})
// ----------------------------------------
// WebSocket
// ----------------------------------------
io.use(passportSocketIo.authorize({
key: 'wikijs.sid',
store: sessionStore,
secret: wiki.config.sessionSecret,
cookieParser,
success: (data, accept) => {
accept()
},
fail: (data, message, error, accept) => {
accept()
}
}))
io.on('connection', ctrl.ws)
// ----------------------------------------
// Graceful shutdown
// ----------------------------------------
graceful.on('exit', () => {
// wiki.logger.info('- SHUTTING DOWN - Terminating Background Agent...')
// bgAgent.kill()
wiki.logger.info('- SHUTTING DOWN - Performing git sync...')
return global.git.resync().then(() => {
wiki.logger.info('- SHUTTING DOWN - Git sync successful. Now safe to exit.')
process.exit()
})
})

7
server/models/_relations.js
View File

@ -4,7 +4,10 @@
* Associate DB Model relations * Associate DB Model relations
*/ */
module.exports = db => { module.exports = db => {
db.User.belongsToMany(db.Group, { through: 'UserGroups' }) db.User.belongsToMany(db.Group, { through: 'userGroups' })
db.Group.hasMany(db.Right, { as: 'GroupRights' }) db.Group.hasMany(db.Right, { as: 'groupRights' })
db.Document.hasMany(db.Tag, { as: 'documentTags' })
db.File.belongsTo(db.Folder) db.File.belongsTo(db.Folder)
db.Comment.belongsTo(db.Document)
db.Comment.belongsTo(db.User, { as: 'author' })
} }

18
server/models/comment.js Normal file
View File

@ -0,0 +1,18 @@
'use strict'
/**
* Comment schema
*/
module.exports = (sequelize, DataTypes) => {
let commentSchema = sequelize.define('comment', {
content: {
type: DataTypes.STRING,
allowNull: false
}
}, {
timestamps: true,
version: true
})
return commentSchema
}

5
server/models/document.js
View File

@ -40,6 +40,11 @@ module.exports = (sequelize, DataTypes) => {
type: DataTypes.BOOLEAN, type: DataTypes.BOOLEAN,
allowNull: false, allowNull: false,
defaultValue: false defaultValue: false
},
searchContent: {
type: DataTypes.TEXT,
allowNull: true,
defaultValue: ''
} }
}, { }, {
timestamps: true, timestamps: true,

24
server/models/tag.js Normal file
View File

@ -0,0 +1,24 @@
'use strict'
/**
* Tags schema
*/
module.exports = (sequelize, DataTypes) => {
let tagSchema = sequelize.define('tag', {
key: {
type: DataTypes.STRING,
allowNull: false
}
}, {
timestamps: true,
version: true,
indexes: [
{
unique: true,
fields: ['key']
}
]
})
return tagSchema
}

3
server/modules/db.js
View File

@ -64,7 +64,8 @@ module.exports = {
// Sync DB // Sync DB
self.onReady = self.inst.sync({ self.onReady = self.inst.sync({
force: false force: false,
logging: false
}) })
return self return self

9
server/modules/logger.js
View File

@ -2,12 +2,10 @@
/* global wiki */ /* global wiki */
module.exports = (processName) => { const cluster = require('cluster')
let winston = require('winston')
if (typeof processName === 'undefined') { module.exports = () => {
processName = 'SERVER' let winston = require('winston')
}
// Console // Console
@ -25,6 +23,7 @@ module.exports = (processName) => {
}) })
logger.filters.push((level, msg) => { logger.filters.push((level, msg) => {
let processName = (cluster.isMaster) ? 'MASTER' : `WORKER-${cluster.worker.id}`
return '[' + processName + '] ' + msg return '[' + processName + '] ' + msg
}) })

81
server/modules/search-index/index.js
View File

@ -1,81 +0,0 @@
const bunyan = require('bunyan')
const level = require('levelup')
const down = require('memdown')
const SearchIndexAdder = require('search-index-adder')
const SearchIndexSearcher = require('search-index-searcher')
module.exports = function (givenOptions, moduleReady) {
const optionsLoaded = function (err, SearchIndex) {
const siUtil = require('./siUtil.js')(SearchIndex.options)
if (err) return moduleReady(err)
SearchIndex.close = siUtil.close
SearchIndex.countDocs = siUtil.countDocs
getAdder(SearchIndex, adderLoaded)
}
const adderLoaded = function (err, SearchIndex) {
if (err) return moduleReady(err)
getSearcher(SearchIndex, searcherLoaded)
}
const searcherLoaded = function (err, SearchIndex) {
if (err) return moduleReady(err)
return moduleReady(err, SearchIndex)
}
getOptions(givenOptions, optionsLoaded)
}
const getAdder = function (SearchIndex, done) {
SearchIndexAdder(SearchIndex.options, function (err, searchIndexAdder) {
SearchIndex.add = searchIndexAdder.add
SearchIndex.callbackyAdd = searchIndexAdder.concurrentAdd // deprecated
SearchIndex.concurrentAdd = searchIndexAdder.concurrentAdd
SearchIndex.createWriteStream = searchIndexAdder.createWriteStream
SearchIndex.dbWriteStream = searchIndexAdder.dbWriteStream
SearchIndex.defaultPipeline = searchIndexAdder.defaultPipeline
SearchIndex.del = searchIndexAdder.deleter
SearchIndex.deleteStream = searchIndexAdder.deleteStream
SearchIndex.flush = searchIndexAdder.flush
done(err, SearchIndex)
})
}
const getSearcher = function (SearchIndex, done) {
SearchIndexSearcher(SearchIndex.options, function (err, searchIndexSearcher) {
SearchIndex.availableFields = searchIndexSearcher.availableFields
SearchIndex.buckets = searchIndexSearcher.bucketStream
SearchIndex.categorize = searchIndexSearcher.categoryStream
SearchIndex.dbReadStream = searchIndexSearcher.dbReadStream
SearchIndex.get = searchIndexSearcher.get
SearchIndex.match = searchIndexSearcher.match
SearchIndex.scan = searchIndexSearcher.scan
SearchIndex.search = searchIndexSearcher.search
SearchIndex.totalHits = searchIndexSearcher.totalHits
done(err, SearchIndex)
})
}
const getOptions = function (options, done) {
var SearchIndex = {}
SearchIndex.options = Object.assign({}, {
indexPath: 'si',
keySeparator: '￮',
logLevel: 'error'
}, options)
options.log = bunyan.createLogger({
name: 'search-index',
level: options.logLevel
})
if (!options.indexes) {
level(SearchIndex.options.indexPath || 'si', {
valueEncoding: 'json',
db: down
}, function (err, db) {
SearchIndex.options.indexes = db
return done(err, SearchIndex)
})
} else {
return done(null, SearchIndex)
}
}

36
server/modules/search-index/siUtil.js
View File

@ -1,36 +0,0 @@
'use strict'
module.exports = function (siOptions) {
var siUtil = {}
siUtil.countDocs = function (callback) {
var count = 0
const gte = 'DOCUMENT' + siOptions.keySeparator
const lte = 'DOCUMENT' + siOptions.keySeparator + siOptions.keySeparator
siOptions.indexes.createReadStream({gte: gte, lte: lte})
.on('data', function (data) {
count++
})
.on('error', function (err) {
return callback(err, null)
})
.on('end', function () {
return callback(null, count)
})
}
siUtil.close = function (callback) {
siOptions.indexes.close(function (err) {
while (!siOptions.indexes.isClosed()) {
// log not always working here- investigate
if (siOptions.log) siOptions.log.info('closing...')
}
if (siOptions.indexes.isClosed()) {
if (siOptions.log) siOptions.log.info('closed...')
callback(err)
}
})
}
return siUtil
}

6
server/modules/search.js
View File

@ -4,7 +4,7 @@
const Promise = require('bluebird') const Promise = require('bluebird')
const _ = require('lodash') const _ = require('lodash')
const searchIndex = require('./search-index') // const searchIndex = require('./search-index')
const stopWord = require('stopword') const stopWord = require('stopword')
const streamToPromise = require('stream-to-promise') const streamToPromise = require('stream-to-promise')
const searchAllowedChars = new RegExp('[^a-z0-9' + wiki.data.regex.cjk + wiki.data.regex.arabic + ' ]', 'g') const searchAllowedChars = new RegExp('[^a-z0-9' + wiki.data.regex.cjk + wiki.data.regex.arabic + ' ]', 'g')
@ -22,7 +22,7 @@ module.exports = {
init () { init () {
let self = this let self = this
self._isReady = new Promise((resolve, reject) => { self._isReady = new Promise((resolve, reject) => {
searchIndex({ /*searchIndex({
deletable: true, deletable: true,
fieldedSearch: true, fieldedSearch: true,
indexPath: 'wiki', indexPath: 'wiki',
@ -39,7 +39,7 @@ module.exports = {
resolve(true) resolve(true)
}) })
} }
}) }) */
}) })
return self return self

86
server/agent.js → server/worker.js
View File

@ -1,39 +1,19 @@
// =========================================== 'use strict'
// Wiki.js - Background Agent
// 1.0.1 /* global wiki */
// Licensed under AGPLv3
// ===========================================
const path = require('path') const path = require('path')
const ROOTPATH = process.cwd()
const SERVERPATH = path.join(ROOTPATH, 'server')
global.ROOTPATH = ROOTPATH
global.SERVERPATH = SERVERPATH
const IS_DEBUG = process.env.NODE_ENV === 'development'
let appconf = require('./modules/config')()
global.appconfig = appconf.config
global.appdata = appconf.data
// ----------------------------------------
// Load Winston
// ----------------------------------------
global.winston = require('./modules/logger')(IS_DEBUG, 'AGENT')
// ---------------------------------------- // ----------------------------------------
// Load global modules // Load global modules
// ---------------------------------------- // ----------------------------------------
global.winston.info('Background Agent is initializing...') wiki.db = require('./modules/db').init()
wiki.upl = require('./modules/uploads-agent').init()
global.db = require('./modules/db').init() wiki.git = require('./modules/git').init()
global.upl = require('./modules/uploads-agent').init() wiki.entries = require('./modules/entries').init()
global.git = require('./modules/git').init() wiki.lang = require('i18next')
global.entries = require('./modules/entries').init() wiki.mark = require('./modules/markdown')
global.lang = require('i18next')
global.mark = require('./modules/markdown')
// ---------------------------------------- // ----------------------------------------
// Load modules // Load modules
@ -52,20 +32,18 @@ const entryHelper = require('./helpers/entry')
// Localization Engine // Localization Engine
// ---------------------------------------- // ----------------------------------------
global.lang wiki.lang.use(i18nBackend).init({
.use(i18nBackend) load: 'languageOnly',
.init({ ns: ['common', 'admin', 'auth', 'errors', 'git'],
load: 'languageOnly', defaultNS: 'common',
ns: ['common', 'admin', 'auth', 'errors', 'git'], saveMissing: false,
defaultNS: 'common', preload: [wiki.config.lang],
saveMissing: false, lng: wiki.config.lang,
preload: [appconfig.lang], fallbackLng: 'en',
lng: appconfig.lang, backend: {
fallbackLng: 'en', loadPath: path.join(wiki.SERVERPATH, 'locales/{{lng}}/{{ns}}.json')
backend: { }
loadPath: path.join(SERVERPATH, 'locales/{{lng}}/{{ns}}.json') })
}
})
// ---------------------------------------- // ----------------------------------------
// Start Cron // Start Cron
@ -75,8 +53,8 @@ let job
let jobIsBusy = false let jobIsBusy = false
let jobUplWatchStarted = false let jobUplWatchStarted = false
global.db.onReady.then(() => { wiki.db.onReady.then(() => {
return global.db.Entry.remove({}) return wiki.db.Entry.remove({})
}).then(() => { }).then(() => {
job = new Cron({ job = new Cron({
cronTime: '0 */5 * * * *', cronTime: '0 */5 * * * *',
@ -84,17 +62,17 @@ global.db.onReady.then(() => {
// Make sure we don't start two concurrent jobs // Make sure we don't start two concurrent jobs
if (jobIsBusy) { if (jobIsBusy) {
global.winston.warn('Previous job has not completed gracefully or is still running! Skipping for now. (This is not normal, you should investigate)') wiki.logger.warn('Previous job has not completed gracefully or is still running! Skipping for now. (This is not normal, you should investigate)')
return return
} }
global.winston.info('Running all jobs...') wiki.logger.info('Running all jobs...')
jobIsBusy = true jobIsBusy = true
// Prepare async job collector // Prepare async job collector
let jobs = [] let jobs = []
let repoPath = path.resolve(ROOTPATH, appconfig.paths.repo) let repoPath = path.resolve(wiki.ROOTPATH, wiki.config.paths.repo)
let dataPath = path.resolve(ROOTPATH, appconfig.paths.data) let dataPath = path.resolve(wiki.ROOTPATH, wiki.config.paths.data)
let uploadsTempPath = path.join(dataPath, 'temp-upload') let uploadsTempPath = path.join(dataPath, 'temp-upload')
// ---------------------------------------- // ----------------------------------------
@ -105,7 +83,7 @@ global.db.onReady.then(() => {
// -> Sync with Git remote // -> Sync with Git remote
//* **************************************** //* ****************************************
jobs.push(global.git.resync().then(() => { jobs.push(wiki.git.resync().then(() => {
// -> Stream all documents // -> Stream all documents
let cacheJobs = [] let cacheJobs = []
@ -185,18 +163,18 @@ global.db.onReady.then(() => {
// ---------------------------------------- // ----------------------------------------
Promise.all(jobs).then(() => { Promise.all(jobs).then(() => {
global.winston.info('All jobs completed successfully! Going to sleep for now.') wiki.logger.info('All jobs completed successfully! Going to sleep for now.')
if (!jobUplWatchStarted) { if (!jobUplWatchStarted) {
jobUplWatchStarted = true jobUplWatchStarted = true
global.upl.initialScan().then(() => { wiki.upl.initialScan().then(() => {
job.start() job.start()
}) })
} }
return true return true
}).catch((err) => { }).catch((err) => {
global.winston.error('One or more jobs have failed: ', err) wiki.logger.error('One or more jobs have failed: ', err)
}).finally(() => { }).finally(() => {
jobIsBusy = false jobIsBusy = false
}) })
@ -212,7 +190,7 @@ global.db.onReady.then(() => {
// ---------------------------------------- // ----------------------------------------
process.on('disconnect', () => { process.on('disconnect', () => {
global.winston.warn('Lost connection to main server. Exiting...') wiki.logger.warn('Lost connection to main server. Exiting...')
job.stop() job.stop()
process.exit() process.exit()
}) })

63
yarn.lock
View File

@ -1036,7 +1036,7 @@ block-stream@*:
dependencies: dependencies:
inherits "~2.0.0" inherits "~2.0.0"
bluebird@^3.1.1, bluebird@^3.3.4, bluebird@^3.4.1, bluebird@^3.4.6, bluebird@~3.5.0: bluebird@^3.1.1, bluebird@^3.3.4, bluebird@^3.4.1, bluebird@^3.4.6, bluebird@^3.5.0, bluebird@~3.5.0:
version "3.5.0" version "3.5.0"
resolved "https://registry.yarnpkg.com/bluebird/-/bluebird-3.5.0.tgz#791420d7f551eea2897453a8a77653f96606d67c" resolved "https://registry.yarnpkg.com/bluebird/-/bluebird-3.5.0.tgz#791420d7f551eea2897453a8a77653f96606d67c"
@ -1124,6 +1124,18 @@ builtin-modules@^1.0.0, builtin-modules@^1.1.1:
version "1.1.1" version "1.1.1"
resolved "https://registry.yarnpkg.com/builtin-modules/-/builtin-modules-1.1.1.tgz#270f076c5a72c02f5b65a47df94c5fe3a278892f" resolved "https://registry.yarnpkg.com/builtin-modules/-/builtin-modules-1.1.1.tgz#270f076c5a72c02f5b65a47df94c5fe3a278892f"
bull@/home/nick3.0.0-rc.4:
version "3.0.0-rc.4"
resolved "https://registry.yarnpkg.com/bull/-/bull-3.0.0-rc.4.tgz#dea18e870787037183849fc0198982ed756589b7"
dependencies:
bluebird "^3.5.0"
cron-parser "^2.4.1"
debuglog "^1.0.0"
ioredis "^3.1.1"
lodash "^4.17.4"
semver "^5.3.0"
uuid "^3.1.0"
bunyan@^1.8.1, bunyan@^1.8.10, bunyan@^1.8.3, bunyan@~1.8.10: bunyan@^1.8.1, bunyan@^1.8.10, bunyan@^1.8.3, bunyan@~1.8.10:
version "1.8.10" version "1.8.10"
resolved "https://registry.yarnpkg.com/bunyan/-/bunyan-1.8.10.tgz#201fedd26c7080b632f416072f53a90b9a52981c" resolved "https://registry.yarnpkg.com/bunyan/-/bunyan-1.8.10.tgz#201fedd26c7080b632f416072f53a90b9a52981c"
@ -1590,6 +1602,13 @@ crc@3.4.4, crc@^3.4.0:
version "3.4.4" version "3.4.4"
resolved "https://registry.yarnpkg.com/crc/-/crc-3.4.4.tgz#9da1e980e3bd44fc5c93bf5ab3da3378d85e466b" resolved "https://registry.yarnpkg.com/crc/-/crc-3.4.4.tgz#9da1e980e3bd44fc5c93bf5ab3da3378d85e466b"
cron-parser@^2.4.1:
version "2.4.1"
resolved "https://registry.yarnpkg.com/cron-parser/-/cron-parser-2.4.1.tgz#022befce1af293e4d3144ff04c2cbd2edb491271"
dependencies:
is-nan "^1.2.1"
moment-timezone "^0.5.0"
cron@1.2.1, cron@~1.2.1: cron@1.2.1, cron@~1.2.1:
version "1.2.1" version "1.2.1"
resolved "https://registry.yarnpkg.com/cron/-/cron-1.2.1.tgz#3a86c09b41b8f261ac863a7cc85ea4735857eab2" resolved "https://registry.yarnpkg.com/cron/-/cron-1.2.1.tgz#3a86c09b41b8f261ac863a7cc85ea4735857eab2"
@ -1702,6 +1721,10 @@ debug@~2.2.0:
dependencies: dependencies:
ms "0.7.1" ms "0.7.1"
debuglog@^1.0.0:
version "1.0.1"
resolved "https://registry.yarnpkg.com/debuglog/-/debuglog-1.0.1.tgz#aa24ffb9ac3df9a2351837cfb2d279360cd78492"
decamelize@^1.0.0, decamelize@^1.1.1, decamelize@^1.1.2: decamelize@^1.0.0, decamelize@^1.1.1, decamelize@^1.1.2:
version "1.2.0" version "1.2.0"
resolved "https://registry.yarnpkg.com/decamelize/-/decamelize-1.2.0.tgz#f6534d15148269b20352e7bee26f501f9a191290" resolved "https://registry.yarnpkg.com/decamelize/-/decamelize-1.2.0.tgz#f6534d15148269b20352e7bee26f501f9a191290"
@ -1732,6 +1755,13 @@ deferred-leveldown@~1.2.1:
dependencies: dependencies:
abstract-leveldown "~2.4.0" abstract-leveldown "~2.4.0"
define-properties@^1.1.1:
version "1.1.2"
resolved "https://registry.yarnpkg.com/define-properties/-/define-properties-1.1.2.tgz#83a73f2fea569898fb737193c8f873caf6d45c94"
dependencies:
foreach "^2.0.5"
object-keys "^1.0.8"
del@^2.0.2: del@^2.0.2:
version "2.2.2" version "2.2.2"
resolved "https://registry.yarnpkg.com/del/-/del-2.2.2.tgz#c12c981d067846c84bcaf862cff930d907ffd1a8" resolved "https://registry.yarnpkg.com/del/-/del-2.2.2.tgz#c12c981d067846c84bcaf862cff930d907ffd1a8"
@ -2573,6 +2603,10 @@ for-own@^0.1.4:
dependencies: dependencies:
for-in "^1.0.1" for-in "^1.0.1"
foreach@^2.0.5:
version "2.0.5"
resolved "https://registry.yarnpkg.com/foreach/-/foreach-2.0.5.tgz#0bee005018aeb260d0a3af3ae658dd0136ec1b99"
forever-agent@~0.6.1: forever-agent@~0.6.1:
version "0.6.1" version "0.6.1"
resolved "https://registry.yarnpkg.com/forever-agent/-/forever-agent-0.6.1.tgz#fbc71f0c41adeb37f96c577ad1ed42d8fdacca91" resolved "https://registry.yarnpkg.com/forever-agent/-/forever-agent-0.6.1.tgz#fbc71f0c41adeb37f96c577ad1ed42d8fdacca91"
@ -3207,6 +3241,19 @@ invert-kv@^1.0.0:
version "1.0.0" version "1.0.0"
resolved "https://registry.yarnpkg.com/invert-kv/-/invert-kv-1.0.0.tgz#104a8e4aaca6d3d8cd157a8ef8bfab2d7a3ffdb6" resolved "https://registry.yarnpkg.com/invert-kv/-/invert-kv-1.0.0.tgz#104a8e4aaca6d3d8cd157a8ef8bfab2d7a3ffdb6"
ioredis@^3.1.1:
version "3.1.2"
resolved "https://registry.yarnpkg.com/ioredis/-/ioredis-3.1.2.tgz#2579e3eba6dc490f68f14c7b51346281332b467b"
dependencies:
bluebird "^3.3.4"
cluster-key-slot "^1.0.6"
debug "^2.2.0"
denque "^1.1.0"
flexbuffer "0.0.6"
lodash "^4.8.2"
redis-commands "^1.2.0"
redis-parser "^2.4.0"
ioredis@~3.1.1: ioredis@~3.1.1:
version "3.1.1" version "3.1.1"
resolved "https://registry.yarnpkg.com/ioredis/-/ioredis-3.1.1.tgz#cc2f1d3232b8c95cc153046bce168f2baa1186e8" resolved "https://registry.yarnpkg.com/ioredis/-/ioredis-3.1.1.tgz#cc2f1d3232b8c95cc153046bce168f2baa1186e8"
@ -3320,6 +3367,12 @@ is-glob@^2.0.0, is-glob@^2.0.1:
dependencies: dependencies:
is-extglob "^1.0.0" is-extglob "^1.0.0"
is-nan@^1.2.1:
version "1.2.1"
resolved "https://registry.yarnpkg.com/is-nan/-/is-nan-1.2.1.tgz#9faf65b6fb6db24b7f5c0628475ea71f988401e2"
dependencies:
define-properties "^1.1.1"
is-npm@^1.0.0: is-npm@^1.0.0:
version "1.0.0" version "1.0.0"
resolved "https://registry.yarnpkg.com/is-npm/-/is-npm-1.0.0.tgz#f2fb63a65e4905b406c86072765a1a4dc793b9f4" resolved "https://registry.yarnpkg.com/is-npm/-/is-npm-1.0.0.tgz#f2fb63a65e4905b406c86072765a1a4dc793b9f4"
@ -4574,7 +4627,7 @@ mkdirp@0.5.1, "mkdirp@>=0.5 0", mkdirp@^0.5.0, mkdirp@^0.5.1, mkdirp@~0.5.1:
dependencies: dependencies:
minimist "0.0.8" minimist "0.0.8"
moment-timezone@^0.5.4, moment-timezone@^0.5.x, moment-timezone@~0.5.13: moment-timezone@^0.5.0, moment-timezone@^0.5.4, moment-timezone@^0.5.x, moment-timezone@~0.5.13:
version "0.5.13" version "0.5.13"
resolved "https://registry.yarnpkg.com/moment-timezone/-/moment-timezone-0.5.13.tgz#99ce5c7d827262eb0f1f702044177f60745d7b90" resolved "https://registry.yarnpkg.com/moment-timezone/-/moment-timezone-0.5.13.tgz#99ce5c7d827262eb0f1f702044177f60745d7b90"
dependencies: dependencies:
@ -4877,6 +4930,10 @@ object-component@0.0.3:
version "0.0.3" version "0.0.3"
resolved "https://registry.yarnpkg.com/object-component/-/object-component-0.0.3.tgz#f0c69aa50efc95b866c186f400a33769cb2f1291" resolved "https://registry.yarnpkg.com/object-component/-/object-component-0.0.3.tgz#f0c69aa50efc95b866c186f400a33769cb2f1291"
object-keys@^1.0.8:
version "1.0.11"
resolved "https://registry.yarnpkg.com/object-keys/-/object-keys-1.0.11.tgz#c54601778ad560f1142ce0e01bcca8b56d13426d"
object.omit@^2.0.0: object.omit@^2.0.0:
version "2.0.1" version "2.0.1"
resolved "https://registry.yarnpkg.com/object.omit/-/object.omit-2.0.1.tgz#1a9c744829f39dbb858c76ca3579ae2a54ebd1fa" resolved "https://registry.yarnpkg.com/object.omit/-/object.omit-2.0.1.tgz#1a9c744829f39dbb858c76ca3579ae2a54ebd1fa"
@ -7045,7 +7102,7 @@ uuid@^2.0.1, uuid@^2.0.2:
version "2.0.3" version "2.0.3"
resolved "https://registry.yarnpkg.com/uuid/-/uuid-2.0.3.tgz#67e2e863797215530dff318e5bf9dcebfd47b21a" resolved "https://registry.yarnpkg.com/uuid/-/uuid-2.0.3.tgz#67e2e863797215530dff318e5bf9dcebfd47b21a"
uuid@^3.0.0, uuid@^3.0.1: uuid@^3.0.0, uuid@^3.0.1, uuid@^3.1.0:
version "3.1.0" version "3.1.0"
resolved "https://registry.yarnpkg.com/uuid/-/uuid-3.1.0.tgz#3dd3d3e790abc24d7b0d3a034ffababe28ebbc04" resolved "https://registry.yarnpkg.com/uuid/-/uuid-3.1.0.tgz#3dd3d3e790abc24d7b0d3a034ffababe28ebbc04"