liz/wikijs-fork
1
0
Fork 0
Code Issues Pull Requests Packages Projects Releases Wiki Activity

fix: telemetry

Browse Source
This commit is contained in:
Nick
2019-02-02 01:17:09 -05:00
parent 3ede499732
commit 44e87cb60b
12 changed files with 139 additions and 104 deletions
Download Patch File Download Diff File Expand all files Collapse all files

1
CHANGELOG.md
View File

@@ -12,6 +12,7 @@ This project adheres to [Semantic Versioning](http://semver.org/).
### Fixed
- Fixed root admin refresh token fail
- Fixed error page metadata title warning
- Fixed telemetry
### Changed
- Moved Insert Media button in Markdown editor

6
client/components/admin/admin-utilities.vue
View File

@@ -95,7 +95,7 @@
v-card-text
v-subheader What is telemetry?
.body-1.pl-3 Telemetry allows the developers of Wiki.js to improve the software by collecting basic anonymized data about its usage and the host info. #[br] This is entirely optional and #[strong absolutely no] private data (such as content or personal data) is collected.
.body-1.pt-3.pl-3 For maximum privacy, a random client ID is generated every time Wiki.js is started. This ID is used to group requests together while keeping complete anonymity.
.body-1.pt-3.pl-3 For maximum privacy, a random client ID is generated during setup. This ID is used to group requests together while keeping complete anonymity. You can reset and generate a new one below at any time.
v-divider.my-3
v-subheader What is collected?
.body-1.pl-3 When telemetry is enabled, only the following data is transmitted:
@@ -105,13 +105,13 @@
v-list-tile-content: v-list-tile-title.caption Version of Wiki.js installed
v-list-tile
v-list-tile-avatar: v-icon info_outline
v-list-tile-content: v-list-tile-title.caption Basic OS information (version, CPU cores, RAM available)
v-list-tile-content: v-list-tile-title.caption Basic OS information (platform, CPU cores count, DB type)
v-list-tile
v-list-tile-avatar: v-icon info_outline
v-list-tile-content: v-list-tile-title.caption Crash debug data
v-list-tile
v-list-tile-avatar: v-icon info_outline
v-list-tile-content: v-list-tile-title.caption Setup analytics (step reached)
v-list-tile-content: v-list-tile-title.caption Setup analytics (installation checkpoint reached)
.body-2.pl-3
v-divider.my-3
v-subheader Settings

6
client/components/setup.vue
View File

@@ -3,7 +3,7 @@
v-toolbar(color='blue darken-2', dark, app, clipped-left, fixed, flat)
v-spacer
v-toolbar-title
span.subheading Wiki.js Setup
span.subheading Wiki.js {{wikiVersion}}
v-spacer
v-content.white
v-progress-linear.ma-0(indeterminate, height='4', :active='loading')
@@ -193,9 +193,7 @@
v-btn(color='success', @click='finish', v-if='loading || final.ok', :disabled='loading') Continue
v-footer.pa-3(app, absolute, color='grey darken-3', height='auto')
.caption.grey--text Wiki.js
v-spacer
.caption.grey--text(v-if='conf.telemetry') Telemetry Client ID: {{telemetryId}}
.caption.grey--text Wiki.js {{wikiVersion}} Installation Wizard
</template>
<script>

44
package.json
View File

@@ -39,15 +39,15 @@
"node": ">=10.12"
},
"dependencies": {
"@bugsnag/js": "5.2.0",
"apollo-fetch": "0.7.0",
"apollo-server": "2.3.2",
"apollo-server-express": "2.3.2",
"apollo-server": "2.3.3",
"apollo-server-express": "2.3.3",
"auto-load": "3.0.4",
"axios": "0.18.0",
"bcryptjs-then": "1.0.1",
"bluebird": "3.5.3",
"body-parser": "1.18.3",
"bugsnag": "2.4.3",
"bull": "3.6.0",
"chalk": "2.4.2",
"cheerio": "1.0.0-rc.2",
@@ -62,7 +62,7 @@
"dependency-graph": "0.8.0",
"diff": "4.0.1",
"diff2html": "2.7.0",
"dotize": "^0.2.0",
"dotize": "^0.3.0",
"execa": "1.0.0",
"express": "4.16.4",
"express-brute": "1.0.1",
@@ -77,7 +77,7 @@
"graphql-list-fields": "2.0.2",
"graphql-subscriptions": "1.0.0",
"graphql-tools": "4.0.4",
"highlight.js": "9.13.1",
"highlight.js": "9.14.2",
"i18next": "14.0.1",
"i18next-express-middleware": "1.7.1",
"i18next-localstorage-cache": "1.1.1",
@@ -157,13 +157,13 @@
"simple-git": "1.107.0",
"sqlite3": "4.0.6",
"subscriptions-transport-ws": "0.9.15",
"twemoji": "11.2.0",
"twemoji": "11.3.0",
"uslug": "1.0.4",
"uuid": "3.3.2",
"validate.js": "0.12.0",
"validator": "10.11.0",
"validator-as-promised": "1.0.2",
"winston": "3.1.0",
"winston": "3.2.1",
"yargs": "12.0.5"
},
"devDependencies": {
@@ -184,14 +184,14 @@
"animated-number-vue": "0.1.4",
"apollo-cache-inmemory": "1.4.2",
"apollo-client": "2.4.12",
"apollo-link": "1.2.6",
"apollo-link-batch-http": "1.2.6",
"apollo-link-error": "1.1.5",
"apollo-link-http": "1.5.9",
"apollo-link": "1.2.8",
"apollo-link-batch-http": "1.2.8",
"apollo-link-error": "1.1.7",
"apollo-link-http": "1.5.11",
"apollo-link-persisted-queries": "0.2.2",
"apollo-link-ws": "1.0.12",
"apollo-link-ws": "1.0.14",
"apollo-utilities": "1.1.2",
"autoprefixer": "9.4.6",
"autoprefixer": "9.4.7",
"babel-eslint": "10.0.1",
"babel-jest": "24.0.0",
"babel-loader": "^8.0.5",
@@ -208,17 +208,17 @@
"cssnano": "4.1.8",
"duplicate-package-checker-webpack-plugin": "3.0.0",
"epic-spinners": "1.0.4",
"eslint": "5.12.1",
"eslint": "5.13.0",
"eslint-config-requarks": "1.0.7",
"eslint-config-standard": "12.0.0",
"eslint-plugin-import": "2.15.0",
"eslint-plugin-import": "2.16.0",
"eslint-plugin-node": "8.0.1",
"eslint-plugin-promise": "4.0.1",
"eslint-plugin-standard": "4.0.0",
"eslint-plugin-vue": "5.1.0",
"file-loader": "3.0.1",
"filesize.js": "1.0.2",
"grapesjs": "0.14.50",
"grapesjs": "0.14.52",
"graphiql": "0.12.0",
"graphql-persisted-document-loader": "1.0.1",
"graphql-tag": "^2.10.1",
@@ -256,7 +256,7 @@
"stylus-loader": "3.0.2",
"twemoji-awesome": "1.0.6",
"url-loader": "1.1.2",
"vee-validate": "2.1.5",
"vee-validate": "2.1.7",
"velocity-animate": "1.5.2",
"viz.js": "2.1.2",
"vue": "2.5.22",
@@ -265,7 +265,7 @@
"vue-clipboards": "1.2.4",
"vue-codemirror": "4.0.6",
"vue-hot-reload-api": "2.3.1",
"vue-loader": "15.6.1",
"vue-loader": "15.6.2",
"vue-material-design-icons": "2.6.0",
"vue-moment": "4.0.0",
"vue-router": "3.0.2",
@@ -275,8 +275,8 @@
"vue-tour": "1.1.0",
"vue2-animate": "2.1.0",
"vuedraggable": "2.17.0",
"vuescroll": "4.9.6",
"vuetify": "1.4.3",
"vuescroll": "4.10.2",
"vuetify": "1.4.4",
"vuex": "3.1.0",
"vuex-pathify": "1.1.3",
"vuex-persistedstate": "2.5.4",
@@ -286,10 +286,10 @@
"webpack-dev-middleware": "3.5.1",
"webpack-hot-middleware": "2.24.3",
"webpack-merge": "4.2.1",
"webpack-subresource-integrity": "1.3.1",
"webpack-subresource-integrity": "1.3.2",
"whatwg-fetch": "3.0.0",
"write-file-webpack-plugin": "4.5.0",
"xterm": "3.10.1",
"xterm": "3.11.0",
"zxcvbn": "4.4.2"
},
"browserslist": [

2
server/app/data.yml
View File

@@ -86,5 +86,5 @@ telemetry:
BUGSNAG_ID: 'bb4b324d0675bcbba10025617fd2cec8'
BUGSNAG_REMOTE: 'https://notify.bugsnag.com'
GA_ID: 'UA-9094100-7'
GA_REMOTE: 'https://www.google-analytics.com/collect'
GA_REMOTE: 'https://www.google-analytics.com/batch'
# ---------------------------------

3
server/core/db.js
View File

@@ -90,9 +90,6 @@ module.exports = {
const models = autoload(path.join(WIKI.SERVERPATH, 'models'))
// Set init tasks
console.info(migrationSource)
let initTasks = {
// -> Migrate DB Schemas
async syncSchemas() {

16
server/core/kernel.js
View File

@@ -40,6 +40,7 @@ module.exports = {
WIKI.logger.info('Starting setup wizard...')
require('../setup')()
} else {
await this.initTelemetry()
await require('../master')()
this.postBootMaster()
}
@@ -61,5 +62,20 @@ module.exports = {
await WIKI.auth.activateStrategies()
await WIKI.queue.start()
},
/**
* Init Telemetry
*/
async initTelemetry() {
require('./telemetry').init()
process.on('unhandledRejection', (err) => {
WIKI.logger.warn(err)
WIKI.telemetry.sendError(err)
})
process.on('uncaughtException', (err) => {
WIKI.logger.warn(err)
WIKI.telemetry.sendError(err)
})
}
}

2
server/core/system.js
View File

@@ -35,8 +35,6 @@ module.exports = {
* @param {Object} opts Options object
*/
async upgradeFromMongo (opts) {
WIKI.telemetry.sendEvent('setup', 'upgradeFromMongo')
WIKI.logger.info('Upgrading from MongoDB...')
let mongo = require('mongodb').MongoClient

92
server/core/telemetry.js
View File

@@ -1,57 +1,89 @@
const _ = require('lodash')
const axios = require('axios')
const bugsnag = require('bugsnag')
const path = require('path')
const bugsnag = require('@bugsnag/node')
const uuid = require('uuid/v4')
const qs = require('querystring')
const os = require('os')
/* global WIKI */
module.exports = {
cid: '',
client: null,
enabled: false,
init() {
this.cid = uuid()
bugsnag.register(WIKI.data.telemetry.BUGSNAG_ID, {
this.client = bugsnag({
apiKey: WIKI.data.telemetry.BUGSNAG_ID,
appType: 'server',
appVersion: WIKI.version,
autoNotify: false,
hostname: this.cid,
collectUserIp: false,
hostname: _.get(WIKI.config, 'telemetry.clientId', uuid()),
notifyReleaseStages: ['production'],
packageJSON: path.join(WIKI.ROOTPATH, 'package.json'),
releaseStage: WIKI.IS_DEBUG ? 'development' : 'production',
projectRoot: WIKI.ROOTPATH,
useSSL: true
})
bugsnag.onBeforeNotify((notification, originalError) => {
if (!this.enabled) { return false }
logger: null,
beforeSend: (report) => {
if (!WIKI.telemetry.enabled) { return false }
}
})
WIKI.telemetry = this
if (_.get(WIKI.config, 'logging.telemetry', false) === true) {
if (_.get(WIKI.config, 'telemetry.isEnabled', false) === true) {
this.enabled = true
this.sendOSInfo()
}
return this
},
sendOSInfo() {
this.sendBatchEvents([
{
eventCategory: 'node-version',
eventAction: process.version
},
{
eventCategory: 'os-platform',
eventAction: os.platform()
},
{
eventCategory: 'cpu-cores',
eventAction: os.cpus().length
},
{
eventCategory: 'db-type',
eventAction: WIKI.config.db.type
}
])
},
sendError(err) {
bugsnag.notify(err, { userId: this.cid })
this.client.notify(err)
},
sendEvent(eventCategory, eventAction, eventLabel) {
if (!this.enabled) { return false }
this.sendBatchEvents([{
eventCategory,
eventAction,
eventLabel
}])
},
sendBatchEvents(events) {
if (!this.enabled || WIKI.IS_DEBUG) { return false }
axios({
method: 'post',
url: WIKI.data.telemetry.GA_REMOTE,
headers: {
'Content-type': 'application/x-www-form-urlencoded'
'Content-type': 'text/plain'
},
params: {
v: 1, // API version
tid: WIKI.data.telemetry.GA_ID, // Tracking ID
aip: 1, // Anonymize IP
ds: 'server', // Data source
cid: this.cid, // Client ID
t: 'event', // Hit Type
ec: eventCategory, // Event Category
ea: eventAction, // Event Action
el: eventLabel // Event Label
}
data: events.map(ev => {
return qs.stringify({
v: 1, // API version
tid: WIKI.data.telemetry.GA_ID, // Tracking ID
aip: 1, // Anonymize IP
ds: 'server', // Data source
cid: WIKI.telemetry.cid, // Client ID
t: 'event', // Hit Type
ec: ev.eventCategory, // Event Category
ea: ev.eventAction, // Event Action
el: ev.eventLabel // Event Label
})
}).join('\n')
}).then(resp => {
if (resp.status !== 200) {
WIKI.logger.warn('Unable to send analytics telemetry request.')
@@ -59,5 +91,9 @@ module.exports = {
}, err => {
WIKI.logger.warn('Unable to send analytics telemetry request.')
})
},
generateClientId() {
_.set(WIKI.config, 'telemetry.clientId', uuid())
return WIKI.config.telemetry.clientId
}
}

15
server/index.js
View File

@@ -24,21 +24,6 @@ WIKI.configSvc.init()
WIKI.logger = require('./core/logger').init('MASTER')
// ----------------------------------------
// Init Telemetry
// ----------------------------------------
WIKI.telemetry = require('./core/telemetry').init()
process.on('unhandledRejection', (err) => {
WIKI.logger.warn(err)
WIKI.telemetry.sendError(err)
})
process.on('uncaughtException', (err) => {
WIKI.logger.warn(err)
WIKI.telemetry.sendError(err)
})
// ----------------------------------------
// Start Kernel
// ----------------------------------------

56
server/setup.js
View File

@@ -71,38 +71,14 @@ module.exports = () => {
app.get('*', async (req, res) => {
let packageObj = await fs.readJson(path.join(WIKI.ROOTPATH, 'package.json'))
res.render('setup', {
packageObj,
telemetryClientID: WIKI.telemetry.cid
})
res.render('setup', { packageObj })
})
/**
* Finalize
*/
app.post('/finalize', async (req, res) => {
WIKI.telemetry.sendEvent('setup', 'finalize')
try {
// Basic checks
if (!semver.satisfies(process.version, '>=10.14')) {
throw new Error('Node.js 10.14.x or later required!')
}
// Upgrade from WIKI.js 1.x?
if (req.body.upgrade) {
await WIKI.system.upgradeFromMongo({
mongoCnStr: cfgHelper.parseConfigValue(req.body.upgMongo)
})
}
// Create directory structure
WIKI.logger.info('Creating data directories...')
const dataPath = path.join(process.cwd(), 'data')
await fs.ensureDir(dataPath)
await fs.emptyDir(path.join(dataPath, 'cache'))
await fs.ensureDir(path.join(dataPath, 'uploads'))
// Set config
_.set(WIKI.config, 'auth', {
audience: 'urn:wiki.js',
@@ -149,7 +125,7 @@ module.exports = () => {
_.set(WIKI.config, 'sessionSecret', (await crypto.randomBytesAsync(32)).toString('hex'))
_.set(WIKI.config, 'telemetry', {
isEnabled: req.body.telemetry === true,
clientId: WIKI.telemetry.cid
clientId: uuid()
})
_.set(WIKI.config, 'theming', {
theme: 'default',
@@ -157,6 +133,30 @@ module.exports = () => {
})
_.set(WIKI.config, 'title', 'Wiki.js')
// Init Telemetry
WIKI.kernel.initTelemetry()
WIKI.telemetry.sendEvent('setup', 'install-start')
// Basic checks
if (!semver.satisfies(process.version, '>=10.14')) {
throw new Error('Node.js 10.14.x or later required!')
}
// Upgrade from WIKI.js 1.x?
if (req.body.upgrade) {
WIKI.telemetry.sendEvent('setup', 'install-mongo-upgrade')
await WIKI.system.upgradeFromMongo({
mongoCnStr: cfgHelper.parseConfigValue(req.body.upgMongo)
})
}
// Create directory structure
WIKI.logger.info('Creating data directories...')
const dataPath = path.join(process.cwd(), 'data')
await fs.ensureDir(dataPath)
await fs.emptyDir(path.join(dataPath, 'cache'))
await fs.ensureDir(path.join(dataPath, 'uploads'))
// Generate certificates
WIKI.logger.info('Generating certificates...')
const certs = crypto.generateKeyPairSync('rsa', {
@@ -244,6 +244,8 @@ module.exports = () => {
await WIKI.models.searchEngines.refreshSearchEnginesFromDisk()
await WIKI.models.searchEngines.query().patch({ isEnabled: true }).where('key', 'db')
WIKI.telemetry.sendEvent('setup', 'install-loadedmodules')
// Load storage targets
await WIKI.models.storage.refreshTargetsFromDisk()
@@ -305,6 +307,7 @@ module.exports = () => {
})
WIKI.logger.info('Setup is complete!')
WIKI.telemetry.sendEvent('setup', 'install-completed')
res.json({
ok: true,
redirectPath: '/',
@@ -321,6 +324,7 @@ module.exports = () => {
}, 1000)
})
} catch (err) {
WIKI.telemetry.sendError(err)
res.json({ ok: false, error: err.message })
}
})

BIN
yarn.lock
View File

Binary file not shown.