feat: rendering modules + admin rendering

server/core/kernel.js

@@ -51,6 +51,7 @@ module.exports = {
   async postBootMaster() {
     await WIKI.models.authentication.refreshStrategiesFromDisk()
     await WIKI.models.editors.refreshEditorsFromDisk()
+    await WIKI.models.renderers.refreshRenderersFromDisk()
     await WIKI.models.storage.refreshTargetsFromDisk()
 
     await WIKI.auth.activateStrategies()

server/db/migrations/2.0.0.js

@@ -96,7 +96,7 @@ exports.up = knex => {
       table.string('createdAt').notNullable()
       table.string('updatedAt').notNullable()
     })
-    // STORAGE -----------------------------
+    // RENDERERS ---------------------------
     .createTable('renderers', table => {
       table.increments('id').primary()
       table.string('key').notNullable().unique()
@@ -199,14 +199,19 @@ exports.up = knex => {
 exports.down = knex => {
   return knex.schema
     .dropTableIfExists('userGroups')
+    .dropTableIfExists('pageHistoryTags')
+    .dropTableIfExists('pageHistory')
     .dropTableIfExists('pageTags')
     .dropTableIfExists('assets')
     .dropTableIfExists('assetFolders')
     .dropTableIfExists('comments')
+    .dropTableIfExists('editors')
     .dropTableIfExists('groups')
     .dropTableIfExists('locales')
     .dropTableIfExists('pages')
+    .dropTableIfExists('renderers')
     .dropTableIfExists('settings')
+    .dropTableIfExists('storage')
     .dropTableIfExists('tags')
     .dropTableIfExists('users')
 }

server/graph/resolvers/rendering.js

@@ -0,0 +1,58 @@
+const _ = require('lodash')
+const graphHelper = require('../../helpers/graph')
+
+/* global WIKI */
+
+module.exports = {
+  Query: {
+    async rendering() { return {} }
+  },
+  Mutation: {
+    async rendering() { return {} }
+  },
+  RenderingQuery: {
+    async renderers(obj, args, context, info) {
+      let renderers = await WIKI.models.renderers.getRenderers()
+      renderers = renderers.map(rdr => {
+        const rendererInfo = _.find(WIKI.data.renderers, ['key', rdr.key]) || {}
+        return {
+          ...rendererInfo,
+          ...rdr,
+          config: _.sortBy(_.transform(rdr.config, (res, value, key) => {
+            const configData = _.get(rendererInfo.props, key, {})
+            res.push({
+              key,
+              value: JSON.stringify({
+                ...configData,
+                value
+              })
+            })
+          }, []), 'key')
+        }
+      })
+      if (args.filter) { renderers = graphHelper.filter(renderers, args.filter) }
+      if (args.orderBy) { renderers = graphHelper.orderBy(renderers, args.orderBy) }
+      return renderers
+    }
+  },
+  RenderingMutation: {
+    async updateRenderers(obj, args, context) {
+      try {
+        for (let rdr of args.renderers) {
+          await WIKI.models.storage.query().patch({
+            isEnabled: rdr.isEnabled,
+            config: _.reduce(rdr.config, (result, value, key) => {
+              _.set(result, `${value.key}`, value.value)
+              return result
+            }, {})
+          }).where('key', rdr.key)
+        }
+        return {
+          responseResult: graphHelper.generateSuccess('Renderers updated successfully')
+        }
+      } catch (err) {
+        return graphHelper.generateError(err)
+      }
+    }
+  }
+}

server/graph/schemas/rendering.graphql

@@ -0,0 +1,55 @@
+# ===============================================
+# RENDERING
+# ===============================================
+
+extend type Query {
+  rendering: RenderingQuery
+}
+
+extend type Mutation {
+  rendering: RenderingMutation
+}
+
+# -----------------------------------------------
+# QUERIES
+# -----------------------------------------------
+
+type RenderingQuery {
+  renderers(
+    filter: String
+    orderBy: String
+  ): [Renderer]
+}
+
+# -----------------------------------------------
+# MUTATIONS
+# -----------------------------------------------
+
+type RenderingMutation {
+  updateRenderers(
+    renderers: [RendererInput]
+  ): DefaultResponse
+}
+
+# -----------------------------------------------
+# TYPES
+# -----------------------------------------------
+
+type Renderer {
+  isEnabled: Boolean!
+  key: String!
+  title: String!
+  description: String
+  icon: String
+  dependsOn: String
+  input: String
+  output: String
+  config: [KeyValuePair]
+}
+
+input RendererInput {
+  isEnabled: Boolean!
+  key: String!
+  mode: String!
+  config: [KeyValuePairInput]
+}

server/models/renderers.js

@@ -0,0 +1,108 @@
+const Model = require('objection').Model
+const path = require('path')
+const fs = require('fs-extra')
+const _ = require('lodash')
+const yaml = require('js-yaml')
+const commonHelper = require('../helpers/common')
+
+/* global WIKI */
+
+/**
+ * Renderer model
+ */
+module.exports = class Renderer extends Model {
+  static get tableName() { return 'renderers' }
+
+  static get jsonSchema () {
+    return {
+      type: 'object',
+      required: ['key', 'isEnabled'],
+
+      properties: {
+        id: {type: 'integer'},
+        key: {type: 'string'},
+        isEnabled: {type: 'boolean'},
+        config: {type: 'object'}
+      }
+    }
+  }
+
+  static async getRenderers() {
+    return WIKI.models.renderers.query()
+  }
+
+  static async refreshRenderersFromDisk() {
+    let trx
+    try {
+      const dbRenderers = await WIKI.models.renderers.query()
+
+      // -> Fetch definitions from disk
+      const rendererDirs = await fs.readdir(path.join(WIKI.SERVERPATH, 'modules/renderer'))
+      let diskRenderers = []
+      for (let dir of rendererDirs) {
+        const def = await fs.readFile(path.join(WIKI.SERVERPATH, 'modules/renderer', dir, 'definition.yml'), 'utf8')
+        diskRenderers.push(yaml.safeLoad(def))
+      }
+      WIKI.data.renderers = diskRenderers.map(renderer => ({
+        ...renderer,
+        props: commonHelper.parseModuleProps(renderer.props)
+      }))
+
+      // -> Insert new Renderers
+      let newRenderers = []
+      for (let renderer of WIKI.data.renderers) {
+        if (!_.some(dbRenderers, ['key', renderer.key])) {
+          newRenderers.push({
+            key: renderer.key,
+            isEnabled: _.get(renderer, 'enabledDefault', true),
+            config: _.transform(renderer.props, (result, value, key) => {
+              _.set(result, key, value.default)
+              return result
+            }, {})
+          })
+        } else {
+          const rendererConfig = _.get(_.find(dbRenderers, ['key', renderer.key]), 'config', {})
+          await WIKI.models.renderers.query().patch({
+            config: _.transform(renderer.props, (result, value, key) => {
+              if (!_.has(result, key)) {
+                _.set(result, key, value.default)
+              }
+              return result
+            }, rendererConfig)
+          }).where('key', renderer.key)
+        }
+      }
+      if (newRenderers.length > 0) {
+        trx = await WIKI.models.Objection.transaction.start(WIKI.models.knex)
+        for (let renderer of newRenderers) {
+          await WIKI.models.renderers.query(trx).insert(renderer)
+        }
+        await trx.commit()
+        WIKI.logger.info(`Loaded ${newRenderers.length} new renderers: [ OK ]`)
+      } else {
+        WIKI.logger.info(`No new renderers found: [ SKIPPED ]`)
+      }
+    } catch (err) {
+      WIKI.logger.error(`Failed to scan or load new renderers: [ FAILED ]`)
+      WIKI.logger.error(err)
+      if (trx) {
+        trx.rollback()
+      }
+    }
+  }
+
+  static async pageEvent({ event, page }) {
+    const targets = await WIKI.models.storage.query().where('isEnabled', true)
+    if (targets && targets.length > 0) {
+      _.forEach(targets, target => {
+        WIKI.queue.job.syncStorage.add({
+          event,
+          target,
+          page
+        }, {
+          removeOnComplete: true
+        })
+      })
+    }
+  }
+}

server/modules/renderer/html-asciinema/definition.yml

@@ -1,7 +1,8 @@
-key: markdownAbbr
-title: Abbreviations
-description: Parse abbreviations into abbr tags
+key: htmlAsciinema
+title: Asciinema
+description: Embed asciinema players from compatible links
 author: requarks.io
-dependsOn:
-  - markdownCore
+icon: subtitles
+enabledDefault: false
+dependsOn: htmlCore
 props: {}

server/modules/renderer/html-blockquotes/definition.yml

@@ -1,7 +1,8 @@
-key: markdownAbbr
-title: Abbreviations
-description: Parse abbreviations into abbr tags
+key: htmlBlockquotes
+title: Blockquotes
+description: Embed audio players for audio content
 author: requarks.io
-dependsOn:
-  - markdownCore
+icon: insert_comment
+enabledDefault: true
+dependsOn: htmlCore
 props: {}

server/modules/renderer/html-core/definition.yml

@@ -0,0 +1,8 @@
+key: htmlCore
+title: Core
+description: Basic HTML Parser
+author: requarks.io
+input: html
+output: html
+icon: crop_free
+props: {}

server/modules/renderer/html-core/renderer.js

@@ -0,0 +1 @@
+module.exports = {}

server/modules/renderer/html-mathjax/definition.yml

@@ -1,7 +1,8 @@
-key: markdownAbbr
-title: Abbreviations
-description: Parse abbreviations into abbr tags
+key: htmlMathjax
+title: Mathjax Processor
+description: TeX/MathML Math Equations Parser
 author: requarks.io
-dependsOn:
-  - markdownCore
+icon: functions
+enabledDefault: false
+dependsOn: htmlCore
 props: {}

server/modules/renderer/html-mediaplayers/definition.yml

@@ -1,7 +1,8 @@
-key: markdownAbbr
-title: Abbreviations
-description: Parse abbreviations into abbr tags
+key: htmlMedia
+title: Media Players
+description: Embed players such as Youtube, Vimeo, Soundcloud, etc.
 author: requarks.io
-dependsOn:
-  - markdownCore
+icon: subscriptions
+enabledDefault: false
+dependsOn: htmlCore
 props: {}

server/modules/renderer/html-security/definition.yml

@@ -1,7 +1,8 @@
-key: markdownAbbr
-title: Abbreviations
-description: Parse abbreviations into abbr tags
+key: htmlSecurity
+title: Security
+description: Filter and strips potentially dangerous content
 author: requarks.io
-dependsOn:
-  - markdownCore
+icon: whatshot
+enabledDefault: true
+dependsOn: htmlCore
 props: {}

server/modules/renderer/markdown-abbr/definition.yml

@@ -2,6 +2,7 @@ key: markdownAbbr
 title: Abbreviations
 description: Parse abbreviations into abbr tags
 author: requarks.io
-dependsOn:
-  - markdownCore
+icon: text_format
+enabledDefault: true
+dependsOn: markdownCore
 props: {}

server/modules/renderer/markdown-core/definition.yml

@@ -2,7 +2,9 @@ key: markdownCore
 title: Core
 description: Basic Markdown Parser
 author: requarks.io
-dependsOn: []
+input: markdown
+output: html
+icon: crop_free
 props:
   linkify:
     type: Boolean

server/modules/renderer/markdown-emoji/definition.yml

@@ -2,6 +2,7 @@ key: markdownEmoji
 title: Emoji
 description: Convert tags to emojis
 author: requarks.io
-dependsOn:
-  - markdownCore
+icon: tag_faces
+enabledDefault: true
+dependsOn: markdownCore
 props: {}

server/modules/renderer/markdown-expandtabs/definition.yml

@@ -2,6 +2,7 @@ key: markdownExpandtabs
 title: Expand Tabs
 description: Replace tabs with spaces in code blocks
 author: requarks.io
-dependsOn:
-  - markdownCore
+icon: space_bar
+enabledDefault: true
+dependsOn: markdownCore
 props: {}

server/modules/renderer/markdown-footnotes/definition.yml

@@ -2,6 +2,7 @@ key: markdownFootnotes
 title: Footnotes
 description: Parse footnotes references
 author: requarks.io
-dependsOn:
-  - markdownCore
+icon: low_priority
+enabledDefault: true
+dependsOn: markdownCore
 props: {}

server/modules/renderer/markdown-mathjax/definition.yml

@@ -2,6 +2,7 @@ key: markdownMathjax
 title: Mathjax Pre-Processor
 description: Pre-parse TeX blocks for Mathjax
 author: requarks.io
-dependsOn:
-  - markdownCore
+icon: functions
+enabledDefault: false
+dependsOn: markdownCore
 props: {}

server/modules/renderer/markdown-mermaid/definition.yml

@@ -2,6 +2,7 @@ key: markdownMermaid
 title: Mermaid
 description: Generate flowcharts from Mermaid syntax
 author: requarks.io
-dependsOn:
-  - markdownCore
+icon: merge_type
+enabledDefault: false
+dependsOn: markdownCore
 props: {}

server/modules/renderer/markdown-plantuml/definition.yml

@@ -2,6 +2,7 @@ key: markdownPlantuml
 title: PlantUML
 description: Generate diagrams from PlantUML syntax
 author: requarks.io
-dependsOn:
-  - markdownCore
+icon: multiline_chart
+enabledDefault: false
+dependsOn: markdownCore
 props: {}

server/modules/renderer/markdown-tasklists/definition.yml

@@ -2,6 +2,7 @@ key: markdownTasklists
 title: Task Lists
 description: Parse task lists to checkboxes
 author: requarks.io
-dependsOn:
-  - markdownCore
+icon: list
+enabledDefault: true
+dependsOn: markdownCore
 props: {}

server/setup.js

@@ -24,7 +24,6 @@ module.exports = () => {
   const yaml = require('js-yaml')
   const _ = require('lodash')
   const cfgHelper = require('./helpers/config')
-  const filesize = require('filesize.js')
   const crypto = Promise.promisifyAll(require('crypto'))
 
   // ----------------------------------------
@@ -76,174 +75,6 @@ module.exports = () => {
     })
   })
 
-  /**
-   * Perform basic system checks
-   */
-  app.post('/syscheck', (req, res) => {
-    WIKI.telemetry.enabled = (req.body.telemetry === true)
-    WIKI.telemetry.sendEvent('setup', 'start')
-
-    Promise.mapSeries([
-      () => {
-        const semver = require('semver')
-        if (!semver.satisfies(semver.clean(process.version), '>=8.9.0')) {
-          throw new Error('Node.js version is too old. Minimum is 8.9.0.')
-        }
-        return {
-          title: 'Node.js ' + process.version + ' detected.',
-          subtitle: ' Minimum is 8.9.0.'
-        }
-      },
-      () => {
-        return Promise.try(() => {
-          require('crypto')
-        }).catch(err => {
-          throw new Error('Crypto Node.js module is not available.')
-        }).return({
-          title: 'Node.js Crypto module is available.',
-          subtitle: 'Crypto module is required.'
-        })
-      },
-      () => {
-        const exec = require('child_process').exec
-        const semver = require('semver')
-        return new Promise((resolve, reject) => {
-          exec('git --version', (err, stdout, stderr) => {
-            if (err || stdout.length < 3) {
-              reject(new Error('Git is not installed or not reachable from PATH.'))
-            }
-            let gitver = _.head(stdout.match(/[\d]+\.[\d]+(\.[\d]+)?/gi))
-            if (!gitver || !semver.satisfies(semver.clean(gitver), '>=2.7.4')) {
-              reject(new Error('Git version is too old. Minimum is 2.7.4.'))
-            }
-            resolve({
-              title: 'Git ' + gitver + ' detected.',
-              subtitle: 'Minimum is 2.7.4.'
-            })
-          })
-        })
-      },
-      () => {
-        const os = require('os')
-        if (os.totalmem() < 1000 * 1000 * 768) {
-          throw new Error('Not enough memory. Minimum is 768 MB.')
-        }
-        return {
-          title: filesize(os.totalmem()) + ' of system memory available.',
-          subtitle: 'Minimum is 768 MB.'
-        }
-      },
-      () => {
-        let fs = require('fs')
-        return Promise.try(() => {
-          fs.accessSync(path.join(WIKI.ROOTPATH, 'config.yml'), (fs.constants || fs).W_OK)
-        }).catch(err => {
-          throw new Error('config.yml file is not writable by Node.js process or was not created properly.')
-        }).return({
-          title: 'config.yml is writable by the setup process.',
-          subtitle: 'Setup will write to this file.'
-        })
-      }
-    ], test => test()).then(results => {
-      res.json({ ok: true, results })
-    }).catch(err => {
-      res.json({ ok: false, error: err.message })
-    })
-  })
-
-  /**
-   * Check the Git connection
-   */
-  app.post('/gitcheck', (req, res) => {
-    WIKI.telemetry.sendEvent('setup', 'gitcheck')
-
-    const exec = require('execa')
-    const url = require('url')
-
-    const dataDir = path.resolve(WIKI.ROOTPATH, cfgHelper.parseConfigValue(req.body.pathData))
-    const gitDir = path.resolve(WIKI.ROOTPATH, cfgHelper.parseConfigValue(req.body.pathRepo))
-
-    let gitRemoteUrl = ''
-
-    if (req.body.gitUseRemote === true) {
-      let urlObj = url.parse(cfgHelper.parseConfigValue(req.body.gitUrl))
-      if (req.body.gitAuthType === 'basic') {
-        urlObj.auth = req.body.gitAuthUser + ':' + req.body.gitAuthPass
-      }
-      gitRemoteUrl = url.format(urlObj)
-    }
-
-    Promise.mapSeries([
-      () => {
-        return fs.ensureDir(dataDir).then(() => 'Data directory path is valid.')
-      },
-      () => {
-        return fs.ensureDir(gitDir).then(() => 'Git directory path is valid.')
-      },
-      () => {
-        return exec.stdout('git', ['init'], { cwd: gitDir }).then(result => {
-          return 'Local git repository has been initialized.'
-        })
-      },
-      () => {
-        if (req.body.gitUseRemote === false) { return false }
-        return exec.stdout('git', ['config', '--local', 'user.name', 'Wiki'], { cwd: gitDir }).then(result => {
-          return 'Git Signature Name has been set successfully.'
-        })
-      },
-      () => {
-        if (req.body.gitUseRemote === false) { return false }
-        return exec.stdout('git', ['config', '--local', 'user.email', req.body.gitServerEmail], { cwd: gitDir }).then(result => {
-          return 'Git Signature Name has been set successfully.'
-        })
-      },
-      () => {
-        if (req.body.gitUseRemote === false) { return false }
-        return exec.stdout('git', ['config', '--local', '--bool', 'http.sslVerify', req.body.gitAuthSSL], { cwd: gitDir }).then(result => {
-          return 'Git SSL Verify flag has been set successfully.'
-        })
-      },
-      () => {
-        if (req.body.gitUseRemote === false) { return false }
-        if (_.includes(['sshenv', 'sshdb'], req.body.gitAuthType)) {
-          req.body.gitAuthSSHKey = path.join(dataDir, 'ssh/key.pem')
-        }
-        if (_.startsWith(req.body.gitAuthType, 'ssh')) {
-          return exec.stdout('git', ['config', '--local', 'core.sshCommand', 'ssh -i "' + req.body.gitAuthSSHKey + '" -o StrictHostKeyChecking=no'], { cwd: gitDir }).then(result => {
-            return 'Git SSH Private Key path has been set successfully.'
-          })
-        } else {
-          return false
-        }
-      },
-      () => {
-        if (req.body.gitUseRemote === false) { return false }
-        return exec.stdout('git', ['remote', 'rm', 'origin'], { cwd: gitDir }).catch(err => {
-          if (_.includes(err.message, 'No such remote') || _.includes(err.message, 'Could not remove')) {
-            return true
-          } else {
-            throw err
-          }
-        }).then(() => {
-          return exec.stdout('git', ['remote', 'add', 'origin', gitRemoteUrl], { cwd: gitDir }).then(result => {
-            return 'Git Remote was added successfully.'
-          })
-        })
-      },
-      () => {
-        if (req.body.gitUseRemote === false) { return false }
-        return exec.stdout('git', ['pull', 'origin', req.body.gitBranch], { cwd: gitDir }).then(result => {
-          return 'Git Pull operation successful.'
-        })
-      }
-    ], step => { return step() }).then(results => {
-      return res.json({ ok: true, results: _.without(results, false) })
-    }).catch(err => {
-      let errMsg = (err.stderr) ? err.stderr.replace(/(error:|warning:|fatal:)/gi, '').replace(/ \s+/g, ' ') : err.message
-      res.json({ ok: false, error: errMsg })
-    })
-  })
-
   /**
    * Finalize
    */
@@ -263,13 +94,6 @@ module.exports = () => {
       let confRaw = await fs.readFileAsync(path.join(WIKI.ROOTPATH, 'config.yml'), 'utf8')
       let conf = yaml.safeLoad(confRaw)
 
-      conf.port = req.body.port
-      conf.paths.data = req.body.pathData
-      conf.paths.content = req.body.pathContent
-
-      confRaw = yaml.safeDump(conf)
-      await fs.writeFileAsync(path.join(WIKI.ROOTPATH, 'config.yml'), confRaw)
-
       // Create directory structure
       await fs.ensureDir(conf.paths.data)
       await fs.ensureDir(path.join(conf.paths.data, 'cache'))
@@ -283,16 +107,13 @@ module.exports = () => {
       _.set(WIKI.config, 'lang.autoUpdate', true)
       _.set(WIKI.config, 'lang.namespacing', false)
       _.set(WIKI.config, 'lang.namespaces', [])
-      _.set(WIKI.config, 'paths.content', req.body.pathContent)
-      _.set(WIKI.config, 'paths.data', req.body.pathData)
-      _.set(WIKI.config, 'port', req.body.port)
-      _.set(WIKI.config, 'public', req.body.public === 'true')
+      _.set(WIKI.config, 'public', false)
       _.set(WIKI.config, 'sessionSecret', (await crypto.randomBytesAsync(32)).toString('hex'))
       _.set(WIKI.config, 'telemetry.isEnabled', req.body.telemetry === 'true')
       _.set(WIKI.config, 'telemetry.clientId', WIKI.telemetry.cid)
       _.set(WIKI.config, 'theming.theme', 'default')
       _.set(WIKI.config, 'theming.darkMode', false)
-      _.set(WIKI.config, 'title', req.body.title)
+      _.set(WIKI.config, 'title', 'Wiki.js')
 
       // Save config to DB
       WIKI.logger.info('Persisting config to DB...')
@@ -325,6 +146,9 @@ module.exports = () => {
       await WIKI.models.editors.refreshEditorsFromDisk()
       await WIKI.models.editors.query().patch({ isEnabled: true }).where('key', 'markdown')
 
+      // Load renderers
+      await WIKI.models.renderers.refreshRenderersFromDisk()
+
       // Load storage targets
       await WIKI.models.storage.refreshTargetsFromDisk()
 
@@ -367,7 +191,7 @@ module.exports = () => {
       WIKI.logger.info('Setup is complete!')
       res.json({
         ok: true,
-        redirectPath: WIKI.config.site.path,
+        redirectPath: '/',
         redirectPort: WIKI.config.port
       }).end()