feat: algolia search engine

package.json

@@ -40,6 +40,7 @@
   },
   "dependencies": {
     "@bugsnag/js": "5.2.0",
+    "algoliasearch": "3.32.1",
     "apollo-fetch": "0.7.0",
     "apollo-server": "2.3.3",
     "apollo-server-express": "2.3.3",

server/modules/search/algolia/definition.yml

@@ -4,7 +4,7 @@ description: Algolia is a powerful search-as-a-service solution, made easy to us
 author: requarks.io
 logo: https://static.requarks.io/logo/algolia.svg
 website: https://www.algolia.com/
-isAvailable: false
+isAvailable: true
 props:
   appId:
     type: String

server/modules/search/algolia/engine.js

@@ -1,26 +1,202 @@
+const _ = require('lodash')
+const algoliasearch = require('algoliasearch')
+const { pipeline, Transform } = require('stream')
+
+/* global WIKI */
+
 module.exports = {
-  activate() {
+  async activate() {
-
+    // not used
   },
-  deactivate() {
+  async deactivate() {
-
+    // not used
   },
-  query() {
+  /**
+   * INIT
+   */
+  async init() {
+    WIKI.logger.info(`(SEARCH/ALGOLIA) Initializing...`)
+    this.client = algoliasearch(this.config.appId, this.config.apiKey)
+    this.index = this.client.initIndex(this.config.indexName)
 
+    // -> Create Search Index
+    WIKI.logger.info(`(SEARCH/ALGOLIA) Setting index configuration...`)
+    await this.index.setSettings({
+      searchableAttributes: [
+        'title',
+        'description',
+        'content'
+      ],
+      attributesToRetrieve: [
+        'locale',
+        'path',
+        'title',
+        'description'
+      ],
+      advancedSyntax: true
+    })
+    WIKI.logger.info(`(SEARCH/ALGOLIA) Initialization completed.`)
   },
-  created() {
+  /**
-
+   * QUERY
+   *
+   * @param {String} q Query
+   * @param {Object} opts Additional options
+   */
+  async query(q, opts) {
+    try {
+      const results = await this.index.search({
+        query: q,
+        hitsPerPage: 50
+      })
+      return {
+        results: _.map(results.hits, r => ({
+          id: r.objectID,
+          locale: r.locale,
+          path: r.path,
+          title: r.title,
+          description: r.description
+        })),
+        suggestions: [],
+        totalHits: results.nbHits
+      }
+    } catch (err) {
+      WIKI.logger.warn('Search Engine Error:')
+      WIKI.logger.warn(err)
+    }
   },
-  updated() {
+  /**
-
+   * CREATE
+   *
+   * @param {Object} page Page to create
+   */
+  async created(page) {
+    await this.index.addObject({
+      objectID: page.hash,
+      locale: page.localeCode,
+      path: page.path,
+      title: page.title,
+      description: page.description,
+      content: page.content
+    })
   },
-  deleted() {
+  /**
-
+   * UPDATE
+   *
+   * @param {Object} page Page to update
+   */
+  async updated(page) {
+    await this.index.partialUpdateObject({
+      objectID: page.hash,
+      title: page.title,
+      description: page.description,
+      content: page.content
+    })
   },
-  renamed() {
+  /**
-
+   * DELETE
+   *
+   * @param {Object} page Page to delete
+   */
+  async deleted(page) {
+    await this.index.deleteObject(page.hash)
   },
-  rebuild() {
+  /**
+   * RENAME
+   *
+   * @param {Object} page Page to rename
+   */
+  async renamed(page) {
+    await this.index.deleteObject(page.sourceHash)
+    await this.index.addObject({
+      objectID: page.destinationHash,
+      locale: page.localeCode,
+      path: page.destinationPath,
+      title: page.title,
+      description: page.description,
+      content: page.content
+    })
+  },
+  /**
+   * REBUILD INDEX
+   */
+  async rebuild() {
+    WIKI.logger.info(`(SEARCH/ALGOLIA) Rebuilding Index...`)
+    await this.index.clearIndex()
 
+    const MAX_DOCUMENT_BYTES = 10 * Math.pow(2, 10) // 10 KB
+    const MAX_INDEXING_BYTES = 10 * Math.pow(2, 20) - Buffer.from('[').byteLength - Buffer.from(']').byteLength // 10 MB
+    const MAX_INDEXING_COUNT = 1000
+    const COMMA_BYTES = Buffer.from(',').byteLength
+
+    let chunks = []
+    let bytes = 0
+
+    const processDocument = async (cb, doc) => {
+      try {
+        if (doc) {
+          const docBytes = Buffer.from(JSON.stringify(doc)).byteLength
+          // -> Document too large
+          if (docBytes >= MAX_DOCUMENT_BYTES) {
+            throw new Error('Document exceeds maximum size allowed by Algolia.')
+          }
+
+          // -> Current batch exceeds size hard limit, flush
+          if (docBytes + COMMA_BYTES + bytes >= MAX_INDEXING_BYTES) {
+            await flushBuffer()
+          }
+
+          if (chunks.length > 0) {
+            bytes += COMMA_BYTES
+          }
+          bytes += docBytes
+          chunks.push(doc)
+
+          // -> Current batch exceeds count soft limit, flush
+          if (chunks.length >= MAX_INDEXING_COUNT) {
+            await flushBuffer()
+          }
+        } else {
+          // -> End of stream, flush
+          await flushBuffer()
+        }
+        cb()
+      } catch (err) {
+        cb(err)
+      }
+    }
+
+    const flushBuffer = async () => {
+      WIKI.logger.info(`(SEARCH/ALGOLIA) Sending batch of ${chunks.length}...`)
+      try {
+        await this.index.addObjects(
+          _.map(chunks, doc => ({
+            objectID: doc.id,
+            locale: doc.locale,
+            path: doc.path,
+            title: doc.title,
+            description: doc.description,
+            content: doc.content
+          }))
+        )
+      } catch (err) {
+        WIKI.logger.warn('(SEARCH/ALGOLIA) Failed to send batch to Algolia: ', err)
+      }
+      chunks.length = 0
+      bytes = 0
+    }
+
+    await pipeline(
+      WIKI.models.knex.column({ id: 'hash' }, 'path', { locale: 'localeCode' }, 'title', 'description', 'content').select().from('pages').where({
+        isPublished: true,
+        isPrivate: false
+      }).stream(),
+      new Transform({
+        objectMode: true,
+        transform: async (chunk, enc, cb) => processDocument(cb, chunk),
+        flush: async (cb) => processDocument(cb)
+      })
+    )
+    WIKI.logger.info(`(SEARCH/ALGOLIA) Index rebuilt successfully.`)
   }
 }

server/modules/search/aws/engine.js

@@ -2,6 +2,8 @@ const _ = require('lodash')
 const AWS = require('aws-sdk')
 const { pipeline, Transform } = require('stream')
 
+/* global WIKI */
+
 module.exports = {
   async activate() {
     // not used
@@ -110,12 +112,12 @@ module.exports = {
       rebuildIndex = true
     }
 
-    //-> Define suggester
+    // -> Define suggester
     const suggesters = await this.client.describeSuggesters({
       DomainName: this.config.domain,
       SuggesterNames: ['default_suggester']
     }).promise()
-    if(_.get(suggesters, 'Suggesters', []).length < 1) {
+    if (_.get(suggesters, 'Suggesters', []).length < 1) {
       WIKI.logger.info(`(SEARCH/AWS) Defining Suggester...`)
       await this.client.defineSuggester({
         DomainName: this.config.domain,
@@ -323,7 +325,7 @@ module.exports = {
     const flushBuffer = async () => {
       WIKI.logger.info(`(SEARCH/AWS) Sending batch of ${chunks.length}...`)
       try {
-        const resp = await this.clientDomain.uploadDocuments({
+        await this.clientDomain.uploadDocuments({
           contentType: 'application/json',
           documents: JSON.stringify(_.map(chunks, doc => ({
             type: 'add',
@@ -351,8 +353,8 @@ module.exports = {
       }).stream(),
       new Transform({
         objectMode: true,
-        transform: async (chunk, enc, cb) => await processDocument(cb, chunk),
+        transform: async (chunk, enc, cb) => processDocument(cb, chunk),
-        flush: async (cb) => await processDocument(cb)
+        flush: async (cb) => processDocument(cb)
       })
     )
 
@@ -364,4 +366,3 @@ module.exports = {
     WIKI.logger.info(`(SEARCH/AWS) Index rebuilt successfully.`)
   }
 }
-

server/modules/search/azure/engine.js

@@ -3,6 +3,8 @@ const { SearchService, QueryType } = require('azure-search-client')
 const request = require('request-promise')
 const { pipeline } = require('stream')
 
+/* global WIKI */
+
 module.exports = {
   async activate() {
     // not used
@@ -20,7 +22,7 @@ module.exports = {
     // -> Create Search Index
     const indexes = await this.client.indexes.list()
     if (!_.find(_.get(indexes, 'result.value', []), ['name', this.config.indexName])) {
-      WIKI.logger.info(`(SEARCH/AWS) Creating index...`)
+      WIKI.logger.info(`(SEARCH/AZURE) Creating index...`)
       await this.client.indexes.create({
         name: this.config.indexName,
         fields: [
@@ -74,7 +76,7 @@ module.exports = {
             searchMode: 'analyzingInfixMatching',
             sourceFields: ['title', 'description', 'content']
           }
-        ],
+        ]
       })
     }
     WIKI.logger.info(`(SEARCH/AZURE) Initialization completed.`)

yarn.lock

@@ -1456,6 +1456,11 @@ agent-base@4, agent-base@^4.1.0, agent-base@^4.2.0, agent-base@~4.2.0:
   dependencies:
     es6-promisify "^5.0.0"
 
+agentkeepalive@^2.2.0:
+  version "2.2.0"
+  resolved "https://registry.yarnpkg.com/agentkeepalive/-/agentkeepalive-2.2.0.tgz#c5d1bd4b129008f1163f236f86e5faea2026e2ef"
+  integrity sha1-xdG9SxKQCPEWPyNvhuX66iAm4u8=
+
 agentkeepalive@^3.4.1:
   version "3.5.2"
   resolved "https://registry.yarnpkg.com/agentkeepalive/-/agentkeepalive-3.5.2.tgz#a113924dd3fa24a0bc3b78108c450c2abee00f67"
@@ -1483,6 +1488,27 @@ ajv@^6.1.0, ajv@^6.1.1, ajv@^6.5.3, ajv@^6.5.5, ajv@^6.6.1:
     json-schema-traverse "^0.4.1"
     uri-js "^4.2.2"
 
+algoliasearch@3.32.1:
+  version "3.32.1"
+  resolved "https://registry.yarnpkg.com/algoliasearch/-/algoliasearch-3.32.1.tgz#605f8a2c17ab8da2af4456110f4d0a02b384e3d0"
+  integrity sha512-NaaHMboU9tKwrU3aim7LlzSDqKb+1TGaC+Lx3NOttSnuMHbPpaf+7LtJL4KlosbRWEwqb9t5wSYMVDrPTH2dNA==
+  dependencies:
+    agentkeepalive "^2.2.0"
+    debug "^2.6.9"
+    envify "^4.0.0"
+    es6-promise "^4.1.0"
+    events "^1.1.0"
+    foreach "^2.0.5"
+    global "^4.3.2"
+    inherits "^2.0.1"
+    isarray "^2.0.1"
+    load-script "^1.0.0"
+    object-keys "^1.0.11"
+    querystring-es3 "^0.2.1"
+    reduce "^1.0.1"
+    semver "^5.1.0"
+    tunnel-agent "^0.6.0"
+
 align-text@^0.1.1, align-text@^0.1.3:
   version "0.1.4"
   resolved "https://registry.yarnpkg.com/align-text/-/align-text-0.1.4.tgz#0cd90a561093f35d0a99256c22b7069433fad117"
@@ -4261,6 +4287,11 @@ dom-serializer@0, dom-serializer@~0.1.0:
     domelementtype "~1.1.1"
     entities "~1.1.1"
 
+dom-walk@^0.1.0:
+  version "0.1.1"
+  resolved "https://registry.yarnpkg.com/dom-walk/-/dom-walk-0.1.1.tgz#672226dc74c8f799ad35307df936aba11acd6018"
+  integrity sha1-ZyIm3HTI95mtNTB9+TaroRrNYBg=
+
 domain-browser@^1.1.1:
   version "1.2.0"
   resolved "https://registry.yarnpkg.com/domain-browser/-/domain-browser-1.2.0.tgz#3d31f50191a6749dd1375a7f522e823d42e54eda"
@@ -4473,6 +4504,14 @@ env-variable@0.0.x:
   resolved "https://registry.yarnpkg.com/env-variable/-/env-variable-0.0.5.tgz#913dd830bef11e96a039c038d4130604eba37f88"
   integrity sha512-zoB603vQReOFvTg5xMl9I1P2PnHsHQQKTEowsKKD7nseUfJq6UWzK+4YtlWUO1nhiQUxe6XMkk+JleSZD1NZFA==
 
+envify@^4.0.0:
+  version "4.1.0"
+  resolved "https://registry.yarnpkg.com/envify/-/envify-4.1.0.tgz#f39ad3db9d6801b4e6b478b61028d3f0b6819f7e"
+  integrity sha512-IKRVVoAYr4pIx4yIWNsz9mOsboxlNXiu7TNBnem/K/uTHdkyzXWDzHCK7UTolqBbgaBz0tQHsD3YNls0uIIjiw==
+  dependencies:
+    esprima "^4.0.0"
+    through "~2.3.4"
+
 epic-spinners@1.0.4:
   version "1.0.4"
   resolved "https://registry.yarnpkg.com/epic-spinners/-/epic-spinners-1.0.4.tgz#6372b4a7748eb7cad562e8b516cc7505d26fb5b3"
@@ -4543,6 +4582,11 @@ es6-promise@^4.0.3:
   resolved "https://registry.yarnpkg.com/es6-promise/-/es6-promise-4.2.5.tgz#da6d0d5692efb461e082c14817fe2427d8f5d054"
   integrity sha512-n6wvpdE43VFtJq+lUDYDBFUwV8TZbuGXLV4D6wKafg13ldznKsyEvatubnmUe31zcvelSzOHF+XbaT+Bl9ObDg==
 
+es6-promise@^4.1.0:
+  version "4.2.6"
+  resolved "https://registry.yarnpkg.com/es6-promise/-/es6-promise-4.2.6.tgz#b685edd8258886365ea62b57d30de28fadcd974f"
+  integrity sha512-aRVgGdnmW2OiySVPUC9e6m+plolMAJKjZnQlCwNSuK5yQ0JN61DZSO1X1Ufd1foqWRAlig0rhduTCHe7sVtK5Q==
+
 es6-promisify@^5.0.0:
   version "5.0.0"
   resolved "https://registry.yarnpkg.com/es6-promisify/-/es6-promisify-5.0.0.tgz#5109d62f3e56ea967c4b63505aef08291c8a5203"
@@ -4814,7 +4858,7 @@ eventemitter3@^3.1.0:
   resolved "https://registry.yarnpkg.com/eventemitter3/-/eventemitter3-3.1.0.tgz#090b4d6cdbd645ed10bf750d4b5407942d7ba163"
   integrity sha512-ivIvhpq/Y0uSjcHDcOIccjmYjGLcP09MFGE7ysAwkAvkXfpZlC985pH2/ui64DKazbTW/4kN3yqozUxlXzI6cA==
 
-events@1.1.1:
+events@1.1.1, events@^1.1.0:
   version "1.1.1"
   resolved "https://registry.yarnpkg.com/events/-/events-1.1.1.tgz#9ebdb7635ad099c70dcc4c2a1f5004288e8bd924"
   integrity sha1-nr23Y1rQmccNzEwqH1AEKI6L2SQ=
@@ -5232,6 +5276,11 @@ for-own@^1.0.0:
   dependencies:
     for-in "^1.0.1"
 
+foreach@^2.0.5:
+  version "2.0.5"
+  resolved "https://registry.yarnpkg.com/foreach/-/foreach-2.0.5.tgz#0bee005018aeb260d0a3af3ae658dd0136ec1b99"
+  integrity sha1-C+4AUBiusmDQo6865ljdATbsG5k=
+
 forever-agent@~0.6.1:
   version "0.6.1"
   resolved "https://registry.yarnpkg.com/forever-agent/-/forever-agent-0.6.1.tgz#fbc71f0c41adeb37f96c577ad1ed42d8fdacca91"
@@ -5521,6 +5570,14 @@ global-prefix@^1.0.1:
     is-windows "^1.0.1"
     which "^1.2.14"
 
+global@^4.3.2:
+  version "4.3.2"
+  resolved "https://registry.yarnpkg.com/global/-/global-4.3.2.tgz#e76989268a6c74c38908b1305b10fc0e394e9d0f"
+  integrity sha1-52mJJopsdMOJCLEwWxD8DjlOnQ8=
+  dependencies:
+    min-document "^2.19.0"
+    process "~0.5.1"
+
 globals@^11.1.0, globals@^11.7.0:
   version "11.10.0"
   resolved "https://registry.yarnpkg.com/globals/-/globals-11.10.0.tgz#1e09776dffda5e01816b3bb4077c8b59c24eaa50"
@@ -6637,6 +6694,11 @@ isarray@1.0.0, isarray@^1.0.0, isarray@~1.0.0:
   resolved "https://registry.yarnpkg.com/isarray/-/isarray-1.0.0.tgz#bb935d48582cba168c06834957a54a3e07124f11"
   integrity sha1-u5NdSFgsuhaMBoNJV6VKPgcSTxE=
 
+isarray@^2.0.1:
+  version "2.0.4"
+  resolved "https://registry.yarnpkg.com/isarray/-/isarray-2.0.4.tgz#38e7bcbb0f3ba1b7933c86ba1894ddfc3781bbb7"
+  integrity sha512-GMxXOiUirWg1xTKRipM0Ek07rX+ubx4nNVElTJdNLYmNO/2YrDkgJGw9CljXn+r4EWiDQg/8lsRdHyg2PJuUaA==
+
 iserror@^0.0.2:
   version "0.0.2"
   resolved "https://registry.yarnpkg.com/iserror/-/iserror-0.0.2.tgz#bd53451fe2f668b9f2402c1966787aaa2c7c0bf5"
@@ -7179,6 +7241,11 @@ load-json-file@^4.0.0:
     pify "^3.0.0"
     strip-bom "^3.0.0"
 
+load-script@^1.0.0:
+  version "1.0.0"
+  resolved "https://registry.yarnpkg.com/load-script/-/load-script-1.0.0.tgz#0491939e0bee5643ee494a7e3da3d2bac70c6ca4"
+  integrity sha1-BJGTngvuVkPuSUp+PaPSuscMbKQ=
+
 loader-runner@^2.3.0:
   version "2.4.0"
   resolved "https://registry.yarnpkg.com/loader-runner/-/loader-runner-2.4.0.tgz#ed47066bfe534d7e84c4c7b9998c2a75607d9357"
@@ -7841,6 +7908,13 @@ mimic-fn@^1.0.0:
   resolved "https://registry.yarnpkg.com/mimic-fn/-/mimic-fn-1.2.0.tgz#820c86a39334640e99516928bd03fca88057d022"
   integrity sha512-jf84uxzwiuiIVKiOLpfYk7N46TSy8ubTonmneY9vrpHNAnp0QBt2BxWV9dO3/j+BoVAb+a5G6YDPW3M5HOdMWQ==
 
+min-document@^2.19.0:
+  version "2.19.0"
+  resolved "https://registry.yarnpkg.com/min-document/-/min-document-2.19.0.tgz#7bd282e3f5842ed295bb748cdd9f1ffa2c824685"
+  integrity sha1-e9KC4/WELtKVu3SM3Z8f+iyCRoU=
+  dependencies:
+    dom-walk "^0.1.0"
+
 mini-css-extract-plugin@0.5.0:
   version "0.5.0"
   resolved "https://registry.yarnpkg.com/mini-css-extract-plugin/-/mini-css-extract-plugin-0.5.0.tgz#ac0059b02b9692515a637115b0cc9fed3a35c7b0"
@@ -8469,6 +8543,11 @@ object-copy@^0.1.0:
     define-property "^0.2.5"
     kind-of "^3.0.3"
 
+object-keys@^1.0.11, object-keys@^1.1.0:
+  version "1.1.0"
+  resolved "https://registry.yarnpkg.com/object-keys/-/object-keys-1.1.0.tgz#11bd22348dd2e096a045ab06f6c85bcc340fa032"
+  integrity sha512-6OO5X1+2tYkNyNEx6TsCxEqFfRWaqx6EtMiSbGrw8Ob8v9Ne+Hl8rBAgLBZn5wjEz3s/s6U1WXFUFOcxxAwUpg==
+
 object-keys@^1.0.12:
   version "1.0.12"
   resolved "https://registry.yarnpkg.com/object-keys/-/object-keys-1.0.12.tgz#09c53855377575310cca62f55bb334abff7b3ed2"
@@ -10389,6 +10468,11 @@ process@^0.11.10:
   resolved "https://registry.yarnpkg.com/process/-/process-0.11.10.tgz#7332300e840161bda3e69a1d1d91a7d4bc16f182"
   integrity sha1-czIwDoQBYb2j5podHZGn1LwW8YI=
 
+process@~0.5.1:
+  version "0.5.2"
+  resolved "https://registry.yarnpkg.com/process/-/process-0.5.2.tgz#1638d8a8e34c2f440a91db95ab9aeb677fc185cf"
+  integrity sha1-FjjYqONML0QKkduVq5rrZ3/Bhc8=
+
 progress@^2.0.0:
   version "2.0.3"
   resolved "https://registry.yarnpkg.com/progress/-/progress-2.0.3.tgz#7e8cf8d8f5b8f239c1bc68beb4eb78567d572ef8"
@@ -10708,7 +10792,7 @@ qs@^6.5.1:
   resolved "https://registry.yarnpkg.com/qs/-/qs-6.6.0.tgz#a99c0f69a8d26bf7ef012f871cdabb0aee4424c2"
   integrity sha512-KIJqT9jQJDQx5h5uAVPimw6yVg2SekOKu959OCtktD3FjzbpvaPr8i4zzg07DOMz+igA4W/aNM7OV8H37pFYfA==
 
-querystring-es3@^0.2.0:
+querystring-es3@^0.2.0, querystring-es3@^0.2.1:
   version "0.2.1"
   resolved "https://registry.yarnpkg.com/querystring-es3/-/querystring-es3-0.2.1.tgz#9ec61f79049875707d69414596fd907a4d711e73"
   integrity sha1-nsYfeQSYdXB9aUFFlv2Qek1xHnM=
@@ -11011,6 +11095,13 @@ reduce-function-call@^1.0.1, reduce-function-call@^1.0.2:
   dependencies:
     balanced-match "^0.4.2"
 
+reduce@^1.0.1:
+  version "1.0.2"
+  resolved "https://registry.yarnpkg.com/reduce/-/reduce-1.0.2.tgz#0cd680ad3ffe0b060e57a5c68bdfce37168d361b"
+  integrity sha512-xX7Fxke/oHO5IfZSk77lvPa/7bjMh9BuCk4OOoX5XTXrM7s0Z+MkPfSDfz0q7r91BhhGSs8gii/VEN/7zhCPpQ==
+  dependencies:
+    object-keys "^1.1.0"
+
 redux@^3.4.0:
   version "3.7.2"
   resolved "https://registry.yarnpkg.com/redux/-/redux-3.7.2.tgz#06b73123215901d25d065be342eb026bc1c8537b"
@@ -12425,7 +12516,7 @@ through2@^2.0.0:
     readable-stream "~2.3.6"
     xtend "~4.0.1"
 
-through@2, through@^2.3.6:
+through@2, through@^2.3.6, through@~2.3.4:
   version "2.3.8"
   resolved "https://registry.yarnpkg.com/through/-/through-2.3.8.tgz#0dd4c9ffaabc357960b1b724115d7e0e86a2e1f5"
   integrity sha1-DdTJ/6q8NXlgsbckEV1+Doai4fU=