From 343d4db0b36f724cb8ff424b40c1637f0f9283c4 Mon Sep 17 00:00:00 2001 From: Nick Date: Sun, 17 Mar 2019 21:52:16 -0400 Subject: [PATCH] feat: algolia search engine --- package.json | 1 + server/modules/search/algolia/definition.yml | 2 +- server/modules/search/algolia/engine.js | 204 +++++++++++++++++-- server/modules/search/aws/engine.js | 13 +- server/modules/search/azure/engine.js | 6 +- yarn.lock | 97 ++++++++- 6 files changed, 297 insertions(+), 26 deletions(-) diff --git a/package.json b/package.json index f2380fd6..a45fd510 100644 --- a/package.json +++ b/package.json @@ -40,6 +40,7 @@ }, "dependencies": { "@bugsnag/js": "5.2.0", + "algoliasearch": "3.32.1", "apollo-fetch": "0.7.0", "apollo-server": "2.3.3", "apollo-server-express": "2.3.3", diff --git a/server/modules/search/algolia/definition.yml b/server/modules/search/algolia/definition.yml index 7edc7bb4..9976925a 100644 --- a/server/modules/search/algolia/definition.yml +++ b/server/modules/search/algolia/definition.yml @@ -4,7 +4,7 @@ description: Algolia is a powerful search-as-a-service solution, made easy to us author: requarks.io logo: https://static.requarks.io/logo/algolia.svg website: https://www.algolia.com/ -isAvailable: false +isAvailable: true props: appId: type: String diff --git a/server/modules/search/algolia/engine.js b/server/modules/search/algolia/engine.js index e7369ccd..217d2632 100644 --- a/server/modules/search/algolia/engine.js +++ b/server/modules/search/algolia/engine.js @@ -1,26 +1,202 @@ +const _ = require('lodash') +const algoliasearch = require('algoliasearch') +const { pipeline, Transform } = require('stream') + +/* global WIKI */ + module.exports = { - activate() { - + async activate() { + // not used }, - deactivate() { - + async deactivate() { + // not used }, - query() { + /** + * INIT + */ + async init() { + WIKI.logger.info(`(SEARCH/ALGOLIA) Initializing...`) + this.client = algoliasearch(this.config.appId, this.config.apiKey) + this.index = this.client.initIndex(this.config.indexName) + // -> Create Search Index + WIKI.logger.info(`(SEARCH/ALGOLIA) Setting index configuration...`) + await this.index.setSettings({ + searchableAttributes: [ + 'title', + 'description', + 'content' + ], + attributesToRetrieve: [ + 'locale', + 'path', + 'title', + 'description' + ], + advancedSyntax: true + }) + WIKI.logger.info(`(SEARCH/ALGOLIA) Initialization completed.`) }, - created() { - + /** + * QUERY + * + * @param {String} q Query + * @param {Object} opts Additional options + */ + async query(q, opts) { + try { + const results = await this.index.search({ + query: q, + hitsPerPage: 50 + }) + return { + results: _.map(results.hits, r => ({ + id: r.objectID, + locale: r.locale, + path: r.path, + title: r.title, + description: r.description + })), + suggestions: [], + totalHits: results.nbHits + } + } catch (err) { + WIKI.logger.warn('Search Engine Error:') + WIKI.logger.warn(err) + } }, - updated() { - + /** + * CREATE + * + * @param {Object} page Page to create + */ + async created(page) { + await this.index.addObject({ + objectID: page.hash, + locale: page.localeCode, + path: page.path, + title: page.title, + description: page.description, + content: page.content + }) }, - deleted() { - + /** + * UPDATE + * + * @param {Object} page Page to update + */ + async updated(page) { + await this.index.partialUpdateObject({ + objectID: page.hash, + title: page.title, + description: page.description, + content: page.content + }) }, - renamed() { - + /** + * DELETE + * + * @param {Object} page Page to delete + */ + async deleted(page) { + await this.index.deleteObject(page.hash) }, - rebuild() { + /** + * RENAME + * + * @param {Object} page Page to rename + */ + async renamed(page) { + await this.index.deleteObject(page.sourceHash) + await this.index.addObject({ + objectID: page.destinationHash, + locale: page.localeCode, + path: page.destinationPath, + title: page.title, + description: page.description, + content: page.content + }) + }, + /** + * REBUILD INDEX + */ + async rebuild() { + WIKI.logger.info(`(SEARCH/ALGOLIA) Rebuilding Index...`) + await this.index.clearIndex() + const MAX_DOCUMENT_BYTES = 10 * Math.pow(2, 10) // 10 KB + const MAX_INDEXING_BYTES = 10 * Math.pow(2, 20) - Buffer.from('[').byteLength - Buffer.from(']').byteLength // 10 MB + const MAX_INDEXING_COUNT = 1000 + const COMMA_BYTES = Buffer.from(',').byteLength + + let chunks = [] + let bytes = 0 + + const processDocument = async (cb, doc) => { + try { + if (doc) { + const docBytes = Buffer.from(JSON.stringify(doc)).byteLength + // -> Document too large + if (docBytes >= MAX_DOCUMENT_BYTES) { + throw new Error('Document exceeds maximum size allowed by Algolia.') + } + + // -> Current batch exceeds size hard limit, flush + if (docBytes + COMMA_BYTES + bytes >= MAX_INDEXING_BYTES) { + await flushBuffer() + } + + if (chunks.length > 0) { + bytes += COMMA_BYTES + } + bytes += docBytes + chunks.push(doc) + + // -> Current batch exceeds count soft limit, flush + if (chunks.length >= MAX_INDEXING_COUNT) { + await flushBuffer() + } + } else { + // -> End of stream, flush + await flushBuffer() + } + cb() + } catch (err) { + cb(err) + } + } + + const flushBuffer = async () => { + WIKI.logger.info(`(SEARCH/ALGOLIA) Sending batch of ${chunks.length}...`) + try { + await this.index.addObjects( + _.map(chunks, doc => ({ + objectID: doc.id, + locale: doc.locale, + path: doc.path, + title: doc.title, + description: doc.description, + content: doc.content + })) + ) + } catch (err) { + WIKI.logger.warn('(SEARCH/ALGOLIA) Failed to send batch to Algolia: ', err) + } + chunks.length = 0 + bytes = 0 + } + + await pipeline( + WIKI.models.knex.column({ id: 'hash' }, 'path', { locale: 'localeCode' }, 'title', 'description', 'content').select().from('pages').where({ + isPublished: true, + isPrivate: false + }).stream(), + new Transform({ + objectMode: true, + transform: async (chunk, enc, cb) => processDocument(cb, chunk), + flush: async (cb) => processDocument(cb) + }) + ) + WIKI.logger.info(`(SEARCH/ALGOLIA) Index rebuilt successfully.`) } } diff --git a/server/modules/search/aws/engine.js b/server/modules/search/aws/engine.js index 916cc572..599f1bae 100644 --- a/server/modules/search/aws/engine.js +++ b/server/modules/search/aws/engine.js @@ -2,6 +2,8 @@ const _ = require('lodash') const AWS = require('aws-sdk') const { pipeline, Transform } = require('stream') +/* global WIKI */ + module.exports = { async activate() { // not used @@ -110,12 +112,12 @@ module.exports = { rebuildIndex = true } - //-> Define suggester + // -> Define suggester const suggesters = await this.client.describeSuggesters({ DomainName: this.config.domain, SuggesterNames: ['default_suggester'] }).promise() - if(_.get(suggesters, 'Suggesters', []).length < 1) { + if (_.get(suggesters, 'Suggesters', []).length < 1) { WIKI.logger.info(`(SEARCH/AWS) Defining Suggester...`) await this.client.defineSuggester({ DomainName: this.config.domain, @@ -323,7 +325,7 @@ module.exports = { const flushBuffer = async () => { WIKI.logger.info(`(SEARCH/AWS) Sending batch of ${chunks.length}...`) try { - const resp = await this.clientDomain.uploadDocuments({ + await this.clientDomain.uploadDocuments({ contentType: 'application/json', documents: JSON.stringify(_.map(chunks, doc => ({ type: 'add', @@ -351,8 +353,8 @@ module.exports = { }).stream(), new Transform({ objectMode: true, - transform: async (chunk, enc, cb) => await processDocument(cb, chunk), - flush: async (cb) => await processDocument(cb) + transform: async (chunk, enc, cb) => processDocument(cb, chunk), + flush: async (cb) => processDocument(cb) }) ) @@ -364,4 +366,3 @@ module.exports = { WIKI.logger.info(`(SEARCH/AWS) Index rebuilt successfully.`) } } - diff --git a/server/modules/search/azure/engine.js b/server/modules/search/azure/engine.js index 0a9c0ef7..f242df58 100644 --- a/server/modules/search/azure/engine.js +++ b/server/modules/search/azure/engine.js @@ -3,6 +3,8 @@ const { SearchService, QueryType } = require('azure-search-client') const request = require('request-promise') const { pipeline } = require('stream') +/* global WIKI */ + module.exports = { async activate() { // not used @@ -20,7 +22,7 @@ module.exports = { // -> Create Search Index const indexes = await this.client.indexes.list() if (!_.find(_.get(indexes, 'result.value', []), ['name', this.config.indexName])) { - WIKI.logger.info(`(SEARCH/AWS) Creating index...`) + WIKI.logger.info(`(SEARCH/AZURE) Creating index...`) await this.client.indexes.create({ name: this.config.indexName, fields: [ @@ -74,7 +76,7 @@ module.exports = { searchMode: 'analyzingInfixMatching', sourceFields: ['title', 'description', 'content'] } - ], + ] }) } WIKI.logger.info(`(SEARCH/AZURE) Initialization completed.`) diff --git a/yarn.lock b/yarn.lock index a7a7a3fc..921b3e56 100644 --- a/yarn.lock +++ b/yarn.lock @@ -1456,6 +1456,11 @@ agent-base@4, agent-base@^4.1.0, agent-base@^4.2.0, agent-base@~4.2.0: dependencies: es6-promisify "^5.0.0" +agentkeepalive@^2.2.0: + version "2.2.0" + resolved "https://registry.yarnpkg.com/agentkeepalive/-/agentkeepalive-2.2.0.tgz#c5d1bd4b129008f1163f236f86e5faea2026e2ef" + integrity sha1-xdG9SxKQCPEWPyNvhuX66iAm4u8= + agentkeepalive@^3.4.1: version "3.5.2" resolved "https://registry.yarnpkg.com/agentkeepalive/-/agentkeepalive-3.5.2.tgz#a113924dd3fa24a0bc3b78108c450c2abee00f67" @@ -1483,6 +1488,27 @@ ajv@^6.1.0, ajv@^6.1.1, ajv@^6.5.3, ajv@^6.5.5, ajv@^6.6.1: json-schema-traverse "^0.4.1" uri-js "^4.2.2" +algoliasearch@3.32.1: + version "3.32.1" + resolved "https://registry.yarnpkg.com/algoliasearch/-/algoliasearch-3.32.1.tgz#605f8a2c17ab8da2af4456110f4d0a02b384e3d0" + integrity sha512-NaaHMboU9tKwrU3aim7LlzSDqKb+1TGaC+Lx3NOttSnuMHbPpaf+7LtJL4KlosbRWEwqb9t5wSYMVDrPTH2dNA== + dependencies: + agentkeepalive "^2.2.0" + debug "^2.6.9" + envify "^4.0.0" + es6-promise "^4.1.0" + events "^1.1.0" + foreach "^2.0.5" + global "^4.3.2" + inherits "^2.0.1" + isarray "^2.0.1" + load-script "^1.0.0" + object-keys "^1.0.11" + querystring-es3 "^0.2.1" + reduce "^1.0.1" + semver "^5.1.0" + tunnel-agent "^0.6.0" + align-text@^0.1.1, align-text@^0.1.3: version "0.1.4" resolved "https://registry.yarnpkg.com/align-text/-/align-text-0.1.4.tgz#0cd90a561093f35d0a99256c22b7069433fad117" @@ -4261,6 +4287,11 @@ dom-serializer@0, dom-serializer@~0.1.0: domelementtype "~1.1.1" entities "~1.1.1" +dom-walk@^0.1.0: + version "0.1.1" + resolved "https://registry.yarnpkg.com/dom-walk/-/dom-walk-0.1.1.tgz#672226dc74c8f799ad35307df936aba11acd6018" + integrity sha1-ZyIm3HTI95mtNTB9+TaroRrNYBg= + domain-browser@^1.1.1: version "1.2.0" resolved "https://registry.yarnpkg.com/domain-browser/-/domain-browser-1.2.0.tgz#3d31f50191a6749dd1375a7f522e823d42e54eda" @@ -4473,6 +4504,14 @@ env-variable@0.0.x: resolved "https://registry.yarnpkg.com/env-variable/-/env-variable-0.0.5.tgz#913dd830bef11e96a039c038d4130604eba37f88" integrity sha512-zoB603vQReOFvTg5xMl9I1P2PnHsHQQKTEowsKKD7nseUfJq6UWzK+4YtlWUO1nhiQUxe6XMkk+JleSZD1NZFA== +envify@^4.0.0: + version "4.1.0" + resolved "https://registry.yarnpkg.com/envify/-/envify-4.1.0.tgz#f39ad3db9d6801b4e6b478b61028d3f0b6819f7e" + integrity sha512-IKRVVoAYr4pIx4yIWNsz9mOsboxlNXiu7TNBnem/K/uTHdkyzXWDzHCK7UTolqBbgaBz0tQHsD3YNls0uIIjiw== + dependencies: + esprima "^4.0.0" + through "~2.3.4" + epic-spinners@1.0.4: version "1.0.4" resolved "https://registry.yarnpkg.com/epic-spinners/-/epic-spinners-1.0.4.tgz#6372b4a7748eb7cad562e8b516cc7505d26fb5b3" @@ -4543,6 +4582,11 @@ es6-promise@^4.0.3: resolved "https://registry.yarnpkg.com/es6-promise/-/es6-promise-4.2.5.tgz#da6d0d5692efb461e082c14817fe2427d8f5d054" integrity sha512-n6wvpdE43VFtJq+lUDYDBFUwV8TZbuGXLV4D6wKafg13ldznKsyEvatubnmUe31zcvelSzOHF+XbaT+Bl9ObDg== +es6-promise@^4.1.0: + version "4.2.6" + resolved "https://registry.yarnpkg.com/es6-promise/-/es6-promise-4.2.6.tgz#b685edd8258886365ea62b57d30de28fadcd974f" + integrity sha512-aRVgGdnmW2OiySVPUC9e6m+plolMAJKjZnQlCwNSuK5yQ0JN61DZSO1X1Ufd1foqWRAlig0rhduTCHe7sVtK5Q== + es6-promisify@^5.0.0: version "5.0.0" resolved "https://registry.yarnpkg.com/es6-promisify/-/es6-promisify-5.0.0.tgz#5109d62f3e56ea967c4b63505aef08291c8a5203" @@ -4814,7 +4858,7 @@ eventemitter3@^3.1.0: resolved "https://registry.yarnpkg.com/eventemitter3/-/eventemitter3-3.1.0.tgz#090b4d6cdbd645ed10bf750d4b5407942d7ba163" integrity sha512-ivIvhpq/Y0uSjcHDcOIccjmYjGLcP09MFGE7ysAwkAvkXfpZlC985pH2/ui64DKazbTW/4kN3yqozUxlXzI6cA== -events@1.1.1: +events@1.1.1, events@^1.1.0: version "1.1.1" resolved "https://registry.yarnpkg.com/events/-/events-1.1.1.tgz#9ebdb7635ad099c70dcc4c2a1f5004288e8bd924" integrity sha1-nr23Y1rQmccNzEwqH1AEKI6L2SQ= @@ -5232,6 +5276,11 @@ for-own@^1.0.0: dependencies: for-in "^1.0.1" +foreach@^2.0.5: + version "2.0.5" + resolved "https://registry.yarnpkg.com/foreach/-/foreach-2.0.5.tgz#0bee005018aeb260d0a3af3ae658dd0136ec1b99" + integrity sha1-C+4AUBiusmDQo6865ljdATbsG5k= + forever-agent@~0.6.1: version "0.6.1" resolved "https://registry.yarnpkg.com/forever-agent/-/forever-agent-0.6.1.tgz#fbc71f0c41adeb37f96c577ad1ed42d8fdacca91" @@ -5521,6 +5570,14 @@ global-prefix@^1.0.1: is-windows "^1.0.1" which "^1.2.14" +global@^4.3.2: + version "4.3.2" + resolved "https://registry.yarnpkg.com/global/-/global-4.3.2.tgz#e76989268a6c74c38908b1305b10fc0e394e9d0f" + integrity sha1-52mJJopsdMOJCLEwWxD8DjlOnQ8= + dependencies: + min-document "^2.19.0" + process "~0.5.1" + globals@^11.1.0, globals@^11.7.0: version "11.10.0" resolved "https://registry.yarnpkg.com/globals/-/globals-11.10.0.tgz#1e09776dffda5e01816b3bb4077c8b59c24eaa50" @@ -6637,6 +6694,11 @@ isarray@1.0.0, isarray@^1.0.0, isarray@~1.0.0: resolved "https://registry.yarnpkg.com/isarray/-/isarray-1.0.0.tgz#bb935d48582cba168c06834957a54a3e07124f11" integrity sha1-u5NdSFgsuhaMBoNJV6VKPgcSTxE= +isarray@^2.0.1: + version "2.0.4" + resolved "https://registry.yarnpkg.com/isarray/-/isarray-2.0.4.tgz#38e7bcbb0f3ba1b7933c86ba1894ddfc3781bbb7" + integrity sha512-GMxXOiUirWg1xTKRipM0Ek07rX+ubx4nNVElTJdNLYmNO/2YrDkgJGw9CljXn+r4EWiDQg/8lsRdHyg2PJuUaA== + iserror@^0.0.2: version "0.0.2" resolved "https://registry.yarnpkg.com/iserror/-/iserror-0.0.2.tgz#bd53451fe2f668b9f2402c1966787aaa2c7c0bf5" @@ -7179,6 +7241,11 @@ load-json-file@^4.0.0: pify "^3.0.0" strip-bom "^3.0.0" +load-script@^1.0.0: + version "1.0.0" + resolved "https://registry.yarnpkg.com/load-script/-/load-script-1.0.0.tgz#0491939e0bee5643ee494a7e3da3d2bac70c6ca4" + integrity sha1-BJGTngvuVkPuSUp+PaPSuscMbKQ= + loader-runner@^2.3.0: version "2.4.0" resolved "https://registry.yarnpkg.com/loader-runner/-/loader-runner-2.4.0.tgz#ed47066bfe534d7e84c4c7b9998c2a75607d9357" @@ -7841,6 +7908,13 @@ mimic-fn@^1.0.0: resolved "https://registry.yarnpkg.com/mimic-fn/-/mimic-fn-1.2.0.tgz#820c86a39334640e99516928bd03fca88057d022" integrity sha512-jf84uxzwiuiIVKiOLpfYk7N46TSy8ubTonmneY9vrpHNAnp0QBt2BxWV9dO3/j+BoVAb+a5G6YDPW3M5HOdMWQ== +min-document@^2.19.0: + version "2.19.0" + resolved "https://registry.yarnpkg.com/min-document/-/min-document-2.19.0.tgz#7bd282e3f5842ed295bb748cdd9f1ffa2c824685" + integrity sha1-e9KC4/WELtKVu3SM3Z8f+iyCRoU= + dependencies: + dom-walk "^0.1.0" + mini-css-extract-plugin@0.5.0: version "0.5.0" resolved "https://registry.yarnpkg.com/mini-css-extract-plugin/-/mini-css-extract-plugin-0.5.0.tgz#ac0059b02b9692515a637115b0cc9fed3a35c7b0" @@ -8469,6 +8543,11 @@ object-copy@^0.1.0: define-property "^0.2.5" kind-of "^3.0.3" +object-keys@^1.0.11, object-keys@^1.1.0: + version "1.1.0" + resolved "https://registry.yarnpkg.com/object-keys/-/object-keys-1.1.0.tgz#11bd22348dd2e096a045ab06f6c85bcc340fa032" + integrity sha512-6OO5X1+2tYkNyNEx6TsCxEqFfRWaqx6EtMiSbGrw8Ob8v9Ne+Hl8rBAgLBZn5wjEz3s/s6U1WXFUFOcxxAwUpg== + object-keys@^1.0.12: version "1.0.12" resolved "https://registry.yarnpkg.com/object-keys/-/object-keys-1.0.12.tgz#09c53855377575310cca62f55bb334abff7b3ed2" @@ -10389,6 +10468,11 @@ process@^0.11.10: resolved "https://registry.yarnpkg.com/process/-/process-0.11.10.tgz#7332300e840161bda3e69a1d1d91a7d4bc16f182" integrity sha1-czIwDoQBYb2j5podHZGn1LwW8YI= +process@~0.5.1: + version "0.5.2" + resolved "https://registry.yarnpkg.com/process/-/process-0.5.2.tgz#1638d8a8e34c2f440a91db95ab9aeb677fc185cf" + integrity sha1-FjjYqONML0QKkduVq5rrZ3/Bhc8= + progress@^2.0.0: version "2.0.3" resolved "https://registry.yarnpkg.com/progress/-/progress-2.0.3.tgz#7e8cf8d8f5b8f239c1bc68beb4eb78567d572ef8" @@ -10708,7 +10792,7 @@ qs@^6.5.1: resolved "https://registry.yarnpkg.com/qs/-/qs-6.6.0.tgz#a99c0f69a8d26bf7ef012f871cdabb0aee4424c2" integrity sha512-KIJqT9jQJDQx5h5uAVPimw6yVg2SekOKu959OCtktD3FjzbpvaPr8i4zzg07DOMz+igA4W/aNM7OV8H37pFYfA== -querystring-es3@^0.2.0: +querystring-es3@^0.2.0, querystring-es3@^0.2.1: version "0.2.1" resolved "https://registry.yarnpkg.com/querystring-es3/-/querystring-es3-0.2.1.tgz#9ec61f79049875707d69414596fd907a4d711e73" integrity sha1-nsYfeQSYdXB9aUFFlv2Qek1xHnM= @@ -11011,6 +11095,13 @@ reduce-function-call@^1.0.1, reduce-function-call@^1.0.2: dependencies: balanced-match "^0.4.2" +reduce@^1.0.1: + version "1.0.2" + resolved "https://registry.yarnpkg.com/reduce/-/reduce-1.0.2.tgz#0cd680ad3ffe0b060e57a5c68bdfce37168d361b" + integrity sha512-xX7Fxke/oHO5IfZSk77lvPa/7bjMh9BuCk4OOoX5XTXrM7s0Z+MkPfSDfz0q7r91BhhGSs8gii/VEN/7zhCPpQ== + dependencies: + object-keys "^1.1.0" + redux@^3.4.0: version "3.7.2" resolved "https://registry.yarnpkg.com/redux/-/redux-3.7.2.tgz#06b73123215901d25d065be342eb026bc1c8537b" @@ -12425,7 +12516,7 @@ through2@^2.0.0: readable-stream "~2.3.6" xtend "~4.0.1" -through@2, through@^2.3.6: +through@2, through@^2.3.6, through@~2.3.4: version "2.3.8" resolved "https://registry.yarnpkg.com/through/-/through-2.3.8.tgz#0dd4c9ffaabc357960b1b724115d7e0e86a2e1f5" integrity sha1-DdTJ/6q8NXlgsbckEV1+Doai4fU=