feat: azure search module + rebuild index

client/components/admin.vue

@@ -104,6 +104,7 @@
         router-view
 
     nav-footer
+    search-results
 </template>
 
 <script>

client/components/admin/admin-search.vue

@@ -28,6 +28,7 @@
                 :key='engine.key'
                 :label='engine.title'
                 :value='engine.key'
+                :disabled='!engine.isAvailable'
                 color='primary'
                 hide-details
               )
@@ -87,6 +88,7 @@ import _ from 'lodash'
 
 import enginesQuery from 'gql/admin/search/search-query-engines.gql'
 import enginesSaveMutation from 'gql/admin/search/search-mutation-save-engines.gql'
+import enginesRebuildMutation from 'gql/admin/search/search-mutation-rebuild-index.gql'
 
 export default {
   data() {
@@ -101,7 +103,7 @@ export default {
       this.engine = _.find(this.engines, ['key', newValue]) || {}
     },
     engines(newValue, oldValue) {
-      this.selectedEngine = 'db'
+      this.selectedEngine = _.get(_.find(this.engines, 'isEnabled'), 'key', 'db')
     }
   },
   methods: {
@@ -115,29 +117,50 @@ export default {
     },
     async save() {
       this.$store.commit(`loadingStart`, 'admin-search-saveengines')
-      await this.$apollo.mutate({
+      try {
-        mutation: enginesSaveMutation,
+        const resp = await this.$apollo.mutate({
-        variables: {
+          mutation: enginesSaveMutation,
-          engines: this.engines.map(tgt => _.pick(tgt, [
+          variables: {
-            'isEnabled',
+            engines: this.engines.map(tgt => ({
-            'key',
+              isEnabled: tgt.key === this.selectedEngine,
-            'config'
+              key: tgt.key,
-          ])).map(str => ({...str, config: str.config.map(cfg => ({...cfg, value: JSON.stringify({ v: cfg.value.value })}))}))
+              config: tgt.config.map(cfg => ({...cfg, value: JSON.stringify({ v: cfg.value.value })}))
+            }))
+          }
+        })
+        if (_.get(resp, 'data.search.updateSearchEngines.responseResult.succeeded', false)) {
+          this.$store.commit('showNotification', {
+            message: 'Search engine configuration saved successfully.',
+            style: 'success',
+            icon: 'check'
+          })
+        } else {
+          throw new Error(_.get(resp, 'data.search.updateSearchEngines.responseResult.message', 'An unexpected error occured'))
         }
-      })
+      } catch (err) {
-      this.$store.commit('showNotification', {
+        this.$store.commit('pushGraphError', err)
-        message: 'Search engine configuration saved successfully.',
+      }
-        style: 'success',
-        icon: 'check'
-      })
       this.$store.commit(`loadingStop`, 'admin-search-saveengines')
     },
     async rebuild () {
-      this.$store.commit('showNotification', {
+      this.$store.commit(`loadingStart`, 'admin-search-rebuildindex')
-        style: 'indigo',
+      try {
-        message: `Coming soon...`,
+        const resp = await this.$apollo.mutate({
-        icon: 'directions_boat'
+          mutation: enginesRebuildMutation
-      })
+        })
+        if (_.get(resp, 'data.search.rebuildIndex.responseResult.succeeded', false)) {
+          this.$store.commit('showNotification', {
+            message: 'Index rebuilt successfully.',
+            style: 'success',
+            icon: 'check'
+          })
+        } else {
+          throw new Error(_.get(resp, 'data.search.rebuildIndex.responseResult.message', 'An unexpected error occured'))
+        }
+      } catch (err) {
+        this.$store.commit('pushGraphError', err)
+      }
+      this.$store.commit(`loadingStop`, 'admin-search-rebuildindex')
     }
   },
   apollo: {

client/components/common/nav-header.vue

@@ -89,29 +89,38 @@
             v-model='searchAdvMenuShown'
             left
             offset-y
-            min-width='350'
+            min-width='450'
             :close-on-content-click='false'
+            nudge-bottom='7'
+            nudge-right='5'
             )
             v-btn.nav-header-search-adv(icon, outline, color='grey darken-2', slot='activator')
               v-icon(color='white') expand_more
-            v-card
+            v-card.radius-0(dark)
-              v-toolbar(flat, :color='$vuetify.dark ? `grey darken-3-d5` : `grey lighten-4`', dense)
+              v-toolbar(flat, color='grey darken-4', dense)
+                v-icon.mr-2 search
                 v-subheader.pl-0 Advanced Search
-              v-card-text
+              v-card-text.pa-4
                 v-checkbox.mt-0(
                   label='Restrict to Current Language'
-                  color='primary'
+                  color='white'
                   v-model='searchRestrictLocale'
                   hide-details
                 )
                 v-checkbox(
                   label='Restrict to Below Current Path'
-                  color='primary'
+                  color='white'
                   v-model='searchRestrictPath'
                   hide-details
                 )
-              v-card-actions
+              v-divider
-                v-btn(outline, small, color='grey') Save as defaults
+              v-card-actions.grey.darken-3-d4
+                v-btn(depressed, color='grey darken-3', block)
+                  v-icon(left) chevron_right
+                  span Save as defaults
+                v-btn(depressed, color='grey darken-3', block)
+                  v-icon(left) cached
+                  span Reset
       v-flex(xs6, :md4='searchIsShown', :md6='!searchIsShown')
         v-toolbar.nav-header-inner(color='black', dark, flat)
           v-spacer

client/components/common/search-results.vue

@@ -206,13 +206,13 @@ export default {
 
   &-items {
     .highlighted {
-      background-color: mc('blue', '50');
+      background: #FFF linear-gradient(to bottom, #FFF, mc('orange', '100'));
     }
   }
 
   &-suggestions {
     .highlighted {
-      background-color: mc('blue', '500');
+      background: transparent linear-gradient(to bottom, mc('blue', '500'), mc('blue', '700'));
     }
   }
 }

client/components/history.vue

@@ -103,6 +103,7 @@
                 v-card.mt-3(light, v-html='diffHTML')
 
     nav-footer
+    search-results
 </template>
 
 <script>

client/components/profile.vue

@@ -23,6 +23,7 @@
         router-view
 
     nav-footer
+    search-results
 </template>
 
 <script>

client/components/source.vue

@@ -16,6 +16,7 @@
                   slot
 
     nav-footer
+    search-results
 </template>
 
 <script>

client/graph/admin/search/search-mutation-rebuild-index.gql

@@ -0,0 +1,12 @@
+mutation {
+  search {
+    rebuildIndex {
+      responseResult {
+        succeeded
+        errorCode
+        slug
+        message
+      }
+    }
+  }
+}

client/graph/admin/search/search-mutation-save-engines.gql

@@ -1,6 +1,6 @@
-mutation($searchEngines: [SearchEngineInput]) {
+mutation($engines: [SearchEngineInput]) {
   search {
-    updateSearchEngines(searchEngines: $searchEngines) {
+    updateSearchEngines(engines: $engines) {
       responseResult {
         succeeded
         errorCode

client/graph/admin/search/search-query-engines.gql

@@ -7,6 +7,7 @@ query {
       description
       logo
       website
+      isAvailable
       config {
         key
         value

package.json

@@ -45,6 +45,7 @@
     "apollo-server-express": "2.3.3",
     "auto-load": "3.0.4",
     "axios": "0.18.0",
+    "azure-search-client": "3.1.5",
     "bcryptjs-then": "1.0.1",
     "bluebird": "3.5.3",
     "body-parser": "1.18.3",
@@ -136,6 +137,7 @@
     "pg": "7.8.0",
     "pg-hstore": "2.3.2",
     "pg-tsquery": "8.0.3",
+    "pg-query-stream": "2.0.0",
     "pm2": "3.2.9",
     "pug": "2.0.3",
     "qr-image": "3.2.0",

server/graph/resolvers/search.js

@@ -38,7 +38,7 @@ module.exports = {
   SearchMutation: {
     async updateSearchEngines(obj, args, context) {
       try {
-        for (let searchEngine of args.searchEngines) {
+        for (let searchEngine of args.engines) {
           await WIKI.models.searchEngines.query().patch({
             isEnabled: searchEngine.isEnabled,
             config: _.reduce(searchEngine.config, (result, value, key) => {
@@ -47,12 +47,23 @@ module.exports = {
             }, {})
           }).where('key', searchEngine.key)
         }
+        await WIKI.models.searchEngines.initEngine({ activate: true })
         return {
           responseResult: graphHelper.generateSuccess('Search Engines updated successfully')
         }
       } catch (err) {
         return graphHelper.generateError(err)
       }
+    },
+    async rebuildIndex (obj, args, context) {
+      try {
+        await WIKI.data.searchEngine.rebuild()
+        return {
+          responseResult: graphHelper.generateSuccess('Index rebuilt successfully')
+        }
+      } catch (err) {
+        return graphHelper.generateError(err)
+      }
     }
   }
 }

server/graph/schemas/page.graphql

@@ -102,7 +102,7 @@ type PageSearchResponse {
 }
 
 type PageSearchResult {
-  id: Int!
+  id: String!
   title: String!
   description: String!
   path: String!

server/graph/schemas/search.graphql

@@ -27,8 +27,10 @@ type SearchQuery {
 
 type SearchMutation {
   updateSearchEngines(
-    searchEngines: [SearchEngineInput]
+    engines: [SearchEngineInput]
   ): DefaultResponse @auth(requires: ["manage:system"])
+
+  rebuildIndex: DefaultResponse @auth(requires: ["manage:system"])
 }
 
 # -----------------------------------------------
@@ -42,6 +44,7 @@ type SearchEngine {
   description: String
   logo: String
   website: String
+  isAvailable: Boolean
   config: [KeyValuePair]
 }
 

server/helpers/error.js

@@ -69,6 +69,10 @@ module.exports = {
     message: 'Invalid locale or namespace.',
     code: 1009
   }),
+  SearchActivationFailed: CustomError('SearchActivationFailed', {
+    message: 'Search Engine activation failed.',
+    code: 1019
+  }),
   UserCreationFailed: CustomError('UserCreationFailed', {
     message: 'An unexpected error occured during user creation.',
     code: 1010

server/models/pages.js

@@ -210,6 +210,7 @@ module.exports = class Page extends Model {
       isPrivate: opts.isPrivate
     })
     await WIKI.models.pages.renderPage(page)
+    await WIKI.data.searchEngine.created(page)
     if (!opts.skipStorage) {
       await WIKI.models.storage.pageEvent({
         event: 'created',
@@ -245,6 +246,7 @@ module.exports = class Page extends Model {
       isPrivate: ogPage.isPrivate
     })
     await WIKI.models.pages.renderPage(page)
+    await WIKI.data.searchEngine.updated(page)
     if (!opts.skipStorage) {
       await WIKI.models.storage.pageEvent({
         event: 'updated',
@@ -273,6 +275,7 @@ module.exports = class Page extends Model {
     })
     await WIKI.models.pages.query().delete().where('id', page.id)
     await WIKI.models.pages.deletePageFromCache(page)
+    await WIKI.data.searchEngine.deleted(page)
     if (!opts.skipStorage) {
       await WIKI.models.storage.pageEvent({
         event: 'deleted',

server/models/searchEngines.js

@@ -95,11 +95,25 @@ module.exports = class SearchEngine extends Model {
     }
   }
 
-  static async initEngine() {
+  static async initEngine({ activate = false } = {}) {
     const searchEngine = await WIKI.models.searchEngines.query().findOne('isEnabled', true)
     if (searchEngine) {
       WIKI.data.searchEngine = require(`../modules/search/${searchEngine.key}/engine`)
       WIKI.data.searchEngine.config = searchEngine.config
+      if (activate) {
+        try {
+          await WIKI.data.searchEngine.activate()
+        } catch (err) {
+          // -> Revert to basic engine
+          if (err instanceof WIKI.Error.SearchActivationFailed) {
+            await WIKI.models.searchEngines.query().patch({ isEnabled: false }).where('key', searchEngine.key)
+            await WIKI.models.searchEngines.query().patch({ isEnabled: true }).where('key', 'db')
+            await WIKI.models.searchEngines.initEngine()
+          }
+          throw err
+        }
+      }
+
       try {
         await WIKI.data.searchEngine.init()
       } catch (err) {
@@ -107,19 +121,4 @@ module.exports = class SearchEngine extends Model {
       }
     }
   }
-
-  static async pageEvent({ event, page }) {
-    const searchEngines = await WIKI.models.storage.query().where('isEnabled', true)
-    if (searchEngines && searchEngines.length > 0) {
-      _.forEach(searchEngines, logger => {
-        WIKI.queue.job.syncStorage.add({
-          event,
-          logger,
-          page
-        }, {
-          removeOnComplete: true
-        })
-      })
-    }
-  }
 }

server/modules/search/algolia/definition.yml

@@ -4,6 +4,7 @@ description: Algolia is a powerful search-as-a-service solution, made easy to us
 author: requarks.io
 logo: https://static.requarks.io/logo/algolia.svg
 website: https://www.algolia.com/
+isAvailable: false
 props:
   appId:
     type: String

server/modules/search/aws/definition.yml

@@ -4,4 +4,5 @@ description: Amazon CloudSearch is a managed service in the AWS Cloud that makes
 author: requarks.io
 logo: https://static.requarks.io/logo/aws-cloudsearch.svg
 website: https://aws.amazon.com/cloudsearch/
+isAvailable: false
 props: {}

server/modules/search/azure/definition.yml

@@ -4,4 +4,21 @@ description: AI-Powered cloud search service for web and mobile app development.
 author: requarks.io
 logo: https://static.requarks.io/logo/azure.svg
 website: https://azure.microsoft.com/services/search/
-props: {}
+isAvailable: true
+props:
+  serviceName:
+    type: String
+    title: Service Name
+    hint: The name of the Azure Search Service. Found under Properties.
+    order: 1
+  adminKey:
+    type: String
+    title: Admin API Key
+    hint: Either the primary or secondary admin key. Found under Keys.
+    order: 2
+  indexName:
+    type: String
+    title: Index Name
+    hint: 'Name to use when creating the index. (default: wiki)'
+    default: wiki
+    order: 3

server/modules/search/azure/engine.js

@@ -1,26 +1,213 @@
+const _ = require('lodash')
+const { SearchService, QueryType } = require('azure-search-client')
+const request = require('request-promise')
+const { pipeline } = require('stream')
+
 module.exports = {
-  activate() {
+  async activate() {
-
+    // not used
   },
-  deactivate() {
+  async deactivate() {
-
+    // not used
   },
-  query() {
+  /**
+   * INIT
+   */
+  async init() {
+    this.client = new SearchService(this.config.serviceName, this.config.adminKey)
 
+    // -> Create Search Index
+    const indexes = await this.client.indexes.list()
+    if (!_.find(_.get(indexes, 'result.value', []), ['name', this.config.indexName])) {
+      await this.client.indexes.create({
+        name: this.config.indexName,
+        fields: [
+          {
+            name: 'id',
+            type: 'Edm.String',
+            key: true,
+            searchable: false
+          },
+          {
+            name: 'locale',
+            type: 'Edm.String',
+            searchable: false
+          },
+          {
+            name: 'path',
+            type: 'Edm.String',
+            searchable: false
+          },
+          {
+            name: 'title',
+            type: 'Edm.String',
+            searchable: true
+          },
+          {
+            name: 'description',
+            type: 'Edm.String',
+            searchable: true
+          },
+          {
+            name: 'content',
+            type: 'Edm.String',
+            searchable: true
+          }
+        ],
+        scoringProfiles: [
+          {
+            name: 'fieldWeights',
+            text: {
+              weights: {
+                title: 4,
+                description: 3,
+                content: 1
+              }
+            }
+          }
+        ],
+        suggesters: [
+          {
+            name: 'suggestions',
+            searchMode: 'analyzingInfixMatching',
+            sourceFields: ['title', 'description', 'content']
+          }
+        ],
+      })
+    }
   },
-  created() {
+  /**
-
+   * QUERY
+   *
+   * @param {String} q Query
+   * @param {Object} opts Additional options
+   */
+  async query(q, opts) {
+    try {
+      let suggestions = []
+      const results = await this.client.indexes.use(this.config.indexName).search({
+        count: true,
+        scoringProfile: 'fieldWeights',
+        search: q,
+        select: 'id, locale, path, title, description',
+        queryType: QueryType.simple,
+        top: 50
+      })
+      if (results.result.value.length < 5) {
+        // Using plain request, not yet available in library...
+        try {
+          const suggestResults = await request({
+            uri: `https://${this.config.serviceName}.search.windows.net/indexes/${this.config.indexName}/docs/autocomplete`,
+            method: 'post',
+            qs: {
+              'api-version': '2017-11-11-Preview'
+            },
+            headers: {
+              'api-key': this.config.adminKey,
+              'Content-Type': 'application/json'
+            },
+            json: true,
+            body: {
+              autocompleteMode: 'oneTermWithContext',
+              search: q,
+              suggesterName: 'suggestions'
+            }
+          })
+          suggestions = suggestResults.value.map(s => s.queryPlusText)
+        } catch (err) {
+          WIKI.logger.warn('Search Engine suggestion failure: ', err)
+        }
+      }
+      return {
+        results: results.result.value,
+        suggestions,
+        totalHits: results.result['@odata.count']
+      }
+    } catch (err) {
+      WIKI.logger.warn('Search Engine Error:')
+      WIKI.logger.warn(err)
+    }
   },
-  updated() {
+  /**
-
+   * CREATE
+   *
+   * @param {Object} page Page to create
+   */
+  async created(page) {
+    await this.client.indexes.use(this.config.indexName).index([
+      {
+        id: page.hash,
+        locale: page.localeCode,
+        path: page.path,
+        title: page.title,
+        description: page.description,
+        content: page.content
+      }
+    ])
   },
-  deleted() {
+  /**
-
+   * UPDATE
+   *
+   * @param {Object} page Page to update
+   */
+  async updated(page) {
+    await this.client.indexes.use(this.config.indexName).index([
+      {
+        id: page.hash,
+        locale: page.localeCode,
+        path: page.path,
+        title: page.title,
+        description: page.description,
+        content: page.content
+      }
+    ])
   },
-  renamed() {
+  /**
-
+   * DELETE
+   *
+   * @param {Object} page Page to delete
+   */
+  async deleted(page) {
+    await this.client.indexes.use(this.config.indexName).index([
+      {
+        '@search.action': 'delete',
+        id: page.hash
+      }
+    ])
   },
-  rebuild() {
+  /**
-
+   * RENAME
+   *
+   * @param {Object} page Page to rename
+   */
+  async renamed(page) {
+    await this.client.indexes.use(this.config.indexName).index([
+      {
+        '@search.action': 'delete',
+        id: page.sourceHash
+      }
+    ])
+    await this.client.indexes.use(this.config.indexName).index([
+      {
+        id: page.destinationHash,
+        locale: page.localeCode,
+        path: page.destinationPath,
+        title: page.title,
+        description: page.description,
+        content: page.content
+      }
+    ])
+  },
+  /**
+   * REBUILD INDEX
+   */
+  async rebuild() {
+    await pipeline(
+      WIKI.models.knex.column({ id: 'hash' }, 'path', { locale: 'localeCode' }, 'title', 'description', 'content').select().from('pages').where({
+        isPublished: true,
+        isPrivate: false
+      }).stream(),
+      this.client.indexes.use(this.config.indexName).createIndexingStream()
+    )
   }
 }

server/modules/search/db/definition.yml

@@ -4,4 +4,5 @@ description: Default basic database-based search engine.
 author: requarks.io
 logo: https://static.requarks.io/logo/database.svg
 website: https://www.requarks.io/
+isAvailable: true
 props: {}

server/modules/search/elasticsearch/definition.yml

@@ -4,6 +4,7 @@ description: Elasticsearch is a distributed, RESTful search and analytics engine
 author: requarks.io
 logo: https://static.requarks.io/logo/elasticsearch.svg
 website: https://www.elastic.co/products/elasticsearch
+isAvailable: false
 props:
   apiVersion:
     type: String

server/modules/search/manticore/definition.yml

@@ -4,4 +4,5 @@ description: High performance full-text search engine with SQL and JSON support.
 author: requarks.io
 logo: https://static.requarks.io/logo/manticore.svg
 website: https://manticoresearch.com/
+isAvailable: false
 props: {}

server/modules/search/postgres/definition.yml

@@ -4,6 +4,7 @@ description: Advanced PostgreSQL-based search engine.
 author: requarks.io
 logo: https://static.requarks.io/logo/postgresql.svg
 website: https://www.requarks.io/
+isAvailable: true
 props:
   dictLanguage:
     type: String

server/modules/search/postgres/engine.js

@@ -3,7 +3,9 @@ const tsquery = require('pg-tsquery')()
 
 module.exports = {
   async activate() {
-    // not used
+    if (WIKI.config.db.type !== 'postgres') {
+      throw new WIKI.Error.SearchActivationFailed('Must use PostgreSQL database to activate this engine!')
+    }
   },
   async deactivate() {
     // not used
@@ -75,7 +77,7 @@ module.exports = {
       INSERT INTO "pagesVector" (path, locale, title, description, tokens) VALUES (
         '?', '?', '?', '?', (setweight(to_tsvector('${this.config.dictLanguage}', '?'), 'A') || setweight(to_tsvector('${this.config.dictLanguage}', '?'), 'B') || setweight(to_tsvector('${this.config.dictLanguage}', '?'), 'C'))
       )
-    `, [page.path, page.locale, page.title, page.description, page.title, page.description, page.content])
+    `, [page.path, page.localeCode, page.title, page.description, page.title, page.description, page.content])
   },
   /**
    * UPDATE
@@ -85,13 +87,13 @@ module.exports = {
   async updated(page) {
     await WIKI.models.knex.raw(`
       UPDATE "pagesVector" SET
-        title = '?',
+        title = ?,
-        description = '?',
+        description = ?,
-        tokens = (setweight(to_tsvector('${this.config.dictLanguage}', '?'), 'A') ||
+        tokens = (setweight(to_tsvector('${this.config.dictLanguage}', ?), 'A') ||
-        setweight(to_tsvector('${this.config.dictLanguage}', '?'), 'B') ||
+        setweight(to_tsvector('${this.config.dictLanguage}', ?), 'B') ||
-        setweight(to_tsvector('${this.config.dictLanguage}', '?'), 'C'))
+        setweight(to_tsvector('${this.config.dictLanguage}', ?), 'C'))
-      WHERE path = '?' AND locale = '?' LIMIT 1
+      WHERE path = ? AND locale = ?
-    `, [page.title, page.description, page.title, page.description, page.content, page.path, page.locale])
+    `, [page.title, page.description, page.title, page.description, page.content, page.path, page.localeCode])
   },
   /**
    * DELETE
@@ -100,7 +102,7 @@ module.exports = {
    */
   async deleted(page) {
     await WIKI.models.knex('pagesVector').where({
-      locale: page.locale,
+      locale: page.localeCode,
       path: page.path
     }).del().limit(1)
   },
@@ -111,12 +113,12 @@ module.exports = {
    */
   async renamed(page) {
     await WIKI.models.knex('pagesVector').where({
-      locale: page.locale,
+      locale: page.localeCode,
       path: page.sourcePath
     }).update({
-      locale: page.locale,
+      locale: page.localeCode,
       path: page.destinationPath
-    }).limit(1)
+    })
   },
   /**
    * REBUILD INDEX

server/modules/search/solr/definition.yml

@@ -4,6 +4,7 @@ description: Solr is the popular, blazing-fast, open source enterprise search pl
 author: requarks.io
 logo: https://static.requarks.io/logo/solr.svg
 website: http://lucene.apache.org/solr/
+isAvailable: false
 props:
   host:
     type: String

server/modules/search/sphinx/definition.yml

@@ -4,4 +4,5 @@ description: Sphinx is an open source full text search server, designed from the
 author: requarks.io
 logo: https://static.requarks.io/logo/sphinx.svg
 website: http://sphinxsearch.com/
+isAvailable: false
 props: {}