Search

nodemon.json

@@ -8,5 +8,5 @@
   "watch": [
     "**/*"
   ],
-  "ext": "js,twig"
+  "ext": "ts,js,twig"
 }

package.json

@@ -47,6 +47,7 @@
     "@babel/polyfill": "^7.12.1",
     "@babel/preset-env": "^7.16.11",
     "@codexteam/misprints": "^1.0.0",
+    "@codexteam/shortcuts": "^1.2.0",
     "@editorjs/checklist": "^1.3.0",
     "@editorjs/code": "^2.7.0",
     "@editorjs/delimiter": "^1.2.0",
@@ -83,6 +84,7 @@
     "@types/sinon": "^10.0.2",
     "@types/twig": "^1.12.6",
     "autoprefixer": "^10.4.2",
+    "axios": "^0.27.2",
     "babel": "^6.23.0",
     "babel-eslint": "^10.0.1",
     "babel-loader": "^8.2.3",

src/backend/controllers/search.ts

@@ -0,0 +1,302 @@
+import PageData from '../models/page.js';
+import Pages from '../controllers/pages.js';
+import urlify from '../utils/urlify.js';
+import Page from '../models/page.js';
+
+let globalWords: { [key: string]: {[key: string]: number} } = Object.create(null);
+let globalPages: PageData[] = [];
+
+class Search {
+  /**
+   * Initialize search
+   */
+  public async init() {
+    if (globalWords && Object.keys(globalWords).length) {
+      return Promise.resolve();
+    }
+
+    await this.syncDB();
+  }
+
+  /**
+   * Load all pages from DB and update globalWords
+   * Use this method when any page was updated
+   */
+  public async syncDB() {
+    globalWords = Object.create(null);
+    globalPages = await this.getPages();
+
+    /**
+     * Process all pages
+     */
+    for await (const page of globalPages) {
+      /**
+       * Read content blocks from page
+       */
+      for await (const block of page.body.blocks) {
+        const blockRatio = this.getBlockRatio(block);
+        const blockContent = this.getCleanTextFromBlock(block);
+        const blockWords: string[] = this.splitTextToWords(blockContent);
+
+        /**
+         * Process list of words in a block
+         */
+        for await (const word of blockWords) {
+          if (!globalWords[word]) {
+            globalWords[word] = Object.create(null);
+          }
+
+          if (page._id) {
+            if (!globalWords[word][page._id]) {
+              globalWords[word][page._id] = 0;
+            }
+
+            /**
+             * Add page id to the list of pages with this word
+             */
+            globalWords[word][page._id] += blockRatio;
+          }
+        }
+      }
+    }
+
+    console.log('Done');
+  }
+
+  /**
+   * Search for pages by given query
+   * @param searchString
+   */
+  public async query(searchString: string) {
+    await this.init();
+
+    const searchWords = this.splitTextToWords(searchString);
+
+    const goodPages = (await this.getPagesByWords(searchWords))
+      .slice(0, 10);
+
+    const returnPages: {[key: string]: string|number, ratio: number}[] = [];
+
+    goodPages.forEach(({ pageId, ratio }) => {
+      const page = globalPages.filter(page => page._id === pageId).pop();
+
+      if (!page) {
+        return;
+      }
+
+      let section = '';
+
+      page.body.blocks.forEach((block: any) => {
+        let koef = 1;
+
+        let blockContent = this.getCleanTextFromBlock(block);
+
+        let shortBody = blockContent;
+
+        if (block.type === 'header') {
+          section = blockContent;
+        }
+
+        searchWords.forEach(word => {
+          if (blockContent.toLowerCase().indexOf(word) !== -1) {
+            koef *= 10;
+          }
+        })
+
+        shortBody = this.highlightSubstring(shortBody, searchWords);
+
+        if (koef > 0) {
+          returnPages.push({
+            ...page,
+            shortBody,
+            anchor: urlify(section),
+            section,
+            ratio: ratio * koef,
+          })
+        }
+      });
+    });
+
+    return {
+      suggestions: ['description', 'about', 'contact'],
+      pages: returnPages
+        .sort((a, b) => b.ratio - a.ratio)
+        .slice(0, 15)
+    }
+  }
+
+  /**
+   *
+   * @private
+   */
+  private async getPages(): Promise<Page[]> {
+    return await Pages.getAll();
+  }
+
+  /**
+   * Return list of pages with a given words
+   * @param words
+   * @private
+   */
+  private async getPagesByWords(words: string[]) {
+    const pagesList: {[key: string]: number} = {};
+
+    /**
+     * Get list of words starting with a words from the search query
+     */
+    const validWords = Object.keys(globalWords)
+      .filter(word => {
+        return !!words.filter(searchWord => word.indexOf(searchWord) !== -1).length
+      });
+
+    /**
+     * For each word get list of pages with this word
+     */
+    validWords.forEach(word => {
+      Object.keys(globalWords[word])
+        .forEach(pageId => {
+          if (!pagesList[pageId]) {
+            pagesList[pageId] = 0;
+          }
+
+          pagesList[pageId] += globalWords[word][pageId]
+        })
+    })
+
+    /**
+     * Sort pages by frequency of given words
+     */
+    const sortedPagesList = Object.keys(pagesList)
+      .map(pageId => {
+        return {
+          pageId,
+          ratio: pagesList[pageId]
+        }
+      })
+      .sort((a, b) => b.ratio - a.ratio);
+
+    return sortedPagesList;
+  }
+
+  /**
+   * Get block's ratio. It is used to calculate the weight of the words in the block
+   * @param block
+   * @private
+   */
+  private getBlockRatio(block: any) {
+    switch (block.type) {
+      case 'header':
+        if (block.data.level === 1) {
+          return 16;
+        } else {
+          return 2;
+        }
+
+      case 'paragraph':
+        return 1.1;
+
+      case 'list':
+        return 1;
+
+      default:
+        return 0;
+    }
+  }
+
+  /**
+   * Return clear text content from block without HTML tags and special characters
+   * @param block
+   * @private
+   */
+  private getCleanTextFromBlock(block: any): string {
+    let blockContent = '';
+
+    switch (block.type) {
+      case 'header':
+        blockContent = block.data.text;
+        break;
+
+      case 'paragraph':
+        blockContent = block.data.text
+        break;
+
+      case 'list':
+        blockContent = block.data.items.join(' ');
+        break;
+
+      default:
+        return blockContent;
+    }
+
+    blockContent = this.removeHTMLTags(blockContent);
+    blockContent = this.removeHTMLSpecialCharacters(blockContent);
+
+    return blockContent;
+  }
+
+  /**
+   * Remove HTML tags from string. Only content inside tags will be left
+   * @param text
+   * @private
+   */
+  private removeHTMLTags(text: string) {
+    return text.replace(/<[^>]*>?/gm, '');
+  }
+
+  /**
+   * Remove special characters from text. For example: &nbsp; &amp; &quot; &lt; &gt;
+   * @param text
+   * @private
+   */
+  private removeHTMLSpecialCharacters(text: string) {
+    return text.replace(/&[^;]*;?/gm, '');
+  }
+
+  /**
+   * Split text to words
+   * @param text
+   * @private
+   */
+  private splitTextToWords(text: string): string[] {
+    return text
+      // lowercase all words
+      .toLowerCase()
+
+      // remove punctuation
+      .replace(/[.,;:]/gi, '')
+
+      // left only letters (+cyrillic) and numbers
+      .replace(/[^a-zа-я0-9]/gi, ' ')
+
+      // remove multiple spaces
+      .replace(/\s+/g, ' ')
+
+      // remove spaces at the beginning and at the end
+      .trim()
+
+      // split to words by spaces
+      .split(' ')
+
+      // ignore words shorter than 3 chars
+      .filter(word => word.length >= 3);
+  }
+
+  /**
+   * Highlight substring in string with a span wrapper
+   */
+  private highlightSubstring(text: string, words: string|string[]) {
+    if (typeof words === 'string') {
+      words = [words];
+    }
+
+    const wordRegExp = new RegExp(words.join('|'), "ig");
+    const CLASS_STYLE = 'search-word';
+
+    return text.replace(wordRegExp, `<span class="${CLASS_STYLE}">$&</span>`);
+  }
+}
+
+/**
+ * Export initialized instance
+ */
+export default new Search();

src/backend/routes/api/index.ts

@@ -1,12 +1,15 @@
 import express from 'express';
+
 import pagesAPI from './pages.js';
 import transportAPI from './transport.js';
 import linksAPI from './links.js';
+import searchAPI from './search.js';
 
 const router = express.Router();
 
 router.use('/', pagesAPI);
 router.use('/', transportAPI);
 router.use('/', linksAPI);
+router.use('/', searchAPI);
 
 export default router;

src/backend/routes/api/pages.ts

@@ -2,6 +2,7 @@ import express, { Request, Response } from 'express';
 import multerFunc from 'multer';
 import Pages from '../../controllers/pages.js';
 import PagesOrder from '../../controllers/pagesOrder.js';
+import Search from '../../controllers/search.js';
 
 const router = express.Router();
 const multer = multerFunc();
@@ -70,6 +71,9 @@ router.put('/page', multer.none(), async (req: Request, res: Response) => {
     /** push to the orders array */
     await PagesOrder.push(parent, page._id);
 
+    /** Update search index */
+    await Search.syncDB();
+
     res.json({
       success: true,
       result: page,
@@ -127,6 +131,10 @@ router.post('/page/:id', multer.none(), async (req: Request, res: Response) => {
       parent,
       uri,
     });
+
+    /** Update search index */
+    await Search.syncDB();
+
     res.json({
       success: true,
       result: page,
@@ -206,6 +214,9 @@ router.delete('/page/:id', async (req: Request, res: Response) => {
     parentPageOrder.remove(req.params.id);
     await parentPageOrder.save();
 
+    /** Update search index */
+    await Search.syncDB();
+
     res.json({
       success: true,
       result: pageToRedirect,