feat: replace internal web search indexer (#3818)

* feat: replace internal search indexer

* chore: add import

* chore: fix tests

* add search dependency

* remove lunr-mutable-indexes

package.json

@@ -24,6 +24,7 @@
     "@verdaccio/local-storage": "10.3.3",
     "@verdaccio/logger-7": "6.0.0-6-next.13",
     "@verdaccio/middleware": "6.0.0-6-next.47",
+    "@verdaccio/search": "6.0.0-6-next.2",
     "@verdaccio/signature": "6.0.0-6-next.2",
     "@verdaccio/streams": "10.2.1",
     "@verdaccio/tarball": "11.0.0-6-next.37",
@@ -48,7 +49,6 @@
     "kleur": "4.1.5",
     "lodash": "4.17.21",
     "lru-cache": "7.18.3",
-    "lunr-mutable-indexes": "2.3.2",
     "mime": "3.0.0",
     "mkdirp": "1.0.4",
     "mv": "2.1.1",
@@ -62,6 +62,7 @@
   "devDependencies": {
     "@babel/cli": "7.21.5",
     "@babel/core": "7.21.8",
+    "@babel/eslint-parser": "7.21.8",
     "@babel/node": "7.20.7",
     "@babel/plugin-proposal-class-properties": "7.18.6",
     "@babel/plugin-proposal-decorators": "7.21.0",
@@ -101,7 +102,6 @@
     "@typescript-eslint/parser": "5.59.5",
     "@verdaccio-scope/verdaccio-auth-foo": "0.0.2",
     "@verdaccio/types": "11.0.0-6-next.25",
-    "@babel/eslint-parser": "7.21.8",
     "babel-jest": "29.5.0",
     "babel-plugin-dynamic-import-node": "2.3.3",
     "cross-env": "7.0.3",

src/api/index.ts

@@ -40,7 +40,7 @@ export function loadTheme(config) {
   }
 }
 
-const defineAPI = function (config: IConfig, storage: Storage): any {
+const defineAPI = async function (config: IConfig, storage: Storage): Promise<any> {
   const auth = new Auth(config);
   const app: Application = express();
 
@@ -105,7 +105,8 @@ const defineAPI = function (config: IConfig, storage: Storage): any {
       res.locals.app_version = version ?? '';
       next();
     });
-    app.use('/', webMiddleware(config, auth, storage));
+    const midl = await webMiddleware(config, auth, storage);
+    app.use('/', midl);
   } else {
     app.get('/', function (req: $RequestExtend, res: $ResponseExtend, next: $NextFunctionVer) {
       next(ErrorCode.getNotFound(API_ERROR.WEB_DISABLED));
@@ -140,5 +141,5 @@ export default (async function (configHash: any): Promise<any> {
   const storage = new Storage(config);
   // waits until init calls have been initialized
   await storage.init(config, filters);
-  return defineAPI(config, storage);
+  return await defineAPI(config, storage);
 });

src/api/web/api/search.ts

@@ -1,10 +1,10 @@
 import { Router } from 'express';
 
-import { Package } from '@verdaccio/types';
+import { SearchMemoryIndexer } from '@verdaccio/search';
+import { Manifest } from '@verdaccio/types';
 
 import Auth from '../../../lib/auth';
 import { DIST_TAGS } from '../../../lib/constants';
-import Search from '../../../lib/search';
 import Storage from '../../../lib/storage';
 import { $NextFunctionVer, $RequestExtend, $ResponseExtend } from '../../../types';
 
@@ -13,16 +13,21 @@ function addSearchWebApi(storage: Storage, auth: Auth): Router {
   // Search package
   route.get(
     '/search/:anything',
-    function (req: $RequestExtend, res: $ResponseExtend, next: $NextFunctionVer): void {
-      const results: any = Search.query(req.params.anything);
-      // FUTURE: figure out here the correct type
+    async function (
+      req: $RequestExtend,
+      _res: $ResponseExtend,
+      next: $NextFunctionVer
+    ): Promise<void> {
+      const term = req.params.anything;
+      const indexer = (await SearchMemoryIndexer.query(term)) as any;
       const packages: any[] = [];
+      const results = indexer.hits;
 
       const getPackageInfo = function (i): void {
         storage.getPackage({
-          name: results[i].ref,
+          name: results[i].id,
           uplinksLook: false,
-          callback: (err, entry: Package): void => {
+          callback: (err, entry: Manifest): void => {
             if (!err && entry) {
               auth.allow_access(
                 { packageName: entry.name },

src/api/web/index.ts

@@ -3,15 +3,16 @@ import express from 'express';
 import _ from 'lodash';
 
 import { webMiddleware } from '@verdaccio/middleware';
+import { SearchMemoryIndexer } from '@verdaccio/search';
 
 import loadPlugin from '../../lib/plugin-loader';
-import Search from '../../lib/search';
 import webApi from './api';
 
 const debug = buildDebug('verdaccio:web');
 
 export function loadTheme(config) {
   if (_.isNil(config.theme) === false) {
+    debug('loading custom ui theme');
     return _.head(
       loadPlugin(
         config,
@@ -26,9 +27,10 @@ export function loadTheme(config) {
   }
 }
 
-export default (config, auth, storage) => {
+export default async (config, auth, storage) => {
   const pluginOptions = loadTheme(config) || require('@verdaccio/ui-theme')();
-  Search.configureStorage(storage);
+  SearchMemoryIndexer.configureStorage(storage);
+  await SearchMemoryIndexer.init();
   // eslint-disable-next-line new-cap
   const router = express.Router();
   // load application

src/lib/storage-utils.ts

@@ -1,21 +1,16 @@
 import _ from 'lodash';
 
 import { pkgUtils } from '@verdaccio/core';
-import {
-  AbbreviatedManifest,
-  AbbreviatedVersions,
-  Manifest,
-  Package,
-  Version,
-} from '@verdaccio/types';
+import { SearchMemoryIndexer } from '@verdaccio/search';
+import { AbbreviatedManifest, AbbreviatedVersions, Manifest, Version } from '@verdaccio/types';
 import { generateRandomHexString } from '@verdaccio/utils';
 
 import { API_ERROR, DIST_TAGS, HTTP_STATUS, STORAGE, USERS } from './constants';
 import LocalStorage from './local-storage';
-import Search from './search';
-import { ErrorCode, isObject, normalizeDistTags, semverSort } from './utils';
+import { logger } from './logger';
+import { ErrorCode, isObject, normalizeDistTags } from './utils';
 
-export function generatePackageTemplate(name: string): Package {
+export function generatePackageTemplate(name: string): Manifest {
   return {
     // standard things
     name,
@@ -34,7 +29,7 @@ export function generatePackageTemplate(name: string): Package {
  * Normalize package properties, tags, revision id.
  * @param {Object} pkg package reference.
  */
-export function normalizePackage(pkg: Package): Package {
+export function normalizePackage(pkg: Manifest): Manifest {
   const pkgProperties = ['versions', 'dist-tags', '_distfiles', '_attachments', '_uplinks', 'time'];
 
   pkgProperties.forEach((key): void => {
@@ -65,7 +60,7 @@ export function generateRevision(rev: string): string {
   return (+_rev[0] || 0) + 1 + '-' + generateRandomHexString();
 }
 
-export function getLatestReadme(pkg: Package): string {
+export function getLatestReadme(pkg: Manifest): string {
   const versions = pkg['versions'] || {};
   const distTags = pkg[DIST_TAGS] || {};
   // FIXME: here is a bit tricky add the types
@@ -107,7 +102,7 @@ export const WHITELIST = [
   'users',
 ];
 
-export function cleanUpLinksRef(keepUpLinkData: boolean, result: Package): Package {
+export function cleanUpLinksRef(keepUpLinkData: boolean, result: Manifest): Manifest {
   const propertyToKeep = [...WHITELIST];
   if (keepUpLinkData === true) {
     propertyToKeep.push('_uplinks');
@@ -152,7 +147,9 @@ export function publishPackage(
       if (!_.isNull(err)) {
         return reject(err);
       } else if (!_.isUndefined(latest)) {
-        Search.add(latest);
+        SearchMemoryIndexer.add(latest).catch((reason) => {
+          logger.error('indexer has failed on add item');
+        });
       }
       return resolve();
     });