feat: Add a new search implementation (#1984)

* fix: working on grammar updates for Words

* fix: restore tests for legacy search, add search2 tests in new file. all tests passing

* fix: cleanup

* fix: re-add slop

* fix: copyright + tests

* fix: pass verify file headers

* fix: index names with simple analyzer, add fields

* fix: symmetric analyzers, default to linter, remove dbeug logs

packages/backend/indices/iex-insights.json

@@ -130,6 +130,10 @@
             "type": "keyword",
             "ignore_above": 512,
             "normalizer": "lowercase_normalizer"
+          },
+          "simple": {
+            "type": "text",
+            "analyzer": "simple"
           }
         }
       },
@@ -278,10 +282,7 @@
       "normalizer": {
         "lowercase_normalizer": {
           "type": "custom",
-          "filter": [
-            "lowercase",
-            "trim"
-          ]
+          "filter": ["lowercase", "trim"]
         }
       }
     }

packages/backend/schema.gql

@@ -332,6 +332,7 @@ input InsightSearch {
   paging: Paging
   query: String!
   sort: [Sort!]
+  useNewSearch: Boolean!
 }
 
 type InsightSearchResults {

packages/backend/src/lib/elasticsearch.ts

@@ -21,7 +21,18 @@ import { GetResponse, MgetResponse, SearchBody, SearchResponse } from '@iex/mode
 import { IndexedInsight } from '@iex/models/indexed/indexed-insight';
 import { ItemType } from '@iex/models/item-type';
 import { getLogger } from '@iex/shared/logger';
-import { parseToElasticsearch, SearchMultiTerm, SearchNestedOrFilter, SearchTerm } from '@iex/shared/search';
+import {
+  parseToElasticsearch as parseToElasticsearchOld,
+  SearchMultiTerm as SearchMultiTermOld,
+  SearchNestedOrFilter as SearchNestedOrFilterOld,
+  SearchTerm as SearchTermOld
+} from '@iex/shared/search';
+import {
+  parseToElasticsearch as parseToElasticsearchNew,
+  SearchMultiTerm as SearchMultiTermNew,
+  SearchNestedOrFilter as SearchNestedOrFilterNew,
+  SearchTerm as SearchTermNew
+} from '@iex/shared/search2';
 import { detailedDiff } from 'deep-object-diff';
 import { DateTime } from 'luxon';
 
@@ -321,6 +332,10 @@ export async function searchInsights(
 
   if (search != null) {
     // Parse an IEX search into Elasticsearch query
+    const parseToElasticsearch = search.useNewSearch ? parseToElasticsearchNew : parseToElasticsearchOld;
+    const SearchMultiTerm = search.useNewSearch ? SearchMultiTermNew : SearchMultiTermOld;
+    const SearchNestedOrFilter = search.useNewSearch ? SearchNestedOrFilterNew : SearchNestedOrFilterOld;
+    const SearchTerm = search.useNewSearch ? SearchTermNew : SearchTermOld;
     query.body!.query = parseToElasticsearch(search.query, (clauses) => {
       // This modifier function runs after parsing but before converting to Elasticsearch
 

packages/backend/src/models/insight-search.ts

@@ -38,6 +38,9 @@ export class InsightSearch {
   @Field()
   query!: string;
 
+  @Field()
+  useNewSearch!: boolean;
+
   @Field(() => [Sort], { nullable: true })
   sort?: Sort[];
 

packages/frontend/src/pages/search-page/components/search-bar/search-bar.tsx

@@ -24,6 +24,7 @@ import {
   MenuItemOption,
   MenuList,
   MenuOptionGroup,
+  Switch,
   Tooltip
 } from '@chakra-ui/react';
 import type { ReactElement } from 'react';
@@ -52,12 +53,18 @@ const availableSortFields = [
 
 export const SearchBar = (): ReactElement => {
   const dispatch = useDispatch<AppDispatch>();
-
-  const { query, sort, showFilters, isFiltered, options } = useSelector((state: RootState) => state.search);
+  const { query, useNewSearch, sort, showFilters, isFiltered, options } = useSelector(
+    (state: RootState) => state.search
+  );
 
   const [internalQuery, setInternalQuery] = useState(query);
+
   const previousQueryRef = useRef(query);
 
+  const setUseNewSearch = (value: boolean) => {
+    dispatch(searchSlice.actions.setUseNewSearch(value));
+  };
+
   const toggleShowFilters = () => {
     dispatch(searchSlice.actions.setShowFilters(!showFilters));
   };
@@ -124,6 +131,16 @@ export const SearchBar = (): ReactElement => {
         canClear={query.length > 0 || sort !== undefined}
       />
 
+      <Tooltip placement="left" label="Use new search" aria-label="Use new search" zIndex="10">
+        <Switch
+          id="enable-new-search"
+          colorScheme="nord8"
+          aria-label="Use New Search"
+          isChecked={useNewSearch}
+          onChange={(event) => setUseNewSearch(event?.target.checked)}
+        />
+      </Tooltip>
+
       <SearchSyntax />
 
       <Menu>

packages/frontend/src/pages/search-page/search-page.tsx

@@ -35,7 +35,7 @@ import { SearchBar } from './components/search-bar/search-bar';
 export const SearchPage = () => {
   const initialized = useRef(false);
   const dispatch = useDispatch<AppDispatch>();
-  const { query, sort, showFilters, options } = useSelector((state: RootState) => state.search);
+  const { query, useNewSearch, sort, showFilters, options } = useSelector((state: RootState) => state.search);
 
   // Load query from params
   const { query: queryFromUrl } = useParams();
@@ -45,7 +45,8 @@ export const SearchPage = () => {
   const [{ data, error, fetching, hasMore, total }, fetchMore] = useSearch({
     query,
     sort,
-    paused: !initialized.current
+    paused: !initialized.current,
+    useNewSearch
   });
 
   const insightResults = data.insights.results.map(({ insight }) => {
@@ -74,9 +75,9 @@ export const SearchPage = () => {
       }
     }
 
-    const url = '/search' + generateSearchUrl(query, sort);
+    const url = '/search' + generateSearchUrl(query, sort, useNewSearch);
     navigate(url, { replace: true });
-  }, [dispatch, navigate, query, queryFromUrl, searchParams, sort]);
+  }, [dispatch, navigate, query, useNewSearch, queryFromUrl, searchParams, sort]);
 
   useDebounce(
     () => {

packages/frontend/src/shared/search-url.ts

@@ -16,7 +16,11 @@
 
 import type { Sort } from '../models/generated/graphql';
 
-export const generateSearchUrl = (query: string | undefined, sort: Sort | undefined) => {
+export const generateSearchUrl = (
+  query: string | undefined,
+  sort: Sort | undefined,
+  useNewSearch?: boolean | undefined
+) => {
   const path = `/${encodeURIComponent(query || '')}`;
   const searchParams: string[] = [];
   if (sort != null) {
@@ -27,6 +31,9 @@ export const generateSearchUrl = (query: string | undefined, sort: Sort | undefi
       searchParams.push(`dir=${sort.direction}`);
     }
   }
+  if (useNewSearch === false) {
+    searchParams.push(`legacySearch=true`);
+  }
 
   return searchParams.length > 0 ? `${path}?${searchParams.join('&')}` : path;
 };

packages/frontend/src/shared/useSearch.ts

@@ -81,6 +81,7 @@ const INSIGHTS_QUERY = gql`
 
 export interface UseSearchProps {
   query?: string;
+  useNewSearch?: boolean;
   sort?: Sort;
   paused?: boolean;
 }
@@ -122,7 +123,7 @@ export type UseSearchReturnType = [SearchResultState, () => Promise<void>];
  * @param {boolean} [options.paused=false] - Whether the search is paused.
  * @returns {UseSearchReturnType} - The search result state and a function to fetch more results.
  */
-export function useSearch({ query, sort, paused = false }: UseSearchProps): UseSearchReturnType {
+export function useSearch({ query, useNewSearch = true, sort, paused = false }: UseSearchProps): UseSearchReturnType {
   const [fetching, setFetching] = useState(true);
 
   const [suggestedFilters, setSuggestedFilters] = useState<AutocompleteResults | undefined>();
@@ -142,7 +143,7 @@ export function useSearch({ query, sort, paused = false }: UseSearchProps): UseS
     }
 
     // Generate a unique ID for this request so we can ignore old responses
-    const requestId = `r||${query}||${sort}||${from.current}`;
+    const requestId = `r||${query}||${sort}||${from.current}||${useNewSearch}`;
     if (latestRequest.current === requestId) {
       // Ignore duplicate requests
       // The infinite scroll component may trigger multiple requests for the same page
@@ -155,6 +156,7 @@ export function useSearch({ query, sort, paused = false }: UseSearchProps): UseS
       .query(INSIGHTS_QUERY, {
         search: {
           query: query || '',
+          useNewSearch,
           sort: sort && [sort],
           paging: {
             from: from.current,
@@ -188,7 +190,7 @@ export function useSearch({ query, sort, paused = false }: UseSearchProps): UseS
 
     // Done!
     setFetching(false);
-  }, [paused, query, sort]);
+  }, [paused, query, useNewSearch, sort]);
 
   useEffect(() => {
     // Reset the scroll state whenever query/sort changes

packages/frontend/src/store/search.slice.ts

@@ -28,6 +28,7 @@ export interface SearchOptions {
 
 export interface SearchState {
   query: string;
+  useNewSearch: boolean;
   sort?: Sort;
   paging?: Paging;
 
@@ -43,6 +44,7 @@ export interface SearchState {
 
 const initialState: SearchState = {
   query: '',
+  useNewSearch: true,
   sort: undefined,
   paging: undefined,
   isFiltered: false,
@@ -62,6 +64,9 @@ export const searchSlice = createSlice({
     setQuery(state, action: PayloadAction<string>) {
       state.query = action.payload;
     },
+    setUseNewSearch(state, action: PayloadAction<boolean>) {
+      state.useNewSearch = action.payload;
+    },
     setSort(state, action: PayloadAction<Sort | undefined>) {
       state.sort = action.payload;
     },
@@ -105,7 +110,10 @@ export const searchSlice = createSlice({
     },
     parseUrlIntoState(
       state,
-      action: PayloadAction<{ query: string | undefined; searchParams: { sort?: string; dir?: string } }>
+      action: PayloadAction<{
+        query: string | undefined;
+        searchParams: { sort?: string; dir?: string; legacySearch?: boolean };
+      }>
     ) {
       const { query, searchParams } = action.payload;
       // Take URL only if not empty
@@ -127,6 +135,11 @@ export const searchSlice = createSlice({
         modified = true;
       }
 
+      if (searchParams.legacySearch) {
+        state.useNewSearch = false;
+        modified = true;
+      }
+
       // If query provided but sort not modified, clear the existing sort options
       if (query && !modified) {
         state.sort = undefined;

packages/shared/src/search.ts

@@ -457,7 +457,6 @@ const lang = Parsimmon.createLanguage({
  */
 export function parseSearchQuery(searchQuery: string): SearchClause[] {
   const clauses: SearchClause[] = lang.Query.tryParse(searchQuery);
-
   return clauses;
 }