From 9631d8d57dc0b8ff0c7ed9748c8777de0253decf Mon Sep 17 00:00:00 2001 From: Niv Greenstein Date: Thu, 16 May 2024 16:20:18 +0300 Subject: [PATCH 01/11] feat: changed config to support multiple elastic clients --- config/default.json | 31 ++++++++++++++++++++++--------- config/test.json | 31 ++++++++++++++++++++++--------- 2 files changed, 44 insertions(+), 18 deletions(-) diff --git a/config/default.json b/config/default.json index 10347881..4e6cb631 100644 --- a/config/default.json +++ b/config/default.json @@ -27,16 +27,29 @@ }, "db": { "elastic": { - "node": "http://localhost:9200", - "auth": { - "username": "elastic", - "password": "changeme" + "searchy": { + "node": "http://localhost:9200", + "auth": { + "username": "elastic", + "password": "changeme" + }, + "requestTimeout": 60000, + "properties": { + "index": "control_gil_v5", + "size": 3 + } }, - "requestTimeout": 60000, - "properties": { - "controlIndex": "control_gil_v5", - "nlpIndex": "nlp_gil_v5", - "size": 3 + "nlp": { + "node": "http://localhost:9200", + "auth": { + "username": "elastic", + "password": "changeme" + }, + "requestTimeout": 60000, + "properties": { + "index": "nlp_gil_v5", + "size": 3 + } } }, "postgresql": { diff --git a/config/test.json b/config/test.json index 3654706c..8ab5e167 100644 --- a/config/test.json +++ b/config/test.json @@ -1,16 +1,29 @@ { "db": { "elastic": { - "node": "http://localhost:9200", - "auth": { - "username": "elastic", - "password": "changeme" + "searchy": { + "node": "http://localhost:9200", + "auth": { + "username": "elastic", + "password": "changeme" + }, + "requestTimeout": 60000, + "properties": { + "index": "control_gil_v5", + "size": 3 + } }, - "requestTimeout": 60000, - "properties": { - "controlIndex": "geocoder_control_tests", - "nlpIndex": "geocoder_control_tests", - "size": 3 + "nlp": { + "node": "http://localhost:9200", + "auth": { + "username": "elastic", + "password": "changeme" + }, + "requestTimeout": 60000, + "properties": { + "index": "nlp_gil_v5", + "size": 3 + } } }, "postgresql": { From 35efd66305598b8c7e186ae660a9d75d8fd6888b Mon Sep 17 00:00:00 2001 From: Niv Greenstein Date: Thu, 16 May 2024 16:20:55 +0300 Subject: [PATCH 02/11] feat: added new types to support multiple clients --- src/common/interfaces.ts | 8 +++++++- 1 file changed, 7 insertions(+), 1 deletion(-) diff --git a/src/common/interfaces.ts b/src/common/interfaces.ts index ca49a24a..73ec269d 100644 --- a/src/common/interfaces.ts +++ b/src/common/interfaces.ts @@ -1,4 +1,4 @@ -import { ClientOptions } from '@elastic/elasticsearch'; +import { Client, ClientOptions } from '@elastic/elasticsearch'; import { DataSourceOptions } from 'typeorm'; import { Feature, FeatureCollection as GeoJSONFeatureCollection } from 'geojson'; @@ -14,6 +14,10 @@ export interface OpenApiConfig { uiPath: string; } +export type ElasticDbClientsConfig = { + [key in 'searchy' | 'nlp']: ElasticDbConfig; +}; + export type ElasticDbConfig = ClientOptions; export type PostgresDbConfig = { @@ -21,6 +25,8 @@ export type PostgresDbConfig = { sslPaths: { ca: string; cert: string; key: string }; } & DataSourceOptions; +export type ElasticClients = Record; + export interface GeoContext { bbox?: number[]; radius?: number; From 14ef8f9cfcb2cb4b192cb0384dfeb6e060b66074 Mon Sep 17 00:00:00 2001 From: Niv Greenstein Date: Thu, 16 May 2024 16:21:56 +0300 Subject: [PATCH 03/11] feat: getting the configs from config file and initing elastic clients --- src/containerConfig.ts | 11 +++++++---- 1 file changed, 7 insertions(+), 4 deletions(-) diff --git a/src/containerConfig.ts b/src/containerConfig.ts index 973bee1d..d375f865 100644 --- a/src/containerConfig.ts +++ b/src/containerConfig.ts @@ -9,7 +9,7 @@ import { SERVICES, SERVICE_NAME } from './common/constants'; import { tracing } from './common/tracing'; import { InjectionObject, registerDependencies } from './common/dependencyRegistration'; import { elasticClientSymbol, initElasticsearchClient } from './common/elastic'; -import { ElasticDbConfig, PostgresDbConfig } from './common/interfaces'; +import { ElasticClients, ElasticDbClientsConfig, ElasticDbConfig, PostgresDbConfig } from './common/interfaces'; import { TILE_REPOSITORY_SYMBOL, tileRepositoryFactory } from './tile/DAL/tileRepository'; import { TILE_ROUTER_SYMBOL, tileRouterFactory } from './tile/routes/tileRouter'; import { ITEM_REPOSITORY_SYMBOL, itemRepositoryFactory } from './item/DAL/itemRepository'; @@ -38,9 +38,12 @@ export const registerExternalValues = async (options?: RegisterOptions): Promise tracing.start(); const tracer = trace.getTracer(SERVICE_NAME); - const elasticDataSourceOptions = config.get('db.elastic'); + const elasticClientsConfig = config.get('db.elastic'); const postgresqlDataSourceOptions = config.get('db.postgresql'); - const elasticClient = await initElasticsearchClient(elasticDataSourceOptions); + const elasticClients = {} as ElasticClients; + for (const [key, value] of Object.entries(elasticClientsConfig)) { + elasticClients[key as keyof ElasticDbClientsConfig] = await initElasticsearchClient(value as ElasticDbConfig); + } const postgresqlConnection = await initDataSource(postgresqlDataSourceOptions); const dependencies: InjectionObject[] = [ @@ -48,7 +51,7 @@ export const registerExternalValues = async (options?: RegisterOptions): Promise { token: SERVICES.LOGGER, provider: { useValue: logger } }, { token: SERVICES.TRACER, provider: { useValue: tracer } }, { token: SERVICES.METER, provider: { useValue: OtelMetrics.getMeterProvider().getMeter(SERVICE_NAME) } }, - { token: elasticClientSymbol, provider: { useValue: elasticClient } }, + { token: elasticClientSymbol, provider: { useValue: elasticClients } }, { token: DataSource, provider: { useValue: postgresqlConnection } }, { token: TILE_REPOSITORY_SYMBOL, provider: { useFactory: tileRepositoryFactory } }, { token: TILE_ROUTER_SYMBOL, provider: { useFactory: tileRouterFactory } }, From cf5a71700e4a5b3ca06b119aeb0a6fdd6e1b3b5b Mon Sep 17 00:00:00 2001 From: Niv Greenstein Date: Thu, 16 May 2024 16:22:31 +0300 Subject: [PATCH 04/11] feat: added util to get the wanted client query size --- src/common/utils.ts | 8 ++++++-- 1 file changed, 6 insertions(+), 2 deletions(-) diff --git a/src/common/utils.ts b/src/common/utils.ts index 2354277a..b6972c34 100644 --- a/src/common/utils.ts +++ b/src/common/utils.ts @@ -6,7 +6,7 @@ import { Tile } from '../tile/models/tile'; import { Route } from '../route/models/route'; import { FIELDS } from './constants'; import { utmProjection, wgs84Projection } from './projections'; -import { FeatureCollection, WGS84Coordinate } from './interfaces'; +import { ElasticClients, FeatureCollection, WGS84Coordinate } from './interfaces'; export const formatResponse = (elasticResponse: estypes.SearchResponse): FeatureCollection => ({ type: 'FeatureCollection', @@ -29,7 +29,7 @@ export const formatResponse = (elasticResponse: e /* eslint-disable @typescript-eslint/naming-convention */ export const additionalSearchProperties = (size: number): { size: number; index: string; _source: string[] } => ({ size, - index: config.get('db.elastic.properties.controlIndex'), + index: config.get('db.elastic.searchy.properties.index'), _source: FIELDS, }); /* eslint-enable @typescript-eslint/naming-convention */ @@ -96,3 +96,7 @@ export const validateTile = (tile: { tileName: string; subTileNumber: number[] } } return true; }; + +export const getElasticClientQuerySize = (key: keyof ElasticClients): number => { + return config.get(`db.elastic.${key}.properties.size`); +}; From 4ede1e3fb37dd9795150421b3672f042a4df47d7 Mon Sep 17 00:00:00 2001 From: Niv Greenstein Date: Thu, 16 May 2024 16:23:05 +0300 Subject: [PATCH 05/11] fix: specifying elastic desired client --- src/item/DAL/itemRepository.ts | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/src/item/DAL/itemRepository.ts b/src/item/DAL/itemRepository.ts index 0c62dad0..212a919a 100644 --- a/src/item/DAL/itemRepository.ts +++ b/src/item/DAL/itemRepository.ts @@ -3,6 +3,7 @@ import { FactoryFunction } from 'tsyringe'; import { elasticClientSymbol } from '../../common/elastic'; import { Item } from '../models/item'; import { additionalSearchProperties } from '../../common/utils'; +import { ElasticClients } from '../../common/interfaces'; import { ItemQueryParams, queryForItems } from './queries'; /* eslint-enable @typescript-eslint/naming-convention */ @@ -25,7 +26,7 @@ const createItemRepository = (client: Client) => { export type ItemRepository = ReturnType; export const itemRepositoryFactory: FactoryFunction = (depContainer) => { - return createItemRepository(depContainer.resolve(elasticClientSymbol)); + return createItemRepository(depContainer.resolve(elasticClientSymbol).searchy); }; export const ITEM_REPOSITORY_SYMBOL = Symbol('ITEM_REPOSITORY_SYMBOL'); From a3d5e91ce08b11d154fa04df6ecc7ce861c87300 Mon Sep 17 00:00:00 2001 From: Niv Greenstein Date: Thu, 16 May 2024 16:23:12 +0300 Subject: [PATCH 06/11] fix: specifying elastic desired client --- src/item/models/itemManager.ts | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/item/models/itemManager.ts b/src/item/models/itemManager.ts index 7833e7de..6720d175 100644 --- a/src/item/models/itemManager.ts +++ b/src/item/models/itemManager.ts @@ -19,7 +19,7 @@ export class ItemManager { public async getItems(itemQueryParams: ItemQueryParams, reduceFuzzyMatch = false, size?: number): Promise> { let elasticResponse: estypes.SearchResponse | undefined = undefined; - elasticResponse = await this.itemRepository.getItems(itemQueryParams, size ?? this.config.get('db.elastic.properties.size')); + elasticResponse = await this.itemRepository.getItems(itemQueryParams, size ?? this.config.get('db.elastic.searchy.properties.size')); const formattedResponse = formatResponse(elasticResponse); From eb05c89014f284ef702de952ebe7eaccfd625073 Mon Sep 17 00:00:00 2001 From: Niv Greenstein Date: Thu, 16 May 2024 16:23:19 +0300 Subject: [PATCH 07/11] fix: specifying elastic desired client --- src/route/DAL/routeRepository.ts | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/src/route/DAL/routeRepository.ts b/src/route/DAL/routeRepository.ts index db17bdd2..54432e3c 100644 --- a/src/route/DAL/routeRepository.ts +++ b/src/route/DAL/routeRepository.ts @@ -3,6 +3,7 @@ import { FactoryFunction } from 'tsyringe'; import { elasticClientSymbol } from '../../common/elastic'; import { Route } from '../models/route'; import { additionalSearchProperties } from '../../common/utils'; +import { ElasticClients } from '../../common/interfaces'; import { RouteQueryParams, queryForControlPointInRoute, queryForRoute } from './queries'; // eslint-disable-next-line @typescript-eslint/explicit-function-return-type @@ -34,7 +35,7 @@ const createRouteRepository = (client: Client) => { export type RouteRepository = ReturnType; export const routeRepositoryFactory: FactoryFunction = (depContainer) => { - return createRouteRepository(depContainer.resolve(elasticClientSymbol)); + return createRouteRepository(depContainer.resolve(elasticClientSymbol).searchy); }; export const ROUTE_REPOSITORY_SYMBOL = Symbol('ROUTE_REPOSITORY_SYMBOL'); From 6ed48dec1ff3338667c3ee55fd1e4299cdc162c6 Mon Sep 17 00:00:00 2001 From: Niv Greenstein Date: Thu, 16 May 2024 16:23:36 +0300 Subject: [PATCH 08/11] fix: specifying elastic desired client --- src/query/DAL/queryRepository.ts | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/src/query/DAL/queryRepository.ts b/src/query/DAL/queryRepository.ts index 0dc80848..0a9db34b 100644 --- a/src/query/DAL/queryRepository.ts +++ b/src/query/DAL/queryRepository.ts @@ -6,6 +6,7 @@ import { cleanQuery, fetchNLPService } from '../utils'; import { PlaceType, TextSearchParams, TokenResponse } from '../interfaces'; import { TextSearchHit } from '../models/textSearchHit'; import { BadRequestError } from '../../common/errors'; +import { ElasticClients } from '../../common/interfaces'; import { esQuery } from './queries'; /* eslint-enable @typescript-eslint/naming-convention */ @@ -29,7 +30,6 @@ const createQueryRepository = (client: Client) => { }, async getPlaceType(endpoint: string, query: string): Promise { - // const endpoint = config.get('services.placeTypeUrl'); const response = await fetchNLPService(endpoint, { text: query, start: 0, @@ -42,7 +42,7 @@ const createQueryRepository = (client: Client) => { }, async queryElastic(params: TextSearchParams): Promise> { - const index = config.get('db.elastic.properties.nlpIndex'); + const index = config.get('db.elastic.nlp.properties.index'); const response = await client.search({ index, @@ -57,7 +57,7 @@ const createQueryRepository = (client: Client) => { export type QueryRepository = ReturnType; export const queryRepositoryFactory: FactoryFunction = (depContainer) => { - return createQueryRepository(depContainer.resolve(elasticClientSymbol)); + return createQueryRepository(depContainer.resolve(elasticClientSymbol).nlp); }; export const QUERY_REPOSITORY_SYMBOL = Symbol('QUERY_REPOSITORY_SYMBOL'); From 22b10bd31757e649b6bb1d0bb0249d3759afcc8b Mon Sep 17 00:00:00 2001 From: Niv Greenstein Date: Thu, 16 May 2024 16:23:51 +0300 Subject: [PATCH 09/11] fix: specifying elastic desired client --- src/tile/DAL/tileRepository.ts | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/src/tile/DAL/tileRepository.ts b/src/tile/DAL/tileRepository.ts index 9232d768..f6caeba9 100644 --- a/src/tile/DAL/tileRepository.ts +++ b/src/tile/DAL/tileRepository.ts @@ -3,6 +3,7 @@ import { FactoryFunction } from 'tsyringe'; import { elasticClientSymbol } from '../../common/elastic'; import { Tile } from '../models/tile'; import { additionalSearchProperties } from '../../common/utils'; +import { ElasticClients } from '../../common/interfaces'; import { queryForTiles, queryForSubTiles, TileQueryParams } from './queries'; // eslint-disable-next-line @typescript-eslint/explicit-function-return-type @@ -32,7 +33,7 @@ const createTileRepository = (client: Client) => { export type TileRepository = ReturnType; export const tileRepositoryFactory: FactoryFunction = (depContainer) => { - return createTileRepository(depContainer.resolve(elasticClientSymbol)); + return createTileRepository(depContainer.resolve(elasticClientSymbol).searchy); }; export const TILE_REPOSITORY_SYMBOL = Symbol('TILE_REPOSITORY_SYMBOL'); From 0287b09ff173117d88127be86d3cf863be2662f0 Mon Sep 17 00:00:00 2001 From: Niv Greenstein Date: Thu, 16 May 2024 16:24:34 +0300 Subject: [PATCH 10/11] fix: utilizing getElasticClientQuerySize function --- src/route/models/routeManager.ts | 6 +++--- src/tile/models/tileManager.ts | 4 ++-- 2 files changed, 5 insertions(+), 5 deletions(-) diff --git a/src/route/models/routeManager.ts b/src/route/models/routeManager.ts index 31a47127..207e4158 100644 --- a/src/route/models/routeManager.ts +++ b/src/route/models/routeManager.ts @@ -5,7 +5,7 @@ import { estypes } from '@elastic/elasticsearch'; import { SERVICES } from '../../common/constants'; import { ROUTE_REPOSITORY_SYMBOL, RouteRepository } from '../DAL/routeRepository'; import { RouteQueryParams } from '../DAL/queries'; -import { formatResponse } from '../../common/utils'; +import { formatResponse, getElasticClientQuerySize } from '../../common/utils'; import { FeatureCollection } from '../../common/interfaces'; import { Route } from './route'; @@ -22,10 +22,10 @@ export class RouteManager { if (routeQueryParams.controlPoint ?? 0) { elasticResponse = await this.routeRepository.getControlPointInRoute( routeQueryParams as RouteQueryParams & Required>, - size ?? this.config.get('db.elastic.properties.size') + size ?? getElasticClientQuerySize('searchy') ); } else { - elasticResponse = await this.routeRepository.getRoutes(routeQueryParams, size ?? this.config.get('db.elastic.properties.size')); + elasticResponse = await this.routeRepository.getRoutes(routeQueryParams, size ?? getElasticClientQuerySize('searchy')); } const formattedResponse = formatResponse(elasticResponse); diff --git a/src/tile/models/tileManager.ts b/src/tile/models/tileManager.ts index 881217dc..0a407acd 100644 --- a/src/tile/models/tileManager.ts +++ b/src/tile/models/tileManager.ts @@ -4,7 +4,7 @@ import { inject, injectable } from 'tsyringe'; import { estypes } from '@elastic/elasticsearch'; import { SERVICES } from '../../common/constants'; import { TILE_REPOSITORY_SYMBOL, TileRepository } from '../DAL/tileRepository'; -import { formatResponse } from '../../common/utils'; +import { formatResponse, getElasticClientQuerySize } from '../../common/utils'; import { TileQueryParams } from '../DAL/queries'; import { FeatureCollection } from '../../common/interfaces'; import { Tile } from './tile'; @@ -19,7 +19,7 @@ export class TileManager { public async getTiles(tileQueryParams: TileQueryParams, reduceFuzzyMatch = false, size?: number): Promise> { let elasticResponse: estypes.SearchResponse | undefined = undefined; - const numberOfResults = size ?? this.config.get('db.elastic.properties.size'); + const numberOfResults = size ?? getElasticClientQuerySize('searchy'); if (tileQueryParams.subTile ?? 0) { elasticResponse = await this.tileRepository.getSubTiles(tileQueryParams as Required, numberOfResults); } else { From 406a38936f24fa1e0c10c5e6aa84c49ee3083599 Mon Sep 17 00:00:00 2001 From: Niv Greenstein Date: Thu, 16 May 2024 16:25:05 +0300 Subject: [PATCH 11/11] fix: specifying searchy --- devScripts/importDataToElastic.ts | 6 +++--- tests/unit/common/utils.spec.ts | 3 ++- 2 files changed, 5 insertions(+), 4 deletions(-) diff --git a/devScripts/importDataToElastic.ts b/devScripts/importDataToElastic.ts index 9836e52d..951964a0 100644 --- a/devScripts/importDataToElastic.ts +++ b/devScripts/importDataToElastic.ts @@ -6,12 +6,12 @@ import config from '../config/test.json'; import data from './elasticsearchData.json'; const main = async (): Promise => { - const client = new Client({ node: config.db.elastic.node as string }); + const client = new Client({ node: config.db.elastic.searchy.node as string }); for (const item of data) { await client.index({ - index: config.db.elastic.properties.controlIndex as string, - id: item._id as string, + index: config.db.elastic.searchy.properties.index as string, + id: item._id, body: item._source, }); } diff --git a/tests/unit/common/utils.spec.ts b/tests/unit/common/utils.spec.ts index df63e54c..1913381b 100644 --- a/tests/unit/common/utils.spec.ts +++ b/tests/unit/common/utils.spec.ts @@ -36,7 +36,8 @@ describe('utils', () => { it('should return additional search properties', () => { const size = 10; const result = additionalSearchProperties(size); - expect(result).toMatchObject({ size, index: config.db.elastic.properties.controlIndex, _source: FIELDS }); + // eslint-disable-next-line @typescript-eslint/no-unsafe-member-access + expect(result).toMatchObject({ size, index: config.db.elastic.searchy.properties.index as string, _source: FIELDS }); }); it('should convert UTM to WGS84', () => {