From 419fe420104c49cdb94c969f104f2da17fdf950b Mon Sep 17 00:00:00 2001 From: Niv Greenstein Date: Tue, 23 Jul 2024 20:26:31 +0300 Subject: [PATCH 01/67] feat: updated config files and added support for custom envs --- config/custom-environment-variables.json | 62 ++++++++++++++++++++++++ config/default.json | 48 ++++++++++++------ 2 files changed, 95 insertions(+), 15 deletions(-) diff --git a/config/custom-environment-variables.json b/config/custom-environment-variables.json index 75b4c82e..3bdd69b8 100644 --- a/config/custom-environment-variables.json +++ b/config/custom-environment-variables.json @@ -38,5 +38,67 @@ } } } + }, + "db": { + "elastic": { + "control": { + "node": "ELASTIC_CONTROL_URL", + "auth": { + "username": "ELASTIC_CONTROL_USERNAME", + "password": "ELASTIC_CONTROL_PASSWORD" + }, + "properties": { + "index": "ELASTIC_CONTROL_INDEX" + } + }, + "geotext": { + "node": "ELASTIC_GEOTEXT_URL", + "auth": { + "username": "ELASTIC_GEOTEXT_USERNAME", + "password": "ELASTIC_GEOTEXT_PASSWORD" + }, + "properties": { + "index": { + "geotext": "ELASTIC_GEOTEXT_DATA_INDEX", + "placetypes": "ELASTIC_GEOTEXT_PLACETYPES_INDEX", + "hierarchies": "ELASTIC_GEOTEXT_HIERARCHIES_INDEX" + }, + "textTermLanguage": "ELASTIC_GEOTEXT_TEXT_TERM_LANGUAGE" + } + } + }, + "postgresql": { + "host": "POSTGRES_URL", + "port": { + "__name": "POSTGRES_PORT", + "__format": "number" + }, + "username": "POSTGRES_USERNAME", + "password": "POSTGRES_PASSWORD", + "enableSslAuth": { + "__name": "POSTGRES_ENABLE_SLL_AUTH", + "__format": "boolean" + }, + "sslPaths": { + "ca": "POSTGRES_CA_PATH", + "key": "POSTGRES_KEY_PATH", + "cert": "POSTGRES_CERT_PATH" + }, + "database": "POSTGRES_DB_NAME", + "schema": "POSTGRES_DB_SCHEMA" + } + }, + "application": { + "services": { + "tokenTypesUrl": "TOKEN_TYPE_URL" + }, + "sources": { + "__name": "GEOTEXT_SOURCES", + "__format": "json" + }, + "regions": { + "__name": "REGIONS", + "__format": "json" + } } } diff --git a/config/default.json b/config/default.json index 4e6cb631..e4d32bb9 100644 --- a/config/default.json +++ b/config/default.json @@ -27,28 +27,33 @@ }, "db": { "elastic": { - "searchy": { + "control": { "node": "http://localhost:9200", "auth": { - "username": "elastic", - "password": "changeme" + "username": "control", + "password": "password" }, "requestTimeout": 60000, "properties": { - "index": "control_gil_v5", - "size": 3 + "index": "control_index", + "defaultResponseLimit": 3 } }, - "nlp": { + "geotext": { "node": "http://localhost:9200", "auth": { - "username": "elastic", - "password": "changeme" + "username": "geotext", + "password": "password" }, "requestTimeout": 60000, "properties": { - "index": "nlp_gil_v5", - "size": 3 + "index": { + "geotext": "geotext_index", + "placetypes": "placetypes_index", + "hierarchies": "hierarchies_index" + }, + "defaultResponseLimit": 3, + "textTermLanguage": "en" } } }, @@ -68,9 +73,22 @@ "schema": "geocoder" } }, - "services": { - "placeTypeUrl": "http://example.com", - "tokenTypesUrl": "http://example.com" - }, - "cronLoadTileLatLonDataPattern": "0 * * * *" + "application": { + "services": { + "tokenTypesUrl": "http://NLP_ANALYSES" + }, + "cronLoadTileLatLonDataPattern": "0 * * * *", + "elasticQueryBoosts": { + "name": 1.1, + "placeType": 1.1, + "subPlaceType": 1.1, + "hierarchy": 1.1, + "viewbox": 1.1 + }, + "sources": { + "SOURCE_A": "a", + "SOURCE_B": "b" + }, + "regions": {} + } } From 55b4c653a7227e08061b731f625964560186f19c Mon Sep 17 00:00:00 2001 From: Niv Greenstein Date: Tue, 23 Jul 2024 20:31:10 +0300 Subject: [PATCH 02/67] fix: added properties to elastic client init and error handling --- src/common/elastic/index.ts | 27 ++++++++++++++++----------- 1 file changed, 16 insertions(+), 11 deletions(-) diff --git a/src/common/elastic/index.ts b/src/common/elastic/index.ts index 364423c9..b3a0d7fd 100644 --- a/src/common/elastic/index.ts +++ b/src/common/elastic/index.ts @@ -1,17 +1,22 @@ import { Client, ClientOptions } from '@elastic/elasticsearch'; -let client: Client | null = null; - -export const initElasticsearchClient = async (clientOptions: ClientOptions): Promise => { - if (!client) { - client = new Client(clientOptions); - try { - await client.ping(); - } catch (error) { - console.error('Elasticsearch cluster is down!', error); - throw error; - } +export const initElasticsearchClient = async (clientOptions: ClientOptions): Promise => { + const client = new Client({ + ...clientOptions, + sniffOnStart: false, + sniffOnConnectionFault: false, + tls: { + rejectUnauthorized: false, + }, + }); + try { + await client.ping(); + console.log('Elastic connection established to:', clientOptions.node); + } catch (error) { + console.error("Can't connect to Elasticseach!", clientOptions.node, error); + return null; } + return client; }; From 6be0e442ed2bf5235a148a07f85ea10ef9400fb4 Mon Sep 17 00:00:00 2001 From: Niv Greenstein Date: Tue, 23 Jul 2024 20:31:59 +0300 Subject: [PATCH 03/67] feat: added application and the path to elastic in the config file --- src/common/constants.ts | 3 +++ 1 file changed, 3 insertions(+) diff --git a/src/common/constants.ts b/src/common/constants.ts index b6a7dd88..16c3d931 100644 --- a/src/common/constants.ts +++ b/src/common/constants.ts @@ -12,6 +12,7 @@ export const SERVICES: Record = { CONFIG: Symbol('Config'), TRACER: Symbol('Tracer'), METER: Symbol('Meter'), + APPLICATION: Symbol('Application'), }; /* eslint-enable @typescript-eslint/naming-convention */ @@ -28,3 +29,5 @@ export const FIELDS = [ 'properties.SUB_TILE_ID', 'properties.SECTION', ]; + +export const elasticConfigPath = 'db.elastic'; From 8339e03a5fbbe8cf8d902ae5f8b2cce5e611ac7e Mon Sep 17 00:00:00 2001 From: Niv Greenstein Date: Tue, 23 Jul 2024 20:32:23 +0300 Subject: [PATCH 04/67] feat: added common types --- src/common/interfaces.ts | 32 +++++++++++++++++++++++++++++++- 1 file changed, 31 insertions(+), 1 deletion(-) diff --git a/src/common/interfaces.ts b/src/common/interfaces.ts index 73ec269d..3d8411cd 100644 --- a/src/common/interfaces.ts +++ b/src/common/interfaces.ts @@ -15,7 +15,17 @@ export interface OpenApiConfig { } export type ElasticDbClientsConfig = { - [key in 'searchy' | 'nlp']: ElasticDbConfig; + [key in 'control' | 'geotext']: ElasticDbConfig & { + properties: { + index: + | string + | { + [key: string]: string; + }; + defaultResponseLimit: number; + textTermLanguage: string; + }; + }; }; export type ElasticDbConfig = ClientOptions; @@ -47,3 +57,23 @@ export interface WGS84Coordinate { lat: number; lon: number; } + +export interface IApplication { + services: { + tokenTypesUrl: string; + }; + cronLoadTileLatLonDataPattern: string; + elasticQueryBoosts: { + name: number; + placeType: number; + subPlaceType: number; + hierarchy: number; + viewbox: number; + }; + sources?: { + [key: string]: string; + }; + regions?: { + [key: string]: string[]; + }; +} From ddf2739ce9de08d554003cf3fd8b0fe1caaca4de Mon Sep 17 00:00:00 2001 From: Niv Greenstein Date: Tue, 23 Jul 2024 20:33:09 +0300 Subject: [PATCH 05/67] fix: added types and constants --- src/common/utils.ts | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/src/common/utils.ts b/src/common/utils.ts index b6972c34..f50b5eae 100644 --- a/src/common/utils.ts +++ b/src/common/utils.ts @@ -4,9 +4,9 @@ import proj4 from 'proj4'; import { Item } from '../item/models/item'; import { Tile } from '../tile/models/tile'; import { Route } from '../route/models/route'; -import { FIELDS } from './constants'; +import { FIELDS, elasticConfigPath } from './constants'; import { utmProjection, wgs84Projection } from './projections'; -import { ElasticClients, FeatureCollection, WGS84Coordinate } from './interfaces'; +import { ElasticClients, ElasticDbClientsConfig, FeatureCollection, WGS84Coordinate } from './interfaces'; export const formatResponse = (elasticResponse: estypes.SearchResponse): FeatureCollection => ({ type: 'FeatureCollection', @@ -29,7 +29,7 @@ export const formatResponse = (elasticResponse: e /* eslint-disable @typescript-eslint/naming-convention */ export const additionalSearchProperties = (size: number): { size: number; index: string; _source: string[] } => ({ size, - index: config.get('db.elastic.searchy.properties.index'), + index: config.get(elasticConfigPath).control.properties.index as string, _source: FIELDS, }); /* eslint-enable @typescript-eslint/naming-convention */ @@ -98,5 +98,5 @@ export const validateTile = (tile: { tileName: string; subTileNumber: number[] } }; export const getElasticClientQuerySize = (key: keyof ElasticClients): number => { - return config.get(`db.elastic.${key}.properties.size`); + return config.get(elasticConfigPath)[key].properties.defaultResponseLimit; }; From c26a0f9bdf13282678c682d91f5713be1754865d Mon Sep 17 00:00:00 2001 From: Niv Greenstein Date: Tue, 23 Jul 2024 20:33:30 +0300 Subject: [PATCH 06/67] feat: created geotext controller --- .../controllers/geotextSearchController.ts | 44 +++++++++++++++++++ 1 file changed, 44 insertions(+) create mode 100644 src/geotextSearch/controllers/geotextSearchController.ts diff --git a/src/geotextSearch/controllers/geotextSearchController.ts b/src/geotextSearch/controllers/geotextSearchController.ts new file mode 100644 index 00000000..b31aaf42 --- /dev/null +++ b/src/geotextSearch/controllers/geotextSearchController.ts @@ -0,0 +1,44 @@ +import { Logger } from "@map-colonies/js-logger"; +import { BoundCounter, Meter } from "@opentelemetry/api-metrics"; +import { RequestHandler } from "express"; +import httpStatus from "http-status-codes"; +import { injectable, inject } from "tsyringe"; +import { SERVICES } from "../../common/constants"; +import { GeotextSearchManager } from "../models/queryManager"; +import { GetQueryQueryParams, QueryResult } from "../interfaces"; + +type GetGeotextSearchHandler = RequestHandler< + unknown, + QueryResult | { message: string; error: string }, //response + undefined, + GetQueryQueryParams +>; + +type GetRegionshHandler = RequestHandler; + +@injectable() +export class geotextSearchController { + private readonly createdResourceCounter: BoundCounter; + + public constructor( + @inject(SERVICES.LOGGER) private readonly logger: Logger, + @inject(GeotextSearchManager) private readonly manager: GeotextSearchManager, + @inject(SERVICES.METER) private readonly meter: Meter + ) { + this.createdResourceCounter = meter.createCounter("created_resource"); + } + + public getGeotextSearch: GetGeotextSearchHandler = async (req, res, next) => { + try { + const response = await this.manager.search(req.query); + return res.status(httpStatus.OK).json(response); + } catch (error: unknown) { + this.logger.error("Error in getGeotextSearch", error); + next(error); + } + }; + + public getRegions: GetRegionshHandler = (req, res, next) => { + return res.status(httpStatus.OK).json(this.manager.regions()); + }; +} From 4301fd9e862b7c283fd7208b5a950917dfaf49fc Mon Sep 17 00:00:00 2001 From: Niv Greenstein Date: Tue, 23 Jul 2024 20:39:42 +0300 Subject: [PATCH 07/67] feat: created queries --- src/geotextSearch/DAL/queries.ts | 198 +++++++++++++++++++++++++++++++ 1 file changed, 198 insertions(+) create mode 100644 src/geotextSearch/DAL/queries.ts diff --git a/src/geotextSearch/DAL/queries.ts b/src/geotextSearch/DAL/queries.ts new file mode 100644 index 00000000..28d98b65 --- /dev/null +++ b/src/geotextSearch/DAL/queries.ts @@ -0,0 +1,198 @@ +/* eslint-disable @typescript-eslint/naming-convention */ +import WKT, { GeoJSONPolygon } from 'wellknown'; +import { estypes } from '@elastic/elasticsearch'; +import { QueryDslQueryContainer } from '@elastic/elasticsearch/lib/api/types'; +import { TextSearchParams } from '../interfaces'; +import { IApplication } from '../../common/interfaces'; + +const TEXT_FIELD = 'text'; +const PLACETYPE_FIELD = 'placetype.keyword'; +const SUB_PLACETYPE_FIELD = 'sub_placetype.keyword'; +const GEOJSON_FIELD = 'geo_json'; +const SOURCE_FIELD = 'source'; +const REGION_FIELD = 'region'; +const HIERARCHY_FIELD = 'heirarchy'; +const PLACETYPE_SEARCH_FIELD = 'sub_placetype_keyword'; + +export const geotextQuery = ( + { query, limit: size, name, placeTypes, subPlaceTypes, hierarchies, regions, viewbox, boundary, sources }: TextSearchParams, + textLanguage: string, + boosts: IApplication['elasticQueryBoosts'] +): estypes.SearchRequest => { + const esQuery: estypes.SearchRequest = { + query: { + function_score: { + functions: [], + query: { + bool: { + must: [], + filter: [], + }, + }, + }, + }, + highlight: { + fields: {}, + }, + size, + }; + + if (!name && subPlaceTypes?.length) { + (esQuery.query?.function_score?.query?.bool?.must as QueryDslQueryContainer[]).push({ + terms: { + [SUB_PLACETYPE_FIELD]: subPlaceTypes, + }, + }); + + (esQuery.query?.function_score?.query?.bool?.must as QueryDslQueryContainer[]).push({ + term: { + text_language: textLanguage, + }, + }); + } else { + (esQuery.query?.function_score?.query?.bool?.must as QueryDslQueryContainer[]).push({ + match: { + [TEXT_FIELD]: { + query, + fuzziness: 'AUTO:3,4', + }, + }, + }); + } + + boundary && + (esQuery.query?.function_score?.query?.bool?.must as QueryDslQueryContainer[]).push({ + geo_shape: { + [GEOJSON_FIELD]: { + shape: boundary, + }, + }, + }); + + sources?.length && + (esQuery.query?.function_score?.query?.bool?.must as QueryDslQueryContainer[]).push({ + terms: { + [SOURCE_FIELD]: sources, + }, + }); + + regions?.length && + (esQuery.query?.function_score?.query?.bool?.must as QueryDslQueryContainer[]).push({ + terms: { + [REGION_FIELD]: regions, + }, + }); + + name && + esQuery.query?.function_score?.functions?.push({ + weight: boosts.name, + filter: { + match: { + [TEXT_FIELD]: name, + }, + }, + }); + + placeTypes?.length && + esQuery.query?.function_score?.functions?.push({ + weight: boosts.placeType, + filter: { + terms: { + [PLACETYPE_FIELD]: placeTypes, + }, + }, + }); + + subPlaceTypes?.length && + esQuery.query?.function_score?.functions?.push({ + weight: boosts.subPlaceType, + filter: { + terms: { + [SUB_PLACETYPE_FIELD]: subPlaceTypes, + }, + }, + }); + + viewbox && + esQuery.query?.function_score?.functions?.push({ + weight: boosts.viewbox, + filter: { + geo_shape: { + [GEOJSON_FIELD]: { + shape: viewbox, + }, + }, + }, + }); + + hierarchies.forEach((hierarchy) => { + const hierarchyGeoJSON = WKT.parse(hierarchy.geo_json); + const hierarchyShape = { + type: hierarchyGeoJSON!.type.toLowerCase(), + coordinates: (hierarchyGeoJSON as GeoJSONPolygon).coordinates, + }; + + esQuery.query?.function_score?.functions?.push({ + weight: hierarchy.weight, + filter: { + geo_shape: { + [GEOJSON_FIELD]: { + shape: hierarchyShape, + }, + }, + }, + }); + }); + + esQuery.highlight!.fields = { [TEXT_FIELD]: {} }; + + return esQuery; +}; + +export const placetypeQuery = (query: string): estypes.SearchRequest => ({ + query: { + bool: { + should: { + match: { + [PLACETYPE_SEARCH_FIELD]: { + query, + fuzziness: 'AUTO:3,4', + }, + }, + }, + }, + }, + size: 2, +}); + +export const hierarchyQuery = (query: string): estypes.SearchRequest => ({ + query: { + function_score: { + functions: [ + { + weight: 1.1, + filter: { + match: { + [HIERARCHY_FIELD]: { + query, + fuzziness: 'AUTO:3,4', + }, + }, + }, + }, + ], + query: { + bool: { + must: { + match: { + text: { + query, + fuzziness: 'AUTO:3,4', + }, + }, + }, + }, + }, + }, + }, +}); From 34f75d61ee8d8c9289bf8c76a0470656c1a69098 Mon Sep 17 00:00:00 2001 From: Niv Greenstein Date: Tue, 23 Jul 2024 20:40:44 +0300 Subject: [PATCH 08/67] feat: created geotext repository --- .../DAL/geotextSearchRepository.ts | 91 +++++++++++++++++++ 1 file changed, 91 insertions(+) create mode 100644 src/geotextSearch/DAL/geotextSearchRepository.ts diff --git a/src/geotextSearch/DAL/geotextSearchRepository.ts b/src/geotextSearch/DAL/geotextSearchRepository.ts new file mode 100644 index 00000000..0de836a6 --- /dev/null +++ b/src/geotextSearch/DAL/geotextSearchRepository.ts @@ -0,0 +1,91 @@ +import { Client, estypes } from '@elastic/elasticsearch'; +import { FactoryFunction } from 'tsyringe'; +import { elasticClientSymbol } from '../../common/elastic'; +import { cleanQuery, fetchNLPService } from '../utils'; +import { TextSearchParams, TokenResponse } from '../interfaces'; +import { PlaceTypeSearchHit, HierarchySearchHit, TextSearchHit } from '../models/textSearchHit'; +import { BadRequestError } from '../../common/errors'; +import { ElasticClients, IApplication } from '../../common/interfaces'; +import { hierarchyQuery, placetypeQuery, geotextQuery } from './queries'; + +/* eslint-enable @typescript-eslint/naming-convention */ + +// eslint-disable-next-line @typescript-eslint/explicit-function-return-type +const createGeotextRepository = (client: Client) => { + return { + async extractName(endpoint: string, query: string): Promise { + const tokensRaw = cleanQuery(query); + const response = await fetchNLPService(endpoint, { tokens: tokensRaw }); + + const { tokens, prediction } = response[0]; + + if (!tokens || !prediction) { + throw new BadRequestError('No tokens or prediction'); + } + + const nameTokens = tokens.filter((_, index) => prediction[index] === 'name'); + + return nameTokens.join(' '); + }, + + async generatePlacetype(index: string, query: string): Promise<{ placeTypes: string[]; subPlaceTypes: string[] }> { + const { + hits: { hits }, + } = await queryElastic(client, index, placetypeQuery(query)); + + if (hits.length == 2 && hits[0]._score! - hits[1]._score! > 0.5) { + hits.pop(); + } + + const [placeTypes, subPlaceTypes] = hits.reduce<[string[], string[]]>( + ([placeTypes, subPlaceTypes], hit) => [placeTypes.concat(hit._source!.placetype), subPlaceTypes.concat(hit._source!.sub_placetype)], + [[], []] + ); + + return { placeTypes, subPlaceTypes }; + }, + + async extractHierarchy(index: string, query: string, hierarchyBoost: number): Promise { + const { + hits: { hits }, + } = await queryElastic(client, index, hierarchyQuery(query)); + + const filteredHits = hits.length > 3 ? hits.filter((hit) => hit._score! >= hits[2]._score!) : hits; + + const highestScore = Math.max(...filteredHits.map((hit) => hit._score!)); + const hierarchies = filteredHits.map((hit) => ({ + ...hit._source!, + weight: (hit._score! / highestScore) * (hierarchyBoost - 1) + 1, + })); + + return hierarchies; + }, + + async geotextSearch( + index: string, + params: TextSearchParams, + textLanguage: string, + elasticQueryBoosts: IApplication['elasticQueryBoosts'] + ): Promise> { + const response = await queryElastic(client, index, geotextQuery(params, textLanguage, elasticQueryBoosts)); + return response; + }, + }; +}; + +const queryElastic = async (client: Client, index: string, body: estypes.SearchRequest): Promise> => { + const response = await client.search({ + index, + ...body, + }); + + return response; +}; + +export type GeotextRepository = ReturnType; + +export const geotextRepositoryFactory: FactoryFunction = (depContainer) => { + return createGeotextRepository(depContainer.resolve(elasticClientSymbol).geotext); +}; + +export const GEOTEXT_REPOSITORY_SYMBOL = Symbol('GEOTEXT_REPOSITORY_SYMBOL'); From d3e33114c9902f73883f755e57dc4d96555d352b Mon Sep 17 00:00:00 2001 From: Niv Greenstein Date: Tue, 23 Jul 2024 20:42:08 +0300 Subject: [PATCH 09/67] feat: created geoTextSearchRepository --- .../controllers/geotextSearchController.ts | 22 +++++++++---------- 1 file changed, 11 insertions(+), 11 deletions(-) diff --git a/src/geotextSearch/controllers/geotextSearchController.ts b/src/geotextSearch/controllers/geotextSearchController.ts index b31aaf42..bd8f022e 100644 --- a/src/geotextSearch/controllers/geotextSearchController.ts +++ b/src/geotextSearch/controllers/geotextSearchController.ts @@ -1,11 +1,11 @@ -import { Logger } from "@map-colonies/js-logger"; -import { BoundCounter, Meter } from "@opentelemetry/api-metrics"; -import { RequestHandler } from "express"; -import httpStatus from "http-status-codes"; -import { injectable, inject } from "tsyringe"; -import { SERVICES } from "../../common/constants"; -import { GeotextSearchManager } from "../models/queryManager"; -import { GetQueryQueryParams, QueryResult } from "../interfaces"; +import { Logger } from '@map-colonies/js-logger'; +import { BoundCounter, Meter } from '@opentelemetry/api-metrics'; +import { RequestHandler } from 'express'; +import httpStatus from 'http-status-codes'; +import { injectable, inject } from 'tsyringe'; +import { SERVICES } from '../../common/constants'; +import { GeotextSearchManager } from '../models/geotextSearchManager'; +import { GetQueryQueryParams, QueryResult } from '../interfaces'; type GetGeotextSearchHandler = RequestHandler< unknown, @@ -17,7 +17,7 @@ type GetGeotextSearchHandler = RequestHandler< type GetRegionshHandler = RequestHandler; @injectable() -export class geotextSearchController { +export class GeotextSearchController { private readonly createdResourceCounter: BoundCounter; public constructor( @@ -25,7 +25,7 @@ export class geotextSearchController { @inject(GeotextSearchManager) private readonly manager: GeotextSearchManager, @inject(SERVICES.METER) private readonly meter: Meter ) { - this.createdResourceCounter = meter.createCounter("created_resource"); + this.createdResourceCounter = meter.createCounter('created_resource'); } public getGeotextSearch: GetGeotextSearchHandler = async (req, res, next) => { @@ -33,7 +33,7 @@ export class geotextSearchController { const response = await this.manager.search(req.query); return res.status(httpStatus.OK).json(response); } catch (error: unknown) { - this.logger.error("Error in getGeotextSearch", error); + this.logger.error('Error in getGeotextSearch', error); next(error); } }; From a92abfd2084d037a04ef057edf882c8b64547073 Mon Sep 17 00:00:00 2001 From: Niv Greenstein Date: Tue, 23 Jul 2024 20:42:32 +0300 Subject: [PATCH 10/67] feat: created geotextSearch manager --- .../models/geotextSearchManager.ts | 68 +++++++++++++++++++ 1 file changed, 68 insertions(+) create mode 100644 src/geotextSearch/models/geotextSearchManager.ts diff --git a/src/geotextSearch/models/geotextSearchManager.ts b/src/geotextSearch/models/geotextSearchManager.ts new file mode 100644 index 00000000..89ab7d64 --- /dev/null +++ b/src/geotextSearch/models/geotextSearchManager.ts @@ -0,0 +1,68 @@ +import { IConfig } from 'config'; +import { Logger } from '@map-colonies/js-logger'; +import { inject, injectable } from 'tsyringe'; +import { SERVICES, elasticConfigPath } from '../../common/constants'; +import { GEOTEXT_REPOSITORY_SYMBOL, GeotextRepository } from '../DAL/geotextSearchRepository'; +import { GetQueryQueryParams, QueryResult, TextSearchParams } from '../interfaces'; +import { convertResult, parseGeo } from '../utils'; +import { IApplication } from '../../common/interfaces'; +import { ElasticDbClientsConfig } from '../../common/interfaces'; + +@injectable() +export class GeotextSearchManager { + public constructor( + @inject(SERVICES.LOGGER) private readonly logger: Logger, + @inject(SERVICES.APPLICATION) private readonly appConfig: IApplication, + @inject(SERVICES.CONFIG) private readonly config: IConfig, + @inject(GEOTEXT_REPOSITORY_SYMBOL) private readonly geotextRepository: GeotextRepository + ) {} + + public async search(params: GetQueryQueryParams): Promise { + const extractNameEndpoint = this.appConfig.services.tokenTypesUrl; + const { + geotext: geotextIndex, + placetypes: placetypesIndex, + hierarchies: hierarchiesIndex, + } = this.config.get(elasticConfigPath).geotext.properties.index as { + [key: string]: string; + }; + const hierarchyBoost = this.appConfig.elasticQueryBoosts.hierarchy; + + const [query, ...hierarchyQuery] = params.query.split(','); + + const promises = Promise.all([ + this.geotextRepository.extractName(extractNameEndpoint, query), + this.geotextRepository.generatePlacetype(placetypesIndex, query), + this.geotextRepository.extractHierarchy(hierarchiesIndex, hierarchyQuery.join(','), hierarchyBoost), + ]); + + const [name, { placeTypes, subPlaceTypes }, hierarchies] = await promises; + + const searchParams: TextSearchParams = { + query, + limit: params.limit, + sources: params.source ? (params.source instanceof Array ? params.source : [params.source]) : undefined, + viewbox: params.viewbox ? parseGeo(params.viewbox) : undefined, + boundary: params.boundary ? parseGeo(params.boundary) : undefined, + name, + placeTypes, + subPlaceTypes, + hierarchies, + regions: params.region, + }; + + const esResult = await this.geotextRepository.geotextSearch( + geotextIndex, + searchParams, + this.config.get(elasticConfigPath).geotext.properties.textTermLanguage, + this.appConfig.elasticQueryBoosts + ); + + console.log(this.appConfig.regions); + return convertResult(searchParams, esResult.hits.hits, { sources: this.appConfig.sources, regionCollection: this.appConfig.regions }); + } + + public regions(): string[] { + return Object.keys(this.appConfig.regions ?? {}); + } +} From 7c320228dfea0d5e83cddb526379a85a2a10df34 Mon Sep 17 00:00:00 2001 From: Niv Greenstein Date: Tue, 23 Jul 2024 20:43:37 +0300 Subject: [PATCH 11/67] feat: created geotextSearch elasticsearch hits expected response --- src/geotextSearch/models/elasticsearchHits.ts | 33 +++++++++++++++++++ 1 file changed, 33 insertions(+) create mode 100644 src/geotextSearch/models/elasticsearchHits.ts diff --git a/src/geotextSearch/models/elasticsearchHits.ts b/src/geotextSearch/models/elasticsearchHits.ts new file mode 100644 index 00000000..48a8a6b7 --- /dev/null +++ b/src/geotextSearch/models/elasticsearchHits.ts @@ -0,0 +1,33 @@ +import { GeoJSON } from "geojson"; + +/* eslint-disable @typescript-eslint/naming-convention */ +export interface TextSearchHit { + name: string; + text: string[]; + translated_text: string[]; + geo_json: GeoJSON; + source: string; + source_id: string[]; + placetype: string; + sub_placetype: string; + layer_name: string; + region: string[]; + sub_region: string[]; +} + +export interface PlaceTypeSearchHit { + placetype: string; + sub_placetype: string; + sub_placetype_keyword: string; +} + +export interface HierarchySearchHit { + text: string; + region: string; + hierarchy: string; + placetype: string; + geo_json: string; + weight?: number; +} + +/* eslint-enable @typescript-eslint/naming-convention */ From 1b3509f083d223e768534e852f31fa176f5902a8 Mon Sep 17 00:00:00 2001 From: Niv Greenstein Date: Tue, 23 Jul 2024 20:43:51 +0300 Subject: [PATCH 12/67] fix: updated import --- src/geotextSearch/DAL/geotextSearchRepository.ts | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/geotextSearch/DAL/geotextSearchRepository.ts b/src/geotextSearch/DAL/geotextSearchRepository.ts index 0de836a6..cc1f236e 100644 --- a/src/geotextSearch/DAL/geotextSearchRepository.ts +++ b/src/geotextSearch/DAL/geotextSearchRepository.ts @@ -3,7 +3,7 @@ import { FactoryFunction } from 'tsyringe'; import { elasticClientSymbol } from '../../common/elastic'; import { cleanQuery, fetchNLPService } from '../utils'; import { TextSearchParams, TokenResponse } from '../interfaces'; -import { PlaceTypeSearchHit, HierarchySearchHit, TextSearchHit } from '../models/textSearchHit'; +import { PlaceTypeSearchHit, HierarchySearchHit, TextSearchHit } from '../models/elasticsearchHits'; import { BadRequestError } from '../../common/errors'; import { ElasticClients, IApplication } from '../../common/interfaces'; import { hierarchyQuery, placetypeQuery, geotextQuery } from './queries'; From c8cb61f2e27df18041e82a2b7d808e81e0c85ebb Mon Sep 17 00:00:00 2001 From: Niv Greenstein Date: Tue, 23 Jul 2024 20:44:22 +0300 Subject: [PATCH 13/67] feat: created geotextSearch router --- src/geotextSearch/routes/geotextSearchRouter.ts | 17 +++++++++++++++++ 1 file changed, 17 insertions(+) create mode 100644 src/geotextSearch/routes/geotextSearchRouter.ts diff --git a/src/geotextSearch/routes/geotextSearchRouter.ts b/src/geotextSearch/routes/geotextSearchRouter.ts new file mode 100644 index 00000000..2c72e165 --- /dev/null +++ b/src/geotextSearch/routes/geotextSearchRouter.ts @@ -0,0 +1,17 @@ +import { Router } from 'express'; +import { FactoryFunction } from 'tsyringe'; +import { GeotextSearchController } from '../controllers/geotextSearchController'; + +const geotextSearchRouterFactory: FactoryFunction = (dependencyContainer) => { + const router = Router(); + const controller = dependencyContainer.resolve(GeotextSearchController); + + router.get('/regions', controller.getRegions); + router.get('/', controller.getGeotextSearch); + + return router; +}; + +export const GEOTEXT_SEARCH_ROUTER_SYMBOL = Symbol('geotextSearchRouterFactory'); + +export { geotextSearchRouterFactory }; From 6d6cd873f59d101b0e20d88cc1f1b537d83ea034 Mon Sep 17 00:00:00 2001 From: Niv Greenstein Date: Tue, 23 Jul 2024 20:44:39 +0300 Subject: [PATCH 14/67] feat: created geotext search interfaces --- src/geotextSearch/interfaces.ts | 65 +++++++++++++++++++++++++++++++++ 1 file changed, 65 insertions(+) create mode 100644 src/geotextSearch/interfaces.ts diff --git a/src/geotextSearch/interfaces.ts b/src/geotextSearch/interfaces.ts new file mode 100644 index 00000000..7f5ee008 --- /dev/null +++ b/src/geotextSearch/interfaces.ts @@ -0,0 +1,65 @@ +import type { GeoJSON } from 'geojson'; +import { HierarchySearchHit } from './models/elasticsearchHits'; + +export interface PlaceType { + placetype: string; + confidence: number; +} + +export interface TokenResponse { + tokens: string[]; + prediction: string[]; +} + +export interface TextSearchParams { + query: string; + viewbox?: GeoJSON; + boundary?: GeoJSON; + sources?: string[]; + regions?: string[]; + name?: string; + placeTypes?: string[]; + subPlaceTypes?: string[]; + hierarchies: HierarchySearchHit[]; + limit: number; +} + +export interface GetQueryQueryParams { + query: string; + limit: number; + source?: string[]; + viewbox?: string; + boundary?: string; + region?: string[]; +} + +/* eslint-disable @typescript-eslint/naming-convention */ + +export interface QueryResult { + type: string; + geocoding: { version?: string; query: TextSearchParams }; + features: { + type: string; + geometry?: GeoJSON; + properties: { + rank: number; + source?: string; + source_id?: string[]; + layer?: string; + name: { + [key: string]: string | string[] | undefined; + }; + highlight?: Record; + placetype?: string; + sub_placetype?: string; + region?: string[]; + sub_region?: string[]; + }; + }[]; +} +/* eslint-enable @typescript-eslint/naming-convention */ + +//Defenitions +export const POINT_LENGTH = 2; +export const BBOX_LENGTH = 4; +export const INDEX_NOT_FOUND = -1; From 88110445062a1a8b48c64656a2ad91ebe8e60cb7 Mon Sep 17 00:00:00 2001 From: Niv Greenstein Date: Tue, 23 Jul 2024 21:23:40 +0300 Subject: [PATCH 15/67] feat: created parsing utils --- src/geotextSearch/parsing.ts | 49 ++++++++++++++++++++++++++++++++++++ 1 file changed, 49 insertions(+) create mode 100644 src/geotextSearch/parsing.ts diff --git a/src/geotextSearch/parsing.ts b/src/geotextSearch/parsing.ts new file mode 100644 index 00000000..29172a94 --- /dev/null +++ b/src/geotextSearch/parsing.ts @@ -0,0 +1,49 @@ +import { XMLParser } from 'fast-xml-parser'; + +type Highlight = { + em: string | string[]; + '#text': string; +}; +const HIERARCHY_OF_INTEREST = 3; + +const HIGHLIGHT_XML_REGEX = /|<\/em>/gi; + +const untagHighlight = (highlight: string) => highlight.replace(HIGHLIGHT_XML_REGEX, ''); + +const calculateHighlightQuality = (highlight: string, queryWordCount: number) => { + const parser = new XMLParser({ numberParseOptions: { skipLike: /[0-9]+/, hex: false, leadingZeros: false } }); + const parsed = parser.parse(highlight) as Highlight; + + if (!(parsed.em instanceof Array)) { + parsed.em = [parsed.em]; + } + + const taggedCount = parsed.em.map((element) => element.split(' ')).flat().length; + + return taggedCount / queryWordCount; +}; + +const compareQualityThenLength = ( + a: { + highlight: string; + quality: number; + }, + b: { + highlight: string; + quality: number; + } +) => a.quality - b.quality || a.highlight.length - b.highlight.length; + +export const generateDisplayName = (highlights: string[], queryWordCount: number, name?: string) => { + const scored = highlights.map((highlight) => ({ + highlight, + quality: calculateHighlightQuality(highlight, queryWordCount), + })); + const filtered = scored.filter(name ? ({ highlight }) => highlight.includes(name) : ({ quality }) => quality < 1); + + const chosen = (filtered.length ? filtered : scored).sort(compareQualityThenLength).pop()?.highlight; + + return chosen && untagHighlight(chosen); +}; + +export const getHierarchyOfInterest = (hierarchy: string): string => hierarchy.split('/')[HIERARCHY_OF_INTEREST]; From c277b7cac419598797e46ac87fc3fc80ddff78c7 Mon Sep 17 00:00:00 2001 From: Niv Greenstein Date: Tue, 23 Jul 2024 21:24:20 +0300 Subject: [PATCH 16/67] fix: changed value to right key --- src/item/DAL/itemRepository.ts | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/item/DAL/itemRepository.ts b/src/item/DAL/itemRepository.ts index 212a919a..4a844b9d 100644 --- a/src/item/DAL/itemRepository.ts +++ b/src/item/DAL/itemRepository.ts @@ -26,7 +26,7 @@ const createItemRepository = (client: Client) => { export type ItemRepository = ReturnType; export const itemRepositoryFactory: FactoryFunction = (depContainer) => { - return createItemRepository(depContainer.resolve(elasticClientSymbol).searchy); + return createItemRepository(depContainer.resolve(elasticClientSymbol).control); }; export const ITEM_REPOSITORY_SYMBOL = Symbol('ITEM_REPOSITORY_SYMBOL'); From 6af50f036eb0a2a260291d95498257e8dde4ba22 Mon Sep 17 00:00:00 2001 From: Niv Greenstein Date: Tue, 23 Jul 2024 21:24:40 +0300 Subject: [PATCH 17/67] fix: fixed types --- src/item/models/itemManager.ts | 9 ++++++--- 1 file changed, 6 insertions(+), 3 deletions(-) diff --git a/src/item/models/itemManager.ts b/src/item/models/itemManager.ts index 6720d175..035dc864 100644 --- a/src/item/models/itemManager.ts +++ b/src/item/models/itemManager.ts @@ -2,11 +2,11 @@ import { IConfig } from 'config'; import { Logger } from '@map-colonies/js-logger'; import { inject, injectable } from 'tsyringe'; import { estypes } from '@elastic/elasticsearch'; -import { SERVICES } from '../../common/constants'; +import { SERVICES, elasticConfigPath } from '../../common/constants'; import { ITEM_REPOSITORY_SYMBOL, ItemRepository } from '../DAL/itemRepository'; import { ItemQueryParams } from '../DAL/queries'; import { formatResponse } from '../../common/utils'; -import { FeatureCollection } from '../../common/interfaces'; +import { ElasticDbClientsConfig, FeatureCollection } from '../../common/interfaces'; import { Item } from './item'; @injectable() @@ -19,7 +19,10 @@ export class ItemManager { public async getItems(itemQueryParams: ItemQueryParams, reduceFuzzyMatch = false, size?: number): Promise> { let elasticResponse: estypes.SearchResponse | undefined = undefined; - elasticResponse = await this.itemRepository.getItems(itemQueryParams, size ?? this.config.get('db.elastic.searchy.properties.size')); + elasticResponse = await this.itemRepository.getItems( + itemQueryParams, + size ?? this.config.get(elasticConfigPath).control.properties.defaultResponseLimit + ); const formattedResponse = formatResponse(elasticResponse); From f5200cfc6ca354c69f3ec1ee614befe9afc58495 Mon Sep 17 00:00:00 2001 From: Niv Greenstein Date: Tue, 23 Jul 2024 21:25:56 +0300 Subject: [PATCH 18/67] feat: upgraded LatLonDal --- src/latLon/DAL/latLonDAL.ts | 96 ++++++++++++++++++++++++++----------- 1 file changed, 68 insertions(+), 28 deletions(-) diff --git a/src/latLon/DAL/latLonDAL.ts b/src/latLon/DAL/latLonDAL.ts index 9b6b9da1..7faf6c6b 100644 --- a/src/latLon/DAL/latLonDAL.ts +++ b/src/latLon/DAL/latLonDAL.ts @@ -1,69 +1,105 @@ -import { IConfig } from 'config'; import { Logger } from '@map-colonies/js-logger'; import cron from 'node-cron'; import { FactoryFunction, inject, injectable } from 'tsyringe'; import { InternalServerError } from '../../common/errors'; +import { IApplication } from '../../common/interfaces'; import { SERVICES } from '../../common/constants'; import { LATLON_CUSTOM_REPOSITORY_SYMBOL, LatLonRepository } from './latLonRepository'; import { LatLon as LatLonDb } from './latLon'; @injectable() export class LatLonDAL { - private latLonMap: Map | null; + private readonly latLonMap: Map; private onGoingUpdate: boolean; - private dataLoad: Promise; + private dataLoad: + | { + promise?: Promise; + resolve?: (value: unknown) => void; + reject?: (reason?: unknown) => void; + } + | undefined; + private dataLoadError: boolean; public constructor( @inject(SERVICES.LOGGER) private readonly logger: Logger, - @inject(LATLON_CUSTOM_REPOSITORY_SYMBOL) private readonly latLonRepository: LatLonRepository, - @inject(SERVICES.CONFIG) private readonly config: IConfig + @inject(LATLON_CUSTOM_REPOSITORY_SYMBOL) private readonly latLonRepository: LatLonRepository ) { this.latLonMap = new Map(); this.onGoingUpdate = true; - this.dataLoad = this.init().catch((error) => { - this.logger.error('Failed to initialize latLon data', error); - throw new InternalServerError(`Failed to initialize latLon data: ${(error as Error).message}`); + this.dataLoad = undefined; + this.dataLoadError = false; + this.init().catch((error) => { + this.logger.error('Failed to initialize lat-lon data', error); + this.dataLoadError = true; }); } + /* istanbul ignore next */ public getOnGoingUpdate(): boolean { return this.onGoingUpdate; } + /* istanbul ignore end */ public async init(): Promise { - this.logger.debug('Initializing latLonData'); - // reset dataLoad promise before overriding latLonMap - this.dataLoad = new Promise((resolve) => resolve()); - this.onGoingUpdate = true; + try { + const dataLoadPromise = new Promise((resolve, reject) => { + this.dataLoadError = false; + this.dataLoad = { resolve, reject }; + }) + .then(() => (this.dataLoad = undefined)) + .catch(() => { + this.dataLoad = undefined; + this.dataLoadError = true; + }); + this.dataLoad = { ...this.dataLoad, promise: dataLoadPromise }; - this.latLonMap = new Map(); + this.onGoingUpdate = true; - await (this.dataLoad = this.loadLatLonData()); - this.onGoingUpdate = false; - this.logger.debug('latLonData initialized'); + this.logger.debug('Initializing latLonData'); + + await this.loadLatLonData(); + this.dataLoad.resolve?.('finished loading data'); + + this.logger.debug('latLonData initialized'); + } catch (error) { + this.logger.error('Failed to initialize latLon data', error); + this.dataLoadError = true; + } finally { + this.onGoingUpdate = false; + this.dataLoad = undefined; + } } public async latLonToTile({ x, y, zone }: { x: number; y: number; zone: number }): Promise { - await this.dataLoad; - return this.latLonMap?.get(`${x},${y},${zone}`); + if (this.dataLoadError) { + throw new InternalServerError('Lat-lon to tile data currently not available'); + } + await this.dataLoad?.promise; + return this.latLonMap.get(`${x},${y},${zone}`); } public async tileToLatLon(tileName: string): Promise { - await this.dataLoad; - return this.latLonMap?.get(tileName); + if (this.dataLoadError) { + throw new InternalServerError('Tile to lat-lon data currently not available'); + } + await this.dataLoad?.promise; + return this.latLonMap.get(tileName); + } + + private clearLatLonMap(): void { + this.logger.debug('Clearing latLon data'); + this.latLonMap.clear(); } private async loadLatLonData(): Promise { this.logger.debug('Loading latLon data'); - if (this.latLonMap === null) { - await this.init(); - } + this.clearLatLonMap(); const latLonData = await this.latLonRepository.getAll(); latLonData.forEach((latLon) => { - this.latLonMap?.set(latLon.tileName, latLon); - this.latLonMap?.set(`${latLon.minX},${latLon.minY},${latLon.zone}`, latLon); + this.latLonMap.set(latLon.tileName, latLon); + this.latLonMap.set(`${latLon.minX},${latLon.minY},${latLon.zone}`, latLon); }); this.logger.debug('latLon data loaded'); } @@ -74,10 +110,14 @@ export const cronLoadTileLatLonDataSymbol = Symbol('cronLoadTileLatLonDataSymbol export const cronLoadTileLatLonDataFactory: FactoryFunction = (dependencyContainer) => { const latLonDAL = dependencyContainer.resolve(LatLonDAL); const logger = dependencyContainer.resolve(SERVICES.LOGGER); - const cronPattern: string | undefined = dependencyContainer.resolve(SERVICES.CONFIG).get('cronLoadTileLatLonDataPattern'); + const cronPattern: string | undefined = dependencyContainer.resolve(SERVICES.APPLICATION).cronLoadTileLatLonDataPattern; + + // eslint-disable-next-line @typescript-eslint/no-unnecessary-condition if (cronPattern === undefined) { - throw new InternalServerError('cron pattern is not defined'); + throw new Error('cron pattern is not defined'); } + + /* istanbul ignore next */ cron.schedule(cronPattern, () => { if (!latLonDAL.getOnGoingUpdate()) { logger.info('cronLoadTileLatLonData: starting update'); @@ -86,10 +126,10 @@ export const cronLoadTileLatLonDataFactory: FactoryFunction = (dependencyC .then(() => logger.info('cronLoadTileLatLonData: update completed')) .catch((error) => { logger.error('cronLoadTileLatLonData: update failed', error); - throw new InternalServerError(`Failed to update latLon data: ${(error as Error).message}`); }); } else { logger.info('cronLoadTileLatLonData: update is already in progress'); } }); + /* istanbul ignore end */ }; From 267c20a842370774e4f3a5197e7d1e8f64cc02df Mon Sep 17 00:00:00 2001 From: Niv Greenstein Date: Tue, 23 Jul 2024 21:26:20 +0300 Subject: [PATCH 19/67] delete: deleted old files because name change --- src/query/DAL/queries.ts | 34 ------- src/query/DAL/queryRepository.ts | 63 ------------- src/query/controllers/queryController.ts | 32 ------- src/query/interfaces.ts | 62 ------------- src/query/models/queryManager.ts | 42 --------- src/query/models/textSearchHit.ts | 17 ---- src/query/routes/queryRouter.ts | 16 ---- src/query/utils.ts | 107 ----------------------- 8 files changed, 373 deletions(-) delete mode 100644 src/query/DAL/queries.ts delete mode 100644 src/query/DAL/queryRepository.ts delete mode 100644 src/query/controllers/queryController.ts delete mode 100644 src/query/interfaces.ts delete mode 100644 src/query/models/queryManager.ts delete mode 100644 src/query/models/textSearchHit.ts delete mode 100644 src/query/routes/queryRouter.ts delete mode 100644 src/query/utils.ts diff --git a/src/query/DAL/queries.ts b/src/query/DAL/queries.ts deleted file mode 100644 index 7356b686..00000000 --- a/src/query/DAL/queries.ts +++ /dev/null @@ -1,34 +0,0 @@ -/* eslint-disable @typescript-eslint/naming-convention */ -import { estypes } from '@elastic/elasticsearch'; -import { TextSearchParams } from '../interfaces'; - -export const esQuery = ({ query, limit, name, placeType, viewbox, boundary, sources }: TextSearchParams): estypes.SearchRequest => ({ - query: { - bool: { - must: { match: { text: query } }, - should: [ - { match_phrase: { text: query } }, - { - match: { - text: { - query, - fuzziness: 'AUTO', - }, - }, - }, - ...(name ?? '' ? [{ match: { text: name } }] : []), - ...(placeType ? [{ term: { placetype: { value: placeType.placetype, boost: placeType.confidence } } }] : []), - ...(viewbox ? [{ geo_shape: { geo_json: { shape: viewbox } } }] : []), - ...(boundary ? [{ geo_shape: { geo_json: { shape: boundary } } }] : []), - ...(sources ? [{ terms: { source: sources } }] : []), - ], - minimum_should_match: 0, - }, - }, - highlight: { - fields: { - text: {}, - }, - }, - size: limit, -}); diff --git a/src/query/DAL/queryRepository.ts b/src/query/DAL/queryRepository.ts deleted file mode 100644 index 0a9db34b..00000000 --- a/src/query/DAL/queryRepository.ts +++ /dev/null @@ -1,63 +0,0 @@ -import config from 'config'; -import { Client, estypes } from '@elastic/elasticsearch'; -import { FactoryFunction } from 'tsyringe'; -import { elasticClientSymbol } from '../../common/elastic'; -import { cleanQuery, fetchNLPService } from '../utils'; -import { PlaceType, TextSearchParams, TokenResponse } from '../interfaces'; -import { TextSearchHit } from '../models/textSearchHit'; -import { BadRequestError } from '../../common/errors'; -import { ElasticClients } from '../../common/interfaces'; -import { esQuery } from './queries'; - -/* eslint-enable @typescript-eslint/naming-convention */ - -// eslint-disable-next-line @typescript-eslint/explicit-function-return-type -const createQueryRepository = (client: Client) => { - return { - async extractName(endpoint: string, query: string): Promise { - const tokensRaw = cleanQuery(query); - const response = await fetchNLPService(endpoint, { tokens: tokensRaw }); - - const { tokens, prediction } = response[0]; - - if (!tokens || !prediction) { - throw new BadRequestError('No tokens or prediction'); - } - - const nameTokens = tokens.filter((_, index) => prediction[index] === 'name'); - - return nameTokens.join(' '); - }, - - async getPlaceType(endpoint: string, query: string): Promise { - const response = await fetchNLPService(endpoint, { - text: query, - start: 0, - end: query.length, - // eslint-disable-next-line @typescript-eslint/naming-convention - entity_text: query, - }); - - return response[0]; - }, - - async queryElastic(params: TextSearchParams): Promise> { - const index = config.get('db.elastic.nlp.properties.index'); - - const response = await client.search({ - index, - body: esQuery(params), - }); - - return response; - }, - }; -}; - -export type QueryRepository = ReturnType; - -export const queryRepositoryFactory: FactoryFunction = (depContainer) => { - return createQueryRepository(depContainer.resolve(elasticClientSymbol).nlp); -}; - -export const QUERY_REPOSITORY_SYMBOL = Symbol('QUERY_REPOSITORY_SYMBOL'); diff --git a/src/query/controllers/queryController.ts b/src/query/controllers/queryController.ts deleted file mode 100644 index 532bcb1c..00000000 --- a/src/query/controllers/queryController.ts +++ /dev/null @@ -1,32 +0,0 @@ -import { Logger } from '@map-colonies/js-logger'; -import { BoundCounter, Meter } from '@opentelemetry/api-metrics'; -import { RequestHandler } from 'express'; -import httpStatus from 'http-status-codes'; -import { injectable, inject } from 'tsyringe'; -import { SERVICES } from '../../common/constants'; -import { QueryManager } from '../models/queryManager'; -import { GetQueryQueryParams, QueryResult } from '../interfaces'; - -type GetQueryHandler = RequestHandler< - unknown, - QueryResult | { message: string; error: string }, //response - undefined, - GetQueryQueryParams ->; -@injectable() -export class QueryController { - private readonly createdResourceCounter: BoundCounter; - - public constructor( - @inject(SERVICES.LOGGER) private readonly logger: Logger, - @inject(QueryManager) private readonly manager: QueryManager, - @inject(SERVICES.METER) private readonly meter: Meter - ) { - this.createdResourceCounter = meter.createCounter('created_resource'); - } - - public getQuery: GetQueryHandler = async (req, res) => { - const response = await this.manager.query(req.query); - return res.status(httpStatus.OK).json(response); - }; -} diff --git a/src/query/interfaces.ts b/src/query/interfaces.ts deleted file mode 100644 index b3e76382..00000000 --- a/src/query/interfaces.ts +++ /dev/null @@ -1,62 +0,0 @@ -import type { GeoJSON } from 'geojson'; - -export interface PlaceType { - placetype: string; - confidence: number; -} - -export interface TokenResponse { - tokens: string[]; - prediction: string[]; -} - -export interface TextSearchParams { - query: string; - viewbox?: GeoJSON; - boundary?: GeoJSON; - sources?: string[]; - name?: string; - placeType?: PlaceType; - limit?: number; -} - -export interface GetQueryQueryParams { - query: string; - limit: number; - sources?: string[]; - viewbox: string; - boundary: string; -} - -/* eslint-disable @typescript-eslint/naming-convention */ - -export interface QueryResult { - type: string; - geocoding: TextSearchParams; - features: { - type: string; - geometry?: GeoJSON; - properties: { - rank: number; - source?: string; - layer?: string; - source_id?: string[]; - name: { - default?: string; - primary?: string[]; - tranlated?: string[]; - }; - highlight?: Record; - placetype?: string; - sub_placetype?: string; - region?: string[]; - sub_region?: string[]; - }; - }[]; -} -/* eslint-enable @typescript-eslint/naming-convention */ - -//Defenitions -export const POINT_LENGTH = 2; -export const BBOX_LENGTH = 4; -export const INDEX_NOT_FOUND = -1; diff --git a/src/query/models/queryManager.ts b/src/query/models/queryManager.ts deleted file mode 100644 index 91d819e9..00000000 --- a/src/query/models/queryManager.ts +++ /dev/null @@ -1,42 +0,0 @@ -import { IConfig } from 'config'; -import { Logger } from '@map-colonies/js-logger'; -import { inject, injectable } from 'tsyringe'; -import { SERVICES } from '../../common/constants'; -import { QUERY_REPOSITORY_SYMBOL, QueryRepository } from '../DAL/queryRepository'; -import { GetQueryQueryParams, TextSearchParams } from '../interfaces'; -import { convertResult, parseGeo } from '../utils'; - -@injectable() -export class QueryManager { - public constructor( - @inject(SERVICES.LOGGER) private readonly logger: Logger, - @inject(SERVICES.CONFIG) private readonly config: IConfig, - @inject(QUERY_REPOSITORY_SYMBOL) private readonly itemRepository: QueryRepository - ) {} - - public async query(params: GetQueryQueryParams) { - const extractNameEndpoint = this.config.get('services.tokenTypesUrl'); - const placeTypeEndpoint = this.config.get('services.placeTypeUrl'); - - const promises = Promise.all([ - this.itemRepository.extractName(extractNameEndpoint, params.query), - this.itemRepository.getPlaceType(placeTypeEndpoint, params.query), - ]); - - const [name, placeType] = await promises; - - const searchParams: TextSearchParams = { - query: params.query, - limit: params.limit, - sources: params.sources ? (params.sources instanceof Array ? params.sources : [params.sources]) : undefined, - viewbox: parseGeo(params.viewbox), - boundary: parseGeo(params.boundary), - name, - placeType, - }; - - const esResult = await this.itemRepository.queryElastic(searchParams); - - return convertResult(searchParams, esResult.hits.hits); - } -} diff --git a/src/query/models/textSearchHit.ts b/src/query/models/textSearchHit.ts deleted file mode 100644 index 0e14d0db..00000000 --- a/src/query/models/textSearchHit.ts +++ /dev/null @@ -1,17 +0,0 @@ -import { GeoJSON } from 'geojson'; - -/* eslint-disable @typescript-eslint/naming-convention */ -export interface TextSearchHit { - name: string; - text: string[]; - tranlated_text: string[]; - geo_json: GeoJSON; - source: string; - source_id: string[]; - placetype: string; - sub_placetype: string; - layer_name: string; - region: string[]; - sub_region: string[]; -} -/* eslint-enable @typescript-eslint/naming-convention */ diff --git a/src/query/routes/queryRouter.ts b/src/query/routes/queryRouter.ts deleted file mode 100644 index 84b7c709..00000000 --- a/src/query/routes/queryRouter.ts +++ /dev/null @@ -1,16 +0,0 @@ -import { Router } from 'express'; -import { FactoryFunction } from 'tsyringe'; -import { QueryController } from '../controllers/queryController'; - -const queryRouterFactory: FactoryFunction = (dependencyContainer) => { - const router = Router(); - const controller = dependencyContainer.resolve(QueryController); - - router.get('/', controller.getQuery); - - return router; -}; - -export const QUERY_ROUTER_SYMBOL = Symbol('queryRouterFactory'); - -export { queryRouterFactory }; diff --git a/src/query/utils.ts b/src/query/utils.ts deleted file mode 100644 index b997949b..00000000 --- a/src/query/utils.ts +++ /dev/null @@ -1,107 +0,0 @@ -import fetch, { Response } from 'node-fetch-commonjs'; -import geojsonValidator from 'geojson-validation'; -import { GeoJSON } from 'geojson'; -import { SearchHit } from '@elastic/elasticsearch/lib/api/types'; -import { InternalServerError, BadRequestError } from '../common/errors'; -import { BBOX_LENGTH, INDEX_NOT_FOUND, POINT_LENGTH, QueryResult, TextSearchParams } from './interfaces'; -import { TextSearchHit } from './models/textSearchHit'; - -const FIND_QUOTES = /["']/g; - -const FIND_SPECIAL = /[`!@#$%^&*()_\-+=|\\/,.<>:[\]{}\n\t\r\s;؛]+/g; - -export const fetchNLPService = async (endpoint: string, requestData: object): Promise => { - let res: Response | null = null, - data: T[] | undefined | null = null; - - try { - res = await fetch(endpoint, { - method: 'POST', - body: JSON.stringify(requestData), - }); - } catch (err: unknown) { - if (err instanceof Error) { - throw new InternalServerError(err.message); - } - throw new InternalServerError('fetchNLPService: Unknown error' + JSON.stringify(err)); - } - - data = (await res.json()) as T[] | undefined; - - if (!res.ok || !data || data.length < 1 || !data[0]) { - throw new InternalServerError(JSON.stringify(data)); - } - return data; -}; - -export const cleanQuery = (query: string): string[] => query.replace(FIND_QUOTES, '').split(FIND_SPECIAL); - -export const parsePoint = (split: string[]): GeoJSON => ({ - type: 'Point', - coordinates: split.map(Number), -}); - -export const parseBbox = (split: string[]): GeoJSON => { - const [xMin, yMin, xMax, yMax] = split.map(Number); - return { - type: 'Polygon', - coordinates: [ - [ - [xMin, yMin], - [xMin, yMax], - [xMax, yMax], - [xMax, yMin], - [xMin, yMin], - ], - ], - }; -}; - -export const parseGeo = (input: string | GeoJSON): GeoJSON => { - if (typeof input === 'string') { - const splitted = input.split(','); - const converted = splitted.map(Number); - - if (converted.findIndex((x) => isNaN(x)) < 0) { - switch (splitted.length) { - case POINT_LENGTH: - // Point - return parsePoint(splitted); - case BBOX_LENGTH: - //BBOX - return parseBbox(splitted); - } - } - } - - return input as GeoJSON; -}; - -/* eslint-disable @typescript-eslint/naming-convention */ -export const convertResult = (params: TextSearchParams, results: SearchHit[]): QueryResult => ({ - type: 'FeatureCollection', - geocoding: { - ...params, - }, - features: results.map(({ highlight, _source: feature }, index) => ({ - type: 'Feature', - geometry: feature?.geo_json, - properties: { - rank: index + 1, - source: feature?.source, // TODO: check if to remove this - layer: feature?.layer_name, - source_id: feature?.source_id.map((id) => id.replace(/(^\{)|(\}$)/g, '')), // TODO: check if to remove this - name: { - default: feature?.name, - primary: feature?.text, - tranlated: feature?.tranlated_text, - }, - highlight, - placetype: feature?.placetype, // TODO: check if to remove this - sub_placetype: feature?.sub_placetype, - region: feature?.region, - sub_region: feature?.sub_region, - }, - })), -}); -/* eslint-enable @typescript-eslint/naming-convention */ From 3d018518fb00454703470475730fc9b875543166 Mon Sep 17 00:00:00 2001 From: Niv Greenstein Date: Tue, 23 Jul 2024 21:26:41 +0300 Subject: [PATCH 20/67] fix: changed value to right key --- src/route/DAL/routeRepository.ts | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/route/DAL/routeRepository.ts b/src/route/DAL/routeRepository.ts index 54432e3c..fc5f3c10 100644 --- a/src/route/DAL/routeRepository.ts +++ b/src/route/DAL/routeRepository.ts @@ -35,7 +35,7 @@ const createRouteRepository = (client: Client) => { export type RouteRepository = ReturnType; export const routeRepositoryFactory: FactoryFunction = (depContainer) => { - return createRouteRepository(depContainer.resolve(elasticClientSymbol).searchy); + return createRouteRepository(depContainer.resolve(elasticClientSymbol).control); }; export const ROUTE_REPOSITORY_SYMBOL = Symbol('ROUTE_REPOSITORY_SYMBOL'); From db872e25a7ed56bf507f500534a1c65b954636cf Mon Sep 17 00:00:00 2001 From: Niv Greenstein Date: Tue, 23 Jul 2024 21:26:49 +0300 Subject: [PATCH 21/67] fix: changed value to right key --- src/route/models/routeManager.ts | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/src/route/models/routeManager.ts b/src/route/models/routeManager.ts index 207e4158..42a680ed 100644 --- a/src/route/models/routeManager.ts +++ b/src/route/models/routeManager.ts @@ -22,10 +22,10 @@ export class RouteManager { if (routeQueryParams.controlPoint ?? 0) { elasticResponse = await this.routeRepository.getControlPointInRoute( routeQueryParams as RouteQueryParams & Required>, - size ?? getElasticClientQuerySize('searchy') + size ?? getElasticClientQuerySize('control') ); } else { - elasticResponse = await this.routeRepository.getRoutes(routeQueryParams, size ?? getElasticClientQuerySize('searchy')); + elasticResponse = await this.routeRepository.getRoutes(routeQueryParams, size ?? getElasticClientQuerySize('control')); } const formattedResponse = formatResponse(elasticResponse); From 3acac4ed0dced22f0df2c1ddb75c3b28ba3f08da Mon Sep 17 00:00:00 2001 From: Niv Greenstein Date: Tue, 23 Jul 2024 21:26:57 +0300 Subject: [PATCH 22/67] fix: changed value to right key --- src/tile/DAL/tileRepository.ts | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/tile/DAL/tileRepository.ts b/src/tile/DAL/tileRepository.ts index f6caeba9..84440b43 100644 --- a/src/tile/DAL/tileRepository.ts +++ b/src/tile/DAL/tileRepository.ts @@ -33,7 +33,7 @@ const createTileRepository = (client: Client) => { export type TileRepository = ReturnType; export const tileRepositoryFactory: FactoryFunction = (depContainer) => { - return createTileRepository(depContainer.resolve(elasticClientSymbol).searchy); + return createTileRepository(depContainer.resolve(elasticClientSymbol).control); }; export const TILE_REPOSITORY_SYMBOL = Symbol('TILE_REPOSITORY_SYMBOL'); From 35fde79ca2af26befb8259f29496ef7cea72fca3 Mon Sep 17 00:00:00 2001 From: Niv Greenstein Date: Tue, 23 Jul 2024 21:27:12 +0300 Subject: [PATCH 23/67] fix: changed value to right key --- src/tile/models/tileManager.ts | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/tile/models/tileManager.ts b/src/tile/models/tileManager.ts index 0a407acd..fb207cbd 100644 --- a/src/tile/models/tileManager.ts +++ b/src/tile/models/tileManager.ts @@ -19,7 +19,7 @@ export class TileManager { public async getTiles(tileQueryParams: TileQueryParams, reduceFuzzyMatch = false, size?: number): Promise> { let elasticResponse: estypes.SearchResponse | undefined = undefined; - const numberOfResults = size ?? getElasticClientQuerySize('searchy'); + const numberOfResults = size ?? getElasticClientQuerySize('control'); if (tileQueryParams.subTile ?? 0) { elasticResponse = await this.tileRepository.getSubTiles(tileQueryParams as Required, numberOfResults); } else { From 830de6095b29bc5900d1e7559f672e1cacdd919e Mon Sep 17 00:00:00 2001 From: Niv Greenstein Date: Tue, 23 Jul 2024 21:30:24 +0300 Subject: [PATCH 24/67] fix: handle types --- src/containerConfig.ts | 22 +++++++++++++--------- 1 file changed, 13 insertions(+), 9 deletions(-) diff --git a/src/containerConfig.ts b/src/containerConfig.ts index d375f865..4e18a334 100644 --- a/src/containerConfig.ts +++ b/src/containerConfig.ts @@ -5,11 +5,11 @@ import { trace, metrics as OtelMetrics } from '@opentelemetry/api'; import { DependencyContainer } from 'tsyringe/dist/typings/types'; import jsLogger, { LoggerOptions } from '@map-colonies/js-logger'; import { Metrics } from '@map-colonies/telemetry'; -import { SERVICES, SERVICE_NAME } from './common/constants'; +import { SERVICES, SERVICE_NAME, elasticConfigPath } from './common/constants'; import { tracing } from './common/tracing'; import { InjectionObject, registerDependencies } from './common/dependencyRegistration'; import { elasticClientSymbol, initElasticsearchClient } from './common/elastic'; -import { ElasticClients, ElasticDbClientsConfig, ElasticDbConfig, PostgresDbConfig } from './common/interfaces'; +import { ElasticClients, ElasticDbClientsConfig, ElasticDbConfig, IApplication, PostgresDbConfig } from './common/interfaces'; import { TILE_REPOSITORY_SYMBOL, tileRepositoryFactory } from './tile/DAL/tileRepository'; import { TILE_ROUTER_SYMBOL, tileRouterFactory } from './tile/routes/tileRouter'; import { ITEM_REPOSITORY_SYMBOL, itemRepositoryFactory } from './item/DAL/itemRepository'; @@ -19,8 +19,8 @@ import { ROUTE_ROUTER_SYMBOL, routeRouterFactory } from './route/routes/routeRou import { initDataSource } from './common/postgresql'; import { LATLON_CUSTOM_REPOSITORY_SYMBOL, latLonRepositoryFactory } from './latLon/DAL/latLonRepository'; import { LAT_LON_ROUTER_SYMBOL, latLonRouterFactory } from './latLon/routes/latLonRouter'; -import { QUERY_REPOSITORY_SYMBOL, queryRepositoryFactory } from './query/DAL/queryRepository'; -import { QUERY_ROUTER_SYMBOL, queryRouterFactory } from './query/routes/queryRouter'; +import { GEOTEXT_REPOSITORY_SYMBOL, geotextRepositoryFactory } from './geotextSearch/DAL/geotextSearchRepository'; +import { GEOTEXT_SEARCH_ROUTER_SYMBOL, geotextSearchRouterFactory } from './geotextSearch/routes/geotextSearchRouter'; import { cronLoadTileLatLonDataFactory, cronLoadTileLatLonDataSymbol } from './latLon/DAL/latLonDAL'; export interface RegisterOptions { @@ -38,11 +38,14 @@ export const registerExternalValues = async (options?: RegisterOptions): Promise tracing.start(); const tracer = trace.getTracer(SERVICE_NAME); - const elasticClientsConfig = config.get('db.elastic'); + const applicationConfig: IApplication = config.get('application'); + + const elasticClientsConfig = config.get(elasticConfigPath); + const postgresqlDataSourceOptions = config.get('db.postgresql'); - const elasticClients = {} as ElasticClients; + const elasticClients = {} as Partial; for (const [key, value] of Object.entries(elasticClientsConfig)) { - elasticClients[key as keyof ElasticDbClientsConfig] = await initElasticsearchClient(value as ElasticDbConfig); + elasticClients[key as keyof ElasticDbClientsConfig] = (await initElasticsearchClient(value as ElasticDbConfig)) ?? undefined; } const postgresqlConnection = await initDataSource(postgresqlDataSourceOptions); @@ -51,6 +54,7 @@ export const registerExternalValues = async (options?: RegisterOptions): Promise { token: SERVICES.LOGGER, provider: { useValue: logger } }, { token: SERVICES.TRACER, provider: { useValue: tracer } }, { token: SERVICES.METER, provider: { useValue: OtelMetrics.getMeterProvider().getMeter(SERVICE_NAME) } }, + { token: SERVICES.APPLICATION, provider: { useValue: applicationConfig } }, { token: elasticClientSymbol, provider: { useValue: elasticClients } }, { token: DataSource, provider: { useValue: postgresqlConnection } }, { token: TILE_REPOSITORY_SYMBOL, provider: { useFactory: tileRepositoryFactory } }, @@ -61,8 +65,8 @@ export const registerExternalValues = async (options?: RegisterOptions): Promise { token: ROUTE_ROUTER_SYMBOL, provider: { useFactory: routeRouterFactory } }, { token: LATLON_CUSTOM_REPOSITORY_SYMBOL, provider: { useFactory: latLonRepositoryFactory } }, { token: LAT_LON_ROUTER_SYMBOL, provider: { useFactory: latLonRouterFactory } }, - { token: QUERY_REPOSITORY_SYMBOL, provider: { useFactory: queryRepositoryFactory } }, - { token: QUERY_ROUTER_SYMBOL, provider: { useFactory: queryRouterFactory } }, + { token: GEOTEXT_REPOSITORY_SYMBOL, provider: { useFactory: geotextRepositoryFactory } }, + { token: GEOTEXT_SEARCH_ROUTER_SYMBOL, provider: { useFactory: geotextSearchRouterFactory } }, { token: cronLoadTileLatLonDataSymbol, provider: { From efc12b0412c0eeee7859c34ba275b74fde6d22ac Mon Sep 17 00:00:00 2001 From: Niv Greenstein Date: Tue, 23 Jul 2024 21:31:14 +0300 Subject: [PATCH 25/67] fix: changed to V1 router --- src/serverBuilder.ts | 26 ++++++++++++++------------ 1 file changed, 14 insertions(+), 12 deletions(-) diff --git a/src/serverBuilder.ts b/src/serverBuilder.ts index 3d366dee..732ba985 100644 --- a/src/serverBuilder.ts +++ b/src/serverBuilder.ts @@ -14,7 +14,7 @@ import { TILE_ROUTER_SYMBOL } from './tile/routes/tileRouter'; import { ITEM_ROUTER_SYMBOL } from './item/routes/itemRouter'; import { ROUTE_ROUTER_SYMBOL } from './route/routes/routeRouter'; import { LAT_LON_ROUTER_SYMBOL } from './latLon/routes/latLonRouter'; -import { QUERY_ROUTER_SYMBOL } from './query/routes/queryRouter'; +import { GEOTEXT_SEARCH_ROUTER_SYMBOL } from './geotextSearch/routes/geotextSearchRouter'; import { cronLoadTileLatLonDataSymbol } from './latLon/DAL/latLonDAL'; @injectable() @@ -28,7 +28,7 @@ export class ServerBuilder { @inject(ITEM_ROUTER_SYMBOL) private readonly itemRouter: Router, @inject(ROUTE_ROUTER_SYMBOL) private readonly routeRouter: Router, @inject(LAT_LON_ROUTER_SYMBOL) private readonly latLonRouter: Router, - @inject(QUERY_ROUTER_SYMBOL) private readonly queryRouter: Router, + @inject(GEOTEXT_SEARCH_ROUTER_SYMBOL) private readonly geotextRouter: Router, @inject(cronLoadTileLatLonDataSymbol) private readonly cronLoadTileLatLonData: void ) { this.serverInstance = express(); @@ -38,28 +38,30 @@ export class ServerBuilder { public build(): express.Application { this.registerPreRoutesMiddleware(); - this.buildRoutes(); + this.buildRoutesV1(); this.registerPostRoutesMiddleware(); return this.serverInstance; } - private buildDocsRoutes(): void { + private buildDocsRoutes(router: Router): void { const openapiRouter = new OpenapiViewerRouter({ ...this.config.get('openapiConfig'), filePathOrSpec: this.config.get('openapiConfig.filePath'), }); openapiRouter.setup(); - this.serverInstance.use(this.config.get('openapiConfig.basePath'), openapiRouter.getRouter()); + router.use(this.config.get('openapiConfig.basePath'), openapiRouter.getRouter()); } - private buildRoutes(): void { - this.serverInstance.use('/search/tiles', this.tileRouter); - this.serverInstance.use('/search/items', this.itemRouter); - this.serverInstance.use('/search/routes', this.routeRouter); - this.serverInstance.use('/lookup', this.latLonRouter); - this.serverInstance.use('/query', this.queryRouter); - this.buildDocsRoutes(); + private buildRoutesV1(): void { + const router = Router(); + router.use('/search/tiles', this.tileRouter); + router.use('/search/items', this.itemRouter); + router.use('/search/routes', this.routeRouter); + router.use('/lookup', this.latLonRouter); + router.use('/query', this.geotextRouter); + this.buildDocsRoutes(router); + this.serverInstance.use('/v1', router); } private registerPreRoutesMiddleware(): void { From 4760272b7b5c1145e3b1962dcd0482c47dba0ceb Mon Sep 17 00:00:00 2001 From: Niv Greenstein Date: Tue, 23 Jul 2024 21:31:47 +0300 Subject: [PATCH 26/67] updated openapi schema --- openapi3.yaml | 81 ++++++++++++++++++++++++++++++++++++++------------- 1 file changed, 60 insertions(+), 21 deletions(-) diff --git a/openapi3.yaml b/openapi3.yaml index 32a381dd..8a19c7db 100644 --- a/openapi3.yaml +++ b/openapi3.yaml @@ -10,13 +10,13 @@ tags: - name: Query Based Search - name: Exact Searches paths: - /query: + /v1/query: get: operationId: searchByQuery tags: - Query Based Search summary: "Search anything by query" - description: "This is for general queries. If known regex is found, the server will return results as if you searched for exact search. Else, the server will search using NLP engine to find a match." + description: "This is for general queries. If known regex is found, the server will return results as if you searched for exact search. Else, the server will search using geotext engine to find a match." parameters: - name: query in: query @@ -28,6 +28,7 @@ paths: description: Text to search title: Query description: Text to search + allowReserved: true - name: limit in: query required: false @@ -52,6 +53,7 @@ paths: - name: viewbox in: query required: false + allowReserved: true schema: type: string description: |- @@ -74,6 +76,7 @@ paths: - name: boundary in: query required: false + allowReserved: true schema: type: string description: |- @@ -89,9 +92,19 @@ paths: Results not touching this geometry will not be returned Possible formats include a valid geoJSON, a valid WKT, an "x,y" point, or a "xmin,ymin,xmax,ymax" bounding box + - name: region + in: query + required: false + schema: + type: array + items: + $ref: "#/components/schemas/Region" + description: Regions to include (if not specified, all Regions will be queried) + title: Region + description: Regions to include (if not specified, all Regions will be queried) responses: 200: - description: "OK (can be one of the following: tilesSchema, routesSchema, itemsSchema, subTileSchema)

WIP: Find out returned schema from nlp" + description: "OK (can be one of the following: tilesSchema, routesSchema, itemsSchema, subTileSchema)

WIP: Find out returned schema from geotext" content: application/json: schema: @@ -110,8 +123,34 @@ paths: "$ref": "#/components/responses/InternalError" security: - X-API-Key: [] - - X-User-ID: [] - /search/tiles: + X-User-ID: [] + /v1/query/regions: + get: + operationId: getRegions + tags: + - Query Based Search + summary: "Get regions" + responses: + 200: + description: "All regions" + content: + application/json: + schema: + type: array + items: + type: string + 400: + "$ref": "#/components/responses/BadRequest" + 401: + "$ref": "#/components/responses/Unauthorized" + 403: + "$ref": "#/components/responses/Forbidden" + 500: + "$ref": "#/components/responses/InternalError" + security: + - X-API-Key: [] + X-User-ID: [] + /v1/search/tiles: get: operationId: getTilesByQueryParams tags: @@ -149,8 +188,8 @@ paths: "$ref": "#/components/responses/InternalError" security: - X-API-Key: [] - - X-User-ID: [] - /search/items: + X-User-ID: [] + /v1/search/items: get: operationId: getItemsByQueryParams tags: @@ -197,8 +236,8 @@ paths: "$ref": "#/components/responses/InternalError" security: - X-API-Key: [] - - X-User-ID: [] - /search/routes: + X-User-ID: [] + /v1/search/routes: get: operationId: getRoutesByQueryParams tags: @@ -239,8 +278,8 @@ paths: "$ref": "#/components/responses/InternalError" security: - X-API-Key: [] - - X-User-ID: [] - /lookup/latlonToTile: + X-User-ID: [] + /v1/lookup/latlonToTile: get: operationId: convertLatlonToTile tags: @@ -280,8 +319,8 @@ paths: "$ref": "#/components/responses/InternalError" security: - X-API-Key: [] - - X-User-ID: [] - /lookup/tileToLatLon: + X-User-ID: [] + /v1/lookup/tileToLatLon: get: operationId: convertTileToLatlon tags: @@ -323,8 +362,8 @@ paths: "$ref": "#/components/responses/InternalError" security: - X-API-Key: [] - - X-User-ID: [] - /lookup/latlonToMgrs: + X-User-ID: [] + /v1/lookup/latlonToMgrs: get: operationId: convertLatLonToMgrs tags: @@ -375,8 +414,8 @@ paths: "$ref": "#/components/responses/InternalError" security: - X-API-Key: [] - - X-User-ID: [] - /lookup/mgrsToLatLon: + X-User-ID: [] + /v1/lookup/mgrsToLatLon: get: operationId: convertMgrsToLatLon tags: @@ -412,7 +451,7 @@ paths: "$ref": "#/components/responses/InternalError" security: - X-API-Key: [] - - X-User-ID: [] + X-User-ID: [] components: responses: BadRequest: @@ -707,10 +746,10 @@ components: $ref: "#/components/schemas/Point3D" Source: type: string - enum: - - moria - - lasagna title: Source + Region: + type: string + title: Region securitySchemes: X-API-Key: type: "apiKey" From 73216a348e33155a14ad877df2a7fbe8da8f9e76 Mon Sep 17 00:00:00 2001 From: Niv Greenstein Date: Thu, 25 Jul 2024 14:47:05 +0300 Subject: [PATCH 27/67] feat: added nameTranslationsKeys, mainLanguageRegex --- config/custom-environment-variables.json | 7 ++++++- config/default.json | 4 +++- src/common/interfaces.ts | 2 ++ 3 files changed, 11 insertions(+), 2 deletions(-) diff --git a/config/custom-environment-variables.json b/config/custom-environment-variables.json index 3bdd69b8..ed72d6e5 100644 --- a/config/custom-environment-variables.json +++ b/config/custom-environment-variables.json @@ -99,6 +99,11 @@ "regions": { "__name": "REGIONS", "__format": "json" - } + }, + "nameTranslationsKeys": { + "__name": "NAME_TRANSLATION_KEYS", + "__format": "json" + }, + "mainLanguageRegex": "MAIN_LANGUAGE_REGEX" } } diff --git a/config/default.json b/config/default.json index e4d32bb9..cf9186b0 100644 --- a/config/default.json +++ b/config/default.json @@ -89,6 +89,8 @@ "SOURCE_A": "a", "SOURCE_B": "b" }, - "regions": {} + "regions": {}, + "nameTranslationsKeys": ["en", "fr"], + "mainLanguageRegex": "[a-zA-Z]" } } diff --git a/src/common/interfaces.ts b/src/common/interfaces.ts index 3d8411cd..d642cd1e 100644 --- a/src/common/interfaces.ts +++ b/src/common/interfaces.ts @@ -76,4 +76,6 @@ export interface IApplication { regions?: { [key: string]: string[]; }; + nameTranslationsKeys: string[]; + mainLanguageRegex: string; } From eb354b20c5e86048b9816cd8e7695fa6baf4b907 Mon Sep 17 00:00:00 2001 From: Niv Greenstein Date: Thu, 25 Jul 2024 14:47:53 +0300 Subject: [PATCH 28/67] fix: added missing required keys and values to convertResult --- src/geotextSearch/models/geotextSearchManager.ts | 7 ++++++- 1 file changed, 6 insertions(+), 1 deletion(-) diff --git a/src/geotextSearch/models/geotextSearchManager.ts b/src/geotextSearch/models/geotextSearchManager.ts index 89ab7d64..75db4608 100644 --- a/src/geotextSearch/models/geotextSearchManager.ts +++ b/src/geotextSearch/models/geotextSearchManager.ts @@ -59,7 +59,12 @@ export class GeotextSearchManager { ); console.log(this.appConfig.regions); - return convertResult(searchParams, esResult.hits.hits, { sources: this.appConfig.sources, regionCollection: this.appConfig.regions }); + return convertResult(searchParams, esResult.hits.hits, { + sources: this.appConfig.sources, + regionCollection: this.appConfig.regions, + nameKeys: this.appConfig.nameTranslationsKeys, + mainLanguageRegex: this.appConfig.mainLanguageRegex, + }); } public regions(): string[] { From 13a049ee85d468a416c5cac0f6cc48cd02253947 Mon Sep 17 00:00:00 2001 From: Niv Greenstein Date: Thu, 25 Jul 2024 14:48:02 +0300 Subject: [PATCH 29/67] feat: utils created --- src/geotextSearch/utils.ts | 139 +++++++++++++++++++++++++++++++++++++ 1 file changed, 139 insertions(+) create mode 100644 src/geotextSearch/utils.ts diff --git a/src/geotextSearch/utils.ts b/src/geotextSearch/utils.ts new file mode 100644 index 00000000..2c497ef6 --- /dev/null +++ b/src/geotextSearch/utils.ts @@ -0,0 +1,139 @@ +// import fetch, { Response } from "node-fetch-commonjs"; +import { GeoJSON } from 'geojson'; +import { SearchHit } from '@elastic/elasticsearch/lib/api/types'; +import { StatusCodes } from 'http-status-codes'; +import axios, { AxiosResponse as Response } from 'axios'; +import { InternalServerError } from '../common/errors'; +import { IApplication } from '../common/interfaces'; +import { BBOX_LENGTH, POINT_LENGTH, QueryResult, TextSearchParams } from './interfaces'; +import { generateDisplayName } from './parsing'; +import { TextSearchHit } from './models/elasticsearchHits'; + +const FIND_QUOTES = /["']/g; + +const FIND_SPECIAL = /[`!@#$%^&*()_\-+=|\\/,.<>:[\]{}\n\t\r\s;؛]+/g; + +export const fetchNLPService = async (endpoint: string, requestData: object): Promise => { + let res: Response | null = null, + data: T[] | undefined | null = null; + try { + res = await axios.post(endpoint, requestData); + } catch (err: unknown) { + if (err instanceof Error) { + throw new InternalServerError(err.message); + } + throw new InternalServerError('fetchNLPService: Unknown error' + JSON.stringify(err)); + } + + try { + // data = (await res.json()) as T[] | undefined; + data = res?.data as T[] | undefined; + } catch (_) { + throw new InternalServerError("Couldn't convert the response from NLP service to JSON"); + } + + if (res?.status !== StatusCodes.OK || !data || data.length < 1 || !data[0]) { + throw new InternalServerError(JSON.stringify(data)); + } + return data; +}; + +export const cleanQuery = (query: string): string[] => query.replace(FIND_QUOTES, '').split(FIND_SPECIAL); + +export const parsePoint = (split: string[]): GeoJSON => ({ + type: 'Point', + coordinates: split.map(Number), +}); + +export const parseBbox = (split: string[]): GeoJSON => { + const [xMin, yMin, xMax, yMax] = split.map(Number); + return { + type: 'Polygon', + coordinates: [ + [ + [xMin, yMin], + [xMin, yMax], + [xMax, yMax], + [xMax, yMin], + [xMin, yMin], + ], + ], + }; +}; + +export const parseGeo = (input: string | GeoJSON): GeoJSON | undefined => { + if (typeof input === 'string') { + const splitted = input.split(','); + const converted = splitted.map(Number); + + if (converted.findIndex((x) => isNaN(x)) < 0) { + switch (splitted.length) { + case POINT_LENGTH: + // Point + return parsePoint(splitted); + case BBOX_LENGTH: + //BBOX + return parseBbox(splitted); + default: + return undefined; + } + } + } + + return input as GeoJSON; +}; + +/* eslint-disable @typescript-eslint/naming-convention */ +export const convertResult = ( + params: TextSearchParams, + results: SearchHit[], + { + sources, + regionCollection, + nameKeys, + mainLanguageRegex, + }: { + sources?: IApplication['sources']; + regionCollection?: IApplication['regions']; + nameKeys: IApplication['nameTranslationsKeys']; + mainLanguageRegex: IApplication['mainLanguageRegex']; + } = { nameKeys: [], mainLanguageRegex: '' } +): QueryResult => ({ + type: 'FeatureCollection', + geocoding: { + version: process.env.npm_package_version, + query: { + ...params, + }, + name: params.name, + }, + features: results.map(({ highlight, _source: feature }, index): QueryResult['features'][number] => { + const allNames = [feature!.text, feature!.translated_text || []]; + return { + type: 'Feature', + geometry: feature?.geo_json, + properties: { + rank: index + 1, + source: (sources ?? {})[feature?.source ?? ''] ?? feature?.source, + layer: feature?.layer_name, + source_id: feature?.source_id.map((id) => id.replace(/(^\{)|(\}$)/g, '')), // TODO: check if to remove this + name: { + [nameKeys[0]]: new RegExp(mainLanguageRegex).test(feature!.text[0]) ? allNames.shift() : allNames.pop(), + [nameKeys[1]]: allNames.pop(), + ['default']: [feature!.name], + display: highlight ? generateDisplayName(highlight.text, params.query!.split(' ').length, params.name) : feature!.name, + }, + highlight, + placetype: feature?.placetype, // TODO: check if to remove this + sub_placetype: feature?.sub_placetype, + regions: feature?.region.map((region) => ({ + region: region, + sub_regions: feature.sub_region.filter((sub_region) => (regionCollection ?? {})[region ?? '']?.includes(sub_region)), + })), + region: feature?.region, + sub_region: feature?.sub_region, + }, + }; + }), +}); +/* eslint-enable @typescript-eslint/naming-convention */ From c114565c721c875d6948659a06cb383761dd0cbc Mon Sep 17 00:00:00 2001 From: Niv Greenstein Date: Thu, 25 Jul 2024 14:48:23 +0300 Subject: [PATCH 30/67] fix: added missing type --- src/geotextSearch/interfaces.ts | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/src/geotextSearch/interfaces.ts b/src/geotextSearch/interfaces.ts index 7f5ee008..f19465cb 100644 --- a/src/geotextSearch/interfaces.ts +++ b/src/geotextSearch/interfaces.ts @@ -37,7 +37,7 @@ export interface GetQueryQueryParams { export interface QueryResult { type: string; - geocoding: { version?: string; query: TextSearchParams }; + geocoding: { version?: string; query: TextSearchParams; name?: string }; features: { type: string; geometry?: GeoJSON; @@ -54,6 +54,7 @@ export interface QueryResult { sub_placetype?: string; region?: string[]; sub_region?: string[]; + regions?: { region: string; sub_regions: string[] }[]; }; }[]; } From d27df1b4d5e9f9903c152452d9bf37f2b6a66d85 Mon Sep 17 00:00:00 2001 From: Niv Greenstein Date: Thu, 25 Jul 2024 14:49:18 +0300 Subject: [PATCH 31/67] feat: added dependencies --- package-lock.json | 200 ++++++++++++++++++++++++++++++++++++++++++++-- package.json | 10 ++- 2 files changed, 200 insertions(+), 10 deletions(-) diff --git a/package-lock.json b/package-lock.json index 6e58eba3..a0ae8830 100644 --- a/package-lock.json +++ b/package-lock.json @@ -1,12 +1,12 @@ { "name": "geocoding", - "version": "1.0.0", + "version": "0.5.0", "lockfileVersion": 2, "requires": true, "packages": { "": { "name": "geocoding", - "version": "1.0.0", + "version": "0.5.0", "hasInstallScript": true, "license": "ISC", "dependencies": { @@ -22,10 +22,12 @@ "@opentelemetry/api-metrics": "0.23.0", "@opentelemetry/instrumentation-express": "0.32.1", "@opentelemetry/instrumentation-http": "0.35.1", + "axios": "^0.21.1", "compression": "^1.7.4", "config": "^3.3.9", "express": "^4.19.2", "express-openapi-validator": "^5.0.4", + "fast-xml-parser": "^4.4.0", "geojson-validation": "^1.0.2", "http-status-codes": "^2.2.0", "mgrs": "^2.0.0", @@ -35,7 +37,8 @@ "proj4": "^2.11.0", "reflect-metadata": "^0.1.13", "tsyringe": "^4.8.0", - "typeorm": "^0.3.20" + "typeorm": "^0.3.20", + "wellknown": "^0.5.0" }, "devDependencies": { "@commitlint/cli": "^17.6.6", @@ -56,6 +59,7 @@ "@types/proj4": "^2.5.5", "@types/supertest": "^2.0.12", "@types/swagger-ui-express": "^4.1.3", + "@types/wellknown": "^0.5.6", "commitlint": "^17.6.6", "copyfiles": "^2.4.1", "cz-conventional-changelog": "^3.3.0", @@ -65,6 +69,7 @@ "jest-create-mock-instance": "^2.0.0", "jest-html-reporters": "^3.1.4", "jest-openapi": "^0.14.2", + "nock": "^13.5.4", "prettier": "^2.8.8", "pretty-quick": "^3.1.3", "rimraf": "^5.0.1", @@ -7034,6 +7039,12 @@ "integrity": "sha512-Q5vtl1W5ue16D+nIaW8JWebSSraJVlK+EthKn7e7UcD4KWsaSJ8BqGPXNaPghgtcn/fhvrN17Tv8ksUsQpiplw==", "dev": true }, + "node_modules/@types/wellknown": { + "version": "0.5.8", + "resolved": "https://registry.npmjs.org/@types/wellknown/-/wellknown-0.5.8.tgz", + "integrity": "sha512-/tXv+IfentaS3F0VplScdl+pwP7biW4RBnOxCTtoWlpNiulFBQWqA8rA7r/knavu9SELsFcNVtdGo6YNIElMdw==", + "dev": true + }, "node_modules/@types/yargs": { "version": "17.0.22", "resolved": "https://registry.npmjs.org/@types/yargs/-/yargs-17.0.22.tgz", @@ -7755,7 +7766,6 @@ "version": "0.21.4", "resolved": "https://registry.npmjs.org/axios/-/axios-0.21.4.tgz", "integrity": "sha512-ut5vewkiu8jjGBdqpM44XxjuCjq9LAKeHVmoVfHVzy8eHgxxq8SbAVQNovDA8mVi05kP0Ea/n/UzcSHcTJQfNg==", - "dev": true, "dependencies": { "follow-redirects": "^1.14.0" } @@ -11104,6 +11114,27 @@ "resolved": "https://registry.npmjs.org/fast-safe-stringify/-/fast-safe-stringify-2.1.1.tgz", "integrity": "sha512-W+KJc2dmILlPplD/H4K9l9LcAHAfPtP6BY84uVLXQ6Evcz9Lcg33Y2z1IVblT6xdY54PXYVHEv+0Wpq8Io6zkA==" }, + "node_modules/fast-xml-parser": { + "version": "4.4.0", + "resolved": "https://registry.npmjs.org/fast-xml-parser/-/fast-xml-parser-4.4.0.tgz", + "integrity": "sha512-kLY3jFlwIYwBNDojclKsNAC12sfD6NwW74QB2CoNGPvtVxjliYehVunB3HYyNi+n4Tt1dAcgwYvmKF/Z18flqg==", + "funding": [ + { + "type": "github", + "url": "https://github.com/sponsors/NaturalIntelligence" + }, + { + "type": "paypal", + "url": "https://paypal.me/naturalintelligence" + } + ], + "dependencies": { + "strnum": "^1.0.5" + }, + "bin": { + "fxparser": "src/cli/cli.js" + } + }, "node_modules/fastify": { "version": "3.29.5", "resolved": "https://registry.npmjs.org/fastify/-/fastify-3.29.5.tgz", @@ -11350,7 +11381,6 @@ "version": "1.15.1", "resolved": "https://registry.npmjs.org/follow-redirects/-/follow-redirects-1.15.1.tgz", "integrity": "sha512-yLAMQs+k0b2m7cVxpS1VKJVvoz7SS9Td1zss3XRwXj+ZDH00RJgnuLx7E44wx02kQLrdM3aOOy+FpzS7+8OizA==", - "dev": true, "funding": [ { "type": "individual", @@ -15508,6 +15538,20 @@ "integrity": "sha512-Yd3UES5mWCSqR+qNT93S3UoYUkqAZ9lLg8a7g9rimsWmYGK8cVToA4/sF3RrshdyV3sAGMXVUmpMYOw+dLpOuw==", "dev": true }, + "node_modules/nock": { + "version": "13.5.4", + "resolved": "https://registry.npmjs.org/nock/-/nock-13.5.4.tgz", + "integrity": "sha512-yAyTfdeNJGGBFxWdzSKCBYxs5FxLbCg5X5Q4ets974hcQzG1+qCxvIyOo4j2Ry6MUlhWVMX4OoYDefAIIwupjw==", + "dev": true, + "dependencies": { + "debug": "^4.1.0", + "json-stringify-safe": "^5.0.1", + "propagate": "^2.0.0" + }, + "engines": { + "node": ">= 10.13" + } + }, "node_modules/node-cron": { "version": "3.0.3", "resolved": "https://registry.npmjs.org/node-cron/-/node-cron-3.0.3.tgz", @@ -16891,6 +16935,15 @@ "react-is": "^16.13.1" } }, + "node_modules/propagate": { + "version": "2.0.1", + "resolved": "https://registry.npmjs.org/propagate/-/propagate-2.0.1.tgz", + "integrity": "sha512-vGrhOavPSTz4QVNuBNdcNXePNdNMaO1xj9yBeH1ScQPjk/rhg9sSlCXPhMkFuaNNW/syTvYqsnbIJxMBfRbbag==", + "dev": true, + "engines": { + "node": ">= 8" + } + }, "node_modules/protobufjs": { "version": "6.11.3", "resolved": "https://registry.npmjs.org/protobufjs/-/protobufjs-6.11.3.tgz", @@ -18186,6 +18239,11 @@ "url": "https://github.com/sponsors/sindresorhus" } }, + "node_modules/strnum": { + "version": "1.0.5", + "resolved": "https://registry.npmjs.org/strnum/-/strnum-1.0.5.tgz", + "integrity": "sha512-J8bbNyKKXl5qYcR36TIO8W3mVGVHrmmxsd5PAItGkmyzwJvybiw2IVq5nqd0i4LSNSkB/sx9VHllbfFdr9k1JA==" + }, "node_modules/superagent": { "version": "8.0.9", "resolved": "https://registry.npmjs.org/superagent/-/superagent-8.0.9.tgz", @@ -19188,6 +19246,54 @@ "integrity": "sha512-2JAn3z8AR6rjK8Sm8orRC0h/bcl/DqL7tRPdGZ4I1CjdF+EaMLmYxBHyXuKL849eucPFhvBoxMsflfOb8kxaeQ==", "dev": true }, + "node_modules/wellknown": { + "version": "0.5.0", + "resolved": "https://registry.npmjs.org/wellknown/-/wellknown-0.5.0.tgz", + "integrity": "sha512-za5vTLuPF9nmrVOovYQwNEWE/PwJCM+yHMAj4xN1WWUvtq9OElsvKiPL0CR9rO8xhrYqL7NpI7IknqR8r6eYOg==", + "dependencies": { + "concat-stream": "~1.5.0", + "minimist": "~1.2.0" + }, + "bin": { + "wellknown": "cli.js" + } + }, + "node_modules/wellknown/node_modules/concat-stream": { + "version": "1.5.2", + "resolved": "https://registry.npmjs.org/concat-stream/-/concat-stream-1.5.2.tgz", + "integrity": "sha512-H6xsIBfQ94aESBG8jGHXQ7i5AEpy5ZeVaLDOisDICiTCKpqEfr34/KmTrspKQNoLKNu9gTkovlpQcUi630AKiQ==", + "engines": [ + "node >= 0.8" + ], + "dependencies": { + "inherits": "~2.0.1", + "readable-stream": "~2.0.0", + "typedarray": "~0.0.5" + } + }, + "node_modules/wellknown/node_modules/process-nextick-args": { + "version": "1.0.7", + "resolved": "https://registry.npmjs.org/process-nextick-args/-/process-nextick-args-1.0.7.tgz", + "integrity": "sha512-yN0WQmuCX63LP/TMvAg31nvT6m4vDqJEiiv2CAZqWOGNWutc9DfDk1NPYYmKUFmaVM2UwDowH4u5AHWYP/jxKw==" + }, + "node_modules/wellknown/node_modules/readable-stream": { + "version": "2.0.6", + "resolved": "https://registry.npmjs.org/readable-stream/-/readable-stream-2.0.6.tgz", + "integrity": "sha512-TXcFfb63BQe1+ySzsHZI/5v1aJPCShfqvWJ64ayNImXMsN1Cd0YGk/wm8KB7/OeessgPc9QvS9Zou8QTkFzsLw==", + "dependencies": { + "core-util-is": "~1.0.0", + "inherits": "~2.0.1", + "isarray": "~1.0.0", + "process-nextick-args": "~1.0.6", + "string_decoder": "~0.10.x", + "util-deprecate": "~1.0.1" + } + }, + "node_modules/wellknown/node_modules/string_decoder": { + "version": "0.10.31", + "resolved": "https://registry.npmjs.org/string_decoder/-/string_decoder-0.10.31.tgz", + "integrity": "sha512-ev2QzSzWPYmy9GuqfIVildA4OdcGLeFZQrq5ys6RtiuF+RQQiZWr8TZNyAcuVXyQRYfEO+MsoB/1BuQVhOJuoQ==" + }, "node_modules/whatwg-url": { "version": "5.0.0", "resolved": "https://registry.npmjs.org/whatwg-url/-/whatwg-url-5.0.0.tgz", @@ -24704,6 +24810,12 @@ "integrity": "sha512-Q5vtl1W5ue16D+nIaW8JWebSSraJVlK+EthKn7e7UcD4KWsaSJ8BqGPXNaPghgtcn/fhvrN17Tv8ksUsQpiplw==", "dev": true }, + "@types/wellknown": { + "version": "0.5.8", + "resolved": "https://registry.npmjs.org/@types/wellknown/-/wellknown-0.5.8.tgz", + "integrity": "sha512-/tXv+IfentaS3F0VplScdl+pwP7biW4RBnOxCTtoWlpNiulFBQWqA8rA7r/knavu9SELsFcNVtdGo6YNIElMdw==", + "dev": true + }, "@types/yargs": { "version": "17.0.22", "resolved": "https://registry.npmjs.org/@types/yargs/-/yargs-17.0.22.tgz", @@ -25189,7 +25301,6 @@ "version": "0.21.4", "resolved": "https://registry.npmjs.org/axios/-/axios-0.21.4.tgz", "integrity": "sha512-ut5vewkiu8jjGBdqpM44XxjuCjq9LAKeHVmoVfHVzy8eHgxxq8SbAVQNovDA8mVi05kP0Ea/n/UzcSHcTJQfNg==", - "dev": true, "requires": { "follow-redirects": "^1.14.0" } @@ -27741,6 +27852,14 @@ "resolved": "https://registry.npmjs.org/fast-safe-stringify/-/fast-safe-stringify-2.1.1.tgz", "integrity": "sha512-W+KJc2dmILlPplD/H4K9l9LcAHAfPtP6BY84uVLXQ6Evcz9Lcg33Y2z1IVblT6xdY54PXYVHEv+0Wpq8Io6zkA==" }, + "fast-xml-parser": { + "version": "4.4.0", + "resolved": "https://registry.npmjs.org/fast-xml-parser/-/fast-xml-parser-4.4.0.tgz", + "integrity": "sha512-kLY3jFlwIYwBNDojclKsNAC12sfD6NwW74QB2CoNGPvtVxjliYehVunB3HYyNi+n4Tt1dAcgwYvmKF/Z18flqg==", + "requires": { + "strnum": "^1.0.5" + } + }, "fastify": { "version": "3.29.5", "resolved": "https://registry.npmjs.org/fastify/-/fastify-3.29.5.tgz", @@ -27956,8 +28075,7 @@ "follow-redirects": { "version": "1.15.1", "resolved": "https://registry.npmjs.org/follow-redirects/-/follow-redirects-1.15.1.tgz", - "integrity": "sha512-yLAMQs+k0b2m7cVxpS1VKJVvoz7SS9Td1zss3XRwXj+ZDH00RJgnuLx7E44wx02kQLrdM3aOOy+FpzS7+8OizA==", - "dev": true + "integrity": "sha512-yLAMQs+k0b2m7cVxpS1VKJVvoz7SS9Td1zss3XRwXj+ZDH00RJgnuLx7E44wx02kQLrdM3aOOy+FpzS7+8OizA==" }, "foreground-child": { "version": "3.1.1", @@ -31051,6 +31169,17 @@ "integrity": "sha512-Yd3UES5mWCSqR+qNT93S3UoYUkqAZ9lLg8a7g9rimsWmYGK8cVToA4/sF3RrshdyV3sAGMXVUmpMYOw+dLpOuw==", "dev": true }, + "nock": { + "version": "13.5.4", + "resolved": "https://registry.npmjs.org/nock/-/nock-13.5.4.tgz", + "integrity": "sha512-yAyTfdeNJGGBFxWdzSKCBYxs5FxLbCg5X5Q4ets974hcQzG1+qCxvIyOo4j2Ry6MUlhWVMX4OoYDefAIIwupjw==", + "dev": true, + "requires": { + "debug": "^4.1.0", + "json-stringify-safe": "^5.0.1", + "propagate": "^2.0.0" + } + }, "node-cron": { "version": "3.0.3", "resolved": "https://registry.npmjs.org/node-cron/-/node-cron-3.0.3.tgz", @@ -32112,6 +32241,12 @@ "react-is": "^16.13.1" } }, + "propagate": { + "version": "2.0.1", + "resolved": "https://registry.npmjs.org/propagate/-/propagate-2.0.1.tgz", + "integrity": "sha512-vGrhOavPSTz4QVNuBNdcNXePNdNMaO1xj9yBeH1ScQPjk/rhg9sSlCXPhMkFuaNNW/syTvYqsnbIJxMBfRbbag==", + "dev": true + }, "protobufjs": { "version": "6.11.3", "resolved": "https://registry.npmjs.org/protobufjs/-/protobufjs-6.11.3.tgz", @@ -33081,6 +33216,11 @@ "resolved": "https://registry.npmjs.org/strip-json-comments/-/strip-json-comments-3.1.1.tgz", "integrity": "sha512-6fPc+R4ihwqP6N/aIv2f1gMH8lOVtWQHoqC4yK6oSDVVocumAsfCqjkXnqiYMhmMwS/mEHLp7Vehlt3ql6lEig==" }, + "strnum": { + "version": "1.0.5", + "resolved": "https://registry.npmjs.org/strnum/-/strnum-1.0.5.tgz", + "integrity": "sha512-J8bbNyKKXl5qYcR36TIO8W3mVGVHrmmxsd5PAItGkmyzwJvybiw2IVq5nqd0i4LSNSkB/sx9VHllbfFdr9k1JA==" + }, "superagent": { "version": "8.0.9", "resolved": "https://registry.npmjs.org/superagent/-/superagent-8.0.9.tgz", @@ -33736,6 +33876,50 @@ "integrity": "sha512-2JAn3z8AR6rjK8Sm8orRC0h/bcl/DqL7tRPdGZ4I1CjdF+EaMLmYxBHyXuKL849eucPFhvBoxMsflfOb8kxaeQ==", "dev": true }, + "wellknown": { + "version": "0.5.0", + "resolved": "https://registry.npmjs.org/wellknown/-/wellknown-0.5.0.tgz", + "integrity": "sha512-za5vTLuPF9nmrVOovYQwNEWE/PwJCM+yHMAj4xN1WWUvtq9OElsvKiPL0CR9rO8xhrYqL7NpI7IknqR8r6eYOg==", + "requires": { + "concat-stream": "~1.5.0", + "minimist": "~1.2.0" + }, + "dependencies": { + "concat-stream": { + "version": "1.5.2", + "resolved": "https://registry.npmjs.org/concat-stream/-/concat-stream-1.5.2.tgz", + "integrity": "sha512-H6xsIBfQ94aESBG8jGHXQ7i5AEpy5ZeVaLDOisDICiTCKpqEfr34/KmTrspKQNoLKNu9gTkovlpQcUi630AKiQ==", + "requires": { + "inherits": "~2.0.1", + "readable-stream": "~2.0.0", + "typedarray": "~0.0.5" + } + }, + "process-nextick-args": { + "version": "1.0.7", + "resolved": "https://registry.npmjs.org/process-nextick-args/-/process-nextick-args-1.0.7.tgz", + "integrity": "sha512-yN0WQmuCX63LP/TMvAg31nvT6m4vDqJEiiv2CAZqWOGNWutc9DfDk1NPYYmKUFmaVM2UwDowH4u5AHWYP/jxKw==" + }, + "readable-stream": { + "version": "2.0.6", + "resolved": "https://registry.npmjs.org/readable-stream/-/readable-stream-2.0.6.tgz", + "integrity": "sha512-TXcFfb63BQe1+ySzsHZI/5v1aJPCShfqvWJ64ayNImXMsN1Cd0YGk/wm8KB7/OeessgPc9QvS9Zou8QTkFzsLw==", + "requires": { + "core-util-is": "~1.0.0", + "inherits": "~2.0.1", + "isarray": "~1.0.0", + "process-nextick-args": "~1.0.6", + "string_decoder": "~0.10.x", + "util-deprecate": "~1.0.1" + } + }, + "string_decoder": { + "version": "0.10.31", + "resolved": "https://registry.npmjs.org/string_decoder/-/string_decoder-0.10.31.tgz", + "integrity": "sha512-ev2QzSzWPYmy9GuqfIVildA4OdcGLeFZQrq5ys6RtiuF+RQQiZWr8TZNyAcuVXyQRYfEO+MsoB/1BuQVhOJuoQ==" + } + } + }, "whatwg-url": { "version": "5.0.0", "resolved": "https://registry.npmjs.org/whatwg-url/-/whatwg-url-5.0.0.tgz", diff --git a/package.json b/package.json index fb4bd0f9..0d74c749 100644 --- a/package.json +++ b/package.json @@ -1,9 +1,10 @@ { "name": "geocoding", - "version": "1.0.0", + "version": "0.5.0", "description": "Geocoding service for MapColonies", "main": "./src/index.ts", "scripts": { + "test:cli": "jest", "test:unit": "jest --config=./tests/configurations/unit/jest.config.js", "test:integration": "jest --config=./tests/configurations/integration/jest.config.js", "format": "prettier --check .", @@ -57,10 +58,12 @@ "@opentelemetry/api-metrics": "0.23.0", "@opentelemetry/instrumentation-express": "0.32.1", "@opentelemetry/instrumentation-http": "0.35.1", + "axios": "^0.21.1", "compression": "^1.7.4", "config": "^3.3.9", "express": "^4.19.2", "express-openapi-validator": "^5.0.4", + "fast-xml-parser": "^4.4.0", "geojson-validation": "^1.0.2", "http-status-codes": "^2.2.0", "mgrs": "^2.0.0", @@ -70,7 +73,8 @@ "proj4": "^2.11.0", "reflect-metadata": "^0.1.13", "tsyringe": "^4.8.0", - "typeorm": "^0.3.20" + "typeorm": "^0.3.20", + "wellknown": "^0.5.0" }, "devDependencies": { "@commitlint/cli": "^17.6.6", @@ -91,6 +95,7 @@ "@types/proj4": "^2.5.5", "@types/supertest": "^2.0.12", "@types/swagger-ui-express": "^4.1.3", + "@types/wellknown": "^0.5.6", "commitlint": "^17.6.6", "copyfiles": "^2.4.1", "cz-conventional-changelog": "^3.3.0", @@ -100,6 +105,7 @@ "jest-create-mock-instance": "^2.0.0", "jest-html-reporters": "^3.1.4", "jest-openapi": "^0.14.2", + "nock": "^13.5.4", "prettier": "^2.8.8", "pretty-quick": "^3.1.3", "rimraf": "^5.0.1", From d27f130c232fce444fd89fafea489afe1fecf4ad Mon Sep 17 00:00:00 2001 From: Niv Greenstein Date: Tue, 30 Jul 2024 10:23:44 +0300 Subject: [PATCH 32/67] delete: removed file --- devScripts/elasticsearchData.json | 145 ------------------------------ 1 file changed, 145 deletions(-) delete mode 100644 devScripts/elasticsearchData.json diff --git a/devScripts/elasticsearchData.json b/devScripts/elasticsearchData.json deleted file mode 100644 index 856ca077..00000000 --- a/devScripts/elasticsearchData.json +++ /dev/null @@ -1,145 +0,0 @@ -[ - { - "_id": "CONTROL.ITEMS", - "_score": 1, - "_source": { - "type": "Feature", - "id": 27, - "geometry": { - "coordinates": [ - [ - [98.96871358832425, 18.77187541003238], - [98.96711613001014, 18.772012912146167], - [98.9668257091372, 18.77957500633211], - [98.96517988589727, 18.783516234000658], - [98.96305002071, 18.786174113473763], - [98.96222710028087, 18.786036643850125], - [98.9615009529004, 18.784478609414165], - [98.96096850521184, 18.78324114944766], - [98.96058105300438, 18.74932410944021], - [98.96029062202649, 18.747169677704846], - [98.9619364723165, 18.74666541646775], - [98.96479250629358, 18.7514784826093], - [98.96401797557468, 18.75193687161918], - [98.96614791571238, 18.754412116891245], - [98.96639000300826, 18.75940841151673], - [98.96779381973744, 18.759133392649602], - [98.96798745662056, 18.76018763174328], - [98.96789063809456, 18.761746062357858], - [98.96905242991687, 18.763487833908357], - [98.96871358832425, 18.77187541003238] - ] - ], - "type": "Polygon" - }, - "properties": { - "OBJECTID": 27, - "F_CODE": 2300, - "F_ATT": 602, - "VIEW_SCALE_50K_CONTROL": 604, - "NAME": null, - "GFID": "{93C8C8E2-6AAB-4A84-BBF0-5B10E6F90938}", - "OBJECT_COMMAND_NAME": "4805", - "SUB_TILE_NAME": null, - "TILE_NAME": "DEF", - "TILE_ID": "36, 7300, 3560", - "SUB_TILE_ID": "36", - "SHAPE.AREA": 2.65, - "SHAPE.LEN": 0.0044192097656775, - "LAYER_NAME": "CONTROL.ITEMS", - "ENTITY_HEB": "airport", - "TYPE": "ITEM" - } - } - }, - { - "_id": "CONTROL.SUB_TILES", - "_score": 1, - "_source": { - "type": "Feature", - "id": 13668, - "geometry": { - "coordinates": [ - [ - [27.149158174343427, 35.63159611670335], - [27.149274355343437, 35.64061707270338], - [27.138786228343463, 35.640716597703374], - [27.13867103934342, 35.631695606703374], - [27.149158174343427, 35.63159611670335] - ] - ], - "type": "Polygon" - }, - "properties": { - "OBJECTID": 13668, - "SUB_TILE_ID": "65", - "TILE_NAME": "GRC", - "NAME": "somePlace", - "ZON": 35, - "LAYER_NAME": "CONTROL.SUB_TILES", - "TYPE": "SUB_TILE" - } - } - }, - { - "_id": "CONTROL.ROUTES", - "_score": 1, - "_source": { - "type": "Feature", - "id": 2, - "geometry": { - "coordinates": [ - [13.448493352142947, 52.31016611400918], - [13.447219581381603, 52.313370282889224], - [13.448088381125075, 52.31631514453963], - [13.450458681234068, 52.31867376333767], - [13.451112278530388, 52.32227665244022], - [13.449728938644029, 52.32463678850752], - [13.445021899434977, 52.32863442881066], - [13.444723882330948, 52.340023400115086], - [13.446229682887974, 52.34532799609971] - ], - "type": "LineString" - }, - "properties": { - "OBJECTID": 2, - "OBJECT_COMMAND_NAME": "route96", - "FIRST_GFID": null, - "SHAPE_Length": 4.1, - "F_CODE": 8754, - "F_ATT": 951, - "LAYER_NAME": "CONTROL.ROUTES", - "ENTITY_HEB": "route96", - "TYPE": "ROUTE" - } - } - }, - { - "_id": "CONTROL.TILES", - "_score": 1, - "_source": { - "type": "Feature", - "id": 52, - "geometry": { - "coordinates": [ - [ - [12.539507865186607, 41.851751203650096], - [12.536787075186538, 41.94185043165008], - [12.42879133518656, 41.93952837265009], - [12.431625055186686, 41.84943698365008], - [12.539507865186607, 41.851751203650096] - ] - ], - "type": "Polygon" - }, - "properties": { - "OBJECTID": 52, - "ZONE": "37", - "TILE_NAME": "RIT", - "TILE_ID": null, - "LAYER_NAME": "CONTROL.TILES", - "TYPE": "TILE" - } - } - } -] From ff5d9d381a5bd6ed4ca700ebe1b3e5e176efc960 Mon Sep 17 00:00:00 2001 From: Niv Greenstein Date: Tue, 30 Jul 2024 10:25:23 +0300 Subject: [PATCH 33/67] feat: created dev data --- devScripts/controlElasticsearchData.json | 145 ++++++++++ devScripts/geotextElasticsearchData.json | 344 +++++++++++++++++++++++ 2 files changed, 489 insertions(+) create mode 100644 devScripts/controlElasticsearchData.json create mode 100644 devScripts/geotextElasticsearchData.json diff --git a/devScripts/controlElasticsearchData.json b/devScripts/controlElasticsearchData.json new file mode 100644 index 00000000..856ca077 --- /dev/null +++ b/devScripts/controlElasticsearchData.json @@ -0,0 +1,145 @@ +[ + { + "_id": "CONTROL.ITEMS", + "_score": 1, + "_source": { + "type": "Feature", + "id": 27, + "geometry": { + "coordinates": [ + [ + [98.96871358832425, 18.77187541003238], + [98.96711613001014, 18.772012912146167], + [98.9668257091372, 18.77957500633211], + [98.96517988589727, 18.783516234000658], + [98.96305002071, 18.786174113473763], + [98.96222710028087, 18.786036643850125], + [98.9615009529004, 18.784478609414165], + [98.96096850521184, 18.78324114944766], + [98.96058105300438, 18.74932410944021], + [98.96029062202649, 18.747169677704846], + [98.9619364723165, 18.74666541646775], + [98.96479250629358, 18.7514784826093], + [98.96401797557468, 18.75193687161918], + [98.96614791571238, 18.754412116891245], + [98.96639000300826, 18.75940841151673], + [98.96779381973744, 18.759133392649602], + [98.96798745662056, 18.76018763174328], + [98.96789063809456, 18.761746062357858], + [98.96905242991687, 18.763487833908357], + [98.96871358832425, 18.77187541003238] + ] + ], + "type": "Polygon" + }, + "properties": { + "OBJECTID": 27, + "F_CODE": 2300, + "F_ATT": 602, + "VIEW_SCALE_50K_CONTROL": 604, + "NAME": null, + "GFID": "{93C8C8E2-6AAB-4A84-BBF0-5B10E6F90938}", + "OBJECT_COMMAND_NAME": "4805", + "SUB_TILE_NAME": null, + "TILE_NAME": "DEF", + "TILE_ID": "36, 7300, 3560", + "SUB_TILE_ID": "36", + "SHAPE.AREA": 2.65, + "SHAPE.LEN": 0.0044192097656775, + "LAYER_NAME": "CONTROL.ITEMS", + "ENTITY_HEB": "airport", + "TYPE": "ITEM" + } + } + }, + { + "_id": "CONTROL.SUB_TILES", + "_score": 1, + "_source": { + "type": "Feature", + "id": 13668, + "geometry": { + "coordinates": [ + [ + [27.149158174343427, 35.63159611670335], + [27.149274355343437, 35.64061707270338], + [27.138786228343463, 35.640716597703374], + [27.13867103934342, 35.631695606703374], + [27.149158174343427, 35.63159611670335] + ] + ], + "type": "Polygon" + }, + "properties": { + "OBJECTID": 13668, + "SUB_TILE_ID": "65", + "TILE_NAME": "GRC", + "NAME": "somePlace", + "ZON": 35, + "LAYER_NAME": "CONTROL.SUB_TILES", + "TYPE": "SUB_TILE" + } + } + }, + { + "_id": "CONTROL.ROUTES", + "_score": 1, + "_source": { + "type": "Feature", + "id": 2, + "geometry": { + "coordinates": [ + [13.448493352142947, 52.31016611400918], + [13.447219581381603, 52.313370282889224], + [13.448088381125075, 52.31631514453963], + [13.450458681234068, 52.31867376333767], + [13.451112278530388, 52.32227665244022], + [13.449728938644029, 52.32463678850752], + [13.445021899434977, 52.32863442881066], + [13.444723882330948, 52.340023400115086], + [13.446229682887974, 52.34532799609971] + ], + "type": "LineString" + }, + "properties": { + "OBJECTID": 2, + "OBJECT_COMMAND_NAME": "route96", + "FIRST_GFID": null, + "SHAPE_Length": 4.1, + "F_CODE": 8754, + "F_ATT": 951, + "LAYER_NAME": "CONTROL.ROUTES", + "ENTITY_HEB": "route96", + "TYPE": "ROUTE" + } + } + }, + { + "_id": "CONTROL.TILES", + "_score": 1, + "_source": { + "type": "Feature", + "id": 52, + "geometry": { + "coordinates": [ + [ + [12.539507865186607, 41.851751203650096], + [12.536787075186538, 41.94185043165008], + [12.42879133518656, 41.93952837265009], + [12.431625055186686, 41.84943698365008], + [12.539507865186607, 41.851751203650096] + ] + ], + "type": "Polygon" + }, + "properties": { + "OBJECTID": 52, + "ZONE": "37", + "TILE_NAME": "RIT", + "TILE_ID": null, + "LAYER_NAME": "CONTROL.TILES", + "TYPE": "TILE" + } + } + } +] diff --git a/devScripts/geotextElasticsearchData.json b/devScripts/geotextElasticsearchData.json new file mode 100644 index 00000000..3d0bddda --- /dev/null +++ b/devScripts/geotextElasticsearchData.json @@ -0,0 +1,344 @@ +[ + { + "_id": "12cd98ee-82eb-415e-a1d8-e1667a6f2e7f", + "_index": "geotext_index", + "_source": { + "source": "OSM", + "layer_name": "osm_airports", + "ingestion_timestamp": "2024-07-17 00:00:00", + "timestamp": "11/12/2023 00:00:00", + "source_id": ["{03ed6d97-fc81-4340-b68a-11993554eef1}"], + "sensitivity": "non-sensitive", + "placetype": "transportation", + "sub_placetype": "airport", + "wkt": "POLYGON ((-73.81278266814672 40.66039916690434, -73.8177430790069 40.66160065836647, -73.82178645734075 40.66043068890016, -73.82341214553732 40.658185554886586, -73.82407909454082 40.65496001884071, -73.8229536180974 40.650532558586235, -73.82211993184252 40.647939195437345, -73.81290769732489 40.643985699842915, -73.79214887014153 40.63414837731818, -73.78339516446982 40.62987771430167, -73.7898562329419 40.62275933562921, -73.78443726476769 40.620069953803636, -73.7791433570518 40.627188619100366, -73.77639219241223 40.62706207167477, -73.77159849644941 40.62336045339214, -73.77209870820208 40.619975033140975, -73.77047302000595 40.61908910045176, -73.76547094984971 40.628422477310664, -73.75338249916041 40.63291467256053, -73.74733827381596 40.63601474373485, -73.7467963777506 40.64208793530722, -73.752548854642 40.64749646458006, -73.76213624656812 40.65309424493557, -73.78181122379466 40.66270746643491, -73.79106514121902 40.66438330508498, -73.7957754685564 40.665205399216404, -73.79856831750864 40.66283394629252, -73.80390390953731 40.66175885985783, -73.8073637074931 40.66039916690434, -73.8109068740743 40.66074699797244, -73.81278266814672 40.66039916690434))", + "geo_json": { + "type": "Polygon", + "coordinates": [ + [ + [-73.81278266814672, 40.66039916690434], + [-73.8177430790069, 40.66160065836647], + [-73.82178645734075, 40.66043068890016], + [-73.82341214553732, 40.658185554886586], + [-73.82407909454082, 40.65496001884071], + [-73.8229536180974, 40.650532558586235], + [-73.82211993184252, 40.647939195437345], + [-73.81290769732489, 40.643985699842915], + [-73.79214887014153, 40.63414837731818], + [-73.78339516446982, 40.62987771430167], + [-73.7898562329419, 40.62275933562921], + [-73.78443726476769, 40.620069953803636], + [-73.7791433570518, 40.627188619100366], + [-73.77639219241223, 40.62706207167477], + [-73.77159849644941, 40.62336045339214], + [-73.77209870820208, 40.619975033140975], + [-73.77047302000595, 40.61908910045176], + [-73.76547094984971, 40.628422477310664], + [-73.75338249916041, 40.63291467256053], + [-73.74733827381596, 40.63601474373485], + [-73.7467963777506, 40.64208793530722], + [-73.752548854642, 40.64749646458006], + [-73.76213624656812, 40.65309424493557], + [-73.78181122379466, 40.66270746643491], + [-73.79106514121902, 40.66438330508498], + [-73.7957754685564, 40.665205399216404], + [-73.79856831750864, 40.66283394629252], + [-73.80390390953731, 40.66175885985783], + [-73.8073637074931, 40.66039916690434], + [-73.8109068740743, 40.66074699797244], + [-73.81278266814672, 40.66039916690434] + ] + ] + }, + "centroid": { + "type": "Point", + "coordinates": [-73.78543773614571, 40.64214724983408] + }, + "geometry_type": "Polygon", + "geometry_hash": "f14be42df6ec5cf2290d764a89e3009b", + "region": ["USA"], + "sub_region": ["New York"], + "name": "JFK", + "text": ["JFK Airport"], + "translated_text": ["Aeropuerto JFK"], + "text_language": "en" + } + }, + { + "_id": "8b5009b4-b643-4856-905d-ac95a2040aca", + "_index": "geotext_index", + "_source": { + "source": "OSM", + "layer_name": "osm_airports", + "ingestion_timestamp": "2024-07-17 00:00:00", + "timestamp": "11/12/2023 00:00:00", + "source_id": ["{009c6b65-3dcb-4c4f-9f02-d766ebb5d808}"], + "sensitivity": "non-sensitive", + "placetype": "transportation", + "sub_placetype": "airport", + "wkt": "POLYGON ((-73.50019138870562 40.76503398530525, -73.49991804403106 40.75762191351754, -73.50172211928987 40.75368779651572, -73.502924836068 40.74958778448408, -73.50095675396722 40.746067370663354, -73.49587254187276 40.74117989901089, -73.48810955136324 40.74010295019133, -73.4862508071566 40.74097279482331, -73.48450140084462 40.74209114977816, -73.48335335295225 40.743250905424986, -73.49368578398266 40.7500847690697, -73.49363111503533 40.75112014169309, -73.49166303293454 40.75087165373341, -73.49128035030373 40.75435040065955, -73.48406404926651 40.75385344795657, -73.4860321313673 40.75840870869581, -73.50019138870562 40.76503398530525))", + "geo_json": { + "coordinates": [ + [ + [-73.50019138870562, 40.76503398530525], + [-73.49991804403106, 40.75762191351754], + [-73.50172211928987, 40.75368779651572], + [-73.502924836068, 40.74958778448408], + [-73.50095675396722, 40.746067370663354], + [-73.49587254187276, 40.74117989901089], + [-73.48810955136324, 40.74010295019133], + [-73.4862508071566, 40.74097279482331], + [-73.48450140084462, 40.74209114977816], + [-73.48335335295225, 40.743250905424986], + [-73.49368578398266, 40.7500847690697], + [-73.49363111503533, 40.75112014169309], + [-73.49166303293454, 40.75087165373341], + [-73.49128035030373, 40.75435040065955], + [-73.48406404926651, 40.75385344795657], + [-73.4860321313673, 40.75840870869581], + [-73.50019138870562, 40.76503398530525] + ] + ], + "type": "Polygon" + }, + "centroid": { + "type": "Point", + "coordinates": [-73.49313909451013, 40.75256846774829] + }, + "geometry_type": "Polygon", + "geometry_hash": "f14be42df6ec5cf2290d764a89e3009b", + "region": ["USA"], + "sub_region": ["New York"], + "name": "Nassau County Police Airport", + "text": ["Nassau County Police Airport"], + "translated_text": ["Aeropuerto de la Policía del Condado de Nassau"], + "text_language": "en" + } + }, + { + "_id": "2bd0d4bb-af93-4211-9499-174c56feddb5", + "_index": "geotext_index", + "_source": { + "source": "OSM", + "layer_name": "osm_airports", + "ingestion_timestamp": "2024-07-17 00:00:00", + "timestamp": "11/12/2023 00:00:00", + "source_id": ["{a4f373ab-b824-41e2-b160-e7729c73bea6}"], + "sensitivity": "non-sensitive", + "placetype": "transportation", + "sub_placetype": "airport", + "wkt": "POLYGON ((-118.42713070992883 33.9512236894319, -118.43440343023548 33.94992163234602, -118.43571147345608 33.94670980636225, -118.43560682999878 33.943107208682775, -118.43325235220159 33.938723119106925, -118.42875268352236 33.9310829780122, -118.41394563426408 33.931820976094315, -118.39756893251248 33.932255089782316, -118.38516868278077 33.935250412835686, -118.37904703905322 33.936509285268926, -118.37899471732432 33.943367642688045, -118.3788377519829 33.94501703885841, -118.39181354073204 33.94527746687625, -118.39510980964849 33.945451085112225, -118.3980398264626 33.94558129855595, -118.39746428744567 33.949748023597365, -118.39720267894361 33.953003135913036, -118.40813791735772 33.95252573110099, -118.42713070992883 33.9512236894319))", + "geo_json": { + "coordinates": [ + [ + [-118.42713070992883, 33.9512236894319], + [-118.43440343023548, 33.94992163234602], + [-118.43571147345608, 33.94670980636225], + [-118.43560682999878, 33.943107208682775], + [-118.43325235220159, 33.938723119106925], + [-118.42875268352236, 33.9310829780122], + [-118.41394563426408, 33.931820976094315], + [-118.39756893251248, 33.932255089782316], + [-118.38516868278077, 33.935250412835686], + [-118.37904703905322, 33.936509285268926], + [-118.37899471732432, 33.943367642688045], + [-118.3788377519829, 33.94501703885841], + [-118.39181354073204, 33.94527746687625], + [-118.39510980964849, 33.945451085112225], + [-118.3980398264626, 33.94558129855595], + [-118.39746428744567, 33.949748023597365], + [-118.39720267894361, 33.953003135913036], + [-118.40813791735772, 33.95252573110099], + [-118.42713070992883, 33.9512236894319] + ] + ], + "type": "Polygon" + }, + "centroid": { + "type": "Point", + "coordinates": [-118.40727461271949, 33.94204305696262] + }, + "geometry_type": "Polygon", + "geometry_hash": "b7fce7d9eb045a2750ae42faff3b8f1b ", + "region": ["USA"], + "sub_region": ["Los Angeles"], + "name": "Los Angeles International Airport", + "text": ["Los Angeles International Airport"], + "translated_text": ["Aeropuerto Internacional de Los Ángeles"], + "text_language": "en" + } + }, + { + "_id": "ba951f2f-08c2-446e-845d-e2b0547d5d8c", + "_index": "geotext_index", + "_source": { + "source": "OSM", + "layer_name": "osm_ports", + "ingestion_timestamp": "2024-07-17 00:00:00", + "timestamp": "11/12/2023 00:00:00", + "source_id": ["{0f36d985-cfbd-4aed-b0cb-ee56600c77f4}"], + "sensitivity": "non-sensitive", + "placetype": "transportation", + "sub_placetype": "port", + "wkt": "POLYGON ((-118.2505781304088 33.7502674389752, -118.25604403409116 33.76075151051916, -118.27057180697577 33.748593059782564, -118.27503083555426 33.741097783576635, -118.2747911028351 33.734798055529765, -118.27215404292296 33.73136889520775, -118.26807858669537 33.720881194108856, -118.26424286318695 33.721997816398385, -118.26640045650717 33.72901625632974, -118.2431463824787 33.735794882347946, -118.24492040460113 33.739303607948656, -118.25072193640723 33.73794798097781, -118.25220827926702 33.74193505797223, -118.24937943317966 33.74508471776615, -118.24798898340768 33.74783559181691, -118.24909175391655 33.74803492708783, -118.25096166912684 33.74600168558719, -118.25326310323155 33.745363795966625, -118.25278363779321 33.74687877606813, -118.2505781304088 33.7502674389752))", + "geo_json": { + "type": "Polygon", + "coordinates": [ + [ + [-118.2505781304088, 33.7502674389752], + [-118.25604403409116, 33.76075151051916], + [-118.27057180697577, 33.748593059782564], + [-118.27503083555426, 33.741097783576635], + [-118.2747911028351, 33.734798055529765], + [-118.27215404292296, 33.73136889520775], + [-118.26807858669537, 33.720881194108856], + [-118.26424286318695, 33.721997816398385], + [-118.26640045650717, 33.72901625632974], + [-118.2431463824787, 33.735794882347946], + [-118.24492040460113, 33.739303607948656], + [-118.25072193640723, 33.73794798097781], + [-118.25220827926702, 33.74193505797223], + [-118.24937943317966, 33.74508471776615], + [-118.24798898340768, 33.74783559181691], + [-118.24909175391655, 33.74803492708783], + [-118.25096166912684, 33.74600168558719], + [-118.25326310323155, 33.745363795966625], + [-118.25278363779321, 33.74687877606813], + [-118.2505781304088, 33.7502674389752] + ] + ] + }, + "centroid": { + "type": "Point", + "coordinates": [-118.25908860901649, 33.740816352314006] + }, + "geometry_type": "Polygon", + "geometry_hash": "ffbc4f66c802719b5bcffaca238ee1d3", + "region": ["USA"], + "sub_region": ["Los Angeles"], + "name": "Port of Los Angeles", + "text": ["Port of Los Angeles"], + "translated_text": ["Puerto de Los Ángeles"], + "text_language": "en" + } + }, + { + "_id": "2b3dd420-d265-4665-94a6-3e13650b7a2d", + "_index": "geotext_index", + "_source": { + "source": "OSM", + "layer_name": "osm_schools", + "ingestion_timestamp": "2024-07-17 00:00:00", + "timestamp": "11/12/2023 00:00:00", + "source_id": ["{1a5b981b-bb0e-44dd-b9e2-424b92f2de49}"], + "sensitivity": "non-sensitive", + "placetype": "education", + "sub_placetype": "elementary school", + "wkt": "POLYGON ((-118.30812263653988 33.71684417247593, -118.30861990876181 33.71674433152869, -118.30879709771484 33.71635922964194, -118.30619642115158 33.71550819588987, -118.30586490633668 33.715921827872904, -118.30587062210924 33.716183318328746, -118.30812263653988 33.71684417247593))", + "geo_json": { + "coordinates": [ + [ + [-118.30812263653988, 33.71684417247593], + [-118.30861990876181, 33.71674433152869], + [-118.30879709771484, 33.71635922964194], + [-118.30619642115158, 33.71550819588987], + [-118.30586490633668, 33.715921827872904], + [-118.30587062210924, 33.716183318328746], + [-118.30812263653988, 33.71684417247593] + ] + ], + "type": "Polygon" + }, + "centroid": { + "type": "Point", + "coordinates": [-118.30733100202576, 33.7161761841829] + }, + "geometry_type": "Polygon", + "geometry_hash": "791f6b24002bbc1d50cdea5015244da9 ", + "region": ["USA"], + "sub_region": ["Los Angeles"], + "name": "White Point Elementary School", + "text": ["White Point Elementary School"], + "translated_text": ["Escuela Primaria White Point"], + "text_language": "en" + } + }, + { + "_id": "5a54ff54-53f6-4ad1-9d2a-cd20f333ee2b", + "_index": "hierarchies_index", + "_source": { + "geo_json": { + "coordinates": [ + [ + [-73.74286189030825, 40.566325396473786], + [-73.47084009765854, 40.56212896709357], + [-73.550927745189, 41.11163279131463], + [-73.74424271181776, 41.225972287315074], + [-73.99969469100891, 41.26438691280978], + [-74.24962338416366, 41.05959508414017], + [-74.13087273437748, 40.7506852054762], + [-74.00659879855483, 40.530651727069795], + [-73.74286189030825, 40.566325396473786] + ] + ], + "type": "Polygon" + }, + "hierarchy": "city", + "placetype": "city", + "region": "USA", + "text": "New York" + } + }, + { + "_id": "a3f38afc-1ce6-443c-a302-a0103c2bcb7d", + "_index": "hierarchies_index", + "_source": { + "geo_json": { + "coordinates": [ + [ + [-118.54430957638033, 34.07939240620722], + [-118.5350828996408, 33.695192367610986], + [-118.04596133238863, 33.47690745532634], + [-117.66265886139905, 33.379872950239346], + [-117.57145361502153, 33.63336904289318], + [-117.67279277766329, 34.23871934668085], + [-118.2605599209839, 34.28059749364003], + [-118.54430957638033, 34.07939240620722] + ] + ], + "type": "Polygon" + }, + "hierarchy": "city", + "placetype": "city", + "region": "USA", + "text": "Los Angeles" + } + }, + { + "_id": "430ddc04-2370-415f-8761-b8c66a142f0a", + "_index": "placetypes_index", + "_source": { + "placetype": "transportation", + "sub_placetype": "airport", + "sub_placetype_keyword": "airport" + } + }, + { + "_id": "6941ab8e-7503-4f8a-94f2-4750b2698db9", + "_index": "placetypes_index", + "_source": { + "placetype": "transportation", + "sub_placetype": "port", + "sub_placetype_keyword": "port" + } + }, + { + "_id": "6941ab8e-7503-4f8a-94f2-4750b2698db9", + "_index": "placetypes_index", + "_source": { + "placetype": "transportation", + "sub_placetype": "education", + "sub_placetype_keyword": "elementary school" + } + } +] From e61574214ac62135cfd6ae73c85d970831cb796b Mon Sep 17 00:00:00 2001 From: Niv Greenstein Date: Tue, 30 Jul 2024 10:31:09 +0300 Subject: [PATCH 34/67] feat: updated dev script --- devScripts/importDataToElastic.ts | 24 ++++++++++++++++++------ 1 file changed, 18 insertions(+), 6 deletions(-) diff --git a/devScripts/importDataToElastic.ts b/devScripts/importDataToElastic.ts index 951964a0..f23aefb0 100644 --- a/devScripts/importDataToElastic.ts +++ b/devScripts/importDataToElastic.ts @@ -2,19 +2,31 @@ /* eslint-disable @typescript-eslint/naming-convention */ /* eslint-disable @typescript-eslint/no-unsafe-member-access */ import { Client } from '@elastic/elasticsearch'; -import config from '../config/test.json'; -import data from './elasticsearchData.json'; +import config from '../config/default.json'; +import controlData from './controlElasticsearchData.json'; +import geotextData from './geotextElasticsearchData.json'; const main = async (): Promise => { - const client = new Client({ node: config.db.elastic.searchy.node as string }); + const controlClient = new Client({ ...config.db.elastic.control }); + const geotextClient = new Client({ ...config.db.elastic.geotext }); - for (const item of data) { - await client.index({ - index: config.db.elastic.searchy.properties.index as string, + for (const item of controlData) { + await controlClient.index({ + index: config.db.elastic.control.properties.index, id: item._id, body: item._source, }); } + + for (const item of geotextData) { + await geotextClient.index({ + index: item._index, + id: item._id, + body: { + ...item._source, + }, + }); + } }; export default main; From b5199a2cea21e17018f0f6c5a220af8019dc7995 Mon Sep 17 00:00:00 2001 From: Niv Greenstein Date: Tue, 30 Jul 2024 10:31:18 +0300 Subject: [PATCH 35/67] feat: update test.json --- config/test.json | 40 +++++++++++++++++++++++++++++++++------- 1 file changed, 33 insertions(+), 7 deletions(-) diff --git a/config/test.json b/config/test.json index 8ab5e167..6c4f3c2f 100644 --- a/config/test.json +++ b/config/test.json @@ -1,7 +1,7 @@ { "db": { "elastic": { - "searchy": { + "control": { "node": "http://localhost:9200", "auth": { "username": "elastic", @@ -9,11 +9,11 @@ }, "requestTimeout": 60000, "properties": { - "index": "control_gil_v5", - "size": 3 + "index": "control_index", + "defaultResponseLimit": 3 } }, - "nlp": { + "geotext": { "node": "http://localhost:9200", "auth": { "username": "elastic", @@ -21,8 +21,13 @@ }, "requestTimeout": 60000, "properties": { - "index": "nlp_gil_v5", - "size": 3 + "index": { + "geotext": "geotext_index", + "placetypes": "placetypes_index", + "hierarchies": "hierarchies_index" + }, + "defaultResponseLimit": 3, + "textTermLanguage": "en" } } }, @@ -39,7 +44,28 @@ "cert": "" }, "database": "postgres", - "schema": "geocoder_test" + "schema": "geocoder" } + }, + "application": { + "services": { + "tokenTypesUrl": "http://localhost:5001/NLP_ANALYSES" + }, + "cronLoadTileLatLonDataPattern": "0 * * * *", + "elasticQueryBoosts": { + "name": 1.1, + "placeType": 1.1, + "subPlaceType": 1.1, + "hierarchy": 1.1, + "viewbox": 1.1 + }, + "sources": { + "OSM": "OSM" + }, + "regions": { + "USA": ["New York", "Los Angeles"] + }, + "nameTranslationsKeys": ["en", "fr"], + "mainLanguageRegex": "[a-zA-Z]" } } From 5ba1cdb7ec40610e4fb2ff205de78c6ba10bbc91 Mon Sep 17 00:00:00 2001 From: Niv Greenstein Date: Tue, 30 Jul 2024 10:31:30 +0300 Subject: [PATCH 36/67] delete: removed console.log --- src/common/elastic/index.ts | 1 - 1 file changed, 1 deletion(-) diff --git a/src/common/elastic/index.ts b/src/common/elastic/index.ts index b3a0d7fd..dffa1841 100644 --- a/src/common/elastic/index.ts +++ b/src/common/elastic/index.ts @@ -11,7 +11,6 @@ export const initElasticsearchClient = async (clientOptions: ClientOptions): Pro }); try { await client.ping(); - console.log('Elastic connection established to:', clientOptions.node); } catch (error) { console.error("Can't connect to Elasticseach!", clientOptions.node, error); return null; From 78edd8233f39b1914c7c757d48281fac8a3213e6 Mon Sep 17 00:00:00 2001 From: Niv Greenstein Date: Tue, 30 Jul 2024 10:31:46 +0300 Subject: [PATCH 37/67] fix: updated import --- src/geotextSearch/controllers/geotextSearchController.ts | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/src/geotextSearch/controllers/geotextSearchController.ts b/src/geotextSearch/controllers/geotextSearchController.ts index bd8f022e..f8b776d7 100644 --- a/src/geotextSearch/controllers/geotextSearchController.ts +++ b/src/geotextSearch/controllers/geotextSearchController.ts @@ -5,13 +5,13 @@ import httpStatus from 'http-status-codes'; import { injectable, inject } from 'tsyringe'; import { SERVICES } from '../../common/constants'; import { GeotextSearchManager } from '../models/geotextSearchManager'; -import { GetQueryQueryParams, QueryResult } from '../interfaces'; +import { GetGeotextSearchParams, QueryResult } from '../interfaces'; type GetGeotextSearchHandler = RequestHandler< unknown, QueryResult | { message: string; error: string }, //response undefined, - GetQueryQueryParams + GetGeotextSearchParams >; type GetRegionshHandler = RequestHandler; From 9a8f2dc6a738a7bb6667e2fce6213b0f2ebe7485 Mon Sep 17 00:00:00 2001 From: Niv Greenstein Date: Tue, 30 Jul 2024 10:32:36 +0300 Subject: [PATCH 38/67] fix: removed unnecessary variable --- src/geotextSearch/DAL/queries.ts | 12 +++++------- 1 file changed, 5 insertions(+), 7 deletions(-) diff --git a/src/geotextSearch/DAL/queries.ts b/src/geotextSearch/DAL/queries.ts index 28d98b65..872d5b3f 100644 --- a/src/geotextSearch/DAL/queries.ts +++ b/src/geotextSearch/DAL/queries.ts @@ -126,18 +126,16 @@ export const geotextQuery = ( }); hierarchies.forEach((hierarchy) => { - const hierarchyGeoJSON = WKT.parse(hierarchy.geo_json); - const hierarchyShape = { - type: hierarchyGeoJSON!.type.toLowerCase(), - coordinates: (hierarchyGeoJSON as GeoJSONPolygon).coordinates, - }; - + const hierarchyShape = typeof hierarchy.geo_json === 'string' ? WKT.parse(hierarchy.geo_json) : hierarchy.geo_json; esQuery.query?.function_score?.functions?.push({ weight: hierarchy.weight, filter: { geo_shape: { [GEOJSON_FIELD]: { - shape: hierarchyShape, + shape: { + type: hierarchyShape?.type, + coordinates: (hierarchyShape as GeoJSONPolygon).coordinates, + }, }, }, }, From 116338bf2bb80dabe613da331fe18b88af1d7ebe Mon Sep 17 00:00:00 2001 From: Niv Greenstein Date: Tue, 30 Jul 2024 10:32:57 +0300 Subject: [PATCH 39/67] fix: updated HierarchySearchHit geo_json type --- src/geotextSearch/models/elasticsearchHits.ts | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/src/geotextSearch/models/elasticsearchHits.ts b/src/geotextSearch/models/elasticsearchHits.ts index 48a8a6b7..5bf98a3a 100644 --- a/src/geotextSearch/models/elasticsearchHits.ts +++ b/src/geotextSearch/models/elasticsearchHits.ts @@ -1,4 +1,4 @@ -import { GeoJSON } from "geojson"; +import { GeoJSON, Geometry } from 'geojson'; /* eslint-disable @typescript-eslint/naming-convention */ export interface TextSearchHit { @@ -26,7 +26,7 @@ export interface HierarchySearchHit { region: string; hierarchy: string; placetype: string; - geo_json: string; + geo_json: string | Geometry; weight?: number; } From 9997c38b98f26f2b476c3925fee932e749808a86 Mon Sep 17 00:00:00 2001 From: Niv Greenstein Date: Tue, 30 Jul 2024 10:33:14 +0300 Subject: [PATCH 40/67] fix: updated import and removed console.log --- src/geotextSearch/models/geotextSearchManager.ts | 5 ++--- 1 file changed, 2 insertions(+), 3 deletions(-) diff --git a/src/geotextSearch/models/geotextSearchManager.ts b/src/geotextSearch/models/geotextSearchManager.ts index 75db4608..eff42ad6 100644 --- a/src/geotextSearch/models/geotextSearchManager.ts +++ b/src/geotextSearch/models/geotextSearchManager.ts @@ -3,7 +3,7 @@ import { Logger } from '@map-colonies/js-logger'; import { inject, injectable } from 'tsyringe'; import { SERVICES, elasticConfigPath } from '../../common/constants'; import { GEOTEXT_REPOSITORY_SYMBOL, GeotextRepository } from '../DAL/geotextSearchRepository'; -import { GetQueryQueryParams, QueryResult, TextSearchParams } from '../interfaces'; +import { GetGeotextSearchParams, QueryResult, TextSearchParams } from '../interfaces'; import { convertResult, parseGeo } from '../utils'; import { IApplication } from '../../common/interfaces'; import { ElasticDbClientsConfig } from '../../common/interfaces'; @@ -17,7 +17,7 @@ export class GeotextSearchManager { @inject(GEOTEXT_REPOSITORY_SYMBOL) private readonly geotextRepository: GeotextRepository ) {} - public async search(params: GetQueryQueryParams): Promise { + public async search(params: GetGeotextSearchParams): Promise { const extractNameEndpoint = this.appConfig.services.tokenTypesUrl; const { geotext: geotextIndex, @@ -58,7 +58,6 @@ export class GeotextSearchManager { this.appConfig.elasticQueryBoosts ); - console.log(this.appConfig.regions); return convertResult(searchParams, esResult.hits.hits, { sources: this.appConfig.sources, regionCollection: this.appConfig.regions, From b28bd708f5116b9d92b7c0b015177cd6d82dc5a0 Mon Sep 17 00:00:00 2001 From: Niv Greenstein Date: Tue, 30 Jul 2024 10:33:26 +0300 Subject: [PATCH 41/67] fix: updated interface name --- src/geotextSearch/interfaces.ts | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/geotextSearch/interfaces.ts b/src/geotextSearch/interfaces.ts index f19465cb..b57fc493 100644 --- a/src/geotextSearch/interfaces.ts +++ b/src/geotextSearch/interfaces.ts @@ -24,7 +24,7 @@ export interface TextSearchParams { limit: number; } -export interface GetQueryQueryParams { +export interface GetGeotextSearchParams { query: string; limit: number; source?: string[]; From 674ad37a1dfefbcd273a102539f85152c7653ab0 Mon Sep 17 00:00:00 2001 From: Niv Greenstein Date: Tue, 30 Jul 2024 10:34:13 +0300 Subject: [PATCH 42/67] fix: change docs route back to be on the main router instead under /v1 --- src/serverBuilder.ts | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/src/serverBuilder.ts b/src/serverBuilder.ts index 732ba985..9e8b387c 100644 --- a/src/serverBuilder.ts +++ b/src/serverBuilder.ts @@ -38,19 +38,20 @@ export class ServerBuilder { public build(): express.Application { this.registerPreRoutesMiddleware(); + this.buildDocsRoutes(); this.buildRoutesV1(); this.registerPostRoutesMiddleware(); return this.serverInstance; } - private buildDocsRoutes(router: Router): void { + private buildDocsRoutes(): void { const openapiRouter = new OpenapiViewerRouter({ ...this.config.get('openapiConfig'), filePathOrSpec: this.config.get('openapiConfig.filePath'), }); openapiRouter.setup(); - router.use(this.config.get('openapiConfig.basePath'), openapiRouter.getRouter()); + this.serverInstance.use(this.config.get('openapiConfig.basePath'), openapiRouter.getRouter()); } private buildRoutesV1(): void { @@ -60,7 +61,6 @@ export class ServerBuilder { router.use('/search/routes', this.routeRouter); router.use('/lookup', this.latLonRouter); router.use('/query', this.geotextRouter); - this.buildDocsRoutes(router); this.serverInstance.use('/v1', router); } From 32825c3c5ac77ff5af9fb0b3803a68e36a0e7be0 Mon Sep 17 00:00:00 2001 From: Niv Greenstein Date: Tue, 30 Jul 2024 10:34:49 +0300 Subject: [PATCH 43/67] fix: added missing x-user-id header --- tests/integration/item/helpers/requestSender.ts | 3 ++- tests/integration/latLon/helpers/requestSender.ts | 6 +++++- tests/integration/route/helpers/requestSender.ts | 3 ++- tests/integration/tile/helpers/requestSender.ts | 3 ++- 4 files changed, 11 insertions(+), 4 deletions(-) diff --git a/tests/integration/item/helpers/requestSender.ts b/tests/integration/item/helpers/requestSender.ts index 6b07c6fd..28f460fa 100644 --- a/tests/integration/item/helpers/requestSender.ts +++ b/tests/integration/item/helpers/requestSender.ts @@ -7,9 +7,10 @@ export class ItemRequestSender { public async getItems(queryParams?: GetItemsQueryParams): Promise { return supertest .agent(this.app) - .get('/search/items/') + .get('/v1/search/items/') .set('Content-Type', 'application/json') .set('X-API-Key', 'abc123') + .set('x-user-id', 'abc123') .query(queryParams ?? {}); } } diff --git a/tests/integration/latLon/helpers/requestSender.ts b/tests/integration/latLon/helpers/requestSender.ts index 1cada784..8ace3344 100644 --- a/tests/integration/latLon/helpers/requestSender.ts +++ b/tests/integration/latLon/helpers/requestSender.ts @@ -6,7 +6,7 @@ import { GetMgrsToLatLonQueryParams, } from '../../../../src/latLon/controllers/latLonController'; -const PREFIX = '/lookup'; +const PREFIX = '/v1/lookup'; export class LatLonRequestSender { public constructor(private readonly app: Express.Application) {} @@ -17,6 +17,7 @@ export class LatLonRequestSender { .get(`${PREFIX}/latlonToTile`) .set('Content-Type', 'application/json') .set('X-API-Key', 'abc123') + .set('x-user-id', 'abc123') .query(queryParams ?? {}); } @@ -26,6 +27,7 @@ export class LatLonRequestSender { .get(`${PREFIX}/tileToLatLon`) .set('Content-Type', 'application/json') .set('X-API-Key', 'abc123') + .set('x-user-id', 'abc123') .query(queryParams ?? {}); } @@ -35,6 +37,7 @@ export class LatLonRequestSender { .get(`${PREFIX}/latlonToMgrs`) .set('Content-Type', 'application/json') .set('X-API-Key', 'abc123') + .set('x-user-id', 'abc123') .query(queryParams ?? {}); } @@ -44,6 +47,7 @@ export class LatLonRequestSender { .get(`${PREFIX}/mgrsToLatLon`) .set('Content-Type', 'application/json') .set('X-API-Key', 'abc123') + .set('x-user-id', 'abc123') .query(queryParams ?? {}); } } diff --git a/tests/integration/route/helpers/requestSender.ts b/tests/integration/route/helpers/requestSender.ts index 767155ee..adac30a7 100644 --- a/tests/integration/route/helpers/requestSender.ts +++ b/tests/integration/route/helpers/requestSender.ts @@ -7,9 +7,10 @@ export class RouteRequestSender { public async getRoutes(queryParams?: GetRoutesQueryParams): Promise { return supertest .agent(this.app) - .get('/search/routes/') + .get('/v1/search/routes/') .set('Content-Type', 'application/json') .set('X-API-Key', 'abc123') + .set('x-user-id', 'abc123') .query(queryParams ?? {}); } } diff --git a/tests/integration/tile/helpers/requestSender.ts b/tests/integration/tile/helpers/requestSender.ts index 2d8ec5b4..0eaed8de 100644 --- a/tests/integration/tile/helpers/requestSender.ts +++ b/tests/integration/tile/helpers/requestSender.ts @@ -7,9 +7,10 @@ export class TileRequestSender { public async getTiles(queryParams?: GetTilesQueryParams): Promise { return supertest .agent(this.app) - .get('/search/tiles/') + .get('/v1/search/tiles/') .set('Content-Type', 'application/json') .set('X-API-Key', 'abc123') + .set('x-user-id', 'abc123') .query(queryParams ?? {}); } } From 3aec2d237bde732af17db860b01c5c97967c81cb Mon Sep 17 00:00:00 2001 From: Niv Greenstein Date: Tue, 30 Jul 2024 11:00:27 +0300 Subject: [PATCH 44/67] feat: created GeotextSearchRequestSender --- .../geotextSearch/helpers/requestSender.ts | 25 +++++++++++++++++++ 1 file changed, 25 insertions(+) create mode 100644 tests/integration/geotextSearch/helpers/requestSender.ts diff --git a/tests/integration/geotextSearch/helpers/requestSender.ts b/tests/integration/geotextSearch/helpers/requestSender.ts new file mode 100644 index 00000000..c924f538 --- /dev/null +++ b/tests/integration/geotextSearch/helpers/requestSender.ts @@ -0,0 +1,25 @@ +import * as supertest from 'supertest'; +import { GetGeotextSearchParams } from '../../../../src/geotextSearch/interfaces'; + +export class GeotextSearchRequestSender { + public constructor(private readonly app: Express.Application) {} + + public async getGeotextSearch(queryParams?: GetGeotextSearchParams): Promise { + return supertest + .agent(this.app) + .get('/v1/query') + .set('Content-Type', 'application/json') + .set('X-API-Key', 'abc123') + .set('x-user-id', 'abc123') + .query(queryParams ?? {}); + } + + public async getRegions(): Promise { + return supertest + .agent(this.app) + .get('/v1/query/regions') + .set('Content-Type', 'application/json') + .set('X-API-Key', 'abc123') + .set('x-user-id', 'abc123'); + } +} From df2fde65d074b0ac9d8cd8a4e3c655170944403d Mon Sep 17 00:00:00 2001 From: Niv Greenstein Date: Tue, 30 Jul 2024 11:00:41 +0300 Subject: [PATCH 45/67] feat: added mock data --- .../geotextSearch/possibleObjects.ts | 167 ++++++++++++++++++ 1 file changed, 167 insertions(+) create mode 100644 tests/integration/geotextSearch/possibleObjects.ts diff --git a/tests/integration/geotextSearch/possibleObjects.ts b/tests/integration/geotextSearch/possibleObjects.ts new file mode 100644 index 00000000..5ddd36e2 --- /dev/null +++ b/tests/integration/geotextSearch/possibleObjects.ts @@ -0,0 +1,167 @@ +/* eslint-disable @typescript-eslint/naming-convention */ +/* eslint-disable @typescript-eslint/no-magic-numbers */ +export const jfkAirport = (rank: number) => ({ + type: 'Feature', + geometry: { + type: 'Polygon', + coordinates: [ + [ + [-73.81278266814672, 40.66039916690434], + [-73.8177430790069, 40.66160065836647], + [-73.82178645734075, 40.66043068890016], + [-73.82341214553732, 40.658185554886586], + [-73.82407909454082, 40.65496001884071], + [-73.8229536180974, 40.650532558586235], + [-73.82211993184252, 40.647939195437345], + [-73.81290769732489, 40.643985699842915], + [-73.79214887014153, 40.63414837731818], + [-73.78339516446982, 40.62987771430167], + [-73.7898562329419, 40.62275933562921], + [-73.78443726476769, 40.620069953803636], + [-73.7791433570518, 40.627188619100366], + [-73.77639219241223, 40.62706207167477], + [-73.77159849644941, 40.62336045339214], + [-73.77209870820208, 40.619975033140975], + [-73.77047302000595, 40.61908910045176], + [-73.76547094984971, 40.628422477310664], + [-73.75338249916041, 40.63291467256053], + [-73.74733827381596, 40.63601474373485], + [-73.7467963777506, 40.64208793530722], + [-73.752548854642, 40.64749646458006], + [-73.76213624656812, 40.65309424493557], + [-73.78181122379466, 40.66270746643491], + [-73.79106514121902, 40.66438330508498], + [-73.7957754685564, 40.665205399216404], + [-73.79856831750864, 40.66283394629252], + [-73.80390390953731, 40.66175885985783], + [-73.8073637074931, 40.66039916690434], + [-73.8109068740743, 40.66074699797244], + [-73.81278266814672, 40.66039916690434], + ], + ], + }, + properties: { + rank, + source: 'OSM', + layer: 'osm_airports', + source_id: ['03ed6d97-fc81-4340-b68a-11993554eef1'], + name: { + en: ['JFK Airport'], + fr: ['Aeropuerto JFK'], + default: ['JFK'], + display: 'JFK', + }, + placetype: 'transportation', + sub_placetype: 'airport', + regions: [ + { + region: 'USA', + sub_regions: ['New York'], + }, + ], + region: ['USA'], + sub_region: ['New York'], + }, +}); + +export const policeAirport = (rank: number) => ({ + type: 'Feature', + geometry: { + coordinates: [ + [ + [-73.50019138870562, 40.76503398530525], + [-73.49991804403106, 40.75762191351754], + [-73.50172211928987, 40.75368779651572], + [-73.502924836068, 40.74958778448408], + [-73.50095675396722, 40.746067370663354], + [-73.49587254187276, 40.74117989901089], + [-73.48810955136324, 40.74010295019133], + [-73.4862508071566, 40.74097279482331], + [-73.48450140084462, 40.74209114977816], + [-73.48335335295225, 40.743250905424986], + [-73.49368578398266, 40.7500847690697], + [-73.49363111503533, 40.75112014169309], + [-73.49166303293454, 40.75087165373341], + [-73.49128035030373, 40.75435040065955], + [-73.48406404926651, 40.75385344795657], + [-73.4860321313673, 40.75840870869581], + [-73.50019138870562, 40.76503398530525], + ], + ], + type: 'Polygon', + }, + properties: { + rank, + source: 'OSM', + layer: 'osm_airports', + source_id: ['009c6b65-3dcb-4c4f-9f02-d766ebb5d808'], + name: { + en: ['Nassau County Police Airport'], + fr: ['Aeropuerto de la Policía del Condado de Nassau'], + default: ['Nassau County Police Airport'], + display: 'Nassau County Police Airport', + }, + placetype: 'transportation', + sub_placetype: 'airport', + regions: [ + { + region: 'USA', + sub_regions: ['New York'], + }, + ], + region: ['USA'], + sub_region: ['New York'], + }, +}); + +export const losAngelesAirport = (rank: number) => ({ + type: 'Feature', + geometry: { + coordinates: [ + [ + [-118.42713070992883, 33.9512236894319], + [-118.43440343023548, 33.94992163234602], + [-118.43571147345608, 33.94670980636225], + [-118.43560682999878, 33.943107208682775], + [-118.43325235220159, 33.938723119106925], + [-118.42875268352236, 33.9310829780122], + [-118.41394563426408, 33.931820976094315], + [-118.39756893251248, 33.932255089782316], + [-118.38516868278077, 33.935250412835686], + [-118.37904703905322, 33.936509285268926], + [-118.37899471732432, 33.943367642688045], + [-118.3788377519829, 33.94501703885841], + [-118.39181354073204, 33.94527746687625], + [-118.39510980964849, 33.945451085112225], + [-118.3980398264626, 33.94558129855595], + [-118.39746428744567, 33.949748023597365], + [-118.39720267894361, 33.953003135913036], + [-118.40813791735772, 33.95252573110099], + [-118.42713070992883, 33.9512236894319], + ], + ], + type: 'Polygon', + }, + properties: { + rank, + source: 'OSM', + layer: 'osm_airports', + source_id: ['a4f373ab-b824-41e2-b160-e7729c73bea6'], + name: { + en: ['Los Angeles International Airport'], + fr: ['Aeropuerto Internacional de Los Ángeles'], + default: ['Los Angeles International Airport'], + display: 'Los Angeles International Airport', + }, + placetype: 'transportation', + sub_placetype: 'airport', + regions: [ + { + region: 'USA', + sub_regions: ['Los Angeles'], + }, + ], + region: ['USA'], + sub_region: ['Los Angeles'], + }, +}); From 98f0d603cd6c16eabc81f77e0592f20bbc4acf54 Mon Sep 17 00:00:00 2001 From: Niv Greenstein Date: Wed, 31 Jul 2024 09:36:57 +0300 Subject: [PATCH 46/67] feat: added tests --- .../integration/geotextSearch/geotext.spec.ts | 172 ++++++++++++++++++ 1 file changed, 172 insertions(+) create mode 100644 tests/integration/geotextSearch/geotext.spec.ts diff --git a/tests/integration/geotextSearch/geotext.spec.ts b/tests/integration/geotextSearch/geotext.spec.ts new file mode 100644 index 00000000..42f0c526 --- /dev/null +++ b/tests/integration/geotextSearch/geotext.spec.ts @@ -0,0 +1,172 @@ +/* eslint-disable @typescript-eslint/naming-convention */ +import config from 'config'; +import jsLogger from '@map-colonies/js-logger'; +import { trace } from '@opentelemetry/api'; +import httpStatusCodes from 'http-status-codes'; +import nock from 'nock'; +import { getApp } from '../../../src/app'; +import { SERVICES } from '../../../src/common/constants'; +import { IApplication } from '../../../src/common/interfaces'; +import { jfkAirport, losAngelesAirport, policeAirport } from './possibleObjects'; +import { GeotextSearchRequestSender } from './helpers/requestSender'; + +describe('/query', function () { + let requestSender: GeotextSearchRequestSender; + + beforeEach(async function () { + const app = await getApp({ + override: [ + { token: SERVICES.LOGGER, provider: { useValue: jsLogger({ enabled: false }) } }, + { token: SERVICES.TRACER, provider: { useValue: trace.getTracer('testTracer') } }, + ], + useChild: true, + }); + requestSender = new GeotextSearchRequestSender(app.app); + }); + + describe('Happy Path', function () { + it('should return 200 status code and the all available regions', async function () { + const response = await requestSender.getRegions(); + + expect(response.status).toBe(httpStatusCodes.OK); + // expect(response).toSatisfyApiSpec(); + + expect(response.body).toEqual(['USA']); + }); + + it('should return 200 status code and all airports', async function () { + const tokenTypesUrlScope = nock(config.get('application').services.tokenTypesUrl) + .post('', { tokens: ['airport'] }) + .reply(httpStatusCodes.OK, [ + { + tokens: ['airport'], + prediction: ['essence'], + }, + ]); + + const response = await requestSender.getGeotextSearch({ query: 'airport', limit: 10 }); + + expect(response.status).toBe(httpStatusCodes.OK); + + // expect(response).toSatisfyApiSpec(); + + expect(response.body).toMatchObject({ + type: 'FeatureCollection', + geocoding: { + version: process.env.npm_package_version, + query: { + query: 'airport', + limit: 10, + name: '', + placeTypes: ['transportation'], + subPlaceTypes: ['airport'], + hierarchies: [], + }, + name: '', + }, + features: [jfkAirport(1), policeAirport(2), losAngelesAirport(3)], + }); + + tokenTypesUrlScope.done(); + }); + }); + + it('should return 200 and los angeles airport while checking the hierarchy system', async () => { + const tokenTypesUrlScope = nock(config.get('application').services.tokenTypesUrl) + .post('', { tokens: ['airport'] }) + .reply(httpStatusCodes.OK, [ + { + tokens: ['airport'], + prediction: ['essence'], + }, + ]); + + const response = await requestSender.getGeotextSearch({ query: 'airport, los angeles', limit: 1 }); + + expect(response.status).toBe(httpStatusCodes.OK); + // expect(response).toSatisfyApiSpec(); + expect(response.body).toMatchObject({ + type: 'FeatureCollection', + geocoding: { + version: process.env.npm_package_version, + query: { + query: 'airport', + limit: 1, + name: '', + placeTypes: ['transportation'], + subPlaceTypes: ['airport'], + hierarchies: [ + { + geo_json: { + coordinates: [ + [ + [-118.54430957638033, 34.07939240620722], + [-118.5350828996408, 33.695192367610986], + [-118.04596133238863, 33.47690745532634], + [-117.66265886139905, 33.379872950239346], + [-117.57145361502153, 33.63336904289318], + [-117.67279277766329, 34.23871934668085], + [-118.2605599209839, 34.28059749364003], + [-118.54430957638033, 34.07939240620722], + ], + ], + type: 'Polygon', + }, + hierarchy: 'city', + placetype: 'city', + region: 'USA', + text: 'Los Angeles', + weight: 1.1, + }, + ], + }, + name: '', + }, + features: [losAngelesAirport(1)], + }); + + tokenTypesUrlScope.done(); + }); + + it('should retrun 200 and los angeles airport while checking the region from NLP analyser', async () => { + const tokenTypesUrlScope = nock(config.get('application').services.tokenTypesUrl) + .post('', { tokens: ['airport', 'los', 'angeles'] }) + .reply(httpStatusCodes.OK, [ + { + tokens: ['airport', 'los', 'angeles'], + prediction: ['essence', 'essence', 'name'], + }, + ]); + + const response = await requestSender.getGeotextSearch({ query: 'airport los angeles', limit: 1 }); + + expect(response.status).toBe(httpStatusCodes.OK); + + // expect(response).toSatisfyApiSpec(); + + expect(response.body).toMatchObject({ + type: 'FeatureCollection', + geocoding: { + version: process.env.npm_package_version, + query: { + query: 'airport los angeles', + limit: 1, + name: 'angeles', + placeTypes: ['transportation'], + subPlaceTypes: ['airport'], + hierarchies: [], + }, + name: 'angeles', + }, + features: [losAngelesAirport(1)], + }); + + tokenTypesUrlScope.done(); + }); + describe('Bad Path', function () { + // All requests with status code of 400 + }); + describe('Sad Path', function () { + // All requests with status code 4XX-5XX + }); +}); From bcc61f0550587e71b5f7f290f756bd84cecfe22e Mon Sep 17 00:00:00 2001 From: Niv Greenstein Date: Tue, 6 Aug 2024 10:56:08 +0300 Subject: [PATCH 47/67] feat(elastic): created elasticClientFactory --- src/common/elastic/index.ts | 55 ++++++++++++++++++++++++++----------- 1 file changed, 39 insertions(+), 16 deletions(-) diff --git a/src/common/elastic/index.ts b/src/common/elastic/index.ts index dffa1841..2c715e6a 100644 --- a/src/common/elastic/index.ts +++ b/src/common/elastic/index.ts @@ -1,22 +1,45 @@ +import { Logger } from '@map-colonies/js-logger'; import { Client, ClientOptions } from '@elastic/elasticsearch'; +import { DependencyContainer, FactoryFunction } from 'tsyringe'; +import { IConfig } from '../interfaces'; +import { elasticConfigPath, SERVICES } from '../constants'; +import { ElasticDbClientsConfig, ElasticDbConfig } from './interfaces'; -export const initElasticsearchClient = async (clientOptions: ClientOptions): Promise => { - const client = new Client({ - ...clientOptions, - sniffOnStart: false, - sniffOnConnectionFault: false, - tls: { - rejectUnauthorized: false, - }, - }); - try { - await client.ping(); - } catch (error) { - console.error("Can't connect to Elasticseach!", clientOptions.node, error); - return null; - } +const createConnectionOptions = (clientOptions: ClientOptions): ClientOptions => ({ + ...clientOptions, + sniffOnStart: false, + sniffOnConnectionFault: false, + tls: { + rejectUnauthorized: false, + }, +}); +const initElasticsearchClient = (clientOptions: ClientOptions): ElasticClient => { + const client = new Client(createConnectionOptions(clientOptions)); return client; }; -export const elasticClientSymbol = Symbol('elasticClient'); +export type ElasticClient = Client | null; + +export const elasticClientFactory: FactoryFunction = (container: DependencyContainer): ElasticClients => { + const config = container.resolve(SERVICES.CONFIG); + const logger = container.resolve(SERVICES.LOGGER); + + const elasticClientsConfig = config.get(elasticConfigPath); + + const elasticClients = {} as ElasticClients; + + for (const [key, value] of Object.entries(elasticClientsConfig)) { + try { + const client = initElasticsearchClient(value as ElasticDbConfig); + elasticClients[key as keyof ElasticDbClientsConfig] = client; + } catch (err) { + logger.error('Failed to connect to Elasticsearch', err); + elasticClients[key as keyof ElasticDbClientsConfig] = null; + } + } + + return elasticClients; +}; + +export type ElasticClients = Record; From f9b2e7bfab436e3044c94b568438fea6c664045d Mon Sep 17 00:00:00 2001 From: Niv Greenstein Date: Tue, 6 Aug 2024 10:57:08 +0300 Subject: [PATCH 48/67] fix(containerConfig): changed elastic clients init to use factory init --- src/containerConfig.ts | 36 ++++++++++++++++++++++++------------ 1 file changed, 24 insertions(+), 12 deletions(-) diff --git a/src/containerConfig.ts b/src/containerConfig.ts index 4e18a334..e3b4d6e5 100644 --- a/src/containerConfig.ts +++ b/src/containerConfig.ts @@ -1,15 +1,16 @@ import config from 'config'; import { getOtelMixin } from '@map-colonies/telemetry'; import { DataSource } from 'typeorm'; +import { instancePerContainerCachingFactory } from 'tsyringe'; import { trace, metrics as OtelMetrics } from '@opentelemetry/api'; import { DependencyContainer } from 'tsyringe/dist/typings/types'; import jsLogger, { LoggerOptions } from '@map-colonies/js-logger'; import { Metrics } from '@map-colonies/telemetry'; -import { SERVICES, SERVICE_NAME, elasticConfigPath } from './common/constants'; +import { SERVICES, SERVICE_NAME } from './common/constants'; import { tracing } from './common/tracing'; import { InjectionObject, registerDependencies } from './common/dependencyRegistration'; -import { elasticClientSymbol, initElasticsearchClient } from './common/elastic'; -import { ElasticClients, ElasticDbClientsConfig, ElasticDbConfig, IApplication, PostgresDbConfig } from './common/interfaces'; +import { elasticClientFactory, ElasticClients } from './common/elastic'; +import { IApplication, PostgresDbConfig } from './common/interfaces'; import { TILE_REPOSITORY_SYMBOL, tileRepositoryFactory } from './tile/DAL/tileRepository'; import { TILE_ROUTER_SYMBOL, tileRouterFactory } from './tile/routes/tileRouter'; import { ITEM_REPOSITORY_SYMBOL, itemRepositoryFactory } from './item/DAL/itemRepository'; @@ -40,13 +41,8 @@ export const registerExternalValues = async (options?: RegisterOptions): Promise const applicationConfig: IApplication = config.get('application'); - const elasticClientsConfig = config.get(elasticConfigPath); - const postgresqlDataSourceOptions = config.get('db.postgresql'); - const elasticClients = {} as Partial; - for (const [key, value] of Object.entries(elasticClientsConfig)) { - elasticClients[key as keyof ElasticDbClientsConfig] = (await initElasticsearchClient(value as ElasticDbConfig)) ?? undefined; - } + const postgresqlConnection = await initDataSource(postgresqlDataSourceOptions); const dependencies: InjectionObject[] = [ @@ -55,7 +51,23 @@ export const registerExternalValues = async (options?: RegisterOptions): Promise { token: SERVICES.TRACER, provider: { useValue: tracer } }, { token: SERVICES.METER, provider: { useValue: OtelMetrics.getMeterProvider().getMeter(SERVICE_NAME) } }, { token: SERVICES.APPLICATION, provider: { useValue: applicationConfig } }, - { token: elasticClientSymbol, provider: { useValue: elasticClients } }, + { + token: SERVICES.ELASTIC_CLIENTS, + provider: { useFactory: instancePerContainerCachingFactory(elasticClientFactory) }, + postInjectionHook: async (deps: DependencyContainer): Promise => { + const elasticClients = deps.resolve(SERVICES.ELASTIC_CLIENTS); + try { + const response = await Promise.all([elasticClients.control?.ping(), elasticClients.geotext?.ping()]); + response.forEach((res) => { + if (!res) { + logger.error('Failed to connect to Elasticsearch', res); + } + }); + } catch (err) { + logger.error('Failed to connect to Elasticsearch', err); + } + }, + }, { token: DataSource, provider: { useValue: postgresqlConnection } }, { token: TILE_REPOSITORY_SYMBOL, provider: { useFactory: tileRepositoryFactory } }, { token: TILE_ROUTER_SYMBOL, provider: { useFactory: tileRouterFactory } }, @@ -84,6 +96,6 @@ export const registerExternalValues = async (options?: RegisterOptions): Promise }, }, ]; - - return registerDependencies(dependencies, options?.override, options?.useChild); + const container = await registerDependencies(dependencies, options?.override, options?.useChild); + return container; }; From 97f6a85eff95753f0b3ffcfa563b1b330d1b8c21 Mon Sep 17 00:00:00 2001 From: Niv Greenstein Date: Tue, 6 Aug 2024 10:57:38 +0300 Subject: [PATCH 49/67] fix(elastic/interfaces): changed ElasticDbClientsConfig location --- src/common/elastic/interfaces.ts | 17 +++++++++++++++++ 1 file changed, 17 insertions(+) create mode 100644 src/common/elastic/interfaces.ts diff --git a/src/common/elastic/interfaces.ts b/src/common/elastic/interfaces.ts new file mode 100644 index 00000000..102332dd --- /dev/null +++ b/src/common/elastic/interfaces.ts @@ -0,0 +1,17 @@ +import { ClientOptions } from '@elastic/elasticsearch'; + +export type ElasticDbClientsConfig = { + [key in 'control' | 'geotext']: ElasticDbConfig & { + properties: { + index: + | string + | { + [key: string]: string; + }; + defaultResponseLimit: number; + textTermLanguage: string; + }; + }; +}; + +export type ElasticDbConfig = ClientOptions; From ae14d8ce41e546da8e5e129869b2b63c4077f373 Mon Sep 17 00:00:00 2001 From: Niv Greenstein Date: Tue, 6 Aug 2024 10:58:15 +0300 Subject: [PATCH 50/67] feat(elastic/utils): created queryElastic common function and additionalControlSearchProperties rename --- src/common/elastic/utils.ts | 27 +++++++++++++++++++++++++++ 1 file changed, 27 insertions(+) diff --git a/src/common/elastic/utils.ts b/src/common/elastic/utils.ts index eb82cd3e..6b4653be 100644 --- a/src/common/elastic/utils.ts +++ b/src/common/elastic/utils.ts @@ -1,4 +1,31 @@ +import config from 'config'; +import { estypes } from '@elastic/elasticsearch'; import { WGS84Coordinate } from '../interfaces'; +import { InternalServerError } from '../errors'; +import { elasticConfigPath, CONTROL_FIELDS } from '../constants'; +import { ElasticDbClientsConfig } from './interfaces'; +import { ElasticClient } from './index'; + +/* eslint-disable @typescript-eslint/naming-convention */ +export const additionalControlSearchProperties = (size: number): { size: number; index: string; _source: string[] } => ({ + size, + index: config.get(elasticConfigPath).control.properties.index as string, + _source: CONTROL_FIELDS, +}); +/* eslint-enable @typescript-eslint/naming-convention */ + +export const queryElastic = async (client: ElasticClient, body: estypes.SearchRequest): Promise> => { + const clientNotAvailableError = new InternalServerError('Elasticsearch client is not available'); + try { + if (!client || !(await client.ping())) { + throw clientNotAvailableError; + } + } catch (error) { + throw clientNotAvailableError; + } + + return client.search(body); +}; /* eslint-disable @typescript-eslint/naming-convention */ export const boundingBox = ( From 55da584aad9c002557136e93888bb5f26ea08d68 Mon Sep 17 00:00:00 2001 From: Niv Greenstein Date: Tue, 6 Aug 2024 10:58:41 +0300 Subject: [PATCH 51/67] feat(constants): added SERVICES.ELASTIC_CLIENTS --- src/common/constants.ts | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/src/common/constants.ts b/src/common/constants.ts index 16c3d931..f5cf88cd 100644 --- a/src/common/constants.ts +++ b/src/common/constants.ts @@ -13,13 +13,14 @@ export const SERVICES: Record = { TRACER: Symbol('Tracer'), METER: Symbol('Meter'), APPLICATION: Symbol('Application'), + ELASTIC_CLIENTS: Symbol('ElasticClients'), }; /* eslint-enable @typescript-eslint/naming-convention */ export const ON_SIGNAL = Symbol('onSignal'); export const HEALTHCHECK = Symbol('healthcheck'); -export const FIELDS = [ +export const CONTROL_FIELDS = [ 'type', 'geometry', 'properties.OBJECT_COMMAND_NAME', From 173619208fda35ce1d7fd018542818de3f2fbf26 Mon Sep 17 00:00:00 2001 From: Niv Greenstein Date: Tue, 6 Aug 2024 10:59:38 +0300 Subject: [PATCH 52/67] fix(dependencyRegistration): changed registerDependencies to be async and added postInjectionHook --- src/common/dependencyRegistration.ts | 24 ++++++++++++------------ 1 file changed, 12 insertions(+), 12 deletions(-) diff --git a/src/common/dependencyRegistration.ts b/src/common/dependencyRegistration.ts index a591ff93..e90304ab 100644 --- a/src/common/dependencyRegistration.ts +++ b/src/common/dependencyRegistration.ts @@ -1,4 +1,4 @@ -import { ClassProvider, container as defaultContainer, FactoryProvider, InjectionToken, ValueProvider } from 'tsyringe'; +import { ClassProvider, container as defaultContainer, FactoryProvider, InjectionToken, ValueProvider, RegistrationOptions } from 'tsyringe'; import { constructor, DependencyContainer } from 'tsyringe/dist/typings/types'; export type Providers = ValueProvider | FactoryProvider | ClassProvider | constructor; @@ -6,22 +6,22 @@ export type Providers = ValueProvider | FactoryProvider | ClassProvider export interface InjectionObject { token: InjectionToken; provider: Providers; + options?: RegistrationOptions; + postInjectionHook?: (container: DependencyContainer) => Promise; } -export const registerDependencies = ( +export const registerDependencies = async ( dependencies: InjectionObject[], override?: InjectionObject[], useChild = false -): DependencyContainer => { +): Promise => { const container = useChild ? defaultContainer.createChildContainer() : defaultContainer; - dependencies.forEach((injectionObj) => { - const inject = override?.find((overrideObj) => overrideObj.token === injectionObj.token) === undefined; - if (inject) { - container.register(injectionObj.token, injectionObj.provider as constructor); - } - }); - override?.forEach((injectionObj) => { - container.register(injectionObj.token, injectionObj.provider as constructor); - }); + + for (const dep of dependencies) { + const injectionObj = override?.find((overrideObj) => overrideObj.token === dep.token) ?? dep; + container.register(injectionObj.token, injectionObj.provider as constructor, injectionObj.options); + await injectionObj.postInjectionHook?.(container); + } + return container; }; From 30f17d058537c553bd672faf2594a91be22ba398 Mon Sep 17 00:00:00 2001 From: Niv Greenstein Date: Tue, 6 Aug 2024 11:00:22 +0300 Subject: [PATCH 53/67] delete(interfaces): removed old elastic interfaces --- src/common/interfaces.ts | 18 ------------------ 1 file changed, 18 deletions(-) diff --git a/src/common/interfaces.ts b/src/common/interfaces.ts index d642cd1e..cadc78ee 100644 --- a/src/common/interfaces.ts +++ b/src/common/interfaces.ts @@ -14,29 +14,11 @@ export interface OpenApiConfig { uiPath: string; } -export type ElasticDbClientsConfig = { - [key in 'control' | 'geotext']: ElasticDbConfig & { - properties: { - index: - | string - | { - [key: string]: string; - }; - defaultResponseLimit: number; - textTermLanguage: string; - }; - }; -}; - -export type ElasticDbConfig = ClientOptions; - export type PostgresDbConfig = { enableSslAuth: boolean; sslPaths: { ca: string; cert: string; key: string }; } & DataSourceOptions; -export type ElasticClients = Record; - export interface GeoContext { bbox?: number[]; radius?: number; From a8a142513cf1f8e9496a1aa34b02cb8ca3655c56 Mon Sep 17 00:00:00 2001 From: Niv Greenstein Date: Tue, 6 Aug 2024 11:07:21 +0300 Subject: [PATCH 54/67] fix: changed function location from common/utils to elastic/utils --- src/common/elastic/utils.ts | 8 ++++++-- src/common/utils.ts | 19 ++++--------------- 2 files changed, 10 insertions(+), 17 deletions(-) diff --git a/src/common/elastic/utils.ts b/src/common/elastic/utils.ts index 6b4653be..f9507217 100644 --- a/src/common/elastic/utils.ts +++ b/src/common/elastic/utils.ts @@ -1,10 +1,14 @@ import config from 'config'; import { estypes } from '@elastic/elasticsearch'; -import { WGS84Coordinate } from '../interfaces'; +import { IConfig, WGS84Coordinate } from '../interfaces'; import { InternalServerError } from '../errors'; import { elasticConfigPath, CONTROL_FIELDS } from '../constants'; import { ElasticDbClientsConfig } from './interfaces'; -import { ElasticClient } from './index'; +import { ElasticClient, ElasticClients } from './index'; + +export const getElasticClientQuerySize = (config: IConfig, key: keyof ElasticClients): number => { + return config.get(elasticConfigPath)[key].properties.defaultResponseLimit; +}; /* eslint-disable @typescript-eslint/naming-convention */ export const additionalControlSearchProperties = (size: number): { size: number; index: string; _source: string[] } => ({ diff --git a/src/common/utils.ts b/src/common/utils.ts index f50b5eae..11359d34 100644 --- a/src/common/utils.ts +++ b/src/common/utils.ts @@ -1,12 +1,13 @@ -import config from 'config'; import { estypes } from '@elastic/elasticsearch'; import proj4 from 'proj4'; import { Item } from '../item/models/item'; import { Tile } from '../tile/models/tile'; import { Route } from '../route/models/route'; -import { FIELDS, elasticConfigPath } from './constants'; +import { elasticConfigPath } from './constants'; import { utmProjection, wgs84Projection } from './projections'; -import { ElasticClients, ElasticDbClientsConfig, FeatureCollection, WGS84Coordinate } from './interfaces'; +import { FeatureCollection, IConfig, WGS84Coordinate } from './interfaces'; +import { ElasticClients } from './elastic'; +import { ElasticDbClientsConfig } from './elastic/interfaces'; export const formatResponse = (elasticResponse: estypes.SearchResponse): FeatureCollection => ({ type: 'FeatureCollection', @@ -26,14 +27,6 @@ export const formatResponse = (elasticResponse: e ], }); -/* eslint-disable @typescript-eslint/naming-convention */ -export const additionalSearchProperties = (size: number): { size: number; index: string; _source: string[] } => ({ - size, - index: config.get(elasticConfigPath).control.properties.index as string, - _source: FIELDS, -}); -/* eslint-enable @typescript-eslint/naming-convention */ - export const validateWGS84Coordinate = (coordinate: { lon: number; lat: number }): boolean => { // eslint-disable-next-line @typescript-eslint/no-magic-numbers const [min, max] = [0, 180]; @@ -96,7 +89,3 @@ export const validateTile = (tile: { tileName: string; subTileNumber: number[] } } return true; }; - -export const getElasticClientQuerySize = (key: keyof ElasticClients): number => { - return config.get(elasticConfigPath)[key].properties.defaultResponseLimit; -}; From 074dcac076b9ea09cf86f7913e0af9a70fe48cf3 Mon Sep 17 00:00:00 2001 From: Niv Greenstein Date: Tue, 6 Aug 2024 11:44:03 +0300 Subject: [PATCH 55/67] fix(elastic): changed elasticClient to be non nullable and removed the try-catch --- src/common/elastic/index.ts | 12 +++--------- 1 file changed, 3 insertions(+), 9 deletions(-) diff --git a/src/common/elastic/index.ts b/src/common/elastic/index.ts index 2c715e6a..51645492 100644 --- a/src/common/elastic/index.ts +++ b/src/common/elastic/index.ts @@ -19,8 +19,6 @@ const initElasticsearchClient = (clientOptions: ClientOptions): ElasticClient => return client; }; -export type ElasticClient = Client | null; - export const elasticClientFactory: FactoryFunction = (container: DependencyContainer): ElasticClients => { const config = container.resolve(SERVICES.CONFIG); const logger = container.resolve(SERVICES.LOGGER); @@ -30,16 +28,12 @@ export const elasticClientFactory: FactoryFunction = (container: const elasticClients = {} as ElasticClients; for (const [key, value] of Object.entries(elasticClientsConfig)) { - try { - const client = initElasticsearchClient(value as ElasticDbConfig); - elasticClients[key as keyof ElasticDbClientsConfig] = client; - } catch (err) { - logger.error('Failed to connect to Elasticsearch', err); - elasticClients[key as keyof ElasticDbClientsConfig] = null; - } + elasticClients[key as keyof ElasticDbClientsConfig] = initElasticsearchClient(value as ElasticDbConfig); + logger.info(`Elasticsearch client for ${key} is initialized`); } return elasticClients; }; +export type ElasticClient = Client; export type ElasticClients = Record; From c64a123594e43cc74f4ed91c8c40491711d58b6b Mon Sep 17 00:00:00 2001 From: Niv Greenstein Date: Tue, 6 Aug 2024 11:44:51 +0300 Subject: [PATCH 56/67] fix(elastic/utils): remove !client condition as it is not relevant anymore --- src/common/elastic/utils.ts | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/common/elastic/utils.ts b/src/common/elastic/utils.ts index f9507217..c0413f46 100644 --- a/src/common/elastic/utils.ts +++ b/src/common/elastic/utils.ts @@ -21,7 +21,7 @@ export const additionalControlSearchProperties = (size: number): { size: number; export const queryElastic = async (client: ElasticClient, body: estypes.SearchRequest): Promise> => { const clientNotAvailableError = new InternalServerError('Elasticsearch client is not available'); try { - if (!client || !(await client.ping())) { + if (!(await client.ping())) { throw clientNotAvailableError; } } catch (error) { From bd268c8efb1c518105c108d686a2b705f1088bd4 Mon Sep 17 00:00:00 2001 From: Niv Greenstein Date: Tue, 6 Aug 2024 11:45:55 +0300 Subject: [PATCH 57/67] fix(geotextSearchRepository): changed queryElastic to common function and injecting logger --- .../DAL/geotextSearchRepository.ts | 35 +++++++++---------- 1 file changed, 17 insertions(+), 18 deletions(-) diff --git a/src/geotextSearch/DAL/geotextSearchRepository.ts b/src/geotextSearch/DAL/geotextSearchRepository.ts index cc1f236e..540b0f1a 100644 --- a/src/geotextSearch/DAL/geotextSearchRepository.ts +++ b/src/geotextSearch/DAL/geotextSearchRepository.ts @@ -1,25 +1,30 @@ -import { Client, estypes } from '@elastic/elasticsearch'; +import { Logger } from '@map-colonies/js-logger'; +import { estypes } from '@elastic/elasticsearch'; import { FactoryFunction } from 'tsyringe'; -import { elasticClientSymbol } from '../../common/elastic'; +import { ElasticClient } from '../../common/elastic'; import { cleanQuery, fetchNLPService } from '../utils'; import { TextSearchParams, TokenResponse } from '../interfaces'; import { PlaceTypeSearchHit, HierarchySearchHit, TextSearchHit } from '../models/elasticsearchHits'; import { BadRequestError } from '../../common/errors'; -import { ElasticClients, IApplication } from '../../common/interfaces'; +import { IApplication } from '../../common/interfaces'; +import { ElasticClients } from '../../common/elastic'; +import { queryElastic } from '../../common/elastic/utils'; +import { SERVICES } from '../../common/constants'; import { hierarchyQuery, placetypeQuery, geotextQuery } from './queries'; /* eslint-enable @typescript-eslint/naming-convention */ // eslint-disable-next-line @typescript-eslint/explicit-function-return-type -const createGeotextRepository = (client: Client) => { +const createGeotextRepository = (client: ElasticClient, logger: Logger) => { return { async extractName(endpoint: string, query: string): Promise { const tokensRaw = cleanQuery(query); - const response = await fetchNLPService(endpoint, { tokens: tokensRaw }); + const response = await fetchNLPService>(endpoint, { tokens: tokensRaw }); const { tokens, prediction } = response[0]; if (!tokens || !prediction) { + logger.error('No tokens or prediction'); throw new BadRequestError('No tokens or prediction'); } @@ -31,7 +36,7 @@ const createGeotextRepository = (client: Client) => { async generatePlacetype(index: string, query: string): Promise<{ placeTypes: string[]; subPlaceTypes: string[] }> { const { hits: { hits }, - } = await queryElastic(client, index, placetypeQuery(query)); + } = await queryElastic(client, { index, ...placetypeQuery(query) }); if (hits.length == 2 && hits[0]._score! - hits[1]._score! > 0.5) { hits.pop(); @@ -48,7 +53,7 @@ const createGeotextRepository = (client: Client) => { async extractHierarchy(index: string, query: string, hierarchyBoost: number): Promise { const { hits: { hits }, - } = await queryElastic(client, index, hierarchyQuery(query)); + } = await queryElastic(client, { index, ...hierarchyQuery(query) }); const filteredHits = hits.length > 3 ? hits.filter((hit) => hit._score! >= hits[2]._score!) : hits; @@ -67,25 +72,19 @@ const createGeotextRepository = (client: Client) => { textLanguage: string, elasticQueryBoosts: IApplication['elasticQueryBoosts'] ): Promise> { - const response = await queryElastic(client, index, geotextQuery(params, textLanguage, elasticQueryBoosts)); + const response = await queryElastic(client, { index, ...geotextQuery(params, textLanguage, elasticQueryBoosts) }); return response; }, }; }; -const queryElastic = async (client: Client, index: string, body: estypes.SearchRequest): Promise> => { - const response = await client.search({ - index, - ...body, - }); - - return response; -}; - export type GeotextRepository = ReturnType; export const geotextRepositoryFactory: FactoryFunction = (depContainer) => { - return createGeotextRepository(depContainer.resolve(elasticClientSymbol).geotext); + return createGeotextRepository( + depContainer.resolve(SERVICES.ELASTIC_CLIENTS).geotext, + depContainer.resolve(SERVICES.LOGGER) + ); }; export const GEOTEXT_REPOSITORY_SYMBOL = Symbol('GEOTEXT_REPOSITORY_SYMBOL'); From a465bf1d4f2668905dda699fff7446882f88226a Mon Sep 17 00:00:00 2001 From: Niv Greenstein Date: Tue, 6 Aug 2024 11:46:36 +0300 Subject: [PATCH 58/67] fix: changed import location --- src/geotextSearch/models/geotextSearchManager.ts | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/geotextSearch/models/geotextSearchManager.ts b/src/geotextSearch/models/geotextSearchManager.ts index eff42ad6..1626406e 100644 --- a/src/geotextSearch/models/geotextSearchManager.ts +++ b/src/geotextSearch/models/geotextSearchManager.ts @@ -6,7 +6,7 @@ import { GEOTEXT_REPOSITORY_SYMBOL, GeotextRepository } from '../DAL/geotextSear import { GetGeotextSearchParams, QueryResult, TextSearchParams } from '../interfaces'; import { convertResult, parseGeo } from '../utils'; import { IApplication } from '../../common/interfaces'; -import { ElasticDbClientsConfig } from '../../common/interfaces'; +import { ElasticDbClientsConfig } from '../../common/elastic/interfaces'; @injectable() export class GeotextSearchManager { From 9915e6f72fa1fa8eb81a6398c1129134dc5b15d6 Mon Sep 17 00:00:00 2001 From: Niv Greenstein Date: Tue, 6 Aug 2024 11:47:22 +0300 Subject: [PATCH 59/67] fix(geotextSearch/utils): changed from instanceof Error to isAxiosError --- src/geotextSearch/utils.ts | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/src/geotextSearch/utils.ts b/src/geotextSearch/utils.ts index 2c497ef6..7324f961 100644 --- a/src/geotextSearch/utils.ts +++ b/src/geotextSearch/utils.ts @@ -19,8 +19,8 @@ export const fetchNLPService = async (endpoint: string, requestData: object): try { res = await axios.post(endpoint, requestData); } catch (err: unknown) { - if (err instanceof Error) { - throw new InternalServerError(err.message); + if (axios.isAxiosError(err)) { + throw new InternalServerError(`NLP analyser is not available - ${err.message}`); } throw new InternalServerError('fetchNLPService: Unknown error' + JSON.stringify(err)); } From 064fd06f9adc86d43409b63a7b7425865e65bbf0 Mon Sep 17 00:00:00 2001 From: Niv Greenstein Date: Tue, 6 Aug 2024 11:48:22 +0300 Subject: [PATCH 60/67] fix(itemRepository): changed queryElastic to common function and injecting logger and config --- src/item/DAL/itemRepository.ts | 25 +++++++++++++++---------- 1 file changed, 15 insertions(+), 10 deletions(-) diff --git a/src/item/DAL/itemRepository.ts b/src/item/DAL/itemRepository.ts index 4a844b9d..6120e57f 100644 --- a/src/item/DAL/itemRepository.ts +++ b/src/item/DAL/itemRepository.ts @@ -1,22 +1,23 @@ -import { Client, estypes } from '@elastic/elasticsearch'; +import { Logger } from '@map-colonies/js-logger'; +import { estypes } from '@elastic/elasticsearch'; import { FactoryFunction } from 'tsyringe'; -import { elasticClientSymbol } from '../../common/elastic'; +import { IConfig } from '../../common/interfaces'; +import { ElasticClients } from '../../common/elastic'; +import { additionalControlSearchProperties, queryElastic } from '../../common/elastic/utils'; +import { ElasticClient } from '../../common/elastic'; import { Item } from '../models/item'; -import { additionalSearchProperties } from '../../common/utils'; -import { ElasticClients } from '../../common/interfaces'; +import { SERVICES } from '../../common/constants'; import { ItemQueryParams, queryForItems } from './queries'; /* eslint-enable @typescript-eslint/naming-convention */ // eslint-disable-next-line @typescript-eslint/explicit-function-return-type -const createItemRepository = (client: Client) => { +const createItemRepository = (client: ElasticClient, config: IConfig, logger: Logger) => { return { // eslint-disable-next-line @typescript-eslint/explicit-module-boundary-types async getItems(itemQueryParams: ItemQueryParams, size: number): Promise> { - const response = await client.search({ - ...additionalSearchProperties(size), - body: queryForItems(itemQueryParams), - }); + logger.info('Querying items from elastic'); + const response = await queryElastic(client, { ...additionalControlSearchProperties(size), ...queryForItems(itemQueryParams) }); return response; }, @@ -26,7 +27,11 @@ const createItemRepository = (client: Client) => { export type ItemRepository = ReturnType; export const itemRepositoryFactory: FactoryFunction = (depContainer) => { - return createItemRepository(depContainer.resolve(elasticClientSymbol).control); + return createItemRepository( + depContainer.resolve(SERVICES.ELASTIC_CLIENTS).control, + depContainer.resolve(SERVICES.CONFIG), + depContainer.resolve(SERVICES.LOGGER) + ); }; export const ITEM_REPOSITORY_SYMBOL = Symbol('ITEM_REPOSITORY_SYMBOL'); From 7c43b92cb2caf20a07663f712fb540bd87869c56 Mon Sep 17 00:00:00 2001 From: Niv Greenstein Date: Tue, 6 Aug 2024 13:21:40 +0300 Subject: [PATCH 61/67] fix: changed import location --- src/item/models/itemManager.ts | 10 ++++------ src/route/models/routeManager.ts | 7 ++++--- src/tile/models/tileManager.ts | 5 +++-- 3 files changed, 11 insertions(+), 11 deletions(-) diff --git a/src/item/models/itemManager.ts b/src/item/models/itemManager.ts index 035dc864..8c17dbac 100644 --- a/src/item/models/itemManager.ts +++ b/src/item/models/itemManager.ts @@ -2,11 +2,12 @@ import { IConfig } from 'config'; import { Logger } from '@map-colonies/js-logger'; import { inject, injectable } from 'tsyringe'; import { estypes } from '@elastic/elasticsearch'; -import { SERVICES, elasticConfigPath } from '../../common/constants'; +import { SERVICES } from '../../common/constants'; import { ITEM_REPOSITORY_SYMBOL, ItemRepository } from '../DAL/itemRepository'; import { ItemQueryParams } from '../DAL/queries'; import { formatResponse } from '../../common/utils'; -import { ElasticDbClientsConfig, FeatureCollection } from '../../common/interfaces'; +import { FeatureCollection } from '../../common/interfaces'; +import { getElasticClientQuerySize } from '../../common/elastic/utils'; import { Item } from './item'; @injectable() @@ -19,10 +20,7 @@ export class ItemManager { public async getItems(itemQueryParams: ItemQueryParams, reduceFuzzyMatch = false, size?: number): Promise> { let elasticResponse: estypes.SearchResponse | undefined = undefined; - elasticResponse = await this.itemRepository.getItems( - itemQueryParams, - size ?? this.config.get(elasticConfigPath).control.properties.defaultResponseLimit - ); + elasticResponse = await this.itemRepository.getItems(itemQueryParams, size ?? getElasticClientQuerySize(this.config, 'control')); const formattedResponse = formatResponse(elasticResponse); diff --git a/src/route/models/routeManager.ts b/src/route/models/routeManager.ts index 42a680ed..619b9a69 100644 --- a/src/route/models/routeManager.ts +++ b/src/route/models/routeManager.ts @@ -5,8 +5,9 @@ import { estypes } from '@elastic/elasticsearch'; import { SERVICES } from '../../common/constants'; import { ROUTE_REPOSITORY_SYMBOL, RouteRepository } from '../DAL/routeRepository'; import { RouteQueryParams } from '../DAL/queries'; -import { formatResponse, getElasticClientQuerySize } from '../../common/utils'; +import { formatResponse } from '../../common/utils'; import { FeatureCollection } from '../../common/interfaces'; +import { getElasticClientQuerySize } from '../../common/elastic/utils'; import { Route } from './route'; @injectable() @@ -22,10 +23,10 @@ export class RouteManager { if (routeQueryParams.controlPoint ?? 0) { elasticResponse = await this.routeRepository.getControlPointInRoute( routeQueryParams as RouteQueryParams & Required>, - size ?? getElasticClientQuerySize('control') + size ?? getElasticClientQuerySize(this.config, 'control') ); } else { - elasticResponse = await this.routeRepository.getRoutes(routeQueryParams, size ?? getElasticClientQuerySize('control')); + elasticResponse = await this.routeRepository.getRoutes(routeQueryParams, size ?? getElasticClientQuerySize(this.config, 'control')); } const formattedResponse = formatResponse(elasticResponse); diff --git a/src/tile/models/tileManager.ts b/src/tile/models/tileManager.ts index fb207cbd..121f4aec 100644 --- a/src/tile/models/tileManager.ts +++ b/src/tile/models/tileManager.ts @@ -4,9 +4,10 @@ import { inject, injectable } from 'tsyringe'; import { estypes } from '@elastic/elasticsearch'; import { SERVICES } from '../../common/constants'; import { TILE_REPOSITORY_SYMBOL, TileRepository } from '../DAL/tileRepository'; -import { formatResponse, getElasticClientQuerySize } from '../../common/utils'; +import { formatResponse } from '../../common/utils'; import { TileQueryParams } from '../DAL/queries'; import { FeatureCollection } from '../../common/interfaces'; +import { getElasticClientQuerySize } from '../../common/elastic/utils'; import { Tile } from './tile'; @injectable() @@ -19,7 +20,7 @@ export class TileManager { public async getTiles(tileQueryParams: TileQueryParams, reduceFuzzyMatch = false, size?: number): Promise> { let elasticResponse: estypes.SearchResponse | undefined = undefined; - const numberOfResults = size ?? getElasticClientQuerySize('control'); + const numberOfResults = size ?? getElasticClientQuerySize(this.config, 'control'); if (tileQueryParams.subTile ?? 0) { elasticResponse = await this.tileRepository.getSubTiles(tileQueryParams as Required, numberOfResults); } else { From 9fa160794b9636790156daeae715571e8ecfd3d6 Mon Sep 17 00:00:00 2001 From: Niv Greenstein Date: Tue, 6 Aug 2024 13:22:09 +0300 Subject: [PATCH 62/67] fix: additionalControlSearchProperties requests config --- src/common/elastic/utils.ts | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/src/common/elastic/utils.ts b/src/common/elastic/utils.ts index c0413f46..5e7e6278 100644 --- a/src/common/elastic/utils.ts +++ b/src/common/elastic/utils.ts @@ -1,4 +1,3 @@ -import config from 'config'; import { estypes } from '@elastic/elasticsearch'; import { IConfig, WGS84Coordinate } from '../interfaces'; import { InternalServerError } from '../errors'; @@ -11,7 +10,7 @@ export const getElasticClientQuerySize = (config: IConfig, key: keyof ElasticCli }; /* eslint-disable @typescript-eslint/naming-convention */ -export const additionalControlSearchProperties = (size: number): { size: number; index: string; _source: string[] } => ({ +export const additionalControlSearchProperties = (config: IConfig, size: number): { size: number; index: string; _source: string[] } => ({ size, index: config.get(elasticConfigPath).control.properties.index as string, _source: CONTROL_FIELDS, From 84f32074500e71a26f27bbf2ea6798eb4eaf24b7 Mon Sep 17 00:00:00 2001 From: Niv Greenstein Date: Tue, 6 Aug 2024 13:23:32 +0300 Subject: [PATCH 63/67] fix(itemRepository): added config to additionalControlSearchProperties --- src/item/DAL/itemRepository.ts | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/item/DAL/itemRepository.ts b/src/item/DAL/itemRepository.ts index 6120e57f..b7df1f41 100644 --- a/src/item/DAL/itemRepository.ts +++ b/src/item/DAL/itemRepository.ts @@ -17,7 +17,7 @@ const createItemRepository = (client: ElasticClient, config: IConfig, logger: Lo // eslint-disable-next-line @typescript-eslint/explicit-module-boundary-types async getItems(itemQueryParams: ItemQueryParams, size: number): Promise> { logger.info('Querying items from elastic'); - const response = await queryElastic(client, { ...additionalControlSearchProperties(size), ...queryForItems(itemQueryParams) }); + const response = await queryElastic(client, { ...additionalControlSearchProperties(config, size), ...queryForItems(itemQueryParams) }); return response; }, From 98331de74ff81cce0bc49ee8a07cece402a45cb4 Mon Sep 17 00:00:00 2001 From: Niv Greenstein Date: Tue, 6 Aug 2024 13:24:04 +0300 Subject: [PATCH 64/67] feat: added logger injection to latLonRepository --- src/latLon/DAL/latLonRepository.ts | 6 ++++-- 1 file changed, 4 insertions(+), 2 deletions(-) diff --git a/src/latLon/DAL/latLonRepository.ts b/src/latLon/DAL/latLonRepository.ts index 28037762..6dc7dcab 100644 --- a/src/latLon/DAL/latLonRepository.ts +++ b/src/latLon/DAL/latLonRepository.ts @@ -1,10 +1,12 @@ /* eslint-disable @typescript-eslint/naming-convention */ +import { Logger } from '@map-colonies/js-logger'; import { DataSource } from 'typeorm'; import { FactoryFunction } from 'tsyringe'; +import { SERVICES } from '../../common/constants'; import { LatLon as LatLonDb } from './latLon'; // eslint-disable-next-line @typescript-eslint/explicit-function-return-type -const createLatLonRepository = (dataSource: DataSource) => { +const createLatLonRepository = (dataSource: DataSource, logger: Logger) => { return dataSource.getRepository(LatLonDb).extend({ async getAll(): Promise { const result = await this.find(); @@ -16,7 +18,7 @@ const createLatLonRepository = (dataSource: DataSource) => { export type LatLonRepository = ReturnType; export const latLonRepositoryFactory: FactoryFunction = (depContainer) => { - return createLatLonRepository(depContainer.resolve(DataSource)); + return createLatLonRepository(depContainer.resolve(DataSource), depContainer.resolve(SERVICES.LOGGER)); }; export const LATLON_CUSTOM_REPOSITORY_SYMBOL = Symbol('LATLON_CUSTOM_REPOSITORY_SYMBOL'); From e6dacc6d086e377c21ada870ea48932f6f4f6557 Mon Sep 17 00:00:00 2001 From: Niv Greenstein Date: Tue, 6 Aug 2024 13:24:47 +0300 Subject: [PATCH 65/67] fix(routeRepository): added config to additionalControlSearchProperties and added config and logger injection --- src/route/DAL/routeRepository.ts | 31 ++++++++++++++++++------------- 1 file changed, 18 insertions(+), 13 deletions(-) diff --git a/src/route/DAL/routeRepository.ts b/src/route/DAL/routeRepository.ts index fc5f3c10..40c013a4 100644 --- a/src/route/DAL/routeRepository.ts +++ b/src/route/DAL/routeRepository.ts @@ -1,19 +1,20 @@ -import { Client, estypes } from '@elastic/elasticsearch'; +import { Logger } from '@map-colonies/js-logger'; +import { estypes } from '@elastic/elasticsearch'; import { FactoryFunction } from 'tsyringe'; -import { elasticClientSymbol } from '../../common/elastic'; +import { ElasticClient } from '../../common/elastic'; +import { elasticConfigPath, SERVICES } from '../../common/constants'; import { Route } from '../models/route'; -import { additionalSearchProperties } from '../../common/utils'; -import { ElasticClients } from '../../common/interfaces'; +import { IConfig } from '../../common/interfaces'; +import { ElasticClients } from '../../common/elastic'; +import { ElasticDbClientsConfig } from '../../common/elastic/interfaces'; +import { additionalControlSearchProperties, queryElastic } from '../../common/elastic/utils'; import { RouteQueryParams, queryForControlPointInRoute, queryForRoute } from './queries'; // eslint-disable-next-line @typescript-eslint/explicit-function-return-type -const createRouteRepository = (client: Client) => { +const createRouteRepository = (client: ElasticClient, config: IConfig, logger: Logger) => { return { async getRoutes(routeQueryParams: RouteQueryParams, size: number): Promise> { - const response = await client.search({ - ...additionalSearchProperties(size), - body: queryForRoute(routeQueryParams), - }); + const response = await queryElastic(client, { ...additionalControlSearchProperties(config, size), ...queryForRoute(routeQueryParams) }); return response; }, @@ -22,9 +23,9 @@ const createRouteRepository = (client: Client) => { routeQueryParams: RouteQueryParams & Required>, size: number ): Promise> { - const response = await client.search({ - ...additionalSearchProperties(size), - body: queryForControlPointInRoute(routeQueryParams), + const response = await queryElastic(client, { + ...additionalControlSearchProperties(config, size), + ...queryForControlPointInRoute(routeQueryParams), }); return response; @@ -35,7 +36,11 @@ const createRouteRepository = (client: Client) => { export type RouteRepository = ReturnType; export const routeRepositoryFactory: FactoryFunction = (depContainer) => { - return createRouteRepository(depContainer.resolve(elasticClientSymbol).control); + return createRouteRepository( + depContainer.resolve(SERVICES.ELASTIC_CLIENTS).control, + depContainer.resolve(SERVICES.CONFIG), + depContainer.resolve(SERVICES.LOGGER) + ); }; export const ROUTE_REPOSITORY_SYMBOL = Symbol('ROUTE_REPOSITORY_SYMBOL'); From 5626e6b275f112282d08fbe17d92a230a03797e6 Mon Sep 17 00:00:00 2001 From: Niv Greenstein Date: Tue, 6 Aug 2024 13:25:14 +0300 Subject: [PATCH 66/67] fix(tileRepository): added config to additionalControlSearchProperties and added config and logger injection --- src/tile/DAL/tileRepository.ts | 28 ++++++++++++++-------------- 1 file changed, 14 insertions(+), 14 deletions(-) diff --git a/src/tile/DAL/tileRepository.ts b/src/tile/DAL/tileRepository.ts index 84440b43..1b1de8b3 100644 --- a/src/tile/DAL/tileRepository.ts +++ b/src/tile/DAL/tileRepository.ts @@ -1,29 +1,25 @@ -import { Client, estypes } from '@elastic/elasticsearch'; +import { Logger } from '@map-colonies/js-logger'; +import { estypes } from '@elastic/elasticsearch'; import { FactoryFunction } from 'tsyringe'; -import { elasticClientSymbol } from '../../common/elastic'; +import { ElasticClient, ElasticClients } from '../../common/elastic'; import { Tile } from '../models/tile'; -import { additionalSearchProperties } from '../../common/utils'; -import { ElasticClients } from '../../common/interfaces'; +import { additionalControlSearchProperties, queryElastic } from '../../common/elastic/utils'; +import { IConfig } from '../../common/interfaces'; +import { SERVICES } from '../../common/constants'; import { queryForTiles, queryForSubTiles, TileQueryParams } from './queries'; // eslint-disable-next-line @typescript-eslint/explicit-function-return-type -const createTileRepository = (client: Client) => { +const createTileRepository = (client: ElasticClient, config: IConfig, logger: Logger) => { return { // eslint-disable-next-line @typescript-eslint/explicit-module-boundary-types async getTiles(tileQueryParams: TileQueryParams, size: number): Promise> { - const response = await client.search({ - ...additionalSearchProperties(size), - body: queryForTiles(tileQueryParams), - }); + const response = await queryElastic(client, { ...additionalControlSearchProperties(config, size), ...queryForTiles(tileQueryParams) }); return response; }, async getSubTiles(tileQueryParams: Required, size: number): Promise> { - const response = await client.search({ - ...additionalSearchProperties(size), - body: queryForSubTiles(tileQueryParams), - }); + const response = await queryElastic(client, { ...additionalControlSearchProperties(config, size), ...queryForSubTiles(tileQueryParams) }); return response; }, @@ -33,7 +29,11 @@ const createTileRepository = (client: Client) => { export type TileRepository = ReturnType; export const tileRepositoryFactory: FactoryFunction = (depContainer) => { - return createTileRepository(depContainer.resolve(elasticClientSymbol).control); + return createTileRepository( + depContainer.resolve(SERVICES.ELASTIC_CLIENTS).control, + depContainer.resolve(SERVICES.CONFIG), + depContainer.resolve(SERVICES.LOGGER) + ); }; export const TILE_REPOSITORY_SYMBOL = Symbol('TILE_REPOSITORY_SYMBOL'); From b1a6d0c51680b1035f932c2c10a46eda9bae1087 Mon Sep 17 00:00:00 2001 From: Niv Greenstein Date: Tue, 6 Aug 2024 13:47:05 +0300 Subject: [PATCH 67/67] fix: boundary, source, regions changed to be filter and fixed coding style --- src/geotextSearch/DAL/queries.ts | 31 +++++++++++++++++-------------- 1 file changed, 17 insertions(+), 14 deletions(-) diff --git a/src/geotextSearch/DAL/queries.ts b/src/geotextSearch/DAL/queries.ts index 872d5b3f..f700acc5 100644 --- a/src/geotextSearch/DAL/queries.ts +++ b/src/geotextSearch/DAL/queries.ts @@ -38,17 +38,20 @@ export const geotextQuery = ( }; if (!name && subPlaceTypes?.length) { - (esQuery.query?.function_score?.query?.bool?.must as QueryDslQueryContainer[]).push({ - terms: { - [SUB_PLACETYPE_FIELD]: subPlaceTypes, - }, - }); - - (esQuery.query?.function_score?.query?.bool?.must as QueryDslQueryContainer[]).push({ - term: { - text_language: textLanguage, - }, - }); + (esQuery.query?.function_score?.query?.bool?.must as QueryDslQueryContainer[]).push( + ...[ + { + terms: { + [SUB_PLACETYPE_FIELD]: subPlaceTypes, + }, + }, + { + term: { + text_language: textLanguage, + }, + }, + ] + ); } else { (esQuery.query?.function_score?.query?.bool?.must as QueryDslQueryContainer[]).push({ match: { @@ -61,7 +64,7 @@ export const geotextQuery = ( } boundary && - (esQuery.query?.function_score?.query?.bool?.must as QueryDslQueryContainer[]).push({ + (esQuery.query?.function_score?.query?.bool?.filter as QueryDslQueryContainer[]).push({ geo_shape: { [GEOJSON_FIELD]: { shape: boundary, @@ -70,14 +73,14 @@ export const geotextQuery = ( }); sources?.length && - (esQuery.query?.function_score?.query?.bool?.must as QueryDslQueryContainer[]).push({ + (esQuery.query?.function_score?.query?.bool?.filter as QueryDslQueryContainer[]).push({ terms: { [SOURCE_FIELD]: sources, }, }); regions?.length && - (esQuery.query?.function_score?.query?.bool?.must as QueryDslQueryContainer[]).push({ + (esQuery.query?.function_score?.query?.bool?.filter as QueryDslQueryContainer[]).push({ terms: { [REGION_FIELD]: regions, },