From 9a9c31360e1e4791bc37569020f37e503432fc38 Mon Sep 17 00:00:00 2001 From: Niv Greenstein Date: Wed, 7 Aug 2024 11:45:48 +0300 Subject: [PATCH 001/262] feat: new openapi based on new architecture --- openapi3.yaml | 688 +++++++++++++++++++++++++++++++------------------- 1 file changed, 434 insertions(+), 254 deletions(-) diff --git a/openapi3.yaml b/openapi3.yaml index 8a19c7db..728b3972 100644 --- a/openapi3.yaml +++ b/openapi3.yaml @@ -1,22 +1,23 @@ openapi: "3.0.1" info: version: "1.0.0" - title: "New Geocoding" - description: "Geocoding api" + title: "Geocoding" + description: |- + MapColonies Vector Geocoding api provides custom geodata search engine uniting multiple sources of data, query tiles, routes, items; Convertion functions - tile to/from WGS84 lat lng, WGS84 to/from US Army MGRS, WGS84 to/from UTM. license: name: MIT url: https://opensource.org/licenses/MIT tags: - - name: Query Based Search - - name: Exact Searches + - name: Location Name Based Search + - name: Control + - name: Convertions paths: - /v1/query: + /search/query: get: - operationId: searchByQuery - tags: - - Query Based Search - summary: "Search anything by query" - description: "This is for general queries. If known regex is found, the server will return results as if you searched for exact search. Else, the server will search using geotext engine to find a match." + operationId: GetSmartQuery + summary: "Search anything" + description: |- + This is for general queries. the services will make a sophisticated guess. parameters: - name: query in: query @@ -24,74 +25,67 @@ paths: schema: type: string minLength: 3 - maxLength: 1000 - description: Text to search - title: Query + maxLength: 100 + title: "Query" description: Text to search allowReserved: true - - name: limit + - $ref: "#/components/parameters/geo_context" + - $ref: "#/components/parameters/geo_context_mode" + - $ref: "#/components/parameters/limit" + - $ref: "#/components/parameters/disable_fuzziness" + responses: + 200: + description: "OK
Will return valid GeoJSON FeatureCollection" + headers: + x-request-id: + schema: + type: string + minLength: 1 + maxLength: 36 + content: + application/json: + schema: + $ref: '#/components/schemas/genericGeocodingResponse' + 400: + "$ref": "#/components/responses/BadRequest" + 401: + "$ref": "#/components/responses/Unauthorized" + 403: + "$ref": "#/components/responses/Forbidden" + 500: + "$ref": "#/components/responses/InternalError" + security: + - x-api-key: [] + x-user-id: [] + + /search/location/query: + get: + operationId: locationGetQuery + summary: "Search engine" + description: "Search for geo-data. It serves data from multiple diverse mapcolonies vector sources and partners." + tags: + - Location Name Based Search + parameters: + - name: query in: query - required: false + required: true schema: - type: integer - maximum: 1000 - minimum: 0 - description: Maximum results the service should return - default: 10 - title: Limit - description: Maximum results the service should return + type: string + minLength: 3 + maxLength: 100 + title: "Query" + description: Text to search + allowReserved: true - name: source in: query required: false schema: type: array items: - $ref: "#/components/schemas/Source" - description: Sources to include (if not specified, all sources will be queried) + type: string title: Source - description: Sources to include (if not specified, all sources will be queried) - - name: viewbox - in: query - required: false - allowReserved: true - schema: - type: string - description: |- - User's viewbox, used for biasing the results, or preferring results closer to the user - - Results close to this geometry will be preferred - - When providing a non-zero-area entity (e.g. polygon or bbox), results inside the entity will be most preferred - - Possible formats include a valid geoJSON, a valid WKT, an "x,y" point, or a "xmin,ymin,xmax,ymax" bounding box - title: Viewbox - description: |- - User's viewbox, used for biasing the results, or preferring results closer to the user - - Results close to this geometry will be preferred - - When providing a non-zero-area entity (e.g. polygon or bbox), results inside the entity will be most preferred - - Possible formats include a valid geoJSON, a valid WKT, an "x,y" point, or a "xmin,ymin,xmax,ymax" bounding box - - name: boundary - in: query - required: false - allowReserved: true - schema: - type: string - description: |- - A filter on the results. - - Results not touching this geometry will not be returned - - Possible formats include a valid geoJSON, a valid WKT, an "x,y" point, or a "xmin,ymin,xmax,ymax" bounding box - title: Boundary - description: |- - A filter on the results. - - Results not touching this geometry will not be returned - - Possible formats include a valid geoJSON, a valid WKT, an "x,y" point, or a "xmin,ymin,xmax,ymax" bounding box + description: | + Sources to include (if not specified, all sources will be queried) - name: region in: query required: false @@ -102,17 +96,22 @@ paths: description: Regions to include (if not specified, all Regions will be queried) title: Region description: Regions to include (if not specified, all Regions will be queried) + - $ref: "#/components/parameters/geo_context" + - $ref: "#/components/parameters/geo_context_mode" + - $ref: "#/components/parameters/limit" + - $ref: "#/components/parameters/disable_fuzziness" responses: 200: - description: "OK (can be one of the following: tilesSchema, routesSchema, itemsSchema, subTileSchema)

WIP: Find out returned schema from geotext" + description: "OK" + x-request-id: + schema: + type: string + minLength: 1 + maxLength: 36 content: application/json: schema: - anyOf: - - $ref: "#/components/schemas/tilesSchema" - - $ref: "#/components/schemas/routesSchema" - - $ref: "#/components/schemas/itemsSchema" - - $ref: "#/components/schemas/subTileSchema" + "$ref": '#/components/schemas/genericGeocodingResponse' 400: "$ref": "#/components/responses/BadRequest" 401: @@ -122,17 +121,45 @@ paths: 500: "$ref": "#/components/responses/InternalError" security: - - X-API-Key: [] - X-User-ID: [] - /v1/query/regions: + - x-api-key: [] + x-user-id: [] + + /search/location/regions: get: - operationId: getRegions + operationId: locationGetRegions tags: - - Query Based Search - summary: "Get regions" + - Location Name Based Search + summary: "Get all available regions to filter on using location query" responses: 200: description: "All regions" + content: + application/json: + schema: + type: array + items: + type: string # return geojson with name property + 400: + "$ref": "#/components/responses/BadRequest" + 401: + "$ref": "#/components/responses/Unauthorized" + 403: + "$ref": "#/components/responses/Forbidden" + 500: + "$ref": "#/components/responses/InternalError" + security: + - x-api-key: [] + x-user-id: [] + + /search/location/sources: + get: + operationId: locationGetSources + tags: + - Location Name Based Search + summary: "Get all available sources to filter on using location query" + responses: + 200: + description: "All sources" content: application/json: schema: @@ -148,29 +175,40 @@ paths: 500: "$ref": "#/components/responses/InternalError" security: - - X-API-Key: [] - X-User-ID: [] - /v1/search/tiles: + - x-api-key: [] + x-user-id: [] + + /search/control/tiles: get: - operationId: getTilesByQueryParams + operationId: controlGetTilesByQueryParams tags: - - Exact Searches + - Control summary: "Search tiles and sub tiles" - description: "Tiles are consist of 3 characters.

You can query and get results from 2 characters.

If you define a tile (full name of it) you can query the sub tile associated with it." + description: |- + Tiles are consisted of 3 characters.
+ You can query and get results from 2 characters.

If you define a tile (full name of it) you can query the sub tile associated with it. parameters: - name: "tile" in: "query" description: "Tile name" schema: type: "string" + minLength: 2 + maxLength: 3 required: true - name: "sub_tile" in: "query" description: "Sub tile number" schema: type: "number" - - $ref: "#/components/parameters/reduce_fuzzy_match" - - $ref: "#/components/parameters/size" + - name: MGRS + in: query + schema: + type: string + - $ref: "#/components/parameters/geo_context" + - $ref: "#/components/parameters/geo_context_mode" + - $ref: "#/components/parameters/limit" + - $ref: "#/components/parameters/disable_fuzziness" responses: 200: description: "OK" @@ -187,13 +225,13 @@ paths: 500: "$ref": "#/components/responses/InternalError" security: - - X-API-Key: [] - X-User-ID: [] - /v1/search/items: + - x-api-key: [] + x-user-id: [] + /search/control/items: get: - operationId: getItemsByQueryParams + operationId: controlGetItemsByQueryParams tags: - - Exact Searches + - Control summary: "Search control items" description: "" parameters: @@ -217,8 +255,9 @@ paths: schema: type: "number" - $ref: "#/components/parameters/geo_context" - - $ref: "#/components/parameters/reduce_fuzzy_match" - - $ref: "#/components/parameters/size" + - $ref: "#/components/parameters/geo_context_mode" + - $ref: "#/components/parameters/limit" + - $ref: "#/components/parameters/disable_fuzziness" responses: 200: description: "OK" @@ -235,13 +274,13 @@ paths: 500: "$ref": "#/components/responses/InternalError" security: - - X-API-Key: [] - X-User-ID: [] - /v1/search/routes: + - x-api-key: [] + x-user-id: [] + /search/control/routes: get: - operationId: getRoutesByQueryParams + operationId: controlGetRoutesByQueryParams tags: - - Exact Searches + - Control summary: "Search routes and report control points" description: "You can query and get control routes.

If you define a route (full name of it) you can query the control points associated with it. " parameters: @@ -257,8 +296,9 @@ paths: schema: type: "number" - $ref: "#/components/parameters/geo_context" - - $ref: "#/components/parameters/reduce_fuzzy_match" - - $ref: "#/components/parameters/size" + - $ref: "#/components/parameters/geo_context_mode" + - $ref: "#/components/parameters/limit" + - $ref: "#/components/parameters/disable_fuzziness" responses: 200: description: "OK (can be routesSchema or itemsSchema)" @@ -277,81 +317,28 @@ paths: 500: "$ref": "#/components/responses/InternalError" security: - - X-API-Key: [] - X-User-ID: [] - /v1/lookup/latlonToTile: - get: - operationId: convertLatlonToTile - tags: - - Exact Searches - summary: "Convert a WGS84 coordinate to a tile" - parameters: - - name: "lat" - in: "query" - description: "Latitude of the coordinate" - schema: - type: "number" - minimum: -90 - maximum: 90 - required: true - - name: "lon" - in: "query" - description: "Longitude of the coordinate" - schema: - type: "number" - minimum: -180 - maximum: 180 - required: true - responses: - 200: - description: "OK" - content: - application/json: - schema: - $ref: "#/components/schemas/subTileSchema" - 400: - "$ref": "#/components/responses/BadRequest" - 401: - "$ref": "#/components/responses/Unauthorized" - 403: - "$ref": "#/components/responses/Forbidden" - 500: - "$ref": "#/components/responses/InternalError" - security: - - X-API-Key: [] - X-User-ID: [] - /v1/lookup/tileToLatLon: + - x-api-key: [] + x-user-id: [] + /search/MGRS/tiles: get: - operationId: convertTileToLatlon + operationId: getMGRSToGeom tags: - - Exact Searches - summary: "Convert a tile to a geojson tile" + - MGRS + summary: "Convert a MGRS string to Geometry in GeoJSON" parameters: - name: "tile" in: "query" - description: "Tile name" + description: "MGRS tile string" schema: type: "string" required: true - - name: "sub_tile_number" - in: "query" - description: "An array describing sub tile number" - schema: - type: "array" - items: - type: "number" - minimum: 0 - maximum: 99 - minItems: 3 - maxItems: 3 - required: true responses: 200: description: "OK" content: application/json: schema: - $ref: "#/components/schemas/tilesSchema" + $ref: "#/components/schemas/mgrsTileSchema" 400: "$ref": "#/components/responses/BadRequest" 401: @@ -361,14 +348,14 @@ paths: 500: "$ref": "#/components/responses/InternalError" security: - - X-API-Key: [] - X-User-ID: [] - /v1/lookup/latlonToMgrs: + - x-api-key: [] + x-user-id: [] + /lookup/coordinates: get: - operationId: convertLatLonToMgrs + operationId: convertionGetLatLonToMgrs tags: - - Exact Searches - summary: "Convert a WGS84 coordinate to US Army MGRS" + - Convertions + summary: "Convert a WGS84 coordinate to US Army MGRS / Control Grid" parameters: - name: "lat" in: "query" @@ -386,13 +373,12 @@ paths: minimum: -180 maximum: 180 required: true - - name: "accuracy" - in: "query" - description: "Accuracy in digits (5 for 1 m, 4 for 10 m, 3 for 100 m, 2 for 1 km, 1 for 10 km or 0 for 100 km). Optional, default is 5." + - name: "target_grid" + in: query + description: Choose target grid schema: - type: "number" - minimum: 0 - maximum: 5 + type: string + enum: ["control", "MGRS"] responses: 200: description: "OK" @@ -401,9 +387,8 @@ paths: schema: type: object properties: - mgrs: - type: string - pattern: ^\d{1,2}[^ABIOYZabioyz][A-Za-z]{2}([0-9][0-9])+$ + geojson: + type: object # Make it geojson 400: "$ref": "#/components/responses/BadRequest" 401: @@ -413,36 +398,39 @@ paths: 500: "$ref": "#/components/responses/InternalError" security: - - X-API-Key: [] - X-User-ID: [] - /v1/lookup/mgrsToLatLon: - get: - operationId: convertMgrsToLatLon + - x-api-key: [] + x-user-id: [] + + /searches/feedback: + post: + operationId: postSearchFeedback tags: - - Exact Searches - summary: "Convert a US Army MGRS to WGS84 coordinate" + - Feedback + summary: "Retrieve feedback from services about quality of results" parameters: - - name: "mgrs" + - name: choosen_result_id in: "query" - description: "MGRS string" + description: "The result ID of chosen search" schema: - type: "string" - pattern: ^\d{1,2}[^ABIOYZabioyz][A-Za-z]{2}([0-9][0-9])+$ + type: string + minimum: 1 + maximum: 100 + required: true + - name: request_id + in: query + description: The request ID retrieved from headers of search API response + schema: + type: string + minimum: 1 + maximum: 100 required: true responses: - 200: - description: "OK" - content: - application/json: - schema: - type: object - properties: - lat: - type: number - lon: - type: number + 204: + description: "OK - No Content" 400: "$ref": "#/components/responses/BadRequest" + 404: + $ref: '#/components/responses/NotFound' 401: "$ref": "#/components/responses/Unauthorized" 403: @@ -450,8 +438,9 @@ paths: 500: "$ref": "#/components/responses/InternalError" security: - - X-API-Key: [] - X-User-ID: [] + - x-api-key: [] + x-user-id: [] + components: responses: BadRequest: @@ -466,6 +455,12 @@ components: application/json: schema: $ref: "#/components/schemas/errorSchema" + NotFound: + description: "Resource Not Found" + content: + application/json: + schema: + $ref: "#/components/schemas/errorSchema" Forbidden: description: "Token is not valid" content: @@ -479,26 +474,44 @@ components: schema: $ref: "#/components/schemas/errorSchema" parameters: - reduce_fuzzy_match: - name: "reduce_fuzzy_match" + disable_fuzziness: + name: "disable_fuzziness" in: "query" - description: "If an accurate result is obtained, only it will be returned (default is false)" + description: |- + If an accurate result is obtained, only it will be returned schema: + default: false type: "boolean" - - size: - name: "size" + limit: + name: "limit" in: "query" - description: "Maximum number of results (default is 3, range: 1-100)" + description: "Maximum number of results" schema: + default: 5 + minimum: 1 + maximum: 100 type: "number" - geo_context: + in: "query" name: "geo_context" + description: |- + Geo context of search. + + Supported queries: + * bounding box:
+ {"bbox":[minX,minY,maxX,maxY]} + * By point and radius (WGS84 and UTM are supported)
+ {"lon":value,"lat":value,"radius":value}
+ {"x":value,"y":value,"zone": number, "radius":value} + schema: + type: "string" # we need to make it oneOf types + geo_context_mode: + name: "geo_context_mode" in: "query" - description: 'Geo context of search.

Supported queries:

*By bounding box, object with these keys: {"bbox":[minX,minY,maxX,maxY]}

*By point and radius, object with these keys: {"lon":value,"lat":value,"radius":value}' + description: "Choose whether geo_context query parameter will be a filter or a bias value" schema: - type: "string" + type: string + enum: ["filter", "bias"] schemas: errorSchema: @@ -527,9 +540,62 @@ components: maximum: 100 minItems: 3 maxItems: 3 + tileSchema: + type: "object" + properties: + type: + type: "string" + enum: ["Feature"] + geometry: + $ref: "#/components/schemas/Polygon" + properties: + type: "object" + required: + - "TYPE" + - "tile_name" + properties: + TYPE: + type: "string" + tile_name: + type: "string" + sub_tile_id: + type: "string" + SUB_TILE_NUMBER: + type: "array" + items: + type: "number" + minimum: 0 + maximum: 100 + minItems: 3 + maxItems: 3 + mgrsTileSchema: + type: "object" + properties: + type: + type: "string" + enum: ["Feature"] + geometry: + $ref: "#/components/schemas/Polygon" + properties: + type: "object" tilesSchema: type: "object" description: "GeoJson feature collection representing a tile (Polygon)" + required: + - "type" + - "features" + properties: + type: + type: "string" + enum: + - "FeatureCollection" + features: + type: "array" + items: + $ref: "#/components/schemas/tileSchema" + itemsSchema: + type: "object" + description: "GeoJson feature collection representing an item" required: - "type" - "features" @@ -543,34 +609,31 @@ components: items: type: "object" properties: - type: - type: "string" - enum: ["Feature"] geometry: - $ref: "#/components/schemas/Geometry" + oneOf: + - $ref: "#/components/schemas/Point" + - $ref: "#/components/schemas/Polygon" properties: type: "object" required: - "TYPE" - - "TILE_NAME" + - "object_command_name" + - "entity_heb" + - "sub_tile_id" properties: TYPE: type: "string" - TILE_NAME: + object_command_name: type: "string" - SUB_TILE_ID: + entity_heb: type: "string" - SUB_TILE_NUMBER: - type: "array" - items: - type: "number" - minimum: 0 - maximum: 100 - minItems: 3 - maxItems: 3 - itemsSchema: + tile_name: + type: "string" + sub_tile_id: + type: "string" + routesSchema: type: "object" - description: "GeoJson feature collection representing an item" + description: "GeoJson feature collection representing a route (MultiLineString, LineString)" required: - "type" - "features" @@ -585,36 +648,91 @@ components: type: "object" properties: geometry: - $ref: "#/components/schemas/Geometry" + oneOf: + - $ref: "#/components/schemas/LineString" + - $ref: "#/components/schemas/MultiLineString" properties: type: "object" required: - "TYPE" - - "OBJECT_COMMAND_NAME" - - "ENTITY_HEB" - - "SUB_TILE_ID" + - "object_command_name" + - "entity_heb" properties: TYPE: type: "string" - OBJECT_COMMAND_NAME: + object_command_name: type: "string" - ENTITY_HEB: + entity_heb: type: "string" - TILE_NAME: - type: "string" - SUB_TILE_ID: + SECTION: type: "string" - routesSchema: + genericGeocodingResponse: # we need discriminator for Control and Location type: "object" - description: "GeoJson feature collection representing a route (Line)" + description: "GeoJson feature collection representing an item" required: - "type" - "features" + - "geocoding" properties: type: type: "string" enum: - "FeatureCollection" + geocoding: + type: object + required: + - "query" + - "response" + properties: + version: + type: string + pattern: '^\d{3}\.\d{3}\.\d{3}$' + example: 1.5.3 + query: + type: object + required: + - "text" + - "limit" + - "geo_context" + - "disable_fuzziness" + properties: + text: + type: string + minLength: 1 + maxLength: 100 + limit: + type: integer + minimum: 1 + maximum: 10 + geo_context: + type: string # fix later by referencing and correct type + disable_fuzziness: + type: boolean # reference later + response: + type: object + required: + - "max_score" + - "results_count" + - "match_latency" + properties: + max_score: + type: number + minimum: 0 + maximum: 100 + results_count: + type: integer + minimum: 0 + maximum: 10 + match_latency_ms: + type: integer + minimum: 0 + maximum: 5000 + bbox: + type: array + items: + type: number + minItems: 4 + maxItems: 4 features: type: "array" items: @@ -624,19 +742,80 @@ components: $ref: "#/components/schemas/Geometry" properties: type: "object" + additionalProperties: true required: - - "TYPE" - - "OBJECT_COMMAND_NAME" - - "ENTITY_HEB" + - id + - score + - matches + - regions + - display_name + - names properties: - TYPE: - type: "string" - OBJECT_COMMAND_NAME: - type: "string" - ENTITY_HEB: - type: "string" - SECTION: - type: "string" + id: + type: string + score: + type: number + minimum: 0 + maximum: 100 + matches: + type: array + minItems: 1 + maxItems: 100 + items: + type: object + required: + - id + - source + - layer + properties: + id: + type: string + source: + type: string + layer: + type: string + display_name: + type: string + names: + type: object + required: + - display + - default + additionalProperties: + type: array + minItems: 1 + maxItems: 100 + items: + type: string + properties: + display: + type: string + default: + type: array + minItems: 1 + items: + type: string + regions: + type: array + minItems: 1 + maxItems: 100 + items: + type: object + required: + - name + - sub_regions_names + properties: + name: + type: string + sub_regions_names: + type: array + minItems: 0 + maxItems: 100 + items: + type: string + minLength: 1 + maxLength: 100 + Geometry: type: object description: Geojson geometry @@ -744,6 +923,7 @@ components: type: array items: $ref: "#/components/schemas/Point3D" + Source: type: string title: Source @@ -751,11 +931,11 @@ components: type: string title: Region securitySchemes: - X-API-Key: + x-api-key: type: "apiKey" - name: "X-API-Key" + name: "x-api-key" in: "header" - X-User-ID: + x-user-id: type: "apiKey" - name: "X-User-ID" + name: "x-user-id" in: "header" From 39ce8fbb897d1498c6aec3a317c940d7f2e9ba27 Mon Sep 17 00:00:00 2001 From: Niv Greenstein Date: Thu, 8 Aug 2024 10:21:52 +0300 Subject: [PATCH 002/262] feat(openapi3): added bbox, UTMCircle and WGSCircle --- openapi3.yaml | 124 ++++++++++++++++++++++++++++++++------------------ 1 file changed, 80 insertions(+), 44 deletions(-) diff --git a/openapi3.yaml b/openapi3.yaml index 728b3972..a2afa9cf 100644 --- a/openapi3.yaml +++ b/openapi3.yaml @@ -45,7 +45,7 @@ paths: content: application/json: schema: - $ref: '#/components/schemas/genericGeocodingResponse' + $ref: "#/components/schemas/genericGeocodingResponse" 400: "$ref": "#/components/responses/BadRequest" 401: @@ -111,7 +111,7 @@ paths: content: application/json: schema: - "$ref": '#/components/schemas/genericGeocodingResponse' + "$ref": "#/components/schemas/genericGeocodingResponse" 400: "$ref": "#/components/responses/BadRequest" 401: @@ -430,7 +430,7 @@ paths: 400: "$ref": "#/components/responses/BadRequest" 404: - $ref: '#/components/responses/NotFound' + $ref: "#/components/responses/NotFound" 401: "$ref": "#/components/responses/Unauthorized" 403: @@ -504,7 +504,10 @@ components: {"lon":value,"lat":value,"radius":value}
{"x":value,"y":value,"zone": number, "radius":value} schema: - type: "string" # we need to make it oneOf types + oneOf: + - $ref: "#/components/schemas/BoundingBox" + - $ref: "#/components/schemas/WGS84Circle" + - $ref: "#/components/schemas/UTMCircle" geo_context_mode: name: "geo_context_mode" in: "query" @@ -543,41 +546,41 @@ components: tileSchema: type: "object" properties: - type: - type: "string" - enum: ["Feature"] - geometry: - $ref: "#/components/schemas/Polygon" + type: + type: "string" + enum: ["Feature"] + geometry: + $ref: "#/components/schemas/Polygon" + properties: + type: "object" + required: + - "TYPE" + - "tile_name" properties: - type: "object" - required: - - "TYPE" - - "tile_name" - properties: - TYPE: - type: "string" - tile_name: - type: "string" - sub_tile_id: - type: "string" - SUB_TILE_NUMBER: - type: "array" - items: - type: "number" - minimum: 0 - maximum: 100 - minItems: 3 - maxItems: 3 + TYPE: + type: "string" + tile_name: + type: "string" + sub_tile_id: + type: "string" + SUB_TILE_NUMBER: + type: "array" + items: + type: "number" + minimum: 0 + maximum: 100 + minItems: 3 + maxItems: 3 mgrsTileSchema: type: "object" properties: - type: - type: "string" - enum: ["Feature"] - geometry: - $ref: "#/components/schemas/Polygon" - properties: - type: "object" + type: + type: "string" + enum: ["Feature"] + geometry: + $ref: "#/components/schemas/Polygon" + properties: + type: "object" tilesSchema: type: "object" description: "GeoJson feature collection representing a tile (Polygon)" @@ -611,8 +614,8 @@ components: properties: geometry: oneOf: - - $ref: "#/components/schemas/Point" - - $ref: "#/components/schemas/Polygon" + - $ref: "#/components/schemas/Point" + - $ref: "#/components/schemas/Polygon" properties: type: "object" required: @@ -649,8 +652,8 @@ components: properties: geometry: oneOf: - - $ref: "#/components/schemas/LineString" - - $ref: "#/components/schemas/MultiLineString" + - $ref: "#/components/schemas/LineString" + - $ref: "#/components/schemas/MultiLineString" properties: type: "object" required: @@ -711,9 +714,9 @@ components: response: type: object required: - - "max_score" - - "results_count" - - "match_latency" + - "max_score" + - "results_count" + - "match_latency" properties: max_score: type: number @@ -802,8 +805,8 @@ components: items: type: object required: - - name - - sub_regions_names + - name + - sub_regions_names properties: name: type: string @@ -930,6 +933,39 @@ components: Region: type: string title: Region + BoundingBox: + type: object + description: "Bounding box array that contains [minX,minY,maxX,maxY]" + example: '{"bbox":[-74.382527,40.477003,-73.322346,40.916383]}' + properties: + bbox: + type: array + items: + type: number + minLength: 4 + maxLength: 4 + WGS84Circle: + type: object + properties: + lat: + type: number + lon: + type: number + radius: + type: number + UTMCircle: + type: object + properties: + x: + type: number + y: + type: number + zone: + type: number + minimum: 1 + maximum: 60 + radius: + type: number securitySchemes: x-api-key: type: "apiKey" From b0717b54745bc61093e0a2734c3264b378dca78c Mon Sep 17 00:00:00 2001 From: Niv Greenstein Date: Thu, 8 Aug 2024 10:22:50 +0300 Subject: [PATCH 003/262] feat(project): moved tile, route and item to control directory --- src/common/utils.ts | 11 +++---- src/containerConfig.ts | 12 ++++---- src/{ => control}/item/DAL/itemRepository.ts | 10 +++---- src/{ => control}/item/DAL/queries.ts | 4 +-- .../item/controllers/itemController.ts | 4 +-- src/{ => control}/item/models/item.ts | 0 src/{ => control}/item/models/itemManager.ts | 8 ++--- src/{ => control}/item/routes/itemRouter.ts | 0 src/{ => control}/route/DAL/queries.ts | 4 +-- .../route/DAL/routeRepository.ts | 11 ++++--- .../route/controllers/routeController.ts | 4 +-- src/{ => control}/route/models/route.ts | 0 .../route/models/routeManager.ts | 8 ++--- src/{ => control}/route/routes/routeRouter.ts | 0 src/{ => control}/tile/DAL/queries.ts | 0 src/{ => control}/tile/DAL/tileRepository.ts | 8 ++--- .../tile/controllers/tileController.ts | 4 +-- src/{ => control}/tile/models/tile.ts | 0 src/{ => control}/tile/models/tileManager.ts | 8 ++--- src/{ => control}/tile/routes/tileRouter.ts | 0 .../controllers/geotextSearchController.ts | 6 ++++ .../routes/geotextSearchRouter.ts | 3 +- src/latLon/controllers/latLonController.ts | 2 +- src/latLon/models/latLonManager.ts | 2 +- src/latLon/utlis/index.ts | 2 +- src/serverBuilder.ts | 29 ++++++++++++------- 26 files changed, 76 insertions(+), 64 deletions(-) rename src/{ => control}/item/DAL/itemRepository.ts (84%) rename src/{ => control}/item/DAL/queries.ts (92%) rename src/{ => control}/item/controllers/itemController.ts (93%) rename src/{ => control}/item/models/item.ts (100%) rename src/{ => control}/item/models/itemManager.ts (84%) rename src/{ => control}/item/routes/itemRouter.ts (100%) rename src/{ => control}/route/DAL/queries.ts (93%) rename src/{ => control}/route/DAL/routeRepository.ts (83%) rename src/{ => control}/route/controllers/routeController.ts (93%) rename src/{ => control}/route/models/route.ts (100%) rename src/{ => control}/route/models/routeManager.ts (88%) rename src/{ => control}/route/routes/routeRouter.ts (100%) rename src/{ => control}/tile/DAL/queries.ts (100%) rename src/{ => control}/tile/DAL/tileRepository.ts (88%) rename src/{ => control}/tile/controllers/tileController.ts (93%) rename src/{ => control}/tile/models/tile.ts (100%) rename src/{ => control}/tile/models/tileManager.ts (87%) rename src/{ => control}/tile/routes/tileRouter.ts (100%) diff --git a/src/common/utils.ts b/src/common/utils.ts index 11359d34..8fba3f28 100644 --- a/src/common/utils.ts +++ b/src/common/utils.ts @@ -1,13 +1,10 @@ import { estypes } from '@elastic/elasticsearch'; import proj4 from 'proj4'; -import { Item } from '../item/models/item'; -import { Tile } from '../tile/models/tile'; -import { Route } from '../route/models/route'; -import { elasticConfigPath } from './constants'; +import { Item } from '../control/item/models/item'; +import { Tile } from '../control/tile/models/tile'; +import { Route } from '../control/route/models/route'; import { utmProjection, wgs84Projection } from './projections'; -import { FeatureCollection, IConfig, WGS84Coordinate } from './interfaces'; -import { ElasticClients } from './elastic'; -import { ElasticDbClientsConfig } from './elastic/interfaces'; +import { FeatureCollection, WGS84Coordinate } from './interfaces'; export const formatResponse = (elasticResponse: estypes.SearchResponse): FeatureCollection => ({ type: 'FeatureCollection', diff --git a/src/containerConfig.ts b/src/containerConfig.ts index e3b4d6e5..512f6a7b 100644 --- a/src/containerConfig.ts +++ b/src/containerConfig.ts @@ -11,18 +11,18 @@ import { tracing } from './common/tracing'; import { InjectionObject, registerDependencies } from './common/dependencyRegistration'; import { elasticClientFactory, ElasticClients } from './common/elastic'; import { IApplication, PostgresDbConfig } from './common/interfaces'; -import { TILE_REPOSITORY_SYMBOL, tileRepositoryFactory } from './tile/DAL/tileRepository'; -import { TILE_ROUTER_SYMBOL, tileRouterFactory } from './tile/routes/tileRouter'; -import { ITEM_REPOSITORY_SYMBOL, itemRepositoryFactory } from './item/DAL/itemRepository'; -import { ITEM_ROUTER_SYMBOL, itemRouterFactory } from './item/routes/itemRouter'; -import { ROUTE_REPOSITORY_SYMBOL, routeRepositoryFactory } from './route/DAL/routeRepository'; -import { ROUTE_ROUTER_SYMBOL, routeRouterFactory } from './route/routes/routeRouter'; +import { TILE_REPOSITORY_SYMBOL, tileRepositoryFactory } from './control/tile/DAL/tileRepository'; +import { TILE_ROUTER_SYMBOL, tileRouterFactory } from './control/tile/routes/tileRouter'; +import { ITEM_ROUTER_SYMBOL, itemRouterFactory } from './control/item/routes/itemRouter'; +import { ROUTE_REPOSITORY_SYMBOL, routeRepositoryFactory } from './control/route/DAL/routeRepository'; +import { ROUTE_ROUTER_SYMBOL, routeRouterFactory } from './control/route/routes/routeRouter'; import { initDataSource } from './common/postgresql'; import { LATLON_CUSTOM_REPOSITORY_SYMBOL, latLonRepositoryFactory } from './latLon/DAL/latLonRepository'; import { LAT_LON_ROUTER_SYMBOL, latLonRouterFactory } from './latLon/routes/latLonRouter'; import { GEOTEXT_REPOSITORY_SYMBOL, geotextRepositoryFactory } from './geotextSearch/DAL/geotextSearchRepository'; import { GEOTEXT_SEARCH_ROUTER_SYMBOL, geotextSearchRouterFactory } from './geotextSearch/routes/geotextSearchRouter'; import { cronLoadTileLatLonDataFactory, cronLoadTileLatLonDataSymbol } from './latLon/DAL/latLonDAL'; +import { ITEM_REPOSITORY_SYMBOL, itemRepositoryFactory } from './control/item/DAL/itemRepository'; export interface RegisterOptions { override?: InjectionObject[]; diff --git a/src/item/DAL/itemRepository.ts b/src/control/item/DAL/itemRepository.ts similarity index 84% rename from src/item/DAL/itemRepository.ts rename to src/control/item/DAL/itemRepository.ts index b7df1f41..bf2a553f 100644 --- a/src/item/DAL/itemRepository.ts +++ b/src/control/item/DAL/itemRepository.ts @@ -1,12 +1,12 @@ import { Logger } from '@map-colonies/js-logger'; import { estypes } from '@elastic/elasticsearch'; import { FactoryFunction } from 'tsyringe'; -import { IConfig } from '../../common/interfaces'; -import { ElasticClients } from '../../common/elastic'; -import { additionalControlSearchProperties, queryElastic } from '../../common/elastic/utils'; -import { ElasticClient } from '../../common/elastic'; +import { IConfig } from '../../../common/interfaces'; +import { ElasticClients } from '../../../common/elastic'; +import { additionalControlSearchProperties, queryElastic } from '../../../common/elastic/utils'; +import { ElasticClient } from '../../../common/elastic'; import { Item } from '../models/item'; -import { SERVICES } from '../../common/constants'; +import { SERVICES } from '../../../common/constants'; import { ItemQueryParams, queryForItems } from './queries'; /* eslint-enable @typescript-eslint/naming-convention */ diff --git a/src/item/DAL/queries.ts b/src/control/item/DAL/queries.ts similarity index 92% rename from src/item/DAL/queries.ts rename to src/control/item/DAL/queries.ts index ec81de64..95911a27 100644 --- a/src/item/DAL/queries.ts +++ b/src/control/item/DAL/queries.ts @@ -1,6 +1,6 @@ import { estypes } from '@elastic/elasticsearch'; -import { boundingBox, geoDistance } from '../../common/elastic/utils'; -import { GeoContext } from '../../common/interfaces'; +import { boundingBox, geoDistance } from '../../../common/elastic/utils'; +import { GeoContext } from '../../../common/interfaces'; export interface ItemQueryParams { commandName: string; diff --git a/src/item/controllers/itemController.ts b/src/control/item/controllers/itemController.ts similarity index 93% rename from src/item/controllers/itemController.ts rename to src/control/item/controllers/itemController.ts index 1a8f5776..e01e0c62 100644 --- a/src/item/controllers/itemController.ts +++ b/src/control/item/controllers/itemController.ts @@ -4,10 +4,10 @@ import { BoundCounter, Meter } from '@opentelemetry/api-metrics'; import { RequestHandler } from 'express'; import httpStatus from 'http-status-codes'; import { injectable, inject } from 'tsyringe'; -import { SERVICES } from '../../common/constants'; +import { SERVICES } from '../../../common/constants'; import { ItemManager } from '../models/itemManager'; import { Item } from '../models/item'; -import { FeatureCollection, GeoContext } from '../../common/interfaces'; +import { FeatureCollection, GeoContext } from '../../../common/interfaces'; type GetItemsHandler = RequestHandler, undefined, GetItemsQueryParams>; diff --git a/src/item/models/item.ts b/src/control/item/models/item.ts similarity index 100% rename from src/item/models/item.ts rename to src/control/item/models/item.ts diff --git a/src/item/models/itemManager.ts b/src/control/item/models/itemManager.ts similarity index 84% rename from src/item/models/itemManager.ts rename to src/control/item/models/itemManager.ts index 8c17dbac..5b5e4a38 100644 --- a/src/item/models/itemManager.ts +++ b/src/control/item/models/itemManager.ts @@ -2,12 +2,12 @@ import { IConfig } from 'config'; import { Logger } from '@map-colonies/js-logger'; import { inject, injectable } from 'tsyringe'; import { estypes } from '@elastic/elasticsearch'; -import { SERVICES } from '../../common/constants'; +import { SERVICES } from '../../../common/constants'; import { ITEM_REPOSITORY_SYMBOL, ItemRepository } from '../DAL/itemRepository'; import { ItemQueryParams } from '../DAL/queries'; -import { formatResponse } from '../../common/utils'; -import { FeatureCollection } from '../../common/interfaces'; -import { getElasticClientQuerySize } from '../../common/elastic/utils'; +import { formatResponse } from '../../../common/utils'; +import { FeatureCollection } from '../../../common/interfaces'; +import { getElasticClientQuerySize } from '../../../common/elastic/utils'; import { Item } from './item'; @injectable() diff --git a/src/item/routes/itemRouter.ts b/src/control/item/routes/itemRouter.ts similarity index 100% rename from src/item/routes/itemRouter.ts rename to src/control/item/routes/itemRouter.ts diff --git a/src/route/DAL/queries.ts b/src/control/route/DAL/queries.ts similarity index 93% rename from src/route/DAL/queries.ts rename to src/control/route/DAL/queries.ts index c07f5309..c6104029 100644 --- a/src/route/DAL/queries.ts +++ b/src/control/route/DAL/queries.ts @@ -1,6 +1,6 @@ import { estypes } from '@elastic/elasticsearch'; -import { boundingBox, geoDistance } from '../../common/elastic/utils'; -import { GeoContext, WGS84Coordinate } from '../../common/interfaces'; +import { boundingBox, geoDistance } from '../../../common/elastic/utils'; +import { GeoContext, WGS84Coordinate } from '../../../common/interfaces'; export interface RouteQueryParams { commandName: string; diff --git a/src/route/DAL/routeRepository.ts b/src/control/route/DAL/routeRepository.ts similarity index 83% rename from src/route/DAL/routeRepository.ts rename to src/control/route/DAL/routeRepository.ts index 40c013a4..36cdf773 100644 --- a/src/route/DAL/routeRepository.ts +++ b/src/control/route/DAL/routeRepository.ts @@ -1,13 +1,12 @@ import { Logger } from '@map-colonies/js-logger'; import { estypes } from '@elastic/elasticsearch'; import { FactoryFunction } from 'tsyringe'; -import { ElasticClient } from '../../common/elastic'; -import { elasticConfigPath, SERVICES } from '../../common/constants'; +import { ElasticClient } from '../../../common/elastic'; +import { SERVICES } from '../../../common/constants'; import { Route } from '../models/route'; -import { IConfig } from '../../common/interfaces'; -import { ElasticClients } from '../../common/elastic'; -import { ElasticDbClientsConfig } from '../../common/elastic/interfaces'; -import { additionalControlSearchProperties, queryElastic } from '../../common/elastic/utils'; +import { IConfig } from '../../../common/interfaces'; +import { ElasticClients } from '../../../common/elastic'; +import { additionalControlSearchProperties, queryElastic } from '../../../common/elastic/utils'; import { RouteQueryParams, queryForControlPointInRoute, queryForRoute } from './queries'; // eslint-disable-next-line @typescript-eslint/explicit-function-return-type diff --git a/src/route/controllers/routeController.ts b/src/control/route/controllers/routeController.ts similarity index 93% rename from src/route/controllers/routeController.ts rename to src/control/route/controllers/routeController.ts index ce388fd3..4cae758a 100644 --- a/src/route/controllers/routeController.ts +++ b/src/control/route/controllers/routeController.ts @@ -4,10 +4,10 @@ import { BoundCounter, Meter } from '@opentelemetry/api-metrics'; import { RequestHandler } from 'express'; import httpStatus from 'http-status-codes'; import { injectable, inject } from 'tsyringe'; -import { SERVICES } from '../../common/constants'; +import { SERVICES } from '../../../common/constants'; import { RouteManager } from '../models/routeManager'; import { Route } from '../models/route'; -import { FeatureCollection, GeoContext } from '../../common/interfaces'; +import { FeatureCollection, GeoContext } from '../../../common/interfaces'; type GetRoutesHandler = RequestHandler, undefined, GetRoutesQueryParams>; diff --git a/src/route/models/route.ts b/src/control/route/models/route.ts similarity index 100% rename from src/route/models/route.ts rename to src/control/route/models/route.ts diff --git a/src/route/models/routeManager.ts b/src/control/route/models/routeManager.ts similarity index 88% rename from src/route/models/routeManager.ts rename to src/control/route/models/routeManager.ts index 619b9a69..53d338a8 100644 --- a/src/route/models/routeManager.ts +++ b/src/control/route/models/routeManager.ts @@ -2,12 +2,12 @@ import { IConfig } from 'config'; import { Logger } from '@map-colonies/js-logger'; import { inject, injectable } from 'tsyringe'; import { estypes } from '@elastic/elasticsearch'; -import { SERVICES } from '../../common/constants'; +import { SERVICES } from '../../../common/constants'; import { ROUTE_REPOSITORY_SYMBOL, RouteRepository } from '../DAL/routeRepository'; import { RouteQueryParams } from '../DAL/queries'; -import { formatResponse } from '../../common/utils'; -import { FeatureCollection } from '../../common/interfaces'; -import { getElasticClientQuerySize } from '../../common/elastic/utils'; +import { formatResponse } from '../../../common/utils'; +import { FeatureCollection } from '../../../common/interfaces'; +import { getElasticClientQuerySize } from '../../../common/elastic/utils'; import { Route } from './route'; @injectable() diff --git a/src/route/routes/routeRouter.ts b/src/control/route/routes/routeRouter.ts similarity index 100% rename from src/route/routes/routeRouter.ts rename to src/control/route/routes/routeRouter.ts diff --git a/src/tile/DAL/queries.ts b/src/control/tile/DAL/queries.ts similarity index 100% rename from src/tile/DAL/queries.ts rename to src/control/tile/DAL/queries.ts diff --git a/src/tile/DAL/tileRepository.ts b/src/control/tile/DAL/tileRepository.ts similarity index 88% rename from src/tile/DAL/tileRepository.ts rename to src/control/tile/DAL/tileRepository.ts index 1b1de8b3..860f6a3e 100644 --- a/src/tile/DAL/tileRepository.ts +++ b/src/control/tile/DAL/tileRepository.ts @@ -1,11 +1,11 @@ import { Logger } from '@map-colonies/js-logger'; import { estypes } from '@elastic/elasticsearch'; import { FactoryFunction } from 'tsyringe'; -import { ElasticClient, ElasticClients } from '../../common/elastic'; +import { ElasticClient, ElasticClients } from '../../../common/elastic'; import { Tile } from '../models/tile'; -import { additionalControlSearchProperties, queryElastic } from '../../common/elastic/utils'; -import { IConfig } from '../../common/interfaces'; -import { SERVICES } from '../../common/constants'; +import { additionalControlSearchProperties, queryElastic } from '../../../common/elastic/utils'; +import { IConfig } from '../../../common/interfaces'; +import { SERVICES } from '../../../common/constants'; import { queryForTiles, queryForSubTiles, TileQueryParams } from './queries'; // eslint-disable-next-line @typescript-eslint/explicit-function-return-type diff --git a/src/tile/controllers/tileController.ts b/src/control/tile/controllers/tileController.ts similarity index 93% rename from src/tile/controllers/tileController.ts rename to src/control/tile/controllers/tileController.ts index 3e86c335..cef76a5b 100644 --- a/src/tile/controllers/tileController.ts +++ b/src/control/tile/controllers/tileController.ts @@ -4,10 +4,10 @@ import { BoundCounter, Meter } from '@opentelemetry/api-metrics'; import { RequestHandler } from 'express'; import httpStatus from 'http-status-codes'; import { injectable, inject } from 'tsyringe'; -import { SERVICES } from '../../common/constants'; +import { SERVICES } from '../../../common/constants'; import { TileManager } from '../models/tileManager'; import { Tile } from '../models/tile'; -import { FeatureCollection } from '../../common/interfaces'; +import { FeatureCollection } from '../../../common/interfaces'; type GetTilesHandler = RequestHandler< undefined, diff --git a/src/tile/models/tile.ts b/src/control/tile/models/tile.ts similarity index 100% rename from src/tile/models/tile.ts rename to src/control/tile/models/tile.ts diff --git a/src/tile/models/tileManager.ts b/src/control/tile/models/tileManager.ts similarity index 87% rename from src/tile/models/tileManager.ts rename to src/control/tile/models/tileManager.ts index 121f4aec..89647139 100644 --- a/src/tile/models/tileManager.ts +++ b/src/control/tile/models/tileManager.ts @@ -2,12 +2,12 @@ import { IConfig } from 'config'; import { Logger } from '@map-colonies/js-logger'; import { inject, injectable } from 'tsyringe'; import { estypes } from '@elastic/elasticsearch'; -import { SERVICES } from '../../common/constants'; +import { SERVICES } from '../../../common/constants'; import { TILE_REPOSITORY_SYMBOL, TileRepository } from '../DAL/tileRepository'; -import { formatResponse } from '../../common/utils'; +import { formatResponse } from '../../../common/utils'; import { TileQueryParams } from '../DAL/queries'; -import { FeatureCollection } from '../../common/interfaces'; -import { getElasticClientQuerySize } from '../../common/elastic/utils'; +import { FeatureCollection } from '../../../common/interfaces'; +import { getElasticClientQuerySize } from '../../../common/elastic/utils'; import { Tile } from './tile'; @injectable() diff --git a/src/tile/routes/tileRouter.ts b/src/control/tile/routes/tileRouter.ts similarity index 100% rename from src/tile/routes/tileRouter.ts rename to src/control/tile/routes/tileRouter.ts diff --git a/src/geotextSearch/controllers/geotextSearchController.ts b/src/geotextSearch/controllers/geotextSearchController.ts index f8b776d7..f42a564c 100644 --- a/src/geotextSearch/controllers/geotextSearchController.ts +++ b/src/geotextSearch/controllers/geotextSearchController.ts @@ -16,6 +16,8 @@ type GetGeotextSearchHandler = RequestHandler< type GetRegionshHandler = RequestHandler; +type GetSourcesHandler = RequestHandler; + @injectable() export class GeotextSearchController { private readonly createdResourceCounter: BoundCounter; @@ -41,4 +43,8 @@ export class GeotextSearchController { public getRegions: GetRegionshHandler = (req, res, next) => { return res.status(httpStatus.OK).json(this.manager.regions()); }; + + public getSources: GetSourcesHandler = (req, res, next) => { + return res.status(httpStatus.NOT_IMPLEMENTED).send(); + }; } diff --git a/src/geotextSearch/routes/geotextSearchRouter.ts b/src/geotextSearch/routes/geotextSearchRouter.ts index 2c72e165..2387d042 100644 --- a/src/geotextSearch/routes/geotextSearchRouter.ts +++ b/src/geotextSearch/routes/geotextSearchRouter.ts @@ -7,7 +7,8 @@ const geotextSearchRouterFactory: FactoryFunction = (dependencyContainer const controller = dependencyContainer.resolve(GeotextSearchController); router.get('/regions', controller.getRegions); - router.get('/', controller.getGeotextSearch); + router.get('/query', controller.getGeotextSearch); + router.get('/sources', controller.getSources); return router; }; diff --git a/src/latLon/controllers/latLonController.ts b/src/latLon/controllers/latLonController.ts index 00633d6a..f39179c3 100644 --- a/src/latLon/controllers/latLonController.ts +++ b/src/latLon/controllers/latLonController.ts @@ -6,7 +6,7 @@ import httpStatus from 'http-status-codes'; import { injectable, inject } from 'tsyringe'; import { SERVICES } from '../../common/constants'; import { LatLonManager } from '../models/latLonManager'; -import { Tile } from '../../tile/models/tile'; +import { Tile } from '../../control/tile/models/tile'; import { FeatureCollection, WGS84Coordinate } from '../../common/interfaces'; type GetLatLonToTileHandler = RequestHandler< diff --git a/src/latLon/models/latLonManager.ts b/src/latLon/models/latLonManager.ts index ee6732a0..c32f4127 100644 --- a/src/latLon/models/latLonManager.ts +++ b/src/latLon/models/latLonManager.ts @@ -7,7 +7,7 @@ import { LatLonDAL } from '../DAL/latLonDAL'; import { convertWgs84ToUTM, validateTile, validateWGS84Coordinate } from '../../common/utils'; import { convertTilesToUTM, getSubTileByBottomLeftUtmCoor, validateResult } from '../utlis'; import { BadRequestError } from '../../common/errors'; -import { Tile } from '../../tile/models/tile'; +import { Tile } from '../../control/tile/models/tile'; import { FeatureCollection, WGS84Coordinate } from '../../common/interfaces'; @injectable() diff --git a/src/latLon/utlis/index.ts b/src/latLon/utlis/index.ts index aedc39ef..6c806eda 100644 --- a/src/latLon/utlis/index.ts +++ b/src/latLon/utlis/index.ts @@ -3,7 +3,7 @@ import { BadRequestError } from '../../common/errors'; import { convertUTMToWgs84 } from '../../common/utils'; import { LatLon } from '../DAL/latLon'; import { FeatureCollection } from '../../common/interfaces'; -import { Tile } from '../../tile/models/tile'; +import { Tile } from '../../control/tile/models/tile'; /* eslint-disable @typescript-eslint/naming-convention */ const geoJsonObjectTemplate = (): FeatureCollection => ({ diff --git a/src/serverBuilder.ts b/src/serverBuilder.ts index 9e8b387c..1320d74d 100644 --- a/src/serverBuilder.ts +++ b/src/serverBuilder.ts @@ -10,9 +10,9 @@ import httpLogger from '@map-colonies/express-access-log-middleware'; import { defaultMetricsMiddleware, getTraceContexHeaderMiddleware } from '@map-colonies/telemetry'; import { SERVICES } from './common/constants'; import { IConfig } from './common/interfaces'; -import { TILE_ROUTER_SYMBOL } from './tile/routes/tileRouter'; -import { ITEM_ROUTER_SYMBOL } from './item/routes/itemRouter'; -import { ROUTE_ROUTER_SYMBOL } from './route/routes/routeRouter'; +import { TILE_ROUTER_SYMBOL } from './control/tile/routes/tileRouter'; +import { ITEM_ROUTER_SYMBOL } from './control/item/routes/itemRouter'; +import { ROUTE_ROUTER_SYMBOL } from './control/route/routes/routeRouter'; import { LAT_LON_ROUTER_SYMBOL } from './latLon/routes/latLonRouter'; import { GEOTEXT_SEARCH_ROUTER_SYMBOL } from './geotextSearch/routes/geotextSearchRouter'; import { cronLoadTileLatLonDataSymbol } from './latLon/DAL/latLonDAL'; @@ -39,7 +39,7 @@ export class ServerBuilder { public build(): express.Application { this.registerPreRoutesMiddleware(); this.buildDocsRoutes(); - this.buildRoutesV1(); + this.buildRoutes(); this.registerPostRoutesMiddleware(); return this.serverInstance; @@ -54,14 +54,23 @@ export class ServerBuilder { this.serverInstance.use(this.config.get('openapiConfig.basePath'), openapiRouter.getRouter()); } - private buildRoutesV1(): void { + private buildRoutes(): void { const router = Router(); - router.use('/search/tiles', this.tileRouter); - router.use('/search/items', this.itemRouter); - router.use('/search/routes', this.routeRouter); router.use('/lookup', this.latLonRouter); - router.use('/query', this.geotextRouter); - this.serverInstance.use('/v1', router); + router.use('/location', this.geotextRouter); + router.use('/control', this.buildControlRoutes()); + + this.serverInstance.use('/search/', router); + } + + private buildControlRoutes(): Router { + const router = Router(); + + router.use('/tiles', this.tileRouter); + router.use('/items', this.itemRouter); + router.use('/routes', this.routeRouter); + + return router; } private registerPreRoutesMiddleware(): void { From 965078fbc11ea99d815328c7ee58000492a55a18 Mon Sep 17 00:00:00 2001 From: Niv Greenstein Date: Sun, 11 Aug 2024 10:00:21 +0300 Subject: [PATCH 004/262] fix: removed required from tile in /search/control/tiles. added MGRS example and description --- openapi3.yaml | 6 ++++-- 1 file changed, 4 insertions(+), 2 deletions(-) diff --git a/openapi3.yaml b/openapi3.yaml index a2afa9cf..d64a04d8 100644 --- a/openapi3.yaml +++ b/openapi3.yaml @@ -186,7 +186,8 @@ paths: summary: "Search tiles and sub tiles" description: |- Tiles are consisted of 3 characters.
- You can query and get results from 2 characters.

If you define a tile (full name of it) you can query the sub tile associated with it. + You can query and get results from 2 characters.

If you define a tile (full name of it) you can query the sub tile associated with it.
+ You may also search tile based on 1 meter precision MGRS tile. parameters: - name: "tile" in: "query" @@ -195,13 +196,14 @@ paths: type: "string" minLength: 2 maxLength: 3 - required: true - name: "sub_tile" in: "query" description: "Sub tile number" schema: type: "number" - name: MGRS + description: "1 meters MGRS Tile" + example: "18SUJ2338907395" in: query schema: type: string From 3f9c8d82d748c51b6903d46b1ffa4d1d939980df Mon Sep 17 00:00:00 2001 From: Niv Greenstein Date: Sun, 11 Aug 2024 10:00:59 +0300 Subject: [PATCH 005/262] fix: setted elastic_keywords instead of writing it right in the query --- src/control/tile/DAL/queries.ts | 21 ++++++++++++++------- 1 file changed, 14 insertions(+), 7 deletions(-) diff --git a/src/control/tile/DAL/queries.ts b/src/control/tile/DAL/queries.ts index d3dfeb92..b04500bb 100644 --- a/src/control/tile/DAL/queries.ts +++ b/src/control/tile/DAL/queries.ts @@ -1,7 +1,14 @@ /* eslint-disable @typescript-eslint/naming-convention */ import { estypes } from '@elastic/elasticsearch'; +import { CommonRequestParameters } from '../../../common/interfaces'; -export interface TileQueryParams { +const ELASTIC_KEYWORDS = { + TYPE: 'properties.TYPE.keyword', + TILE_NAME: 'properties.TILE_NAME.keyword', + SUB_TILE_ID: 'properties.SUB_TILE_ID.keyword', +}; + +export interface TileQueryParams extends CommonRequestParameters { tile: string; subTile?: number; } @@ -12,14 +19,14 @@ export const queryForTiles = (params: Omit): estypes must: [ { term: { - 'properties.TYPE.keyword': 'TILE', + [ELASTIC_KEYWORDS.TYPE]: 'TILE', }, }, { match: { - 'properties.TILE_NAME.keyword': { + [ELASTIC_KEYWORDS.TILE_NAME]: { query: params.tile, - fuzziness: 1, + fuzziness: !params.disable_fuzziness ? 1 : undefined, prefix_length: 1, }, }, @@ -35,17 +42,17 @@ export const queryForSubTiles = (params: Required): estypes.Sea must: [ { term: { - 'properties.TYPE.keyword': 'SUB_TILE', + [ELASTIC_KEYWORDS.TYPE]: 'SUB_TILE', }, }, { term: { - 'properties.TILE_NAME.keyword': params.tile, + [ELASTIC_KEYWORDS.TILE_NAME]: params.tile, }, }, { match: { - 'properties.SUB_TILE_ID.keyword': params.subTile, + [ELASTIC_KEYWORDS.SUB_TILE_ID]: params.subTile, }, }, ], From d64a4694a76abeaa89d8131775493ec44aeb9713 Mon Sep 17 00:00:00 2001 From: Niv Greenstein Date: Sun, 11 Aug 2024 10:01:41 +0300 Subject: [PATCH 006/262] delete: removed getElasticClientQuerySize as it will be given as default from Swagger --- src/common/elastic/utils.ts | 6 +----- 1 file changed, 1 insertion(+), 5 deletions(-) diff --git a/src/common/elastic/utils.ts b/src/common/elastic/utils.ts index 5e7e6278..b0a59c76 100644 --- a/src/common/elastic/utils.ts +++ b/src/common/elastic/utils.ts @@ -3,11 +3,7 @@ import { IConfig, WGS84Coordinate } from '../interfaces'; import { InternalServerError } from '../errors'; import { elasticConfigPath, CONTROL_FIELDS } from '../constants'; import { ElasticDbClientsConfig } from './interfaces'; -import { ElasticClient, ElasticClients } from './index'; - -export const getElasticClientQuerySize = (config: IConfig, key: keyof ElasticClients): number => { - return config.get(elasticConfigPath)[key].properties.defaultResponseLimit; -}; +import { ElasticClient } from './index'; /* eslint-disable @typescript-eslint/naming-convention */ export const additionalControlSearchProperties = (config: IConfig, size: number): { size: number; index: string; _source: string[] } => ({ From 9053435bbf7c5f5da19d438e5eb8e8ce5a6b259f Mon Sep 17 00:00:00 2001 From: Niv Greenstein Date: Sun, 11 Aug 2024 10:02:06 +0300 Subject: [PATCH 007/262] feat: created CommonRequestParameters interface --- src/common/interfaces.ts | 15 ++++++++++++++- 1 file changed, 14 insertions(+), 1 deletion(-) diff --git a/src/common/interfaces.ts b/src/common/interfaces.ts index cadc78ee..d376ec40 100644 --- a/src/common/interfaces.ts +++ b/src/common/interfaces.ts @@ -1,4 +1,3 @@ -import { Client, ClientOptions } from '@elastic/elasticsearch'; import { DataSourceOptions } from 'typeorm'; import { Feature, FeatureCollection as GeoJSONFeatureCollection } from 'geojson'; @@ -61,3 +60,17 @@ export interface IApplication { nameTranslationsKeys: string[]; mainLanguageRegex: string; } + +export enum GeoContextMode { + FILTER = 'filter', + BIAS = 'bias', +} + +/* eslint-disable @typescript-eslint/naming-convention */ +export interface CommonRequestParameters { + geo_context?: GeoContext; + geo_context_mode?: GeoContextMode; + limit: number; + disable_fuzziness: boolean; +} +/* eslint-enable @typescript-eslint/naming-convention */ From c5f66d66769e84b4e31f18a257cf4c85f7c26206 Mon Sep 17 00:00:00 2001 From: Niv Greenstein Date: Sun, 11 Aug 2024 10:02:25 +0300 Subject: [PATCH 008/262] feat: import CommonRequestParameters --- .../tile/controllers/tileController.ts | 21 ++++++++++--------- src/control/tile/models/tileManager.ts | 14 ++++++------- 2 files changed, 18 insertions(+), 17 deletions(-) diff --git a/src/control/tile/controllers/tileController.ts b/src/control/tile/controllers/tileController.ts index cef76a5b..63e05d50 100644 --- a/src/control/tile/controllers/tileController.ts +++ b/src/control/tile/controllers/tileController.ts @@ -7,7 +7,7 @@ import { injectable, inject } from 'tsyringe'; import { SERVICES } from '../../../common/constants'; import { TileManager } from '../models/tileManager'; import { Tile } from '../models/tile'; -import { FeatureCollection } from '../../../common/interfaces'; +import { CommonRequestParameters, FeatureCollection } from '../../../common/interfaces'; type GetTilesHandler = RequestHandler< undefined, @@ -20,11 +20,9 @@ type GetTilesHandler = RequestHandler< GetTilesQueryParams >; -export interface GetTilesQueryParams { +export interface GetTilesQueryParams extends CommonRequestParameters { tile: string; sub_tile?: string; - reduce_fuzzy_match?: string; - size?: string; } @injectable() @@ -41,12 +39,15 @@ export class TileController { public getTiles: GetTilesHandler = async (req, res, next) => { try { - const { tile, sub_tile, reduce_fuzzy_match, size } = req.query; - const response = await this.manager.getTiles( - { tile, subTile: sub_tile ? parseInt(sub_tile) : undefined }, - reduce_fuzzy_match == 'true', - size ? parseInt(size) : undefined - ); + const { tile, sub_tile, disable_fuzziness, geo_context, geo_context_mode, limit } = req.query; + const response = await this.manager.getTiles({ + tile, + subTile: sub_tile ? parseInt(sub_tile) : undefined, + disable_fuzziness, + geo_context, + geo_context_mode, + limit, + }); return res.status(httpStatus.OK).json(response); } catch (error: unknown) { this.logger.warn('tileController.getTiles Error:', error); diff --git a/src/control/tile/models/tileManager.ts b/src/control/tile/models/tileManager.ts index 89647139..1394053a 100644 --- a/src/control/tile/models/tileManager.ts +++ b/src/control/tile/models/tileManager.ts @@ -6,8 +6,7 @@ import { SERVICES } from '../../../common/constants'; import { TILE_REPOSITORY_SYMBOL, TileRepository } from '../DAL/tileRepository'; import { formatResponse } from '../../../common/utils'; import { TileQueryParams } from '../DAL/queries'; -import { FeatureCollection } from '../../../common/interfaces'; -import { getElasticClientQuerySize } from '../../../common/elastic/utils'; +import { CommonRequestParameters, FeatureCollection } from '../../../common/interfaces'; import { Tile } from './tile'; @injectable() @@ -18,18 +17,19 @@ export class TileManager { @inject(TILE_REPOSITORY_SYMBOL) private readonly tileRepository: TileRepository ) {} - public async getTiles(tileQueryParams: TileQueryParams, reduceFuzzyMatch = false, size?: number): Promise> { + public async getTiles(tileQueryParams: TileQueryParams & CommonRequestParameters): Promise> { + const { limit, disable_fuzziness: disableFuzziness } = tileQueryParams; + let elasticResponse: estypes.SearchResponse | undefined = undefined; - const numberOfResults = size ?? getElasticClientQuerySize(this.config, 'control'); if (tileQueryParams.subTile ?? 0) { - elasticResponse = await this.tileRepository.getSubTiles(tileQueryParams as Required, numberOfResults); + elasticResponse = await this.tileRepository.getSubTiles(tileQueryParams as Required, limit); } else { - elasticResponse = await this.tileRepository.getTiles(tileQueryParams, numberOfResults); + elasticResponse = await this.tileRepository.getTiles(tileQueryParams, limit); } const formattedResponse = formatResponse(elasticResponse); - if (reduceFuzzyMatch && formattedResponse.features.length > 0) { + if (!disableFuzziness && formattedResponse.features.length > 0) { const filterFunction = tileQueryParams.subTile ?? 0 ? (hit: Tile | undefined): hit is Tile => From 3e7b3bae771e5a8930a1a939cd8f8f14a28911ca Mon Sep 17 00:00:00 2001 From: Niv Greenstein Date: Sun, 11 Aug 2024 12:47:29 +0300 Subject: [PATCH 009/262] Update openapi3.yaml fix(openapi): fixed bbox, geo_context, disable_fuzziness referencing --- openapi3.yaml | 23 +++++++++-------------- 1 file changed, 9 insertions(+), 14 deletions(-) diff --git a/openapi3.yaml b/openapi3.yaml index d64a04d8..0eaafde2 100644 --- a/openapi3.yaml +++ b/openapi3.yaml @@ -507,7 +507,10 @@ components: {"x":value,"y":value,"zone": number, "radius":value} schema: oneOf: - - $ref: "#/components/schemas/BoundingBox" + - type: object + properties: + bbox: + $ref: "#/components/schemas/BoundingBox" - $ref: "#/components/schemas/WGS84Circle" - $ref: "#/components/schemas/UTMCircle" geo_context_mode: @@ -710,9 +713,9 @@ components: minimum: 1 maximum: 10 geo_context: - type: string # fix later by referencing and correct type + $ref: "#/components/parameters/geo_context" disable_fuzziness: - type: boolean # reference later + $ref: "#/components/parameters/disable_fuzziness" response: type: object required: @@ -733,11 +736,7 @@ components: minimum: 0 maximum: 5000 bbox: - type: array - items: - type: number - minItems: 4 - maxItems: 4 + $ref: "#/components/schemas/BoundingBox" features: type: "array" items: @@ -938,14 +937,10 @@ components: BoundingBox: type: object description: "Bounding box array that contains [minX,minY,maxX,maxY]" - example: '{"bbox":[-74.382527,40.477003,-73.322346,40.916383]}' + example: '[-74.382527,40.477003,-73.322346,40.916383]' properties: bbox: - type: array - items: - type: number - minLength: 4 - maxLength: 4 + $ref: "#/components/schemas/BoundingBox" WGS84Circle: type: object properties: From 86f89eee61455dfbeeec63bc77e7ba913f3d5f23 Mon Sep 17 00:00:00 2001 From: Niv Greenstein Date: Sun, 11 Aug 2024 13:50:49 +0300 Subject: [PATCH 010/262] delete(dev/scripts/controlElasticsearchData): deleted score --- devScripts/controlElasticsearchData.json | 4 ---- 1 file changed, 4 deletions(-) diff --git a/devScripts/controlElasticsearchData.json b/devScripts/controlElasticsearchData.json index 856ca077..1b4b2654 100644 --- a/devScripts/controlElasticsearchData.json +++ b/devScripts/controlElasticsearchData.json @@ -1,7 +1,6 @@ [ { "_id": "CONTROL.ITEMS", - "_score": 1, "_source": { "type": "Feature", "id": 27, @@ -54,7 +53,6 @@ }, { "_id": "CONTROL.SUB_TILES", - "_score": 1, "_source": { "type": "Feature", "id": 13668, @@ -83,7 +81,6 @@ }, { "_id": "CONTROL.ROUTES", - "_score": 1, "_source": { "type": "Feature", "id": 2, @@ -116,7 +113,6 @@ }, { "_id": "CONTROL.TILES", - "_score": 1, "_source": { "type": "Feature", "id": 52, From 0c1b189b9aec8fea07acc54d3756c88cc8844360 Mon Sep 17 00:00:00 2001 From: Niv Greenstein Date: Sun, 11 Aug 2024 13:51:20 +0300 Subject: [PATCH 011/262] feat(devscripts/elasticsearch): added index creation and geo_shape mapping --- devScripts/importDataToElastic.ts | 31 +++++++++++++++++++++++++++++++ 1 file changed, 31 insertions(+) diff --git a/devScripts/importDataToElastic.ts b/devScripts/importDataToElastic.ts index f23aefb0..9eb88f40 100644 --- a/devScripts/importDataToElastic.ts +++ b/devScripts/importDataToElastic.ts @@ -10,6 +10,37 @@ const main = async (): Promise => { const controlClient = new Client({ ...config.db.elastic.control }); const geotextClient = new Client({ ...config.db.elastic.geotext }); + for (const { client, index, key } of [ + { + client: controlClient, + index: config.db.elastic.control.properties.index, + key: 'geometry', + }, + { + client: geotextClient, + index: config.db.elastic.geotext.properties.index.geotext, + key: 'geo_json', + }, + { + client: geotextClient, + index: config.db.elastic.geotext.properties.index.hierarchies, + key: 'geo_json', + }, + ]) { + await client.indices.create({ + index: index, + body: { + mappings: { + properties: { + [key]: { + type: 'geo_shape', + }, + }, + }, + }, + }); + } + for (const item of controlData) { await controlClient.index({ index: config.db.elastic.control.properties.index, From a94dabb045b60b28b05107d4a645e499386af1d4 Mon Sep 17 00:00:00 2001 From: Niv Greenstein Date: Sun, 11 Aug 2024 14:46:18 +0300 Subject: [PATCH 012/262] Update openapi3.yaml fix: created geo_context schema and correctly referencing it --- openapi3.yaml | 18 ++++++++++-------- 1 file changed, 10 insertions(+), 8 deletions(-) diff --git a/openapi3.yaml b/openapi3.yaml index 0eaafde2..b33b40dd 100644 --- a/openapi3.yaml +++ b/openapi3.yaml @@ -506,13 +506,7 @@ components: {"lon":value,"lat":value,"radius":value}
{"x":value,"y":value,"zone": number, "radius":value} schema: - oneOf: - - type: object - properties: - bbox: - $ref: "#/components/schemas/BoundingBox" - - $ref: "#/components/schemas/WGS84Circle" - - $ref: "#/components/schemas/UTMCircle" + $ref: "#/components/schemas/geo_context" geo_context_mode: name: "geo_context_mode" in: "query" @@ -713,7 +707,7 @@ components: minimum: 1 maximum: 10 geo_context: - $ref: "#/components/parameters/geo_context" + $ref: "#/components/schemas/geo_context" disable_fuzziness: $ref: "#/components/parameters/disable_fuzziness" response: @@ -963,6 +957,14 @@ components: maximum: 60 radius: type: number + geo_context: + oneOf: + - type: object + properties: + bbox: + $ref: "#/components/schemas/BoundingBox" + - $ref: "#/components/schemas/WGS84Circle" + - $ref: "#/components/schemas/UTMCircle" securitySchemes: x-api-key: type: "apiKey" From 00321fe0baebe65fdc8efdd1c894d999a6b0bdeb Mon Sep 17 00:00:00 2001 From: Niv Greenstein Date: Sun, 11 Aug 2024 14:52:23 +0300 Subject: [PATCH 013/262] Update openapi3.yaml fix: created disable_fuzziness schema and reference it --- openapi3.yaml | 8 +++++--- 1 file changed, 5 insertions(+), 3 deletions(-) diff --git a/openapi3.yaml b/openapi3.yaml index b33b40dd..5caf5113 100644 --- a/openapi3.yaml +++ b/openapi3.yaml @@ -482,8 +482,7 @@ components: description: |- If an accurate result is obtained, only it will be returned schema: - default: false - type: "boolean" + $ref: "#/components/schemas/disable_fuzziness" limit: name: "limit" in: "query" @@ -709,7 +708,7 @@ components: geo_context: $ref: "#/components/schemas/geo_context" disable_fuzziness: - $ref: "#/components/parameters/disable_fuzziness" + $ref: "#/components/schemas/disable_fuzziness" response: type: object required: @@ -965,6 +964,9 @@ components: $ref: "#/components/schemas/BoundingBox" - $ref: "#/components/schemas/WGS84Circle" - $ref: "#/components/schemas/UTMCircle" + disable_fuzziness: + type: boolean + default: false securitySchemes: x-api-key: type: "apiKey" From 6ff261564ea75b5dcb7f9534702012452b360347 Mon Sep 17 00:00:00 2001 From: Niv Greenstein Date: Sun, 11 Aug 2024 15:08:33 +0300 Subject: [PATCH 014/262] fix: removed unneccecery export --- src/geotextSearch/utils.ts | 51 +++++++++++++++++++------------------- 1 file changed, 26 insertions(+), 25 deletions(-) diff --git a/src/geotextSearch/utils.ts b/src/geotextSearch/utils.ts index 7324f961..2f0a7188 100644 --- a/src/geotextSearch/utils.ts +++ b/src/geotextSearch/utils.ts @@ -1,10 +1,10 @@ // import fetch, { Response } from "node-fetch-commonjs"; -import { GeoJSON } from 'geojson'; +import { GeoJSON, Geometry, Feature, FeatureCollection, Position } from 'geojson'; import { SearchHit } from '@elastic/elasticsearch/lib/api/types'; import { StatusCodes } from 'http-status-codes'; import axios, { AxiosResponse as Response } from 'axios'; import { InternalServerError } from '../common/errors'; -import { IApplication } from '../common/interfaces'; +import { GeoContext, IApplication } from '../common/interfaces'; import { BBOX_LENGTH, POINT_LENGTH, QueryResult, TextSearchParams } from './interfaces'; import { generateDisplayName } from './parsing'; import { TextSearchHit } from './models/elasticsearchHits'; @@ -13,6 +13,27 @@ const FIND_QUOTES = /["']/g; const FIND_SPECIAL = /[`!@#$%^&*()_\-+=|\\/,.<>:[\]{}\n\t\r\s;؛]+/g; +const parsePoint = (split: string[]): GeoJSON => ({ + type: 'Point', + coordinates: split.map(Number), +}); + +const parseBbox = (split: string[]): GeoJSON => { + const [xMin, yMin, xMax, yMax] = split.map(Number); + return { + type: 'Polygon', + coordinates: [ + [ + [xMin, yMin], + [xMin, yMax], + [xMax, yMax], + [xMax, yMin], + [xMin, yMin], + ], + ], + }; +}; + export const fetchNLPService = async (endpoint: string, requestData: object): Promise => { let res: Response | null = null, data: T[] | undefined | null = null; @@ -40,28 +61,8 @@ export const fetchNLPService = async (endpoint: string, requestData: object): export const cleanQuery = (query: string): string[] => query.replace(FIND_QUOTES, '').split(FIND_SPECIAL); -export const parsePoint = (split: string[]): GeoJSON => ({ - type: 'Point', - coordinates: split.map(Number), -}); - -export const parseBbox = (split: string[]): GeoJSON => { - const [xMin, yMin, xMax, yMax] = split.map(Number); - return { - type: 'Polygon', - coordinates: [ - [ - [xMin, yMin], - [xMin, yMax], - [xMax, yMax], - [xMax, yMin], - [xMin, yMin], - ], - ], - }; -}; - -export const parseGeo = (input: string | GeoJSON): GeoJSON | undefined => { +export const parseGeo = (input: string | GeoJSON | GeoContext): GeoJSON | undefined => { + //TODO: remove string | GeoJson as accepted types if (typeof input === 'string') { const splitted = input.split(','); const converted = splitted.map(Number); @@ -79,7 +80,7 @@ export const parseGeo = (input: string | GeoJSON): GeoJSON | undefined => { } } } - + // TODO: Add geojson validation return input as GeoJSON; }; From 7f7deb887fe04a2095d9beebbce4e7b1b9654f1c Mon Sep 17 00:00:00 2001 From: Niv Greenstein Date: Sun, 11 Aug 2024 16:28:04 +0300 Subject: [PATCH 015/262] fix: updated boundingBox as parameter and schema --- openapi3.yaml | 22 +++++++++++++++------- 1 file changed, 15 insertions(+), 7 deletions(-) diff --git a/openapi3.yaml b/openapi3.yaml index 5caf5113..d49288ed 100644 --- a/openapi3.yaml +++ b/openapi3.yaml @@ -928,12 +928,20 @@ components: type: string title: Region BoundingBox: - type: object + type: array description: "Bounding box array that contains [minX,minY,maxX,maxY]" - example: '[-74.382527,40.477003,-73.322346,40.916383]' - properties: - bbox: - $ref: "#/components/schemas/BoundingBox" + # example: '[-74.382527,40.477003,-73.322346,40.916383]' + items: + type: number + minLength: 4 + maxLength: 4 + # properties: + # bbox: + # type: array + # items: + # type: number + # minLength: 4 + # maxLength: 4 WGS84Circle: type: object properties: @@ -956,8 +964,8 @@ components: maximum: 60 radius: type: number - geo_context: - oneOf: + geo_context: + anyOf: - type: object properties: bbox: From 483224cad6bfcea271a880f19a96a5a61133e156 Mon Sep 17 00:00:00 2001 From: Niv Greenstein Date: Sun, 11 Aug 2024 16:28:56 +0300 Subject: [PATCH 016/262] feat: added support for parsing geoContext --- src/geotextSearch/utils.ts | 25 +++++++++++++++++++++---- 1 file changed, 21 insertions(+), 4 deletions(-) diff --git a/src/geotextSearch/utils.ts b/src/geotextSearch/utils.ts index 2f0a7188..9b35debd 100644 --- a/src/geotextSearch/utils.ts +++ b/src/geotextSearch/utils.ts @@ -1,10 +1,11 @@ // import fetch, { Response } from "node-fetch-commonjs"; -import { GeoJSON, Geometry, Feature, FeatureCollection, Position } from 'geojson'; +import { GeoJSON, Point } from 'geojson'; import { SearchHit } from '@elastic/elasticsearch/lib/api/types'; import { StatusCodes } from 'http-status-codes'; import axios, { AxiosResponse as Response } from 'axios'; import { InternalServerError } from '../common/errors'; import { GeoContext, IApplication } from '../common/interfaces'; +import { convertUTMToWgs84, convertWgs84ToUTM } from '../common/utils'; import { BBOX_LENGTH, POINT_LENGTH, QueryResult, TextSearchParams } from './interfaces'; import { generateDisplayName } from './parsing'; import { TextSearchHit } from './models/elasticsearchHits'; @@ -13,12 +14,12 @@ const FIND_QUOTES = /["']/g; const FIND_SPECIAL = /[`!@#$%^&*()_\-+=|\\/,.<>:[\]{}\n\t\r\s;؛]+/g; -const parsePoint = (split: string[]): GeoJSON => ({ +const parsePoint = (split: string[] | number[]): GeoJSON => ({ type: 'Point', coordinates: split.map(Number), }); -const parseBbox = (split: string[]): GeoJSON => { +const parseBbox = (split: string[] | number[]): GeoJSON => { const [xMin, yMin, xMax, yMax] = split.map(Number); return { type: 'Polygon', @@ -63,6 +64,8 @@ export const cleanQuery = (query: string): string[] => query.replace(FIND_QUOTES export const parseGeo = (input: string | GeoJSON | GeoContext): GeoJSON | undefined => { //TODO: remove string | GeoJson as accepted types + //TODO: Add geojson validation + //TODO: refactor this function if (typeof input === 'string') { const splitted = input.split(','); const converted = splitted.map(Number); @@ -79,8 +82,22 @@ export const parseGeo = (input: string | GeoJSON | GeoContext): GeoJSON | undefi return undefined; } } + } else if (input.bbox !== undefined) { + return parseBbox(input.bbox); + } else if ( + ((input as GeoContext).x !== undefined && + (input as GeoContext).y !== undefined && + (input as GeoContext).zone !== undefined && + (input as GeoContext).zone !== undefined) || + ((input as GeoContext).lon !== undefined && (input as GeoContext).lat !== undefined) + ) { + const { x, y, zone, radius } = input as GeoContext; + const { lon, lat } = x && y && zone ? convertUTMToWgs84(x, y, zone) : (input as Required>); + + // console.log(convertWgs84ToUTM(lat, lon)); + + return { type: 'Circle', coordinates: (parsePoint([lon, lat]) as Point).coordinates, radius: `${radius ?? ''}` } as unknown as GeoJSON; } - // TODO: Add geojson validation return input as GeoJSON; }; From c62c63a2f69d40f236251505cded0cd765441b25 Mon Sep 17 00:00:00 2001 From: Niv Greenstein Date: Sun, 11 Aug 2024 16:29:14 +0300 Subject: [PATCH 017/262] feat: added missing x,y,zone --- src/common/interfaces.ts | 3 +++ 1 file changed, 3 insertions(+) diff --git a/src/common/interfaces.ts b/src/common/interfaces.ts index d376ec40..671e8293 100644 --- a/src/common/interfaces.ts +++ b/src/common/interfaces.ts @@ -23,6 +23,9 @@ export interface GeoContext { radius?: number; lon?: number; lat?: number; + x?: number; + y?: number; + zone?: number; } export interface Geometry { From 9e18e9d3cb7909dd0886d76dfbd43f54f2388d79 Mon Sep 17 00:00:00 2001 From: Niv Greenstein Date: Sun, 11 Aug 2024 16:29:37 +0300 Subject: [PATCH 018/262] fix: fixed query to support new params --- src/control/tile/DAL/queries.ts | 72 ++++++++++++++++++++++++--------- 1 file changed, 53 insertions(+), 19 deletions(-) diff --git a/src/control/tile/DAL/queries.ts b/src/control/tile/DAL/queries.ts index b04500bb..b02f6e9c 100644 --- a/src/control/tile/DAL/queries.ts +++ b/src/control/tile/DAL/queries.ts @@ -1,11 +1,14 @@ /* eslint-disable @typescript-eslint/naming-convention */ import { estypes } from '@elastic/elasticsearch'; -import { CommonRequestParameters } from '../../../common/interfaces'; +import { CommonRequestParameters, GeoContextMode } from '../../../common/interfaces'; +import { parseGeo } from '../../../geotextSearch/utils'; +import { BadRequestError } from '../../../common/errors'; const ELASTIC_KEYWORDS = { TYPE: 'properties.TYPE.keyword', TILE_NAME: 'properties.TILE_NAME.keyword', SUB_TILE_ID: 'properties.SUB_TILE_ID.keyword', + GEOMETRY: 'geometry', }; export interface TileQueryParams extends CommonRequestParameters { @@ -13,28 +16,59 @@ export interface TileQueryParams extends CommonRequestParameters { subTile?: number; } -export const queryForTiles = (params: Omit): estypes.SearchRequest => ({ - query: { - bool: { - must: [ - { - term: { - [ELASTIC_KEYWORDS.TYPE]: 'TILE', +export const queryForTiles = (params: Omit): estypes.SearchRequest => { + const { tile, geo_context, geo_context_mode, disable_fuzziness } = params; + if ((geo_context_mode !== undefined && geo_context === undefined) || (geo_context_mode === undefined && geo_context !== undefined)) { + throw new BadRequestError('/control/tiles/queryForTiles: geo_context and geo_context_mode must be both defined or both undefined'); + } + + const esQuery: estypes.SearchRequest = { + query: { + bool: { + must: [ + { + term: { + [ELASTIC_KEYWORDS.TYPE]: 'TILE', + }, }, - }, - { - match: { - [ELASTIC_KEYWORDS.TILE_NAME]: { - query: params.tile, - fuzziness: !params.disable_fuzziness ? 1 : undefined, - prefix_length: 1, + { + match: { + [ELASTIC_KEYWORDS.TILE_NAME]: { + query: tile, + fuzziness: disable_fuzziness ? undefined : 1, + prefix_length: 1, + }, }, }, - }, - ], + ], + // filter: [ + // { + // geo_shape: { + // geometry: { + // shape: parseGeo(geo_context!), + // }, + // boost: geo_context_mode === GeoContextMode.BIAS ? 1.1 : 1, //TODO: change magic number + // }, + // }, + // ], + }, }, - }, -}); + }; + + if (geo_context !== undefined) { + esQuery.query!.bool![geo_context_mode === GeoContextMode.FILTER ? 'filter' : 'should'] = [ + { + geo_shape: { + [ELASTIC_KEYWORDS.GEOMETRY]: { + shape: parseGeo(geo_context), + }, + boost: geo_context_mode === GeoContextMode.BIAS ? 1.1 : 1, //TODO: change magic number + }, + }, + ]; + } + return esQuery; +}; export const queryForSubTiles = (params: Required): estypes.SearchRequest => ({ query: { From d30e0467784a17717471eeded2a43b1637b6b1a3 Mon Sep 17 00:00:00 2001 From: Niv Greenstein Date: Sun, 11 Aug 2024 16:30:46 +0300 Subject: [PATCH 019/262] fix: fuzziness is true by default. added ! to if --- src/control/tile/models/tileManager.ts | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/src/control/tile/models/tileManager.ts b/src/control/tile/models/tileManager.ts index 1394053a..b3807492 100644 --- a/src/control/tile/models/tileManager.ts +++ b/src/control/tile/models/tileManager.ts @@ -17,7 +17,7 @@ export class TileManager { @inject(TILE_REPOSITORY_SYMBOL) private readonly tileRepository: TileRepository ) {} - public async getTiles(tileQueryParams: TileQueryParams & CommonRequestParameters): Promise> { + public async getTiles(tileQueryParams: TileQueryParams): Promise> { const { limit, disable_fuzziness: disableFuzziness } = tileQueryParams; let elasticResponse: estypes.SearchResponse | undefined = undefined; @@ -29,7 +29,7 @@ export class TileManager { const formattedResponse = formatResponse(elasticResponse); - if (!disableFuzziness && formattedResponse.features.length > 0) { + if (disableFuzziness && formattedResponse.features.length > 0) { const filterFunction = tileQueryParams.subTile ?? 0 ? (hit: Tile | undefined): hit is Tile => From 01aef2309ce1a13bfb92c6e78827770a90ddb88b Mon Sep 17 00:00:00 2001 From: Niv Greenstein Date: Sun, 11 Aug 2024 17:05:58 +0300 Subject: [PATCH 020/262] fix: changed maximum features in response to 15 --- openapi3.yaml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/openapi3.yaml b/openapi3.yaml index d49288ed..5da7c15c 100644 --- a/openapi3.yaml +++ b/openapi3.yaml @@ -490,7 +490,7 @@ components: schema: default: 5 minimum: 1 - maximum: 100 + maximum: 15 type: "number" geo_context: in: "query" From ea3bcd8a1377e81b406dcfb878aac1b519cb130b Mon Sep 17 00:00:00 2001 From: Niv Greenstein Date: Sun, 11 Aug 2024 17:06:15 +0300 Subject: [PATCH 021/262] feat: added new mock tile --- devScripts/controlElasticsearchData.json | 26 ++++++++++++++++++++++++ 1 file changed, 26 insertions(+) diff --git a/devScripts/controlElasticsearchData.json b/devScripts/controlElasticsearchData.json index 1b4b2654..1cec3dde 100644 --- a/devScripts/controlElasticsearchData.json +++ b/devScripts/controlElasticsearchData.json @@ -137,5 +137,31 @@ "TYPE": "TILE" } } + }, + { + "_id": "CONTROL.TILES1", + "_source": { + "type": "Feature", + "properties": { + "OBJECTID": 53, + "ZONE": "37", + "TILE_NAME": "RIC", + "TILE_ID": null, + "LAYER_NAME": "CONTROL.TILES", + "TYPE": "TILE" + }, + "geometry": { + "coordinates": [ + [ + [12.64750356570994, 41.854073129598774], + [12.64478277570987, 41.94417235759876], + [12.536787035709892, 41.941850298598766], + [12.539620755710018, 41.85175890959876], + [12.64750356570994, 41.854073129598774] + ] + ], + "type": "Polygon" + } + } } ] From c51bbe0e7258d92e13839f066a1d120949adcb41 Mon Sep 17 00:00:00 2001 From: Niv Greenstein Date: Sun, 11 Aug 2024 17:07:17 +0300 Subject: [PATCH 022/262] fix(devscripts): generate id as uuid --- devScripts/importDataToElastic.ts | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/devScripts/importDataToElastic.ts b/devScripts/importDataToElastic.ts index 9eb88f40..849b34c7 100644 --- a/devScripts/importDataToElastic.ts +++ b/devScripts/importDataToElastic.ts @@ -1,6 +1,7 @@ /* eslint-disable @typescript-eslint/no-unsafe-assignment */ /* eslint-disable @typescript-eslint/naming-convention */ /* eslint-disable @typescript-eslint/no-unsafe-member-access */ +import crypto from 'crypto'; import { Client } from '@elastic/elasticsearch'; import config from '../config/default.json'; import controlData from './controlElasticsearchData.json'; @@ -44,7 +45,7 @@ const main = async (): Promise => { for (const item of controlData) { await controlClient.index({ index: config.db.elastic.control.properties.index, - id: item._id, + id: crypto.randomUUID(), body: item._source, }); } @@ -52,7 +53,7 @@ const main = async (): Promise => { for (const item of geotextData) { await geotextClient.index({ index: item._index, - id: item._id, + id: crypto.randomUUID(), body: { ...item._source, }, From 8e7b3a3c84fd028b79149dda24289a44af4f0df2 Mon Sep 17 00:00:00 2001 From: Niv Greenstein Date: Sun, 11 Aug 2024 17:14:10 +0300 Subject: [PATCH 023/262] delete: removed unused import --- src/geotextSearch/utils.ts | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/geotextSearch/utils.ts b/src/geotextSearch/utils.ts index 9b35debd..cea8f1a1 100644 --- a/src/geotextSearch/utils.ts +++ b/src/geotextSearch/utils.ts @@ -5,7 +5,7 @@ import { StatusCodes } from 'http-status-codes'; import axios, { AxiosResponse as Response } from 'axios'; import { InternalServerError } from '../common/errors'; import { GeoContext, IApplication } from '../common/interfaces'; -import { convertUTMToWgs84, convertWgs84ToUTM } from '../common/utils'; +import { convertUTMToWgs84 } from '../common/utils'; import { BBOX_LENGTH, POINT_LENGTH, QueryResult, TextSearchParams } from './interfaces'; import { generateDisplayName } from './parsing'; import { TextSearchHit } from './models/elasticsearchHits'; From c81d273f9282c93ce0fab3febc40caa842ceb154 Mon Sep 17 00:00:00 2001 From: Niv Greenstein Date: Sun, 11 Aug 2024 17:15:00 +0300 Subject: [PATCH 024/262] feat: added missing mgrs property --- src/control/tile/DAL/queries.ts | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/src/control/tile/DAL/queries.ts b/src/control/tile/DAL/queries.ts index b02f6e9c..1213963b 100644 --- a/src/control/tile/DAL/queries.ts +++ b/src/control/tile/DAL/queries.ts @@ -12,7 +12,8 @@ const ELASTIC_KEYWORDS = { }; export interface TileQueryParams extends CommonRequestParameters { - tile: string; + tile?: string; + mgrs?: string; subTile?: number; } From 5b22a0c32b0d28fcc0b47f7d54703a5a6f868515 Mon Sep 17 00:00:00 2001 From: Niv Greenstein Date: Sun, 11 Aug 2024 17:15:27 +0300 Subject: [PATCH 025/262] fix: fixed params type and removed unused code --- src/control/tile/DAL/queries.ts | 14 +++----------- 1 file changed, 3 insertions(+), 11 deletions(-) diff --git a/src/control/tile/DAL/queries.ts b/src/control/tile/DAL/queries.ts index 1213963b..2a09f9ad 100644 --- a/src/control/tile/DAL/queries.ts +++ b/src/control/tile/DAL/queries.ts @@ -17,7 +17,9 @@ export interface TileQueryParams extends CommonRequestParameters { subTile?: number; } -export const queryForTiles = (params: Omit): estypes.SearchRequest => { +export const queryForTiles = ( + params: Omit & Required> +): estypes.SearchRequest => { const { tile, geo_context, geo_context_mode, disable_fuzziness } = params; if ((geo_context_mode !== undefined && geo_context === undefined) || (geo_context_mode === undefined && geo_context !== undefined)) { throw new BadRequestError('/control/tiles/queryForTiles: geo_context and geo_context_mode must be both defined or both undefined'); @@ -42,16 +44,6 @@ export const queryForTiles = (params: Omit }, }, ], - // filter: [ - // { - // geo_shape: { - // geometry: { - // shape: parseGeo(geo_context!), - // }, - // boost: geo_context_mode === GeoContextMode.BIAS ? 1.1 : 1, //TODO: change magic number - // }, - // }, - // ], }, }, }; From bf25f8e4823d9a673e19f14efad655925a49aa8b Mon Sep 17 00:00:00 2001 From: Niv Greenstein Date: Sun, 11 Aug 2024 17:34:28 +0300 Subject: [PATCH 026/262] fix: added missig mgrs --- src/control/tile/controllers/tileController.ts | 6 ++++-- src/control/tile/models/tileManager.ts | 7 ++++++- 2 files changed, 10 insertions(+), 3 deletions(-) diff --git a/src/control/tile/controllers/tileController.ts b/src/control/tile/controllers/tileController.ts index 63e05d50..2f497402 100644 --- a/src/control/tile/controllers/tileController.ts +++ b/src/control/tile/controllers/tileController.ts @@ -21,7 +21,8 @@ type GetTilesHandler = RequestHandler< >; export interface GetTilesQueryParams extends CommonRequestParameters { - tile: string; + tile?: string; + mgrs?: string; sub_tile?: string; } @@ -39,7 +40,7 @@ export class TileController { public getTiles: GetTilesHandler = async (req, res, next) => { try { - const { tile, sub_tile, disable_fuzziness, geo_context, geo_context_mode, limit } = req.query; + const { tile, sub_tile, disable_fuzziness, geo_context, geo_context_mode, limit, mgrs } = req.query; const response = await this.manager.getTiles({ tile, subTile: sub_tile ? parseInt(sub_tile) : undefined, @@ -47,6 +48,7 @@ export class TileController { geo_context, geo_context_mode, limit, + mgrs, }); return res.status(httpStatus.OK).json(response); } catch (error: unknown) { diff --git a/src/control/tile/models/tileManager.ts b/src/control/tile/models/tileManager.ts index b3807492..a96b9b88 100644 --- a/src/control/tile/models/tileManager.ts +++ b/src/control/tile/models/tileManager.ts @@ -6,7 +6,8 @@ import { SERVICES } from '../../../common/constants'; import { TILE_REPOSITORY_SYMBOL, TileRepository } from '../DAL/tileRepository'; import { formatResponse } from '../../../common/utils'; import { TileQueryParams } from '../DAL/queries'; -import { CommonRequestParameters, FeatureCollection } from '../../../common/interfaces'; +import { FeatureCollection } from '../../../common/interfaces'; +import { BadRequestError } from '../../../common/errors'; import { Tile } from './tile'; @injectable() @@ -18,7 +19,11 @@ export class TileManager { ) {} public async getTiles(tileQueryParams: TileQueryParams): Promise> { + //TODO: Handle MGRS query const { limit, disable_fuzziness: disableFuzziness } = tileQueryParams; + if (tileQueryParams.tile === undefined && tileQueryParams.mgrs === undefined) { + throw new BadRequestError('/control/tiles/queryForTiles: tile or mgrs must be defined'); + } let elasticResponse: estypes.SearchResponse | undefined = undefined; if (tileQueryParams.subTile ?? 0) { From 93e1c93421dba6763d032718699235a58a70a53d Mon Sep 17 00:00:00 2001 From: Niv Greenstein Date: Sun, 11 Aug 2024 17:34:56 +0300 Subject: [PATCH 027/262] fix: changed mgrs to lowercase --- openapi3.yaml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/openapi3.yaml b/openapi3.yaml index 5da7c15c..2ced08d1 100644 --- a/openapi3.yaml +++ b/openapi3.yaml @@ -201,7 +201,7 @@ paths: description: "Sub tile number" schema: type: "number" - - name: MGRS + - name: mgrs description: "1 meters MGRS Tile" example: "18SUJ2338907395" in: query From e8d48edb89e93e5ce48343136464c03ec4eff60e Mon Sep 17 00:00:00 2001 From: Niv Greenstein Date: Sun, 11 Aug 2024 17:35:06 +0300 Subject: [PATCH 028/262] feat: added NotImplementedError --- src/common/errors.ts | 8 ++++++++ 1 file changed, 8 insertions(+) diff --git a/src/common/errors.ts b/src/common/errors.ts index c93dc054..8f3cf50f 100644 --- a/src/common/errors.ts +++ b/src/common/errors.ts @@ -33,3 +33,11 @@ export class NotFoundError extends Error implements HttpError { super(message); } } + +export class NotImplementedError extends Error implements HttpError { + public readonly status = httpStatus.NOT_IMPLEMENTED; + + public constructor(message: string) { + super(message); + } +} From 72bca0e0046b71e2b83f6bc5b90485579c102b7b Mon Sep 17 00:00:00 2001 From: Niv Greenstein Date: Sun, 11 Aug 2024 17:35:30 +0300 Subject: [PATCH 029/262] fix(tileRepository): fixed requested type --- src/control/tile/DAL/tileRepository.ts | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/control/tile/DAL/tileRepository.ts b/src/control/tile/DAL/tileRepository.ts index 860f6a3e..29a8bc7d 100644 --- a/src/control/tile/DAL/tileRepository.ts +++ b/src/control/tile/DAL/tileRepository.ts @@ -12,7 +12,7 @@ import { queryForTiles, queryForSubTiles, TileQueryParams } from './queries'; const createTileRepository = (client: ElasticClient, config: IConfig, logger: Logger) => { return { // eslint-disable-next-line @typescript-eslint/explicit-module-boundary-types - async getTiles(tileQueryParams: TileQueryParams, size: number): Promise> { + async getTiles(tileQueryParams: TileQueryParams & Required>, size: number): Promise> { const response = await queryElastic(client, { ...additionalControlSearchProperties(config, size), ...queryForTiles(tileQueryParams) }); return response; From 7ce6ca7d20ff54e20cd4410b076af7f169d700ee Mon Sep 17 00:00:00 2001 From: Niv Greenstein Date: Sun, 11 Aug 2024 17:36:59 +0300 Subject: [PATCH 030/262] fix(tileManager): throw BadRequestError if both tile and mgrs are not provided --- src/control/tile/models/tileManager.ts | 9 +++++++-- 1 file changed, 7 insertions(+), 2 deletions(-) diff --git a/src/control/tile/models/tileManager.ts b/src/control/tile/models/tileManager.ts index a96b9b88..b4430bde 100644 --- a/src/control/tile/models/tileManager.ts +++ b/src/control/tile/models/tileManager.ts @@ -21,8 +21,13 @@ export class TileManager { public async getTiles(tileQueryParams: TileQueryParams): Promise> { //TODO: Handle MGRS query const { limit, disable_fuzziness: disableFuzziness } = tileQueryParams; - if (tileQueryParams.tile === undefined && tileQueryParams.mgrs === undefined) { - throw new BadRequestError('/control/tiles/queryForTiles: tile or mgrs must be defined'); + + if ( + (tileQueryParams.tile === undefined && tileQueryParams.mgrs === undefined) || + (tileQueryParams.tile !== undefined && tileQueryParams.mgrs !== undefined) + ) { + throw new BadRequestError("/control/tiles/queryForTiles: only one of 'tile' or 'mgrs' query parameter must be defined"); + } } let elasticResponse: estypes.SearchResponse | undefined = undefined; From 0a7e459cbc483870c218c68594a7244a2d50a605 Mon Sep 17 00:00:00 2001 From: Niv Greenstein Date: Sun, 11 Aug 2024 17:38:00 +0300 Subject: [PATCH 031/262] fix(tileManager): throw NotImplementedError if mgrs is provided --- src/control/tile/models/tileManager.ts | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/src/control/tile/models/tileManager.ts b/src/control/tile/models/tileManager.ts index b4430bde..f8be8f2d 100644 --- a/src/control/tile/models/tileManager.ts +++ b/src/control/tile/models/tileManager.ts @@ -28,6 +28,10 @@ export class TileManager { ) { throw new BadRequestError("/control/tiles/queryForTiles: only one of 'tile' or 'mgrs' query parameter must be defined"); } + + //TODO: Handle MGRS query + if (tileQueryParams.mgrs !== undefined) { + throw new NotImplementedError('MGRS query is not implemented yet'); } let elasticResponse: estypes.SearchResponse | undefined = undefined; From cc32252af8c7b6d2e4111162ceb6b24db72a7c39 Mon Sep 17 00:00:00 2001 From: Niv Greenstein Date: Sun, 11 Aug 2024 17:38:29 +0300 Subject: [PATCH 032/262] fix(tileManager): throw BadRequestError tile is undefined but tries to query it --- src/control/tile/models/tileManager.ts | 9 ++++++--- 1 file changed, 6 insertions(+), 3 deletions(-) diff --git a/src/control/tile/models/tileManager.ts b/src/control/tile/models/tileManager.ts index f8be8f2d..44e44bae 100644 --- a/src/control/tile/models/tileManager.ts +++ b/src/control/tile/models/tileManager.ts @@ -7,7 +7,7 @@ import { TILE_REPOSITORY_SYMBOL, TileRepository } from '../DAL/tileRepository'; import { formatResponse } from '../../../common/utils'; import { TileQueryParams } from '../DAL/queries'; import { FeatureCollection } from '../../../common/interfaces'; -import { BadRequestError } from '../../../common/errors'; +import { BadRequestError, NotImplementedError } from '../../../common/errors'; import { Tile } from './tile'; @injectable() @@ -19,7 +19,6 @@ export class TileManager { ) {} public async getTiles(tileQueryParams: TileQueryParams): Promise> { - //TODO: Handle MGRS query const { limit, disable_fuzziness: disableFuzziness } = tileQueryParams; if ( @@ -35,10 +34,14 @@ export class TileManager { } let elasticResponse: estypes.SearchResponse | undefined = undefined; + if (tileQueryParams.tile === undefined) { + throw new BadRequestError('/control/tiles/queryForTiles: tile must be defined'); + } + if (tileQueryParams.subTile ?? 0) { elasticResponse = await this.tileRepository.getSubTiles(tileQueryParams as Required, limit); } else { - elasticResponse = await this.tileRepository.getTiles(tileQueryParams, limit); + elasticResponse = await this.tileRepository.getTiles(tileQueryParams as TileQueryParams & Required>, limit); } const formattedResponse = formatResponse(elasticResponse); From ab963287b9106faabe6c8f7f53a49fc91a11f182 Mon Sep 17 00:00:00 2001 From: Niv Greenstein Date: Sun, 11 Aug 2024 19:25:53 +0300 Subject: [PATCH 033/262] feat(control/utils): extracted logic of geoContext query to a common reusable function --- src/control/utils.ts | 28 ++++++++++++++++++++++++++++ 1 file changed, 28 insertions(+) create mode 100644 src/control/utils.ts diff --git a/src/control/utils.ts b/src/control/utils.ts new file mode 100644 index 00000000..b854c9c0 --- /dev/null +++ b/src/control/utils.ts @@ -0,0 +1,28 @@ +import { estypes } from '@elastic/elasticsearch'; +import { parseGeo } from '../geotextSearch/utils'; +import { GeoContext, GeoContextMode } from '../common/interfaces'; +import { BadRequestError } from '../common/errors'; +import { ELASTIC_KEYWORDS } from './constants'; + +export const geoContextQuery = (geoContext?: GeoContext, geoContextMode?: GeoContextMode): estypes.SearchRequest => { + if (geoContext === undefined && geoContextMode === undefined) { + return {}; + } + if ((geoContext !== undefined && geoContextMode === undefined) || (geoContext === undefined && geoContextMode !== undefined)) { + throw new BadRequestError('/control/tiles/queryForTiles: geo_context and geo_context_mode must be both defined or both undefined'); + } + + return { + [geoContextMode === GeoContextMode.FILTER ? 'filter' : 'should']: [ + { + // eslint-disable-next-line @typescript-eslint/naming-convention + geo_shape: { + [ELASTIC_KEYWORDS.geometry]: { + shape: parseGeo(geoContext!), + }, + boost: geoContextMode === GeoContextMode.BIAS ? 1.1 : 1, //TODO: change magic number + }, + }, + ], + }; +}; From 176fb1a8f2d79d34cfcacacf000fd46f5ad93a6b Mon Sep 17 00:00:00 2001 From: Niv Greenstein Date: Sun, 11 Aug 2024 19:26:29 +0300 Subject: [PATCH 034/262] feat(control/constants): moved CONTROL_FIELDS to this file and created ELASTIC_KEYWORDS as exported const --- src/control/constants.ts | 18 ++++++++++++++++++ 1 file changed, 18 insertions(+) create mode 100644 src/control/constants.ts diff --git a/src/control/constants.ts b/src/control/constants.ts new file mode 100644 index 00000000..0f5a8fd7 --- /dev/null +++ b/src/control/constants.ts @@ -0,0 +1,18 @@ +export const ELASTIC_KEYWORDS = { + type: 'properties.TYPE.keyword', + tileName: 'properties.TILE_NAME.keyword', + subTileId: 'properties.SUB_TILE_ID.keyword', + geometry: 'geometry', + objectCommandName: 'properties.OBJECT_COMMAND_NAME.keyword', +}; + +export const CONTROL_FIELDS = [ + 'type', + 'geometry', + 'properties.OBJECT_COMMAND_NAME', + 'properties.TILE_NAME', + 'properties.TYPE', + 'properties.ENTITY_HEB', + 'properties.SUB_TILE_ID', + 'properties.SECTION', +]; From 80667da05b11fb7f1c86e1cdb480e9dad1df3b5d Mon Sep 17 00:00:00 2001 From: Niv Greenstein Date: Sun, 11 Aug 2024 19:29:18 +0300 Subject: [PATCH 035/262] fix(control/utils): moved additionalControlSearchProperties to this file --- src/control/utils.ts | 14 ++++++++++++-- 1 file changed, 12 insertions(+), 2 deletions(-) diff --git a/src/control/utils.ts b/src/control/utils.ts index b854c9c0..c697c5af 100644 --- a/src/control/utils.ts +++ b/src/control/utils.ts @@ -1,8 +1,10 @@ import { estypes } from '@elastic/elasticsearch'; import { parseGeo } from '../geotextSearch/utils'; -import { GeoContext, GeoContextMode } from '../common/interfaces'; +import { GeoContext, GeoContextMode, IConfig } from '../common/interfaces'; import { BadRequestError } from '../common/errors'; -import { ELASTIC_KEYWORDS } from './constants'; +import { elasticConfigPath } from '../common/constants'; +import { ElasticDbClientsConfig } from '../common/elastic/interfaces'; +import { CONTROL_FIELDS, ELASTIC_KEYWORDS } from './constants'; export const geoContextQuery = (geoContext?: GeoContext, geoContextMode?: GeoContextMode): estypes.SearchRequest => { if (geoContext === undefined && geoContextMode === undefined) { @@ -26,3 +28,11 @@ export const geoContextQuery = (geoContext?: GeoContext, geoContextMode?: GeoCon ], }; }; + +/* eslint-disable @typescript-eslint/naming-convention */ +export const additionalControlSearchProperties = (config: IConfig, size: number): { size: number; index: string; _source: string[] } => ({ + size, + index: config.get(elasticConfigPath).control.properties.index as string, + _source: CONTROL_FIELDS, +}); +/* eslint-enable @typescript-eslint/naming-convention */ From 85c1342c6f9c7e4852706099995d5446402786cf Mon Sep 17 00:00:00 2001 From: Niv Greenstein Date: Sun, 11 Aug 2024 19:29:36 +0300 Subject: [PATCH 036/262] delete(elastic/utils): delete additionalControlSearchProperties --- src/common/elastic/utils.ts | 11 ++--------- 1 file changed, 2 insertions(+), 9 deletions(-) diff --git a/src/common/elastic/utils.ts b/src/common/elastic/utils.ts index b0a59c76..31bf4b20 100644 --- a/src/common/elastic/utils.ts +++ b/src/common/elastic/utils.ts @@ -1,18 +1,11 @@ import { estypes } from '@elastic/elasticsearch'; import { IConfig, WGS84Coordinate } from '../interfaces'; import { InternalServerError } from '../errors'; -import { elasticConfigPath, CONTROL_FIELDS } from '../constants'; +import { elasticConfigPath } from '../constants'; +import { CONTROL_FIELDS } from '../../control/constants'; import { ElasticDbClientsConfig } from './interfaces'; import { ElasticClient } from './index'; -/* eslint-disable @typescript-eslint/naming-convention */ -export const additionalControlSearchProperties = (config: IConfig, size: number): { size: number; index: string; _source: string[] } => ({ - size, - index: config.get(elasticConfigPath).control.properties.index as string, - _source: CONTROL_FIELDS, -}); -/* eslint-enable @typescript-eslint/naming-convention */ - export const queryElastic = async (client: ElasticClient, body: estypes.SearchRequest): Promise> => { const clientNotAvailableError = new InternalServerError('Elasticsearch client is not available'); try { From 0777861f8ff6381ccb962796e322505437136cb1 Mon Sep 17 00:00:00 2001 From: Niv Greenstein Date: Sun, 11 Aug 2024 19:29:57 +0300 Subject: [PATCH 037/262] delete(common/constants): removed CONTROL_FIELDS --- src/common/constants.ts | 11 ----------- 1 file changed, 11 deletions(-) diff --git a/src/common/constants.ts b/src/common/constants.ts index f5cf88cd..5ab1d8b1 100644 --- a/src/common/constants.ts +++ b/src/common/constants.ts @@ -20,15 +20,4 @@ export const SERVICES: Record = { export const ON_SIGNAL = Symbol('onSignal'); export const HEALTHCHECK = Symbol('healthcheck'); -export const CONTROL_FIELDS = [ - 'type', - 'geometry', - 'properties.OBJECT_COMMAND_NAME', - 'properties.TILE_NAME', - 'properties.TYPE', - 'properties.ENTITY_HEB', - 'properties.SUB_TILE_ID', - 'properties.SECTION', -]; - export const elasticConfigPath = 'db.elastic'; From a9daf2cf1cf020975f54e6ee432b658f032e64e4 Mon Sep 17 00:00:00 2001 From: Niv Greenstein Date: Sun, 11 Aug 2024 19:33:45 +0300 Subject: [PATCH 038/262] feat: use geoContextQuery in all queries. use Elastic_keywords in subtile query. --- src/control/tile/DAL/queries.ts | 95 ++++++++++++++------------------- 1 file changed, 39 insertions(+), 56 deletions(-) diff --git a/src/control/tile/DAL/queries.ts b/src/control/tile/DAL/queries.ts index 2a09f9ad..c17ed6c5 100644 --- a/src/control/tile/DAL/queries.ts +++ b/src/control/tile/DAL/queries.ts @@ -1,15 +1,7 @@ -/* eslint-disable @typescript-eslint/naming-convention */ import { estypes } from '@elastic/elasticsearch'; -import { CommonRequestParameters, GeoContextMode } from '../../../common/interfaces'; -import { parseGeo } from '../../../geotextSearch/utils'; -import { BadRequestError } from '../../../common/errors'; - -const ELASTIC_KEYWORDS = { - TYPE: 'properties.TYPE.keyword', - TILE_NAME: 'properties.TILE_NAME.keyword', - SUB_TILE_ID: 'properties.SUB_TILE_ID.keyword', - GEOMETRY: 'geometry', -}; +import { CommonRequestParameters } from '../../../common/interfaces'; +import { ELASTIC_KEYWORDS } from '../../constants'; +import { geoContextQuery } from '../../utils'; export interface TileQueryParams extends CommonRequestParameters { tile?: string; @@ -17,72 +9,63 @@ export interface TileQueryParams extends CommonRequestParameters { subTile?: number; } -export const queryForTiles = ( - params: Omit & Required> -): estypes.SearchRequest => { - const { tile, geo_context, geo_context_mode, disable_fuzziness } = params; - if ((geo_context_mode !== undefined && geo_context === undefined) || (geo_context_mode === undefined && geo_context !== undefined)) { - throw new BadRequestError('/control/tiles/queryForTiles: geo_context and geo_context_mode must be both defined or both undefined'); - } - - const esQuery: estypes.SearchRequest = { - query: { - bool: { - must: [ - { - term: { - [ELASTIC_KEYWORDS.TYPE]: 'TILE', - }, +export const queryForTiles = ({ + tile, + geo_context: geoContext, + geo_context_mode: geoContextMode, + disable_fuzziness: disableFuzziness, +}: Omit & Required>): estypes.SearchRequest => ({ + query: { + bool: { + must: [ + { + term: { + [ELASTIC_KEYWORDS.type]: 'TILE', }, - { - match: { - [ELASTIC_KEYWORDS.TILE_NAME]: { - query: tile, - fuzziness: disable_fuzziness ? undefined : 1, - prefix_length: 1, - }, + }, + { + match: { + [ELASTIC_KEYWORDS.tileName]: { + query: tile, + fuzziness: disableFuzziness ? undefined : 1, + // eslint-disable-next-line @typescript-eslint/naming-convention + prefix_length: 1, }, }, - ], - }, - }, - }; - - if (geo_context !== undefined) { - esQuery.query!.bool![geo_context_mode === GeoContextMode.FILTER ? 'filter' : 'should'] = [ - { - geo_shape: { - [ELASTIC_KEYWORDS.GEOMETRY]: { - shape: parseGeo(geo_context), - }, - boost: geo_context_mode === GeoContextMode.BIAS ? 1.1 : 1, //TODO: change magic number }, - }, - ]; - } - return esQuery; -}; + ], + ...geoContextQuery(geoContext, geoContextMode), + }, + }, +}); -export const queryForSubTiles = (params: Required): estypes.SearchRequest => ({ +export const queryForSubTiles = ({ + tile, + geo_context: geoContext, + geo_context_mode: geoContextMode, + disable_fuzziness: disableFuzziness, + subTile, +}: Required): estypes.SearchRequest => ({ query: { bool: { must: [ { term: { - [ELASTIC_KEYWORDS.TYPE]: 'SUB_TILE', + [ELASTIC_KEYWORDS.type]: 'SUB_TILE', }, }, { term: { - [ELASTIC_KEYWORDS.TILE_NAME]: params.tile, + [ELASTIC_KEYWORDS.tileName]: tile, }, }, { match: { - [ELASTIC_KEYWORDS.SUB_TILE_ID]: params.subTile, + [ELASTIC_KEYWORDS.subTileId]: subTile, }, }, ], + ...geoContextQuery(geoContext, geoContextMode), }, }, }); From f03c6667225c18cc2522cebb7a94a89e23006657 Mon Sep 17 00:00:00 2001 From: Niv Greenstein Date: Sun, 11 Aug 2024 19:34:05 +0300 Subject: [PATCH 039/262] fix(tileRepository): updated imports --- src/control/tile/DAL/tileRepository.ts | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/src/control/tile/DAL/tileRepository.ts b/src/control/tile/DAL/tileRepository.ts index 29a8bc7d..c7f764ab 100644 --- a/src/control/tile/DAL/tileRepository.ts +++ b/src/control/tile/DAL/tileRepository.ts @@ -3,9 +3,10 @@ import { estypes } from '@elastic/elasticsearch'; import { FactoryFunction } from 'tsyringe'; import { ElasticClient, ElasticClients } from '../../../common/elastic'; import { Tile } from '../models/tile'; -import { additionalControlSearchProperties, queryElastic } from '../../../common/elastic/utils'; +import { queryElastic } from '../../../common/elastic/utils'; import { IConfig } from '../../../common/interfaces'; import { SERVICES } from '../../../common/constants'; +import { additionalControlSearchProperties } from '../../utils'; import { queryForTiles, queryForSubTiles, TileQueryParams } from './queries'; // eslint-disable-next-line @typescript-eslint/explicit-function-return-type From e7e1c30bfb431475f3bb7d8a6bd43886a86664a5 Mon Sep 17 00:00:00 2001 From: Niv Greenstein Date: Sun, 11 Aug 2024 20:01:26 +0300 Subject: [PATCH 040/262] fix(control/utils): fixed returned type of geoContextQuery --- src/control/utils.ts | 5 ++++- 1 file changed, 4 insertions(+), 1 deletion(-) diff --git a/src/control/utils.ts b/src/control/utils.ts index c697c5af..ff752582 100644 --- a/src/control/utils.ts +++ b/src/control/utils.ts @@ -6,7 +6,10 @@ import { elasticConfigPath } from '../common/constants'; import { ElasticDbClientsConfig } from '../common/elastic/interfaces'; import { CONTROL_FIELDS, ELASTIC_KEYWORDS } from './constants'; -export const geoContextQuery = (geoContext?: GeoContext, geoContextMode?: GeoContextMode): estypes.SearchRequest => { +export const geoContextQuery = ( + geoContext?: GeoContext, + geoContextMode?: GeoContextMode +): { [key in 'filter' | 'should']?: estypes.QueryDslQueryContainer[] } => { if (geoContext === undefined && geoContextMode === undefined) { return {}; } From 6ef58439decb9096ed7009d430cbca32484dc3f1 Mon Sep 17 00:00:00 2001 From: Niv Greenstein Date: Sun, 11 Aug 2024 20:01:50 +0300 Subject: [PATCH 041/262] feat(control/constants): added layerName and tiedTo to ELASTIC_KEYWORDS --- src/control/constants.ts | 2 ++ 1 file changed, 2 insertions(+) diff --git a/src/control/constants.ts b/src/control/constants.ts index 0f5a8fd7..1b8e99e9 100644 --- a/src/control/constants.ts +++ b/src/control/constants.ts @@ -4,6 +4,8 @@ export const ELASTIC_KEYWORDS = { subTileId: 'properties.SUB_TILE_ID.keyword', geometry: 'geometry', objectCommandName: 'properties.OBJECT_COMMAND_NAME.keyword', + layerName: 'properties.LAYER_NAME.keyword', + tiedTo: 'properties.TIED_TO', }; export const CONTROL_FIELDS = [ From 2aaa190de2e245e02b22c46cd4494c6c59e289ab Mon Sep 17 00:00:00 2001 From: Niv Greenstein Date: Sun, 11 Aug 2024 20:02:14 +0300 Subject: [PATCH 042/262] delete(elastic/utils): removed unused imports --- src/common/elastic/utils.ts | 5 +---- 1 file changed, 1 insertion(+), 4 deletions(-) diff --git a/src/common/elastic/utils.ts b/src/common/elastic/utils.ts index 31bf4b20..61931d70 100644 --- a/src/common/elastic/utils.ts +++ b/src/common/elastic/utils.ts @@ -1,9 +1,6 @@ import { estypes } from '@elastic/elasticsearch'; -import { IConfig, WGS84Coordinate } from '../interfaces'; +import { WGS84Coordinate } from '../interfaces'; import { InternalServerError } from '../errors'; -import { elasticConfigPath } from '../constants'; -import { CONTROL_FIELDS } from '../../control/constants'; -import { ElasticDbClientsConfig } from './interfaces'; import { ElasticClient } from './index'; export const queryElastic = async (client: ElasticClient, body: estypes.SearchRequest): Promise> => { From f76a470f7490a19abe974584c8e519a88ed06cfd Mon Sep 17 00:00:00 2001 From: Niv Greenstein Date: Sun, 11 Aug 2024 20:03:17 +0300 Subject: [PATCH 043/262] feat(itemQuery): changed query to new parameters --- src/control/item/DAL/queries.ts | 50 +++++++++++++++------------------ 1 file changed, 22 insertions(+), 28 deletions(-) diff --git a/src/control/item/DAL/queries.ts b/src/control/item/DAL/queries.ts index 95911a27..53713ffc 100644 --- a/src/control/item/DAL/queries.ts +++ b/src/control/item/DAL/queries.ts @@ -1,66 +1,60 @@ import { estypes } from '@elastic/elasticsearch'; -import { boundingBox, geoDistance } from '../../../common/elastic/utils'; -import { GeoContext } from '../../../common/interfaces'; +import { CommonRequestParameters, GeoContext } from '../../../common/interfaces'; +import { ELASTIC_KEYWORDS } from '../../constants'; +import { geoContextQuery } from '../../utils'; -export interface ItemQueryParams { +export interface ItemQueryParams extends CommonRequestParameters { commandName: string; tile?: string; subTile?: number; geo?: GeoContext; } - /* eslint-disable @typescript-eslint/naming-convention */ -export const queryForItems = (params: ItemQueryParams): estypes.SearchRequest => ({ +export const queryForItems = ({ + commandName, + tile, + subTile, + geo_context: geoContext, + geo_context_mode: geoContextMode, + disable_fuzziness: disableFuzziness, +}: ItemQueryParams): estypes.SearchRequest => ({ query: { bool: { - should: [ + must: [ { term: { - 'properties.TYPE.keyword': 'ITEM', + [ELASTIC_KEYWORDS.type]: 'ITEM', }, }, - ], - must: [ { match: { - 'properties.OBJECT_COMMAND_NAME.keyword': { - query: params.commandName, - fuzziness: 1, + [ELASTIC_KEYWORDS.objectCommandName]: { + query: commandName, + fuzziness: disableFuzziness ? undefined : 1, prefix_length: 1, }, }, }, - ...(params.tile ?? '' + ...(tile ?? '' ? [ { term: { - 'properties.TILE_NAME.keyword': params.tile, + [ELASTIC_KEYWORDS.tileName]: tile, }, }, ] : []), - ...(params.subTile ?? 0 + ...(subTile ?? 0 ? [ { term: { - 'properties.SUB_TILE_ID.keyword': params.subTile, + [ELASTIC_KEYWORDS.subTileId]: subTile, }, }, ] : []), - ...(params.geo?.bbox ? [boundingBox(params.geo.bbox)] : []), - ...((params.geo?.radius ?? 0) && (params.geo?.lat ?? 0) && (params.geo?.lon ?? 0) - ? [ - geoDistance( - params.geo as { - radius: number; - lon: number; - lat: number; - } - ), - ] - : []), ], + ...geoContextQuery(geoContext, geoContextMode), }, }, }); From 41c202b2ae524e73401e9251c2745f73e190d5fd Mon Sep 17 00:00:00 2001 From: Niv Greenstein Date: Sun, 11 Aug 2024 20:03:32 +0300 Subject: [PATCH 044/262] fix(itemRepository): updated import --- src/control/item/DAL/itemRepository.ts | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/src/control/item/DAL/itemRepository.ts b/src/control/item/DAL/itemRepository.ts index bf2a553f..f384e778 100644 --- a/src/control/item/DAL/itemRepository.ts +++ b/src/control/item/DAL/itemRepository.ts @@ -3,10 +3,11 @@ import { estypes } from '@elastic/elasticsearch'; import { FactoryFunction } from 'tsyringe'; import { IConfig } from '../../../common/interfaces'; import { ElasticClients } from '../../../common/elastic'; -import { additionalControlSearchProperties, queryElastic } from '../../../common/elastic/utils'; +import { queryElastic } from '../../../common/elastic/utils'; import { ElasticClient } from '../../../common/elastic'; import { Item } from '../models/item'; import { SERVICES } from '../../../common/constants'; +import { additionalControlSearchProperties } from '../../utils'; import { ItemQueryParams, queryForItems } from './queries'; /* eslint-enable @typescript-eslint/naming-convention */ From 87c6516b83c3b2bd7115b5959257533dfc6ae139 Mon Sep 17 00:00:00 2001 From: Niv Greenstein Date: Sun, 11 Aug 2024 20:03:51 +0300 Subject: [PATCH 045/262] fix(itemManager): updated manager --- src/control/item/models/itemManager.ts | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/src/control/item/models/itemManager.ts b/src/control/item/models/itemManager.ts index 5b5e4a38..9ab47236 100644 --- a/src/control/item/models/itemManager.ts +++ b/src/control/item/models/itemManager.ts @@ -7,7 +7,6 @@ import { ITEM_REPOSITORY_SYMBOL, ItemRepository } from '../DAL/itemRepository'; import { ItemQueryParams } from '../DAL/queries'; import { formatResponse } from '../../../common/utils'; import { FeatureCollection } from '../../../common/interfaces'; -import { getElasticClientQuerySize } from '../../../common/elastic/utils'; import { Item } from './item'; @injectable() @@ -18,13 +17,14 @@ export class ItemManager { @inject(ITEM_REPOSITORY_SYMBOL) private readonly itemRepository: ItemRepository ) {} - public async getItems(itemQueryParams: ItemQueryParams, reduceFuzzyMatch = false, size?: number): Promise> { + public async getItems(itemQueryParams: ItemQueryParams): Promise> { + const { limit, disable_fuzziness: disableFuzziness } = itemQueryParams; let elasticResponse: estypes.SearchResponse | undefined = undefined; - elasticResponse = await this.itemRepository.getItems(itemQueryParams, size ?? getElasticClientQuerySize(this.config, 'control')); + elasticResponse = await this.itemRepository.getItems(itemQueryParams, limit); const formattedResponse = formatResponse(elasticResponse); - if (reduceFuzzyMatch && formattedResponse.features.length > 0) { + if (disableFuzziness && formattedResponse.features.length > 0) { const filterFunction = (hit: Item | undefined): hit is Item => hit?.properties?.OBJECT_COMMAND_NAME === itemQueryParams.commandName; formattedResponse.features = formattedResponse.features.filter(filterFunction); } From 4d5d951707860bf9b879ff100b0a1667542f9b3a Mon Sep 17 00:00:00 2001 From: Niv Greenstein Date: Sun, 11 Aug 2024 20:04:16 +0300 Subject: [PATCH 046/262] fix(itemController): updated controller based on new openapi --- .../item/controllers/itemController.ts | 27 ++++++++----------- 1 file changed, 11 insertions(+), 16 deletions(-) diff --git a/src/control/item/controllers/itemController.ts b/src/control/item/controllers/itemController.ts index e01e0c62..97b581d5 100644 --- a/src/control/item/controllers/itemController.ts +++ b/src/control/item/controllers/itemController.ts @@ -7,17 +7,14 @@ import { injectable, inject } from 'tsyringe'; import { SERVICES } from '../../../common/constants'; import { ItemManager } from '../models/itemManager'; import { Item } from '../models/item'; -import { FeatureCollection, GeoContext } from '../../../common/interfaces'; +import { CommonRequestParameters, FeatureCollection, GeoContext } from '../../../common/interfaces'; type GetItemsHandler = RequestHandler, undefined, GetItemsQueryParams>; -export interface GetItemsQueryParams { +export interface GetItemsQueryParams extends CommonRequestParameters { command_name: string; tile?: string; sub_tile?: string; - geo_context?: string; - reduce_fuzzy_match?: string; - size?: string; } @injectable() @@ -34,17 +31,15 @@ export class ItemController { public getItems: GetItemsHandler = async (req, res, next) => { try { - const { command_name: commandName, tile, sub_tile, geo_context, reduce_fuzzy_match, size } = req.query; - const response = await this.manager.getItems( - { - commandName, - tile, - subTile: sub_tile ? parseInt(sub_tile) : undefined, - geo: geo_context ? (JSON.parse(geo_context) as GeoContext) : undefined, - }, - reduce_fuzzy_match == 'true', - size ? parseInt(size) : undefined - ); + const { command_name: commandName, tile, sub_tile, geo_context, disable_fuzziness, limit } = req.query; + const response = await this.manager.getItems({ + tile, + subTile: sub_tile ? parseInt(sub_tile) : undefined, + commandName, + geo: geo_context, + limit, + disable_fuzziness, + }); return res.status(httpStatus.OK).json(response); } catch (error: unknown) { this.logger.warn('itemController.getItems Error:', error); From 2528bd86a7d34c44bc53d04f9561cff2c5b6b272 Mon Sep 17 00:00:00 2001 From: Niv Greenstein Date: Sun, 11 Aug 2024 20:04:33 +0300 Subject: [PATCH 047/262] feat(routeController): updated controller based on new openapi --- .../route/controllers/routeController.ts | 25 ++++++++----------- 1 file changed, 10 insertions(+), 15 deletions(-) diff --git a/src/control/route/controllers/routeController.ts b/src/control/route/controllers/routeController.ts index 4cae758a..785d496f 100644 --- a/src/control/route/controllers/routeController.ts +++ b/src/control/route/controllers/routeController.ts @@ -7,16 +7,13 @@ import { injectable, inject } from 'tsyringe'; import { SERVICES } from '../../../common/constants'; import { RouteManager } from '../models/routeManager'; import { Route } from '../models/route'; -import { FeatureCollection, GeoContext } from '../../../common/interfaces'; +import { CommonRequestParameters, FeatureCollection, GeoContext } from '../../../common/interfaces'; type GetRoutesHandler = RequestHandler, undefined, GetRoutesQueryParams>; -export interface GetRoutesQueryParams { +export interface GetRoutesQueryParams extends CommonRequestParameters { command_name: string; control_point?: string; - geo_context?: string; - reduce_fuzzy_match?: string; - size?: string; } @injectable() @@ -33,16 +30,14 @@ export class RouteController { public getRoutes: GetRoutesHandler = async (req, res, next) => { try { - const { command_name: commandName, control_point, geo_context, reduce_fuzzy_match, size } = req.query; - const response = await this.manager.getRoutes( - { - commandName, - controlPoint: control_point ? parseInt(control_point) : undefined, - geo: geo_context ? (JSON.parse(geo_context) as GeoContext) : undefined, - }, - reduce_fuzzy_match == 'true', - size ? parseInt(size) : undefined - ); + const { command_name: commandName, control_point, geo_context, disable_fuzziness, limit } = req.query; + const response = await this.manager.getRoutes({ + commandName, + controlPoint: control_point ? parseInt(control_point) : undefined, + geo_context, + disable_fuzziness, + limit, + }); return res.status(httpStatus.OK).json(response); } catch (error: unknown) { this.logger.warn('routeController.getRoutes Error:', error); From 23c8d8140dcd1891a142ec4f01fbb38ea0c9a8f2 Mon Sep 17 00:00:00 2001 From: Niv Greenstein Date: Sun, 11 Aug 2024 20:04:54 +0300 Subject: [PATCH 048/262] fix(routeManager): updated imports based on types --- src/control/route/models/routeManager.ts | 11 ++++++----- 1 file changed, 6 insertions(+), 5 deletions(-) diff --git a/src/control/route/models/routeManager.ts b/src/control/route/models/routeManager.ts index 53d338a8..89f35548 100644 --- a/src/control/route/models/routeManager.ts +++ b/src/control/route/models/routeManager.ts @@ -7,7 +7,6 @@ import { ROUTE_REPOSITORY_SYMBOL, RouteRepository } from '../DAL/routeRepository import { RouteQueryParams } from '../DAL/queries'; import { formatResponse } from '../../../common/utils'; import { FeatureCollection } from '../../../common/interfaces'; -import { getElasticClientQuerySize } from '../../../common/elastic/utils'; import { Route } from './route'; @injectable() @@ -18,20 +17,22 @@ export class RouteManager { @inject(ROUTE_REPOSITORY_SYMBOL) private readonly routeRepository: RouteRepository ) {} - public async getRoutes(routeQueryParams: RouteQueryParams, reduceFuzzyMatch = false, size?: number): Promise> { + public async getRoutes(routeQueryParams: RouteQueryParams): Promise> { + const { limit, disable_fuzziness: disableFuzziness } = routeQueryParams; + let elasticResponse: estypes.SearchResponse | undefined = undefined; if (routeQueryParams.controlPoint ?? 0) { elasticResponse = await this.routeRepository.getControlPointInRoute( routeQueryParams as RouteQueryParams & Required>, - size ?? getElasticClientQuerySize(this.config, 'control') + limit ); } else { - elasticResponse = await this.routeRepository.getRoutes(routeQueryParams, size ?? getElasticClientQuerySize(this.config, 'control')); + elasticResponse = await this.routeRepository.getRoutes(routeQueryParams, limit); } const formattedResponse = formatResponse(elasticResponse); - if (reduceFuzzyMatch && formattedResponse.features.length > 0) { + if (disableFuzziness && formattedResponse.features.length > 0) { const filterFunction = routeQueryParams.controlPoint ?? 0 ? (hit: Route | undefined): hit is Route => hit?.properties?.OBJECT_COMMAND_NAME === routeQueryParams.controlPoint From 61c1697acec6fd8ee2f2ac15cd56740847764b3d Mon Sep 17 00:00:00 2001 From: Niv Greenstein Date: Sun, 11 Aug 2024 20:05:13 +0300 Subject: [PATCH 049/262] fix(routeRepository): updated import --- src/control/route/DAL/routeRepository.ts | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/src/control/route/DAL/routeRepository.ts b/src/control/route/DAL/routeRepository.ts index 36cdf773..d1bfd737 100644 --- a/src/control/route/DAL/routeRepository.ts +++ b/src/control/route/DAL/routeRepository.ts @@ -6,7 +6,8 @@ import { SERVICES } from '../../../common/constants'; import { Route } from '../models/route'; import { IConfig } from '../../../common/interfaces'; import { ElasticClients } from '../../../common/elastic'; -import { additionalControlSearchProperties, queryElastic } from '../../../common/elastic/utils'; +import { queryElastic } from '../../../common/elastic/utils'; +import { additionalControlSearchProperties } from '../../utils'; import { RouteQueryParams, queryForControlPointInRoute, queryForRoute } from './queries'; // eslint-disable-next-line @typescript-eslint/explicit-function-return-type From 2e7ef58a425cb4411bda40ec58347110fa05c0ba Mon Sep 17 00:00:00 2001 From: Niv Greenstein Date: Sun, 11 Aug 2024 20:05:57 +0300 Subject: [PATCH 050/262] fix(route query): updated query to use geoContextQuery --- src/control/route/DAL/queries.ts | 103 ++++++++++++++++--------------- 1 file changed, 53 insertions(+), 50 deletions(-) diff --git a/src/control/route/DAL/queries.ts b/src/control/route/DAL/queries.ts index c6104029..79707263 100644 --- a/src/control/route/DAL/queries.ts +++ b/src/control/route/DAL/queries.ts @@ -1,79 +1,82 @@ import { estypes } from '@elastic/elasticsearch'; -import { boundingBox, geoDistance } from '../../../common/elastic/utils'; -import { GeoContext, WGS84Coordinate } from '../../../common/interfaces'; +import { CommonRequestParameters } from '../../../common/interfaces'; +import { geoContextQuery } from '../../utils'; +import { ELASTIC_KEYWORDS } from '../../constants'; -export interface RouteQueryParams { +export interface RouteQueryParams extends CommonRequestParameters { commandName: string; controlPoint?: number; - geo?: GeoContext; } /* eslint-disable @typescript-eslint/naming-convention */ -export const queryForRoute = (params: RouteQueryParams): estypes.SearchRequest => ({ +export const queryForRoute = ({ + geo_context: geoContext, + geo_context_mode: geoContextMode, + commandName, + disable_fuzziness: disableFuzziness, +}: RouteQueryParams): estypes.SearchRequest => ({ query: { bool: { - should: [ + must: [ { term: { - 'properties.TYPE.keyword': 'ROUTE', + [ELASTIC_KEYWORDS.type]: 'ROUTE', }, }, - ], - must: [ { match: { - 'properties.OBJECT_COMMAND_NAME.keyword': { - query: params.commandName, - fuzziness: 1, + [ELASTIC_KEYWORDS.objectCommandName]: { + query: commandName, + fuzziness: disableFuzziness ? undefined : 1, prefix_length: 1, }, }, }, - ...(params.geo?.bbox ? [boundingBox(params.geo.bbox)] : []), - ...(params.geo?.radius - ? [ - geoDistance( - params.geo as { - radius: number; - lon: number; - lat: number; - } - ), - ] - : []), ], + ...geoContextQuery(geoContext, geoContextMode), }, }, }); -export const queryForControlPointInRoute = (params: RouteQueryParams & Required>): estypes.SearchRequest => ({ - query: { - bool: { - must: [ - { - match: { - 'properties.OBJECT_COMMAND_NAME.keyword': { - query: params.controlPoint, - fuzziness: 1, - prefix_length: 1, +export const queryForControlPointInRoute = ({ + controlPoint, + commandName, + geo_context: geoContext, + geo_context_mode: geoContextMode, +}: RouteQueryParams & Required>): estypes.SearchRequest => { + const geoContextOperation = geoContextQuery(geoContext, geoContextMode); + + const esQuery: estypes.SearchRequest = { + query: { + bool: { + must: [ + { + match: { + [ELASTIC_KEYWORDS.objectCommandName]: { + query: controlPoint, + fuzziness: 1, + prefix_length: 1, + }, }, }, - }, - ...(params.geo?.bbox ? [boundingBox(params.geo.bbox)] : []), - ...(params.geo?.radius ?? 0 ? [geoDistance(params.geo as WGS84Coordinate & { radius: number })] : []), - ], - filter: [ - { - terms: { - 'properties.LAYER_NAME.keyword': ['CONTROL_GIL_GDB.CTR_CONTROL_POINT_N', 'CONTROL_GIL_GDB.CTR_CONTROL_POINT_CROSS_N'], + ], + filter: [ + { + terms: { + [ELASTIC_KEYWORDS.layerName]: ['CONTROL_GIL_GDB.CTR_CONTROL_POINT_N', 'CONTROL_GIL_GDB.CTR_CONTROL_POINT_CROSS_N'], + }, }, - }, - { - term: { - 'properties.TIED_TO': params.commandName, + { + term: { + [ELASTIC_KEYWORDS.tiedTo]: commandName, + }, }, - }, - ], + ...(geoContextOperation.filter ?? []), + ], + should: geoContextOperation.should ?? [], + }, }, - }, -}); + }; + + return esQuery; +}; From 2c579449885f664bbb8e603e8ad11405c01777b9 Mon Sep 17 00:00:00 2001 From: Niv Greenstein Date: Mon, 12 Aug 2024 09:37:39 +0300 Subject: [PATCH 051/262] fix(routeController): added missing geo_context_mode --- src/control/route/controllers/routeController.ts | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/src/control/route/controllers/routeController.ts b/src/control/route/controllers/routeController.ts index 785d496f..287f1616 100644 --- a/src/control/route/controllers/routeController.ts +++ b/src/control/route/controllers/routeController.ts @@ -30,11 +30,12 @@ export class RouteController { public getRoutes: GetRoutesHandler = async (req, res, next) => { try { - const { command_name: commandName, control_point, geo_context, disable_fuzziness, limit } = req.query; + const { command_name: commandName, control_point, geo_context, geo_context_mode, disable_fuzziness, limit } = req.query; const response = await this.manager.getRoutes({ commandName, controlPoint: control_point ? parseInt(control_point) : undefined, geo_context, + geo_context_mode, disable_fuzziness, limit, }); From 026754bfda3ef78ef0c73eef10dda00775ac6d2e Mon Sep 17 00:00:00 2001 From: Niv Greenstein Date: Mon, 12 Aug 2024 10:01:27 +0300 Subject: [PATCH 052/262] feat(common/utils): added Type util that infers snake_case interface to camelCase interface --- src/common/utils.ts | 6 ++++++ 1 file changed, 6 insertions(+) diff --git a/src/common/utils.ts b/src/common/utils.ts index 8fba3f28..91375ec5 100644 --- a/src/common/utils.ts +++ b/src/common/utils.ts @@ -6,6 +6,12 @@ import { Route } from '../control/route/models/route'; import { utmProjection, wgs84Projection } from './projections'; import { FeatureCollection, WGS84Coordinate } from './interfaces'; +type SnakeToCamelCase = S extends `${infer T}_${infer U}` ? `${T}${Capitalize>}` : S; + +export type ConvertSnakeToCamelCase = { + [K in keyof T as SnakeToCamelCase]: T[K]; +}; + export const formatResponse = (elasticResponse: estypes.SearchResponse): FeatureCollection => ({ type: 'FeatureCollection', features: [ From 49a280caa27259e821fac5ac9938257269772d75 Mon Sep 17 00:00:00 2001 From: Niv Greenstein Date: Mon, 12 Aug 2024 10:01:51 +0300 Subject: [PATCH 053/262] delete(common/interfaces): removed unused interface --- src/common/interfaces.ts | 5 ----- 1 file changed, 5 deletions(-) diff --git a/src/common/interfaces.ts b/src/common/interfaces.ts index 671e8293..b8d65a2c 100644 --- a/src/common/interfaces.ts +++ b/src/common/interfaces.ts @@ -28,11 +28,6 @@ export interface GeoContext { zone?: number; } -export interface Geometry { - type: string; - coordinates: number[][][]; -} - export interface FeatureCollection extends GeoJSONFeatureCollection { features: T[]; } From a68ce8676cd3630a2a05bb9cf33f39be5cbecf61 Mon Sep 17 00:00:00 2001 From: Niv Greenstein Date: Mon, 12 Aug 2024 10:04:39 +0300 Subject: [PATCH 054/262] feat(item/queries): ItemQueryParams now extends converted snake-case to camelCase CommonRequestParameters infered type --- src/control/item/DAL/queries.ts | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/src/control/item/DAL/queries.ts b/src/control/item/DAL/queries.ts index 53713ffc..0379ea4f 100644 --- a/src/control/item/DAL/queries.ts +++ b/src/control/item/DAL/queries.ts @@ -2,12 +2,12 @@ import { estypes } from '@elastic/elasticsearch'; import { CommonRequestParameters, GeoContext } from '../../../common/interfaces'; import { ELASTIC_KEYWORDS } from '../../constants'; import { geoContextQuery } from '../../utils'; +import { ConvertSnakeToCamelCase } from '../../../common/utils'; -export interface ItemQueryParams extends CommonRequestParameters { +export interface ItemQueryParams extends ConvertSnakeToCamelCase { commandName: string; tile?: string; subTile?: number; - geo?: GeoContext; } /* eslint-disable @typescript-eslint/naming-convention */ export const queryForItems = ({ From cf9792dfa0768d406c6a9b395d4b2976e7128647 Mon Sep 17 00:00:00 2001 From: Niv Greenstein Date: Mon, 12 Aug 2024 10:05:07 +0300 Subject: [PATCH 055/262] feat(route/queries): ItemQueryParams now extends converted snake-case to camelCase CommonRequestParameters infered type --- src/control/route/DAL/queries.ts | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/src/control/route/DAL/queries.ts b/src/control/route/DAL/queries.ts index 79707263..ea0c5790 100644 --- a/src/control/route/DAL/queries.ts +++ b/src/control/route/DAL/queries.ts @@ -2,8 +2,9 @@ import { estypes } from '@elastic/elasticsearch'; import { CommonRequestParameters } from '../../../common/interfaces'; import { geoContextQuery } from '../../utils'; import { ELASTIC_KEYWORDS } from '../../constants'; +import { ConvertSnakeToCamelCase } from '../../../common/utils'; -export interface RouteQueryParams extends CommonRequestParameters { +export interface RouteQueryParams extends ConvertSnakeToCamelCase { commandName: string; controlPoint?: number; } From 2b94a847ec00586da4950e9fff3a9c30088a3d66 Mon Sep 17 00:00:00 2001 From: Niv Greenstein Date: Mon, 12 Aug 2024 10:05:23 +0300 Subject: [PATCH 056/262] feat(tile/queries): ItemQueryParams now extends converted snake-case to camelCase CommonRequestParameters infered type --- src/control/tile/DAL/queries.ts | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/src/control/tile/DAL/queries.ts b/src/control/tile/DAL/queries.ts index c17ed6c5..a11d4f50 100644 --- a/src/control/tile/DAL/queries.ts +++ b/src/control/tile/DAL/queries.ts @@ -2,8 +2,9 @@ import { estypes } from '@elastic/elasticsearch'; import { CommonRequestParameters } from '../../../common/interfaces'; import { ELASTIC_KEYWORDS } from '../../constants'; import { geoContextQuery } from '../../utils'; +import { ConvertSnakeToCamelCase } from '../../../common/utils'; -export interface TileQueryParams extends CommonRequestParameters { +export interface TileQueryParams extends ConvertSnakeToCamelCase { tile?: string; mgrs?: string; subTile?: number; From 8c8f7ec820da5f47ed25125617bc74c58585f9ce Mon Sep 17 00:00:00 2001 From: Niv Greenstein Date: Mon, 12 Aug 2024 10:06:10 +0300 Subject: [PATCH 057/262] fix(control/item): fixed imports based on new type --- src/control/item/DAL/itemRepository.ts | 2 -- src/control/item/DAL/queries.ts | 8 ++++---- src/control/item/controllers/itemController.ts | 9 +++++---- src/control/item/models/itemManager.ts | 2 +- 4 files changed, 10 insertions(+), 11 deletions(-) diff --git a/src/control/item/DAL/itemRepository.ts b/src/control/item/DAL/itemRepository.ts index f384e778..b7070433 100644 --- a/src/control/item/DAL/itemRepository.ts +++ b/src/control/item/DAL/itemRepository.ts @@ -10,8 +10,6 @@ import { SERVICES } from '../../../common/constants'; import { additionalControlSearchProperties } from '../../utils'; import { ItemQueryParams, queryForItems } from './queries'; -/* eslint-enable @typescript-eslint/naming-convention */ - // eslint-disable-next-line @typescript-eslint/explicit-function-return-type const createItemRepository = (client: ElasticClient, config: IConfig, logger: Logger) => { return { diff --git a/src/control/item/DAL/queries.ts b/src/control/item/DAL/queries.ts index 0379ea4f..4ea5e421 100644 --- a/src/control/item/DAL/queries.ts +++ b/src/control/item/DAL/queries.ts @@ -1,5 +1,5 @@ import { estypes } from '@elastic/elasticsearch'; -import { CommonRequestParameters, GeoContext } from '../../../common/interfaces'; +import { CommonRequestParameters } from '../../../common/interfaces'; import { ELASTIC_KEYWORDS } from '../../constants'; import { geoContextQuery } from '../../utils'; import { ConvertSnakeToCamelCase } from '../../../common/utils'; @@ -14,9 +14,9 @@ export const queryForItems = ({ commandName, tile, subTile, - geo_context: geoContext, - geo_context_mode: geoContextMode, - disable_fuzziness: disableFuzziness, + geoContext, + geoContextMode, + disableFuzziness, }: ItemQueryParams): estypes.SearchRequest => ({ query: { bool: { diff --git a/src/control/item/controllers/itemController.ts b/src/control/item/controllers/itemController.ts index 97b581d5..47ecc6b0 100644 --- a/src/control/item/controllers/itemController.ts +++ b/src/control/item/controllers/itemController.ts @@ -7,7 +7,7 @@ import { injectable, inject } from 'tsyringe'; import { SERVICES } from '../../../common/constants'; import { ItemManager } from '../models/itemManager'; import { Item } from '../models/item'; -import { CommonRequestParameters, FeatureCollection, GeoContext } from '../../../common/interfaces'; +import { CommonRequestParameters, FeatureCollection } from '../../../common/interfaces'; type GetItemsHandler = RequestHandler, undefined, GetItemsQueryParams>; @@ -31,14 +31,15 @@ export class ItemController { public getItems: GetItemsHandler = async (req, res, next) => { try { - const { command_name: commandName, tile, sub_tile, geo_context, disable_fuzziness, limit } = req.query; + const { command_name: commandName, tile, sub_tile, geo_context, geo_context_mode, disable_fuzziness, limit } = req.query; const response = await this.manager.getItems({ tile, subTile: sub_tile ? parseInt(sub_tile) : undefined, commandName, - geo: geo_context, + geoContext: geo_context, + geoContextMode: geo_context_mode, limit, - disable_fuzziness, + disableFuzziness: disable_fuzziness, }); return res.status(httpStatus.OK).json(response); } catch (error: unknown) { diff --git a/src/control/item/models/itemManager.ts b/src/control/item/models/itemManager.ts index 9ab47236..f8cf8ff8 100644 --- a/src/control/item/models/itemManager.ts +++ b/src/control/item/models/itemManager.ts @@ -18,7 +18,7 @@ export class ItemManager { ) {} public async getItems(itemQueryParams: ItemQueryParams): Promise> { - const { limit, disable_fuzziness: disableFuzziness } = itemQueryParams; + const { limit, disableFuzziness } = itemQueryParams; let elasticResponse: estypes.SearchResponse | undefined = undefined; elasticResponse = await this.itemRepository.getItems(itemQueryParams, limit); From cc22ded9a48c7fc9e16f64c735d1c77c010e1c19 Mon Sep 17 00:00:00 2001 From: Niv Greenstein Date: Mon, 12 Aug 2024 10:06:17 +0300 Subject: [PATCH 058/262] fix(control/route): fixed imports based on new type --- src/control/route/DAL/queries.ts | 11 +++-------- src/control/route/controllers/routeController.ts | 8 ++++---- src/control/route/models/routeManager.ts | 2 +- 3 files changed, 8 insertions(+), 13 deletions(-) diff --git a/src/control/route/DAL/queries.ts b/src/control/route/DAL/queries.ts index ea0c5790..4e3ac02e 100644 --- a/src/control/route/DAL/queries.ts +++ b/src/control/route/DAL/queries.ts @@ -10,12 +10,7 @@ export interface RouteQueryParams extends ConvertSnakeToCamelCase ({ +export const queryForRoute = ({ geoContext, geoContextMode, commandName, disableFuzziness }: RouteQueryParams): estypes.SearchRequest => ({ query: { bool: { must: [ @@ -42,8 +37,8 @@ export const queryForRoute = ({ export const queryForControlPointInRoute = ({ controlPoint, commandName, - geo_context: geoContext, - geo_context_mode: geoContextMode, + geoContext, + geoContextMode, }: RouteQueryParams & Required>): estypes.SearchRequest => { const geoContextOperation = geoContextQuery(geoContext, geoContextMode); diff --git a/src/control/route/controllers/routeController.ts b/src/control/route/controllers/routeController.ts index 287f1616..d8a00604 100644 --- a/src/control/route/controllers/routeController.ts +++ b/src/control/route/controllers/routeController.ts @@ -7,7 +7,7 @@ import { injectable, inject } from 'tsyringe'; import { SERVICES } from '../../../common/constants'; import { RouteManager } from '../models/routeManager'; import { Route } from '../models/route'; -import { CommonRequestParameters, FeatureCollection, GeoContext } from '../../../common/interfaces'; +import { CommonRequestParameters, FeatureCollection } from '../../../common/interfaces'; type GetRoutesHandler = RequestHandler, undefined, GetRoutesQueryParams>; @@ -34,9 +34,9 @@ export class RouteController { const response = await this.manager.getRoutes({ commandName, controlPoint: control_point ? parseInt(control_point) : undefined, - geo_context, - geo_context_mode, - disable_fuzziness, + geoContext: geo_context, + geoContextMode: geo_context_mode, + disableFuzziness: disable_fuzziness, limit, }); return res.status(httpStatus.OK).json(response); diff --git a/src/control/route/models/routeManager.ts b/src/control/route/models/routeManager.ts index 89f35548..c02796e1 100644 --- a/src/control/route/models/routeManager.ts +++ b/src/control/route/models/routeManager.ts @@ -18,7 +18,7 @@ export class RouteManager { ) {} public async getRoutes(routeQueryParams: RouteQueryParams): Promise> { - const { limit, disable_fuzziness: disableFuzziness } = routeQueryParams; + const { limit, disableFuzziness } = routeQueryParams; let elasticResponse: estypes.SearchResponse | undefined = undefined; if (routeQueryParams.controlPoint ?? 0) { From 635ff664f8b04d633f54a2c922045d021741100a Mon Sep 17 00:00:00 2001 From: Niv Greenstein Date: Mon, 12 Aug 2024 10:06:24 +0300 Subject: [PATCH 059/262] fix(control/tile): fixed imports based on new type --- src/control/tile/DAL/queries.ts | 14 ++++---------- src/control/tile/controllers/tileController.ts | 6 +++--- src/control/tile/models/tileManager.ts | 2 +- 3 files changed, 8 insertions(+), 14 deletions(-) diff --git a/src/control/tile/DAL/queries.ts b/src/control/tile/DAL/queries.ts index a11d4f50..9097dbab 100644 --- a/src/control/tile/DAL/queries.ts +++ b/src/control/tile/DAL/queries.ts @@ -12,9 +12,9 @@ export interface TileQueryParams extends ConvertSnakeToCamelCase & Required>): estypes.SearchRequest => ({ query: { bool: { @@ -40,13 +40,7 @@ export const queryForTiles = ({ }, }); -export const queryForSubTiles = ({ - tile, - geo_context: geoContext, - geo_context_mode: geoContextMode, - disable_fuzziness: disableFuzziness, - subTile, -}: Required): estypes.SearchRequest => ({ +export const queryForSubTiles = ({ tile, geoContext, geoContextMode, subTile }: Required): estypes.SearchRequest => ({ query: { bool: { must: [ diff --git a/src/control/tile/controllers/tileController.ts b/src/control/tile/controllers/tileController.ts index 2f497402..7d1cf952 100644 --- a/src/control/tile/controllers/tileController.ts +++ b/src/control/tile/controllers/tileController.ts @@ -44,9 +44,9 @@ export class TileController { const response = await this.manager.getTiles({ tile, subTile: sub_tile ? parseInt(sub_tile) : undefined, - disable_fuzziness, - geo_context, - geo_context_mode, + disableFuzziness: disable_fuzziness, + geoContext: geo_context, + geoContextMode: geo_context_mode, limit, mgrs, }); diff --git a/src/control/tile/models/tileManager.ts b/src/control/tile/models/tileManager.ts index 44e44bae..4aecff9c 100644 --- a/src/control/tile/models/tileManager.ts +++ b/src/control/tile/models/tileManager.ts @@ -19,7 +19,7 @@ export class TileManager { ) {} public async getTiles(tileQueryParams: TileQueryParams): Promise> { - const { limit, disable_fuzziness: disableFuzziness } = tileQueryParams; + const { limit, disableFuzziness } = tileQueryParams; if ( (tileQueryParams.tile === undefined && tileQueryParams.mgrs === undefined) || From cb1383890a01462622bd437b3540e62ab4a7dfe9 Mon Sep 17 00:00:00 2001 From: Niv Greenstein Date: Mon, 12 Aug 2024 10:07:03 +0300 Subject: [PATCH 060/262] fix(control/utils): changed thrown message error at geoContextQuery --- src/control/utils.ts | 5 ++--- 1 file changed, 2 insertions(+), 3 deletions(-) diff --git a/src/control/utils.ts b/src/control/utils.ts index ff752582..707854cf 100644 --- a/src/control/utils.ts +++ b/src/control/utils.ts @@ -14,7 +14,7 @@ export const geoContextQuery = ( return {}; } if ((geoContext !== undefined && geoContextMode === undefined) || (geoContext === undefined && geoContextMode !== undefined)) { - throw new BadRequestError('/control/tiles/queryForTiles: geo_context and geo_context_mode must be both defined or both undefined'); + throw new BadRequestError('/control/utils/geoContextQuery: geo_context and geo_context_mode must be both defined or both undefined'); } return { @@ -32,10 +32,9 @@ export const geoContextQuery = ( }; }; -/* eslint-disable @typescript-eslint/naming-convention */ export const additionalControlSearchProperties = (config: IConfig, size: number): { size: number; index: string; _source: string[] } => ({ size, index: config.get(elasticConfigPath).control.properties.index as string, + // eslint-disable-next-line @typescript-eslint/naming-convention _source: CONTROL_FIELDS, }); -/* eslint-enable @typescript-eslint/naming-convention */ From 4432c03cd14ac9adf4f8920c8ec931258292ca46 Mon Sep 17 00:00:00 2001 From: Niv Greenstein Date: Mon, 12 Aug 2024 10:30:23 +0300 Subject: [PATCH 061/262] feat: moved formatResponse function from common/utils to control/utils --- src/common/utils.ts | 24 +----------------------- src/control/utils.ts | 30 +++++++++++++++++++++++++++++- 2 files changed, 30 insertions(+), 24 deletions(-) diff --git a/src/common/utils.ts b/src/common/utils.ts index 91375ec5..fdd312c0 100644 --- a/src/common/utils.ts +++ b/src/common/utils.ts @@ -1,10 +1,6 @@ -import { estypes } from '@elastic/elasticsearch'; import proj4 from 'proj4'; -import { Item } from '../control/item/models/item'; -import { Tile } from '../control/tile/models/tile'; -import { Route } from '../control/route/models/route'; import { utmProjection, wgs84Projection } from './projections'; -import { FeatureCollection, WGS84Coordinate } from './interfaces'; +import { WGS84Coordinate } from './interfaces'; type SnakeToCamelCase = S extends `${infer T}_${infer U}` ? `${T}${Capitalize>}` : S; @@ -12,24 +8,6 @@ export type ConvertSnakeToCamelCase = { [K in keyof T as SnakeToCamelCase]: T[K]; }; -export const formatResponse = (elasticResponse: estypes.SearchResponse): FeatureCollection => ({ - type: 'FeatureCollection', - features: [ - ...(elasticResponse.hits.hits.map((item) => { - const source = item._source; - if (source?.properties) { - Object.keys(source.properties).forEach((key) => { - // eslint-disable-next-line @typescript-eslint/no-unnecessary-condition - if (source.properties !== null && source.properties[key as keyof typeof source.properties] == null) { - delete source.properties[key as keyof typeof source.properties]; - } - }); - } - return source; - }) as T[]), - ], -}); - export const validateWGS84Coordinate = (coordinate: { lon: number; lat: number }): boolean => { // eslint-disable-next-line @typescript-eslint/no-magic-numbers const [min, max] = [0, 180]; diff --git a/src/control/utils.ts b/src/control/utils.ts index 707854cf..264e71c3 100644 --- a/src/control/utils.ts +++ b/src/control/utils.ts @@ -1,11 +1,38 @@ import { estypes } from '@elastic/elasticsearch'; import { parseGeo } from '../geotextSearch/utils'; -import { GeoContext, GeoContextMode, IConfig } from '../common/interfaces'; +import { FeatureCollection, GeoContext, GeoContextMode, IConfig } from '../common/interfaces'; import { BadRequestError } from '../common/errors'; import { elasticConfigPath } from '../common/constants'; import { ElasticDbClientsConfig } from '../common/elastic/interfaces'; +import { Item } from '../control/item/models/item'; +import { Tile } from '../control/tile/models/tile'; +import { Route } from '../control/route/models/route'; import { CONTROL_FIELDS, ELASTIC_KEYWORDS } from './constants'; +type SnakeToCamelCase = S extends `${infer T}_${infer U}` ? `${T}${Capitalize>}` : S; + +export type ConvertSnakeToCamelCase = { + [K in keyof T as SnakeToCamelCase]: T[K]; +}; + +export const formatResponse = (elasticResponse: estypes.SearchResponse): FeatureCollection => ({ + type: 'FeatureCollection', + features: [ + ...(elasticResponse.hits.hits.map((item) => { + const source = item._source; + if (source?.properties) { + Object.keys(source.properties).forEach((key) => { + // eslint-disable-next-line @typescript-eslint/no-unnecessary-condition + if (source.properties !== null && source.properties[key as keyof typeof source.properties] == null) { + delete source.properties[key as keyof typeof source.properties]; + } + }); + } + return source; + }) as T[]), + ], +}); + export const geoContextQuery = ( geoContext?: GeoContext, geoContextMode?: GeoContextMode @@ -32,6 +59,7 @@ export const geoContextQuery = ( }; }; +// eslint-disable-next-line @typescript-eslint/naming-convention export const additionalControlSearchProperties = (config: IConfig, size: number): { size: number; index: string; _source: string[] } => ({ size, index: config.get(elasticConfigPath).control.properties.index as string, From 059bcb56c1ee84c8b5ec10e1506c2e0b79d8618e Mon Sep 17 00:00:00 2001 From: Niv Greenstein Date: Mon, 12 Aug 2024 10:30:42 +0300 Subject: [PATCH 062/262] fix: updated imports --- src/control/item/models/itemManager.ts | 2 +- src/control/route/models/routeManager.ts | 2 +- src/control/tile/models/tileManager.ts | 2 +- 3 files changed, 3 insertions(+), 3 deletions(-) diff --git a/src/control/item/models/itemManager.ts b/src/control/item/models/itemManager.ts index f8cf8ff8..a1766d4d 100644 --- a/src/control/item/models/itemManager.ts +++ b/src/control/item/models/itemManager.ts @@ -5,7 +5,7 @@ import { estypes } from '@elastic/elasticsearch'; import { SERVICES } from '../../../common/constants'; import { ITEM_REPOSITORY_SYMBOL, ItemRepository } from '../DAL/itemRepository'; import { ItemQueryParams } from '../DAL/queries'; -import { formatResponse } from '../../../common/utils'; +import { formatResponse } from '../../utils'; import { FeatureCollection } from '../../../common/interfaces'; import { Item } from './item'; diff --git a/src/control/route/models/routeManager.ts b/src/control/route/models/routeManager.ts index c02796e1..e9fbee73 100644 --- a/src/control/route/models/routeManager.ts +++ b/src/control/route/models/routeManager.ts @@ -5,7 +5,7 @@ import { estypes } from '@elastic/elasticsearch'; import { SERVICES } from '../../../common/constants'; import { ROUTE_REPOSITORY_SYMBOL, RouteRepository } from '../DAL/routeRepository'; import { RouteQueryParams } from '../DAL/queries'; -import { formatResponse } from '../../../common/utils'; +import { formatResponse } from '../../utils'; import { FeatureCollection } from '../../../common/interfaces'; import { Route } from './route'; diff --git a/src/control/tile/models/tileManager.ts b/src/control/tile/models/tileManager.ts index 4aecff9c..818224d6 100644 --- a/src/control/tile/models/tileManager.ts +++ b/src/control/tile/models/tileManager.ts @@ -4,7 +4,7 @@ import { inject, injectable } from 'tsyringe'; import { estypes } from '@elastic/elasticsearch'; import { SERVICES } from '../../../common/constants'; import { TILE_REPOSITORY_SYMBOL, TileRepository } from '../DAL/tileRepository'; -import { formatResponse } from '../../../common/utils'; +import { formatResponse } from '../../utils'; import { TileQueryParams } from '../DAL/queries'; import { FeatureCollection } from '../../../common/interfaces'; import { BadRequestError, NotImplementedError } from '../../../common/errors'; From c662baac59fcbef11c157f732f99f0104e7dfe19 Mon Sep 17 00:00:00 2001 From: Niv Greenstein Date: Mon, 12 Aug 2024 10:30:51 +0300 Subject: [PATCH 063/262] fix: updated import --- tests/unit/common/utils.spec.ts | 10 ++-------- 1 file changed, 2 insertions(+), 8 deletions(-) diff --git a/tests/unit/common/utils.spec.ts b/tests/unit/common/utils.spec.ts index 1913381b..26c1bb44 100644 --- a/tests/unit/common/utils.spec.ts +++ b/tests/unit/common/utils.spec.ts @@ -1,13 +1,7 @@ /* eslint-disable @typescript-eslint/naming-convention */ import { estypes } from '@elastic/elasticsearch'; -import { - formatResponse, - additionalSearchProperties, - convertUTMToWgs84, - convertWgs84ToUTM, - validateTile, - validateWGS84Coordinate, -} from '../../../src/common/utils'; +import { additionalSearchProperties, convertUTMToWgs84, convertWgs84ToUTM, validateTile, validateWGS84Coordinate } from '../../../src/common/utils'; +import { formatResponse } from '../../../src/control/utils'; import config from '../../../config/test.json'; import { FIELDS } from '../../../src/common/constants'; import { WGS84Coordinate } from '../../../src/common/interfaces'; From 622c77c675816ce1c09bf37de6e6ad6153e77a8c Mon Sep 17 00:00:00 2001 From: Niv Greenstein Date: Mon, 12 Aug 2024 14:57:39 +0300 Subject: [PATCH 064/262] feat(control/interfaces): created ControlResponse interface --- src/control/interfaces.ts | 17 +++++++++++++++++ 1 file changed, 17 insertions(+) create mode 100644 src/control/interfaces.ts diff --git a/src/control/interfaces.ts b/src/control/interfaces.ts new file mode 100644 index 00000000..65922bac --- /dev/null +++ b/src/control/interfaces.ts @@ -0,0 +1,17 @@ +import { Feature } from 'geojson'; +import { CommonRequestParameters, FeatureCollection } from '../common/interfaces'; +import { ConvertSnakeToCamelCase } from '../common/utils'; + +export interface ControlResponse extends FeatureCollection { + geocoding?: { + version?: string; + query?: ConvertSnakeToCamelCase; + response: { + /* eslint-disable @typescript-eslint/naming-convention */ + max_score: number; + results_count: number; + match_latency_ms: number; + /* eslint-enable @typescript-eslint/naming-convention */ + }; + }; +} From 1dd898aaa63fd88e31cd7d3d32e6feb07c6875af Mon Sep 17 00:00:00 2001 From: Niv Greenstein Date: Mon, 12 Aug 2024 14:58:20 +0300 Subject: [PATCH 065/262] feat(control/utils): added geocoding debug and stats data to response --- src/control/utils.ts | 27 +++++++++++++++++++-------- 1 file changed, 19 insertions(+), 8 deletions(-) diff --git a/src/control/utils.ts b/src/control/utils.ts index 264e71c3..1c5c35d0 100644 --- a/src/control/utils.ts +++ b/src/control/utils.ts @@ -1,22 +1,33 @@ +/* eslint-disable @typescript-eslint/naming-convention */ import { estypes } from '@elastic/elasticsearch'; import { parseGeo } from '../geotextSearch/utils'; -import { FeatureCollection, GeoContext, GeoContextMode, IConfig } from '../common/interfaces'; +import { CommonRequestParameters, GeoContext, GeoContextMode, IConfig } from '../common/interfaces'; import { BadRequestError } from '../common/errors'; import { elasticConfigPath } from '../common/constants'; import { ElasticDbClientsConfig } from '../common/elastic/interfaces'; import { Item } from '../control/item/models/item'; import { Tile } from '../control/tile/models/tile'; import { Route } from '../control/route/models/route'; +import { ConvertSnakeToCamelCase } from '../common/utils'; import { CONTROL_FIELDS, ELASTIC_KEYWORDS } from './constants'; +import { ControlResponse } from './interfaces'; -type SnakeToCamelCase = S extends `${infer T}_${infer U}` ? `${T}${Capitalize>}` : S; - -export type ConvertSnakeToCamelCase = { - [K in keyof T as SnakeToCamelCase]: T[K]; -}; - -export const formatResponse = (elasticResponse: estypes.SearchResponse): FeatureCollection => ({ +export const formatResponse = ( + elasticResponse: estypes.SearchResponse, + requestParams?: ConvertSnakeToCamelCase | undefined +): ControlResponse => ({ type: 'FeatureCollection', + geocoding: { + version: process.env.npm_package_version, + query: requestParams, + response: { + /* eslint-disable @typescript-eslint/naming-convention */ + results_count: elasticResponse.hits.hits.length, + max_score: elasticResponse.hits.max_score ?? 0, + match_latency_ms: elasticResponse.took, + /* eslint-enable @typescript-eslint/naming-convention */ + }, + }, features: [ ...(elasticResponse.hits.hits.map((item) => { const source = item._source; From 163c2fec0806d3282dfb5640ed9f2e0286c21fb2 Mon Sep 17 00:00:00 2001 From: Niv Greenstein Date: Mon, 12 Aug 2024 14:58:44 +0300 Subject: [PATCH 066/262] feat: added requestParams to formatResponse --- src/control/item/models/itemManager.ts | 2 +- src/control/route/models/routeManager.ts | 2 +- src/control/tile/models/tileManager.ts | 2 +- 3 files changed, 3 insertions(+), 3 deletions(-) diff --git a/src/control/item/models/itemManager.ts b/src/control/item/models/itemManager.ts index a1766d4d..2701f83e 100644 --- a/src/control/item/models/itemManager.ts +++ b/src/control/item/models/itemManager.ts @@ -22,7 +22,7 @@ export class ItemManager { let elasticResponse: estypes.SearchResponse | undefined = undefined; elasticResponse = await this.itemRepository.getItems(itemQueryParams, limit); - const formattedResponse = formatResponse(elasticResponse); + const formattedResponse = formatResponse(elasticResponse, itemQueryParams); if (disableFuzziness && formattedResponse.features.length > 0) { const filterFunction = (hit: Item | undefined): hit is Item => hit?.properties?.OBJECT_COMMAND_NAME === itemQueryParams.commandName; diff --git a/src/control/route/models/routeManager.ts b/src/control/route/models/routeManager.ts index e9fbee73..6b6aae1c 100644 --- a/src/control/route/models/routeManager.ts +++ b/src/control/route/models/routeManager.ts @@ -30,7 +30,7 @@ export class RouteManager { elasticResponse = await this.routeRepository.getRoutes(routeQueryParams, limit); } - const formattedResponse = formatResponse(elasticResponse); + const formattedResponse = formatResponse(elasticResponse, routeQueryParams); if (disableFuzziness && formattedResponse.features.length > 0) { const filterFunction = diff --git a/src/control/tile/models/tileManager.ts b/src/control/tile/models/tileManager.ts index 818224d6..17391f89 100644 --- a/src/control/tile/models/tileManager.ts +++ b/src/control/tile/models/tileManager.ts @@ -44,7 +44,7 @@ export class TileManager { elasticResponse = await this.tileRepository.getTiles(tileQueryParams as TileQueryParams & Required>, limit); } - const formattedResponse = formatResponse(elasticResponse); + const formattedResponse = formatResponse(elasticResponse, tileQueryParams); if (disableFuzziness && formattedResponse.features.length > 0) { const filterFunction = From 999c035ce98851a154e31036a7752086361c6e2b Mon Sep 17 00:00:00 2001 From: Niv Greenstein Date: Tue, 13 Aug 2024 16:14:18 +0300 Subject: [PATCH 067/262] feat: added mock data --- devScripts/controlElasticsearchData.json | 172 ++++++++++++++++++++--- 1 file changed, 152 insertions(+), 20 deletions(-) diff --git a/devScripts/controlElasticsearchData.json b/devScripts/controlElasticsearchData.json index 1cec3dde..0935f2d6 100644 --- a/devScripts/controlElasticsearchData.json +++ b/devScripts/controlElasticsearchData.json @@ -52,30 +52,96 @@ } }, { - "_id": "CONTROL.SUB_TILES", + "_id": "CONTROL.ITEMS1", "_source": { "type": "Feature", - "id": 13668, + "id": 211, "geometry": { "coordinates": [ [ - [27.149158174343427, 35.63159611670335], - [27.149274355343437, 35.64061707270338], - [27.138786228343463, 35.640716597703374], - [27.13867103934342, 35.631695606703374], - [27.149158174343427, 35.63159611670335] + [12.432792582620323, 41.9327692706986], + [12.432648028923637, 41.93209008126263], + [12.43295235249525, 41.93189198298137], + [12.435105441764364, 41.93235609798671], + [12.435516278586334, 41.93274663116725], + [12.43637599267521, 41.93308056343986], + [12.436026020567567, 41.93386161853829], + [12.432792582620323, 41.9327692706986] ] ], "type": "Polygon" }, + "properties": { + "OBJECTID": 211, + "F_CODE": 23000, + "F_ATT": 6020, + "VIEW_SCALE_50K_CONTROL": 6040, + "NAME": null, + "GFID": "{656eb8b8-1056-4eb5-955e-b670996cbb78}", + "OBJECT_COMMAND_NAME": "1234", + "SUB_TILE_NAME": null, + "TILE_NAME": "RIT", + "TILE_ID": "36, 7300, 3560", + "SUB_TILE_ID": "37", + "SHAPE.AREA": 2.65, + "SHAPE.LEN": 0.0044192097656775, + "LAYER_NAME": "CONTROL.ITEMS", + "ENTITY_HEB": "hotel", + "TYPE": "ITEM" + } + } + }, + { + "_id": "CONTROL.SUB_TILES", + "_source": { + "type": "Feature", "properties": { "OBJECTID": 13668, "SUB_TILE_ID": "65", - "TILE_NAME": "GRC", + "TILE_NAME": "RIT", "NAME": "somePlace", - "ZON": 35, + "ZON": 37, "LAYER_NAME": "CONTROL.SUB_TILES", "TYPE": "SUB_TILE" + }, + "geometry": { + "coordinates": [ + [ + [12.439530324602458, 41.93031190061167], + [12.439646505602468, 41.9393328566117], + [12.429158378602494, 41.939432381611695], + [12.429043189602453, 41.930411390611695], + [12.439530324602458, 41.93031190061167] + ] + ], + "type": "Polygon" + } + } + }, + { + "_id": "CONTROL.SUB_TILES1", + "_source": { + "type": "Feature", + "properties": { + "OBJECTID": 13669, + "SUB_TILE_ID": "66", + "TILE_NAME": "RIT", + "NAME": "somePlace", + "ZON": 37, + "LAYER_NAME": "CONTROL.SUB_TILES", + "TYPE": "SUB_TILE" + }, + "geometry": { + "coordinates": [ + [ + [12.44999804325252, 41.930226156898485], + [12.45011422425253, 41.939247112898514], + [12.439626097252557, 41.93934663789851], + [12.439510908252515, 41.93032564689851], + [12.44999804325252, 41.930226156898485] + ] + ], + "type": "Polygon" } } }, @@ -86,31 +152,97 @@ "id": 2, "geometry": { "coordinates": [ - [13.448493352142947, 52.31016611400918], - [13.447219581381603, 52.313370282889224], - [13.448088381125075, 52.31631514453963], - [13.450458681234068, 52.31867376333767], - [13.451112278530388, 52.32227665244022], - [13.449728938644029, 52.32463678850752], - [13.445021899434977, 52.32863442881066], - [13.444723882330948, 52.340023400115086], - [13.446229682887974, 52.34532799609971] + [12.443243654365062, 41.93890891937724], + [12.442636325462843, 41.93804302794496], + [12.442828646282095, 41.93725242115204], + [12.443405608739027, 41.93556576056804], + [12.44388471483822, 41.93370245333628], + [12.445132826188399, 41.93084467089824], + [12.445812354584433, 41.93031849813403], + [12.445819951137906, 41.930296776658196] ], "type": "LineString" }, "properties": { "OBJECTID": 2, - "OBJECT_COMMAND_NAME": "route96", + "OBJECT_COMMAND_NAME": "camilluccia", "FIRST_GFID": null, "SHAPE_Length": 4.1, "F_CODE": 8754, "F_ATT": 951, "LAYER_NAME": "CONTROL.ROUTES", - "ENTITY_HEB": "route96", + "ENTITY_HEB": "route", + "TYPE": "ROUTE" + } + } + }, + { + "_id": "CONTROL.ROUTES1", + "_source": { + "type": "Feature", + "id": 3, + "geometry": { + "coordinates": [ + [12.472759211857749, 41.932073169074016], + [12.475880836863752, 41.93239148035457] + ], + "type": "LineString" + }, + "properties": { + "OBJECTID": 3, + "OBJECT_COMMAND_NAME": "olimpiade", + "FIRST_GFID": null, + "SHAPE_Length": 4.1, + "F_CODE": 87541, + "F_ATT": 9511, + "LAYER_NAME": "CONTROL.ROUTES", + "ENTITY_HEB": "route", "TYPE": "ROUTE" } } }, + { + "_id": "CONTROL.ROUTES2", + "_source": { + "type": "Feature", + "id": 4, + "geometry": { + "coordinates": [12.475638293442415, 41.932360642739155], + "type": "Point" + }, + "properties": { + "OBJECTID": 4, + "OBJECT_COMMAND_NAME": "111", + "TIED_TO": "olimpiade", + "F_CODE": 8754111, + "F_ATT": 951111, + "ENTITY_HEB": "control point", + "LAYER_NAME": "CONTROL_GIL_GDB.CTR_CONTROL_POINT_CROSS_N", + "TYPE": "ITEM" + } + } + }, + { + "_id": "CONTROL.ROUTES3", + "_source": { + "type": "Feature", + "id": 5, + "geometry": { + "coordinates": [12.474175672012962, 41.932217551210556], + "type": "Point" + }, + "properties": { + "OBJECTID": 4, + "OBJECT_COMMAND_NAME": "112", + "TIED_TO": "olimpiade", + "F_CODE": 875411, + "F_ATT": 95111, + "ENTITY_HEB": "control point", + "LAYER_NAME": "CONTROL_GIL_GDB.CTR_CONTROL_POINT_CROSS_N", + "TYPE": "ITEM" + } + } + }, { "_id": "CONTROL.TILES", "_source": { From 26f3bea3b00f94cb17b78f04591315ddbfe98adc Mon Sep 17 00:00:00 2001 From: Niv Greenstein Date: Tue, 13 Aug 2024 16:15:49 +0300 Subject: [PATCH 068/262] fix(control/utils): additionalControlSearchProperties refined returned type --- src/control/utils.ts | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/control/utils.ts b/src/control/utils.ts index 1c5c35d0..052f9c42 100644 --- a/src/control/utils.ts +++ b/src/control/utils.ts @@ -71,7 +71,7 @@ export const geoContextQuery = ( }; // eslint-disable-next-line @typescript-eslint/naming-convention -export const additionalControlSearchProperties = (config: IConfig, size: number): { size: number; index: string; _source: string[] } => ({ +export const additionalControlSearchProperties = (config: IConfig, size: number): Pick => ({ size, index: config.get(elasticConfigPath).control.properties.index as string, // eslint-disable-next-line @typescript-eslint/naming-convention From ade19d197df44eef29a129954bdf92b75347b401 Mon Sep 17 00:00:00 2001 From: Niv Greenstein Date: Wed, 14 Aug 2024 13:49:06 +0300 Subject: [PATCH 069/262] feat(tile/queries): added fuzziness to sub_tile query --- src/control/tile/DAL/queries.ts | 15 +++++++++++++-- 1 file changed, 13 insertions(+), 2 deletions(-) diff --git a/src/control/tile/DAL/queries.ts b/src/control/tile/DAL/queries.ts index 9097dbab..7f09f763 100644 --- a/src/control/tile/DAL/queries.ts +++ b/src/control/tile/DAL/queries.ts @@ -40,7 +40,13 @@ export const queryForTiles = ({ }, }); -export const queryForSubTiles = ({ tile, geoContext, geoContextMode, subTile }: Required): estypes.SearchRequest => ({ +export const queryForSubTiles = ({ + tile, + geoContext, + geoContextMode, + subTile, + disableFuzziness, +}: Required): estypes.SearchRequest => ({ query: { bool: { must: [ @@ -56,7 +62,12 @@ export const queryForSubTiles = ({ tile, geoContext, geoContextMode, subTile }: }, { match: { - [ELASTIC_KEYWORDS.subTileId]: subTile, + [ELASTIC_KEYWORDS.subTileId]: { + query: subTile, + fuzziness: disableFuzziness ? undefined : 1, + // eslint-disable-next-line @typescript-eslint/naming-convention + prefix_length: 1, + }, }, }, ], From d225c00b67b614ede067c0b0f9b526980507b547 Mon Sep 17 00:00:00 2001 From: Niv Greenstein Date: Wed, 14 Aug 2024 14:48:56 +0300 Subject: [PATCH 070/262] feat: added _score to each returned feature --- src/control/utils.ts | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/src/control/utils.ts b/src/control/utils.ts index 052f9c42..15ed10bb 100644 --- a/src/control/utils.ts +++ b/src/control/utils.ts @@ -39,8 +39,8 @@ export const formatResponse = ( } }); } - return source; - }) as T[]), + return { ...source, _score: item._score }; + }) as (T & Pick, '_score'>)[]), ], }); From e98c8395a93210cd1cfc16fe7364b47957645a71 Mon Sep 17 00:00:00 2001 From: Niv Greenstein Date: Wed, 14 Aug 2024 14:50:02 +0300 Subject: [PATCH 071/262] fix(openapi3): changed sub_tile type to string --- openapi3.yaml | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/openapi3.yaml b/openapi3.yaml index 2ced08d1..685dfc2c 100644 --- a/openapi3.yaml +++ b/openapi3.yaml @@ -255,7 +255,9 @@ paths: in: "query" description: "The sub tile the item in it (required if tile is defined)" schema: - type: "number" + type: "string" + pattern: "^[1-9][0-9]*$" + example: "66" - $ref: "#/components/parameters/geo_context" - $ref: "#/components/parameters/geo_context_mode" - $ref: "#/components/parameters/limit" From 6518983eec7b0000dc03d56bca621b0d7ee62f2f Mon Sep 17 00:00:00 2001 From: Niv Greenstein Date: Wed, 14 Aug 2024 14:50:41 +0300 Subject: [PATCH 072/262] fix(item/queries): changed subTile from number to string --- src/control/item/DAL/queries.ts | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/src/control/item/DAL/queries.ts b/src/control/item/DAL/queries.ts index 4ea5e421..f1205691 100644 --- a/src/control/item/DAL/queries.ts +++ b/src/control/item/DAL/queries.ts @@ -7,7 +7,7 @@ import { ConvertSnakeToCamelCase } from '../../../common/utils'; export interface ItemQueryParams extends ConvertSnakeToCamelCase { commandName: string; tile?: string; - subTile?: number; + subTile?: string; } /* eslint-disable @typescript-eslint/naming-convention */ export const queryForItems = ({ @@ -44,7 +44,7 @@ export const queryForItems = ({ }, ] : []), - ...(subTile ?? 0 + ...(subTile ?? '' ? [ { term: { From ae2d06a964c26c1693723bf37cb46a075536c16f Mon Sep 17 00:00:00 2001 From: Niv Greenstein Date: Wed, 14 Aug 2024 14:51:31 +0300 Subject: [PATCH 073/262] fix(tileManager): removed filter function after response --- src/control/tile/models/tileManager.ts | 15 ++------------- 1 file changed, 2 insertions(+), 13 deletions(-) diff --git a/src/control/tile/models/tileManager.ts b/src/control/tile/models/tileManager.ts index 17391f89..458036c1 100644 --- a/src/control/tile/models/tileManager.ts +++ b/src/control/tile/models/tileManager.ts @@ -19,7 +19,7 @@ export class TileManager { ) {} public async getTiles(tileQueryParams: TileQueryParams): Promise> { - const { limit, disableFuzziness } = tileQueryParams; + const { limit } = tileQueryParams; if ( (tileQueryParams.tile === undefined && tileQueryParams.mgrs === undefined) || @@ -44,17 +44,6 @@ export class TileManager { elasticResponse = await this.tileRepository.getTiles(tileQueryParams as TileQueryParams & Required>, limit); } - const formattedResponse = formatResponse(elasticResponse, tileQueryParams); - - if (disableFuzziness && formattedResponse.features.length > 0) { - const filterFunction = - tileQueryParams.subTile ?? 0 - ? (hit: Tile | undefined): hit is Tile => - hit?.properties?.SUB_TILE_ID === tileQueryParams.subTile && hit?.properties?.TILE_NAME === tileQueryParams.tile - : (hit: Tile | undefined): hit is Tile => hit?.properties?.TILE_NAME === tileQueryParams.tile; - formattedResponse.features = formattedResponse.features.filter(filterFunction); - } - - return formattedResponse; + return formatResponse(elasticResponse, tileQueryParams); } } From 6a37102c1b4ba394fad65f3ad91c055559e63b3d Mon Sep 17 00:00:00 2001 From: Niv Greenstein Date: Wed, 14 Aug 2024 14:55:14 +0300 Subject: [PATCH 074/262] fix(openapi3): changed other sub_tile query type from number to numeric string --- openapi3.yaml | 3 ++- src/control/item/controllers/itemController.ts | 2 +- 2 files changed, 3 insertions(+), 2 deletions(-) diff --git a/openapi3.yaml b/openapi3.yaml index 685dfc2c..6fc5570b 100644 --- a/openapi3.yaml +++ b/openapi3.yaml @@ -200,7 +200,8 @@ paths: in: "query" description: "Sub tile number" schema: - type: "number" + type: "string" + pattern: "^[1-9][0-9]*$" - name: mgrs description: "1 meters MGRS Tile" example: "18SUJ2338907395" diff --git a/src/control/item/controllers/itemController.ts b/src/control/item/controllers/itemController.ts index 47ecc6b0..1d76bd7a 100644 --- a/src/control/item/controllers/itemController.ts +++ b/src/control/item/controllers/itemController.ts @@ -34,7 +34,7 @@ export class ItemController { const { command_name: commandName, tile, sub_tile, geo_context, geo_context_mode, disable_fuzziness, limit } = req.query; const response = await this.manager.getItems({ tile, - subTile: sub_tile ? parseInt(sub_tile) : undefined, + subTile: sub_tile, commandName, geoContext: geo_context, geoContextMode: geo_context_mode, From d1ef322c32fd014fbe73f6ab2fb9cdf925cc4841 Mon Sep 17 00:00:00 2001 From: Niv Greenstein Date: Wed, 14 Aug 2024 14:55:38 +0300 Subject: [PATCH 075/262] fix(tile/queries): changed subTile from number to string --- src/control/tile/DAL/queries.ts | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/control/tile/DAL/queries.ts b/src/control/tile/DAL/queries.ts index 7f09f763..85f2c221 100644 --- a/src/control/tile/DAL/queries.ts +++ b/src/control/tile/DAL/queries.ts @@ -7,7 +7,7 @@ import { ConvertSnakeToCamelCase } from '../../../common/utils'; export interface TileQueryParams extends ConvertSnakeToCamelCase { tile?: string; mgrs?: string; - subTile?: number; + subTile?: string; } export const queryForTiles = ({ From 559a3cd32939b2b261f566558e2024a91d8bacf6 Mon Sep 17 00:00:00 2001 From: Niv Greenstein Date: Wed, 14 Aug 2024 14:56:33 +0300 Subject: [PATCH 076/262] fix(tile/controller): fixed passed value to subTile --- src/control/tile/controllers/tileController.ts | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/control/tile/controllers/tileController.ts b/src/control/tile/controllers/tileController.ts index 7d1cf952..1bd158e1 100644 --- a/src/control/tile/controllers/tileController.ts +++ b/src/control/tile/controllers/tileController.ts @@ -43,7 +43,7 @@ export class TileController { const { tile, sub_tile, disable_fuzziness, geo_context, geo_context_mode, limit, mgrs } = req.query; const response = await this.manager.getTiles({ tile, - subTile: sub_tile ? parseInt(sub_tile) : undefined, + subTile: sub_tile, disableFuzziness: disable_fuzziness, geoContext: geo_context, geoContextMode: geo_context_mode, From 339d8a9b3c3cae71bf8130c148467b3dfdaa2d55 Mon Sep 17 00:00:00 2001 From: Niv Greenstein Date: Wed, 14 Aug 2024 14:57:10 +0300 Subject: [PATCH 077/262] fix(control/interfaces): fixed ControlResponse.features returned type --- src/control/interfaces.ts | 2 ++ 1 file changed, 2 insertions(+) diff --git a/src/control/interfaces.ts b/src/control/interfaces.ts index 65922bac..0066e9a5 100644 --- a/src/control/interfaces.ts +++ b/src/control/interfaces.ts @@ -1,4 +1,5 @@ import { Feature } from 'geojson'; +import { estypes } from '@elastic/elasticsearch'; import { CommonRequestParameters, FeatureCollection } from '../common/interfaces'; import { ConvertSnakeToCamelCase } from '../common/utils'; @@ -14,4 +15,5 @@ export interface ControlResponse extends FeatureCollection /* eslint-enable @typescript-eslint/naming-convention */ }; }; + features: (T & Pick, '_score'>)[]; } From 668a5bb50767e4910f1d5685dfad44444b9b3d5a Mon Sep 17 00:00:00 2001 From: Niv Greenstein Date: Wed, 14 Aug 2024 15:11:18 +0300 Subject: [PATCH 078/262] fix(openapi3): changed control_point type to numeric string --- openapi3.yaml | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/openapi3.yaml b/openapi3.yaml index 6fc5570b..6053600b 100644 --- a/openapi3.yaml +++ b/openapi3.yaml @@ -299,7 +299,8 @@ paths: in: "query" description: "The associated report control point of the route" schema: - type: "number" + type: "string" + pattern: "^[1-9][0-9]*$" - $ref: "#/components/parameters/geo_context" - $ref: "#/components/parameters/geo_context_mode" - $ref: "#/components/parameters/limit" From d6fb58892e7bb7e775b7faf07b2c3c89ebe4ad83 Mon Sep 17 00:00:00 2001 From: Niv Greenstein Date: Wed, 14 Aug 2024 15:12:11 +0300 Subject: [PATCH 079/262] fix(routeManager): removed fuzzy filter --- src/control/route/models/routeManager.ts | 16 +++------------- 1 file changed, 3 insertions(+), 13 deletions(-) diff --git a/src/control/route/models/routeManager.ts b/src/control/route/models/routeManager.ts index 6b6aae1c..5d6a54ea 100644 --- a/src/control/route/models/routeManager.ts +++ b/src/control/route/models/routeManager.ts @@ -18,10 +18,10 @@ export class RouteManager { ) {} public async getRoutes(routeQueryParams: RouteQueryParams): Promise> { - const { limit, disableFuzziness } = routeQueryParams; + const { limit } = routeQueryParams; let elasticResponse: estypes.SearchResponse | undefined = undefined; - if (routeQueryParams.controlPoint ?? 0) { + if (routeQueryParams.controlPoint ?? '') { elasticResponse = await this.routeRepository.getControlPointInRoute( routeQueryParams as RouteQueryParams & Required>, limit @@ -30,16 +30,6 @@ export class RouteManager { elasticResponse = await this.routeRepository.getRoutes(routeQueryParams, limit); } - const formattedResponse = formatResponse(elasticResponse, routeQueryParams); - - if (disableFuzziness && formattedResponse.features.length > 0) { - const filterFunction = - routeQueryParams.controlPoint ?? 0 - ? (hit: Route | undefined): hit is Route => hit?.properties?.OBJECT_COMMAND_NAME === routeQueryParams.controlPoint - : (hit: Route | undefined): hit is Route => hit?.properties?.OBJECT_COMMAND_NAME === routeQueryParams.commandName; - formattedResponse.features = formattedResponse.features.filter(filterFunction); - } - - return formattedResponse; + return formatResponse(elasticResponse, routeQueryParams); } } From a6ce7872ebfbd5255995119d096d90f0fc3235c6 Mon Sep 17 00:00:00 2001 From: Niv Greenstein Date: Wed, 14 Aug 2024 15:12:25 +0300 Subject: [PATCH 080/262] fix: changed controlPoint type to string --- src/control/route/DAL/queries.ts | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/src/control/route/DAL/queries.ts b/src/control/route/DAL/queries.ts index 4e3ac02e..9c70090f 100644 --- a/src/control/route/DAL/queries.ts +++ b/src/control/route/DAL/queries.ts @@ -6,7 +6,7 @@ import { ConvertSnakeToCamelCase } from '../../../common/utils'; export interface RouteQueryParams extends ConvertSnakeToCamelCase { commandName: string; - controlPoint?: number; + controlPoint?: string; } /* eslint-disable @typescript-eslint/naming-convention */ @@ -39,6 +39,7 @@ export const queryForControlPointInRoute = ({ commandName, geoContext, geoContextMode, + disableFuzziness, }: RouteQueryParams & Required>): estypes.SearchRequest => { const geoContextOperation = geoContextQuery(geoContext, geoContextMode); @@ -50,7 +51,7 @@ export const queryForControlPointInRoute = ({ match: { [ELASTIC_KEYWORDS.objectCommandName]: { query: controlPoint, - fuzziness: 1, + fuzziness: disableFuzziness ? undefined : 1, prefix_length: 1, }, }, From ac4e3bf91d1495592541cf79e568008e6549dfe2 Mon Sep 17 00:00:00 2001 From: Niv Greenstein Date: Wed, 14 Aug 2024 15:12:39 +0300 Subject: [PATCH 081/262] fix: changed controlPoint passed value --- src/control/route/controllers/routeController.ts | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/control/route/controllers/routeController.ts b/src/control/route/controllers/routeController.ts index d8a00604..87983fd3 100644 --- a/src/control/route/controllers/routeController.ts +++ b/src/control/route/controllers/routeController.ts @@ -33,7 +33,7 @@ export class RouteController { const { command_name: commandName, control_point, geo_context, geo_context_mode, disable_fuzziness, limit } = req.query; const response = await this.manager.getRoutes({ commandName, - controlPoint: control_point ? parseInt(control_point) : undefined, + controlPoint: control_point, geoContext: geo_context, geoContextMode: geo_context_mode, disableFuzziness: disable_fuzziness, From 0ffe24751a2bbad40318ecfa938174fea993025f Mon Sep 17 00:00:00 2001 From: Niv Greenstein Date: Wed, 14 Aug 2024 15:15:28 +0300 Subject: [PATCH 082/262] fix(itemManager): removed filter as disable fuzzy is on the query itself --- src/control/item/models/itemManager.ts | 11 ++--------- 1 file changed, 2 insertions(+), 9 deletions(-) diff --git a/src/control/item/models/itemManager.ts b/src/control/item/models/itemManager.ts index 2701f83e..25a33547 100644 --- a/src/control/item/models/itemManager.ts +++ b/src/control/item/models/itemManager.ts @@ -18,17 +18,10 @@ export class ItemManager { ) {} public async getItems(itemQueryParams: ItemQueryParams): Promise> { - const { limit, disableFuzziness } = itemQueryParams; + const { limit } = itemQueryParams; let elasticResponse: estypes.SearchResponse | undefined = undefined; elasticResponse = await this.itemRepository.getItems(itemQueryParams, limit); - const formattedResponse = formatResponse(elasticResponse, itemQueryParams); - - if (disableFuzziness && formattedResponse.features.length > 0) { - const filterFunction = (hit: Item | undefined): hit is Item => hit?.properties?.OBJECT_COMMAND_NAME === itemQueryParams.commandName; - formattedResponse.features = formattedResponse.features.filter(filterFunction); - } - - return formattedResponse; + return formatResponse(elasticResponse, itemQueryParams); } } From 3f1208ed2a6c977adfb3ff3ef71001f9ded3bb9b Mon Sep 17 00:00:00 2001 From: Niv Greenstein Date: Wed, 14 Aug 2024 15:16:16 +0300 Subject: [PATCH 083/262] fix: fixed eslint error --- src/control/tile/models/tileManager.ts | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/control/tile/models/tileManager.ts b/src/control/tile/models/tileManager.ts index 458036c1..a8ec5cad 100644 --- a/src/control/tile/models/tileManager.ts +++ b/src/control/tile/models/tileManager.ts @@ -38,7 +38,7 @@ export class TileManager { throw new BadRequestError('/control/tiles/queryForTiles: tile must be defined'); } - if (tileQueryParams.subTile ?? 0) { + if (tileQueryParams.subTile ?? '') { elasticResponse = await this.tileRepository.getSubTiles(tileQueryParams as Required, limit); } else { elasticResponse = await this.tileRepository.getTiles(tileQueryParams as TileQueryParams & Required>, limit); From 029251ba79be4805c9452fc613fed4b418576e86 Mon Sep 17 00:00:00 2001 From: Niv Greenstein Date: Wed, 14 Aug 2024 15:21:27 +0300 Subject: [PATCH 084/262] fix: changed geotext*.ts file names and imports to location*.ts --- src/containerConfig.ts | 4 ++-- src/control/utils.ts | 2 +- .../DAL/locationRepository.ts} | 0 src/{geotextSearch => location}/DAL/queries.ts | 0 .../controllers/locationController.ts} | 2 +- src/{geotextSearch => location}/interfaces.ts | 0 src/{geotextSearch => location}/models/elasticsearchHits.ts | 0 .../models/locationManager.ts} | 2 +- src/{geotextSearch => location}/parsing.ts | 0 .../routes/locationRouter.ts} | 2 +- src/{geotextSearch => location}/utils.ts | 0 src/serverBuilder.ts | 2 +- 12 files changed, 7 insertions(+), 7 deletions(-) rename src/{geotextSearch/DAL/geotextSearchRepository.ts => location/DAL/locationRepository.ts} (100%) rename src/{geotextSearch => location}/DAL/queries.ts (100%) rename src/{geotextSearch/controllers/geotextSearchController.ts => location/controllers/locationController.ts} (96%) rename src/{geotextSearch => location}/interfaces.ts (100%) rename src/{geotextSearch => location}/models/elasticsearchHits.ts (100%) rename src/{geotextSearch/models/geotextSearchManager.ts => location/models/locationManager.ts} (98%) rename src/{geotextSearch => location}/parsing.ts (100%) rename src/{geotextSearch/routes/geotextSearchRouter.ts => location/routes/locationRouter.ts} (87%) rename src/{geotextSearch => location}/utils.ts (100%) diff --git a/src/containerConfig.ts b/src/containerConfig.ts index 512f6a7b..203b7372 100644 --- a/src/containerConfig.ts +++ b/src/containerConfig.ts @@ -19,8 +19,8 @@ import { ROUTE_ROUTER_SYMBOL, routeRouterFactory } from './control/route/routes/ import { initDataSource } from './common/postgresql'; import { LATLON_CUSTOM_REPOSITORY_SYMBOL, latLonRepositoryFactory } from './latLon/DAL/latLonRepository'; import { LAT_LON_ROUTER_SYMBOL, latLonRouterFactory } from './latLon/routes/latLonRouter'; -import { GEOTEXT_REPOSITORY_SYMBOL, geotextRepositoryFactory } from './geotextSearch/DAL/geotextSearchRepository'; -import { GEOTEXT_SEARCH_ROUTER_SYMBOL, geotextSearchRouterFactory } from './geotextSearch/routes/geotextSearchRouter'; +import { GEOTEXT_REPOSITORY_SYMBOL, geotextRepositoryFactory } from './location/DAL/locationRepository'; +import { GEOTEXT_SEARCH_ROUTER_SYMBOL, geotextSearchRouterFactory } from './location/routes/locationRouter'; import { cronLoadTileLatLonDataFactory, cronLoadTileLatLonDataSymbol } from './latLon/DAL/latLonDAL'; import { ITEM_REPOSITORY_SYMBOL, itemRepositoryFactory } from './control/item/DAL/itemRepository'; diff --git a/src/control/utils.ts b/src/control/utils.ts index 15ed10bb..d52a44c5 100644 --- a/src/control/utils.ts +++ b/src/control/utils.ts @@ -1,6 +1,6 @@ /* eslint-disable @typescript-eslint/naming-convention */ import { estypes } from '@elastic/elasticsearch'; -import { parseGeo } from '../geotextSearch/utils'; +import { parseGeo } from '../location/utils'; import { CommonRequestParameters, GeoContext, GeoContextMode, IConfig } from '../common/interfaces'; import { BadRequestError } from '../common/errors'; import { elasticConfigPath } from '../common/constants'; diff --git a/src/geotextSearch/DAL/geotextSearchRepository.ts b/src/location/DAL/locationRepository.ts similarity index 100% rename from src/geotextSearch/DAL/geotextSearchRepository.ts rename to src/location/DAL/locationRepository.ts diff --git a/src/geotextSearch/DAL/queries.ts b/src/location/DAL/queries.ts similarity index 100% rename from src/geotextSearch/DAL/queries.ts rename to src/location/DAL/queries.ts diff --git a/src/geotextSearch/controllers/geotextSearchController.ts b/src/location/controllers/locationController.ts similarity index 96% rename from src/geotextSearch/controllers/geotextSearchController.ts rename to src/location/controllers/locationController.ts index f42a564c..ebf7af43 100644 --- a/src/geotextSearch/controllers/geotextSearchController.ts +++ b/src/location/controllers/locationController.ts @@ -4,7 +4,7 @@ import { RequestHandler } from 'express'; import httpStatus from 'http-status-codes'; import { injectable, inject } from 'tsyringe'; import { SERVICES } from '../../common/constants'; -import { GeotextSearchManager } from '../models/geotextSearchManager'; +import { GeotextSearchManager } from '../models/locationManager'; import { GetGeotextSearchParams, QueryResult } from '../interfaces'; type GetGeotextSearchHandler = RequestHandler< diff --git a/src/geotextSearch/interfaces.ts b/src/location/interfaces.ts similarity index 100% rename from src/geotextSearch/interfaces.ts rename to src/location/interfaces.ts diff --git a/src/geotextSearch/models/elasticsearchHits.ts b/src/location/models/elasticsearchHits.ts similarity index 100% rename from src/geotextSearch/models/elasticsearchHits.ts rename to src/location/models/elasticsearchHits.ts diff --git a/src/geotextSearch/models/geotextSearchManager.ts b/src/location/models/locationManager.ts similarity index 98% rename from src/geotextSearch/models/geotextSearchManager.ts rename to src/location/models/locationManager.ts index 1626406e..7fd7544d 100644 --- a/src/geotextSearch/models/geotextSearchManager.ts +++ b/src/location/models/locationManager.ts @@ -2,7 +2,7 @@ import { IConfig } from 'config'; import { Logger } from '@map-colonies/js-logger'; import { inject, injectable } from 'tsyringe'; import { SERVICES, elasticConfigPath } from '../../common/constants'; -import { GEOTEXT_REPOSITORY_SYMBOL, GeotextRepository } from '../DAL/geotextSearchRepository'; +import { GEOTEXT_REPOSITORY_SYMBOL, GeotextRepository } from '../DAL/locationRepository'; import { GetGeotextSearchParams, QueryResult, TextSearchParams } from '../interfaces'; import { convertResult, parseGeo } from '../utils'; import { IApplication } from '../../common/interfaces'; diff --git a/src/geotextSearch/parsing.ts b/src/location/parsing.ts similarity index 100% rename from src/geotextSearch/parsing.ts rename to src/location/parsing.ts diff --git a/src/geotextSearch/routes/geotextSearchRouter.ts b/src/location/routes/locationRouter.ts similarity index 87% rename from src/geotextSearch/routes/geotextSearchRouter.ts rename to src/location/routes/locationRouter.ts index 2387d042..430e42fe 100644 --- a/src/geotextSearch/routes/geotextSearchRouter.ts +++ b/src/location/routes/locationRouter.ts @@ -1,6 +1,6 @@ import { Router } from 'express'; import { FactoryFunction } from 'tsyringe'; -import { GeotextSearchController } from '../controllers/geotextSearchController'; +import { GeotextSearchController } from '../controllers/locationController'; const geotextSearchRouterFactory: FactoryFunction = (dependencyContainer) => { const router = Router(); diff --git a/src/geotextSearch/utils.ts b/src/location/utils.ts similarity index 100% rename from src/geotextSearch/utils.ts rename to src/location/utils.ts diff --git a/src/serverBuilder.ts b/src/serverBuilder.ts index 1320d74d..c9c04c50 100644 --- a/src/serverBuilder.ts +++ b/src/serverBuilder.ts @@ -14,7 +14,7 @@ import { TILE_ROUTER_SYMBOL } from './control/tile/routes/tileRouter'; import { ITEM_ROUTER_SYMBOL } from './control/item/routes/itemRouter'; import { ROUTE_ROUTER_SYMBOL } from './control/route/routes/routeRouter'; import { LAT_LON_ROUTER_SYMBOL } from './latLon/routes/latLonRouter'; -import { GEOTEXT_SEARCH_ROUTER_SYMBOL } from './geotextSearch/routes/geotextSearchRouter'; +import { GEOTEXT_SEARCH_ROUTER_SYMBOL } from './location/routes/locationRouter'; import { cronLoadTileLatLonDataSymbol } from './latLon/DAL/latLonDAL'; @injectable() From bc22327bb5da4d6ec455d793166045210e91a5ae Mon Sep 17 00:00:00 2001 From: Niv Greenstein Date: Wed, 14 Aug 2024 17:34:42 +0300 Subject: [PATCH 085/262] feat(location/interfaces): updated types to match new openapi --- src/location/interfaces.ts | 26 ++++++++++++-------------- 1 file changed, 12 insertions(+), 14 deletions(-) diff --git a/src/location/interfaces.ts b/src/location/interfaces.ts index b57fc493..ee36a83d 100644 --- a/src/location/interfaces.ts +++ b/src/location/interfaces.ts @@ -1,4 +1,7 @@ import type { GeoJSON } from 'geojson'; +import { estypes } from '@elastic/elasticsearch'; +import { CommonRequestParameters } from '../common/interfaces'; +import { ConvertSnakeToCamelCase } from '../common/utils'; import { HierarchySearchHit } from './models/elasticsearchHits'; export interface PlaceType { @@ -11,25 +14,16 @@ export interface TokenResponse { prediction: string[]; } -export interface TextSearchParams { - query: string; - viewbox?: GeoJSON; - boundary?: GeoJSON; - sources?: string[]; - regions?: string[]; +export interface TextSearchParams extends ConvertSnakeToCamelCase { name?: string; placeTypes?: string[]; subPlaceTypes?: string[]; hierarchies: HierarchySearchHit[]; - limit: number; } -export interface GetGeotextSearchParams { +export interface GetGeotextSearchParams extends CommonRequestParameters { query: string; - limit: number; source?: string[]; - viewbox?: string; - boundary?: string; region?: string[]; } @@ -37,8 +31,12 @@ export interface GetGeotextSearchParams { export interface QueryResult { type: string; - geocoding: { version?: string; query: TextSearchParams; name?: string }; - features: { + geocoding: { + version?: string; + query: TextSearchParams & { response: { max_score: number; results_count: number; match_latency_ms: number } }; + name?: string; + }; + features: ({ type: string; geometry?: GeoJSON; properties: { @@ -56,7 +54,7 @@ export interface QueryResult { sub_region?: string[]; regions?: { region: string; sub_regions: string[] }[]; }; - }[]; + } & Pick)[]; } /* eslint-enable @typescript-eslint/naming-convention */ From f1a21a52e8e82a93b0ea73e550df66577037f281 Mon Sep 17 00:00:00 2001 From: Niv Greenstein Date: Wed, 14 Aug 2024 17:35:21 +0300 Subject: [PATCH 086/262] feat(location/utils): added es-response data and score to returned response --- src/location/utils.ts | 16 ++++++++++++---- 1 file changed, 12 insertions(+), 4 deletions(-) diff --git a/src/location/utils.ts b/src/location/utils.ts index cea8f1a1..6bd3e03d 100644 --- a/src/location/utils.ts +++ b/src/location/utils.ts @@ -1,6 +1,6 @@ // import fetch, { Response } from "node-fetch-commonjs"; import { GeoJSON, Point } from 'geojson'; -import { SearchHit } from '@elastic/elasticsearch/lib/api/types'; +import { SearchResponse } from '@elastic/elasticsearch/lib/api/types'; import { StatusCodes } from 'http-status-codes'; import axios, { AxiosResponse as Response } from 'axios'; import { InternalServerError } from '../common/errors'; @@ -104,7 +104,7 @@ export const parseGeo = (input: string | GeoJSON | GeoContext): GeoJSON | undefi /* eslint-disable @typescript-eslint/naming-convention */ export const convertResult = ( params: TextSearchParams, - results: SearchHit[], + results: SearchResponse, { sources, regionCollection, @@ -122,14 +122,22 @@ export const convertResult = ( version: process.env.npm_package_version, query: { ...params, + response: { + /* eslint-disable @typescript-eslint/naming-convention */ + results_count: results.hits.hits.length, + max_score: results.hits.max_score ?? 0, + match_latency_ms: results.took, + /* eslint-enable @typescript-eslint/naming-convention */ + }, }, - name: params.name, + name: params.name || undefined, }, - features: results.map(({ highlight, _source: feature }, index): QueryResult['features'][number] => { + features: results.hits.hits.map(({ highlight, _source: feature, _score }, index): QueryResult['features'][number] => { const allNames = [feature!.text, feature!.translated_text || []]; return { type: 'Feature', geometry: feature?.geo_json, + _score, properties: { rank: index + 1, source: (sources ?? {})[feature?.source ?? ''] ?? feature?.source, From b5ade165210071be0174bada445ec43e80bb70a3 Mon Sep 17 00:00:00 2001 From: Niv Greenstein Date: Wed, 14 Aug 2024 17:36:06 +0300 Subject: [PATCH 087/262] fix(location/queries): updated queries accroding to new openapi --- src/location/DAL/queries.ts | 84 ++++++++++++++++++++++--------------- 1 file changed, 51 insertions(+), 33 deletions(-) diff --git a/src/location/DAL/queries.ts b/src/location/DAL/queries.ts index f700acc5..2b9586f9 100644 --- a/src/location/DAL/queries.ts +++ b/src/location/DAL/queries.ts @@ -1,9 +1,12 @@ +/* eslint-disable @typescript-eslint/no-unused-expressions */ /* eslint-disable @typescript-eslint/naming-convention */ import WKT, { GeoJSONPolygon } from 'wellknown'; import { estypes } from '@elastic/elasticsearch'; import { QueryDslQueryContainer } from '@elastic/elasticsearch/lib/api/types'; import { TextSearchParams } from '../interfaces'; -import { IApplication } from '../../common/interfaces'; +import { GeoContextMode, IApplication } from '../../common/interfaces'; +import { BadRequestError } from '../../common/errors'; +import { parseGeo } from '../utils'; const TEXT_FIELD = 'text'; const PLACETYPE_FIELD = 'placetype.keyword'; @@ -15,10 +18,26 @@ const HIERARCHY_FIELD = 'heirarchy'; const PLACETYPE_SEARCH_FIELD = 'sub_placetype_keyword'; export const geotextQuery = ( - { query, limit: size, name, placeTypes, subPlaceTypes, hierarchies, regions, viewbox, boundary, sources }: TextSearchParams, + { + query, + limit: size, + name, + placeTypes, + subPlaceTypes, + hierarchies, + region, + source, + geoContext, + geoContextMode, + disableFuzziness, + }: TextSearchParams, textLanguage: string, boosts: IApplication['elasticQueryBoosts'] ): estypes.SearchRequest => { + if ((geoContext !== undefined && geoContextMode === undefined) || (geoContext === undefined && geoContextMode !== undefined)) { + throw new BadRequestError('/location/geotextQuery: geo_context and geo_context_mode must be both defined or both undefined'); + } + const esQuery: estypes.SearchRequest = { query: { function_score: { @@ -37,6 +56,26 @@ export const geotextQuery = ( size, }; + if (geoContext && geoContextMode) { + const geo_shape = { + [GEOJSON_FIELD]: { + shape: parseGeo(geoContext), + }, + }; + if (geoContextMode === GeoContextMode.FILTER) { + (esQuery.query?.function_score?.query?.bool?.filter as QueryDslQueryContainer[]).push({ + geo_shape: geo_shape, + }); + } else { + esQuery.query?.function_score?.functions?.push({ + weight: boosts.viewbox, + filter: { + geo_shape, + }, + }); + } + } + if (!name && subPlaceTypes?.length) { (esQuery.query?.function_score?.query?.bool?.must as QueryDslQueryContainer[]).push( ...[ @@ -57,32 +96,23 @@ export const geotextQuery = ( match: { [TEXT_FIELD]: { query, - fuzziness: 'AUTO:3,4', + fuzziness: disableFuzziness ? undefined : 'AUTO:3,4', }, }, }); } - boundary && - (esQuery.query?.function_score?.query?.bool?.filter as QueryDslQueryContainer[]).push({ - geo_shape: { - [GEOJSON_FIELD]: { - shape: boundary, - }, - }, - }); - - sources?.length && + source?.length && (esQuery.query?.function_score?.query?.bool?.filter as QueryDslQueryContainer[]).push({ terms: { - [SOURCE_FIELD]: sources, + [SOURCE_FIELD]: source, }, }); - regions?.length && + region?.length && (esQuery.query?.function_score?.query?.bool?.filter as QueryDslQueryContainer[]).push({ terms: { - [REGION_FIELD]: regions, + [REGION_FIELD]: region, }, }); @@ -116,18 +146,6 @@ export const geotextQuery = ( }, }); - viewbox && - esQuery.query?.function_score?.functions?.push({ - weight: boosts.viewbox, - filter: { - geo_shape: { - [GEOJSON_FIELD]: { - shape: viewbox, - }, - }, - }, - }); - hierarchies.forEach((hierarchy) => { const hierarchyShape = typeof hierarchy.geo_json === 'string' ? WKT.parse(hierarchy.geo_json) : hierarchy.geo_json; esQuery.query?.function_score?.functions?.push({ @@ -150,14 +168,14 @@ export const geotextQuery = ( return esQuery; }; -export const placetypeQuery = (query: string): estypes.SearchRequest => ({ +export const placetypeQuery = (query: string, disableFuzziness: boolean): estypes.SearchRequest => ({ query: { bool: { should: { match: { [PLACETYPE_SEARCH_FIELD]: { query, - fuzziness: 'AUTO:3,4', + fuzziness: disableFuzziness ? undefined : 'AUTO:3,4', }, }, }, @@ -166,7 +184,7 @@ export const placetypeQuery = (query: string): estypes.SearchRequest => ({ size: 2, }); -export const hierarchyQuery = (query: string): estypes.SearchRequest => ({ +export const hierarchyQuery = (query: string, disableFuzziness: boolean): estypes.SearchRequest => ({ query: { function_score: { functions: [ @@ -176,7 +194,7 @@ export const hierarchyQuery = (query: string): estypes.SearchRequest => ({ match: { [HIERARCHY_FIELD]: { query, - fuzziness: 'AUTO:3,4', + fuzziness: disableFuzziness ? undefined : 'AUTO:3,4', }, }, }, @@ -188,7 +206,7 @@ export const hierarchyQuery = (query: string): estypes.SearchRequest => ({ match: { text: { query, - fuzziness: 'AUTO:3,4', + fuzziness: disableFuzziness ? undefined : 'AUTO:3,4', }, }, }, From fdf24233780cf6547cb6107af28e8d6e87f98b99 Mon Sep 17 00:00:00 2001 From: Niv Greenstein Date: Wed, 14 Aug 2024 17:36:28 +0300 Subject: [PATCH 088/262] feat(locationRepository): added disableFuzziness to params --- src/location/DAL/locationRepository.ts | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/src/location/DAL/locationRepository.ts b/src/location/DAL/locationRepository.ts index 540b0f1a..c646233e 100644 --- a/src/location/DAL/locationRepository.ts +++ b/src/location/DAL/locationRepository.ts @@ -33,10 +33,10 @@ const createGeotextRepository = (client: ElasticClient, logger: Logger) => { return nameTokens.join(' '); }, - async generatePlacetype(index: string, query: string): Promise<{ placeTypes: string[]; subPlaceTypes: string[] }> { + async generatePlacetype(index: string, query: string, disableFuzziness: boolean): Promise<{ placeTypes: string[]; subPlaceTypes: string[] }> { const { hits: { hits }, - } = await queryElastic(client, { index, ...placetypeQuery(query) }); + } = await queryElastic(client, { index, ...placetypeQuery(query, disableFuzziness) }); if (hits.length == 2 && hits[0]._score! - hits[1]._score! > 0.5) { hits.pop(); @@ -50,10 +50,10 @@ const createGeotextRepository = (client: ElasticClient, logger: Logger) => { return { placeTypes, subPlaceTypes }; }, - async extractHierarchy(index: string, query: string, hierarchyBoost: number): Promise { + async extractHierarchy(index: string, query: string, hierarchyBoost: number, disableFuzziness: boolean): Promise { const { hits: { hits }, - } = await queryElastic(client, { index, ...hierarchyQuery(query) }); + } = await queryElastic(client, { index, ...hierarchyQuery(query, disableFuzziness) }); const filteredHits = hits.length > 3 ? hits.filter((hit) => hit._score! >= hits[2]._score!) : hits; From c900baa3eb648e265bce351be31157eaa8c4d0b5 Mon Sep 17 00:00:00 2001 From: Niv Greenstein Date: Wed, 14 Aug 2024 17:36:55 +0300 Subject: [PATCH 089/262] fix(locationManager): updated types and passed values to functions --- src/location/models/locationManager.ts | 18 +++++++----------- 1 file changed, 7 insertions(+), 11 deletions(-) diff --git a/src/location/models/locationManager.ts b/src/location/models/locationManager.ts index 7fd7544d..6440cf14 100644 --- a/src/location/models/locationManager.ts +++ b/src/location/models/locationManager.ts @@ -4,9 +4,10 @@ import { inject, injectable } from 'tsyringe'; import { SERVICES, elasticConfigPath } from '../../common/constants'; import { GEOTEXT_REPOSITORY_SYMBOL, GeotextRepository } from '../DAL/locationRepository'; import { GetGeotextSearchParams, QueryResult, TextSearchParams } from '../interfaces'; -import { convertResult, parseGeo } from '../utils'; +import { convertResult } from '../utils'; import { IApplication } from '../../common/interfaces'; import { ElasticDbClientsConfig } from '../../common/elastic/interfaces'; +import { ConvertSnakeToCamelCase } from '../../common/utils'; @injectable() export class GeotextSearchManager { @@ -17,7 +18,7 @@ export class GeotextSearchManager { @inject(GEOTEXT_REPOSITORY_SYMBOL) private readonly geotextRepository: GeotextRepository ) {} - public async search(params: GetGeotextSearchParams): Promise { + public async search(params: ConvertSnakeToCamelCase): Promise { const extractNameEndpoint = this.appConfig.services.tokenTypesUrl; const { geotext: geotextIndex, @@ -32,23 +33,18 @@ export class GeotextSearchManager { const promises = Promise.all([ this.geotextRepository.extractName(extractNameEndpoint, query), - this.geotextRepository.generatePlacetype(placetypesIndex, query), - this.geotextRepository.extractHierarchy(hierarchiesIndex, hierarchyQuery.join(','), hierarchyBoost), + this.geotextRepository.generatePlacetype(placetypesIndex, query, params.disableFuzziness), + this.geotextRepository.extractHierarchy(hierarchiesIndex, hierarchyQuery.join(','), hierarchyBoost, params.disableFuzziness), ]); const [name, { placeTypes, subPlaceTypes }, hierarchies] = await promises; const searchParams: TextSearchParams = { - query, - limit: params.limit, - sources: params.source ? (params.source instanceof Array ? params.source : [params.source]) : undefined, - viewbox: params.viewbox ? parseGeo(params.viewbox) : undefined, - boundary: params.boundary ? parseGeo(params.boundary) : undefined, + ...params, name, placeTypes, subPlaceTypes, hierarchies, - regions: params.region, }; const esResult = await this.geotextRepository.geotextSearch( @@ -58,7 +54,7 @@ export class GeotextSearchManager { this.appConfig.elasticQueryBoosts ); - return convertResult(searchParams, esResult.hits.hits, { + return convertResult(searchParams, esResult, { sources: this.appConfig.sources, regionCollection: this.appConfig.regions, nameKeys: this.appConfig.nameTranslationsKeys, From 4ec66fa3f33f99d617fc0a06767b8ee913ce198e Mon Sep 17 00:00:00 2001 From: Niv Greenstein Date: Wed, 14 Aug 2024 17:37:32 +0300 Subject: [PATCH 090/262] fix(locationController): updated values passed to search function --- src/location/controllers/locationController.ts | 12 +++++++++++- 1 file changed, 11 insertions(+), 1 deletion(-) diff --git a/src/location/controllers/locationController.ts b/src/location/controllers/locationController.ts index ebf7af43..6230fd46 100644 --- a/src/location/controllers/locationController.ts +++ b/src/location/controllers/locationController.ts @@ -31,8 +31,18 @@ export class GeotextSearchController { } public getGeotextSearch: GetGeotextSearchHandler = async (req, res, next) => { + const { + disable_fuzziness: disableFuzziness, + geo_context: geoContext, + geo_context_mode: geoContextMode, + query, + region, + source, + limit, + } = req.query; + try { - const response = await this.manager.search(req.query); + const response = await this.manager.search({ query, region, source, disableFuzziness, geoContext, geoContextMode, limit }); return res.status(httpStatus.OK).json(response); } catch (error: unknown) { this.logger.error('Error in getGeotextSearch', error); From fe9e01c754dbc261c09dc930d49a4ec5389629d2 Mon Sep 17 00:00:00 2001 From: Niv Greenstein Date: Wed, 14 Aug 2024 18:22:57 +0300 Subject: [PATCH 091/262] fix: fixed school mock data --- devScripts/geotextElasticsearchData.json | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/devScripts/geotextElasticsearchData.json b/devScripts/geotextElasticsearchData.json index 3d0bddda..56b0e4be 100644 --- a/devScripts/geotextElasticsearchData.json +++ b/devScripts/geotextElasticsearchData.json @@ -233,7 +233,7 @@ "source_id": ["{1a5b981b-bb0e-44dd-b9e2-424b92f2de49}"], "sensitivity": "non-sensitive", "placetype": "education", - "sub_placetype": "elementary school", + "sub_placetype": "school", "wkt": "POLYGON ((-118.30812263653988 33.71684417247593, -118.30861990876181 33.71674433152869, -118.30879709771484 33.71635922964194, -118.30619642115158 33.71550819588987, -118.30586490633668 33.715921827872904, -118.30587062210924 33.716183318328746, -118.30812263653988 33.71684417247593))", "geo_json": { "coordinates": [ @@ -336,9 +336,9 @@ "_id": "6941ab8e-7503-4f8a-94f2-4750b2698db9", "_index": "placetypes_index", "_source": { - "placetype": "transportation", - "sub_placetype": "education", - "sub_placetype_keyword": "elementary school" + "placetype": "education", + "sub_placetype": "school", + "sub_placetype_keyword": "school" } } ] From 5d0b68f6babee93c102e08e3d9afa428a81e989a Mon Sep 17 00:00:00 2001 From: Niv Greenstein Date: Thu, 15 Aug 2024 11:06:31 +0300 Subject: [PATCH 092/262] feat(locationManager): added sources function --- src/location/models/locationManager.ts | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/src/location/models/locationManager.ts b/src/location/models/locationManager.ts index 6440cf14..135cf2b4 100644 --- a/src/location/models/locationManager.ts +++ b/src/location/models/locationManager.ts @@ -65,4 +65,8 @@ export class GeotextSearchManager { public regions(): string[] { return Object.keys(this.appConfig.regions ?? {}); } + + public sources(): string[] { + return Object.keys(this.appConfig.sources ?? {}); + } } From dceca468059344494d70864b184fbd99af9ee44d Mon Sep 17 00:00:00 2001 From: Niv Greenstein Date: Thu, 15 Aug 2024 11:07:06 +0300 Subject: [PATCH 093/262] feat(locationController): added sources --- src/location/controllers/locationController.ts | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/src/location/controllers/locationController.ts b/src/location/controllers/locationController.ts index 6230fd46..d1efeecd 100644 --- a/src/location/controllers/locationController.ts +++ b/src/location/controllers/locationController.ts @@ -16,7 +16,7 @@ type GetGeotextSearchHandler = RequestHandler< type GetRegionshHandler = RequestHandler; -type GetSourcesHandler = RequestHandler; +type GetSourcesHandler = RequestHandler; @injectable() export class GeotextSearchController { @@ -55,6 +55,6 @@ export class GeotextSearchController { }; public getSources: GetSourcesHandler = (req, res, next) => { - return res.status(httpStatus.NOT_IMPLEMENTED).send(); + return res.status(httpStatus.OK).json(this.manager.sources()); }; } From ced894806dd2117f01a6f1191e83779308a5d6af Mon Sep 17 00:00:00 2001 From: Niv Greenstein Date: Thu, 15 Aug 2024 11:07:43 +0300 Subject: [PATCH 094/262] feat: changed postgres init to functionfactory init --- src/common/postgresql/index.ts | 21 ++++++++++----------- src/containerConfig.ts | 22 +++++++++++++++------- 2 files changed, 25 insertions(+), 18 deletions(-) diff --git a/src/common/postgresql/index.ts b/src/common/postgresql/index.ts index 23f4c835..e131e61d 100644 --- a/src/common/postgresql/index.ts +++ b/src/common/postgresql/index.ts @@ -1,12 +1,12 @@ import { readFileSync } from 'fs'; import { HealthCheck } from '@godaddy/terminus'; import { DataSource, DataSourceOptions, QueryFailedError } from 'typeorm'; -import { PostgresDbConfig } from '../interfaces'; +import { DependencyContainer, FactoryFunction } from 'tsyringe'; +import { IConfig, PostgresDbConfig } from '../interfaces'; import { LatLon } from '../../latLon/DAL/latLon'; +import { SERVICES } from '../constants'; import { promiseTimeout } from './promiseTimeout'; -let connectionSingleton: DataSource | undefined; - const DB_TIMEOUT = 5000; enum TransactionFailure { @@ -18,6 +18,13 @@ interface QueryFailedErrorWithCode extends QueryFailedError { code: string | undefined; } +export const postgresClientFactory: FactoryFunction = (container: DependencyContainer): DataSource => { + const config = container.resolve(SERVICES.CONFIG); + + const dbConfig = config.get('db.postgresql'); + return new DataSource(createConnectionOptions(dbConfig)); +}; + export enum TransactionName { TRY_CLOSING_FILE = 'TryClosingFile', CREATE_RERUN = 'CreateRerun', @@ -44,14 +51,6 @@ export const createConnectionOptions = (dbConfig: PostgresDbConfig): DataSourceO return { entities: [...DB_ENTITIES, '**/DAL/*.js'], ...connectionOptions }; }; -export const initDataSource = async (dbConfig: PostgresDbConfig): Promise => { - if (connectionSingleton === undefined || !connectionSingleton.isInitialized) { - connectionSingleton = new DataSource(createConnectionOptions(dbConfig)); - await connectionSingleton.initialize(); - } - return connectionSingleton; -}; - export const getDbHealthCheckFunction = (connection: DataSource): HealthCheck => { return async (): Promise => { const check = connection.query('SELECT 1').then(() => { diff --git a/src/containerConfig.ts b/src/containerConfig.ts index 203b7372..e6e49d95 100644 --- a/src/containerConfig.ts +++ b/src/containerConfig.ts @@ -10,13 +10,13 @@ import { SERVICES, SERVICE_NAME } from './common/constants'; import { tracing } from './common/tracing'; import { InjectionObject, registerDependencies } from './common/dependencyRegistration'; import { elasticClientFactory, ElasticClients } from './common/elastic'; -import { IApplication, PostgresDbConfig } from './common/interfaces'; +import { IApplication } from './common/interfaces'; import { TILE_REPOSITORY_SYMBOL, tileRepositoryFactory } from './control/tile/DAL/tileRepository'; import { TILE_ROUTER_SYMBOL, tileRouterFactory } from './control/tile/routes/tileRouter'; import { ITEM_ROUTER_SYMBOL, itemRouterFactory } from './control/item/routes/itemRouter'; import { ROUTE_REPOSITORY_SYMBOL, routeRepositoryFactory } from './control/route/DAL/routeRepository'; import { ROUTE_ROUTER_SYMBOL, routeRouterFactory } from './control/route/routes/routeRouter'; -import { initDataSource } from './common/postgresql'; +import { postgresClientFactory } from './common/postgresql'; import { LATLON_CUSTOM_REPOSITORY_SYMBOL, latLonRepositoryFactory } from './latLon/DAL/latLonRepository'; import { LAT_LON_ROUTER_SYMBOL, latLonRouterFactory } from './latLon/routes/latLonRouter'; import { GEOTEXT_REPOSITORY_SYMBOL, geotextRepositoryFactory } from './location/DAL/locationRepository'; @@ -41,10 +41,6 @@ export const registerExternalValues = async (options?: RegisterOptions): Promise const applicationConfig: IApplication = config.get('application'); - const postgresqlDataSourceOptions = config.get('db.postgresql'); - - const postgresqlConnection = await initDataSource(postgresqlDataSourceOptions); - const dependencies: InjectionObject[] = [ { token: SERVICES.CONFIG, provider: { useValue: config } }, { token: SERVICES.LOGGER, provider: { useValue: logger } }, @@ -68,7 +64,19 @@ export const registerExternalValues = async (options?: RegisterOptions): Promise } }, }, - { token: DataSource, provider: { useValue: postgresqlConnection } }, + { + token: DataSource, + provider: { useFactory: postgresClientFactory }, + postInjectionHook: async (deps: DependencyContainer): Promise => { + const connection = deps.resolve(DataSource); + try { + await connection.initialize(); + logger.info('Connected to Postgres'); + } catch (err) { + logger.error('Failed to connect to Postgres', err); + } + }, + }, { token: TILE_REPOSITORY_SYMBOL, provider: { useFactory: tileRepositoryFactory } }, { token: TILE_ROUTER_SYMBOL, provider: { useFactory: tileRouterFactory } }, { token: ITEM_REPOSITORY_SYMBOL, provider: { useFactory: itemRepositoryFactory } }, From bc8832a79c1a463eb806167b758c54070e5f33d8 Mon Sep 17 00:00:00 2001 From: Niv Greenstein Date: Thu, 15 Aug 2024 11:07:55 +0300 Subject: [PATCH 095/262] feat(errors): added ServiceUnavailableError --- src/common/errors.ts | 8 ++++++++ 1 file changed, 8 insertions(+) diff --git a/src/common/errors.ts b/src/common/errors.ts index 8f3cf50f..244c0576 100644 --- a/src/common/errors.ts +++ b/src/common/errors.ts @@ -26,6 +26,14 @@ export class InternalServerError extends Error implements HttpError { } } +export class ServiceUnavailableError extends Error implements HttpError { + public readonly status = httpStatus.SERVICE_UNAVAILABLE; + + public constructor(message: string) { + super(message); + } +} + export class NotFoundError extends Error implements HttpError { public readonly status = httpStatus.NOT_FOUND; From 15bfd84d8ae96b00dafb7fb13be34f8d216964c9 Mon Sep 17 00:00:00 2001 From: Niv Greenstein Date: Thu, 15 Aug 2024 11:08:18 +0300 Subject: [PATCH 096/262] fix: changed version --- package.json | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/package.json b/package.json index 0d74c749..eaf1c718 100644 --- a/package.json +++ b/package.json @@ -1,6 +1,6 @@ { "name": "geocoding", - "version": "0.5.0", + "version": "0.1.0-prealpha", "description": "Geocoding service for MapColonies", "main": "./src/index.ts", "scripts": { From ddc23760ff334211e34946d90a65d7d0fc197d63 Mon Sep 17 00:00:00 2001 From: Niv Greenstein Date: Thu, 15 Aug 2024 11:39:11 +0300 Subject: [PATCH 097/262] fix(latlonRepository): added try-catch to getAll Request and thow ServiceUnavailableError --- src/latLon/DAL/latLonRepository.ts | 10 ++++++++-- 1 file changed, 8 insertions(+), 2 deletions(-) diff --git a/src/latLon/DAL/latLonRepository.ts b/src/latLon/DAL/latLonRepository.ts index 6dc7dcab..a7f9dbfc 100644 --- a/src/latLon/DAL/latLonRepository.ts +++ b/src/latLon/DAL/latLonRepository.ts @@ -3,14 +3,20 @@ import { Logger } from '@map-colonies/js-logger'; import { DataSource } from 'typeorm'; import { FactoryFunction } from 'tsyringe'; import { SERVICES } from '../../common/constants'; +import { ServiceUnavailableError } from '../../common/errors'; import { LatLon as LatLonDb } from './latLon'; // eslint-disable-next-line @typescript-eslint/explicit-function-return-type const createLatLonRepository = (dataSource: DataSource, logger: Logger) => { return dataSource.getRepository(LatLonDb).extend({ async getAll(): Promise { - const result = await this.find(); - return result; + try { + const result = await this.find(); + return result; + } catch (error: unknown) { + logger.error('Error in getAll function in latLonRepository', error); + throw new ServiceUnavailableError('Postgres client is not available. As for, the getAll request cannot be executed.'); + } }, }); }; From 18ba1a8047ffb2a73c9f123b3af7b5f06ed2dd64 Mon Sep 17 00:00:00 2001 From: Niv Greenstein Date: Thu, 15 Aug 2024 11:40:01 +0300 Subject: [PATCH 098/262] fix(utils/queryElastic): throw ServiceUnavailable instead of InternalServer --- src/common/elastic/utils.ts | 6 ++++-- 1 file changed, 4 insertions(+), 2 deletions(-) diff --git a/src/common/elastic/utils.ts b/src/common/elastic/utils.ts index 61931d70..5dcb413a 100644 --- a/src/common/elastic/utils.ts +++ b/src/common/elastic/utils.ts @@ -1,10 +1,12 @@ import { estypes } from '@elastic/elasticsearch'; import { WGS84Coordinate } from '../interfaces'; -import { InternalServerError } from '../errors'; +import { ServiceUnavailableError } from '../errors'; import { ElasticClient } from './index'; export const queryElastic = async (client: ElasticClient, body: estypes.SearchRequest): Promise> => { - const clientNotAvailableError = new InternalServerError('Elasticsearch client is not available'); + const clientNotAvailableError = new ServiceUnavailableError( + 'Elasticsearch client is not available. As for, the search request cannot be executed.' + ); try { if (!(await client.ping())) { throw clientNotAvailableError; From 246b49f78bee9955e21ba8659b28f4a6155bf536 Mon Sep 17 00:00:00 2001 From: Niv Greenstein Date: Thu, 15 Aug 2024 11:41:24 +0300 Subject: [PATCH 099/262] feat: added /lookup/coordinates --- src/latLon/DAL/latLonRepository.ts | 2 ++ src/latLon/controllers/latLonController.ts | 31 ++++++++++++++++++++++ src/latLon/models/latLonManager.ts | 26 ++++++++++++++---- src/latLon/routes/latLonRouter.ts | 1 + 4 files changed, 55 insertions(+), 5 deletions(-) diff --git a/src/latLon/DAL/latLonRepository.ts b/src/latLon/DAL/latLonRepository.ts index a7f9dbfc..a67e3c0b 100644 --- a/src/latLon/DAL/latLonRepository.ts +++ b/src/latLon/DAL/latLonRepository.ts @@ -14,6 +14,8 @@ const createLatLonRepository = (dataSource: DataSource, logger: Logger) => { const result = await this.find(); return result; } catch (error: unknown) { + //TODO: check if error is due to postgres client not being available + console.log(error); logger.error('Error in getAll function in latLonRepository', error); throw new ServiceUnavailableError('Postgres client is not available. As for, the getAll request cannot be executed.'); } diff --git a/src/latLon/controllers/latLonController.ts b/src/latLon/controllers/latLonController.ts index f39179c3..27cd396a 100644 --- a/src/latLon/controllers/latLonController.ts +++ b/src/latLon/controllers/latLonController.ts @@ -1,4 +1,5 @@ /* eslint-disable @typescript-eslint/naming-convention */ +import { Feature } from 'geojson'; import { Logger } from '@map-colonies/js-logger'; import { BoundCounter, Meter } from '@opentelemetry/api-metrics'; import { RequestHandler } from 'express'; @@ -25,6 +26,13 @@ type GetLatLonToMgrsHandler = RequestHandler; +type GetCoordinatesHandler = RequestHandler< + undefined, + { [key: string]: unknown } & Feature, + undefined, + WGS84Coordinate & { target_gird: 'control' | 'MGRS' } +>; + export interface GetLatLonToTileQueryParams extends WGS84Coordinate {} export interface GetTileToLatLonQueryParams { @@ -103,4 +111,27 @@ export class LatLonController { next(error); } }; + + public getCoordinates: GetCoordinatesHandler = async (req, res, next) => { + try { + const { lat, lon, target_gird } = req.query; + + let response: + | ({ + [key: string]: unknown; + } & Feature) + | undefined = undefined; + + if (target_gird === 'control') { + response = await this.manager.latLonToTile({ lat, lon }); + } else { + response = this.manager.latLonToMGRS({ lat, lon }); + } + + return res.status(httpStatus.OK).json(response); + } catch (error: unknown) { + this.logger.warn('latLonController.getCoordinates Error:', error); + next(error); + } + }; } diff --git a/src/latLon/models/latLonManager.ts b/src/latLon/models/latLonManager.ts index c32f4127..628458aa 100644 --- a/src/latLon/models/latLonManager.ts +++ b/src/latLon/models/latLonManager.ts @@ -9,6 +9,8 @@ import { convertTilesToUTM, getSubTileByBottomLeftUtmCoor, validateResult } from import { BadRequestError } from '../../common/errors'; import { Tile } from '../../control/tile/models/tile'; import { FeatureCollection, WGS84Coordinate } from '../../common/interfaces'; +import { Feature } from 'geojson'; +import { parseGeo } from '../../location/utils'; @injectable() export class LatLonManager { @@ -22,10 +24,7 @@ export class LatLonManager { this.dbSchema = this.config.get('db.postgresql.schema'); } - public async latLonToTile({ lat, lon }: WGS84Coordinate): Promise<{ - tileName: string; - subTileNumber: number[]; - }> { + public async latLonToTile({ lat, lon }: WGS84Coordinate): Promise<{ [key: string]: unknown } & Feature> { if (!validateWGS84Coordinate({ lat, lon })) { this.logger.warn("LatLonManager.latLonToTile: Invalid lat lon, check 'lat' and 'lon' keys exists and their values are legal"); throw new BadRequestError("Invalid lat lon, check 'lat' and 'lon' keys exists and their values are legal"); @@ -59,6 +58,15 @@ export class LatLonManager { .padStart(4, '0'); return { + type: 'Feature', + properties: {}, + geometry: parseGeo({ + bbox: [tileCoordinateData.extMinX, tileCoordinateData.extMinY, tileCoordinateData.extMaxX, tileCoordinateData.extMaxY], + }) ?? { + type: 'Point', + coordinates: [lon, lat], + }, + bbox: [tileCoordinateData.extMinX, tileCoordinateData.extMinY, tileCoordinateData.extMaxX, tileCoordinateData.extMaxY], tileName: tileCoordinateData.tileName, subTileNumber: new Array(3).fill('').map(function (value, i) { return +(xNumber[i] + yNumber[i]); @@ -88,9 +96,17 @@ export class LatLonManager { return geojsonRes; } - public latLonToMGRS({ lat, lon, accuracy = 5 }: { lat: number; lon: number; accuracy?: number }): { mgrs: string } { + public latLonToMGRS({ lat, lon, accuracy = 5 }: { lat: number; lon: number; accuracy?: number }): { [key: string]: unknown } & Feature { return { + type: 'Feature', mgrs: mgrs.forward([lon, lat], accuracy), + geometry: { + type: 'Point', + coordinates: [lon, lat], + }, + properties: { + accuracy, + }, }; } diff --git a/src/latLon/routes/latLonRouter.ts b/src/latLon/routes/latLonRouter.ts index 534f765e..8357157d 100644 --- a/src/latLon/routes/latLonRouter.ts +++ b/src/latLon/routes/latLonRouter.ts @@ -10,6 +10,7 @@ const latLonRouterFactory: FactoryFunction = (dependencyContainer) => { router.get('/tileToLatLon', controller.tileToLatLon); router.get('/latlonToMgrs', controller.latlonToMgrs); router.get('/mgrsToLatlon', controller.mgrsToLatlon); + router.get('coordinates', controller.getCoordinates); return router; }; From aee18ac3129085e5e87a9f6a6217c3cb9728c8a7 Mon Sep 17 00:00:00 2001 From: Niv Greenstein Date: Thu, 15 Aug 2024 11:42:29 +0300 Subject: [PATCH 100/262] fix: changed type from Geometry --- src/location/utils.ts | 12 ++++++------ 1 file changed, 6 insertions(+), 6 deletions(-) diff --git a/src/location/utils.ts b/src/location/utils.ts index 6bd3e03d..4f26d5c1 100644 --- a/src/location/utils.ts +++ b/src/location/utils.ts @@ -1,5 +1,5 @@ // import fetch, { Response } from "node-fetch-commonjs"; -import { GeoJSON, Point } from 'geojson'; +import { GeoJSON, Geometry, Point } from 'geojson'; import { SearchResponse } from '@elastic/elasticsearch/lib/api/types'; import { StatusCodes } from 'http-status-codes'; import axios, { AxiosResponse as Response } from 'axios'; @@ -14,12 +14,12 @@ const FIND_QUOTES = /["']/g; const FIND_SPECIAL = /[`!@#$%^&*()_\-+=|\\/,.<>:[\]{}\n\t\r\s;؛]+/g; -const parsePoint = (split: string[] | number[]): GeoJSON => ({ +const parsePoint = (split: string[] | number[]): Geometry => ({ type: 'Point', coordinates: split.map(Number), }); -const parseBbox = (split: string[] | number[]): GeoJSON => { +const parseBbox = (split: string[] | number[]): Geometry => { const [xMin, yMin, xMax, yMax] = split.map(Number); return { type: 'Polygon', @@ -62,7 +62,7 @@ export const fetchNLPService = async (endpoint: string, requestData: object): export const cleanQuery = (query: string): string[] => query.replace(FIND_QUOTES, '').split(FIND_SPECIAL); -export const parseGeo = (input: string | GeoJSON | GeoContext): GeoJSON | undefined => { +export const parseGeo = (input: string | GeoJSON | GeoContext): Geometry | undefined => { //TODO: remove string | GeoJson as accepted types //TODO: Add geojson validation //TODO: refactor this function @@ -96,9 +96,9 @@ export const parseGeo = (input: string | GeoJSON | GeoContext): GeoJSON | undefi // console.log(convertWgs84ToUTM(lat, lon)); - return { type: 'Circle', coordinates: (parsePoint([lon, lat]) as Point).coordinates, radius: `${radius ?? ''}` } as unknown as GeoJSON; + return { type: 'Circle', coordinates: (parsePoint([lon, lat]) as Point).coordinates, radius: `${radius ?? ''}` } as unknown as Geometry; } - return input as GeoJSON; + return input as Geometry; }; /* eslint-disable @typescript-eslint/naming-convention */ From 6e88bf09f7e8af60d8f2e425a7a33f1f2c416ecb Mon Sep 17 00:00:00 2001 From: Niv Greenstein Date: Thu, 15 Aug 2024 11:43:26 +0300 Subject: [PATCH 101/262] delete(location/convertResult):removed old region and sub_regions --- src/location/utils.ts | 2 -- 1 file changed, 2 deletions(-) diff --git a/src/location/utils.ts b/src/location/utils.ts index 4f26d5c1..ef2f7fca 100644 --- a/src/location/utils.ts +++ b/src/location/utils.ts @@ -156,8 +156,6 @@ export const convertResult = ( region: region, sub_regions: feature.sub_region.filter((sub_region) => (regionCollection ?? {})[region ?? '']?.includes(sub_region)), })), - region: feature?.region, - sub_region: feature?.sub_region, }, }; }), From a66c254a5581a2f76fdbe7ee1b334305aa9b4d33 Mon Sep 17 00:00:00 2001 From: Niv Greenstein Date: Sun, 18 Aug 2024 10:03:08 +0300 Subject: [PATCH 102/262] fix(openapi3): re --- openapi3.yaml | 11 ++--------- 1 file changed, 2 insertions(+), 9 deletions(-) diff --git a/openapi3.yaml b/openapi3.yaml index 6053600b..7ac03933 100644 --- a/openapi3.yaml +++ b/openapi3.yaml @@ -82,7 +82,7 @@ paths: schema: type: array items: - type: string + $ref: "#/components/schemas/Source" title: Source description: | Sources to include (if not specified, all sources will be queried) @@ -934,18 +934,11 @@ components: BoundingBox: type: array description: "Bounding box array that contains [minX,minY,maxX,maxY]" - # example: '[-74.382527,40.477003,-73.322346,40.916383]' + example: '[-74.382527,40.477003,-73.322346,40.916383]' items: type: number minLength: 4 maxLength: 4 - # properties: - # bbox: - # type: array - # items: - # type: number - # minLength: 4 - # maxLength: 4 WGS84Circle: type: object properties: From fa84efc805905766cac917d8662b418653a9f4a6 Mon Sep 17 00:00:00 2001 From: Niv Greenstein Date: Sun, 18 Aug 2024 13:10:18 +0300 Subject: [PATCH 103/262] fix: updatd geocoding version --- openapi3.yaml | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/openapi3.yaml b/openapi3.yaml index 7ac03933..67c3edc4 100644 --- a/openapi3.yaml +++ b/openapi3.yaml @@ -1,6 +1,6 @@ openapi: "3.0.1" info: - version: "1.0.0" + version: "0.1.0" title: "Geocoding" description: |- MapColonies Vector Geocoding api provides custom geodata search engine uniting multiple sources of data, query tiles, routes, items; Convertion functions - tile to/from WGS84 lat lng, WGS84 to/from US Army MGRS, WGS84 to/from UTM. @@ -934,7 +934,7 @@ components: BoundingBox: type: array description: "Bounding box array that contains [minX,minY,maxX,maxY]" - example: '[-74.382527,40.477003,-73.322346,40.916383]' + example: "[-74.382527,40.477003,-73.322346,40.916383]" items: type: number minLength: 4 From 42434a641914851efddf3c37969bebb1b007ba92 Mon Sep 17 00:00:00 2001 From: Niv Greenstein Date: Tue, 20 Aug 2024 15:40:06 +0300 Subject: [PATCH 104/262] fix: changed convertWgs84ToUTM params to accept object to be more readable and expectable --- src/common/utils.ts | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/src/common/utils.ts b/src/common/utils.ts index fdd312c0..510e9ed1 100644 --- a/src/common/utils.ts +++ b/src/common/utils.ts @@ -30,8 +30,7 @@ export const validateWGS84Coordinate = (coordinate: { lon: number; lat: number } /* eslint-disable @typescript-eslint/naming-convention */ export const convertWgs84ToUTM = ( - latitude: number, - longitude: number, + { longitude, latitude }: { longitude: number; latitude: number }, utmPrecision = 0 ): | string From c0861dcd755cea074cbd543e45c3778f0261b2b8 Mon Sep 17 00:00:00 2001 From: Niv Greenstein Date: Tue, 20 Aug 2024 15:40:48 +0300 Subject: [PATCH 105/262] feat(control/interfaces): added G= any generic to be more typesafe at testing level. Not required when coding --- src/control/interfaces.ts | 12 +++++------- 1 file changed, 5 insertions(+), 7 deletions(-) diff --git a/src/control/interfaces.ts b/src/control/interfaces.ts index 0066e9a5..688cd1be 100644 --- a/src/control/interfaces.ts +++ b/src/control/interfaces.ts @@ -1,17 +1,15 @@ import { Feature } from 'geojson'; import { estypes } from '@elastic/elasticsearch'; import { CommonRequestParameters, FeatureCollection } from '../common/interfaces'; -import { ConvertSnakeToCamelCase } from '../common/utils'; -export interface ControlResponse extends FeatureCollection { +export interface ControlResponse extends FeatureCollection { geocoding?: { version?: string; - query?: ConvertSnakeToCamelCase; - response: { + query?: G & CommonRequestParameters; + response: Pick & { /* eslint-disable @typescript-eslint/naming-convention */ - max_score: number; - results_count: number; - match_latency_ms: number; + results_count?: estypes.SearchHitsMetadata['total']; + match_latency_ms?: estypes.SearchResponse['took']; /* eslint-enable @typescript-eslint/naming-convention */ }; }; From eac5bd5580f28f9a6bab54a1678bd64066169632 Mon Sep 17 00:00:00 2001 From: Niv Greenstein Date: Tue, 20 Aug 2024 15:41:20 +0300 Subject: [PATCH 106/262] delete: removed console.log --- src/location/utils.ts | 2 -- 1 file changed, 2 deletions(-) diff --git a/src/location/utils.ts b/src/location/utils.ts index ef2f7fca..c9edd151 100644 --- a/src/location/utils.ts +++ b/src/location/utils.ts @@ -94,8 +94,6 @@ export const parseGeo = (input: string | GeoJSON | GeoContext): Geometry | undef const { x, y, zone, radius } = input as GeoContext; const { lon, lat } = x && y && zone ? convertUTMToWgs84(x, y, zone) : (input as Required>); - // console.log(convertWgs84ToUTM(lat, lon)); - return { type: 'Circle', coordinates: (parsePoint([lon, lat]) as Point).coordinates, radius: `${radius ?? ''}` } as unknown as Geometry; } return input as Geometry; From c3a09288c076212896d9159148f1f09a76b4ee1d Mon Sep 17 00:00:00 2001 From: Niv Greenstein Date: Tue, 20 Aug 2024 15:42:20 +0300 Subject: [PATCH 107/262] feat: added convertCamelToSnakeCase util. on control response, in geocoding.query, it will send all variable names from camelCase to snake_case --- src/control/utils.ts | 15 +++++++++++++-- 1 file changed, 13 insertions(+), 2 deletions(-) diff --git a/src/control/utils.ts b/src/control/utils.ts index d52a44c5..1d31d9e9 100644 --- a/src/control/utils.ts +++ b/src/control/utils.ts @@ -12,14 +12,25 @@ import { ConvertSnakeToCamelCase } from '../common/utils'; import { CONTROL_FIELDS, ELASTIC_KEYWORDS } from './constants'; import { ControlResponse } from './interfaces'; +export const convertCamelToSnakeCase = (obj: Record): Record => { + const snakeCaseObj: Record = {}; + for (const key in obj) { + if (Object.prototype.hasOwnProperty.call(obj, key)) { + const snakeCaseKey = key.replace(/[A-Z]/g, (match) => `_${match.toLowerCase()}`); + snakeCaseObj[snakeCaseKey] = obj[key]; + } + } + return snakeCaseObj; +}; + export const formatResponse = ( elasticResponse: estypes.SearchResponse, - requestParams?: ConvertSnakeToCamelCase | undefined + requestParams?: CommonRequestParameters | ConvertSnakeToCamelCase | undefined ): ControlResponse => ({ type: 'FeatureCollection', geocoding: { version: process.env.npm_package_version, - query: requestParams, + query: requestParams ? convertCamelToSnakeCase(requestParams as Record) : undefined, response: { /* eslint-disable @typescript-eslint/naming-convention */ results_count: elasticResponse.hits.hits.length, From 8aedbc47b81c3afcd54164eee40fe213cc17e373 Mon Sep 17 00:00:00 2001 From: Niv Greenstein Date: Tue, 20 Aug 2024 15:43:45 +0300 Subject: [PATCH 108/262] fix: changed control tests location --- .../item/helpers/requestSender.ts | 0 .../{ => control}/item/item.spec.ts | 0 .../route/helpers/requestSender.ts | 0 .../{ => control}/route/route.spec.ts | 4 +- .../integration/tile/helpers/requestSender.ts | 16 --- tests/integration/tile/tile.spec.ts | 124 ------------------ tests/unit/common/utils.spec.ts | 10 +- 7 files changed, 7 insertions(+), 147 deletions(-) rename tests/integration/{ => control}/item/helpers/requestSender.ts (100%) rename tests/integration/{ => control}/item/item.spec.ts (100%) rename tests/integration/{ => control}/route/helpers/requestSender.ts (100%) rename tests/integration/{ => control}/route/route.spec.ts (96%) delete mode 100644 tests/integration/tile/helpers/requestSender.ts delete mode 100644 tests/integration/tile/tile.spec.ts diff --git a/tests/integration/item/helpers/requestSender.ts b/tests/integration/control/item/helpers/requestSender.ts similarity index 100% rename from tests/integration/item/helpers/requestSender.ts rename to tests/integration/control/item/helpers/requestSender.ts diff --git a/tests/integration/item/item.spec.ts b/tests/integration/control/item/item.spec.ts similarity index 100% rename from tests/integration/item/item.spec.ts rename to tests/integration/control/item/item.spec.ts diff --git a/tests/integration/route/helpers/requestSender.ts b/tests/integration/control/route/helpers/requestSender.ts similarity index 100% rename from tests/integration/route/helpers/requestSender.ts rename to tests/integration/control/route/helpers/requestSender.ts diff --git a/tests/integration/route/route.spec.ts b/tests/integration/control/route/route.spec.ts similarity index 96% rename from tests/integration/route/route.spec.ts rename to tests/integration/control/route/route.spec.ts index 6e195f95..72278b01 100644 --- a/tests/integration/route/route.spec.ts +++ b/tests/integration/control/route/route.spec.ts @@ -2,8 +2,8 @@ import jsLogger from '@map-colonies/js-logger'; import { trace } from '@opentelemetry/api'; import httpStatusCodes from 'http-status-codes'; -import { getApp } from '../../../src/app'; -import { SERVICES } from '../../../src/common/constants'; +import { getApp } from '../../../../src/app'; +import { SERVICES } from '../../../../src/common/constants'; import { GetRoutesQueryParams } from '../../../src/route/controllers/routeController'; import { RouteRequestSender } from './helpers/requestSender'; diff --git a/tests/integration/tile/helpers/requestSender.ts b/tests/integration/tile/helpers/requestSender.ts deleted file mode 100644 index 0eaed8de..00000000 --- a/tests/integration/tile/helpers/requestSender.ts +++ /dev/null @@ -1,16 +0,0 @@ -import * as supertest from 'supertest'; -import { GetTilesQueryParams } from '../../../../src/tile/controllers/tileController'; - -export class TileRequestSender { - public constructor(private readonly app: Express.Application) {} - - public async getTiles(queryParams?: GetTilesQueryParams): Promise { - return supertest - .agent(this.app) - .get('/v1/search/tiles/') - .set('Content-Type', 'application/json') - .set('X-API-Key', 'abc123') - .set('x-user-id', 'abc123') - .query(queryParams ?? {}); - } -} diff --git a/tests/integration/tile/tile.spec.ts b/tests/integration/tile/tile.spec.ts deleted file mode 100644 index abd5bd6e..00000000 --- a/tests/integration/tile/tile.spec.ts +++ /dev/null @@ -1,124 +0,0 @@ -/* eslint-disable @typescript-eslint/naming-convention */ -import jsLogger from '@map-colonies/js-logger'; -import { trace } from '@opentelemetry/api'; -import httpStatusCodes from 'http-status-codes'; -import { getApp } from '../../../src/app'; -import { SERVICES } from '../../../src/common/constants'; -import { GetTilesQueryParams } from '../../../src/tile/controllers/tileController'; -import { TileRequestSender } from './helpers/requestSender'; - -describe('/tiles', function () { - let requestSender: TileRequestSender; - - beforeEach(async function () { - const app = await getApp({ - override: [ - { token: SERVICES.LOGGER, provider: { useValue: jsLogger({ enabled: false }) } }, - { token: SERVICES.TRACER, provider: { useValue: trace.getTracer('testTracer') } }, - ], - useChild: true, - }); - requestSender = new TileRequestSender(app.app); - }); - - describe('Happy Path', function () { - it('should return 200 status code and the tile', async function () { - const response = await requestSender.getTiles({ tile: 'RIT' }); - - expect(response.status).toBe(httpStatusCodes.OK); - expect(response).toSatisfyApiSpec(); - expect(response.body).toMatchObject({ - type: 'FeatureCollection', - features: [ - { - type: 'Feature', - geometry: { - coordinates: [ - [ - [12.539507865186607, 41.851751203650096], - [12.536787075186538, 41.94185043165008], - [12.42879133518656, 41.93952837265009], - [12.431625055186686, 41.84943698365008], - [12.539507865186607, 41.851751203650096], - ], - ], - type: 'Polygon', - }, - properties: { - TILE_NAME: 'RIT', - TYPE: 'TILE', - }, - }, - ], - }); - }); - - it('should return 200 status code and the subtile', async function () { - const response = await requestSender.getTiles({ tile: 'GRC', sub_tile: '65' }); - - expect(response.status).toBe(httpStatusCodes.OK); - expect(response).toSatisfyApiSpec(); - expect(response.body).toMatchObject({ - type: 'FeatureCollection', - features: [ - { - type: 'Feature', - geometry: { - coordinates: [ - [ - [27.149158174343427, 35.63159611670335], - [27.149274355343437, 35.64061707270338], - [27.138786228343463, 35.640716597703374], - [27.13867103934342, 35.631695606703374], - [27.149158174343427, 35.63159611670335], - ], - ], - type: 'Polygon', - }, - properties: { - SUB_TILE_ID: '65', - TILE_NAME: 'GRC', - TYPE: 'SUB_TILE', - }, - }, - ], - }); - }); - - it('should return 200 status code and response empty array', async function () { - const response = await requestSender.getTiles({ tile: 'xyz' }); - - expect(response.status).toBe(httpStatusCodes.OK); - expect(response).toSatisfyApiSpec(); - expect(response.body).toMatchObject({ - type: 'FeatureCollection', - features: [], - }); - }); - }); - describe('Bad Path', function () { - // All requests with status code of 400 - it('Should return 400 status code and meessage "request/query must have required property \'tile\'"', async function () { - const message = "request/query must have required property 'tile'"; - - const response = await requestSender.getTiles(); - - expect(response.status).toBe(httpStatusCodes.BAD_REQUEST); - expect(response.body).toMatchObject({ message }); - expect(response).toSatisfyApiSpec(); - }); - it('Should return 400 status code for unknown parameter', async function () { - const parameter = 'test1234'; - const message = `Unknown query parameter '${parameter}'`; - - const response = await requestSender.getTiles({ [parameter]: parameter } as unknown as GetTilesQueryParams); - - expect(response.status).toBe(httpStatusCodes.BAD_REQUEST); - expect(response.body).toMatchObject({ message }); - expect(response).toSatisfyApiSpec(); - }); - }); - describe('Sad Path', function () { - // All requests with status code 4XX-5XX - }); -}); diff --git a/tests/unit/common/utils.spec.ts b/tests/unit/common/utils.spec.ts index 26c1bb44..596d9a19 100644 --- a/tests/unit/common/utils.spec.ts +++ b/tests/unit/common/utils.spec.ts @@ -1,10 +1,10 @@ /* eslint-disable @typescript-eslint/naming-convention */ import { estypes } from '@elastic/elasticsearch'; -import { additionalSearchProperties, convertUTMToWgs84, convertWgs84ToUTM, validateTile, validateWGS84Coordinate } from '../../../src/common/utils'; -import { formatResponse } from '../../../src/control/utils'; +import { convertUTMToWgs84, convertWgs84ToUTM, validateTile, validateWGS84Coordinate } from '../../../src/common/utils'; +import { additionalControlSearchProperties, formatResponse } from '../../../src/control/utils'; import config from '../../../config/test.json'; -import { FIELDS } from '../../../src/common/constants'; import { WGS84Coordinate } from '../../../src/common/interfaces'; +import { CONTROL_FIELDS } from '../../../src/control/constants'; import { itemElasticResponse, itemExpectedFormattedResponse, @@ -29,9 +29,9 @@ describe('utils', () => { it('should return additional search properties', () => { const size = 10; - const result = additionalSearchProperties(size); + const result = additionalControlSearchProperties(size); // eslint-disable-next-line @typescript-eslint/no-unsafe-member-access - expect(result).toMatchObject({ size, index: config.db.elastic.searchy.properties.index as string, _source: FIELDS }); + expect(result).toMatchObject({ size, index: config.db.elastic.searchy.properties.index as string, _source: CONTROL_FIELDS }); }); it('should convert UTM to WGS84', () => { From a7264285b621d50d745bf449a152fb2a2af09a8b Mon Sep 17 00:00:00 2001 From: Niv Greenstein Date: Tue, 20 Aug 2024 15:44:13 +0300 Subject: [PATCH 109/262] feat(test/control/tiles): written new happy path tiles tests --- .../control/tile/helpers/requestSender.ts | 16 + tests/integration/control/tile/tile.spec.ts | 279 ++++++++++++++++++ tests/integration/control/tile/utils.ts | 116 ++++++++ 3 files changed, 411 insertions(+) create mode 100644 tests/integration/control/tile/helpers/requestSender.ts create mode 100644 tests/integration/control/tile/tile.spec.ts create mode 100644 tests/integration/control/tile/utils.ts diff --git a/tests/integration/control/tile/helpers/requestSender.ts b/tests/integration/control/tile/helpers/requestSender.ts new file mode 100644 index 00000000..d15778c1 --- /dev/null +++ b/tests/integration/control/tile/helpers/requestSender.ts @@ -0,0 +1,16 @@ +import * as supertest from 'supertest'; +import { GetTilesQueryParams } from '../../../../../src/control/tile/controllers/tileController'; + +export class TileRequestSender { + public constructor(private readonly app: Express.Application) {} + + public async getTiles(queryParams?: GetTilesQueryParams): Promise { + return supertest + .agent(this.app) + .get('/search/control/tiles') + .set('Content-Type', 'application/json') + .set('x-api-key', 'abc123') + .set('x-user-id', 'abc123') + .query(queryParams ?? {}); + } +} diff --git a/tests/integration/control/tile/tile.spec.ts b/tests/integration/control/tile/tile.spec.ts new file mode 100644 index 00000000..9e80d808 --- /dev/null +++ b/tests/integration/control/tile/tile.spec.ts @@ -0,0 +1,279 @@ +/* eslint-disable @typescript-eslint/naming-convention */ +import jsLogger from '@map-colonies/js-logger'; +import { trace } from '@opentelemetry/api'; +import httpStatusCodes from 'http-status-codes'; +import { DataSource } from 'typeorm'; +import { getApp } from '../../../../src/app'; +import { SERVICES } from '../../../../src/common/constants'; +import { GetTilesQueryParams } from '../../../../src/control/tile/controllers/tileController'; +import { Tile } from '../../../../src/control/tile/models/tile'; +import { ControlResponse } from '../../../../src/control/interfaces'; +import { CommonRequestParameters, GeoContextMode } from '../../../../src/common/interfaces'; +import { LATLON_CUSTOM_REPOSITORY_SYMBOL } from '../../../../src/latLon/DAL/latLonRepository'; +import { cronLoadTileLatLonDataSymbol } from '../../../../src/latLon/DAL/latLonDAL'; +import { TileRequestSender } from './helpers/requestSender'; +import { RIC_TILE, RIT_TILE, SUB_TILE_65, SUB_TILE_66, expectedResponse } from './utils'; + +describe('/tiles', function () { + let requestSender: TileRequestSender; + + beforeEach(async function () { + const app = await getApp({ + override: [ + { token: SERVICES.LOGGER, provider: { useValue: jsLogger({ enabled: false }) } }, + { token: SERVICES.TRACER, provider: { useValue: trace.getTracer('testTracer') } }, + { token: LATLON_CUSTOM_REPOSITORY_SYMBOL, provider: { useValue: {} } }, + { token: DataSource, provider: { useValue: {} } }, + { token: cronLoadTileLatLonDataSymbol, provider: { useValue: {} } }, + ], + useChild: true, + }); + + requestSender = new TileRequestSender(app.app); + }); + + describe('Happy Path', function () { + it('should return 200 status code and tiles', async function () { + const requestParams: GetTilesQueryParams = { tile: 'RIT', limit: 5, disable_fuzziness: false }; + + const response = await requestSender.getTiles(requestParams); + + expect(response.status).toBe(httpStatusCodes.OK); + // expect(response).toSatisfyApiSpec(); + expect(response.body).toMatchObject>>( + expectedResponse(requestParams, [RIT_TILE, RIC_TILE], expect) + ); + }); + + it('should return 200 status code and tiles biased by geo_context (bbox)', async function () { + const requestParams: GetTilesQueryParams = { + tile: 'RI', + geo_context: { bbox: [12.554407132912445, 41.84962590648513, 12.652837919839953, 41.94545380230761] }, + geo_context_mode: GeoContextMode.BIAS, + limit: 5, + disable_fuzziness: false, + }; + const response = await requestSender.getTiles(requestParams); + + expect(response.status).toBe(httpStatusCodes.OK); + // expect(response).toSatisfyApiSpec(); + expect(response.body).toMatchObject>>( + expectedResponse(requestParams, [RIC_TILE, RIT_TILE], expect) + ); + }); + + it('should return 200 status code and tiles filtered by geo_context (bbox)', async function () { + const requestParams: GetTilesQueryParams = { + tile: 'RI', + geo_context: { bbox: [12.554407132912445, 41.84962590648513, 12.652837919839953, 41.94545380230761] }, + geo_context_mode: GeoContextMode.FILTER, + limit: 5, + disable_fuzziness: false, + }; + const response = await requestSender.getTiles(requestParams); + + expect(response.status).toBe(httpStatusCodes.OK); + // expect(response).toSatisfyApiSpec(); + expect(response.body).toMatchObject>>( + expectedResponse(requestParams, [RIC_TILE], expect) + ); + }); + + it('should return 200 status code and tiles biased by geo_context (WGS84)', async function () { + const geo_context = { lon: 12.598899687444742, lat: 41.90667824634701, radius: 10 }; + const requestParams: GetTilesQueryParams = { + tile: 'RI', + geo_context, + geo_context_mode: GeoContextMode.BIAS, + limit: 5, + disable_fuzziness: false, + }; + const response = await requestSender.getTiles({ + ...requestParams, + geo_context: JSON.stringify(geo_context) as unknown as GetTilesQueryParams['geo_context'], + }); + + expect(response.status).toBe(httpStatusCodes.OK); + // expect(response).toSatisfyApiSpec(); + expect(response.body).toMatchObject>>( + expectedResponse(requestParams, [RIC_TILE, RIT_TILE], expect) + ); + }); + + it('should return 200 status code and tiles filtered by geo_context (WGS84)', async function () { + const geo_context = { lon: 12.598899687444742, lat: 41.90667824634701, radius: 10 }; + const requestParams: GetTilesQueryParams = { + tile: 'RI', + geo_context, + geo_context_mode: GeoContextMode.FILTER, + limit: 5, + disable_fuzziness: false, + }; + const response = await requestSender.getTiles({ + ...requestParams, + geo_context: JSON.stringify(geo_context) as unknown as GetTilesQueryParams['geo_context'], + }); + + expect(response.status).toBe(httpStatusCodes.OK); + // expect(response).toSatisfyApiSpec(); + expect(response.body).toMatchObject>>( + expectedResponse(requestParams, [RIC_TILE], expect) + ); + }); + + it('should return 200 status code and tiles biased by geo_context (UTM)', async function () { + const geo_context = { x: 300850, y: 4642203, zone: 33, radius: 100 }; + const requestParams: GetTilesQueryParams = { + tile: 'RI', + geo_context, + geo_context_mode: GeoContextMode.BIAS, + limit: 5, + disable_fuzziness: false, + }; + const response = await requestSender.getTiles({ + ...requestParams, + geo_context: JSON.stringify(geo_context) as unknown as GetTilesQueryParams['geo_context'], + }); + + expect(response.status).toBe(httpStatusCodes.OK); + // expect(response).toSatisfyApiSpec(); + expect(response.body).toMatchObject>>( + expectedResponse(requestParams, [RIC_TILE, RIT_TILE], expect) + ); + }); + + it('should return 200 status code and tiles filtered by geo_context (UTM)', async function () { + const geo_context = { x: 300850, y: 4642203, zone: 33, radius: 100 }; + const requestParams: GetTilesQueryParams = { + tile: 'RI', + geo_context, + geo_context_mode: GeoContextMode.FILTER, + limit: 5, + disable_fuzziness: false, + }; + const response = await requestSender.getTiles({ + ...requestParams, + geo_context: JSON.stringify(geo_context) as unknown as GetTilesQueryParams['geo_context'], + }); + + expect(response.status).toBe(httpStatusCodes.OK); + // expect(response).toSatisfyApiSpec(); + expect(response.body).toMatchObject>>( + expectedResponse(requestParams, [RIC_TILE], expect) + ); + }); + + it('should return 200 status code and send RIC tile as disable_fuzziness is true', async function () { + const requestParams: GetTilesQueryParams = { + tile: 'RIC', + limit: 5, + disable_fuzziness: true, + }; + const response = await requestSender.getTiles(requestParams); + + expect(response.status).toBe(httpStatusCodes.OK); + // expect(response).toSatisfyApiSpec(); + expect(response.body).toMatchObject>>( + expectedResponse(requestParams, [RIC_TILE], expect) + ); + }); + + it('should return 200 status code and no tile as disable_fuzziness is true', async function () { + const requestParams: GetTilesQueryParams = { + tile: 'RI', + limit: 5, + disable_fuzziness: true, + }; + const response = await requestSender.getTiles(requestParams); + + expect(response.status).toBe(httpStatusCodes.OK); + // expect(response).toSatisfyApiSpec(); + expect(response.body).toMatchObject>>( + expectedResponse(requestParams, [], expect) + ); + }); + + it('should return 200 status code and sub tiles', async function () { + const requestParams: GetTilesQueryParams = { + tile: 'RIT', + sub_tile: '66', + limit: 5, + disable_fuzziness: false, + }; + const response = await requestSender.getTiles(requestParams); + + expect(response.status).toBe(httpStatusCodes.OK); + // expect(response).toSatisfyApiSpec(); + expect(response.body).toMatchObject>>( + expectedResponse(requestParams, [SUB_TILE_66, SUB_TILE_65], expect) + ); + }); + + it('should return 200 status code and sub_tile "66" filtered by geo_context (UTM)', async function () { + const geo_context = { x: 288240, y: 4645787, zone: 33, radius: 100 }; + const requestParams: GetTilesQueryParams = { + tile: 'RIT', + sub_tile: '66', + geo_context, + geo_context_mode: GeoContextMode.FILTER, + limit: 5, + disable_fuzziness: false, + }; + const response = await requestSender.getTiles({ + ...requestParams, + geo_context: JSON.stringify(geo_context) as unknown as GetTilesQueryParams['geo_context'], + }); + + // expect(response.status).toBe(httpStatusCodes.OK); + // expect(response).toSatisfyApiSpec(); + expect(response.body).toMatchObject>>( + expectedResponse(requestParams, [SUB_TILE_66], expect) + ); + }); + + it('should return 200 status code and sub_tile "66" filtered by geo_context (WGS84)', async function () { + const geo_context = { lon: 12.445715777842793, lat: 41.935651320498835, radius: 100 }; + const requestParams: GetTilesQueryParams = { + tile: 'RIT', + sub_tile: '66', + geo_context, + geo_context_mode: GeoContextMode.FILTER, + limit: 5, + disable_fuzziness: false, + }; + const response = await requestSender.getTiles({ + ...requestParams, + geo_context: JSON.stringify(geo_context) as unknown as GetTilesQueryParams['geo_context'], + }); + + // expect(response.status).toBe(httpStatusCodes.OK); + // expect(response).toSatisfyApiSpec(); + expect(response.body).toMatchObject>>( + expectedResponse(requestParams, [SUB_TILE_66], expect) + ); + }); + it('should return 200 status code and sub_tile "66" filtered by geo_context (bbox)', async function () { + const requestParams: GetTilesQueryParams = { + tile: 'RIT', + sub_tile: '66', + geo_context: { bbox: [12.442900073406776, 41.92988103633658, 12.450060653825801, 41.939618057332865] }, + geo_context_mode: GeoContextMode.FILTER, + limit: 5, + disable_fuzziness: false, + }; + const response = await requestSender.getTiles(requestParams); + + expect(response.status).toBe(httpStatusCodes.OK); + // // expect(response).toSatisfyApiSpec(); + expect(response.body).toMatchObject>>( + expectedResponse(requestParams, [SUB_TILE_66], expect) + ); + }); + }); + describe('Bad Path', function () { + // All requests with status code of 400 + }); + describe('Sad Path', function () { + // All requests with status code 4XX-5XX + }); +}); diff --git a/tests/integration/control/tile/utils.ts b/tests/integration/control/tile/utils.ts new file mode 100644 index 00000000..24586aa9 --- /dev/null +++ b/tests/integration/control/tile/utils.ts @@ -0,0 +1,116 @@ +/* eslint-disable @typescript-eslint/naming-convention */ +/* eslint-disable @typescript-eslint/no-magic-numbers */ +import { CommonRequestParameters } from '../../../../src/common/interfaces'; +import { ControlResponse } from '../../../../src/control/interfaces'; +import { GetTilesQueryParams } from '../../../../src/control/tile/controllers/tileController'; +import { Tile } from '../../../../src/control/tile/models/tile'; + +const expectedTileWithScore = (tile: Tile, expect: jest.Expect): ControlResponse['features'][number] => ({ + ...tile, + _score: expect.any(Number) as number, +}); + +const expectedGeocodingElasticResponseMetrics = ( + resultsCount: number, + expect: jest.Expect +): NonNullable['geocoding']>['response'] => ({ + results_count: resultsCount, + max_score: expect.any(Number) as number, + match_latency_ms: expect.any(Number) as number, +}); + +export const RIT_TILE: Tile = { + type: 'Feature', + geometry: { + coordinates: [ + [ + [12.539507865186607, 41.851751203650096], + [12.536787075186538, 41.94185043165008], + [12.42879133518656, 41.93952837265009], + [12.431625055186686, 41.84943698365008], + [12.539507865186607, 41.851751203650096], + ], + ], + type: 'Polygon', + }, + properties: { + TILE_NAME: 'RIT', + TYPE: 'TILE', + }, +}; + +export const RIC_TILE: Tile = { + type: 'Feature', + properties: { + TILE_NAME: 'RIC', + TYPE: 'TILE', + }, + geometry: { + coordinates: [ + [ + [12.64750356570994, 41.854073129598774], + [12.64478277570987, 41.94417235759876], + [12.536787035709892, 41.941850298598766], + [12.539620755710018, 41.85175890959876], + [12.64750356570994, 41.854073129598774], + ], + ], + type: 'Polygon', + }, +}; + +export const SUB_TILE_66: Tile = { + type: 'Feature', + properties: { + SUB_TILE_ID: '66', + TILE_NAME: 'RIT', + TYPE: 'SUB_TILE', + }, + geometry: { + coordinates: [ + [ + [12.44999804325252, 41.930226156898485], + [12.45011422425253, 41.939247112898514], + [12.439626097252557, 41.93934663789851], + [12.439510908252515, 41.93032564689851], + [12.44999804325252, 41.930226156898485], + ], + ], + type: 'Polygon', + }, +}; + +export const SUB_TILE_65: Tile = { + type: 'Feature', + properties: { + SUB_TILE_ID: '65', + TILE_NAME: 'RIT', + TYPE: 'SUB_TILE', + }, + geometry: { + coordinates: [ + [ + [12.439530324602458, 41.93031190061167], + [12.439646505602468, 41.9393328566117], + [12.429158378602494, 41.939432381611695], + [12.429043189602453, 41.930411390611695], + [12.439530324602458, 41.93031190061167], + ], + ], + type: 'Polygon', + }, +}; + +export const expectedResponse = ( + requestParams: GetTilesQueryParams, + tiles: Tile[], + expect: jest.Expect +): ControlResponse> => ({ + type: 'FeatureCollection', + geocoding: { + version: process.env.npm_package_version, + query: requestParams, + response: expectedGeocodingElasticResponseMetrics(tiles.length, expect), + }, + features: tiles.map((tile) => expectedTileWithScore(tile, expect)), +}); From 4e197c3caacfd4001954bd3d268e244e1feca39e Mon Sep 17 00:00:00 2001 From: Niv Greenstein Date: Tue, 20 Aug 2024 15:44:46 +0300 Subject: [PATCH 110/262] chore: changed syntax --- src/app.ts | 8 +++----- 1 file changed, 3 insertions(+), 5 deletions(-) diff --git a/src/app.ts b/src/app.ts index 805d3dfc..57da10f1 100644 --- a/src/app.ts +++ b/src/app.ts @@ -1,12 +1,10 @@ import { Application } from 'express'; import { DependencyContainer } from 'tsyringe'; -import { RegisterOptions, registerExternalValues } from './containerConfig'; +import { registerExternalValues, RegisterOptions } from './containerConfig'; import { ServerBuilder } from './serverBuilder'; -async function getApp(registerOptions?: RegisterOptions): Promise<{ app: Application; container: DependencyContainer }> { +export const getApp = async (registerOptions?: RegisterOptions): Promise<{ app: Application; container: DependencyContainer }> => { const container = await registerExternalValues(registerOptions); const app = container.resolve(ServerBuilder).build(); return { app, container }; -} - -export { getApp }; +}; From f5d1c2e3c3f8c8b4f44eaed92c8aa0393c590920 Mon Sep 17 00:00:00 2001 From: Niv Greenstein Date: Tue, 20 Aug 2024 15:45:16 +0300 Subject: [PATCH 111/262] fix: added required for WGS84Circle, UTMCircle, geo_context --- openapi3.yaml | 11 +++++++++++ 1 file changed, 11 insertions(+) diff --git a/openapi3.yaml b/openapi3.yaml index 67c3edc4..f9ebb92d 100644 --- a/openapi3.yaml +++ b/openapi3.yaml @@ -948,6 +948,10 @@ components: type: number radius: type: number + required: + - lat + - lon + - radius UTMCircle: type: object properties: @@ -961,12 +965,19 @@ components: maximum: 60 radius: type: number + required: + - x + - y + - zone + - radius geo_context: anyOf: - type: object properties: bbox: $ref: "#/components/schemas/BoundingBox" + required: + - bbox - $ref: "#/components/schemas/WGS84Circle" - $ref: "#/components/schemas/UTMCircle" disable_fuzziness: From 3732cd0a2d655cf7fe7d650e804069cdcd91053b Mon Sep 17 00:00:00 2001 From: Niv Greenstein Date: Tue, 20 Aug 2024 18:48:09 +0300 Subject: [PATCH 112/262] chore: changed import location according eslint --- src/latLon/models/latLonManager.ts | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/src/latLon/models/latLonManager.ts b/src/latLon/models/latLonManager.ts index 628458aa..4f012af2 100644 --- a/src/latLon/models/latLonManager.ts +++ b/src/latLon/models/latLonManager.ts @@ -1,6 +1,7 @@ import { IConfig } from 'config'; import { Logger } from '@map-colonies/js-logger'; import { inject, injectable } from 'tsyringe'; +import { Feature } from 'geojson'; import * as mgrs from 'mgrs'; import { SERVICES } from '../../common/constants'; import { LatLonDAL } from '../DAL/latLonDAL'; @@ -9,7 +10,6 @@ import { convertTilesToUTM, getSubTileByBottomLeftUtmCoor, validateResult } from import { BadRequestError } from '../../common/errors'; import { Tile } from '../../control/tile/models/tile'; import { FeatureCollection, WGS84Coordinate } from '../../common/interfaces'; -import { Feature } from 'geojson'; import { parseGeo } from '../../location/utils'; @injectable() @@ -30,7 +30,7 @@ export class LatLonManager { throw new BadRequestError("Invalid lat lon, check 'lat' and 'lon' keys exists and their values are legal"); } - const utm = convertWgs84ToUTM(lat, lon); + const utm = convertWgs84ToUTM({ longitude: lon, latitude: lat }); if (typeof utm === 'string') { this.logger.warn('LatLonManager.latLonToTile: utm is string'); From 81a975bdbda129458d6c9e504b86ac42429b718e Mon Sep 17 00:00:00 2001 From: Niv Greenstein Date: Tue, 20 Aug 2024 18:48:44 +0300 Subject: [PATCH 113/262] feat(control/utils): added validateGeoContext --- src/control/utils.ts | 23 +++++++++++++++++++++++ 1 file changed, 23 insertions(+) diff --git a/src/control/utils.ts b/src/control/utils.ts index 1d31d9e9..50070735 100644 --- a/src/control/utils.ts +++ b/src/control/utils.ts @@ -9,6 +9,7 @@ import { Item } from '../control/item/models/item'; import { Tile } from '../control/tile/models/tile'; import { Route } from '../control/route/models/route'; import { ConvertSnakeToCamelCase } from '../common/utils'; +import { BBOX_LENGTH } from '../location/interfaces'; import { CONTROL_FIELDS, ELASTIC_KEYWORDS } from './constants'; import { ControlResponse } from './interfaces'; @@ -81,6 +82,28 @@ export const geoContextQuery = ( }; }; +export const validateGeoContext = (geoContext: GeoContext): boolean => { + //TODO: Add validation for possible values + + const messagePrefix = 'geo_context validation: '; + + const validPairs = [['bbox'], ['lat', 'lon', 'radius'], ['x', 'y', 'zone', 'radius']]; + + if (geoContext.bbox !== undefined && geoContext.bbox.length !== BBOX_LENGTH) { + throw new BadRequestError(messagePrefix + 'bbox must contain 4 values'); + } + + if ( + !validPairs.some( + (pair) => pair.every((key) => geoContext[key as keyof GeoContext] !== undefined) && Object.keys(geoContext).length === pair.length + ) + ) { + throw new BadRequestError(messagePrefix + 'geo_context must contain one of the following: {bbox}, {lat, lon, radius}, or {x, y, zone, radius}'); + } + + return true; +}; + // eslint-disable-next-line @typescript-eslint/naming-convention export const additionalControlSearchProperties = (config: IConfig, size: number): Pick => ({ size, From 1b7ed53a879258b9c6c5de694257ce339e34f138 Mon Sep 17 00:00:00 2001 From: Niv Greenstein Date: Tue, 20 Aug 2024 18:49:07 +0300 Subject: [PATCH 114/262] feat(tileManager): use validateGeoContext --- src/control/tile/models/tileManager.ts | 10 +++++----- 1 file changed, 5 insertions(+), 5 deletions(-) diff --git a/src/control/tile/models/tileManager.ts b/src/control/tile/models/tileManager.ts index a8ec5cad..56692a75 100644 --- a/src/control/tile/models/tileManager.ts +++ b/src/control/tile/models/tileManager.ts @@ -4,7 +4,7 @@ import { inject, injectable } from 'tsyringe'; import { estypes } from '@elastic/elasticsearch'; import { SERVICES } from '../../../common/constants'; import { TILE_REPOSITORY_SYMBOL, TileRepository } from '../DAL/tileRepository'; -import { formatResponse } from '../../utils'; +import { formatResponse, validateGeoContext } from '../../utils'; import { TileQueryParams } from '../DAL/queries'; import { FeatureCollection } from '../../../common/interfaces'; import { BadRequestError, NotImplementedError } from '../../../common/errors'; @@ -19,13 +19,16 @@ export class TileManager { ) {} public async getTiles(tileQueryParams: TileQueryParams): Promise> { + if (tileQueryParams.geoContext) { + validateGeoContext(tileQueryParams.geoContext); + } const { limit } = tileQueryParams; if ( (tileQueryParams.tile === undefined && tileQueryParams.mgrs === undefined) || (tileQueryParams.tile !== undefined && tileQueryParams.mgrs !== undefined) ) { - throw new BadRequestError("/control/tiles/queryForTiles: only one of 'tile' or 'mgrs' query parameter must be defined"); + throw new BadRequestError("/control/tiles: only one of 'tile' or 'mgrs' query parameter must be defined"); } //TODO: Handle MGRS query @@ -34,9 +37,6 @@ export class TileManager { } let elasticResponse: estypes.SearchResponse | undefined = undefined; - if (tileQueryParams.tile === undefined) { - throw new BadRequestError('/control/tiles/queryForTiles: tile must be defined'); - } if (tileQueryParams.subTile ?? '') { elasticResponse = await this.tileRepository.getSubTiles(tileQueryParams as Required, limit); From b01e071453b1337e70eec0175db0ca6009f50b51 Mon Sep 17 00:00:00 2001 From: Niv Greenstein Date: Tue, 20 Aug 2024 18:49:53 +0300 Subject: [PATCH 115/262] fix(latLonController): fixed response types --- src/latLon/controllers/latLonController.ts | 17 +++++------------ 1 file changed, 5 insertions(+), 12 deletions(-) diff --git a/src/latLon/controllers/latLonController.ts b/src/latLon/controllers/latLonController.ts index 27cd396a..184e322c 100644 --- a/src/latLon/controllers/latLonController.ts +++ b/src/latLon/controllers/latLonController.ts @@ -8,21 +8,14 @@ import { injectable, inject } from 'tsyringe'; import { SERVICES } from '../../common/constants'; import { LatLonManager } from '../models/latLonManager'; import { Tile } from '../../control/tile/models/tile'; -import { FeatureCollection, WGS84Coordinate } from '../../common/interfaces'; +import { WGS84Coordinate } from '../../common/interfaces'; +import { ControlResponse } from '../../control/interfaces'; -type GetLatLonToTileHandler = RequestHandler< - undefined, - { - tileName: string; - subTileNumber: number[]; - }, - undefined, - GetLatLonToTileQueryParams ->; +type GetLatLonToTileHandler = RequestHandler; -type GetTileToLatLonHandler = RequestHandler, undefined, GetTileToLatLonQueryParams>; +type GetTileToLatLonHandler = RequestHandler, undefined, GetTileToLatLonQueryParams>; -type GetLatLonToMgrsHandler = RequestHandler; +type GetLatLonToMgrsHandler = RequestHandler; type GetMgrsToLatLonHandler = RequestHandler; From 5a9464bc08eabae7c6e1751c9efaeeae6e6fc790 Mon Sep 17 00:00:00 2001 From: Niv Greenstein Date: Tue, 20 Aug 2024 18:50:09 +0300 Subject: [PATCH 116/262] fix: removed required fields. --- openapi3.yaml | 11 ----------- 1 file changed, 11 deletions(-) diff --git a/openapi3.yaml b/openapi3.yaml index f9ebb92d..67c3edc4 100644 --- a/openapi3.yaml +++ b/openapi3.yaml @@ -948,10 +948,6 @@ components: type: number radius: type: number - required: - - lat - - lon - - radius UTMCircle: type: object properties: @@ -965,19 +961,12 @@ components: maximum: 60 radius: type: number - required: - - x - - y - - zone - - radius geo_context: anyOf: - type: object properties: bbox: $ref: "#/components/schemas/BoundingBox" - required: - - bbox - $ref: "#/components/schemas/WGS84Circle" - $ref: "#/components/schemas/UTMCircle" disable_fuzziness: From 0ec25712fde73534f38c6b69564c2b24fcfa39a6 Mon Sep 17 00:00:00 2001 From: Niv Greenstein Date: Tue, 20 Aug 2024 18:50:34 +0300 Subject: [PATCH 117/262] chore: changed version --- package-lock.json | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/package-lock.json b/package-lock.json index a0ae8830..0c1d7e64 100644 --- a/package-lock.json +++ b/package-lock.json @@ -1,12 +1,12 @@ { "name": "geocoding", - "version": "0.5.0", + "version": "0.1.0-prealpha", "lockfileVersion": 2, "requires": true, "packages": { "": { "name": "geocoding", - "version": "0.5.0", + "version": "0.1.0-prealpha", "hasInstallScript": true, "license": "ISC", "dependencies": { From 8527d094437df581270a3e922610b7d10e82255b Mon Sep 17 00:00:00 2001 From: Niv Greenstein Date: Tue, 20 Aug 2024 18:51:11 +0300 Subject: [PATCH 118/262] feat: added bad path tests --- tests/integration/control/tile/tile.spec.ts | 191 ++++++++++++++++++++ 1 file changed, 191 insertions(+) diff --git a/tests/integration/control/tile/tile.spec.ts b/tests/integration/control/tile/tile.spec.ts index 9e80d808..f5ddf68a 100644 --- a/tests/integration/control/tile/tile.spec.ts +++ b/tests/integration/control/tile/tile.spec.ts @@ -272,6 +272,197 @@ describe('/tiles', function () { }); describe('Bad Path', function () { // All requests with status code of 400 + it('should return 400 status code and error message when tile or mgrs is not defined', async function () { + const response = await requestSender.getTiles({} as unknown as GetTilesQueryParams); + + expect(response.status).toBe(httpStatusCodes.BAD_REQUEST); + expect(response.body).toMatchObject({ + message: "/control/tiles: only one of 'tile' or 'mgrs' query parameter must be defined", + }); + }); + + it('should return 400 status code and error message when tile value is empty', async function () { + const response = await requestSender.getTiles({ tile: '', limit: 5, disable_fuzziness: false }); + + expect(response.status).toBe(httpStatusCodes.BAD_REQUEST); + expect(response.body).toMatchObject({ + message: "Empty value found for query parameter 'tile'", + }); + }); + + it('should return 400 status code and error message when tile value is invalid', async function () { + let response = await requestSender.getTiles({ tile: 'invalid', limit: 5, disable_fuzziness: false }); + + expect(response.status).toBe(httpStatusCodes.BAD_REQUEST); + expect(response.body).toMatchObject({ + message: 'request/query/tile must NOT have more than 3 characters', + }); + + response = await requestSender.getTiles({ tile: 'i', limit: 5, disable_fuzziness: false }); + + expect(response.status).toBe(httpStatusCodes.BAD_REQUEST); + expect(response.body).toMatchObject({ + message: 'request/query/tile must NOT have fewer than 2 characters', + }); + }); + + test.each(['invalid', '6a6', '06', '-11', '6 ', ' 6', ' ', ' 6 ', ''])( + 'should return 400 status code and error message when sub_tile value is invalid', + async (sub_tile) => { + const response = await requestSender.getTiles({ tile: 'RIT', sub_tile, limit: 5, disable_fuzziness: false }); + + const message = sub_tile ? 'request/query/sub_tile must match pattern "^[1-9][0-9]*$"' : "Empty value found for query parameter 'sub_tile'"; + + expect(response.status).toBe(httpStatusCodes.BAD_REQUEST); + expect(response.body).toMatchObject({ + message, + }); + } + ); + + test.each>([ + { + geo_context: { x: 300850, y: 4642203, zone: 33, radius: 100 }, + }, + { + geo_context: { lon: 12.598899687444742, lat: 41.90667824634701, radius: 10 }, + }, + { + geo_context: { bbox: [12.554407132912445, 41.84962590648513, 12.652837919839953, 41.94545380230761] }, + }, + { + geo_context_mode: GeoContextMode.BIAS, + }, + { + geo_context_mode: GeoContextMode.FILTER, + }, + ])('should return 400 and message that geo_context and geo_context_mode must be both defined or both undefined', async function (requestParams) { + const response = await requestSender.getTiles({ tile: 'RIT', limit: 5, disable_fuzziness: false, ...requestParams }); + + expect(response.status).toBe(httpStatusCodes.BAD_REQUEST); + expect(response.body).toMatchObject({ + message: '/control/utils/geoContextQuery: geo_context and geo_context_mode must be both defined or both undefined', + }); + }); + + test.each>([ + { + geo_context: { x: 300850, y: 4642203, zone: 33, radius: 100 }, + }, + { + geo_context: { lon: 12.598899687444742, lat: 41.90667824634701, radius: 10 }, + }, + { + geo_context: { bbox: [12.554407132912445, 41.84962590648513, 12.652837919839953, 41.94545380230761] }, + }, + { + geo_context_mode: GeoContextMode.BIAS, + }, + { + geo_context_mode: GeoContextMode.FILTER, + }, + ])('should return 400 and message that geo_context and geo_context_mode must be both defined or both undefined', async function (requestParams) { + const response = await requestSender.getTiles({ tile: 'RIT', limit: 5, disable_fuzziness: false, ...requestParams }); + + expect(response.status).toBe(httpStatusCodes.BAD_REQUEST); + expect(response.body).toMatchObject({ + message: '/control/utils/geoContextQuery: geo_context and geo_context_mode must be both defined or both undefined', + }); + }); + + test.each>([ + { + disable_fuzziness: '' as unknown as boolean, + }, + { + disable_fuzziness: 'True' as unknown as boolean, + }, + { + disable_fuzziness: 'False' as unknown as boolean, + }, + ])('should return 400 status code and error message when disable_fuzziness value is invalid', async function (requestParams) { + const response = await requestSender.getTiles({ tile: 'RIT', limit: 5, ...requestParams }); + + const message = + (requestParams.disable_fuzziness as unknown as string) === '' + ? "Empty value found for query parameter 'disable_fuzziness'" + : 'request/query/disable_fuzziness must be boolean'; + + expect(response.status).toBe(httpStatusCodes.BAD_REQUEST); + expect(response.body).toMatchObject({ + message, + }); + }); + + test.each>([ + { + limit: '' as unknown as number, + }, + { + limit: 0, + }, + { + limit: 101, + }, + ])('should return 400 status code and error message when limit value is invalid', async function (requestParams) { + const response = await requestSender.getTiles({ tile: 'RIT', disable_fuzziness: false, ...requestParams }); + + const message = + (requestParams.limit as unknown as string) === '' + ? "Empty value found for query parameter 'limit'" + : requestParams.limit < 1 + ? 'request/query/limit must be >= 1' + : 'request/query/limit must be <= 15'; + + expect(response.status).toBe(httpStatusCodes.BAD_REQUEST); + expect(response.body).toMatchObject({ + message, + }); + }); + + it('should return 400 status code and error message when geo_context object is invalid', async function () { + const response = await requestSender.getTiles({ + tile: 'RIT', + limit: 5, + disable_fuzziness: false, + geo_context: {} as unknown as GetTilesQueryParams['geo_context'], + }); + + expect(response.status).toBe(httpStatusCodes.BAD_REQUEST); + expect(response.body).toMatchObject({ + message: 'geo_context validation: geocontext must contain one of the following: {bbox}, {lat, lon, radius}, or {x, y, zone, radius}', + }); + }); + + test.each>([ + {}, + { geo_context: { radius: 1 } }, + { geo_context: { lat: 1 } }, + { geo_context: { lon: 1 } }, + { geo_context: { lat: 1, lon: 1 } }, + { geo_context: { lat: 1, radius: 1 } }, + { geo_context: { lon: 1, radius: 1 } }, + { geo_context: { x: 1 } }, + { geo_context: { y: 1 } }, + { geo_context: { zone: 1 } }, + { geo_context: { x: 1, y: 1 } }, + { geo_context: { x: 1, zone: 1 } }, + { geo_context: { y: 1, zone: 1 } }, + { geo_context: { x: 1, radius: 1 } }, + { geo_context: { y: 1, radius: 1 } }, + { geo_context: { zone: 1, radius: 1 } }, + { geo_context: { x: 1, y: 1, zone: 1 } }, + { geo_context: { x: 1, y: 1, radius: 1 } }, + { geo_context: { x: 1, zone: 1, radius: 1 } }, + { geo_context: { y: 1, zone: 1, radius: 1 } }, + ])('should return 400 status code and not pass geo_context validation', async function (requestParams) { + const response = await requestSender.getTiles({ tile: 'RIT', limit: 5, disable_fuzziness: false, ...requestParams }); + + expect(response.status).toBe(httpStatusCodes.BAD_REQUEST); + expect(response.body).toMatchObject({ + message: 'geo_context validation: geo_context must contain one of the following: {bbox}, {lat, lon, radius}, or {x, y, zone, radius}', + }); + }); }); describe('Sad Path', function () { // All requests with status code 4XX-5XX From d30ddf081c9a81c752b0b7bae2060c5184c85cb9 Mon Sep 17 00:00:00 2001 From: Niv Greenstein Date: Wed, 21 Aug 2024 11:45:16 +0300 Subject: [PATCH 119/262] feat(geoContextQuery): added validateGeoContext --- src/control/utils.ts | 2 ++ 1 file changed, 2 insertions(+) diff --git a/src/control/utils.ts b/src/control/utils.ts index 50070735..71bc5ccc 100644 --- a/src/control/utils.ts +++ b/src/control/utils.ts @@ -67,6 +67,8 @@ export const geoContextQuery = ( throw new BadRequestError('/control/utils/geoContextQuery: geo_context and geo_context_mode must be both defined or both undefined'); } + validateGeoContext(geoContext!); + return { [geoContextMode === GeoContextMode.FILTER ? 'filter' : 'should']: [ { From 23cacdcfc4e1ea0b712b20dd25d77013c32b5db0 Mon Sep 17 00:00:00 2001 From: Niv Greenstein Date: Wed, 21 Aug 2024 11:46:42 +0300 Subject: [PATCH 120/262] delete(tileManager): removed validateGeoContext --- src/control/tile/models/tileManager.ts | 5 +---- 1 file changed, 1 insertion(+), 4 deletions(-) diff --git a/src/control/tile/models/tileManager.ts b/src/control/tile/models/tileManager.ts index 56692a75..1e419d32 100644 --- a/src/control/tile/models/tileManager.ts +++ b/src/control/tile/models/tileManager.ts @@ -4,7 +4,7 @@ import { inject, injectable } from 'tsyringe'; import { estypes } from '@elastic/elasticsearch'; import { SERVICES } from '../../../common/constants'; import { TILE_REPOSITORY_SYMBOL, TileRepository } from '../DAL/tileRepository'; -import { formatResponse, validateGeoContext } from '../../utils'; +import { formatResponse } from '../../utils'; import { TileQueryParams } from '../DAL/queries'; import { FeatureCollection } from '../../../common/interfaces'; import { BadRequestError, NotImplementedError } from '../../../common/errors'; @@ -19,9 +19,6 @@ export class TileManager { ) {} public async getTiles(tileQueryParams: TileQueryParams): Promise> { - if (tileQueryParams.geoContext) { - validateGeoContext(tileQueryParams.geoContext); - } const { limit } = tileQueryParams; if ( From f0f80f96b1f3b1af6245298806fddb0b3d75a29d Mon Sep 17 00:00:00 2001 From: Niv Greenstein Date: Wed, 21 Aug 2024 11:47:33 +0300 Subject: [PATCH 121/262] feat(test/control/tile): added tests for invalid geo_context object. --- tests/integration/control/tile/tile.spec.ts | 90 +++++++++++---------- 1 file changed, 46 insertions(+), 44 deletions(-) diff --git a/tests/integration/control/tile/tile.spec.ts b/tests/integration/control/tile/tile.spec.ts index f5ddf68a..2fcfc51d 100644 --- a/tests/integration/control/tile/tile.spec.ts +++ b/tests/integration/control/tile/tile.spec.ts @@ -8,7 +8,7 @@ import { SERVICES } from '../../../../src/common/constants'; import { GetTilesQueryParams } from '../../../../src/control/tile/controllers/tileController'; import { Tile } from '../../../../src/control/tile/models/tile'; import { ControlResponse } from '../../../../src/control/interfaces'; -import { CommonRequestParameters, GeoContextMode } from '../../../../src/common/interfaces'; +import { CommonRequestParameters, GeoContext, GeoContextMode } from '../../../../src/common/interfaces'; import { LATLON_CUSTOM_REPOSITORY_SYMBOL } from '../../../../src/latLon/DAL/latLonRepository'; import { cronLoadTileLatLonDataSymbol } from '../../../../src/latLon/DAL/latLonDAL'; import { TileRequestSender } from './helpers/requestSender'; @@ -420,49 +420,51 @@ describe('/tiles', function () { }); }); - it('should return 400 status code and error message when geo_context object is invalid', async function () { - const response = await requestSender.getTiles({ - tile: 'RIT', - limit: 5, - disable_fuzziness: false, - geo_context: {} as unknown as GetTilesQueryParams['geo_context'], - }); - - expect(response.status).toBe(httpStatusCodes.BAD_REQUEST); - expect(response.body).toMatchObject({ - message: 'geo_context validation: geocontext must contain one of the following: {bbox}, {lat, lon, radius}, or {x, y, zone, radius}', - }); - }); - - test.each>([ - {}, - { geo_context: { radius: 1 } }, - { geo_context: { lat: 1 } }, - { geo_context: { lon: 1 } }, - { geo_context: { lat: 1, lon: 1 } }, - { geo_context: { lat: 1, radius: 1 } }, - { geo_context: { lon: 1, radius: 1 } }, - { geo_context: { x: 1 } }, - { geo_context: { y: 1 } }, - { geo_context: { zone: 1 } }, - { geo_context: { x: 1, y: 1 } }, - { geo_context: { x: 1, zone: 1 } }, - { geo_context: { y: 1, zone: 1 } }, - { geo_context: { x: 1, radius: 1 } }, - { geo_context: { y: 1, radius: 1 } }, - { geo_context: { zone: 1, radius: 1 } }, - { geo_context: { x: 1, y: 1, zone: 1 } }, - { geo_context: { x: 1, y: 1, radius: 1 } }, - { geo_context: { x: 1, zone: 1, radius: 1 } }, - { geo_context: { y: 1, zone: 1, radius: 1 } }, - ])('should return 400 status code and not pass geo_context validation', async function (requestParams) { - const response = await requestSender.getTiles({ tile: 'RIT', limit: 5, disable_fuzziness: false, ...requestParams }); - - expect(response.status).toBe(httpStatusCodes.BAD_REQUEST); - expect(response.body).toMatchObject({ - message: 'geo_context validation: geo_context must contain one of the following: {bbox}, {lat, lon, radius}, or {x, y, zone, radius}', - }); - }); + test.each<(keyof GeoContext)[][]>([[['lat', 'lon', 'radius']], [['x', 'y', 'zone', 'radius']]])( + 'should return 400 for all invalid geo_context object for the keys %s', + async function (keys) { + function generateCombinations(keys: (keyof GeoContext)[]): GeoContext[] { + const combinations: object[] = []; + + function backtrack(current: object, remainingKeys: string[]): void { + if (remainingKeys.length === 0) { + combinations.push(current); + return; + } + + const key = remainingKeys[0]; + const remaining = remainingKeys.slice(1); + + backtrack({ ...current, [key]: 1 }, remaining); + backtrack(current, remaining); + } + + backtrack({}, keys); + + return combinations; + } + + const geoContexts = generateCombinations(keys); + + for (const geo_context of geoContexts) { + if (Object.keys(geo_context).length === keys.length) { + continue; + } + const response = await requestSender.getTiles({ + tile: 'RIT', + limit: 5, + disable_fuzziness: false, + geo_context: JSON.stringify(geo_context) as unknown as GetTilesQueryParams['geo_context'], + geo_context_mode: GeoContextMode.BIAS, + }); + + expect(response.status).toBe(httpStatusCodes.BAD_REQUEST); + expect(response.body).toMatchObject({ + message: 'geo_context validation: geo_context must contain one of the following: {bbox}, {lat, lon, radius}, or {x, y, zone, radius}', + }); + } + } + ); }); describe('Sad Path', function () { // All requests with status code 4XX-5XX From 887b1ef2584bce762710defcec523d2084568611 Mon Sep 17 00:00:00 2001 From: Niv Greenstein Date: Wed, 21 Aug 2024 16:44:04 +0300 Subject: [PATCH 122/262] feat: added tests to item --- .../control/item/helpers/requestSender.ts | 6 +- tests/integration/control/item/item.spec.ts | 394 ++++++++++++++++-- tests/integration/control/item/mockObjects.ts | 96 +++++ 3 files changed, 469 insertions(+), 27 deletions(-) create mode 100644 tests/integration/control/item/mockObjects.ts diff --git a/tests/integration/control/item/helpers/requestSender.ts b/tests/integration/control/item/helpers/requestSender.ts index 28f460fa..78a17c20 100644 --- a/tests/integration/control/item/helpers/requestSender.ts +++ b/tests/integration/control/item/helpers/requestSender.ts @@ -1,5 +1,5 @@ import * as supertest from 'supertest'; -import { GetItemsQueryParams } from '../../../../src/item/controllers/itemController'; +import { GetItemsQueryParams } from '../../../../../src/control/item/controllers/itemController'; export class ItemRequestSender { public constructor(private readonly app: Express.Application) {} @@ -7,9 +7,9 @@ export class ItemRequestSender { public async getItems(queryParams?: GetItemsQueryParams): Promise { return supertest .agent(this.app) - .get('/v1/search/items/') + .get('/search/control/items') .set('Content-Type', 'application/json') - .set('X-API-Key', 'abc123') + .set('x-api-key', 'abc123') .set('x-user-id', 'abc123') .query(queryParams ?? {}); } diff --git a/tests/integration/control/item/item.spec.ts b/tests/integration/control/item/item.spec.ts index d1eb396b..95615992 100644 --- a/tests/integration/control/item/item.spec.ts +++ b/tests/integration/control/item/item.spec.ts @@ -2,12 +2,20 @@ import jsLogger from '@map-colonies/js-logger'; import { trace } from '@opentelemetry/api'; import httpStatusCodes from 'http-status-codes'; -import { getApp } from '../../../src/app'; -import { SERVICES } from '../../../src/common/constants'; -import { GetItemsQueryParams } from '../../../src/item/controllers/itemController'; +import { DataSource } from 'typeorm'; +import { getApp } from '../../../../src/app'; +import { SERVICES } from '../../../../src/common/constants'; +import { GetItemsQueryParams } from '../../../../src/control/item/controllers/itemController'; +import { Item } from '../../../../src/control/item/models/item'; +import { ControlResponse } from '../../../../src/control/interfaces'; +import { CommonRequestParameters, GeoContext, GeoContextMode } from '../../../../src/common/interfaces'; +import { LATLON_CUSTOM_REPOSITORY_SYMBOL } from '../../../../src/latLon/DAL/latLonRepository'; +import { cronLoadTileLatLonDataSymbol } from '../../../../src/latLon/DAL/latLonDAL'; +import { expectedResponse } from '../utils'; import { ItemRequestSender } from './helpers/requestSender'; +import { ITEM_1234, ITEM_1235, ITEM_1236 } from './mockObjects'; -describe('/items', function () { +describe('/search/control/items', function () { let requestSender: ItemRequestSender; beforeEach(async function () { @@ -15,51 +23,389 @@ describe('/items', function () { override: [ { token: SERVICES.LOGGER, provider: { useValue: jsLogger({ enabled: false }) } }, { token: SERVICES.TRACER, provider: { useValue: trace.getTracer('testTracer') } }, + { token: LATLON_CUSTOM_REPOSITORY_SYMBOL, provider: { useValue: {} } }, + { token: DataSource, provider: { useValue: {} } }, + { token: cronLoadTileLatLonDataSymbol, provider: { useValue: {} } }, ], useChild: true, }); + requestSender = new ItemRequestSender(app.app); }); describe('Happy Path', function () { - it('should return 200 status code and the item', async function () { - const response = await requestSender.getItems({ command_name: '4805' }); + it('should return 200 status code and items', async function () { + const requestParams: GetItemsQueryParams = { command_name: '123', limit: 5, disable_fuzziness: false }; + + const response = await requestSender.getItems(requestParams); expect(response.status).toBe(httpStatusCodes.OK); - expect(response).toSatisfyApiSpec(); + // expect(response).toSatisfyApiSpec(); + expect(response.body).toMatchObject>>( + expectedResponse(requestParams, [ITEM_1234, ITEM_1235, ITEM_1236], expect) + ); }); - it('should return 200 status code and empty response', async function () { - const response = await requestSender.getItems({ command_name: '48054805' }); + + it('should return 200 status code and tiles biased by geo_context (bbox)', async function () { + const requestParams: GetItemsQueryParams = { + command_name: '123', + geo_context: { bbox: [12.507611205446722, 41.90406708449768, 12.517586703397825, 41.90966573813128] }, + geo_context_mode: GeoContextMode.BIAS, + limit: 5, + disable_fuzziness: false, + }; + const response = await requestSender.getItems(requestParams); expect(response.status).toBe(httpStatusCodes.OK); - expect(response).toSatisfyApiSpec(); - expect(response.body).toMatchObject({ - type: 'FeatureCollection', - features: [], + // expect(response).toSatisfyApiSpec(); + expect(response.body).toMatchObject>>( + expectedResponse(requestParams, [ITEM_1236, ITEM_1234, ITEM_1235], expect) + ); + }); + + it('should return 200 status code and tiles filtered by geo_context (bbox)', async function () { + const requestParams: GetItemsQueryParams = { + command_name: '123', + geo_context: { bbox: [12.507611205446722, 41.90406708449768, 12.517586703397825, 41.90966573813128] }, + geo_context_mode: GeoContextMode.FILTER, + limit: 5, + disable_fuzziness: false, + }; + const response = await requestSender.getItems(requestParams); + + expect(response.status).toBe(httpStatusCodes.OK); + // expect(response).toSatisfyApiSpec(); + expect(response.body).toMatchObject>>( + expectedResponse(requestParams, [ITEM_1236], expect) + ); + }); + + it('should return 200 status code and tiles biased by geo_context (WGS84)', async function () { + const geo_context = { lon: 12.512345456194254, lat: 41.90673389969385, radius: 10 }; + + const requestParams: GetItemsQueryParams = { + command_name: '123', + geo_context, + geo_context_mode: GeoContextMode.BIAS, + limit: 5, + disable_fuzziness: false, + }; + const response = await requestSender.getItems({ + ...requestParams, + geo_context: JSON.stringify(geo_context) as unknown as GetItemsQueryParams['geo_context'], + }); + + expect(response.status).toBe(httpStatusCodes.OK); + // expect(response).toSatisfyApiSpec(); + expect(response.body).toMatchObject>>( + expectedResponse(requestParams, [ITEM_1236, ITEM_1234, ITEM_1235], expect) + ); + }); + + it('should return 200 status code and tiles filtered by geo_context (WGS84)', async function () { + const geo_context = { lon: 12.512345456194254, lat: 41.90673389969385, radius: 10 }; + const requestParams: GetItemsQueryParams = { + command_name: '123', + geo_context, + geo_context_mode: GeoContextMode.FILTER, + limit: 5, + disable_fuzziness: false, + }; + const response = await requestSender.getItems({ + ...requestParams, + geo_context: JSON.stringify(geo_context) as unknown as GetItemsQueryParams['geo_context'], + }); + + expect(response.status).toBe(httpStatusCodes.OK); + // expect(response).toSatisfyApiSpec(); + expect(response.body).toMatchObject>>( + expectedResponse(requestParams, [ITEM_1236], expect) + ); + }); + + it('should return 200 status code and tiles biased by geo_context (UTM)', async function () { + const geo_context = { x: 293671, y: 4642414, zone: 33, radius: 10 }; + const requestParams: GetItemsQueryParams = { + command_name: '123', + geo_context, + geo_context_mode: GeoContextMode.BIAS, + limit: 5, + disable_fuzziness: false, + }; + const response = await requestSender.getItems({ + ...requestParams, + geo_context: JSON.stringify(geo_context) as unknown as GetItemsQueryParams['geo_context'], + }); + + expect(response.status).toBe(httpStatusCodes.OK); + // expect(response).toSatisfyApiSpec(); + expect(response.body).toMatchObject>>( + expectedResponse(requestParams, [ITEM_1236, ITEM_1234, ITEM_1235], expect) + ); + }); + + it('should return 200 status code and tiles filtered by geo_context (UTM)', async function () { + const geo_context = { x: 293671, y: 4642414, zone: 33, radius: 10 }; + const requestParams: GetItemsQueryParams = { + command_name: '123', + geo_context, + geo_context_mode: GeoContextMode.FILTER, + limit: 5, + disable_fuzziness: false, + }; + const response = await requestSender.getItems({ + ...requestParams, + geo_context: JSON.stringify(geo_context) as unknown as GetItemsQueryParams['geo_context'], }); + + expect(response.status).toBe(httpStatusCodes.OK); + // expect(response).toSatisfyApiSpec(); + expect(response.body).toMatchObject>>( + expectedResponse(requestParams, [ITEM_1236], expect) + ); + }); + + it('should return 200 status code and send 1234 item as disable_fuzziness is true', async function () { + const requestParams: GetItemsQueryParams = { + command_name: '1234', + limit: 5, + disable_fuzziness: true, + }; + const response = await requestSender.getItems(requestParams); + + expect(response.status).toBe(httpStatusCodes.OK); + // expect(response).toSatisfyApiSpec(); + expect(response.body).toMatchObject>>( + expectedResponse(requestParams, [ITEM_1234], expect) + ); + }); + + it('should return 200 status code and no item as disable_fuzziness is true', async function () { + const requestParams: GetItemsQueryParams = { + command_name: '123', + limit: 5, + disable_fuzziness: true, + }; + const response = await requestSender.getItems(requestParams); + + expect(response.status).toBe(httpStatusCodes.OK); + // expect(response).toSatisfyApiSpec(); + expect(response.body).toMatchObject>>( + expectedResponse(requestParams, [], expect) + ); + }); + + it('should return 200 status code and items in sub_tile', async function () { + const requestParams: GetItemsQueryParams = { + command_name: '123', + sub_tile: '37', + limit: 5, + disable_fuzziness: false, + }; + const response = await requestSender.getItems(requestParams); + + expect(response.status).toBe(httpStatusCodes.OK); + // expect(response).toSatisfyApiSpec(); + expect(response.body).toMatchObject>>( + expectedResponse(requestParams, [ITEM_1234, ITEM_1235], expect) + ); }); }); describe('Bad Path', function () { // All requests with status code of 400 - it('Should return 400 status code and meessage "request/query must have required property \'command_name\'"', async function () { - const message = "request/query must have required property 'command_name'"; + it("should return 400 status code and error message when item's command_name", async function () { + const response = await requestSender.getItems({} as unknown as GetItemsQueryParams); + + expect(response.status).toBe(httpStatusCodes.BAD_REQUEST); + expect(response.body).toMatchObject({ + message: "request/query must have required property 'command_name'", + }); + }); + + it('should return 400 status code and error message when command_name value is empty', async function () { + const response = await requestSender.getItems({ command_name: '', limit: 5, disable_fuzziness: false }); - const response = await requestSender.getItems(); + expect(response.status).toBe(httpStatusCodes.BAD_REQUEST); + expect(response.body).toMatchObject({ + message: "Empty value found for query parameter 'command_name'", + }); + }); + + it('should return 400 status code and error message when tile value is invalid', async function () { + let response = await requestSender.getItems({ command_name: '1234', tile: 'invalid', limit: 5, disable_fuzziness: false }); + + expect(response.status).toBe(httpStatusCodes.BAD_REQUEST); + expect(response.body).toMatchObject({ + message: 'request/query/tile must NOT have more than 3 characters', + }); + + response = await requestSender.getItems({ command_name: '1234', tile: 'i', limit: 5, disable_fuzziness: false }); + + expect(response.status).toBe(httpStatusCodes.BAD_REQUEST); + expect(response.body).toMatchObject({ + message: 'request/query/tile must NOT have fewer than 3 characters', + }); + }); + + test.each(['invalid', '6a6', '06', '-11', '6 ', ' 6', ' ', ' 6 ', ''])( + 'should return 400 status code and error message when sub_tile value is invalid', + async (sub_tile) => { + const response = await requestSender.getItems({ command_name: '1234', tile: 'RIT', sub_tile, limit: 5, disable_fuzziness: false }); + + const message = sub_tile ? 'request/query/sub_tile must match pattern "^[1-9][0-9]*$"' : "Empty value found for query parameter 'sub_tile'"; + + expect(response.status).toBe(httpStatusCodes.BAD_REQUEST); + expect(response.body).toMatchObject({ + message, + }); + } + ); + + test.each>([ + { + geo_context: { x: 300850, y: 4642203, zone: 33, radius: 100 }, + }, + { + geo_context: { lon: 12.598899687444742, lat: 41.90667824634701, radius: 10 }, + }, + { + geo_context: { bbox: [12.554407132912445, 41.84962590648513, 12.652837919839953, 41.94545380230761] }, + }, + { + geo_context_mode: GeoContextMode.BIAS, + }, + { + geo_context_mode: GeoContextMode.FILTER, + }, + ])('should return 400 and message that geo_context and geo_context_mode must be both defined or both undefined', async function (requestParams) { + const response = await requestSender.getItems({ command_name: '1234', limit: 5, disable_fuzziness: false, ...requestParams }); + + expect(response.status).toBe(httpStatusCodes.BAD_REQUEST); + expect(response.body).toMatchObject({ + message: '/control/utils/geoContextQuery: geo_context and geo_context_mode must be both defined or both undefined', + }); + }); + + test.each>([ + { + geo_context: { x: 300850, y: 4642203, zone: 33, radius: 100 }, + }, + { + geo_context: { lon: 12.598899687444742, lat: 41.90667824634701, radius: 10 }, + }, + { + geo_context: { bbox: [12.554407132912445, 41.84962590648513, 12.652837919839953, 41.94545380230761] }, + }, + { + geo_context_mode: GeoContextMode.BIAS, + }, + { + geo_context_mode: GeoContextMode.FILTER, + }, + ])('should return 400 and message that geo_context and geo_context_mode must be both defined or both undefined', async function (requestParams) { + const response = await requestSender.getItems({ command_name: '1234', limit: 5, disable_fuzziness: false, ...requestParams }); expect(response.status).toBe(httpStatusCodes.BAD_REQUEST); - expect(response.body).toMatchObject({ message }); - expect(response).toSatisfyApiSpec(); + expect(response.body).toMatchObject({ + message: '/control/utils/geoContextQuery: geo_context and geo_context_mode must be both defined or both undefined', + }); }); - it('Should return 400 status code for unknown parameter', async function () { - const parameter = 'test1234'; - const message = `Unknown query parameter '${parameter}'`; - const response = await requestSender.getItems({ [parameter]: parameter } as unknown as GetItemsQueryParams); + test.each>([ + { + disable_fuzziness: '' as unknown as boolean, + }, + { + disable_fuzziness: 'True' as unknown as boolean, + }, + { + disable_fuzziness: 'False' as unknown as boolean, + }, + ])('should return 400 status code and error message when disable_fuzziness value is invalid', async function (requestParams) { + const response = await requestSender.getItems({ command_name: '1234', limit: 5, ...requestParams }); + + const message = + (requestParams.disable_fuzziness as unknown as string) === '' + ? "Empty value found for query parameter 'disable_fuzziness'" + : 'request/query/disable_fuzziness must be boolean'; expect(response.status).toBe(httpStatusCodes.BAD_REQUEST); - expect(response.body).toMatchObject({ message }); - expect(response).toSatisfyApiSpec(); + expect(response.body).toMatchObject({ + message, + }); }); + + test.each>([ + { + limit: '' as unknown as number, + }, + { + limit: 0, + }, + { + limit: 101, + }, + ])('should return 400 status code and error message when limit value is invalid', async function (requestParams) { + const response = await requestSender.getItems({ command_name: '1234', disable_fuzziness: false, ...requestParams }); + + const message = + (requestParams.limit as unknown as string) === '' + ? "Empty value found for query parameter 'limit'" + : requestParams.limit < 1 + ? 'request/query/limit must be >= 1' + : 'request/query/limit must be <= 15'; + + expect(response.status).toBe(httpStatusCodes.BAD_REQUEST); + expect(response.body).toMatchObject({ + message, + }); + }); + + test.each<(keyof GeoContext)[][]>([[['lat', 'lon', 'radius']], [['x', 'y', 'zone', 'radius']]])( + 'should return 400 for all invalid geo_context object for the keys %s', + async function (keys) { + function generateCombinations(keys: (keyof GeoContext)[]): GeoContext[] { + const combinations: object[] = []; + + function backtrack(current: object, remainingKeys: string[]): void { + if (remainingKeys.length === 0) { + combinations.push(current); + return; + } + + const key = remainingKeys[0]; + const remaining = remainingKeys.slice(1); + + backtrack({ ...current, [key]: 1 }, remaining); + backtrack(current, remaining); + } + + backtrack({}, keys); + + return combinations; + } + + const geoContexts = generateCombinations(keys); + + for (const geo_context of geoContexts) { + if (Object.keys(geo_context).length === keys.length) { + continue; + } + const response = await requestSender.getItems({ + command_name: '1234', + limit: 5, + disable_fuzziness: false, + geo_context: JSON.stringify(geo_context) as unknown as GetItemsQueryParams['geo_context'], + geo_context_mode: GeoContextMode.BIAS, + }); + + expect(response.status).toBe(httpStatusCodes.BAD_REQUEST); + expect(response.body).toMatchObject({ + message: 'geo_context validation: geo_context must contain one of the following: {bbox}, {lat, lon, radius}, or {x, y, zone, radius}', + }); + } + } + ); }); describe('Sad Path', function () { // All requests with status code 4XX-5XX diff --git a/tests/integration/control/item/mockObjects.ts b/tests/integration/control/item/mockObjects.ts new file mode 100644 index 00000000..de32974d --- /dev/null +++ b/tests/integration/control/item/mockObjects.ts @@ -0,0 +1,96 @@ +/* eslint-disable @typescript-eslint/naming-convention */ +/* eslint-disable @typescript-eslint/no-magic-numbers */ +import { Item } from '../../../../src/control/item/models/item'; + +export const ITEM_1234: Item = { + type: 'Feature', + geometry: { + coordinates: [ + [ + [12.432792582620323, 41.9327692706986], + [12.432648028923637, 41.93209008126263], + [12.43295235249525, 41.93189198298137], + [12.435105441764364, 41.93235609798671], + [12.435516278586334, 41.93274663116725], + [12.43637599267521, 41.93308056343986], + [12.436026020567567, 41.93386161853829], + [12.432792582620323, 41.9327692706986], + ], + ], + type: 'Polygon', + }, + properties: { + OBJECT_COMMAND_NAME: '1234', + TILE_NAME: 'RIT', + SUB_TILE_ID: '37', + ENTITY_HEB: 'hotel', + TYPE: 'ITEM', + }, +}; + +export const ITEM_1235: Item = { + type: 'Feature', + geometry: { + coordinates: [ + [ + [12.454224879037952, 41.93533128513971], + [12.4536294370983, 41.93493944442386], + [12.453354617740985, 41.934479454425286], + [12.453308814515196, 41.93386612926582], + [12.453789748389482, 41.933065390320394], + [12.454660009686364, 41.93255427509931], + [12.455667680662373, 41.93265649847106], + [12.456217319376208, 41.933355020461335], + [12.456217319376208, 41.934309086916585], + [12.45569058227531, 41.935092773686364], + [12.454957730656645, 41.93534832163783], + [12.454224879037952, 41.93533128513971], + ], + ], + type: 'Polygon', + }, + properties: { + OBJECT_COMMAND_NAME: '1235', + TILE_NAME: 'RIT', + SUB_TILE_ID: '37', + ENTITY_HEB: 'oplympic stadium', + TYPE: 'ITEM', + }, +}; + +export const ITEM_1236: Item = { + type: 'Feature', + geometry: { + coordinates: [ + [ + [12.508591887289015, 41.908719199148294], + [12.5109416268715, 41.90414674778023], + [12.515619548794433, 41.905558627360875], + [12.514541686600694, 41.906697734719614], + [12.514929038978266, 41.9065233923005], + [12.515546733877699, 41.90711121610428], + [12.51436197480865, 41.90815873484965], + [12.513612640013008, 41.90876915151631], + [12.512974692823434, 41.90872393566693], + [12.512863305218872, 41.90807583830912], + [12.513561316395936, 41.90754911551221], + [12.513962675013943, 41.907221129768516], + [12.51419876831929, 41.906986853205694], + [12.513954805237745, 41.906893142339726], + [12.512516460769376, 41.90804005137895], + [12.511855399516293, 41.90857302052004], + [12.51176096219396, 41.908660872149284], + [12.508762577222939, 41.908924426310676], + [12.508591887289015, 41.908719199148294], + ], + ], + type: 'Polygon', + }, + properties: { + OBJECT_COMMAND_NAME: '1236', + TILE_NAME: 'RIT', + SUB_TILE_ID: '38', + ENTITY_HEB: 'hospital', + TYPE: 'ITEM', + }, +}; From d3b84eda6c9addd2a776a5770cc323487d8800bf Mon Sep 17 00:00:00 2001 From: Niv Greenstein Date: Wed, 21 Aug 2024 16:44:26 +0300 Subject: [PATCH 123/262] feat: added mock data to control mock --- devScripts/controlElasticsearchData.json | 125 ++++++++++++++++++++++- 1 file changed, 124 insertions(+), 1 deletion(-) diff --git a/devScripts/controlElasticsearchData.json b/devScripts/controlElasticsearchData.json index 0935f2d6..73da7efd 100644 --- a/devScripts/controlElasticsearchData.json +++ b/devScripts/controlElasticsearchData.json @@ -91,6 +91,101 @@ } } }, + { + "_id": "CONTROL.ITEMS2", + "_source": { + "type": "Feature", + "id": 212, + "geometry": { + "coordinates": [ + [ + [12.454224879037952, 41.93533128513971], + [12.4536294370983, 41.93493944442386], + [12.453354617740985, 41.934479454425286], + [12.453308814515196, 41.93386612926582], + [12.453789748389482, 41.933065390320394], + [12.454660009686364, 41.93255427509931], + [12.455667680662373, 41.93265649847106], + [12.456217319376208, 41.933355020461335], + [12.456217319376208, 41.934309086916585], + [12.45569058227531, 41.935092773686364], + [12.454957730656645, 41.93534832163783], + [12.454224879037952, 41.93533128513971] + ] + ], + "type": "Polygon" + }, + "properties": { + "OBJECTID": 212, + "F_CODE": 23001, + "F_ATT": 6021, + "VIEW_SCALE_50K_CONTROL": 6041, + "NAME": null, + "GFID": "{69a2805c-3977-4123-b97d-c149287af9d4}", + "OBJECT_COMMAND_NAME": "1235", + "SUB_TILE_NAME": null, + "TILE_NAME": "RIT", + "TILE_ID": "36, 7310, 3561", + "SUB_TILE_ID": "37", + "SHAPE.AREA": 2.65, + "SHAPE.LEN": 0.0044192097656775, + "LAYER_NAME": "CONTROL.ITEMS", + "ENTITY_HEB": "oplympic stadium", + "TYPE": "ITEM" + } + } + }, + { + "_id": "CONTROL.ITEMS3", + "_source": { + "type": "Feature", + "id": 213, + "geometry": { + "coordinates": [ + [ + [12.508591887289015, 41.908719199148294], + [12.5109416268715, 41.90414674778023], + [12.515619548794433, 41.905558627360875], + [12.514541686600694, 41.906697734719614], + [12.514929038978266, 41.9065233923005], + [12.515546733877699, 41.90711121610428], + [12.51436197480865, 41.90815873484965], + [12.513612640013008, 41.90876915151631], + [12.512974692823434, 41.90872393566693], + [12.512863305218872, 41.90807583830912], + [12.513561316395936, 41.90754911551221], + [12.513962675013943, 41.907221129768516], + [12.51419876831929, 41.906986853205694], + [12.513954805237745, 41.906893142339726], + [12.512516460769376, 41.90804005137895], + [12.511855399516293, 41.90857302052004], + [12.51176096219396, 41.908660872149284], + [12.508762577222939, 41.908924426310676], + [12.508591887289015, 41.908719199148294] + ] + ], + "type": "Polygon" + }, + "properties": { + "OBJECTID": 213, + "F_CODE": 23002, + "F_ATT": 6022, + "VIEW_SCALE_50K_CONTROL": 6042, + "NAME": null, + "GFID": "{935265e6-402f-4824-a193-e8725f73212e}", + "OBJECT_COMMAND_NAME": "1236", + "SUB_TILE_NAME": null, + "TILE_NAME": "RIT", + "TILE_ID": "36, 7310, 3581", + "SUB_TILE_ID": "38", + "SHAPE.AREA": 2.65, + "SHAPE.LEN": 0.0044192097656775, + "LAYER_NAME": "CONTROL.ITEMS", + "ENTITY_HEB": "hospital", + "TYPE": "ITEM" + } + } + }, { "_id": "CONTROL.SUB_TILES", "_source": { @@ -165,7 +260,35 @@ }, "properties": { "OBJECTID": 2, - "OBJECT_COMMAND_NAME": "camilluccia", + "OBJECT_COMMAND_NAME": "via camillucciaA", + "FIRST_GFID": null, + "SHAPE_Length": 4.1, + "F_CODE": 8754, + "F_ATT": 951, + "LAYER_NAME": "CONTROL.ROUTES", + "ENTITY_HEB": "route", + "TYPE": "ROUTE" + } + } + }, + { + "_id": "CONTROL.ROUTES2", + "_source": { + "type": "Feature", + "id": 3, + "geometry": { + "coordinates": [ + [12.445818466287847, 41.93029376141277], + [12.446047161911423, 41.930040913942264], + [12.446171038707206, 41.9297762500957], + [12.446167862379838, 41.92945014491775], + [12.446075748864388, 41.9286136066203] + ], + "type": "LineString" + }, + "properties": { + "OBJECTID": 3, + "OBJECT_COMMAND_NAME": "via camillucciaB", "FIRST_GFID": null, "SHAPE_Length": 4.1, "F_CODE": 8754, From 45d6252ba382bd57539081c5af3e7548c8e123fe Mon Sep 17 00:00:00 2001 From: Niv Greenstein Date: Wed, 21 Aug 2024 16:44:53 +0300 Subject: [PATCH 124/262] feat(CONTROL_FIELDS): added properties.TIED_TO --- src/control/constants.ts | 1 + 1 file changed, 1 insertion(+) diff --git a/src/control/constants.ts b/src/control/constants.ts index 1b8e99e9..755994cd 100644 --- a/src/control/constants.ts +++ b/src/control/constants.ts @@ -17,4 +17,5 @@ export const CONTROL_FIELDS = [ 'properties.ENTITY_HEB', 'properties.SUB_TILE_ID', 'properties.SECTION', + 'properties.TIED_TO', ]; From c17d2e1ca0e25c85555e29489549e2accb63bbdd Mon Sep 17 00:00:00 2001 From: Niv Greenstein Date: Wed, 21 Aug 2024 16:45:24 +0300 Subject: [PATCH 125/262] fix(tests/tiles): changed place of mock data --- .../control/tile/{utils.ts => mockObjects.ts} | 31 ------------------- tests/integration/control/tile/tile.spec.ts | 5 +-- 2 files changed, 3 insertions(+), 33 deletions(-) rename tests/integration/control/tile/{utils.ts => mockObjects.ts} (63%) diff --git a/tests/integration/control/tile/utils.ts b/tests/integration/control/tile/mockObjects.ts similarity index 63% rename from tests/integration/control/tile/utils.ts rename to tests/integration/control/tile/mockObjects.ts index 24586aa9..2292a2cd 100644 --- a/tests/integration/control/tile/utils.ts +++ b/tests/integration/control/tile/mockObjects.ts @@ -1,24 +1,7 @@ /* eslint-disable @typescript-eslint/naming-convention */ /* eslint-disable @typescript-eslint/no-magic-numbers */ -import { CommonRequestParameters } from '../../../../src/common/interfaces'; -import { ControlResponse } from '../../../../src/control/interfaces'; -import { GetTilesQueryParams } from '../../../../src/control/tile/controllers/tileController'; import { Tile } from '../../../../src/control/tile/models/tile'; -const expectedTileWithScore = (tile: Tile, expect: jest.Expect): ControlResponse['features'][number] => ({ - ...tile, - _score: expect.any(Number) as number, -}); - -const expectedGeocodingElasticResponseMetrics = ( - resultsCount: number, - expect: jest.Expect -): NonNullable['geocoding']>['response'] => ({ - results_count: resultsCount, - max_score: expect.any(Number) as number, - match_latency_ms: expect.any(Number) as number, -}); - export const RIT_TILE: Tile = { type: 'Feature', geometry: { @@ -100,17 +83,3 @@ export const SUB_TILE_65: Tile = { type: 'Polygon', }, }; - -export const expectedResponse = ( - requestParams: GetTilesQueryParams, - tiles: Tile[], - expect: jest.Expect -): ControlResponse> => ({ - type: 'FeatureCollection', - geocoding: { - version: process.env.npm_package_version, - query: requestParams, - response: expectedGeocodingElasticResponseMetrics(tiles.length, expect), - }, - features: tiles.map((tile) => expectedTileWithScore(tile, expect)), -}); diff --git a/tests/integration/control/tile/tile.spec.ts b/tests/integration/control/tile/tile.spec.ts index 2fcfc51d..10c8ffcc 100644 --- a/tests/integration/control/tile/tile.spec.ts +++ b/tests/integration/control/tile/tile.spec.ts @@ -11,10 +11,11 @@ import { ControlResponse } from '../../../../src/control/interfaces'; import { CommonRequestParameters, GeoContext, GeoContextMode } from '../../../../src/common/interfaces'; import { LATLON_CUSTOM_REPOSITORY_SYMBOL } from '../../../../src/latLon/DAL/latLonRepository'; import { cronLoadTileLatLonDataSymbol } from '../../../../src/latLon/DAL/latLonDAL'; +import { expectedResponse } from '../utils'; import { TileRequestSender } from './helpers/requestSender'; -import { RIC_TILE, RIT_TILE, SUB_TILE_65, SUB_TILE_66, expectedResponse } from './utils'; +import { RIC_TILE, RIT_TILE, SUB_TILE_65, SUB_TILE_66 } from './mockObjects'; -describe('/tiles', function () { +describe('/search/control/tiles', function () { let requestSender: TileRequestSender; beforeEach(async function () { From 54bb9b70e19957ba3b771e33ae6842e99202efa4 Mon Sep 17 00:00:00 2001 From: Niv Greenstein Date: Wed, 21 Aug 2024 16:46:02 +0300 Subject: [PATCH 126/262] fix(openapi): added minLength and maxLength to tile query in /search/control/items --- openapi3.yaml | 2 ++ 1 file changed, 2 insertions(+) diff --git a/openapi3.yaml b/openapi3.yaml index 67c3edc4..f3b5cc9e 100644 --- a/openapi3.yaml +++ b/openapi3.yaml @@ -250,6 +250,8 @@ paths: description: "The tile the item in it (full name of it)" schema: type: "string" + minLength: 3 + maxLength: 3 required: false - name: "sub_tile" From 1c13b33af74c2e68877d2e35c24760400a560f01 Mon Sep 17 00:00:00 2001 From: Niv Greenstein Date: Wed, 21 Aug 2024 16:46:19 +0300 Subject: [PATCH 127/262] feat: added common utils to control testing --- tests/integration/control/utils.ts | 37 ++++++++++++++++++++++++++++++ 1 file changed, 37 insertions(+) create mode 100644 tests/integration/control/utils.ts diff --git a/tests/integration/control/utils.ts b/tests/integration/control/utils.ts new file mode 100644 index 00000000..5f03ecd0 --- /dev/null +++ b/tests/integration/control/utils.ts @@ -0,0 +1,37 @@ +/* eslint-disable @typescript-eslint/naming-convention */ +import { CommonRequestParameters } from '../../../src/common/interfaces'; +import { ControlResponse } from '../../../src/control/interfaces'; +import { GetItemsQueryParams } from '../../../src/control/item/controllers/itemController'; +import { Item } from '../../../src/control/item/models/item'; +import { GetRoutesQueryParams } from '../../../src/control/route/controllers/routeController'; +import { Route } from '../../../src/control/route/models/route'; +import { GetTilesQueryParams } from '../../../src/control/tile/controllers/tileController'; +import { Tile } from '../../../src/control/tile/models/tile'; + +const expectedObjectWithScore = (obj: T, expect: jest.Expect): ControlResponse['features'][number] => ({ + ...obj, + _score: expect.any(Number) as number, +}); + +const expectedGeocodingElasticResponseMetrics = ( + resultsCount: number, + expect: jest.Expect +): NonNullable['geocoding']>['response'] => ({ + results_count: resultsCount, + max_score: expect.any(Number) as number, + match_latency_ms: expect.any(Number) as number, +}); + +export const expectedResponse = ( + requestParams: U, + arr: T[], + expect: jest.Expect +): ControlResponse> => ({ + type: 'FeatureCollection', + geocoding: { + version: process.env.npm_package_version, + query: requestParams, + response: expectedGeocodingElasticResponseMetrics(arr.length, expect), + }, + features: arr.map((item) => expectedObjectWithScore(item, expect)), +}); From e12f3ee2b724ce3b967ec025d70176b1a4d9976d Mon Sep 17 00:00:00 2001 From: Niv Greenstein Date: Wed, 21 Aug 2024 17:41:18 +0300 Subject: [PATCH 128/262] delete(control/utils): removed unused foreach --- src/control/utils.ts | 20 ++++++-------------- 1 file changed, 6 insertions(+), 14 deletions(-) diff --git a/src/control/utils.ts b/src/control/utils.ts index 71bc5ccc..31aea8f2 100644 --- a/src/control/utils.ts +++ b/src/control/utils.ts @@ -26,12 +26,12 @@ export const convertCamelToSnakeCase = (obj: Record): Record( elasticResponse: estypes.SearchResponse, - requestParams?: CommonRequestParameters | ConvertSnakeToCamelCase | undefined + requestParams: CommonRequestParameters | ConvertSnakeToCamelCase ): ControlResponse => ({ type: 'FeatureCollection', geocoding: { version: process.env.npm_package_version, - query: requestParams ? convertCamelToSnakeCase(requestParams as Record) : undefined, + query: convertCamelToSnakeCase(requestParams as Record), response: { /* eslint-disable @typescript-eslint/naming-convention */ results_count: elasticResponse.hits.hits.length, @@ -41,18 +41,10 @@ export const formatResponse = ( }, }, features: [ - ...(elasticResponse.hits.hits.map((item) => { - const source = item._source; - if (source?.properties) { - Object.keys(source.properties).forEach((key) => { - // eslint-disable-next-line @typescript-eslint/no-unnecessary-condition - if (source.properties !== null && source.properties[key as keyof typeof source.properties] == null) { - delete source.properties[key as keyof typeof source.properties]; - } - }); - } - return { ...source, _score: item._score }; - }) as (T & Pick, '_score'>)[]), + ...(elasticResponse.hits.hits.map((item) => ({ + ...item._source, + _score: item._score, + })) as (T & Pick, '_score'>)[]), ], }); From d40744b86a123a6f6b8feb3a654dde33cd43d64f Mon Sep 17 00:00:00 2001 From: Niv Greenstein Date: Wed, 21 Aug 2024 17:41:43 +0300 Subject: [PATCH 129/262] feat: added test for items inside tile --- tests/integration/control/item/item.spec.ts | 24 +++++++++++++++++++++ 1 file changed, 24 insertions(+) diff --git a/tests/integration/control/item/item.spec.ts b/tests/integration/control/item/item.spec.ts index 95615992..18627d34 100644 --- a/tests/integration/control/item/item.spec.ts +++ b/tests/integration/control/item/item.spec.ts @@ -46,6 +46,30 @@ describe('/search/control/items', function () { ); }); + it('should return 200 status code and items in tile RIT', async function () { + const requestParams: GetItemsQueryParams = { command_name: '123', tile: 'RIT', limit: 5, disable_fuzziness: false }; + + const response = await requestSender.getItems(requestParams); + + expect(response.status).toBe(httpStatusCodes.OK); + // expect(response).toSatisfyApiSpec(); + expect(response.body).toMatchObject>>( + expectedResponse(requestParams, [ITEM_1234, ITEM_1235, ITEM_1236], expect) + ); + }); + + it('should return 200 status code and no items in tile RIC', async function () { + const requestParams: GetItemsQueryParams = { command_name: '123', tile: 'RIC', limit: 5, disable_fuzziness: false }; + + const response = await requestSender.getItems(requestParams); + + expect(response.status).toBe(httpStatusCodes.OK); + // expect(response).toSatisfyApiSpec(); + expect(response.body).toMatchObject>>( + expectedResponse(requestParams, [], expect) + ); + }); + it('should return 200 status code and tiles biased by geo_context (bbox)', async function () { const requestParams: GetItemsQueryParams = { command_name: '123', From a49060d0a9d28bbadf5e8e88877b6b6691954aef Mon Sep 17 00:00:00 2001 From: Niv Greenstein Date: Wed, 21 Aug 2024 17:42:37 +0300 Subject: [PATCH 130/262] feat(test/tile): added test for no subtiles as disable_fuzziness true not providing exact match --- tests/integration/control/tile/tile.spec.ts | 34 +++++++++++++++++++++ 1 file changed, 34 insertions(+) diff --git a/tests/integration/control/tile/tile.spec.ts b/tests/integration/control/tile/tile.spec.ts index 10c8ffcc..5db433f3 100644 --- a/tests/integration/control/tile/tile.spec.ts +++ b/tests/integration/control/tile/tile.spec.ts @@ -210,6 +210,22 @@ describe('/search/control/tiles', function () { ); }); + it('should return 200 status code and no sub tiles as disable_fuzziness: true', async function () { + const requestParams: GetTilesQueryParams = { + tile: 'RIT', + sub_tile: '11', + limit: 5, + disable_fuzziness: true, + }; + const response = await requestSender.getTiles(requestParams); + + expect(response.status).toBe(httpStatusCodes.OK); + // expect(response).toSatisfyApiSpec(); + expect(response.body).toMatchObject>>( + expectedResponse(requestParams, [], expect) + ); + }); + it('should return 200 status code and sub_tile "66" filtered by geo_context (UTM)', async function () { const geo_context = { x: 288240, y: 4645787, zone: 33, radius: 100 }; const requestParams: GetTilesQueryParams = { @@ -307,6 +323,24 @@ describe('/search/control/tiles', function () { }); }); + test.each([[[1]], [[1, 1]], [[1, 1, 1]], [[1, 1, 1, 1, 1]]])( + 'should return 400 status code and error message when bbox not containing 4 values', + async function (bbox) { + const response = await requestSender.getTiles({ + tile: 'RIT', + limit: 5, + disable_fuzziness: false, + geo_context: { bbox }, + geo_context_mode: GeoContextMode.FILTER, + }); + + expect(response.status).toBe(httpStatusCodes.BAD_REQUEST); + expect(response.body).toMatchObject({ + message: 'geo_context validation: bbox must contain 4 values', + }); + } + ); + test.each(['invalid', '6a6', '06', '-11', '6 ', ' 6', ' ', ' 6 ', ''])( 'should return 400 status code and error message when sub_tile value is invalid', async (sub_tile) => { From ad8946eb698319f8c66f894b71e3492a3c3c34a7 Mon Sep 17 00:00:00 2001 From: Niv Greenstein Date: Wed, 21 Aug 2024 17:43:05 +0300 Subject: [PATCH 131/262] feat(test/route): created tests for routes --- .../control/route/helpers/requestSender.ts | 6 +- .../integration/control/route/mockObjects.ts | 71 +++ tests/integration/control/route/route.spec.ts | 530 +++++++++++++++++- 3 files changed, 582 insertions(+), 25 deletions(-) create mode 100644 tests/integration/control/route/mockObjects.ts diff --git a/tests/integration/control/route/helpers/requestSender.ts b/tests/integration/control/route/helpers/requestSender.ts index adac30a7..d5dd4b68 100644 --- a/tests/integration/control/route/helpers/requestSender.ts +++ b/tests/integration/control/route/helpers/requestSender.ts @@ -1,5 +1,5 @@ import * as supertest from 'supertest'; -import { GetRoutesQueryParams } from '../../../../src/route/controllers/routeController'; +import { GetRoutesQueryParams } from '../../../../../src/control/route/controllers/routeController'; export class RouteRequestSender { public constructor(private readonly app: Express.Application) {} @@ -7,9 +7,9 @@ export class RouteRequestSender { public async getRoutes(queryParams?: GetRoutesQueryParams): Promise { return supertest .agent(this.app) - .get('/v1/search/routes/') + .get('/search/control/routes') .set('Content-Type', 'application/json') - .set('X-API-Key', 'abc123') + .set('x-api-key', 'abc123') .set('x-user-id', 'abc123') .query(queryParams ?? {}); } diff --git a/tests/integration/control/route/mockObjects.ts b/tests/integration/control/route/mockObjects.ts new file mode 100644 index 00000000..34b25845 --- /dev/null +++ b/tests/integration/control/route/mockObjects.ts @@ -0,0 +1,71 @@ +/* eslint-disable @typescript-eslint/naming-convention */ +/* eslint-disable @typescript-eslint/no-magic-numbers */ +import { Route } from '../../../../src/control/route/models/route'; + +export const ROUTE_VIA_CAMILLUCCIA_A: Route = { + type: 'Feature', + geometry: { + coordinates: [ + [12.443243654365062, 41.93890891937724], + [12.442636325462843, 41.93804302794496], + [12.442828646282095, 41.93725242115204], + [12.443405608739027, 41.93556576056804], + [12.44388471483822, 41.93370245333628], + [12.445132826188399, 41.93084467089824], + [12.445812354584433, 41.93031849813403], + [12.445819951137906, 41.930296776658196], + ], + type: 'LineString', + }, + properties: { + OBJECT_COMMAND_NAME: 'via camillucciaA', + ENTITY_HEB: 'route', + TYPE: 'ROUTE', + }, +}; +export const ROUTE_VIA_CAMILLUCCIA_B: Route = { + type: 'Feature', + geometry: { + coordinates: [ + [12.445818466287847, 41.93029376141277], + [12.446047161911423, 41.930040913942264], + [12.446171038707206, 41.9297762500957], + [12.446167862379838, 41.92945014491775], + [12.446075748864388, 41.9286136066203], + ], + type: 'LineString', + }, + properties: { + OBJECT_COMMAND_NAME: 'via camillucciaB', + ENTITY_HEB: 'route', + TYPE: 'ROUTE', + }, +}; + +export const CONTROL_POINT_OLIMPIADE_111: Route = { + type: 'Feature', + geometry: { + coordinates: [12.475638293442415, 41.932360642739155], + type: 'Point', + }, + properties: { + OBJECT_COMMAND_NAME: '111', + ENTITY_HEB: 'control point', + TIED_TO: 'olimpiade', + TYPE: 'ITEM', + }, +}; + +export const CONTROL_POINT_OLIMPIADE_112: Route = { + type: 'Feature', + geometry: { + coordinates: [12.474175672012962, 41.932217551210556], + type: 'Point', + }, + properties: { + OBJECT_COMMAND_NAME: '112', + ENTITY_HEB: 'control point', + TIED_TO: 'olimpiade', + TYPE: 'ITEM', + }, +}; diff --git a/tests/integration/control/route/route.spec.ts b/tests/integration/control/route/route.spec.ts index 72278b01..a0cd68c9 100644 --- a/tests/integration/control/route/route.spec.ts +++ b/tests/integration/control/route/route.spec.ts @@ -2,12 +2,20 @@ import jsLogger from '@map-colonies/js-logger'; import { trace } from '@opentelemetry/api'; import httpStatusCodes from 'http-status-codes'; +import { DataSource } from 'typeorm'; import { getApp } from '../../../../src/app'; import { SERVICES } from '../../../../src/common/constants'; -import { GetRoutesQueryParams } from '../../../src/route/controllers/routeController'; +import { GetRoutesQueryParams } from '../../../../src/control/route/controllers/routeController'; +import { Route } from '../../../../src/control/route/models/route'; +import { ControlResponse } from '../../../../src/control/interfaces'; +import { CommonRequestParameters, GeoContext, GeoContextMode } from '../../../../src/common/interfaces'; +import { LATLON_CUSTOM_REPOSITORY_SYMBOL } from '../../../../src/latLon/DAL/latLonRepository'; +import { cronLoadTileLatLonDataSymbol } from '../../../../src/latLon/DAL/latLonDAL'; +import { expectedResponse } from '../utils'; import { RouteRequestSender } from './helpers/requestSender'; +import { ROUTE_VIA_CAMILLUCCIA_A, ROUTE_VIA_CAMILLUCCIA_B, CONTROL_POINT_OLIMPIADE_111, CONTROL_POINT_OLIMPIADE_112 } from './mockObjects'; -describe('/routes', function () { +describe('/search/control/route', function () { let requestSender: RouteRequestSender; beforeEach(async function () { @@ -15,52 +23,530 @@ describe('/routes', function () { override: [ { token: SERVICES.LOGGER, provider: { useValue: jsLogger({ enabled: false }) } }, { token: SERVICES.TRACER, provider: { useValue: trace.getTracer('testTracer') } }, + { token: LATLON_CUSTOM_REPOSITORY_SYMBOL, provider: { useValue: {} } }, + { token: DataSource, provider: { useValue: {} } }, + { token: cronLoadTileLatLonDataSymbol, provider: { useValue: {} } }, ], useChild: true, }); + requestSender = new RouteRequestSender(app.app); }); describe('Happy Path', function () { - it('should return 200 status code and the route', async function () { - const response = await requestSender.getRoutes({ command_name: 'route96' }); + it('should return 200 status code and routes', async function () { + const requestParams: GetRoutesQueryParams = { command_name: 'via camilluccia', limit: 5, disable_fuzziness: false }; + + const response = await requestSender.getRoutes(requestParams); expect(response.status).toBe(httpStatusCodes.OK); - expect(response).toSatisfyApiSpec(); + // expect(response).toSatisfyApiSpec(); + expect(response.body).toMatchObject>>( + expectedResponse(requestParams, [ROUTE_VIA_CAMILLUCCIA_A, ROUTE_VIA_CAMILLUCCIA_B], expect) + ); }); - it('should return 200 status code and empty response', async function () { - const response = await requestSender.getRoutes({ command_name: '48054805' }); + + it('should return 200 status code and routes biased by geo_context (bbox)', async function () { + const requestParams: GetRoutesQueryParams = { + command_name: 'via camilluccia', + geo_context: { bbox: [12.445945519411595, 41.92899524904075, 12.446385440853476, 41.9292587345808] }, + geo_context_mode: GeoContextMode.BIAS, + limit: 5, + disable_fuzziness: false, + }; + const response = await requestSender.getRoutes(requestParams); expect(response.status).toBe(httpStatusCodes.OK); + // expect(response).toSatisfyApiSpec(); + expect(response.body).toMatchObject>>( + expectedResponse(requestParams, [ROUTE_VIA_CAMILLUCCIA_B, ROUTE_VIA_CAMILLUCCIA_A], expect) + ); + }); - expect(response).toSatisfyApiSpec(); - expect(response.body).toMatchObject({ - type: 'FeatureCollection', - features: [], + it('should return 200 status code and routes filtered by geo_context (bbox)', async function () { + const requestParams: GetRoutesQueryParams = { + command_name: 'via camilluccia', + geo_context: { bbox: [12.445945519411595, 41.92899524904075, 12.446385440853476, 41.9292587345808] }, + geo_context_mode: GeoContextMode.FILTER, + limit: 5, + disable_fuzziness: false, + }; + const response = await requestSender.getRoutes(requestParams); + + expect(response.status).toBe(httpStatusCodes.OK); + // expect(response).toSatisfyApiSpec(); + expect(response.body).toMatchObject>>( + expectedResponse(requestParams, [ROUTE_VIA_CAMILLUCCIA_B], expect) + ); + }); + + it('should return 200 status code and routes biased by geo_context (WGS84)', async function () { + const geo_context = { lon: 12.446085277848027, lat: 41.928658505847835, radius: 10 }; + const requestParams: GetRoutesQueryParams = { + command_name: 'via camilluccia', + geo_context, + geo_context_mode: GeoContextMode.BIAS, + limit: 5, + disable_fuzziness: false, + }; + const response = await requestSender.getRoutes({ + ...requestParams, + geo_context: JSON.stringify(geo_context) as unknown as GetRoutesQueryParams['geo_context'], + }); + + expect(response.status).toBe(httpStatusCodes.OK); + // expect(response).toSatisfyApiSpec(); + expect(response.body).toMatchObject>>( + expectedResponse(requestParams, [ROUTE_VIA_CAMILLUCCIA_B, ROUTE_VIA_CAMILLUCCIA_A], expect) + ); + }); + + it('should return 200 status code and routes filtered by geo_context (WGS84)', async function () { + const geo_context = { lon: 12.446085277848027, lat: 41.928658505847835, radius: 10 }; + const requestParams: GetRoutesQueryParams = { + command_name: 'via camilluccia', + geo_context, + geo_context_mode: GeoContextMode.FILTER, + limit: 5, + disable_fuzziness: false, + }; + const response = await requestSender.getRoutes({ + ...requestParams, + geo_context: JSON.stringify(geo_context) as unknown as GetRoutesQueryParams['geo_context'], + }); + + expect(response.status).toBe(httpStatusCodes.OK); + // expect(response).toSatisfyApiSpec(); + expect(response.body).toMatchObject>>( + expectedResponse(requestParams, [ROUTE_VIA_CAMILLUCCIA_B], expect) + ); + }); + + it('should return 200 status code and routes biased by geo_context (UTM)', async function () { + const geo_context = { x: 288247, y: 4645010, zone: 33, radius: 10 }; + const requestParams: GetRoutesQueryParams = { + command_name: 'via camilluccia', + geo_context, + geo_context_mode: GeoContextMode.BIAS, + limit: 5, + disable_fuzziness: false, + }; + const response = await requestSender.getRoutes({ + ...requestParams, + geo_context: JSON.stringify(geo_context) as unknown as GetRoutesQueryParams['geo_context'], + }); + + expect(response.status).toBe(httpStatusCodes.OK); + // expect(response).toSatisfyApiSpec(); + expect(response.body).toMatchObject>>( + expectedResponse(requestParams, [ROUTE_VIA_CAMILLUCCIA_B, ROUTE_VIA_CAMILLUCCIA_A], expect) + ); + }); + + it('should return 200 status code and routes filtered by geo_context (UTM)', async function () { + const geo_context = { x: 288247, y: 4645010, zone: 33, radius: 10 }; + const requestParams: GetRoutesQueryParams = { + command_name: 'via camilluccia', + geo_context, + geo_context_mode: GeoContextMode.FILTER, + limit: 5, + disable_fuzziness: false, + }; + const response = await requestSender.getRoutes({ + ...requestParams, + geo_context: JSON.stringify(geo_context) as unknown as GetRoutesQueryParams['geo_context'], + }); + + expect(response.status).toBe(httpStatusCodes.OK); + // expect(response).toSatisfyApiSpec(); + expect(response.body).toMatchObject>>( + expectedResponse(requestParams, [ROUTE_VIA_CAMILLUCCIA_B], expect) + ); + }); + + it('should return 200 status code and send via camillucciaB route as disable_fuzziness is true', async function () { + const requestParams: GetRoutesQueryParams = { + command_name: 'via camillucciaB', + limit: 5, + disable_fuzziness: true, + }; + const response = await requestSender.getRoutes(requestParams); + + expect(response.status).toBe(httpStatusCodes.OK); + // expect(response).toSatisfyApiSpec(); + expect(response.body).toMatchObject>>( + expectedResponse(requestParams, [ROUTE_VIA_CAMILLUCCIA_B], expect) + ); + }); + + it('should return 200 status code and no route as disable_fuzziness is true', async function () { + const requestParams: GetRoutesQueryParams = { + command_name: 'via camilluccia', + limit: 5, + disable_fuzziness: true, + }; + const response = await requestSender.getRoutes(requestParams); + + expect(response.status).toBe(httpStatusCodes.OK); + // expect(response).toSatisfyApiSpec(); + expect(response.body).toMatchObject>>( + expectedResponse(requestParams, [], expect) + ); + }); + + it('should return 200 status code and control points biased by geo_context (bbox)', async function () { + const requestParams: GetRoutesQueryParams = { + command_name: 'olimpiade', + control_point: '11', + geo_context: { bbox: [12.473743746822265, 41.93195262135879, 12.474626229200709, 41.93249150688004] }, + geo_context_mode: GeoContextMode.BIAS, + limit: 5, + disable_fuzziness: false, + }; + const response = await requestSender.getRoutes(requestParams); + + expect(response.status).toBe(httpStatusCodes.OK); + // expect(response).toSatisfyApiSpec(); + expect(response.body).toMatchObject>>( + expectedResponse(requestParams, [CONTROL_POINT_OLIMPIADE_112, CONTROL_POINT_OLIMPIADE_111], expect) + ); + }); + + it('should return 200 status code and control points filtered by geo_context (bbox)', async function () { + const requestParams: GetRoutesQueryParams = { + command_name: 'olimpiade', + control_point: '11', + geo_context: { bbox: [12.473743746822265, 41.93195262135879, 12.474626229200709, 41.93249150688004] }, + geo_context_mode: GeoContextMode.FILTER, + limit: 5, + disable_fuzziness: false, + }; + const response = await requestSender.getRoutes(requestParams); + + expect(response.status).toBe(httpStatusCodes.OK); + // expect(response).toSatisfyApiSpec(); + expect(response.body).toMatchObject>>( + expectedResponse(requestParams, [CONTROL_POINT_OLIMPIADE_112], expect) + ); + }); + + it('should return 200 status code and control points biased by geo_context (WGS84)', async function () { + const geo_context = { lon: 12.474175672012962, lat: 41.932217551210556, radius: 10 }; + const requestParams: GetRoutesQueryParams = { + command_name: 'olimpiade', + control_point: '11', + geo_context, + geo_context_mode: GeoContextMode.BIAS, + limit: 5, + disable_fuzziness: false, + }; + const response = await requestSender.getRoutes({ + ...requestParams, + geo_context: JSON.stringify(geo_context) as unknown as GetRoutesQueryParams['geo_context'], + }); + + expect(response.status).toBe(httpStatusCodes.OK); + // expect(response).toSatisfyApiSpec(); + expect(response.body).toMatchObject>>( + expectedResponse(requestParams, [CONTROL_POINT_OLIMPIADE_112, CONTROL_POINT_OLIMPIADE_111], expect) + ); + }); + + it('should return 200 status code and control points filtered by geo_context (WGS84)', async function () { + const geo_context = { lon: 12.474175672012962, lat: 41.932217551210556, radius: 10 }; + const requestParams: GetRoutesQueryParams = { + command_name: 'olimpiade', + control_point: '11', + geo_context, + geo_context_mode: GeoContextMode.FILTER, + limit: 5, + disable_fuzziness: false, + }; + const response = await requestSender.getRoutes({ + ...requestParams, + geo_context: JSON.stringify(geo_context) as unknown as GetRoutesQueryParams['geo_context'], + }); + + expect(response.status).toBe(httpStatusCodes.OK); + // expect(response).toSatisfyApiSpec(); + expect(response.body).toMatchObject>>( + expectedResponse(requestParams, [CONTROL_POINT_OLIMPIADE_112], expect) + ); + }); + + it('should return 200 status code and control points biased by geo_context (UTM)', async function () { + const geo_context = { x: 290588, y: 4645336, zone: 33, radius: 10 }; + const requestParams: GetRoutesQueryParams = { + command_name: 'olimpiade', + control_point: '11', + geo_context, + geo_context_mode: GeoContextMode.BIAS, + limit: 5, + disable_fuzziness: false, + }; + const response = await requestSender.getRoutes({ + ...requestParams, + geo_context: JSON.stringify(geo_context) as unknown as GetRoutesQueryParams['geo_context'], }); + + expect(response.status).toBe(httpStatusCodes.OK); + // expect(response).toSatisfyApiSpec(); + expect(response.body).toMatchObject>>( + expectedResponse(requestParams, [CONTROL_POINT_OLIMPIADE_112, CONTROL_POINT_OLIMPIADE_111], expect) + ); + }); + + it('should return 200 status code and control points filtered by geo_context (UTM)', async function () { + const geo_context = { x: 290588, y: 4645336, zone: 33, radius: 10 }; + const requestParams: GetRoutesQueryParams = { + command_name: 'olimpiade', + control_point: '11', + geo_context, + geo_context_mode: GeoContextMode.FILTER, + limit: 5, + disable_fuzziness: false, + }; + const response = await requestSender.getRoutes({ + ...requestParams, + geo_context: JSON.stringify(geo_context) as unknown as GetRoutesQueryParams['geo_context'], + }); + + expect(response.status).toBe(httpStatusCodes.OK); + // expect(response).toSatisfyApiSpec(); + expect(response.body).toMatchObject>>( + expectedResponse(requestParams, [CONTROL_POINT_OLIMPIADE_112], expect) + ); + }); + + it('should return 200 status code and control points', async function () { + const requestParams: GetRoutesQueryParams = { + command_name: 'olimpiade', + control_point: '111', + limit: 5, + disable_fuzziness: false, + }; + const response = await requestSender.getRoutes(requestParams); + + expect(response.status).toBe(httpStatusCodes.OK); + // expect(response).toSatisfyApiSpec(); + expect(response.body).toMatchObject>>( + expectedResponse(requestParams, [CONTROL_POINT_OLIMPIADE_111, CONTROL_POINT_OLIMPIADE_112], expect) + ); + }); + + it('should return 200 status code and 111 control_point with disable_fuzziness true', async function () { + const requestParams: GetRoutesQueryParams = { + command_name: 'olimpiade', + control_point: '111', + limit: 5, + disable_fuzziness: true, + }; + const response = await requestSender.getRoutes(requestParams); + + expect(response.status).toBe(httpStatusCodes.OK); + // expect(response).toSatisfyApiSpec(); + expect(response.body).toMatchObject>>( + expectedResponse(requestParams, [CONTROL_POINT_OLIMPIADE_111], expect) + ); + }); + + it('should return 200 status code and return no control points as no control points with disable_fuzziness true', async function () { + const requestParams: GetRoutesQueryParams = { + command_name: 'camilluccia', + control_point: '111', + limit: 5, + disable_fuzziness: true, + }; + const response = await requestSender.getRoutes(requestParams); + + expect(response.status).toBe(httpStatusCodes.OK); + // expect(response).toSatisfyApiSpec(); + expect(response.body).toMatchObject>>( + expectedResponse(requestParams, [], expect) + ); }); }); describe('Bad Path', function () { // All requests with status code of 400 - it('Should return 400 status code and meessage "request/query must have required property \'command_name\'"', async function () { - const message = "request/query must have required property 'command_name'"; + it('should return 400 status code and error message when empty object is passed', async function () { + const response = await requestSender.getRoutes({} as unknown as GetRoutesQueryParams); - const response = await requestSender.getRoutes(); + expect(response.status).toBe(httpStatusCodes.BAD_REQUEST); + expect(response.body).toMatchObject({ + message: "request/query must have required property 'command_name'", + }); + }); + + it('should return 400 status code and error message when command_name is empty', async function () { + const response = await requestSender.getRoutes({ command_name: '', limit: 5, disable_fuzziness: false }); expect(response.status).toBe(httpStatusCodes.BAD_REQUEST); - expect(response.body).toMatchObject({ message }); - expect(response).toSatisfyApiSpec(); + expect(response.body).toMatchObject({ + message: "Empty value found for query parameter 'command_name'", + }); }); - it('Should return 400 status code for unknown parameter', async function () { - const parameter = 'test1234'; - const message = `Unknown query parameter '${parameter}'`; - const response = await requestSender.getRoutes({ [parameter]: parameter } as unknown as GetRoutesQueryParams); + test.each(['invalid', '6a6', '06', '-11', '6 ', ' 6', ' ', ' 6 ', ''])( + 'should return 400 status code and error message when value: "%s" for control_point value is invalid', + async (control_point) => { + const response = await requestSender.getRoutes({ command_name: 'olimpiade', control_point, limit: 5, disable_fuzziness: false }); + + const message = control_point + ? 'request/query/control_point must match pattern "^[1-9][0-9]*$"' + : "Empty value found for query parameter 'control_point'"; + + expect(response.status).toBe(httpStatusCodes.BAD_REQUEST); + expect(response.body).toMatchObject({ + message, + }); + } + ); + + test.each>([ + { + geo_context: { x: 300850, y: 4642203, zone: 33, radius: 100 }, + }, + { + geo_context: { lon: 12.598899687444742, lat: 41.90667824634701, radius: 10 }, + }, + { + geo_context: { bbox: [12.554407132912445, 41.84962590648513, 12.652837919839953, 41.94545380230761] }, + }, + { + geo_context_mode: GeoContextMode.BIAS, + }, + { + geo_context_mode: GeoContextMode.FILTER, + }, + ])('should return 400 and message that geo_context and geo_context_mode must be both defined or both undefined', async function (requestParams) { + const response = await requestSender.getRoutes({ command_name: '1234', limit: 5, disable_fuzziness: false, ...requestParams }); expect(response.status).toBe(httpStatusCodes.BAD_REQUEST); - expect(response.body).toMatchObject({ message }); - expect(response).toSatisfyApiSpec(); + expect(response.body).toMatchObject({ + message: '/control/utils/geoContextQuery: geo_context and geo_context_mode must be both defined or both undefined', + }); }); + + test.each>([ + { + geo_context: { x: 300850, y: 4642203, zone: 33, radius: 100 }, + }, + { + geo_context: { lon: 12.598899687444742, lat: 41.90667824634701, radius: 10 }, + }, + { + geo_context: { bbox: [12.554407132912445, 41.84962590648513, 12.652837919839953, 41.94545380230761] }, + }, + { + geo_context_mode: GeoContextMode.BIAS, + }, + { + geo_context_mode: GeoContextMode.FILTER, + }, + ])('should return 400 and message that geo_context and geo_context_mode must be both defined or both undefined', async function (requestParams) { + const response = await requestSender.getRoutes({ command_name: '1234', limit: 5, disable_fuzziness: false, ...requestParams }); + + expect(response.status).toBe(httpStatusCodes.BAD_REQUEST); + expect(response.body).toMatchObject({ + message: '/control/utils/geoContextQuery: geo_context and geo_context_mode must be both defined or both undefined', + }); + }); + + test.each>([ + { + disable_fuzziness: '' as unknown as boolean, + }, + { + disable_fuzziness: 'True' as unknown as boolean, + }, + { + disable_fuzziness: 'False' as unknown as boolean, + }, + ])('should return 400 status code and error message when disable_fuzziness value is invalid', async function (requestParams) { + const response = await requestSender.getRoutes({ command_name: '1234', limit: 5, ...requestParams }); + + const message = + (requestParams.disable_fuzziness as unknown as string) === '' + ? "Empty value found for query parameter 'disable_fuzziness'" + : 'request/query/disable_fuzziness must be boolean'; + + expect(response.status).toBe(httpStatusCodes.BAD_REQUEST); + expect(response.body).toMatchObject({ + message, + }); + }); + + test.each>([ + { + limit: '' as unknown as number, + }, + { + limit: 0, + }, + { + limit: 101, + }, + ])('should return 400 status code and error message when limit value is invalid', async function (requestParams) { + const response = await requestSender.getRoutes({ command_name: '1234', disable_fuzziness: false, ...requestParams }); + + const message = + (requestParams.limit as unknown as string) === '' + ? "Empty value found for query parameter 'limit'" + : requestParams.limit < 1 + ? 'request/query/limit must be >= 1' + : 'request/query/limit must be <= 15'; + + expect(response.status).toBe(httpStatusCodes.BAD_REQUEST); + expect(response.body).toMatchObject({ + message, + }); + }); + + test.each<(keyof GeoContext)[][]>([[['lat', 'lon', 'radius']], [['x', 'y', 'zone', 'radius']]])( + 'should return 400 for all invalid geo_context object for the keys %s', + async function (keys) { + function generateCombinations(keys: (keyof GeoContext)[]): GeoContext[] { + const combinations: object[] = []; + + function backtrack(current: object, remainingKeys: string[]): void { + if (remainingKeys.length === 0) { + combinations.push(current); + return; + } + + const key = remainingKeys[0]; + const remaining = remainingKeys.slice(1); + + backtrack({ ...current, [key]: 1 }, remaining); + backtrack(current, remaining); + } + + backtrack({}, keys); + + return combinations; + } + + const geoContexts = generateCombinations(keys); + + for (const geo_context of geoContexts) { + if (Object.keys(geo_context).length === keys.length) { + continue; + } + const response = await requestSender.getRoutes({ + command_name: '1234', + limit: 5, + disable_fuzziness: false, + geo_context: JSON.stringify(geo_context) as unknown as GetRoutesQueryParams['geo_context'], + geo_context_mode: GeoContextMode.BIAS, + }); + + expect(response.status).toBe(httpStatusCodes.BAD_REQUEST); + expect(response.body).toMatchObject({ + message: 'geo_context validation: geo_context must contain one of the following: {bbox}, {lat, lon, radius}, or {x, y, zone, radius}', + }); + } + } + ); }); describe('Sad Path', function () { // All requests with status code 4XX-5XX From e8ca56e17ead74daed9b06410f228d7e752f3a7e Mon Sep 17 00:00:00 2001 From: Niv Greenstein Date: Thu, 22 Aug 2024 17:53:50 +0300 Subject: [PATCH 132/262] feat: added utils to Convert Camel To SnakeCase --- src/common/utils.ts | 10 ++++++++++ 1 file changed, 10 insertions(+) diff --git a/src/common/utils.ts b/src/common/utils.ts index 510e9ed1..6f75dec6 100644 --- a/src/common/utils.ts +++ b/src/common/utils.ts @@ -4,10 +4,20 @@ import { WGS84Coordinate } from './interfaces'; type SnakeToCamelCase = S extends `${infer T}_${infer U}` ? `${T}${Capitalize>}` : S; +type CamelToSnakeCase = S extends `${infer T}${infer U}` + ? U extends Uncapitalize + ? `${Lowercase}${CamelToSnakeCase}` + : `${Lowercase}_${CamelToSnakeCase>}` + : S; + export type ConvertSnakeToCamelCase = { [K in keyof T as SnakeToCamelCase]: T[K]; }; +export type ConvertCamelToSnakeCase = { + [K in keyof T as CamelToSnakeCase]: T[K]; +}; + export const validateWGS84Coordinate = (coordinate: { lon: number; lat: number }): boolean => { // eslint-disable-next-line @typescript-eslint/no-magic-numbers const [min, max] = [0, 180]; From 76fffcbfbc2f2f6d0b0ca51b51a1da0a1fe26b5a Mon Sep 17 00:00:00 2001 From: Niv Greenstein Date: Thu, 22 Aug 2024 17:54:13 +0300 Subject: [PATCH 133/262] fix: converting camel to snake case in geocoding.query --- src/location/utils.ts | 20 ++++++++++---------- 1 file changed, 10 insertions(+), 10 deletions(-) diff --git a/src/location/utils.ts b/src/location/utils.ts index c9edd151..86c36192 100644 --- a/src/location/utils.ts +++ b/src/location/utils.ts @@ -5,7 +5,8 @@ import { StatusCodes } from 'http-status-codes'; import axios, { AxiosResponse as Response } from 'axios'; import { InternalServerError } from '../common/errors'; import { GeoContext, IApplication } from '../common/interfaces'; -import { convertUTMToWgs84 } from '../common/utils'; +import { ConvertCamelToSnakeCase, convertUTMToWgs84 } from '../common/utils'; +import { convertCamelToSnakeCase } from '../control/utils'; import { BBOX_LENGTH, POINT_LENGTH, QueryResult, TextSearchParams } from './interfaces'; import { generateDisplayName } from './parsing'; import { TextSearchHit } from './models/elasticsearchHits'; @@ -119,16 +120,15 @@ export const convertResult = ( geocoding: { version: process.env.npm_package_version, query: { - ...params, - response: { - /* eslint-disable @typescript-eslint/naming-convention */ - results_count: results.hits.hits.length, - max_score: results.hits.max_score ?? 0, - match_latency_ms: results.took, - /* eslint-enable @typescript-eslint/naming-convention */ - }, + ...(convertCamelToSnakeCase(params as unknown as Record) as ConvertCamelToSnakeCase), + }, + response: { + /* eslint-disable @typescript-eslint/naming-convention */ + results_count: results.hits.hits.length, + max_score: results.hits.max_score ?? 0, + match_latency_ms: results.took, + /* eslint-enable @typescript-eslint/naming-convention */ }, - name: params.name || undefined, }, features: results.hits.hits.map(({ highlight, _source: feature, _score }, index): QueryResult['features'][number] => { const allNames = [feature!.text, feature!.translated_text || []]; From 9f1a44573050ccd6f89b1f5ef88cc1915f03e50e Mon Sep 17 00:00:00 2001 From: Niv Greenstein Date: Thu, 22 Aug 2024 17:54:22 +0300 Subject: [PATCH 134/262] feat: added mock data --- devScripts/geotextElasticsearchData.json | 54 ++++++++++++++++++++++++ 1 file changed, 54 insertions(+) diff --git a/devScripts/geotextElasticsearchData.json b/devScripts/geotextElasticsearchData.json index 56b0e4be..99f6f4a3 100644 --- a/devScripts/geotextElasticsearchData.json +++ b/devScripts/geotextElasticsearchData.json @@ -222,6 +222,60 @@ "text_language": "en" } }, + { + "_id": "1bb11f54-939e-457b-bf68-a3920ccf629c", + "_index": "geotext_index", + "_source": { + "source": "GOOGLE", + "layer_name": "google_ports", + "ingestion_timestamp": "2024-07-17 00:00:00", + "timestamp": "11/12/2023 00:00:00", + "source_id": ["{1bb11f54-939e-457b-bf68-a3920ccf629c}"], + "sensitivity": "non-sensitive", + "placetype": "transportation", + "sub_placetype": "port", + "wkt": "POLYGON ((-118.2505781304088 33.7502674389752, -118.25604403409116 33.76075151051916, -118.27057180697577 33.748593059782564, -118.27503083555426 33.741097783576635, -118.2747911028351 33.734798055529765, -118.27215404292296 33.73136889520775, -118.26807858669537 33.720881194108856, -118.26424286318695 33.721997816398385, -118.26640045650717 33.72901625632974, -118.2431463824787 33.735794882347946, -118.24492040460113 33.739303607948656, -118.25072193640723 33.73794798097781, -118.25220827926702 33.74193505797223, -118.24937943317966 33.74508471776615, -118.24798898340768 33.74783559181691, -118.24909175391655 33.74803492708783, -118.25096166912684 33.74600168558719, -118.25326310323155 33.745363795966625, -118.25278363779321 33.74687877606813, -118.2505781304088 33.7502674389752))", + "geo_json": { + "type": "Polygon", + "coordinates": [ + [ + [-118.2505781304088, 33.7502674389752], + [-118.25604403409116, 33.76075151051916], + [-118.27057180697577, 33.748593059782564], + [-118.27503083555426, 33.741097783576635], + [-118.2747911028351, 33.734798055529765], + [-118.27215404292296, 33.73136889520775], + [-118.26807858669537, 33.720881194108856], + [-118.26424286318695, 33.721997816398385], + [-118.26640045650717, 33.72901625632974], + [-118.2431463824787, 33.735794882347946], + [-118.24492040460113, 33.739303607948656], + [-118.25072193640723, 33.73794798097781], + [-118.25220827926702, 33.74193505797223], + [-118.24937943317966, 33.74508471776615], + [-118.24798898340768, 33.74783559181691], + [-118.24909175391655, 33.74803492708783], + [-118.25096166912684, 33.74600168558719], + [-118.25326310323155, 33.745363795966625], + [-118.25278363779321, 33.74687877606813], + [-118.2505781304088, 33.7502674389752] + ] + ] + }, + "centroid": { + "type": "Point", + "coordinates": [-118.25908860901649, 33.740816352314006] + }, + "geometry_type": "Polygon", + "geometry_hash": "ffbc4f66c802719b5bcffaca238ee1d3", + "region": ["USA"], + "sub_region": ["Los Angeles"], + "name": "Port of Los Angeles", + "text": ["Port of Los Angeles"], + "translated_text": ["Puerto de Los Ángeles"], + "text_language": "en" + } + }, { "_id": "2b3dd420-d265-4665-94a6-3e13650b7a2d", "_index": "geotext_index", From 3f9bdfdfa9adba320a7cc6ead8026e76deb23cd3 Mon Sep 17 00:00:00 2001 From: Niv Greenstein Date: Thu, 22 Aug 2024 17:54:53 +0300 Subject: [PATCH 135/262] fix: fixed QueryResult interface --- src/location/interfaces.ts | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/src/location/interfaces.ts b/src/location/interfaces.ts index ee36a83d..379c1f26 100644 --- a/src/location/interfaces.ts +++ b/src/location/interfaces.ts @@ -1,7 +1,7 @@ import type { GeoJSON } from 'geojson'; import { estypes } from '@elastic/elasticsearch'; import { CommonRequestParameters } from '../common/interfaces'; -import { ConvertSnakeToCamelCase } from '../common/utils'; +import { ConvertCamelToSnakeCase, ConvertSnakeToCamelCase } from '../common/utils'; import { HierarchySearchHit } from './models/elasticsearchHits'; export interface PlaceType { @@ -33,8 +33,8 @@ export interface QueryResult { type: string; geocoding: { version?: string; - query: TextSearchParams & { response: { max_score: number; results_count: number; match_latency_ms: number } }; - name?: string; + query: ConvertCamelToSnakeCase; + response: { max_score: number; results_count: number; match_latency_ms: number }; }; features: ({ type: string; From 3c085c3e000c92b88ccaaed8e3368936c32bfcf9 Mon Sep 17 00:00:00 2001 From: Niv Greenstein Date: Thu, 22 Aug 2024 17:55:15 +0300 Subject: [PATCH 136/262] delete: removed old geotext tests --- .../integration/geotextSearch/geotext.spec.ts | 172 ------------------ .../geotextSearch/helpers/requestSender.ts | 25 --- .../geotextSearch/possibleObjects.ts | 167 ----------------- 3 files changed, 364 deletions(-) delete mode 100644 tests/integration/geotextSearch/geotext.spec.ts delete mode 100644 tests/integration/geotextSearch/helpers/requestSender.ts delete mode 100644 tests/integration/geotextSearch/possibleObjects.ts diff --git a/tests/integration/geotextSearch/geotext.spec.ts b/tests/integration/geotextSearch/geotext.spec.ts deleted file mode 100644 index 42f0c526..00000000 --- a/tests/integration/geotextSearch/geotext.spec.ts +++ /dev/null @@ -1,172 +0,0 @@ -/* eslint-disable @typescript-eslint/naming-convention */ -import config from 'config'; -import jsLogger from '@map-colonies/js-logger'; -import { trace } from '@opentelemetry/api'; -import httpStatusCodes from 'http-status-codes'; -import nock from 'nock'; -import { getApp } from '../../../src/app'; -import { SERVICES } from '../../../src/common/constants'; -import { IApplication } from '../../../src/common/interfaces'; -import { jfkAirport, losAngelesAirport, policeAirport } from './possibleObjects'; -import { GeotextSearchRequestSender } from './helpers/requestSender'; - -describe('/query', function () { - let requestSender: GeotextSearchRequestSender; - - beforeEach(async function () { - const app = await getApp({ - override: [ - { token: SERVICES.LOGGER, provider: { useValue: jsLogger({ enabled: false }) } }, - { token: SERVICES.TRACER, provider: { useValue: trace.getTracer('testTracer') } }, - ], - useChild: true, - }); - requestSender = new GeotextSearchRequestSender(app.app); - }); - - describe('Happy Path', function () { - it('should return 200 status code and the all available regions', async function () { - const response = await requestSender.getRegions(); - - expect(response.status).toBe(httpStatusCodes.OK); - // expect(response).toSatisfyApiSpec(); - - expect(response.body).toEqual(['USA']); - }); - - it('should return 200 status code and all airports', async function () { - const tokenTypesUrlScope = nock(config.get('application').services.tokenTypesUrl) - .post('', { tokens: ['airport'] }) - .reply(httpStatusCodes.OK, [ - { - tokens: ['airport'], - prediction: ['essence'], - }, - ]); - - const response = await requestSender.getGeotextSearch({ query: 'airport', limit: 10 }); - - expect(response.status).toBe(httpStatusCodes.OK); - - // expect(response).toSatisfyApiSpec(); - - expect(response.body).toMatchObject({ - type: 'FeatureCollection', - geocoding: { - version: process.env.npm_package_version, - query: { - query: 'airport', - limit: 10, - name: '', - placeTypes: ['transportation'], - subPlaceTypes: ['airport'], - hierarchies: [], - }, - name: '', - }, - features: [jfkAirport(1), policeAirport(2), losAngelesAirport(3)], - }); - - tokenTypesUrlScope.done(); - }); - }); - - it('should return 200 and los angeles airport while checking the hierarchy system', async () => { - const tokenTypesUrlScope = nock(config.get('application').services.tokenTypesUrl) - .post('', { tokens: ['airport'] }) - .reply(httpStatusCodes.OK, [ - { - tokens: ['airport'], - prediction: ['essence'], - }, - ]); - - const response = await requestSender.getGeotextSearch({ query: 'airport, los angeles', limit: 1 }); - - expect(response.status).toBe(httpStatusCodes.OK); - // expect(response).toSatisfyApiSpec(); - expect(response.body).toMatchObject({ - type: 'FeatureCollection', - geocoding: { - version: process.env.npm_package_version, - query: { - query: 'airport', - limit: 1, - name: '', - placeTypes: ['transportation'], - subPlaceTypes: ['airport'], - hierarchies: [ - { - geo_json: { - coordinates: [ - [ - [-118.54430957638033, 34.07939240620722], - [-118.5350828996408, 33.695192367610986], - [-118.04596133238863, 33.47690745532634], - [-117.66265886139905, 33.379872950239346], - [-117.57145361502153, 33.63336904289318], - [-117.67279277766329, 34.23871934668085], - [-118.2605599209839, 34.28059749364003], - [-118.54430957638033, 34.07939240620722], - ], - ], - type: 'Polygon', - }, - hierarchy: 'city', - placetype: 'city', - region: 'USA', - text: 'Los Angeles', - weight: 1.1, - }, - ], - }, - name: '', - }, - features: [losAngelesAirport(1)], - }); - - tokenTypesUrlScope.done(); - }); - - it('should retrun 200 and los angeles airport while checking the region from NLP analyser', async () => { - const tokenTypesUrlScope = nock(config.get('application').services.tokenTypesUrl) - .post('', { tokens: ['airport', 'los', 'angeles'] }) - .reply(httpStatusCodes.OK, [ - { - tokens: ['airport', 'los', 'angeles'], - prediction: ['essence', 'essence', 'name'], - }, - ]); - - const response = await requestSender.getGeotextSearch({ query: 'airport los angeles', limit: 1 }); - - expect(response.status).toBe(httpStatusCodes.OK); - - // expect(response).toSatisfyApiSpec(); - - expect(response.body).toMatchObject({ - type: 'FeatureCollection', - geocoding: { - version: process.env.npm_package_version, - query: { - query: 'airport los angeles', - limit: 1, - name: 'angeles', - placeTypes: ['transportation'], - subPlaceTypes: ['airport'], - hierarchies: [], - }, - name: 'angeles', - }, - features: [losAngelesAirport(1)], - }); - - tokenTypesUrlScope.done(); - }); - describe('Bad Path', function () { - // All requests with status code of 400 - }); - describe('Sad Path', function () { - // All requests with status code 4XX-5XX - }); -}); diff --git a/tests/integration/geotextSearch/helpers/requestSender.ts b/tests/integration/geotextSearch/helpers/requestSender.ts deleted file mode 100644 index c924f538..00000000 --- a/tests/integration/geotextSearch/helpers/requestSender.ts +++ /dev/null @@ -1,25 +0,0 @@ -import * as supertest from 'supertest'; -import { GetGeotextSearchParams } from '../../../../src/geotextSearch/interfaces'; - -export class GeotextSearchRequestSender { - public constructor(private readonly app: Express.Application) {} - - public async getGeotextSearch(queryParams?: GetGeotextSearchParams): Promise { - return supertest - .agent(this.app) - .get('/v1/query') - .set('Content-Type', 'application/json') - .set('X-API-Key', 'abc123') - .set('x-user-id', 'abc123') - .query(queryParams ?? {}); - } - - public async getRegions(): Promise { - return supertest - .agent(this.app) - .get('/v1/query/regions') - .set('Content-Type', 'application/json') - .set('X-API-Key', 'abc123') - .set('x-user-id', 'abc123'); - } -} diff --git a/tests/integration/geotextSearch/possibleObjects.ts b/tests/integration/geotextSearch/possibleObjects.ts deleted file mode 100644 index 5ddd36e2..00000000 --- a/tests/integration/geotextSearch/possibleObjects.ts +++ /dev/null @@ -1,167 +0,0 @@ -/* eslint-disable @typescript-eslint/naming-convention */ -/* eslint-disable @typescript-eslint/no-magic-numbers */ -export const jfkAirport = (rank: number) => ({ - type: 'Feature', - geometry: { - type: 'Polygon', - coordinates: [ - [ - [-73.81278266814672, 40.66039916690434], - [-73.8177430790069, 40.66160065836647], - [-73.82178645734075, 40.66043068890016], - [-73.82341214553732, 40.658185554886586], - [-73.82407909454082, 40.65496001884071], - [-73.8229536180974, 40.650532558586235], - [-73.82211993184252, 40.647939195437345], - [-73.81290769732489, 40.643985699842915], - [-73.79214887014153, 40.63414837731818], - [-73.78339516446982, 40.62987771430167], - [-73.7898562329419, 40.62275933562921], - [-73.78443726476769, 40.620069953803636], - [-73.7791433570518, 40.627188619100366], - [-73.77639219241223, 40.62706207167477], - [-73.77159849644941, 40.62336045339214], - [-73.77209870820208, 40.619975033140975], - [-73.77047302000595, 40.61908910045176], - [-73.76547094984971, 40.628422477310664], - [-73.75338249916041, 40.63291467256053], - [-73.74733827381596, 40.63601474373485], - [-73.7467963777506, 40.64208793530722], - [-73.752548854642, 40.64749646458006], - [-73.76213624656812, 40.65309424493557], - [-73.78181122379466, 40.66270746643491], - [-73.79106514121902, 40.66438330508498], - [-73.7957754685564, 40.665205399216404], - [-73.79856831750864, 40.66283394629252], - [-73.80390390953731, 40.66175885985783], - [-73.8073637074931, 40.66039916690434], - [-73.8109068740743, 40.66074699797244], - [-73.81278266814672, 40.66039916690434], - ], - ], - }, - properties: { - rank, - source: 'OSM', - layer: 'osm_airports', - source_id: ['03ed6d97-fc81-4340-b68a-11993554eef1'], - name: { - en: ['JFK Airport'], - fr: ['Aeropuerto JFK'], - default: ['JFK'], - display: 'JFK', - }, - placetype: 'transportation', - sub_placetype: 'airport', - regions: [ - { - region: 'USA', - sub_regions: ['New York'], - }, - ], - region: ['USA'], - sub_region: ['New York'], - }, -}); - -export const policeAirport = (rank: number) => ({ - type: 'Feature', - geometry: { - coordinates: [ - [ - [-73.50019138870562, 40.76503398530525], - [-73.49991804403106, 40.75762191351754], - [-73.50172211928987, 40.75368779651572], - [-73.502924836068, 40.74958778448408], - [-73.50095675396722, 40.746067370663354], - [-73.49587254187276, 40.74117989901089], - [-73.48810955136324, 40.74010295019133], - [-73.4862508071566, 40.74097279482331], - [-73.48450140084462, 40.74209114977816], - [-73.48335335295225, 40.743250905424986], - [-73.49368578398266, 40.7500847690697], - [-73.49363111503533, 40.75112014169309], - [-73.49166303293454, 40.75087165373341], - [-73.49128035030373, 40.75435040065955], - [-73.48406404926651, 40.75385344795657], - [-73.4860321313673, 40.75840870869581], - [-73.50019138870562, 40.76503398530525], - ], - ], - type: 'Polygon', - }, - properties: { - rank, - source: 'OSM', - layer: 'osm_airports', - source_id: ['009c6b65-3dcb-4c4f-9f02-d766ebb5d808'], - name: { - en: ['Nassau County Police Airport'], - fr: ['Aeropuerto de la Policía del Condado de Nassau'], - default: ['Nassau County Police Airport'], - display: 'Nassau County Police Airport', - }, - placetype: 'transportation', - sub_placetype: 'airport', - regions: [ - { - region: 'USA', - sub_regions: ['New York'], - }, - ], - region: ['USA'], - sub_region: ['New York'], - }, -}); - -export const losAngelesAirport = (rank: number) => ({ - type: 'Feature', - geometry: { - coordinates: [ - [ - [-118.42713070992883, 33.9512236894319], - [-118.43440343023548, 33.94992163234602], - [-118.43571147345608, 33.94670980636225], - [-118.43560682999878, 33.943107208682775], - [-118.43325235220159, 33.938723119106925], - [-118.42875268352236, 33.9310829780122], - [-118.41394563426408, 33.931820976094315], - [-118.39756893251248, 33.932255089782316], - [-118.38516868278077, 33.935250412835686], - [-118.37904703905322, 33.936509285268926], - [-118.37899471732432, 33.943367642688045], - [-118.3788377519829, 33.94501703885841], - [-118.39181354073204, 33.94527746687625], - [-118.39510980964849, 33.945451085112225], - [-118.3980398264626, 33.94558129855595], - [-118.39746428744567, 33.949748023597365], - [-118.39720267894361, 33.953003135913036], - [-118.40813791735772, 33.95252573110099], - [-118.42713070992883, 33.9512236894319], - ], - ], - type: 'Polygon', - }, - properties: { - rank, - source: 'OSM', - layer: 'osm_airports', - source_id: ['a4f373ab-b824-41e2-b160-e7729c73bea6'], - name: { - en: ['Los Angeles International Airport'], - fr: ['Aeropuerto Internacional de Los Ángeles'], - default: ['Los Angeles International Airport'], - display: 'Los Angeles International Airport', - }, - placetype: 'transportation', - sub_placetype: 'airport', - regions: [ - { - region: 'USA', - sub_regions: ['Los Angeles'], - }, - ], - region: ['USA'], - sub_region: ['Los Angeles'], - }, -}); From 1b03e864c22a435ae4625dca02f5731a50f9bb5d Mon Sep 17 00:00:00 2001 From: Niv Greenstein Date: Thu, 22 Aug 2024 17:55:45 +0300 Subject: [PATCH 137/262] feat: added tests for location/query --- .../location/helpers/requestSender.ts | 36 ++ tests/integration/location/location.spec.ts | 166 +++++++++ tests/integration/location/mockObjects.ts | 351 ++++++++++++++++++ tests/integration/location/utils.ts | 32 ++ 4 files changed, 585 insertions(+) create mode 100644 tests/integration/location/helpers/requestSender.ts create mode 100644 tests/integration/location/location.spec.ts create mode 100644 tests/integration/location/mockObjects.ts create mode 100644 tests/integration/location/utils.ts diff --git a/tests/integration/location/helpers/requestSender.ts b/tests/integration/location/helpers/requestSender.ts new file mode 100644 index 00000000..56b867f9 --- /dev/null +++ b/tests/integration/location/helpers/requestSender.ts @@ -0,0 +1,36 @@ +import * as supertest from 'supertest'; +import { GetGeotextSearchParams } from '../../../../src/location/interfaces'; + +export class LocationRequestSender { + private readonly pathPrefix = '/search/location'; + + public constructor(private readonly app: Express.Application) {} + + public async getQuery(queryParams?: GetGeotextSearchParams): Promise { + return supertest + .agent(this.app) + .get(`${this.pathPrefix}/query`) + .set('Content-Type', 'application/json') + .set('x-api-key', 'abc123') + .set('x-user-id', 'abc123') + .query(queryParams ?? {}); + } + + public async getRegions(): Promise { + return supertest + .agent(this.app) + .get(`${this.pathPrefix}/regions`) + .set('Content-Type', 'application/json') + .set('x-api-key', 'abc123') + .set('x-user-id', 'abc123'); + } + + public async getSources(): Promise { + return supertest + .agent(this.app) + .get(`${this.pathPrefix}/sources`) + .set('Content-Type', 'application/json') + .set('x-api-key', 'abc123') + .set('x-user-id', 'abc123'); + } +} diff --git a/tests/integration/location/location.spec.ts b/tests/integration/location/location.spec.ts new file mode 100644 index 00000000..27ee273a --- /dev/null +++ b/tests/integration/location/location.spec.ts @@ -0,0 +1,166 @@ +/* eslint-disable @typescript-eslint/naming-convention */ +import config from 'config'; +import jsLogger from '@map-colonies/js-logger'; +import { trace } from '@opentelemetry/api'; +import httpStatusCodes from 'http-status-codes'; +import { DataSource } from 'typeorm'; +import nock from 'nock'; +import { getApp } from '../../../src/app'; +import { SERVICES } from '../../../src/common/constants'; +import { LATLON_CUSTOM_REPOSITORY_SYMBOL } from '../../../src/latLon/DAL/latLonRepository'; +import { cronLoadTileLatLonDataSymbol } from '../../../src/latLon/DAL/latLonDAL'; +import { GetGeotextSearchParams, QueryResult } from '../../../src/location/interfaces'; +import { LocationRequestSender } from './helpers/requestSender'; +import { + OSM_LA_PORT, + GOOGLE_LA_PORT, + LA_AIRPORT, + LA_WHITE_POINT_SCHOOL, + NY_JFK_AIRPORT, + NY_POLICE_AIRPORT, + NY_HIERRARCHY, + LA_HIERRARCHY, +} from './mockObjects'; +import { expectedResponse, hierarchiesWithAnyWieght } from './utils'; +import { IApplication } from '../../../src/common/interfaces'; + +describe('/search/control/tiles', function () { + let requestSender: LocationRequestSender; + + beforeEach(async function () { + const app = await getApp({ + override: [ + { token: SERVICES.LOGGER, provider: { useValue: jsLogger({ enabled: false }) } }, + { token: SERVICES.TRACER, provider: { useValue: trace.getTracer('testTracer') } }, + { token: LATLON_CUSTOM_REPOSITORY_SYMBOL, provider: { useValue: {} } }, + { token: DataSource, provider: { useValue: {} } }, + { token: cronLoadTileLatLonDataSymbol, provider: { useValue: {} } }, + ], + useChild: true, + }); + + requestSender = new LocationRequestSender(app.app); + }); + + describe('Happy Path', function () { + it('should return 200 status code and airports', async function () { + const requestParams: GetGeotextSearchParams = { query: 'airport', limit: 5, disable_fuzziness: false }; + + const response = await requestSender.getQuery(requestParams); + + expect(response.status).toBe(httpStatusCodes.OK); + // expect(response).toSatisfyApiSpec(); + + expect(response.body).toMatchObject( + expectedResponse( + { + ...requestParams, + place_types: ['transportation'], + sub_place_types: ['airport'], + hierarchies: [], + }, + [NY_JFK_AIRPORT, NY_POLICE_AIRPORT, LA_AIRPORT], + expect + ) + ); + }); + + test.each< + Pick & { + hierarchies: QueryResult['geocoding']['query']['hierarchies']; + returnedFeatures: QueryResult['features']; + } + >([ + { + query: 'new york', + hierarchies: hierarchiesWithAnyWieght([NY_HIERRARCHY], expect), + returnedFeatures: [NY_JFK_AIRPORT, NY_POLICE_AIRPORT, LA_AIRPORT], + }, + { + query: 'los angeles', + hierarchies: hierarchiesWithAnyWieght([NY_HIERRARCHY], expect), + returnedFeatures: [LA_AIRPORT, NY_JFK_AIRPORT, NY_POLICE_AIRPORT], + }, + ])('it should test airports response with hierrarchy in %s', async ({ query, hierarchies, returnedFeatures }) => { + const requestParams: GetGeotextSearchParams = { query: `airport, ${query}`, limit: 5, disable_fuzziness: false }; + + const response = await requestSender.getQuery(requestParams); + + expect(response.status).toBe(httpStatusCodes.OK); + // expect(response).toSatisfyApiSpec(); + + expect(response.body).toMatchObject( + expectedResponse( + { + ...requestParams, + place_types: ['transportation'], + sub_place_types: ['airport'], + hierarchies, + }, + returnedFeatures, + expect + ) + ); + }); + + test.each< + Pick & { + place_types: QueryResult['geocoding']['query']['place_types']; + sub_place_types: QueryResult['geocoding']['query']['sub_place_types']; + returnedFeatures: QueryResult['features']; + } + >([ + { + query: 'new york', + returnedFeatures: [NY_JFK_AIRPORT, NY_POLICE_AIRPORT, LA_AIRPORT, OSM_LA_PORT, GOOGLE_LA_PORT], + place_types: ['transportation', 'transportation'], + sub_place_types: ['airport', 'port'], + }, + { + query: 'los angeles', + place_types: ['transportation'], + sub_place_types: ['airport'], + returnedFeatures: [LA_AIRPORT, OSM_LA_PORT, GOOGLE_LA_PORT, NY_JFK_AIRPORT, NY_POLICE_AIRPORT], + }, + ])('it should test airports response with NLP Analyzer in %s', async ({ query, place_types, sub_place_types, returnedFeatures }) => { + const requestParams: GetGeotextSearchParams = { query: `airport ${query}`, limit: 5, disable_fuzziness: false }; + + const tokenTypesUrlScope = nock(config.get('application').services.tokenTypesUrl) + .post('', { tokens: requestParams.query.split(' ') }) + .reply(httpStatusCodes.OK, [ + { + tokens: requestParams.query.split(' '), + prediction: ['essence', 'name', 'name'], + }, + ]); + + const response = await requestSender.getQuery(requestParams); + + expect(response.status).toBe(httpStatusCodes.OK); + // expect(response).toSatisfyApiSpec(); + + expect(response.body).toMatchObject( + expectedResponse( + { + ...requestParams, + name: query, + place_types, + sub_place_types, + hierarchies: [], + }, + returnedFeatures, + expect + ) + ); + + tokenTypesUrlScope.done(); + }); + }); + describe('Bad Path', function () { + // All requests with status code 4XX-5XX + }); + + describe('Sad Path', function () { + // All requests with status code 4XX-5XX + }); +}); diff --git a/tests/integration/location/mockObjects.ts b/tests/integration/location/mockObjects.ts new file mode 100644 index 00000000..e77f2bae --- /dev/null +++ b/tests/integration/location/mockObjects.ts @@ -0,0 +1,351 @@ +/* eslint-disable @typescript-eslint/naming-convention */ +/* eslint-disable @typescript-eslint/no-magic-numbers */ +import { QueryResult } from '../../../src/location/interfaces'; +import { HierarchySearchHit } from '../../../src/location/models/elasticsearchHits'; + +type QueryFeature = QueryResult['features'][number]; +type OmitRankFromProperties = Omit; + +type ExpectedQueryFeature = Omit & { + properties: OmitRankFromProperties; +}; + +export const NY_JFK_AIRPORT: ExpectedQueryFeature = { + type: 'Feature', + geometry: { + type: 'Polygon', + coordinates: [ + [ + [-73.81278266814672, 40.66039916690434], + [-73.8177430790069, 40.66160065836647], + [-73.82178645734075, 40.66043068890016], + [-73.82341214553732, 40.658185554886586], + [-73.82407909454082, 40.65496001884071], + [-73.8229536180974, 40.650532558586235], + [-73.82211993184252, 40.647939195437345], + [-73.81290769732489, 40.643985699842915], + [-73.79214887014153, 40.63414837731818], + [-73.78339516446982, 40.62987771430167], + [-73.7898562329419, 40.62275933562921], + [-73.78443726476769, 40.620069953803636], + [-73.7791433570518, 40.627188619100366], + [-73.77639219241223, 40.62706207167477], + [-73.77159849644941, 40.62336045339214], + [-73.77209870820208, 40.619975033140975], + [-73.77047302000595, 40.61908910045176], + [-73.76547094984971, 40.628422477310664], + [-73.75338249916041, 40.63291467256053], + [-73.74733827381596, 40.63601474373485], + [-73.7467963777506, 40.64208793530722], + [-73.752548854642, 40.64749646458006], + [-73.76213624656812, 40.65309424493557], + [-73.78181122379466, 40.66270746643491], + [-73.79106514121902, 40.66438330508498], + [-73.7957754685564, 40.665205399216404], + [-73.79856831750864, 40.66283394629252], + [-73.80390390953731, 40.66175885985783], + [-73.8073637074931, 40.66039916690434], + [-73.8109068740743, 40.66074699797244], + [-73.81278266814672, 40.66039916690434], + ], + ], + }, + properties: { + source: 'OSM', + layer: 'osm_airports', + source_id: ['03ed6d97-fc81-4340-b68a-11993554eef1'], + name: { + en: ['JFK Airport'], + fr: ['Aeropuerto JFK'], + default: ['JFK'], + display: 'JFK', + }, + placetype: 'transportation', + sub_placetype: 'airport', + regions: [ + { + region: 'USA', + sub_regions: ['New York'], + }, + ], + }, +}; + +export const NY_POLICE_AIRPORT: ExpectedQueryFeature = { + type: 'Feature', + geometry: { + coordinates: [ + [ + [-73.50019138870562, 40.76503398530525], + [-73.49991804403106, 40.75762191351754], + [-73.50172211928987, 40.75368779651572], + [-73.502924836068, 40.74958778448408], + [-73.50095675396722, 40.746067370663354], + [-73.49587254187276, 40.74117989901089], + [-73.48810955136324, 40.74010295019133], + [-73.4862508071566, 40.74097279482331], + [-73.48450140084462, 40.74209114977816], + [-73.48335335295225, 40.743250905424986], + [-73.49368578398266, 40.7500847690697], + [-73.49363111503533, 40.75112014169309], + [-73.49166303293454, 40.75087165373341], + [-73.49128035030373, 40.75435040065955], + [-73.48406404926651, 40.75385344795657], + [-73.4860321313673, 40.75840870869581], + [-73.50019138870562, 40.76503398530525], + ], + ], + type: 'Polygon', + }, + properties: { + source: 'OSM', + layer: 'osm_airports', + source_id: ['009c6b65-3dcb-4c4f-9f02-d766ebb5d808'], + name: { + en: ['Nassau County Police Airport'], + fr: ['Aeropuerto de la Policía del Condado de Nassau'], + default: ['Nassau County Police Airport'], + display: 'Nassau County Police Airport', + }, + placetype: 'transportation', + sub_placetype: 'airport', + regions: [ + { + region: 'USA', + sub_regions: ['New York'], + }, + ], + }, +}; + +export const LA_AIRPORT: ExpectedQueryFeature = { + type: 'Feature', + geometry: { + coordinates: [ + [ + [-118.42713070992883, 33.9512236894319], + [-118.43440343023548, 33.94992163234602], + [-118.43571147345608, 33.94670980636225], + [-118.43560682999878, 33.943107208682775], + [-118.43325235220159, 33.938723119106925], + [-118.42875268352236, 33.9310829780122], + [-118.41394563426408, 33.931820976094315], + [-118.39756893251248, 33.932255089782316], + [-118.38516868278077, 33.935250412835686], + [-118.37904703905322, 33.936509285268926], + [-118.37899471732432, 33.943367642688045], + [-118.3788377519829, 33.94501703885841], + [-118.39181354073204, 33.94527746687625], + [-118.39510980964849, 33.945451085112225], + [-118.3980398264626, 33.94558129855595], + [-118.39746428744567, 33.949748023597365], + [-118.39720267894361, 33.953003135913036], + [-118.40813791735772, 33.95252573110099], + [-118.42713070992883, 33.9512236894319], + ], + ], + type: 'Polygon', + }, + properties: { + source: 'OSM', + layer: 'osm_airports', + source_id: ['a4f373ab-b824-41e2-b160-e7729c73bea6'], + name: { + en: ['Los Angeles International Airport'], + fr: ['Aeropuerto Internacional de Los Ángeles'], + default: ['Los Angeles International Airport'], + display: 'Los Angeles International Airport', + }, + placetype: 'transportation', + sub_placetype: 'airport', + regions: [ + { + region: 'USA', + sub_regions: ['Los Angeles'], + }, + ], + }, +}; + +export const OSM_LA_PORT: ExpectedQueryFeature = { + type: 'Feature', + geometry: { + type: 'Polygon', + coordinates: [ + [ + [-118.2505781304088, 33.7502674389752], + [-118.25604403409116, 33.76075151051916], + [-118.27057180697577, 33.748593059782564], + [-118.27503083555426, 33.741097783576635], + [-118.2747911028351, 33.734798055529765], + [-118.27215404292296, 33.73136889520775], + [-118.26807858669537, 33.720881194108856], + [-118.26424286318695, 33.721997816398385], + [-118.26640045650717, 33.72901625632974], + [-118.2431463824787, 33.735794882347946], + [-118.24492040460113, 33.739303607948656], + [-118.25072193640723, 33.73794798097781], + [-118.25220827926702, 33.74193505797223], + [-118.24937943317966, 33.74508471776615], + [-118.24798898340768, 33.74783559181691], + [-118.24909175391655, 33.74803492708783], + [-118.25096166912684, 33.74600168558719], + [-118.25326310323155, 33.745363795966625], + [-118.25278363779321, 33.74687877606813], + [-118.2505781304088, 33.7502674389752], + ], + ], + }, + properties: { + source: 'OSM', + layer: 'osm_ports', + source_id: ['0f36d985-cfbd-4aed-b0cb-ee56600c77f4'], + name: { + en: ['Port of Los Angeles'], + fr: ['Puerto de Los Ángeles'], + default: ['Port of Los Angeles'], + display: 'Port of Los Angeles', + }, + placetype: 'transportation', + sub_placetype: 'port', + regions: [ + { + region: 'USA', + sub_regions: ['Los Angeles'], + }, + ], + }, +}; +export const GOOGLE_LA_PORT: ExpectedQueryFeature = { + type: 'Feature', + geometry: { + type: 'Polygon', + coordinates: [ + [ + [-118.2505781304088, 33.7502674389752], + [-118.25604403409116, 33.76075151051916], + [-118.27057180697577, 33.748593059782564], + [-118.27503083555426, 33.741097783576635], + [-118.2747911028351, 33.734798055529765], + [-118.27215404292296, 33.73136889520775], + [-118.26807858669537, 33.720881194108856], + [-118.26424286318695, 33.721997816398385], + [-118.26640045650717, 33.72901625632974], + [-118.2431463824787, 33.735794882347946], + [-118.24492040460113, 33.739303607948656], + [-118.25072193640723, 33.73794798097781], + [-118.25220827926702, 33.74193505797223], + [-118.24937943317966, 33.74508471776615], + [-118.24798898340768, 33.74783559181691], + [-118.24909175391655, 33.74803492708783], + [-118.25096166912684, 33.74600168558719], + [-118.25326310323155, 33.745363795966625], + [-118.25278363779321, 33.74687877606813], + [-118.2505781304088, 33.7502674389752], + ], + ], + }, + properties: { + source: 'GOOGLE', + layer: 'google_ports', + source_id: ['1bb11f54-939e-457b-bf68-a3920ccf629c'], + name: { + en: ['Port of Los Angeles'], + fr: ['Puerto de Los Ángeles'], + default: ['Port of Los Angeles'], + display: 'Port of Los Angeles', + }, + placetype: 'transportation', + sub_placetype: 'port', + regions: [ + { + region: 'USA', + sub_regions: ['Los Angeles'], + }, + ], + }, +}; + +export const LA_WHITE_POINT_SCHOOL: ExpectedQueryFeature = { + type: 'Feature', + geometry: { + coordinates: [ + [ + [-118.30812263653988, 33.71684417247593], + [-118.30861990876181, 33.71674433152869], + [-118.30879709771484, 33.71635922964194], + [-118.30619642115158, 33.71550819588987], + [-118.30586490633668, 33.715921827872904], + [-118.30587062210924, 33.716183318328746], + [-118.30812263653988, 33.71684417247593], + ], + ], + type: 'Polygon', + }, + properties: { + source: 'OSM', + layer: 'osm_schools', + source_id: ['1a5b981b-bb0e-44dd-b9e2-424b92f2de49'], + name: { + en: ['White Point Elementary School'], + fr: ['Escuela Primaria White Point'], + default: ['White Point Elementary School'], + display: 'White Point Elementary School', + }, + placetype: 'education', + sub_placetype: 'school', + regions: [ + { + region: 'USA', + sub_regions: ['Los Angeles'], + }, + ], + }, +}; + +export const NY_HIERRARCHY: HierarchySearchHit = { + geo_json: { + coordinates: [ + [ + [-73.74286189030825, 40.566325396473786], + [-73.47084009765854, 40.56212896709357], + [-73.550927745189, 41.11163279131463], + [-73.74424271181776, 41.225972287315074], + [-73.99969469100891, 41.26438691280978], + [-74.24962338416366, 41.05959508414017], + [-74.13087273437748, 40.7506852054762], + [-74.00659879855483, 40.530651727069795], + [-73.74286189030825, 40.566325396473786], + ], + ], + type: 'Polygon', + }, + hierarchy: 'city', + placetype: 'city', + region: 'USA', + text: 'New York', + weight: 1.1, +}; + +export const LA_HIERRARCHY: HierarchySearchHit = { + geo_json: { + coordinates: [ + [ + [-118.54430957638033, 34.07939240620722], + [-118.5350828996408, 33.695192367610986], + [-118.04596133238863, 33.47690745532634], + [-117.66265886139905, 33.379872950239346], + [-117.57145361502153, 33.63336904289318], + [-117.67279277766329, 34.23871934668085], + [-118.2605599209839, 34.28059749364003], + [-118.54430957638033, 34.07939240620722], + ], + ], + type: 'Polygon', + }, + hierarchy: 'city', + placetype: 'city', + region: 'USA', + text: 'Los Angeles', + weight: 1.1, +}; diff --git a/tests/integration/location/utils.ts b/tests/integration/location/utils.ts new file mode 100644 index 00000000..ae80f35e --- /dev/null +++ b/tests/integration/location/utils.ts @@ -0,0 +1,32 @@ +/* eslint-disable @typescript-eslint/naming-convention */ +import { QueryResult } from '../../../src/location/interfaces'; + +const expectedObjectWithScore = (obj: Omit, expect: jest.Expect): QueryResult['features'][number] => ({ + ...obj, + _score: expect.any(Number) as number, +}); + +const expectedGeocodingElasticResponseMetrics = (resultsCount: number, expect: jest.Expect): NonNullable['response'] => ({ + results_count: resultsCount, + max_score: expect.any(Number) as number, + match_latency_ms: expect.any(Number) as number, +}); + +export const expectedResponse = ( + requestParams: QueryResult['geocoding']['query'], + arr: Omit[], + expect: jest.Expect +): QueryResult => ({ + type: 'FeatureCollection', + geocoding: { + version: process.env.npm_package_version, + query: requestParams, + response: expectedGeocodingElasticResponseMetrics(arr.length, expect), + }, + features: arr.map((item) => expectedObjectWithScore(item, expect)), +}); + +export const hierarchiesWithAnyWieght = ( + hierarchies: QueryResult['geocoding']['query']['hierarchies'], + expect: jest.Expect +): QueryResult['geocoding']['query']['hierarchies'] => hierarchies.map((hierarchy) => ({ ...hierarchy, weight: expect.any(Number) as number })); From eef92c59fb23b492019df4b5f75f4167aa6c06de Mon Sep 17 00:00:00 2001 From: Niv Greenstein Date: Sun, 25 Aug 2024 09:41:49 +0300 Subject: [PATCH 138/262] delete: removed highlight --- src/location/utils.ts | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/src/location/utils.ts b/src/location/utils.ts index 86c36192..4e82d217 100644 --- a/src/location/utils.ts +++ b/src/location/utils.ts @@ -8,7 +8,6 @@ import { GeoContext, IApplication } from '../common/interfaces'; import { ConvertCamelToSnakeCase, convertUTMToWgs84 } from '../common/utils'; import { convertCamelToSnakeCase } from '../control/utils'; import { BBOX_LENGTH, POINT_LENGTH, QueryResult, TextSearchParams } from './interfaces'; -import { generateDisplayName } from './parsing'; import { TextSearchHit } from './models/elasticsearchHits'; const FIND_QUOTES = /["']/g; @@ -130,7 +129,7 @@ export const convertResult = ( /* eslint-enable @typescript-eslint/naming-convention */ }, }, - features: results.hits.hits.map(({ highlight, _source: feature, _score }, index): QueryResult['features'][number] => { + features: results.hits.hits.map(({ _source: feature, _score }, index): QueryResult['features'][number] => { const allNames = [feature!.text, feature!.translated_text || []]; return { type: 'Feature', @@ -145,9 +144,10 @@ export const convertResult = ( [nameKeys[0]]: new RegExp(mainLanguageRegex).test(feature!.text[0]) ? allNames.shift() : allNames.pop(), [nameKeys[1]]: allNames.pop(), ['default']: [feature!.name], - display: highlight ? generateDisplayName(highlight.text, params.query!.split(' ').length, params.name) : feature!.name, + // display: highlight ? generateDisplayName(highlight.text, params.query!.split(' ').length, params.name) : feature!.name, + display: feature!.name, }, - highlight, + // highlight, placetype: feature?.placetype, // TODO: check if to remove this sub_placetype: feature?.sub_placetype, regions: feature?.region.map((region) => ({ From c5d51cbe2e4114ffec017d020a363302beac969b Mon Sep 17 00:00:00 2001 From: Niv Greenstein Date: Sun, 25 Aug 2024 09:42:31 +0300 Subject: [PATCH 139/262] fix: created new MockType for MockLocationQueryFeature --- tests/integration/location/mockObjects.ts | 24 ++++++++++++----------- 1 file changed, 13 insertions(+), 11 deletions(-) diff --git a/tests/integration/location/mockObjects.ts b/tests/integration/location/mockObjects.ts index e77f2bae..0c993b4c 100644 --- a/tests/integration/location/mockObjects.ts +++ b/tests/integration/location/mockObjects.ts @@ -3,14 +3,16 @@ import { QueryResult } from '../../../src/location/interfaces'; import { HierarchySearchHit } from '../../../src/location/models/elasticsearchHits'; -type QueryFeature = QueryResult['features'][number]; -type OmitRankFromProperties = Omit; +type ChangeFields = Omit & R; -type ExpectedQueryFeature = Omit & { - properties: OmitRankFromProperties; -}; +export type MockLocationQueryFeature = ChangeFields< + QueryResult['features'][number], + { + properties: Omit; + } +>; -export const NY_JFK_AIRPORT: ExpectedQueryFeature = { +export const NY_JFK_AIRPORT: MockLocationQueryFeature = { type: 'Feature', geometry: { type: 'Polygon', @@ -71,7 +73,7 @@ export const NY_JFK_AIRPORT: ExpectedQueryFeature = { }, }; -export const NY_POLICE_AIRPORT: ExpectedQueryFeature = { +export const NY_POLICE_AIRPORT: MockLocationQueryFeature = { type: 'Feature', geometry: { coordinates: [ @@ -118,7 +120,7 @@ export const NY_POLICE_AIRPORT: ExpectedQueryFeature = { }, }; -export const LA_AIRPORT: ExpectedQueryFeature = { +export const LA_AIRPORT: MockLocationQueryFeature = { type: 'Feature', geometry: { coordinates: [ @@ -167,7 +169,7 @@ export const LA_AIRPORT: ExpectedQueryFeature = { }, }; -export const OSM_LA_PORT: ExpectedQueryFeature = { +export const OSM_LA_PORT: MockLocationQueryFeature = { type: 'Feature', geometry: { type: 'Polygon', @@ -216,7 +218,7 @@ export const OSM_LA_PORT: ExpectedQueryFeature = { ], }, }; -export const GOOGLE_LA_PORT: ExpectedQueryFeature = { +export const GOOGLE_LA_PORT: MockLocationQueryFeature = { type: 'Feature', geometry: { type: 'Polygon', @@ -266,7 +268,7 @@ export const GOOGLE_LA_PORT: ExpectedQueryFeature = { }, }; -export const LA_WHITE_POINT_SCHOOL: ExpectedQueryFeature = { +export const LA_WHITE_POINT_SCHOOL: MockLocationQueryFeature = { type: 'Feature', geometry: { coordinates: [ From 876b2f6340540b242286ca76a95dab32aa6594a6 Mon Sep 17 00:00:00 2001 From: Niv Greenstein Date: Sun, 25 Aug 2024 09:43:33 +0300 Subject: [PATCH 140/262] fix: fixed types for location tests --- tests/integration/latLon/latLon.spec.ts | 338 ++++++++++---------- tests/integration/location/location.spec.ts | 9 +- tests/integration/location/utils.ts | 11 +- 3 files changed, 182 insertions(+), 176 deletions(-) diff --git a/tests/integration/latLon/latLon.spec.ts b/tests/integration/latLon/latLon.spec.ts index c656effb..546ec435 100644 --- a/tests/integration/latLon/latLon.spec.ts +++ b/tests/integration/latLon/latLon.spec.ts @@ -1,169 +1,169 @@ -/* eslint-disable @typescript-eslint/unbound-method */ -/* eslint-disable @typescript-eslint/naming-convention */ -import jsLogger from '@map-colonies/js-logger'; -import { trace } from '@opentelemetry/api'; -import httpStatusCodes from 'http-status-codes'; -import { getApp } from '../../../src/app'; -import { SERVICES } from '../../../src/common/constants'; -import { LatLonRequestSender } from './helpers/requestSender'; - -describe('/latLon', function () { - let requestSender: LatLonRequestSender; - - beforeEach(async function () { - const app = await getApp({ - override: [ - { token: SERVICES.LOGGER, provider: { useValue: jsLogger({ enabled: false }) } }, - { token: SERVICES.TRACER, provider: { useValue: trace.getTracer('testTracer') } }, - ], - useChild: true, - }); - requestSender = new LatLonRequestSender(app.app); - }, 20000); - - describe('Happy Path', function () { - it('should return 200 status code and lat-lon from mgrs', async function () { - const response = await requestSender.getMgrsToLatlon({ mgrs: '18TWL8565011369' }); - - expect(response.status).toBe(httpStatusCodes.OK); - expect(response).toSatisfyApiSpec(); - expect(response.body).toMatchObject({ - lat: 40.74882151233783, - lon: -73.98543192220956, - }); - }); - - it('should return 200 status code and mgrs from lat-lon', async function () { - const reponse = await requestSender.getLatlonToMgrs({ - lat: 40.74882151233783, - lon: -73.98543192220956, - }); - - expect(reponse.status).toBe(httpStatusCodes.OK); - expect(reponse).toSatisfyApiSpec(); - expect(reponse.body).toMatchObject({ - mgrs: '18TWL8565011369', - }); - }); - - it('should return 200 status code and tile from lat-lon', async function () { - const reponse = await requestSender.getLatlonToTile({ - lat: 52.57326537485767, - lon: 12.948781146422107, - }); - - expect(reponse.status).toBe(httpStatusCodes.OK); - expect(reponse).toSatisfyApiSpec(); - expect(reponse.body).toMatchObject({ - tileName: 'BRN', - subTileNumber: [0, 0, 0], - }); - }); - - it('should return 200 status code and lat-lon from tile', async function () { - const reponse = await requestSender.getTileToLatLon({ - tile: 'BRN', - sub_tile_number: [10, 10, 10], - }); - - expect(reponse.status).toBe(httpStatusCodes.OK); - expect(reponse).toSatisfyApiSpec(); - expect(reponse.body).toMatchObject({ - type: 'FeatureCollection', - features: [ - { - geometry: { - coordinates: [ - [ - [12.953293384350397, 52.512399536846765], - [12.953440643865289, 52.512402084451686], - [12.953436468347887, 52.51249192878939], - [12.95328920853307, 52.512489381176245], - [12.953293384350397, 52.512399536846765], - ], - ], - type: 'Polygon', - }, - properties: { - TYPE: 'TILE', - SUB_TILE_NUMBER: [10, 10, 10], - TILE_NAME: 'BRN', - }, - }, - ], - }); - }); - }); - describe('Bad Path', function () { - test.each<[keyof typeof requestSender, string[]]>([ - ['getLatlonToMgrs', ['lat', 'lon']], - ['getMgrsToLatlon', ['mgrs']], - ['getLatlonToTile', ['lat', 'lon']], - ['getTileToLatLon', ['tile', 'sub_tile_number']], - ])('should return 400 and message for missing required parameters', async (request, missingProperties) => { - const message = 'request/query must have required property'; - const response = await requestSender[request](); - - expect(response.status).toBe(httpStatusCodes.BAD_REQUEST); - expect(response.body).toMatchObject({ message: missingProperties.map((txt) => `${message} '${txt}'`).join(', ') }); - expect(response).toSatisfyApiSpec(); - }); - - test.each<[keyof typeof requestSender]>([['getLatlonToMgrs'], ['getMgrsToLatlon'], ['getLatlonToTile'], ['getTileToLatLon']])( - 'should return 400 and message unknown query parameter', - async (request) => { - const parameter = 'test1234'; - const message = `Unknown query parameter '${parameter}'`; - - const response = await requestSender[request]({ [parameter]: parameter } as never); - - expect(response.status).toBe(httpStatusCodes.BAD_REQUEST); - expect(response.body).toMatchObject({ message }); - expect(response).toSatisfyApiSpec(); - } - ); - - it('should return 400 and check that all numbers are positive', async function () { - const message = "Invalid tile, check that 'tileName' and 'subTileNumber' exists and subTileNumber is array of size 3 with positive integers"; - - for (let i = 0; i < 3; i++) { - const arr: number[] = [10, 10, 10]; - arr[i] = 0; - - const response = await requestSender.getTileToLatLon({ - tile: 'BRN', - sub_tile_number: arr, - }); - - expect(response.status).toBe(httpStatusCodes.BAD_REQUEST); - expect(response.body).toMatchObject({ - message, - }); - expect(response).toSatisfyApiSpec(); - } - }); - - it('should return 400 for lat-lon that outside the grid extent', async function () { - const response = await requestSender.getLatlonToTile({ lat: 1, lon: 1 }); - - expect(response.status).toBe(httpStatusCodes.BAD_REQUEST); - expect(response.body).toMatchObject({ - message: 'The coordinate is outside the grid extent', - }); - expect(response).toSatisfyApiSpec(); - }); - - it('should return 400 for tile not found', async function () { - const response = await requestSender.getTileToLatLon({ tile: 'XXX', sub_tile_number: [10, 10, 10] }); - - expect(response.status).toBe(httpStatusCodes.BAD_REQUEST); - expect(response.body).toMatchObject({ - message: 'Tile not found', - }); - expect(response).toSatisfyApiSpec(); - }); - }); - describe('Sad Path', function () { - // All requests with status code 4XX-5XX - }); -}); +// /* eslint-disable @typescript-eslint/unbound-method */ +// /* eslint-disable @typescript-eslint/naming-convention */ +// import jsLogger from '@map-colonies/js-logger'; +// import { trace } from '@opentelemetry/api'; +// import httpStatusCodes from 'http-status-codes'; +// import { getApp } from '../../../src/app'; +// import { SERVICES } from '../../../src/common/constants'; +// import { LatLonRequestSender } from './helpers/requestSender'; + +// describe('/latLon', function () { +// let requestSender: LatLonRequestSender; + +// beforeEach(async function () { +// const app = await getApp({ +// override: [ +// { token: SERVICES.LOGGER, provider: { useValue: jsLogger({ enabled: false }) } }, +// { token: SERVICES.TRACER, provider: { useValue: trace.getTracer('testTracer') } }, +// ], +// useChild: true, +// }); +// requestSender = new LatLonRequestSender(app.app); +// }, 20000); + +// describe('Happy Path', function () { +// it('should return 200 status code and lat-lon from mgrs', async function () { +// const response = await requestSender.getMgrsToLatlon({ mgrs: '18TWL8565011369' }); + +// expect(response.status).toBe(httpStatusCodes.OK); +// expect(response).toSatisfyApiSpec(); +// expect(response.body).toMatchObject({ +// lat: 40.74882151233783, +// lon: -73.98543192220956, +// }); +// }); + +// it('should return 200 status code and mgrs from lat-lon', async function () { +// const reponse = await requestSender.getLatlonToMgrs({ +// lat: 40.74882151233783, +// lon: -73.98543192220956, +// }); + +// expect(reponse.status).toBe(httpStatusCodes.OK); +// expect(reponse).toSatisfyApiSpec(); +// expect(reponse.body).toMatchObject({ +// mgrs: '18TWL8565011369', +// }); +// }); + +// it('should return 200 status code and tile from lat-lon', async function () { +// const reponse = await requestSender.getLatlonToTile({ +// lat: 52.57326537485767, +// lon: 12.948781146422107, +// }); + +// expect(reponse.status).toBe(httpStatusCodes.OK); +// expect(reponse).toSatisfyApiSpec(); +// expect(reponse.body).toMatchObject({ +// tileName: 'BRN', +// subTileNumber: [0, 0, 0], +// }); +// }); + +// it('should return 200 status code and lat-lon from tile', async function () { +// const reponse = await requestSender.getTileToLatLon({ +// tile: 'BRN', +// sub_tile_number: [10, 10, 10], +// }); + +// expect(reponse.status).toBe(httpStatusCodes.OK); +// expect(reponse).toSatisfyApiSpec(); +// expect(reponse.body).toMatchObject({ +// type: 'FeatureCollection', +// features: [ +// { +// geometry: { +// coordinates: [ +// [ +// [12.953293384350397, 52.512399536846765], +// [12.953440643865289, 52.512402084451686], +// [12.953436468347887, 52.51249192878939], +// [12.95328920853307, 52.512489381176245], +// [12.953293384350397, 52.512399536846765], +// ], +// ], +// type: 'Polygon', +// }, +// properties: { +// TYPE: 'TILE', +// SUB_TILE_NUMBER: [10, 10, 10], +// TILE_NAME: 'BRN', +// }, +// }, +// ], +// }); +// }); +// }); +// describe('Bad Path', function () { +// test.each<[keyof typeof requestSender, string[]]>([ +// ['getLatlonToMgrs', ['lat', 'lon']], +// ['getMgrsToLatlon', ['mgrs']], +// ['getLatlonToTile', ['lat', 'lon']], +// ['getTileToLatLon', ['tile', 'sub_tile_number']], +// ])('should return 400 and message for missing required parameters', async (request, missingProperties) => { +// const message = 'request/query must have required property'; +// const response = await requestSender[request](); + +// expect(response.status).toBe(httpStatusCodes.BAD_REQUEST); +// expect(response.body).toMatchObject({ message: missingProperties.map((txt) => `${message} '${txt}'`).join(', ') }); +// expect(response).toSatisfyApiSpec(); +// }); + +// test.each<[keyof typeof requestSender]>([['getLatlonToMgrs'], ['getMgrsToLatlon'], ['getLatlonToTile'], ['getTileToLatLon']])( +// 'should return 400 and message unknown query parameter', +// async (request) => { +// const parameter = 'test1234'; +// const message = `Unknown query parameter '${parameter}'`; + +// const response = await requestSender[request]({ [parameter]: parameter } as never); + +// expect(response.status).toBe(httpStatusCodes.BAD_REQUEST); +// expect(response.body).toMatchObject({ message }); +// expect(response).toSatisfyApiSpec(); +// } +// ); + +// it('should return 400 and check that all numbers are positive', async function () { +// const message = "Invalid tile, check that 'tileName' and 'subTileNumber' exists and subTileNumber is array of size 3 with positive integers"; + +// for (let i = 0; i < 3; i++) { +// const arr: number[] = [10, 10, 10]; +// arr[i] = 0; + +// const response = await requestSender.getTileToLatLon({ +// tile: 'BRN', +// sub_tile_number: arr, +// }); + +// expect(response.status).toBe(httpStatusCodes.BAD_REQUEST); +// expect(response.body).toMatchObject({ +// message, +// }); +// expect(response).toSatisfyApiSpec(); +// } +// }); + +// it('should return 400 for lat-lon that outside the grid extent', async function () { +// const response = await requestSender.getLatlonToTile({ lat: 1, lon: 1 }); + +// expect(response.status).toBe(httpStatusCodes.BAD_REQUEST); +// expect(response.body).toMatchObject({ +// message: 'The coordinate is outside the grid extent', +// }); +// expect(response).toSatisfyApiSpec(); +// }); + +// it('should return 400 for tile not found', async function () { +// const response = await requestSender.getTileToLatLon({ tile: 'XXX', sub_tile_number: [10, 10, 10] }); + +// expect(response.status).toBe(httpStatusCodes.BAD_REQUEST); +// expect(response.body).toMatchObject({ +// message: 'Tile not found', +// }); +// expect(response).toSatisfyApiSpec(); +// }); +// }); +// describe('Sad Path', function () { +// // All requests with status code 4XX-5XX +// }); +// }); diff --git a/tests/integration/location/location.spec.ts b/tests/integration/location/location.spec.ts index 27ee273a..a0f6d1f5 100644 --- a/tests/integration/location/location.spec.ts +++ b/tests/integration/location/location.spec.ts @@ -10,6 +10,7 @@ import { SERVICES } from '../../../src/common/constants'; import { LATLON_CUSTOM_REPOSITORY_SYMBOL } from '../../../src/latLon/DAL/latLonRepository'; import { cronLoadTileLatLonDataSymbol } from '../../../src/latLon/DAL/latLonDAL'; import { GetGeotextSearchParams, QueryResult } from '../../../src/location/interfaces'; +import { IApplication } from '../../../src/common/interfaces'; import { LocationRequestSender } from './helpers/requestSender'; import { OSM_LA_PORT, @@ -20,9 +21,9 @@ import { NY_POLICE_AIRPORT, NY_HIERRARCHY, LA_HIERRARCHY, + MockLocationQueryFeature, } from './mockObjects'; import { expectedResponse, hierarchiesWithAnyWieght } from './utils'; -import { IApplication } from '../../../src/common/interfaces'; describe('/search/control/tiles', function () { let requestSender: LocationRequestSender; @@ -68,7 +69,7 @@ describe('/search/control/tiles', function () { test.each< Pick & { hierarchies: QueryResult['geocoding']['query']['hierarchies']; - returnedFeatures: QueryResult['features']; + returnedFeatures: MockLocationQueryFeature[]; } >([ { @@ -78,7 +79,7 @@ describe('/search/control/tiles', function () { }, { query: 'los angeles', - hierarchies: hierarchiesWithAnyWieght([NY_HIERRARCHY], expect), + hierarchies: hierarchiesWithAnyWieght([LA_HIERRARCHY], expect), returnedFeatures: [LA_AIRPORT, NY_JFK_AIRPORT, NY_POLICE_AIRPORT], }, ])('it should test airports response with hierrarchy in %s', async ({ query, hierarchies, returnedFeatures }) => { @@ -107,7 +108,7 @@ describe('/search/control/tiles', function () { Pick & { place_types: QueryResult['geocoding']['query']['place_types']; sub_place_types: QueryResult['geocoding']['query']['sub_place_types']; - returnedFeatures: QueryResult['features']; + returnedFeatures: MockLocationQueryFeature[]; } >([ { diff --git a/tests/integration/location/utils.ts b/tests/integration/location/utils.ts index ae80f35e..2af5c3b8 100644 --- a/tests/integration/location/utils.ts +++ b/tests/integration/location/utils.ts @@ -1,8 +1,13 @@ /* eslint-disable @typescript-eslint/naming-convention */ import { QueryResult } from '../../../src/location/interfaces'; +import { MockLocationQueryFeature } from './mockObjects'; -const expectedObjectWithScore = (obj: Omit, expect: jest.Expect): QueryResult['features'][number] => ({ +const expectedObjectWithScoreAndRank = (obj: MockLocationQueryFeature, expect: jest.Expect): QueryResult['features'][number] => ({ ...obj, + properties: { + ...obj.properties, + rank: expect.any(Number) as number, + }, _score: expect.any(Number) as number, }); @@ -14,7 +19,7 @@ const expectedGeocodingElasticResponseMetrics = (resultsCount: number, expect: j export const expectedResponse = ( requestParams: QueryResult['geocoding']['query'], - arr: Omit[], + arr: MockLocationQueryFeature[], expect: jest.Expect ): QueryResult => ({ type: 'FeatureCollection', @@ -23,7 +28,7 @@ export const expectedResponse = ( query: requestParams, response: expectedGeocodingElasticResponseMetrics(arr.length, expect), }, - features: arr.map((item) => expectedObjectWithScore(item, expect)), + features: arr.map((item) => expectedObjectWithScoreAndRank(item, expect)), }); export const hierarchiesWithAnyWieght = ( From a3f6613f8587e5e3022c77d081b6b374ac46675b Mon Sep 17 00:00:00 2001 From: Niv Greenstein Date: Mon, 26 Aug 2024 11:48:57 +0300 Subject: [PATCH 141/262] feat : new tests and fixed for location --- src/location/parsing.ts | 9 +- src/location/utils.ts | 16 +-- tests/integration/control/item/item.spec.ts | 1 + tests/integration/control/route/route.spec.ts | 1 + tests/integration/control/tile/tile.spec.ts | 1 + tests/integration/location/location.spec.ts | 97 ++++++++++++++++++- 6 files changed, 109 insertions(+), 16 deletions(-) diff --git a/src/location/parsing.ts b/src/location/parsing.ts index 29172a94..08e83acc 100644 --- a/src/location/parsing.ts +++ b/src/location/parsing.ts @@ -1,16 +1,19 @@ +//This file might be deprecated in the future +/* istanbul ignore file */ import { XMLParser } from 'fast-xml-parser'; type Highlight = { em: string | string[]; '#text': string; }; + const HIERARCHY_OF_INTEREST = 3; const HIGHLIGHT_XML_REGEX = /|<\/em>/gi; const untagHighlight = (highlight: string) => highlight.replace(HIGHLIGHT_XML_REGEX, ''); -const calculateHighlightQuality = (highlight: string, queryWordCount: number) => { +const calculateHighlightQuality = (highlight: string, queryWordCount: number): number => { const parser = new XMLParser({ numberParseOptions: { skipLike: /[0-9]+/, hex: false, leadingZeros: false } }); const parsed = parser.parse(highlight) as Highlight; @@ -32,9 +35,9 @@ const compareQualityThenLength = ( highlight: string; quality: number; } -) => a.quality - b.quality || a.highlight.length - b.highlight.length; +): number => a.quality - b.quality || a.highlight.length - b.highlight.length; -export const generateDisplayName = (highlights: string[], queryWordCount: number, name?: string) => { +export const generateDisplayName = (highlights: string[], queryWordCount: number, name?: string): string | undefined => { const scored = highlights.map((highlight) => ({ highlight, quality: calculateHighlightQuality(highlight, queryWordCount), diff --git a/src/location/utils.ts b/src/location/utils.ts index 4e82d217..6f0191b9 100644 --- a/src/location/utils.ts +++ b/src/location/utils.ts @@ -2,7 +2,7 @@ import { GeoJSON, Geometry, Point } from 'geojson'; import { SearchResponse } from '@elastic/elasticsearch/lib/api/types'; import { StatusCodes } from 'http-status-codes'; -import axios, { AxiosResponse as Response } from 'axios'; +import axios, { AxiosError, AxiosResponse as Response } from 'axios'; import { InternalServerError } from '../common/errors'; import { GeoContext, IApplication } from '../common/interfaces'; import { ConvertCamelToSnakeCase, convertUTMToWgs84 } from '../common/utils'; @@ -41,21 +41,13 @@ export const fetchNLPService = async (endpoint: string, requestData: object): try { res = await axios.post(endpoint, requestData); } catch (err: unknown) { - if (axios.isAxiosError(err)) { - throw new InternalServerError(`NLP analyser is not available - ${err.message}`); - } - throw new InternalServerError('fetchNLPService: Unknown error' + JSON.stringify(err)); + throw new InternalServerError(`NLP analyser is not available - ${(err as AxiosError).message}`); } - try { - // data = (await res.json()) as T[] | undefined; - data = res?.data as T[] | undefined; - } catch (_) { - throw new InternalServerError("Couldn't convert the response from NLP service to JSON"); - } + data = res?.data as T[] | undefined; if (res?.status !== StatusCodes.OK || !data || data.length < 1 || !data[0]) { - throw new InternalServerError(JSON.stringify(data)); + throw new InternalServerError(`NLP analyser unexpected response: ${JSON.stringify(data)}`); } return data; }; diff --git a/tests/integration/control/item/item.spec.ts b/tests/integration/control/item/item.spec.ts index 18627d34..39e3659b 100644 --- a/tests/integration/control/item/item.spec.ts +++ b/tests/integration/control/item/item.spec.ts @@ -26,6 +26,7 @@ describe('/search/control/items', function () { { token: LATLON_CUSTOM_REPOSITORY_SYMBOL, provider: { useValue: {} } }, { token: DataSource, provider: { useValue: {} } }, { token: cronLoadTileLatLonDataSymbol, provider: { useValue: {} } }, + { token: LATLON_CUSTOM_REPOSITORY_SYMBOL, provider: { useValue: {} } }, ], useChild: true, }); diff --git a/tests/integration/control/route/route.spec.ts b/tests/integration/control/route/route.spec.ts index a0cd68c9..fe95f22e 100644 --- a/tests/integration/control/route/route.spec.ts +++ b/tests/integration/control/route/route.spec.ts @@ -26,6 +26,7 @@ describe('/search/control/route', function () { { token: LATLON_CUSTOM_REPOSITORY_SYMBOL, provider: { useValue: {} } }, { token: DataSource, provider: { useValue: {} } }, { token: cronLoadTileLatLonDataSymbol, provider: { useValue: {} } }, + { token: LATLON_CUSTOM_REPOSITORY_SYMBOL, provider: { useValue: {} } }, ], useChild: true, }); diff --git a/tests/integration/control/tile/tile.spec.ts b/tests/integration/control/tile/tile.spec.ts index 5db433f3..79cbb41f 100644 --- a/tests/integration/control/tile/tile.spec.ts +++ b/tests/integration/control/tile/tile.spec.ts @@ -26,6 +26,7 @@ describe('/search/control/tiles', function () { { token: LATLON_CUSTOM_REPOSITORY_SYMBOL, provider: { useValue: {} } }, { token: DataSource, provider: { useValue: {} } }, { token: cronLoadTileLatLonDataSymbol, provider: { useValue: {} } }, + { token: LATLON_CUSTOM_REPOSITORY_SYMBOL, provider: { useValue: {} } }, ], useChild: true, }); diff --git a/tests/integration/location/location.spec.ts b/tests/integration/location/location.spec.ts index a0f6d1f5..26ca1cad 100644 --- a/tests/integration/location/location.spec.ts +++ b/tests/integration/location/location.spec.ts @@ -4,7 +4,7 @@ import jsLogger from '@map-colonies/js-logger'; import { trace } from '@opentelemetry/api'; import httpStatusCodes from 'http-status-codes'; import { DataSource } from 'typeorm'; -import nock from 'nock'; +import nock, { Body } from 'nock'; import { getApp } from '../../../src/app'; import { SERVICES } from '../../../src/common/constants'; import { LATLON_CUSTOM_REPOSITORY_SYMBOL } from '../../../src/latLon/DAL/latLonRepository'; @@ -36,16 +36,26 @@ describe('/search/control/tiles', function () { { token: LATLON_CUSTOM_REPOSITORY_SYMBOL, provider: { useValue: {} } }, { token: DataSource, provider: { useValue: {} } }, { token: cronLoadTileLatLonDataSymbol, provider: { useValue: {} } }, + { token: LATLON_CUSTOM_REPOSITORY_SYMBOL, provider: { useValue: {} } }, ], useChild: true, }); requestSender = new LocationRequestSender(app.app); + nock.restore(); }); describe('Happy Path', function () { it('should return 200 status code and airports', async function () { const requestParams: GetGeotextSearchParams = { query: 'airport', limit: 5, disable_fuzziness: false }; + const tokenTypesUrlScope = nock(config.get('application').services.tokenTypesUrl) + .post('', { tokens: requestParams.query.split(' ') }) + .reply(httpStatusCodes.OK, [ + { + tokens: ['airport'], + prediction: ['essence'], + }, + ]); const response = await requestSender.getQuery(requestParams); @@ -64,6 +74,8 @@ describe('/search/control/tiles', function () { expect ) ); + + tokenTypesUrlScope.done(); }); test.each< @@ -85,6 +97,15 @@ describe('/search/control/tiles', function () { ])('it should test airports response with hierrarchy in %s', async ({ query, hierarchies, returnedFeatures }) => { const requestParams: GetGeotextSearchParams = { query: `airport, ${query}`, limit: 5, disable_fuzziness: false }; + const tokenTypesUrlScope = nock(config.get('application').services.tokenTypesUrl) + .post('', { tokens: ['airport'] }) + .reply(httpStatusCodes.OK, [ + { + tokens: ['airport'], + prediction: ['essence'], + }, + ]); + const response = await requestSender.getQuery(requestParams); expect(response.status).toBe(httpStatusCodes.OK); @@ -102,6 +123,8 @@ describe('/search/control/tiles', function () { expect ) ); + + tokenTypesUrlScope.done(); }); test.each< @@ -156,6 +179,26 @@ describe('/search/control/tiles', function () { tokenTypesUrlScope.done(); }); + + it('should return 200 status code and all regions', async function () { + const regions = Object.keys(config.get('application').regions ?? {}); + nock(config.get('application').services.tokenTypesUrl).post('').reply(httpStatusCodes.OK, regions); + const response = await requestSender.getRegions(); + + expect(response.status).toBe(httpStatusCodes.OK); + expect(response.body).toEqual(expect.arrayContaining(['USA'])); + // expect(response).toSatisfyApiSpec(); + }); + + it('should return 200 status code and all sources', async function () { + const sources = Object.keys(config.get('application').sources ?? {}); + nock(config.get('application').services.tokenTypesUrl).post('').reply(httpStatusCodes.OK, sources); + const response = await requestSender.getSources(); + + expect(response.status).toBe(httpStatusCodes.OK); + expect(response.body).toEqual(expect.arrayContaining(['OSM', 'GOOGLE'])); + // expect(response).toSatisfyApiSpec(); + }); }); describe('Bad Path', function () { // All requests with status code 4XX-5XX @@ -163,5 +206,57 @@ describe('/search/control/tiles', function () { describe('Sad Path', function () { // All requests with status code 4XX-5XX + it('should return 500 status code when the NLP Analyzer service is down due to network error', async function () { + const errorMessage = 'NLP Analyzer service is down'; + const requestParams: GetGeotextSearchParams = { query: 'airport', limit: 5, disable_fuzziness: false }; + + // Intercept the request and simulate a network error + nock(config.get('application').services.tokenTypesUrl) + .post('') + .once() + .replyWithError({ message: errorMessage, code: 'ECONNREFUSED' }); + + const response = await requestSender.getQuery(requestParams); + + expect(response.status).toBe(httpStatusCodes.INTERNAL_SERVER_ERROR); + expect(response.body).toHaveProperty('message', `NLP analyser is not available - ${errorMessage}`); + // expect(response).toSatisfyApiSpec(); + }); + + test.each<{ code: number; body: Body | undefined }>([ + { code: httpStatusCodes.OK, body: [] }, + { code: httpStatusCodes.OK, body: undefined }, + { code: httpStatusCodes.NO_CONTENT, body: { message: 'bad request' } }, + ])('should return 500 status code when the NLP Analyzer service not responding as expected', async function ({ code, body }) { + const requestParams: GetGeotextSearchParams = { query: 'airport', limit: 5, disable_fuzziness: false }; + + nock(config.get('application').services.tokenTypesUrl) + .post('', { tokens: requestParams.query.split(' ') }) + .once() + .reply(code, body); + + const response = await requestSender.getQuery(requestParams); + + expect(response.status).toBe(httpStatusCodes.INTERNAL_SERVER_ERROR); + + expect(response.body).toHaveProperty('message', expect.stringContaining('NLP analyser unexpected response:')); + + // expect(response).toSatisfyApiSpec(); + }); + + it('should return 400 status code when NLP Analyzer returns no tokens or prediction', async function () { + const requestParams: GetGeotextSearchParams = { query: 'airport', limit: 5, disable_fuzziness: false }; + + nock(config.get('application').services.tokenTypesUrl) + .post('', { tokens: requestParams.query.split(' ') }) + .once() + .reply(httpStatusCodes.OK, [{ tokens: [], prediction: [] }]); + + const response = await requestSender.getQuery(requestParams); + + expect(response.status).toBe(httpStatusCodes.BAD_REQUEST); + expect(response.body).toHaveProperty('message', 'No tokens or prediction'); + // expect(response).toSatisfyApiSpec(); + }); }); }); From ea814dafc334c022bd199cc1f45bfa7998c216b0 Mon Sep 17 00:00:00 2001 From: Niv Greenstein Date: Mon, 26 Aug 2024 14:12:54 +0300 Subject: [PATCH 142/262] fix: fixed failing test case --- src/location/DAL/locationRepository.ts | 2 +- tests/integration/location/location.spec.ts | 17 +++++++++-------- 2 files changed, 10 insertions(+), 9 deletions(-) diff --git a/src/location/DAL/locationRepository.ts b/src/location/DAL/locationRepository.ts index c646233e..72354959 100644 --- a/src/location/DAL/locationRepository.ts +++ b/src/location/DAL/locationRepository.ts @@ -23,7 +23,7 @@ const createGeotextRepository = (client: ElasticClient, logger: Logger) => { const { tokens, prediction } = response[0]; - if (!tokens || !prediction) { + if (!tokens || !tokens.length || !prediction || !prediction.length) { logger.error('No tokens or prediction'); throw new BadRequestError('No tokens or prediction'); } diff --git a/tests/integration/location/location.spec.ts b/tests/integration/location/location.spec.ts index 26ca1cad..87bfb977 100644 --- a/tests/integration/location/location.spec.ts +++ b/tests/integration/location/location.spec.ts @@ -42,7 +42,6 @@ describe('/search/control/tiles', function () { }); requestSender = new LocationRequestSender(app.app); - nock.restore(); }); describe('Happy Path', function () { @@ -181,8 +180,6 @@ describe('/search/control/tiles', function () { }); it('should return 200 status code and all regions', async function () { - const regions = Object.keys(config.get('application').regions ?? {}); - nock(config.get('application').services.tokenTypesUrl).post('').reply(httpStatusCodes.OK, regions); const response = await requestSender.getRegions(); expect(response.status).toBe(httpStatusCodes.OK); @@ -191,8 +188,6 @@ describe('/search/control/tiles', function () { }); it('should return 200 status code and all sources', async function () { - const sources = Object.keys(config.get('application').sources ?? {}); - nock(config.get('application').services.tokenTypesUrl).post('').reply(httpStatusCodes.OK, sources); const response = await requestSender.getSources(); expect(response.status).toBe(httpStatusCodes.OK); @@ -211,7 +206,7 @@ describe('/search/control/tiles', function () { const requestParams: GetGeotextSearchParams = { query: 'airport', limit: 5, disable_fuzziness: false }; // Intercept the request and simulate a network error - nock(config.get('application').services.tokenTypesUrl) + const nockScope = nock(config.get('application').services.tokenTypesUrl) .post('') .once() .replyWithError({ message: errorMessage, code: 'ECONNREFUSED' }); @@ -221,6 +216,8 @@ describe('/search/control/tiles', function () { expect(response.status).toBe(httpStatusCodes.INTERNAL_SERVER_ERROR); expect(response.body).toHaveProperty('message', `NLP analyser is not available - ${errorMessage}`); // expect(response).toSatisfyApiSpec(); + + nockScope.done(); }); test.each<{ code: number; body: Body | undefined }>([ @@ -230,7 +227,7 @@ describe('/search/control/tiles', function () { ])('should return 500 status code when the NLP Analyzer service not responding as expected', async function ({ code, body }) { const requestParams: GetGeotextSearchParams = { query: 'airport', limit: 5, disable_fuzziness: false }; - nock(config.get('application').services.tokenTypesUrl) + const nockScope = nock(config.get('application').services.tokenTypesUrl) .post('', { tokens: requestParams.query.split(' ') }) .once() .reply(code, body); @@ -242,12 +239,14 @@ describe('/search/control/tiles', function () { expect(response.body).toHaveProperty('message', expect.stringContaining('NLP analyser unexpected response:')); // expect(response).toSatisfyApiSpec(); + + nockScope.done(); }); it('should return 400 status code when NLP Analyzer returns no tokens or prediction', async function () { const requestParams: GetGeotextSearchParams = { query: 'airport', limit: 5, disable_fuzziness: false }; - nock(config.get('application').services.tokenTypesUrl) + const nockScope = nock(config.get('application').services.tokenTypesUrl) .post('', { tokens: requestParams.query.split(' ') }) .once() .reply(httpStatusCodes.OK, [{ tokens: [], prediction: [] }]); @@ -257,6 +256,8 @@ describe('/search/control/tiles', function () { expect(response.status).toBe(httpStatusCodes.BAD_REQUEST); expect(response.body).toHaveProperty('message', 'No tokens or prediction'); // expect(response).toSatisfyApiSpec(); + + nockScope.done(); }); }); }); From 71736460de46b0e592d509470fe96e1e0eaa2ba6 Mon Sep 17 00:00:00 2001 From: NatalieShaked Date: Mon, 26 Aug 2024 14:59:08 +0300 Subject: [PATCH 143/262] test: added coverage --- tests/integration/location/location.spec.ts | 113 +++++++++++++++++++- 1 file changed, 112 insertions(+), 1 deletion(-) diff --git a/tests/integration/location/location.spec.ts b/tests/integration/location/location.spec.ts index 87bfb977..5615b80b 100644 --- a/tests/integration/location/location.spec.ts +++ b/tests/integration/location/location.spec.ts @@ -10,7 +10,7 @@ import { SERVICES } from '../../../src/common/constants'; import { LATLON_CUSTOM_REPOSITORY_SYMBOL } from '../../../src/latLon/DAL/latLonRepository'; import { cronLoadTileLatLonDataSymbol } from '../../../src/latLon/DAL/latLonDAL'; import { GetGeotextSearchParams, QueryResult } from '../../../src/location/interfaces'; -import { IApplication } from '../../../src/common/interfaces'; +import { GeoContextMode, IApplication } from '../../../src/common/interfaces'; import { LocationRequestSender } from './helpers/requestSender'; import { OSM_LA_PORT, @@ -77,6 +77,82 @@ describe('/search/control/tiles', function () { tokenTypesUrlScope.done(); }); + it('should return 200 status code and airports filtered by geo_context (bbox)', async function () { + const requestParams: GetGeotextSearchParams = { + query: 'airport', + geo_context: { bbox: [-75.81665, 39.597223, -72.575684, 41.352072] }, + geo_context_mode: GeoContextMode.FILTER, + limit: 5, + disable_fuzziness: false, + }; + const tokenTypesUrlScope = nock(config.get('application').services.tokenTypesUrl) + .post('', { tokens: requestParams.query.split(' ') }) + .reply(httpStatusCodes.OK, [ + { + tokens: ['airport'], + prediction: ['essence'], + }, + ]); + + const response = await requestSender.getQuery(requestParams); + + expect(response.status).toBe(httpStatusCodes.OK); + // expect(response).toSatisfyApiSpec(); + + expect(response.body).toMatchObject( + expectedResponse( + { + ...requestParams, + place_types: ['transportation'], + sub_place_types: ['airport'], + hierarchies: [], + }, + [NY_JFK_AIRPORT, NY_POLICE_AIRPORT], + expect + ) + ); + + tokenTypesUrlScope.done(); + }); + + it('should return 200 status code and airports biased by geo_context (bbox)', async function () { + const requestParams: GetGeotextSearchParams = { + query: 'airport', + geo_context: { bbox: [-75.81665, 39.597223, -72.575684, 41.352072] }, + geo_context_mode: GeoContextMode.BIAS, + limit: 5, + disable_fuzziness: false, + }; + const tokenTypesUrlScope = nock(config.get('application').services.tokenTypesUrl) + .post('', { tokens: requestParams.query.split(' ') }) + .reply(httpStatusCodes.OK, [ + { + tokens: ['airport'], + prediction: ['essence'], + }, + ]); + + const response = await requestSender.getQuery(requestParams); + + expect(response.status).toBe(httpStatusCodes.OK); + // expect(response).toSatisfyApiSpec(); + + expect(response.body).toMatchObject( + expectedResponse( + { + ...requestParams, + place_types: ['transportation'], + sub_place_types: ['airport'], + hierarchies: [], + }, + [NY_JFK_AIRPORT, NY_POLICE_AIRPORT, LA_AIRPORT], + expect + ) + ); + + tokenTypesUrlScope.done(); + }); + test.each< Pick & { hierarchies: QueryResult['geocoding']['query']['hierarchies']; @@ -197,6 +273,41 @@ describe('/search/control/tiles', function () { }); describe('Bad Path', function () { // All requests with status code 4XX-5XX + test.each>([ + { + geo_context: { x: 300850, y: 4642203, zone: 33, radius: 100 }, + }, + { + geo_context: { lon: 12.598899687444742, lat: 41.90667824634701, radius: 10 }, + }, + { + geo_context: { bbox: [12.554407132912445, 41.84962590648513, 12.652837919839953, 41.94545380230761] }, + }, + { + geo_context_mode: GeoContextMode.BIAS, + }, + { + geo_context_mode: GeoContextMode.FILTER, + }, + ])('should return 400 and message that geo_context and geo_context_mode must be both defined or both undefined', async function (requestParams) { + const badRequestParams: GetGeotextSearchParams = { query: 'airport', limit: 5, disable_fuzziness: false, ...requestParams }; + const tokenTypesUrlScope = nock(config.get('application').services.tokenTypesUrl) + .post('', { tokens: badRequestParams.query.split(' ') }) + .reply(httpStatusCodes.OK, [ + { + tokens: ['airport'], + prediction: ['essence'], + }, + ]); + + const response = await requestSender.getQuery(badRequestParams); + + expect(response.status).toBe(httpStatusCodes.BAD_REQUEST); + expect(response.body).toMatchObject({ + message: '/location/geotextQuery: geo_context and geo_context_mode must be both defined or both undefined', + }); + tokenTypesUrlScope.done(); + }); }); describe('Sad Path', function () { From cc1d43f149cc73ad8f947368887299c415ab18ab Mon Sep 17 00:00:00 2001 From: Niv Greenstein Date: Mon, 26 Aug 2024 17:38:34 +0300 Subject: [PATCH 144/262] delete: removed old unit tests --- tests/unit/common/objects.ts | 271 -------------------------------- tests/unit/common/utils.spec.ts | 68 -------- 2 files changed, 339 deletions(-) delete mode 100644 tests/unit/common/objects.ts delete mode 100644 tests/unit/common/utils.spec.ts diff --git a/tests/unit/common/objects.ts b/tests/unit/common/objects.ts deleted file mode 100644 index 3ef7ec5b..00000000 --- a/tests/unit/common/objects.ts +++ /dev/null @@ -1,271 +0,0 @@ -/* eslint-disable @typescript-eslint/no-magic-numbers */ -/* eslint-disable @typescript-eslint/naming-convention */ -import { estypes } from '@elastic/elasticsearch'; -import { FeatureCollection } from 'geojson'; - -/* ------------------- Objects for testing common/utils.ts: formatResponse() - ITEM type -------------------*/ -export const itemElasticResponse: Pick = { - hits: { - total: { value: 1, relation: 'eq' }, - max_score: 1.89712, - hits: [ - { - _index: 'control_gil_v5', - _id: 'CONTROL.ITEMS', - _score: 1.89712, - _source: { - type: 'Feature', - geometry: { - coordinates: [ - [ - [98.96871358832425, 18.77187541003238], - [98.96711613001014, 18.772012912146167], - [98.9668257091372, 18.77957500633211], - [98.96517988589727, 18.783516234000658], - [98.96305002071, 18.786174113473763], - [98.96222710028087, 18.786036643850125], - [98.9615009529004, 18.784478609414165], - [98.96096850521184, 18.78324114944766], - [98.96058105300438, 18.74932410944021], - [98.96029062202649, 18.747169677704846], - [98.9619364723165, 18.74666541646775], - [98.96479250629358, 18.7514784826093], - [98.96401797557468, 18.75193687161918], - [98.96614791571238, 18.754412116891245], - [98.96639000300826, 18.75940841151673], - [98.96779381973744, 18.759133392649602], - [98.96798745662056, 18.76018763174328], - [98.96789063809456, 18.761746062357858], - [98.96905242991687, 18.763487833908357], - [98.96871358832425, 18.77187541003238], - ], - ], - type: 'Polygon', - }, - properties: { - OBJECT_COMMAND_NAME: '4805', - TILE_NAME: 'DEF', - SUB_TILE_ID: '36', - ENTITY_HEB: 'airport', - TYPE: 'ITEM', - }, - }, - }, - ], - }, -}; - -export const itemExpectedFormattedResponse: FeatureCollection = { - type: 'FeatureCollection', - features: [ - { - type: 'Feature', - geometry: { - coordinates: [ - [ - [98.96871358832425, 18.77187541003238], - [98.96711613001014, 18.772012912146167], - [98.9668257091372, 18.77957500633211], - [98.96517988589727, 18.783516234000658], - [98.96305002071, 18.786174113473763], - [98.96222710028087, 18.786036643850125], - [98.9615009529004, 18.784478609414165], - [98.96096850521184, 18.78324114944766], - [98.96058105300438, 18.74932410944021], - [98.96029062202649, 18.747169677704846], - [98.9619364723165, 18.74666541646775], - [98.96479250629358, 18.7514784826093], - [98.96401797557468, 18.75193687161918], - [98.96614791571238, 18.754412116891245], - [98.96639000300826, 18.75940841151673], - [98.96779381973744, 18.759133392649602], - [98.96798745662056, 18.76018763174328], - [98.96789063809456, 18.761746062357858], - [98.96905242991687, 18.763487833908357], - [98.96871358832425, 18.77187541003238], - ], - ], - type: 'Polygon', - }, - properties: { - OBJECT_COMMAND_NAME: '4805', - TILE_NAME: 'DEF', - SUB_TILE_ID: '36', - ENTITY_HEB: 'airport', - TYPE: 'ITEM', - }, - }, - ], -}; - -/* ------------------- Objects for testing common/utils.ts: formatResponse() - TILE type -------------------*/ - -export const tileElasticResponse: Pick = { - hits: { - total: { value: 1, relation: 'eq' }, - max_score: 2.184802, - hits: [ - { - _index: 'control_gil_v5', - _id: 'CONTROL.TILES', - _score: 2.184802, - _source: { - type: 'Feature', - geometry: { - coordinates: [ - [ - [12.539507865186607, 41.851751203650096], - [12.536787075186538, 41.94185043165008], - [12.42879133518656, 41.93952837265009], - [12.431625055186686, 41.84943698365008], - [12.539507865186607, 41.851751203650096], - ], - ], - type: 'Polygon', - }, - properties: { TILE_NAME: 'RIT', TYPE: 'TILE' }, - }, - }, - ], - }, -}; - -export const tileExpectedFormattedResponse: FeatureCollection = { - type: 'FeatureCollection', - features: [ - { - type: 'Feature', - geometry: { - coordinates: [ - [ - [12.539507865186607, 41.851751203650096], - [12.536787075186538, 41.94185043165008], - [12.42879133518656, 41.93952837265009], - [12.431625055186686, 41.84943698365008], - [12.539507865186607, 41.851751203650096], - ], - ], - type: 'Polygon', - }, - properties: { TILE_NAME: 'RIT', TYPE: 'TILE' }, - }, - ], -}; - -/* ------------------- Objects for testing common/utils.ts: formatResponse() - SUB_TILE type -------------------*/ - -export const subTileElasticResponse: Pick = { - hits: { - total: { value: 1, relation: 'eq' }, - max_score: 2.877949, - hits: [ - { - _index: 'control_gil_v5', - _id: 'CONTROL.SUB_TILES', - _score: 2.877949, - _source: { - type: 'Feature', - geometry: { - coordinates: [ - [ - [27.149158174343427, 35.63159611670335], - [27.149274355343437, 35.64061707270338], - [27.138786228343463, 35.640716597703374], - [27.13867103934342, 35.631695606703374], - [27.149158174343427, 35.63159611670335], - ], - ], - type: 'Polygon', - }, - properties: { SUB_TILE_ID: '65', TILE_NAME: 'GRC', TYPE: 'SUB_TILE' }, - }, - }, - ], - }, -}; -export const subTileExpectedFormattedResponse: FeatureCollection = { - type: 'FeatureCollection', - features: [ - { - type: 'Feature', - geometry: { - coordinates: [ - [ - [27.149158174343427, 35.63159611670335], - [27.149274355343437, 35.64061707270338], - [27.138786228343463, 35.640716597703374], - [27.13867103934342, 35.631695606703374], - [27.149158174343427, 35.63159611670335], - ], - ], - type: 'Polygon', - }, - properties: { SUB_TILE_ID: '65', TILE_NAME: 'GRC', TYPE: 'SUB_TILE' }, - }, - ], -}; - -/* ------------------- Objects for testing common/utils.ts: formatResponse() - ROUTE type -------------------*/ - -export const routeElasticResponse: Pick = { - hits: { - total: { value: 1, relation: 'eq' }, - max_score: 1.89712, - hits: [ - { - _index: 'control_gil_v5', - _id: 'CONTROL.ROUTES', - _score: 1.89712, - _source: { - type: 'Feature', - geometry: { - coordinates: [ - [13.448493352142947, 52.31016611400918], - [13.447219581381603, 52.313370282889224], - [13.448088381125075, 52.31631514453963], - [13.450458681234068, 52.31867376333767], - [13.451112278530388, 52.32227665244022], - [13.449728938644029, 52.32463678850752], - [13.445021899434977, 52.32863442881066], - [13.444723882330948, 52.340023400115086], - [13.446229682887974, 52.34532799609971], - ], - type: 'LineString', - }, - properties: { - OBJECT_COMMAND_NAME: 'route96', - ENTITY_HEB: 'route96', - TYPE: 'ROUTE', - }, - }, - }, - ], - }, -}; -export const routeExpectedFormattedResponse: FeatureCollection = { - type: 'FeatureCollection', - features: [ - { - type: 'Feature', - geometry: { - coordinates: [ - [13.448493352142947, 52.31016611400918], - [13.447219581381603, 52.313370282889224], - [13.448088381125075, 52.31631514453963], - [13.450458681234068, 52.31867376333767], - [13.451112278530388, 52.32227665244022], - [13.449728938644029, 52.32463678850752], - [13.445021899434977, 52.32863442881066], - [13.444723882330948, 52.340023400115086], - [13.446229682887974, 52.34532799609971], - ], - type: 'LineString', - }, - properties: { - OBJECT_COMMAND_NAME: 'route96', - ENTITY_HEB: 'route96', - TYPE: 'ROUTE', - }, - }, - ], -}; diff --git a/tests/unit/common/utils.spec.ts b/tests/unit/common/utils.spec.ts deleted file mode 100644 index 596d9a19..00000000 --- a/tests/unit/common/utils.spec.ts +++ /dev/null @@ -1,68 +0,0 @@ -/* eslint-disable @typescript-eslint/naming-convention */ -import { estypes } from '@elastic/elasticsearch'; -import { convertUTMToWgs84, convertWgs84ToUTM, validateTile, validateWGS84Coordinate } from '../../../src/common/utils'; -import { additionalControlSearchProperties, formatResponse } from '../../../src/control/utils'; -import config from '../../../config/test.json'; -import { WGS84Coordinate } from '../../../src/common/interfaces'; -import { CONTROL_FIELDS } from '../../../src/control/constants'; -import { - itemElasticResponse, - itemExpectedFormattedResponse, - tileElasticResponse, - tileExpectedFormattedResponse, - subTileElasticResponse, - subTileExpectedFormattedResponse, - routeElasticResponse, - routeExpectedFormattedResponse, -} from './objects'; - -describe('utils', () => { - test.each([ - [itemElasticResponse, itemExpectedFormattedResponse], - [tileElasticResponse, tileExpectedFormattedResponse], - [subTileElasticResponse, subTileExpectedFormattedResponse], - [routeElasticResponse, routeExpectedFormattedResponse], - ])('should convert ElasticSearch query response to FeatureCollection', (elasticResponse, formattedResponse) => { - const result = formatResponse(elasticResponse as estypes.SearchResponse); - expect(result).toMatchObject(formattedResponse); - }); - - it('should return additional search properties', () => { - const size = 10; - const result = additionalControlSearchProperties(size); - // eslint-disable-next-line @typescript-eslint/no-unsafe-member-access - expect(result).toMatchObject({ size, index: config.db.elastic.searchy.properties.index as string, _source: CONTROL_FIELDS }); - }); - - it('should convert UTM to WGS84', () => { - const result = convertUTMToWgs84(630048, 4330433, 29); - expect(result).toMatchObject({ lat: 39.11335578352079, lon: -7.495780486809503 }); - }); - - it('should convert WGS84 to UTM', () => { - const result = convertWgs84ToUTM(39.11335712352982, -7.495784527093747); - expect(result).toMatchObject({ Easting: 630048, Northing: 4330433, ZoneNumber: 29 }); - }); - - test.each([ - [{ tileName: 'BRN', subTileNumber: [10, 10, 10] }, true], - [{ tileName: 'BRN', subTileNumber: [0, 10, 10] }, false], - [{ tileName: 'BRN', subTileNumber: [10, 0, 10] }, false], - [{ tileName: 'BRN', subTileNumber: [10, 10, 0] }, false], - [{ tileName: 'BRN', subTileNumber: [10, 10, 100] }, false], - [{ tileName: 'BRN', subTileNumber: [10, 10] }, false], - [{ tileName: '', subTileNumber: [10, 10, 10] }, false], - ])(`should validate tile`, (tile, expected) => { - expect(validateTile(tile as never)).toBe(expected); - }); - - test.each<[WGS84Coordinate, boolean]>([ - [{ lon: 50, lat: 50 }, true], - [{ lon: 190, lat: 50 }, false], - [{ lon: 50, lat: 190 }, false], - [{ lon: -10, lat: 50 }, false], - [{ lon: 50, lat: -10 }, false], - ])('should validate WGS84 coordinate', (coordinate, expected) => { - expect(validateWGS84Coordinate(coordinate as never)).toBe(expected); - }); -}); From 4689aa9009b81e6d2098dae9f0940d44c6fdb4da Mon Sep 17 00:00:00 2001 From: NatalieShaked Date: Mon, 26 Aug 2024 17:39:25 +0300 Subject: [PATCH 145/262] test: added mock data to tests in order to test regions properly --- config/test.json | 6 ++- devScripts/geotextElasticsearchData.json | 64 +++++++++++++++++++++++ tests/integration/location/mockObjects.ts | 58 ++++++++++++++++++++ 3 files changed, 126 insertions(+), 2 deletions(-) diff --git a/config/test.json b/config/test.json index 6c4f3c2f..55384c41 100644 --- a/config/test.json +++ b/config/test.json @@ -60,10 +60,12 @@ "viewbox": 1.1 }, "sources": { - "OSM": "OSM" + "OSM": "OSM", + "GOOGLE": "GOOGLE" }, "regions": { - "USA": ["New York", "Los Angeles"] + "USA": ["New York", "Los Angeles"], + "FRANCE": ["Paris"] }, "nameTranslationsKeys": ["en", "fr"], "mainLanguageRegex": "[a-zA-Z]" diff --git a/devScripts/geotextElasticsearchData.json b/devScripts/geotextElasticsearchData.json index 99f6f4a3..c91160a3 100644 --- a/devScripts/geotextElasticsearchData.json +++ b/devScripts/geotextElasticsearchData.json @@ -317,6 +317,45 @@ "text_language": "en" } }, + { + "_id": "dc02a3f9-156a-4f61-85bd-fd040cd322a3", + "_index": "geotext_index", + "_source": { + "source": "OSM", + "layer_name": "osm_schools", + "ingestion_timestamp": "2024-07-17 00:00:00", + "timestamp": "11/12/2023 00:00:00", + "source_id": ["{dc02a3f9-156a-4f61-85bd-fd040cd322a3}"], + "sensitivity": "non-sensitive", + "placetype": "education", + "sub_placetype": "school", + "wkt": "POLYGON ((2.346441270696971 48.88088750665477, 2.3462780852304945 48.88018258877358, 2.347503576087604 48.87999951892243, 2.347737155284733 48.88070864783427, 2.346441270696971 48.88088750665477))", + "geo_json": { + "coordinates": [ + [ + [2.346441270696971, 48.88088750665477], + [2.3462780852304945, 48.88018258877358], + [2.347503576087604, 48.87999951892243], + [2.347737155284733, 48.88070864783427], + [2.346441270696971, 48.88088750665477] + ] + ], + "type": "Polygon" + }, + "centroid": { + "type": "Point", + "coordinates": [2.346966023961727, 48.88044772323158] + }, + "geometry_type": "Polygon", + "geometry_hash": "791f6b24002bbc1d50cdea5015244da9 ", + "region": ["FRANCE"], + "sub_region": ["Paris"], + "name": "Wi School Paris 9", + "text": ["Wi School Paris 9"], + "translated_text": ["Ecole Wi Paris 9"], + "text_language": "en" + } + }, { "_id": "5a54ff54-53f6-4ad1-9d2a-cd20f333ee2b", "_index": "hierarchies_index", @@ -368,6 +407,31 @@ "text": "Los Angeles" } }, + { + "_id": "a3f38afc-1ce6-443c-a302-a0103c2bcb7d", + "_index": "hierarchies_index", + "_source": { + "geo_json": { + "coordinates": [ + [ + [2.226678539753607, 49.06838747927134], + [1.9344166918067742, 48.906487548202136], + [2.014124468519668, 48.56855190252173], + [2.6536844864307625, 48.53463335095324], + [2.902296837606201, 48.82159183126478], + [2.6460932696008683, 49.0124047223114], + [2.384196288972504, 49.05097737411208], + [2.226678539753607, 49.06838747927134] + ] + ], + "type": "Polygon" + }, + "hierarchy": "city", + "placetype": "city", + "region": "FRANCE", + "text": "Paris" + } + }, { "_id": "430ddc04-2370-415f-8761-b8c66a142f0a", "_index": "placetypes_index", diff --git a/tests/integration/location/mockObjects.ts b/tests/integration/location/mockObjects.ts index 0c993b4c..39745b7e 100644 --- a/tests/integration/location/mockObjects.ts +++ b/tests/integration/location/mockObjects.ts @@ -305,6 +305,41 @@ export const LA_WHITE_POINT_SCHOOL: MockLocationQueryFeature = { }, }; +export const PARIS_WI_SCHOOL: MockLocationQueryFeature = { + type: 'Feature', + geometry: { + coordinates: [ + [ + [2.346441270696971, 48.88088750665477], + [2.3462780852304945, 48.88018258877358], + [2.347503576087604, 48.87999951892243], + [2.347737155284733, 48.88070864783427], + [2.346441270696971, 48.88088750665477], + ], + ], + type: 'Polygon', + }, + properties: { + source: 'OSM', + layer: 'osm_schools', + source_id: ['dc02a3f9-156a-4f61-85bd-fd040cd322a3'], + name: { + en: ['Wi School Paris 9'], + fr: ['Ecole Wi Paris 9'], + default: ['Wi School Paris 9'], + display: 'Wi School Paris 9', + }, + placetype: 'education', + sub_placetype: 'school', + regions: [ + { + region: 'FRANCE', + sub_regions: ['Paris'], + }, + ], + }, +}; + export const NY_HIERRARCHY: HierarchySearchHit = { geo_json: { coordinates: [ @@ -351,3 +386,26 @@ export const LA_HIERRARCHY: HierarchySearchHit = { text: 'Los Angeles', weight: 1.1, }; + +export const PARIS_HIERRARCHY: HierarchySearchHit = { + geo_json: { + coordinates: [ + [ + [2.226678539753607, 49.06838747927134], + [1.9344166918067742, 48.906487548202136], + [2.014124468519668, 48.56855190252173], + [2.6536844864307625, 48.53463335095324], + [2.902296837606201, 48.82159183126478], + [2.6460932696008683, 49.0124047223114], + [2.384196288972504, 49.05097737411208], + [2.226678539753607, 49.06838747927134], + ], + ], + type: 'Polygon', + }, + hierarchy: 'city', + placetype: 'city', + region: 'FRANCE', + text: 'Paris', + weight: 1.1, +}; \ No newline at end of file From d613ccd1acaab4d64728b2e3fc98ce8b4dd6750d Mon Sep 17 00:00:00 2001 From: Niv Greenstein Date: Mon, 26 Aug 2024 17:39:29 +0300 Subject: [PATCH 146/262] fix(location): changed location feature response --- src/common/utils.ts | 4 ++ src/control/interfaces.ts | 3 +- src/control/utils.ts | 10 ++-- src/latLon/DAL/latLonRepository.ts | 2 - src/latLon/controllers/latLonController.ts | 1 + src/latLon/utlis/index.ts | 1 + tests/integration/control/utils.ts | 2 +- tests/integration/location/location.spec.ts | 57 ++++++++++++++++++--- tests/integration/location/mockObjects.ts | 42 ++++++--------- tests/integration/location/utils.ts | 21 +++++--- 10 files changed, 95 insertions(+), 48 deletions(-) diff --git a/src/common/utils.ts b/src/common/utils.ts index 6f75dec6..50ad1458 100644 --- a/src/common/utils.ts +++ b/src/common/utils.ts @@ -18,6 +18,10 @@ export type ConvertCamelToSnakeCase = { [K in keyof T as CamelToSnakeCase]: T[K]; }; +export type RemoveUnderscore = { + [K in keyof T as K extends `_${infer Rest}` ? Rest : K]: T[K]; +}; + export const validateWGS84Coordinate = (coordinate: { lon: number; lat: number }): boolean => { // eslint-disable-next-line @typescript-eslint/no-magic-numbers const [min, max] = [0, 180]; diff --git a/src/control/interfaces.ts b/src/control/interfaces.ts index 688cd1be..930f0646 100644 --- a/src/control/interfaces.ts +++ b/src/control/interfaces.ts @@ -1,6 +1,7 @@ import { Feature } from 'geojson'; import { estypes } from '@elastic/elasticsearch'; import { CommonRequestParameters, FeatureCollection } from '../common/interfaces'; +import { RemoveUnderscore } from '../common/utils'; export interface ControlResponse extends FeatureCollection { geocoding?: { @@ -13,5 +14,5 @@ export interface ControlResponse extends FeatureColl /* eslint-enable @typescript-eslint/naming-convention */ }; }; - features: (T & Pick, '_score'>)[]; + features: (T & RemoveUnderscore, '_score'>>)[]; } diff --git a/src/control/utils.ts b/src/control/utils.ts index 31aea8f2..0a730eb8 100644 --- a/src/control/utils.ts +++ b/src/control/utils.ts @@ -8,7 +8,7 @@ import { ElasticDbClientsConfig } from '../common/elastic/interfaces'; import { Item } from '../control/item/models/item'; import { Tile } from '../control/tile/models/tile'; import { Route } from '../control/route/models/route'; -import { ConvertSnakeToCamelCase } from '../common/utils'; +import { ConvertSnakeToCamelCase, RemoveUnderscore } from '../common/utils'; import { BBOX_LENGTH } from '../location/interfaces'; import { CONTROL_FIELDS, ELASTIC_KEYWORDS } from './constants'; import { ControlResponse } from './interfaces'; @@ -41,10 +41,10 @@ export const formatResponse = ( }, }, features: [ - ...(elasticResponse.hits.hits.map((item) => ({ - ...item._source, - _score: item._score, - })) as (T & Pick, '_score'>)[]), + ...(elasticResponse.hits.hits.map(({ _source: source, _score: score }) => ({ + ...source, + score, + })) as (T & RemoveUnderscore, '_score'>>)[]), ], }); diff --git a/src/latLon/DAL/latLonRepository.ts b/src/latLon/DAL/latLonRepository.ts index a67e3c0b..a7f9dbfc 100644 --- a/src/latLon/DAL/latLonRepository.ts +++ b/src/latLon/DAL/latLonRepository.ts @@ -14,8 +14,6 @@ const createLatLonRepository = (dataSource: DataSource, logger: Logger) => { const result = await this.find(); return result; } catch (error: unknown) { - //TODO: check if error is due to postgres client not being available - console.log(error); logger.error('Error in getAll function in latLonRepository', error); throw new ServiceUnavailableError('Postgres client is not available. As for, the getAll request cannot be executed.'); } diff --git a/src/latLon/controllers/latLonController.ts b/src/latLon/controllers/latLonController.ts index 184e322c..a62d7716 100644 --- a/src/latLon/controllers/latLonController.ts +++ b/src/latLon/controllers/latLonController.ts @@ -10,6 +10,7 @@ import { LatLonManager } from '../models/latLonManager'; import { Tile } from '../../control/tile/models/tile'; import { WGS84Coordinate } from '../../common/interfaces'; import { ControlResponse } from '../../control/interfaces'; +/* istanbul ignore file */ type GetLatLonToTileHandler = RequestHandler; diff --git a/src/latLon/utlis/index.ts b/src/latLon/utlis/index.ts index 6c806eda..128b4881 100644 --- a/src/latLon/utlis/index.ts +++ b/src/latLon/utlis/index.ts @@ -1,3 +1,4 @@ +/* istanbul ignore file */ import { Polygon } from 'geojson'; import { BadRequestError } from '../../common/errors'; import { convertUTMToWgs84 } from '../../common/utils'; diff --git a/tests/integration/control/utils.ts b/tests/integration/control/utils.ts index 5f03ecd0..0c19f48c 100644 --- a/tests/integration/control/utils.ts +++ b/tests/integration/control/utils.ts @@ -10,7 +10,7 @@ import { Tile } from '../../../src/control/tile/models/tile'; const expectedObjectWithScore = (obj: T, expect: jest.Expect): ControlResponse['features'][number] => ({ ...obj, - _score: expect.any(Number) as number, + score: expect.any(Number) as number, }); const expectedGeocodingElasticResponseMetrics = ( diff --git a/tests/integration/location/location.spec.ts b/tests/integration/location/location.spec.ts index 87bfb977..fa5dce50 100644 --- a/tests/integration/location/location.spec.ts +++ b/tests/integration/location/location.spec.ts @@ -65,11 +65,26 @@ describe('/search/control/tiles', function () { expectedResponse( { ...requestParams, + }, + { place_types: ['transportation'], sub_place_types: ['airport'], hierarchies: [], }, - [NY_JFK_AIRPORT, NY_POLICE_AIRPORT, LA_AIRPORT], + [ + { + ...NY_JFK_AIRPORT, + properties: { + ...NY_JFK_AIRPORT.properties, + name: { + ...NY_JFK_AIRPORT.properties.name, + display: expect.stringContaining('JFK') as string, + }, + }, + }, + NY_POLICE_AIRPORT, + LA_AIRPORT, + ], expect ) ); @@ -79,19 +94,45 @@ describe('/search/control/tiles', function () { test.each< Pick & { - hierarchies: QueryResult['geocoding']['query']['hierarchies']; + hierarchies: QueryResult['geocoding']['response']['hierarchies']; returnedFeatures: MockLocationQueryFeature[]; } >([ { query: 'new york', hierarchies: hierarchiesWithAnyWieght([NY_HIERRARCHY], expect), - returnedFeatures: [NY_JFK_AIRPORT, NY_POLICE_AIRPORT, LA_AIRPORT], + returnedFeatures: [ + { + ...NY_JFK_AIRPORT, + properties: { + ...NY_JFK_AIRPORT.properties, + name: { + ...NY_JFK_AIRPORT.properties.name, + display: expect.stringContaining('JFK') as string, + }, + }, + }, + NY_POLICE_AIRPORT, + LA_AIRPORT, + ], }, { query: 'los angeles', hierarchies: hierarchiesWithAnyWieght([LA_HIERRARCHY], expect), - returnedFeatures: [LA_AIRPORT, NY_JFK_AIRPORT, NY_POLICE_AIRPORT], + returnedFeatures: [ + LA_AIRPORT, + { + ...NY_JFK_AIRPORT, + properties: { + ...NY_JFK_AIRPORT.properties, + name: { + ...NY_JFK_AIRPORT.properties.name, + display: expect.stringContaining('JFK') as string, + }, + }, + }, + NY_POLICE_AIRPORT, + ], }, ])('it should test airports response with hierrarchy in %s', async ({ query, hierarchies, returnedFeatures }) => { const requestParams: GetGeotextSearchParams = { query: `airport, ${query}`, limit: 5, disable_fuzziness: false }; @@ -114,6 +155,8 @@ describe('/search/control/tiles', function () { expectedResponse( { ...requestParams, + }, + { place_types: ['transportation'], sub_place_types: ['airport'], hierarchies, @@ -128,8 +171,8 @@ describe('/search/control/tiles', function () { test.each< Pick & { - place_types: QueryResult['geocoding']['query']['place_types']; - sub_place_types: QueryResult['geocoding']['query']['sub_place_types']; + place_types: QueryResult['geocoding']['response']['place_types']; + sub_place_types: QueryResult['geocoding']['response']['sub_place_types']; returnedFeatures: MockLocationQueryFeature[]; } >([ @@ -166,6 +209,8 @@ describe('/search/control/tiles', function () { expectedResponse( { ...requestParams, + }, + { name: query, place_types, sub_place_types, diff --git a/tests/integration/location/mockObjects.ts b/tests/integration/location/mockObjects.ts index 0c993b4c..ea6060ec 100644 --- a/tests/integration/location/mockObjects.ts +++ b/tests/integration/location/mockObjects.ts @@ -53,21 +53,19 @@ export const NY_JFK_AIRPORT: MockLocationQueryFeature = { ], }, properties: { - source: 'OSM', - layer: 'osm_airports', - source_id: ['03ed6d97-fc81-4340-b68a-11993554eef1'], + matches: { source: 'OSM', layer: 'osm_airports', source_id: ['03ed6d97-fc81-4340-b68a-11993554eef1'] }, name: { en: ['JFK Airport'], fr: ['Aeropuerto JFK'], default: ['JFK'], - display: 'JFK', + display: 'JFK Airport', }, placetype: 'transportation', sub_placetype: 'airport', regions: [ { region: 'USA', - sub_regions: ['New York'], + sub_region_names: ['New York'], }, ], }, @@ -100,9 +98,7 @@ export const NY_POLICE_AIRPORT: MockLocationQueryFeature = { type: 'Polygon', }, properties: { - source: 'OSM', - layer: 'osm_airports', - source_id: ['009c6b65-3dcb-4c4f-9f02-d766ebb5d808'], + matches: { source: 'OSM', layer: 'osm_airports', source_id: ['009c6b65-3dcb-4c4f-9f02-d766ebb5d808'] }, name: { en: ['Nassau County Police Airport'], fr: ['Aeropuerto de la Policía del Condado de Nassau'], @@ -114,7 +110,7 @@ export const NY_POLICE_AIRPORT: MockLocationQueryFeature = { regions: [ { region: 'USA', - sub_regions: ['New York'], + sub_region_names: ['New York'], }, ], }, @@ -149,9 +145,7 @@ export const LA_AIRPORT: MockLocationQueryFeature = { type: 'Polygon', }, properties: { - source: 'OSM', - layer: 'osm_airports', - source_id: ['a4f373ab-b824-41e2-b160-e7729c73bea6'], + matches: { source: 'OSM', layer: 'osm_airports', source_id: ['a4f373ab-b824-41e2-b160-e7729c73bea6'] }, name: { en: ['Los Angeles International Airport'], fr: ['Aeropuerto Internacional de Los Ángeles'], @@ -163,7 +157,7 @@ export const LA_AIRPORT: MockLocationQueryFeature = { regions: [ { region: 'USA', - sub_regions: ['Los Angeles'], + sub_region_names: ['Los Angeles'], }, ], }, @@ -199,9 +193,7 @@ export const OSM_LA_PORT: MockLocationQueryFeature = { ], }, properties: { - source: 'OSM', - layer: 'osm_ports', - source_id: ['0f36d985-cfbd-4aed-b0cb-ee56600c77f4'], + matches: { source: 'OSM', layer: 'osm_ports', source_id: ['0f36d985-cfbd-4aed-b0cb-ee56600c77f4'] }, name: { en: ['Port of Los Angeles'], fr: ['Puerto de Los Ángeles'], @@ -213,7 +205,7 @@ export const OSM_LA_PORT: MockLocationQueryFeature = { regions: [ { region: 'USA', - sub_regions: ['Los Angeles'], + sub_region_names: ['Los Angeles'], }, ], }, @@ -248,9 +240,7 @@ export const GOOGLE_LA_PORT: MockLocationQueryFeature = { ], }, properties: { - source: 'GOOGLE', - layer: 'google_ports', - source_id: ['1bb11f54-939e-457b-bf68-a3920ccf629c'], + matches: { source: 'GOOGLE', layer: 'google_ports', source_id: ['1bb11f54-939e-457b-bf68-a3920ccf629c'] }, name: { en: ['Port of Los Angeles'], fr: ['Puerto de Los Ángeles'], @@ -262,7 +252,7 @@ export const GOOGLE_LA_PORT: MockLocationQueryFeature = { regions: [ { region: 'USA', - sub_regions: ['Los Angeles'], + sub_region_names: ['Los Angeles'], }, ], }, @@ -285,9 +275,11 @@ export const LA_WHITE_POINT_SCHOOL: MockLocationQueryFeature = { type: 'Polygon', }, properties: { - source: 'OSM', - layer: 'osm_schools', - source_id: ['1a5b981b-bb0e-44dd-b9e2-424b92f2de49'], + matches: { + source: 'OSM', + layer: 'osm_schools', + source_id: ['1a5b981b-bb0e-44dd-b9e2-424b92f2de49'], + }, name: { en: ['White Point Elementary School'], fr: ['Escuela Primaria White Point'], @@ -299,7 +291,7 @@ export const LA_WHITE_POINT_SCHOOL: MockLocationQueryFeature = { regions: [ { region: 'USA', - sub_regions: ['Los Angeles'], + sub_region_names: ['Los Angeles'], }, ], }, diff --git a/tests/integration/location/utils.ts b/tests/integration/location/utils.ts index 2af5c3b8..1a04cc08 100644 --- a/tests/integration/location/utils.ts +++ b/tests/integration/location/utils.ts @@ -2,23 +2,28 @@ import { QueryResult } from '../../../src/location/interfaces'; import { MockLocationQueryFeature } from './mockObjects'; -const expectedObjectWithScoreAndRank = (obj: MockLocationQueryFeature, expect: jest.Expect): QueryResult['features'][number] => ({ +const expectedObjectWithScore = (obj: MockLocationQueryFeature, expect: jest.Expect): QueryResult['features'][number] => ({ ...obj, properties: { ...obj.properties, - rank: expect.any(Number) as number, }, - _score: expect.any(Number) as number, + score: expect.any(Number) as number, }); -const expectedGeocodingElasticResponseMetrics = (resultsCount: number, expect: jest.Expect): NonNullable['response'] => ({ +const expectedGeocodingElasticResponseMetrics = ( + responseParams: Partial, + resultsCount: number, + expect: jest.Expect +): NonNullable['response'] => ({ results_count: resultsCount, max_score: expect.any(Number) as number, match_latency_ms: expect.any(Number) as number, + ...responseParams, }); export const expectedResponse = ( requestParams: QueryResult['geocoding']['query'], + responseParams: Partial, arr: MockLocationQueryFeature[], expect: jest.Expect ): QueryResult => ({ @@ -26,12 +31,12 @@ export const expectedResponse = ( geocoding: { version: process.env.npm_package_version, query: requestParams, - response: expectedGeocodingElasticResponseMetrics(arr.length, expect), + response: expectedGeocodingElasticResponseMetrics(responseParams, arr.length, expect), }, - features: arr.map((item) => expectedObjectWithScoreAndRank(item, expect)), + features: arr.map((item) => expectedObjectWithScore(item, expect)), }); export const hierarchiesWithAnyWieght = ( - hierarchies: QueryResult['geocoding']['query']['hierarchies'], + hierarchies: QueryResult['geocoding']['response']['hierarchies'], expect: jest.Expect -): QueryResult['geocoding']['query']['hierarchies'] => hierarchies.map((hierarchy) => ({ ...hierarchy, weight: expect.any(Number) as number })); +): QueryResult['geocoding']['response']['hierarchies'] => hierarchies?.map((hierarchy) => ({ ...hierarchy, weight: expect.any(Number) as number })); From 9c51555f6b0d0f37dae5d7c7f6ff335d8e8aaf4f Mon Sep 17 00:00:00 2001 From: NatalieShaked Date: Mon, 26 Aug 2024 17:39:54 +0300 Subject: [PATCH 147/262] test: added coverage --- tests/integration/location/location.spec.ts | 67 +++++++++++++++++++++ 1 file changed, 67 insertions(+) diff --git a/tests/integration/location/location.spec.ts b/tests/integration/location/location.spec.ts index 5615b80b..1fb47785 100644 --- a/tests/integration/location/location.spec.ts +++ b/tests/integration/location/location.spec.ts @@ -22,6 +22,8 @@ import { NY_HIERRARCHY, LA_HIERRARCHY, MockLocationQueryFeature, + PARIS_WI_SCHOOL, + PARIS_HIERRARCHY, } from './mockObjects'; import { expectedResponse, hierarchiesWithAnyWieght } from './utils'; @@ -270,7 +272,72 @@ describe('/search/control/tiles', function () { expect(response.body).toEqual(expect.arrayContaining(['OSM', 'GOOGLE'])); // expect(response).toSatisfyApiSpec(); }); + + it('should return 200 status code and ports from the corresponding source', async function () { + const requestParams: GetGeotextSearchParams = { query: 'port', source: ['google'], limit: 5, disable_fuzziness: false }; + const tokenTypesUrlScope = nock(config.get('application').services.tokenTypesUrl) + .post('', { tokens: requestParams.query.split(' ') }) + .reply(httpStatusCodes.OK, [ + { + tokens: ['port'], + prediction: ['essence'], + }, + ]); + + const response = await requestSender.getQuery(requestParams); + + expect(response.status).toBe(httpStatusCodes.OK); + // expect(response).toSatisfyApiSpec(); + + expect(response.body).toMatchObject( + expectedResponse( + { + ...requestParams, + place_types: ['transportation'], + sub_place_types: ['port'], + hierarchies: [], + }, + [GOOGLE_LA_PORT], + expect + ) + ); + + tokenTypesUrlScope.done(); + }); + + it('should return 200 status code and schools in specified region', async function () { + const requestParams: GetGeotextSearchParams = { query: 'school', region: ['france'], limit: 5, disable_fuzziness: false }; + const tokenTypesUrlScope = nock(config.get('application').services.tokenTypesUrl) + .post('', { tokens: requestParams.query.split(' ') }) + .reply(httpStatusCodes.OK, [ + { + tokens: ['school'], + prediction: ['essence'], + }, + ]); + + const response = await requestSender.getQuery(requestParams); + + expect(response.status).toBe(httpStatusCodes.OK); + // expect(response).toSatisfyApiSpec(); + + expect(response.body).toMatchObject( + expectedResponse( + { + ...requestParams, + place_types: ['education'], + sub_place_types: ['school'], + hierarchies: [], + }, + [PARIS_WI_SCHOOL], + expect + ) + ); + + tokenTypesUrlScope.done(); + }); }); + describe('Bad Path', function () { // All requests with status code 4XX-5XX test.each>([ From 6189f8d2c77ef2ca88ee2cfc08b729d10979799a Mon Sep 17 00:00:00 2001 From: Niv Greenstein Date: Tue, 27 Aug 2024 10:20:39 +0300 Subject: [PATCH 148/262] feat: changed location response to match geocodingGenereicResponse schema --- src/control/interfaces.ts | 12 +++++++++++- src/location/interfaces.ts | 26 ++++++++++++++------------ src/location/parsing.ts | 18 +++++++----------- src/location/utils.ts | 36 ++++++++++++++++++++++-------------- 4 files changed, 54 insertions(+), 38 deletions(-) diff --git a/src/control/interfaces.ts b/src/control/interfaces.ts index 930f0646..a7de812a 100644 --- a/src/control/interfaces.ts +++ b/src/control/interfaces.ts @@ -14,5 +14,15 @@ export interface ControlResponse extends FeatureColl /* eslint-enable @typescript-eslint/naming-convention */ }; }; - features: (T & RemoveUnderscore, '_score'>>)[]; + features: (T & { + matches: { + source: string; + layer: string; + }[]; + names: { + [key: string]: string; + display: string; + default: string; + }; + } & RemoveUnderscore, '_score'>>)[]; } diff --git a/src/location/interfaces.ts b/src/location/interfaces.ts index 379c1f26..7bec8721 100644 --- a/src/location/interfaces.ts +++ b/src/location/interfaces.ts @@ -1,7 +1,7 @@ import type { GeoJSON } from 'geojson'; import { estypes } from '@elastic/elasticsearch'; import { CommonRequestParameters } from '../common/interfaces'; -import { ConvertCamelToSnakeCase, ConvertSnakeToCamelCase } from '../common/utils'; +import { ConvertCamelToSnakeCase, ConvertSnakeToCamelCase, RemoveUnderscore } from '../common/utils'; import { HierarchySearchHit } from './models/elasticsearchHits'; export interface PlaceType { @@ -33,28 +33,30 @@ export interface QueryResult { type: string; geocoding: { version?: string; - query: ConvertCamelToSnakeCase; - response: { max_score: number; results_count: number; match_latency_ms: number }; + query: ConvertCamelToSnakeCase; + response: { max_score: number; results_count: number; match_latency_ms: number } & Partial< + ConvertCamelToSnakeCase> + >; }; features: ({ type: string; geometry?: GeoJSON; properties: { - rank: number; - source?: string; - source_id?: string[]; - layer?: string; + matches: { + source?: string; + source_id?: string[]; + layer?: string; + }; name: { [key: string]: string | string[] | undefined; + display: string; + default: string[]; }; - highlight?: Record; placetype?: string; sub_placetype?: string; - region?: string[]; - sub_region?: string[]; - regions?: { region: string; sub_regions: string[] }[]; + regions?: { region: string; sub_region_names: string[] }[]; }; - } & Pick)[]; + } & RemoveUnderscore>)[]; } /* eslint-enable @typescript-eslint/naming-convention */ diff --git a/src/location/parsing.ts b/src/location/parsing.ts index 08e83acc..99657aa9 100644 --- a/src/location/parsing.ts +++ b/src/location/parsing.ts @@ -1,21 +1,17 @@ //This file might be deprecated in the future /* istanbul ignore file */ +import { SearchHit } from '@elastic/elasticsearch/lib/api/types'; import { XMLParser } from 'fast-xml-parser'; -type Highlight = { - em: string | string[]; - '#text': string; -}; - const HIERARCHY_OF_INTEREST = 3; const HIGHLIGHT_XML_REGEX = /|<\/em>/gi; -const untagHighlight = (highlight: string) => highlight.replace(HIGHLIGHT_XML_REGEX, ''); +const untagHighlight = (highlight: string): string => highlight.replace(HIGHLIGHT_XML_REGEX, ''); const calculateHighlightQuality = (highlight: string, queryWordCount: number): number => { const parser = new XMLParser({ numberParseOptions: { skipLike: /[0-9]+/, hex: false, leadingZeros: false } }); - const parsed = parser.parse(highlight) as Highlight; + const parsed = parser.parse(highlight) as NonNullable; if (!(parsed.em instanceof Array)) { parsed.em = [parsed.em]; @@ -37,16 +33,16 @@ const compareQualityThenLength = ( } ): number => a.quality - b.quality || a.highlight.length - b.highlight.length; -export const generateDisplayName = (highlights: string[], queryWordCount: number, name?: string): string | undefined => { +export const generateDisplayName = ({ text: highlights }: NonNullable, queryWordCount: number, name?: string): string => { const scored = highlights.map((highlight) => ({ highlight, quality: calculateHighlightQuality(highlight, queryWordCount), })); - const filtered = scored.filter(name ? ({ highlight }) => highlight.includes(name) : ({ quality }) => quality < 1); + const filtered = scored.filter(name !== undefined ? ({ highlight }): boolean => highlight.includes(name) : ({ quality }): boolean => quality < 1); - const chosen = (filtered.length ? filtered : scored).sort(compareQualityThenLength).pop()?.highlight; + const chosen = (filtered.length ? filtered : scored).sort(compareQualityThenLength).pop()!.highlight; - return chosen && untagHighlight(chosen); + return untagHighlight(chosen); }; export const getHierarchyOfInterest = (hierarchy: string): string => hierarchy.split('/')[HIERARCHY_OF_INTEREST]; diff --git a/src/location/utils.ts b/src/location/utils.ts index 6f0191b9..87f6f474 100644 --- a/src/location/utils.ts +++ b/src/location/utils.ts @@ -7,8 +7,9 @@ import { InternalServerError } from '../common/errors'; import { GeoContext, IApplication } from '../common/interfaces'; import { ConvertCamelToSnakeCase, convertUTMToWgs84 } from '../common/utils'; import { convertCamelToSnakeCase } from '../control/utils'; -import { BBOX_LENGTH, POINT_LENGTH, QueryResult, TextSearchParams } from './interfaces'; +import { BBOX_LENGTH, GetGeotextSearchParams, POINT_LENGTH, QueryResult, TextSearchParams } from './interfaces'; import { TextSearchHit } from './models/elasticsearchHits'; +import { generateDisplayName } from './parsing'; const FIND_QUOTES = /["']/g; @@ -111,40 +112,47 @@ export const convertResult = ( geocoding: { version: process.env.npm_package_version, query: { - ...(convertCamelToSnakeCase(params as unknown as Record) as ConvertCamelToSnakeCase), + query: params.query, + region: params.region, + source: params.source, + geo_context: params.geoContext, + geo_context_mode: params.geoContextMode, + disable_fuzziness: params.disableFuzziness, + limit: params.limit, }, response: { - /* eslint-disable @typescript-eslint/naming-convention */ results_count: results.hits.hits.length, max_score: results.hits.max_score ?? 0, match_latency_ms: results.took, - /* eslint-enable @typescript-eslint/naming-convention */ + name: params.name ?? undefined, + place_types: params.placeTypes, + sub_place_types: params.subPlaceTypes, + hierarchies: params.hierarchies, }, }, - features: results.hits.hits.map(({ _source: feature, _score }, index): QueryResult['features'][number] => { + features: results.hits.hits.map(({ _source: feature, _score: score, highlight }, index): QueryResult['features'][number] => { const allNames = [feature!.text, feature!.translated_text || []]; return { type: 'Feature', geometry: feature?.geo_json, - _score, + score, properties: { - rank: index + 1, - source: (sources ?? {})[feature?.source ?? ''] ?? feature?.source, - layer: feature?.layer_name, - source_id: feature?.source_id.map((id) => id.replace(/(^\{)|(\}$)/g, '')), // TODO: check if to remove this + matches: { + layer: feature?.layer_name, + source: (sources ?? {})[feature?.source ?? ''] ?? feature?.source, + source_id: feature?.source_id.map((id) => id.replace(/(^\{)|(\}$)/g, '')), // TODO: check if to remove this + }, name: { [nameKeys[0]]: new RegExp(mainLanguageRegex).test(feature!.text[0]) ? allNames.shift() : allNames.pop(), [nameKeys[1]]: allNames.pop(), ['default']: [feature!.name], - // display: highlight ? generateDisplayName(highlight.text, params.query!.split(' ').length, params.name) : feature!.name, - display: feature!.name, + display: highlight ? generateDisplayName(highlight, params.query.split(' ').length, params.name) : feature!.name, }, - // highlight, placetype: feature?.placetype, // TODO: check if to remove this sub_placetype: feature?.sub_placetype, regions: feature?.region.map((region) => ({ region: region, - sub_regions: feature.sub_region.filter((sub_region) => (regionCollection ?? {})[region ?? '']?.includes(sub_region)), + sub_region_names: feature.sub_region.filter((sub_region) => (regionCollection ?? {})[region ?? '']?.includes(sub_region)), })), }, }; From de51484a247769e1a2737c1215672c488a6192aa Mon Sep 17 00:00:00 2001 From: NatalieShaked Date: Tue, 27 Aug 2024 11:45:51 +0300 Subject: [PATCH 149/262] delete: removed duplicated test --- tests/integration/control/tile/tile.spec.ts | 25 --------------------- 1 file changed, 25 deletions(-) diff --git a/tests/integration/control/tile/tile.spec.ts b/tests/integration/control/tile/tile.spec.ts index 79cbb41f..44685fcc 100644 --- a/tests/integration/control/tile/tile.spec.ts +++ b/tests/integration/control/tile/tile.spec.ts @@ -381,31 +381,6 @@ describe('/search/control/tiles', function () { }); }); - test.each>([ - { - geo_context: { x: 300850, y: 4642203, zone: 33, radius: 100 }, - }, - { - geo_context: { lon: 12.598899687444742, lat: 41.90667824634701, radius: 10 }, - }, - { - geo_context: { bbox: [12.554407132912445, 41.84962590648513, 12.652837919839953, 41.94545380230761] }, - }, - { - geo_context_mode: GeoContextMode.BIAS, - }, - { - geo_context_mode: GeoContextMode.FILTER, - }, - ])('should return 400 and message that geo_context and geo_context_mode must be both defined or both undefined', async function (requestParams) { - const response = await requestSender.getTiles({ tile: 'RIT', limit: 5, disable_fuzziness: false, ...requestParams }); - - expect(response.status).toBe(httpStatusCodes.BAD_REQUEST); - expect(response.body).toMatchObject({ - message: '/control/utils/geoContextQuery: geo_context and geo_context_mode must be both defined or both undefined', - }); - }); - test.each>([ { disable_fuzziness: '' as unknown as boolean, From 1dabf220c5b3820f9485fd79f1c39ed6d6d7a05d Mon Sep 17 00:00:00 2001 From: NatalieShaked Date: Tue, 27 Aug 2024 11:46:13 +0300 Subject: [PATCH 150/262] feat: added coverage --- tests/integration/location/location.spec.ts | 16 ++++++++-------- 1 file changed, 8 insertions(+), 8 deletions(-) diff --git a/tests/integration/location/location.spec.ts b/tests/integration/location/location.spec.ts index 1fb47785..e61cdc0b 100644 --- a/tests/integration/location/location.spec.ts +++ b/tests/integration/location/location.spec.ts @@ -48,7 +48,7 @@ describe('/search/control/tiles', function () { describe('Happy Path', function () { it('should return 200 status code and airports', async function () { - const requestParams: GetGeotextSearchParams = { query: 'airport', limit: 5, disable_fuzziness: false }; + const requestParams: GetGeotextSearchParams = { query: 'airport', limit: 5, disable_fuzziness: true }; const tokenTypesUrlScope = nock(config.get('application').services.tokenTypesUrl) .post('', { tokens: requestParams.query.split(' ') }) .reply(httpStatusCodes.OK, [ @@ -85,7 +85,7 @@ describe('/search/control/tiles', function () { geo_context: { bbox: [-75.81665, 39.597223, -72.575684, 41.352072] }, geo_context_mode: GeoContextMode.FILTER, limit: 5, - disable_fuzziness: false, + disable_fuzziness: true, }; const tokenTypesUrlScope = nock(config.get('application').services.tokenTypesUrl) .post('', { tokens: requestParams.query.split(' ') }) @@ -123,7 +123,7 @@ describe('/search/control/tiles', function () { geo_context: { bbox: [-75.81665, 39.597223, -72.575684, 41.352072] }, geo_context_mode: GeoContextMode.BIAS, limit: 5, - disable_fuzziness: false, + disable_fuzziness: true, }; const tokenTypesUrlScope = nock(config.get('application').services.tokenTypesUrl) .post('', { tokens: requestParams.query.split(' ') }) @@ -172,7 +172,7 @@ describe('/search/control/tiles', function () { returnedFeatures: [LA_AIRPORT, NY_JFK_AIRPORT, NY_POLICE_AIRPORT], }, ])('it should test airports response with hierrarchy in %s', async ({ query, hierarchies, returnedFeatures }) => { - const requestParams: GetGeotextSearchParams = { query: `airport, ${query}`, limit: 5, disable_fuzziness: false }; + const requestParams: GetGeotextSearchParams = { query: `airport, ${query}`, limit: 5, disable_fuzziness: true }; const tokenTypesUrlScope = nock(config.get('application').services.tokenTypesUrl) .post('', { tokens: ['airport'] }) @@ -274,7 +274,7 @@ describe('/search/control/tiles', function () { }); it('should return 200 status code and ports from the corresponding source', async function () { - const requestParams: GetGeotextSearchParams = { query: 'port', source: ['google'], limit: 5, disable_fuzziness: false }; + const requestParams: GetGeotextSearchParams = { query: 'port', source: ['google'], limit: 5, disable_fuzziness: true }; const tokenTypesUrlScope = nock(config.get('application').services.tokenTypesUrl) .post('', { tokens: requestParams.query.split(' ') }) .reply(httpStatusCodes.OK, [ @@ -306,7 +306,7 @@ describe('/search/control/tiles', function () { }); it('should return 200 status code and schools in specified region', async function () { - const requestParams: GetGeotextSearchParams = { query: 'school', region: ['france'], limit: 5, disable_fuzziness: false }; + const requestParams: GetGeotextSearchParams = { query: 'school', region: ['france'], limit: 5, disable_fuzziness: true }; const tokenTypesUrlScope = nock(config.get('application').services.tokenTypesUrl) .post('', { tokens: requestParams.query.split(' ') }) .reply(httpStatusCodes.OK, [ @@ -357,7 +357,7 @@ describe('/search/control/tiles', function () { geo_context_mode: GeoContextMode.FILTER, }, ])('should return 400 and message that geo_context and geo_context_mode must be both defined or both undefined', async function (requestParams) { - const badRequestParams: GetGeotextSearchParams = { query: 'airport', limit: 5, disable_fuzziness: false, ...requestParams }; + const badRequestParams: GetGeotextSearchParams = { query: 'airport', limit: 5, disable_fuzziness: true, ...requestParams }; const tokenTypesUrlScope = nock(config.get('application').services.tokenTypesUrl) .post('', { tokens: badRequestParams.query.split(' ') }) .reply(httpStatusCodes.OK, [ @@ -381,7 +381,7 @@ describe('/search/control/tiles', function () { // All requests with status code 4XX-5XX it('should return 500 status code when the NLP Analyzer service is down due to network error', async function () { const errorMessage = 'NLP Analyzer service is down'; - const requestParams: GetGeotextSearchParams = { query: 'airport', limit: 5, disable_fuzziness: false }; + const requestParams: GetGeotextSearchParams = { query: 'airport', limit: 5, disable_fuzziness: true }; // Intercept the request and simulate a network error const nockScope = nock(config.get('application').services.tokenTypesUrl) From 7749d3f3f04b386a9624f8463171fbb04f5226dc Mon Sep 17 00:00:00 2001 From: NatalieShaked Date: Tue, 27 Aug 2024 14:46:25 +0300 Subject: [PATCH 151/262] feat: added redis configuration --- config/custom-environment-variables.json | 17 +++ config/default.json | 6 + config/test.json | 6 + package-lock.json | 138 +++++++++++++++++++++ package.json | 1 + src/common/constants.ts | 4 + src/common/interfaces.ts | 4 + src/common/redis/domainFieldsRepository.ts | 5 + src/common/redis/index.ts | 33 +++++ src/common/redis/keys.ts | 5 + src/common/redis/redisManager.ts | 31 +++++ src/containerConfig.ts | 52 +++++++- 12 files changed, 296 insertions(+), 6 deletions(-) create mode 100644 src/common/redis/domainFieldsRepository.ts create mode 100644 src/common/redis/index.ts create mode 100644 src/common/redis/keys.ts create mode 100644 src/common/redis/redisManager.ts diff --git a/config/custom-environment-variables.json b/config/custom-environment-variables.json index ed72d6e5..c6ef51dc 100644 --- a/config/custom-environment-variables.json +++ b/config/custom-environment-variables.json @@ -86,6 +86,23 @@ }, "database": "POSTGRES_DB_NAME", "schema": "POSTGRES_DB_SCHEMA" + }, + "redis": { + "host": "REDIS_HOST", + "port": { + "__name": "REDIS_PORT", + "__format": "number" + }, + "db": { + "__name": "REDIS_INDEX", + "__format": "number" + }, + "username": "REDIS_USERNAME", + "password": "REDIS_PASSWORD", + "maxRetriesPerRequest": { + "__name": "REDIS_MAX_RETRIES_PER_REQUEST", + "__format": "number" + } } }, "application": { diff --git a/config/default.json b/config/default.json index cf9186b0..4e0dd4ad 100644 --- a/config/default.json +++ b/config/default.json @@ -71,6 +71,12 @@ }, "database": "postgres", "schema": "geocoder" + }, + "redis": { + "host": "localhost", + "port": 6379, + "db": 0, + "maxRetriesPerRequest": 1 } }, "application": { diff --git a/config/test.json b/config/test.json index 55384c41..f267cd35 100644 --- a/config/test.json +++ b/config/test.json @@ -45,6 +45,12 @@ }, "database": "postgres", "schema": "geocoder" + }, + "redis": { + "host": "localhost", + "port": 6379, + "db": 0, + "maxRetriesPerRequest": 1 } }, "application": { diff --git a/package-lock.json b/package-lock.json index 0c1d7e64..527fc462 100644 --- a/package-lock.json +++ b/package-lock.json @@ -30,6 +30,7 @@ "fast-xml-parser": "^4.4.0", "geojson-validation": "^1.0.2", "http-status-codes": "^2.2.0", + "ioredis": "^5.4.1", "mgrs": "^2.0.0", "node-cron": "^3.0.3", "node-fetch-commonjs": "^3.3.2", @@ -2953,6 +2954,11 @@ "node": ">=6.9.0" } }, + "node_modules/@ioredis/commands": { + "version": "1.2.0", + "resolved": "https://registry.npmjs.org/@ioredis/commands/-/commands-1.2.0.tgz", + "integrity": "sha512-Sx1pU8EM64o2BrqNpEO1CNLtKQwyhuXuqyfH7oGKCk+1a33d2r5saW8zNwm3j6BTExtjrv2BxTgzzkMwts6vGg==" + }, "node_modules/@isaacs/cliui": { "version": "8.0.2", "resolved": "https://registry.npmjs.org/@isaacs/cliui/-/cliui-8.0.2.tgz", @@ -8576,6 +8582,14 @@ "node": ">=0.8" } }, + "node_modules/cluster-key-slot": { + "version": "1.1.2", + "resolved": "https://registry.npmjs.org/cluster-key-slot/-/cluster-key-slot-1.1.2.tgz", + "integrity": "sha512-RMr0FhtfXemyinomL4hrWcYJxmX6deFdCxpJzhDttxgO1+bcCnkk+9drydLVDmAMG7NE6aN/fl4F7ucU/90gAA==", + "engines": { + "node": ">=0.10.0" + } + }, "node_modules/co": { "version": "4.6.0", "resolved": "https://registry.npmjs.org/co/-/co-4.6.0.tgz", @@ -9671,6 +9685,14 @@ "node": ">=0.4.0" } }, + "node_modules/denque": { + "version": "2.1.0", + "resolved": "https://registry.npmjs.org/denque/-/denque-2.1.0.tgz", + "integrity": "sha512-HVQE3AAb/pxF8fQAoiqpvg9i3evqug3hoiwakOyZAwJm+6vZehbkYXZ0l4JxS+I3QxM97v5aaRNhj8v5oBhekw==", + "engines": { + "node": ">=0.10" + } + }, "node_modules/depd": { "version": "2.0.0", "resolved": "https://registry.npmjs.org/depd/-/depd-2.0.0.tgz", @@ -12389,6 +12411,29 @@ "node": ">= 0.4" } }, + "node_modules/ioredis": { + "version": "5.4.1", + "resolved": "https://registry.npmjs.org/ioredis/-/ioredis-5.4.1.tgz", + "integrity": "sha512-2YZsvl7jopIa1gaePkeMtd9rAcSjOOjPtpcLlOeusyO+XH2SK5ZcT+UCrElPP+WVIInh2TzeI4XW9ENaSLVVHA==", + "dependencies": { + "@ioredis/commands": "^1.1.1", + "cluster-key-slot": "^1.1.0", + "debug": "^4.3.4", + "denque": "^2.1.0", + "lodash.defaults": "^4.2.0", + "lodash.isarguments": "^3.1.0", + "redis-errors": "^1.2.0", + "redis-parser": "^3.0.0", + "standard-as-callback": "^2.1.0" + }, + "engines": { + "node": ">=12.22.0" + }, + "funding": { + "type": "opencollective", + "url": "https://opencollective.com/ioredis" + } + }, "node_modules/ipaddr.js": { "version": "1.9.1", "resolved": "https://registry.npmjs.org/ipaddr.js/-/ipaddr.js-1.9.1.tgz", @@ -14983,11 +15028,21 @@ "integrity": "sha512-FT1yDzDYEoYWhnSGnpE/4Kj1fLZkDFyqRb7fNt6FdYOSxlUWAtp42Eh6Wb0rGIv/m9Bgo7x4GhQbm5Ys4SG5ow==", "dev": true }, + "node_modules/lodash.defaults": { + "version": "4.2.0", + "resolved": "https://registry.npmjs.org/lodash.defaults/-/lodash.defaults-4.2.0.tgz", + "integrity": "sha512-qjxPLHd3r5DnsdGacqOMU6pb/avJzdh9tFX2ymgoZE27BmjXrNy/y4LoaiTeAb+O3gL8AfpJGtqfX/ae2leYYQ==" + }, "node_modules/lodash.get": { "version": "4.4.2", "resolved": "https://registry.npmjs.org/lodash.get/-/lodash.get-4.4.2.tgz", "integrity": "sha512-z+Uw/vLuy6gQe8cfaFWD7p0wVv8fJl3mbzXh33RS+0oW2wvUqiRXiQ69gLWSLpgB5/6sU+r6BlQR0MBILadqTQ==" }, + "node_modules/lodash.isarguments": { + "version": "3.1.0", + "resolved": "https://registry.npmjs.org/lodash.isarguments/-/lodash.isarguments-3.1.0.tgz", + "integrity": "sha512-chi4NHZlZqZD18a0imDHnZPrDeBbTtVN7GXMwuGdRH9qotxAjYs3aVLKc7zNOG9eddR5Ksd8rvFEBc9SsggPpg==" + }, "node_modules/lodash.isequal": { "version": "4.5.0", "resolved": "https://registry.npmjs.org/lodash.isequal/-/lodash.isequal-4.5.0.tgz", @@ -17231,6 +17286,25 @@ "node": ">=8" } }, + "node_modules/redis-errors": { + "version": "1.2.0", + "resolved": "https://registry.npmjs.org/redis-errors/-/redis-errors-1.2.0.tgz", + "integrity": "sha512-1qny3OExCf0UvUV/5wpYKf2YwPcOqXzkwKKSmKHiE6ZMQs5heeE/c8eXK+PNllPvmjgAbfnsbpkGZWy8cBpn9w==", + "engines": { + "node": ">=4" + } + }, + "node_modules/redis-parser": { + "version": "3.0.0", + "resolved": "https://registry.npmjs.org/redis-parser/-/redis-parser-3.0.0.tgz", + "integrity": "sha512-DJnGAeenTdpMEH6uAJRK/uiyEIH9WVsUmoLwzudwGJUwZPp80PDBWPHXSAGNPwNvIXAbe7MSUB1zQFugFml66A==", + "dependencies": { + "redis-errors": "^1.0.0" + }, + "engines": { + "node": ">=4" + } + }, "node_modules/reflect-metadata": { "version": "0.1.13", "resolved": "https://registry.npmjs.org/reflect-metadata/-/reflect-metadata-0.1.13.tgz", @@ -17926,6 +18000,11 @@ "node": ">=8" } }, + "node_modules/standard-as-callback": { + "version": "2.1.0", + "resolved": "https://registry.npmjs.org/standard-as-callback/-/standard-as-callback-2.1.0.tgz", + "integrity": "sha512-qoRRSyROncaz1z0mvYqIE4lCd9p2R90i6GxW3uZv5ucSu8tU7B5HXUP1gG8pVZsYNVaXjk8ClXHPttLyxAL48A==" + }, "node_modules/standard-version": { "version": "9.5.0", "resolved": "https://registry.npmjs.org/standard-version/-/standard-version-9.5.0.tgz", @@ -21658,6 +21737,11 @@ "integrity": "sha512-H9XAx3hc0BQHY6l+IFSWHDySypcXsvsuLhgYLUGywmJ5pswRVQJUHpOsobnLYp2ZUaUlKiKDrgWWhosOwAEM8Q==", "dev": true }, + "@ioredis/commands": { + "version": "1.2.0", + "resolved": "https://registry.npmjs.org/@ioredis/commands/-/commands-1.2.0.tgz", + "integrity": "sha512-Sx1pU8EM64o2BrqNpEO1CNLtKQwyhuXuqyfH7oGKCk+1a33d2r5saW8zNwm3j6BTExtjrv2BxTgzzkMwts6vGg==" + }, "@isaacs/cliui": { "version": "8.0.2", "resolved": "https://registry.npmjs.org/@isaacs/cliui/-/cliui-8.0.2.tgz", @@ -25891,6 +25975,11 @@ "integrity": "sha512-JQHZ2QMW6l3aH/j6xCqQThY/9OH4D/9ls34cgkUBiEeocRTU04tHfKPBsUK1PqZCUQM7GiA0IIXJSuXHI64Kbg==", "dev": true }, + "cluster-key-slot": { + "version": "1.1.2", + "resolved": "https://registry.npmjs.org/cluster-key-slot/-/cluster-key-slot-1.1.2.tgz", + "integrity": "sha512-RMr0FhtfXemyinomL4hrWcYJxmX6deFdCxpJzhDttxgO1+bcCnkk+9drydLVDmAMG7NE6aN/fl4F7ucU/90gAA==" + }, "co": { "version": "4.6.0", "resolved": "https://registry.npmjs.org/co/-/co-4.6.0.tgz", @@ -26773,6 +26862,11 @@ "integrity": "sha512-ZySD7Nf91aLB0RxL4KGrKHBXl7Eds1DAmEdcoVawXnLD7SDhpNgtuII2aAkg7a7QS41jxPSZ17p4VdGnMHk3MQ==", "dev": true }, + "denque": { + "version": "2.1.0", + "resolved": "https://registry.npmjs.org/denque/-/denque-2.1.0.tgz", + "integrity": "sha512-HVQE3AAb/pxF8fQAoiqpvg9i3evqug3hoiwakOyZAwJm+6vZehbkYXZ0l4JxS+I3QxM97v5aaRNhj8v5oBhekw==" + }, "depd": { "version": "2.0.0", "resolved": "https://registry.npmjs.org/depd/-/depd-2.0.0.tgz", @@ -28802,6 +28896,22 @@ "side-channel": "^1.0.4" } }, + "ioredis": { + "version": "5.4.1", + "resolved": "https://registry.npmjs.org/ioredis/-/ioredis-5.4.1.tgz", + "integrity": "sha512-2YZsvl7jopIa1gaePkeMtd9rAcSjOOjPtpcLlOeusyO+XH2SK5ZcT+UCrElPP+WVIInh2TzeI4XW9ENaSLVVHA==", + "requires": { + "@ioredis/commands": "^1.1.1", + "cluster-key-slot": "^1.1.0", + "debug": "^4.3.4", + "denque": "^2.1.0", + "lodash.defaults": "^4.2.0", + "lodash.isarguments": "^3.1.0", + "redis-errors": "^1.2.0", + "redis-parser": "^3.0.0", + "standard-as-callback": "^2.1.0" + } + }, "ipaddr.js": { "version": "1.9.1", "resolved": "https://registry.npmjs.org/ipaddr.js/-/ipaddr.js-1.9.1.tgz", @@ -30729,11 +30839,21 @@ "integrity": "sha512-FT1yDzDYEoYWhnSGnpE/4Kj1fLZkDFyqRb7fNt6FdYOSxlUWAtp42Eh6Wb0rGIv/m9Bgo7x4GhQbm5Ys4SG5ow==", "dev": true }, + "lodash.defaults": { + "version": "4.2.0", + "resolved": "https://registry.npmjs.org/lodash.defaults/-/lodash.defaults-4.2.0.tgz", + "integrity": "sha512-qjxPLHd3r5DnsdGacqOMU6pb/avJzdh9tFX2ymgoZE27BmjXrNy/y4LoaiTeAb+O3gL8AfpJGtqfX/ae2leYYQ==" + }, "lodash.get": { "version": "4.4.2", "resolved": "https://registry.npmjs.org/lodash.get/-/lodash.get-4.4.2.tgz", "integrity": "sha512-z+Uw/vLuy6gQe8cfaFWD7p0wVv8fJl3mbzXh33RS+0oW2wvUqiRXiQ69gLWSLpgB5/6sU+r6BlQR0MBILadqTQ==" }, + "lodash.isarguments": { + "version": "3.1.0", + "resolved": "https://registry.npmjs.org/lodash.isarguments/-/lodash.isarguments-3.1.0.tgz", + "integrity": "sha512-chi4NHZlZqZD18a0imDHnZPrDeBbTtVN7GXMwuGdRH9qotxAjYs3aVLKc7zNOG9eddR5Ksd8rvFEBc9SsggPpg==" + }, "lodash.isequal": { "version": "4.5.0", "resolved": "https://registry.npmjs.org/lodash.isequal/-/lodash.isequal-4.5.0.tgz", @@ -32456,6 +32576,19 @@ "strip-indent": "^3.0.0" } }, + "redis-errors": { + "version": "1.2.0", + "resolved": "https://registry.npmjs.org/redis-errors/-/redis-errors-1.2.0.tgz", + "integrity": "sha512-1qny3OExCf0UvUV/5wpYKf2YwPcOqXzkwKKSmKHiE6ZMQs5heeE/c8eXK+PNllPvmjgAbfnsbpkGZWy8cBpn9w==" + }, + "redis-parser": { + "version": "3.0.0", + "resolved": "https://registry.npmjs.org/redis-parser/-/redis-parser-3.0.0.tgz", + "integrity": "sha512-DJnGAeenTdpMEH6uAJRK/uiyEIH9WVsUmoLwzudwGJUwZPp80PDBWPHXSAGNPwNvIXAbe7MSUB1zQFugFml66A==", + "requires": { + "redis-errors": "^1.0.0" + } + }, "reflect-metadata": { "version": "0.1.13", "resolved": "https://registry.npmjs.org/reflect-metadata/-/reflect-metadata-0.1.13.tgz", @@ -32985,6 +33118,11 @@ } } }, + "standard-as-callback": { + "version": "2.1.0", + "resolved": "https://registry.npmjs.org/standard-as-callback/-/standard-as-callback-2.1.0.tgz", + "integrity": "sha512-qoRRSyROncaz1z0mvYqIE4lCd9p2R90i6GxW3uZv5ucSu8tU7B5HXUP1gG8pVZsYNVaXjk8ClXHPttLyxAL48A==" + }, "standard-version": { "version": "9.5.0", "resolved": "https://registry.npmjs.org/standard-version/-/standard-version-9.5.0.tgz", diff --git a/package.json b/package.json index eaf1c718..98795cf1 100644 --- a/package.json +++ b/package.json @@ -66,6 +66,7 @@ "fast-xml-parser": "^4.4.0", "geojson-validation": "^1.0.2", "http-status-codes": "^2.2.0", + "ioredis": "^5.4.1", "mgrs": "^2.0.0", "node-cron": "^3.0.3", "node-fetch-commonjs": "^3.3.2", diff --git a/src/common/constants.ts b/src/common/constants.ts index 5ab1d8b1..39f7b888 100644 --- a/src/common/constants.ts +++ b/src/common/constants.ts @@ -1,7 +1,9 @@ +import { hostname } from 'os'; import { readPackageJsonSync } from '@map-colonies/read-pkg'; export const SERVICE_NAME = readPackageJsonSync().name ?? 'unknown_service'; export const DEFAULT_SERVER_PORT = 80; +export const HOSTNAME = hostname(); export const IGNORED_OUTGOING_TRACE_ROUTES = [/^.*\/v1\/metrics.*$/]; export const IGNORED_INCOMING_TRACE_ROUTES = [/^.*\/docs.*$/]; @@ -19,5 +21,7 @@ export const SERVICES: Record = { export const ON_SIGNAL = Symbol('onSignal'); export const HEALTHCHECK = Symbol('healthcheck'); +export const REDIS_SYMBOL = Symbol('REDIS'); export const elasticConfigPath = 'db.elastic'; +export const REDIS_KEYS_SEPARATOR = ':'; diff --git a/src/common/interfaces.ts b/src/common/interfaces.ts index b8d65a2c..d140f1a0 100644 --- a/src/common/interfaces.ts +++ b/src/common/interfaces.ts @@ -49,6 +49,10 @@ export interface IApplication { hierarchy: number; viewbox: number; }; + hashKey: { + value?: string; + enabled: boolean; + }; sources?: { [key: string]: string; }; diff --git a/src/common/redis/domainFieldsRepository.ts b/src/common/redis/domainFieldsRepository.ts new file mode 100644 index 00000000..9cf9e3cc --- /dev/null +++ b/src/common/redis/domainFieldsRepository.ts @@ -0,0 +1,5 @@ +export const IDOMAIN_FIELDS_REPO_SYMBOL = Symbol('DOMAINFIELDSREPO'); + +export interface IDomainFieldsRepository { + getFields: (fields: string[]) => Promise<(string | null)[]>; +} diff --git a/src/common/redis/index.ts b/src/common/redis/index.ts new file mode 100644 index 00000000..fb3396dc --- /dev/null +++ b/src/common/redis/index.ts @@ -0,0 +1,33 @@ +import Redis, { RedisOptions } from 'ioredis'; +import { HOSTNAME } from '../constants'; + +const RETRY_DELAY_INCREASE = 50; +const RETRY_DELAY_TOP = 2000; +let redis: Redis; + +const retryFunction = (times: number): number => { + const delay = Math.min(times * RETRY_DELAY_INCREASE, RETRY_DELAY_TOP); + return delay; +}; + +export const createRedisConnection = async (redisOptions: RedisOptions): Promise => { + try { + redisOptions = { + ...redisOptions, + retryStrategy: retryFunction, + lazyConnect: true, + connectionName: HOSTNAME, + }; + + redis = new Redis(redisOptions); + await redis.connect(); + return redis; + } catch (err) { + redis.disconnect(); + let errorMessage = 'Redis connection failed'; + if (err instanceof Error) { + errorMessage += ` with the following error: ${err.message}`; + } + throw new Error(errorMessage); + } +}; diff --git a/src/common/redis/keys.ts b/src/common/redis/keys.ts new file mode 100644 index 00000000..e268631c --- /dev/null +++ b/src/common/redis/keys.ts @@ -0,0 +1,5 @@ +import { REDIS_KEYS_SEPARATOR } from '../../common/constants'; + +export const keyConstructor = (...vals: string[]): string => { + return vals.filter((v) => v).join(REDIS_KEYS_SEPARATOR); // drop undefined, null and empty string from a key +}; diff --git a/src/common/redis/redisManager.ts b/src/common/redis/redisManager.ts new file mode 100644 index 00000000..0ea22c07 --- /dev/null +++ b/src/common/redis/redisManager.ts @@ -0,0 +1,31 @@ +import Redis from 'ioredis'; +import { inject, injectable } from 'tsyringe'; +import { REDIS_SYMBOL, SERVICES } from '../constants'; +import { IApplication } from '../interfaces'; +import { IDomainFieldsRepository } from './domainFieldsRepository'; + +@injectable() +export class RedisManager implements IDomainFieldsRepository { + public getData: (fields: string[]) => Promise<(string | null)[]>; + + public constructor(@inject(REDIS_SYMBOL) private readonly redis: Redis, @inject(SERVICES.APPLICATION) appConfig: IApplication) { + const { value, enabled } = appConfig.hashKey; + if (enabled && value !== undefined) { + this.getData = async (fields: string[]): Promise<(string | null)[]> => { + return this.redis.hmget(value, ...fields); + }; + } else { + this.getData = async (fields: string[]): Promise<(string | null)[]> => { + return this.redis.mget(fields); + }; + } + } + + public async getFields(fields: string[]): Promise<(string | null)[]> { + try { + return await this.getData(fields); + } catch (e) { + throw new Error('redis: failed to fetch keys'); + } + } +} diff --git a/src/containerConfig.ts b/src/containerConfig.ts index e6e49d95..658dc442 100644 --- a/src/containerConfig.ts +++ b/src/containerConfig.ts @@ -1,4 +1,5 @@ import config from 'config'; +import Redis, { RedisOptions } from 'ioredis'; import { getOtelMixin } from '@map-colonies/telemetry'; import { DataSource } from 'typeorm'; import { instancePerContainerCachingFactory } from 'tsyringe'; @@ -6,10 +7,11 @@ import { trace, metrics as OtelMetrics } from '@opentelemetry/api'; import { DependencyContainer } from 'tsyringe/dist/typings/types'; import jsLogger, { LoggerOptions } from '@map-colonies/js-logger'; import { Metrics } from '@map-colonies/telemetry'; -import { SERVICES, SERVICE_NAME } from './common/constants'; +import { SERVICES, SERVICE_NAME, REDIS_SYMBOL } from './common/constants'; import { tracing } from './common/tracing'; import { InjectionObject, registerDependencies } from './common/dependencyRegistration'; import { elasticClientFactory, ElasticClients } from './common/elastic'; +import { RedisManager } from './common/redis/redisManager'; import { IApplication } from './common/interfaces'; import { TILE_REPOSITORY_SYMBOL, tileRepositoryFactory } from './control/tile/DAL/tileRepository'; import { TILE_ROUTER_SYMBOL, tileRouterFactory } from './control/tile/routes/tileRouter'; @@ -23,6 +25,8 @@ import { GEOTEXT_REPOSITORY_SYMBOL, geotextRepositoryFactory } from './location/ import { GEOTEXT_SEARCH_ROUTER_SYMBOL, geotextSearchRouterFactory } from './location/routes/locationRouter'; import { cronLoadTileLatLonDataFactory, cronLoadTileLatLonDataSymbol } from './latLon/DAL/latLonDAL'; import { ITEM_REPOSITORY_SYMBOL, itemRepositoryFactory } from './control/item/DAL/itemRepository'; +import { createRedisConnection } from './common/redis/index'; +import { IDOMAIN_FIELDS_REPO_SYMBOL } from './common/redis/domainFieldsRepository'; export interface RegisterOptions { override?: InjectionObject[]; @@ -41,6 +45,8 @@ export const registerExternalValues = async (options?: RegisterOptions): Promise const applicationConfig: IApplication = config.get('application'); + let redisConnection: Redis | undefined; + const dependencies: InjectionObject[] = [ { token: SERVICES.CONFIG, provider: { useValue: config } }, { token: SERVICES.LOGGER, provider: { useValue: logger } }, @@ -77,6 +83,29 @@ export const registerExternalValues = async (options?: RegisterOptions): Promise } }, }, + { + token: REDIS_SYMBOL, + provider: { + useFactory: instancePerContainerCachingFactory(async () => { + redisConnection = await createRedisConnection(config.get('db')); + + redisConnection.on('connect', () => { + logger.info(`redis client is connected.`); + }); + + redisConnection.on('error', (err: Error) => { + logger.error({ err: err, msg: 'redis client got an error' }); + }); + + redisConnection.on('reconnecting', (delay: number) => { + logger.info(`redis client reconnecting, next reconnection attemp in ${delay}ms`); + }); + container.register(SERVICES.LOGGER, { useValue: logger }); + container.register(IDOMAIN_FIELDS_REPO_SYMBOL, { useClass: RedisManager }); + return redisConnection; + }), + }, + }, { token: TILE_REPOSITORY_SYMBOL, provider: { useFactory: tileRepositoryFactory } }, { token: TILE_ROUTER_SYMBOL, provider: { useFactory: tileRouterFactory } }, { token: ITEM_REPOSITORY_SYMBOL, provider: { useFactory: itemRepositoryFactory } }, @@ -96,11 +125,22 @@ export const registerExternalValues = async (options?: RegisterOptions): Promise { token: 'onSignal', provider: { - useValue: { - useValue: async (): Promise => { - await Promise.all([tracing.stop(), metrics.stop()]); - }, - }, + useFactory: instancePerContainerCachingFactory(async (): Promise => { + const promises: Promise[] = [tracing.stop(), metrics.stop()]; + if (redisConnection !== undefined) { + redisConnection.disconnect(); + + const promisifyQuit = new Promise((resolve) => { + redisConnection = redisConnection as Redis; + redisConnection.once('end', () => { + resolve(); + }); + void redisConnection.quit(); + }); + promises.push(promisifyQuit); + } + await Promise.all(promises); + }), }, }, ]; From 5bb7d11999385f5a2cf63f8a3a4963961fdbdd1d Mon Sep 17 00:00:00 2001 From: NatalieShaked Date: Tue, 27 Aug 2024 14:46:59 +0300 Subject: [PATCH 152/262] test: added redis to tests --- tests/integration/control/item/item.spec.ts | 30 +++++++++++++++++-- tests/integration/control/route/route.spec.ts | 28 ++++++++++++++++- tests/integration/control/tile/tile.spec.ts | 28 ++++++++++++++++- tests/integration/location/location.spec.ts | 27 ++++++++++++++++- 4 files changed, 108 insertions(+), 5 deletions(-) diff --git a/tests/integration/control/item/item.spec.ts b/tests/integration/control/item/item.spec.ts index 39e3659b..01327f88 100644 --- a/tests/integration/control/item/item.spec.ts +++ b/tests/integration/control/item/item.spec.ts @@ -3,8 +3,12 @@ import jsLogger from '@map-colonies/js-logger'; import { trace } from '@opentelemetry/api'; import httpStatusCodes from 'http-status-codes'; import { DataSource } from 'typeorm'; +import config from 'config'; +import Redis, { RedisOptions } from 'ioredis'; +import { DependencyContainer } from 'tsyringe'; +import { Application } from 'express'; import { getApp } from '../../../../src/app'; -import { SERVICES } from '../../../../src/common/constants'; +import { REDIS_SYMBOL, SERVICES } from '../../../../src/common/constants'; import { GetItemsQueryParams } from '../../../../src/control/item/controllers/itemController'; import { Item } from '../../../../src/control/item/models/item'; import { ControlResponse } from '../../../../src/control/interfaces'; @@ -12,14 +16,30 @@ import { CommonRequestParameters, GeoContext, GeoContextMode } from '../../../.. import { LATLON_CUSTOM_REPOSITORY_SYMBOL } from '../../../../src/latLon/DAL/latLonRepository'; import { cronLoadTileLatLonDataSymbol } from '../../../../src/latLon/DAL/latLonDAL'; import { expectedResponse } from '../utils'; +import { RedisManager } from '../../../../src/common/redis/redisManager'; +import { IDOMAIN_FIELDS_REPO_SYMBOL } from '../../../../src/common/redis/domainFieldsRepository'; +import { createRedisConnection } from '../../../../src/common/redis'; import { ItemRequestSender } from './helpers/requestSender'; import { ITEM_1234, ITEM_1235, ITEM_1236 } from './mockObjects'; describe('/search/control/items', function () { let requestSender: ItemRequestSender; + let app: { app: Application; container?: DependencyContainer }; + let redisConnection: Redis; + + beforeAll(async function () { + redisConnection = await createRedisConnection(config.get('db')); + app = await getApp({ + override: [ + { token: REDIS_SYMBOL, provider: { useValue: redisConnection } }, + { token: IDOMAIN_FIELDS_REPO_SYMBOL, provider: { useClass: RedisManager } }, + ], + useChild: true, + }); + }); beforeEach(async function () { - const app = await getApp({ + app = await getApp({ override: [ { token: SERVICES.LOGGER, provider: { useValue: jsLogger({ enabled: false }) } }, { token: SERVICES.TRACER, provider: { useValue: trace.getTracer('testTracer') } }, @@ -34,6 +54,12 @@ describe('/search/control/items', function () { requestSender = new ItemRequestSender(app.app); }); + afterAll(async function () { + if (!['end'].includes(redisConnection.status)) { + await redisConnection.quit(); + } + }); + describe('Happy Path', function () { it('should return 200 status code and items', async function () { const requestParams: GetItemsQueryParams = { command_name: '123', limit: 5, disable_fuzziness: false }; diff --git a/tests/integration/control/route/route.spec.ts b/tests/integration/control/route/route.spec.ts index fe95f22e..1dd5b394 100644 --- a/tests/integration/control/route/route.spec.ts +++ b/tests/integration/control/route/route.spec.ts @@ -3,20 +3,40 @@ import jsLogger from '@map-colonies/js-logger'; import { trace } from '@opentelemetry/api'; import httpStatusCodes from 'http-status-codes'; import { DataSource } from 'typeorm'; +import config from 'config'; +import Redis, { RedisOptions } from 'ioredis'; +import { DependencyContainer } from 'tsyringe'; +import { Application } from 'express'; import { getApp } from '../../../../src/app'; -import { SERVICES } from '../../../../src/common/constants'; +import { REDIS_SYMBOL, SERVICES } from '../../../../src/common/constants'; import { GetRoutesQueryParams } from '../../../../src/control/route/controllers/routeController'; import { Route } from '../../../../src/control/route/models/route'; import { ControlResponse } from '../../../../src/control/interfaces'; import { CommonRequestParameters, GeoContext, GeoContextMode } from '../../../../src/common/interfaces'; import { LATLON_CUSTOM_REPOSITORY_SYMBOL } from '../../../../src/latLon/DAL/latLonRepository'; import { cronLoadTileLatLonDataSymbol } from '../../../../src/latLon/DAL/latLonDAL'; +import { RedisManager } from '../../../../src/common/redis/redisManager'; +import { IDOMAIN_FIELDS_REPO_SYMBOL } from '../../../../src/common/redis/domainFieldsRepository'; +import { createRedisConnection } from '../../../../src/common/redis'; import { expectedResponse } from '../utils'; import { RouteRequestSender } from './helpers/requestSender'; import { ROUTE_VIA_CAMILLUCCIA_A, ROUTE_VIA_CAMILLUCCIA_B, CONTROL_POINT_OLIMPIADE_111, CONTROL_POINT_OLIMPIADE_112 } from './mockObjects'; describe('/search/control/route', function () { let requestSender: RouteRequestSender; + let app: { app: Application; container?: DependencyContainer }; + let redisConnection: Redis; + + beforeAll(async function () { + redisConnection = await createRedisConnection(config.get('db')); + app = await getApp({ + override: [ + { token: REDIS_SYMBOL, provider: { useValue: redisConnection } }, + { token: IDOMAIN_FIELDS_REPO_SYMBOL, provider: { useClass: RedisManager } }, + ], + useChild: true, + }); + }); beforeEach(async function () { const app = await getApp({ @@ -34,6 +54,12 @@ describe('/search/control/route', function () { requestSender = new RouteRequestSender(app.app); }); + afterAll(async function () { + if (!['end'].includes(redisConnection.status)) { + await redisConnection.quit(); + } + }); + describe('Happy Path', function () { it('should return 200 status code and routes', async function () { const requestParams: GetRoutesQueryParams = { command_name: 'via camilluccia', limit: 5, disable_fuzziness: false }; diff --git a/tests/integration/control/tile/tile.spec.ts b/tests/integration/control/tile/tile.spec.ts index 44685fcc..6564fd99 100644 --- a/tests/integration/control/tile/tile.spec.ts +++ b/tests/integration/control/tile/tile.spec.ts @@ -3,13 +3,20 @@ import jsLogger from '@map-colonies/js-logger'; import { trace } from '@opentelemetry/api'; import httpStatusCodes from 'http-status-codes'; import { DataSource } from 'typeorm'; +import config from 'config'; +import Redis, { RedisOptions } from 'ioredis'; +import { DependencyContainer } from 'tsyringe'; +import { Application } from 'express'; import { getApp } from '../../../../src/app'; -import { SERVICES } from '../../../../src/common/constants'; +import { REDIS_SYMBOL, SERVICES } from '../../../../src/common/constants'; import { GetTilesQueryParams } from '../../../../src/control/tile/controllers/tileController'; import { Tile } from '../../../../src/control/tile/models/tile'; import { ControlResponse } from '../../../../src/control/interfaces'; import { CommonRequestParameters, GeoContext, GeoContextMode } from '../../../../src/common/interfaces'; import { LATLON_CUSTOM_REPOSITORY_SYMBOL } from '../../../../src/latLon/DAL/latLonRepository'; +import { RedisManager } from '../../../../src/common/redis/redisManager'; +import { IDOMAIN_FIELDS_REPO_SYMBOL } from '../../../../src/common/redis/domainFieldsRepository'; +import { createRedisConnection } from '../../../../src/common/redis'; import { cronLoadTileLatLonDataSymbol } from '../../../../src/latLon/DAL/latLonDAL'; import { expectedResponse } from '../utils'; import { TileRequestSender } from './helpers/requestSender'; @@ -17,6 +24,19 @@ import { RIC_TILE, RIT_TILE, SUB_TILE_65, SUB_TILE_66 } from './mockObjects'; describe('/search/control/tiles', function () { let requestSender: TileRequestSender; + let app: { app: Application; container?: DependencyContainer }; + let redisConnection: Redis; + + beforeAll(async function () { + redisConnection = await createRedisConnection(config.get('db')); + app = await getApp({ + override: [ + { token: REDIS_SYMBOL, provider: { useValue: redisConnection } }, + { token: IDOMAIN_FIELDS_REPO_SYMBOL, provider: { useClass: RedisManager } }, + ], + useChild: true, + }); + }); beforeEach(async function () { const app = await getApp({ @@ -34,6 +54,12 @@ describe('/search/control/tiles', function () { requestSender = new TileRequestSender(app.app); }); + afterAll(async function () { + if (!['end'].includes(redisConnection.status)) { + await redisConnection.quit(); + } + }); + describe('Happy Path', function () { it('should return 200 status code and tiles', async function () { const requestParams: GetTilesQueryParams = { tile: 'RIT', limit: 5, disable_fuzziness: false }; diff --git a/tests/integration/location/location.spec.ts b/tests/integration/location/location.spec.ts index e61cdc0b..385c4f59 100644 --- a/tests/integration/location/location.spec.ts +++ b/tests/integration/location/location.spec.ts @@ -3,12 +3,18 @@ import config from 'config'; import jsLogger from '@map-colonies/js-logger'; import { trace } from '@opentelemetry/api'; import httpStatusCodes from 'http-status-codes'; +import Redis, { RedisOptions } from 'ioredis'; +import { DependencyContainer } from 'tsyringe'; +import { Application } from 'express'; import { DataSource } from 'typeorm'; import nock, { Body } from 'nock'; import { getApp } from '../../../src/app'; -import { SERVICES } from '../../../src/common/constants'; +import { REDIS_SYMBOL, SERVICES } from '../../../src/common/constants'; import { LATLON_CUSTOM_REPOSITORY_SYMBOL } from '../../../src/latLon/DAL/latLonRepository'; import { cronLoadTileLatLonDataSymbol } from '../../../src/latLon/DAL/latLonDAL'; +import { RedisManager } from '../../../src/common/redis/redisManager'; +import { IDOMAIN_FIELDS_REPO_SYMBOL } from '../../../src/common/redis/domainFieldsRepository'; +import { createRedisConnection } from '../../../src/common/redis'; import { GetGeotextSearchParams, QueryResult } from '../../../src/location/interfaces'; import { GeoContextMode, IApplication } from '../../../src/common/interfaces'; import { LocationRequestSender } from './helpers/requestSender'; @@ -29,6 +35,19 @@ import { expectedResponse, hierarchiesWithAnyWieght } from './utils'; describe('/search/control/tiles', function () { let requestSender: LocationRequestSender; + let app: { app: Application; container?: DependencyContainer }; + let redisConnection: Redis; + + beforeAll(async function () { + redisConnection = await createRedisConnection(config.get('db')); + app = await getApp({ + override: [ + { token: REDIS_SYMBOL, provider: { useValue: redisConnection } }, + { token: IDOMAIN_FIELDS_REPO_SYMBOL, provider: { useClass: RedisManager } }, + ], + useChild: true, + }); + }); beforeEach(async function () { const app = await getApp({ @@ -46,6 +65,12 @@ describe('/search/control/tiles', function () { requestSender = new LocationRequestSender(app.app); }); + afterAll(async function () { + if (!['end'].includes(redisConnection.status)) { + await redisConnection.quit(); + } + }); + describe('Happy Path', function () { it('should return 200 status code and airports', async function () { const requestParams: GetGeotextSearchParams = { query: 'airport', limit: 5, disable_fuzziness: true }; From fc2990cafad7ed3c83ab6e0077f8db57bbc3f049 Mon Sep 17 00:00:00 2001 From: Niv Greenstein Date: Tue, 27 Aug 2024 17:07:05 +0300 Subject: [PATCH 153/262] feat: changed returned response to be genericGeocodingResponse --- src/common/interfaces.ts | 3 + src/control/constants.ts | 1 + src/control/interfaces.ts | 27 +++++---- src/control/item/models/item.ts | 12 +++- src/control/item/models/itemManager.ts | 5 +- src/control/route/models/route.ts | 12 +++- src/control/route/models/routeManager.ts | 5 +- src/control/tile/models/tile.ts | 12 +++- src/control/tile/models/tileManager.ts | 5 +- src/control/utils.ts | 70 ++++++++++++++++++++---- src/location/interfaces.ts | 2 +- src/location/utils.ts | 12 ++-- 12 files changed, 127 insertions(+), 39 deletions(-) diff --git a/src/common/interfaces.ts b/src/common/interfaces.ts index b8d65a2c..58ec287e 100644 --- a/src/common/interfaces.ts +++ b/src/common/interfaces.ts @@ -57,6 +57,9 @@ export interface IApplication { }; nameTranslationsKeys: string[]; mainLanguageRegex: string; + controlObjectDisplayNamePrefixes: { + [key: string]: string; + }; } export enum GeoContextMode { diff --git a/src/control/constants.ts b/src/control/constants.ts index 755994cd..620d816e 100644 --- a/src/control/constants.ts +++ b/src/control/constants.ts @@ -18,4 +18,5 @@ export const CONTROL_FIELDS = [ 'properties.SUB_TILE_ID', 'properties.SECTION', 'properties.TIED_TO', + 'properties.LAYER_NAME', ]; diff --git a/src/control/interfaces.ts b/src/control/interfaces.ts index a7de812a..6ff17d97 100644 --- a/src/control/interfaces.ts +++ b/src/control/interfaces.ts @@ -1,4 +1,4 @@ -import { Feature } from 'geojson'; +import { Feature, GeoJsonProperties } from 'geojson'; import { estypes } from '@elastic/elasticsearch'; import { CommonRequestParameters, FeatureCollection } from '../common/interfaces'; import { RemoveUnderscore } from '../common/utils'; @@ -15,14 +15,19 @@ export interface ControlResponse extends FeatureColl }; }; features: (T & { - matches: { - source: string; - layer: string; - }[]; - names: { - [key: string]: string; - display: string; - default: string; - }; - } & RemoveUnderscore, '_score'>>)[]; + properties: RemoveUnderscore, '_score'>> & + GeoJsonProperties & { + matches: { + layer: string; + source: string; + // eslint-disable-next-line @typescript-eslint/naming-convention + source_id: string[]; + }[]; + name: { + [key: string]: string | string[] | undefined; + display: string; + default: string[]; + }; + }; + })[]; } diff --git a/src/control/item/models/item.ts b/src/control/item/models/item.ts index 3869c449..5b57db3a 100644 --- a/src/control/item/models/item.ts +++ b/src/control/item/models/item.ts @@ -1,3 +1,11 @@ -import { Feature } from 'geojson'; +/* eslint-disable @typescript-eslint/naming-convention */ +import { Feature, GeoJsonProperties } from 'geojson'; -export interface Item extends Feature {} +export interface Item extends Feature { + properties: GeoJsonProperties & { + TYPE: 'ITEM'; + OBJECT_COMMAND_NAME: string; + LAYER_NAME: string; + TIED_TO?: string; + }; +} diff --git a/src/control/item/models/itemManager.ts b/src/control/item/models/itemManager.ts index 25a33547..b751e8a9 100644 --- a/src/control/item/models/itemManager.ts +++ b/src/control/item/models/itemManager.ts @@ -6,7 +6,7 @@ import { SERVICES } from '../../../common/constants'; import { ITEM_REPOSITORY_SYMBOL, ItemRepository } from '../DAL/itemRepository'; import { ItemQueryParams } from '../DAL/queries'; import { formatResponse } from '../../utils'; -import { FeatureCollection } from '../../../common/interfaces'; +import { FeatureCollection, IApplication } from '../../../common/interfaces'; import { Item } from './item'; @injectable() @@ -14,6 +14,7 @@ export class ItemManager { public constructor( @inject(SERVICES.LOGGER) private readonly logger: Logger, @inject(SERVICES.CONFIG) private readonly config: IConfig, + @inject(SERVICES.APPLICATION) private readonly application: IApplication, @inject(ITEM_REPOSITORY_SYMBOL) private readonly itemRepository: ItemRepository ) {} @@ -22,6 +23,6 @@ export class ItemManager { let elasticResponse: estypes.SearchResponse | undefined = undefined; elasticResponse = await this.itemRepository.getItems(itemQueryParams, limit); - return formatResponse(elasticResponse, itemQueryParams); + return formatResponse(elasticResponse, itemQueryParams, this.application.controlObjectDisplayNamePrefixes); } } diff --git a/src/control/route/models/route.ts b/src/control/route/models/route.ts index 4860b131..607fcd12 100644 --- a/src/control/route/models/route.ts +++ b/src/control/route/models/route.ts @@ -1,3 +1,11 @@ -import { Feature } from 'geojson'; +/* eslint-disable @typescript-eslint/naming-convention */ +import { Feature, GeoJsonProperties } from 'geojson'; -export interface Route extends Feature {} +export interface Route extends Feature { + properties: GeoJsonProperties & { + TYPE: 'ROUTE'; + OBJECT_COMMAND_NAME: string; + ENTITY_HEB: string; + LAYER_NAME: string; + }; +} diff --git a/src/control/route/models/routeManager.ts b/src/control/route/models/routeManager.ts index 5d6a54ea..ccc5934f 100644 --- a/src/control/route/models/routeManager.ts +++ b/src/control/route/models/routeManager.ts @@ -6,7 +6,7 @@ import { SERVICES } from '../../../common/constants'; import { ROUTE_REPOSITORY_SYMBOL, RouteRepository } from '../DAL/routeRepository'; import { RouteQueryParams } from '../DAL/queries'; import { formatResponse } from '../../utils'; -import { FeatureCollection } from '../../../common/interfaces'; +import { FeatureCollection, IApplication } from '../../../common/interfaces'; import { Route } from './route'; @injectable() @@ -14,6 +14,7 @@ export class RouteManager { public constructor( @inject(SERVICES.LOGGER) private readonly logger: Logger, @inject(SERVICES.CONFIG) private readonly config: IConfig, + @inject(SERVICES.APPLICATION) private readonly application: IApplication, @inject(ROUTE_REPOSITORY_SYMBOL) private readonly routeRepository: RouteRepository ) {} @@ -30,6 +31,6 @@ export class RouteManager { elasticResponse = await this.routeRepository.getRoutes(routeQueryParams, limit); } - return formatResponse(elasticResponse, routeQueryParams); + return formatResponse(elasticResponse, routeQueryParams, this.application.controlObjectDisplayNamePrefixes); } } diff --git a/src/control/tile/models/tile.ts b/src/control/tile/models/tile.ts index 91cc6452..5b58d885 100644 --- a/src/control/tile/models/tile.ts +++ b/src/control/tile/models/tile.ts @@ -1,3 +1,11 @@ -import { Feature } from 'geojson'; +/* eslint-disable @typescript-eslint/naming-convention */ +import { Feature, GeoJsonProperties } from 'geojson'; -export interface Tile extends Feature {} +export interface Tile extends Feature { + properties: GeoJsonProperties & { + TYPE: 'TILE' | 'SUB_TILE'; + TILE_NAME?: string; + SUB_TILE_ID?: string; + LAYER_NAME?: string; + }; +} diff --git a/src/control/tile/models/tileManager.ts b/src/control/tile/models/tileManager.ts index 1e419d32..1ca48507 100644 --- a/src/control/tile/models/tileManager.ts +++ b/src/control/tile/models/tileManager.ts @@ -6,7 +6,7 @@ import { SERVICES } from '../../../common/constants'; import { TILE_REPOSITORY_SYMBOL, TileRepository } from '../DAL/tileRepository'; import { formatResponse } from '../../utils'; import { TileQueryParams } from '../DAL/queries'; -import { FeatureCollection } from '../../../common/interfaces'; +import { FeatureCollection, IApplication } from '../../../common/interfaces'; import { BadRequestError, NotImplementedError } from '../../../common/errors'; import { Tile } from './tile'; @@ -15,6 +15,7 @@ export class TileManager { public constructor( @inject(SERVICES.LOGGER) private readonly logger: Logger, @inject(SERVICES.CONFIG) private readonly config: IConfig, + @inject(SERVICES.APPLICATION) private readonly application: IApplication, @inject(TILE_REPOSITORY_SYMBOL) private readonly tileRepository: TileRepository ) {} @@ -41,6 +42,6 @@ export class TileManager { elasticResponse = await this.tileRepository.getTiles(tileQueryParams as TileQueryParams & Required>, limit); } - return formatResponse(elasticResponse, tileQueryParams); + return formatResponse(elasticResponse, tileQueryParams, this.application.controlObjectDisplayNamePrefixes); } } diff --git a/src/control/utils.ts b/src/control/utils.ts index 0a730eb8..9c92afca 100644 --- a/src/control/utils.ts +++ b/src/control/utils.ts @@ -1,18 +1,50 @@ +/* eslint-disable @typescript-eslint/no-unused-expressions */ /* eslint-disable @typescript-eslint/naming-convention */ import { estypes } from '@elastic/elasticsearch'; import { parseGeo } from '../location/utils'; -import { CommonRequestParameters, GeoContext, GeoContextMode, IConfig } from '../common/interfaces'; +import { CommonRequestParameters, GeoContext, GeoContextMode, IApplication, IConfig } from '../common/interfaces'; import { BadRequestError } from '../common/errors'; import { elasticConfigPath } from '../common/constants'; import { ElasticDbClientsConfig } from '../common/elastic/interfaces'; import { Item } from '../control/item/models/item'; import { Tile } from '../control/tile/models/tile'; import { Route } from '../control/route/models/route'; -import { ConvertSnakeToCamelCase, RemoveUnderscore } from '../common/utils'; +import { ConvertSnakeToCamelCase } from '../common/utils'; import { BBOX_LENGTH } from '../location/interfaces'; import { CONTROL_FIELDS, ELASTIC_KEYWORDS } from './constants'; import { ControlResponse } from './interfaces'; +const LAST_ELEMENT_INDEX = -1; + +const generateDisplayName = ( + source: T, + displayNamePrefixes: IApplication['controlObjectDisplayNamePrefixes'] +): (string | undefined)[] => { + const sourceType = + source.properties.TYPE === 'ITEM' && (source as Item).properties.TIED_TO !== undefined ? 'CONTROL_POINT' : source.properties.TYPE; + + const name: (string | undefined)[] = []; + sourceType === 'TILE' && name.unshift((source as Tile).properties.TILE_NAME); + sourceType === 'SUB_TILE' && name.unshift((source as Tile).properties.SUB_TILE_ID); + sourceType === 'ITEM' && name.unshift((source as Item).properties.OBJECT_COMMAND_NAME); + sourceType === 'ROUTE' && name.unshift((source as Route).properties.OBJECT_COMMAND_NAME); + sourceType === 'CONTROL_POINT' && name.unshift((source as Item).properties.OBJECT_COMMAND_NAME); + + name.unshift(displayNamePrefixes[sourceType]); + + if (sourceType === 'SUB_TILE') { + name.unshift((source as Tile).properties.TILE_NAME); + name.unshift(displayNamePrefixes['TILE']); + } + + if (sourceType === 'CONTROL_POINT') { + name.unshift((source as Item).properties.TIED_TO); + name.unshift(displayNamePrefixes['ROUTE']); + } + + return name; +}; + export const convertCamelToSnakeCase = (obj: Record): Record => { const snakeCaseObj: Record = {}; for (const key in obj) { @@ -24,27 +56,45 @@ export const convertCamelToSnakeCase = (obj: Record): Record( +export const formatResponse = ( elasticResponse: estypes.SearchResponse, - requestParams: CommonRequestParameters | ConvertSnakeToCamelCase + requestParams: CommonRequestParameters | ConvertSnakeToCamelCase, + displayNamePrefixes: IApplication['controlObjectDisplayNamePrefixes'] ): ControlResponse => ({ type: 'FeatureCollection', geocoding: { version: process.env.npm_package_version, query: convertCamelToSnakeCase(requestParams as Record), response: { - /* eslint-disable @typescript-eslint/naming-convention */ results_count: elasticResponse.hits.hits.length, max_score: elasticResponse.hits.max_score ?? 0, match_latency_ms: elasticResponse.took, - /* eslint-enable @typescript-eslint/naming-convention */ }, }, features: [ - ...(elasticResponse.hits.hits.map(({ _source: source, _score: score }) => ({ - ...source, - score, - })) as (T & RemoveUnderscore, '_score'>>)[]), + ...(elasticResponse.hits.hits.map(({ _source: source, _score: score, _index: index }) => { + if (source !== undefined) { + const generatedDisplayName = generateDisplayName(source, displayNamePrefixes); + return { + ...source, + properties: { + ...source.properties, + matches: [ + { + layer: source.properties.LAYER_NAME, + source: index, + source_id: [], + }, + ], + name: { + default: [generatedDisplayName.at(LAST_ELEMENT_INDEX)], + display: generatedDisplayName.join(' '), + }, + }, + score, + }; + } + }) as ControlResponse['features']), ], }); diff --git a/src/location/interfaces.ts b/src/location/interfaces.ts index 7bec8721..7611d7a0 100644 --- a/src/location/interfaces.ts +++ b/src/location/interfaces.ts @@ -46,7 +46,7 @@ export interface QueryResult { source?: string; source_id?: string[]; layer?: string; - }; + }[]; name: { [key: string]: string | string[] | undefined; display: string; diff --git a/src/location/utils.ts b/src/location/utils.ts index 87f6f474..2ae9f12c 100644 --- a/src/location/utils.ts +++ b/src/location/utils.ts @@ -137,11 +137,13 @@ export const convertResult = ( geometry: feature?.geo_json, score, properties: { - matches: { - layer: feature?.layer_name, - source: (sources ?? {})[feature?.source ?? ''] ?? feature?.source, - source_id: feature?.source_id.map((id) => id.replace(/(^\{)|(\}$)/g, '')), // TODO: check if to remove this - }, + matches: [ + { + layer: feature?.layer_name, + source: (sources ?? {})[feature?.source ?? ''] ?? feature?.source, + source_id: feature?.source_id.map((id) => id.replace(/(^\{)|(\}$)/g, '')), // TODO: check if to remove this + }, + ], name: { [nameKeys[0]]: new RegExp(mainLanguageRegex).test(feature!.text[0]) ? allNames.shift() : allNames.pop(), [nameKeys[1]]: allNames.pop(), From e608ce8e1d84aa3c2cf9f904005c292aa28cb568 Mon Sep 17 00:00:00 2001 From: Niv Greenstein Date: Tue, 27 Aug 2024 17:09:04 +0300 Subject: [PATCH 154/262] fix: fixed map source elements to lowercase because of elasticsearch analysis search --- src/location/DAL/queries.ts | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/src/location/DAL/queries.ts b/src/location/DAL/queries.ts index 2b9586f9..22b51bf2 100644 --- a/src/location/DAL/queries.ts +++ b/src/location/DAL/queries.ts @@ -105,14 +105,14 @@ export const geotextQuery = ( source?.length && (esQuery.query?.function_score?.query?.bool?.filter as QueryDslQueryContainer[]).push({ terms: { - [SOURCE_FIELD]: source, + [SOURCE_FIELD]: source.map((s) => s.toLowerCase()), }, }); region?.length && (esQuery.query?.function_score?.query?.bool?.filter as QueryDslQueryContainer[]).push({ terms: { - [REGION_FIELD]: region, + [REGION_FIELD]: region.map((r) => r.toLowerCase()), }, }); From b4a56ae15c32df27097fe29c69591e5bbcefab9f Mon Sep 17 00:00:00 2001 From: Niv Greenstein Date: Tue, 27 Aug 2024 17:31:45 +0300 Subject: [PATCH 155/262] fix: changed scopre to be returned under properties --- src/control/utils.ts | 2 +- src/location/interfaces.ts | 6 +++--- src/location/utils.ts | 2 +- 3 files changed, 5 insertions(+), 5 deletions(-) diff --git a/src/control/utils.ts b/src/control/utils.ts index 9c92afca..6e5a189f 100644 --- a/src/control/utils.ts +++ b/src/control/utils.ts @@ -90,8 +90,8 @@ export const formatResponse = ( default: [generatedDisplayName.at(LAST_ELEMENT_INDEX)], display: generatedDisplayName.join(' '), }, + score, }, - score, }; } }) as ControlResponse['features']), diff --git a/src/location/interfaces.ts b/src/location/interfaces.ts index 7611d7a0..aff76c38 100644 --- a/src/location/interfaces.ts +++ b/src/location/interfaces.ts @@ -38,7 +38,7 @@ export interface QueryResult { ConvertCamelToSnakeCase> >; }; - features: ({ + features: { type: string; geometry?: GeoJSON; properties: { @@ -55,8 +55,8 @@ export interface QueryResult { placetype?: string; sub_placetype?: string; regions?: { region: string; sub_region_names: string[] }[]; - }; - } & RemoveUnderscore>)[]; + } & RemoveUnderscore>; + }[]; } /* eslint-enable @typescript-eslint/naming-convention */ diff --git a/src/location/utils.ts b/src/location/utils.ts index 2ae9f12c..9459bdc8 100644 --- a/src/location/utils.ts +++ b/src/location/utils.ts @@ -135,8 +135,8 @@ export const convertResult = ( return { type: 'Feature', geometry: feature?.geo_json, - score, properties: { + score, matches: [ { layer: feature?.layer_name, From 448ac0cbba1a1488097a698b25353df44ef5735b Mon Sep 17 00:00:00 2001 From: Niv Greenstein Date: Tue, 27 Aug 2024 17:32:22 +0300 Subject: [PATCH 156/262] fix: fixed mock objects and utils to return as expected in genericGeocodingResponse --- tests/integration/control/item/mockObjects.ts | 3 +++ .../integration/control/route/mockObjects.ts | 8 +++++-- tests/integration/control/utils.ts | 20 ++++++++++++++--- tests/integration/location/mockObjects.ts | 22 ++++++++++--------- tests/integration/location/utils.ts | 2 +- 5 files changed, 39 insertions(+), 16 deletions(-) diff --git a/tests/integration/control/item/mockObjects.ts b/tests/integration/control/item/mockObjects.ts index de32974d..96a62178 100644 --- a/tests/integration/control/item/mockObjects.ts +++ b/tests/integration/control/item/mockObjects.ts @@ -20,6 +20,7 @@ export const ITEM_1234: Item = { type: 'Polygon', }, properties: { + LAYER_NAME: 'CONTROL.ITEMS', OBJECT_COMMAND_NAME: '1234', TILE_NAME: 'RIT', SUB_TILE_ID: '37', @@ -50,6 +51,7 @@ export const ITEM_1235: Item = { type: 'Polygon', }, properties: { + LAYER_NAME: 'CONTROL.ITEMS', OBJECT_COMMAND_NAME: '1235', TILE_NAME: 'RIT', SUB_TILE_ID: '37', @@ -87,6 +89,7 @@ export const ITEM_1236: Item = { type: 'Polygon', }, properties: { + LAYER_NAME: 'CONTROL.ITEMS', OBJECT_COMMAND_NAME: '1236', TILE_NAME: 'RIT', SUB_TILE_ID: '38', diff --git a/tests/integration/control/route/mockObjects.ts b/tests/integration/control/route/mockObjects.ts index 34b25845..eb60440b 100644 --- a/tests/integration/control/route/mockObjects.ts +++ b/tests/integration/control/route/mockObjects.ts @@ -21,6 +21,7 @@ export const ROUTE_VIA_CAMILLUCCIA_A: Route = { OBJECT_COMMAND_NAME: 'via camillucciaA', ENTITY_HEB: 'route', TYPE: 'ROUTE', + LAYER_NAME: 'CONTROL.ROUTES', }, }; export const ROUTE_VIA_CAMILLUCCIA_B: Route = { @@ -39,6 +40,7 @@ export const ROUTE_VIA_CAMILLUCCIA_B: Route = { OBJECT_COMMAND_NAME: 'via camillucciaB', ENTITY_HEB: 'route', TYPE: 'ROUTE', + LAYER_NAME: 'CONTROL.ROUTES', }, }; @@ -52,7 +54,8 @@ export const CONTROL_POINT_OLIMPIADE_111: Route = { OBJECT_COMMAND_NAME: '111', ENTITY_HEB: 'control point', TIED_TO: 'olimpiade', - TYPE: 'ITEM', + TYPE: 'ITEM' as never, + LAYER_NAME: 'CONTROL_GIL_GDB.CTR_CONTROL_POINT_CROSS_N', }, }; @@ -66,6 +69,7 @@ export const CONTROL_POINT_OLIMPIADE_112: Route = { OBJECT_COMMAND_NAME: '112', ENTITY_HEB: 'control point', TIED_TO: 'olimpiade', - TYPE: 'ITEM', + TYPE: 'ITEM' as never, + LAYER_NAME: 'CONTROL_GIL_GDB.CTR_CONTROL_POINT_CROSS_N', }, }; diff --git a/tests/integration/control/utils.ts b/tests/integration/control/utils.ts index 0c19f48c..e68d4a1e 100644 --- a/tests/integration/control/utils.ts +++ b/tests/integration/control/utils.ts @@ -8,9 +8,23 @@ import { Route } from '../../../src/control/route/models/route'; import { GetTilesQueryParams } from '../../../src/control/tile/controllers/tileController'; import { Tile } from '../../../src/control/tile/models/tile'; -const expectedObjectWithScore = (obj: T, expect: jest.Expect): ControlResponse['features'][number] => ({ - ...obj, - score: expect.any(Number) as number, +const expectedObjectWithScore = (source: T, expect: jest.Expect): ControlResponse['features'][number] => ({ + ...source, + properties: { + ...source.properties, + score: expect.any(Number) as number, + matches: [ + { + layer: expect.any(String) as string, + source: expect.any(String) as string, + source_id: [], + }, + ], + name: { + default: [expect.any(String) as string], + display: expect.any(String) as string, + }, + }, }); const expectedGeocodingElasticResponseMetrics = ( diff --git a/tests/integration/location/mockObjects.ts b/tests/integration/location/mockObjects.ts index ea6060ec..cc6c518d 100644 --- a/tests/integration/location/mockObjects.ts +++ b/tests/integration/location/mockObjects.ts @@ -53,7 +53,7 @@ export const NY_JFK_AIRPORT: MockLocationQueryFeature = { ], }, properties: { - matches: { source: 'OSM', layer: 'osm_airports', source_id: ['03ed6d97-fc81-4340-b68a-11993554eef1'] }, + matches: [{ source: 'OSM', layer: 'osm_airports', source_id: ['03ed6d97-fc81-4340-b68a-11993554eef1'] }], name: { en: ['JFK Airport'], fr: ['Aeropuerto JFK'], @@ -98,7 +98,7 @@ export const NY_POLICE_AIRPORT: MockLocationQueryFeature = { type: 'Polygon', }, properties: { - matches: { source: 'OSM', layer: 'osm_airports', source_id: ['009c6b65-3dcb-4c4f-9f02-d766ebb5d808'] }, + matches: [{ source: 'OSM', layer: 'osm_airports', source_id: ['009c6b65-3dcb-4c4f-9f02-d766ebb5d808'] }], name: { en: ['Nassau County Police Airport'], fr: ['Aeropuerto de la Policía del Condado de Nassau'], @@ -145,7 +145,7 @@ export const LA_AIRPORT: MockLocationQueryFeature = { type: 'Polygon', }, properties: { - matches: { source: 'OSM', layer: 'osm_airports', source_id: ['a4f373ab-b824-41e2-b160-e7729c73bea6'] }, + matches: [{ source: 'OSM', layer: 'osm_airports', source_id: ['a4f373ab-b824-41e2-b160-e7729c73bea6'] }], name: { en: ['Los Angeles International Airport'], fr: ['Aeropuerto Internacional de Los Ángeles'], @@ -193,7 +193,7 @@ export const OSM_LA_PORT: MockLocationQueryFeature = { ], }, properties: { - matches: { source: 'OSM', layer: 'osm_ports', source_id: ['0f36d985-cfbd-4aed-b0cb-ee56600c77f4'] }, + matches: [{ source: 'OSM', layer: 'osm_ports', source_id: ['0f36d985-cfbd-4aed-b0cb-ee56600c77f4'] }], name: { en: ['Port of Los Angeles'], fr: ['Puerto de Los Ángeles'], @@ -240,7 +240,7 @@ export const GOOGLE_LA_PORT: MockLocationQueryFeature = { ], }, properties: { - matches: { source: 'GOOGLE', layer: 'google_ports', source_id: ['1bb11f54-939e-457b-bf68-a3920ccf629c'] }, + matches: [{ source: 'GOOGLE', layer: 'google_ports', source_id: ['1bb11f54-939e-457b-bf68-a3920ccf629c'] }], name: { en: ['Port of Los Angeles'], fr: ['Puerto de Los Ángeles'], @@ -275,11 +275,13 @@ export const LA_WHITE_POINT_SCHOOL: MockLocationQueryFeature = { type: 'Polygon', }, properties: { - matches: { - source: 'OSM', - layer: 'osm_schools', - source_id: ['1a5b981b-bb0e-44dd-b9e2-424b92f2de49'], - }, + matches: [ + { + source: 'OSM', + layer: 'osm_schools', + source_id: ['1a5b981b-bb0e-44dd-b9e2-424b92f2de49'], + }, + ], name: { en: ['White Point Elementary School'], fr: ['Escuela Primaria White Point'], diff --git a/tests/integration/location/utils.ts b/tests/integration/location/utils.ts index 1a04cc08..4c5269da 100644 --- a/tests/integration/location/utils.ts +++ b/tests/integration/location/utils.ts @@ -6,8 +6,8 @@ const expectedObjectWithScore = (obj: MockLocationQueryFeature, expect: jest.Exp ...obj, properties: { ...obj.properties, + score: expect.any(Number) as number, }, - score: expect.any(Number) as number, }); const expectedGeocodingElasticResponseMetrics = ( From 46d0d06d96fda4db9f884d97ec6b92ba47a5ea4b Mon Sep 17 00:00:00 2001 From: Niv Greenstein Date: Tue, 27 Aug 2024 17:47:16 +0300 Subject: [PATCH 157/262] fix: fixed location tests --- tests/integration/location/location.spec.ts | 37 ++++++++++++++++++--- tests/integration/location/mockObjects.ts | 14 +++++--- 2 files changed, 41 insertions(+), 10 deletions(-) diff --git a/tests/integration/location/location.spec.ts b/tests/integration/location/location.spec.ts index a30ce7d4..77427c8c 100644 --- a/tests/integration/location/location.spec.ts +++ b/tests/integration/location/location.spec.ts @@ -67,11 +67,26 @@ describe('/search/control/tiles', function () { expectedResponse( { ...requestParams, + }, + { place_types: ['transportation'], sub_place_types: ['airport'], hierarchies: [], }, - [NY_JFK_AIRPORT, NY_POLICE_AIRPORT, LA_AIRPORT], + [ + { + ...NY_JFK_AIRPORT, + properties: { + ...NY_JFK_AIRPORT.properties, + name: { + ...NY_JFK_AIRPORT.properties.name, + display: expect.stringContaining('JFK') as string, + }, + }, + }, + NY_POLICE_AIRPORT, + LA_AIRPORT, + ], expect ) ); @@ -103,13 +118,25 @@ describe('/search/control/tiles', function () { expect(response.body).toMatchObject( expectedResponse( + requestParams, { - ...requestParams, place_types: ['transportation'], sub_place_types: ['airport'], hierarchies: [], }, - [NY_JFK_AIRPORT, NY_POLICE_AIRPORT], + [ + { + ...NY_JFK_AIRPORT, + properties: { + ...NY_JFK_AIRPORT.properties, + name: { + ...NY_JFK_AIRPORT.properties.name, + display: expect.stringContaining('JFK') as string, + }, + }, + }, + NY_POLICE_AIRPORT, + ], expect ) ); @@ -336,8 +363,8 @@ describe('/search/control/tiles', function () { expect(response.body).toMatchObject( expectedResponse( + requestParams, { - ...requestParams, place_types: ['transportation'], sub_place_types: ['port'], hierarchies: [], @@ -368,8 +395,8 @@ describe('/search/control/tiles', function () { expect(response.body).toMatchObject( expectedResponse( + requestParams, { - ...requestParams, place_types: ['education'], sub_place_types: ['school'], hierarchies: [], diff --git a/tests/integration/location/mockObjects.ts b/tests/integration/location/mockObjects.ts index cbaf51c3..9195e135 100644 --- a/tests/integration/location/mockObjects.ts +++ b/tests/integration/location/mockObjects.ts @@ -314,9 +314,13 @@ export const PARIS_WI_SCHOOL: MockLocationQueryFeature = { type: 'Polygon', }, properties: { - source: 'OSM', - layer: 'osm_schools', - source_id: ['dc02a3f9-156a-4f61-85bd-fd040cd322a3'], + matches: [ + { + source: 'OSM', + layer: 'osm_schools', + source_id: ['dc02a3f9-156a-4f61-85bd-fd040cd322a3'], + }, + ], name: { en: ['Wi School Paris 9'], fr: ['Ecole Wi Paris 9'], @@ -328,7 +332,7 @@ export const PARIS_WI_SCHOOL: MockLocationQueryFeature = { regions: [ { region: 'FRANCE', - sub_regions: ['Paris'], + sub_region_names: ['Paris'], }, ], }, @@ -402,4 +406,4 @@ export const PARIS_HIERRARCHY: HierarchySearchHit = { region: 'FRANCE', text: 'Paris', weight: 1.1, -}; \ No newline at end of file +}; From aaf433e6064168e928cfb970d07f5a80cfceba9e Mon Sep 17 00:00:00 2001 From: Niv Greenstein Date: Tue, 27 Aug 2024 17:49:09 +0300 Subject: [PATCH 158/262] delete: removed removed parse geo string| geojson and deprecated code --- src/location/utils.ts | 37 ++++++++----------------------------- 1 file changed, 8 insertions(+), 29 deletions(-) diff --git a/src/location/utils.ts b/src/location/utils.ts index 9459bdc8..04a87db5 100644 --- a/src/location/utils.ts +++ b/src/location/utils.ts @@ -1,13 +1,12 @@ // import fetch, { Response } from "node-fetch-commonjs"; -import { GeoJSON, Geometry, Point } from 'geojson'; +import { Geometry, Point } from 'geojson'; import { SearchResponse } from '@elastic/elasticsearch/lib/api/types'; import { StatusCodes } from 'http-status-codes'; import axios, { AxiosError, AxiosResponse as Response } from 'axios'; import { InternalServerError } from '../common/errors'; import { GeoContext, IApplication } from '../common/interfaces'; -import { ConvertCamelToSnakeCase, convertUTMToWgs84 } from '../common/utils'; -import { convertCamelToSnakeCase } from '../control/utils'; -import { BBOX_LENGTH, GetGeotextSearchParams, POINT_LENGTH, QueryResult, TextSearchParams } from './interfaces'; +import { convertUTMToWgs84 } from '../common/utils'; +import { QueryResult, TextSearchParams } from './interfaces'; import { TextSearchHit } from './models/elasticsearchHits'; import { generateDisplayName } from './parsing'; @@ -55,36 +54,16 @@ export const fetchNLPService = async (endpoint: string, requestData: object): export const cleanQuery = (query: string): string[] => query.replace(FIND_QUOTES, '').split(FIND_SPECIAL); -export const parseGeo = (input: string | GeoJSON | GeoContext): Geometry | undefined => { - //TODO: remove string | GeoJson as accepted types +export const parseGeo = (input: GeoContext): Geometry | undefined => { //TODO: Add geojson validation //TODO: refactor this function - if (typeof input === 'string') { - const splitted = input.split(','); - const converted = splitted.map(Number); - - if (converted.findIndex((x) => isNaN(x)) < 0) { - switch (splitted.length) { - case POINT_LENGTH: - // Point - return parsePoint(splitted); - case BBOX_LENGTH: - //BBOX - return parseBbox(splitted); - default: - return undefined; - } - } - } else if (input.bbox !== undefined) { + if (input.bbox !== undefined) { return parseBbox(input.bbox); } else if ( - ((input as GeoContext).x !== undefined && - (input as GeoContext).y !== undefined && - (input as GeoContext).zone !== undefined && - (input as GeoContext).zone !== undefined) || - ((input as GeoContext).lon !== undefined && (input as GeoContext).lat !== undefined) + (input.x !== undefined && input.y !== undefined && input.zone !== undefined && input.zone !== undefined) || + (input.lon !== undefined && input.lat !== undefined) ) { - const { x, y, zone, radius } = input as GeoContext; + const { x, y, zone, radius } = input; const { lon, lat } = x && y && zone ? convertUTMToWgs84(x, y, zone) : (input as Required>); return { type: 'Circle', coordinates: (parsePoint([lon, lat]) as Point).coordinates, radius: `${radius ?? ''}` } as unknown as Geometry; From 24b4602d170f62d30364d1e4719bd49729a672d1 Mon Sep 17 00:00:00 2001 From: Niv Greenstein Date: Tue, 27 Aug 2024 19:02:54 +0300 Subject: [PATCH 159/262] feat: upload new default and test configs --- config/default.json | 32 ++++++++++++++-------- config/test.json | 13 ++++++--- src/latLon/controllers/latLonController.ts | 3 ++ src/location/utils.ts | 1 - 4 files changed, 32 insertions(+), 17 deletions(-) diff --git a/config/default.json b/config/default.json index cf9186b0..470bddec 100644 --- a/config/default.json +++ b/config/default.json @@ -30,20 +30,19 @@ "control": { "node": "http://localhost:9200", "auth": { - "username": "control", - "password": "password" + "username": "elastic", + "password": "changeme" }, "requestTimeout": 60000, "properties": { - "index": "control_index", - "defaultResponseLimit": 3 + "index": "control_gil_v5" } }, "geotext": { "node": "http://localhost:9200", "auth": { - "username": "geotext", - "password": "password" + "username": "elastic", + "password": "changeme" }, "requestTimeout": 60000, "properties": { @@ -52,7 +51,6 @@ "placetypes": "placetypes_index", "hierarchies": "hierarchies_index" }, - "defaultResponseLimit": 3, "textTermLanguage": "en" } } @@ -70,12 +68,12 @@ "cert": "" }, "database": "postgres", - "schema": "geocoder" + "schema": "geocoding" } }, "application": { "services": { - "tokenTypesUrl": "http://NLP_ANALYSES" + "tokenTypesUrl": "http://localhost:5001/NLP_ANALYSES" }, "cronLoadTileLatLonDataPattern": "0 * * * *", "elasticQueryBoosts": { @@ -86,10 +84,20 @@ "viewbox": 1.1 }, "sources": { - "SOURCE_A": "a", - "SOURCE_B": "b" + "OSM": "OSM", + "GOOGLE": "GOOGLE" + }, + "regions": { + "USA": ["New York", "Los Angeles"], + "FRANCE": ["Paris"] + }, + "controlObjectDisplayNamePrefixes": { + "TILE": "Tile", + "SUB_TILE": "Sub Tile", + "ROUTE": "Route", + "ITEM": "Item", + "CONTROL_POINT": "Control Point" }, - "regions": {}, "nameTranslationsKeys": ["en", "fr"], "mainLanguageRegex": "[a-zA-Z]" } diff --git a/config/test.json b/config/test.json index 55384c41..85008963 100644 --- a/config/test.json +++ b/config/test.json @@ -9,8 +9,7 @@ }, "requestTimeout": 60000, "properties": { - "index": "control_index", - "defaultResponseLimit": 3 + "index": "control_gil_v5" } }, "geotext": { @@ -26,7 +25,6 @@ "placetypes": "placetypes_index", "hierarchies": "hierarchies_index" }, - "defaultResponseLimit": 3, "textTermLanguage": "en" } } @@ -44,7 +42,7 @@ "cert": "" }, "database": "postgres", - "schema": "geocoder" + "schema": "geocoding" } }, "application": { @@ -67,6 +65,13 @@ "USA": ["New York", "Los Angeles"], "FRANCE": ["Paris"] }, + "controlObjectDisplayNamePrefixes": { + "TILE": "Tile", + "SUB_TILE": "Sub Tile", + "ROUTE": "Route", + "ITEM": "Item", + "CONTROL_POINT": "Control Point" + }, "nameTranslationsKeys": ["en", "fr"], "mainLanguageRegex": "[a-zA-Z]" } diff --git a/src/latLon/controllers/latLonController.ts b/src/latLon/controllers/latLonController.ts index a62d7716..e90b607c 100644 --- a/src/latLon/controllers/latLonController.ts +++ b/src/latLon/controllers/latLonController.ts @@ -73,6 +73,9 @@ export class LatLonController { tileName, subTileNumber: sub_tile_number, }); + + // TODO: REMOVE TS IGNORE + //@ts-ignore return res.status(httpStatus.OK).json(response); } catch (error: unknown) { this.logger.warn('latLonController.tileToLatLon Error:', error); diff --git a/src/location/utils.ts b/src/location/utils.ts index 04a87db5..bc6224b9 100644 --- a/src/location/utils.ts +++ b/src/location/utils.ts @@ -68,7 +68,6 @@ export const parseGeo = (input: GeoContext): Geometry | undefined => { return { type: 'Circle', coordinates: (parsePoint([lon, lat]) as Point).coordinates, radius: `${radius ?? ''}` } as unknown as Geometry; } - return input as Geometry; }; /* eslint-disable @typescript-eslint/naming-convention */ From 92171080d18ec839453425719418fc2a99057c45 Mon Sep 17 00:00:00 2001 From: Niv Greenstein Date: Wed, 28 Aug 2024 15:23:38 +0300 Subject: [PATCH 160/262] feat(config): added NAME_TRANSLATION_KEYS env --- config/custom-environment-variables.json | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/config/custom-environment-variables.json b/config/custom-environment-variables.json index ed72d6e5..aaf6aec0 100644 --- a/config/custom-environment-variables.json +++ b/config/custom-environment-variables.json @@ -104,6 +104,10 @@ "__name": "NAME_TRANSLATION_KEYS", "__format": "json" }, + "controlObjectDisplayNamePrefixes": { + "__name": "NAME_TRANSLATION_KEYS", + "__format": "json" + }, "mainLanguageRegex": "MAIN_LANGUAGE_REGEX" } } From a1953300af8ec85a5f9d4f3626f1deb4d37d4edd Mon Sep 17 00:00:00 2001 From: Niv Greenstein Date: Wed, 28 Aug 2024 15:24:02 +0300 Subject: [PATCH 161/262] delete(tests/latlon): removed old tests --- .../latLon/helpers/requestSender.ts | 53 ------ tests/integration/latLon/latLon.spec.ts | 169 ------------------ 2 files changed, 222 deletions(-) delete mode 100644 tests/integration/latLon/helpers/requestSender.ts delete mode 100644 tests/integration/latLon/latLon.spec.ts diff --git a/tests/integration/latLon/helpers/requestSender.ts b/tests/integration/latLon/helpers/requestSender.ts deleted file mode 100644 index 8ace3344..00000000 --- a/tests/integration/latLon/helpers/requestSender.ts +++ /dev/null @@ -1,53 +0,0 @@ -import * as supertest from 'supertest'; -import { - GetLatLonToTileQueryParams, - GetTileToLatLonQueryParams, - GetLatLonToMgrsQueryParams, - GetMgrsToLatLonQueryParams, -} from '../../../../src/latLon/controllers/latLonController'; - -const PREFIX = '/v1/lookup'; - -export class LatLonRequestSender { - public constructor(private readonly app: Express.Application) {} - - public async getLatlonToTile(queryParams?: GetLatLonToTileQueryParams): Promise { - return supertest - .agent(this.app) - .get(`${PREFIX}/latlonToTile`) - .set('Content-Type', 'application/json') - .set('X-API-Key', 'abc123') - .set('x-user-id', 'abc123') - .query(queryParams ?? {}); - } - - public async getTileToLatLon(queryParams?: GetTileToLatLonQueryParams): Promise { - return supertest - .agent(this.app) - .get(`${PREFIX}/tileToLatLon`) - .set('Content-Type', 'application/json') - .set('X-API-Key', 'abc123') - .set('x-user-id', 'abc123') - .query(queryParams ?? {}); - } - - public async getLatlonToMgrs(queryParams?: GetLatLonToMgrsQueryParams): Promise { - return supertest - .agent(this.app) - .get(`${PREFIX}/latlonToMgrs`) - .set('Content-Type', 'application/json') - .set('X-API-Key', 'abc123') - .set('x-user-id', 'abc123') - .query(queryParams ?? {}); - } - - public async getMgrsToLatlon(queryParams?: GetMgrsToLatLonQueryParams): Promise { - return supertest - .agent(this.app) - .get(`${PREFIX}/mgrsToLatLon`) - .set('Content-Type', 'application/json') - .set('X-API-Key', 'abc123') - .set('x-user-id', 'abc123') - .query(queryParams ?? {}); - } -} diff --git a/tests/integration/latLon/latLon.spec.ts b/tests/integration/latLon/latLon.spec.ts deleted file mode 100644 index 546ec435..00000000 --- a/tests/integration/latLon/latLon.spec.ts +++ /dev/null @@ -1,169 +0,0 @@ -// /* eslint-disable @typescript-eslint/unbound-method */ -// /* eslint-disable @typescript-eslint/naming-convention */ -// import jsLogger from '@map-colonies/js-logger'; -// import { trace } from '@opentelemetry/api'; -// import httpStatusCodes from 'http-status-codes'; -// import { getApp } from '../../../src/app'; -// import { SERVICES } from '../../../src/common/constants'; -// import { LatLonRequestSender } from './helpers/requestSender'; - -// describe('/latLon', function () { -// let requestSender: LatLonRequestSender; - -// beforeEach(async function () { -// const app = await getApp({ -// override: [ -// { token: SERVICES.LOGGER, provider: { useValue: jsLogger({ enabled: false }) } }, -// { token: SERVICES.TRACER, provider: { useValue: trace.getTracer('testTracer') } }, -// ], -// useChild: true, -// }); -// requestSender = new LatLonRequestSender(app.app); -// }, 20000); - -// describe('Happy Path', function () { -// it('should return 200 status code and lat-lon from mgrs', async function () { -// const response = await requestSender.getMgrsToLatlon({ mgrs: '18TWL8565011369' }); - -// expect(response.status).toBe(httpStatusCodes.OK); -// expect(response).toSatisfyApiSpec(); -// expect(response.body).toMatchObject({ -// lat: 40.74882151233783, -// lon: -73.98543192220956, -// }); -// }); - -// it('should return 200 status code and mgrs from lat-lon', async function () { -// const reponse = await requestSender.getLatlonToMgrs({ -// lat: 40.74882151233783, -// lon: -73.98543192220956, -// }); - -// expect(reponse.status).toBe(httpStatusCodes.OK); -// expect(reponse).toSatisfyApiSpec(); -// expect(reponse.body).toMatchObject({ -// mgrs: '18TWL8565011369', -// }); -// }); - -// it('should return 200 status code and tile from lat-lon', async function () { -// const reponse = await requestSender.getLatlonToTile({ -// lat: 52.57326537485767, -// lon: 12.948781146422107, -// }); - -// expect(reponse.status).toBe(httpStatusCodes.OK); -// expect(reponse).toSatisfyApiSpec(); -// expect(reponse.body).toMatchObject({ -// tileName: 'BRN', -// subTileNumber: [0, 0, 0], -// }); -// }); - -// it('should return 200 status code and lat-lon from tile', async function () { -// const reponse = await requestSender.getTileToLatLon({ -// tile: 'BRN', -// sub_tile_number: [10, 10, 10], -// }); - -// expect(reponse.status).toBe(httpStatusCodes.OK); -// expect(reponse).toSatisfyApiSpec(); -// expect(reponse.body).toMatchObject({ -// type: 'FeatureCollection', -// features: [ -// { -// geometry: { -// coordinates: [ -// [ -// [12.953293384350397, 52.512399536846765], -// [12.953440643865289, 52.512402084451686], -// [12.953436468347887, 52.51249192878939], -// [12.95328920853307, 52.512489381176245], -// [12.953293384350397, 52.512399536846765], -// ], -// ], -// type: 'Polygon', -// }, -// properties: { -// TYPE: 'TILE', -// SUB_TILE_NUMBER: [10, 10, 10], -// TILE_NAME: 'BRN', -// }, -// }, -// ], -// }); -// }); -// }); -// describe('Bad Path', function () { -// test.each<[keyof typeof requestSender, string[]]>([ -// ['getLatlonToMgrs', ['lat', 'lon']], -// ['getMgrsToLatlon', ['mgrs']], -// ['getLatlonToTile', ['lat', 'lon']], -// ['getTileToLatLon', ['tile', 'sub_tile_number']], -// ])('should return 400 and message for missing required parameters', async (request, missingProperties) => { -// const message = 'request/query must have required property'; -// const response = await requestSender[request](); - -// expect(response.status).toBe(httpStatusCodes.BAD_REQUEST); -// expect(response.body).toMatchObject({ message: missingProperties.map((txt) => `${message} '${txt}'`).join(', ') }); -// expect(response).toSatisfyApiSpec(); -// }); - -// test.each<[keyof typeof requestSender]>([['getLatlonToMgrs'], ['getMgrsToLatlon'], ['getLatlonToTile'], ['getTileToLatLon']])( -// 'should return 400 and message unknown query parameter', -// async (request) => { -// const parameter = 'test1234'; -// const message = `Unknown query parameter '${parameter}'`; - -// const response = await requestSender[request]({ [parameter]: parameter } as never); - -// expect(response.status).toBe(httpStatusCodes.BAD_REQUEST); -// expect(response.body).toMatchObject({ message }); -// expect(response).toSatisfyApiSpec(); -// } -// ); - -// it('should return 400 and check that all numbers are positive', async function () { -// const message = "Invalid tile, check that 'tileName' and 'subTileNumber' exists and subTileNumber is array of size 3 with positive integers"; - -// for (let i = 0; i < 3; i++) { -// const arr: number[] = [10, 10, 10]; -// arr[i] = 0; - -// const response = await requestSender.getTileToLatLon({ -// tile: 'BRN', -// sub_tile_number: arr, -// }); - -// expect(response.status).toBe(httpStatusCodes.BAD_REQUEST); -// expect(response.body).toMatchObject({ -// message, -// }); -// expect(response).toSatisfyApiSpec(); -// } -// }); - -// it('should return 400 for lat-lon that outside the grid extent', async function () { -// const response = await requestSender.getLatlonToTile({ lat: 1, lon: 1 }); - -// expect(response.status).toBe(httpStatusCodes.BAD_REQUEST); -// expect(response.body).toMatchObject({ -// message: 'The coordinate is outside the grid extent', -// }); -// expect(response).toSatisfyApiSpec(); -// }); - -// it('should return 400 for tile not found', async function () { -// const response = await requestSender.getTileToLatLon({ tile: 'XXX', sub_tile_number: [10, 10, 10] }); - -// expect(response.status).toBe(httpStatusCodes.BAD_REQUEST); -// expect(response.body).toMatchObject({ -// message: 'Tile not found', -// }); -// expect(response).toSatisfyApiSpec(); -// }); -// }); -// describe('Sad Path', function () { -// // All requests with status code 4XX-5XX -// }); -// }); From 237fb95e4ec7441338c15b059d55963162ff5cde Mon Sep 17 00:00:00 2001 From: Niv Greenstein Date: Wed, 28 Aug 2024 15:25:51 +0300 Subject: [PATCH 162/262] feat(location/utils): added rejectUnauthorized: false to axios --- src/location/utils.ts | 8 ++++++-- 1 file changed, 6 insertions(+), 2 deletions(-) diff --git a/src/location/utils.ts b/src/location/utils.ts index bc6224b9..2c96645b 100644 --- a/src/location/utils.ts +++ b/src/location/utils.ts @@ -1,4 +1,4 @@ -// import fetch, { Response } from "node-fetch-commonjs"; +import https from 'https'; import { Geometry, Point } from 'geojson'; import { SearchResponse } from '@elastic/elasticsearch/lib/api/types'; import { StatusCodes } from 'http-status-codes'; @@ -14,6 +14,10 @@ const FIND_QUOTES = /["']/g; const FIND_SPECIAL = /[`!@#$%^&*()_\-+=|\\/,.<>:[\]{}\n\t\r\s;؛]+/g; +const axiosInstance = axios.create({ + httpsAgent: new https.Agent({ rejectUnauthorized: false }), +}); + const parsePoint = (split: string[] | number[]): Geometry => ({ type: 'Point', coordinates: split.map(Number), @@ -39,7 +43,7 @@ export const fetchNLPService = async (endpoint: string, requestData: object): let res: Response | null = null, data: T[] | undefined | null = null; try { - res = await axios.post(endpoint, requestData); + res = await axiosInstance.post(endpoint, requestData); } catch (err: unknown) { throw new InternalServerError(`NLP analyser is not available - ${(err as AxiosError).message}`); } From 4fa68b94d8b7039fd89c04bf427d2a85eaec40ad Mon Sep 17 00:00:00 2001 From: Niv Greenstein Date: Wed, 28 Aug 2024 17:43:22 +0300 Subject: [PATCH 163/262] fixed env for controlObjectDisplayNamePrefixes. changed version --- config/custom-environment-variables.json | 2 +- package.json | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/config/custom-environment-variables.json b/config/custom-environment-variables.json index aaf6aec0..d596858d 100644 --- a/config/custom-environment-variables.json +++ b/config/custom-environment-variables.json @@ -105,7 +105,7 @@ "__format": "json" }, "controlObjectDisplayNamePrefixes": { - "__name": "NAME_TRANSLATION_KEYS", + "__name": "CONTROL_DISPLAY_KEY_TRANSLATIONS", "__format": "json" }, "mainLanguageRegex": "MAIN_LANGUAGE_REGEX" diff --git a/package.json b/package.json index eaf1c718..23fc3403 100644 --- a/package.json +++ b/package.json @@ -1,6 +1,6 @@ { "name": "geocoding", - "version": "0.1.0-prealpha", + "version": "0.1.0-prealpha.2", "description": "Geocoding service for MapColonies", "main": "./src/index.ts", "scripts": { From 3536e6e38e50029029aef6792056521ac3b9f4da Mon Sep 17 00:00:00 2001 From: NatalieShaked Date: Thu, 29 Aug 2024 16:13:03 +0300 Subject: [PATCH 164/262] fix: fixed redis configuration --- config/custom-environment-variables.json | 17 +- config/default.json | 11 +- config/test.json | 11 +- helm/templates/configmap.yaml | 13 ++ helm/templates/deployment.yaml | 10 ++ helm/values.yaml | 13 ++ package-lock.json | 182 +++++++++++++++++++++ package.json | 3 + src/common/constants.ts | 2 +- src/common/errors.ts | 2 + src/common/interfaces.ts | 8 + src/common/redis/domainFieldsRepository.ts | 5 - src/common/redis/index.ts | 79 ++++++--- src/common/redis/keys.ts | 5 - src/common/redis/redisManager.ts | 31 ---- src/common/utils.ts | 14 ++ src/containerConfig.ts | 60 +++---- src/serverBuilder.ts | 8 +- 18 files changed, 354 insertions(+), 120 deletions(-) delete mode 100644 src/common/redis/domainFieldsRepository.ts delete mode 100644 src/common/redis/keys.ts delete mode 100644 src/common/redis/redisManager.ts diff --git a/config/custom-environment-variables.json b/config/custom-environment-variables.json index c6ef51dc..d341e4ca 100644 --- a/config/custom-environment-variables.json +++ b/config/custom-environment-variables.json @@ -93,14 +93,19 @@ "__name": "REDIS_PORT", "__format": "number" }, - "db": { - "__name": "REDIS_INDEX", - "__format": "number" - }, "username": "REDIS_USERNAME", "password": "REDIS_PASSWORD", - "maxRetriesPerRequest": { - "__name": "REDIS_MAX_RETRIES_PER_REQUEST", + "enableSslAuth": { + "__name": "REDIS_ENABLE_SSL_AUTH", + "__format": "boolean" + }, + "sslPaths": { + "ca": "REDIS_CA_PATH", + "key": "REDIS_KEY_PATH", + "cert": "REDIS_CERT_PATH" + }, + "database": { + "__name": "REDIS_DATABASE", "__format": "number" } } diff --git a/config/default.json b/config/default.json index 4e0dd4ad..1a507876 100644 --- a/config/default.json +++ b/config/default.json @@ -75,8 +75,15 @@ "redis": { "host": "localhost", "port": 6379, - "db": 0, - "maxRetriesPerRequest": 1 + "username": "", + "password": "", + "enableSslAuth": false, + "sslPaths": { + "ca": "", + "key": "", + "cert": "" + }, + "database": 0 } }, "application": { diff --git a/config/test.json b/config/test.json index f267cd35..dc66d247 100644 --- a/config/test.json +++ b/config/test.json @@ -49,8 +49,15 @@ "redis": { "host": "localhost", "port": 6379, - "db": 0, - "maxRetriesPerRequest": 1 + "username": "", + "password": "", + "enableSslAuth": false, + "sslPaths": { + "ca": "", + "key": "", + "cert": "" + }, + "database": 0 } }, "application": { diff --git a/helm/templates/configmap.yaml b/helm/templates/configmap.yaml index 69202f7f..b83572d1 100644 --- a/helm/templates/configmap.yaml +++ b/helm/templates/configmap.yaml @@ -18,5 +18,18 @@ data: TELEMETRY_METRICS_ENABLED: 'true' TELEMETRY_METRICS_URL: {{ $metricsUrl }} {{ end }} + {{- with .Values.redisConfig }} + REDIS_HOST: {{ .host }} + REDIS_DATABASE: {{ .database | quote}} + REDIS_PORT: {{ .port | quote }} + {{- if .sslAuth.enabled }} + REDIS_ENABLE_SSL_AUTH: "true" + REDIS_CERT_PATH: /tmp/certs/{{ .sslAuth.certFileName }} + REDIS_KEY_PATH: /tmp/certs/{{ .sslAuth.keyFileName }} + REDIS_CA_PATH: /tmp/certs/{{ .sslAuth.caFileName }} + {{- else }} + REDIS_ENABLE_SSL_AUTH: "false" + {{- end }} + {{- end }} npm_config_cache: /tmp/ {{- end }} diff --git a/helm/templates/deployment.yaml b/helm/templates/deployment.yaml index 007efa2a..5484cfd7 100644 --- a/helm/templates/deployment.yaml +++ b/helm/templates/deployment.yaml @@ -53,6 +53,11 @@ spec: name: root-ca subPath: {{ quote .Values.caKey }} {{- end }} + {{- if .Values.redisConfig.sslAuth.enabled }} + - name: cert-conf + mountPath: /tmp/certs + readOnly: true + {{- end }} {{- if .Values.extraVolumeMounts -}} {{ toYaml .Values.extraVolumeMounts | nindent 12 }} {{- end }} @@ -109,6 +114,11 @@ spec: secret: secretName: {{ .Values.caSecretName }} {{- end }} + {{- if .Values.redisConfig.sslAuth.enabled }} + - name: cert-conf + secret: + secretName: {{ .Values.redisConfig.sslAuth.secretName }} + {{- end }} {{- if .Values.extraVolumes -}} {{ tpl (toYaml .Values.extraVolumes) . | nindent 8 }} {{- end }} diff --git a/helm/values.yaml b/helm/values.yaml index 25fd7688..3adb3d5b 100644 --- a/helm/values.yaml +++ b/helm/values.yaml @@ -71,6 +71,19 @@ env: enabled: false url: http://localhost:55681/v1/metrics +redisConfig: + host: localhost + username: "" + password: "" + database: 0 + port: 6379 + sslAuth: + enabled: false + secretName: secret-name + certFileName: postgresql.crt + keyFileName: postgresql.key + caFileName: root.crt + resources: enabled: true value: diff --git a/package-lock.json b/package-lock.json index 527fc462..1b6ec40d 100644 --- a/package-lock.json +++ b/package-lock.json @@ -12,6 +12,8 @@ "dependencies": { "@elastic/elasticsearch": "^8.13.1", "@godaddy/terminus": "^4.12.1", + "@map-colonies/cleanup-registry": "^1.1.0", + "@map-colonies/detiler-common": "^1.0.0", "@map-colonies/error-express-handler": "^2.1.0", "@map-colonies/express-access-log-middleware": "^2.0.1", "@map-colonies/js-logger": "^1.0.1", @@ -36,6 +38,7 @@ "node-fetch-commonjs": "^3.3.2", "pg": "^8.11.5", "proj4": "^2.11.0", + "redis": "^4.7.0", "reflect-metadata": "^0.1.13", "tsyringe": "^4.8.0", "typeorm": "^0.3.20", @@ -3790,6 +3793,20 @@ "resolved": "https://registry.npmjs.org/@jsdevtools/ono/-/ono-7.1.3.tgz", "integrity": "sha512-4JQNk+3mVzK3xh2rqd6RB4J46qUR19azEHBneZyTZM+c456qOrbbM/5xcR8huNCCcbVt7+UmizG6GuUvPvKUYg==" }, + "node_modules/@map-colonies/cleanup-registry": { + "version": "1.1.0", + "resolved": "https://registry.npmjs.org/@map-colonies/cleanup-registry/-/cleanup-registry-1.1.0.tgz", + "integrity": "sha512-/lhIGklWPZSY37JwzhFJEtBlqwXDRhHSeCBwpPaGMxpycpt5ZRIVQxUt6Og4mt6c5GoRoX9dZYHY0qV3UMGvtQ==", + "dependencies": { + "nanoid": "^3.3.4", + "tiny-typed-emitter": "^2.1.0" + } + }, + "node_modules/@map-colonies/detiler-common": { + "version": "1.0.0", + "resolved": "https://registry.npmjs.org/@map-colonies/detiler-common/-/detiler-common-1.0.0.tgz", + "integrity": "sha512-5gxowEjHBD9CcgqonYQGyarIXRElYnhO4qO/ffxN3IFieWyUWUuKFhc+U6rnvy53TN0UorPfTuOrCO85W9ZsKw==" + }, "node_modules/@map-colonies/error-express-handler": { "version": "2.1.0", "resolved": "https://registry.npmjs.org/@map-colonies/error-express-handler/-/error-express-handler-2.1.0.tgz", @@ -6288,6 +6305,59 @@ "resolved": "https://registry.npmjs.org/@protobufjs/utf8/-/utf8-1.1.0.tgz", "integrity": "sha512-Vvn3zZrhQZkkBE8LSuW3em98c0FwgO4nxzv6OdSxPKJIEKY2bGbHn+mhGIPerzI4twdxaP8/0+06HBpwf345Lw==" }, + "node_modules/@redis/bloom": { + "version": "1.2.0", + "resolved": "https://registry.npmjs.org/@redis/bloom/-/bloom-1.2.0.tgz", + "integrity": "sha512-HG2DFjYKbpNmVXsa0keLHp/3leGJz1mjh09f2RLGGLQZzSHpkmZWuwJbAvo3QcRY8p80m5+ZdXZdYOSBLlp7Cg==", + "peerDependencies": { + "@redis/client": "^1.0.0" + } + }, + "node_modules/@redis/client": { + "version": "1.6.0", + "resolved": "https://registry.npmjs.org/@redis/client/-/client-1.6.0.tgz", + "integrity": "sha512-aR0uffYI700OEEH4gYnitAnv3vzVGXCFvYfdpu/CJKvk4pHfLPEy/JSZyrpQ+15WhXe1yJRXLtfQ84s4mEXnPg==", + "dependencies": { + "cluster-key-slot": "1.1.2", + "generic-pool": "3.9.0", + "yallist": "4.0.0" + }, + "engines": { + "node": ">=14" + } + }, + "node_modules/@redis/graph": { + "version": "1.1.1", + "resolved": "https://registry.npmjs.org/@redis/graph/-/graph-1.1.1.tgz", + "integrity": "sha512-FEMTcTHZozZciLRl6GiiIB4zGm5z5F3F6a6FZCyrfxdKOhFlGkiAqlexWMBzCi4DcRoyiOsuLfW+cjlGWyExOw==", + "peerDependencies": { + "@redis/client": "^1.0.0" + } + }, + "node_modules/@redis/json": { + "version": "1.0.7", + "resolved": "https://registry.npmjs.org/@redis/json/-/json-1.0.7.tgz", + "integrity": "sha512-6UyXfjVaTBTJtKNG4/9Z8PSpKE6XgSyEb8iwaqDcy+uKrd/DGYHTWkUdnQDyzm727V7p21WUMhsqz5oy65kPcQ==", + "peerDependencies": { + "@redis/client": "^1.0.0" + } + }, + "node_modules/@redis/search": { + "version": "1.2.0", + "resolved": "https://registry.npmjs.org/@redis/search/-/search-1.2.0.tgz", + "integrity": "sha512-tYoDBbtqOVigEDMAcTGsRlMycIIjwMCgD8eR2t0NANeQmgK/lvxNAvYyb6bZDD4frHRhIHkJu2TBRvB0ERkOmw==", + "peerDependencies": { + "@redis/client": "^1.0.0" + } + }, + "node_modules/@redis/time-series": { + "version": "1.1.0", + "resolved": "https://registry.npmjs.org/@redis/time-series/-/time-series-1.1.0.tgz", + "integrity": "sha512-c1Q99M5ljsIuc4YdaCwfUEXsofakb9c8+Zse2qxTadu8TalLXuAESzLvFAvNVbkmSlvlzIQOLpBCmWI9wTOt+g==", + "peerDependencies": { + "@redis/client": "^1.0.0" + } + }, "node_modules/@redocly/ajv": { "version": "8.6.4", "resolved": "https://registry.npmjs.org/@redocly/ajv/-/ajv-8.6.4.tgz", @@ -15573,6 +15643,23 @@ "thenify-all": "^1.0.0" } }, + "node_modules/nanoid": { + "version": "3.3.7", + "resolved": "https://registry.npmjs.org/nanoid/-/nanoid-3.3.7.tgz", + "integrity": "sha512-eSRppjcPIatRIMC1U6UngP8XFcz8MQWGQdt1MTBQ7NaAmvXDfvNxbvWV3x2y6CdEUciCSsDHDQZbhYaB8QEo2g==", + "funding": [ + { + "type": "github", + "url": "https://github.com/sponsors/ai" + } + ], + "bin": { + "nanoid": "bin/nanoid.cjs" + }, + "engines": { + "node": "^10 || ^12 || ^13.7 || ^14 || >=15.0.1" + } + }, "node_modules/natural-compare": { "version": "1.4.0", "resolved": "https://registry.npmjs.org/natural-compare/-/natural-compare-1.4.0.tgz", @@ -17286,6 +17373,19 @@ "node": ">=8" } }, + "node_modules/redis": { + "version": "4.7.0", + "resolved": "https://registry.npmjs.org/redis/-/redis-4.7.0.tgz", + "integrity": "sha512-zvmkHEAdGMn+hMRXuMBtu4Vo5P6rHQjLoHftu+lBqq8ZTA3RCVC/WzD790bkKKiNFp7d5/9PcSD19fJyyRvOdQ==", + "dependencies": { + "@redis/bloom": "1.2.0", + "@redis/client": "1.6.0", + "@redis/graph": "1.1.1", + "@redis/json": "1.0.7", + "@redis/search": "1.2.0", + "@redis/time-series": "1.1.0" + } + }, "node_modules/redis-errors": { "version": "1.2.0", "resolved": "https://registry.npmjs.org/redis-errors/-/redis-errors-1.2.0.tgz", @@ -18512,6 +18612,11 @@ "node": ">=6" } }, + "node_modules/tiny-typed-emitter": { + "version": "2.1.0", + "resolved": "https://registry.npmjs.org/tiny-typed-emitter/-/tiny-typed-emitter-2.1.0.tgz", + "integrity": "sha512-qVtvMxeXbVej0cQWKqVSSAHmKZEHAvxdF8HEUBFWts8h+xEo5m/lEiPakuyZ3BnCBjOD8i24kzNOiOLLgsSxhA==" + }, "node_modules/tmp": { "version": "0.0.33", "resolved": "https://registry.npmjs.org/tmp/-/tmp-0.0.33.tgz", @@ -22370,6 +22475,20 @@ "resolved": "https://registry.npmjs.org/@jsdevtools/ono/-/ono-7.1.3.tgz", "integrity": "sha512-4JQNk+3mVzK3xh2rqd6RB4J46qUR19azEHBneZyTZM+c456qOrbbM/5xcR8huNCCcbVt7+UmizG6GuUvPvKUYg==" }, + "@map-colonies/cleanup-registry": { + "version": "1.1.0", + "resolved": "https://registry.npmjs.org/@map-colonies/cleanup-registry/-/cleanup-registry-1.1.0.tgz", + "integrity": "sha512-/lhIGklWPZSY37JwzhFJEtBlqwXDRhHSeCBwpPaGMxpycpt5ZRIVQxUt6Og4mt6c5GoRoX9dZYHY0qV3UMGvtQ==", + "requires": { + "nanoid": "^3.3.4", + "tiny-typed-emitter": "^2.1.0" + } + }, + "@map-colonies/detiler-common": { + "version": "1.0.0", + "resolved": "https://registry.npmjs.org/@map-colonies/detiler-common/-/detiler-common-1.0.0.tgz", + "integrity": "sha512-5gxowEjHBD9CcgqonYQGyarIXRElYnhO4qO/ffxN3IFieWyUWUuKFhc+U6rnvy53TN0UorPfTuOrCO85W9ZsKw==" + }, "@map-colonies/error-express-handler": { "version": "2.1.0", "resolved": "https://registry.npmjs.org/@map-colonies/error-express-handler/-/error-express-handler-2.1.0.tgz", @@ -24161,6 +24280,46 @@ "resolved": "https://registry.npmjs.org/@protobufjs/utf8/-/utf8-1.1.0.tgz", "integrity": "sha512-Vvn3zZrhQZkkBE8LSuW3em98c0FwgO4nxzv6OdSxPKJIEKY2bGbHn+mhGIPerzI4twdxaP8/0+06HBpwf345Lw==" }, + "@redis/bloom": { + "version": "1.2.0", + "resolved": "https://registry.npmjs.org/@redis/bloom/-/bloom-1.2.0.tgz", + "integrity": "sha512-HG2DFjYKbpNmVXsa0keLHp/3leGJz1mjh09f2RLGGLQZzSHpkmZWuwJbAvo3QcRY8p80m5+ZdXZdYOSBLlp7Cg==", + "requires": {} + }, + "@redis/client": { + "version": "1.6.0", + "resolved": "https://registry.npmjs.org/@redis/client/-/client-1.6.0.tgz", + "integrity": "sha512-aR0uffYI700OEEH4gYnitAnv3vzVGXCFvYfdpu/CJKvk4pHfLPEy/JSZyrpQ+15WhXe1yJRXLtfQ84s4mEXnPg==", + "requires": { + "cluster-key-slot": "1.1.2", + "generic-pool": "3.9.0", + "yallist": "4.0.0" + } + }, + "@redis/graph": { + "version": "1.1.1", + "resolved": "https://registry.npmjs.org/@redis/graph/-/graph-1.1.1.tgz", + "integrity": "sha512-FEMTcTHZozZciLRl6GiiIB4zGm5z5F3F6a6FZCyrfxdKOhFlGkiAqlexWMBzCi4DcRoyiOsuLfW+cjlGWyExOw==", + "requires": {} + }, + "@redis/json": { + "version": "1.0.7", + "resolved": "https://registry.npmjs.org/@redis/json/-/json-1.0.7.tgz", + "integrity": "sha512-6UyXfjVaTBTJtKNG4/9Z8PSpKE6XgSyEb8iwaqDcy+uKrd/DGYHTWkUdnQDyzm727V7p21WUMhsqz5oy65kPcQ==", + "requires": {} + }, + "@redis/search": { + "version": "1.2.0", + "resolved": "https://registry.npmjs.org/@redis/search/-/search-1.2.0.tgz", + "integrity": "sha512-tYoDBbtqOVigEDMAcTGsRlMycIIjwMCgD8eR2t0NANeQmgK/lvxNAvYyb6bZDD4frHRhIHkJu2TBRvB0ERkOmw==", + "requires": {} + }, + "@redis/time-series": { + "version": "1.1.0", + "resolved": "https://registry.npmjs.org/@redis/time-series/-/time-series-1.1.0.tgz", + "integrity": "sha512-c1Q99M5ljsIuc4YdaCwfUEXsofakb9c8+Zse2qxTadu8TalLXuAESzLvFAvNVbkmSlvlzIQOLpBCmWI9wTOt+g==", + "requires": {} + }, "@redocly/ajv": { "version": "8.6.4", "resolved": "https://registry.npmjs.org/@redocly/ajv/-/ajv-8.6.4.tgz", @@ -31272,6 +31431,11 @@ "thenify-all": "^1.0.0" } }, + "nanoid": { + "version": "3.3.7", + "resolved": "https://registry.npmjs.org/nanoid/-/nanoid-3.3.7.tgz", + "integrity": "sha512-eSRppjcPIatRIMC1U6UngP8XFcz8MQWGQdt1MTBQ7NaAmvXDfvNxbvWV3x2y6CdEUciCSsDHDQZbhYaB8QEo2g==" + }, "natural-compare": { "version": "1.4.0", "resolved": "https://registry.npmjs.org/natural-compare/-/natural-compare-1.4.0.tgz", @@ -32576,6 +32740,19 @@ "strip-indent": "^3.0.0" } }, + "redis": { + "version": "4.7.0", + "resolved": "https://registry.npmjs.org/redis/-/redis-4.7.0.tgz", + "integrity": "sha512-zvmkHEAdGMn+hMRXuMBtu4Vo5P6rHQjLoHftu+lBqq8ZTA3RCVC/WzD790bkKKiNFp7d5/9PcSD19fJyyRvOdQ==", + "requires": { + "@redis/bloom": "1.2.0", + "@redis/client": "1.6.0", + "@redis/graph": "1.1.1", + "@redis/json": "1.0.7", + "@redis/search": "1.2.0", + "@redis/time-series": "1.1.0" + } + }, "redis-errors": { "version": "1.2.0", "resolved": "https://registry.npmjs.org/redis-errors/-/redis-errors-1.2.0.tgz", @@ -33508,6 +33685,11 @@ "resolved": "https://registry.npmjs.org/tiny-lru/-/tiny-lru-8.0.2.tgz", "integrity": "sha512-ApGvZ6vVvTNdsmt676grvCkUCGwzG9IqXma5Z07xJgiC5L7akUMof5U8G2JTI9Rz/ovtVhJBlY6mNhEvtjzOIg==" }, + "tiny-typed-emitter": { + "version": "2.1.0", + "resolved": "https://registry.npmjs.org/tiny-typed-emitter/-/tiny-typed-emitter-2.1.0.tgz", + "integrity": "sha512-qVtvMxeXbVej0cQWKqVSSAHmKZEHAvxdF8HEUBFWts8h+xEo5m/lEiPakuyZ3BnCBjOD8i24kzNOiOLLgsSxhA==" + }, "tmp": { "version": "0.0.33", "resolved": "https://registry.npmjs.org/tmp/-/tmp-0.0.33.tgz", diff --git a/package.json b/package.json index 98795cf1..a9e21788 100644 --- a/package.json +++ b/package.json @@ -48,6 +48,8 @@ "dependencies": { "@elastic/elasticsearch": "^8.13.1", "@godaddy/terminus": "^4.12.1", + "@map-colonies/cleanup-registry": "^1.1.0", + "@map-colonies/detiler-common": "^1.0.0", "@map-colonies/error-express-handler": "^2.1.0", "@map-colonies/express-access-log-middleware": "^2.0.1", "@map-colonies/js-logger": "^1.0.1", @@ -72,6 +74,7 @@ "node-fetch-commonjs": "^3.3.2", "pg": "^8.11.5", "proj4": "^2.11.0", + "redis": "^4.7.0", "reflect-metadata": "^0.1.13", "tsyringe": "^4.8.0", "typeorm": "^0.3.20", diff --git a/src/common/constants.ts b/src/common/constants.ts index 39f7b888..83b3a228 100644 --- a/src/common/constants.ts +++ b/src/common/constants.ts @@ -16,6 +16,7 @@ export const SERVICES: Record = { METER: Symbol('Meter'), APPLICATION: Symbol('Application'), ELASTIC_CLIENTS: Symbol('ElasticClients'), + REDIS: Symbol('Redis'), }; /* eslint-enable @typescript-eslint/naming-convention */ @@ -24,4 +25,3 @@ export const HEALTHCHECK = Symbol('healthcheck'); export const REDIS_SYMBOL = Symbol('REDIS'); export const elasticConfigPath = 'db.elastic'; -export const REDIS_KEYS_SEPARATOR = ':'; diff --git a/src/common/errors.ts b/src/common/errors.ts index 244c0576..c56a5641 100644 --- a/src/common/errors.ts +++ b/src/common/errors.ts @@ -49,3 +49,5 @@ export class NotImplementedError extends Error implements HttpError { super(message); } } + +export class TimeoutError extends Error {} \ No newline at end of file diff --git a/src/common/interfaces.ts b/src/common/interfaces.ts index d140f1a0..3cc33eac 100644 --- a/src/common/interfaces.ts +++ b/src/common/interfaces.ts @@ -1,4 +1,5 @@ import { DataSourceOptions } from 'typeorm'; +import { RedisClientOptions } from 'redis'; import { Feature, FeatureCollection as GeoJSONFeatureCollection } from 'geojson'; export interface IConfig { @@ -13,6 +14,13 @@ export interface OpenApiConfig { uiPath: string; } +export type RedisConfig = { + host: string; + port: number; + enableSslAuth: boolean; + sslPaths: { ca: string; cert: string; key: string }; +} & RedisClientOptions; + export type PostgresDbConfig = { enableSslAuth: boolean; sslPaths: { ca: string; cert: string; key: string }; diff --git a/src/common/redis/domainFieldsRepository.ts b/src/common/redis/domainFieldsRepository.ts deleted file mode 100644 index 9cf9e3cc..00000000 --- a/src/common/redis/domainFieldsRepository.ts +++ /dev/null @@ -1,5 +0,0 @@ -export const IDOMAIN_FIELDS_REPO_SYMBOL = Symbol('DOMAINFIELDSREPO'); - -export interface IDomainFieldsRepository { - getFields: (fields: string[]) => Promise<(string | null)[]>; -} diff --git a/src/common/redis/index.ts b/src/common/redis/index.ts index fb3396dc..6e4cf92f 100644 --- a/src/common/redis/index.ts +++ b/src/common/redis/index.ts @@ -1,33 +1,58 @@ -import Redis, { RedisOptions } from 'ioredis'; -import { HOSTNAME } from '../constants'; +import { readFileSync } from 'fs'; +import { ILogger } from '@map-colonies/detiler-common'; +import { HealthCheck } from '@godaddy/terminus'; +import { createClient, RedisClientOptions } from 'redis'; +import { DependencyContainer, FactoryFunction } from 'tsyringe'; +import { SERVICES } from '../constants'; +import { RedisConfig, IConfig } from '../interfaces'; +import { promiseTimeout } from '../utils'; -const RETRY_DELAY_INCREASE = 50; -const RETRY_DELAY_TOP = 2000; -let redis: Redis; +const DEFAULT_LIMIT_FROM = 0; +const DEFAULT_LIMIT_SIZE = 1000; -const retryFunction = (times: number): number => { - const delay = Math.min(times * RETRY_DELAY_INCREASE, RETRY_DELAY_TOP); - return delay; +const createConnectionOptions = (redisConfig: RedisConfig): Partial => { + const { host, port, enableSslAuth, sslPaths, ...clientOptions } = redisConfig; + clientOptions.socket = { host, port }; + if (enableSslAuth) { + clientOptions.socket = { + ...clientOptions.socket, + tls: true, + key: sslPaths.key !== '' ? readFileSync(sslPaths.key) : undefined, + cert: sslPaths.cert !== '' ? readFileSync(sslPaths.cert) : undefined, + ca: sslPaths.ca !== '' ? readFileSync(sslPaths.ca) : undefined, + }; + } + + return clientOptions; }; -export const createRedisConnection = async (redisOptions: RedisOptions): Promise => { - try { - redisOptions = { - ...redisOptions, - retryStrategy: retryFunction, - lazyConnect: true, - connectionName: HOSTNAME, - }; +export const CONNECTION_TIMEOUT = 5000; - redis = new Redis(redisOptions); - await redis.connect(); - return redis; - } catch (err) { - redis.disconnect(); - let errorMessage = 'Redis connection failed'; - if (err instanceof Error) { - errorMessage += ` with the following error: ${err.message}`; - } - throw new Error(errorMessage); - } +export const DEFAULT_LIMIT = { from: DEFAULT_LIMIT_FROM, size: DEFAULT_LIMIT_SIZE }; + +export type RedisClient = ReturnType; + +export const redisClientFactory: FactoryFunction = (container: DependencyContainer): RedisClient => { + const logger = container.resolve(SERVICES.LOGGER); + const config = container.resolve(SERVICES.CONFIG); + const dbConfig = config.get('db.redis'); + const connectionOptions = createConnectionOptions(dbConfig); + + const redisClient = createClient(connectionOptions) + .on('error', (error: Error) => logger.error({ msg: 'redis client errored', err: error })) + .on('reconnecting', (...args) => logger.warn({ msg: 'redis client reconnecting', ...args })) + .on('end', (...args) => logger.info({ msg: 'redis client end', ...args })) + .on('connect', (...args) => logger.debug({ msg: 'redis client connected', ...args })) + .on('ready', (...args) => logger.debug({ msg: 'redis client is ready', ...args })); + + return redisClient; +}; + +export const healthCheckFunctionFactory = (redis: RedisClient): HealthCheck => { + return async (): Promise => { + const check = redis.ping().then(() => { + return; + }); + return promiseTimeout(CONNECTION_TIMEOUT, check); + }; }; diff --git a/src/common/redis/keys.ts b/src/common/redis/keys.ts deleted file mode 100644 index e268631c..00000000 --- a/src/common/redis/keys.ts +++ /dev/null @@ -1,5 +0,0 @@ -import { REDIS_KEYS_SEPARATOR } from '../../common/constants'; - -export const keyConstructor = (...vals: string[]): string => { - return vals.filter((v) => v).join(REDIS_KEYS_SEPARATOR); // drop undefined, null and empty string from a key -}; diff --git a/src/common/redis/redisManager.ts b/src/common/redis/redisManager.ts deleted file mode 100644 index 0ea22c07..00000000 --- a/src/common/redis/redisManager.ts +++ /dev/null @@ -1,31 +0,0 @@ -import Redis from 'ioredis'; -import { inject, injectable } from 'tsyringe'; -import { REDIS_SYMBOL, SERVICES } from '../constants'; -import { IApplication } from '../interfaces'; -import { IDomainFieldsRepository } from './domainFieldsRepository'; - -@injectable() -export class RedisManager implements IDomainFieldsRepository { - public getData: (fields: string[]) => Promise<(string | null)[]>; - - public constructor(@inject(REDIS_SYMBOL) private readonly redis: Redis, @inject(SERVICES.APPLICATION) appConfig: IApplication) { - const { value, enabled } = appConfig.hashKey; - if (enabled && value !== undefined) { - this.getData = async (fields: string[]): Promise<(string | null)[]> => { - return this.redis.hmget(value, ...fields); - }; - } else { - this.getData = async (fields: string[]): Promise<(string | null)[]> => { - return this.redis.mget(fields); - }; - } - } - - public async getFields(fields: string[]): Promise<(string | null)[]> { - try { - return await this.getData(fields); - } catch (e) { - throw new Error('redis: failed to fetch keys'); - } - } -} diff --git a/src/common/utils.ts b/src/common/utils.ts index 6f75dec6..0b64a1c1 100644 --- a/src/common/utils.ts +++ b/src/common/utils.ts @@ -1,6 +1,7 @@ import proj4 from 'proj4'; import { utmProjection, wgs84Projection } from './projections'; import { WGS84Coordinate } from './interfaces'; +import { TimeoutError } from './errors'; type SnakeToCamelCase = S extends `${infer T}_${infer U}` ? `${T}${Capitalize>}` : S; @@ -79,3 +80,16 @@ export const validateTile = (tile: { tileName: string; subTileNumber: number[] } } return true; }; + +export const promiseTimeout = async (ms: number, promise: Promise): Promise => { + // create a promise that rejects in milliseconds + const timeout = new Promise((_, reject) => { + const id = setTimeout(() => { + clearTimeout(id); + reject(new TimeoutError(`Timed out in + ${ms} + ms.`)); + }, ms); + }); + + // returns a race between our timeout and the passed in promise + return Promise.race([promise, timeout]); +}; \ No newline at end of file diff --git a/src/containerConfig.ts b/src/containerConfig.ts index 658dc442..4424fa4b 100644 --- a/src/containerConfig.ts +++ b/src/containerConfig.ts @@ -1,17 +1,16 @@ import config from 'config'; -import Redis, { RedisOptions } from 'ioredis'; import { getOtelMixin } from '@map-colonies/telemetry'; import { DataSource } from 'typeorm'; import { instancePerContainerCachingFactory } from 'tsyringe'; import { trace, metrics as OtelMetrics } from '@opentelemetry/api'; +import { CleanupRegistry } from '@map-colonies/cleanup-registry'; import { DependencyContainer } from 'tsyringe/dist/typings/types'; import jsLogger, { LoggerOptions } from '@map-colonies/js-logger'; import { Metrics } from '@map-colonies/telemetry'; -import { SERVICES, SERVICE_NAME, REDIS_SYMBOL } from './common/constants'; +import { HEALTHCHECK, SERVICES, SERVICE_NAME } from './common/constants'; import { tracing } from './common/tracing'; import { InjectionObject, registerDependencies } from './common/dependencyRegistration'; import { elasticClientFactory, ElasticClients } from './common/elastic'; -import { RedisManager } from './common/redis/redisManager'; import { IApplication } from './common/interfaces'; import { TILE_REPOSITORY_SYMBOL, tileRepositoryFactory } from './control/tile/DAL/tileRepository'; import { TILE_ROUTER_SYMBOL, tileRouterFactory } from './control/tile/routes/tileRouter'; @@ -25,8 +24,8 @@ import { GEOTEXT_REPOSITORY_SYMBOL, geotextRepositoryFactory } from './location/ import { GEOTEXT_SEARCH_ROUTER_SYMBOL, geotextSearchRouterFactory } from './location/routes/locationRouter'; import { cronLoadTileLatLonDataFactory, cronLoadTileLatLonDataSymbol } from './latLon/DAL/latLonDAL'; import { ITEM_REPOSITORY_SYMBOL, itemRepositoryFactory } from './control/item/DAL/itemRepository'; -import { createRedisConnection } from './common/redis/index'; -import { IDOMAIN_FIELDS_REPO_SYMBOL } from './common/redis/domainFieldsRepository'; +import { healthCheckFunctionFactory, RedisClient, redisClientFactory } from './common/redis'; +import { HealthCheck } from '@godaddy/terminus'; export interface RegisterOptions { override?: InjectionObject[]; @@ -34,6 +33,8 @@ export interface RegisterOptions { } export const registerExternalValues = async (options?: RegisterOptions): Promise => { + const cleanupRegistry = new CleanupRegistry(); + const loggerConfig = config.get('telemetry.logger'); const logger = jsLogger({ ...loggerConfig, prettyPrint: loggerConfig.prettyPrint, mixin: getOtelMixin() }); @@ -45,8 +46,6 @@ export const registerExternalValues = async (options?: RegisterOptions): Promise const applicationConfig: IApplication = config.get('application'); - let redisConnection: Redis | undefined; - const dependencies: InjectionObject[] = [ { token: SERVICES.CONFIG, provider: { useValue: config } }, { token: SERVICES.LOGGER, provider: { useValue: logger } }, @@ -84,26 +83,21 @@ export const registerExternalValues = async (options?: RegisterOptions): Promise }, }, { - token: REDIS_SYMBOL, + token: SERVICES.REDIS, + provider: { useFactory: instancePerContainerCachingFactory(redisClientFactory) }, + postInjectionHook: async (deps: DependencyContainer): Promise => { + const redis = deps.resolve(SERVICES.REDIS); + cleanupRegistry.register({ func: redis.disconnect.bind(redis), id: SERVICES.REDIS }); + await redis.connect(); + }, + }, + { + token: HEALTHCHECK, provider: { - useFactory: instancePerContainerCachingFactory(async () => { - redisConnection = await createRedisConnection(config.get('db')); - - redisConnection.on('connect', () => { - logger.info(`redis client is connected.`); - }); - - redisConnection.on('error', (err: Error) => { - logger.error({ err: err, msg: 'redis client got an error' }); - }); - - redisConnection.on('reconnecting', (delay: number) => { - logger.info(`redis client reconnecting, next reconnection attemp in ${delay}ms`); - }); - container.register(SERVICES.LOGGER, { useValue: logger }); - container.register(IDOMAIN_FIELDS_REPO_SYMBOL, { useClass: RedisManager }); - return redisConnection; - }), + useFactory: (container): HealthCheck => { + const redis = container.resolve(SERVICES.REDIS); + return healthCheckFunctionFactory(redis); + }, }, }, { token: TILE_REPOSITORY_SYMBOL, provider: { useFactory: tileRepositoryFactory } }, @@ -127,20 +121,8 @@ export const registerExternalValues = async (options?: RegisterOptions): Promise provider: { useFactory: instancePerContainerCachingFactory(async (): Promise => { const promises: Promise[] = [tracing.stop(), metrics.stop()]; - if (redisConnection !== undefined) { - redisConnection.disconnect(); - - const promisifyQuit = new Promise((resolve) => { - redisConnection = redisConnection as Redis; - redisConnection.once('end', () => { - resolve(); - }); - void redisConnection.quit(); - }); - promises.push(promisifyQuit); - } - await Promise.all(promises); }), + useValue: cleanupRegistry.trigger.bind(cleanupRegistry), }, }, ]; diff --git a/src/serverBuilder.ts b/src/serverBuilder.ts index c9c04c50..5d6740ae 100644 --- a/src/serverBuilder.ts +++ b/src/serverBuilder.ts @@ -16,6 +16,7 @@ import { ROUTE_ROUTER_SYMBOL } from './control/route/routes/routeRouter'; import { LAT_LON_ROUTER_SYMBOL } from './latLon/routes/latLonRouter'; import { GEOTEXT_SEARCH_ROUTER_SYMBOL } from './location/routes/locationRouter'; import { cronLoadTileLatLonDataSymbol } from './latLon/DAL/latLonDAL'; +import { feedbackApiMiddlewareManager } from './common/middlewares/feedbackApi.middleware'; @injectable() export class ServerBuilder { @@ -29,7 +30,8 @@ export class ServerBuilder { @inject(ROUTE_ROUTER_SYMBOL) private readonly routeRouter: Router, @inject(LAT_LON_ROUTER_SYMBOL) private readonly latLonRouter: Router, @inject(GEOTEXT_SEARCH_ROUTER_SYMBOL) private readonly geotextRouter: Router, - @inject(cronLoadTileLatLonDataSymbol) private readonly cronLoadTileLatLonData: void + @inject(cronLoadTileLatLonDataSymbol) private readonly cronLoadTileLatLonData: void, + @inject(feedbackApiMiddlewareManager) private readonly feedbackApiMiddleware: feedbackApiMiddlewareManager ) { this.serverInstance = express(); // eslint-disable-next-line @typescript-eslint/no-unused-expressions @@ -55,12 +57,14 @@ export class ServerBuilder { } private buildRoutes(): void { + this.serverInstance.use(this.feedbackApiMiddleware.saveResponses); const router = Router(); + router.use('/lookup', this.latLonRouter); router.use('/location', this.geotextRouter); router.use('/control', this.buildControlRoutes()); - this.serverInstance.use('/search/', router); + this.serverInstance.use('/search', router); } private buildControlRoutes(): Router { From a560876820d4fd78f448690d1a2ea1f05aff3736 Mon Sep 17 00:00:00 2001 From: NatalieShaked Date: Thu, 29 Aug 2024 17:41:19 +0300 Subject: [PATCH 165/262] feat: api responses get sent to redis --- src/common/interfaces.ts | 10 +++-- .../middlewares/feedbackApi.middleware.ts | 38 +++++++++++++++++++ 2 files changed, 44 insertions(+), 4 deletions(-) create mode 100644 src/common/middlewares/feedbackApi.middleware.ts diff --git a/src/common/interfaces.ts b/src/common/interfaces.ts index 3cc33eac..3c0052f3 100644 --- a/src/common/interfaces.ts +++ b/src/common/interfaces.ts @@ -57,10 +57,6 @@ export interface IApplication { hierarchy: number; viewbox: number; }; - hashKey: { - value?: string; - enabled: boolean; - }; sources?: { [key: string]: string; }; @@ -76,6 +72,12 @@ export enum GeoContextMode { BIAS = 'bias', } +export interface GeocodingResponse { + userId: string; + response: JSON; + respondedAt: Date; +} + /* eslint-disable @typescript-eslint/naming-convention */ export interface CommonRequestParameters { geo_context?: GeoContext; diff --git a/src/common/middlewares/feedbackApi.middleware.ts b/src/common/middlewares/feedbackApi.middleware.ts new file mode 100644 index 00000000..bcf7b9aa --- /dev/null +++ b/src/common/middlewares/feedbackApi.middleware.ts @@ -0,0 +1,38 @@ +import { Request, Response, NextFunction } from 'express'; +import { SERVICES } from '../constants'; +import { Logger } from '@map-colonies/js-logger'; +import { inject, injectable } from 'tsyringe'; +import { RedisClient } from '../redis'; +import * as crypto from 'node:crypto'; +import { GeocodingResponse } from '../interfaces'; + +@injectable() +export class feedbackApiMiddlewareManager { + public constructor(@inject(SERVICES.LOGGER) private readonly logger: Logger, @inject(SERVICES.REDIS) private readonly redis: RedisClient) {} + + saveResponses = async (req: Request, res: Response, next: NextFunction) => { + this.logger.info({ msg: 'saving response to redis' }); + const reqId = crypto.randomUUID(); + const redisClient = this.redis + + // let response: JSON = JSON.parse('{}'); + let geocodingResponseDetails: GeocodingResponse = { + userId: req.headers['x-user-id'] as string, + response: JSON.parse('{}'), + respondedAt: new Date(), + }; + + const originalJson = res.json; + const logJson = async function (this: Response, body: any): Promise>> { + console.log('Response:', body); + // response = await body; + geocodingResponseDetails.response = await body; + await redisClient.set(reqId, JSON.stringify(geocodingResponseDetails)); + + return originalJson.call(this, body); + }; + + res.json = logJson as unknown as Response['json']; + next(); + }; +} From 1f97c76e08294c2413a17fe8e4b0b6565e3643b0 Mon Sep 17 00:00:00 2001 From: NatalieShaked Date: Thu, 29 Aug 2024 18:04:05 +0300 Subject: [PATCH 166/262] feat: added ttl of 5 mins to redis set --- .../middlewares/feedbackApi.middleware.ts | 30 +++++++++++++++---- 1 file changed, 25 insertions(+), 5 deletions(-) diff --git a/src/common/middlewares/feedbackApi.middleware.ts b/src/common/middlewares/feedbackApi.middleware.ts index bcf7b9aa..f22bc77e 100644 --- a/src/common/middlewares/feedbackApi.middleware.ts +++ b/src/common/middlewares/feedbackApi.middleware.ts @@ -5,16 +5,19 @@ import { inject, injectable } from 'tsyringe'; import { RedisClient } from '../redis'; import * as crypto from 'node:crypto'; import { GeocodingResponse } from '../interfaces'; +import { RedisClientType } from '@redis/client'; +import { RedisModules, RedisFunctions, RedisScripts } from 'redis'; @injectable() export class feedbackApiMiddlewareManager { public constructor(@inject(SERVICES.LOGGER) private readonly logger: Logger, @inject(SERVICES.REDIS) private readonly redis: RedisClient) {} saveResponses = async (req: Request, res: Response, next: NextFunction) => { - this.logger.info({ msg: 'saving response to redis' }); const reqId = crypto.randomUUID(); - const redisClient = this.redis + const redisClient = this.redis; + const logger = this.logger; + logger.info({ msg: 'saving response to redis' }); // let response: JSON = JSON.parse('{}'); let geocodingResponseDetails: GeocodingResponse = { userId: req.headers['x-user-id'] as string, @@ -25,14 +28,31 @@ export class feedbackApiMiddlewareManager { const originalJson = res.json; const logJson = async function (this: Response, body: any): Promise>> { console.log('Response:', body); - // response = await body; geocodingResponseDetails.response = await body; - await redisClient.set(reqId, JSON.stringify(geocodingResponseDetails)); + + try { + await redisClient.setEx(reqId, 300, JSON.stringify(geocodingResponseDetails)); + logger.info({ msg: 'saving response to redis' }); + } catch (err) { + logger.error('Error setting key:', err); + } + + //await setKeyWithTTL(reqId, JSON.stringify(geocodingResponseDetails), redisClient); return originalJson.call(this, body); }; - res.json = logJson as unknown as Response['json']; next(); }; } + +async function setKeyWithTTL(key: string, value: string, redis: RedisClient) { + try { + await redis.set(key, value, { + EX: 300, // 5 minutes in seconds + }); + console.log(`Key '${key}' set with a TTL of 5 minutes.`); + } catch (err) { + console.error('Error setting key:', err); + } +} From b5a4031de4e43450af06b859d1de76277680cb39 Mon Sep 17 00:00:00 2001 From: NatalieShaked Date: Sun, 1 Sep 2024 12:13:50 +0300 Subject: [PATCH 167/262] fix: removed unnecessary redis configs --- .../middlewares/feedbackApi.middleware.ts | 5 +--- tests/configurations/unit/jest.config.js | 1 + tests/integration/control/item/item.spec.ts | 30 ++----------------- tests/integration/control/route/route.spec.ts | 28 +---------------- tests/integration/control/tile/tile.spec.ts | 28 +---------------- tests/integration/location/location.spec.ts | 29 +----------------- 6 files changed, 7 insertions(+), 114 deletions(-) diff --git a/src/common/middlewares/feedbackApi.middleware.ts b/src/common/middlewares/feedbackApi.middleware.ts index f22bc77e..c3a178dd 100644 --- a/src/common/middlewares/feedbackApi.middleware.ts +++ b/src/common/middlewares/feedbackApi.middleware.ts @@ -5,8 +5,6 @@ import { inject, injectable } from 'tsyringe'; import { RedisClient } from '../redis'; import * as crypto from 'node:crypto'; import { GeocodingResponse } from '../interfaces'; -import { RedisClientType } from '@redis/client'; -import { RedisModules, RedisFunctions, RedisScripts } from 'redis'; @injectable() export class feedbackApiMiddlewareManager { @@ -18,7 +16,6 @@ export class feedbackApiMiddlewareManager { const logger = this.logger; logger.info({ msg: 'saving response to redis' }); - // let response: JSON = JSON.parse('{}'); let geocodingResponseDetails: GeocodingResponse = { userId: req.headers['x-user-id'] as string, response: JSON.parse('{}'), @@ -27,7 +24,7 @@ export class feedbackApiMiddlewareManager { const originalJson = res.json; const logJson = async function (this: Response, body: any): Promise>> { - console.log('Response:', body); + // console.log('Response:', body); geocodingResponseDetails.response = await body; try { diff --git a/tests/configurations/unit/jest.config.js b/tests/configurations/unit/jest.config.js index 07af12e5..bfd03fc4 100644 --- a/tests/configurations/unit/jest.config.js +++ b/tests/configurations/unit/jest.config.js @@ -13,6 +13,7 @@ module.exports = { '!*/common/**', '!**/controllers/**', '!**/routes/**', + '!**/redis/**', '!/src/*', ], coverageDirectory: '/coverage', diff --git a/tests/integration/control/item/item.spec.ts b/tests/integration/control/item/item.spec.ts index 01327f88..39e3659b 100644 --- a/tests/integration/control/item/item.spec.ts +++ b/tests/integration/control/item/item.spec.ts @@ -3,12 +3,8 @@ import jsLogger from '@map-colonies/js-logger'; import { trace } from '@opentelemetry/api'; import httpStatusCodes from 'http-status-codes'; import { DataSource } from 'typeorm'; -import config from 'config'; -import Redis, { RedisOptions } from 'ioredis'; -import { DependencyContainer } from 'tsyringe'; -import { Application } from 'express'; import { getApp } from '../../../../src/app'; -import { REDIS_SYMBOL, SERVICES } from '../../../../src/common/constants'; +import { SERVICES } from '../../../../src/common/constants'; import { GetItemsQueryParams } from '../../../../src/control/item/controllers/itemController'; import { Item } from '../../../../src/control/item/models/item'; import { ControlResponse } from '../../../../src/control/interfaces'; @@ -16,30 +12,14 @@ import { CommonRequestParameters, GeoContext, GeoContextMode } from '../../../.. import { LATLON_CUSTOM_REPOSITORY_SYMBOL } from '../../../../src/latLon/DAL/latLonRepository'; import { cronLoadTileLatLonDataSymbol } from '../../../../src/latLon/DAL/latLonDAL'; import { expectedResponse } from '../utils'; -import { RedisManager } from '../../../../src/common/redis/redisManager'; -import { IDOMAIN_FIELDS_REPO_SYMBOL } from '../../../../src/common/redis/domainFieldsRepository'; -import { createRedisConnection } from '../../../../src/common/redis'; import { ItemRequestSender } from './helpers/requestSender'; import { ITEM_1234, ITEM_1235, ITEM_1236 } from './mockObjects'; describe('/search/control/items', function () { let requestSender: ItemRequestSender; - let app: { app: Application; container?: DependencyContainer }; - let redisConnection: Redis; - - beforeAll(async function () { - redisConnection = await createRedisConnection(config.get('db')); - app = await getApp({ - override: [ - { token: REDIS_SYMBOL, provider: { useValue: redisConnection } }, - { token: IDOMAIN_FIELDS_REPO_SYMBOL, provider: { useClass: RedisManager } }, - ], - useChild: true, - }); - }); beforeEach(async function () { - app = await getApp({ + const app = await getApp({ override: [ { token: SERVICES.LOGGER, provider: { useValue: jsLogger({ enabled: false }) } }, { token: SERVICES.TRACER, provider: { useValue: trace.getTracer('testTracer') } }, @@ -54,12 +34,6 @@ describe('/search/control/items', function () { requestSender = new ItemRequestSender(app.app); }); - afterAll(async function () { - if (!['end'].includes(redisConnection.status)) { - await redisConnection.quit(); - } - }); - describe('Happy Path', function () { it('should return 200 status code and items', async function () { const requestParams: GetItemsQueryParams = { command_name: '123', limit: 5, disable_fuzziness: false }; diff --git a/tests/integration/control/route/route.spec.ts b/tests/integration/control/route/route.spec.ts index 1dd5b394..fe95f22e 100644 --- a/tests/integration/control/route/route.spec.ts +++ b/tests/integration/control/route/route.spec.ts @@ -3,40 +3,20 @@ import jsLogger from '@map-colonies/js-logger'; import { trace } from '@opentelemetry/api'; import httpStatusCodes from 'http-status-codes'; import { DataSource } from 'typeorm'; -import config from 'config'; -import Redis, { RedisOptions } from 'ioredis'; -import { DependencyContainer } from 'tsyringe'; -import { Application } from 'express'; import { getApp } from '../../../../src/app'; -import { REDIS_SYMBOL, SERVICES } from '../../../../src/common/constants'; +import { SERVICES } from '../../../../src/common/constants'; import { GetRoutesQueryParams } from '../../../../src/control/route/controllers/routeController'; import { Route } from '../../../../src/control/route/models/route'; import { ControlResponse } from '../../../../src/control/interfaces'; import { CommonRequestParameters, GeoContext, GeoContextMode } from '../../../../src/common/interfaces'; import { LATLON_CUSTOM_REPOSITORY_SYMBOL } from '../../../../src/latLon/DAL/latLonRepository'; import { cronLoadTileLatLonDataSymbol } from '../../../../src/latLon/DAL/latLonDAL'; -import { RedisManager } from '../../../../src/common/redis/redisManager'; -import { IDOMAIN_FIELDS_REPO_SYMBOL } from '../../../../src/common/redis/domainFieldsRepository'; -import { createRedisConnection } from '../../../../src/common/redis'; import { expectedResponse } from '../utils'; import { RouteRequestSender } from './helpers/requestSender'; import { ROUTE_VIA_CAMILLUCCIA_A, ROUTE_VIA_CAMILLUCCIA_B, CONTROL_POINT_OLIMPIADE_111, CONTROL_POINT_OLIMPIADE_112 } from './mockObjects'; describe('/search/control/route', function () { let requestSender: RouteRequestSender; - let app: { app: Application; container?: DependencyContainer }; - let redisConnection: Redis; - - beforeAll(async function () { - redisConnection = await createRedisConnection(config.get('db')); - app = await getApp({ - override: [ - { token: REDIS_SYMBOL, provider: { useValue: redisConnection } }, - { token: IDOMAIN_FIELDS_REPO_SYMBOL, provider: { useClass: RedisManager } }, - ], - useChild: true, - }); - }); beforeEach(async function () { const app = await getApp({ @@ -54,12 +34,6 @@ describe('/search/control/route', function () { requestSender = new RouteRequestSender(app.app); }); - afterAll(async function () { - if (!['end'].includes(redisConnection.status)) { - await redisConnection.quit(); - } - }); - describe('Happy Path', function () { it('should return 200 status code and routes', async function () { const requestParams: GetRoutesQueryParams = { command_name: 'via camilluccia', limit: 5, disable_fuzziness: false }; diff --git a/tests/integration/control/tile/tile.spec.ts b/tests/integration/control/tile/tile.spec.ts index 6564fd99..44685fcc 100644 --- a/tests/integration/control/tile/tile.spec.ts +++ b/tests/integration/control/tile/tile.spec.ts @@ -3,20 +3,13 @@ import jsLogger from '@map-colonies/js-logger'; import { trace } from '@opentelemetry/api'; import httpStatusCodes from 'http-status-codes'; import { DataSource } from 'typeorm'; -import config from 'config'; -import Redis, { RedisOptions } from 'ioredis'; -import { DependencyContainer } from 'tsyringe'; -import { Application } from 'express'; import { getApp } from '../../../../src/app'; -import { REDIS_SYMBOL, SERVICES } from '../../../../src/common/constants'; +import { SERVICES } from '../../../../src/common/constants'; import { GetTilesQueryParams } from '../../../../src/control/tile/controllers/tileController'; import { Tile } from '../../../../src/control/tile/models/tile'; import { ControlResponse } from '../../../../src/control/interfaces'; import { CommonRequestParameters, GeoContext, GeoContextMode } from '../../../../src/common/interfaces'; import { LATLON_CUSTOM_REPOSITORY_SYMBOL } from '../../../../src/latLon/DAL/latLonRepository'; -import { RedisManager } from '../../../../src/common/redis/redisManager'; -import { IDOMAIN_FIELDS_REPO_SYMBOL } from '../../../../src/common/redis/domainFieldsRepository'; -import { createRedisConnection } from '../../../../src/common/redis'; import { cronLoadTileLatLonDataSymbol } from '../../../../src/latLon/DAL/latLonDAL'; import { expectedResponse } from '../utils'; import { TileRequestSender } from './helpers/requestSender'; @@ -24,19 +17,6 @@ import { RIC_TILE, RIT_TILE, SUB_TILE_65, SUB_TILE_66 } from './mockObjects'; describe('/search/control/tiles', function () { let requestSender: TileRequestSender; - let app: { app: Application; container?: DependencyContainer }; - let redisConnection: Redis; - - beforeAll(async function () { - redisConnection = await createRedisConnection(config.get('db')); - app = await getApp({ - override: [ - { token: REDIS_SYMBOL, provider: { useValue: redisConnection } }, - { token: IDOMAIN_FIELDS_REPO_SYMBOL, provider: { useClass: RedisManager } }, - ], - useChild: true, - }); - }); beforeEach(async function () { const app = await getApp({ @@ -54,12 +34,6 @@ describe('/search/control/tiles', function () { requestSender = new TileRequestSender(app.app); }); - afterAll(async function () { - if (!['end'].includes(redisConnection.status)) { - await redisConnection.quit(); - } - }); - describe('Happy Path', function () { it('should return 200 status code and tiles', async function () { const requestParams: GetTilesQueryParams = { tile: 'RIT', limit: 5, disable_fuzziness: false }; diff --git a/tests/integration/location/location.spec.ts b/tests/integration/location/location.spec.ts index 385c4f59..c561aa42 100644 --- a/tests/integration/location/location.spec.ts +++ b/tests/integration/location/location.spec.ts @@ -3,18 +3,12 @@ import config from 'config'; import jsLogger from '@map-colonies/js-logger'; import { trace } from '@opentelemetry/api'; import httpStatusCodes from 'http-status-codes'; -import Redis, { RedisOptions } from 'ioredis'; -import { DependencyContainer } from 'tsyringe'; -import { Application } from 'express'; import { DataSource } from 'typeorm'; import nock, { Body } from 'nock'; import { getApp } from '../../../src/app'; -import { REDIS_SYMBOL, SERVICES } from '../../../src/common/constants'; +import { SERVICES } from '../../../src/common/constants'; import { LATLON_CUSTOM_REPOSITORY_SYMBOL } from '../../../src/latLon/DAL/latLonRepository'; import { cronLoadTileLatLonDataSymbol } from '../../../src/latLon/DAL/latLonDAL'; -import { RedisManager } from '../../../src/common/redis/redisManager'; -import { IDOMAIN_FIELDS_REPO_SYMBOL } from '../../../src/common/redis/domainFieldsRepository'; -import { createRedisConnection } from '../../../src/common/redis'; import { GetGeotextSearchParams, QueryResult } from '../../../src/location/interfaces'; import { GeoContextMode, IApplication } from '../../../src/common/interfaces'; import { LocationRequestSender } from './helpers/requestSender'; @@ -22,32 +16,17 @@ import { OSM_LA_PORT, GOOGLE_LA_PORT, LA_AIRPORT, - LA_WHITE_POINT_SCHOOL, NY_JFK_AIRPORT, NY_POLICE_AIRPORT, NY_HIERRARCHY, LA_HIERRARCHY, MockLocationQueryFeature, PARIS_WI_SCHOOL, - PARIS_HIERRARCHY, } from './mockObjects'; import { expectedResponse, hierarchiesWithAnyWieght } from './utils'; describe('/search/control/tiles', function () { let requestSender: LocationRequestSender; - let app: { app: Application; container?: DependencyContainer }; - let redisConnection: Redis; - - beforeAll(async function () { - redisConnection = await createRedisConnection(config.get('db')); - app = await getApp({ - override: [ - { token: REDIS_SYMBOL, provider: { useValue: redisConnection } }, - { token: IDOMAIN_FIELDS_REPO_SYMBOL, provider: { useClass: RedisManager } }, - ], - useChild: true, - }); - }); beforeEach(async function () { const app = await getApp({ @@ -65,12 +44,6 @@ describe('/search/control/tiles', function () { requestSender = new LocationRequestSender(app.app); }); - afterAll(async function () { - if (!['end'].includes(redisConnection.status)) { - await redisConnection.quit(); - } - }); - describe('Happy Path', function () { it('should return 200 status code and airports', async function () { const requestParams: GetGeotextSearchParams = { query: 'airport', limit: 5, disable_fuzziness: true }; From e0c9f30cb7d0be38f0bcf3752e752cf76fffa55b Mon Sep 17 00:00:00 2001 From: NatalieShaked Date: Sun, 1 Sep 2024 13:49:57 +0300 Subject: [PATCH 168/262] fix: eslint fixes --- openapi3.yaml | 580 +++++++++--------- src/common/constants.ts | 1 + src/common/errors.ts | 2 +- .../middlewares/feedbackApi.middleware.ts | 41 +- src/common/utils.ts | 2 +- src/containerConfig.ts | 13 +- src/control/route/DAL/routeRepository.ts | 6 +- src/control/tile/DAL/tileRepository.ts | 9 +- src/control/utils.ts | 2 +- src/index.ts | 2 +- .../controllers/locationController.ts | 4 +- src/location/parsing.ts | 4 +- src/location/utils.ts | 2 +- src/serverBuilder.ts | 4 +- tests/integration/location/mockObjects.ts | 2 +- 15 files changed, 335 insertions(+), 339 deletions(-) diff --git a/openapi3.yaml b/openapi3.yaml index f3b5cc9e..4c0948e5 100644 --- a/openapi3.yaml +++ b/openapi3.yaml @@ -1,7 +1,7 @@ -openapi: "3.0.1" +openapi: '3.0.1' info: - version: "0.1.0" - title: "Geocoding" + version: '0.1.0' + title: 'Geocoding' description: |- MapColonies Vector Geocoding api provides custom geodata search engine uniting multiple sources of data, query tiles, routes, items; Convertion functions - tile to/from WGS84 lat lng, WGS84 to/from US Army MGRS, WGS84 to/from UTM. license: @@ -15,7 +15,7 @@ paths: /search/query: get: operationId: GetSmartQuery - summary: "Search anything" + summary: 'Search anything' description: |- This is for general queries. the services will make a sophisticated guess. parameters: @@ -26,16 +26,16 @@ paths: type: string minLength: 3 maxLength: 100 - title: "Query" + title: 'Query' description: Text to search allowReserved: true - - $ref: "#/components/parameters/geo_context" - - $ref: "#/components/parameters/geo_context_mode" - - $ref: "#/components/parameters/limit" - - $ref: "#/components/parameters/disable_fuzziness" + - $ref: '#/components/parameters/geo_context' + - $ref: '#/components/parameters/geo_context_mode' + - $ref: '#/components/parameters/limit' + - $ref: '#/components/parameters/disable_fuzziness' responses: 200: - description: "OK
Will return valid GeoJSON FeatureCollection" + description: 'OK
Will return valid GeoJSON FeatureCollection' headers: x-request-id: schema: @@ -45,15 +45,15 @@ paths: content: application/json: schema: - $ref: "#/components/schemas/genericGeocodingResponse" + $ref: '#/components/schemas/genericGeocodingResponse' 400: - "$ref": "#/components/responses/BadRequest" + '$ref': '#/components/responses/BadRequest' 401: - "$ref": "#/components/responses/Unauthorized" + '$ref': '#/components/responses/Unauthorized' 403: - "$ref": "#/components/responses/Forbidden" + '$ref': '#/components/responses/Forbidden' 500: - "$ref": "#/components/responses/InternalError" + '$ref': '#/components/responses/InternalError' security: - x-api-key: [] x-user-id: [] @@ -61,8 +61,8 @@ paths: /search/location/query: get: operationId: locationGetQuery - summary: "Search engine" - description: "Search for geo-data. It serves data from multiple diverse mapcolonies vector sources and partners." + summary: 'Search engine' + description: 'Search for geo-data. It serves data from multiple diverse mapcolonies vector sources and partners.' tags: - Location Name Based Search parameters: @@ -73,7 +73,7 @@ paths: type: string minLength: 3 maxLength: 100 - title: "Query" + title: 'Query' description: Text to search allowReserved: true - name: source @@ -82,7 +82,7 @@ paths: schema: type: array items: - $ref: "#/components/schemas/Source" + $ref: '#/components/schemas/Source' title: Source description: | Sources to include (if not specified, all sources will be queried) @@ -92,17 +92,17 @@ paths: schema: type: array items: - $ref: "#/components/schemas/Region" + $ref: '#/components/schemas/Region' description: Regions to include (if not specified, all Regions will be queried) title: Region description: Regions to include (if not specified, all Regions will be queried) - - $ref: "#/components/parameters/geo_context" - - $ref: "#/components/parameters/geo_context_mode" - - $ref: "#/components/parameters/limit" - - $ref: "#/components/parameters/disable_fuzziness" + - $ref: '#/components/parameters/geo_context' + - $ref: '#/components/parameters/geo_context_mode' + - $ref: '#/components/parameters/limit' + - $ref: '#/components/parameters/disable_fuzziness' responses: 200: - description: "OK" + description: 'OK' x-request-id: schema: type: string @@ -111,15 +111,15 @@ paths: content: application/json: schema: - "$ref": "#/components/schemas/genericGeocodingResponse" + '$ref': '#/components/schemas/genericGeocodingResponse' 400: - "$ref": "#/components/responses/BadRequest" + '$ref': '#/components/responses/BadRequest' 401: - "$ref": "#/components/responses/Unauthorized" + '$ref': '#/components/responses/Unauthorized' 403: - "$ref": "#/components/responses/Forbidden" + '$ref': '#/components/responses/Forbidden' 500: - "$ref": "#/components/responses/InternalError" + '$ref': '#/components/responses/InternalError' security: - x-api-key: [] x-user-id: [] @@ -129,10 +129,10 @@ paths: operationId: locationGetRegions tags: - Location Name Based Search - summary: "Get all available regions to filter on using location query" + summary: 'Get all available regions to filter on using location query' responses: 200: - description: "All regions" + description: 'All regions' content: application/json: schema: @@ -140,13 +140,13 @@ paths: items: type: string # return geojson with name property 400: - "$ref": "#/components/responses/BadRequest" + '$ref': '#/components/responses/BadRequest' 401: - "$ref": "#/components/responses/Unauthorized" + '$ref': '#/components/responses/Unauthorized' 403: - "$ref": "#/components/responses/Forbidden" + '$ref': '#/components/responses/Forbidden' 500: - "$ref": "#/components/responses/InternalError" + '$ref': '#/components/responses/InternalError' security: - x-api-key: [] x-user-id: [] @@ -156,10 +156,10 @@ paths: operationId: locationGetSources tags: - Location Name Based Search - summary: "Get all available sources to filter on using location query" + summary: 'Get all available sources to filter on using location query' responses: 200: - description: "All sources" + description: 'All sources' content: application/json: schema: @@ -167,13 +167,13 @@ paths: items: type: string 400: - "$ref": "#/components/responses/BadRequest" + '$ref': '#/components/responses/BadRequest' 401: - "$ref": "#/components/responses/Unauthorized" + '$ref': '#/components/responses/Unauthorized' 403: - "$ref": "#/components/responses/Forbidden" + '$ref': '#/components/responses/Forbidden' 500: - "$ref": "#/components/responses/InternalError" + '$ref': '#/components/responses/InternalError' security: - x-api-key: [] x-user-id: [] @@ -183,50 +183,50 @@ paths: operationId: controlGetTilesByQueryParams tags: - Control - summary: "Search tiles and sub tiles" + summary: 'Search tiles and sub tiles' description: |- Tiles are consisted of 3 characters.
You can query and get results from 2 characters.

If you define a tile (full name of it) you can query the sub tile associated with it.
You may also search tile based on 1 meter precision MGRS tile. parameters: - - name: "tile" - in: "query" - description: "Tile name" + - name: 'tile' + in: 'query' + description: 'Tile name' schema: - type: "string" + type: 'string' minLength: 2 maxLength: 3 - - name: "sub_tile" - in: "query" - description: "Sub tile number" + - name: 'sub_tile' + in: 'query' + description: 'Sub tile number' schema: - type: "string" - pattern: "^[1-9][0-9]*$" + type: 'string' + pattern: '^[1-9][0-9]*$' - name: mgrs - description: "1 meters MGRS Tile" - example: "18SUJ2338907395" + description: '1 meters MGRS Tile' + example: '18SUJ2338907395' in: query schema: type: string - - $ref: "#/components/parameters/geo_context" - - $ref: "#/components/parameters/geo_context_mode" - - $ref: "#/components/parameters/limit" - - $ref: "#/components/parameters/disable_fuzziness" + - $ref: '#/components/parameters/geo_context' + - $ref: '#/components/parameters/geo_context_mode' + - $ref: '#/components/parameters/limit' + - $ref: '#/components/parameters/disable_fuzziness' responses: 200: - description: "OK" + description: 'OK' content: application/json: schema: - $ref: "#/components/schemas/tilesSchema" + $ref: '#/components/schemas/tilesSchema' 400: - "$ref": "#/components/responses/BadRequest" + '$ref': '#/components/responses/BadRequest' 401: - "$ref": "#/components/responses/Unauthorized" + '$ref': '#/components/responses/Unauthorized' 403: - "$ref": "#/components/responses/Forbidden" + '$ref': '#/components/responses/Forbidden' 500: - "$ref": "#/components/responses/InternalError" + '$ref': '#/components/responses/InternalError' security: - x-api-key: [] x-user-id: [] @@ -235,51 +235,51 @@ paths: operationId: controlGetItemsByQueryParams tags: - Control - summary: "Search control items" - description: "" + summary: 'Search control items' + description: '' parameters: - - name: "command_name" - in: "query" - description: "Object command name of the item" + - name: 'command_name' + in: 'query' + description: 'Object command name of the item' schema: - type: "string" + type: 'string' required: true - - name: "tile" - in: "query" - description: "The tile the item in it (full name of it)" + - name: 'tile' + in: 'query' + description: 'The tile the item in it (full name of it)' schema: - type: "string" + type: 'string' minLength: 3 maxLength: 3 required: false - - name: "sub_tile" - in: "query" - description: "The sub tile the item in it (required if tile is defined)" + - name: 'sub_tile' + in: 'query' + description: 'The sub tile the item in it (required if tile is defined)' schema: - type: "string" - pattern: "^[1-9][0-9]*$" - example: "66" - - $ref: "#/components/parameters/geo_context" - - $ref: "#/components/parameters/geo_context_mode" - - $ref: "#/components/parameters/limit" - - $ref: "#/components/parameters/disable_fuzziness" + type: 'string' + pattern: '^[1-9][0-9]*$' + example: '66' + - $ref: '#/components/parameters/geo_context' + - $ref: '#/components/parameters/geo_context_mode' + - $ref: '#/components/parameters/limit' + - $ref: '#/components/parameters/disable_fuzziness' responses: 200: - description: "OK" + description: 'OK' content: application/json: schema: - $ref: "#/components/schemas/itemsSchema" + $ref: '#/components/schemas/itemsSchema' 400: - "$ref": "#/components/responses/BadRequest" + '$ref': '#/components/responses/BadRequest' 401: - "$ref": "#/components/responses/Unauthorized" + '$ref': '#/components/responses/Unauthorized' 403: - "$ref": "#/components/responses/Forbidden" + '$ref': '#/components/responses/Forbidden' 500: - "$ref": "#/components/responses/InternalError" + '$ref': '#/components/responses/InternalError' security: - x-api-key: [] x-user-id: [] @@ -288,42 +288,42 @@ paths: operationId: controlGetRoutesByQueryParams tags: - Control - summary: "Search routes and report control points" - description: "You can query and get control routes.

If you define a route (full name of it) you can query the control points associated with it. " + summary: 'Search routes and report control points' + description: 'You can query and get control routes.

If you define a route (full name of it) you can query the control points associated with it. ' parameters: - - name: "command_name" - in: "query" - description: "Object command name of the item" + - name: 'command_name' + in: 'query' + description: 'Object command name of the item' schema: - type: "string" + type: 'string' required: true - - name: "control_point" - in: "query" - description: "The associated report control point of the route" + - name: 'control_point' + in: 'query' + description: 'The associated report control point of the route' schema: - type: "string" - pattern: "^[1-9][0-9]*$" - - $ref: "#/components/parameters/geo_context" - - $ref: "#/components/parameters/geo_context_mode" - - $ref: "#/components/parameters/limit" - - $ref: "#/components/parameters/disable_fuzziness" + type: 'string' + pattern: '^[1-9][0-9]*$' + - $ref: '#/components/parameters/geo_context' + - $ref: '#/components/parameters/geo_context_mode' + - $ref: '#/components/parameters/limit' + - $ref: '#/components/parameters/disable_fuzziness' responses: 200: - description: "OK (can be routesSchema or itemsSchema)" + description: 'OK (can be routesSchema or itemsSchema)' content: application/json: schema: anyOf: - - $ref: "#/components/schemas/routesSchema" - - $ref: "#/components/schemas/itemsSchema" + - $ref: '#/components/schemas/routesSchema' + - $ref: '#/components/schemas/itemsSchema' 400: - "$ref": "#/components/responses/BadRequest" + '$ref': '#/components/responses/BadRequest' 401: - "$ref": "#/components/responses/Unauthorized" + '$ref': '#/components/responses/Unauthorized' 403: - "$ref": "#/components/responses/Forbidden" + '$ref': '#/components/responses/Forbidden' 500: - "$ref": "#/components/responses/InternalError" + '$ref': '#/components/responses/InternalError' security: - x-api-key: [] x-user-id: [] @@ -332,29 +332,29 @@ paths: operationId: getMGRSToGeom tags: - MGRS - summary: "Convert a MGRS string to Geometry in GeoJSON" + summary: 'Convert a MGRS string to Geometry in GeoJSON' parameters: - - name: "tile" - in: "query" - description: "MGRS tile string" + - name: 'tile' + in: 'query' + description: 'MGRS tile string' schema: - type: "string" + type: 'string' required: true responses: 200: - description: "OK" + description: 'OK' content: application/json: schema: - $ref: "#/components/schemas/mgrsTileSchema" + $ref: '#/components/schemas/mgrsTileSchema' 400: - "$ref": "#/components/responses/BadRequest" + '$ref': '#/components/responses/BadRequest' 401: - "$ref": "#/components/responses/Unauthorized" + '$ref': '#/components/responses/Unauthorized' 403: - "$ref": "#/components/responses/Forbidden" + '$ref': '#/components/responses/Forbidden' 500: - "$ref": "#/components/responses/InternalError" + '$ref': '#/components/responses/InternalError' security: - x-api-key: [] x-user-id: [] @@ -363,33 +363,33 @@ paths: operationId: convertionGetLatLonToMgrs tags: - Convertions - summary: "Convert a WGS84 coordinate to US Army MGRS / Control Grid" + summary: 'Convert a WGS84 coordinate to US Army MGRS / Control Grid' parameters: - - name: "lat" - in: "query" - description: "Latitude of the coordinate" + - name: 'lat' + in: 'query' + description: 'Latitude of the coordinate' schema: - type: "number" + type: 'number' minimum: -90 maximum: 90 required: true - - name: "lon" - in: "query" - description: "Longitude of the coordinate" + - name: 'lon' + in: 'query' + description: 'Longitude of the coordinate' schema: - type: "number" + type: 'number' minimum: -180 maximum: 180 required: true - - name: "target_grid" + - name: 'target_grid' in: query description: Choose target grid schema: type: string - enum: ["control", "MGRS"] + enum: ['control', 'MGRS'] responses: 200: - description: "OK" + description: 'OK' content: application/json: schema: @@ -398,13 +398,13 @@ paths: geojson: type: object # Make it geojson 400: - "$ref": "#/components/responses/BadRequest" + '$ref': '#/components/responses/BadRequest' 401: - "$ref": "#/components/responses/Unauthorized" + '$ref': '#/components/responses/Unauthorized' 403: - "$ref": "#/components/responses/Forbidden" + '$ref': '#/components/responses/Forbidden' 500: - "$ref": "#/components/responses/InternalError" + '$ref': '#/components/responses/InternalError' security: - x-api-key: [] x-user-id: [] @@ -414,11 +414,11 @@ paths: operationId: postSearchFeedback tags: - Feedback - summary: "Retrieve feedback from services about quality of results" + summary: 'Retrieve feedback from services about quality of results' parameters: - name: choosen_result_id - in: "query" - description: "The result ID of chosen search" + in: 'query' + description: 'The result ID of chosen search' schema: type: string minimum: 1 @@ -434,17 +434,17 @@ paths: required: true responses: 204: - description: "OK - No Content" + description: 'OK - No Content' 400: - "$ref": "#/components/responses/BadRequest" + '$ref': '#/components/responses/BadRequest' 404: - $ref: "#/components/responses/NotFound" + $ref: '#/components/responses/NotFound' 401: - "$ref": "#/components/responses/Unauthorized" + '$ref': '#/components/responses/Unauthorized' 403: - "$ref": "#/components/responses/Forbidden" + '$ref': '#/components/responses/Forbidden' 500: - "$ref": "#/components/responses/InternalError" + '$ref': '#/components/responses/InternalError' security: - x-api-key: [] x-user-id: [] @@ -452,55 +452,55 @@ paths: components: responses: BadRequest: - description: "Invalid Request" + description: 'Invalid Request' content: application/json: schema: - $ref: "#/components/schemas/errorSchema" + $ref: '#/components/schemas/errorSchema' Unauthorized: - description: "Please provide a valid token" + description: 'Please provide a valid token' content: application/json: schema: - $ref: "#/components/schemas/errorSchema" + $ref: '#/components/schemas/errorSchema' NotFound: - description: "Resource Not Found" + description: 'Resource Not Found' content: application/json: schema: - $ref: "#/components/schemas/errorSchema" + $ref: '#/components/schemas/errorSchema' Forbidden: - description: "Token is not valid" + description: 'Token is not valid' content: application/json: schema: - $ref: "#/components/schemas/errorSchema" + $ref: '#/components/schemas/errorSchema' InternalError: - description: "Invalid Request" + description: 'Invalid Request' content: application/json: schema: - $ref: "#/components/schemas/errorSchema" + $ref: '#/components/schemas/errorSchema' parameters: disable_fuzziness: - name: "disable_fuzziness" - in: "query" + name: 'disable_fuzziness' + in: 'query' description: |- If an accurate result is obtained, only it will be returned schema: - $ref: "#/components/schemas/disable_fuzziness" + $ref: '#/components/schemas/disable_fuzziness' limit: - name: "limit" - in: "query" - description: "Maximum number of results" + name: 'limit' + in: 'query' + description: 'Maximum number of results' schema: default: 5 minimum: 1 maximum: 15 - type: "number" + type: 'number' geo_context: - in: "query" - name: "geo_context" + in: 'query' + name: 'geo_context' description: |- Geo context of search. @@ -511,185 +511,185 @@ components: {"lon":value,"lat":value,"radius":value}
{"x":value,"y":value,"zone": number, "radius":value} schema: - $ref: "#/components/schemas/geo_context" + $ref: '#/components/schemas/geo_context' geo_context_mode: - name: "geo_context_mode" - in: "query" - description: "Choose whether geo_context query parameter will be a filter or a bias value" + name: 'geo_context_mode' + in: 'query' + description: 'Choose whether geo_context query parameter will be a filter or a bias value' schema: type: string - enum: ["filter", "bias"] + enum: ['filter', 'bias'] schemas: errorSchema: - type: "object" + type: 'object' required: - - "message" + - 'message' properties: message: - type: "string" + type: 'string' status: - type: "number" + type: 'number' subTileSchema: - type: "object" - description: "An object represnting a sub tile" + type: 'object' + description: 'An object represnting a sub tile' required: - tileName - subTileNumber properties: tileName: - type: "string" + type: 'string' subTileNumber: - type: "array" + type: 'array' items: - type: "number" + type: 'number' minimum: 0 maximum: 100 minItems: 3 maxItems: 3 tileSchema: - type: "object" + type: 'object' properties: type: - type: "string" - enum: ["Feature"] + type: 'string' + enum: ['Feature'] geometry: - $ref: "#/components/schemas/Polygon" + $ref: '#/components/schemas/Polygon' properties: - type: "object" + type: 'object' required: - - "TYPE" - - "tile_name" + - 'TYPE' + - 'tile_name' properties: TYPE: - type: "string" + type: 'string' tile_name: - type: "string" + type: 'string' sub_tile_id: - type: "string" + type: 'string' SUB_TILE_NUMBER: - type: "array" + type: 'array' items: - type: "number" + type: 'number' minimum: 0 maximum: 100 minItems: 3 maxItems: 3 mgrsTileSchema: - type: "object" + type: 'object' properties: type: - type: "string" - enum: ["Feature"] + type: 'string' + enum: ['Feature'] geometry: - $ref: "#/components/schemas/Polygon" + $ref: '#/components/schemas/Polygon' properties: - type: "object" + type: 'object' tilesSchema: - type: "object" - description: "GeoJson feature collection representing a tile (Polygon)" + type: 'object' + description: 'GeoJson feature collection representing a tile (Polygon)' required: - - "type" - - "features" + - 'type' + - 'features' properties: type: - type: "string" + type: 'string' enum: - - "FeatureCollection" + - 'FeatureCollection' features: - type: "array" + type: 'array' items: - $ref: "#/components/schemas/tileSchema" + $ref: '#/components/schemas/tileSchema' itemsSchema: - type: "object" - description: "GeoJson feature collection representing an item" + type: 'object' + description: 'GeoJson feature collection representing an item' required: - - "type" - - "features" + - 'type' + - 'features' properties: type: - type: "string" + type: 'string' enum: - - "FeatureCollection" + - 'FeatureCollection' features: - type: "array" + type: 'array' items: - type: "object" + type: 'object' properties: geometry: oneOf: - - $ref: "#/components/schemas/Point" - - $ref: "#/components/schemas/Polygon" + - $ref: '#/components/schemas/Point' + - $ref: '#/components/schemas/Polygon' properties: - type: "object" + type: 'object' required: - - "TYPE" - - "object_command_name" - - "entity_heb" - - "sub_tile_id" + - 'TYPE' + - 'object_command_name' + - 'entity_heb' + - 'sub_tile_id' properties: TYPE: - type: "string" + type: 'string' object_command_name: - type: "string" + type: 'string' entity_heb: - type: "string" + type: 'string' tile_name: - type: "string" + type: 'string' sub_tile_id: - type: "string" + type: 'string' routesSchema: - type: "object" - description: "GeoJson feature collection representing a route (MultiLineString, LineString)" + type: 'object' + description: 'GeoJson feature collection representing a route (MultiLineString, LineString)' required: - - "type" - - "features" + - 'type' + - 'features' properties: type: - type: "string" + type: 'string' enum: - - "FeatureCollection" + - 'FeatureCollection' features: - type: "array" + type: 'array' items: - type: "object" + type: 'object' properties: geometry: oneOf: - - $ref: "#/components/schemas/LineString" - - $ref: "#/components/schemas/MultiLineString" + - $ref: '#/components/schemas/LineString' + - $ref: '#/components/schemas/MultiLineString' properties: - type: "object" + type: 'object' required: - - "TYPE" - - "object_command_name" - - "entity_heb" + - 'TYPE' + - 'object_command_name' + - 'entity_heb' properties: TYPE: - type: "string" + type: 'string' object_command_name: - type: "string" + type: 'string' entity_heb: - type: "string" + type: 'string' SECTION: - type: "string" + type: 'string' genericGeocodingResponse: # we need discriminator for Control and Location - type: "object" - description: "GeoJson feature collection representing an item" + type: 'object' + description: 'GeoJson feature collection representing an item' required: - - "type" - - "features" - - "geocoding" + - 'type' + - 'features' + - 'geocoding' properties: type: - type: "string" + type: 'string' enum: - - "FeatureCollection" + - 'FeatureCollection' geocoding: type: object required: - - "query" - - "response" + - 'query' + - 'response' properties: version: type: string @@ -698,10 +698,10 @@ components: query: type: object required: - - "text" - - "limit" - - "geo_context" - - "disable_fuzziness" + - 'text' + - 'limit' + - 'geo_context' + - 'disable_fuzziness' properties: text: type: string @@ -712,15 +712,15 @@ components: minimum: 1 maximum: 10 geo_context: - $ref: "#/components/schemas/geo_context" + $ref: '#/components/schemas/geo_context' disable_fuzziness: - $ref: "#/components/schemas/disable_fuzziness" + $ref: '#/components/schemas/disable_fuzziness' response: type: object required: - - "max_score" - - "results_count" - - "match_latency" + - 'max_score' + - 'results_count' + - 'match_latency' properties: max_score: type: number @@ -735,16 +735,16 @@ components: minimum: 0 maximum: 5000 bbox: - $ref: "#/components/schemas/BoundingBox" + $ref: '#/components/schemas/BoundingBox' features: - type: "array" + type: 'array' items: - type: "object" + type: 'object' properties: geometry: - $ref: "#/components/schemas/Geometry" + $ref: '#/components/schemas/Geometry' properties: - type: "object" + type: 'object' additionalProperties: true required: - id @@ -854,69 +854,69 @@ components: externalDocs: url: http://geojson.org/geojson-spec.html#id2 allOf: - - $ref: "#/components/schemas/Geometry" + - $ref: '#/components/schemas/Geometry' - properties: coordinates: - $ref: "#/components/schemas/Point3D" + $ref: '#/components/schemas/Point3D' LineString: type: object description: Geojson geometry externalDocs: url: http://geojson.org/geojson-spec.html#id3 allOf: - - $ref: "#/components/schemas/Geometry" + - $ref: '#/components/schemas/Geometry' - properties: coordinates: type: array items: - $ref: "#/components/schemas/Point3D" + $ref: '#/components/schemas/Point3D' Polygon: type: object description: Geojson geometry externalDocs: url: http://geojson.org/geojson-spec.html#id4 allOf: - - $ref: "#/components/schemas/Geometry" + - $ref: '#/components/schemas/Geometry' - properties: coordinates: type: array items: type: array items: - $ref: "#/components/schemas/Point3D" + $ref: '#/components/schemas/Point3D' MultiPoint: type: object description: Geojson geometry externalDocs: url: http://geojson.org/geojson-spec.html#id5 allOf: - - $ref: "#/components/schemas/Geometry" + - $ref: '#/components/schemas/Geometry' - properties: coordinates: type: array items: - $ref: "#/components/schemas/Point3D" + $ref: '#/components/schemas/Point3D' MultiLineString: type: object description: Geojson geometry externalDocs: url: http://geojson.org/geojson-spec.html#id6 allOf: - - $ref: "#/components/schemas/Geometry" + - $ref: '#/components/schemas/Geometry' - properties: coordinates: type: array items: type: array items: - $ref: "#/components/schemas/Point3D" + $ref: '#/components/schemas/Point3D' MultiPolygon: type: object description: Geojson geometry externalDocs: url: http://geojson.org/geojson-spec.html#id6 allOf: - - $ref: "#/components/schemas/Geometry" + - $ref: '#/components/schemas/Geometry' - properties: coordinates: type: array @@ -925,7 +925,7 @@ components: items: type: array items: - $ref: "#/components/schemas/Point3D" + $ref: '#/components/schemas/Point3D' Source: type: string @@ -935,8 +935,8 @@ components: title: Region BoundingBox: type: array - description: "Bounding box array that contains [minX,minY,maxX,maxY]" - example: "[-74.382527,40.477003,-73.322346,40.916383]" + description: 'Bounding box array that contains [minX,minY,maxX,maxY]' + example: '[-74.382527,40.477003,-73.322346,40.916383]' items: type: number minLength: 4 @@ -968,18 +968,18 @@ components: - type: object properties: bbox: - $ref: "#/components/schemas/BoundingBox" - - $ref: "#/components/schemas/WGS84Circle" - - $ref: "#/components/schemas/UTMCircle" + $ref: '#/components/schemas/BoundingBox' + - $ref: '#/components/schemas/WGS84Circle' + - $ref: '#/components/schemas/UTMCircle' disable_fuzziness: type: boolean default: false securitySchemes: x-api-key: - type: "apiKey" - name: "x-api-key" - in: "header" + type: 'apiKey' + name: 'x-api-key' + in: 'header' x-user-id: - type: "apiKey" - name: "x-user-id" - in: "header" + type: 'apiKey' + name: 'x-user-id' + in: 'header' diff --git a/src/common/constants.ts b/src/common/constants.ts index 83b3a228..ac72a036 100644 --- a/src/common/constants.ts +++ b/src/common/constants.ts @@ -24,4 +24,5 @@ export const ON_SIGNAL = Symbol('onSignal'); export const HEALTHCHECK = Symbol('healthcheck'); export const REDIS_SYMBOL = Symbol('REDIS'); +export const REDIS_TTL = 300; export const elasticConfigPath = 'db.elastic'; diff --git a/src/common/errors.ts b/src/common/errors.ts index c56a5641..374b146c 100644 --- a/src/common/errors.ts +++ b/src/common/errors.ts @@ -50,4 +50,4 @@ export class NotImplementedError extends Error implements HttpError { } } -export class TimeoutError extends Error {} \ No newline at end of file +export class TimeoutError extends Error {} diff --git a/src/common/middlewares/feedbackApi.middleware.ts b/src/common/middlewares/feedbackApi.middleware.ts index c3a178dd..6064e86a 100644 --- a/src/common/middlewares/feedbackApi.middleware.ts +++ b/src/common/middlewares/feedbackApi.middleware.ts @@ -1,34 +1,35 @@ +import * as crypto from 'node:crypto'; import { Request, Response, NextFunction } from 'express'; -import { SERVICES } from '../constants'; import { Logger } from '@map-colonies/js-logger'; import { inject, injectable } from 'tsyringe'; +import { REDIS_TTL, SERVICES } from '../constants'; import { RedisClient } from '../redis'; -import * as crypto from 'node:crypto'; import { GeocodingResponse } from '../interfaces'; @injectable() -export class feedbackApiMiddlewareManager { +export class FeedbackApiMiddlewareManager { public constructor(@inject(SERVICES.LOGGER) private readonly logger: Logger, @inject(SERVICES.REDIS) private readonly redis: RedisClient) {} - saveResponses = async (req: Request, res: Response, next: NextFunction) => { + // eslint-disable-next-line @typescript-eslint/explicit-function-return-type + public saveResponses = (req: Request, res: Response, next: NextFunction) => { const reqId = crypto.randomUUID(); const redisClient = this.redis; const logger = this.logger; logger.info({ msg: 'saving response to redis' }); - let geocodingResponseDetails: GeocodingResponse = { + const geocodingResponseDetails: GeocodingResponse = { userId: req.headers['x-user-id'] as string, - response: JSON.parse('{}'), + response: JSON.parse('{}') as JSON, respondedAt: new Date(), }; const originalJson = res.json; - const logJson = async function (this: Response, body: any): Promise>> { - // console.log('Response:', body); - geocodingResponseDetails.response = await body; + const logJson = async function (this: Response, body: JSON): Promise { + // console.log('Response:', body); + geocodingResponseDetails.response = body; try { - await redisClient.setEx(reqId, 300, JSON.stringify(geocodingResponseDetails)); + await redisClient.setEx(reqId, REDIS_TTL, JSON.stringify(geocodingResponseDetails)); logger.info({ msg: 'saving response to redis' }); } catch (err) { logger.error('Error setting key:', err); @@ -43,13 +44,13 @@ export class feedbackApiMiddlewareManager { }; } -async function setKeyWithTTL(key: string, value: string, redis: RedisClient) { - try { - await redis.set(key, value, { - EX: 300, // 5 minutes in seconds - }); - console.log(`Key '${key}' set with a TTL of 5 minutes.`); - } catch (err) { - console.error('Error setting key:', err); - } -} +// async function setKeyWithTTL(key: string, value: string, redis: RedisClient) { +// try { +// await redis.set(key, value, { +// EX: 300, // 5 minutes in seconds +// }); +// console.log(`Key '${key}' set with a TTL of 5 minutes.`); +// } catch (err) { +// console.error('Error setting key:', err); +// } +// } diff --git a/src/common/utils.ts b/src/common/utils.ts index 0b64a1c1..8421389d 100644 --- a/src/common/utils.ts +++ b/src/common/utils.ts @@ -92,4 +92,4 @@ export const promiseTimeout = async (ms: number, promise: Promise): Promis // returns a race between our timeout and the passed in promise return Promise.race([promise, timeout]); -}; \ No newline at end of file +}; diff --git a/src/containerConfig.ts b/src/containerConfig.ts index 4424fa4b..14dbe465 100644 --- a/src/containerConfig.ts +++ b/src/containerConfig.ts @@ -7,6 +7,7 @@ import { CleanupRegistry } from '@map-colonies/cleanup-registry'; import { DependencyContainer } from 'tsyringe/dist/typings/types'; import jsLogger, { LoggerOptions } from '@map-colonies/js-logger'; import { Metrics } from '@map-colonies/telemetry'; +import { HealthCheck } from '@godaddy/terminus'; import { HEALTHCHECK, SERVICES, SERVICE_NAME } from './common/constants'; import { tracing } from './common/tracing'; import { InjectionObject, registerDependencies } from './common/dependencyRegistration'; @@ -25,7 +26,6 @@ import { GEOTEXT_SEARCH_ROUTER_SYMBOL, geotextSearchRouterFactory } from './loca import { cronLoadTileLatLonDataFactory, cronLoadTileLatLonDataSymbol } from './latLon/DAL/latLonDAL'; import { ITEM_REPOSITORY_SYMBOL, itemRepositoryFactory } from './control/item/DAL/itemRepository'; import { healthCheckFunctionFactory, RedisClient, redisClientFactory } from './common/redis'; -import { HealthCheck } from '@godaddy/terminus'; export interface RegisterOptions { override?: InjectionObject[]; @@ -58,7 +58,7 @@ export const registerExternalValues = async (options?: RegisterOptions): Promise postInjectionHook: async (deps: DependencyContainer): Promise => { const elasticClients = deps.resolve(SERVICES.ELASTIC_CLIENTS); try { - const response = await Promise.all([elasticClients.control?.ping(), elasticClients.geotext?.ping()]); + const response = await Promise.all([elasticClients.control.ping(), elasticClients.geotext.ping()]); response.forEach((res) => { if (!res) { logger.error('Failed to connect to Elasticsearch', res); @@ -119,10 +119,11 @@ export const registerExternalValues = async (options?: RegisterOptions): Promise { token: 'onSignal', provider: { - useFactory: instancePerContainerCachingFactory(async (): Promise => { - const promises: Promise[] = [tracing.stop(), metrics.stop()]; - }), - useValue: cleanupRegistry.trigger.bind(cleanupRegistry), + useValue: { + useValue: async (): Promise => { + await Promise.all([tracing.stop(), metrics.stop()]); + }, + }, }, }, ]; diff --git a/src/control/route/DAL/routeRepository.ts b/src/control/route/DAL/routeRepository.ts index d1bfd737..34da5b2e 100644 --- a/src/control/route/DAL/routeRepository.ts +++ b/src/control/route/DAL/routeRepository.ts @@ -1,4 +1,3 @@ -import { Logger } from '@map-colonies/js-logger'; import { estypes } from '@elastic/elasticsearch'; import { FactoryFunction } from 'tsyringe'; import { ElasticClient } from '../../../common/elastic'; @@ -11,7 +10,7 @@ import { additionalControlSearchProperties } from '../../utils'; import { RouteQueryParams, queryForControlPointInRoute, queryForRoute } from './queries'; // eslint-disable-next-line @typescript-eslint/explicit-function-return-type -const createRouteRepository = (client: ElasticClient, config: IConfig, logger: Logger) => { +const createRouteRepository = (client: ElasticClient, config: IConfig) => { return { async getRoutes(routeQueryParams: RouteQueryParams, size: number): Promise> { const response = await queryElastic(client, { ...additionalControlSearchProperties(config, size), ...queryForRoute(routeQueryParams) }); @@ -38,8 +37,7 @@ export type RouteRepository = ReturnType; export const routeRepositoryFactory: FactoryFunction = (depContainer) => { return createRouteRepository( depContainer.resolve(SERVICES.ELASTIC_CLIENTS).control, - depContainer.resolve(SERVICES.CONFIG), - depContainer.resolve(SERVICES.LOGGER) + depContainer.resolve(SERVICES.CONFIG) ); }; diff --git a/src/control/tile/DAL/tileRepository.ts b/src/control/tile/DAL/tileRepository.ts index c7f764ab..a3063b08 100644 --- a/src/control/tile/DAL/tileRepository.ts +++ b/src/control/tile/DAL/tileRepository.ts @@ -1,4 +1,3 @@ -import { Logger } from '@map-colonies/js-logger'; import { estypes } from '@elastic/elasticsearch'; import { FactoryFunction } from 'tsyringe'; import { ElasticClient, ElasticClients } from '../../../common/elastic'; @@ -10,7 +9,7 @@ import { additionalControlSearchProperties } from '../../utils'; import { queryForTiles, queryForSubTiles, TileQueryParams } from './queries'; // eslint-disable-next-line @typescript-eslint/explicit-function-return-type -const createTileRepository = (client: ElasticClient, config: IConfig, logger: Logger) => { +const createTileRepository = (client: ElasticClient, config: IConfig) => { return { // eslint-disable-next-line @typescript-eslint/explicit-module-boundary-types async getTiles(tileQueryParams: TileQueryParams & Required>, size: number): Promise> { @@ -30,11 +29,7 @@ const createTileRepository = (client: ElasticClient, config: IConfig, logger: Lo export type TileRepository = ReturnType; export const tileRepositoryFactory: FactoryFunction = (depContainer) => { - return createTileRepository( - depContainer.resolve(SERVICES.ELASTIC_CLIENTS).control, - depContainer.resolve(SERVICES.CONFIG), - depContainer.resolve(SERVICES.LOGGER) - ); + return createTileRepository(depContainer.resolve(SERVICES.ELASTIC_CLIENTS).control, depContainer.resolve(SERVICES.CONFIG)); }; export const TILE_REPOSITORY_SYMBOL = Symbol('TILE_REPOSITORY_SYMBOL'); diff --git a/src/control/utils.ts b/src/control/utils.ts index 31aea8f2..13c47fa6 100644 --- a/src/control/utils.ts +++ b/src/control/utils.ts @@ -104,4 +104,4 @@ export const additionalControlSearchProperties = (config: IConfig, size: number) index: config.get(elasticConfigPath).control.properties.index as string, // eslint-disable-next-line @typescript-eslint/naming-convention _source: CONTROL_FIELDS, -}); +}); \ No newline at end of file diff --git a/src/index.ts b/src/index.ts index 4ee6fbac..adad9e8d 100644 --- a/src/index.ts +++ b/src/index.ts @@ -7,7 +7,7 @@ import { DependencyContainer } from 'tsyringe'; import { createTerminus } from '@godaddy/terminus'; import { Logger } from '@map-colonies/js-logger'; import config from 'config'; -import { DEFAULT_SERVER_PORT, HEALTHCHECK, ON_SIGNAL, SERVICES } from './common/constants'; +import { DEFAULT_SERVER_PORT, ON_SIGNAL, SERVICES } from './common/constants'; import { getApp } from './app'; let depContainer: DependencyContainer | undefined; diff --git a/src/location/controllers/locationController.ts b/src/location/controllers/locationController.ts index d1efeecd..a8ec476e 100644 --- a/src/location/controllers/locationController.ts +++ b/src/location/controllers/locationController.ts @@ -50,11 +50,11 @@ export class GeotextSearchController { } }; - public getRegions: GetRegionshHandler = (req, res, next) => { + public getRegions: GetRegionshHandler = (req, res) => { return res.status(httpStatus.OK).json(this.manager.regions()); }; - public getSources: GetSourcesHandler = (req, res, next) => { + public getSources: GetSourcesHandler = (req, res) => { return res.status(httpStatus.OK).json(this.manager.sources()); }; } diff --git a/src/location/parsing.ts b/src/location/parsing.ts index 08e83acc..a84d4cd6 100644 --- a/src/location/parsing.ts +++ b/src/location/parsing.ts @@ -2,10 +2,10 @@ /* istanbul ignore file */ import { XMLParser } from 'fast-xml-parser'; -type Highlight = { +interface Highlight { em: string | string[]; '#text': string; -}; +} const HIERARCHY_OF_INTEREST = 3; diff --git a/src/location/utils.ts b/src/location/utils.ts index 6f0191b9..679245b9 100644 --- a/src/location/utils.ts +++ b/src/location/utils.ts @@ -144,7 +144,7 @@ export const convertResult = ( sub_placetype: feature?.sub_placetype, regions: feature?.region.map((region) => ({ region: region, - sub_regions: feature.sub_region.filter((sub_region) => (regionCollection ?? {})[region ?? '']?.includes(sub_region)), + sub_regions: feature.sub_region.filter((sub_region) => (regionCollection ?? {})[region ?? ''].includes(sub_region)), })), }, }; diff --git a/src/serverBuilder.ts b/src/serverBuilder.ts index 5d6740ae..a2450759 100644 --- a/src/serverBuilder.ts +++ b/src/serverBuilder.ts @@ -16,7 +16,7 @@ import { ROUTE_ROUTER_SYMBOL } from './control/route/routes/routeRouter'; import { LAT_LON_ROUTER_SYMBOL } from './latLon/routes/latLonRouter'; import { GEOTEXT_SEARCH_ROUTER_SYMBOL } from './location/routes/locationRouter'; import { cronLoadTileLatLonDataSymbol } from './latLon/DAL/latLonDAL'; -import { feedbackApiMiddlewareManager } from './common/middlewares/feedbackApi.middleware'; +import { FeedbackApiMiddlewareManager } from './common/middlewares/feedbackApi.middleware'; @injectable() export class ServerBuilder { @@ -31,7 +31,7 @@ export class ServerBuilder { @inject(LAT_LON_ROUTER_SYMBOL) private readonly latLonRouter: Router, @inject(GEOTEXT_SEARCH_ROUTER_SYMBOL) private readonly geotextRouter: Router, @inject(cronLoadTileLatLonDataSymbol) private readonly cronLoadTileLatLonData: void, - @inject(feedbackApiMiddlewareManager) private readonly feedbackApiMiddleware: feedbackApiMiddlewareManager + @inject(FeedbackApiMiddlewareManager) private readonly feedbackApiMiddleware: FeedbackApiMiddlewareManager ) { this.serverInstance = express(); // eslint-disable-next-line @typescript-eslint/no-unused-expressions diff --git a/tests/integration/location/mockObjects.ts b/tests/integration/location/mockObjects.ts index 39745b7e..9f6c4867 100644 --- a/tests/integration/location/mockObjects.ts +++ b/tests/integration/location/mockObjects.ts @@ -408,4 +408,4 @@ export const PARIS_HIERRARCHY: HierarchySearchHit = { region: 'FRANCE', text: 'Paris', weight: 1.1, -}; \ No newline at end of file +}; From 4ea3e2c818874873d27a5acf024a2650c0e5da9a Mon Sep 17 00:00:00 2001 From: Niv Greenstein Date: Wed, 4 Sep 2024 15:49:26 +0300 Subject: [PATCH 169/262] feat: created s3 repository --- src/common/constants.ts | 1 + src/common/s3/index.ts | 48 +++++++++++++++++++++++++++ src/common/s3/s3Repository.ts | 62 +++++++++++++++++++++++++++++++++++ 3 files changed, 111 insertions(+) create mode 100644 src/common/s3/index.ts create mode 100644 src/common/s3/s3Repository.ts diff --git a/src/common/constants.ts b/src/common/constants.ts index 5ab1d8b1..1d45c6d8 100644 --- a/src/common/constants.ts +++ b/src/common/constants.ts @@ -14,6 +14,7 @@ export const SERVICES: Record = { METER: Symbol('Meter'), APPLICATION: Symbol('Application'), ELASTIC_CLIENTS: Symbol('ElasticClients'), + S3_CLIENT: Symbol('S3Client'), }; /* eslint-enable @typescript-eslint/naming-convention */ diff --git a/src/common/s3/index.ts b/src/common/s3/index.ts new file mode 100644 index 00000000..4b145662 --- /dev/null +++ b/src/common/s3/index.ts @@ -0,0 +1,48 @@ +import * as https from 'https'; +import * as http from 'http'; +import { S3Client, S3ClientConfig } from '@aws-sdk/client-s3'; +import { NodeHttpHandler } from '@smithy/node-http-handler'; +import { Logger } from '@map-colonies/js-logger'; +import { DependencyContainer, FactoryFunction } from 'tsyringe'; +import { IConfig } from '../interfaces'; +import { SERVICES } from '../constants'; + +const createConnectionOptions = (clientOptions: S3ClientConfig): S3ClientConfig => ({ + ...clientOptions, + forcePathStyle: true, + requestHandler: new NodeHttpHandler({ + httpsAgent: new https.Agent({ + rejectUnauthorized: false, + }), + httpAgent: new http.Agent(), + }), +}); + +const initS3Client = (clientOptions: S3ClientConfig): S3Client => { + const client = new S3Client(createConnectionOptions(clientOptions)); + return client; +}; + +export const s3ClientFactory: FactoryFunction = (container: DependencyContainer): S3Client => { + const config = container.resolve(SERVICES.CONFIG); + const logger = container.resolve(SERVICES.LOGGER); + + const s3Config = config.get(s3ConfigPath); + + const client = initS3Client(s3Config); + logger.info(`S3 Client is initialized`); + + return client; +}; + +export type S3Config = S3ClientConfig & { bucketName: string } & { + files: { + [key: string]: + | { + bucket: string; + fileName: string; + } + | undefined; + }; +}; +export const s3ConfigPath = 'db.s3'; diff --git a/src/common/s3/s3Repository.ts b/src/common/s3/s3Repository.ts new file mode 100644 index 00000000..c247379b --- /dev/null +++ b/src/common/s3/s3Repository.ts @@ -0,0 +1,62 @@ +/* eslint-disable @typescript-eslint/naming-convention */ +import fs from 'fs'; +import { Logger } from '@map-colonies/js-logger'; +import { FactoryFunction } from 'tsyringe'; +import { GetObjectCommand, S3Client } from '@aws-sdk/client-s3'; +import { SERVICES } from '../constants'; +import { IConfig } from '../interfaces'; +import { S3Config, s3ConfigPath } from '.'; + +// eslint-disable-next-line @typescript-eslint/explicit-function-return-type +const createS3Repository = (s3Client: S3Client, config: IConfig, logger: Logger) => { + return { + async downloadFile(key: keyof S3Config['files']): Promise { + try { + const fileData = config.get(s3ConfigPath).files[key]; + if (!fileData) { + throw new Error(`${key} data is missing in the configuration`); + } + const { bucket: Bucket, fileName: Key } = fileData; + logger.info(`Downloading ${Key} file from S3`); + + const command = new GetObjectCommand({ + Bucket, + Key, + }); + + const { Body } = await s3Client.send(command); + + const filePath = __dirname + '/table.json'; + + await new Promise((resolve, reject) => { + (Body as NodeJS.ReadableStream) + .pipe(fs.createWriteStream(filePath)) + .on('error', (err: Error) => { + reject(err); + }) + .on('close', () => { + logger.info('table.json file was downloaded successfully'); + resolve(); + }); + }); + + return filePath; + } catch (error) { + logger.error(`Error while downloading ${key}'s data from S3. Error: ${(error as Error).message}`); + throw error; + } + }, + }; +}; + +export const s3RepositoryFactory: FactoryFunction = (depContainer) => { + return createS3Repository( + depContainer.resolve(SERVICES.S3_CLIENT), + depContainer.resolve(SERVICES.CONFIG), + depContainer.resolve(SERVICES.LOGGER) + ); +}; + +export type S3Repository = ReturnType; + +export const S3_REPOSITORY_SYMBOL = Symbol('S3_REPOSITORY_SYMBOL'); From 6c0aed31fdbd11e996c223178bf33ee15a7ecde7 Mon Sep 17 00:00:00 2001 From: Niv Greenstein Date: Wed, 4 Sep 2024 15:52:10 +0300 Subject: [PATCH 170/262] fix: fixed target_grid typo --- src/latLon/controllers/latLonController.ts | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/src/latLon/controllers/latLonController.ts b/src/latLon/controllers/latLonController.ts index e90b607c..85175bb8 100644 --- a/src/latLon/controllers/latLonController.ts +++ b/src/latLon/controllers/latLonController.ts @@ -24,7 +24,7 @@ type GetCoordinatesHandler = RequestHandler< undefined, { [key: string]: unknown } & Feature, undefined, - WGS84Coordinate & { target_gird: 'control' | 'MGRS' } + WGS84Coordinate & { target_grid: 'control' | 'MGRS' } >; export interface GetLatLonToTileQueryParams extends WGS84Coordinate {} @@ -111,7 +111,7 @@ export class LatLonController { public getCoordinates: GetCoordinatesHandler = async (req, res, next) => { try { - const { lat, lon, target_gird } = req.query; + const { lat, lon, target_grid } = req.query; let response: | ({ @@ -119,7 +119,7 @@ export class LatLonController { } & Feature) | undefined = undefined; - if (target_gird === 'control') { + if (target_grid === 'control') { response = await this.manager.latLonToTile({ lat, lon }); } else { response = this.manager.latLonToMGRS({ lat, lon }); From 58e8c7c78e03c5b307ed33ba56a0c65af96b78ba Mon Sep 17 00:00:00 2001 From: Niv Greenstein Date: Wed, 4 Sep 2024 15:58:26 +0300 Subject: [PATCH 171/262] feat: inject s3 --- src/containerConfig.ts | 23 +++++++++++++---------- 1 file changed, 13 insertions(+), 10 deletions(-) diff --git a/src/containerConfig.ts b/src/containerConfig.ts index e6e49d95..c145d8ea 100644 --- a/src/containerConfig.ts +++ b/src/containerConfig.ts @@ -1,6 +1,6 @@ import config from 'config'; import { getOtelMixin } from '@map-colonies/telemetry'; -import { DataSource } from 'typeorm'; +import { ListBucketsCommand, S3Client } from '@aws-sdk/client-s3'; import { instancePerContainerCachingFactory } from 'tsyringe'; import { trace, metrics as OtelMetrics } from '@opentelemetry/api'; import { DependencyContainer } from 'tsyringe/dist/typings/types'; @@ -16,13 +16,13 @@ import { TILE_ROUTER_SYMBOL, tileRouterFactory } from './control/tile/routes/til import { ITEM_ROUTER_SYMBOL, itemRouterFactory } from './control/item/routes/itemRouter'; import { ROUTE_REPOSITORY_SYMBOL, routeRepositoryFactory } from './control/route/DAL/routeRepository'; import { ROUTE_ROUTER_SYMBOL, routeRouterFactory } from './control/route/routes/routeRouter'; -import { postgresClientFactory } from './common/postgresql'; -import { LATLON_CUSTOM_REPOSITORY_SYMBOL, latLonRepositoryFactory } from './latLon/DAL/latLonRepository'; import { LAT_LON_ROUTER_SYMBOL, latLonRouterFactory } from './latLon/routes/latLonRouter'; import { GEOTEXT_REPOSITORY_SYMBOL, geotextRepositoryFactory } from './location/DAL/locationRepository'; import { GEOTEXT_SEARCH_ROUTER_SYMBOL, geotextSearchRouterFactory } from './location/routes/locationRouter'; import { cronLoadTileLatLonDataFactory, cronLoadTileLatLonDataSymbol } from './latLon/DAL/latLonDAL'; import { ITEM_REPOSITORY_SYMBOL, itemRepositoryFactory } from './control/item/DAL/itemRepository'; +import { s3ClientFactory } from './common/s3'; +import { S3_REPOSITORY_SYMBOL, s3RepositoryFactory } from './common/s3/s3Repository'; export interface RegisterOptions { override?: InjectionObject[]; @@ -65,25 +65,28 @@ export const registerExternalValues = async (options?: RegisterOptions): Promise }, }, { - token: DataSource, - provider: { useFactory: postgresClientFactory }, + token: SERVICES.S3_CLIENT, + provider: { useFactory: s3ClientFactory }, postInjectionHook: async (deps: DependencyContainer): Promise => { - const connection = deps.resolve(DataSource); + const s3Client = deps.resolve(SERVICES.S3_CLIENT); try { - await connection.initialize(); - logger.info('Connected to Postgres'); + await s3Client.send(new ListBucketsCommand({})); + logger.info('Connected to S3'); } catch (err) { - logger.error('Failed to connect to Postgres', err); + logger.error('Failed to connect to S3', err); } }, }, + { + token: S3_REPOSITORY_SYMBOL, + provider: { useFactory: s3RepositoryFactory }, + }, { token: TILE_REPOSITORY_SYMBOL, provider: { useFactory: tileRepositoryFactory } }, { token: TILE_ROUTER_SYMBOL, provider: { useFactory: tileRouterFactory } }, { token: ITEM_REPOSITORY_SYMBOL, provider: { useFactory: itemRepositoryFactory } }, { token: ITEM_ROUTER_SYMBOL, provider: { useFactory: itemRouterFactory } }, { token: ROUTE_REPOSITORY_SYMBOL, provider: { useFactory: routeRepositoryFactory } }, { token: ROUTE_ROUTER_SYMBOL, provider: { useFactory: routeRouterFactory } }, - { token: LATLON_CUSTOM_REPOSITORY_SYMBOL, provider: { useFactory: latLonRepositoryFactory } }, { token: LAT_LON_ROUTER_SYMBOL, provider: { useFactory: latLonRouterFactory } }, { token: GEOTEXT_REPOSITORY_SYMBOL, provider: { useFactory: geotextRepositoryFactory } }, { token: GEOTEXT_SEARCH_ROUTER_SYMBOL, provider: { useFactory: geotextSearchRouterFactory } }, From ec4be33036817f8decccbe0d719b4427c7069454 Mon Sep 17 00:00:00 2001 From: Niv Greenstein Date: Wed, 4 Sep 2024 15:59:33 +0300 Subject: [PATCH 172/262] feat: changed to latLon from models/latLon --- src/latLon/DAL/latLon.ts | 32 ------------------------------ src/latLon/DAL/latLonDAL.ts | 24 ++++++++++++---------- src/latLon/DAL/latLonRepository.ts | 30 ---------------------------- src/latLon/models/latLon.ts | 1 - src/latLon/utlis/index.ts | 14 ++++++------- 5 files changed, 21 insertions(+), 80 deletions(-) delete mode 100644 src/latLon/DAL/latLon.ts delete mode 100644 src/latLon/DAL/latLonRepository.ts diff --git a/src/latLon/DAL/latLon.ts b/src/latLon/DAL/latLon.ts deleted file mode 100644 index 7f057e7b..00000000 --- a/src/latLon/DAL/latLon.ts +++ /dev/null @@ -1,32 +0,0 @@ -import { Column, Entity, PrimaryColumn } from 'typeorm'; -import { LatLon as ILatLon } from '../models/latLon'; - -@Entity({ name: 'tile_lat_lon' }) -export class LatLon implements ILatLon { - @PrimaryColumn({ name: 'pk' }) - public primaryKey!: number; - - @Column({ name: 'tile_name' }) - public tileName!: string; - - @Column({ name: 'zone' }) - public zone!: string; - - @Column({ name: 'min_x' }) - public minX!: string; - - @Column({ name: 'min_y' }) - public minY!: string; - - @Column({ name: 'ext_min_x' }) - public extMinX!: number; - - @Column({ name: 'ext_min_y' }) - public extMinY!: number; - - @Column({ name: 'ext_max_x' }) - public extMaxX!: number; - - @Column({ name: 'ext_max_y' }) - public extMaxY!: number; -} diff --git a/src/latLon/DAL/latLonDAL.ts b/src/latLon/DAL/latLonDAL.ts index 7faf6c6b..503aa81b 100644 --- a/src/latLon/DAL/latLonDAL.ts +++ b/src/latLon/DAL/latLonDAL.ts @@ -1,15 +1,18 @@ +import fs from 'fs'; import { Logger } from '@map-colonies/js-logger'; import cron from 'node-cron'; import { FactoryFunction, inject, injectable } from 'tsyringe'; import { InternalServerError } from '../../common/errors'; import { IApplication } from '../../common/interfaces'; import { SERVICES } from '../../common/constants'; -import { LATLON_CUSTOM_REPOSITORY_SYMBOL, LatLonRepository } from './latLonRepository'; -import { LatLon as LatLonDb } from './latLon'; +import { LatLon as ILatLon } from '../models/latLon'; +import { ConvertCamelToSnakeCase } from '../../common/utils'; +import { S3_REPOSITORY_SYMBOL, S3Repository } from '../../common/s3/s3Repository'; +type LatLon = ConvertCamelToSnakeCase; @injectable() export class LatLonDAL { - private readonly latLonMap: Map; + private readonly latLonMap: Map; private onGoingUpdate: boolean; private dataLoad: | { @@ -22,9 +25,9 @@ export class LatLonDAL { public constructor( @inject(SERVICES.LOGGER) private readonly logger: Logger, - @inject(LATLON_CUSTOM_REPOSITORY_SYMBOL) private readonly latLonRepository: LatLonRepository + @inject(S3_REPOSITORY_SYMBOL) private readonly latLonRepository: S3Repository ) { - this.latLonMap = new Map(); + this.latLonMap = new Map(); this.onGoingUpdate = true; this.dataLoad = undefined; this.dataLoadError = false; @@ -70,7 +73,7 @@ export class LatLonDAL { } } - public async latLonToTile({ x, y, zone }: { x: number; y: number; zone: number }): Promise { + public async latLonToTile({ x, y, zone }: { x: number; y: number; zone: number }): Promise { if (this.dataLoadError) { throw new InternalServerError('Lat-lon to tile data currently not available'); } @@ -78,7 +81,7 @@ export class LatLonDAL { return this.latLonMap.get(`${x},${y},${zone}`); } - public async tileToLatLon(tileName: string): Promise { + public async tileToLatLon(tileName: string): Promise { if (this.dataLoadError) { throw new InternalServerError('Tile to lat-lon data currently not available'); } @@ -96,10 +99,11 @@ export class LatLonDAL { this.clearLatLonMap(); - const latLonData = await this.latLonRepository.getAll(); + const latLonDataPath = await this.latLonRepository.downloadFile('latLonConvertionTable'); + const { items: latLonData } = JSON.parse(fs.readFileSync(latLonDataPath, 'utf8')) as { items: LatLon[] }; + latLonData.forEach((latLon) => { - this.latLonMap.set(latLon.tileName, latLon); - this.latLonMap.set(`${latLon.minX},${latLon.minY},${latLon.zone}`, latLon); + this.latLonMap.set(`${latLon.min_x},${latLon.min_y},${latLon.zone}`, latLon); }); this.logger.debug('latLon data loaded'); } diff --git a/src/latLon/DAL/latLonRepository.ts b/src/latLon/DAL/latLonRepository.ts deleted file mode 100644 index a7f9dbfc..00000000 --- a/src/latLon/DAL/latLonRepository.ts +++ /dev/null @@ -1,30 +0,0 @@ -/* eslint-disable @typescript-eslint/naming-convention */ -import { Logger } from '@map-colonies/js-logger'; -import { DataSource } from 'typeorm'; -import { FactoryFunction } from 'tsyringe'; -import { SERVICES } from '../../common/constants'; -import { ServiceUnavailableError } from '../../common/errors'; -import { LatLon as LatLonDb } from './latLon'; - -// eslint-disable-next-line @typescript-eslint/explicit-function-return-type -const createLatLonRepository = (dataSource: DataSource, logger: Logger) => { - return dataSource.getRepository(LatLonDb).extend({ - async getAll(): Promise { - try { - const result = await this.find(); - return result; - } catch (error: unknown) { - logger.error('Error in getAll function in latLonRepository', error); - throw new ServiceUnavailableError('Postgres client is not available. As for, the getAll request cannot be executed.'); - } - }, - }); -}; - -export type LatLonRepository = ReturnType; - -export const latLonRepositoryFactory: FactoryFunction = (depContainer) => { - return createLatLonRepository(depContainer.resolve(DataSource), depContainer.resolve(SERVICES.LOGGER)); -}; - -export const LATLON_CUSTOM_REPOSITORY_SYMBOL = Symbol('LATLON_CUSTOM_REPOSITORY_SYMBOL'); diff --git a/src/latLon/models/latLon.ts b/src/latLon/models/latLon.ts index ab1176ba..3fc70537 100644 --- a/src/latLon/models/latLon.ts +++ b/src/latLon/models/latLon.ts @@ -1,5 +1,4 @@ export interface LatLon { - primaryKey: number; tileName: string; zone: string; minX: string; diff --git a/src/latLon/utlis/index.ts b/src/latLon/utlis/index.ts index 128b4881..617f97d2 100644 --- a/src/latLon/utlis/index.ts +++ b/src/latLon/utlis/index.ts @@ -1,10 +1,10 @@ /* istanbul ignore file */ import { Polygon } from 'geojson'; import { BadRequestError } from '../../common/errors'; -import { convertUTMToWgs84 } from '../../common/utils'; -import { LatLon } from '../DAL/latLon'; +import { ConvertCamelToSnakeCase, convertUTMToWgs84 } from '../../common/utils'; import { FeatureCollection } from '../../common/interfaces'; import { Tile } from '../../control/tile/models/tile'; +import { LatLon } from '../models/latLon'; /* eslint-disable @typescript-eslint/naming-convention */ const geoJsonObjectTemplate = (): FeatureCollection => ({ @@ -26,14 +26,14 @@ const geoJsonObjectTemplate = (): FeatureCollection => ({ export const convertTilesToUTM = ( tile: { tileName: string; subTileNumber: number[] }, - tileObject: LatLon + tileObject: ConvertCamelToSnakeCase ): { x: number; y: number; zone: number; } => { - const xCoordinate = parseInt(tileObject.minX); - const yCoordinate = parseInt(tileObject.minY); + const xCoordinate = parseInt(tileObject.min_x); + const yCoordinate = parseInt(tileObject.min_y); const xCoordinatePart = tile.subTileNumber .map((x) => { @@ -50,14 +50,14 @@ export const convertTilesToUTM = ( }; export const validateResult = ( - tile: LatLon, + tile: ConvertCamelToSnakeCase, utmCoor: { x: number; y: number; zone: number; } ): void => { - if (tile.extMinX > utmCoor.x || tile.extMaxX < utmCoor.x || tile.extMinY > utmCoor.y || tile.extMaxY < utmCoor.y) { + if (tile.ext_min_x > utmCoor.x || tile.ext_max_x < utmCoor.x || tile.ext_min_y > utmCoor.y || tile.ext_max_y < utmCoor.y) { throw new BadRequestError("Tile is found, sub tile is not in tile's extent"); } }; From 0a2ede96199c8d0eea7542e885bf9e23bb6e106b Mon Sep 17 00:00:00 2001 From: Niv Greenstein Date: Wed, 4 Sep 2024 16:00:28 +0300 Subject: [PATCH 173/262] fix: lookup routes --- src/latLon/routes/latLonRouter.ts | 2 +- src/serverBuilder.ts | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/src/latLon/routes/latLonRouter.ts b/src/latLon/routes/latLonRouter.ts index 8357157d..cd4ada62 100644 --- a/src/latLon/routes/latLonRouter.ts +++ b/src/latLon/routes/latLonRouter.ts @@ -10,7 +10,7 @@ const latLonRouterFactory: FactoryFunction = (dependencyContainer) => { router.get('/tileToLatLon', controller.tileToLatLon); router.get('/latlonToMgrs', controller.latlonToMgrs); router.get('/mgrsToLatlon', controller.mgrsToLatlon); - router.get('coordinates', controller.getCoordinates); + router.get('/coordinates', controller.getCoordinates); return router; }; diff --git a/src/serverBuilder.ts b/src/serverBuilder.ts index c9c04c50..b67b2853 100644 --- a/src/serverBuilder.ts +++ b/src/serverBuilder.ts @@ -56,11 +56,11 @@ export class ServerBuilder { private buildRoutes(): void { const router = Router(); - router.use('/lookup', this.latLonRouter); router.use('/location', this.geotextRouter); router.use('/control', this.buildControlRoutes()); this.serverInstance.use('/search/', router); + this.serverInstance.use('/lookup', this.latLonRouter); } private buildControlRoutes(): Router { From 82784ebeca081ea890f5f26f55338ec8342cd05f Mon Sep 17 00:00:00 2001 From: Niv Greenstein Date: Wed, 4 Sep 2024 16:00:44 +0300 Subject: [PATCH 174/262] fix: fixed logic at latLonToTile --- src/latLon/models/latLonManager.ts | 44 +++++++++++++++++++++--------- 1 file changed, 31 insertions(+), 13 deletions(-) diff --git a/src/latLon/models/latLonManager.ts b/src/latLon/models/latLonManager.ts index 4f012af2..8d189417 100644 --- a/src/latLon/models/latLonManager.ts +++ b/src/latLon/models/latLonManager.ts @@ -1,11 +1,11 @@ import { IConfig } from 'config'; import { Logger } from '@map-colonies/js-logger'; import { inject, injectable } from 'tsyringe'; -import { Feature } from 'geojson'; +import { BBox, Feature } from 'geojson'; import * as mgrs from 'mgrs'; import { SERVICES } from '../../common/constants'; import { LatLonDAL } from '../DAL/latLonDAL'; -import { convertWgs84ToUTM, validateTile, validateWGS84Coordinate } from '../../common/utils'; +import { convertUTMToWgs84, convertWgs84ToUTM, validateTile, validateWGS84Coordinate } from '../../common/utils'; import { convertTilesToUTM, getSubTileByBottomLeftUtmCoor, validateResult } from '../utlis'; import { BadRequestError } from '../../common/errors'; import { Tile } from '../../control/tile/models/tile'; @@ -50,27 +50,37 @@ export class LatLonManager { throw new BadRequestError('The coordinate is outside the grid extent'); } - const xNumber = Math.abs(Math.trunc((coordinatesUTM.x % 10000) / 10) * 10) + const xNumber = Math.abs(Math.trunc((utm.Easting % 10000) / 10) * 10) .toString() .padStart(4, '0'); - const yNumber = Math.abs(Math.trunc((coordinatesUTM.y % 10000) / 10) * 10) + const yNumber = Math.abs(Math.trunc((utm.Northing % 10000) / 10) * 10) .toString() .padStart(4, '0'); + const bbox = [ + ...( + Object.values(convertUTMToWgs84(tileCoordinateData.ext_min_x, tileCoordinateData.ext_min_y, +tileCoordinateData.zone)) as number[] + ).reverse(), + ...( + Object.values(convertUTMToWgs84(tileCoordinateData.ext_max_x, tileCoordinateData.ext_max_y, +tileCoordinateData.zone)) as number[] + ).reverse(), + ] as BBox; + return { type: 'Feature', - properties: {}, + properties: { + tileName: tileCoordinateData.tile_name, + subTileNumber: new Array(3).fill('').map(function (value, i) { + return xNumber[i] + yNumber[i]; + }), + }, geometry: parseGeo({ - bbox: [tileCoordinateData.extMinX, tileCoordinateData.extMinY, tileCoordinateData.extMaxX, tileCoordinateData.extMaxY], + bbox, }) ?? { type: 'Point', coordinates: [lon, lat], }, - bbox: [tileCoordinateData.extMinX, tileCoordinateData.extMinY, tileCoordinateData.extMaxX, tileCoordinateData.extMaxY], - tileName: tileCoordinateData.tileName, - subTileNumber: new Array(3).fill('').map(function (value, i) { - return +(xNumber[i] + yNumber[i]); - }), + bbox, }; } @@ -97,15 +107,23 @@ export class LatLonManager { } public latLonToMGRS({ lat, lon, accuracy = 5 }: { lat: number; lon: number; accuracy?: number }): { [key: string]: unknown } & Feature { + const accuracyString: Record = { + [0]: '100km', + [1]: '10km', + [2]: '1km', + [3]: '100m', + [4]: '10m', + [5]: '1m', + }; return { type: 'Feature', - mgrs: mgrs.forward([lon, lat], accuracy), geometry: { type: 'Point', coordinates: [lon, lat], }, properties: { - accuracy, + accuracy: accuracyString[accuracy], + mgrs: mgrs.forward([lon, lat], accuracy), }, }; } From 19b89dfdc3a31859d778cc5ce8ae3e2e24180060 Mon Sep 17 00:00:00 2001 From: Niv Greenstein Date: Wed, 4 Sep 2024 16:01:02 +0300 Subject: [PATCH 175/262] delete: deleted postgresql --- src/common/interfaces.ts | 6 --- src/common/postgresql/index.ts | 67 ------------------------- src/common/postgresql/promiseTimeout.ts | 14 ------ 3 files changed, 87 deletions(-) delete mode 100644 src/common/postgresql/index.ts delete mode 100644 src/common/postgresql/promiseTimeout.ts diff --git a/src/common/interfaces.ts b/src/common/interfaces.ts index 58ec287e..ad25417c 100644 --- a/src/common/interfaces.ts +++ b/src/common/interfaces.ts @@ -1,4 +1,3 @@ -import { DataSourceOptions } from 'typeorm'; import { Feature, FeatureCollection as GeoJSONFeatureCollection } from 'geojson'; export interface IConfig { @@ -13,11 +12,6 @@ export interface OpenApiConfig { uiPath: string; } -export type PostgresDbConfig = { - enableSslAuth: boolean; - sslPaths: { ca: string; cert: string; key: string }; -} & DataSourceOptions; - export interface GeoContext { bbox?: number[]; radius?: number; diff --git a/src/common/postgresql/index.ts b/src/common/postgresql/index.ts deleted file mode 100644 index e131e61d..00000000 --- a/src/common/postgresql/index.ts +++ /dev/null @@ -1,67 +0,0 @@ -import { readFileSync } from 'fs'; -import { HealthCheck } from '@godaddy/terminus'; -import { DataSource, DataSourceOptions, QueryFailedError } from 'typeorm'; -import { DependencyContainer, FactoryFunction } from 'tsyringe'; -import { IConfig, PostgresDbConfig } from '../interfaces'; -import { LatLon } from '../../latLon/DAL/latLon'; -import { SERVICES } from '../constants'; -import { promiseTimeout } from './promiseTimeout'; - -const DB_TIMEOUT = 5000; - -enum TransactionFailure { - SERIALIZATION_FAILURE = '40001', - DEADLOCK_DETECTED = '40P01', -} - -interface QueryFailedErrorWithCode extends QueryFailedError { - code: string | undefined; -} - -export const postgresClientFactory: FactoryFunction = (container: DependencyContainer): DataSource => { - const config = container.resolve(SERVICES.CONFIG); - - const dbConfig = config.get('db.postgresql'); - return new DataSource(createConnectionOptions(dbConfig)); -}; - -export enum TransactionName { - TRY_CLOSING_FILE = 'TryClosingFile', - CREATE_RERUN = 'CreateRerun', - TRY_CLOSING_CHANGESET = 'TryClosingChangeset', - TRY_CLOSING_CHANGESETS = 'TryClosingChangesets', -} - -export const isTransactionFailure = (error: unknown): boolean => { - if (error instanceof QueryFailedError) { - const code = (error as QueryFailedErrorWithCode).code; - return code === TransactionFailure.SERIALIZATION_FAILURE || code === TransactionFailure.DEADLOCK_DETECTED; - } - return false; -}; - -export const DB_ENTITIES = [LatLon]; - -export const createConnectionOptions = (dbConfig: PostgresDbConfig): DataSourceOptions => { - const { enableSslAuth, sslPaths, ...connectionOptions } = dbConfig; - if (enableSslAuth && connectionOptions.type === 'postgres') { - connectionOptions.password = undefined; - connectionOptions.ssl = { key: readFileSync(sslPaths.key), cert: readFileSync(sslPaths.cert), ca: readFileSync(sslPaths.ca) }; - } - return { entities: [...DB_ENTITIES, '**/DAL/*.js'], ...connectionOptions }; -}; - -export const getDbHealthCheckFunction = (connection: DataSource): HealthCheck => { - return async (): Promise => { - const check = connection.query('SELECT 1').then(() => { - return; - }); - return promiseTimeout(DB_TIMEOUT, check); - }; -}; - -export interface ReturningId { - id: string; -} - -export type ReturningResult = [T[], number]; diff --git a/src/common/postgresql/promiseTimeout.ts b/src/common/postgresql/promiseTimeout.ts deleted file mode 100644 index 7f6255c4..00000000 --- a/src/common/postgresql/promiseTimeout.ts +++ /dev/null @@ -1,14 +0,0 @@ -export class TimeoutError extends Error {} - -export const promiseTimeout = async (ms: number, promise: Promise): Promise => { - // Create a promise that rejects in milliseconds - const timeout = new Promise((_, reject) => { - const id = setTimeout(() => { - clearTimeout(id); - reject(new TimeoutError(`Timed out in + ${ms} + ms.`)); - }, ms); - }); - - // Returns a race between our timeout and the passed in promise - return Promise.race([promise, timeout]); -}; From 4d2b0377bfebf8dba3edcc6f96e82b2529a34ae8 Mon Sep 17 00:00:00 2001 From: Niv Greenstein Date: Wed, 4 Sep 2024 16:01:54 +0300 Subject: [PATCH 176/262] feat: removed proj4 --- src/common/projections.ts | 1 + src/common/utils.ts | 24 ++++++++++++------------ 2 files changed, 13 insertions(+), 12 deletions(-) diff --git a/src/common/projections.ts b/src/common/projections.ts index f36a7501..71824891 100644 --- a/src/common/projections.ts +++ b/src/common/projections.ts @@ -1,3 +1,4 @@ +//TODO: REMOVE THIS FILE - if proj4 is not in use? export const wgs84Projection = '+proj=longlat +ellps=WGS84 +datum=WGS84 +no_defs'; export const utmProjection = (zone: number): string => `+proj=utm +zone=${zone} +ellps=WGS84 +datum=WGS84 +units=m +no_defs`; diff --git a/src/common/utils.ts b/src/common/utils.ts index 50ad1458..dceab4c3 100644 --- a/src/common/utils.ts +++ b/src/common/utils.ts @@ -1,5 +1,4 @@ -import proj4 from 'proj4'; -import { utmProjection, wgs84Projection } from './projections'; +import utm from 'utm-latlng'; import { WGS84Coordinate } from './interfaces'; type SnakeToCamelCase = S extends `${infer T}_${infer U}` ? `${T}${Capitalize>}` : S; @@ -53,21 +52,22 @@ export const convertWgs84ToUTM = ( Northing: number; ZoneNumber: number; } => { - const zone = Math.floor((longitude + 180) / 6) + 1; - - const [easting, northing] = proj4(wgs84Projection, utmProjection(zone), [longitude, latitude]); - - return { - Easting: +easting.toFixed(utmPrecision), - Northing: +northing.toFixed(utmPrecision), - ZoneNumber: zone, + //@ts-expect-error: utm has problem with types. Need to ignore ts error here + // eslint-disable-next-line @typescript-eslint/no-unsafe-call + const UTMcoordinates = new utm().convertLatLngToUtm(latitude, longitude, utmPrecision) as { + Easting: number; + Northing: number; + ZoneNumber: number; + ZoneLetter: string; }; + + return UTMcoordinates; }; /* eslint-enable @typescript-eslint/naming-convention */ export const convertUTMToWgs84 = (x: number, y: number, zone: number): WGS84Coordinate => { - const [longitude, latitude] = proj4(utmProjection(zone), wgs84Projection, [x, y]); - return { lat: latitude, lon: longitude }; + const { lat, lng: lon } = new utm().convertUtmToLatLng(x, y, zone, 'N') as { lat: number; lng: number }; + return { lat, lon }; }; export const validateTile = (tile: { tileName: string; subTileNumber: number[] }): boolean => { From 09c7fcb2faa1affa656e644bf8767ead794d5c52 Mon Sep 17 00:00:00 2001 From: Niv Greenstein Date: Wed, 4 Sep 2024 16:04:21 +0300 Subject: [PATCH 177/262] fix: added s3 config. removed postgres config --- config/custom-environment-variables.json | 28 +++++++++--------------- config/default.json | 25 ++++++++++----------- 2 files changed, 22 insertions(+), 31 deletions(-) diff --git a/config/custom-environment-variables.json b/config/custom-environment-variables.json index d596858d..2be63e7b 100644 --- a/config/custom-environment-variables.json +++ b/config/custom-environment-variables.json @@ -67,25 +67,17 @@ } } }, - "postgresql": { - "host": "POSTGRES_URL", - "port": { - "__name": "POSTGRES_PORT", - "__format": "number" + "s3": { + "endpoint": "S3_ENDPOINT", + "credentials": { + "accessKeyId": "S3_ACCESS_KEY", + "secretAccessKey": "S3_SECRET_KEY" }, - "username": "POSTGRES_USERNAME", - "password": "POSTGRES_PASSWORD", - "enableSslAuth": { - "__name": "POSTGRES_ENABLE_SLL_AUTH", - "__format": "boolean" - }, - "sslPaths": { - "ca": "POSTGRES_CA_PATH", - "key": "POSTGRES_KEY_PATH", - "cert": "POSTGRES_CERT_PATH" - }, - "database": "POSTGRES_DB_NAME", - "schema": "POSTGRES_DB_SCHEMA" + "region": "S3_REGION", + "files": { + "__name": "S3_FILES_DATA", + "__format": "json" + } } }, "application": { diff --git a/config/default.json b/config/default.json index 470bddec..43b6cb32 100644 --- a/config/default.json +++ b/config/default.json @@ -55,20 +55,19 @@ } } }, - "postgresql": { - "type": "postgres", - "host": "localhost", - "port": 5432, - "username": "postgres", - "password": "postgres", - "enableSslAuth": false, - "sslPaths": { - "ca": "", - "key": "", - "cert": "" + "s3": { + "endpoint": "http://localhost:9000", + "credentials": { + "accessKeyId": "O8JnyGPmTtIz69uZihyh", + "secretAccessKey": "1uzitcz85XFpZ8oxedJMoQpCuDSiJrhfDPbAvPOM" }, - "database": "postgres", - "schema": "geocoding" + "region": "local", + "files": { + "latLonConvertionTable": { + "bucket": "geocoding", + "fileName": "table.json" + } + } } }, "application": { From 941778ee33d9a18f3b5c77583f5ee19511aa3677 Mon Sep 17 00:00:00 2001 From: Niv Greenstein Date: Wed, 4 Sep 2024 16:04:46 +0300 Subject: [PATCH 178/262] feat: added aws-sdk dependencies --- package-lock.json | 4776 ++++++++++++++++++++++++++++++++++++--------- package.json | 4 +- 2 files changed, 3904 insertions(+), 876 deletions(-) diff --git a/package-lock.json b/package-lock.json index 0c1d7e64..fac554d8 100644 --- a/package-lock.json +++ b/package-lock.json @@ -1,15 +1,16 @@ { "name": "geocoding", - "version": "0.1.0-prealpha", + "version": "0.1.0-prealpha.2", "lockfileVersion": 2, "requires": true, "packages": { "": { "name": "geocoding", - "version": "0.1.0-prealpha", + "version": "0.1.0-prealpha.2", "hasInstallScript": true, "license": "ISC", "dependencies": { + "@aws-sdk/client-s3": "^3.637.0", "@elastic/elasticsearch": "^8.13.1", "@godaddy/terminus": "^4.12.1", "@map-colonies/error-express-handler": "^2.1.0", @@ -22,6 +23,7 @@ "@opentelemetry/api-metrics": "0.23.0", "@opentelemetry/instrumentation-express": "0.32.1", "@opentelemetry/instrumentation-http": "0.35.1", + "@smithy/node-http-handler": "^3.1.4", "axios": "^0.21.1", "compression": "^1.7.4", "config": "^3.3.9", @@ -37,7 +39,7 @@ "proj4": "^2.11.0", "reflect-metadata": "^0.1.13", "tsyringe": "^4.8.0", - "typeorm": "^0.3.20", + "utm-latlng": "^1.0.8", "wellknown": "^0.5.0" }, "devDependencies": { @@ -103,6 +105,1051 @@ "js-yaml": "^4.1.0" } }, + "node_modules/@aws-crypto/crc32": { + "version": "5.2.0", + "resolved": "https://registry.npmjs.org/@aws-crypto/crc32/-/crc32-5.2.0.tgz", + "integrity": "sha512-nLbCWqQNgUiwwtFsen1AdzAtvuLRsQS8rYgMuxCrdKf9kOssamGLuPwyTY9wyYblNr9+1XM8v6zoDTPPSIeANg==", + "dependencies": { + "@aws-crypto/util": "^5.2.0", + "@aws-sdk/types": "^3.222.0", + "tslib": "^2.6.2" + }, + "engines": { + "node": ">=16.0.0" + } + }, + "node_modules/@aws-crypto/crc32/node_modules/tslib": { + "version": "2.7.0", + "resolved": "https://registry.npmjs.org/tslib/-/tslib-2.7.0.tgz", + "integrity": "sha512-gLXCKdN1/j47AiHiOkJN69hJmcbGTHI0ImLmbYLHykhgeN0jVGola9yVjFgzCUklsZQMW55o+dW7IXv3RCXDzA==" + }, + "node_modules/@aws-crypto/crc32c": { + "version": "5.2.0", + "resolved": "https://registry.npmjs.org/@aws-crypto/crc32c/-/crc32c-5.2.0.tgz", + "integrity": "sha512-+iWb8qaHLYKrNvGRbiYRHSdKRWhto5XlZUEBwDjYNf+ly5SVYG6zEoYIdxvf5R3zyeP16w4PLBn3rH1xc74Rag==", + "dependencies": { + "@aws-crypto/util": "^5.2.0", + "@aws-sdk/types": "^3.222.0", + "tslib": "^2.6.2" + } + }, + "node_modules/@aws-crypto/crc32c/node_modules/tslib": { + "version": "2.7.0", + "resolved": "https://registry.npmjs.org/tslib/-/tslib-2.7.0.tgz", + "integrity": "sha512-gLXCKdN1/j47AiHiOkJN69hJmcbGTHI0ImLmbYLHykhgeN0jVGola9yVjFgzCUklsZQMW55o+dW7IXv3RCXDzA==" + }, + "node_modules/@aws-crypto/sha1-browser": { + "version": "5.2.0", + "resolved": "https://registry.npmjs.org/@aws-crypto/sha1-browser/-/sha1-browser-5.2.0.tgz", + "integrity": "sha512-OH6lveCFfcDjX4dbAvCFSYUjJZjDr/3XJ3xHtjn3Oj5b9RjojQo8npoLeA/bNwkOkrSQ0wgrHzXk4tDRxGKJeg==", + "dependencies": { + "@aws-crypto/supports-web-crypto": "^5.2.0", + "@aws-crypto/util": "^5.2.0", + "@aws-sdk/types": "^3.222.0", + "@aws-sdk/util-locate-window": "^3.0.0", + "@smithy/util-utf8": "^2.0.0", + "tslib": "^2.6.2" + } + }, + "node_modules/@aws-crypto/sha1-browser/node_modules/@smithy/is-array-buffer": { + "version": "2.2.0", + "resolved": "https://registry.npmjs.org/@smithy/is-array-buffer/-/is-array-buffer-2.2.0.tgz", + "integrity": "sha512-GGP3O9QFD24uGeAXYUjwSTXARoqpZykHadOmA8G5vfJPK0/DC67qa//0qvqrJzL1xc8WQWX7/yc7fwudjPHPhA==", + "dependencies": { + "tslib": "^2.6.2" + }, + "engines": { + "node": ">=14.0.0" + } + }, + "node_modules/@aws-crypto/sha1-browser/node_modules/@smithy/util-buffer-from": { + "version": "2.2.0", + "resolved": "https://registry.npmjs.org/@smithy/util-buffer-from/-/util-buffer-from-2.2.0.tgz", + "integrity": "sha512-IJdWBbTcMQ6DA0gdNhh/BwrLkDR+ADW5Kr1aZmd4k3DIF6ezMV4R2NIAmT08wQJ3yUK82thHWmC/TnK/wpMMIA==", + "dependencies": { + "@smithy/is-array-buffer": "^2.2.0", + "tslib": "^2.6.2" + }, + "engines": { + "node": ">=14.0.0" + } + }, + "node_modules/@aws-crypto/sha1-browser/node_modules/@smithy/util-utf8": { + "version": "2.3.0", + "resolved": "https://registry.npmjs.org/@smithy/util-utf8/-/util-utf8-2.3.0.tgz", + "integrity": "sha512-R8Rdn8Hy72KKcebgLiv8jQcQkXoLMOGGv5uI1/k0l+snqkOzQ1R0ChUBCxWMlBsFMekWjq0wRudIweFs7sKT5A==", + "dependencies": { + "@smithy/util-buffer-from": "^2.2.0", + "tslib": "^2.6.2" + }, + "engines": { + "node": ">=14.0.0" + } + }, + "node_modules/@aws-crypto/sha1-browser/node_modules/tslib": { + "version": "2.7.0", + "resolved": "https://registry.npmjs.org/tslib/-/tslib-2.7.0.tgz", + "integrity": "sha512-gLXCKdN1/j47AiHiOkJN69hJmcbGTHI0ImLmbYLHykhgeN0jVGola9yVjFgzCUklsZQMW55o+dW7IXv3RCXDzA==" + }, + "node_modules/@aws-crypto/sha256-browser": { + "version": "5.2.0", + "resolved": "https://registry.npmjs.org/@aws-crypto/sha256-browser/-/sha256-browser-5.2.0.tgz", + "integrity": "sha512-AXfN/lGotSQwu6HNcEsIASo7kWXZ5HYWvfOmSNKDsEqC4OashTp8alTmaz+F7TC2L083SFv5RdB+qU3Vs1kZqw==", + "dependencies": { + "@aws-crypto/sha256-js": "^5.2.0", + "@aws-crypto/supports-web-crypto": "^5.2.0", + "@aws-crypto/util": "^5.2.0", + "@aws-sdk/types": "^3.222.0", + "@aws-sdk/util-locate-window": "^3.0.0", + "@smithy/util-utf8": "^2.0.0", + "tslib": "^2.6.2" + } + }, + "node_modules/@aws-crypto/sha256-browser/node_modules/@smithy/is-array-buffer": { + "version": "2.2.0", + "resolved": "https://registry.npmjs.org/@smithy/is-array-buffer/-/is-array-buffer-2.2.0.tgz", + "integrity": "sha512-GGP3O9QFD24uGeAXYUjwSTXARoqpZykHadOmA8G5vfJPK0/DC67qa//0qvqrJzL1xc8WQWX7/yc7fwudjPHPhA==", + "dependencies": { + "tslib": "^2.6.2" + }, + "engines": { + "node": ">=14.0.0" + } + }, + "node_modules/@aws-crypto/sha256-browser/node_modules/@smithy/util-buffer-from": { + "version": "2.2.0", + "resolved": "https://registry.npmjs.org/@smithy/util-buffer-from/-/util-buffer-from-2.2.0.tgz", + "integrity": "sha512-IJdWBbTcMQ6DA0gdNhh/BwrLkDR+ADW5Kr1aZmd4k3DIF6ezMV4R2NIAmT08wQJ3yUK82thHWmC/TnK/wpMMIA==", + "dependencies": { + "@smithy/is-array-buffer": "^2.2.0", + "tslib": "^2.6.2" + }, + "engines": { + "node": ">=14.0.0" + } + }, + "node_modules/@aws-crypto/sha256-browser/node_modules/@smithy/util-utf8": { + "version": "2.3.0", + "resolved": "https://registry.npmjs.org/@smithy/util-utf8/-/util-utf8-2.3.0.tgz", + "integrity": "sha512-R8Rdn8Hy72KKcebgLiv8jQcQkXoLMOGGv5uI1/k0l+snqkOzQ1R0ChUBCxWMlBsFMekWjq0wRudIweFs7sKT5A==", + "dependencies": { + "@smithy/util-buffer-from": "^2.2.0", + "tslib": "^2.6.2" + }, + "engines": { + "node": ">=14.0.0" + } + }, + "node_modules/@aws-crypto/sha256-browser/node_modules/tslib": { + "version": "2.7.0", + "resolved": "https://registry.npmjs.org/tslib/-/tslib-2.7.0.tgz", + "integrity": "sha512-gLXCKdN1/j47AiHiOkJN69hJmcbGTHI0ImLmbYLHykhgeN0jVGola9yVjFgzCUklsZQMW55o+dW7IXv3RCXDzA==" + }, + "node_modules/@aws-crypto/sha256-js": { + "version": "5.2.0", + "resolved": "https://registry.npmjs.org/@aws-crypto/sha256-js/-/sha256-js-5.2.0.tgz", + "integrity": "sha512-FFQQyu7edu4ufvIZ+OadFpHHOt+eSTBaYaki44c+akjg7qZg9oOQeLlk77F6tSYqjDAFClrHJk9tMf0HdVyOvA==", + "dependencies": { + "@aws-crypto/util": "^5.2.0", + "@aws-sdk/types": "^3.222.0", + "tslib": "^2.6.2" + }, + "engines": { + "node": ">=16.0.0" + } + }, + "node_modules/@aws-crypto/sha256-js/node_modules/tslib": { + "version": "2.7.0", + "resolved": "https://registry.npmjs.org/tslib/-/tslib-2.7.0.tgz", + "integrity": "sha512-gLXCKdN1/j47AiHiOkJN69hJmcbGTHI0ImLmbYLHykhgeN0jVGola9yVjFgzCUklsZQMW55o+dW7IXv3RCXDzA==" + }, + "node_modules/@aws-crypto/supports-web-crypto": { + "version": "5.2.0", + "resolved": "https://registry.npmjs.org/@aws-crypto/supports-web-crypto/-/supports-web-crypto-5.2.0.tgz", + "integrity": "sha512-iAvUotm021kM33eCdNfwIN//F77/IADDSs58i+MDaOqFrVjZo9bAal0NK7HurRuWLLpF1iLX7gbWrjHjeo+YFg==", + "dependencies": { + "tslib": "^2.6.2" + } + }, + "node_modules/@aws-crypto/supports-web-crypto/node_modules/tslib": { + "version": "2.7.0", + "resolved": "https://registry.npmjs.org/tslib/-/tslib-2.7.0.tgz", + "integrity": "sha512-gLXCKdN1/j47AiHiOkJN69hJmcbGTHI0ImLmbYLHykhgeN0jVGola9yVjFgzCUklsZQMW55o+dW7IXv3RCXDzA==" + }, + "node_modules/@aws-crypto/util": { + "version": "5.2.0", + "resolved": "https://registry.npmjs.org/@aws-crypto/util/-/util-5.2.0.tgz", + "integrity": "sha512-4RkU9EsI6ZpBve5fseQlGNUWKMa1RLPQ1dnjnQoe07ldfIzcsGb5hC5W0Dm7u423KWzawlrpbjXBrXCEv9zazQ==", + "dependencies": { + "@aws-sdk/types": "^3.222.0", + "@smithy/util-utf8": "^2.0.0", + "tslib": "^2.6.2" + } + }, + "node_modules/@aws-crypto/util/node_modules/@smithy/is-array-buffer": { + "version": "2.2.0", + "resolved": "https://registry.npmjs.org/@smithy/is-array-buffer/-/is-array-buffer-2.2.0.tgz", + "integrity": "sha512-GGP3O9QFD24uGeAXYUjwSTXARoqpZykHadOmA8G5vfJPK0/DC67qa//0qvqrJzL1xc8WQWX7/yc7fwudjPHPhA==", + "dependencies": { + "tslib": "^2.6.2" + }, + "engines": { + "node": ">=14.0.0" + } + }, + "node_modules/@aws-crypto/util/node_modules/@smithy/util-buffer-from": { + "version": "2.2.0", + "resolved": "https://registry.npmjs.org/@smithy/util-buffer-from/-/util-buffer-from-2.2.0.tgz", + "integrity": "sha512-IJdWBbTcMQ6DA0gdNhh/BwrLkDR+ADW5Kr1aZmd4k3DIF6ezMV4R2NIAmT08wQJ3yUK82thHWmC/TnK/wpMMIA==", + "dependencies": { + "@smithy/is-array-buffer": "^2.2.0", + "tslib": "^2.6.2" + }, + "engines": { + "node": ">=14.0.0" + } + }, + "node_modules/@aws-crypto/util/node_modules/@smithy/util-utf8": { + "version": "2.3.0", + "resolved": "https://registry.npmjs.org/@smithy/util-utf8/-/util-utf8-2.3.0.tgz", + "integrity": "sha512-R8Rdn8Hy72KKcebgLiv8jQcQkXoLMOGGv5uI1/k0l+snqkOzQ1R0ChUBCxWMlBsFMekWjq0wRudIweFs7sKT5A==", + "dependencies": { + "@smithy/util-buffer-from": "^2.2.0", + "tslib": "^2.6.2" + }, + "engines": { + "node": ">=14.0.0" + } + }, + "node_modules/@aws-crypto/util/node_modules/tslib": { + "version": "2.7.0", + "resolved": "https://registry.npmjs.org/tslib/-/tslib-2.7.0.tgz", + "integrity": "sha512-gLXCKdN1/j47AiHiOkJN69hJmcbGTHI0ImLmbYLHykhgeN0jVGola9yVjFgzCUklsZQMW55o+dW7IXv3RCXDzA==" + }, + "node_modules/@aws-sdk/client-s3": { + "version": "3.637.0", + "resolved": "https://registry.npmjs.org/@aws-sdk/client-s3/-/client-s3-3.637.0.tgz", + "integrity": "sha512-y6UC94fsMvhKbf0dzfnjVP1HePeGjplfcYfilZU1COIJLyTkMcUv4XcT4I407CGIrvgEafONHkiC09ygqUauNA==", + "dependencies": { + "@aws-crypto/sha1-browser": "5.2.0", + "@aws-crypto/sha256-browser": "5.2.0", + "@aws-crypto/sha256-js": "5.2.0", + "@aws-sdk/client-sso-oidc": "3.637.0", + "@aws-sdk/client-sts": "3.637.0", + "@aws-sdk/core": "3.635.0", + "@aws-sdk/credential-provider-node": "3.637.0", + "@aws-sdk/middleware-bucket-endpoint": "3.620.0", + "@aws-sdk/middleware-expect-continue": "3.620.0", + "@aws-sdk/middleware-flexible-checksums": "3.620.0", + "@aws-sdk/middleware-host-header": "3.620.0", + "@aws-sdk/middleware-location-constraint": "3.609.0", + "@aws-sdk/middleware-logger": "3.609.0", + "@aws-sdk/middleware-recursion-detection": "3.620.0", + "@aws-sdk/middleware-sdk-s3": "3.635.0", + "@aws-sdk/middleware-ssec": "3.609.0", + "@aws-sdk/middleware-user-agent": "3.637.0", + "@aws-sdk/region-config-resolver": "3.614.0", + "@aws-sdk/signature-v4-multi-region": "3.635.0", + "@aws-sdk/types": "3.609.0", + "@aws-sdk/util-endpoints": "3.637.0", + "@aws-sdk/util-user-agent-browser": "3.609.0", + "@aws-sdk/util-user-agent-node": "3.614.0", + "@aws-sdk/xml-builder": "3.609.0", + "@smithy/config-resolver": "^3.0.5", + "@smithy/core": "^2.4.0", + "@smithy/eventstream-serde-browser": "^3.0.6", + "@smithy/eventstream-serde-config-resolver": "^3.0.3", + "@smithy/eventstream-serde-node": "^3.0.5", + "@smithy/fetch-http-handler": "^3.2.4", + "@smithy/hash-blob-browser": "^3.1.2", + "@smithy/hash-node": "^3.0.3", + "@smithy/hash-stream-node": "^3.1.2", + "@smithy/invalid-dependency": "^3.0.3", + "@smithy/md5-js": "^3.0.3", + "@smithy/middleware-content-length": "^3.0.5", + "@smithy/middleware-endpoint": "^3.1.0", + "@smithy/middleware-retry": "^3.0.15", + "@smithy/middleware-serde": "^3.0.3", + "@smithy/middleware-stack": "^3.0.3", + "@smithy/node-config-provider": "^3.1.4", + "@smithy/node-http-handler": "^3.1.4", + "@smithy/protocol-http": "^4.1.0", + "@smithy/smithy-client": "^3.2.0", + "@smithy/types": "^3.3.0", + "@smithy/url-parser": "^3.0.3", + "@smithy/util-base64": "^3.0.0", + "@smithy/util-body-length-browser": "^3.0.0", + "@smithy/util-body-length-node": "^3.0.0", + "@smithy/util-defaults-mode-browser": "^3.0.15", + "@smithy/util-defaults-mode-node": "^3.0.15", + "@smithy/util-endpoints": "^2.0.5", + "@smithy/util-middleware": "^3.0.3", + "@smithy/util-retry": "^3.0.3", + "@smithy/util-stream": "^3.1.3", + "@smithy/util-utf8": "^3.0.0", + "@smithy/util-waiter": "^3.1.2", + "tslib": "^2.6.2" + }, + "engines": { + "node": ">=16.0.0" + } + }, + "node_modules/@aws-sdk/client-s3/node_modules/tslib": { + "version": "2.7.0", + "resolved": "https://registry.npmjs.org/tslib/-/tslib-2.7.0.tgz", + "integrity": "sha512-gLXCKdN1/j47AiHiOkJN69hJmcbGTHI0ImLmbYLHykhgeN0jVGola9yVjFgzCUklsZQMW55o+dW7IXv3RCXDzA==" + }, + "node_modules/@aws-sdk/client-sso": { + "version": "3.637.0", + "resolved": "https://registry.npmjs.org/@aws-sdk/client-sso/-/client-sso-3.637.0.tgz", + "integrity": "sha512-+KjLvgX5yJYROWo3TQuwBJlHCY0zz9PsLuEolmXQn0BVK1L/m9GteZHtd+rEdAoDGBpE0Xqjy1oz5+SmtsaRUw==", + "dependencies": { + "@aws-crypto/sha256-browser": "5.2.0", + "@aws-crypto/sha256-js": "5.2.0", + "@aws-sdk/core": "3.635.0", + "@aws-sdk/middleware-host-header": "3.620.0", + "@aws-sdk/middleware-logger": "3.609.0", + "@aws-sdk/middleware-recursion-detection": "3.620.0", + "@aws-sdk/middleware-user-agent": "3.637.0", + "@aws-sdk/region-config-resolver": "3.614.0", + "@aws-sdk/types": "3.609.0", + "@aws-sdk/util-endpoints": "3.637.0", + "@aws-sdk/util-user-agent-browser": "3.609.0", + "@aws-sdk/util-user-agent-node": "3.614.0", + "@smithy/config-resolver": "^3.0.5", + "@smithy/core": "^2.4.0", + "@smithy/fetch-http-handler": "^3.2.4", + "@smithy/hash-node": "^3.0.3", + "@smithy/invalid-dependency": "^3.0.3", + "@smithy/middleware-content-length": "^3.0.5", + "@smithy/middleware-endpoint": "^3.1.0", + "@smithy/middleware-retry": "^3.0.15", + "@smithy/middleware-serde": "^3.0.3", + "@smithy/middleware-stack": "^3.0.3", + "@smithy/node-config-provider": "^3.1.4", + "@smithy/node-http-handler": "^3.1.4", + "@smithy/protocol-http": "^4.1.0", + "@smithy/smithy-client": "^3.2.0", + "@smithy/types": "^3.3.0", + "@smithy/url-parser": "^3.0.3", + "@smithy/util-base64": "^3.0.0", + "@smithy/util-body-length-browser": "^3.0.0", + "@smithy/util-body-length-node": "^3.0.0", + "@smithy/util-defaults-mode-browser": "^3.0.15", + "@smithy/util-defaults-mode-node": "^3.0.15", + "@smithy/util-endpoints": "^2.0.5", + "@smithy/util-middleware": "^3.0.3", + "@smithy/util-retry": "^3.0.3", + "@smithy/util-utf8": "^3.0.0", + "tslib": "^2.6.2" + }, + "engines": { + "node": ">=16.0.0" + } + }, + "node_modules/@aws-sdk/client-sso-oidc": { + "version": "3.637.0", + "resolved": "https://registry.npmjs.org/@aws-sdk/client-sso-oidc/-/client-sso-oidc-3.637.0.tgz", + "integrity": "sha512-27bHALN6Qb6m6KZmPvRieJ/QRlj1lyac/GT2Rn5kJpre8Mpp+yxrtvp3h9PjNBty4lCeFEENfY4dGNSozBuBcw==", + "dependencies": { + "@aws-crypto/sha256-browser": "5.2.0", + "@aws-crypto/sha256-js": "5.2.0", + "@aws-sdk/core": "3.635.0", + "@aws-sdk/credential-provider-node": "3.637.0", + "@aws-sdk/middleware-host-header": "3.620.0", + "@aws-sdk/middleware-logger": "3.609.0", + "@aws-sdk/middleware-recursion-detection": "3.620.0", + "@aws-sdk/middleware-user-agent": "3.637.0", + "@aws-sdk/region-config-resolver": "3.614.0", + "@aws-sdk/types": "3.609.0", + "@aws-sdk/util-endpoints": "3.637.0", + "@aws-sdk/util-user-agent-browser": "3.609.0", + "@aws-sdk/util-user-agent-node": "3.614.0", + "@smithy/config-resolver": "^3.0.5", + "@smithy/core": "^2.4.0", + "@smithy/fetch-http-handler": "^3.2.4", + "@smithy/hash-node": "^3.0.3", + "@smithy/invalid-dependency": "^3.0.3", + "@smithy/middleware-content-length": "^3.0.5", + "@smithy/middleware-endpoint": "^3.1.0", + "@smithy/middleware-retry": "^3.0.15", + "@smithy/middleware-serde": "^3.0.3", + "@smithy/middleware-stack": "^3.0.3", + "@smithy/node-config-provider": "^3.1.4", + "@smithy/node-http-handler": "^3.1.4", + "@smithy/protocol-http": "^4.1.0", + "@smithy/smithy-client": "^3.2.0", + "@smithy/types": "^3.3.0", + "@smithy/url-parser": "^3.0.3", + "@smithy/util-base64": "^3.0.0", + "@smithy/util-body-length-browser": "^3.0.0", + "@smithy/util-body-length-node": "^3.0.0", + "@smithy/util-defaults-mode-browser": "^3.0.15", + "@smithy/util-defaults-mode-node": "^3.0.15", + "@smithy/util-endpoints": "^2.0.5", + "@smithy/util-middleware": "^3.0.3", + "@smithy/util-retry": "^3.0.3", + "@smithy/util-utf8": "^3.0.0", + "tslib": "^2.6.2" + }, + "engines": { + "node": ">=16.0.0" + }, + "peerDependencies": { + "@aws-sdk/client-sts": "^3.637.0" + } + }, + "node_modules/@aws-sdk/client-sso-oidc/node_modules/tslib": { + "version": "2.7.0", + "resolved": "https://registry.npmjs.org/tslib/-/tslib-2.7.0.tgz", + "integrity": "sha512-gLXCKdN1/j47AiHiOkJN69hJmcbGTHI0ImLmbYLHykhgeN0jVGola9yVjFgzCUklsZQMW55o+dW7IXv3RCXDzA==" + }, + "node_modules/@aws-sdk/client-sso/node_modules/tslib": { + "version": "2.7.0", + "resolved": "https://registry.npmjs.org/tslib/-/tslib-2.7.0.tgz", + "integrity": "sha512-gLXCKdN1/j47AiHiOkJN69hJmcbGTHI0ImLmbYLHykhgeN0jVGola9yVjFgzCUklsZQMW55o+dW7IXv3RCXDzA==" + }, + "node_modules/@aws-sdk/client-sts": { + "version": "3.637.0", + "resolved": "https://registry.npmjs.org/@aws-sdk/client-sts/-/client-sts-3.637.0.tgz", + "integrity": "sha512-xUi7x4qDubtA8QREtlblPuAcn91GS/09YVEY/RwU7xCY0aqGuFwgszAANlha4OUIqva8oVj2WO4gJuG+iaSnhw==", + "dependencies": { + "@aws-crypto/sha256-browser": "5.2.0", + "@aws-crypto/sha256-js": "5.2.0", + "@aws-sdk/client-sso-oidc": "3.637.0", + "@aws-sdk/core": "3.635.0", + "@aws-sdk/credential-provider-node": "3.637.0", + "@aws-sdk/middleware-host-header": "3.620.0", + "@aws-sdk/middleware-logger": "3.609.0", + "@aws-sdk/middleware-recursion-detection": "3.620.0", + "@aws-sdk/middleware-user-agent": "3.637.0", + "@aws-sdk/region-config-resolver": "3.614.0", + "@aws-sdk/types": "3.609.0", + "@aws-sdk/util-endpoints": "3.637.0", + "@aws-sdk/util-user-agent-browser": "3.609.0", + "@aws-sdk/util-user-agent-node": "3.614.0", + "@smithy/config-resolver": "^3.0.5", + "@smithy/core": "^2.4.0", + "@smithy/fetch-http-handler": "^3.2.4", + "@smithy/hash-node": "^3.0.3", + "@smithy/invalid-dependency": "^3.0.3", + "@smithy/middleware-content-length": "^3.0.5", + "@smithy/middleware-endpoint": "^3.1.0", + "@smithy/middleware-retry": "^3.0.15", + "@smithy/middleware-serde": "^3.0.3", + "@smithy/middleware-stack": "^3.0.3", + "@smithy/node-config-provider": "^3.1.4", + "@smithy/node-http-handler": "^3.1.4", + "@smithy/protocol-http": "^4.1.0", + "@smithy/smithy-client": "^3.2.0", + "@smithy/types": "^3.3.0", + "@smithy/url-parser": "^3.0.3", + "@smithy/util-base64": "^3.0.0", + "@smithy/util-body-length-browser": "^3.0.0", + "@smithy/util-body-length-node": "^3.0.0", + "@smithy/util-defaults-mode-browser": "^3.0.15", + "@smithy/util-defaults-mode-node": "^3.0.15", + "@smithy/util-endpoints": "^2.0.5", + "@smithy/util-middleware": "^3.0.3", + "@smithy/util-retry": "^3.0.3", + "@smithy/util-utf8": "^3.0.0", + "tslib": "^2.6.2" + }, + "engines": { + "node": ">=16.0.0" + } + }, + "node_modules/@aws-sdk/client-sts/node_modules/tslib": { + "version": "2.7.0", + "resolved": "https://registry.npmjs.org/tslib/-/tslib-2.7.0.tgz", + "integrity": "sha512-gLXCKdN1/j47AiHiOkJN69hJmcbGTHI0ImLmbYLHykhgeN0jVGola9yVjFgzCUklsZQMW55o+dW7IXv3RCXDzA==" + }, + "node_modules/@aws-sdk/core": { + "version": "3.635.0", + "resolved": "https://registry.npmjs.org/@aws-sdk/core/-/core-3.635.0.tgz", + "integrity": "sha512-i1x/E/sgA+liUE1XJ7rj1dhyXpAKO1UKFUcTTHXok2ARjWTvszHnSXMOsB77aPbmn0fUp1JTx2kHUAZ1LVt5Bg==", + "dependencies": { + "@smithy/core": "^2.4.0", + "@smithy/node-config-provider": "^3.1.4", + "@smithy/property-provider": "^3.1.3", + "@smithy/protocol-http": "^4.1.0", + "@smithy/signature-v4": "^4.1.0", + "@smithy/smithy-client": "^3.2.0", + "@smithy/types": "^3.3.0", + "@smithy/util-middleware": "^3.0.3", + "fast-xml-parser": "4.4.1", + "tslib": "^2.6.2" + }, + "engines": { + "node": ">=16.0.0" + } + }, + "node_modules/@aws-sdk/core/node_modules/tslib": { + "version": "2.7.0", + "resolved": "https://registry.npmjs.org/tslib/-/tslib-2.7.0.tgz", + "integrity": "sha512-gLXCKdN1/j47AiHiOkJN69hJmcbGTHI0ImLmbYLHykhgeN0jVGola9yVjFgzCUklsZQMW55o+dW7IXv3RCXDzA==" + }, + "node_modules/@aws-sdk/credential-provider-env": { + "version": "3.620.1", + "resolved": "https://registry.npmjs.org/@aws-sdk/credential-provider-env/-/credential-provider-env-3.620.1.tgz", + "integrity": "sha512-ExuILJ2qLW5ZO+rgkNRj0xiAipKT16Rk77buvPP8csR7kkCflT/gXTyzRe/uzIiETTxM7tr8xuO9MP/DQXqkfg==", + "dependencies": { + "@aws-sdk/types": "3.609.0", + "@smithy/property-provider": "^3.1.3", + "@smithy/types": "^3.3.0", + "tslib": "^2.6.2" + }, + "engines": { + "node": ">=16.0.0" + } + }, + "node_modules/@aws-sdk/credential-provider-env/node_modules/tslib": { + "version": "2.7.0", + "resolved": "https://registry.npmjs.org/tslib/-/tslib-2.7.0.tgz", + "integrity": "sha512-gLXCKdN1/j47AiHiOkJN69hJmcbGTHI0ImLmbYLHykhgeN0jVGola9yVjFgzCUklsZQMW55o+dW7IXv3RCXDzA==" + }, + "node_modules/@aws-sdk/credential-provider-http": { + "version": "3.635.0", + "resolved": "https://registry.npmjs.org/@aws-sdk/credential-provider-http/-/credential-provider-http-3.635.0.tgz", + "integrity": "sha512-iJyRgEjOCQlBMXqtwPLIKYc7Bsc6nqjrZybdMDenPDa+kmLg7xh8LxHsu9088e+2/wtLicE34FsJJIfzu3L82g==", + "dependencies": { + "@aws-sdk/types": "3.609.0", + "@smithy/fetch-http-handler": "^3.2.4", + "@smithy/node-http-handler": "^3.1.4", + "@smithy/property-provider": "^3.1.3", + "@smithy/protocol-http": "^4.1.0", + "@smithy/smithy-client": "^3.2.0", + "@smithy/types": "^3.3.0", + "@smithy/util-stream": "^3.1.3", + "tslib": "^2.6.2" + }, + "engines": { + "node": ">=16.0.0" + } + }, + "node_modules/@aws-sdk/credential-provider-http/node_modules/tslib": { + "version": "2.7.0", + "resolved": "https://registry.npmjs.org/tslib/-/tslib-2.7.0.tgz", + "integrity": "sha512-gLXCKdN1/j47AiHiOkJN69hJmcbGTHI0ImLmbYLHykhgeN0jVGola9yVjFgzCUklsZQMW55o+dW7IXv3RCXDzA==" + }, + "node_modules/@aws-sdk/credential-provider-ini": { + "version": "3.637.0", + "resolved": "https://registry.npmjs.org/@aws-sdk/credential-provider-ini/-/credential-provider-ini-3.637.0.tgz", + "integrity": "sha512-h+PFCWfZ0Q3Dx84SppET/TFpcQHmxFW8/oV9ArEvMilw4EBN+IlxgbL0CnHwjHW64szcmrM0mbebjEfHf4FXmw==", + "dependencies": { + "@aws-sdk/credential-provider-env": "3.620.1", + "@aws-sdk/credential-provider-http": "3.635.0", + "@aws-sdk/credential-provider-process": "3.620.1", + "@aws-sdk/credential-provider-sso": "3.637.0", + "@aws-sdk/credential-provider-web-identity": "3.621.0", + "@aws-sdk/types": "3.609.0", + "@smithy/credential-provider-imds": "^3.2.0", + "@smithy/property-provider": "^3.1.3", + "@smithy/shared-ini-file-loader": "^3.1.4", + "@smithy/types": "^3.3.0", + "tslib": "^2.6.2" + }, + "engines": { + "node": ">=16.0.0" + }, + "peerDependencies": { + "@aws-sdk/client-sts": "^3.637.0" + } + }, + "node_modules/@aws-sdk/credential-provider-ini/node_modules/tslib": { + "version": "2.7.0", + "resolved": "https://registry.npmjs.org/tslib/-/tslib-2.7.0.tgz", + "integrity": "sha512-gLXCKdN1/j47AiHiOkJN69hJmcbGTHI0ImLmbYLHykhgeN0jVGola9yVjFgzCUklsZQMW55o+dW7IXv3RCXDzA==" + }, + "node_modules/@aws-sdk/credential-provider-node": { + "version": "3.637.0", + "resolved": "https://registry.npmjs.org/@aws-sdk/credential-provider-node/-/credential-provider-node-3.637.0.tgz", + "integrity": "sha512-yoEhoxJJfs7sPVQ6Is939BDQJZpZCoUgKr/ySse4YKOZ24t4VqgHA6+wV7rYh+7IW24Rd91UTvEzSuHYTlxlNA==", + "dependencies": { + "@aws-sdk/credential-provider-env": "3.620.1", + "@aws-sdk/credential-provider-http": "3.635.0", + "@aws-sdk/credential-provider-ini": "3.637.0", + "@aws-sdk/credential-provider-process": "3.620.1", + "@aws-sdk/credential-provider-sso": "3.637.0", + "@aws-sdk/credential-provider-web-identity": "3.621.0", + "@aws-sdk/types": "3.609.0", + "@smithy/credential-provider-imds": "^3.2.0", + "@smithy/property-provider": "^3.1.3", + "@smithy/shared-ini-file-loader": "^3.1.4", + "@smithy/types": "^3.3.0", + "tslib": "^2.6.2" + }, + "engines": { + "node": ">=16.0.0" + } + }, + "node_modules/@aws-sdk/credential-provider-node/node_modules/tslib": { + "version": "2.7.0", + "resolved": "https://registry.npmjs.org/tslib/-/tslib-2.7.0.tgz", + "integrity": "sha512-gLXCKdN1/j47AiHiOkJN69hJmcbGTHI0ImLmbYLHykhgeN0jVGola9yVjFgzCUklsZQMW55o+dW7IXv3RCXDzA==" + }, + "node_modules/@aws-sdk/credential-provider-process": { + "version": "3.620.1", + "resolved": "https://registry.npmjs.org/@aws-sdk/credential-provider-process/-/credential-provider-process-3.620.1.tgz", + "integrity": "sha512-hWqFMidqLAkaV9G460+1at6qa9vySbjQKKc04p59OT7lZ5cO5VH5S4aI05e+m4j364MBROjjk2ugNvfNf/8ILg==", + "dependencies": { + "@aws-sdk/types": "3.609.0", + "@smithy/property-provider": "^3.1.3", + "@smithy/shared-ini-file-loader": "^3.1.4", + "@smithy/types": "^3.3.0", + "tslib": "^2.6.2" + }, + "engines": { + "node": ">=16.0.0" + } + }, + "node_modules/@aws-sdk/credential-provider-process/node_modules/tslib": { + "version": "2.7.0", + "resolved": "https://registry.npmjs.org/tslib/-/tslib-2.7.0.tgz", + "integrity": "sha512-gLXCKdN1/j47AiHiOkJN69hJmcbGTHI0ImLmbYLHykhgeN0jVGola9yVjFgzCUklsZQMW55o+dW7IXv3RCXDzA==" + }, + "node_modules/@aws-sdk/credential-provider-sso": { + "version": "3.637.0", + "resolved": "https://registry.npmjs.org/@aws-sdk/credential-provider-sso/-/credential-provider-sso-3.637.0.tgz", + "integrity": "sha512-Mvz+h+e62/tl+dVikLafhv+qkZJ9RUb8l2YN/LeKMWkxQylPT83CPk9aimVhCV89zth1zpREArl97+3xsfgQvA==", + "dependencies": { + "@aws-sdk/client-sso": "3.637.0", + "@aws-sdk/token-providers": "3.614.0", + "@aws-sdk/types": "3.609.0", + "@smithy/property-provider": "^3.1.3", + "@smithy/shared-ini-file-loader": "^3.1.4", + "@smithy/types": "^3.3.0", + "tslib": "^2.6.2" + }, + "engines": { + "node": ">=16.0.0" + } + }, + "node_modules/@aws-sdk/credential-provider-sso/node_modules/tslib": { + "version": "2.7.0", + "resolved": "https://registry.npmjs.org/tslib/-/tslib-2.7.0.tgz", + "integrity": "sha512-gLXCKdN1/j47AiHiOkJN69hJmcbGTHI0ImLmbYLHykhgeN0jVGola9yVjFgzCUklsZQMW55o+dW7IXv3RCXDzA==" + }, + "node_modules/@aws-sdk/credential-provider-web-identity": { + "version": "3.621.0", + "resolved": "https://registry.npmjs.org/@aws-sdk/credential-provider-web-identity/-/credential-provider-web-identity-3.621.0.tgz", + "integrity": "sha512-w7ASSyfNvcx7+bYGep3VBgC3K6vEdLmlpjT7nSIHxxQf+WSdvy+HynwJosrpZax0sK5q0D1Jpn/5q+r5lwwW6w==", + "dependencies": { + "@aws-sdk/types": "3.609.0", + "@smithy/property-provider": "^3.1.3", + "@smithy/types": "^3.3.0", + "tslib": "^2.6.2" + }, + "engines": { + "node": ">=16.0.0" + }, + "peerDependencies": { + "@aws-sdk/client-sts": "^3.621.0" + } + }, + "node_modules/@aws-sdk/credential-provider-web-identity/node_modules/tslib": { + "version": "2.7.0", + "resolved": "https://registry.npmjs.org/tslib/-/tslib-2.7.0.tgz", + "integrity": "sha512-gLXCKdN1/j47AiHiOkJN69hJmcbGTHI0ImLmbYLHykhgeN0jVGola9yVjFgzCUklsZQMW55o+dW7IXv3RCXDzA==" + }, + "node_modules/@aws-sdk/middleware-bucket-endpoint": { + "version": "3.620.0", + "resolved": "https://registry.npmjs.org/@aws-sdk/middleware-bucket-endpoint/-/middleware-bucket-endpoint-3.620.0.tgz", + "integrity": "sha512-eGLL0W6L3HDb3OACyetZYOWpHJ+gLo0TehQKeQyy2G8vTYXqNTeqYhuI6up9HVjBzU9eQiULVQETmgQs7TFaRg==", + "dependencies": { + "@aws-sdk/types": "3.609.0", + "@aws-sdk/util-arn-parser": "3.568.0", + "@smithy/node-config-provider": "^3.1.4", + "@smithy/protocol-http": "^4.1.0", + "@smithy/types": "^3.3.0", + "@smithy/util-config-provider": "^3.0.0", + "tslib": "^2.6.2" + }, + "engines": { + "node": ">=16.0.0" + } + }, + "node_modules/@aws-sdk/middleware-bucket-endpoint/node_modules/tslib": { + "version": "2.7.0", + "resolved": "https://registry.npmjs.org/tslib/-/tslib-2.7.0.tgz", + "integrity": "sha512-gLXCKdN1/j47AiHiOkJN69hJmcbGTHI0ImLmbYLHykhgeN0jVGola9yVjFgzCUklsZQMW55o+dW7IXv3RCXDzA==" + }, + "node_modules/@aws-sdk/middleware-expect-continue": { + "version": "3.620.0", + "resolved": "https://registry.npmjs.org/@aws-sdk/middleware-expect-continue/-/middleware-expect-continue-3.620.0.tgz", + "integrity": "sha512-QXeRFMLfyQ31nAHLbiTLtk0oHzG9QLMaof5jIfqcUwnOkO8YnQdeqzakrg1Alpy/VQ7aqzIi8qypkBe2KXZz0A==", + "dependencies": { + "@aws-sdk/types": "3.609.0", + "@smithy/protocol-http": "^4.1.0", + "@smithy/types": "^3.3.0", + "tslib": "^2.6.2" + }, + "engines": { + "node": ">=16.0.0" + } + }, + "node_modules/@aws-sdk/middleware-expect-continue/node_modules/tslib": { + "version": "2.7.0", + "resolved": "https://registry.npmjs.org/tslib/-/tslib-2.7.0.tgz", + "integrity": "sha512-gLXCKdN1/j47AiHiOkJN69hJmcbGTHI0ImLmbYLHykhgeN0jVGola9yVjFgzCUklsZQMW55o+dW7IXv3RCXDzA==" + }, + "node_modules/@aws-sdk/middleware-flexible-checksums": { + "version": "3.620.0", + "resolved": "https://registry.npmjs.org/@aws-sdk/middleware-flexible-checksums/-/middleware-flexible-checksums-3.620.0.tgz", + "integrity": "sha512-ftz+NW7qka2sVuwnnO1IzBku5ccP+s5qZGeRTPgrKB7OzRW85gthvIo1vQR2w+OwHFk7WJbbhhWwbCbktnP4UA==", + "dependencies": { + "@aws-crypto/crc32": "5.2.0", + "@aws-crypto/crc32c": "5.2.0", + "@aws-sdk/types": "3.609.0", + "@smithy/is-array-buffer": "^3.0.0", + "@smithy/protocol-http": "^4.1.0", + "@smithy/types": "^3.3.0", + "@smithy/util-utf8": "^3.0.0", + "tslib": "^2.6.2" + }, + "engines": { + "node": ">=16.0.0" + } + }, + "node_modules/@aws-sdk/middleware-flexible-checksums/node_modules/tslib": { + "version": "2.7.0", + "resolved": "https://registry.npmjs.org/tslib/-/tslib-2.7.0.tgz", + "integrity": "sha512-gLXCKdN1/j47AiHiOkJN69hJmcbGTHI0ImLmbYLHykhgeN0jVGola9yVjFgzCUklsZQMW55o+dW7IXv3RCXDzA==" + }, + "node_modules/@aws-sdk/middleware-host-header": { + "version": "3.620.0", + "resolved": "https://registry.npmjs.org/@aws-sdk/middleware-host-header/-/middleware-host-header-3.620.0.tgz", + "integrity": "sha512-VMtPEZwqYrII/oUkffYsNWY9PZ9xpNJpMgmyU0rlDQ25O1c0Hk3fJmZRe6pEkAJ0omD7kLrqGl1DUjQVxpd/Rg==", + "dependencies": { + "@aws-sdk/types": "3.609.0", + "@smithy/protocol-http": "^4.1.0", + "@smithy/types": "^3.3.0", + "tslib": "^2.6.2" + }, + "engines": { + "node": ">=16.0.0" + } + }, + "node_modules/@aws-sdk/middleware-host-header/node_modules/tslib": { + "version": "2.7.0", + "resolved": "https://registry.npmjs.org/tslib/-/tslib-2.7.0.tgz", + "integrity": "sha512-gLXCKdN1/j47AiHiOkJN69hJmcbGTHI0ImLmbYLHykhgeN0jVGola9yVjFgzCUklsZQMW55o+dW7IXv3RCXDzA==" + }, + "node_modules/@aws-sdk/middleware-location-constraint": { + "version": "3.609.0", + "resolved": "https://registry.npmjs.org/@aws-sdk/middleware-location-constraint/-/middleware-location-constraint-3.609.0.tgz", + "integrity": "sha512-xzsdoTkszGVqGVPjUmgoP7TORiByLueMHieI1fhQL888WPdqctwAx3ES6d/bA9Q/i8jnc6hs+Fjhy8UvBTkE9A==", + "dependencies": { + "@aws-sdk/types": "3.609.0", + "@smithy/types": "^3.3.0", + "tslib": "^2.6.2" + }, + "engines": { + "node": ">=16.0.0" + } + }, + "node_modules/@aws-sdk/middleware-location-constraint/node_modules/tslib": { + "version": "2.7.0", + "resolved": "https://registry.npmjs.org/tslib/-/tslib-2.7.0.tgz", + "integrity": "sha512-gLXCKdN1/j47AiHiOkJN69hJmcbGTHI0ImLmbYLHykhgeN0jVGola9yVjFgzCUklsZQMW55o+dW7IXv3RCXDzA==" + }, + "node_modules/@aws-sdk/middleware-logger": { + "version": "3.609.0", + "resolved": "https://registry.npmjs.org/@aws-sdk/middleware-logger/-/middleware-logger-3.609.0.tgz", + "integrity": "sha512-S62U2dy4jMDhDFDK5gZ4VxFdWzCtLzwbYyFZx2uvPYTECkepLUfzLic2BHg2Qvtu4QjX+oGE3P/7fwaGIsGNuQ==", + "dependencies": { + "@aws-sdk/types": "3.609.0", + "@smithy/types": "^3.3.0", + "tslib": "^2.6.2" + }, + "engines": { + "node": ">=16.0.0" + } + }, + "node_modules/@aws-sdk/middleware-logger/node_modules/tslib": { + "version": "2.7.0", + "resolved": "https://registry.npmjs.org/tslib/-/tslib-2.7.0.tgz", + "integrity": "sha512-gLXCKdN1/j47AiHiOkJN69hJmcbGTHI0ImLmbYLHykhgeN0jVGola9yVjFgzCUklsZQMW55o+dW7IXv3RCXDzA==" + }, + "node_modules/@aws-sdk/middleware-recursion-detection": { + "version": "3.620.0", + "resolved": "https://registry.npmjs.org/@aws-sdk/middleware-recursion-detection/-/middleware-recursion-detection-3.620.0.tgz", + "integrity": "sha512-nh91S7aGK3e/o1ck64sA/CyoFw+gAYj2BDOnoNa6ouyCrVJED96ZXWbhye/fz9SgmNUZR2g7GdVpiLpMKZoI5w==", + "dependencies": { + "@aws-sdk/types": "3.609.0", + "@smithy/protocol-http": "^4.1.0", + "@smithy/types": "^3.3.0", + "tslib": "^2.6.2" + }, + "engines": { + "node": ">=16.0.0" + } + }, + "node_modules/@aws-sdk/middleware-recursion-detection/node_modules/tslib": { + "version": "2.7.0", + "resolved": "https://registry.npmjs.org/tslib/-/tslib-2.7.0.tgz", + "integrity": "sha512-gLXCKdN1/j47AiHiOkJN69hJmcbGTHI0ImLmbYLHykhgeN0jVGola9yVjFgzCUklsZQMW55o+dW7IXv3RCXDzA==" + }, + "node_modules/@aws-sdk/middleware-sdk-s3": { + "version": "3.635.0", + "resolved": "https://registry.npmjs.org/@aws-sdk/middleware-sdk-s3/-/middleware-sdk-s3-3.635.0.tgz", + "integrity": "sha512-RLdYJPEV4JL/7NBoFUs7VlP90X++5FlJdxHz0DzCjmiD3qCviKy+Cym3qg1gBgHwucs5XisuClxDrGokhAdTQw==", + "dependencies": { + "@aws-sdk/core": "3.635.0", + "@aws-sdk/types": "3.609.0", + "@aws-sdk/util-arn-parser": "3.568.0", + "@smithy/core": "^2.4.0", + "@smithy/node-config-provider": "^3.1.4", + "@smithy/protocol-http": "^4.1.0", + "@smithy/signature-v4": "^4.1.0", + "@smithy/smithy-client": "^3.2.0", + "@smithy/types": "^3.3.0", + "@smithy/util-config-provider": "^3.0.0", + "@smithy/util-middleware": "^3.0.3", + "@smithy/util-stream": "^3.1.3", + "@smithy/util-utf8": "^3.0.0", + "tslib": "^2.6.2" + }, + "engines": { + "node": ">=16.0.0" + } + }, + "node_modules/@aws-sdk/middleware-sdk-s3/node_modules/tslib": { + "version": "2.7.0", + "resolved": "https://registry.npmjs.org/tslib/-/tslib-2.7.0.tgz", + "integrity": "sha512-gLXCKdN1/j47AiHiOkJN69hJmcbGTHI0ImLmbYLHykhgeN0jVGola9yVjFgzCUklsZQMW55o+dW7IXv3RCXDzA==" + }, + "node_modules/@aws-sdk/middleware-ssec": { + "version": "3.609.0", + "resolved": "https://registry.npmjs.org/@aws-sdk/middleware-ssec/-/middleware-ssec-3.609.0.tgz", + "integrity": "sha512-GZSD1s7+JswWOTamVap79QiDaIV7byJFssBW68GYjyRS5EBjNfwA/8s+6uE6g39R3ojyTbYOmvcANoZEhSULXg==", + "dependencies": { + "@aws-sdk/types": "3.609.0", + "@smithy/types": "^3.3.0", + "tslib": "^2.6.2" + }, + "engines": { + "node": ">=16.0.0" + } + }, + "node_modules/@aws-sdk/middleware-ssec/node_modules/tslib": { + "version": "2.7.0", + "resolved": "https://registry.npmjs.org/tslib/-/tslib-2.7.0.tgz", + "integrity": "sha512-gLXCKdN1/j47AiHiOkJN69hJmcbGTHI0ImLmbYLHykhgeN0jVGola9yVjFgzCUklsZQMW55o+dW7IXv3RCXDzA==" + }, + "node_modules/@aws-sdk/middleware-user-agent": { + "version": "3.637.0", + "resolved": "https://registry.npmjs.org/@aws-sdk/middleware-user-agent/-/middleware-user-agent-3.637.0.tgz", + "integrity": "sha512-EYo0NE9/da/OY8STDsK2LvM4kNa79DBsf4YVtaG4P5pZ615IeFsD8xOHZeuJmUrSMlVQ8ywPRX7WMucUybsKug==", + "dependencies": { + "@aws-sdk/types": "3.609.0", + "@aws-sdk/util-endpoints": "3.637.0", + "@smithy/protocol-http": "^4.1.0", + "@smithy/types": "^3.3.0", + "tslib": "^2.6.2" + }, + "engines": { + "node": ">=16.0.0" + } + }, + "node_modules/@aws-sdk/middleware-user-agent/node_modules/tslib": { + "version": "2.7.0", + "resolved": "https://registry.npmjs.org/tslib/-/tslib-2.7.0.tgz", + "integrity": "sha512-gLXCKdN1/j47AiHiOkJN69hJmcbGTHI0ImLmbYLHykhgeN0jVGola9yVjFgzCUklsZQMW55o+dW7IXv3RCXDzA==" + }, + "node_modules/@aws-sdk/region-config-resolver": { + "version": "3.614.0", + "resolved": "https://registry.npmjs.org/@aws-sdk/region-config-resolver/-/region-config-resolver-3.614.0.tgz", + "integrity": "sha512-vDCeMXvic/LU0KFIUjpC3RiSTIkkvESsEfbVHiHH0YINfl8HnEqR5rj+L8+phsCeVg2+LmYwYxd5NRz4PHxt5g==", + "dependencies": { + "@aws-sdk/types": "3.609.0", + "@smithy/node-config-provider": "^3.1.4", + "@smithy/types": "^3.3.0", + "@smithy/util-config-provider": "^3.0.0", + "@smithy/util-middleware": "^3.0.3", + "tslib": "^2.6.2" + }, + "engines": { + "node": ">=16.0.0" + } + }, + "node_modules/@aws-sdk/region-config-resolver/node_modules/tslib": { + "version": "2.7.0", + "resolved": "https://registry.npmjs.org/tslib/-/tslib-2.7.0.tgz", + "integrity": "sha512-gLXCKdN1/j47AiHiOkJN69hJmcbGTHI0ImLmbYLHykhgeN0jVGola9yVjFgzCUklsZQMW55o+dW7IXv3RCXDzA==" + }, + "node_modules/@aws-sdk/signature-v4-multi-region": { + "version": "3.635.0", + "resolved": "https://registry.npmjs.org/@aws-sdk/signature-v4-multi-region/-/signature-v4-multi-region-3.635.0.tgz", + "integrity": "sha512-J6QY4/invOkpogCHjSaDON1hF03viPpOnsrzVuCvJMmclS/iG62R4EY0wq1alYll0YmSdmKlpJwHMWwGtqK63Q==", + "dependencies": { + "@aws-sdk/middleware-sdk-s3": "3.635.0", + "@aws-sdk/types": "3.609.0", + "@smithy/protocol-http": "^4.1.0", + "@smithy/signature-v4": "^4.1.0", + "@smithy/types": "^3.3.0", + "tslib": "^2.6.2" + }, + "engines": { + "node": ">=16.0.0" + } + }, + "node_modules/@aws-sdk/signature-v4-multi-region/node_modules/tslib": { + "version": "2.7.0", + "resolved": "https://registry.npmjs.org/tslib/-/tslib-2.7.0.tgz", + "integrity": "sha512-gLXCKdN1/j47AiHiOkJN69hJmcbGTHI0ImLmbYLHykhgeN0jVGola9yVjFgzCUklsZQMW55o+dW7IXv3RCXDzA==" + }, + "node_modules/@aws-sdk/token-providers": { + "version": "3.614.0", + "resolved": "https://registry.npmjs.org/@aws-sdk/token-providers/-/token-providers-3.614.0.tgz", + "integrity": "sha512-okItqyY6L9IHdxqs+Z116y5/nda7rHxLvROxtAJdLavWTYDydxrZstImNgGWTeVdmc0xX2gJCI77UYUTQWnhRw==", + "dependencies": { + "@aws-sdk/types": "3.609.0", + "@smithy/property-provider": "^3.1.3", + "@smithy/shared-ini-file-loader": "^3.1.4", + "@smithy/types": "^3.3.0", + "tslib": "^2.6.2" + }, + "engines": { + "node": ">=16.0.0" + }, + "peerDependencies": { + "@aws-sdk/client-sso-oidc": "^3.614.0" + } + }, + "node_modules/@aws-sdk/token-providers/node_modules/tslib": { + "version": "2.7.0", + "resolved": "https://registry.npmjs.org/tslib/-/tslib-2.7.0.tgz", + "integrity": "sha512-gLXCKdN1/j47AiHiOkJN69hJmcbGTHI0ImLmbYLHykhgeN0jVGola9yVjFgzCUklsZQMW55o+dW7IXv3RCXDzA==" + }, + "node_modules/@aws-sdk/types": { + "version": "3.609.0", + "resolved": "https://registry.npmjs.org/@aws-sdk/types/-/types-3.609.0.tgz", + "integrity": "sha512-+Tqnh9w0h2LcrUsdXyT1F8mNhXz+tVYBtP19LpeEGntmvHwa2XzvLUCWpoIAIVsHp5+HdB2X9Sn0KAtmbFXc2Q==", + "dependencies": { + "@smithy/types": "^3.3.0", + "tslib": "^2.6.2" + }, + "engines": { + "node": ">=16.0.0" + } + }, + "node_modules/@aws-sdk/types/node_modules/tslib": { + "version": "2.7.0", + "resolved": "https://registry.npmjs.org/tslib/-/tslib-2.7.0.tgz", + "integrity": "sha512-gLXCKdN1/j47AiHiOkJN69hJmcbGTHI0ImLmbYLHykhgeN0jVGola9yVjFgzCUklsZQMW55o+dW7IXv3RCXDzA==" + }, + "node_modules/@aws-sdk/util-arn-parser": { + "version": "3.568.0", + "resolved": "https://registry.npmjs.org/@aws-sdk/util-arn-parser/-/util-arn-parser-3.568.0.tgz", + "integrity": "sha512-XUKJWWo+KOB7fbnPP0+g/o5Ulku/X53t7i/h+sPHr5xxYTJJ9CYnbToo95mzxe7xWvkLrsNtJ8L+MnNn9INs2w==", + "dependencies": { + "tslib": "^2.6.2" + }, + "engines": { + "node": ">=16.0.0" + } + }, + "node_modules/@aws-sdk/util-arn-parser/node_modules/tslib": { + "version": "2.7.0", + "resolved": "https://registry.npmjs.org/tslib/-/tslib-2.7.0.tgz", + "integrity": "sha512-gLXCKdN1/j47AiHiOkJN69hJmcbGTHI0ImLmbYLHykhgeN0jVGola9yVjFgzCUklsZQMW55o+dW7IXv3RCXDzA==" + }, + "node_modules/@aws-sdk/util-endpoints": { + "version": "3.637.0", + "resolved": "https://registry.npmjs.org/@aws-sdk/util-endpoints/-/util-endpoints-3.637.0.tgz", + "integrity": "sha512-pAqOKUHeVWHEXXDIp/qoMk/6jyxIb6GGjnK1/f8dKHtKIEs4tKsnnL563gceEvdad53OPXIt86uoevCcCzmBnw==", + "dependencies": { + "@aws-sdk/types": "3.609.0", + "@smithy/types": "^3.3.0", + "@smithy/util-endpoints": "^2.0.5", + "tslib": "^2.6.2" + }, + "engines": { + "node": ">=16.0.0" + } + }, + "node_modules/@aws-sdk/util-endpoints/node_modules/tslib": { + "version": "2.7.0", + "resolved": "https://registry.npmjs.org/tslib/-/tslib-2.7.0.tgz", + "integrity": "sha512-gLXCKdN1/j47AiHiOkJN69hJmcbGTHI0ImLmbYLHykhgeN0jVGola9yVjFgzCUklsZQMW55o+dW7IXv3RCXDzA==" + }, + "node_modules/@aws-sdk/util-locate-window": { + "version": "3.568.0", + "resolved": "https://registry.npmjs.org/@aws-sdk/util-locate-window/-/util-locate-window-3.568.0.tgz", + "integrity": "sha512-3nh4TINkXYr+H41QaPelCceEB2FXP3fxp93YZXB/kqJvX0U9j0N0Uk45gvsjmEPzG8XxkPEeLIfT2I1M7A6Lig==", + "dependencies": { + "tslib": "^2.6.2" + }, + "engines": { + "node": ">=16.0.0" + } + }, + "node_modules/@aws-sdk/util-locate-window/node_modules/tslib": { + "version": "2.7.0", + "resolved": "https://registry.npmjs.org/tslib/-/tslib-2.7.0.tgz", + "integrity": "sha512-gLXCKdN1/j47AiHiOkJN69hJmcbGTHI0ImLmbYLHykhgeN0jVGola9yVjFgzCUklsZQMW55o+dW7IXv3RCXDzA==" + }, + "node_modules/@aws-sdk/util-user-agent-browser": { + "version": "3.609.0", + "resolved": "https://registry.npmjs.org/@aws-sdk/util-user-agent-browser/-/util-user-agent-browser-3.609.0.tgz", + "integrity": "sha512-fojPU+mNahzQ0YHYBsx0ZIhmMA96H+ZIZ665ObU9tl+SGdbLneVZVikGve+NmHTQwHzwkFsZYYnVKAkreJLAtA==", + "dependencies": { + "@aws-sdk/types": "3.609.0", + "@smithy/types": "^3.3.0", + "bowser": "^2.11.0", + "tslib": "^2.6.2" + } + }, + "node_modules/@aws-sdk/util-user-agent-browser/node_modules/tslib": { + "version": "2.7.0", + "resolved": "https://registry.npmjs.org/tslib/-/tslib-2.7.0.tgz", + "integrity": "sha512-gLXCKdN1/j47AiHiOkJN69hJmcbGTHI0ImLmbYLHykhgeN0jVGola9yVjFgzCUklsZQMW55o+dW7IXv3RCXDzA==" + }, + "node_modules/@aws-sdk/util-user-agent-node": { + "version": "3.614.0", + "resolved": "https://registry.npmjs.org/@aws-sdk/util-user-agent-node/-/util-user-agent-node-3.614.0.tgz", + "integrity": "sha512-15ElZT88peoHnq5TEoEtZwoXTXRxNrk60TZNdpl/TUBJ5oNJ9Dqb5Z4ryb8ofN6nm9aFf59GVAerFDz8iUoHBA==", + "dependencies": { + "@aws-sdk/types": "3.609.0", + "@smithy/node-config-provider": "^3.1.4", + "@smithy/types": "^3.3.0", + "tslib": "^2.6.2" + }, + "engines": { + "node": ">=16.0.0" + }, + "peerDependencies": { + "aws-crt": ">=1.0.0" + }, + "peerDependenciesMeta": { + "aws-crt": { + "optional": true + } + } + }, + "node_modules/@aws-sdk/util-user-agent-node/node_modules/tslib": { + "version": "2.7.0", + "resolved": "https://registry.npmjs.org/tslib/-/tslib-2.7.0.tgz", + "integrity": "sha512-gLXCKdN1/j47AiHiOkJN69hJmcbGTHI0ImLmbYLHykhgeN0jVGola9yVjFgzCUklsZQMW55o+dW7IXv3RCXDzA==" + }, + "node_modules/@aws-sdk/xml-builder": { + "version": "3.609.0", + "resolved": "https://registry.npmjs.org/@aws-sdk/xml-builder/-/xml-builder-3.609.0.tgz", + "integrity": "sha512-l9XxNcA4HX98rwCC2/KoiWcmEiRfZe4G+mYwDbCFT87JIMj6GBhLDkAzr/W8KAaA2IDr8Vc6J8fZPgVulxxfMA==", + "dependencies": { + "@smithy/types": "^3.3.0", + "tslib": "^2.6.2" + }, + "engines": { + "node": ">=16.0.0" + } + }, + "node_modules/@aws-sdk/xml-builder/node_modules/tslib": { + "version": "2.7.0", + "resolved": "https://registry.npmjs.org/tslib/-/tslib-2.7.0.tgz", + "integrity": "sha512-gLXCKdN1/j47AiHiOkJN69hJmcbGTHI0ImLmbYLHykhgeN0jVGola9yVjFgzCUklsZQMW55o+dW7IXv3RCXDzA==" + }, "node_modules/@babel/code-frame": { "version": "7.16.7", "resolved": "https://registry.npmjs.org/@babel/code-frame/-/code-frame-7.16.7.tgz", @@ -2619,7 +3666,7 @@ "version": "0.8.1", "resolved": "https://registry.npmjs.org/@cspotcode/source-map-support/-/source-map-support-0.8.1.tgz", "integrity": "sha512-IchNf6dN4tHoMFIn/7OE8LWZ19Y6q/67Bmf6vnGREv8RSbBVb9LPJxEcnwrcwX6ixSvaiGoomAUvu4YSxXrVgw==", - "devOptional": true, + "dev": true, "dependencies": { "@jridgewell/trace-mapping": "0.3.9" }, @@ -2631,7 +3678,7 @@ "version": "0.3.9", "resolved": "https://registry.npmjs.org/@jridgewell/trace-mapping/-/trace-mapping-0.3.9.tgz", "integrity": "sha512-3Belt6tdc8bPgAtbcmdtNJlirVoTmEb5e2gC94PnkwEW9jI6CAHUeoG85tjWP5WquqfavoMtMwiG4P926ZKKuQ==", - "devOptional": true, + "dev": true, "dependencies": { "@jridgewell/resolve-uri": "^3.0.3", "@jridgewell/sourcemap-codec": "^1.4.10" @@ -2957,6 +4004,7 @@ "version": "8.0.2", "resolved": "https://registry.npmjs.org/@isaacs/cliui/-/cliui-8.0.2.tgz", "integrity": "sha512-O8jcjabXaleOG9DQ0+ARXWZBTfnP4WNAqzuiJK7ll44AmxGKv/J2M4TPjxjY3znBCfvBXFzucm1twdyFybFqEA==", + "dev": true, "dependencies": { "string-width": "^5.1.2", "string-width-cjs": "npm:string-width@^4.2.0", @@ -2973,6 +4021,7 @@ "version": "6.0.1", "resolved": "https://registry.npmjs.org/ansi-regex/-/ansi-regex-6.0.1.tgz", "integrity": "sha512-n5M855fKb2SsfMIiFFoVrABHJC8QtHwVx+mHWP3QcEqBHYienj5dHSgjbxtC0WEZXYt4wcD6zrQElDPhFuZgfA==", + "dev": true, "engines": { "node": ">=12" }, @@ -2984,6 +4033,7 @@ "version": "6.2.1", "resolved": "https://registry.npmjs.org/ansi-styles/-/ansi-styles-6.2.1.tgz", "integrity": "sha512-bN798gFfQX+viw3R7yrGWRqnrN2oRkEkUjjl4JNn4E8GxxbjtG3FbrEIIY3l8/hrwUwIeCZvi4QuOTP4MErVug==", + "dev": true, "engines": { "node": ">=12" }, @@ -2994,12 +4044,14 @@ "node_modules/@isaacs/cliui/node_modules/emoji-regex": { "version": "9.2.2", "resolved": "https://registry.npmjs.org/emoji-regex/-/emoji-regex-9.2.2.tgz", - "integrity": "sha512-L18DaJsXSUk2+42pv8mLs5jJT2hqFkFE4j21wOmgbUqsZ2hL72NsUU785g9RXgo3s0ZNgVl42TiHp3ZtOv/Vyg==" + "integrity": "sha512-L18DaJsXSUk2+42pv8mLs5jJT2hqFkFE4j21wOmgbUqsZ2hL72NsUU785g9RXgo3s0ZNgVl42TiHp3ZtOv/Vyg==", + "dev": true }, "node_modules/@isaacs/cliui/node_modules/string-width": { "version": "5.1.2", "resolved": "https://registry.npmjs.org/string-width/-/string-width-5.1.2.tgz", "integrity": "sha512-HnLOCR3vjcY8beoNLtcjZ5/nxn2afmME6lhrDrebokqMap+XbeW8n9TXpPDOqdGK5qcI3oT0GKTW6wC7EMiVqA==", + "dev": true, "dependencies": { "eastasianwidth": "^0.2.0", "emoji-regex": "^9.2.2", @@ -3016,6 +4068,7 @@ "version": "7.1.0", "resolved": "https://registry.npmjs.org/strip-ansi/-/strip-ansi-7.1.0.tgz", "integrity": "sha512-iq6eVVI64nQQTRYq2KtEg2d2uU7LElhTJwsH4YzIHZshxlgZms/wIc4VoDQTlG/IvVIrBKG06CrZnp0qv7hkcQ==", + "dev": true, "dependencies": { "ansi-regex": "^6.0.1" }, @@ -3030,6 +4083,7 @@ "version": "8.1.0", "resolved": "https://registry.npmjs.org/wrap-ansi/-/wrap-ansi-8.1.0.tgz", "integrity": "sha512-si7QWI6zUMq56bESFvagtmzMdGOtoxfR+Sez11Mobfc7tm+VkUckk9bW2UeffTGVUbOksxmSw0AA2gs8g71NCQ==", + "dev": true, "dependencies": { "ansi-styles": "^6.1.0", "string-width": "^5.0.1", @@ -3749,7 +4803,7 @@ "version": "3.1.0", "resolved": "https://registry.npmjs.org/@jridgewell/resolve-uri/-/resolve-uri-3.1.0.tgz", "integrity": "sha512-F2msla3tad+Mfht5cJq7LSXcdudKTWCVYUgw6pLFOOHSTtZlj6SWNYAp+AhuqLmWdBO2X5hPrLcu8cVP8fy28w==", - "devOptional": true, + "dev": true, "engines": { "node": ">=6.0.0" } @@ -3767,7 +4821,7 @@ "version": "1.4.14", "resolved": "https://registry.npmjs.org/@jridgewell/sourcemap-codec/-/sourcemap-codec-1.4.14.tgz", "integrity": "sha512-XPSJHWmi394fuUuzDnGz1wiKqWfo1yXecHQMRf2l6hztTO+nPru658AyDngaBe7isIxEkRsPR3FZh+s7iVa4Uw==", - "devOptional": true + "dev": true }, "node_modules/@jridgewell/trace-mapping": { "version": "0.3.17", @@ -6223,6 +7277,7 @@ "version": "0.11.0", "resolved": "https://registry.npmjs.org/@pkgjs/parseargs/-/parseargs-0.11.0.tgz", "integrity": "sha512-+1VkjdD0QBLPodGrJUeqarH8VAIvQODIbwh9XpP5Syisf7YoQgsJKPNFoqqLQlu+VQ/tVSshMR6loPMn8U+dPg==", + "dev": true, "optional": true, "engines": { "node": ">=14" @@ -6461,34 +7516,914 @@ "@sinonjs/commons": "^2.0.0" } }, - "node_modules/@sqltools/formatter": { - "version": "1.2.5", - "resolved": "https://registry.npmjs.org/@sqltools/formatter/-/formatter-1.2.5.tgz", - "integrity": "sha512-Uy0+khmZqUrUGm5dmMqVlnvufZRSK0FbYzVgp0UMstm+F5+W2/jnEEQyc9vo1ZR/E5ZI/B1WjjoTqBqwJL6Krw==" + "node_modules/@smithy/abort-controller": { + "version": "3.1.1", + "resolved": "https://registry.npmjs.org/@smithy/abort-controller/-/abort-controller-3.1.1.tgz", + "integrity": "sha512-MBJBiidoe+0cTFhyxT8g+9g7CeVccLM0IOKKUMCNQ1CNMJ/eIfoo0RTfVrXOONEI1UCN1W+zkiHSbzUNE9dZtQ==", + "dependencies": { + "@smithy/types": "^3.3.0", + "tslib": "^2.6.2" + }, + "engines": { + "node": ">=16.0.0" + } + }, + "node_modules/@smithy/abort-controller/node_modules/tslib": { + "version": "2.7.0", + "resolved": "https://registry.npmjs.org/tslib/-/tslib-2.7.0.tgz", + "integrity": "sha512-gLXCKdN1/j47AiHiOkJN69hJmcbGTHI0ImLmbYLHykhgeN0jVGola9yVjFgzCUklsZQMW55o+dW7IXv3RCXDzA==" + }, + "node_modules/@smithy/chunked-blob-reader": { + "version": "3.0.0", + "resolved": "https://registry.npmjs.org/@smithy/chunked-blob-reader/-/chunked-blob-reader-3.0.0.tgz", + "integrity": "sha512-sbnURCwjF0gSToGlsBiAmd1lRCmSn72nu9axfJu5lIx6RUEgHu6GwTMbqCdhQSi0Pumcm5vFxsi9XWXb2mTaoA==", + "dependencies": { + "tslib": "^2.6.2" + } + }, + "node_modules/@smithy/chunked-blob-reader-native": { + "version": "3.0.0", + "resolved": "https://registry.npmjs.org/@smithy/chunked-blob-reader-native/-/chunked-blob-reader-native-3.0.0.tgz", + "integrity": "sha512-VDkpCYW+peSuM4zJip5WDfqvg2Mo/e8yxOv3VF1m11y7B8KKMKVFtmZWDe36Fvk8rGuWrPZHHXZ7rR7uM5yWyg==", + "dependencies": { + "@smithy/util-base64": "^3.0.0", + "tslib": "^2.6.2" + } + }, + "node_modules/@smithy/chunked-blob-reader-native/node_modules/tslib": { + "version": "2.7.0", + "resolved": "https://registry.npmjs.org/tslib/-/tslib-2.7.0.tgz", + "integrity": "sha512-gLXCKdN1/j47AiHiOkJN69hJmcbGTHI0ImLmbYLHykhgeN0jVGola9yVjFgzCUklsZQMW55o+dW7IXv3RCXDzA==" + }, + "node_modules/@smithy/chunked-blob-reader/node_modules/tslib": { + "version": "2.7.0", + "resolved": "https://registry.npmjs.org/tslib/-/tslib-2.7.0.tgz", + "integrity": "sha512-gLXCKdN1/j47AiHiOkJN69hJmcbGTHI0ImLmbYLHykhgeN0jVGola9yVjFgzCUklsZQMW55o+dW7IXv3RCXDzA==" + }, + "node_modules/@smithy/config-resolver": { + "version": "3.0.5", + "resolved": "https://registry.npmjs.org/@smithy/config-resolver/-/config-resolver-3.0.5.tgz", + "integrity": "sha512-SkW5LxfkSI1bUC74OtfBbdz+grQXYiPYolyu8VfpLIjEoN/sHVBlLeGXMQ1vX4ejkgfv6sxVbQJ32yF2cl1veA==", + "dependencies": { + "@smithy/node-config-provider": "^3.1.4", + "@smithy/types": "^3.3.0", + "@smithy/util-config-provider": "^3.0.0", + "@smithy/util-middleware": "^3.0.3", + "tslib": "^2.6.2" + }, + "engines": { + "node": ">=16.0.0" + } + }, + "node_modules/@smithy/config-resolver/node_modules/tslib": { + "version": "2.7.0", + "resolved": "https://registry.npmjs.org/tslib/-/tslib-2.7.0.tgz", + "integrity": "sha512-gLXCKdN1/j47AiHiOkJN69hJmcbGTHI0ImLmbYLHykhgeN0jVGola9yVjFgzCUklsZQMW55o+dW7IXv3RCXDzA==" + }, + "node_modules/@smithy/core": { + "version": "2.4.0", + "resolved": "https://registry.npmjs.org/@smithy/core/-/core-2.4.0.tgz", + "integrity": "sha512-cHXq+FneIF/KJbt4q4pjN186+Jf4ZB0ZOqEaZMBhT79srEyGDDBV31NqBRBjazz8ppQ1bJbDJMY9ba5wKFV36w==", + "dependencies": { + "@smithy/middleware-endpoint": "^3.1.0", + "@smithy/middleware-retry": "^3.0.15", + "@smithy/middleware-serde": "^3.0.3", + "@smithy/protocol-http": "^4.1.0", + "@smithy/smithy-client": "^3.2.0", + "@smithy/types": "^3.3.0", + "@smithy/util-body-length-browser": "^3.0.0", + "@smithy/util-middleware": "^3.0.3", + "@smithy/util-utf8": "^3.0.0", + "tslib": "^2.6.2" + }, + "engines": { + "node": ">=16.0.0" + } + }, + "node_modules/@smithy/core/node_modules/tslib": { + "version": "2.7.0", + "resolved": "https://registry.npmjs.org/tslib/-/tslib-2.7.0.tgz", + "integrity": "sha512-gLXCKdN1/j47AiHiOkJN69hJmcbGTHI0ImLmbYLHykhgeN0jVGola9yVjFgzCUklsZQMW55o+dW7IXv3RCXDzA==" + }, + "node_modules/@smithy/credential-provider-imds": { + "version": "3.2.0", + "resolved": "https://registry.npmjs.org/@smithy/credential-provider-imds/-/credential-provider-imds-3.2.0.tgz", + "integrity": "sha512-0SCIzgd8LYZ9EJxUjLXBmEKSZR/P/w6l7Rz/pab9culE/RWuqelAKGJvn5qUOl8BgX8Yj5HWM50A5hiB/RzsgA==", + "dependencies": { + "@smithy/node-config-provider": "^3.1.4", + "@smithy/property-provider": "^3.1.3", + "@smithy/types": "^3.3.0", + "@smithy/url-parser": "^3.0.3", + "tslib": "^2.6.2" + }, + "engines": { + "node": ">=16.0.0" + } + }, + "node_modules/@smithy/credential-provider-imds/node_modules/tslib": { + "version": "2.7.0", + "resolved": "https://registry.npmjs.org/tslib/-/tslib-2.7.0.tgz", + "integrity": "sha512-gLXCKdN1/j47AiHiOkJN69hJmcbGTHI0ImLmbYLHykhgeN0jVGola9yVjFgzCUklsZQMW55o+dW7IXv3RCXDzA==" + }, + "node_modules/@smithy/eventstream-codec": { + "version": "3.1.2", + "resolved": "https://registry.npmjs.org/@smithy/eventstream-codec/-/eventstream-codec-3.1.2.tgz", + "integrity": "sha512-0mBcu49JWt4MXhrhRAlxASNy0IjDRFU+aWNDRal9OtUJvJNiwDuyKMUONSOjLjSCeGwZaE0wOErdqULer8r7yw==", + "dependencies": { + "@aws-crypto/crc32": "5.2.0", + "@smithy/types": "^3.3.0", + "@smithy/util-hex-encoding": "^3.0.0", + "tslib": "^2.6.2" + } + }, + "node_modules/@smithy/eventstream-codec/node_modules/tslib": { + "version": "2.7.0", + "resolved": "https://registry.npmjs.org/tslib/-/tslib-2.7.0.tgz", + "integrity": "sha512-gLXCKdN1/j47AiHiOkJN69hJmcbGTHI0ImLmbYLHykhgeN0jVGola9yVjFgzCUklsZQMW55o+dW7IXv3RCXDzA==" + }, + "node_modules/@smithy/eventstream-serde-browser": { + "version": "3.0.6", + "resolved": "https://registry.npmjs.org/@smithy/eventstream-serde-browser/-/eventstream-serde-browser-3.0.6.tgz", + "integrity": "sha512-2hM54UWQUOrki4BtsUI1WzmD13/SeaqT/AB3EUJKbcver/WgKNaiJ5y5F5XXuVe6UekffVzuUDrBZVAA3AWRpQ==", + "dependencies": { + "@smithy/eventstream-serde-universal": "^3.0.5", + "@smithy/types": "^3.3.0", + "tslib": "^2.6.2" + }, + "engines": { + "node": ">=16.0.0" + } + }, + "node_modules/@smithy/eventstream-serde-browser/node_modules/tslib": { + "version": "2.7.0", + "resolved": "https://registry.npmjs.org/tslib/-/tslib-2.7.0.tgz", + "integrity": "sha512-gLXCKdN1/j47AiHiOkJN69hJmcbGTHI0ImLmbYLHykhgeN0jVGola9yVjFgzCUklsZQMW55o+dW7IXv3RCXDzA==" + }, + "node_modules/@smithy/eventstream-serde-config-resolver": { + "version": "3.0.3", + "resolved": "https://registry.npmjs.org/@smithy/eventstream-serde-config-resolver/-/eventstream-serde-config-resolver-3.0.3.tgz", + "integrity": "sha512-NVTYjOuYpGfrN/VbRQgn31x73KDLfCXCsFdad8DiIc3IcdxL+dYA9zEQPyOP7Fy2QL8CPy2WE4WCUD+ZsLNfaQ==", + "dependencies": { + "@smithy/types": "^3.3.0", + "tslib": "^2.6.2" + }, + "engines": { + "node": ">=16.0.0" + } + }, + "node_modules/@smithy/eventstream-serde-config-resolver/node_modules/tslib": { + "version": "2.7.0", + "resolved": "https://registry.npmjs.org/tslib/-/tslib-2.7.0.tgz", + "integrity": "sha512-gLXCKdN1/j47AiHiOkJN69hJmcbGTHI0ImLmbYLHykhgeN0jVGola9yVjFgzCUklsZQMW55o+dW7IXv3RCXDzA==" + }, + "node_modules/@smithy/eventstream-serde-node": { + "version": "3.0.5", + "resolved": "https://registry.npmjs.org/@smithy/eventstream-serde-node/-/eventstream-serde-node-3.0.5.tgz", + "integrity": "sha512-+upXvnHNyZP095s11jF5dhGw/Ihzqwl5G+/KtMnoQOpdfC3B5HYCcDVG9EmgkhJMXJlM64PyN5gjJl0uXFQehQ==", + "dependencies": { + "@smithy/eventstream-serde-universal": "^3.0.5", + "@smithy/types": "^3.3.0", + "tslib": "^2.6.2" + }, + "engines": { + "node": ">=16.0.0" + } + }, + "node_modules/@smithy/eventstream-serde-node/node_modules/tslib": { + "version": "2.7.0", + "resolved": "https://registry.npmjs.org/tslib/-/tslib-2.7.0.tgz", + "integrity": "sha512-gLXCKdN1/j47AiHiOkJN69hJmcbGTHI0ImLmbYLHykhgeN0jVGola9yVjFgzCUklsZQMW55o+dW7IXv3RCXDzA==" + }, + "node_modules/@smithy/eventstream-serde-universal": { + "version": "3.0.5", + "resolved": "https://registry.npmjs.org/@smithy/eventstream-serde-universal/-/eventstream-serde-universal-3.0.5.tgz", + "integrity": "sha512-5u/nXbyoh1s4QxrvNre9V6vfyoLWuiVvvd5TlZjGThIikc3G+uNiG9uOTCWweSRjv1asdDIWK7nOmN7le4RYHQ==", + "dependencies": { + "@smithy/eventstream-codec": "^3.1.2", + "@smithy/types": "^3.3.0", + "tslib": "^2.6.2" + }, + "engines": { + "node": ">=16.0.0" + } + }, + "node_modules/@smithy/eventstream-serde-universal/node_modules/tslib": { + "version": "2.7.0", + "resolved": "https://registry.npmjs.org/tslib/-/tslib-2.7.0.tgz", + "integrity": "sha512-gLXCKdN1/j47AiHiOkJN69hJmcbGTHI0ImLmbYLHykhgeN0jVGola9yVjFgzCUklsZQMW55o+dW7IXv3RCXDzA==" + }, + "node_modules/@smithy/fetch-http-handler": { + "version": "3.2.4", + "resolved": "https://registry.npmjs.org/@smithy/fetch-http-handler/-/fetch-http-handler-3.2.4.tgz", + "integrity": "sha512-kBprh5Gs5h7ug4nBWZi1FZthdqSM+T7zMmsZxx0IBvWUn7dK3diz2SHn7Bs4dQGFDk8plDv375gzenDoNwrXjg==", + "dependencies": { + "@smithy/protocol-http": "^4.1.0", + "@smithy/querystring-builder": "^3.0.3", + "@smithy/types": "^3.3.0", + "@smithy/util-base64": "^3.0.0", + "tslib": "^2.6.2" + } + }, + "node_modules/@smithy/fetch-http-handler/node_modules/tslib": { + "version": "2.7.0", + "resolved": "https://registry.npmjs.org/tslib/-/tslib-2.7.0.tgz", + "integrity": "sha512-gLXCKdN1/j47AiHiOkJN69hJmcbGTHI0ImLmbYLHykhgeN0jVGola9yVjFgzCUklsZQMW55o+dW7IXv3RCXDzA==" + }, + "node_modules/@smithy/hash-blob-browser": { + "version": "3.1.2", + "resolved": "https://registry.npmjs.org/@smithy/hash-blob-browser/-/hash-blob-browser-3.1.2.tgz", + "integrity": "sha512-hAbfqN2UbISltakCC2TP0kx4LqXBttEv2MqSPE98gVuDFMf05lU+TpC41QtqGP3Ff5A3GwZMPfKnEy0VmEUpmg==", + "dependencies": { + "@smithy/chunked-blob-reader": "^3.0.0", + "@smithy/chunked-blob-reader-native": "^3.0.0", + "@smithy/types": "^3.3.0", + "tslib": "^2.6.2" + } + }, + "node_modules/@smithy/hash-blob-browser/node_modules/tslib": { + "version": "2.7.0", + "resolved": "https://registry.npmjs.org/tslib/-/tslib-2.7.0.tgz", + "integrity": "sha512-gLXCKdN1/j47AiHiOkJN69hJmcbGTHI0ImLmbYLHykhgeN0jVGola9yVjFgzCUklsZQMW55o+dW7IXv3RCXDzA==" + }, + "node_modules/@smithy/hash-node": { + "version": "3.0.3", + "resolved": "https://registry.npmjs.org/@smithy/hash-node/-/hash-node-3.0.3.tgz", + "integrity": "sha512-2ctBXpPMG+B3BtWSGNnKELJ7SH9e4TNefJS0cd2eSkOOROeBnnVBnAy9LtJ8tY4vUEoe55N4CNPxzbWvR39iBw==", + "dependencies": { + "@smithy/types": "^3.3.0", + "@smithy/util-buffer-from": "^3.0.0", + "@smithy/util-utf8": "^3.0.0", + "tslib": "^2.6.2" + }, + "engines": { + "node": ">=16.0.0" + } + }, + "node_modules/@smithy/hash-node/node_modules/tslib": { + "version": "2.7.0", + "resolved": "https://registry.npmjs.org/tslib/-/tslib-2.7.0.tgz", + "integrity": "sha512-gLXCKdN1/j47AiHiOkJN69hJmcbGTHI0ImLmbYLHykhgeN0jVGola9yVjFgzCUklsZQMW55o+dW7IXv3RCXDzA==" + }, + "node_modules/@smithy/hash-stream-node": { + "version": "3.1.2", + "resolved": "https://registry.npmjs.org/@smithy/hash-stream-node/-/hash-stream-node-3.1.2.tgz", + "integrity": "sha512-PBgDMeEdDzi6JxKwbfBtwQG9eT9cVwsf0dZzLXoJF4sHKHs5HEo/3lJWpn6jibfJwT34I1EBXpBnZE8AxAft6g==", + "dependencies": { + "@smithy/types": "^3.3.0", + "@smithy/util-utf8": "^3.0.0", + "tslib": "^2.6.2" + }, + "engines": { + "node": ">=16.0.0" + } + }, + "node_modules/@smithy/hash-stream-node/node_modules/tslib": { + "version": "2.7.0", + "resolved": "https://registry.npmjs.org/tslib/-/tslib-2.7.0.tgz", + "integrity": "sha512-gLXCKdN1/j47AiHiOkJN69hJmcbGTHI0ImLmbYLHykhgeN0jVGola9yVjFgzCUklsZQMW55o+dW7IXv3RCXDzA==" + }, + "node_modules/@smithy/invalid-dependency": { + "version": "3.0.3", + "resolved": "https://registry.npmjs.org/@smithy/invalid-dependency/-/invalid-dependency-3.0.3.tgz", + "integrity": "sha512-ID1eL/zpDULmHJbflb864k72/SNOZCADRc9i7Exq3RUNJw6raWUSlFEQ+3PX3EYs++bTxZB2dE9mEHTQLv61tw==", + "dependencies": { + "@smithy/types": "^3.3.0", + "tslib": "^2.6.2" + } + }, + "node_modules/@smithy/invalid-dependency/node_modules/tslib": { + "version": "2.7.0", + "resolved": "https://registry.npmjs.org/tslib/-/tslib-2.7.0.tgz", + "integrity": "sha512-gLXCKdN1/j47AiHiOkJN69hJmcbGTHI0ImLmbYLHykhgeN0jVGola9yVjFgzCUklsZQMW55o+dW7IXv3RCXDzA==" + }, + "node_modules/@smithy/is-array-buffer": { + "version": "3.0.0", + "resolved": "https://registry.npmjs.org/@smithy/is-array-buffer/-/is-array-buffer-3.0.0.tgz", + "integrity": "sha512-+Fsu6Q6C4RSJiy81Y8eApjEB5gVtM+oFKTffg+jSuwtvomJJrhUJBu2zS8wjXSgH/g1MKEWrzyChTBe6clb5FQ==", + "dependencies": { + "tslib": "^2.6.2" + }, + "engines": { + "node": ">=16.0.0" + } + }, + "node_modules/@smithy/is-array-buffer/node_modules/tslib": { + "version": "2.7.0", + "resolved": "https://registry.npmjs.org/tslib/-/tslib-2.7.0.tgz", + "integrity": "sha512-gLXCKdN1/j47AiHiOkJN69hJmcbGTHI0ImLmbYLHykhgeN0jVGola9yVjFgzCUklsZQMW55o+dW7IXv3RCXDzA==" + }, + "node_modules/@smithy/md5-js": { + "version": "3.0.3", + "resolved": "https://registry.npmjs.org/@smithy/md5-js/-/md5-js-3.0.3.tgz", + "integrity": "sha512-O/SAkGVwpWmelpj/8yDtsaVe6sINHLB1q8YE/+ZQbDxIw3SRLbTZuRaI10K12sVoENdnHqzPp5i3/H+BcZ3m3Q==", + "dependencies": { + "@smithy/types": "^3.3.0", + "@smithy/util-utf8": "^3.0.0", + "tslib": "^2.6.2" + } + }, + "node_modules/@smithy/md5-js/node_modules/tslib": { + "version": "2.7.0", + "resolved": "https://registry.npmjs.org/tslib/-/tslib-2.7.0.tgz", + "integrity": "sha512-gLXCKdN1/j47AiHiOkJN69hJmcbGTHI0ImLmbYLHykhgeN0jVGola9yVjFgzCUklsZQMW55o+dW7IXv3RCXDzA==" + }, + "node_modules/@smithy/middleware-content-length": { + "version": "3.0.5", + "resolved": "https://registry.npmjs.org/@smithy/middleware-content-length/-/middleware-content-length-3.0.5.tgz", + "integrity": "sha512-ILEzC2eyxx6ncej3zZSwMpB5RJ0zuqH7eMptxC4KN3f+v9bqT8ohssKbhNR78k/2tWW+KS5Spw+tbPF4Ejyqvw==", + "dependencies": { + "@smithy/protocol-http": "^4.1.0", + "@smithy/types": "^3.3.0", + "tslib": "^2.6.2" + }, + "engines": { + "node": ">=16.0.0" + } + }, + "node_modules/@smithy/middleware-content-length/node_modules/tslib": { + "version": "2.7.0", + "resolved": "https://registry.npmjs.org/tslib/-/tslib-2.7.0.tgz", + "integrity": "sha512-gLXCKdN1/j47AiHiOkJN69hJmcbGTHI0ImLmbYLHykhgeN0jVGola9yVjFgzCUklsZQMW55o+dW7IXv3RCXDzA==" + }, + "node_modules/@smithy/middleware-endpoint": { + "version": "3.1.0", + "resolved": "https://registry.npmjs.org/@smithy/middleware-endpoint/-/middleware-endpoint-3.1.0.tgz", + "integrity": "sha512-5y5aiKCEwg9TDPB4yFE7H6tYvGFf1OJHNczeY10/EFF8Ir8jZbNntQJxMWNfeQjC1mxPsaQ6mR9cvQbf+0YeMw==", + "dependencies": { + "@smithy/middleware-serde": "^3.0.3", + "@smithy/node-config-provider": "^3.1.4", + "@smithy/shared-ini-file-loader": "^3.1.4", + "@smithy/types": "^3.3.0", + "@smithy/url-parser": "^3.0.3", + "@smithy/util-middleware": "^3.0.3", + "tslib": "^2.6.2" + }, + "engines": { + "node": ">=16.0.0" + } + }, + "node_modules/@smithy/middleware-endpoint/node_modules/tslib": { + "version": "2.7.0", + "resolved": "https://registry.npmjs.org/tslib/-/tslib-2.7.0.tgz", + "integrity": "sha512-gLXCKdN1/j47AiHiOkJN69hJmcbGTHI0ImLmbYLHykhgeN0jVGola9yVjFgzCUklsZQMW55o+dW7IXv3RCXDzA==" + }, + "node_modules/@smithy/middleware-retry": { + "version": "3.0.15", + "resolved": "https://registry.npmjs.org/@smithy/middleware-retry/-/middleware-retry-3.0.15.tgz", + "integrity": "sha512-iTMedvNt1ApdvkaoE8aSDuwaoc+BhvHqttbA/FO4Ty+y/S5hW6Ci/CTScG7vam4RYJWZxdTElc3MEfHRVH6cgQ==", + "dependencies": { + "@smithy/node-config-provider": "^3.1.4", + "@smithy/protocol-http": "^4.1.0", + "@smithy/service-error-classification": "^3.0.3", + "@smithy/smithy-client": "^3.2.0", + "@smithy/types": "^3.3.0", + "@smithy/util-middleware": "^3.0.3", + "@smithy/util-retry": "^3.0.3", + "tslib": "^2.6.2", + "uuid": "^9.0.1" + }, + "engines": { + "node": ">=16.0.0" + } + }, + "node_modules/@smithy/middleware-retry/node_modules/tslib": { + "version": "2.7.0", + "resolved": "https://registry.npmjs.org/tslib/-/tslib-2.7.0.tgz", + "integrity": "sha512-gLXCKdN1/j47AiHiOkJN69hJmcbGTHI0ImLmbYLHykhgeN0jVGola9yVjFgzCUklsZQMW55o+dW7IXv3RCXDzA==" + }, + "node_modules/@smithy/middleware-serde": { + "version": "3.0.3", + "resolved": "https://registry.npmjs.org/@smithy/middleware-serde/-/middleware-serde-3.0.3.tgz", + "integrity": "sha512-puUbyJQBcg9eSErFXjKNiGILJGtiqmuuNKEYNYfUD57fUl4i9+mfmThtQhvFXU0hCVG0iEJhvQUipUf+/SsFdA==", + "dependencies": { + "@smithy/types": "^3.3.0", + "tslib": "^2.6.2" + }, + "engines": { + "node": ">=16.0.0" + } + }, + "node_modules/@smithy/middleware-serde/node_modules/tslib": { + "version": "2.7.0", + "resolved": "https://registry.npmjs.org/tslib/-/tslib-2.7.0.tgz", + "integrity": "sha512-gLXCKdN1/j47AiHiOkJN69hJmcbGTHI0ImLmbYLHykhgeN0jVGola9yVjFgzCUklsZQMW55o+dW7IXv3RCXDzA==" + }, + "node_modules/@smithy/middleware-stack": { + "version": "3.0.3", + "resolved": "https://registry.npmjs.org/@smithy/middleware-stack/-/middleware-stack-3.0.3.tgz", + "integrity": "sha512-r4klY9nFudB0r9UdSMaGSyjyQK5adUyPnQN/ZM6M75phTxOdnc/AhpvGD1fQUvgmqjQEBGCwpnPbDm8pH5PapA==", + "dependencies": { + "@smithy/types": "^3.3.0", + "tslib": "^2.6.2" + }, + "engines": { + "node": ">=16.0.0" + } + }, + "node_modules/@smithy/middleware-stack/node_modules/tslib": { + "version": "2.7.0", + "resolved": "https://registry.npmjs.org/tslib/-/tslib-2.7.0.tgz", + "integrity": "sha512-gLXCKdN1/j47AiHiOkJN69hJmcbGTHI0ImLmbYLHykhgeN0jVGola9yVjFgzCUklsZQMW55o+dW7IXv3RCXDzA==" + }, + "node_modules/@smithy/node-config-provider": { + "version": "3.1.4", + "resolved": "https://registry.npmjs.org/@smithy/node-config-provider/-/node-config-provider-3.1.4.tgz", + "integrity": "sha512-YvnElQy8HR4vDcAjoy7Xkx9YT8xZP4cBXcbJSgm/kxmiQu08DwUwj8rkGnyoJTpfl/3xYHH+d8zE+eHqoDCSdQ==", + "dependencies": { + "@smithy/property-provider": "^3.1.3", + "@smithy/shared-ini-file-loader": "^3.1.4", + "@smithy/types": "^3.3.0", + "tslib": "^2.6.2" + }, + "engines": { + "node": ">=16.0.0" + } + }, + "node_modules/@smithy/node-config-provider/node_modules/tslib": { + "version": "2.7.0", + "resolved": "https://registry.npmjs.org/tslib/-/tslib-2.7.0.tgz", + "integrity": "sha512-gLXCKdN1/j47AiHiOkJN69hJmcbGTHI0ImLmbYLHykhgeN0jVGola9yVjFgzCUklsZQMW55o+dW7IXv3RCXDzA==" + }, + "node_modules/@smithy/node-http-handler": { + "version": "3.1.4", + "resolved": "https://registry.npmjs.org/@smithy/node-http-handler/-/node-http-handler-3.1.4.tgz", + "integrity": "sha512-+UmxgixgOr/yLsUxcEKGH0fMNVteJFGkmRltYFHnBMlogyFdpzn2CwqWmxOrfJELhV34v0WSlaqG1UtE1uXlJg==", + "dependencies": { + "@smithy/abort-controller": "^3.1.1", + "@smithy/protocol-http": "^4.1.0", + "@smithy/querystring-builder": "^3.0.3", + "@smithy/types": "^3.3.0", + "tslib": "^2.6.2" + }, + "engines": { + "node": ">=16.0.0" + } + }, + "node_modules/@smithy/node-http-handler/node_modules/tslib": { + "version": "2.7.0", + "resolved": "https://registry.npmjs.org/tslib/-/tslib-2.7.0.tgz", + "integrity": "sha512-gLXCKdN1/j47AiHiOkJN69hJmcbGTHI0ImLmbYLHykhgeN0jVGola9yVjFgzCUklsZQMW55o+dW7IXv3RCXDzA==" + }, + "node_modules/@smithy/property-provider": { + "version": "3.1.3", + "resolved": "https://registry.npmjs.org/@smithy/property-provider/-/property-provider-3.1.3.tgz", + "integrity": "sha512-zahyOVR9Q4PEoguJ/NrFP4O7SMAfYO1HLhB18M+q+Z4KFd4V2obiMnlVoUFzFLSPeVt1POyNWneHHrZaTMoc/g==", + "dependencies": { + "@smithy/types": "^3.3.0", + "tslib": "^2.6.2" + }, + "engines": { + "node": ">=16.0.0" + } + }, + "node_modules/@smithy/property-provider/node_modules/tslib": { + "version": "2.7.0", + "resolved": "https://registry.npmjs.org/tslib/-/tslib-2.7.0.tgz", + "integrity": "sha512-gLXCKdN1/j47AiHiOkJN69hJmcbGTHI0ImLmbYLHykhgeN0jVGola9yVjFgzCUklsZQMW55o+dW7IXv3RCXDzA==" + }, + "node_modules/@smithy/protocol-http": { + "version": "4.1.0", + "resolved": "https://registry.npmjs.org/@smithy/protocol-http/-/protocol-http-4.1.0.tgz", + "integrity": "sha512-dPVoHYQ2wcHooGXg3LQisa1hH0e4y0pAddPMeeUPipI1tEOqL6A4N0/G7abeq+K8wrwSgjk4C0wnD1XZpJm5aA==", + "dependencies": { + "@smithy/types": "^3.3.0", + "tslib": "^2.6.2" + }, + "engines": { + "node": ">=16.0.0" + } + }, + "node_modules/@smithy/protocol-http/node_modules/tslib": { + "version": "2.7.0", + "resolved": "https://registry.npmjs.org/tslib/-/tslib-2.7.0.tgz", + "integrity": "sha512-gLXCKdN1/j47AiHiOkJN69hJmcbGTHI0ImLmbYLHykhgeN0jVGola9yVjFgzCUklsZQMW55o+dW7IXv3RCXDzA==" + }, + "node_modules/@smithy/querystring-builder": { + "version": "3.0.3", + "resolved": "https://registry.npmjs.org/@smithy/querystring-builder/-/querystring-builder-3.0.3.tgz", + "integrity": "sha512-vyWckeUeesFKzCDaRwWLUA1Xym9McaA6XpFfAK5qI9DKJ4M33ooQGqvM4J+LalH4u/Dq9nFiC8U6Qn1qi0+9zw==", + "dependencies": { + "@smithy/types": "^3.3.0", + "@smithy/util-uri-escape": "^3.0.0", + "tslib": "^2.6.2" + }, + "engines": { + "node": ">=16.0.0" + } + }, + "node_modules/@smithy/querystring-builder/node_modules/tslib": { + "version": "2.7.0", + "resolved": "https://registry.npmjs.org/tslib/-/tslib-2.7.0.tgz", + "integrity": "sha512-gLXCKdN1/j47AiHiOkJN69hJmcbGTHI0ImLmbYLHykhgeN0jVGola9yVjFgzCUklsZQMW55o+dW7IXv3RCXDzA==" + }, + "node_modules/@smithy/querystring-parser": { + "version": "3.0.3", + "resolved": "https://registry.npmjs.org/@smithy/querystring-parser/-/querystring-parser-3.0.3.tgz", + "integrity": "sha512-zahM1lQv2YjmznnfQsWbYojFe55l0SLG/988brlLv1i8z3dubloLF+75ATRsqPBboUXsW6I9CPGE5rQgLfY0vQ==", + "dependencies": { + "@smithy/types": "^3.3.0", + "tslib": "^2.6.2" + }, + "engines": { + "node": ">=16.0.0" + } + }, + "node_modules/@smithy/querystring-parser/node_modules/tslib": { + "version": "2.7.0", + "resolved": "https://registry.npmjs.org/tslib/-/tslib-2.7.0.tgz", + "integrity": "sha512-gLXCKdN1/j47AiHiOkJN69hJmcbGTHI0ImLmbYLHykhgeN0jVGola9yVjFgzCUklsZQMW55o+dW7IXv3RCXDzA==" + }, + "node_modules/@smithy/service-error-classification": { + "version": "3.0.3", + "resolved": "https://registry.npmjs.org/@smithy/service-error-classification/-/service-error-classification-3.0.3.tgz", + "integrity": "sha512-Jn39sSl8cim/VlkLsUhRFq/dKDnRUFlfRkvhOJaUbLBXUsLRLNf9WaxDv/z9BjuQ3A6k/qE8af1lsqcwm7+DaQ==", + "dependencies": { + "@smithy/types": "^3.3.0" + }, + "engines": { + "node": ">=16.0.0" + } + }, + "node_modules/@smithy/shared-ini-file-loader": { + "version": "3.1.4", + "resolved": "https://registry.npmjs.org/@smithy/shared-ini-file-loader/-/shared-ini-file-loader-3.1.4.tgz", + "integrity": "sha512-qMxS4hBGB8FY2GQqshcRUy1K6k8aBWP5vwm8qKkCT3A9K2dawUwOIJfqh9Yste/Bl0J2lzosVyrXDj68kLcHXQ==", + "dependencies": { + "@smithy/types": "^3.3.0", + "tslib": "^2.6.2" + }, + "engines": { + "node": ">=16.0.0" + } + }, + "node_modules/@smithy/shared-ini-file-loader/node_modules/tslib": { + "version": "2.7.0", + "resolved": "https://registry.npmjs.org/tslib/-/tslib-2.7.0.tgz", + "integrity": "sha512-gLXCKdN1/j47AiHiOkJN69hJmcbGTHI0ImLmbYLHykhgeN0jVGola9yVjFgzCUklsZQMW55o+dW7IXv3RCXDzA==" + }, + "node_modules/@smithy/signature-v4": { + "version": "4.1.0", + "resolved": "https://registry.npmjs.org/@smithy/signature-v4/-/signature-v4-4.1.0.tgz", + "integrity": "sha512-aRryp2XNZeRcOtuJoxjydO6QTaVhxx/vjaR+gx7ZjaFgrgPRyZ3HCTbfwqYj6ZWEBHkCSUfcaymKPURaByukag==", + "dependencies": { + "@smithy/is-array-buffer": "^3.0.0", + "@smithy/protocol-http": "^4.1.0", + "@smithy/types": "^3.3.0", + "@smithy/util-hex-encoding": "^3.0.0", + "@smithy/util-middleware": "^3.0.3", + "@smithy/util-uri-escape": "^3.0.0", + "@smithy/util-utf8": "^3.0.0", + "tslib": "^2.6.2" + }, + "engines": { + "node": ">=16.0.0" + } + }, + "node_modules/@smithy/signature-v4/node_modules/tslib": { + "version": "2.7.0", + "resolved": "https://registry.npmjs.org/tslib/-/tslib-2.7.0.tgz", + "integrity": "sha512-gLXCKdN1/j47AiHiOkJN69hJmcbGTHI0ImLmbYLHykhgeN0jVGola9yVjFgzCUklsZQMW55o+dW7IXv3RCXDzA==" + }, + "node_modules/@smithy/smithy-client": { + "version": "3.2.0", + "resolved": "https://registry.npmjs.org/@smithy/smithy-client/-/smithy-client-3.2.0.tgz", + "integrity": "sha512-pDbtxs8WOhJLJSeaF/eAbPgXg4VVYFlRcL/zoNYA5WbG3wBL06CHtBSg53ppkttDpAJ/hdiede+xApip1CwSLw==", + "dependencies": { + "@smithy/middleware-endpoint": "^3.1.0", + "@smithy/middleware-stack": "^3.0.3", + "@smithy/protocol-http": "^4.1.0", + "@smithy/types": "^3.3.0", + "@smithy/util-stream": "^3.1.3", + "tslib": "^2.6.2" + }, + "engines": { + "node": ">=16.0.0" + } + }, + "node_modules/@smithy/smithy-client/node_modules/tslib": { + "version": "2.7.0", + "resolved": "https://registry.npmjs.org/tslib/-/tslib-2.7.0.tgz", + "integrity": "sha512-gLXCKdN1/j47AiHiOkJN69hJmcbGTHI0ImLmbYLHykhgeN0jVGola9yVjFgzCUklsZQMW55o+dW7IXv3RCXDzA==" + }, + "node_modules/@smithy/types": { + "version": "3.3.0", + "resolved": "https://registry.npmjs.org/@smithy/types/-/types-3.3.0.tgz", + "integrity": "sha512-IxvBBCTFDHbVoK7zIxqA1ZOdc4QfM5HM7rGleCuHi7L1wnKv5Pn69xXJQ9hgxH60ZVygH9/JG0jRgtUncE3QUA==", + "dependencies": { + "tslib": "^2.6.2" + }, + "engines": { + "node": ">=16.0.0" + } + }, + "node_modules/@smithy/types/node_modules/tslib": { + "version": "2.7.0", + "resolved": "https://registry.npmjs.org/tslib/-/tslib-2.7.0.tgz", + "integrity": "sha512-gLXCKdN1/j47AiHiOkJN69hJmcbGTHI0ImLmbYLHykhgeN0jVGola9yVjFgzCUklsZQMW55o+dW7IXv3RCXDzA==" + }, + "node_modules/@smithy/url-parser": { + "version": "3.0.3", + "resolved": "https://registry.npmjs.org/@smithy/url-parser/-/url-parser-3.0.3.tgz", + "integrity": "sha512-pw3VtZtX2rg+s6HMs6/+u9+hu6oY6U7IohGhVNnjbgKy86wcIsSZwgHrFR+t67Uyxvp4Xz3p3kGXXIpTNisq8A==", + "dependencies": { + "@smithy/querystring-parser": "^3.0.3", + "@smithy/types": "^3.3.0", + "tslib": "^2.6.2" + } + }, + "node_modules/@smithy/url-parser/node_modules/tslib": { + "version": "2.7.0", + "resolved": "https://registry.npmjs.org/tslib/-/tslib-2.7.0.tgz", + "integrity": "sha512-gLXCKdN1/j47AiHiOkJN69hJmcbGTHI0ImLmbYLHykhgeN0jVGola9yVjFgzCUklsZQMW55o+dW7IXv3RCXDzA==" + }, + "node_modules/@smithy/util-base64": { + "version": "3.0.0", + "resolved": "https://registry.npmjs.org/@smithy/util-base64/-/util-base64-3.0.0.tgz", + "integrity": "sha512-Kxvoh5Qtt0CDsfajiZOCpJxgtPHXOKwmM+Zy4waD43UoEMA+qPxxa98aE/7ZhdnBFZFXMOiBR5xbcaMhLtznQQ==", + "dependencies": { + "@smithy/util-buffer-from": "^3.0.0", + "@smithy/util-utf8": "^3.0.0", + "tslib": "^2.6.2" + }, + "engines": { + "node": ">=16.0.0" + } + }, + "node_modules/@smithy/util-base64/node_modules/tslib": { + "version": "2.7.0", + "resolved": "https://registry.npmjs.org/tslib/-/tslib-2.7.0.tgz", + "integrity": "sha512-gLXCKdN1/j47AiHiOkJN69hJmcbGTHI0ImLmbYLHykhgeN0jVGola9yVjFgzCUklsZQMW55o+dW7IXv3RCXDzA==" + }, + "node_modules/@smithy/util-body-length-browser": { + "version": "3.0.0", + "resolved": "https://registry.npmjs.org/@smithy/util-body-length-browser/-/util-body-length-browser-3.0.0.tgz", + "integrity": "sha512-cbjJs2A1mLYmqmyVl80uoLTJhAcfzMOyPgjwAYusWKMdLeNtzmMz9YxNl3/jRLoxSS3wkqkf0jwNdtXWtyEBaQ==", + "dependencies": { + "tslib": "^2.6.2" + } + }, + "node_modules/@smithy/util-body-length-browser/node_modules/tslib": { + "version": "2.7.0", + "resolved": "https://registry.npmjs.org/tslib/-/tslib-2.7.0.tgz", + "integrity": "sha512-gLXCKdN1/j47AiHiOkJN69hJmcbGTHI0ImLmbYLHykhgeN0jVGola9yVjFgzCUklsZQMW55o+dW7IXv3RCXDzA==" + }, + "node_modules/@smithy/util-body-length-node": { + "version": "3.0.0", + "resolved": "https://registry.npmjs.org/@smithy/util-body-length-node/-/util-body-length-node-3.0.0.tgz", + "integrity": "sha512-Tj7pZ4bUloNUP6PzwhN7K386tmSmEET9QtQg0TgdNOnxhZvCssHji+oZTUIuzxECRfG8rdm2PMw2WCFs6eIYkA==", + "dependencies": { + "tslib": "^2.6.2" + }, + "engines": { + "node": ">=16.0.0" + } + }, + "node_modules/@smithy/util-body-length-node/node_modules/tslib": { + "version": "2.7.0", + "resolved": "https://registry.npmjs.org/tslib/-/tslib-2.7.0.tgz", + "integrity": "sha512-gLXCKdN1/j47AiHiOkJN69hJmcbGTHI0ImLmbYLHykhgeN0jVGola9yVjFgzCUklsZQMW55o+dW7IXv3RCXDzA==" + }, + "node_modules/@smithy/util-buffer-from": { + "version": "3.0.0", + "resolved": "https://registry.npmjs.org/@smithy/util-buffer-from/-/util-buffer-from-3.0.0.tgz", + "integrity": "sha512-aEOHCgq5RWFbP+UDPvPot26EJHjOC+bRgse5A8V3FSShqd5E5UN4qc7zkwsvJPPAVsf73QwYcHN1/gt/rtLwQA==", + "dependencies": { + "@smithy/is-array-buffer": "^3.0.0", + "tslib": "^2.6.2" + }, + "engines": { + "node": ">=16.0.0" + } + }, + "node_modules/@smithy/util-buffer-from/node_modules/tslib": { + "version": "2.7.0", + "resolved": "https://registry.npmjs.org/tslib/-/tslib-2.7.0.tgz", + "integrity": "sha512-gLXCKdN1/j47AiHiOkJN69hJmcbGTHI0ImLmbYLHykhgeN0jVGola9yVjFgzCUklsZQMW55o+dW7IXv3RCXDzA==" + }, + "node_modules/@smithy/util-config-provider": { + "version": "3.0.0", + "resolved": "https://registry.npmjs.org/@smithy/util-config-provider/-/util-config-provider-3.0.0.tgz", + "integrity": "sha512-pbjk4s0fwq3Di/ANL+rCvJMKM5bzAQdE5S/6RL5NXgMExFAi6UgQMPOm5yPaIWPpr+EOXKXRonJ3FoxKf4mCJQ==", + "dependencies": { + "tslib": "^2.6.2" + }, + "engines": { + "node": ">=16.0.0" + } + }, + "node_modules/@smithy/util-config-provider/node_modules/tslib": { + "version": "2.7.0", + "resolved": "https://registry.npmjs.org/tslib/-/tslib-2.7.0.tgz", + "integrity": "sha512-gLXCKdN1/j47AiHiOkJN69hJmcbGTHI0ImLmbYLHykhgeN0jVGola9yVjFgzCUklsZQMW55o+dW7IXv3RCXDzA==" + }, + "node_modules/@smithy/util-defaults-mode-browser": { + "version": "3.0.15", + "resolved": "https://registry.npmjs.org/@smithy/util-defaults-mode-browser/-/util-defaults-mode-browser-3.0.15.tgz", + "integrity": "sha512-FZ4Psa3vjp8kOXcd3HJOiDPBCWtiilLl57r0cnNtq/Ga9RSDrM5ERL6xt+tO43+2af6Pn5Yp92x2n5vPuduNfg==", + "dependencies": { + "@smithy/property-provider": "^3.1.3", + "@smithy/smithy-client": "^3.2.0", + "@smithy/types": "^3.3.0", + "bowser": "^2.11.0", + "tslib": "^2.6.2" + }, + "engines": { + "node": ">= 10.0.0" + } + }, + "node_modules/@smithy/util-defaults-mode-browser/node_modules/tslib": { + "version": "2.7.0", + "resolved": "https://registry.npmjs.org/tslib/-/tslib-2.7.0.tgz", + "integrity": "sha512-gLXCKdN1/j47AiHiOkJN69hJmcbGTHI0ImLmbYLHykhgeN0jVGola9yVjFgzCUklsZQMW55o+dW7IXv3RCXDzA==" + }, + "node_modules/@smithy/util-defaults-mode-node": { + "version": "3.0.15", + "resolved": "https://registry.npmjs.org/@smithy/util-defaults-mode-node/-/util-defaults-mode-node-3.0.15.tgz", + "integrity": "sha512-KSyAAx2q6d0t6f/S4XB2+3+6aQacm3aLMhs9aLMqn18uYGUepbdssfogW5JQZpc6lXNBnp0tEnR5e9CEKmEd7A==", + "dependencies": { + "@smithy/config-resolver": "^3.0.5", + "@smithy/credential-provider-imds": "^3.2.0", + "@smithy/node-config-provider": "^3.1.4", + "@smithy/property-provider": "^3.1.3", + "@smithy/smithy-client": "^3.2.0", + "@smithy/types": "^3.3.0", + "tslib": "^2.6.2" + }, + "engines": { + "node": ">= 10.0.0" + } + }, + "node_modules/@smithy/util-defaults-mode-node/node_modules/tslib": { + "version": "2.7.0", + "resolved": "https://registry.npmjs.org/tslib/-/tslib-2.7.0.tgz", + "integrity": "sha512-gLXCKdN1/j47AiHiOkJN69hJmcbGTHI0ImLmbYLHykhgeN0jVGola9yVjFgzCUklsZQMW55o+dW7IXv3RCXDzA==" + }, + "node_modules/@smithy/util-endpoints": { + "version": "2.0.5", + "resolved": "https://registry.npmjs.org/@smithy/util-endpoints/-/util-endpoints-2.0.5.tgz", + "integrity": "sha512-ReQP0BWihIE68OAblC/WQmDD40Gx+QY1Ez8mTdFMXpmjfxSyz2fVQu3A4zXRfQU9sZXtewk3GmhfOHswvX+eNg==", + "dependencies": { + "@smithy/node-config-provider": "^3.1.4", + "@smithy/types": "^3.3.0", + "tslib": "^2.6.2" + }, + "engines": { + "node": ">=16.0.0" + } + }, + "node_modules/@smithy/util-endpoints/node_modules/tslib": { + "version": "2.7.0", + "resolved": "https://registry.npmjs.org/tslib/-/tslib-2.7.0.tgz", + "integrity": "sha512-gLXCKdN1/j47AiHiOkJN69hJmcbGTHI0ImLmbYLHykhgeN0jVGola9yVjFgzCUklsZQMW55o+dW7IXv3RCXDzA==" + }, + "node_modules/@smithy/util-hex-encoding": { + "version": "3.0.0", + "resolved": "https://registry.npmjs.org/@smithy/util-hex-encoding/-/util-hex-encoding-3.0.0.tgz", + "integrity": "sha512-eFndh1WEK5YMUYvy3lPlVmYY/fZcQE1D8oSf41Id2vCeIkKJXPcYDCZD+4+xViI6b1XSd7tE+s5AmXzz5ilabQ==", + "dependencies": { + "tslib": "^2.6.2" + }, + "engines": { + "node": ">=16.0.0" + } + }, + "node_modules/@smithy/util-hex-encoding/node_modules/tslib": { + "version": "2.7.0", + "resolved": "https://registry.npmjs.org/tslib/-/tslib-2.7.0.tgz", + "integrity": "sha512-gLXCKdN1/j47AiHiOkJN69hJmcbGTHI0ImLmbYLHykhgeN0jVGola9yVjFgzCUklsZQMW55o+dW7IXv3RCXDzA==" + }, + "node_modules/@smithy/util-middleware": { + "version": "3.0.3", + "resolved": "https://registry.npmjs.org/@smithy/util-middleware/-/util-middleware-3.0.3.tgz", + "integrity": "sha512-l+StyYYK/eO3DlVPbU+4Bi06Jjal+PFLSMmlWM1BEwyLxZ3aKkf1ROnoIakfaA7mC6uw3ny7JBkau4Yc+5zfWw==", + "dependencies": { + "@smithy/types": "^3.3.0", + "tslib": "^2.6.2" + }, + "engines": { + "node": ">=16.0.0" + } + }, + "node_modules/@smithy/util-middleware/node_modules/tslib": { + "version": "2.7.0", + "resolved": "https://registry.npmjs.org/tslib/-/tslib-2.7.0.tgz", + "integrity": "sha512-gLXCKdN1/j47AiHiOkJN69hJmcbGTHI0ImLmbYLHykhgeN0jVGola9yVjFgzCUklsZQMW55o+dW7IXv3RCXDzA==" + }, + "node_modules/@smithy/util-retry": { + "version": "3.0.3", + "resolved": "https://registry.npmjs.org/@smithy/util-retry/-/util-retry-3.0.3.tgz", + "integrity": "sha512-AFw+hjpbtVApzpNDhbjNG5NA3kyoMs7vx0gsgmlJF4s+yz1Zlepde7J58zpIRIsdjc+emhpAITxA88qLkPF26w==", + "dependencies": { + "@smithy/service-error-classification": "^3.0.3", + "@smithy/types": "^3.3.0", + "tslib": "^2.6.2" + }, + "engines": { + "node": ">=16.0.0" + } + }, + "node_modules/@smithy/util-retry/node_modules/tslib": { + "version": "2.7.0", + "resolved": "https://registry.npmjs.org/tslib/-/tslib-2.7.0.tgz", + "integrity": "sha512-gLXCKdN1/j47AiHiOkJN69hJmcbGTHI0ImLmbYLHykhgeN0jVGola9yVjFgzCUklsZQMW55o+dW7IXv3RCXDzA==" + }, + "node_modules/@smithy/util-stream": { + "version": "3.1.3", + "resolved": "https://registry.npmjs.org/@smithy/util-stream/-/util-stream-3.1.3.tgz", + "integrity": "sha512-FIv/bRhIlAxC0U7xM1BCnF2aDRPq0UaelqBHkM2lsCp26mcBbgI0tCVTv+jGdsQLUmAMybua/bjDsSu8RQHbmw==", + "dependencies": { + "@smithy/fetch-http-handler": "^3.2.4", + "@smithy/node-http-handler": "^3.1.4", + "@smithy/types": "^3.3.0", + "@smithy/util-base64": "^3.0.0", + "@smithy/util-buffer-from": "^3.0.0", + "@smithy/util-hex-encoding": "^3.0.0", + "@smithy/util-utf8": "^3.0.0", + "tslib": "^2.6.2" + }, + "engines": { + "node": ">=16.0.0" + } + }, + "node_modules/@smithy/util-stream/node_modules/tslib": { + "version": "2.7.0", + "resolved": "https://registry.npmjs.org/tslib/-/tslib-2.7.0.tgz", + "integrity": "sha512-gLXCKdN1/j47AiHiOkJN69hJmcbGTHI0ImLmbYLHykhgeN0jVGola9yVjFgzCUklsZQMW55o+dW7IXv3RCXDzA==" + }, + "node_modules/@smithy/util-uri-escape": { + "version": "3.0.0", + "resolved": "https://registry.npmjs.org/@smithy/util-uri-escape/-/util-uri-escape-3.0.0.tgz", + "integrity": "sha512-LqR7qYLgZTD7nWLBecUi4aqolw8Mhza9ArpNEQ881MJJIU2sE5iHCK6TdyqqzcDLy0OPe10IY4T8ctVdtynubg==", + "dependencies": { + "tslib": "^2.6.2" + }, + "engines": { + "node": ">=16.0.0" + } + }, + "node_modules/@smithy/util-uri-escape/node_modules/tslib": { + "version": "2.7.0", + "resolved": "https://registry.npmjs.org/tslib/-/tslib-2.7.0.tgz", + "integrity": "sha512-gLXCKdN1/j47AiHiOkJN69hJmcbGTHI0ImLmbYLHykhgeN0jVGola9yVjFgzCUklsZQMW55o+dW7IXv3RCXDzA==" + }, + "node_modules/@smithy/util-utf8": { + "version": "3.0.0", + "resolved": "https://registry.npmjs.org/@smithy/util-utf8/-/util-utf8-3.0.0.tgz", + "integrity": "sha512-rUeT12bxFnplYDe815GXbq/oixEGHfRFFtcTF3YdDi/JaENIM6aSYYLJydG83UNzLXeRI5K8abYd/8Sp/QM0kA==", + "dependencies": { + "@smithy/util-buffer-from": "^3.0.0", + "tslib": "^2.6.2" + }, + "engines": { + "node": ">=16.0.0" + } + }, + "node_modules/@smithy/util-utf8/node_modules/tslib": { + "version": "2.7.0", + "resolved": "https://registry.npmjs.org/tslib/-/tslib-2.7.0.tgz", + "integrity": "sha512-gLXCKdN1/j47AiHiOkJN69hJmcbGTHI0ImLmbYLHykhgeN0jVGola9yVjFgzCUklsZQMW55o+dW7IXv3RCXDzA==" + }, + "node_modules/@smithy/util-waiter": { + "version": "3.1.2", + "resolved": "https://registry.npmjs.org/@smithy/util-waiter/-/util-waiter-3.1.2.tgz", + "integrity": "sha512-4pP0EV3iTsexDx+8PPGAKCQpd/6hsQBaQhqWzU4hqKPHN5epPsxKbvUTIiYIHTxaKt6/kEaqPBpu/ufvfbrRzw==", + "dependencies": { + "@smithy/abort-controller": "^3.1.1", + "@smithy/types": "^3.3.0", + "tslib": "^2.6.2" + }, + "engines": { + "node": ">=16.0.0" + } + }, + "node_modules/@smithy/util-waiter/node_modules/tslib": { + "version": "2.7.0", + "resolved": "https://registry.npmjs.org/tslib/-/tslib-2.7.0.tgz", + "integrity": "sha512-gLXCKdN1/j47AiHiOkJN69hJmcbGTHI0ImLmbYLHykhgeN0jVGola9yVjFgzCUklsZQMW55o+dW7IXv3RCXDzA==" }, "node_modules/@tsconfig/node10": { "version": "1.0.9", "resolved": "https://registry.npmjs.org/@tsconfig/node10/-/node10-1.0.9.tgz", "integrity": "sha512-jNsYVVxU8v5g43Erja32laIDHXeoNvFEpX33OK4d6hljo3jDhCBDhx5dhCCTMWUojscpAagGiRkBKxpdl9fxqA==", - "devOptional": true + "dev": true }, "node_modules/@tsconfig/node12": { "version": "1.0.11", "resolved": "https://registry.npmjs.org/@tsconfig/node12/-/node12-1.0.11.tgz", "integrity": "sha512-cqefuRsh12pWyGsIoBKJA9luFu3mRxCA+ORZvA4ktLSzIuCUtWVxGIuXigEwO5/ywWFMZ2QEGKWvkZG1zDMTag==", - "devOptional": true + "dev": true }, "node_modules/@tsconfig/node14": { "version": "1.0.3", "resolved": "https://registry.npmjs.org/@tsconfig/node14/-/node14-1.0.3.tgz", "integrity": "sha512-ysT8mhdixWK6Hw3i1V2AeRqZ5WfXg1G43mqoYlM2nc6388Fq5jcXyr5mRsqViLx/GJYdoL0bfXD8nmF+Zn/Iow==", - "devOptional": true + "dev": true }, "node_modules/@tsconfig/node16": { "version": "1.0.3", "resolved": "https://registry.npmjs.org/@tsconfig/node16/-/node16-1.0.3.tgz", "integrity": "sha512-yOlFc+7UtL/89t2ZhjPvvB/DeAr3r+Dq58IgzsFkOAvVC6NMJXmCGjbptdXdR9qsX7pKcTL+s87FtYREi2dEEQ==", - "devOptional": true + "dev": true }, "node_modules/@types/accepts": { "version": "1.3.5", @@ -7393,7 +9328,7 @@ "version": "8.8.2", "resolved": "https://registry.npmjs.org/acorn/-/acorn-8.8.2.tgz", "integrity": "sha512-xjIYgE8HBrkpd/sJqOGNspf8uHG+NOHGOw6a/Urj8taM2EXfdNAH2oFcPeIFfsv3+kz/mJrS5VuMqbNLjCa2vw==", - "devOptional": true, + "dev": true, "bin": { "acorn": "bin/acorn" }, @@ -7414,7 +9349,7 @@ "version": "8.2.0", "resolved": "https://registry.npmjs.org/acorn-walk/-/acorn-walk-8.2.0.tgz", "integrity": "sha512-k+iyHEuPgSw6SbuDpGQM+06HQUa04DZ3o+F6CSzXMvvI5KMvnaEqXe+YVe555R9nn6GPt404fos4wcgpw12SDA==", - "devOptional": true, + "dev": true, "engines": { "node": ">=0.4.0" } @@ -7523,11 +9458,6 @@ "node": ">=4" } }, - "node_modules/any-promise": { - "version": "1.3.0", - "resolved": "https://registry.npmjs.org/any-promise/-/any-promise-1.3.0.tgz", - "integrity": "sha512-7UvmKalWRt1wgjL1RrGxoSJW/0QZFIegpeGvZG9kjp8vrRu55XTHbwnqq2GpXm9uLbcuhxm3IqX9OB4MZR1b2A==" - }, "node_modules/anymatch": { "version": "3.1.2", "resolved": "https://registry.npmjs.org/anymatch/-/anymatch-3.1.2.tgz", @@ -7541,14 +9471,6 @@ "node": ">= 8" } }, - "node_modules/app-root-path": { - "version": "3.1.0", - "resolved": "https://registry.npmjs.org/app-root-path/-/app-root-path-3.1.0.tgz", - "integrity": "sha512-biN3PwB2gUtjaYy/isrU3aNWI5w+fAfvHkSvCKeQGxhmYpwKFUxudR3Yya+KqVRHBmEDYh+/lTozYCFbmzX4nA==", - "engines": { - "node": ">= 6.0.0" - } - }, "node_modules/append-field": { "version": "1.0.0", "resolved": "https://registry.npmjs.org/append-field/-/append-field-1.0.0.tgz", @@ -7563,7 +9485,7 @@ "version": "4.1.3", "resolved": "https://registry.npmjs.org/arg/-/arg-4.1.3.tgz", "integrity": "sha512-58S9QDqG0Xx27YwPSt9fJxivjYl432YCwfDMfZ+71RAqUrZef7LrKQZ3LHLOwCS4FLNBplP533Zx895SeOCHvA==", - "devOptional": true + "dev": true }, "node_modules/argparse": { "version": "2.0.1", @@ -8132,6 +10054,11 @@ "resolved": "https://registry.npmjs.org/ms/-/ms-2.0.0.tgz", "integrity": "sha512-Tpp60P6IUJDTuOq/5Z8cdskzJujfwqfOTkrwIwj7IRISpnkJnT6SyJ4PCPnGMoFjC9ddhal5KVIYtAt97ix05A==" }, + "node_modules/bowser": { + "version": "2.11.0", + "resolved": "https://registry.npmjs.org/bowser/-/bowser-2.11.0.tgz", + "integrity": "sha512-AlcaJBi/pqqJBIQ8U9Mcpc9i8Aqxn88Skv5d+xBX006BY5u8N3mGLHa5Lgppa7L/HfwgwLgZ6NYs+Ag6uUmJRA==" + }, "node_modules/brace-expansion": { "version": "1.1.11", "resolved": "https://registry.npmjs.org/brace-expansion/-/brace-expansion-1.1.11.tgz", @@ -8435,107 +10362,6 @@ "node": ">=8" } }, - "node_modules/cli-highlight": { - "version": "2.1.11", - "resolved": "https://registry.npmjs.org/cli-highlight/-/cli-highlight-2.1.11.tgz", - "integrity": "sha512-9KDcoEVwyUXrjcJNvHD0NFc/hiwe/WPVYIleQh2O1N2Zro5gWJZ/K+3DGn8w8P/F6FxOgzyC5bxDyHIgCSPhGg==", - "dependencies": { - "chalk": "^4.0.0", - "highlight.js": "^10.7.1", - "mz": "^2.4.0", - "parse5": "^5.1.1", - "parse5-htmlparser2-tree-adapter": "^6.0.0", - "yargs": "^16.0.0" - }, - "bin": { - "highlight": "bin/highlight" - }, - "engines": { - "node": ">=8.0.0", - "npm": ">=5.0.0" - } - }, - "node_modules/cli-highlight/node_modules/ansi-styles": { - "version": "4.3.0", - "resolved": "https://registry.npmjs.org/ansi-styles/-/ansi-styles-4.3.0.tgz", - "integrity": "sha512-zbB9rCJAT1rbjiVDb2hqKFHNYLxgtk8NURxZ3IZwD3F6NtxbXZQCnnSi1Lkx+IDohdPlFp222wVALIheZJQSEg==", - "dependencies": { - "color-convert": "^2.0.1" - }, - "engines": { - "node": ">=8" - }, - "funding": { - "url": "https://github.com/chalk/ansi-styles?sponsor=1" - } - }, - "node_modules/cli-highlight/node_modules/chalk": { - "version": "4.1.2", - "resolved": "https://registry.npmjs.org/chalk/-/chalk-4.1.2.tgz", - "integrity": "sha512-oKnbhFyRIXpUuez8iBMmyEa4nbj4IOQyuhc/wy9kY7/WVPcwIO9VA668Pu8RkO7+0G76SLROeyw9CpQ061i4mA==", - "dependencies": { - "ansi-styles": "^4.1.0", - "supports-color": "^7.1.0" - }, - "engines": { - "node": ">=10" - }, - "funding": { - "url": "https://github.com/chalk/chalk?sponsor=1" - } - }, - "node_modules/cli-highlight/node_modules/color-convert": { - "version": "2.0.1", - "resolved": "https://registry.npmjs.org/color-convert/-/color-convert-2.0.1.tgz", - "integrity": "sha512-RRECPsj7iu/xb5oKYcsFHSppFNnsj/52OVTRKb4zP5onXwVF3zVmmToNcOfGC+CRDpfK/U584fMg38ZHCaElKQ==", - "dependencies": { - "color-name": "~1.1.4" - }, - "engines": { - "node": ">=7.0.0" - } - }, - "node_modules/cli-highlight/node_modules/color-name": { - "version": "1.1.4", - "resolved": "https://registry.npmjs.org/color-name/-/color-name-1.1.4.tgz", - "integrity": "sha512-dOy+3AuW3a2wNbZHIuMZpTcgjGuLU/uBL/ubcZF9OXbDo8ff4O8yVp5Bf0efS8uEoYo5q4Fx7dY9OgQGXgAsQA==" - }, - "node_modules/cli-highlight/node_modules/has-flag": { - "version": "4.0.0", - "resolved": "https://registry.npmjs.org/has-flag/-/has-flag-4.0.0.tgz", - "integrity": "sha512-EykJT/Q1KjTWctppgIAgfSO0tKVuZUjhgMr17kqTumMl6Afv3EISleU7qZUzoXDFTAHTDC4NOoG/ZxU3EvlMPQ==", - "engines": { - "node": ">=8" - } - }, - "node_modules/cli-highlight/node_modules/supports-color": { - "version": "7.2.0", - "resolved": "https://registry.npmjs.org/supports-color/-/supports-color-7.2.0.tgz", - "integrity": "sha512-qpCAvRl9stuOHveKsn7HncJRvv501qIacKzQlO/+Lwxc9+0q2wLyv4Dfvt80/DPn2pqOBsJdDiogXGR9+OvwRw==", - "dependencies": { - "has-flag": "^4.0.0" - }, - "engines": { - "node": ">=8" - } - }, - "node_modules/cli-highlight/node_modules/yargs": { - "version": "16.2.0", - "resolved": "https://registry.npmjs.org/yargs/-/yargs-16.2.0.tgz", - "integrity": "sha512-D1mvvtDG0L5ft/jGWkLpG1+m0eQxOfaBvTNELraWj22wSVUMWxZUvYgJYcKh6jGGIkJFhH4IZPQhR4TKpc8mBw==", - "dependencies": { - "cliui": "^7.0.2", - "escalade": "^3.1.1", - "get-caller-file": "^2.0.5", - "require-directory": "^2.1.1", - "string-width": "^4.2.0", - "y18n": "^5.0.5", - "yargs-parser": "^20.2.2" - }, - "engines": { - "node": ">=10" - } - }, "node_modules/cli-spinners": { "version": "2.7.0", "resolved": "https://registry.npmjs.org/cli-spinners/-/cli-spinners-2.7.0.tgz", @@ -9495,12 +11321,13 @@ "version": "1.1.1", "resolved": "https://registry.npmjs.org/create-require/-/create-require-1.1.1.tgz", "integrity": "sha512-dcKFX3jn0MpIaXjisoRvexIJVEKzaq7z2rZKxf+MSr9TkdmHmsU4m2lcLojrj/FHl8mk5VxMmYA+ftRkP/3oKQ==", - "devOptional": true + "dev": true }, "node_modules/cross-spawn": { "version": "7.0.3", "resolved": "https://registry.npmjs.org/cross-spawn/-/cross-spawn-7.0.3.tgz", "integrity": "sha512-iRDPJKUPVEND7dHPO8rkbOnPpyDygcDFtWjpeWNCgy8WP2rXcxXL8TskReQl6OrB2G7+UJrags1q15Fudc7G6w==", + "dev": true, "dependencies": { "path-key": "^3.1.0", "shebang-command": "^2.0.0", @@ -9553,11 +11380,6 @@ "node": "*" } }, - "node_modules/dayjs": { - "version": "1.11.10", - "resolved": "https://registry.npmjs.org/dayjs/-/dayjs-1.11.10.tgz", - "integrity": "sha512-vjAczensTgRcqDERK0SR2XMwsF/tSvnvlv6VcF2GIhg6Sx4yOIt/irsr1RDJsKiIyBzJDpCoXiWWq28MqH2cnQ==" - }, "node_modules/debug": { "version": "4.3.4", "resolved": "https://registry.npmjs.org/debug/-/debug-4.3.4.tgz", @@ -9729,7 +11551,7 @@ "version": "4.0.2", "resolved": "https://registry.npmjs.org/diff/-/diff-4.0.2.tgz", "integrity": "sha512-58lmxKSA4BNyLz+HHMUzlOEpg09FV+ev6ZMe3vJihgdxzgcwZ8VoEEPmALCZG9LmqfVoNMMKpttIYTVG6uDY7A==", - "devOptional": true, + "dev": true, "engines": { "node": ">=0.3.1" } @@ -9779,17 +11601,6 @@ "node": ">=8" } }, - "node_modules/dotenv": { - "version": "16.4.5", - "resolved": "https://registry.npmjs.org/dotenv/-/dotenv-16.4.5.tgz", - "integrity": "sha512-ZmdL2rui+eB2YwhsWzjInR8LldtZHGDoQ1ugH85ppHKwpUHL7j7rN0Ti9NCnGiQbhaZ11FpR+7ao1dNsmduNUg==", - "engines": { - "node": ">=12" - }, - "funding": { - "url": "https://dotenvx.com" - } - }, "node_modules/dotgitignore": { "version": "2.1.0", "resolved": "https://registry.npmjs.org/dotgitignore/-/dotgitignore-2.1.0.tgz", @@ -9863,7 +11674,8 @@ "node_modules/eastasianwidth": { "version": "0.2.0", "resolved": "https://registry.npmjs.org/eastasianwidth/-/eastasianwidth-0.2.0.tgz", - "integrity": "sha512-I88TYZWc9XiYHRQ4/3c5rjjfgkjhLyW2luGIheGERbNQ6OY7yTybanSpDXZa8y7VUP9YmDcYa+eyq4ca7iLqWA==" + "integrity": "sha512-I88TYZWc9XiYHRQ4/3c5rjjfgkjhLyW2luGIheGERbNQ6OY7yTybanSpDXZa8y7VUP9YmDcYa+eyq4ca7iLqWA==", + "dev": true }, "node_modules/ee-first": { "version": "1.1.1", @@ -11115,9 +12927,9 @@ "integrity": "sha512-W+KJc2dmILlPplD/H4K9l9LcAHAfPtP6BY84uVLXQ6Evcz9Lcg33Y2z1IVblT6xdY54PXYVHEv+0Wpq8Io6zkA==" }, "node_modules/fast-xml-parser": { - "version": "4.4.0", - "resolved": "https://registry.npmjs.org/fast-xml-parser/-/fast-xml-parser-4.4.0.tgz", - "integrity": "sha512-kLY3jFlwIYwBNDojclKsNAC12sfD6NwW74QB2CoNGPvtVxjliYehVunB3HYyNi+n4Tt1dAcgwYvmKF/Z18flqg==", + "version": "4.4.1", + "resolved": "https://registry.npmjs.org/fast-xml-parser/-/fast-xml-parser-4.4.1.tgz", + "integrity": "sha512-xkjOecfnKGkSsOwtZ5Pz7Us/T6mrbPQrq0nh+aCO5V9nk5NLWmasAHumTKjiPJPWANe+kAZ84Jc8ooJkzZ88Sw==", "funding": [ { "type": "github", @@ -11400,6 +13212,7 @@ "version": "3.1.1", "resolved": "https://registry.npmjs.org/foreground-child/-/foreground-child-3.1.1.tgz", "integrity": "sha512-TMKDUnIte6bfb5nWv7V/caI169OHgvwjb7V4WkeUvbQQdjr5rWKqHFiKWb/fcOwB+CzBT+qbWjvj+DVwRskpIg==", + "dev": true, "dependencies": { "cross-spawn": "^7.0.0", "signal-exit": "^4.0.1" @@ -11415,6 +13228,7 @@ "version": "4.0.2", "resolved": "https://registry.npmjs.org/signal-exit/-/signal-exit-4.0.2.tgz", "integrity": "sha512-MY2/qGx4enyjprQnFaZsHib3Yadh3IXyV2C321GY0pjGfVBu4un0uDJkwgdxqO+Rdx8JMT8IfJIRwbYVz3Ob3Q==", + "dev": true, "engines": { "node": ">=14" }, @@ -12068,14 +13882,6 @@ "node": ">=8" } }, - "node_modules/highlight.js": { - "version": "10.7.3", - "resolved": "https://registry.npmjs.org/highlight.js/-/highlight.js-10.7.3.tgz", - "integrity": "sha512-tzcUFauisWKNHaRkN4Wjl/ZA07gENAjFl3J/c480dprkGTg5EQstgaNFqBfUqCq54kZRIEcreTsAgF/m2quD7A==", - "engines": { - "node": "*" - } - }, "node_modules/homedir-polyfill": { "version": "1.0.3", "resolved": "https://registry.npmjs.org/homedir-polyfill/-/homedir-polyfill-1.0.3.tgz", @@ -12747,7 +14553,8 @@ "node_modules/isexe": { "version": "2.0.0", "resolved": "https://registry.npmjs.org/isexe/-/isexe-2.0.0.tgz", - "integrity": "sha512-RHxMLp9lnKHGHRng9QFhRCMbYAcVpn69smSGcq3f36xjgVVWThj4qqLbTLlq7Ssj8B+fIQ1EuCEGI2lKsyQeIw==" + "integrity": "sha512-RHxMLp9lnKHGHRng9QFhRCMbYAcVpn69smSGcq3f36xjgVVWThj4qqLbTLlq7Ssj8B+fIQ1EuCEGI2lKsyQeIw==", + "dev": true }, "node_modules/istanbul-lib-coverage": { "version": "3.2.0", @@ -12849,6 +14656,7 @@ "version": "2.3.6", "resolved": "https://registry.npmjs.org/jackspeak/-/jackspeak-2.3.6.tgz", "integrity": "sha512-N3yCS/NegsOBokc8GAdM8UcmfsKiSS8cipheD/nivzr700H+nsMOxJjQnvwOcRYVuFkdH0wGUvW2WbXGmrZGbQ==", + "dev": true, "dependencies": { "@isaacs/cliui": "^8.0.2" }, @@ -15220,7 +17028,7 @@ "version": "1.3.6", "resolved": "https://registry.npmjs.org/make-error/-/make-error-1.3.6.tgz", "integrity": "sha512-s8UhlNe7vPKomQhC1qFelMokr/Sc3AgNbso3n74mVPA5LTZwkB9NlXf4XPamLxJE8h0gh73rM94xvwRT2CVInw==", - "devOptional": true + "dev": true }, "node_modules/makeerror": { "version": "1.0.12", @@ -15426,6 +17234,7 @@ "version": "6.0.2", "resolved": "https://registry.npmjs.org/minipass/-/minipass-6.0.2.tgz", "integrity": "sha512-MzWSV5nYVT7mVyWCwn2o7JH13w2TBRmmSqSRCKzTw+lmft9X4z+3wjvs06Tzijo5z4W/kahUCDpRXTF+ZrmF/w==", + "dev": true, "engines": { "node": ">=16 || 14 >=14.17" } @@ -15508,16 +17317,6 @@ "integrity": "sha512-nnbWWOkoWyUsTjKrhgD0dcz22mdkSnpYqbEjIm2nhwhuxlSkpywJmBo8h0ZqJdkp73mb90SssHkN4rsRaBAfAA==", "dev": true }, - "node_modules/mz": { - "version": "2.7.0", - "resolved": "https://registry.npmjs.org/mz/-/mz-2.7.0.tgz", - "integrity": "sha512-z81GNO7nnYMEhrGh9LeymoE4+Yr0Wn5McHIZMK5cfQCl+NDX08sCZgUc9/6MHni9IWuFLm1Z3HTCXu2z9fN62Q==", - "dependencies": { - "any-promise": "^1.0.0", - "object-assign": "^4.0.1", - "thenify-all": "^1.0.0" - } - }, "node_modules/natural-compare": { "version": "1.4.0", "resolved": "https://registry.npmjs.org/natural-compare/-/natural-compare-1.4.0.tgz", @@ -16162,24 +17961,6 @@ "node": ">=0.10.0" } }, - "node_modules/parse5": { - "version": "5.1.1", - "resolved": "https://registry.npmjs.org/parse5/-/parse5-5.1.1.tgz", - "integrity": "sha512-ugq4DFI0Ptb+WWjAdOK16+u/nHfiIrcE+sh8kZMaM0WllQKLI9rOUq6c2b7cwPkXdzfQESqvoqK6ug7U/Yyzug==" - }, - "node_modules/parse5-htmlparser2-tree-adapter": { - "version": "6.0.1", - "resolved": "https://registry.npmjs.org/parse5-htmlparser2-tree-adapter/-/parse5-htmlparser2-tree-adapter-6.0.1.tgz", - "integrity": "sha512-qPuWvbLgvDGilKc5BoicRovlT4MtYT6JfJyBOMDsKoiT+GiuP5qyrPCnR9HcPECIJJmZh5jRndyNThnhhb/vlA==", - "dependencies": { - "parse5": "^6.0.1" - } - }, - "node_modules/parse5-htmlparser2-tree-adapter/node_modules/parse5": { - "version": "6.0.1", - "resolved": "https://registry.npmjs.org/parse5/-/parse5-6.0.1.tgz", - "integrity": "sha512-Ofn/CTFzRGTTxwpNEs9PP93gXShHcTq255nzRYSKe8AkVpZY7e1fpmTfOyoIvjP5HG7Z2ZM7VS9PPhQGW2pOpw==" - }, "node_modules/parseurl": { "version": "1.3.3", "resolved": "https://registry.npmjs.org/parseurl/-/parseurl-1.3.3.tgz", @@ -16210,6 +17991,7 @@ "version": "3.1.1", "resolved": "https://registry.npmjs.org/path-key/-/path-key-3.1.1.tgz", "integrity": "sha512-ojmeN0qd+y0jszEtoY48r0Peq5dwMEkIlCOu6Q5f41lfkswXuKtYrhgoTpLnyIcHm24Uhqx+5Tqm2InSwLhE6Q==", + "dev": true, "engines": { "node": ">=8" } @@ -16233,6 +18015,7 @@ "version": "1.10.2", "resolved": "https://registry.npmjs.org/path-scurry/-/path-scurry-1.10.2.tgz", "integrity": "sha512-7xTavNy5RQXnsjANvVvMkEjvloOinkAjv/Z6Ildz9v2RinZ4SBKTWFOVRbaF8p0vpHnyjV/UwNDdKuUv6M5qcA==", + "dev": true, "dependencies": { "lru-cache": "^10.2.0", "minipass": "^5.0.0 || ^6.0.2 || ^7.0.0" @@ -16248,6 +18031,7 @@ "version": "10.2.0", "resolved": "https://registry.npmjs.org/lru-cache/-/lru-cache-10.2.0.tgz", "integrity": "sha512-2bIM8x+VAf6JT4bKAljS1qUWgMsqZRPGJS6FSahIMPVvctcNhyVp7AJu7quxOW9jwkryBReKZY5tY5JYv2n/7Q==", + "dev": true, "engines": { "node": "14 || >=16.14" } @@ -17723,22 +19507,11 @@ "resolved": "https://registry.npmjs.org/setprototypeof/-/setprototypeof-1.2.0.tgz", "integrity": "sha512-E5LDX7Wrp85Kil5bhZv46j8jOeboKq5JMmYM3gVGdGH8xFpPWXUMsNrlODCrkoxMEeNi/XZIwuRvY4XNwYMJpw==" }, - "node_modules/sha.js": { - "version": "2.4.11", - "resolved": "https://registry.npmjs.org/sha.js/-/sha.js-2.4.11.tgz", - "integrity": "sha512-QMEp5B7cftE7APOjk5Y6xgrbWu+WkLVQwk8JNjZ8nKRciZaByEW6MubieAiToS7+dwvrjGhH8jRXz3MVd0AYqQ==", - "dependencies": { - "inherits": "^2.0.1", - "safe-buffer": "^5.0.1" - }, - "bin": { - "sha.js": "bin.js" - } - }, "node_modules/shebang-command": { "version": "2.0.0", "resolved": "https://registry.npmjs.org/shebang-command/-/shebang-command-2.0.0.tgz", "integrity": "sha512-kHxr2zZpYtdmrN1qDjrrX/Z1rR1kG8Dx+gkpK1G4eXmvXswmcE1hTWBWYUzlraYw1/yZp6YuDY77YtvbN0dmDA==", + "dev": true, "dependencies": { "shebang-regex": "^3.0.0" }, @@ -17750,6 +19523,7 @@ "version": "3.0.0", "resolved": "https://registry.npmjs.org/shebang-regex/-/shebang-regex-3.0.0.tgz", "integrity": "sha512-7++dFhtcx3353uBaq8DDR4NuxBetBzC7ZQOhmTQInHEd6bSrXdiEyzCvG07Z44UYdLShWUyXt5M/yhz8ekcb1A==", + "dev": true, "engines": { "node": ">=8" } @@ -18113,6 +19887,7 @@ "version": "4.2.3", "resolved": "https://registry.npmjs.org/string-width/-/string-width-4.2.3.tgz", "integrity": "sha512-wKyQRQpjJ0sIp62ErSZdGsjMJWsap5oRNihHhu6G7JVO/9jIB6UyevL+tXuOqrng8j/cxKTWyWUwvSTriiZz/g==", + "dev": true, "dependencies": { "emoji-regex": "^8.0.0", "is-fullwidth-code-point": "^3.0.0", @@ -18191,6 +19966,7 @@ "version": "6.0.1", "resolved": "https://registry.npmjs.org/strip-ansi/-/strip-ansi-6.0.1.tgz", "integrity": "sha512-Y38VPSHcqkFrCpFnQ9vuSXmquuv5oXOKpGeT6aGrr3o3Gc9AlVa6JBfUSOCnbxGGZF+/0ooI7KrPuUSztUdU5A==", + "dev": true, "dependencies": { "ansi-regex": "^5.0.1" }, @@ -18383,25 +20159,6 @@ "integrity": "sha1-f17oI66AUgfACvLfSoTsP8+lcLQ=", "dev": true }, - "node_modules/thenify": { - "version": "3.3.1", - "resolved": "https://registry.npmjs.org/thenify/-/thenify-3.3.1.tgz", - "integrity": "sha512-RVZSIV5IG10Hk3enotrhvz0T9em6cyHBLkH/YAZuKqd8hRkKhSfCGIcP2KUY0EPxndzANBmNllzWPwak+bheSw==", - "dependencies": { - "any-promise": "^1.0.0" - } - }, - "node_modules/thenify-all": { - "version": "1.6.0", - "resolved": "https://registry.npmjs.org/thenify-all/-/thenify-all-1.6.0.tgz", - "integrity": "sha512-RNxQH/qI8/t3thXJDwcstUO4zeqo64+Uy/+sNVRBx4Xn2OX+OZ9oP+iJnNFqplFra2ZUVeKCSa2oVWi3T4uVmA==", - "dependencies": { - "thenify": ">= 3.1.0 < 4" - }, - "engines": { - "node": ">=0.8" - } - }, "node_modules/thread-stream": { "version": "0.15.2", "resolved": "https://registry.npmjs.org/thread-stream/-/thread-stream-0.15.2.tgz", @@ -18563,7 +20320,7 @@ "version": "10.9.1", "resolved": "https://registry.npmjs.org/ts-node/-/ts-node-10.9.1.tgz", "integrity": "sha512-NtVysVPkxxrwFGUUxGYhfux8k78pQB3JqYBXlLRZgdGUqTO5wU/UyHop5p70iEbGhB7q5KmiZiU0Y3KlJrScEw==", - "devOptional": true, + "dev": true, "dependencies": { "@cspotcode/source-map-support": "^0.8.0", "@tsconfig/node10": "^1.0.7", @@ -18712,316 +20469,11 @@ "integrity": "sha1-nIRAPyMjrlOZFnJ1SXY46h0vJEA=", "dev": true }, - "node_modules/typeorm": { - "version": "0.3.20", - "resolved": "https://registry.npmjs.org/typeorm/-/typeorm-0.3.20.tgz", - "integrity": "sha512-sJ0T08dV5eoZroaq9uPKBoNcGslHBR4E4y+EBHs//SiGbblGe7IeduP/IH4ddCcj0qp3PHwDwGnuvqEAnKlq/Q==", - "dependencies": { - "@sqltools/formatter": "^1.2.5", - "app-root-path": "^3.1.0", - "buffer": "^6.0.3", - "chalk": "^4.1.2", - "cli-highlight": "^2.1.11", - "dayjs": "^1.11.9", - "debug": "^4.3.4", - "dotenv": "^16.0.3", - "glob": "^10.3.10", - "mkdirp": "^2.1.3", - "reflect-metadata": "^0.2.1", - "sha.js": "^2.4.11", - "tslib": "^2.5.0", - "uuid": "^9.0.0", - "yargs": "^17.6.2" - }, - "bin": { - "typeorm": "cli.js", - "typeorm-ts-node-commonjs": "cli-ts-node-commonjs.js", - "typeorm-ts-node-esm": "cli-ts-node-esm.js" - }, - "engines": { - "node": ">=16.13.0" - }, - "funding": { - "url": "https://opencollective.com/typeorm" - }, - "peerDependencies": { - "@google-cloud/spanner": "^5.18.0", - "@sap/hana-client": "^2.12.25", - "better-sqlite3": "^7.1.2 || ^8.0.0 || ^9.0.0", - "hdb-pool": "^0.1.6", - "ioredis": "^5.0.4", - "mongodb": "^5.8.0", - "mssql": "^9.1.1 || ^10.0.1", - "mysql2": "^2.2.5 || ^3.0.1", - "oracledb": "^6.3.0", - "pg": "^8.5.1", - "pg-native": "^3.0.0", - "pg-query-stream": "^4.0.0", - "redis": "^3.1.1 || ^4.0.0", - "sql.js": "^1.4.0", - "sqlite3": "^5.0.3", - "ts-node": "^10.7.0", - "typeorm-aurora-data-api-driver": "^2.0.0" - }, - "peerDependenciesMeta": { - "@google-cloud/spanner": { - "optional": true - }, - "@sap/hana-client": { - "optional": true - }, - "better-sqlite3": { - "optional": true - }, - "hdb-pool": { - "optional": true - }, - "ioredis": { - "optional": true - }, - "mongodb": { - "optional": true - }, - "mssql": { - "optional": true - }, - "mysql2": { - "optional": true - }, - "oracledb": { - "optional": true - }, - "pg": { - "optional": true - }, - "pg-native": { - "optional": true - }, - "pg-query-stream": { - "optional": true - }, - "redis": { - "optional": true - }, - "sql.js": { - "optional": true - }, - "sqlite3": { - "optional": true - }, - "ts-node": { - "optional": true - }, - "typeorm-aurora-data-api-driver": { - "optional": true - } - } - }, - "node_modules/typeorm/node_modules/ansi-styles": { - "version": "4.3.0", - "resolved": "https://registry.npmjs.org/ansi-styles/-/ansi-styles-4.3.0.tgz", - "integrity": "sha512-zbB9rCJAT1rbjiVDb2hqKFHNYLxgtk8NURxZ3IZwD3F6NtxbXZQCnnSi1Lkx+IDohdPlFp222wVALIheZJQSEg==", - "dependencies": { - "color-convert": "^2.0.1" - }, - "engines": { - "node": ">=8" - }, - "funding": { - "url": "https://github.com/chalk/ansi-styles?sponsor=1" - } - }, - "node_modules/typeorm/node_modules/brace-expansion": { - "version": "2.0.1", - "resolved": "https://registry.npmjs.org/brace-expansion/-/brace-expansion-2.0.1.tgz", - "integrity": "sha512-XnAIvQ8eM+kC6aULx6wuQiwVsnzsi9d3WxzV3FpWTGA19F621kwdbsAcFKXgKUHZWsy+mY6iL1sHTxWEFCytDA==", - "dependencies": { - "balanced-match": "^1.0.0" - } - }, - "node_modules/typeorm/node_modules/buffer": { - "version": "6.0.3", - "resolved": "https://registry.npmjs.org/buffer/-/buffer-6.0.3.tgz", - "integrity": "sha512-FTiCpNxtwiZZHEZbcbTIcZjERVICn9yq/pDFkTl95/AxzD1naBctN7YO68riM/gLSDY7sdrMby8hofADYuuqOA==", - "funding": [ - { - "type": "github", - "url": "https://github.com/sponsors/feross" - }, - { - "type": "patreon", - "url": "https://www.patreon.com/feross" - }, - { - "type": "consulting", - "url": "https://feross.org/support" - } - ], - "dependencies": { - "base64-js": "^1.3.1", - "ieee754": "^1.2.1" - } - }, - "node_modules/typeorm/node_modules/chalk": { - "version": "4.1.2", - "resolved": "https://registry.npmjs.org/chalk/-/chalk-4.1.2.tgz", - "integrity": "sha512-oKnbhFyRIXpUuez8iBMmyEa4nbj4IOQyuhc/wy9kY7/WVPcwIO9VA668Pu8RkO7+0G76SLROeyw9CpQ061i4mA==", - "dependencies": { - "ansi-styles": "^4.1.0", - "supports-color": "^7.1.0" - }, - "engines": { - "node": ">=10" - }, - "funding": { - "url": "https://github.com/chalk/chalk?sponsor=1" - } - }, - "node_modules/typeorm/node_modules/cliui": { - "version": "8.0.1", - "resolved": "https://registry.npmjs.org/cliui/-/cliui-8.0.1.tgz", - "integrity": "sha512-BSeNnyus75C4//NQ9gQt1/csTXyo/8Sb+afLAkzAptFuMsod9HFokGNudZpi/oQV73hnVK+sR+5PVRMd+Dr7YQ==", - "dependencies": { - "string-width": "^4.2.0", - "strip-ansi": "^6.0.1", - "wrap-ansi": "^7.0.0" - }, - "engines": { - "node": ">=12" - } - }, - "node_modules/typeorm/node_modules/color-convert": { - "version": "2.0.1", - "resolved": "https://registry.npmjs.org/color-convert/-/color-convert-2.0.1.tgz", - "integrity": "sha512-RRECPsj7iu/xb5oKYcsFHSppFNnsj/52OVTRKb4zP5onXwVF3zVmmToNcOfGC+CRDpfK/U584fMg38ZHCaElKQ==", - "dependencies": { - "color-name": "~1.1.4" - }, - "engines": { - "node": ">=7.0.0" - } - }, - "node_modules/typeorm/node_modules/color-name": { - "version": "1.1.4", - "resolved": "https://registry.npmjs.org/color-name/-/color-name-1.1.4.tgz", - "integrity": "sha512-dOy+3AuW3a2wNbZHIuMZpTcgjGuLU/uBL/ubcZF9OXbDo8ff4O8yVp5Bf0efS8uEoYo5q4Fx7dY9OgQGXgAsQA==" - }, - "node_modules/typeorm/node_modules/glob": { - "version": "10.3.12", - "resolved": "https://registry.npmjs.org/glob/-/glob-10.3.12.tgz", - "integrity": "sha512-TCNv8vJ+xz4QiqTpfOJA7HvYv+tNIRHKfUWw/q+v2jdgN4ebz+KY9tGx5J4rHP0o84mNP+ApH66HRX8us3Khqg==", - "dependencies": { - "foreground-child": "^3.1.0", - "jackspeak": "^2.3.6", - "minimatch": "^9.0.1", - "minipass": "^7.0.4", - "path-scurry": "^1.10.2" - }, - "bin": { - "glob": "dist/esm/bin.mjs" - }, - "engines": { - "node": ">=16 || 14 >=14.17" - }, - "funding": { - "url": "https://github.com/sponsors/isaacs" - } - }, - "node_modules/typeorm/node_modules/has-flag": { - "version": "4.0.0", - "resolved": "https://registry.npmjs.org/has-flag/-/has-flag-4.0.0.tgz", - "integrity": "sha512-EykJT/Q1KjTWctppgIAgfSO0tKVuZUjhgMr17kqTumMl6Afv3EISleU7qZUzoXDFTAHTDC4NOoG/ZxU3EvlMPQ==", - "engines": { - "node": ">=8" - } - }, - "node_modules/typeorm/node_modules/minimatch": { - "version": "9.0.4", - "resolved": "https://registry.npmjs.org/minimatch/-/minimatch-9.0.4.tgz", - "integrity": "sha512-KqWh+VchfxcMNRAJjj2tnsSJdNbHsVgnkBhTNrW7AjVo6OvLtxw8zfT9oLw1JSohlFzJ8jCoTgaoXvJ+kHt6fw==", - "dependencies": { - "brace-expansion": "^2.0.1" - }, - "engines": { - "node": ">=16 || 14 >=14.17" - }, - "funding": { - "url": "https://github.com/sponsors/isaacs" - } - }, - "node_modules/typeorm/node_modules/minipass": { - "version": "7.0.4", - "resolved": "https://registry.npmjs.org/minipass/-/minipass-7.0.4.tgz", - "integrity": "sha512-jYofLM5Dam9279rdkWzqHozUo4ybjdZmCsDHePy5V/PbBcVMiSZR97gmAy45aqi8CK1lG2ECd356FU86avfwUQ==", - "engines": { - "node": ">=16 || 14 >=14.17" - } - }, - "node_modules/typeorm/node_modules/mkdirp": { - "version": "2.1.6", - "resolved": "https://registry.npmjs.org/mkdirp/-/mkdirp-2.1.6.tgz", - "integrity": "sha512-+hEnITedc8LAtIP9u3HJDFIdcLV2vXP33sqLLIzkv1Db1zO/1OxbvYf0Y1OC/S/Qo5dxHXepofhmxL02PsKe+A==", - "bin": { - "mkdirp": "dist/cjs/src/bin.js" - }, - "engines": { - "node": ">=10" - }, - "funding": { - "url": "https://github.com/sponsors/isaacs" - } - }, - "node_modules/typeorm/node_modules/reflect-metadata": { - "version": "0.2.2", - "resolved": "https://registry.npmjs.org/reflect-metadata/-/reflect-metadata-0.2.2.tgz", - "integrity": "sha512-urBwgfrvVP/eAyXx4hluJivBKzuEbSQs9rKWCrCkbSxNv8mxPcUZKeuoF3Uy4mJl3Lwprp6yy5/39VWigZ4K6Q==" - }, - "node_modules/typeorm/node_modules/supports-color": { - "version": "7.2.0", - "resolved": "https://registry.npmjs.org/supports-color/-/supports-color-7.2.0.tgz", - "integrity": "sha512-qpCAvRl9stuOHveKsn7HncJRvv501qIacKzQlO/+Lwxc9+0q2wLyv4Dfvt80/DPn2pqOBsJdDiogXGR9+OvwRw==", - "dependencies": { - "has-flag": "^4.0.0" - }, - "engines": { - "node": ">=8" - } - }, - "node_modules/typeorm/node_modules/tslib": { - "version": "2.6.2", - "resolved": "https://registry.npmjs.org/tslib/-/tslib-2.6.2.tgz", - "integrity": "sha512-AEYxH93jGFPn/a2iVAwW87VuUIkR1FVUKB77NwMF7nBTDkDrrT/Hpt/IrCJ0QXhW27jTBDcf5ZY7w6RiqTMw2Q==" - }, - "node_modules/typeorm/node_modules/yargs": { - "version": "17.7.2", - "resolved": "https://registry.npmjs.org/yargs/-/yargs-17.7.2.tgz", - "integrity": "sha512-7dSzzRQ++CKnNI/krKnYRV7JKKPUXMEh61soaHKg9mrWEhzFWhFnxPxGl+69cD1Ou63C13NUPCnmIcrvqCuM6w==", - "dependencies": { - "cliui": "^8.0.1", - "escalade": "^3.1.1", - "get-caller-file": "^2.0.5", - "require-directory": "^2.1.1", - "string-width": "^4.2.3", - "y18n": "^5.0.5", - "yargs-parser": "^21.1.1" - }, - "engines": { - "node": ">=12" - } - }, - "node_modules/typeorm/node_modules/yargs-parser": { - "version": "21.1.1", - "resolved": "https://registry.npmjs.org/yargs-parser/-/yargs-parser-21.1.1.tgz", - "integrity": "sha512-tVpsJW7DdjecAiFpbIB1e3qxIQsE6NoPc5/eTdrbbIC4h0LVsWhnoa3g+m2HclBIujHzsxZ4VJVA+GUuc2/LBw==", - "engines": { - "node": ">=12" - } - }, "node_modules/typescript": { "version": "5.1.3", "resolved": "https://registry.npmjs.org/typescript/-/typescript-5.1.3.tgz", "integrity": "sha512-XH627E9vkeqhlZFQuL+UsyAXEnibT0kWR2FWONlr4sTjvxyJYnyefgrkyECLzM5NenmKzRAy2rR/OlYLA1HkZw==", - "devOptional": true, + "dev": true, "bin": { "tsc": "bin/tsc", "tsserver": "bin/tsserver" @@ -19164,6 +20616,11 @@ "node": ">= 0.4.0" } }, + "node_modules/utm-latlng": { + "version": "1.0.8", + "resolved": "https://registry.npmjs.org/utm-latlng/-/utm-latlng-1.0.8.tgz", + "integrity": "sha512-jQ5l1BNTzNgas7jXOIh0jU0m5PU5LtmjYvRiLwR4iymYYAhEhQ6N0MGCnLjv5iO958b9tLMMSmzdbIIhLtXqCg==" + }, "node_modules/uuid": { "version": "9.0.1", "resolved": "https://registry.npmjs.org/uuid/-/uuid-9.0.1.tgz", @@ -19180,7 +20637,7 @@ "version": "3.0.1", "resolved": "https://registry.npmjs.org/v8-compile-cache-lib/-/v8-compile-cache-lib-3.0.1.tgz", "integrity": "sha512-wa7YjyUGfNZngI/vtK0UHAN+lgDCxBPCylVXGp0zu59Fz5aiGtNXaq3DhIov063MorB+VfufLh3JlF2KdTK3xg==", - "devOptional": true + "dev": true }, "node_modules/v8-to-istanbul": { "version": "9.1.0", @@ -19308,6 +20765,7 @@ "version": "2.0.2", "resolved": "https://registry.npmjs.org/which/-/which-2.0.2.tgz", "integrity": "sha512-BLI3Tl1TW3Pvl70l3yq3Y64i+awpwXqsGBYWkkqMtnbXgrMD+yj7rhW0kuEDxzJaYXGjEW5ogapKNMEKNMjibA==", + "dev": true, "dependencies": { "isexe": "^2.0.0" }, @@ -19375,6 +20833,7 @@ "version": "7.0.0", "resolved": "https://registry.npmjs.org/wrap-ansi/-/wrap-ansi-7.0.0.tgz", "integrity": "sha512-YVGIj2kamLSTxw6NsZjoBxfSwsn0ycdesmc4p+Q21c5zPuZ1pl+NfxVdxPtdHvmNVOQ6XSYG4AUtyt/Fi7D16Q==", + "dev": true, "dependencies": { "ansi-styles": "^4.0.0", "string-width": "^4.1.0", @@ -19391,6 +20850,7 @@ "version": "4.3.0", "resolved": "https://registry.npmjs.org/ansi-styles/-/ansi-styles-4.3.0.tgz", "integrity": "sha512-zbB9rCJAT1rbjiVDb2hqKFHNYLxgtk8NURxZ3IZwD3F6NtxbXZQCnnSi1Lkx+IDohdPlFp222wVALIheZJQSEg==", + "dev": true, "dependencies": { "color-convert": "^2.0.1" }, @@ -19405,6 +20865,7 @@ "version": "2.0.1", "resolved": "https://registry.npmjs.org/color-convert/-/color-convert-2.0.1.tgz", "integrity": "sha512-RRECPsj7iu/xb5oKYcsFHSppFNnsj/52OVTRKb4zP5onXwVF3zVmmToNcOfGC+CRDpfK/U584fMg38ZHCaElKQ==", + "dev": true, "dependencies": { "color-name": "~1.1.4" }, @@ -19415,7 +20876,8 @@ "node_modules/wrap-ansi-cjs/node_modules/color-name": { "version": "1.1.4", "resolved": "https://registry.npmjs.org/color-name/-/color-name-1.1.4.tgz", - "integrity": "sha512-dOy+3AuW3a2wNbZHIuMZpTcgjGuLU/uBL/ubcZF9OXbDo8ff4O8yVp5Bf0efS8uEoYo5q4Fx7dY9OgQGXgAsQA==" + "integrity": "sha512-dOy+3AuW3a2wNbZHIuMZpTcgjGuLU/uBL/ubcZF9OXbDo8ff4O8yVp5Bf0efS8uEoYo5q4Fx7dY9OgQGXgAsQA==", + "dev": true }, "node_modules/wrap-ansi/node_modules/ansi-styles": { "version": "4.3.0", @@ -19561,7 +21023,7 @@ "version": "3.1.1", "resolved": "https://registry.npmjs.org/yn/-/yn-3.1.1.tgz", "integrity": "sha512-Ux4ygGWsu2c7isFWe8Yu1YluJmqVhxqK2cLXNQA5AcC3QfbGNpM7fu0Y8b/z16pXLnFxZYvWhd3fhBY9DLmC6Q==", - "devOptional": true, + "dev": true, "engines": { "node": ">=6" } @@ -19601,6 +21063,983 @@ "js-yaml": "^4.1.0" } }, + "@aws-crypto/crc32": { + "version": "5.2.0", + "resolved": "https://registry.npmjs.org/@aws-crypto/crc32/-/crc32-5.2.0.tgz", + "integrity": "sha512-nLbCWqQNgUiwwtFsen1AdzAtvuLRsQS8rYgMuxCrdKf9kOssamGLuPwyTY9wyYblNr9+1XM8v6zoDTPPSIeANg==", + "requires": { + "@aws-crypto/util": "^5.2.0", + "@aws-sdk/types": "^3.222.0", + "tslib": "^2.6.2" + }, + "dependencies": { + "tslib": { + "version": "2.7.0", + "resolved": "https://registry.npmjs.org/tslib/-/tslib-2.7.0.tgz", + "integrity": "sha512-gLXCKdN1/j47AiHiOkJN69hJmcbGTHI0ImLmbYLHykhgeN0jVGola9yVjFgzCUklsZQMW55o+dW7IXv3RCXDzA==" + } + } + }, + "@aws-crypto/crc32c": { + "version": "5.2.0", + "resolved": "https://registry.npmjs.org/@aws-crypto/crc32c/-/crc32c-5.2.0.tgz", + "integrity": "sha512-+iWb8qaHLYKrNvGRbiYRHSdKRWhto5XlZUEBwDjYNf+ly5SVYG6zEoYIdxvf5R3zyeP16w4PLBn3rH1xc74Rag==", + "requires": { + "@aws-crypto/util": "^5.2.0", + "@aws-sdk/types": "^3.222.0", + "tslib": "^2.6.2" + }, + "dependencies": { + "tslib": { + "version": "2.7.0", + "resolved": "https://registry.npmjs.org/tslib/-/tslib-2.7.0.tgz", + "integrity": "sha512-gLXCKdN1/j47AiHiOkJN69hJmcbGTHI0ImLmbYLHykhgeN0jVGola9yVjFgzCUklsZQMW55o+dW7IXv3RCXDzA==" + } + } + }, + "@aws-crypto/sha1-browser": { + "version": "5.2.0", + "resolved": "https://registry.npmjs.org/@aws-crypto/sha1-browser/-/sha1-browser-5.2.0.tgz", + "integrity": "sha512-OH6lveCFfcDjX4dbAvCFSYUjJZjDr/3XJ3xHtjn3Oj5b9RjojQo8npoLeA/bNwkOkrSQ0wgrHzXk4tDRxGKJeg==", + "requires": { + "@aws-crypto/supports-web-crypto": "^5.2.0", + "@aws-crypto/util": "^5.2.0", + "@aws-sdk/types": "^3.222.0", + "@aws-sdk/util-locate-window": "^3.0.0", + "@smithy/util-utf8": "^2.0.0", + "tslib": "^2.6.2" + }, + "dependencies": { + "@smithy/is-array-buffer": { + "version": "2.2.0", + "resolved": "https://registry.npmjs.org/@smithy/is-array-buffer/-/is-array-buffer-2.2.0.tgz", + "integrity": "sha512-GGP3O9QFD24uGeAXYUjwSTXARoqpZykHadOmA8G5vfJPK0/DC67qa//0qvqrJzL1xc8WQWX7/yc7fwudjPHPhA==", + "requires": { + "tslib": "^2.6.2" + } + }, + "@smithy/util-buffer-from": { + "version": "2.2.0", + "resolved": "https://registry.npmjs.org/@smithy/util-buffer-from/-/util-buffer-from-2.2.0.tgz", + "integrity": "sha512-IJdWBbTcMQ6DA0gdNhh/BwrLkDR+ADW5Kr1aZmd4k3DIF6ezMV4R2NIAmT08wQJ3yUK82thHWmC/TnK/wpMMIA==", + "requires": { + "@smithy/is-array-buffer": "^2.2.0", + "tslib": "^2.6.2" + } + }, + "@smithy/util-utf8": { + "version": "2.3.0", + "resolved": "https://registry.npmjs.org/@smithy/util-utf8/-/util-utf8-2.3.0.tgz", + "integrity": "sha512-R8Rdn8Hy72KKcebgLiv8jQcQkXoLMOGGv5uI1/k0l+snqkOzQ1R0ChUBCxWMlBsFMekWjq0wRudIweFs7sKT5A==", + "requires": { + "@smithy/util-buffer-from": "^2.2.0", + "tslib": "^2.6.2" + } + }, + "tslib": { + "version": "2.7.0", + "resolved": "https://registry.npmjs.org/tslib/-/tslib-2.7.0.tgz", + "integrity": "sha512-gLXCKdN1/j47AiHiOkJN69hJmcbGTHI0ImLmbYLHykhgeN0jVGola9yVjFgzCUklsZQMW55o+dW7IXv3RCXDzA==" + } + } + }, + "@aws-crypto/sha256-browser": { + "version": "5.2.0", + "resolved": "https://registry.npmjs.org/@aws-crypto/sha256-browser/-/sha256-browser-5.2.0.tgz", + "integrity": "sha512-AXfN/lGotSQwu6HNcEsIASo7kWXZ5HYWvfOmSNKDsEqC4OashTp8alTmaz+F7TC2L083SFv5RdB+qU3Vs1kZqw==", + "requires": { + "@aws-crypto/sha256-js": "^5.2.0", + "@aws-crypto/supports-web-crypto": "^5.2.0", + "@aws-crypto/util": "^5.2.0", + "@aws-sdk/types": "^3.222.0", + "@aws-sdk/util-locate-window": "^3.0.0", + "@smithy/util-utf8": "^2.0.0", + "tslib": "^2.6.2" + }, + "dependencies": { + "@smithy/is-array-buffer": { + "version": "2.2.0", + "resolved": "https://registry.npmjs.org/@smithy/is-array-buffer/-/is-array-buffer-2.2.0.tgz", + "integrity": "sha512-GGP3O9QFD24uGeAXYUjwSTXARoqpZykHadOmA8G5vfJPK0/DC67qa//0qvqrJzL1xc8WQWX7/yc7fwudjPHPhA==", + "requires": { + "tslib": "^2.6.2" + } + }, + "@smithy/util-buffer-from": { + "version": "2.2.0", + "resolved": "https://registry.npmjs.org/@smithy/util-buffer-from/-/util-buffer-from-2.2.0.tgz", + "integrity": "sha512-IJdWBbTcMQ6DA0gdNhh/BwrLkDR+ADW5Kr1aZmd4k3DIF6ezMV4R2NIAmT08wQJ3yUK82thHWmC/TnK/wpMMIA==", + "requires": { + "@smithy/is-array-buffer": "^2.2.0", + "tslib": "^2.6.2" + } + }, + "@smithy/util-utf8": { + "version": "2.3.0", + "resolved": "https://registry.npmjs.org/@smithy/util-utf8/-/util-utf8-2.3.0.tgz", + "integrity": "sha512-R8Rdn8Hy72KKcebgLiv8jQcQkXoLMOGGv5uI1/k0l+snqkOzQ1R0ChUBCxWMlBsFMekWjq0wRudIweFs7sKT5A==", + "requires": { + "@smithy/util-buffer-from": "^2.2.0", + "tslib": "^2.6.2" + } + }, + "tslib": { + "version": "2.7.0", + "resolved": "https://registry.npmjs.org/tslib/-/tslib-2.7.0.tgz", + "integrity": "sha512-gLXCKdN1/j47AiHiOkJN69hJmcbGTHI0ImLmbYLHykhgeN0jVGola9yVjFgzCUklsZQMW55o+dW7IXv3RCXDzA==" + } + } + }, + "@aws-crypto/sha256-js": { + "version": "5.2.0", + "resolved": "https://registry.npmjs.org/@aws-crypto/sha256-js/-/sha256-js-5.2.0.tgz", + "integrity": "sha512-FFQQyu7edu4ufvIZ+OadFpHHOt+eSTBaYaki44c+akjg7qZg9oOQeLlk77F6tSYqjDAFClrHJk9tMf0HdVyOvA==", + "requires": { + "@aws-crypto/util": "^5.2.0", + "@aws-sdk/types": "^3.222.0", + "tslib": "^2.6.2" + }, + "dependencies": { + "tslib": { + "version": "2.7.0", + "resolved": "https://registry.npmjs.org/tslib/-/tslib-2.7.0.tgz", + "integrity": "sha512-gLXCKdN1/j47AiHiOkJN69hJmcbGTHI0ImLmbYLHykhgeN0jVGola9yVjFgzCUklsZQMW55o+dW7IXv3RCXDzA==" + } + } + }, + "@aws-crypto/supports-web-crypto": { + "version": "5.2.0", + "resolved": "https://registry.npmjs.org/@aws-crypto/supports-web-crypto/-/supports-web-crypto-5.2.0.tgz", + "integrity": "sha512-iAvUotm021kM33eCdNfwIN//F77/IADDSs58i+MDaOqFrVjZo9bAal0NK7HurRuWLLpF1iLX7gbWrjHjeo+YFg==", + "requires": { + "tslib": "^2.6.2" + }, + "dependencies": { + "tslib": { + "version": "2.7.0", + "resolved": "https://registry.npmjs.org/tslib/-/tslib-2.7.0.tgz", + "integrity": "sha512-gLXCKdN1/j47AiHiOkJN69hJmcbGTHI0ImLmbYLHykhgeN0jVGola9yVjFgzCUklsZQMW55o+dW7IXv3RCXDzA==" + } + } + }, + "@aws-crypto/util": { + "version": "5.2.0", + "resolved": "https://registry.npmjs.org/@aws-crypto/util/-/util-5.2.0.tgz", + "integrity": "sha512-4RkU9EsI6ZpBve5fseQlGNUWKMa1RLPQ1dnjnQoe07ldfIzcsGb5hC5W0Dm7u423KWzawlrpbjXBrXCEv9zazQ==", + "requires": { + "@aws-sdk/types": "^3.222.0", + "@smithy/util-utf8": "^2.0.0", + "tslib": "^2.6.2" + }, + "dependencies": { + "@smithy/is-array-buffer": { + "version": "2.2.0", + "resolved": "https://registry.npmjs.org/@smithy/is-array-buffer/-/is-array-buffer-2.2.0.tgz", + "integrity": "sha512-GGP3O9QFD24uGeAXYUjwSTXARoqpZykHadOmA8G5vfJPK0/DC67qa//0qvqrJzL1xc8WQWX7/yc7fwudjPHPhA==", + "requires": { + "tslib": "^2.6.2" + } + }, + "@smithy/util-buffer-from": { + "version": "2.2.0", + "resolved": "https://registry.npmjs.org/@smithy/util-buffer-from/-/util-buffer-from-2.2.0.tgz", + "integrity": "sha512-IJdWBbTcMQ6DA0gdNhh/BwrLkDR+ADW5Kr1aZmd4k3DIF6ezMV4R2NIAmT08wQJ3yUK82thHWmC/TnK/wpMMIA==", + "requires": { + "@smithy/is-array-buffer": "^2.2.0", + "tslib": "^2.6.2" + } + }, + "@smithy/util-utf8": { + "version": "2.3.0", + "resolved": "https://registry.npmjs.org/@smithy/util-utf8/-/util-utf8-2.3.0.tgz", + "integrity": "sha512-R8Rdn8Hy72KKcebgLiv8jQcQkXoLMOGGv5uI1/k0l+snqkOzQ1R0ChUBCxWMlBsFMekWjq0wRudIweFs7sKT5A==", + "requires": { + "@smithy/util-buffer-from": "^2.2.0", + "tslib": "^2.6.2" + } + }, + "tslib": { + "version": "2.7.0", + "resolved": "https://registry.npmjs.org/tslib/-/tslib-2.7.0.tgz", + "integrity": "sha512-gLXCKdN1/j47AiHiOkJN69hJmcbGTHI0ImLmbYLHykhgeN0jVGola9yVjFgzCUklsZQMW55o+dW7IXv3RCXDzA==" + } + } + }, + "@aws-sdk/client-s3": { + "version": "3.637.0", + "resolved": "https://registry.npmjs.org/@aws-sdk/client-s3/-/client-s3-3.637.0.tgz", + "integrity": "sha512-y6UC94fsMvhKbf0dzfnjVP1HePeGjplfcYfilZU1COIJLyTkMcUv4XcT4I407CGIrvgEafONHkiC09ygqUauNA==", + "requires": { + "@aws-crypto/sha1-browser": "5.2.0", + "@aws-crypto/sha256-browser": "5.2.0", + "@aws-crypto/sha256-js": "5.2.0", + "@aws-sdk/client-sso-oidc": "3.637.0", + "@aws-sdk/client-sts": "3.637.0", + "@aws-sdk/core": "3.635.0", + "@aws-sdk/credential-provider-node": "3.637.0", + "@aws-sdk/middleware-bucket-endpoint": "3.620.0", + "@aws-sdk/middleware-expect-continue": "3.620.0", + "@aws-sdk/middleware-flexible-checksums": "3.620.0", + "@aws-sdk/middleware-host-header": "3.620.0", + "@aws-sdk/middleware-location-constraint": "3.609.0", + "@aws-sdk/middleware-logger": "3.609.0", + "@aws-sdk/middleware-recursion-detection": "3.620.0", + "@aws-sdk/middleware-sdk-s3": "3.635.0", + "@aws-sdk/middleware-ssec": "3.609.0", + "@aws-sdk/middleware-user-agent": "3.637.0", + "@aws-sdk/region-config-resolver": "3.614.0", + "@aws-sdk/signature-v4-multi-region": "3.635.0", + "@aws-sdk/types": "3.609.0", + "@aws-sdk/util-endpoints": "3.637.0", + "@aws-sdk/util-user-agent-browser": "3.609.0", + "@aws-sdk/util-user-agent-node": "3.614.0", + "@aws-sdk/xml-builder": "3.609.0", + "@smithy/config-resolver": "^3.0.5", + "@smithy/core": "^2.4.0", + "@smithy/eventstream-serde-browser": "^3.0.6", + "@smithy/eventstream-serde-config-resolver": "^3.0.3", + "@smithy/eventstream-serde-node": "^3.0.5", + "@smithy/fetch-http-handler": "^3.2.4", + "@smithy/hash-blob-browser": "^3.1.2", + "@smithy/hash-node": "^3.0.3", + "@smithy/hash-stream-node": "^3.1.2", + "@smithy/invalid-dependency": "^3.0.3", + "@smithy/md5-js": "^3.0.3", + "@smithy/middleware-content-length": "^3.0.5", + "@smithy/middleware-endpoint": "^3.1.0", + "@smithy/middleware-retry": "^3.0.15", + "@smithy/middleware-serde": "^3.0.3", + "@smithy/middleware-stack": "^3.0.3", + "@smithy/node-config-provider": "^3.1.4", + "@smithy/node-http-handler": "^3.1.4", + "@smithy/protocol-http": "^4.1.0", + "@smithy/smithy-client": "^3.2.0", + "@smithy/types": "^3.3.0", + "@smithy/url-parser": "^3.0.3", + "@smithy/util-base64": "^3.0.0", + "@smithy/util-body-length-browser": "^3.0.0", + "@smithy/util-body-length-node": "^3.0.0", + "@smithy/util-defaults-mode-browser": "^3.0.15", + "@smithy/util-defaults-mode-node": "^3.0.15", + "@smithy/util-endpoints": "^2.0.5", + "@smithy/util-middleware": "^3.0.3", + "@smithy/util-retry": "^3.0.3", + "@smithy/util-stream": "^3.1.3", + "@smithy/util-utf8": "^3.0.0", + "@smithy/util-waiter": "^3.1.2", + "tslib": "^2.6.2" + }, + "dependencies": { + "tslib": { + "version": "2.7.0", + "resolved": "https://registry.npmjs.org/tslib/-/tslib-2.7.0.tgz", + "integrity": "sha512-gLXCKdN1/j47AiHiOkJN69hJmcbGTHI0ImLmbYLHykhgeN0jVGola9yVjFgzCUklsZQMW55o+dW7IXv3RCXDzA==" + } + } + }, + "@aws-sdk/client-sso": { + "version": "3.637.0", + "resolved": "https://registry.npmjs.org/@aws-sdk/client-sso/-/client-sso-3.637.0.tgz", + "integrity": "sha512-+KjLvgX5yJYROWo3TQuwBJlHCY0zz9PsLuEolmXQn0BVK1L/m9GteZHtd+rEdAoDGBpE0Xqjy1oz5+SmtsaRUw==", + "requires": { + "@aws-crypto/sha256-browser": "5.2.0", + "@aws-crypto/sha256-js": "5.2.0", + "@aws-sdk/core": "3.635.0", + "@aws-sdk/middleware-host-header": "3.620.0", + "@aws-sdk/middleware-logger": "3.609.0", + "@aws-sdk/middleware-recursion-detection": "3.620.0", + "@aws-sdk/middleware-user-agent": "3.637.0", + "@aws-sdk/region-config-resolver": "3.614.0", + "@aws-sdk/types": "3.609.0", + "@aws-sdk/util-endpoints": "3.637.0", + "@aws-sdk/util-user-agent-browser": "3.609.0", + "@aws-sdk/util-user-agent-node": "3.614.0", + "@smithy/config-resolver": "^3.0.5", + "@smithy/core": "^2.4.0", + "@smithy/fetch-http-handler": "^3.2.4", + "@smithy/hash-node": "^3.0.3", + "@smithy/invalid-dependency": "^3.0.3", + "@smithy/middleware-content-length": "^3.0.5", + "@smithy/middleware-endpoint": "^3.1.0", + "@smithy/middleware-retry": "^3.0.15", + "@smithy/middleware-serde": "^3.0.3", + "@smithy/middleware-stack": "^3.0.3", + "@smithy/node-config-provider": "^3.1.4", + "@smithy/node-http-handler": "^3.1.4", + "@smithy/protocol-http": "^4.1.0", + "@smithy/smithy-client": "^3.2.0", + "@smithy/types": "^3.3.0", + "@smithy/url-parser": "^3.0.3", + "@smithy/util-base64": "^3.0.0", + "@smithy/util-body-length-browser": "^3.0.0", + "@smithy/util-body-length-node": "^3.0.0", + "@smithy/util-defaults-mode-browser": "^3.0.15", + "@smithy/util-defaults-mode-node": "^3.0.15", + "@smithy/util-endpoints": "^2.0.5", + "@smithy/util-middleware": "^3.0.3", + "@smithy/util-retry": "^3.0.3", + "@smithy/util-utf8": "^3.0.0", + "tslib": "^2.6.2" + }, + "dependencies": { + "tslib": { + "version": "2.7.0", + "resolved": "https://registry.npmjs.org/tslib/-/tslib-2.7.0.tgz", + "integrity": "sha512-gLXCKdN1/j47AiHiOkJN69hJmcbGTHI0ImLmbYLHykhgeN0jVGola9yVjFgzCUklsZQMW55o+dW7IXv3RCXDzA==" + } + } + }, + "@aws-sdk/client-sso-oidc": { + "version": "3.637.0", + "resolved": "https://registry.npmjs.org/@aws-sdk/client-sso-oidc/-/client-sso-oidc-3.637.0.tgz", + "integrity": "sha512-27bHALN6Qb6m6KZmPvRieJ/QRlj1lyac/GT2Rn5kJpre8Mpp+yxrtvp3h9PjNBty4lCeFEENfY4dGNSozBuBcw==", + "requires": { + "@aws-crypto/sha256-browser": "5.2.0", + "@aws-crypto/sha256-js": "5.2.0", + "@aws-sdk/core": "3.635.0", + "@aws-sdk/credential-provider-node": "3.637.0", + "@aws-sdk/middleware-host-header": "3.620.0", + "@aws-sdk/middleware-logger": "3.609.0", + "@aws-sdk/middleware-recursion-detection": "3.620.0", + "@aws-sdk/middleware-user-agent": "3.637.0", + "@aws-sdk/region-config-resolver": "3.614.0", + "@aws-sdk/types": "3.609.0", + "@aws-sdk/util-endpoints": "3.637.0", + "@aws-sdk/util-user-agent-browser": "3.609.0", + "@aws-sdk/util-user-agent-node": "3.614.0", + "@smithy/config-resolver": "^3.0.5", + "@smithy/core": "^2.4.0", + "@smithy/fetch-http-handler": "^3.2.4", + "@smithy/hash-node": "^3.0.3", + "@smithy/invalid-dependency": "^3.0.3", + "@smithy/middleware-content-length": "^3.0.5", + "@smithy/middleware-endpoint": "^3.1.0", + "@smithy/middleware-retry": "^3.0.15", + "@smithy/middleware-serde": "^3.0.3", + "@smithy/middleware-stack": "^3.0.3", + "@smithy/node-config-provider": "^3.1.4", + "@smithy/node-http-handler": "^3.1.4", + "@smithy/protocol-http": "^4.1.0", + "@smithy/smithy-client": "^3.2.0", + "@smithy/types": "^3.3.0", + "@smithy/url-parser": "^3.0.3", + "@smithy/util-base64": "^3.0.0", + "@smithy/util-body-length-browser": "^3.0.0", + "@smithy/util-body-length-node": "^3.0.0", + "@smithy/util-defaults-mode-browser": "^3.0.15", + "@smithy/util-defaults-mode-node": "^3.0.15", + "@smithy/util-endpoints": "^2.0.5", + "@smithy/util-middleware": "^3.0.3", + "@smithy/util-retry": "^3.0.3", + "@smithy/util-utf8": "^3.0.0", + "tslib": "^2.6.2" + }, + "dependencies": { + "tslib": { + "version": "2.7.0", + "resolved": "https://registry.npmjs.org/tslib/-/tslib-2.7.0.tgz", + "integrity": "sha512-gLXCKdN1/j47AiHiOkJN69hJmcbGTHI0ImLmbYLHykhgeN0jVGola9yVjFgzCUklsZQMW55o+dW7IXv3RCXDzA==" + } + } + }, + "@aws-sdk/client-sts": { + "version": "3.637.0", + "resolved": "https://registry.npmjs.org/@aws-sdk/client-sts/-/client-sts-3.637.0.tgz", + "integrity": "sha512-xUi7x4qDubtA8QREtlblPuAcn91GS/09YVEY/RwU7xCY0aqGuFwgszAANlha4OUIqva8oVj2WO4gJuG+iaSnhw==", + "requires": { + "@aws-crypto/sha256-browser": "5.2.0", + "@aws-crypto/sha256-js": "5.2.0", + "@aws-sdk/client-sso-oidc": "3.637.0", + "@aws-sdk/core": "3.635.0", + "@aws-sdk/credential-provider-node": "3.637.0", + "@aws-sdk/middleware-host-header": "3.620.0", + "@aws-sdk/middleware-logger": "3.609.0", + "@aws-sdk/middleware-recursion-detection": "3.620.0", + "@aws-sdk/middleware-user-agent": "3.637.0", + "@aws-sdk/region-config-resolver": "3.614.0", + "@aws-sdk/types": "3.609.0", + "@aws-sdk/util-endpoints": "3.637.0", + "@aws-sdk/util-user-agent-browser": "3.609.0", + "@aws-sdk/util-user-agent-node": "3.614.0", + "@smithy/config-resolver": "^3.0.5", + "@smithy/core": "^2.4.0", + "@smithy/fetch-http-handler": "^3.2.4", + "@smithy/hash-node": "^3.0.3", + "@smithy/invalid-dependency": "^3.0.3", + "@smithy/middleware-content-length": "^3.0.5", + "@smithy/middleware-endpoint": "^3.1.0", + "@smithy/middleware-retry": "^3.0.15", + "@smithy/middleware-serde": "^3.0.3", + "@smithy/middleware-stack": "^3.0.3", + "@smithy/node-config-provider": "^3.1.4", + "@smithy/node-http-handler": "^3.1.4", + "@smithy/protocol-http": "^4.1.0", + "@smithy/smithy-client": "^3.2.0", + "@smithy/types": "^3.3.0", + "@smithy/url-parser": "^3.0.3", + "@smithy/util-base64": "^3.0.0", + "@smithy/util-body-length-browser": "^3.0.0", + "@smithy/util-body-length-node": "^3.0.0", + "@smithy/util-defaults-mode-browser": "^3.0.15", + "@smithy/util-defaults-mode-node": "^3.0.15", + "@smithy/util-endpoints": "^2.0.5", + "@smithy/util-middleware": "^3.0.3", + "@smithy/util-retry": "^3.0.3", + "@smithy/util-utf8": "^3.0.0", + "tslib": "^2.6.2" + }, + "dependencies": { + "tslib": { + "version": "2.7.0", + "resolved": "https://registry.npmjs.org/tslib/-/tslib-2.7.0.tgz", + "integrity": "sha512-gLXCKdN1/j47AiHiOkJN69hJmcbGTHI0ImLmbYLHykhgeN0jVGola9yVjFgzCUklsZQMW55o+dW7IXv3RCXDzA==" + } + } + }, + "@aws-sdk/core": { + "version": "3.635.0", + "resolved": "https://registry.npmjs.org/@aws-sdk/core/-/core-3.635.0.tgz", + "integrity": "sha512-i1x/E/sgA+liUE1XJ7rj1dhyXpAKO1UKFUcTTHXok2ARjWTvszHnSXMOsB77aPbmn0fUp1JTx2kHUAZ1LVt5Bg==", + "requires": { + "@smithy/core": "^2.4.0", + "@smithy/node-config-provider": "^3.1.4", + "@smithy/property-provider": "^3.1.3", + "@smithy/protocol-http": "^4.1.0", + "@smithy/signature-v4": "^4.1.0", + "@smithy/smithy-client": "^3.2.0", + "@smithy/types": "^3.3.0", + "@smithy/util-middleware": "^3.0.3", + "fast-xml-parser": "4.4.1", + "tslib": "^2.6.2" + }, + "dependencies": { + "tslib": { + "version": "2.7.0", + "resolved": "https://registry.npmjs.org/tslib/-/tslib-2.7.0.tgz", + "integrity": "sha512-gLXCKdN1/j47AiHiOkJN69hJmcbGTHI0ImLmbYLHykhgeN0jVGola9yVjFgzCUklsZQMW55o+dW7IXv3RCXDzA==" + } + } + }, + "@aws-sdk/credential-provider-env": { + "version": "3.620.1", + "resolved": "https://registry.npmjs.org/@aws-sdk/credential-provider-env/-/credential-provider-env-3.620.1.tgz", + "integrity": "sha512-ExuILJ2qLW5ZO+rgkNRj0xiAipKT16Rk77buvPP8csR7kkCflT/gXTyzRe/uzIiETTxM7tr8xuO9MP/DQXqkfg==", + "requires": { + "@aws-sdk/types": "3.609.0", + "@smithy/property-provider": "^3.1.3", + "@smithy/types": "^3.3.0", + "tslib": "^2.6.2" + }, + "dependencies": { + "tslib": { + "version": "2.7.0", + "resolved": "https://registry.npmjs.org/tslib/-/tslib-2.7.0.tgz", + "integrity": "sha512-gLXCKdN1/j47AiHiOkJN69hJmcbGTHI0ImLmbYLHykhgeN0jVGola9yVjFgzCUklsZQMW55o+dW7IXv3RCXDzA==" + } + } + }, + "@aws-sdk/credential-provider-http": { + "version": "3.635.0", + "resolved": "https://registry.npmjs.org/@aws-sdk/credential-provider-http/-/credential-provider-http-3.635.0.tgz", + "integrity": "sha512-iJyRgEjOCQlBMXqtwPLIKYc7Bsc6nqjrZybdMDenPDa+kmLg7xh8LxHsu9088e+2/wtLicE34FsJJIfzu3L82g==", + "requires": { + "@aws-sdk/types": "3.609.0", + "@smithy/fetch-http-handler": "^3.2.4", + "@smithy/node-http-handler": "^3.1.4", + "@smithy/property-provider": "^3.1.3", + "@smithy/protocol-http": "^4.1.0", + "@smithy/smithy-client": "^3.2.0", + "@smithy/types": "^3.3.0", + "@smithy/util-stream": "^3.1.3", + "tslib": "^2.6.2" + }, + "dependencies": { + "tslib": { + "version": "2.7.0", + "resolved": "https://registry.npmjs.org/tslib/-/tslib-2.7.0.tgz", + "integrity": "sha512-gLXCKdN1/j47AiHiOkJN69hJmcbGTHI0ImLmbYLHykhgeN0jVGola9yVjFgzCUklsZQMW55o+dW7IXv3RCXDzA==" + } + } + }, + "@aws-sdk/credential-provider-ini": { + "version": "3.637.0", + "resolved": "https://registry.npmjs.org/@aws-sdk/credential-provider-ini/-/credential-provider-ini-3.637.0.tgz", + "integrity": "sha512-h+PFCWfZ0Q3Dx84SppET/TFpcQHmxFW8/oV9ArEvMilw4EBN+IlxgbL0CnHwjHW64szcmrM0mbebjEfHf4FXmw==", + "requires": { + "@aws-sdk/credential-provider-env": "3.620.1", + "@aws-sdk/credential-provider-http": "3.635.0", + "@aws-sdk/credential-provider-process": "3.620.1", + "@aws-sdk/credential-provider-sso": "3.637.0", + "@aws-sdk/credential-provider-web-identity": "3.621.0", + "@aws-sdk/types": "3.609.0", + "@smithy/credential-provider-imds": "^3.2.0", + "@smithy/property-provider": "^3.1.3", + "@smithy/shared-ini-file-loader": "^3.1.4", + "@smithy/types": "^3.3.0", + "tslib": "^2.6.2" + }, + "dependencies": { + "tslib": { + "version": "2.7.0", + "resolved": "https://registry.npmjs.org/tslib/-/tslib-2.7.0.tgz", + "integrity": "sha512-gLXCKdN1/j47AiHiOkJN69hJmcbGTHI0ImLmbYLHykhgeN0jVGola9yVjFgzCUklsZQMW55o+dW7IXv3RCXDzA==" + } + } + }, + "@aws-sdk/credential-provider-node": { + "version": "3.637.0", + "resolved": "https://registry.npmjs.org/@aws-sdk/credential-provider-node/-/credential-provider-node-3.637.0.tgz", + "integrity": "sha512-yoEhoxJJfs7sPVQ6Is939BDQJZpZCoUgKr/ySse4YKOZ24t4VqgHA6+wV7rYh+7IW24Rd91UTvEzSuHYTlxlNA==", + "requires": { + "@aws-sdk/credential-provider-env": "3.620.1", + "@aws-sdk/credential-provider-http": "3.635.0", + "@aws-sdk/credential-provider-ini": "3.637.0", + "@aws-sdk/credential-provider-process": "3.620.1", + "@aws-sdk/credential-provider-sso": "3.637.0", + "@aws-sdk/credential-provider-web-identity": "3.621.0", + "@aws-sdk/types": "3.609.0", + "@smithy/credential-provider-imds": "^3.2.0", + "@smithy/property-provider": "^3.1.3", + "@smithy/shared-ini-file-loader": "^3.1.4", + "@smithy/types": "^3.3.0", + "tslib": "^2.6.2" + }, + "dependencies": { + "tslib": { + "version": "2.7.0", + "resolved": "https://registry.npmjs.org/tslib/-/tslib-2.7.0.tgz", + "integrity": "sha512-gLXCKdN1/j47AiHiOkJN69hJmcbGTHI0ImLmbYLHykhgeN0jVGola9yVjFgzCUklsZQMW55o+dW7IXv3RCXDzA==" + } + } + }, + "@aws-sdk/credential-provider-process": { + "version": "3.620.1", + "resolved": "https://registry.npmjs.org/@aws-sdk/credential-provider-process/-/credential-provider-process-3.620.1.tgz", + "integrity": "sha512-hWqFMidqLAkaV9G460+1at6qa9vySbjQKKc04p59OT7lZ5cO5VH5S4aI05e+m4j364MBROjjk2ugNvfNf/8ILg==", + "requires": { + "@aws-sdk/types": "3.609.0", + "@smithy/property-provider": "^3.1.3", + "@smithy/shared-ini-file-loader": "^3.1.4", + "@smithy/types": "^3.3.0", + "tslib": "^2.6.2" + }, + "dependencies": { + "tslib": { + "version": "2.7.0", + "resolved": "https://registry.npmjs.org/tslib/-/tslib-2.7.0.tgz", + "integrity": "sha512-gLXCKdN1/j47AiHiOkJN69hJmcbGTHI0ImLmbYLHykhgeN0jVGola9yVjFgzCUklsZQMW55o+dW7IXv3RCXDzA==" + } + } + }, + "@aws-sdk/credential-provider-sso": { + "version": "3.637.0", + "resolved": "https://registry.npmjs.org/@aws-sdk/credential-provider-sso/-/credential-provider-sso-3.637.0.tgz", + "integrity": "sha512-Mvz+h+e62/tl+dVikLafhv+qkZJ9RUb8l2YN/LeKMWkxQylPT83CPk9aimVhCV89zth1zpREArl97+3xsfgQvA==", + "requires": { + "@aws-sdk/client-sso": "3.637.0", + "@aws-sdk/token-providers": "3.614.0", + "@aws-sdk/types": "3.609.0", + "@smithy/property-provider": "^3.1.3", + "@smithy/shared-ini-file-loader": "^3.1.4", + "@smithy/types": "^3.3.0", + "tslib": "^2.6.2" + }, + "dependencies": { + "tslib": { + "version": "2.7.0", + "resolved": "https://registry.npmjs.org/tslib/-/tslib-2.7.0.tgz", + "integrity": "sha512-gLXCKdN1/j47AiHiOkJN69hJmcbGTHI0ImLmbYLHykhgeN0jVGola9yVjFgzCUklsZQMW55o+dW7IXv3RCXDzA==" + } + } + }, + "@aws-sdk/credential-provider-web-identity": { + "version": "3.621.0", + "resolved": "https://registry.npmjs.org/@aws-sdk/credential-provider-web-identity/-/credential-provider-web-identity-3.621.0.tgz", + "integrity": "sha512-w7ASSyfNvcx7+bYGep3VBgC3K6vEdLmlpjT7nSIHxxQf+WSdvy+HynwJosrpZax0sK5q0D1Jpn/5q+r5lwwW6w==", + "requires": { + "@aws-sdk/types": "3.609.0", + "@smithy/property-provider": "^3.1.3", + "@smithy/types": "^3.3.0", + "tslib": "^2.6.2" + }, + "dependencies": { + "tslib": { + "version": "2.7.0", + "resolved": "https://registry.npmjs.org/tslib/-/tslib-2.7.0.tgz", + "integrity": "sha512-gLXCKdN1/j47AiHiOkJN69hJmcbGTHI0ImLmbYLHykhgeN0jVGola9yVjFgzCUklsZQMW55o+dW7IXv3RCXDzA==" + } + } + }, + "@aws-sdk/middleware-bucket-endpoint": { + "version": "3.620.0", + "resolved": "https://registry.npmjs.org/@aws-sdk/middleware-bucket-endpoint/-/middleware-bucket-endpoint-3.620.0.tgz", + "integrity": "sha512-eGLL0W6L3HDb3OACyetZYOWpHJ+gLo0TehQKeQyy2G8vTYXqNTeqYhuI6up9HVjBzU9eQiULVQETmgQs7TFaRg==", + "requires": { + "@aws-sdk/types": "3.609.0", + "@aws-sdk/util-arn-parser": "3.568.0", + "@smithy/node-config-provider": "^3.1.4", + "@smithy/protocol-http": "^4.1.0", + "@smithy/types": "^3.3.0", + "@smithy/util-config-provider": "^3.0.0", + "tslib": "^2.6.2" + }, + "dependencies": { + "tslib": { + "version": "2.7.0", + "resolved": "https://registry.npmjs.org/tslib/-/tslib-2.7.0.tgz", + "integrity": "sha512-gLXCKdN1/j47AiHiOkJN69hJmcbGTHI0ImLmbYLHykhgeN0jVGola9yVjFgzCUklsZQMW55o+dW7IXv3RCXDzA==" + } + } + }, + "@aws-sdk/middleware-expect-continue": { + "version": "3.620.0", + "resolved": "https://registry.npmjs.org/@aws-sdk/middleware-expect-continue/-/middleware-expect-continue-3.620.0.tgz", + "integrity": "sha512-QXeRFMLfyQ31nAHLbiTLtk0oHzG9QLMaof5jIfqcUwnOkO8YnQdeqzakrg1Alpy/VQ7aqzIi8qypkBe2KXZz0A==", + "requires": { + "@aws-sdk/types": "3.609.0", + "@smithy/protocol-http": "^4.1.0", + "@smithy/types": "^3.3.0", + "tslib": "^2.6.2" + }, + "dependencies": { + "tslib": { + "version": "2.7.0", + "resolved": "https://registry.npmjs.org/tslib/-/tslib-2.7.0.tgz", + "integrity": "sha512-gLXCKdN1/j47AiHiOkJN69hJmcbGTHI0ImLmbYLHykhgeN0jVGola9yVjFgzCUklsZQMW55o+dW7IXv3RCXDzA==" + } + } + }, + "@aws-sdk/middleware-flexible-checksums": { + "version": "3.620.0", + "resolved": "https://registry.npmjs.org/@aws-sdk/middleware-flexible-checksums/-/middleware-flexible-checksums-3.620.0.tgz", + "integrity": "sha512-ftz+NW7qka2sVuwnnO1IzBku5ccP+s5qZGeRTPgrKB7OzRW85gthvIo1vQR2w+OwHFk7WJbbhhWwbCbktnP4UA==", + "requires": { + "@aws-crypto/crc32": "5.2.0", + "@aws-crypto/crc32c": "5.2.0", + "@aws-sdk/types": "3.609.0", + "@smithy/is-array-buffer": "^3.0.0", + "@smithy/protocol-http": "^4.1.0", + "@smithy/types": "^3.3.0", + "@smithy/util-utf8": "^3.0.0", + "tslib": "^2.6.2" + }, + "dependencies": { + "tslib": { + "version": "2.7.0", + "resolved": "https://registry.npmjs.org/tslib/-/tslib-2.7.0.tgz", + "integrity": "sha512-gLXCKdN1/j47AiHiOkJN69hJmcbGTHI0ImLmbYLHykhgeN0jVGola9yVjFgzCUklsZQMW55o+dW7IXv3RCXDzA==" + } + } + }, + "@aws-sdk/middleware-host-header": { + "version": "3.620.0", + "resolved": "https://registry.npmjs.org/@aws-sdk/middleware-host-header/-/middleware-host-header-3.620.0.tgz", + "integrity": "sha512-VMtPEZwqYrII/oUkffYsNWY9PZ9xpNJpMgmyU0rlDQ25O1c0Hk3fJmZRe6pEkAJ0omD7kLrqGl1DUjQVxpd/Rg==", + "requires": { + "@aws-sdk/types": "3.609.0", + "@smithy/protocol-http": "^4.1.0", + "@smithy/types": "^3.3.0", + "tslib": "^2.6.2" + }, + "dependencies": { + "tslib": { + "version": "2.7.0", + "resolved": "https://registry.npmjs.org/tslib/-/tslib-2.7.0.tgz", + "integrity": "sha512-gLXCKdN1/j47AiHiOkJN69hJmcbGTHI0ImLmbYLHykhgeN0jVGola9yVjFgzCUklsZQMW55o+dW7IXv3RCXDzA==" + } + } + }, + "@aws-sdk/middleware-location-constraint": { + "version": "3.609.0", + "resolved": "https://registry.npmjs.org/@aws-sdk/middleware-location-constraint/-/middleware-location-constraint-3.609.0.tgz", + "integrity": "sha512-xzsdoTkszGVqGVPjUmgoP7TORiByLueMHieI1fhQL888WPdqctwAx3ES6d/bA9Q/i8jnc6hs+Fjhy8UvBTkE9A==", + "requires": { + "@aws-sdk/types": "3.609.0", + "@smithy/types": "^3.3.0", + "tslib": "^2.6.2" + }, + "dependencies": { + "tslib": { + "version": "2.7.0", + "resolved": "https://registry.npmjs.org/tslib/-/tslib-2.7.0.tgz", + "integrity": "sha512-gLXCKdN1/j47AiHiOkJN69hJmcbGTHI0ImLmbYLHykhgeN0jVGola9yVjFgzCUklsZQMW55o+dW7IXv3RCXDzA==" + } + } + }, + "@aws-sdk/middleware-logger": { + "version": "3.609.0", + "resolved": "https://registry.npmjs.org/@aws-sdk/middleware-logger/-/middleware-logger-3.609.0.tgz", + "integrity": "sha512-S62U2dy4jMDhDFDK5gZ4VxFdWzCtLzwbYyFZx2uvPYTECkepLUfzLic2BHg2Qvtu4QjX+oGE3P/7fwaGIsGNuQ==", + "requires": { + "@aws-sdk/types": "3.609.0", + "@smithy/types": "^3.3.0", + "tslib": "^2.6.2" + }, + "dependencies": { + "tslib": { + "version": "2.7.0", + "resolved": "https://registry.npmjs.org/tslib/-/tslib-2.7.0.tgz", + "integrity": "sha512-gLXCKdN1/j47AiHiOkJN69hJmcbGTHI0ImLmbYLHykhgeN0jVGola9yVjFgzCUklsZQMW55o+dW7IXv3RCXDzA==" + } + } + }, + "@aws-sdk/middleware-recursion-detection": { + "version": "3.620.0", + "resolved": "https://registry.npmjs.org/@aws-sdk/middleware-recursion-detection/-/middleware-recursion-detection-3.620.0.tgz", + "integrity": "sha512-nh91S7aGK3e/o1ck64sA/CyoFw+gAYj2BDOnoNa6ouyCrVJED96ZXWbhye/fz9SgmNUZR2g7GdVpiLpMKZoI5w==", + "requires": { + "@aws-sdk/types": "3.609.0", + "@smithy/protocol-http": "^4.1.0", + "@smithy/types": "^3.3.0", + "tslib": "^2.6.2" + }, + "dependencies": { + "tslib": { + "version": "2.7.0", + "resolved": "https://registry.npmjs.org/tslib/-/tslib-2.7.0.tgz", + "integrity": "sha512-gLXCKdN1/j47AiHiOkJN69hJmcbGTHI0ImLmbYLHykhgeN0jVGola9yVjFgzCUklsZQMW55o+dW7IXv3RCXDzA==" + } + } + }, + "@aws-sdk/middleware-sdk-s3": { + "version": "3.635.0", + "resolved": "https://registry.npmjs.org/@aws-sdk/middleware-sdk-s3/-/middleware-sdk-s3-3.635.0.tgz", + "integrity": "sha512-RLdYJPEV4JL/7NBoFUs7VlP90X++5FlJdxHz0DzCjmiD3qCviKy+Cym3qg1gBgHwucs5XisuClxDrGokhAdTQw==", + "requires": { + "@aws-sdk/core": "3.635.0", + "@aws-sdk/types": "3.609.0", + "@aws-sdk/util-arn-parser": "3.568.0", + "@smithy/core": "^2.4.0", + "@smithy/node-config-provider": "^3.1.4", + "@smithy/protocol-http": "^4.1.0", + "@smithy/signature-v4": "^4.1.0", + "@smithy/smithy-client": "^3.2.0", + "@smithy/types": "^3.3.0", + "@smithy/util-config-provider": "^3.0.0", + "@smithy/util-middleware": "^3.0.3", + "@smithy/util-stream": "^3.1.3", + "@smithy/util-utf8": "^3.0.0", + "tslib": "^2.6.2" + }, + "dependencies": { + "tslib": { + "version": "2.7.0", + "resolved": "https://registry.npmjs.org/tslib/-/tslib-2.7.0.tgz", + "integrity": "sha512-gLXCKdN1/j47AiHiOkJN69hJmcbGTHI0ImLmbYLHykhgeN0jVGola9yVjFgzCUklsZQMW55o+dW7IXv3RCXDzA==" + } + } + }, + "@aws-sdk/middleware-ssec": { + "version": "3.609.0", + "resolved": "https://registry.npmjs.org/@aws-sdk/middleware-ssec/-/middleware-ssec-3.609.0.tgz", + "integrity": "sha512-GZSD1s7+JswWOTamVap79QiDaIV7byJFssBW68GYjyRS5EBjNfwA/8s+6uE6g39R3ojyTbYOmvcANoZEhSULXg==", + "requires": { + "@aws-sdk/types": "3.609.0", + "@smithy/types": "^3.3.0", + "tslib": "^2.6.2" + }, + "dependencies": { + "tslib": { + "version": "2.7.0", + "resolved": "https://registry.npmjs.org/tslib/-/tslib-2.7.0.tgz", + "integrity": "sha512-gLXCKdN1/j47AiHiOkJN69hJmcbGTHI0ImLmbYLHykhgeN0jVGola9yVjFgzCUklsZQMW55o+dW7IXv3RCXDzA==" + } + } + }, + "@aws-sdk/middleware-user-agent": { + "version": "3.637.0", + "resolved": "https://registry.npmjs.org/@aws-sdk/middleware-user-agent/-/middleware-user-agent-3.637.0.tgz", + "integrity": "sha512-EYo0NE9/da/OY8STDsK2LvM4kNa79DBsf4YVtaG4P5pZ615IeFsD8xOHZeuJmUrSMlVQ8ywPRX7WMucUybsKug==", + "requires": { + "@aws-sdk/types": "3.609.0", + "@aws-sdk/util-endpoints": "3.637.0", + "@smithy/protocol-http": "^4.1.0", + "@smithy/types": "^3.3.0", + "tslib": "^2.6.2" + }, + "dependencies": { + "tslib": { + "version": "2.7.0", + "resolved": "https://registry.npmjs.org/tslib/-/tslib-2.7.0.tgz", + "integrity": "sha512-gLXCKdN1/j47AiHiOkJN69hJmcbGTHI0ImLmbYLHykhgeN0jVGola9yVjFgzCUklsZQMW55o+dW7IXv3RCXDzA==" + } + } + }, + "@aws-sdk/region-config-resolver": { + "version": "3.614.0", + "resolved": "https://registry.npmjs.org/@aws-sdk/region-config-resolver/-/region-config-resolver-3.614.0.tgz", + "integrity": "sha512-vDCeMXvic/LU0KFIUjpC3RiSTIkkvESsEfbVHiHH0YINfl8HnEqR5rj+L8+phsCeVg2+LmYwYxd5NRz4PHxt5g==", + "requires": { + "@aws-sdk/types": "3.609.0", + "@smithy/node-config-provider": "^3.1.4", + "@smithy/types": "^3.3.0", + "@smithy/util-config-provider": "^3.0.0", + "@smithy/util-middleware": "^3.0.3", + "tslib": "^2.6.2" + }, + "dependencies": { + "tslib": { + "version": "2.7.0", + "resolved": "https://registry.npmjs.org/tslib/-/tslib-2.7.0.tgz", + "integrity": "sha512-gLXCKdN1/j47AiHiOkJN69hJmcbGTHI0ImLmbYLHykhgeN0jVGola9yVjFgzCUklsZQMW55o+dW7IXv3RCXDzA==" + } + } + }, + "@aws-sdk/signature-v4-multi-region": { + "version": "3.635.0", + "resolved": "https://registry.npmjs.org/@aws-sdk/signature-v4-multi-region/-/signature-v4-multi-region-3.635.0.tgz", + "integrity": "sha512-J6QY4/invOkpogCHjSaDON1hF03viPpOnsrzVuCvJMmclS/iG62R4EY0wq1alYll0YmSdmKlpJwHMWwGtqK63Q==", + "requires": { + "@aws-sdk/middleware-sdk-s3": "3.635.0", + "@aws-sdk/types": "3.609.0", + "@smithy/protocol-http": "^4.1.0", + "@smithy/signature-v4": "^4.1.0", + "@smithy/types": "^3.3.0", + "tslib": "^2.6.2" + }, + "dependencies": { + "tslib": { + "version": "2.7.0", + "resolved": "https://registry.npmjs.org/tslib/-/tslib-2.7.0.tgz", + "integrity": "sha512-gLXCKdN1/j47AiHiOkJN69hJmcbGTHI0ImLmbYLHykhgeN0jVGola9yVjFgzCUklsZQMW55o+dW7IXv3RCXDzA==" + } + } + }, + "@aws-sdk/token-providers": { + "version": "3.614.0", + "resolved": "https://registry.npmjs.org/@aws-sdk/token-providers/-/token-providers-3.614.0.tgz", + "integrity": "sha512-okItqyY6L9IHdxqs+Z116y5/nda7rHxLvROxtAJdLavWTYDydxrZstImNgGWTeVdmc0xX2gJCI77UYUTQWnhRw==", + "requires": { + "@aws-sdk/types": "3.609.0", + "@smithy/property-provider": "^3.1.3", + "@smithy/shared-ini-file-loader": "^3.1.4", + "@smithy/types": "^3.3.0", + "tslib": "^2.6.2" + }, + "dependencies": { + "tslib": { + "version": "2.7.0", + "resolved": "https://registry.npmjs.org/tslib/-/tslib-2.7.0.tgz", + "integrity": "sha512-gLXCKdN1/j47AiHiOkJN69hJmcbGTHI0ImLmbYLHykhgeN0jVGola9yVjFgzCUklsZQMW55o+dW7IXv3RCXDzA==" + } + } + }, + "@aws-sdk/types": { + "version": "3.609.0", + "resolved": "https://registry.npmjs.org/@aws-sdk/types/-/types-3.609.0.tgz", + "integrity": "sha512-+Tqnh9w0h2LcrUsdXyT1F8mNhXz+tVYBtP19LpeEGntmvHwa2XzvLUCWpoIAIVsHp5+HdB2X9Sn0KAtmbFXc2Q==", + "requires": { + "@smithy/types": "^3.3.0", + "tslib": "^2.6.2" + }, + "dependencies": { + "tslib": { + "version": "2.7.0", + "resolved": "https://registry.npmjs.org/tslib/-/tslib-2.7.0.tgz", + "integrity": "sha512-gLXCKdN1/j47AiHiOkJN69hJmcbGTHI0ImLmbYLHykhgeN0jVGola9yVjFgzCUklsZQMW55o+dW7IXv3RCXDzA==" + } + } + }, + "@aws-sdk/util-arn-parser": { + "version": "3.568.0", + "resolved": "https://registry.npmjs.org/@aws-sdk/util-arn-parser/-/util-arn-parser-3.568.0.tgz", + "integrity": "sha512-XUKJWWo+KOB7fbnPP0+g/o5Ulku/X53t7i/h+sPHr5xxYTJJ9CYnbToo95mzxe7xWvkLrsNtJ8L+MnNn9INs2w==", + "requires": { + "tslib": "^2.6.2" + }, + "dependencies": { + "tslib": { + "version": "2.7.0", + "resolved": "https://registry.npmjs.org/tslib/-/tslib-2.7.0.tgz", + "integrity": "sha512-gLXCKdN1/j47AiHiOkJN69hJmcbGTHI0ImLmbYLHykhgeN0jVGola9yVjFgzCUklsZQMW55o+dW7IXv3RCXDzA==" + } + } + }, + "@aws-sdk/util-endpoints": { + "version": "3.637.0", + "resolved": "https://registry.npmjs.org/@aws-sdk/util-endpoints/-/util-endpoints-3.637.0.tgz", + "integrity": "sha512-pAqOKUHeVWHEXXDIp/qoMk/6jyxIb6GGjnK1/f8dKHtKIEs4tKsnnL563gceEvdad53OPXIt86uoevCcCzmBnw==", + "requires": { + "@aws-sdk/types": "3.609.0", + "@smithy/types": "^3.3.0", + "@smithy/util-endpoints": "^2.0.5", + "tslib": "^2.6.2" + }, + "dependencies": { + "tslib": { + "version": "2.7.0", + "resolved": "https://registry.npmjs.org/tslib/-/tslib-2.7.0.tgz", + "integrity": "sha512-gLXCKdN1/j47AiHiOkJN69hJmcbGTHI0ImLmbYLHykhgeN0jVGola9yVjFgzCUklsZQMW55o+dW7IXv3RCXDzA==" + } + } + }, + "@aws-sdk/util-locate-window": { + "version": "3.568.0", + "resolved": "https://registry.npmjs.org/@aws-sdk/util-locate-window/-/util-locate-window-3.568.0.tgz", + "integrity": "sha512-3nh4TINkXYr+H41QaPelCceEB2FXP3fxp93YZXB/kqJvX0U9j0N0Uk45gvsjmEPzG8XxkPEeLIfT2I1M7A6Lig==", + "requires": { + "tslib": "^2.6.2" + }, + "dependencies": { + "tslib": { + "version": "2.7.0", + "resolved": "https://registry.npmjs.org/tslib/-/tslib-2.7.0.tgz", + "integrity": "sha512-gLXCKdN1/j47AiHiOkJN69hJmcbGTHI0ImLmbYLHykhgeN0jVGola9yVjFgzCUklsZQMW55o+dW7IXv3RCXDzA==" + } + } + }, + "@aws-sdk/util-user-agent-browser": { + "version": "3.609.0", + "resolved": "https://registry.npmjs.org/@aws-sdk/util-user-agent-browser/-/util-user-agent-browser-3.609.0.tgz", + "integrity": "sha512-fojPU+mNahzQ0YHYBsx0ZIhmMA96H+ZIZ665ObU9tl+SGdbLneVZVikGve+NmHTQwHzwkFsZYYnVKAkreJLAtA==", + "requires": { + "@aws-sdk/types": "3.609.0", + "@smithy/types": "^3.3.0", + "bowser": "^2.11.0", + "tslib": "^2.6.2" + }, + "dependencies": { + "tslib": { + "version": "2.7.0", + "resolved": "https://registry.npmjs.org/tslib/-/tslib-2.7.0.tgz", + "integrity": "sha512-gLXCKdN1/j47AiHiOkJN69hJmcbGTHI0ImLmbYLHykhgeN0jVGola9yVjFgzCUklsZQMW55o+dW7IXv3RCXDzA==" + } + } + }, + "@aws-sdk/util-user-agent-node": { + "version": "3.614.0", + "resolved": "https://registry.npmjs.org/@aws-sdk/util-user-agent-node/-/util-user-agent-node-3.614.0.tgz", + "integrity": "sha512-15ElZT88peoHnq5TEoEtZwoXTXRxNrk60TZNdpl/TUBJ5oNJ9Dqb5Z4ryb8ofN6nm9aFf59GVAerFDz8iUoHBA==", + "requires": { + "@aws-sdk/types": "3.609.0", + "@smithy/node-config-provider": "^3.1.4", + "@smithy/types": "^3.3.0", + "tslib": "^2.6.2" + }, + "dependencies": { + "tslib": { + "version": "2.7.0", + "resolved": "https://registry.npmjs.org/tslib/-/tslib-2.7.0.tgz", + "integrity": "sha512-gLXCKdN1/j47AiHiOkJN69hJmcbGTHI0ImLmbYLHykhgeN0jVGola9yVjFgzCUklsZQMW55o+dW7IXv3RCXDzA==" + } + } + }, + "@aws-sdk/xml-builder": { + "version": "3.609.0", + "resolved": "https://registry.npmjs.org/@aws-sdk/xml-builder/-/xml-builder-3.609.0.tgz", + "integrity": "sha512-l9XxNcA4HX98rwCC2/KoiWcmEiRfZe4G+mYwDbCFT87JIMj6GBhLDkAzr/W8KAaA2IDr8Vc6J8fZPgVulxxfMA==", + "requires": { + "@smithy/types": "^3.3.0", + "tslib": "^2.6.2" + }, + "dependencies": { + "tslib": { + "version": "2.7.0", + "resolved": "https://registry.npmjs.org/tslib/-/tslib-2.7.0.tgz", + "integrity": "sha512-gLXCKdN1/j47AiHiOkJN69hJmcbGTHI0ImLmbYLHykhgeN0jVGola9yVjFgzCUklsZQMW55o+dW7IXv3RCXDzA==" + } + } + }, "@babel/code-frame": { "version": "7.16.7", "resolved": "https://registry.npmjs.org/@babel/code-frame/-/code-frame-7.16.7.tgz", @@ -21384,7 +23823,7 @@ "version": "0.8.1", "resolved": "https://registry.npmjs.org/@cspotcode/source-map-support/-/source-map-support-0.8.1.tgz", "integrity": "sha512-IchNf6dN4tHoMFIn/7OE8LWZ19Y6q/67Bmf6vnGREv8RSbBVb9LPJxEcnwrcwX6ixSvaiGoomAUvu4YSxXrVgw==", - "devOptional": true, + "dev": true, "requires": { "@jridgewell/trace-mapping": "0.3.9" }, @@ -21393,7 +23832,7 @@ "version": "0.3.9", "resolved": "https://registry.npmjs.org/@jridgewell/trace-mapping/-/trace-mapping-0.3.9.tgz", "integrity": "sha512-3Belt6tdc8bPgAtbcmdtNJlirVoTmEb5e2gC94PnkwEW9jI6CAHUeoG85tjWP5WquqfavoMtMwiG4P926ZKKuQ==", - "devOptional": true, + "dev": true, "requires": { "@jridgewell/resolve-uri": "^3.0.3", "@jridgewell/sourcemap-codec": "^1.4.10" @@ -21662,6 +24101,7 @@ "version": "8.0.2", "resolved": "https://registry.npmjs.org/@isaacs/cliui/-/cliui-8.0.2.tgz", "integrity": "sha512-O8jcjabXaleOG9DQ0+ARXWZBTfnP4WNAqzuiJK7ll44AmxGKv/J2M4TPjxjY3znBCfvBXFzucm1twdyFybFqEA==", + "dev": true, "requires": { "string-width": "^5.1.2", "string-width-cjs": "npm:string-width@^4.2.0", @@ -21674,22 +24114,26 @@ "ansi-regex": { "version": "6.0.1", "resolved": "https://registry.npmjs.org/ansi-regex/-/ansi-regex-6.0.1.tgz", - "integrity": "sha512-n5M855fKb2SsfMIiFFoVrABHJC8QtHwVx+mHWP3QcEqBHYienj5dHSgjbxtC0WEZXYt4wcD6zrQElDPhFuZgfA==" + "integrity": "sha512-n5M855fKb2SsfMIiFFoVrABHJC8QtHwVx+mHWP3QcEqBHYienj5dHSgjbxtC0WEZXYt4wcD6zrQElDPhFuZgfA==", + "dev": true }, "ansi-styles": { "version": "6.2.1", "resolved": "https://registry.npmjs.org/ansi-styles/-/ansi-styles-6.2.1.tgz", - "integrity": "sha512-bN798gFfQX+viw3R7yrGWRqnrN2oRkEkUjjl4JNn4E8GxxbjtG3FbrEIIY3l8/hrwUwIeCZvi4QuOTP4MErVug==" + "integrity": "sha512-bN798gFfQX+viw3R7yrGWRqnrN2oRkEkUjjl4JNn4E8GxxbjtG3FbrEIIY3l8/hrwUwIeCZvi4QuOTP4MErVug==", + "dev": true }, "emoji-regex": { "version": "9.2.2", "resolved": "https://registry.npmjs.org/emoji-regex/-/emoji-regex-9.2.2.tgz", - "integrity": "sha512-L18DaJsXSUk2+42pv8mLs5jJT2hqFkFE4j21wOmgbUqsZ2hL72NsUU785g9RXgo3s0ZNgVl42TiHp3ZtOv/Vyg==" + "integrity": "sha512-L18DaJsXSUk2+42pv8mLs5jJT2hqFkFE4j21wOmgbUqsZ2hL72NsUU785g9RXgo3s0ZNgVl42TiHp3ZtOv/Vyg==", + "dev": true }, "string-width": { "version": "5.1.2", "resolved": "https://registry.npmjs.org/string-width/-/string-width-5.1.2.tgz", "integrity": "sha512-HnLOCR3vjcY8beoNLtcjZ5/nxn2afmME6lhrDrebokqMap+XbeW8n9TXpPDOqdGK5qcI3oT0GKTW6wC7EMiVqA==", + "dev": true, "requires": { "eastasianwidth": "^0.2.0", "emoji-regex": "^9.2.2", @@ -21700,6 +24144,7 @@ "version": "7.1.0", "resolved": "https://registry.npmjs.org/strip-ansi/-/strip-ansi-7.1.0.tgz", "integrity": "sha512-iq6eVVI64nQQTRYq2KtEg2d2uU7LElhTJwsH4YzIHZshxlgZms/wIc4VoDQTlG/IvVIrBKG06CrZnp0qv7hkcQ==", + "dev": true, "requires": { "ansi-regex": "^6.0.1" } @@ -21708,6 +24153,7 @@ "version": "8.1.0", "resolved": "https://registry.npmjs.org/wrap-ansi/-/wrap-ansi-8.1.0.tgz", "integrity": "sha512-si7QWI6zUMq56bESFvagtmzMdGOtoxfR+Sez11Mobfc7tm+VkUckk9bW2UeffTGVUbOksxmSw0AA2gs8g71NCQ==", + "dev": true, "requires": { "ansi-styles": "^6.1.0", "string-width": "^5.0.1", @@ -22257,7 +24703,7 @@ "version": "3.1.0", "resolved": "https://registry.npmjs.org/@jridgewell/resolve-uri/-/resolve-uri-3.1.0.tgz", "integrity": "sha512-F2msla3tad+Mfht5cJq7LSXcdudKTWCVYUgw6pLFOOHSTtZlj6SWNYAp+AhuqLmWdBO2X5hPrLcu8cVP8fy28w==", - "devOptional": true + "dev": true }, "@jridgewell/set-array": { "version": "1.1.1", @@ -22269,7 +24715,7 @@ "version": "1.4.14", "resolved": "https://registry.npmjs.org/@jridgewell/sourcemap-codec/-/sourcemap-codec-1.4.14.tgz", "integrity": "sha512-XPSJHWmi394fuUuzDnGz1wiKqWfo1yXecHQMRf2l6hztTO+nPru658AyDngaBe7isIxEkRsPR3FZh+s7iVa4Uw==", - "devOptional": true + "dev": true }, "@jridgewell/trace-mapping": { "version": "0.3.17", @@ -24021,6 +26467,7 @@ "version": "0.11.0", "resolved": "https://registry.npmjs.org/@pkgjs/parseargs/-/parseargs-0.11.0.tgz", "integrity": "sha512-+1VkjdD0QBLPodGrJUeqarH8VAIvQODIbwh9XpP5Syisf7YoQgsJKPNFoqqLQlu+VQ/tVSshMR6loPMn8U+dPg==", + "dev": true, "optional": true }, "@protobufjs/aspromise": { @@ -24234,34 +26681,889 @@ "@sinonjs/commons": "^2.0.0" } }, - "@sqltools/formatter": { - "version": "1.2.5", - "resolved": "https://registry.npmjs.org/@sqltools/formatter/-/formatter-1.2.5.tgz", - "integrity": "sha512-Uy0+khmZqUrUGm5dmMqVlnvufZRSK0FbYzVgp0UMstm+F5+W2/jnEEQyc9vo1ZR/E5ZI/B1WjjoTqBqwJL6Krw==" + "@smithy/abort-controller": { + "version": "3.1.1", + "resolved": "https://registry.npmjs.org/@smithy/abort-controller/-/abort-controller-3.1.1.tgz", + "integrity": "sha512-MBJBiidoe+0cTFhyxT8g+9g7CeVccLM0IOKKUMCNQ1CNMJ/eIfoo0RTfVrXOONEI1UCN1W+zkiHSbzUNE9dZtQ==", + "requires": { + "@smithy/types": "^3.3.0", + "tslib": "^2.6.2" + }, + "dependencies": { + "tslib": { + "version": "2.7.0", + "resolved": "https://registry.npmjs.org/tslib/-/tslib-2.7.0.tgz", + "integrity": "sha512-gLXCKdN1/j47AiHiOkJN69hJmcbGTHI0ImLmbYLHykhgeN0jVGola9yVjFgzCUklsZQMW55o+dW7IXv3RCXDzA==" + } + } + }, + "@smithy/chunked-blob-reader": { + "version": "3.0.0", + "resolved": "https://registry.npmjs.org/@smithy/chunked-blob-reader/-/chunked-blob-reader-3.0.0.tgz", + "integrity": "sha512-sbnURCwjF0gSToGlsBiAmd1lRCmSn72nu9axfJu5lIx6RUEgHu6GwTMbqCdhQSi0Pumcm5vFxsi9XWXb2mTaoA==", + "requires": { + "tslib": "^2.6.2" + }, + "dependencies": { + "tslib": { + "version": "2.7.0", + "resolved": "https://registry.npmjs.org/tslib/-/tslib-2.7.0.tgz", + "integrity": "sha512-gLXCKdN1/j47AiHiOkJN69hJmcbGTHI0ImLmbYLHykhgeN0jVGola9yVjFgzCUklsZQMW55o+dW7IXv3RCXDzA==" + } + } + }, + "@smithy/chunked-blob-reader-native": { + "version": "3.0.0", + "resolved": "https://registry.npmjs.org/@smithy/chunked-blob-reader-native/-/chunked-blob-reader-native-3.0.0.tgz", + "integrity": "sha512-VDkpCYW+peSuM4zJip5WDfqvg2Mo/e8yxOv3VF1m11y7B8KKMKVFtmZWDe36Fvk8rGuWrPZHHXZ7rR7uM5yWyg==", + "requires": { + "@smithy/util-base64": "^3.0.0", + "tslib": "^2.6.2" + }, + "dependencies": { + "tslib": { + "version": "2.7.0", + "resolved": "https://registry.npmjs.org/tslib/-/tslib-2.7.0.tgz", + "integrity": "sha512-gLXCKdN1/j47AiHiOkJN69hJmcbGTHI0ImLmbYLHykhgeN0jVGola9yVjFgzCUklsZQMW55o+dW7IXv3RCXDzA==" + } + } + }, + "@smithy/config-resolver": { + "version": "3.0.5", + "resolved": "https://registry.npmjs.org/@smithy/config-resolver/-/config-resolver-3.0.5.tgz", + "integrity": "sha512-SkW5LxfkSI1bUC74OtfBbdz+grQXYiPYolyu8VfpLIjEoN/sHVBlLeGXMQ1vX4ejkgfv6sxVbQJ32yF2cl1veA==", + "requires": { + "@smithy/node-config-provider": "^3.1.4", + "@smithy/types": "^3.3.0", + "@smithy/util-config-provider": "^3.0.0", + "@smithy/util-middleware": "^3.0.3", + "tslib": "^2.6.2" + }, + "dependencies": { + "tslib": { + "version": "2.7.0", + "resolved": "https://registry.npmjs.org/tslib/-/tslib-2.7.0.tgz", + "integrity": "sha512-gLXCKdN1/j47AiHiOkJN69hJmcbGTHI0ImLmbYLHykhgeN0jVGola9yVjFgzCUklsZQMW55o+dW7IXv3RCXDzA==" + } + } + }, + "@smithy/core": { + "version": "2.4.0", + "resolved": "https://registry.npmjs.org/@smithy/core/-/core-2.4.0.tgz", + "integrity": "sha512-cHXq+FneIF/KJbt4q4pjN186+Jf4ZB0ZOqEaZMBhT79srEyGDDBV31NqBRBjazz8ppQ1bJbDJMY9ba5wKFV36w==", + "requires": { + "@smithy/middleware-endpoint": "^3.1.0", + "@smithy/middleware-retry": "^3.0.15", + "@smithy/middleware-serde": "^3.0.3", + "@smithy/protocol-http": "^4.1.0", + "@smithy/smithy-client": "^3.2.0", + "@smithy/types": "^3.3.0", + "@smithy/util-body-length-browser": "^3.0.0", + "@smithy/util-middleware": "^3.0.3", + "@smithy/util-utf8": "^3.0.0", + "tslib": "^2.6.2" + }, + "dependencies": { + "tslib": { + "version": "2.7.0", + "resolved": "https://registry.npmjs.org/tslib/-/tslib-2.7.0.tgz", + "integrity": "sha512-gLXCKdN1/j47AiHiOkJN69hJmcbGTHI0ImLmbYLHykhgeN0jVGola9yVjFgzCUklsZQMW55o+dW7IXv3RCXDzA==" + } + } + }, + "@smithy/credential-provider-imds": { + "version": "3.2.0", + "resolved": "https://registry.npmjs.org/@smithy/credential-provider-imds/-/credential-provider-imds-3.2.0.tgz", + "integrity": "sha512-0SCIzgd8LYZ9EJxUjLXBmEKSZR/P/w6l7Rz/pab9culE/RWuqelAKGJvn5qUOl8BgX8Yj5HWM50A5hiB/RzsgA==", + "requires": { + "@smithy/node-config-provider": "^3.1.4", + "@smithy/property-provider": "^3.1.3", + "@smithy/types": "^3.3.0", + "@smithy/url-parser": "^3.0.3", + "tslib": "^2.6.2" + }, + "dependencies": { + "tslib": { + "version": "2.7.0", + "resolved": "https://registry.npmjs.org/tslib/-/tslib-2.7.0.tgz", + "integrity": "sha512-gLXCKdN1/j47AiHiOkJN69hJmcbGTHI0ImLmbYLHykhgeN0jVGola9yVjFgzCUklsZQMW55o+dW7IXv3RCXDzA==" + } + } + }, + "@smithy/eventstream-codec": { + "version": "3.1.2", + "resolved": "https://registry.npmjs.org/@smithy/eventstream-codec/-/eventstream-codec-3.1.2.tgz", + "integrity": "sha512-0mBcu49JWt4MXhrhRAlxASNy0IjDRFU+aWNDRal9OtUJvJNiwDuyKMUONSOjLjSCeGwZaE0wOErdqULer8r7yw==", + "requires": { + "@aws-crypto/crc32": "5.2.0", + "@smithy/types": "^3.3.0", + "@smithy/util-hex-encoding": "^3.0.0", + "tslib": "^2.6.2" + }, + "dependencies": { + "tslib": { + "version": "2.7.0", + "resolved": "https://registry.npmjs.org/tslib/-/tslib-2.7.0.tgz", + "integrity": "sha512-gLXCKdN1/j47AiHiOkJN69hJmcbGTHI0ImLmbYLHykhgeN0jVGola9yVjFgzCUklsZQMW55o+dW7IXv3RCXDzA==" + } + } + }, + "@smithy/eventstream-serde-browser": { + "version": "3.0.6", + "resolved": "https://registry.npmjs.org/@smithy/eventstream-serde-browser/-/eventstream-serde-browser-3.0.6.tgz", + "integrity": "sha512-2hM54UWQUOrki4BtsUI1WzmD13/SeaqT/AB3EUJKbcver/WgKNaiJ5y5F5XXuVe6UekffVzuUDrBZVAA3AWRpQ==", + "requires": { + "@smithy/eventstream-serde-universal": "^3.0.5", + "@smithy/types": "^3.3.0", + "tslib": "^2.6.2" + }, + "dependencies": { + "tslib": { + "version": "2.7.0", + "resolved": "https://registry.npmjs.org/tslib/-/tslib-2.7.0.tgz", + "integrity": "sha512-gLXCKdN1/j47AiHiOkJN69hJmcbGTHI0ImLmbYLHykhgeN0jVGola9yVjFgzCUklsZQMW55o+dW7IXv3RCXDzA==" + } + } + }, + "@smithy/eventstream-serde-config-resolver": { + "version": "3.0.3", + "resolved": "https://registry.npmjs.org/@smithy/eventstream-serde-config-resolver/-/eventstream-serde-config-resolver-3.0.3.tgz", + "integrity": "sha512-NVTYjOuYpGfrN/VbRQgn31x73KDLfCXCsFdad8DiIc3IcdxL+dYA9zEQPyOP7Fy2QL8CPy2WE4WCUD+ZsLNfaQ==", + "requires": { + "@smithy/types": "^3.3.0", + "tslib": "^2.6.2" + }, + "dependencies": { + "tslib": { + "version": "2.7.0", + "resolved": "https://registry.npmjs.org/tslib/-/tslib-2.7.0.tgz", + "integrity": "sha512-gLXCKdN1/j47AiHiOkJN69hJmcbGTHI0ImLmbYLHykhgeN0jVGola9yVjFgzCUklsZQMW55o+dW7IXv3RCXDzA==" + } + } + }, + "@smithy/eventstream-serde-node": { + "version": "3.0.5", + "resolved": "https://registry.npmjs.org/@smithy/eventstream-serde-node/-/eventstream-serde-node-3.0.5.tgz", + "integrity": "sha512-+upXvnHNyZP095s11jF5dhGw/Ihzqwl5G+/KtMnoQOpdfC3B5HYCcDVG9EmgkhJMXJlM64PyN5gjJl0uXFQehQ==", + "requires": { + "@smithy/eventstream-serde-universal": "^3.0.5", + "@smithy/types": "^3.3.0", + "tslib": "^2.6.2" + }, + "dependencies": { + "tslib": { + "version": "2.7.0", + "resolved": "https://registry.npmjs.org/tslib/-/tslib-2.7.0.tgz", + "integrity": "sha512-gLXCKdN1/j47AiHiOkJN69hJmcbGTHI0ImLmbYLHykhgeN0jVGola9yVjFgzCUklsZQMW55o+dW7IXv3RCXDzA==" + } + } + }, + "@smithy/eventstream-serde-universal": { + "version": "3.0.5", + "resolved": "https://registry.npmjs.org/@smithy/eventstream-serde-universal/-/eventstream-serde-universal-3.0.5.tgz", + "integrity": "sha512-5u/nXbyoh1s4QxrvNre9V6vfyoLWuiVvvd5TlZjGThIikc3G+uNiG9uOTCWweSRjv1asdDIWK7nOmN7le4RYHQ==", + "requires": { + "@smithy/eventstream-codec": "^3.1.2", + "@smithy/types": "^3.3.0", + "tslib": "^2.6.2" + }, + "dependencies": { + "tslib": { + "version": "2.7.0", + "resolved": "https://registry.npmjs.org/tslib/-/tslib-2.7.0.tgz", + "integrity": "sha512-gLXCKdN1/j47AiHiOkJN69hJmcbGTHI0ImLmbYLHykhgeN0jVGola9yVjFgzCUklsZQMW55o+dW7IXv3RCXDzA==" + } + } + }, + "@smithy/fetch-http-handler": { + "version": "3.2.4", + "resolved": "https://registry.npmjs.org/@smithy/fetch-http-handler/-/fetch-http-handler-3.2.4.tgz", + "integrity": "sha512-kBprh5Gs5h7ug4nBWZi1FZthdqSM+T7zMmsZxx0IBvWUn7dK3diz2SHn7Bs4dQGFDk8plDv375gzenDoNwrXjg==", + "requires": { + "@smithy/protocol-http": "^4.1.0", + "@smithy/querystring-builder": "^3.0.3", + "@smithy/types": "^3.3.0", + "@smithy/util-base64": "^3.0.0", + "tslib": "^2.6.2" + }, + "dependencies": { + "tslib": { + "version": "2.7.0", + "resolved": "https://registry.npmjs.org/tslib/-/tslib-2.7.0.tgz", + "integrity": "sha512-gLXCKdN1/j47AiHiOkJN69hJmcbGTHI0ImLmbYLHykhgeN0jVGola9yVjFgzCUklsZQMW55o+dW7IXv3RCXDzA==" + } + } + }, + "@smithy/hash-blob-browser": { + "version": "3.1.2", + "resolved": "https://registry.npmjs.org/@smithy/hash-blob-browser/-/hash-blob-browser-3.1.2.tgz", + "integrity": "sha512-hAbfqN2UbISltakCC2TP0kx4LqXBttEv2MqSPE98gVuDFMf05lU+TpC41QtqGP3Ff5A3GwZMPfKnEy0VmEUpmg==", + "requires": { + "@smithy/chunked-blob-reader": "^3.0.0", + "@smithy/chunked-blob-reader-native": "^3.0.0", + "@smithy/types": "^3.3.0", + "tslib": "^2.6.2" + }, + "dependencies": { + "tslib": { + "version": "2.7.0", + "resolved": "https://registry.npmjs.org/tslib/-/tslib-2.7.0.tgz", + "integrity": "sha512-gLXCKdN1/j47AiHiOkJN69hJmcbGTHI0ImLmbYLHykhgeN0jVGola9yVjFgzCUklsZQMW55o+dW7IXv3RCXDzA==" + } + } + }, + "@smithy/hash-node": { + "version": "3.0.3", + "resolved": "https://registry.npmjs.org/@smithy/hash-node/-/hash-node-3.0.3.tgz", + "integrity": "sha512-2ctBXpPMG+B3BtWSGNnKELJ7SH9e4TNefJS0cd2eSkOOROeBnnVBnAy9LtJ8tY4vUEoe55N4CNPxzbWvR39iBw==", + "requires": { + "@smithy/types": "^3.3.0", + "@smithy/util-buffer-from": "^3.0.0", + "@smithy/util-utf8": "^3.0.0", + "tslib": "^2.6.2" + }, + "dependencies": { + "tslib": { + "version": "2.7.0", + "resolved": "https://registry.npmjs.org/tslib/-/tslib-2.7.0.tgz", + "integrity": "sha512-gLXCKdN1/j47AiHiOkJN69hJmcbGTHI0ImLmbYLHykhgeN0jVGola9yVjFgzCUklsZQMW55o+dW7IXv3RCXDzA==" + } + } + }, + "@smithy/hash-stream-node": { + "version": "3.1.2", + "resolved": "https://registry.npmjs.org/@smithy/hash-stream-node/-/hash-stream-node-3.1.2.tgz", + "integrity": "sha512-PBgDMeEdDzi6JxKwbfBtwQG9eT9cVwsf0dZzLXoJF4sHKHs5HEo/3lJWpn6jibfJwT34I1EBXpBnZE8AxAft6g==", + "requires": { + "@smithy/types": "^3.3.0", + "@smithy/util-utf8": "^3.0.0", + "tslib": "^2.6.2" + }, + "dependencies": { + "tslib": { + "version": "2.7.0", + "resolved": "https://registry.npmjs.org/tslib/-/tslib-2.7.0.tgz", + "integrity": "sha512-gLXCKdN1/j47AiHiOkJN69hJmcbGTHI0ImLmbYLHykhgeN0jVGola9yVjFgzCUklsZQMW55o+dW7IXv3RCXDzA==" + } + } + }, + "@smithy/invalid-dependency": { + "version": "3.0.3", + "resolved": "https://registry.npmjs.org/@smithy/invalid-dependency/-/invalid-dependency-3.0.3.tgz", + "integrity": "sha512-ID1eL/zpDULmHJbflb864k72/SNOZCADRc9i7Exq3RUNJw6raWUSlFEQ+3PX3EYs++bTxZB2dE9mEHTQLv61tw==", + "requires": { + "@smithy/types": "^3.3.0", + "tslib": "^2.6.2" + }, + "dependencies": { + "tslib": { + "version": "2.7.0", + "resolved": "https://registry.npmjs.org/tslib/-/tslib-2.7.0.tgz", + "integrity": "sha512-gLXCKdN1/j47AiHiOkJN69hJmcbGTHI0ImLmbYLHykhgeN0jVGola9yVjFgzCUklsZQMW55o+dW7IXv3RCXDzA==" + } + } + }, + "@smithy/is-array-buffer": { + "version": "3.0.0", + "resolved": "https://registry.npmjs.org/@smithy/is-array-buffer/-/is-array-buffer-3.0.0.tgz", + "integrity": "sha512-+Fsu6Q6C4RSJiy81Y8eApjEB5gVtM+oFKTffg+jSuwtvomJJrhUJBu2zS8wjXSgH/g1MKEWrzyChTBe6clb5FQ==", + "requires": { + "tslib": "^2.6.2" + }, + "dependencies": { + "tslib": { + "version": "2.7.0", + "resolved": "https://registry.npmjs.org/tslib/-/tslib-2.7.0.tgz", + "integrity": "sha512-gLXCKdN1/j47AiHiOkJN69hJmcbGTHI0ImLmbYLHykhgeN0jVGola9yVjFgzCUklsZQMW55o+dW7IXv3RCXDzA==" + } + } + }, + "@smithy/md5-js": { + "version": "3.0.3", + "resolved": "https://registry.npmjs.org/@smithy/md5-js/-/md5-js-3.0.3.tgz", + "integrity": "sha512-O/SAkGVwpWmelpj/8yDtsaVe6sINHLB1q8YE/+ZQbDxIw3SRLbTZuRaI10K12sVoENdnHqzPp5i3/H+BcZ3m3Q==", + "requires": { + "@smithy/types": "^3.3.0", + "@smithy/util-utf8": "^3.0.0", + "tslib": "^2.6.2" + }, + "dependencies": { + "tslib": { + "version": "2.7.0", + "resolved": "https://registry.npmjs.org/tslib/-/tslib-2.7.0.tgz", + "integrity": "sha512-gLXCKdN1/j47AiHiOkJN69hJmcbGTHI0ImLmbYLHykhgeN0jVGola9yVjFgzCUklsZQMW55o+dW7IXv3RCXDzA==" + } + } + }, + "@smithy/middleware-content-length": { + "version": "3.0.5", + "resolved": "https://registry.npmjs.org/@smithy/middleware-content-length/-/middleware-content-length-3.0.5.tgz", + "integrity": "sha512-ILEzC2eyxx6ncej3zZSwMpB5RJ0zuqH7eMptxC4KN3f+v9bqT8ohssKbhNR78k/2tWW+KS5Spw+tbPF4Ejyqvw==", + "requires": { + "@smithy/protocol-http": "^4.1.0", + "@smithy/types": "^3.3.0", + "tslib": "^2.6.2" + }, + "dependencies": { + "tslib": { + "version": "2.7.0", + "resolved": "https://registry.npmjs.org/tslib/-/tslib-2.7.0.tgz", + "integrity": "sha512-gLXCKdN1/j47AiHiOkJN69hJmcbGTHI0ImLmbYLHykhgeN0jVGola9yVjFgzCUklsZQMW55o+dW7IXv3RCXDzA==" + } + } + }, + "@smithy/middleware-endpoint": { + "version": "3.1.0", + "resolved": "https://registry.npmjs.org/@smithy/middleware-endpoint/-/middleware-endpoint-3.1.0.tgz", + "integrity": "sha512-5y5aiKCEwg9TDPB4yFE7H6tYvGFf1OJHNczeY10/EFF8Ir8jZbNntQJxMWNfeQjC1mxPsaQ6mR9cvQbf+0YeMw==", + "requires": { + "@smithy/middleware-serde": "^3.0.3", + "@smithy/node-config-provider": "^3.1.4", + "@smithy/shared-ini-file-loader": "^3.1.4", + "@smithy/types": "^3.3.0", + "@smithy/url-parser": "^3.0.3", + "@smithy/util-middleware": "^3.0.3", + "tslib": "^2.6.2" + }, + "dependencies": { + "tslib": { + "version": "2.7.0", + "resolved": "https://registry.npmjs.org/tslib/-/tslib-2.7.0.tgz", + "integrity": "sha512-gLXCKdN1/j47AiHiOkJN69hJmcbGTHI0ImLmbYLHykhgeN0jVGola9yVjFgzCUklsZQMW55o+dW7IXv3RCXDzA==" + } + } + }, + "@smithy/middleware-retry": { + "version": "3.0.15", + "resolved": "https://registry.npmjs.org/@smithy/middleware-retry/-/middleware-retry-3.0.15.tgz", + "integrity": "sha512-iTMedvNt1ApdvkaoE8aSDuwaoc+BhvHqttbA/FO4Ty+y/S5hW6Ci/CTScG7vam4RYJWZxdTElc3MEfHRVH6cgQ==", + "requires": { + "@smithy/node-config-provider": "^3.1.4", + "@smithy/protocol-http": "^4.1.0", + "@smithy/service-error-classification": "^3.0.3", + "@smithy/smithy-client": "^3.2.0", + "@smithy/types": "^3.3.0", + "@smithy/util-middleware": "^3.0.3", + "@smithy/util-retry": "^3.0.3", + "tslib": "^2.6.2", + "uuid": "^9.0.1" + }, + "dependencies": { + "tslib": { + "version": "2.7.0", + "resolved": "https://registry.npmjs.org/tslib/-/tslib-2.7.0.tgz", + "integrity": "sha512-gLXCKdN1/j47AiHiOkJN69hJmcbGTHI0ImLmbYLHykhgeN0jVGola9yVjFgzCUklsZQMW55o+dW7IXv3RCXDzA==" + } + } + }, + "@smithy/middleware-serde": { + "version": "3.0.3", + "resolved": "https://registry.npmjs.org/@smithy/middleware-serde/-/middleware-serde-3.0.3.tgz", + "integrity": "sha512-puUbyJQBcg9eSErFXjKNiGILJGtiqmuuNKEYNYfUD57fUl4i9+mfmThtQhvFXU0hCVG0iEJhvQUipUf+/SsFdA==", + "requires": { + "@smithy/types": "^3.3.0", + "tslib": "^2.6.2" + }, + "dependencies": { + "tslib": { + "version": "2.7.0", + "resolved": "https://registry.npmjs.org/tslib/-/tslib-2.7.0.tgz", + "integrity": "sha512-gLXCKdN1/j47AiHiOkJN69hJmcbGTHI0ImLmbYLHykhgeN0jVGola9yVjFgzCUklsZQMW55o+dW7IXv3RCXDzA==" + } + } + }, + "@smithy/middleware-stack": { + "version": "3.0.3", + "resolved": "https://registry.npmjs.org/@smithy/middleware-stack/-/middleware-stack-3.0.3.tgz", + "integrity": "sha512-r4klY9nFudB0r9UdSMaGSyjyQK5adUyPnQN/ZM6M75phTxOdnc/AhpvGD1fQUvgmqjQEBGCwpnPbDm8pH5PapA==", + "requires": { + "@smithy/types": "^3.3.0", + "tslib": "^2.6.2" + }, + "dependencies": { + "tslib": { + "version": "2.7.0", + "resolved": "https://registry.npmjs.org/tslib/-/tslib-2.7.0.tgz", + "integrity": "sha512-gLXCKdN1/j47AiHiOkJN69hJmcbGTHI0ImLmbYLHykhgeN0jVGola9yVjFgzCUklsZQMW55o+dW7IXv3RCXDzA==" + } + } + }, + "@smithy/node-config-provider": { + "version": "3.1.4", + "resolved": "https://registry.npmjs.org/@smithy/node-config-provider/-/node-config-provider-3.1.4.tgz", + "integrity": "sha512-YvnElQy8HR4vDcAjoy7Xkx9YT8xZP4cBXcbJSgm/kxmiQu08DwUwj8rkGnyoJTpfl/3xYHH+d8zE+eHqoDCSdQ==", + "requires": { + "@smithy/property-provider": "^3.1.3", + "@smithy/shared-ini-file-loader": "^3.1.4", + "@smithy/types": "^3.3.0", + "tslib": "^2.6.2" + }, + "dependencies": { + "tslib": { + "version": "2.7.0", + "resolved": "https://registry.npmjs.org/tslib/-/tslib-2.7.0.tgz", + "integrity": "sha512-gLXCKdN1/j47AiHiOkJN69hJmcbGTHI0ImLmbYLHykhgeN0jVGola9yVjFgzCUklsZQMW55o+dW7IXv3RCXDzA==" + } + } + }, + "@smithy/node-http-handler": { + "version": "3.1.4", + "resolved": "https://registry.npmjs.org/@smithy/node-http-handler/-/node-http-handler-3.1.4.tgz", + "integrity": "sha512-+UmxgixgOr/yLsUxcEKGH0fMNVteJFGkmRltYFHnBMlogyFdpzn2CwqWmxOrfJELhV34v0WSlaqG1UtE1uXlJg==", + "requires": { + "@smithy/abort-controller": "^3.1.1", + "@smithy/protocol-http": "^4.1.0", + "@smithy/querystring-builder": "^3.0.3", + "@smithy/types": "^3.3.0", + "tslib": "^2.6.2" + }, + "dependencies": { + "tslib": { + "version": "2.7.0", + "resolved": "https://registry.npmjs.org/tslib/-/tslib-2.7.0.tgz", + "integrity": "sha512-gLXCKdN1/j47AiHiOkJN69hJmcbGTHI0ImLmbYLHykhgeN0jVGola9yVjFgzCUklsZQMW55o+dW7IXv3RCXDzA==" + } + } + }, + "@smithy/property-provider": { + "version": "3.1.3", + "resolved": "https://registry.npmjs.org/@smithy/property-provider/-/property-provider-3.1.3.tgz", + "integrity": "sha512-zahyOVR9Q4PEoguJ/NrFP4O7SMAfYO1HLhB18M+q+Z4KFd4V2obiMnlVoUFzFLSPeVt1POyNWneHHrZaTMoc/g==", + "requires": { + "@smithy/types": "^3.3.0", + "tslib": "^2.6.2" + }, + "dependencies": { + "tslib": { + "version": "2.7.0", + "resolved": "https://registry.npmjs.org/tslib/-/tslib-2.7.0.tgz", + "integrity": "sha512-gLXCKdN1/j47AiHiOkJN69hJmcbGTHI0ImLmbYLHykhgeN0jVGola9yVjFgzCUklsZQMW55o+dW7IXv3RCXDzA==" + } + } + }, + "@smithy/protocol-http": { + "version": "4.1.0", + "resolved": "https://registry.npmjs.org/@smithy/protocol-http/-/protocol-http-4.1.0.tgz", + "integrity": "sha512-dPVoHYQ2wcHooGXg3LQisa1hH0e4y0pAddPMeeUPipI1tEOqL6A4N0/G7abeq+K8wrwSgjk4C0wnD1XZpJm5aA==", + "requires": { + "@smithy/types": "^3.3.0", + "tslib": "^2.6.2" + }, + "dependencies": { + "tslib": { + "version": "2.7.0", + "resolved": "https://registry.npmjs.org/tslib/-/tslib-2.7.0.tgz", + "integrity": "sha512-gLXCKdN1/j47AiHiOkJN69hJmcbGTHI0ImLmbYLHykhgeN0jVGola9yVjFgzCUklsZQMW55o+dW7IXv3RCXDzA==" + } + } + }, + "@smithy/querystring-builder": { + "version": "3.0.3", + "resolved": "https://registry.npmjs.org/@smithy/querystring-builder/-/querystring-builder-3.0.3.tgz", + "integrity": "sha512-vyWckeUeesFKzCDaRwWLUA1Xym9McaA6XpFfAK5qI9DKJ4M33ooQGqvM4J+LalH4u/Dq9nFiC8U6Qn1qi0+9zw==", + "requires": { + "@smithy/types": "^3.3.0", + "@smithy/util-uri-escape": "^3.0.0", + "tslib": "^2.6.2" + }, + "dependencies": { + "tslib": { + "version": "2.7.0", + "resolved": "https://registry.npmjs.org/tslib/-/tslib-2.7.0.tgz", + "integrity": "sha512-gLXCKdN1/j47AiHiOkJN69hJmcbGTHI0ImLmbYLHykhgeN0jVGola9yVjFgzCUklsZQMW55o+dW7IXv3RCXDzA==" + } + } + }, + "@smithy/querystring-parser": { + "version": "3.0.3", + "resolved": "https://registry.npmjs.org/@smithy/querystring-parser/-/querystring-parser-3.0.3.tgz", + "integrity": "sha512-zahM1lQv2YjmznnfQsWbYojFe55l0SLG/988brlLv1i8z3dubloLF+75ATRsqPBboUXsW6I9CPGE5rQgLfY0vQ==", + "requires": { + "@smithy/types": "^3.3.0", + "tslib": "^2.6.2" + }, + "dependencies": { + "tslib": { + "version": "2.7.0", + "resolved": "https://registry.npmjs.org/tslib/-/tslib-2.7.0.tgz", + "integrity": "sha512-gLXCKdN1/j47AiHiOkJN69hJmcbGTHI0ImLmbYLHykhgeN0jVGola9yVjFgzCUklsZQMW55o+dW7IXv3RCXDzA==" + } + } + }, + "@smithy/service-error-classification": { + "version": "3.0.3", + "resolved": "https://registry.npmjs.org/@smithy/service-error-classification/-/service-error-classification-3.0.3.tgz", + "integrity": "sha512-Jn39sSl8cim/VlkLsUhRFq/dKDnRUFlfRkvhOJaUbLBXUsLRLNf9WaxDv/z9BjuQ3A6k/qE8af1lsqcwm7+DaQ==", + "requires": { + "@smithy/types": "^3.3.0" + } + }, + "@smithy/shared-ini-file-loader": { + "version": "3.1.4", + "resolved": "https://registry.npmjs.org/@smithy/shared-ini-file-loader/-/shared-ini-file-loader-3.1.4.tgz", + "integrity": "sha512-qMxS4hBGB8FY2GQqshcRUy1K6k8aBWP5vwm8qKkCT3A9K2dawUwOIJfqh9Yste/Bl0J2lzosVyrXDj68kLcHXQ==", + "requires": { + "@smithy/types": "^3.3.0", + "tslib": "^2.6.2" + }, + "dependencies": { + "tslib": { + "version": "2.7.0", + "resolved": "https://registry.npmjs.org/tslib/-/tslib-2.7.0.tgz", + "integrity": "sha512-gLXCKdN1/j47AiHiOkJN69hJmcbGTHI0ImLmbYLHykhgeN0jVGola9yVjFgzCUklsZQMW55o+dW7IXv3RCXDzA==" + } + } + }, + "@smithy/signature-v4": { + "version": "4.1.0", + "resolved": "https://registry.npmjs.org/@smithy/signature-v4/-/signature-v4-4.1.0.tgz", + "integrity": "sha512-aRryp2XNZeRcOtuJoxjydO6QTaVhxx/vjaR+gx7ZjaFgrgPRyZ3HCTbfwqYj6ZWEBHkCSUfcaymKPURaByukag==", + "requires": { + "@smithy/is-array-buffer": "^3.0.0", + "@smithy/protocol-http": "^4.1.0", + "@smithy/types": "^3.3.0", + "@smithy/util-hex-encoding": "^3.0.0", + "@smithy/util-middleware": "^3.0.3", + "@smithy/util-uri-escape": "^3.0.0", + "@smithy/util-utf8": "^3.0.0", + "tslib": "^2.6.2" + }, + "dependencies": { + "tslib": { + "version": "2.7.0", + "resolved": "https://registry.npmjs.org/tslib/-/tslib-2.7.0.tgz", + "integrity": "sha512-gLXCKdN1/j47AiHiOkJN69hJmcbGTHI0ImLmbYLHykhgeN0jVGola9yVjFgzCUklsZQMW55o+dW7IXv3RCXDzA==" + } + } + }, + "@smithy/smithy-client": { + "version": "3.2.0", + "resolved": "https://registry.npmjs.org/@smithy/smithy-client/-/smithy-client-3.2.0.tgz", + "integrity": "sha512-pDbtxs8WOhJLJSeaF/eAbPgXg4VVYFlRcL/zoNYA5WbG3wBL06CHtBSg53ppkttDpAJ/hdiede+xApip1CwSLw==", + "requires": { + "@smithy/middleware-endpoint": "^3.1.0", + "@smithy/middleware-stack": "^3.0.3", + "@smithy/protocol-http": "^4.1.0", + "@smithy/types": "^3.3.0", + "@smithy/util-stream": "^3.1.3", + "tslib": "^2.6.2" + }, + "dependencies": { + "tslib": { + "version": "2.7.0", + "resolved": "https://registry.npmjs.org/tslib/-/tslib-2.7.0.tgz", + "integrity": "sha512-gLXCKdN1/j47AiHiOkJN69hJmcbGTHI0ImLmbYLHykhgeN0jVGola9yVjFgzCUklsZQMW55o+dW7IXv3RCXDzA==" + } + } + }, + "@smithy/types": { + "version": "3.3.0", + "resolved": "https://registry.npmjs.org/@smithy/types/-/types-3.3.0.tgz", + "integrity": "sha512-IxvBBCTFDHbVoK7zIxqA1ZOdc4QfM5HM7rGleCuHi7L1wnKv5Pn69xXJQ9hgxH60ZVygH9/JG0jRgtUncE3QUA==", + "requires": { + "tslib": "^2.6.2" + }, + "dependencies": { + "tslib": { + "version": "2.7.0", + "resolved": "https://registry.npmjs.org/tslib/-/tslib-2.7.0.tgz", + "integrity": "sha512-gLXCKdN1/j47AiHiOkJN69hJmcbGTHI0ImLmbYLHykhgeN0jVGola9yVjFgzCUklsZQMW55o+dW7IXv3RCXDzA==" + } + } + }, + "@smithy/url-parser": { + "version": "3.0.3", + "resolved": "https://registry.npmjs.org/@smithy/url-parser/-/url-parser-3.0.3.tgz", + "integrity": "sha512-pw3VtZtX2rg+s6HMs6/+u9+hu6oY6U7IohGhVNnjbgKy86wcIsSZwgHrFR+t67Uyxvp4Xz3p3kGXXIpTNisq8A==", + "requires": { + "@smithy/querystring-parser": "^3.0.3", + "@smithy/types": "^3.3.0", + "tslib": "^2.6.2" + }, + "dependencies": { + "tslib": { + "version": "2.7.0", + "resolved": "https://registry.npmjs.org/tslib/-/tslib-2.7.0.tgz", + "integrity": "sha512-gLXCKdN1/j47AiHiOkJN69hJmcbGTHI0ImLmbYLHykhgeN0jVGola9yVjFgzCUklsZQMW55o+dW7IXv3RCXDzA==" + } + } + }, + "@smithy/util-base64": { + "version": "3.0.0", + "resolved": "https://registry.npmjs.org/@smithy/util-base64/-/util-base64-3.0.0.tgz", + "integrity": "sha512-Kxvoh5Qtt0CDsfajiZOCpJxgtPHXOKwmM+Zy4waD43UoEMA+qPxxa98aE/7ZhdnBFZFXMOiBR5xbcaMhLtznQQ==", + "requires": { + "@smithy/util-buffer-from": "^3.0.0", + "@smithy/util-utf8": "^3.0.0", + "tslib": "^2.6.2" + }, + "dependencies": { + "tslib": { + "version": "2.7.0", + "resolved": "https://registry.npmjs.org/tslib/-/tslib-2.7.0.tgz", + "integrity": "sha512-gLXCKdN1/j47AiHiOkJN69hJmcbGTHI0ImLmbYLHykhgeN0jVGola9yVjFgzCUklsZQMW55o+dW7IXv3RCXDzA==" + } + } + }, + "@smithy/util-body-length-browser": { + "version": "3.0.0", + "resolved": "https://registry.npmjs.org/@smithy/util-body-length-browser/-/util-body-length-browser-3.0.0.tgz", + "integrity": "sha512-cbjJs2A1mLYmqmyVl80uoLTJhAcfzMOyPgjwAYusWKMdLeNtzmMz9YxNl3/jRLoxSS3wkqkf0jwNdtXWtyEBaQ==", + "requires": { + "tslib": "^2.6.2" + }, + "dependencies": { + "tslib": { + "version": "2.7.0", + "resolved": "https://registry.npmjs.org/tslib/-/tslib-2.7.0.tgz", + "integrity": "sha512-gLXCKdN1/j47AiHiOkJN69hJmcbGTHI0ImLmbYLHykhgeN0jVGola9yVjFgzCUklsZQMW55o+dW7IXv3RCXDzA==" + } + } + }, + "@smithy/util-body-length-node": { + "version": "3.0.0", + "resolved": "https://registry.npmjs.org/@smithy/util-body-length-node/-/util-body-length-node-3.0.0.tgz", + "integrity": "sha512-Tj7pZ4bUloNUP6PzwhN7K386tmSmEET9QtQg0TgdNOnxhZvCssHji+oZTUIuzxECRfG8rdm2PMw2WCFs6eIYkA==", + "requires": { + "tslib": "^2.6.2" + }, + "dependencies": { + "tslib": { + "version": "2.7.0", + "resolved": "https://registry.npmjs.org/tslib/-/tslib-2.7.0.tgz", + "integrity": "sha512-gLXCKdN1/j47AiHiOkJN69hJmcbGTHI0ImLmbYLHykhgeN0jVGola9yVjFgzCUklsZQMW55o+dW7IXv3RCXDzA==" + } + } + }, + "@smithy/util-buffer-from": { + "version": "3.0.0", + "resolved": "https://registry.npmjs.org/@smithy/util-buffer-from/-/util-buffer-from-3.0.0.tgz", + "integrity": "sha512-aEOHCgq5RWFbP+UDPvPot26EJHjOC+bRgse5A8V3FSShqd5E5UN4qc7zkwsvJPPAVsf73QwYcHN1/gt/rtLwQA==", + "requires": { + "@smithy/is-array-buffer": "^3.0.0", + "tslib": "^2.6.2" + }, + "dependencies": { + "tslib": { + "version": "2.7.0", + "resolved": "https://registry.npmjs.org/tslib/-/tslib-2.7.0.tgz", + "integrity": "sha512-gLXCKdN1/j47AiHiOkJN69hJmcbGTHI0ImLmbYLHykhgeN0jVGola9yVjFgzCUklsZQMW55o+dW7IXv3RCXDzA==" + } + } + }, + "@smithy/util-config-provider": { + "version": "3.0.0", + "resolved": "https://registry.npmjs.org/@smithy/util-config-provider/-/util-config-provider-3.0.0.tgz", + "integrity": "sha512-pbjk4s0fwq3Di/ANL+rCvJMKM5bzAQdE5S/6RL5NXgMExFAi6UgQMPOm5yPaIWPpr+EOXKXRonJ3FoxKf4mCJQ==", + "requires": { + "tslib": "^2.6.2" + }, + "dependencies": { + "tslib": { + "version": "2.7.0", + "resolved": "https://registry.npmjs.org/tslib/-/tslib-2.7.0.tgz", + "integrity": "sha512-gLXCKdN1/j47AiHiOkJN69hJmcbGTHI0ImLmbYLHykhgeN0jVGola9yVjFgzCUklsZQMW55o+dW7IXv3RCXDzA==" + } + } + }, + "@smithy/util-defaults-mode-browser": { + "version": "3.0.15", + "resolved": "https://registry.npmjs.org/@smithy/util-defaults-mode-browser/-/util-defaults-mode-browser-3.0.15.tgz", + "integrity": "sha512-FZ4Psa3vjp8kOXcd3HJOiDPBCWtiilLl57r0cnNtq/Ga9RSDrM5ERL6xt+tO43+2af6Pn5Yp92x2n5vPuduNfg==", + "requires": { + "@smithy/property-provider": "^3.1.3", + "@smithy/smithy-client": "^3.2.0", + "@smithy/types": "^3.3.0", + "bowser": "^2.11.0", + "tslib": "^2.6.2" + }, + "dependencies": { + "tslib": { + "version": "2.7.0", + "resolved": "https://registry.npmjs.org/tslib/-/tslib-2.7.0.tgz", + "integrity": "sha512-gLXCKdN1/j47AiHiOkJN69hJmcbGTHI0ImLmbYLHykhgeN0jVGola9yVjFgzCUklsZQMW55o+dW7IXv3RCXDzA==" + } + } + }, + "@smithy/util-defaults-mode-node": { + "version": "3.0.15", + "resolved": "https://registry.npmjs.org/@smithy/util-defaults-mode-node/-/util-defaults-mode-node-3.0.15.tgz", + "integrity": "sha512-KSyAAx2q6d0t6f/S4XB2+3+6aQacm3aLMhs9aLMqn18uYGUepbdssfogW5JQZpc6lXNBnp0tEnR5e9CEKmEd7A==", + "requires": { + "@smithy/config-resolver": "^3.0.5", + "@smithy/credential-provider-imds": "^3.2.0", + "@smithy/node-config-provider": "^3.1.4", + "@smithy/property-provider": "^3.1.3", + "@smithy/smithy-client": "^3.2.0", + "@smithy/types": "^3.3.0", + "tslib": "^2.6.2" + }, + "dependencies": { + "tslib": { + "version": "2.7.0", + "resolved": "https://registry.npmjs.org/tslib/-/tslib-2.7.0.tgz", + "integrity": "sha512-gLXCKdN1/j47AiHiOkJN69hJmcbGTHI0ImLmbYLHykhgeN0jVGola9yVjFgzCUklsZQMW55o+dW7IXv3RCXDzA==" + } + } + }, + "@smithy/util-endpoints": { + "version": "2.0.5", + "resolved": "https://registry.npmjs.org/@smithy/util-endpoints/-/util-endpoints-2.0.5.tgz", + "integrity": "sha512-ReQP0BWihIE68OAblC/WQmDD40Gx+QY1Ez8mTdFMXpmjfxSyz2fVQu3A4zXRfQU9sZXtewk3GmhfOHswvX+eNg==", + "requires": { + "@smithy/node-config-provider": "^3.1.4", + "@smithy/types": "^3.3.0", + "tslib": "^2.6.2" + }, + "dependencies": { + "tslib": { + "version": "2.7.0", + "resolved": "https://registry.npmjs.org/tslib/-/tslib-2.7.0.tgz", + "integrity": "sha512-gLXCKdN1/j47AiHiOkJN69hJmcbGTHI0ImLmbYLHykhgeN0jVGola9yVjFgzCUklsZQMW55o+dW7IXv3RCXDzA==" + } + } + }, + "@smithy/util-hex-encoding": { + "version": "3.0.0", + "resolved": "https://registry.npmjs.org/@smithy/util-hex-encoding/-/util-hex-encoding-3.0.0.tgz", + "integrity": "sha512-eFndh1WEK5YMUYvy3lPlVmYY/fZcQE1D8oSf41Id2vCeIkKJXPcYDCZD+4+xViI6b1XSd7tE+s5AmXzz5ilabQ==", + "requires": { + "tslib": "^2.6.2" + }, + "dependencies": { + "tslib": { + "version": "2.7.0", + "resolved": "https://registry.npmjs.org/tslib/-/tslib-2.7.0.tgz", + "integrity": "sha512-gLXCKdN1/j47AiHiOkJN69hJmcbGTHI0ImLmbYLHykhgeN0jVGola9yVjFgzCUklsZQMW55o+dW7IXv3RCXDzA==" + } + } + }, + "@smithy/util-middleware": { + "version": "3.0.3", + "resolved": "https://registry.npmjs.org/@smithy/util-middleware/-/util-middleware-3.0.3.tgz", + "integrity": "sha512-l+StyYYK/eO3DlVPbU+4Bi06Jjal+PFLSMmlWM1BEwyLxZ3aKkf1ROnoIakfaA7mC6uw3ny7JBkau4Yc+5zfWw==", + "requires": { + "@smithy/types": "^3.3.0", + "tslib": "^2.6.2" + }, + "dependencies": { + "tslib": { + "version": "2.7.0", + "resolved": "https://registry.npmjs.org/tslib/-/tslib-2.7.0.tgz", + "integrity": "sha512-gLXCKdN1/j47AiHiOkJN69hJmcbGTHI0ImLmbYLHykhgeN0jVGola9yVjFgzCUklsZQMW55o+dW7IXv3RCXDzA==" + } + } + }, + "@smithy/util-retry": { + "version": "3.0.3", + "resolved": "https://registry.npmjs.org/@smithy/util-retry/-/util-retry-3.0.3.tgz", + "integrity": "sha512-AFw+hjpbtVApzpNDhbjNG5NA3kyoMs7vx0gsgmlJF4s+yz1Zlepde7J58zpIRIsdjc+emhpAITxA88qLkPF26w==", + "requires": { + "@smithy/service-error-classification": "^3.0.3", + "@smithy/types": "^3.3.0", + "tslib": "^2.6.2" + }, + "dependencies": { + "tslib": { + "version": "2.7.0", + "resolved": "https://registry.npmjs.org/tslib/-/tslib-2.7.0.tgz", + "integrity": "sha512-gLXCKdN1/j47AiHiOkJN69hJmcbGTHI0ImLmbYLHykhgeN0jVGola9yVjFgzCUklsZQMW55o+dW7IXv3RCXDzA==" + } + } + }, + "@smithy/util-stream": { + "version": "3.1.3", + "resolved": "https://registry.npmjs.org/@smithy/util-stream/-/util-stream-3.1.3.tgz", + "integrity": "sha512-FIv/bRhIlAxC0U7xM1BCnF2aDRPq0UaelqBHkM2lsCp26mcBbgI0tCVTv+jGdsQLUmAMybua/bjDsSu8RQHbmw==", + "requires": { + "@smithy/fetch-http-handler": "^3.2.4", + "@smithy/node-http-handler": "^3.1.4", + "@smithy/types": "^3.3.0", + "@smithy/util-base64": "^3.0.0", + "@smithy/util-buffer-from": "^3.0.0", + "@smithy/util-hex-encoding": "^3.0.0", + "@smithy/util-utf8": "^3.0.0", + "tslib": "^2.6.2" + }, + "dependencies": { + "tslib": { + "version": "2.7.0", + "resolved": "https://registry.npmjs.org/tslib/-/tslib-2.7.0.tgz", + "integrity": "sha512-gLXCKdN1/j47AiHiOkJN69hJmcbGTHI0ImLmbYLHykhgeN0jVGola9yVjFgzCUklsZQMW55o+dW7IXv3RCXDzA==" + } + } + }, + "@smithy/util-uri-escape": { + "version": "3.0.0", + "resolved": "https://registry.npmjs.org/@smithy/util-uri-escape/-/util-uri-escape-3.0.0.tgz", + "integrity": "sha512-LqR7qYLgZTD7nWLBecUi4aqolw8Mhza9ArpNEQ881MJJIU2sE5iHCK6TdyqqzcDLy0OPe10IY4T8ctVdtynubg==", + "requires": { + "tslib": "^2.6.2" + }, + "dependencies": { + "tslib": { + "version": "2.7.0", + "resolved": "https://registry.npmjs.org/tslib/-/tslib-2.7.0.tgz", + "integrity": "sha512-gLXCKdN1/j47AiHiOkJN69hJmcbGTHI0ImLmbYLHykhgeN0jVGola9yVjFgzCUklsZQMW55o+dW7IXv3RCXDzA==" + } + } + }, + "@smithy/util-utf8": { + "version": "3.0.0", + "resolved": "https://registry.npmjs.org/@smithy/util-utf8/-/util-utf8-3.0.0.tgz", + "integrity": "sha512-rUeT12bxFnplYDe815GXbq/oixEGHfRFFtcTF3YdDi/JaENIM6aSYYLJydG83UNzLXeRI5K8abYd/8Sp/QM0kA==", + "requires": { + "@smithy/util-buffer-from": "^3.0.0", + "tslib": "^2.6.2" + }, + "dependencies": { + "tslib": { + "version": "2.7.0", + "resolved": "https://registry.npmjs.org/tslib/-/tslib-2.7.0.tgz", + "integrity": "sha512-gLXCKdN1/j47AiHiOkJN69hJmcbGTHI0ImLmbYLHykhgeN0jVGola9yVjFgzCUklsZQMW55o+dW7IXv3RCXDzA==" + } + } + }, + "@smithy/util-waiter": { + "version": "3.1.2", + "resolved": "https://registry.npmjs.org/@smithy/util-waiter/-/util-waiter-3.1.2.tgz", + "integrity": "sha512-4pP0EV3iTsexDx+8PPGAKCQpd/6hsQBaQhqWzU4hqKPHN5epPsxKbvUTIiYIHTxaKt6/kEaqPBpu/ufvfbrRzw==", + "requires": { + "@smithy/abort-controller": "^3.1.1", + "@smithy/types": "^3.3.0", + "tslib": "^2.6.2" + }, + "dependencies": { + "tslib": { + "version": "2.7.0", + "resolved": "https://registry.npmjs.org/tslib/-/tslib-2.7.0.tgz", + "integrity": "sha512-gLXCKdN1/j47AiHiOkJN69hJmcbGTHI0ImLmbYLHykhgeN0jVGola9yVjFgzCUklsZQMW55o+dW7IXv3RCXDzA==" + } + } }, "@tsconfig/node10": { "version": "1.0.9", "resolved": "https://registry.npmjs.org/@tsconfig/node10/-/node10-1.0.9.tgz", "integrity": "sha512-jNsYVVxU8v5g43Erja32laIDHXeoNvFEpX33OK4d6hljo3jDhCBDhx5dhCCTMWUojscpAagGiRkBKxpdl9fxqA==", - "devOptional": true + "dev": true }, "@tsconfig/node12": { "version": "1.0.11", "resolved": "https://registry.npmjs.org/@tsconfig/node12/-/node12-1.0.11.tgz", "integrity": "sha512-cqefuRsh12pWyGsIoBKJA9luFu3mRxCA+ORZvA4ktLSzIuCUtWVxGIuXigEwO5/ywWFMZ2QEGKWvkZG1zDMTag==", - "devOptional": true + "dev": true }, "@tsconfig/node14": { "version": "1.0.3", "resolved": "https://registry.npmjs.org/@tsconfig/node14/-/node14-1.0.3.tgz", "integrity": "sha512-ysT8mhdixWK6Hw3i1V2AeRqZ5WfXg1G43mqoYlM2nc6388Fq5jcXyr5mRsqViLx/GJYdoL0bfXD8nmF+Zn/Iow==", - "devOptional": true + "dev": true }, "@tsconfig/node16": { "version": "1.0.3", "resolved": "https://registry.npmjs.org/@tsconfig/node16/-/node16-1.0.3.tgz", "integrity": "sha512-yOlFc+7UtL/89t2ZhjPvvB/DeAr3r+Dq58IgzsFkOAvVC6NMJXmCGjbptdXdR9qsX7pKcTL+s87FtYREi2dEEQ==", - "devOptional": true + "dev": true }, "@types/accepts": { "version": "1.3.5", @@ -25018,7 +28320,7 @@ "version": "8.8.2", "resolved": "https://registry.npmjs.org/acorn/-/acorn-8.8.2.tgz", "integrity": "sha512-xjIYgE8HBrkpd/sJqOGNspf8uHG+NOHGOw6a/Urj8taM2EXfdNAH2oFcPeIFfsv3+kz/mJrS5VuMqbNLjCa2vw==", - "devOptional": true + "dev": true }, "acorn-jsx": { "version": "5.3.2", @@ -25031,7 +28333,7 @@ "version": "8.2.0", "resolved": "https://registry.npmjs.org/acorn-walk/-/acorn-walk-8.2.0.tgz", "integrity": "sha512-k+iyHEuPgSw6SbuDpGQM+06HQUa04DZ3o+F6CSzXMvvI5KMvnaEqXe+YVe555R9nn6GPt404fos4wcgpw12SDA==", - "devOptional": true + "dev": true }, "add-stream": { "version": "1.0.0", @@ -25107,11 +28409,6 @@ "color-convert": "^1.9.0" } }, - "any-promise": { - "version": "1.3.0", - "resolved": "https://registry.npmjs.org/any-promise/-/any-promise-1.3.0.tgz", - "integrity": "sha512-7UvmKalWRt1wgjL1RrGxoSJW/0QZFIegpeGvZG9kjp8vrRu55XTHbwnqq2GpXm9uLbcuhxm3IqX9OB4MZR1b2A==" - }, "anymatch": { "version": "3.1.2", "resolved": "https://registry.npmjs.org/anymatch/-/anymatch-3.1.2.tgz", @@ -25122,11 +28419,6 @@ "picomatch": "^2.0.4" } }, - "app-root-path": { - "version": "3.1.0", - "resolved": "https://registry.npmjs.org/app-root-path/-/app-root-path-3.1.0.tgz", - "integrity": "sha512-biN3PwB2gUtjaYy/isrU3aNWI5w+fAfvHkSvCKeQGxhmYpwKFUxudR3Yya+KqVRHBmEDYh+/lTozYCFbmzX4nA==" - }, "append-field": { "version": "1.0.0", "resolved": "https://registry.npmjs.org/append-field/-/append-field-1.0.0.tgz", @@ -25141,7 +28433,7 @@ "version": "4.1.3", "resolved": "https://registry.npmjs.org/arg/-/arg-4.1.3.tgz", "integrity": "sha512-58S9QDqG0Xx27YwPSt9fJxivjYl432YCwfDMfZ+71RAqUrZef7LrKQZ3LHLOwCS4FLNBplP533Zx895SeOCHvA==", - "devOptional": true + "dev": true }, "argparse": { "version": "2.0.1", @@ -25591,6 +28883,11 @@ } } }, + "bowser": { + "version": "2.11.0", + "resolved": "https://registry.npmjs.org/bowser/-/bowser-2.11.0.tgz", + "integrity": "sha512-AlcaJBi/pqqJBIQ8U9Mcpc9i8Aqxn88Skv5d+xBX006BY5u8N3mGLHa5Lgppa7L/HfwgwLgZ6NYs+Ag6uUmJRA==" + }, "brace-expansion": { "version": "1.1.11", "resolved": "https://registry.npmjs.org/brace-expansion/-/brace-expansion-1.1.11.tgz", @@ -25791,78 +29088,6 @@ "restore-cursor": "^3.1.0" } }, - "cli-highlight": { - "version": "2.1.11", - "resolved": "https://registry.npmjs.org/cli-highlight/-/cli-highlight-2.1.11.tgz", - "integrity": "sha512-9KDcoEVwyUXrjcJNvHD0NFc/hiwe/WPVYIleQh2O1N2Zro5gWJZ/K+3DGn8w8P/F6FxOgzyC5bxDyHIgCSPhGg==", - "requires": { - "chalk": "^4.0.0", - "highlight.js": "^10.7.1", - "mz": "^2.4.0", - "parse5": "^5.1.1", - "parse5-htmlparser2-tree-adapter": "^6.0.0", - "yargs": "^16.0.0" - }, - "dependencies": { - "ansi-styles": { - "version": "4.3.0", - "resolved": "https://registry.npmjs.org/ansi-styles/-/ansi-styles-4.3.0.tgz", - "integrity": "sha512-zbB9rCJAT1rbjiVDb2hqKFHNYLxgtk8NURxZ3IZwD3F6NtxbXZQCnnSi1Lkx+IDohdPlFp222wVALIheZJQSEg==", - "requires": { - "color-convert": "^2.0.1" - } - }, - "chalk": { - "version": "4.1.2", - "resolved": "https://registry.npmjs.org/chalk/-/chalk-4.1.2.tgz", - "integrity": "sha512-oKnbhFyRIXpUuez8iBMmyEa4nbj4IOQyuhc/wy9kY7/WVPcwIO9VA668Pu8RkO7+0G76SLROeyw9CpQ061i4mA==", - "requires": { - "ansi-styles": "^4.1.0", - "supports-color": "^7.1.0" - } - }, - "color-convert": { - "version": "2.0.1", - "resolved": "https://registry.npmjs.org/color-convert/-/color-convert-2.0.1.tgz", - "integrity": "sha512-RRECPsj7iu/xb5oKYcsFHSppFNnsj/52OVTRKb4zP5onXwVF3zVmmToNcOfGC+CRDpfK/U584fMg38ZHCaElKQ==", - "requires": { - "color-name": "~1.1.4" - } - }, - "color-name": { - "version": "1.1.4", - "resolved": "https://registry.npmjs.org/color-name/-/color-name-1.1.4.tgz", - "integrity": "sha512-dOy+3AuW3a2wNbZHIuMZpTcgjGuLU/uBL/ubcZF9OXbDo8ff4O8yVp5Bf0efS8uEoYo5q4Fx7dY9OgQGXgAsQA==" - }, - "has-flag": { - "version": "4.0.0", - "resolved": "https://registry.npmjs.org/has-flag/-/has-flag-4.0.0.tgz", - "integrity": "sha512-EykJT/Q1KjTWctppgIAgfSO0tKVuZUjhgMr17kqTumMl6Afv3EISleU7qZUzoXDFTAHTDC4NOoG/ZxU3EvlMPQ==" - }, - "supports-color": { - "version": "7.2.0", - "resolved": "https://registry.npmjs.org/supports-color/-/supports-color-7.2.0.tgz", - "integrity": "sha512-qpCAvRl9stuOHveKsn7HncJRvv501qIacKzQlO/+Lwxc9+0q2wLyv4Dfvt80/DPn2pqOBsJdDiogXGR9+OvwRw==", - "requires": { - "has-flag": "^4.0.0" - } - }, - "yargs": { - "version": "16.2.0", - "resolved": "https://registry.npmjs.org/yargs/-/yargs-16.2.0.tgz", - "integrity": "sha512-D1mvvtDG0L5ft/jGWkLpG1+m0eQxOfaBvTNELraWj22wSVUMWxZUvYgJYcKh6jGGIkJFhH4IZPQhR4TKpc8mBw==", - "requires": { - "cliui": "^7.0.2", - "escalade": "^3.1.1", - "get-caller-file": "^2.0.5", - "require-directory": "^2.1.1", - "string-width": "^4.2.0", - "y18n": "^5.0.5", - "yargs-parser": "^20.2.2" - } - } - } - }, "cli-spinners": { "version": "2.7.0", "resolved": "https://registry.npmjs.org/cli-spinners/-/cli-spinners-2.7.0.tgz", @@ -26644,12 +29869,13 @@ "version": "1.1.1", "resolved": "https://registry.npmjs.org/create-require/-/create-require-1.1.1.tgz", "integrity": "sha512-dcKFX3jn0MpIaXjisoRvexIJVEKzaq7z2rZKxf+MSr9TkdmHmsU4m2lcLojrj/FHl8mk5VxMmYA+ftRkP/3oKQ==", - "devOptional": true + "dev": true }, "cross-spawn": { "version": "7.0.3", "resolved": "https://registry.npmjs.org/cross-spawn/-/cross-spawn-7.0.3.tgz", "integrity": "sha512-iRDPJKUPVEND7dHPO8rkbOnPpyDygcDFtWjpeWNCgy8WP2rXcxXL8TskReQl6OrB2G7+UJrags1q15Fudc7G6w==", + "dev": true, "requires": { "path-key": "^3.1.0", "shebang-command": "^2.0.0", @@ -26688,11 +29914,6 @@ "resolved": "https://registry.npmjs.org/dateformat/-/dateformat-4.6.3.tgz", "integrity": "sha512-2P0p0pFGzHS5EMnhdxQi7aJN+iMheud0UhG4dlE1DLAlvL8JHjJJTX/CSm4JXwV0Ka5nGk3zC5mcb5bUQUxxMA==" }, - "dayjs": { - "version": "1.11.10", - "resolved": "https://registry.npmjs.org/dayjs/-/dayjs-1.11.10.tgz", - "integrity": "sha512-vjAczensTgRcqDERK0SR2XMwsF/tSvnvlv6VcF2GIhg6Sx4yOIt/irsr1RDJsKiIyBzJDpCoXiWWq28MqH2cnQ==" - }, "debug": { "version": "4.3.4", "resolved": "https://registry.npmjs.org/debug/-/debug-4.3.4.tgz", @@ -26815,7 +30036,7 @@ "version": "4.0.2", "resolved": "https://registry.npmjs.org/diff/-/diff-4.0.2.tgz", "integrity": "sha512-58lmxKSA4BNyLz+HHMUzlOEpg09FV+ev6ZMe3vJihgdxzgcwZ8VoEEPmALCZG9LmqfVoNMMKpttIYTVG6uDY7A==", - "devOptional": true + "dev": true }, "diff-sequences": { "version": "29.4.3", @@ -26850,11 +30071,6 @@ "is-obj": "^2.0.0" } }, - "dotenv": { - "version": "16.4.5", - "resolved": "https://registry.npmjs.org/dotenv/-/dotenv-16.4.5.tgz", - "integrity": "sha512-ZmdL2rui+eB2YwhsWzjInR8LldtZHGDoQ1ugH85ppHKwpUHL7j7rN0Ti9NCnGiQbhaZ11FpR+7ao1dNsmduNUg==" - }, "dotgitignore": { "version": "2.1.0", "resolved": "https://registry.npmjs.org/dotgitignore/-/dotgitignore-2.1.0.tgz", @@ -26915,7 +30131,8 @@ "eastasianwidth": { "version": "0.2.0", "resolved": "https://registry.npmjs.org/eastasianwidth/-/eastasianwidth-0.2.0.tgz", - "integrity": "sha512-I88TYZWc9XiYHRQ4/3c5rjjfgkjhLyW2luGIheGERbNQ6OY7yTybanSpDXZa8y7VUP9YmDcYa+eyq4ca7iLqWA==" + "integrity": "sha512-I88TYZWc9XiYHRQ4/3c5rjjfgkjhLyW2luGIheGERbNQ6OY7yTybanSpDXZa8y7VUP9YmDcYa+eyq4ca7iLqWA==", + "dev": true }, "ee-first": { "version": "1.1.1", @@ -27853,9 +31070,9 @@ "integrity": "sha512-W+KJc2dmILlPplD/H4K9l9LcAHAfPtP6BY84uVLXQ6Evcz9Lcg33Y2z1IVblT6xdY54PXYVHEv+0Wpq8Io6zkA==" }, "fast-xml-parser": { - "version": "4.4.0", - "resolved": "https://registry.npmjs.org/fast-xml-parser/-/fast-xml-parser-4.4.0.tgz", - "integrity": "sha512-kLY3jFlwIYwBNDojclKsNAC12sfD6NwW74QB2CoNGPvtVxjliYehVunB3HYyNi+n4Tt1dAcgwYvmKF/Z18flqg==", + "version": "4.4.1", + "resolved": "https://registry.npmjs.org/fast-xml-parser/-/fast-xml-parser-4.4.1.tgz", + "integrity": "sha512-xkjOecfnKGkSsOwtZ5Pz7Us/T6mrbPQrq0nh+aCO5V9nk5NLWmasAHumTKjiPJPWANe+kAZ84Jc8ooJkzZ88Sw==", "requires": { "strnum": "^1.0.5" } @@ -28081,6 +31298,7 @@ "version": "3.1.1", "resolved": "https://registry.npmjs.org/foreground-child/-/foreground-child-3.1.1.tgz", "integrity": "sha512-TMKDUnIte6bfb5nWv7V/caI169OHgvwjb7V4WkeUvbQQdjr5rWKqHFiKWb/fcOwB+CzBT+qbWjvj+DVwRskpIg==", + "dev": true, "requires": { "cross-spawn": "^7.0.0", "signal-exit": "^4.0.1" @@ -28089,7 +31307,8 @@ "signal-exit": { "version": "4.0.2", "resolved": "https://registry.npmjs.org/signal-exit/-/signal-exit-4.0.2.tgz", - "integrity": "sha512-MY2/qGx4enyjprQnFaZsHib3Yadh3IXyV2C321GY0pjGfVBu4un0uDJkwgdxqO+Rdx8JMT8IfJIRwbYVz3Ob3Q==" + "integrity": "sha512-MY2/qGx4enyjprQnFaZsHib3Yadh3IXyV2C321GY0pjGfVBu4un0uDJkwgdxqO+Rdx8JMT8IfJIRwbYVz3Ob3Q==", + "dev": true } } }, @@ -28575,11 +31794,6 @@ "integrity": "sha512-QFLV0taWQOZtvIRIAdBChesmogZrtuXvVWsFHZTk2SU+anspqZ2vMnoLg7IE1+Uk16N19APic1BuF8bC8c2m5g==", "dev": true }, - "highlight.js": { - "version": "10.7.3", - "resolved": "https://registry.npmjs.org/highlight.js/-/highlight.js-10.7.3.tgz", - "integrity": "sha512-tzcUFauisWKNHaRkN4Wjl/ZA07gENAjFl3J/c480dprkGTg5EQstgaNFqBfUqCq54kZRIEcreTsAgF/m2quD7A==" - }, "homedir-polyfill": { "version": "1.0.3", "resolved": "https://registry.npmjs.org/homedir-polyfill/-/homedir-polyfill-1.0.3.tgz", @@ -29037,7 +32251,8 @@ "isexe": { "version": "2.0.0", "resolved": "https://registry.npmjs.org/isexe/-/isexe-2.0.0.tgz", - "integrity": "sha512-RHxMLp9lnKHGHRng9QFhRCMbYAcVpn69smSGcq3f36xjgVVWThj4qqLbTLlq7Ssj8B+fIQ1EuCEGI2lKsyQeIw==" + "integrity": "sha512-RHxMLp9lnKHGHRng9QFhRCMbYAcVpn69smSGcq3f36xjgVVWThj4qqLbTLlq7Ssj8B+fIQ1EuCEGI2lKsyQeIw==", + "dev": true }, "istanbul-lib-coverage": { "version": "3.2.0", @@ -29119,6 +32334,7 @@ "version": "2.3.6", "resolved": "https://registry.npmjs.org/jackspeak/-/jackspeak-2.3.6.tgz", "integrity": "sha512-N3yCS/NegsOBokc8GAdM8UcmfsKiSS8cipheD/nivzr700H+nsMOxJjQnvwOcRYVuFkdH0wGUvW2WbXGmrZGbQ==", + "dev": true, "requires": { "@isaacs/cliui": "^8.0.2", "@pkgjs/parseargs": "^0.11.0" @@ -30925,7 +34141,7 @@ "version": "1.3.6", "resolved": "https://registry.npmjs.org/make-error/-/make-error-1.3.6.tgz", "integrity": "sha512-s8UhlNe7vPKomQhC1qFelMokr/Sc3AgNbso3n74mVPA5LTZwkB9NlXf4XPamLxJE8h0gh73rM94xvwRT2CVInw==", - "devOptional": true + "dev": true }, "makeerror": { "version": "1.0.12", @@ -31075,7 +34291,8 @@ "minipass": { "version": "6.0.2", "resolved": "https://registry.npmjs.org/minipass/-/minipass-6.0.2.tgz", - "integrity": "sha512-MzWSV5nYVT7mVyWCwn2o7JH13w2TBRmmSqSRCKzTw+lmft9X4z+3wjvs06Tzijo5z4W/kahUCDpRXTF+ZrmF/w==" + "integrity": "sha512-MzWSV5nYVT7mVyWCwn2o7JH13w2TBRmmSqSRCKzTw+lmft9X4z+3wjvs06Tzijo5z4W/kahUCDpRXTF+ZrmF/w==", + "dev": true }, "mkdirp": { "version": "0.5.6", @@ -31142,16 +34359,6 @@ "integrity": "sha512-nnbWWOkoWyUsTjKrhgD0dcz22mdkSnpYqbEjIm2nhwhuxlSkpywJmBo8h0ZqJdkp73mb90SssHkN4rsRaBAfAA==", "dev": true }, - "mz": { - "version": "2.7.0", - "resolved": "https://registry.npmjs.org/mz/-/mz-2.7.0.tgz", - "integrity": "sha512-z81GNO7nnYMEhrGh9LeymoE4+Yr0Wn5McHIZMK5cfQCl+NDX08sCZgUc9/6MHni9IWuFLm1Z3HTCXu2z9fN62Q==", - "requires": { - "any-promise": "^1.0.0", - "object-assign": "^4.0.1", - "thenify-all": "^1.0.0" - } - }, "natural-compare": { "version": "1.4.0", "resolved": "https://registry.npmjs.org/natural-compare/-/natural-compare-1.4.0.tgz", @@ -31648,26 +34855,6 @@ "integrity": "sha1-bVuTSkVpk7I9N/QKOC1vFmao5cY=", "dev": true }, - "parse5": { - "version": "5.1.1", - "resolved": "https://registry.npmjs.org/parse5/-/parse5-5.1.1.tgz", - "integrity": "sha512-ugq4DFI0Ptb+WWjAdOK16+u/nHfiIrcE+sh8kZMaM0WllQKLI9rOUq6c2b7cwPkXdzfQESqvoqK6ug7U/Yyzug==" - }, - "parse5-htmlparser2-tree-adapter": { - "version": "6.0.1", - "resolved": "https://registry.npmjs.org/parse5-htmlparser2-tree-adapter/-/parse5-htmlparser2-tree-adapter-6.0.1.tgz", - "integrity": "sha512-qPuWvbLgvDGilKc5BoicRovlT4MtYT6JfJyBOMDsKoiT+GiuP5qyrPCnR9HcPECIJJmZh5jRndyNThnhhb/vlA==", - "requires": { - "parse5": "^6.0.1" - }, - "dependencies": { - "parse5": { - "version": "6.0.1", - "resolved": "https://registry.npmjs.org/parse5/-/parse5-6.0.1.tgz", - "integrity": "sha512-Ofn/CTFzRGTTxwpNEs9PP93gXShHcTq255nzRYSKe8AkVpZY7e1fpmTfOyoIvjP5HG7Z2ZM7VS9PPhQGW2pOpw==" - } - } - }, "parseurl": { "version": "1.3.3", "resolved": "https://registry.npmjs.org/parseurl/-/parseurl-1.3.3.tgz", @@ -31688,7 +34875,8 @@ "path-key": { "version": "3.1.1", "resolved": "https://registry.npmjs.org/path-key/-/path-key-3.1.1.tgz", - "integrity": "sha512-ojmeN0qd+y0jszEtoY48r0Peq5dwMEkIlCOu6Q5f41lfkswXuKtYrhgoTpLnyIcHm24Uhqx+5Tqm2InSwLhE6Q==" + "integrity": "sha512-ojmeN0qd+y0jszEtoY48r0Peq5dwMEkIlCOu6Q5f41lfkswXuKtYrhgoTpLnyIcHm24Uhqx+5Tqm2InSwLhE6Q==", + "dev": true }, "path-parse": { "version": "1.0.7", @@ -31709,6 +34897,7 @@ "version": "1.10.2", "resolved": "https://registry.npmjs.org/path-scurry/-/path-scurry-1.10.2.tgz", "integrity": "sha512-7xTavNy5RQXnsjANvVvMkEjvloOinkAjv/Z6Ildz9v2RinZ4SBKTWFOVRbaF8p0vpHnyjV/UwNDdKuUv6M5qcA==", + "dev": true, "requires": { "lru-cache": "^10.2.0", "minipass": "^5.0.0 || ^6.0.2 || ^7.0.0" @@ -31717,7 +34906,8 @@ "lru-cache": { "version": "10.2.0", "resolved": "https://registry.npmjs.org/lru-cache/-/lru-cache-10.2.0.tgz", - "integrity": "sha512-2bIM8x+VAf6JT4bKAljS1qUWgMsqZRPGJS6FSahIMPVvctcNhyVp7AJu7quxOW9jwkryBReKZY5tY5JYv2n/7Q==" + "integrity": "sha512-2bIM8x+VAf6JT4bKAljS1qUWgMsqZRPGJS6FSahIMPVvctcNhyVp7AJu7quxOW9jwkryBReKZY5tY5JYv2n/7Q==", + "dev": true } } }, @@ -32824,19 +36014,11 @@ "resolved": "https://registry.npmjs.org/setprototypeof/-/setprototypeof-1.2.0.tgz", "integrity": "sha512-E5LDX7Wrp85Kil5bhZv46j8jOeboKq5JMmYM3gVGdGH8xFpPWXUMsNrlODCrkoxMEeNi/XZIwuRvY4XNwYMJpw==" }, - "sha.js": { - "version": "2.4.11", - "resolved": "https://registry.npmjs.org/sha.js/-/sha.js-2.4.11.tgz", - "integrity": "sha512-QMEp5B7cftE7APOjk5Y6xgrbWu+WkLVQwk8JNjZ8nKRciZaByEW6MubieAiToS7+dwvrjGhH8jRXz3MVd0AYqQ==", - "requires": { - "inherits": "^2.0.1", - "safe-buffer": "^5.0.1" - } - }, "shebang-command": { "version": "2.0.0", "resolved": "https://registry.npmjs.org/shebang-command/-/shebang-command-2.0.0.tgz", "integrity": "sha512-kHxr2zZpYtdmrN1qDjrrX/Z1rR1kG8Dx+gkpK1G4eXmvXswmcE1hTWBWYUzlraYw1/yZp6YuDY77YtvbN0dmDA==", + "dev": true, "requires": { "shebang-regex": "^3.0.0" } @@ -32844,7 +36026,8 @@ "shebang-regex": { "version": "3.0.0", "resolved": "https://registry.npmjs.org/shebang-regex/-/shebang-regex-3.0.0.tgz", - "integrity": "sha512-7++dFhtcx3353uBaq8DDR4NuxBetBzC7ZQOhmTQInHEd6bSrXdiEyzCvG07Z44UYdLShWUyXt5M/yhz8ekcb1A==" + "integrity": "sha512-7++dFhtcx3353uBaq8DDR4NuxBetBzC7ZQOhmTQInHEd6bSrXdiEyzCvG07Z44UYdLShWUyXt5M/yhz8ekcb1A==", + "dev": true }, "shimmer": { "version": "1.2.1", @@ -33124,6 +36307,7 @@ "version": "npm:string-width@4.2.3", "resolved": "https://registry.npmjs.org/string-width/-/string-width-4.2.3.tgz", "integrity": "sha512-wKyQRQpjJ0sIp62ErSZdGsjMJWsap5oRNihHhu6G7JVO/9jIB6UyevL+tXuOqrng8j/cxKTWyWUwvSTriiZz/g==", + "dev": true, "requires": { "emoji-regex": "^8.0.0", "is-fullwidth-code-point": "^3.0.0", @@ -33186,6 +36370,7 @@ "version": "npm:strip-ansi@6.0.1", "resolved": "https://registry.npmjs.org/strip-ansi/-/strip-ansi-6.0.1.tgz", "integrity": "sha512-Y38VPSHcqkFrCpFnQ9vuSXmquuv5oXOKpGeT6aGrr3o3Gc9AlVa6JBfUSOCnbxGGZF+/0ooI7KrPuUSztUdU5A==", + "dev": true, "requires": { "ansi-regex": "^5.0.1" } @@ -33326,22 +36511,6 @@ "integrity": "sha1-f17oI66AUgfACvLfSoTsP8+lcLQ=", "dev": true }, - "thenify": { - "version": "3.3.1", - "resolved": "https://registry.npmjs.org/thenify/-/thenify-3.3.1.tgz", - "integrity": "sha512-RVZSIV5IG10Hk3enotrhvz0T9em6cyHBLkH/YAZuKqd8hRkKhSfCGIcP2KUY0EPxndzANBmNllzWPwak+bheSw==", - "requires": { - "any-promise": "^1.0.0" - } - }, - "thenify-all": { - "version": "1.6.0", - "resolved": "https://registry.npmjs.org/thenify-all/-/thenify-all-1.6.0.tgz", - "integrity": "sha512-RNxQH/qI8/t3thXJDwcstUO4zeqo64+Uy/+sNVRBx4Xn2OX+OZ9oP+iJnNFqplFra2ZUVeKCSa2oVWi3T4uVmA==", - "requires": { - "thenify": ">= 3.1.0 < 4" - } - }, "thread-stream": { "version": "0.15.2", "resolved": "https://registry.npmjs.org/thread-stream/-/thread-stream-0.15.2.tgz", @@ -33452,7 +36621,7 @@ "version": "10.9.1", "resolved": "https://registry.npmjs.org/ts-node/-/ts-node-10.9.1.tgz", "integrity": "sha512-NtVysVPkxxrwFGUUxGYhfux8k78pQB3JqYBXlLRZgdGUqTO5wU/UyHop5p70iEbGhB7q5KmiZiU0Y3KlJrScEw==", - "devOptional": true, + "dev": true, "requires": { "@cspotcode/source-map-support": "^0.8.0", "@tsconfig/node10": "^1.0.7", @@ -33554,164 +36723,11 @@ "integrity": "sha1-nIRAPyMjrlOZFnJ1SXY46h0vJEA=", "dev": true }, - "typeorm": { - "version": "0.3.20", - "resolved": "https://registry.npmjs.org/typeorm/-/typeorm-0.3.20.tgz", - "integrity": "sha512-sJ0T08dV5eoZroaq9uPKBoNcGslHBR4E4y+EBHs//SiGbblGe7IeduP/IH4ddCcj0qp3PHwDwGnuvqEAnKlq/Q==", - "requires": { - "@sqltools/formatter": "^1.2.5", - "app-root-path": "^3.1.0", - "buffer": "^6.0.3", - "chalk": "^4.1.2", - "cli-highlight": "^2.1.11", - "dayjs": "^1.11.9", - "debug": "^4.3.4", - "dotenv": "^16.0.3", - "glob": "^10.3.10", - "mkdirp": "^2.1.3", - "reflect-metadata": "^0.2.1", - "sha.js": "^2.4.11", - "tslib": "^2.5.0", - "uuid": "^9.0.0", - "yargs": "^17.6.2" - }, - "dependencies": { - "ansi-styles": { - "version": "4.3.0", - "resolved": "https://registry.npmjs.org/ansi-styles/-/ansi-styles-4.3.0.tgz", - "integrity": "sha512-zbB9rCJAT1rbjiVDb2hqKFHNYLxgtk8NURxZ3IZwD3F6NtxbXZQCnnSi1Lkx+IDohdPlFp222wVALIheZJQSEg==", - "requires": { - "color-convert": "^2.0.1" - } - }, - "brace-expansion": { - "version": "2.0.1", - "resolved": "https://registry.npmjs.org/brace-expansion/-/brace-expansion-2.0.1.tgz", - "integrity": "sha512-XnAIvQ8eM+kC6aULx6wuQiwVsnzsi9d3WxzV3FpWTGA19F621kwdbsAcFKXgKUHZWsy+mY6iL1sHTxWEFCytDA==", - "requires": { - "balanced-match": "^1.0.0" - } - }, - "buffer": { - "version": "6.0.3", - "resolved": "https://registry.npmjs.org/buffer/-/buffer-6.0.3.tgz", - "integrity": "sha512-FTiCpNxtwiZZHEZbcbTIcZjERVICn9yq/pDFkTl95/AxzD1naBctN7YO68riM/gLSDY7sdrMby8hofADYuuqOA==", - "requires": { - "base64-js": "^1.3.1", - "ieee754": "^1.2.1" - } - }, - "chalk": { - "version": "4.1.2", - "resolved": "https://registry.npmjs.org/chalk/-/chalk-4.1.2.tgz", - "integrity": "sha512-oKnbhFyRIXpUuez8iBMmyEa4nbj4IOQyuhc/wy9kY7/WVPcwIO9VA668Pu8RkO7+0G76SLROeyw9CpQ061i4mA==", - "requires": { - "ansi-styles": "^4.1.0", - "supports-color": "^7.1.0" - } - }, - "cliui": { - "version": "8.0.1", - "resolved": "https://registry.npmjs.org/cliui/-/cliui-8.0.1.tgz", - "integrity": "sha512-BSeNnyus75C4//NQ9gQt1/csTXyo/8Sb+afLAkzAptFuMsod9HFokGNudZpi/oQV73hnVK+sR+5PVRMd+Dr7YQ==", - "requires": { - "string-width": "^4.2.0", - "strip-ansi": "^6.0.1", - "wrap-ansi": "^7.0.0" - } - }, - "color-convert": { - "version": "2.0.1", - "resolved": "https://registry.npmjs.org/color-convert/-/color-convert-2.0.1.tgz", - "integrity": "sha512-RRECPsj7iu/xb5oKYcsFHSppFNnsj/52OVTRKb4zP5onXwVF3zVmmToNcOfGC+CRDpfK/U584fMg38ZHCaElKQ==", - "requires": { - "color-name": "~1.1.4" - } - }, - "color-name": { - "version": "1.1.4", - "resolved": "https://registry.npmjs.org/color-name/-/color-name-1.1.4.tgz", - "integrity": "sha512-dOy+3AuW3a2wNbZHIuMZpTcgjGuLU/uBL/ubcZF9OXbDo8ff4O8yVp5Bf0efS8uEoYo5q4Fx7dY9OgQGXgAsQA==" - }, - "glob": { - "version": "10.3.12", - "resolved": "https://registry.npmjs.org/glob/-/glob-10.3.12.tgz", - "integrity": "sha512-TCNv8vJ+xz4QiqTpfOJA7HvYv+tNIRHKfUWw/q+v2jdgN4ebz+KY9tGx5J4rHP0o84mNP+ApH66HRX8us3Khqg==", - "requires": { - "foreground-child": "^3.1.0", - "jackspeak": "^2.3.6", - "minimatch": "^9.0.1", - "minipass": "^7.0.4", - "path-scurry": "^1.10.2" - } - }, - "has-flag": { - "version": "4.0.0", - "resolved": "https://registry.npmjs.org/has-flag/-/has-flag-4.0.0.tgz", - "integrity": "sha512-EykJT/Q1KjTWctppgIAgfSO0tKVuZUjhgMr17kqTumMl6Afv3EISleU7qZUzoXDFTAHTDC4NOoG/ZxU3EvlMPQ==" - }, - "minimatch": { - "version": "9.0.4", - "resolved": "https://registry.npmjs.org/minimatch/-/minimatch-9.0.4.tgz", - "integrity": "sha512-KqWh+VchfxcMNRAJjj2tnsSJdNbHsVgnkBhTNrW7AjVo6OvLtxw8zfT9oLw1JSohlFzJ8jCoTgaoXvJ+kHt6fw==", - "requires": { - "brace-expansion": "^2.0.1" - } - }, - "minipass": { - "version": "7.0.4", - "resolved": "https://registry.npmjs.org/minipass/-/minipass-7.0.4.tgz", - "integrity": "sha512-jYofLM5Dam9279rdkWzqHozUo4ybjdZmCsDHePy5V/PbBcVMiSZR97gmAy45aqi8CK1lG2ECd356FU86avfwUQ==" - }, - "mkdirp": { - "version": "2.1.6", - "resolved": "https://registry.npmjs.org/mkdirp/-/mkdirp-2.1.6.tgz", - "integrity": "sha512-+hEnITedc8LAtIP9u3HJDFIdcLV2vXP33sqLLIzkv1Db1zO/1OxbvYf0Y1OC/S/Qo5dxHXepofhmxL02PsKe+A==" - }, - "reflect-metadata": { - "version": "0.2.2", - "resolved": "https://registry.npmjs.org/reflect-metadata/-/reflect-metadata-0.2.2.tgz", - "integrity": "sha512-urBwgfrvVP/eAyXx4hluJivBKzuEbSQs9rKWCrCkbSxNv8mxPcUZKeuoF3Uy4mJl3Lwprp6yy5/39VWigZ4K6Q==" - }, - "supports-color": { - "version": "7.2.0", - "resolved": "https://registry.npmjs.org/supports-color/-/supports-color-7.2.0.tgz", - "integrity": "sha512-qpCAvRl9stuOHveKsn7HncJRvv501qIacKzQlO/+Lwxc9+0q2wLyv4Dfvt80/DPn2pqOBsJdDiogXGR9+OvwRw==", - "requires": { - "has-flag": "^4.0.0" - } - }, - "tslib": { - "version": "2.6.2", - "resolved": "https://registry.npmjs.org/tslib/-/tslib-2.6.2.tgz", - "integrity": "sha512-AEYxH93jGFPn/a2iVAwW87VuUIkR1FVUKB77NwMF7nBTDkDrrT/Hpt/IrCJ0QXhW27jTBDcf5ZY7w6RiqTMw2Q==" - }, - "yargs": { - "version": "17.7.2", - "resolved": "https://registry.npmjs.org/yargs/-/yargs-17.7.2.tgz", - "integrity": "sha512-7dSzzRQ++CKnNI/krKnYRV7JKKPUXMEh61soaHKg9mrWEhzFWhFnxPxGl+69cD1Ou63C13NUPCnmIcrvqCuM6w==", - "requires": { - "cliui": "^8.0.1", - "escalade": "^3.1.1", - "get-caller-file": "^2.0.5", - "require-directory": "^2.1.1", - "string-width": "^4.2.3", - "y18n": "^5.0.5", - "yargs-parser": "^21.1.1" - } - }, - "yargs-parser": { - "version": "21.1.1", - "resolved": "https://registry.npmjs.org/yargs-parser/-/yargs-parser-21.1.1.tgz", - "integrity": "sha512-tVpsJW7DdjecAiFpbIB1e3qxIQsE6NoPc5/eTdrbbIC4h0LVsWhnoa3g+m2HclBIujHzsxZ4VJVA+GUuc2/LBw==" - } - } - }, "typescript": { "version": "5.1.3", "resolved": "https://registry.npmjs.org/typescript/-/typescript-5.1.3.tgz", "integrity": "sha512-XH627E9vkeqhlZFQuL+UsyAXEnibT0kWR2FWONlr4sTjvxyJYnyefgrkyECLzM5NenmKzRAy2rR/OlYLA1HkZw==", - "devOptional": true + "dev": true }, "uglify-js": { "version": "3.15.5", @@ -33810,6 +36826,11 @@ "resolved": "https://registry.npmjs.org/utils-merge/-/utils-merge-1.0.1.tgz", "integrity": "sha1-n5VxD1CiZ5R7LMwSR0HBAoQn5xM=" }, + "utm-latlng": { + "version": "1.0.8", + "resolved": "https://registry.npmjs.org/utm-latlng/-/utm-latlng-1.0.8.tgz", + "integrity": "sha512-jQ5l1BNTzNgas7jXOIh0jU0m5PU5LtmjYvRiLwR4iymYYAhEhQ6N0MGCnLjv5iO958b9tLMMSmzdbIIhLtXqCg==" + }, "uuid": { "version": "9.0.1", "resolved": "https://registry.npmjs.org/uuid/-/uuid-9.0.1.tgz", @@ -33819,7 +36840,7 @@ "version": "3.0.1", "resolved": "https://registry.npmjs.org/v8-compile-cache-lib/-/v8-compile-cache-lib-3.0.1.tgz", "integrity": "sha512-wa7YjyUGfNZngI/vtK0UHAN+lgDCxBPCylVXGp0zu59Fz5aiGtNXaq3DhIov063MorB+VfufLh3JlF2KdTK3xg==", - "devOptional": true + "dev": true }, "v8-to-istanbul": { "version": "9.1.0", @@ -33934,6 +36955,7 @@ "version": "2.0.2", "resolved": "https://registry.npmjs.org/which/-/which-2.0.2.tgz", "integrity": "sha512-BLI3Tl1TW3Pvl70l3yq3Y64i+awpwXqsGBYWkkqMtnbXgrMD+yj7rhW0kuEDxzJaYXGjEW5ogapKNMEKNMjibA==", + "dev": true, "requires": { "isexe": "^2.0.0" } @@ -34005,6 +37027,7 @@ "version": "npm:wrap-ansi@7.0.0", "resolved": "https://registry.npmjs.org/wrap-ansi/-/wrap-ansi-7.0.0.tgz", "integrity": "sha512-YVGIj2kamLSTxw6NsZjoBxfSwsn0ycdesmc4p+Q21c5zPuZ1pl+NfxVdxPtdHvmNVOQ6XSYG4AUtyt/Fi7D16Q==", + "dev": true, "requires": { "ansi-styles": "^4.0.0", "string-width": "^4.1.0", @@ -34015,6 +37038,7 @@ "version": "4.3.0", "resolved": "https://registry.npmjs.org/ansi-styles/-/ansi-styles-4.3.0.tgz", "integrity": "sha512-zbB9rCJAT1rbjiVDb2hqKFHNYLxgtk8NURxZ3IZwD3F6NtxbXZQCnnSi1Lkx+IDohdPlFp222wVALIheZJQSEg==", + "dev": true, "requires": { "color-convert": "^2.0.1" } @@ -34023,6 +37047,7 @@ "version": "2.0.1", "resolved": "https://registry.npmjs.org/color-convert/-/color-convert-2.0.1.tgz", "integrity": "sha512-RRECPsj7iu/xb5oKYcsFHSppFNnsj/52OVTRKb4zP5onXwVF3zVmmToNcOfGC+CRDpfK/U584fMg38ZHCaElKQ==", + "dev": true, "requires": { "color-name": "~1.1.4" } @@ -34030,7 +37055,8 @@ "color-name": { "version": "1.1.4", "resolved": "https://registry.npmjs.org/color-name/-/color-name-1.1.4.tgz", - "integrity": "sha512-dOy+3AuW3a2wNbZHIuMZpTcgjGuLU/uBL/ubcZF9OXbDo8ff4O8yVp5Bf0efS8uEoYo5q4Fx7dY9OgQGXgAsQA==" + "integrity": "sha512-dOy+3AuW3a2wNbZHIuMZpTcgjGuLU/uBL/ubcZF9OXbDo8ff4O8yVp5Bf0efS8uEoYo5q4Fx7dY9OgQGXgAsQA==", + "dev": true } } }, @@ -34115,7 +37141,7 @@ "version": "3.1.1", "resolved": "https://registry.npmjs.org/yn/-/yn-3.1.1.tgz", "integrity": "sha512-Ux4ygGWsu2c7isFWe8Yu1YluJmqVhxqK2cLXNQA5AcC3QfbGNpM7fu0Y8b/z16pXLnFxZYvWhd3fhBY9DLmC6Q==", - "devOptional": true + "dev": true }, "yocto-queue": { "version": "0.1.0", diff --git a/package.json b/package.json index 23fc3403..7348f928 100644 --- a/package.json +++ b/package.json @@ -46,6 +46,7 @@ } }, "dependencies": { + "@aws-sdk/client-s3": "^3.637.0", "@elastic/elasticsearch": "^8.13.1", "@godaddy/terminus": "^4.12.1", "@map-colonies/error-express-handler": "^2.1.0", @@ -58,6 +59,7 @@ "@opentelemetry/api-metrics": "0.23.0", "@opentelemetry/instrumentation-express": "0.32.1", "@opentelemetry/instrumentation-http": "0.35.1", + "@smithy/node-http-handler": "^3.1.4", "axios": "^0.21.1", "compression": "^1.7.4", "config": "^3.3.9", @@ -73,7 +75,7 @@ "proj4": "^2.11.0", "reflect-metadata": "^0.1.13", "tsyringe": "^4.8.0", - "typeorm": "^0.3.20", + "utm-latlng": "^1.0.8", "wellknown": "^0.5.0" }, "devDependencies": { From 229b1c52d8f0c4da9273342e28401aa8ee4eff73 Mon Sep 17 00:00:00 2001 From: Niv Greenstein Date: Wed, 4 Sep 2024 16:05:03 +0300 Subject: [PATCH 179/262] fix: require target_grid in openapi --- openapi3.yaml | 1 + 1 file changed, 1 insertion(+) diff --git a/openapi3.yaml b/openapi3.yaml index f3b5cc9e..d8f9cbb1 100644 --- a/openapi3.yaml +++ b/openapi3.yaml @@ -387,6 +387,7 @@ paths: schema: type: string enum: ["control", "MGRS"] + required: true responses: 200: description: "OK" From eeefd86ca9d4b09e543a2b98032b308f71a225c9 Mon Sep 17 00:00:00 2001 From: Niv Greenstein Date: Wed, 4 Sep 2024 16:12:33 +0300 Subject: [PATCH 180/262] added todo comment --- src/common/utils.ts | 1 + 1 file changed, 1 insertion(+) diff --git a/src/common/utils.ts b/src/common/utils.ts index dceab4c3..0902a7b9 100644 --- a/src/common/utils.ts +++ b/src/common/utils.ts @@ -66,6 +66,7 @@ export const convertWgs84ToUTM = ( /* eslint-enable @typescript-eslint/naming-convention */ export const convertUTMToWgs84 = (x: number, y: number, zone: number): WGS84Coordinate => { + //TODO: change ZONE Letter to relevent letter. Currently it is hardcoded to 'N' const { lat, lng: lon } = new utm().convertUtmToLatLng(x, y, zone, 'N') as { lat: number; lng: number }; return { lat, lon }; }; From 852c0c40e9fa0d9a0af7757792d600abdd3b0e3e Mon Sep 17 00:00:00 2001 From: Niv Greenstein Date: Wed, 4 Sep 2024 16:40:45 +0300 Subject: [PATCH 181/262] delete: removed postgres references --- devScripts/importToPostgres.ts | 47 ------------------------------ src/latLon/models/latLonManager.ts | 6 +--- 2 files changed, 1 insertion(+), 52 deletions(-) delete mode 100644 devScripts/importToPostgres.ts diff --git a/devScripts/importToPostgres.ts b/devScripts/importToPostgres.ts deleted file mode 100644 index 9e81b7a6..00000000 --- a/devScripts/importToPostgres.ts +++ /dev/null @@ -1,47 +0,0 @@ -import { DataSource } from 'typeorm'; -import { createConnectionOptions } from '../src/common/postgresql'; -import config from '../config/test.json'; -import { PostgresDbConfig } from '../src/common/interfaces'; - -export default async function createDatabaseClient(): Promise { - const connectionOptions = config.db.postgresql as PostgresDbConfig; - const connection = new DataSource({ - ...createConnectionOptions(connectionOptions), - }); - - await connection.initialize(); - - await connection.query(`CREATE SCHEMA IF NOT EXISTS ${config.db.postgresql.schema};`); - - await connection.query(` - CREATE TABLE IF NOT EXISTS ${config.db.postgresql.schema}.tile_lat_lon - ( - pk integer NOT NULL, - tile_name text COLLATE pg_catalog."default", - zone text COLLATE pg_catalog."default", - min_x text COLLATE pg_catalog."default", - min_y text COLLATE pg_catalog."default", - ext_min_x integer, - ext_min_y integer, - ext_max_x integer, - ext_max_y integer, - CONSTRAINT tile_lat_lon_pkey PRIMARY KEY (pk) - ) - `); - - await connection.query(` - INSERT INTO ${config.db.postgresql.schema}.tile_lat_lon( - pk, tile_name, zone, min_x, min_y, ext_min_x, ext_min_y, ext_max_x, ext_max_y) - VALUES (1, 'BRN', '33', '360000', '5820000', 360000, 5820000, 370000, 5830000); - `); - - await connection.query(` - INSERT INTO ${config.db.postgresql.schema}.tile_lat_lon( - pk, tile_name, zone, min_x, min_y, ext_min_x, ext_min_y, ext_max_x, ext_max_y) - VALUES (2, 'BMN', '32', '480000', '5880000', 480000, 5880000, 490000, 5890000); - `); - - console.log('PGSQL: Table created and data inserted'); - - await connection.destroy(); -} diff --git a/src/latLon/models/latLonManager.ts b/src/latLon/models/latLonManager.ts index 8d189417..184bbd1a 100644 --- a/src/latLon/models/latLonManager.ts +++ b/src/latLon/models/latLonManager.ts @@ -14,15 +14,11 @@ import { parseGeo } from '../../location/utils'; @injectable() export class LatLonManager { - private readonly dbSchema: string; - public constructor( @inject(SERVICES.LOGGER) private readonly logger: Logger, @inject(LatLonDAL) private readonly latLonDAL: LatLonDAL, @inject(SERVICES.CONFIG) private readonly config: IConfig - ) { - this.dbSchema = this.config.get('db.postgresql.schema'); - } + ) {} public async latLonToTile({ lat, lon }: WGS84Coordinate): Promise<{ [key: string]: unknown } & Feature> { if (!validateWGS84Coordinate({ lat, lon })) { From 91ff30f1d94b1b23be1204bbf5b0fa431a4f1775 Mon Sep 17 00:00:00 2001 From: Niv Greenstein Date: Wed, 4 Sep 2024 16:41:20 +0300 Subject: [PATCH 182/262] fix: inject noting to s3 in order not to initialize it --- tests/integration/control/item/item.spec.ts | 8 +++----- tests/integration/control/route/route.spec.ts | 8 +++----- tests/integration/control/tile/tile.spec.ts | 8 +++----- tests/integration/docs/docs.spec.ts | 5 +++++ tests/integration/location/location.spec.ts | 8 +++----- 5 files changed, 17 insertions(+), 20 deletions(-) diff --git a/tests/integration/control/item/item.spec.ts b/tests/integration/control/item/item.spec.ts index 39e3659b..70d00a39 100644 --- a/tests/integration/control/item/item.spec.ts +++ b/tests/integration/control/item/item.spec.ts @@ -2,15 +2,14 @@ import jsLogger from '@map-colonies/js-logger'; import { trace } from '@opentelemetry/api'; import httpStatusCodes from 'http-status-codes'; -import { DataSource } from 'typeorm'; import { getApp } from '../../../../src/app'; import { SERVICES } from '../../../../src/common/constants'; import { GetItemsQueryParams } from '../../../../src/control/item/controllers/itemController'; import { Item } from '../../../../src/control/item/models/item'; import { ControlResponse } from '../../../../src/control/interfaces'; import { CommonRequestParameters, GeoContext, GeoContextMode } from '../../../../src/common/interfaces'; -import { LATLON_CUSTOM_REPOSITORY_SYMBOL } from '../../../../src/latLon/DAL/latLonRepository'; import { cronLoadTileLatLonDataSymbol } from '../../../../src/latLon/DAL/latLonDAL'; +import { S3_REPOSITORY_SYMBOL } from '../../../../src/common/s3/s3Repository'; import { expectedResponse } from '../utils'; import { ItemRequestSender } from './helpers/requestSender'; import { ITEM_1234, ITEM_1235, ITEM_1236 } from './mockObjects'; @@ -23,10 +22,9 @@ describe('/search/control/items', function () { override: [ { token: SERVICES.LOGGER, provider: { useValue: jsLogger({ enabled: false }) } }, { token: SERVICES.TRACER, provider: { useValue: trace.getTracer('testTracer') } }, - { token: LATLON_CUSTOM_REPOSITORY_SYMBOL, provider: { useValue: {} } }, - { token: DataSource, provider: { useValue: {} } }, + { token: S3_REPOSITORY_SYMBOL, provider: { useValue: {} } }, + { token: SERVICES.S3_CLIENT, provider: { useValue: {} } }, { token: cronLoadTileLatLonDataSymbol, provider: { useValue: {} } }, - { token: LATLON_CUSTOM_REPOSITORY_SYMBOL, provider: { useValue: {} } }, ], useChild: true, }); diff --git a/tests/integration/control/route/route.spec.ts b/tests/integration/control/route/route.spec.ts index fe95f22e..2e1dcc6c 100644 --- a/tests/integration/control/route/route.spec.ts +++ b/tests/integration/control/route/route.spec.ts @@ -2,14 +2,13 @@ import jsLogger from '@map-colonies/js-logger'; import { trace } from '@opentelemetry/api'; import httpStatusCodes from 'http-status-codes'; -import { DataSource } from 'typeorm'; import { getApp } from '../../../../src/app'; import { SERVICES } from '../../../../src/common/constants'; import { GetRoutesQueryParams } from '../../../../src/control/route/controllers/routeController'; import { Route } from '../../../../src/control/route/models/route'; import { ControlResponse } from '../../../../src/control/interfaces'; import { CommonRequestParameters, GeoContext, GeoContextMode } from '../../../../src/common/interfaces'; -import { LATLON_CUSTOM_REPOSITORY_SYMBOL } from '../../../../src/latLon/DAL/latLonRepository'; +import { S3_REPOSITORY_SYMBOL } from '../../../../src/common/s3/s3Repository'; import { cronLoadTileLatLonDataSymbol } from '../../../../src/latLon/DAL/latLonDAL'; import { expectedResponse } from '../utils'; import { RouteRequestSender } from './helpers/requestSender'; @@ -23,10 +22,9 @@ describe('/search/control/route', function () { override: [ { token: SERVICES.LOGGER, provider: { useValue: jsLogger({ enabled: false }) } }, { token: SERVICES.TRACER, provider: { useValue: trace.getTracer('testTracer') } }, - { token: LATLON_CUSTOM_REPOSITORY_SYMBOL, provider: { useValue: {} } }, - { token: DataSource, provider: { useValue: {} } }, + { token: S3_REPOSITORY_SYMBOL, provider: { useValue: {} } }, + { token: SERVICES.S3_CLIENT, provider: { useValue: {} } }, { token: cronLoadTileLatLonDataSymbol, provider: { useValue: {} } }, - { token: LATLON_CUSTOM_REPOSITORY_SYMBOL, provider: { useValue: {} } }, ], useChild: true, }); diff --git a/tests/integration/control/tile/tile.spec.ts b/tests/integration/control/tile/tile.spec.ts index 44685fcc..5f2e679c 100644 --- a/tests/integration/control/tile/tile.spec.ts +++ b/tests/integration/control/tile/tile.spec.ts @@ -2,14 +2,13 @@ import jsLogger from '@map-colonies/js-logger'; import { trace } from '@opentelemetry/api'; import httpStatusCodes from 'http-status-codes'; -import { DataSource } from 'typeorm'; import { getApp } from '../../../../src/app'; import { SERVICES } from '../../../../src/common/constants'; import { GetTilesQueryParams } from '../../../../src/control/tile/controllers/tileController'; import { Tile } from '../../../../src/control/tile/models/tile'; import { ControlResponse } from '../../../../src/control/interfaces'; import { CommonRequestParameters, GeoContext, GeoContextMode } from '../../../../src/common/interfaces'; -import { LATLON_CUSTOM_REPOSITORY_SYMBOL } from '../../../../src/latLon/DAL/latLonRepository'; +import { S3_REPOSITORY_SYMBOL } from '../../../../src/common/s3/s3Repository'; import { cronLoadTileLatLonDataSymbol } from '../../../../src/latLon/DAL/latLonDAL'; import { expectedResponse } from '../utils'; import { TileRequestSender } from './helpers/requestSender'; @@ -23,10 +22,9 @@ describe('/search/control/tiles', function () { override: [ { token: SERVICES.LOGGER, provider: { useValue: jsLogger({ enabled: false }) } }, { token: SERVICES.TRACER, provider: { useValue: trace.getTracer('testTracer') } }, - { token: LATLON_CUSTOM_REPOSITORY_SYMBOL, provider: { useValue: {} } }, - { token: DataSource, provider: { useValue: {} } }, + { token: S3_REPOSITORY_SYMBOL, provider: { useValue: {} } }, + { token: SERVICES.S3_CLIENT, provider: { useValue: {} } }, { token: cronLoadTileLatLonDataSymbol, provider: { useValue: {} } }, - { token: LATLON_CUSTOM_REPOSITORY_SYMBOL, provider: { useValue: {} } }, ], useChild: true, }); diff --git a/tests/integration/docs/docs.spec.ts b/tests/integration/docs/docs.spec.ts index 88f06840..e818ec69 100644 --- a/tests/integration/docs/docs.spec.ts +++ b/tests/integration/docs/docs.spec.ts @@ -3,6 +3,8 @@ import { trace } from '@opentelemetry/api'; import httpStatusCodes from 'http-status-codes'; import { getApp } from '../../../src/app'; import { SERVICES } from '../../../src/common/constants'; +import { S3_REPOSITORY_SYMBOL } from '../../../src/common/s3/s3Repository'; +import { cronLoadTileLatLonDataSymbol } from '../../../src/latLon/DAL/latLonDAL'; import { DocsRequestSender } from './helpers/docsRequestSender'; describe('docs', function () { @@ -12,6 +14,9 @@ describe('docs', function () { override: [ { token: SERVICES.LOGGER, provider: { useValue: jsLogger({ enabled: false }) } }, { token: SERVICES.TRACER, provider: { useValue: trace.getTracer('testTracer') } }, + { token: S3_REPOSITORY_SYMBOL, provider: { useValue: {} } }, + { token: SERVICES.S3_CLIENT, provider: { useValue: {} } }, + { token: cronLoadTileLatLonDataSymbol, provider: { useValue: {} } }, ], useChild: true, }); diff --git a/tests/integration/location/location.spec.ts b/tests/integration/location/location.spec.ts index 77427c8c..ec542d62 100644 --- a/tests/integration/location/location.spec.ts +++ b/tests/integration/location/location.spec.ts @@ -3,11 +3,10 @@ import config from 'config'; import jsLogger from '@map-colonies/js-logger'; import { trace } from '@opentelemetry/api'; import httpStatusCodes from 'http-status-codes'; -import { DataSource } from 'typeorm'; import nock, { Body } from 'nock'; import { getApp } from '../../../src/app'; import { SERVICES } from '../../../src/common/constants'; -import { LATLON_CUSTOM_REPOSITORY_SYMBOL } from '../../../src/latLon/DAL/latLonRepository'; +import { S3_REPOSITORY_SYMBOL } from '../../../src/common/s3/s3Repository'; import { cronLoadTileLatLonDataSymbol } from '../../../src/latLon/DAL/latLonDAL'; import { GetGeotextSearchParams, QueryResult } from '../../../src/location/interfaces'; import { GeoContextMode, IApplication } from '../../../src/common/interfaces'; @@ -35,10 +34,9 @@ describe('/search/control/tiles', function () { override: [ { token: SERVICES.LOGGER, provider: { useValue: jsLogger({ enabled: false }) } }, { token: SERVICES.TRACER, provider: { useValue: trace.getTracer('testTracer') } }, - { token: LATLON_CUSTOM_REPOSITORY_SYMBOL, provider: { useValue: {} } }, - { token: DataSource, provider: { useValue: {} } }, + { token: S3_REPOSITORY_SYMBOL, provider: { useValue: {} } }, + { token: SERVICES.S3_CLIENT, provider: { useValue: {} } }, { token: cronLoadTileLatLonDataSymbol, provider: { useValue: {} } }, - { token: LATLON_CUSTOM_REPOSITORY_SYMBOL, provider: { useValue: {} } }, ], useChild: true, }); From c47d07f3eb9cada2d90654c853fe6acad04b6491 Mon Sep 17 00:00:00 2001 From: Niv Greenstein Date: Wed, 4 Sep 2024 17:34:46 +0300 Subject: [PATCH 183/262] delete: removed unused code --- src/latLon/controllers/latLonController.ts | 45 --------- src/latLon/routes/latLonRouter.ts | 4 - src/latLon/utlis/index.ts | 101 --------------------- 3 files changed, 150 deletions(-) delete mode 100644 src/latLon/utlis/index.ts diff --git a/src/latLon/controllers/latLonController.ts b/src/latLon/controllers/latLonController.ts index 85175bb8..0df87b0f 100644 --- a/src/latLon/controllers/latLonController.ts +++ b/src/latLon/controllers/latLonController.ts @@ -7,19 +7,13 @@ import httpStatus from 'http-status-codes'; import { injectable, inject } from 'tsyringe'; import { SERVICES } from '../../common/constants'; import { LatLonManager } from '../models/latLonManager'; -import { Tile } from '../../control/tile/models/tile'; import { WGS84Coordinate } from '../../common/interfaces'; -import { ControlResponse } from '../../control/interfaces'; /* istanbul ignore file */ type GetLatLonToTileHandler = RequestHandler; -type GetTileToLatLonHandler = RequestHandler, undefined, GetTileToLatLonQueryParams>; - type GetLatLonToMgrsHandler = RequestHandler; -type GetMgrsToLatLonHandler = RequestHandler; - type GetCoordinatesHandler = RequestHandler< undefined, { [key: string]: unknown } & Feature, @@ -29,17 +23,9 @@ type GetCoordinatesHandler = RequestHandler< export interface GetLatLonToTileQueryParams extends WGS84Coordinate {} -export interface GetTileToLatLonQueryParams { - tile: string; - sub_tile_number: number[]; -} - export interface GetLatLonToMgrsQueryParams extends WGS84Coordinate { accuracy?: number; } -export interface GetMgrsToLatLonQueryParams { - mgrs: string; -} @injectable() export class LatLonController { @@ -65,24 +51,6 @@ export class LatLonController { } }; - public tileToLatLon: GetTileToLatLonHandler = async (req, res, next) => { - try { - const { tile: tileName, sub_tile_number } = req.query; - - const response = await this.manager.tileToLatLon({ - tileName, - subTileNumber: sub_tile_number, - }); - - // TODO: REMOVE TS IGNORE - //@ts-ignore - return res.status(httpStatus.OK).json(response); - } catch (error: unknown) { - this.logger.warn('latLonController.tileToLatLon Error:', error); - next(error); - } - }; - public latlonToMgrs: GetLatLonToMgrsHandler = (req, res, next) => { try { const { lat, lon, accuracy } = req.query; @@ -96,19 +64,6 @@ export class LatLonController { } }; - public mgrsToLatlon: GetMgrsToLatLonHandler = (req, res, next) => { - try { - const { mgrs } = req.query; - - const response = this.manager.mgrsToLatLon(mgrs); - - return res.status(httpStatus.OK).json(response); - } catch (error: unknown) { - this.logger.warn('latLonController.mgrsToLatlon Error:', error); - next(error); - } - }; - public getCoordinates: GetCoordinatesHandler = async (req, res, next) => { try { const { lat, lon, target_grid } = req.query; diff --git a/src/latLon/routes/latLonRouter.ts b/src/latLon/routes/latLonRouter.ts index cd4ada62..517178d0 100644 --- a/src/latLon/routes/latLonRouter.ts +++ b/src/latLon/routes/latLonRouter.ts @@ -6,10 +6,6 @@ const latLonRouterFactory: FactoryFunction = (dependencyContainer) => { const router = Router(); const controller = dependencyContainer.resolve(LatLonController); - router.get('/latlonToTile', controller.latlonToTile); - router.get('/tileToLatLon', controller.tileToLatLon); - router.get('/latlonToMgrs', controller.latlonToMgrs); - router.get('/mgrsToLatlon', controller.mgrsToLatlon); router.get('/coordinates', controller.getCoordinates); return router; diff --git a/src/latLon/utlis/index.ts b/src/latLon/utlis/index.ts deleted file mode 100644 index 617f97d2..00000000 --- a/src/latLon/utlis/index.ts +++ /dev/null @@ -1,101 +0,0 @@ -/* istanbul ignore file */ -import { Polygon } from 'geojson'; -import { BadRequestError } from '../../common/errors'; -import { ConvertCamelToSnakeCase, convertUTMToWgs84 } from '../../common/utils'; -import { FeatureCollection } from '../../common/interfaces'; -import { Tile } from '../../control/tile/models/tile'; -import { LatLon } from '../models/latLon'; - -/* eslint-disable @typescript-eslint/naming-convention */ -const geoJsonObjectTemplate = (): FeatureCollection => ({ - type: 'FeatureCollection', - features: [ - { - type: 'Feature', - geometry: { - type: 'Polygon', - coordinates: [[]], - }, - properties: { - TYPE: 'TILE', - }, - }, - ], -}); -/* eslint-enable @typescript-eslint/naming-convention */ - -export const convertTilesToUTM = ( - tile: { tileName: string; subTileNumber: number[] }, - tileObject: ConvertCamelToSnakeCase -): { - x: number; - y: number; - zone: number; -} => { - const xCoordinate = parseInt(tileObject.min_x); - const yCoordinate = parseInt(tileObject.min_y); - - const xCoordinatePart = tile.subTileNumber - .map((x) => { - return x.toString()[0]; - }) - .join(''); - const yCoordinatePart = tile.subTileNumber - .map(function (y) { - return y.toString()[1]; - }) - .join(''); - - return { x: xCoordinate + parseInt(xCoordinatePart) * 10, y: yCoordinate + parseInt(yCoordinatePart) * 10, zone: parseInt(tileObject.zone) }; -}; - -export const validateResult = ( - tile: ConvertCamelToSnakeCase, - utmCoor: { - x: number; - y: number; - zone: number; - } -): void => { - if (tile.ext_min_x > utmCoor.x || tile.ext_max_x < utmCoor.x || tile.ext_min_y > utmCoor.y || tile.ext_max_y < utmCoor.y) { - throw new BadRequestError("Tile is found, sub tile is not in tile's extent"); - } -}; - -export const getSubTileByBottomLeftUtmCoor = ( - utmCoor: { - x: number; - y: number; - zone: number; - }, - tile: { tileName: string; subTileNumber: number[] } -): FeatureCollection => { - const result = geoJsonObjectTemplate(); - - const multiplyByOrder = [ - [0, 0], - [1, 0], - [1, 1], - [0, 1], - [0, 0], - ]; // bottom left -> bottom right -> top right -> top left -> bottom left - const distance = 10; // 10 meters - const polygon: Polygon = { - type: 'Polygon', - coordinates: [[]], - }; - - for (const multiply of multiplyByOrder) { - const coordiante = convertUTMToWgs84(utmCoor.x + distance * multiply[0], utmCoor.y + distance * multiply[1], utmCoor.zone); - if (typeof coordiante === 'string') { - throw new Error('coordinate is string'); - } - polygon.coordinates[0].push([coordiante.lon, coordiante.lat]); - } - - result.features[0].geometry = polygon; - // eslint-disable-next-line @typescript-eslint/naming-convention - result.features[0].properties = { ...result.features[0].properties, TILE_NAME: tile.tileName, SUB_TILE_NUMBER: tile.subTileNumber }; - - return result; -}; From 258b6de2ec910b1d7c68aa198287266af3c5fdf6 Mon Sep 17 00:00:00 2001 From: Niv Greenstein Date: Wed, 4 Sep 2024 17:35:09 +0300 Subject: [PATCH 184/262] feat: added healthCheckFactory. removed unused validateTile --- src/common/utils.ts | 31 +++++++++++++++++++++---------- 1 file changed, 21 insertions(+), 10 deletions(-) diff --git a/src/common/utils.ts b/src/common/utils.ts index 0902a7b9..8c45db35 100644 --- a/src/common/utils.ts +++ b/src/common/utils.ts @@ -1,5 +1,10 @@ import utm from 'utm-latlng'; +import { ListBucketsCommand, S3Client } from '@aws-sdk/client-s3'; +import { DependencyContainer, FactoryFunction } from 'tsyringe'; +import { Logger } from '@map-colonies/js-logger'; import { WGS84Coordinate } from './interfaces'; +import { SERVICES } from './constants'; +import { ElasticClients } from './elastic'; type SnakeToCamelCase = S extends `${infer T}_${infer U}` ? `${T}${Capitalize>}` : S; @@ -71,16 +76,22 @@ export const convertUTMToWgs84 = (x: number, y: number, zone: number): WGS84Coor return { lat, lon }; }; -export const validateTile = (tile: { tileName: string; subTileNumber: number[] }): boolean => { - if (!tile.tileName || !Array.isArray(tile.subTileNumber) || tile.subTileNumber.length !== 3) { - return false; - } - //regex = /^-?d+$/; - const regex = /^(\d\d)$/; - for (const subTileNumber of tile.subTileNumber) { - if (!regex.test(`${subTileNumber}`)) { - return false; +export const healthCheckFactory: FactoryFunction = (container: DependencyContainer): void => { + const logger = container.resolve(SERVICES.LOGGER); + const elasticClients = container.resolve(SERVICES.ELASTIC_CLIENTS); + const s3Client = container.resolve(SERVICES.S3_CLIENT); + logger.info('Healthcheck is running'); + + try { + for (const [key, client] of Object.entries(elasticClients)) { + logger.info(`Checking health of ${key}`); + void client.cluster.health({}); } + + void s3Client.send(new ListBucketsCommand({})); + + logger.info('healthcheck passed'); + } catch (error) { + logger.error(`Healthcheck failed. Error: ${(error as Error).message}`); } - return true; }; From b70bb9200e268803bd0997aad7f924ddf1efaf4e Mon Sep 17 00:00:00 2001 From: Niv Greenstein Date: Wed, 4 Sep 2024 17:35:30 +0300 Subject: [PATCH 185/262] feat: remove file when finish --- src/latLon/DAL/latLonDAL.ts | 25 +++++++++++++++---------- 1 file changed, 15 insertions(+), 10 deletions(-) diff --git a/src/latLon/DAL/latLonDAL.ts b/src/latLon/DAL/latLonDAL.ts index 503aa81b..38bccb9c 100644 --- a/src/latLon/DAL/latLonDAL.ts +++ b/src/latLon/DAL/latLonDAL.ts @@ -100,18 +100,25 @@ export class LatLonDAL { this.clearLatLonMap(); const latLonDataPath = await this.latLonRepository.downloadFile('latLonConvertionTable'); - const { items: latLonData } = JSON.parse(fs.readFileSync(latLonDataPath, 'utf8')) as { items: LatLon[] }; + + const { items: latLonData } = JSON.parse(await fs.promises.readFile(latLonDataPath, 'utf8')) as { items: LatLon[] }; latLonData.forEach((latLon) => { this.latLonMap.set(`${latLon.min_x},${latLon.min_y},${latLon.zone}`, latLon); }); - this.logger.debug('latLon data loaded'); + + try { + await fs.promises.unlink(latLonDataPath); + } catch (error) { + this.logger.error(`Failed to delete latLonData file ${latLonDataPath}. Error: ${(error as Error).message}`); + } + this.logger.info('cronLoadTileLatLonData: update completed'); } } export const cronLoadTileLatLonDataSymbol = Symbol('cronLoadTileLatLonDataSymbol'); -export const cronLoadTileLatLonDataFactory: FactoryFunction = (dependencyContainer) => { +export const cronLoadTileLatLonDataFactory: FactoryFunction = (dependencyContainer) => { const latLonDAL = dependencyContainer.resolve(LatLonDAL); const logger = dependencyContainer.resolve(SERVICES.LOGGER); const cronPattern: string | undefined = dependencyContainer.resolve(SERVICES.APPLICATION).cronLoadTileLatLonDataPattern; @@ -122,18 +129,16 @@ export const cronLoadTileLatLonDataFactory: FactoryFunction = (dependencyC } /* istanbul ignore next */ - cron.schedule(cronPattern, () => { + const scheduledTask = cron.schedule(cronPattern, () => { if (!latLonDAL.getOnGoingUpdate()) { logger.info('cronLoadTileLatLonData: starting update'); - latLonDAL - .init() - .then(() => logger.info('cronLoadTileLatLonData: update completed')) - .catch((error) => { - logger.error('cronLoadTileLatLonData: update failed', error); - }); + latLonDAL.init().catch((error) => { + logger.error('cronLoadTileLatLonData: update failed', error); + }); } else { logger.info('cronLoadTileLatLonData: update is already in progress'); } }); + return scheduledTask; /* istanbul ignore end */ }; From 44ddbfd3cfeb7055e51e90ac02f45551b4eeb9d8 Mon Sep 17 00:00:00 2001 From: Niv Greenstein Date: Wed, 4 Sep 2024 17:35:43 +0300 Subject: [PATCH 186/262] delete: removed unused code --- src/latLon/models/latLonManager.ts | 33 ++---------------------------- 1 file changed, 2 insertions(+), 31 deletions(-) diff --git a/src/latLon/models/latLonManager.ts b/src/latLon/models/latLonManager.ts index 184bbd1a..b97d1878 100644 --- a/src/latLon/models/latLonManager.ts +++ b/src/latLon/models/latLonManager.ts @@ -5,11 +5,9 @@ import { BBox, Feature } from 'geojson'; import * as mgrs from 'mgrs'; import { SERVICES } from '../../common/constants'; import { LatLonDAL } from '../DAL/latLonDAL'; -import { convertUTMToWgs84, convertWgs84ToUTM, validateTile, validateWGS84Coordinate } from '../../common/utils'; -import { convertTilesToUTM, getSubTileByBottomLeftUtmCoor, validateResult } from '../utlis'; +import { convertUTMToWgs84, convertWgs84ToUTM, validateWGS84Coordinate } from '../../common/utils'; import { BadRequestError } from '../../common/errors'; -import { Tile } from '../../control/tile/models/tile'; -import { FeatureCollection, WGS84Coordinate } from '../../common/interfaces'; +import { WGS84Coordinate } from '../../common/interfaces'; import { parseGeo } from '../../location/utils'; @injectable() @@ -80,28 +78,6 @@ export class LatLonManager { }; } - public async tileToLatLon({ tileName, subTileNumber }: { tileName: string; subTileNumber: number[] }): Promise> { - if (!validateTile({ tileName, subTileNumber })) { - const message = "Invalid tile, check that 'tileName' and 'subTileNumber' exists and subTileNumber is array of size 3 with positive integers"; - this.logger.warn(`LatLonManager.tileToLatLon: ${message}`); - throw new BadRequestError(message); - } - - const tile = await this.latLonDAL.tileToLatLon(tileName); - - if (!tile) { - const meessage = 'Tile not found'; - this.logger.warn(`LatLonManager.tileToLatLon: ${meessage}`); - throw new BadRequestError(meessage); - } - - const utmCoor = convertTilesToUTM({ tileName, subTileNumber }, tile); - validateResult(tile, utmCoor); - - const geojsonRes = getSubTileByBottomLeftUtmCoor(utmCoor, { tileName, subTileNumber }); - return geojsonRes; - } - public latLonToMGRS({ lat, lon, accuracy = 5 }: { lat: number; lon: number; accuracy?: number }): { [key: string]: unknown } & Feature { const accuracyString: Record = { [0]: '100km', @@ -123,9 +99,4 @@ export class LatLonManager { }, }; } - - public mgrsToLatLon(mgrsStr: string): { lat: number; lon: number } { - const [lon, lat] = mgrs.toPoint(mgrsStr); - return { lat, lon }; - } } From c6bb6138ae61e7b3b9968cf2e4a0f565796c8e1a Mon Sep 17 00:00:00 2001 From: Niv Greenstein Date: Wed, 4 Sep 2024 17:36:21 +0300 Subject: [PATCH 187/262] feat: added liveness and cleanup on signal --- src/containerConfig.ts | 170 ++++++++++++++++++++++++----------------- src/index.ts | 4 +- 2 files changed, 101 insertions(+), 73 deletions(-) diff --git a/src/containerConfig.ts b/src/containerConfig.ts index c145d8ea..25e60c0b 100644 --- a/src/containerConfig.ts +++ b/src/containerConfig.ts @@ -6,7 +6,9 @@ import { trace, metrics as OtelMetrics } from '@opentelemetry/api'; import { DependencyContainer } from 'tsyringe/dist/typings/types'; import jsLogger, { LoggerOptions } from '@map-colonies/js-logger'; import { Metrics } from '@map-colonies/telemetry'; -import { SERVICES, SERVICE_NAME } from './common/constants'; +import { CleanupRegistry } from '@map-colonies/cleanup-registry'; +import { ScheduledTask } from 'node-cron'; +import { HEALTHCHECK, ON_SIGNAL, SERVICES, SERVICE_NAME } from './common/constants'; import { tracing } from './common/tracing'; import { InjectionObject, registerDependencies } from './common/dependencyRegistration'; import { elasticClientFactory, ElasticClients } from './common/elastic'; @@ -23,6 +25,7 @@ import { cronLoadTileLatLonDataFactory, cronLoadTileLatLonDataSymbol } from './l import { ITEM_REPOSITORY_SYMBOL, itemRepositoryFactory } from './control/item/DAL/itemRepository'; import { s3ClientFactory } from './common/s3'; import { S3_REPOSITORY_SYMBOL, s3RepositoryFactory } from './common/s3/s3Repository'; +import { healthCheckFactory } from './common/utils'; export interface RegisterOptions { override?: InjectionObject[]; @@ -30,83 +33,108 @@ export interface RegisterOptions { } export const registerExternalValues = async (options?: RegisterOptions): Promise => { - const loggerConfig = config.get('telemetry.logger'); - const logger = jsLogger({ ...loggerConfig, prettyPrint: loggerConfig.prettyPrint, mixin: getOtelMixin() }); + const cleanupRegistry = new CleanupRegistry(); + try { + const loggerConfig = config.get('telemetry.logger'); + const logger = jsLogger({ ...loggerConfig, prettyPrint: loggerConfig.prettyPrint, mixin: getOtelMixin() }); + const cleanupRegistryLogger = logger.child({ subComponent: 'cleanupRegistry' }); - const metrics = new Metrics(); - metrics.start(); + cleanupRegistry.on('itemFailed', (id, error, msg) => cleanupRegistryLogger.error({ msg, itemId: id, err: error })); + cleanupRegistry.on('finished', (status) => cleanupRegistryLogger.info({ msg: `cleanup registry finished cleanup`, status })); - tracing.start(); - const tracer = trace.getTracer(SERVICE_NAME); + const metrics = new Metrics(); + cleanupRegistry.register({ func: metrics.stop.bind(metrics), id: SERVICES.METER }); + metrics.start(); - const applicationConfig: IApplication = config.get('application'); + cleanupRegistry.register({ func: tracing.stop.bind(tracing), id: SERVICES.TRACER }); + tracing.start(); + const tracer = trace.getTracer(SERVICE_NAME); - const dependencies: InjectionObject[] = [ - { token: SERVICES.CONFIG, provider: { useValue: config } }, - { token: SERVICES.LOGGER, provider: { useValue: logger } }, - { token: SERVICES.TRACER, provider: { useValue: tracer } }, - { token: SERVICES.METER, provider: { useValue: OtelMetrics.getMeterProvider().getMeter(SERVICE_NAME) } }, - { token: SERVICES.APPLICATION, provider: { useValue: applicationConfig } }, - { - token: SERVICES.ELASTIC_CLIENTS, - provider: { useFactory: instancePerContainerCachingFactory(elasticClientFactory) }, - postInjectionHook: async (deps: DependencyContainer): Promise => { - const elasticClients = deps.resolve(SERVICES.ELASTIC_CLIENTS); - try { - const response = await Promise.all([elasticClients.control?.ping(), elasticClients.geotext?.ping()]); - response.forEach((res) => { - if (!res) { - logger.error('Failed to connect to Elasticsearch', res); - } - }); - } catch (err) { - logger.error('Failed to connect to Elasticsearch', err); - } + const applicationConfig: IApplication = config.get('application'); + + const dependencies: InjectionObject[] = [ + { token: SERVICES.CONFIG, provider: { useValue: config } }, + { token: SERVICES.LOGGER, provider: { useValue: logger } }, + { token: SERVICES.TRACER, provider: { useValue: tracer } }, + { token: SERVICES.METER, provider: { useValue: OtelMetrics.getMeterProvider().getMeter(SERVICE_NAME) } }, + { token: SERVICES.APPLICATION, provider: { useValue: applicationConfig } }, + { token: HEALTHCHECK, provider: { useFactory: healthCheckFactory } }, + { + token: ON_SIGNAL, + provider: { + useValue: cleanupRegistry.trigger.bind(cleanupRegistry), + }, + }, + { + token: SERVICES.ELASTIC_CLIENTS, + provider: { useFactory: instancePerContainerCachingFactory(elasticClientFactory) }, + postInjectionHook: async (deps: DependencyContainer): Promise => { + const elasticClients = deps.resolve(SERVICES.ELASTIC_CLIENTS); + try { + const response = await Promise.all([elasticClients.control?.ping(), elasticClients.geotext?.ping()]); + response.forEach((res) => { + if (!res) { + logger.error('Failed to connect to Elasticsearch', res); + } + }); + cleanupRegistry.register({ + func: async () => { + await elasticClients.control.close(); + await elasticClients.geotext.close(); + }, + id: SERVICES.ELASTIC_CLIENTS, + }); + } catch (err) { + logger.error('Failed to connect to Elasticsearch', err); + } + }, }, - }, - { - token: SERVICES.S3_CLIENT, - provider: { useFactory: s3ClientFactory }, - postInjectionHook: async (deps: DependencyContainer): Promise => { - const s3Client = deps.resolve(SERVICES.S3_CLIENT); - try { - await s3Client.send(new ListBucketsCommand({})); - logger.info('Connected to S3'); - } catch (err) { - logger.error('Failed to connect to S3', err); - } + { + token: SERVICES.S3_CLIENT, + provider: { useFactory: s3ClientFactory }, + postInjectionHook: async (deps: DependencyContainer): Promise => { + const s3Client = deps.resolve(SERVICES.S3_CLIENT); + try { + await s3Client.send(new ListBucketsCommand({})); + logger.info('Connected to S3'); + } catch (err) { + logger.error('Failed to connect to S3', err); + } + cleanupRegistry.register({ func: async () => s3Client.destroy(), id: SERVICES.S3_CLIENT }); + }, }, - }, - { - token: S3_REPOSITORY_SYMBOL, - provider: { useFactory: s3RepositoryFactory }, - }, - { token: TILE_REPOSITORY_SYMBOL, provider: { useFactory: tileRepositoryFactory } }, - { token: TILE_ROUTER_SYMBOL, provider: { useFactory: tileRouterFactory } }, - { token: ITEM_REPOSITORY_SYMBOL, provider: { useFactory: itemRepositoryFactory } }, - { token: ITEM_ROUTER_SYMBOL, provider: { useFactory: itemRouterFactory } }, - { token: ROUTE_REPOSITORY_SYMBOL, provider: { useFactory: routeRepositoryFactory } }, - { token: ROUTE_ROUTER_SYMBOL, provider: { useFactory: routeRouterFactory } }, - { token: LAT_LON_ROUTER_SYMBOL, provider: { useFactory: latLonRouterFactory } }, - { token: GEOTEXT_REPOSITORY_SYMBOL, provider: { useFactory: geotextRepositoryFactory } }, - { token: GEOTEXT_SEARCH_ROUTER_SYMBOL, provider: { useFactory: geotextSearchRouterFactory } }, - { - token: cronLoadTileLatLonDataSymbol, - provider: { - useFactory: cronLoadTileLatLonDataFactory, + { + token: S3_REPOSITORY_SYMBOL, + provider: { useFactory: s3RepositoryFactory }, }, - }, - { - token: 'onSignal', - provider: { - useValue: { - useValue: async (): Promise => { - await Promise.all([tracing.stop(), metrics.stop()]); - }, + { token: TILE_REPOSITORY_SYMBOL, provider: { useFactory: tileRepositoryFactory } }, + { token: TILE_ROUTER_SYMBOL, provider: { useFactory: tileRouterFactory } }, + { token: ITEM_REPOSITORY_SYMBOL, provider: { useFactory: itemRepositoryFactory } }, + { token: ITEM_ROUTER_SYMBOL, provider: { useFactory: itemRouterFactory } }, + { token: ROUTE_REPOSITORY_SYMBOL, provider: { useFactory: routeRepositoryFactory } }, + { token: ROUTE_ROUTER_SYMBOL, provider: { useFactory: routeRouterFactory } }, + { token: LAT_LON_ROUTER_SYMBOL, provider: { useFactory: latLonRouterFactory } }, + { token: GEOTEXT_REPOSITORY_SYMBOL, provider: { useFactory: geotextRepositoryFactory } }, + { token: GEOTEXT_SEARCH_ROUTER_SYMBOL, provider: { useFactory: geotextSearchRouterFactory } }, + { + token: cronLoadTileLatLonDataSymbol, + provider: { + useFactory: cronLoadTileLatLonDataFactory, + }, + postInjectionHook: async (deps: DependencyContainer): Promise => { + const cronLoadTileLatLonData = deps.resolve(cronLoadTileLatLonDataSymbol); + cronLoadTileLatLonData.start(); + cleanupRegistry.register({ + func: async () => cronLoadTileLatLonData.stop.bind(cronLoadTileLatLonData), + id: cronLoadTileLatLonDataSymbol, + }); }, }, - }, - ]; - const container = await registerDependencies(dependencies, options?.override, options?.useChild); - return container; + ]; + const container = await registerDependencies(dependencies, options?.override, options?.useChild); + return container; + } catch (error) { + await cleanupRegistry.trigger(); + throw error; + } }; diff --git a/src/index.ts b/src/index.ts index 4ee6fbac..a2f0f74b 100644 --- a/src/index.ts +++ b/src/index.ts @@ -21,8 +21,8 @@ void getApp() const logger = container.resolve(SERVICES.LOGGER); const server = createTerminus(createServer(app), { // eslint-disable-next-line @typescript-eslint/naming-convention - // healthChecks: { '/liveness': container.resolve(HEALTHCHECK) }, - // onSignal: container.resolve(ON_SIGNAL), + healthChecks: { '/liveness': container.resolve(HEALTHCHECK) }, + onSignal: container.resolve(ON_SIGNAL), }); server.listen(port, () => { From bd3ac9cacfaa8980cac74fd16a9c822749ebec27 Mon Sep 17 00:00:00 2001 From: Niv Greenstein Date: Wed, 4 Sep 2024 17:36:31 +0300 Subject: [PATCH 188/262] feat: added cleanup-registry --- package-lock.json | 51 +++++++++++++++++++++++++++++++++++++++++++++++ package.json | 1 + 2 files changed, 52 insertions(+) diff --git a/package-lock.json b/package-lock.json index fac554d8..b857e52e 100644 --- a/package-lock.json +++ b/package-lock.json @@ -13,6 +13,7 @@ "@aws-sdk/client-s3": "^3.637.0", "@elastic/elasticsearch": "^8.13.1", "@godaddy/terminus": "^4.12.1", + "@map-colonies/cleanup-registry": "^1.1.0", "@map-colonies/error-express-handler": "^2.1.0", "@map-colonies/express-access-log-middleware": "^2.0.1", "@map-colonies/js-logger": "^1.0.1", @@ -4838,6 +4839,15 @@ "resolved": "https://registry.npmjs.org/@jsdevtools/ono/-/ono-7.1.3.tgz", "integrity": "sha512-4JQNk+3mVzK3xh2rqd6RB4J46qUR19azEHBneZyTZM+c456qOrbbM/5xcR8huNCCcbVt7+UmizG6GuUvPvKUYg==" }, + "node_modules/@map-colonies/cleanup-registry": { + "version": "1.1.0", + "resolved": "https://registry.npmjs.org/@map-colonies/cleanup-registry/-/cleanup-registry-1.1.0.tgz", + "integrity": "sha512-/lhIGklWPZSY37JwzhFJEtBlqwXDRhHSeCBwpPaGMxpycpt5ZRIVQxUt6Og4mt6c5GoRoX9dZYHY0qV3UMGvtQ==", + "dependencies": { + "nanoid": "^3.3.4", + "tiny-typed-emitter": "^2.1.0" + } + }, "node_modules/@map-colonies/error-express-handler": { "version": "2.1.0", "resolved": "https://registry.npmjs.org/@map-colonies/error-express-handler/-/error-express-handler-2.1.0.tgz", @@ -17317,6 +17327,23 @@ "integrity": "sha512-nnbWWOkoWyUsTjKrhgD0dcz22mdkSnpYqbEjIm2nhwhuxlSkpywJmBo8h0ZqJdkp73mb90SssHkN4rsRaBAfAA==", "dev": true }, + "node_modules/nanoid": { + "version": "3.3.7", + "resolved": "https://registry.npmjs.org/nanoid/-/nanoid-3.3.7.tgz", + "integrity": "sha512-eSRppjcPIatRIMC1U6UngP8XFcz8MQWGQdt1MTBQ7NaAmvXDfvNxbvWV3x2y6CdEUciCSsDHDQZbhYaB8QEo2g==", + "funding": [ + { + "type": "github", + "url": "https://github.com/sponsors/ai" + } + ], + "bin": { + "nanoid": "bin/nanoid.cjs" + }, + "engines": { + "node": "^10 || ^12 || ^13.7 || ^14 || >=15.0.1" + } + }, "node_modules/natural-compare": { "version": "1.4.0", "resolved": "https://registry.npmjs.org/natural-compare/-/natural-compare-1.4.0.tgz", @@ -20190,6 +20217,11 @@ "node": ">=6" } }, + "node_modules/tiny-typed-emitter": { + "version": "2.1.0", + "resolved": "https://registry.npmjs.org/tiny-typed-emitter/-/tiny-typed-emitter-2.1.0.tgz", + "integrity": "sha512-qVtvMxeXbVej0cQWKqVSSAHmKZEHAvxdF8HEUBFWts8h+xEo5m/lEiPakuyZ3BnCBjOD8i24kzNOiOLLgsSxhA==" + }, "node_modules/tmp": { "version": "0.0.33", "resolved": "https://registry.npmjs.org/tmp/-/tmp-0.0.33.tgz", @@ -24732,6 +24764,15 @@ "resolved": "https://registry.npmjs.org/@jsdevtools/ono/-/ono-7.1.3.tgz", "integrity": "sha512-4JQNk+3mVzK3xh2rqd6RB4J46qUR19azEHBneZyTZM+c456qOrbbM/5xcR8huNCCcbVt7+UmizG6GuUvPvKUYg==" }, + "@map-colonies/cleanup-registry": { + "version": "1.1.0", + "resolved": "https://registry.npmjs.org/@map-colonies/cleanup-registry/-/cleanup-registry-1.1.0.tgz", + "integrity": "sha512-/lhIGklWPZSY37JwzhFJEtBlqwXDRhHSeCBwpPaGMxpycpt5ZRIVQxUt6Og4mt6c5GoRoX9dZYHY0qV3UMGvtQ==", + "requires": { + "nanoid": "^3.3.4", + "tiny-typed-emitter": "^2.1.0" + } + }, "@map-colonies/error-express-handler": { "version": "2.1.0", "resolved": "https://registry.npmjs.org/@map-colonies/error-express-handler/-/error-express-handler-2.1.0.tgz", @@ -34359,6 +34400,11 @@ "integrity": "sha512-nnbWWOkoWyUsTjKrhgD0dcz22mdkSnpYqbEjIm2nhwhuxlSkpywJmBo8h0ZqJdkp73mb90SssHkN4rsRaBAfAA==", "dev": true }, + "nanoid": { + "version": "3.3.7", + "resolved": "https://registry.npmjs.org/nanoid/-/nanoid-3.3.7.tgz", + "integrity": "sha512-eSRppjcPIatRIMC1U6UngP8XFcz8MQWGQdt1MTBQ7NaAmvXDfvNxbvWV3x2y6CdEUciCSsDHDQZbhYaB8QEo2g==" + }, "natural-compare": { "version": "1.4.0", "resolved": "https://registry.npmjs.org/natural-compare/-/natural-compare-1.4.0.tgz", @@ -36539,6 +36585,11 @@ "resolved": "https://registry.npmjs.org/tiny-lru/-/tiny-lru-8.0.2.tgz", "integrity": "sha512-ApGvZ6vVvTNdsmt676grvCkUCGwzG9IqXma5Z07xJgiC5L7akUMof5U8G2JTI9Rz/ovtVhJBlY6mNhEvtjzOIg==" }, + "tiny-typed-emitter": { + "version": "2.1.0", + "resolved": "https://registry.npmjs.org/tiny-typed-emitter/-/tiny-typed-emitter-2.1.0.tgz", + "integrity": "sha512-qVtvMxeXbVej0cQWKqVSSAHmKZEHAvxdF8HEUBFWts8h+xEo5m/lEiPakuyZ3BnCBjOD8i24kzNOiOLLgsSxhA==" + }, "tmp": { "version": "0.0.33", "resolved": "https://registry.npmjs.org/tmp/-/tmp-0.0.33.tgz", diff --git a/package.json b/package.json index 7348f928..230d0157 100644 --- a/package.json +++ b/package.json @@ -49,6 +49,7 @@ "@aws-sdk/client-s3": "^3.637.0", "@elastic/elasticsearch": "^8.13.1", "@godaddy/terminus": "^4.12.1", + "@map-colonies/cleanup-registry": "^1.1.0", "@map-colonies/error-express-handler": "^2.1.0", "@map-colonies/express-access-log-middleware": "^2.0.1", "@map-colonies/js-logger": "^1.0.1", From 92f19a8fd8ab1f070e5abf394e38dae080df8279 Mon Sep 17 00:00:00 2001 From: Niv Greenstein Date: Wed, 4 Sep 2024 18:37:19 +0300 Subject: [PATCH 189/262] delete: removed unused code --- src/latLon/controllers/latLonController.ts | 42 ++-------------------- 1 file changed, 2 insertions(+), 40 deletions(-) diff --git a/src/latLon/controllers/latLonController.ts b/src/latLon/controllers/latLonController.ts index 0df87b0f..cff3e100 100644 --- a/src/latLon/controllers/latLonController.ts +++ b/src/latLon/controllers/latLonController.ts @@ -10,22 +10,9 @@ import { LatLonManager } from '../models/latLonManager'; import { WGS84Coordinate } from '../../common/interfaces'; /* istanbul ignore file */ -type GetLatLonToTileHandler = RequestHandler; +type GetCoordinatesHandler = RequestHandler; -type GetLatLonToMgrsHandler = RequestHandler; - -type GetCoordinatesHandler = RequestHandler< - undefined, - { [key: string]: unknown } & Feature, - undefined, - WGS84Coordinate & { target_grid: 'control' | 'MGRS' } ->; - -export interface GetLatLonToTileQueryParams extends WGS84Coordinate {} - -export interface GetLatLonToMgrsQueryParams extends WGS84Coordinate { - accuracy?: number; -} +export type GetCoordinatesRequestParams = WGS84Coordinate & { target_grid: 'control' | 'MGRS' }; @injectable() export class LatLonController { @@ -39,31 +26,6 @@ export class LatLonController { this.createdResourceCounter = meter.createCounter('created_resource'); } - public latlonToTile: GetLatLonToTileHandler = async (req, res, next) => { - try { - const { lat, lon } = req.query; - - const response = await this.manager.latLonToTile({ lat, lon }); - return res.status(httpStatus.OK).json(response); - } catch (error: unknown) { - this.logger.warn('latLonController.latlonToTile Error:', error); - next(error); - } - }; - - public latlonToMgrs: GetLatLonToMgrsHandler = (req, res, next) => { - try { - const { lat, lon, accuracy } = req.query; - - const response = this.manager.latLonToMGRS({ lat, lon, accuracy }); - - return res.status(httpStatus.OK).json(response); - } catch (error: unknown) { - this.logger.warn('latLonController.latlonToMgrs Error:', error); - next(error); - } - }; - public getCoordinates: GetCoordinatesHandler = async (req, res, next) => { try { const { lat, lon, target_grid } = req.query; From 55fa2880f0fb3678de8706a585b6950a50c47c17 Mon Sep 17 00:00:00 2001 From: Niv Greenstein Date: Wed, 4 Sep 2024 18:37:37 +0300 Subject: [PATCH 190/262] fix: added missing query and response properties --- src/latLon/models/latLonManager.ts | 22 ++++++++++++++++------ 1 file changed, 16 insertions(+), 6 deletions(-) diff --git a/src/latLon/models/latLonManager.ts b/src/latLon/models/latLonManager.ts index b97d1878..26ff4991 100644 --- a/src/latLon/models/latLonManager.ts +++ b/src/latLon/models/latLonManager.ts @@ -62,19 +62,24 @@ export class LatLonManager { return { type: 'Feature', - properties: { - tileName: tileCoordinateData.tile_name, - subTileNumber: new Array(3).fill('').map(function (value, i) { - return xNumber[i] + yNumber[i]; - }), + query: { + lat, + lon, }, + response: {}, + bbox, geometry: parseGeo({ bbox, }) ?? { type: 'Point', coordinates: [lon, lat], }, - bbox, + properties: { + tileName: tileCoordinateData.tile_name, + subTileNumber: new Array(3).fill('').map(function (value, i) { + return xNumber[i] + yNumber[i]; + }), + }, }; } @@ -89,6 +94,11 @@ export class LatLonManager { }; return { type: 'Feature', + query: { + lat, + lon, + }, + response: {}, geometry: { type: 'Point', coordinates: [lon, lat], From ae2ce64d919a51de705790c431d10deb39c55e20 Mon Sep 17 00:00:00 2001 From: Niv Greenstein Date: Wed, 4 Sep 2024 18:37:50 +0300 Subject: [PATCH 191/262] feat: added getIsDataLoadError --- src/latLon/DAL/latLonDAL.ts | 14 +++++--------- 1 file changed, 5 insertions(+), 9 deletions(-) diff --git a/src/latLon/DAL/latLonDAL.ts b/src/latLon/DAL/latLonDAL.ts index 38bccb9c..5326a18f 100644 --- a/src/latLon/DAL/latLonDAL.ts +++ b/src/latLon/DAL/latLonDAL.ts @@ -41,6 +41,10 @@ export class LatLonDAL { public getOnGoingUpdate(): boolean { return this.onGoingUpdate; } + + public getIsDataLoadError(): boolean { + return this.dataLoadError; + } /* istanbul ignore end */ public async init(): Promise { @@ -74,21 +78,13 @@ export class LatLonDAL { } public async latLonToTile({ x, y, zone }: { x: number; y: number; zone: number }): Promise { - if (this.dataLoadError) { + if (this.getIsDataLoadError()) { throw new InternalServerError('Lat-lon to tile data currently not available'); } await this.dataLoad?.promise; return this.latLonMap.get(`${x},${y},${zone}`); } - public async tileToLatLon(tileName: string): Promise { - if (this.dataLoadError) { - throw new InternalServerError('Tile to lat-lon data currently not available'); - } - await this.dataLoad?.promise; - return this.latLonMap.get(tileName); - } - private clearLatLonMap(): void { this.logger.debug('Clearing latLon data'); this.latLonMap.clear(); From 3a3e63d7f10e17e9c9f34698ad2b057d770d987a Mon Sep 17 00:00:00 2001 From: Niv Greenstein Date: Wed, 4 Sep 2024 18:38:06 +0300 Subject: [PATCH 192/262] feat: added latLon tests --- config/test.json | 25 ++- devScripts/latLonConvertions.json | 24 +++ .../latLon/helpers/requestSender.ts | 16 ++ tests/integration/latLon/latLon.spec.ts | 197 ++++++++++++++++++ tests/integration/location/location.spec.ts | 2 +- 5 files changed, 250 insertions(+), 14 deletions(-) create mode 100644 devScripts/latLonConvertions.json create mode 100644 tests/integration/latLon/helpers/requestSender.ts create mode 100644 tests/integration/latLon/latLon.spec.ts diff --git a/config/test.json b/config/test.json index 85008963..4fac3063 100644 --- a/config/test.json +++ b/config/test.json @@ -29,20 +29,19 @@ } } }, - "postgresql": { - "type": "postgres", - "host": "localhost", - "port": 5432, - "username": "postgres", - "password": "postgres", - "enableSslAuth": false, - "sslPaths": { - "ca": "", - "key": "", - "cert": "" + "s3": { + "endpoint": "http://localhost:9000", + "credentials": { + "accessKeyId": "O8JnyGPmTtIz69uZihyh", + "secretAccessKey": "1uzitcz85XFpZ8oxedJMoQpCuDSiJrhfDPbAvPOM" }, - "database": "postgres", - "schema": "geocoding" + "region": "local", + "files": { + "latLonConvertionTable": { + "bucket": "geocoding", + "fileName": "table.json" + } + } } }, "application": { diff --git a/devScripts/latLonConvertions.json b/devScripts/latLonConvertions.json new file mode 100644 index 00000000..c406ea34 --- /dev/null +++ b/devScripts/latLonConvertions.json @@ -0,0 +1,24 @@ +{ + "items": [ + { + "tile_name": "BRN", + "zone": "33", + "min_x": "360000", + "min_y": "5820000", + "ext_min_x": 360000, + "ext_min_y": 5820000, + "ext_max_x": 370000, + "ext_max_y": 5830000 + }, + { + "tile_name": "BMN", + "zone": "32", + "min_x": "480000", + "min_y": "5880000", + "ext_min_x": 480000, + "ext_min_y": 5880000, + "ext_max_x": 490000, + "ext_max_y": 5890000 + } + ] +} diff --git a/tests/integration/latLon/helpers/requestSender.ts b/tests/integration/latLon/helpers/requestSender.ts new file mode 100644 index 00000000..2766e47c --- /dev/null +++ b/tests/integration/latLon/helpers/requestSender.ts @@ -0,0 +1,16 @@ +import * as supertest from 'supertest'; +import { GetCoordinatesRequestParams } from '../../../../src/latLon/controllers/latLonController'; + +export class LatLonRequestSender { + public constructor(private readonly app: Express.Application) {} + + public async convertCoordinatesToGrid(queryParams?: GetCoordinatesRequestParams): Promise { + return supertest + .agent(this.app) + .get(`/lookup/coordinates`) + .set('Content-Type', 'application/json') + .set('x-api-key', 'abc123') + .set('x-user-id', 'abc123') + .query(queryParams ?? {}); + } +} diff --git a/tests/integration/latLon/latLon.spec.ts b/tests/integration/latLon/latLon.spec.ts new file mode 100644 index 00000000..e41bf913 --- /dev/null +++ b/tests/integration/latLon/latLon.spec.ts @@ -0,0 +1,197 @@ +/* eslint-disable @typescript-eslint/naming-convention */ +import { IConfig } from 'config'; +import { PutObjectCommand, S3Client } from '@aws-sdk/client-s3'; +import jsLogger from '@map-colonies/js-logger'; +import { trace } from '@opentelemetry/api'; +import httpStatusCodes from 'http-status-codes'; +import { getApp } from '../../../src/app'; +import { SERVICES } from '../../../src/common/constants'; +import { S3Config, s3ConfigPath } from '../../../src/common/s3'; +import mockDataJson from '../../../devScripts/latLonConvertions.json'; +import { LatLonRequestSender } from './helpers/requestSender'; +import { LatLonDAL } from '../../../src/latLon/DAL/latLonDAL'; + +describe('/lookup', function () { + let requestSender: LatLonRequestSender; + + beforeAll(async function () { + const app = await getApp({ + override: [ + { token: SERVICES.LOGGER, provider: { useValue: jsLogger({ enabled: false }) } }, + { token: SERVICES.TRACER, provider: { useValue: trace.getTracer('testTracer') } }, + ], + useChild: true, + }); + const config = app.container.resolve(SERVICES.CONFIG); + const s3Client = app.container.resolve(SERVICES.S3_CLIENT); + + const s3Config = config.get(s3ConfigPath); + + if (s3Config === undefined || s3Config.files.latLonConvertionTable === undefined) { + throw new Error('S3 configuration is missing'); + } + + const { bucket: Bucket, fileName: Key } = s3Config.files.latLonConvertionTable; + + const command = new PutObjectCommand({ + Bucket, + Key, + Body: Buffer.from(JSON.stringify(mockDataJson), 'utf-8'), + }); + + try { + await s3Client.send(command); + } catch (error) { + console.error(error); + throw error; + } + }); + + beforeEach(async function () { + const app = await getApp({ + override: [ + { token: SERVICES.LOGGER, provider: { useValue: jsLogger({ enabled: false }) } }, + { token: SERVICES.TRACER, provider: { useValue: trace.getTracer('testTracer') } }, + ], + useChild: true, + }); + + requestSender = new LatLonRequestSender(app.app); + }); + + describe('Happy Path', function () { + it('should return 200 status code and tile from lat-lon', async function () { + const response = await requestSender.convertCoordinatesToGrid({ + lat: 52.57326537485767, + lon: 12.948781146422107, + target_grid: 'control', + }); + + expect(response.status).toBe(httpStatusCodes.OK); + // expect(response).toSatisfyApiSpec(); + expect(response.body).toEqual({ + type: 'Feature', + query: { + lat: 52.57326537485767, + lon: 12.948781146422107, + }, + response: {}, + properties: { + tileName: 'BRN', + subTileNumber: ['06', '97', '97'], + }, + geometry: { + type: 'Polygon', + coordinates: [ + [ + [12.93694771534361, 52.51211561266182], + [12.93694771534361, 52.60444267653175], + [13.080296161196031, 52.60444267653175], + [13.080296161196031, 52.51211561266182], + [12.93694771534361, 52.51211561266182], + ], + ], + }, + bbox: [12.93694771534361, 52.51211561266182, 13.080296161196031, 52.60444267653175], + }); + }); + + it('should return 200 status code and MGRS from lat-lon', async function () { + const response = await requestSender.convertCoordinatesToGrid({ + lat: 52.57326537485767, + lon: 12.948781146422107, + target_grid: 'MGRS', + }); + + expect(response.status).toBe(httpStatusCodes.OK); + // expect(response).toSatisfyApiSpec(); + expect(response.body).toEqual({ + type: 'Feature', + query: { + lat: 52.57326537485767, + lon: 12.948781146422107, + }, + response: {}, + geometry: { + type: 'Point', + coordinates: [12.948781146422107, 52.57326537485767], + }, + properties: { + accuracy: '1m', + mgrs: '33UUU6099626777', + }, + }); + }); + }); + + describe('Bad Path', function () { + it('should retrun 400 status code when invalid lat lon', async function () { + const response = await requestSender.convertCoordinatesToGrid({ + lat: 67.9435100890131, + lon: -84.41903041752825, + target_grid: 'control', + }); + + expect(response.status).toBe(httpStatusCodes.BAD_REQUEST); + // expect(response).toSatisfyApiSpec(); + expect(response.body).toEqual({ + message: "Invalid lat lon, check 'lat' and 'lon' keys exists and their values are legal", + }); + }); + + it('should return 400 status code when the coordinate is outside the grid extent', async function () { + const response = await requestSender.convertCoordinatesToGrid({ + lat: 32.57326537485767, + lon: 12.948781146422107, + target_grid: 'control', + }); + + expect(response.status).toBe(httpStatusCodes.BAD_REQUEST); + // expect(response).toSatisfyApiSpec(); + expect(response.body).toEqual({ + message: 'The coordinate is outside the grid extent', + }); + }); + }); + + describe('Sad Path', function () { + it('should return 500 as isDataLoadError is true for control', async function () { + const dataLoadErrorSpy = jest.spyOn(LatLonDAL.prototype, 'getIsDataLoadError').mockReturnValue(true); + + const response = await requestSender.convertCoordinatesToGrid({ + lat: 32.57326537485767, + lon: 12.948781146422107, + target_grid: 'control', + }); + + expect(response.status).toBe(httpStatusCodes.INTERNAL_SERVER_ERROR); + // expect(response).toSatisfyApiSpec(); + expect(response.body).toEqual({ + message: 'Lat-lon to tile data currently not available', + stacktrace: expect.any(String) as string, + }); + + dataLoadErrorSpy.mockRestore(); + }); + + // it('should return 500 as init is errored', async function () { + // const dataLoadErrorSpy = jest.spyOn(LatLonDAL.prototype as any, 'loadLatLonData').mockRejectedValue(new Error('some error')); + + // const response = await requestSender.convertCoordinatesToGrid({ + // lat: 52.57326537485767, + // lon: 12.948781146422107, + // target_grid: 'control', + // }); + + // console.log(response.status, response.body); + // expect(response.status).toBe(httpStatusCodes.BAD_REQUEST); + // // expect(response).toSatisfyApiSpec(); + // expect(response.body).toEqual({ + // message: 'Lat-lon to tile data currently not available', + // stacktrace: expect.any(String) as string, + // }); + + // dataLoadErrorSpy.mockRestore(); + // }); + }); +}); diff --git a/tests/integration/location/location.spec.ts b/tests/integration/location/location.spec.ts index ec542d62..ee6d9831 100644 --- a/tests/integration/location/location.spec.ts +++ b/tests/integration/location/location.spec.ts @@ -26,7 +26,7 @@ import { } from './mockObjects'; import { expectedResponse, hierarchiesWithAnyWieght } from './utils'; -describe('/search/control/tiles', function () { +describe('/search/location', function () { let requestSender: LocationRequestSender; beforeEach(async function () { From a8850c3b5e9eeaaaa93ac42ae5d9f67aa9ef2642 Mon Sep 17 00:00:00 2001 From: Niv Greenstein Date: Thu, 5 Sep 2024 10:43:46 +0300 Subject: [PATCH 193/262] feat: added create bucket and cleanup after test --- tests/integration/latLon/latLon.spec.ts | 42 +++++++++++++++++++------ 1 file changed, 32 insertions(+), 10 deletions(-) diff --git a/tests/integration/latLon/latLon.spec.ts b/tests/integration/latLon/latLon.spec.ts index e41bf913..e31f877c 100644 --- a/tests/integration/latLon/latLon.spec.ts +++ b/tests/integration/latLon/latLon.spec.ts @@ -1,6 +1,6 @@ /* eslint-disable @typescript-eslint/naming-convention */ import { IConfig } from 'config'; -import { PutObjectCommand, S3Client } from '@aws-sdk/client-s3'; +import { PutObjectCommand, CreateBucketCommand, DeleteBucketCommand, DeleteObjectCommand, S3Client } from '@aws-sdk/client-s3'; import jsLogger from '@map-colonies/js-logger'; import { trace } from '@opentelemetry/api'; import httpStatusCodes from 'http-status-codes'; @@ -8,11 +8,13 @@ import { getApp } from '../../../src/app'; import { SERVICES } from '../../../src/common/constants'; import { S3Config, s3ConfigPath } from '../../../src/common/s3'; import mockDataJson from '../../../devScripts/latLonConvertions.json'; -import { LatLonRequestSender } from './helpers/requestSender'; import { LatLonDAL } from '../../../src/latLon/DAL/latLonDAL'; +import { LatLonRequestSender } from './helpers/requestSender'; describe('/lookup', function () { let requestSender: LatLonRequestSender; + let s3Client: S3Client; + let s3Config: S3Config | undefined; beforeAll(async function () { const app = await getApp({ @@ -23,9 +25,9 @@ describe('/lookup', function () { useChild: true, }); const config = app.container.resolve(SERVICES.CONFIG); - const s3Client = app.container.resolve(SERVICES.S3_CLIENT); + s3Client = app.container.resolve(SERVICES.S3_CLIENT); - const s3Config = config.get(s3ConfigPath); + s3Config = config.get(s3ConfigPath); if (s3Config === undefined || s3Config.files.latLonConvertionTable === undefined) { throw new Error('S3 configuration is missing'); @@ -33,14 +35,34 @@ describe('/lookup', function () { const { bucket: Bucket, fileName: Key } = s3Config.files.latLonConvertionTable; - const command = new PutObjectCommand({ - Bucket, - Key, - Body: Buffer.from(JSON.stringify(mockDataJson), 'utf-8'), - }); + try { + await s3Client.send(new CreateBucketCommand({ Bucket, ACL: 'public-read' })); + } catch (error) { + console.error(error); + } + + try { + await s3Client.send( + new PutObjectCommand({ + Bucket, + Key, + Body: Buffer.from(JSON.stringify(mockDataJson), 'utf-8'), + }) + ); + } catch (error) { + console.error(error); + throw error; + } + }); + afterAll(async function () { try { - await s3Client.send(command); + if (s3Config?.files.latLonConvertionTable !== undefined) { + await s3Client.send( + new DeleteObjectCommand({ Bucket: s3Config.files.latLonConvertionTable.bucket, Key: s3Config.files.latLonConvertionTable.fileName }) + ); + await s3Client.send(new DeleteBucketCommand({ Bucket: s3Config.files.latLonConvertionTable.bucket })); + } } catch (error) { console.error(error); throw error; From f65e7e7fe82d6a3f2ff8e333a04937d6b31ff391 Mon Sep 17 00:00:00 2001 From: Niv Greenstein Date: Thu, 5 Sep 2024 17:23:45 +0300 Subject: [PATCH 194/262] feat: added forcePathStyle true --- config/default.json | 1 + config/test.json | 1 + 2 files changed, 2 insertions(+) diff --git a/config/default.json b/config/default.json index 43b6cb32..ad101143 100644 --- a/config/default.json +++ b/config/default.json @@ -61,6 +61,7 @@ "accessKeyId": "O8JnyGPmTtIz69uZihyh", "secretAccessKey": "1uzitcz85XFpZ8oxedJMoQpCuDSiJrhfDPbAvPOM" }, + "forcePathStyle": true, "region": "local", "files": { "latLonConvertionTable": { diff --git a/config/test.json b/config/test.json index 4fac3063..e2579370 100644 --- a/config/test.json +++ b/config/test.json @@ -36,6 +36,7 @@ "secretAccessKey": "1uzitcz85XFpZ8oxedJMoQpCuDSiJrhfDPbAvPOM" }, "region": "local", + "forcePathStyle": true, "files": { "latLonConvertionTable": { "bucket": "geocoding", From 62f9b587ad673336c352adcd63549fed0c871a95 Mon Sep 17 00:00:00 2001 From: Niv Greenstein Date: Thu, 5 Sep 2024 17:24:32 +0300 Subject: [PATCH 195/262] feat: changed indexes to have _index suffix --- config/test.json | 14 +++++++------- 1 file changed, 7 insertions(+), 7 deletions(-) diff --git a/config/test.json b/config/test.json index e2579370..8b97a838 100644 --- a/config/test.json +++ b/config/test.json @@ -9,7 +9,7 @@ }, "requestTimeout": 60000, "properties": { - "index": "control_gil_v5" + "index": "control_gil_v5_test" } }, "geotext": { @@ -21,9 +21,9 @@ "requestTimeout": 60000, "properties": { "index": { - "geotext": "geotext_index", - "placetypes": "placetypes_index", - "hierarchies": "hierarchies_index" + "geotext": "geotext_index_test", + "placetypes": "placetypes_index_test", + "hierarchies": "hierarchies_index_test" }, "textTermLanguage": "en" } @@ -32,14 +32,14 @@ "s3": { "endpoint": "http://localhost:9000", "credentials": { - "accessKeyId": "O8JnyGPmTtIz69uZihyh", - "secretAccessKey": "1uzitcz85XFpZ8oxedJMoQpCuDSiJrhfDPbAvPOM" + "accessKeyId": "minio", + "secretAccessKey": "minio123" }, "region": "local", "forcePathStyle": true, "files": { "latLonConvertionTable": { - "bucket": "geocoding", + "bucket": "geocoding-test", "fileName": "table.json" } } From 6caae7df6455057f5a39afc3a5149da844c238aa Mon Sep 17 00:00:00 2001 From: Niv Greenstein Date: Thu, 5 Sep 2024 17:24:42 +0300 Subject: [PATCH 196/262] feat: updated dev scripts --- devScripts/geotextElasticsearchData.json | 26 ++++++++-------- devScripts/importDataToElastic.ts | 22 +++++++------ devScripts/importDataToS3.ts | 39 ++++++++++++++++++++++++ devScripts/index.ts | 9 +++--- 4 files changed, 70 insertions(+), 26 deletions(-) create mode 100644 devScripts/importDataToS3.ts diff --git a/devScripts/geotextElasticsearchData.json b/devScripts/geotextElasticsearchData.json index c91160a3..0581ea2c 100644 --- a/devScripts/geotextElasticsearchData.json +++ b/devScripts/geotextElasticsearchData.json @@ -1,7 +1,7 @@ [ { "_id": "12cd98ee-82eb-415e-a1d8-e1667a6f2e7f", - "_index": "geotext_index", + "_index": "geotext", "_source": { "source": "OSM", "layer_name": "osm_airports", @@ -66,7 +66,7 @@ }, { "_id": "8b5009b4-b643-4856-905d-ac95a2040aca", - "_index": "geotext_index", + "_index": "geotext", "_source": { "source": "OSM", "layer_name": "osm_airports", @@ -117,7 +117,7 @@ }, { "_id": "2bd0d4bb-af93-4211-9499-174c56feddb5", - "_index": "geotext_index", + "_index": "geotext", "_source": { "source": "OSM", "layer_name": "osm_airports", @@ -170,7 +170,7 @@ }, { "_id": "ba951f2f-08c2-446e-845d-e2b0547d5d8c", - "_index": "geotext_index", + "_index": "geotext", "_source": { "source": "OSM", "layer_name": "osm_ports", @@ -224,7 +224,7 @@ }, { "_id": "1bb11f54-939e-457b-bf68-a3920ccf629c", - "_index": "geotext_index", + "_index": "geotext", "_source": { "source": "GOOGLE", "layer_name": "google_ports", @@ -278,7 +278,7 @@ }, { "_id": "2b3dd420-d265-4665-94a6-3e13650b7a2d", - "_index": "geotext_index", + "_index": "geotext", "_source": { "source": "OSM", "layer_name": "osm_schools", @@ -319,7 +319,7 @@ }, { "_id": "dc02a3f9-156a-4f61-85bd-fd040cd322a3", - "_index": "geotext_index", + "_index": "geotext", "_source": { "source": "OSM", "layer_name": "osm_schools", @@ -358,7 +358,7 @@ }, { "_id": "5a54ff54-53f6-4ad1-9d2a-cd20f333ee2b", - "_index": "hierarchies_index", + "_index": "hierarchies", "_source": { "geo_json": { "coordinates": [ @@ -384,7 +384,7 @@ }, { "_id": "a3f38afc-1ce6-443c-a302-a0103c2bcb7d", - "_index": "hierarchies_index", + "_index": "hierarchies", "_source": { "geo_json": { "coordinates": [ @@ -409,7 +409,7 @@ }, { "_id": "a3f38afc-1ce6-443c-a302-a0103c2bcb7d", - "_index": "hierarchies_index", + "_index": "hierarchies", "_source": { "geo_json": { "coordinates": [ @@ -434,7 +434,7 @@ }, { "_id": "430ddc04-2370-415f-8761-b8c66a142f0a", - "_index": "placetypes_index", + "_index": "placetypes", "_source": { "placetype": "transportation", "sub_placetype": "airport", @@ -443,7 +443,7 @@ }, { "_id": "6941ab8e-7503-4f8a-94f2-4750b2698db9", - "_index": "placetypes_index", + "_index": "placetypes", "_source": { "placetype": "transportation", "sub_placetype": "port", @@ -452,7 +452,7 @@ }, { "_id": "6941ab8e-7503-4f8a-94f2-4750b2698db9", - "_index": "placetypes_index", + "_index": "placetypes", "_source": { "placetype": "education", "sub_placetype": "school", diff --git a/devScripts/importDataToElastic.ts b/devScripts/importDataToElastic.ts index 849b34c7..3679b505 100644 --- a/devScripts/importDataToElastic.ts +++ b/devScripts/importDataToElastic.ts @@ -3,28 +3,31 @@ /* eslint-disable @typescript-eslint/no-unsafe-member-access */ import crypto from 'crypto'; import { Client } from '@elastic/elasticsearch'; -import config from '../config/default.json'; +import { ElasticDbClientsConfig } from '../src/common/elastic/interfaces'; +import { elasticConfigPath } from '../src/common/constants'; +import { IConfig } from '../src/common/interfaces'; import controlData from './controlElasticsearchData.json'; import geotextData from './geotextElasticsearchData.json'; -const main = async (): Promise => { - const controlClient = new Client({ ...config.db.elastic.control }); - const geotextClient = new Client({ ...config.db.elastic.geotext }); +const main = async (config: IConfig): Promise => { + const elasticConfig = config.get(elasticConfigPath); + const controlClient = new Client({ ...elasticConfig.control }); + const geotextClient = new Client({ ...elasticConfig.geotext }); for (const { client, index, key } of [ { client: controlClient, - index: config.db.elastic.control.properties.index, + index: elasticConfig.control.properties.index as string, key: 'geometry', }, { client: geotextClient, - index: config.db.elastic.geotext.properties.index.geotext, + index: (elasticConfig.geotext.properties.index as { [key: string]: string }).geotext, key: 'geo_json', }, { client: geotextClient, - index: config.db.elastic.geotext.properties.index.hierarchies, + index: (elasticConfig.geotext.properties.index as { [key: string]: string }).hierarchies, key: 'geo_json', }, ]) { @@ -44,7 +47,7 @@ const main = async (): Promise => { for (const item of controlData) { await controlClient.index({ - index: config.db.elastic.control.properties.index, + index: elasticConfig.control.properties.index as string, id: crypto.randomUUID(), body: item._source, }); @@ -52,7 +55,8 @@ const main = async (): Promise => { for (const item of geotextData) { await geotextClient.index({ - index: item._index, + // eslint-disable-next-line @typescript-eslint/no-unnecessary-type-assertion + index: (elasticConfig.geotext.properties.index as { [key: string]: string })[item._index] as string, id: crypto.randomUUID(), body: { ...item._source, diff --git a/devScripts/importDataToS3.ts b/devScripts/importDataToS3.ts new file mode 100644 index 00000000..74aacee0 --- /dev/null +++ b/devScripts/importDataToS3.ts @@ -0,0 +1,39 @@ +/* eslint-disable @typescript-eslint/no-unsafe-assignment */ +/* eslint-disable @typescript-eslint/naming-convention */ +/* eslint-disable @typescript-eslint/no-unsafe-member-access */ +import { CreateBucketCommand, PutObjectCommand, S3Client } from '@aws-sdk/client-s3'; +import { S3Config, s3ConfigPath } from '../src/common/s3'; +import { IConfig } from '../src/common/interfaces'; +import mockDataJson from './latLonConvertions.json'; + +const main = async (config: IConfig): Promise => { + const s3Config = config.get(s3ConfigPath); + const s3Client = new S3Client({ ...s3Config }); + + if (!s3Config.files.latLonConvertionTable) { + throw new Error('No latLonConvertionTable file path provided'); + } + + const { bucket: Bucket, fileName: Key } = s3Config.files.latLonConvertionTable; + + try { + await s3Client.send(new CreateBucketCommand({ Bucket, ACL: 'public-read' })); + } catch (error) { + console.error(error); + } + + try { + await s3Client.send( + new PutObjectCommand({ + Bucket, + Key, + Body: Buffer.from(JSON.stringify(mockDataJson), 'utf-8'), + }) + ); + } catch (error) { + console.error(error); + throw error; + } +}; + +export default main; diff --git a/devScripts/index.ts b/devScripts/index.ts index 1ec5acdc..7a759df3 100644 --- a/devScripts/index.ts +++ b/devScripts/index.ts @@ -1,10 +1,11 @@ +import config from 'config'; import importDataToElastic from './importDataToElastic'; -import importToPostgres from './importToPostgres'; +import importDataToS3 from './importDataToS3'; -importDataToElastic() +importDataToElastic(config) .then(() => console.log('Success import data to elastic')) .catch(console.error); -importToPostgres() - .then(() => console.log('Success import data to postgres')) +importDataToS3(config) + .then(() => console.log('Success import data to s3')) .catch(console.error); From 0b6d3183f098b7b911d5f599da987487f14632dd Mon Sep 17 00:00:00 2001 From: Niv Greenstein Date: Thu, 5 Sep 2024 17:24:50 +0300 Subject: [PATCH 197/262] fix: fixed typo --- src/common/elastic/index.ts | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/common/elastic/index.ts b/src/common/elastic/index.ts index 51645492..da80c831 100644 --- a/src/common/elastic/index.ts +++ b/src/common/elastic/index.ts @@ -19,7 +19,7 @@ const initElasticsearchClient = (clientOptions: ClientOptions): ElasticClient => return client; }; -export const elasticClientFactory: FactoryFunction = (container: DependencyContainer): ElasticClients => { +export const elasticClientsFactory: FactoryFunction = (container: DependencyContainer): ElasticClients => { const config = container.resolve(SERVICES.CONFIG); const logger = container.resolve(SERVICES.LOGGER); From 26791c45a02bc086c1b185fd052582bb9373972d Mon Sep 17 00:00:00 2001 From: Niv Greenstein Date: Thu, 5 Sep 2024 17:25:01 +0300 Subject: [PATCH 198/262] feat: added CLEANUP_REGISTRY symbol --- src/common/constants.ts | 1 + 1 file changed, 1 insertion(+) diff --git a/src/common/constants.ts b/src/common/constants.ts index 1d45c6d8..b570de47 100644 --- a/src/common/constants.ts +++ b/src/common/constants.ts @@ -15,6 +15,7 @@ export const SERVICES: Record = { APPLICATION: Symbol('Application'), ELASTIC_CLIENTS: Symbol('ElasticClients'), S3_CLIENT: Symbol('S3Client'), + CLEANUP_REGISTRY: Symbol('CleanupRegistry'), }; /* eslint-enable @typescript-eslint/naming-convention */ From 3dc6ded8eeb2d9218781da6b860a9b1eef1d6afd Mon Sep 17 00:00:00 2001 From: Niv Greenstein Date: Thu, 5 Sep 2024 17:28:04 +0300 Subject: [PATCH 199/262] feat: update imports,provide cleanup registry, change cleanupRegistry func to return Promise.resolve --- src/containerConfig.ts | 26 +++++++++++++++++++++----- 1 file changed, 21 insertions(+), 5 deletions(-) diff --git a/src/containerConfig.ts b/src/containerConfig.ts index 25e60c0b..75c27816 100644 --- a/src/containerConfig.ts +++ b/src/containerConfig.ts @@ -11,7 +11,7 @@ import { ScheduledTask } from 'node-cron'; import { HEALTHCHECK, ON_SIGNAL, SERVICES, SERVICE_NAME } from './common/constants'; import { tracing } from './common/tracing'; import { InjectionObject, registerDependencies } from './common/dependencyRegistration'; -import { elasticClientFactory, ElasticClients } from './common/elastic'; +import { elasticClientsFactory, ElasticClients } from './common/elastic'; import { IApplication } from './common/interfaces'; import { TILE_REPOSITORY_SYMBOL, tileRepositoryFactory } from './control/tile/DAL/tileRepository'; import { TILE_ROUTER_SYMBOL, tileRouterFactory } from './control/tile/routes/tileRouter'; @@ -62,12 +62,15 @@ export const registerExternalValues = async (options?: RegisterOptions): Promise { token: ON_SIGNAL, provider: { - useValue: cleanupRegistry.trigger.bind(cleanupRegistry), + useFactory: instancePerContainerCachingFactory((container) => { + const cleanupRegistry = container.resolve(SERVICES.CLEANUP_REGISTRY); + return cleanupRegistry.trigger.bind(cleanupRegistry); + }), }, }, { token: SERVICES.ELASTIC_CLIENTS, - provider: { useFactory: instancePerContainerCachingFactory(elasticClientFactory) }, + provider: { useFactory: instancePerContainerCachingFactory(elasticClientsFactory) }, postInjectionHook: async (deps: DependencyContainer): Promise => { const elasticClients = deps.resolve(SERVICES.ELASTIC_CLIENTS); try { @@ -100,13 +103,23 @@ export const registerExternalValues = async (options?: RegisterOptions): Promise } catch (err) { logger.error('Failed to connect to S3', err); } - cleanupRegistry.register({ func: async () => s3Client.destroy(), id: SERVICES.S3_CLIENT }); + cleanupRegistry.register({ + func: async () => { + s3Client.destroy(); + return Promise.resolve(); + }, + id: SERVICES.S3_CLIENT, + }); }, }, { token: S3_REPOSITORY_SYMBOL, provider: { useFactory: s3RepositoryFactory }, }, + { + token: SERVICES.CLEANUP_REGISTRY, + provider: { useValue: cleanupRegistry }, + }, { token: TILE_REPOSITORY_SYMBOL, provider: { useFactory: tileRepositoryFactory } }, { token: TILE_ROUTER_SYMBOL, provider: { useFactory: tileRouterFactory } }, { token: ITEM_REPOSITORY_SYMBOL, provider: { useFactory: itemRepositoryFactory } }, @@ -125,7 +138,10 @@ export const registerExternalValues = async (options?: RegisterOptions): Promise const cronLoadTileLatLonData = deps.resolve(cronLoadTileLatLonDataSymbol); cronLoadTileLatLonData.start(); cleanupRegistry.register({ - func: async () => cronLoadTileLatLonData.stop.bind(cronLoadTileLatLonData), + func: async () => { + cronLoadTileLatLonData.stop(); + return Promise.resolve(); + }, id: cronLoadTileLatLonDataSymbol, }); }, From be5b38b42ac4343c3d02fda2318e2517663bcee8 Mon Sep 17 00:00:00 2001 From: Niv Greenstein Date: Thu, 5 Sep 2024 17:30:08 +0300 Subject: [PATCH 200/262] feat: added globalTeardown and globalSetup for tests. --- .dockerignore | 1 + .../configurations/integration/jest.config.js | 2 + tests/integration/control/item/item.spec.ts | 14 +++- tests/integration/control/route/route.spec.ts | 14 +++- tests/integration/control/tile/tile.spec.ts | 14 +++- tests/integration/globalSetup.ts | 39 +++++++++ tests/integration/globalTeardown.ts | 84 +++++++++++++++++++ tests/integration/latLon/latLon.spec.ts | 71 +++------------- tests/integration/location/location.spec.ts | 15 +++- tests/integration/setupMockData.ts | 84 +++++++++++++++++++ 10 files changed, 275 insertions(+), 63 deletions(-) create mode 100644 tests/integration/globalSetup.ts create mode 100644 tests/integration/globalTeardown.ts create mode 100644 tests/integration/setupMockData.ts diff --git a/.dockerignore b/.dockerignore index bfbdee64..4cb8c68a 100644 --- a/.dockerignore +++ b/.dockerignore @@ -16,3 +16,4 @@ commitlint.config.json dist build reports +devScripts \ No newline at end of file diff --git a/tests/configurations/integration/jest.config.js b/tests/configurations/integration/jest.config.js index 3ae834e9..673b558e 100644 --- a/tests/configurations/integration/jest.config.js +++ b/tests/configurations/integration/jest.config.js @@ -11,6 +11,8 @@ module.exports = { testMatch: ['/tests/integration/**/*.spec.ts'], setupFiles: ['/tests/configurations/jest.setup.ts'], setupFilesAfterEnv: ['jest-openapi', '/tests/configurations/initJestOpenapi.setup.ts'], + globalSetup: '/tests/integration/globalSetup.ts', + globalTeardown: '/tests/integration/globalTeardown.ts', reporters: [ 'default', [ diff --git a/tests/integration/control/item/item.spec.ts b/tests/integration/control/item/item.spec.ts index 70d00a39..350353ea 100644 --- a/tests/integration/control/item/item.spec.ts +++ b/tests/integration/control/item/item.spec.ts @@ -1,6 +1,9 @@ /* eslint-disable @typescript-eslint/naming-convention */ import jsLogger from '@map-colonies/js-logger'; import { trace } from '@opentelemetry/api'; +import { CleanupRegistry } from '@map-colonies/cleanup-registry'; +import { Application } from 'express'; +import { DependencyContainer } from 'tsyringe'; import httpStatusCodes from 'http-status-codes'; import { getApp } from '../../../../src/app'; import { SERVICES } from '../../../../src/common/constants'; @@ -16,9 +19,10 @@ import { ITEM_1234, ITEM_1235, ITEM_1236 } from './mockObjects'; describe('/search/control/items', function () { let requestSender: ItemRequestSender; + let app: { app: Application; container: DependencyContainer }; beforeEach(async function () { - const app = await getApp({ + app = await getApp({ override: [ { token: SERVICES.LOGGER, provider: { useValue: jsLogger({ enabled: false }) } }, { token: SERVICES.TRACER, provider: { useValue: trace.getTracer('testTracer') } }, @@ -32,6 +36,14 @@ describe('/search/control/items', function () { requestSender = new ItemRequestSender(app.app); }); + afterAll(async function () { + const cleanupRegistry = app.container.resolve(SERVICES.CLEANUP_REGISTRY); + await cleanupRegistry.trigger(); + app.container.reset(); + + jest.clearAllTimers(); + }); + describe('Happy Path', function () { it('should return 200 status code and items', async function () { const requestParams: GetItemsQueryParams = { command_name: '123', limit: 5, disable_fuzziness: false }; diff --git a/tests/integration/control/route/route.spec.ts b/tests/integration/control/route/route.spec.ts index 2e1dcc6c..a7097bc3 100644 --- a/tests/integration/control/route/route.spec.ts +++ b/tests/integration/control/route/route.spec.ts @@ -2,6 +2,9 @@ import jsLogger from '@map-colonies/js-logger'; import { trace } from '@opentelemetry/api'; import httpStatusCodes from 'http-status-codes'; +import { Application } from 'express'; +import { DependencyContainer } from 'tsyringe'; +import { CleanupRegistry } from '@map-colonies/cleanup-registry'; import { getApp } from '../../../../src/app'; import { SERVICES } from '../../../../src/common/constants'; import { GetRoutesQueryParams } from '../../../../src/control/route/controllers/routeController'; @@ -16,9 +19,10 @@ import { ROUTE_VIA_CAMILLUCCIA_A, ROUTE_VIA_CAMILLUCCIA_B, CONTROL_POINT_OLIMPIA describe('/search/control/route', function () { let requestSender: RouteRequestSender; + let app: { app: Application; container: DependencyContainer }; beforeEach(async function () { - const app = await getApp({ + app = await getApp({ override: [ { token: SERVICES.LOGGER, provider: { useValue: jsLogger({ enabled: false }) } }, { token: SERVICES.TRACER, provider: { useValue: trace.getTracer('testTracer') } }, @@ -32,6 +36,14 @@ describe('/search/control/route', function () { requestSender = new RouteRequestSender(app.app); }); + afterAll(async function () { + const cleanupRegistry = app.container.resolve(SERVICES.CLEANUP_REGISTRY); + await cleanupRegistry.trigger(); + app.container.reset(); + + jest.clearAllTimers(); + }); + describe('Happy Path', function () { it('should return 200 status code and routes', async function () { const requestParams: GetRoutesQueryParams = { command_name: 'via camilluccia', limit: 5, disable_fuzziness: false }; diff --git a/tests/integration/control/tile/tile.spec.ts b/tests/integration/control/tile/tile.spec.ts index 5f2e679c..7ae212a8 100644 --- a/tests/integration/control/tile/tile.spec.ts +++ b/tests/integration/control/tile/tile.spec.ts @@ -1,6 +1,9 @@ /* eslint-disable @typescript-eslint/naming-convention */ import jsLogger from '@map-colonies/js-logger'; import { trace } from '@opentelemetry/api'; +import { DependencyContainer } from 'tsyringe'; +import { Application } from 'express'; +import { CleanupRegistry } from '@map-colonies/cleanup-registry'; import httpStatusCodes from 'http-status-codes'; import { getApp } from '../../../../src/app'; import { SERVICES } from '../../../../src/common/constants'; @@ -16,9 +19,10 @@ import { RIC_TILE, RIT_TILE, SUB_TILE_65, SUB_TILE_66 } from './mockObjects'; describe('/search/control/tiles', function () { let requestSender: TileRequestSender; + let app: { app: Application; container: DependencyContainer }; beforeEach(async function () { - const app = await getApp({ + app = await getApp({ override: [ { token: SERVICES.LOGGER, provider: { useValue: jsLogger({ enabled: false }) } }, { token: SERVICES.TRACER, provider: { useValue: trace.getTracer('testTracer') } }, @@ -32,6 +36,14 @@ describe('/search/control/tiles', function () { requestSender = new TileRequestSender(app.app); }); + afterAll(async function () { + const cleanupRegistry = app.container.resolve(SERVICES.CLEANUP_REGISTRY); + await cleanupRegistry.trigger(); + app.container.reset(); + + jest.clearAllTimers(); + }); + describe('Happy Path', function () { it('should return 200 status code and tiles', async function () { const requestParams: GetTilesQueryParams = { tile: 'RIT', limit: 5, disable_fuzziness: false }; diff --git a/tests/integration/globalSetup.ts b/tests/integration/globalSetup.ts new file mode 100644 index 00000000..ee34e0d7 --- /dev/null +++ b/tests/integration/globalSetup.ts @@ -0,0 +1,39 @@ +import 'reflect-metadata'; +import jsLogger from '@map-colonies/js-logger'; +import { CleanupRegistry } from '@map-colonies/cleanup-registry'; +import { trace } from '@opentelemetry/api'; +import { getApp } from '../../src/app'; +import { SERVICES } from '../../src/common/constants'; +import { IConfig } from '../../src/common/interfaces'; +import importDataToS3 from '../../devScripts/importDataToS3'; +import importDataToElastic from '../../devScripts/importDataToElastic'; +import { cronLoadTileLatLonDataSymbol } from '../../src/latLon/DAL/latLonDAL'; + +export default async () => { + const app = await getApp({ + override: [ + { token: SERVICES.LOGGER, provider: { useValue: jsLogger({ enabled: false }) } }, + { token: SERVICES.TRACER, provider: { useValue: trace.getTracer('testTracer') } }, + { + token: cronLoadTileLatLonDataSymbol, + provider: { useValue: {} }, + }, + ], + useChild: true, + }); + + const config = app.container.resolve(SERVICES.CONFIG); + + await Promise.allSettled([await importDataToS3(config), await importDataToElastic(config)]).then((results) => { + results.forEach((result) => { + if (result.status === 'rejected') { + throw result.reason; + } + }); + }); + + const cleanupRegistry = app.container.resolve(SERVICES.CLEANUP_REGISTRY); + await cleanupRegistry.trigger(); + app.container.reset(); + return; +}; diff --git a/tests/integration/globalTeardown.ts b/tests/integration/globalTeardown.ts new file mode 100644 index 00000000..96003dc9 --- /dev/null +++ b/tests/integration/globalTeardown.ts @@ -0,0 +1,84 @@ +/* eslint-disable @typescript-eslint/naming-convention */ +import 'reflect-metadata'; +import jsLogger from '@map-colonies/js-logger'; +import { CleanupRegistry } from '@map-colonies/cleanup-registry'; +import { trace } from '@opentelemetry/api'; +import { DeleteBucketCommand, DeleteObjectCommand, S3Client } from '@aws-sdk/client-s3'; +import { getApp } from '../../src/app'; +import { elasticConfigPath, SERVICES } from '../../src/common/constants'; +import { IConfig } from '../../src/common/interfaces'; +import { S3Config, s3ConfigPath } from '../../src/common/s3'; +import { ElasticDbClientsConfig } from '../../src/common/elastic/interfaces'; +import { ElasticClients } from '../../src/common/elastic'; +import { cronLoadTileLatLonDataSymbol } from '../../src/latLon/DAL/latLonDAL'; + +export default async (): Promise => { + const app = await getApp({ + override: [ + { token: SERVICES.LOGGER, provider: { useValue: jsLogger({ enabled: false }) } }, + { token: SERVICES.TRACER, provider: { useValue: trace.getTracer('testTracer') } }, + { + token: cronLoadTileLatLonDataSymbol, + provider: { useValue: {} }, + }, + ], + useChild: true, + }); + + const config = app.container.resolve(SERVICES.CONFIG); + + const s3Client = app.container.resolve(SERVICES.S3_CLIENT); + const elasticClients = app.container.resolve(SERVICES.ELASTIC_CLIENTS); + + const s3Config = config.get(s3ConfigPath); + const elasticClientsConfig = config.get(elasticConfigPath); + + const clearS3Data = new Promise((resolve, reject) => { + void (async (): Promise => { + try { + if (s3Config.files.latLonConvertionTable !== undefined) { + await s3Client.send( + new DeleteObjectCommand({ Bucket: s3Config.files.latLonConvertionTable.bucket, Key: s3Config.files.latLonConvertionTable.fileName }) + ); + await s3Client.send(new DeleteBucketCommand({ Bucket: s3Config.files.latLonConvertionTable.bucket })); + } + resolve(); + } catch (error) { + console.error(error); + reject(error); + } + })(); + }); + + const clearElasticData = new Promise((resolve, reject) => { + void (async (): Promise => { + try { + for (const [key, value] of Object.entries(elasticClientsConfig)) { + await elasticClients[key as keyof ElasticDbClientsConfig].indices.delete({ + index: typeof value.properties.index === 'string' ? value.properties.index : Object.values(value.properties.index), + }); + } + resolve(); + } catch (error) { + console.error(error); + reject(error); + } + })(); + }); + + await Promise.allSettled([clearS3Data, clearElasticData]).then((results) => { + results.forEach((result) => { + if (result.status === 'rejected') { + throw result.reason; + } + }); + }); + + const cleanupRegistry = app.container.resolve(SERVICES.CLEANUP_REGISTRY); + await cleanupRegistry.trigger(); + app.container.reset(); + await app.container.dispose(); + + console.log('Global Teardown completed'); + return; +}; diff --git a/tests/integration/latLon/latLon.spec.ts b/tests/integration/latLon/latLon.spec.ts index e31f877c..b91192e2 100644 --- a/tests/integration/latLon/latLon.spec.ts +++ b/tests/integration/latLon/latLon.spec.ts @@ -1,84 +1,37 @@ /* eslint-disable @typescript-eslint/naming-convention */ -import { IConfig } from 'config'; -import { PutObjectCommand, CreateBucketCommand, DeleteBucketCommand, DeleteObjectCommand, S3Client } from '@aws-sdk/client-s3'; import jsLogger from '@map-colonies/js-logger'; import { trace } from '@opentelemetry/api'; +import { Application } from 'express'; +import { DependencyContainer } from 'tsyringe'; +import { CleanupRegistry } from '@map-colonies/cleanup-registry'; import httpStatusCodes from 'http-status-codes'; import { getApp } from '../../../src/app'; import { SERVICES } from '../../../src/common/constants'; -import { S3Config, s3ConfigPath } from '../../../src/common/s3'; -import mockDataJson from '../../../devScripts/latLonConvertions.json'; import { LatLonDAL } from '../../../src/latLon/DAL/latLonDAL'; import { LatLonRequestSender } from './helpers/requestSender'; describe('/lookup', function () { let requestSender: LatLonRequestSender; - let s3Client: S3Client; - let s3Config: S3Config | undefined; + let app: { app: Application; container: DependencyContainer }; - beforeAll(async function () { - const app = await getApp({ + beforeEach(async function () { + app = await getApp({ override: [ { token: SERVICES.LOGGER, provider: { useValue: jsLogger({ enabled: false }) } }, { token: SERVICES.TRACER, provider: { useValue: trace.getTracer('testTracer') } }, ], useChild: true, }); - const config = app.container.resolve(SERVICES.CONFIG); - s3Client = app.container.resolve(SERVICES.S3_CLIENT); - - s3Config = config.get(s3ConfigPath); - - if (s3Config === undefined || s3Config.files.latLonConvertionTable === undefined) { - throw new Error('S3 configuration is missing'); - } - - const { bucket: Bucket, fileName: Key } = s3Config.files.latLonConvertionTable; - - try { - await s3Client.send(new CreateBucketCommand({ Bucket, ACL: 'public-read' })); - } catch (error) { - console.error(error); - } - - try { - await s3Client.send( - new PutObjectCommand({ - Bucket, - Key, - Body: Buffer.from(JSON.stringify(mockDataJson), 'utf-8'), - }) - ); - } catch (error) { - console.error(error); - throw error; - } - }); - afterAll(async function () { - try { - if (s3Config?.files.latLonConvertionTable !== undefined) { - await s3Client.send( - new DeleteObjectCommand({ Bucket: s3Config.files.latLonConvertionTable.bucket, Key: s3Config.files.latLonConvertionTable.fileName }) - ); - await s3Client.send(new DeleteBucketCommand({ Bucket: s3Config.files.latLonConvertionTable.bucket })); - } - } catch (error) { - console.error(error); - throw error; - } + requestSender = new LatLonRequestSender(app.app); }); - beforeEach(async function () { - const app = await getApp({ - override: [ - { token: SERVICES.LOGGER, provider: { useValue: jsLogger({ enabled: false }) } }, - { token: SERVICES.TRACER, provider: { useValue: trace.getTracer('testTracer') } }, - ], - useChild: true, - }); + afterAll(async function () { + const cleanupRegistry = app.container.resolve(SERVICES.CLEANUP_REGISTRY); + await cleanupRegistry.trigger(); + app.container.reset(); - requestSender = new LatLonRequestSender(app.app); + jest.clearAllTimers(); }); describe('Happy Path', function () { diff --git a/tests/integration/location/location.spec.ts b/tests/integration/location/location.spec.ts index ee6d9831..2beb3926 100644 --- a/tests/integration/location/location.spec.ts +++ b/tests/integration/location/location.spec.ts @@ -1,5 +1,8 @@ /* eslint-disable @typescript-eslint/naming-convention */ import config from 'config'; +import { DependencyContainer } from 'tsyringe'; +import { Application } from 'express'; +import { CleanupRegistry } from '@map-colonies/cleanup-registry'; import jsLogger from '@map-colonies/js-logger'; import { trace } from '@opentelemetry/api'; import httpStatusCodes from 'http-status-codes'; @@ -28,9 +31,10 @@ import { expectedResponse, hierarchiesWithAnyWieght } from './utils'; describe('/search/location', function () { let requestSender: LocationRequestSender; + let app: { app: Application; container: DependencyContainer }; beforeEach(async function () { - const app = await getApp({ + app = await getApp({ override: [ { token: SERVICES.LOGGER, provider: { useValue: jsLogger({ enabled: false }) } }, { token: SERVICES.TRACER, provider: { useValue: trace.getTracer('testTracer') } }, @@ -44,6 +48,15 @@ describe('/search/location', function () { requestSender = new LocationRequestSender(app.app); }); + afterAll(async function () { + const cleanupRegistry = app.container.resolve(SERVICES.CLEANUP_REGISTRY); + await cleanupRegistry.trigger(); + nock.cleanAll(); + app.container.reset(); + + jest.clearAllTimers(); + }); + describe('Happy Path', function () { it('should return 200 status code and airports', async function () { const requestParams: GetGeotextSearchParams = { query: 'airport', limit: 5, disable_fuzziness: true }; diff --git a/tests/integration/setupMockData.ts b/tests/integration/setupMockData.ts new file mode 100644 index 00000000..464d2eea --- /dev/null +++ b/tests/integration/setupMockData.ts @@ -0,0 +1,84 @@ +// /* eslint-disable @typescript-eslint/naming-convention */ +// import jsLogger from '@map-colonies/js-logger'; +// import { DeleteBucketCommand, DeleteObjectCommand, S3Client } from '@aws-sdk/client-s3'; +// import { trace } from '@opentelemetry/api'; +// import { getApp } from '../../src/app'; +// import { elasticConfigPath, SERVICES } from '../../src/common/constants'; +// import { S3Config, s3ConfigPath } from '../../src/common/s3'; +// import { IConfig } from '../../src/common/interfaces'; +// import importDataToS3 from '../../devScripts/importDataToS3'; +// import importDataToElastic from '../../devScripts/importDataToElastic'; +// import { ElasticClients } from '../../src/common/elastic'; +// import { ElasticDbClientsConfig } from '../../src/common/elastic/interfaces'; + +// let s3Client: S3Client; +// let s3Config: S3Config | undefined; +// let elasticClients: ElasticClients; +// let elasticClientsConfig: ElasticDbClientsConfig; + +// beforeAll(async function () { +// const app = await getApp({ +// override: [ +// { token: SERVICES.LOGGER, provider: { useValue: jsLogger({ enabled: false }) } }, +// { token: SERVICES.TRACER, provider: { useValue: trace.getTracer('testTracer') } }, +// ], +// useChild: true, +// }); + +// const config = app.container.resolve(SERVICES.CONFIG); + +// s3Client = app.container.resolve(SERVICES.S3_CLIENT); +// s3Config = config.get(s3ConfigPath); + +// elasticClients = app.container.resolve(SERVICES.ELASTIC_CLIENTS); +// elasticClientsConfig = config.get(elasticConfigPath); + +// await Promise.allSettled([await importDataToS3(config), await importDataToElastic(config)]).then((results) => { +// results.forEach((result) => { +// if (result.status === 'rejected') { +// throw result.reason; +// } +// }); +// }); +// }); + +// afterAll(async function () { +// const clearS3Data = new Promise((resolve, reject) => { +// void (async (): Promise => { +// try { +// if (s3Config?.files.latLonConvertionTable !== undefined) { +// await s3Client.send( +// new DeleteObjectCommand({ Bucket: s3Config.files.latLonConvertionTable.bucket, Key: s3Config.files.latLonConvertionTable.fileName }) +// ); +// await s3Client.send(new DeleteBucketCommand({ Bucket: s3Config.files.latLonConvertionTable.bucket })); +// } +// resolve(); +// } catch (error) { +// console.error(error); +// reject(error); +// } +// })(); +// }); + +// const clearElasticData = new Promise((resolve, reject) => { +// void (async (): Promise => { +// try { +// for (const [key, _] of Object.entries(elasticClientsConfig)) { +// await elasticClients[key as keyof ElasticDbClientsConfig].indices.delete({ index: '*' }); +// } +// resolve(); +// } catch (error) { +// console.error(error); +// reject(error); +// } +// })(); +// }); + +// await Promise.allSettled([clearS3Data, clearElasticData]).then((results) => { +// results.forEach((result) => { +// if (result.status === 'rejected') { +// throw result.reason; +// } +// }); +// }); +// }); From 1c25ef953dd1dab97ceea5c9ac60e3a1b0c8c0f8 Mon Sep 17 00:00:00 2001 From: Niv Greenstein Date: Thu, 5 Sep 2024 17:46:02 +0300 Subject: [PATCH 201/262] Update pull_request.yaml --- .github/workflows/pull_request.yaml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/pull_request.yaml b/.github/workflows/pull_request.yaml index bc691d93..af8fec8e 100644 --- a/.github/workflows/pull_request.yaml +++ b/.github/workflows/pull_request.yaml @@ -60,7 +60,7 @@ jobs: minio: # Docker Hub image - image: quay.io/minio/minio:RELEASE.2024-05-28T17-19-04Z + image: minio/minio:edge-cicd env: MINIO_ROOT_USER: minio MINIO_ROOT_PASSWORD: minio From 179574cbe7d0b4862df6ad8ab5cf66bc0383a76d Mon Sep 17 00:00:00 2001 From: Niv Greenstein Date: Thu, 5 Sep 2024 17:50:22 +0300 Subject: [PATCH 202/262] changed version --- package.json | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/package.json b/package.json index 230d0157..538c30c1 100644 --- a/package.json +++ b/package.json @@ -1,6 +1,6 @@ { "name": "geocoding", - "version": "0.1.0-prealpha.2", + "version": "0.1.0-prealpha.3", "description": "Geocoding service for MapColonies", "main": "./src/index.ts", "scripts": { From d52f9d3cd22aa3f7abaca469300f7fd5deb84914 Mon Sep 17 00:00:00 2001 From: Niv Greenstein Date: Thu, 5 Sep 2024 18:05:14 +0300 Subject: [PATCH 203/262] Update pull_request.yaml --- .github/workflows/pull_request.yaml | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/.github/workflows/pull_request.yaml b/.github/workflows/pull_request.yaml index 9674cd6b..af8fec8e 100644 --- a/.github/workflows/pull_request.yaml +++ b/.github/workflows/pull_request.yaml @@ -60,7 +60,7 @@ jobs: minio: # Docker Hub image - image: quay.io/minio/minio:RELEASE.2024-05-28T17-19-04Z + image: minio/minio:edge-cicd env: MINIO_ROOT_USER: minio MINIO_ROOT_PASSWORD: minio @@ -126,4 +126,4 @@ jobs: run: docker build -t test-build:latest . - name: build migrations Docker image - run: docker build -f ./migrations.Dockerfile -t test-migrations-build:latest . \ No newline at end of file + run: docker build -f ./migrations.Dockerfile -t test-migrations-build:latest . From 872ef3b5a2b5c4cc4f84fdd8c3c12d10031ba128 Mon Sep 17 00:00:00 2001 From: Niv Greenstein Date: Thu, 5 Sep 2024 18:07:31 +0300 Subject: [PATCH 204/262] Update pull_request.yaml --- .github/workflows/pull_request.yaml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/pull_request.yaml b/.github/workflows/pull_request.yaml index af8fec8e..a971ec96 100644 --- a/.github/workflows/pull_request.yaml +++ b/.github/workflows/pull_request.yaml @@ -63,7 +63,7 @@ jobs: image: minio/minio:edge-cicd env: MINIO_ROOT_USER: minio - MINIO_ROOT_PASSWORD: minio + MINIO_ROOT_PASSWORD: minio123 ports: - 9000:9000 # Set health checks to wait until elastic has started From cb81f3faf9d3353916e25381821d4e3ce4bd5d82 Mon Sep 17 00:00:00 2001 From: Niv Greenstein Date: Thu, 5 Sep 2024 18:10:20 +0300 Subject: [PATCH 205/262] Update pull_request.yaml --- .github/workflows/pull_request.yaml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/pull_request.yaml b/.github/workflows/pull_request.yaml index a971ec96..d29ae051 100644 --- a/.github/workflows/pull_request.yaml +++ b/.github/workflows/pull_request.yaml @@ -75,7 +75,7 @@ jobs: strategy: matrix: - node: [14.x, 16.x] + node: [18.x, 20.x] steps: - name: Check out Git repository From 6632a63f2ea22ef23b43218454c1bc0b422276f0 Mon Sep 17 00:00:00 2001 From: Niv Greenstein Date: Thu, 5 Sep 2024 18:20:56 +0300 Subject: [PATCH 206/262] chore: removed data from default.json and changed it to be generic. --- config/default.json | 34 +++++++++++----------------------- 1 file changed, 11 insertions(+), 23 deletions(-) diff --git a/config/default.json b/config/default.json index ad101143..4c6af3f2 100644 --- a/config/default.json +++ b/config/default.json @@ -28,18 +28,18 @@ "db": { "elastic": { "control": { - "node": "http://localhost:9200", + "node": "http://control_elastic:9200", "auth": { "username": "elastic", "password": "changeme" }, "requestTimeout": 60000, "properties": { - "index": "control_gil_v5" + "index": "control_index" } }, "geotext": { - "node": "http://localhost:9200", + "node": "http://geotext_elastic:9200", "auth": { "username": "elastic", "password": "changeme" @@ -56,10 +56,10 @@ } }, "s3": { - "endpoint": "http://localhost:9000", + "endpoint": "http://s3:9000", "credentials": { - "accessKeyId": "O8JnyGPmTtIz69uZihyh", - "secretAccessKey": "1uzitcz85XFpZ8oxedJMoQpCuDSiJrhfDPbAvPOM" + "accessKeyId": "accessKeyId", + "secretAccessKey": "secretAccessKey" }, "forcePathStyle": true, "region": "local", @@ -73,7 +73,7 @@ }, "application": { "services": { - "tokenTypesUrl": "http://localhost:5001/NLP_ANALYSES" + "tokenTypesUrl": "http://NLP_ANALYSES" }, "cronLoadTileLatLonDataPattern": "0 * * * *", "elasticQueryBoosts": { @@ -83,22 +83,10 @@ "hierarchy": 1.1, "viewbox": 1.1 }, - "sources": { - "OSM": "OSM", - "GOOGLE": "GOOGLE" - }, - "regions": { - "USA": ["New York", "Los Angeles"], - "FRANCE": ["Paris"] - }, - "controlObjectDisplayNamePrefixes": { - "TILE": "Tile", - "SUB_TILE": "Sub Tile", - "ROUTE": "Route", - "ITEM": "Item", - "CONTROL_POINT": "Control Point" - }, - "nameTranslationsKeys": ["en", "fr"], + "sources": {}, + "regions": {}, + "controlObjectDisplayNamePrefixes": {}, + "nameTranslationsKeys": [], "mainLanguageRegex": "[a-zA-Z]" } } From 1c383fe264d4706dc016e8466038af18cb299cc1 Mon Sep 17 00:00:00 2001 From: Niv Greenstein Date: Fri, 6 Sep 2024 17:08:32 +0300 Subject: [PATCH 207/262] Update pull_request.yaml --- .github/workflows/pull_request.yaml | 3 --- 1 file changed, 3 deletions(-) diff --git a/.github/workflows/pull_request.yaml b/.github/workflows/pull_request.yaml index d29ae051..8315f335 100644 --- a/.github/workflows/pull_request.yaml +++ b/.github/workflows/pull_request.yaml @@ -124,6 +124,3 @@ jobs: - name: build Docker image run: docker build -t test-build:latest . - - - name: build migrations Docker image - run: docker build -f ./migrations.Dockerfile -t test-migrations-build:latest . From 588e673b08ca4dd5af14d85c8ab8cd67b5802cb0 Mon Sep 17 00:00:00 2001 From: Niv Greenstein Date: Sun, 8 Sep 2024 17:46:59 +0300 Subject: [PATCH 208/262] feat: added MGRS to shape --- src/containerConfig.ts | 2 + src/mgrs/controllers/mgrsController.ts | 48 ++++++++ src/mgrs/models/mgrsManager.ts | 54 +++++++++ src/mgrs/routers/mgrsRouter.ts | 16 +++ src/serverBuilder.ts | 5 +- .../integration/mgrs/helpers/requestSender.ts | 16 +++ tests/integration/mgrs/mgrs.spec.ts | 113 ++++++++++++++++++ 7 files changed, 253 insertions(+), 1 deletion(-) create mode 100644 src/mgrs/controllers/mgrsController.ts create mode 100644 src/mgrs/models/mgrsManager.ts create mode 100644 src/mgrs/routers/mgrsRouter.ts create mode 100644 tests/integration/mgrs/helpers/requestSender.ts create mode 100644 tests/integration/mgrs/mgrs.spec.ts diff --git a/src/containerConfig.ts b/src/containerConfig.ts index 75c27816..7f34d4e9 100644 --- a/src/containerConfig.ts +++ b/src/containerConfig.ts @@ -26,6 +26,7 @@ import { ITEM_REPOSITORY_SYMBOL, itemRepositoryFactory } from './control/item/DA import { s3ClientFactory } from './common/s3'; import { S3_REPOSITORY_SYMBOL, s3RepositoryFactory } from './common/s3/s3Repository'; import { healthCheckFactory } from './common/utils'; +import { MGRS_ROUTER_SYMBOL, mgrsRouterFactory } from './mgrs/routers/mgrsRouter'; export interface RegisterOptions { override?: InjectionObject[]; @@ -129,6 +130,7 @@ export const registerExternalValues = async (options?: RegisterOptions): Promise { token: LAT_LON_ROUTER_SYMBOL, provider: { useFactory: latLonRouterFactory } }, { token: GEOTEXT_REPOSITORY_SYMBOL, provider: { useFactory: geotextRepositoryFactory } }, { token: GEOTEXT_SEARCH_ROUTER_SYMBOL, provider: { useFactory: geotextSearchRouterFactory } }, + { token: MGRS_ROUTER_SYMBOL, provider: { useFactory: mgrsRouterFactory } }, { token: cronLoadTileLatLonDataSymbol, provider: { diff --git a/src/mgrs/controllers/mgrsController.ts b/src/mgrs/controllers/mgrsController.ts new file mode 100644 index 00000000..c482fc51 --- /dev/null +++ b/src/mgrs/controllers/mgrsController.ts @@ -0,0 +1,48 @@ +/* eslint-disable @typescript-eslint/naming-convention */ +import { Logger } from '@map-colonies/js-logger'; +import { BoundCounter, Meter } from '@opentelemetry/api-metrics'; +import { RequestHandler } from 'express'; +import { Feature } from 'geojson'; +import httpStatus from 'http-status-codes'; +import { injectable, inject } from 'tsyringe'; +import { SERVICES } from '../../common/constants'; +import { MgrsManager } from '../models/mgrsManager'; + +type GetTilesHandler = RequestHandler< + undefined, + | Feature + | { + type: string; + message: string; + }, + undefined, + GetTileQueryParams +>; + +export interface GetTileQueryParams { + tile: string; +} + +@injectable() +export class MgrsController { + private readonly createdResourceCounter: BoundCounter; + + public constructor( + @inject(SERVICES.LOGGER) private readonly logger: Logger, + @inject(MgrsManager) private readonly manager: MgrsManager, + @inject(SERVICES.METER) private readonly meter: Meter + ) { + this.createdResourceCounter = meter.createCounter('created_resource'); + } + + public getTile: GetTilesHandler = (req, res, next) => { + try { + const { tile } = req.query; + const response = this.manager.getTile({ tile }); + return res.status(httpStatus.OK).json(response); + } catch (error: unknown) { + this.logger.error(`MgrsController.getTile Error: ${(error as Error).message}`); + next(error); + } + }; +} diff --git a/src/mgrs/models/mgrsManager.ts b/src/mgrs/models/mgrsManager.ts new file mode 100644 index 00000000..23490801 --- /dev/null +++ b/src/mgrs/models/mgrsManager.ts @@ -0,0 +1,54 @@ +/* eslint-disable @typescript-eslint/naming-convention */ +import { IConfig } from 'config'; +import { Logger } from '@map-colonies/js-logger'; +import { BBox, Feature, Geometry } from 'geojson'; +import { inject, injectable } from 'tsyringe'; +import * as mgrs from 'mgrs'; +import { SERVICES } from '../../common/constants'; +import { IApplication } from '../../common/interfaces'; +import { GetTileQueryParams } from '../controllers/mgrsController'; +import { parseGeo } from '../../location/utils'; +import { BadRequestError } from '../../common/errors'; + +@injectable() +export class MgrsManager { + public constructor( + @inject(SERVICES.LOGGER) private readonly logger: Logger, + @inject(SERVICES.CONFIG) private readonly config: IConfig, + @inject(SERVICES.APPLICATION) private readonly application: IApplication + ) {} + + public getTile({ tile }: GetTileQueryParams): Feature & { geocoding: { [key: string]: unknown } } { + let bbox: BBox | undefined; + try { + bbox = mgrs.inverse(tile); + } catch (error) { + if ((error as Error).message.includes('MGRSPoint bad conversion')) { + throw new BadRequestError('Invalid MGRS tile'); + } + this.logger.error(`Failed to convert MGRS tile to bbox. Error: ${(error as Error).message}`); + throw error; + } + + const geometry = parseGeo({ bbox }) as Geometry; + + return { + type: 'Feature', + geocoding: { + query: { + tile, + }, + response: { + max_score: 1, + results_count: 1, + match_latency_ms: 0, + }, + }, + bbox, + geometry, + properties: { + score: 1, + }, + }; + } +} diff --git a/src/mgrs/routers/mgrsRouter.ts b/src/mgrs/routers/mgrsRouter.ts new file mode 100644 index 00000000..77ac0714 --- /dev/null +++ b/src/mgrs/routers/mgrsRouter.ts @@ -0,0 +1,16 @@ +import { Router } from 'express'; +import { FactoryFunction } from 'tsyringe'; +import { MgrsController } from '../controllers/mgrsController'; + +const mgrsRouterFactory: FactoryFunction = (dependencyContainer) => { + const router = Router(); + const controller = dependencyContainer.resolve(MgrsController); + + router.get('/tiles', controller.getTile); + + return router; +}; + +export const MGRS_ROUTER_SYMBOL = Symbol('mgrsRouterFactory'); + +export { mgrsRouterFactory }; diff --git a/src/serverBuilder.ts b/src/serverBuilder.ts index b67b2853..729fd433 100644 --- a/src/serverBuilder.ts +++ b/src/serverBuilder.ts @@ -16,6 +16,7 @@ import { ROUTE_ROUTER_SYMBOL } from './control/route/routes/routeRouter'; import { LAT_LON_ROUTER_SYMBOL } from './latLon/routes/latLonRouter'; import { GEOTEXT_SEARCH_ROUTER_SYMBOL } from './location/routes/locationRouter'; import { cronLoadTileLatLonDataSymbol } from './latLon/DAL/latLonDAL'; +import { MGRS_ROUTER_SYMBOL } from './mgrs/routers/mgrsRouter'; @injectable() export class ServerBuilder { @@ -29,7 +30,8 @@ export class ServerBuilder { @inject(ROUTE_ROUTER_SYMBOL) private readonly routeRouter: Router, @inject(LAT_LON_ROUTER_SYMBOL) private readonly latLonRouter: Router, @inject(GEOTEXT_SEARCH_ROUTER_SYMBOL) private readonly geotextRouter: Router, - @inject(cronLoadTileLatLonDataSymbol) private readonly cronLoadTileLatLonData: void + @inject(cronLoadTileLatLonDataSymbol) private readonly cronLoadTileLatLonData: void, + @inject(MGRS_ROUTER_SYMBOL) private readonly mgrsRouter: Router ) { this.serverInstance = express(); // eslint-disable-next-line @typescript-eslint/no-unused-expressions @@ -58,6 +60,7 @@ export class ServerBuilder { const router = Router(); router.use('/location', this.geotextRouter); router.use('/control', this.buildControlRoutes()); + router.use('/MGRS', this.mgrsRouter); this.serverInstance.use('/search/', router); this.serverInstance.use('/lookup', this.latLonRouter); diff --git a/tests/integration/mgrs/helpers/requestSender.ts b/tests/integration/mgrs/helpers/requestSender.ts new file mode 100644 index 00000000..f12c9483 --- /dev/null +++ b/tests/integration/mgrs/helpers/requestSender.ts @@ -0,0 +1,16 @@ +import * as supertest from 'supertest'; +import { GetTileQueryParams } from '../../../../src/mgrs/controllers/mgrsController'; + +export class MgrsRequestSender { + public constructor(private readonly app: Express.Application) {} + + public async getTile(queryParams?: GetTileQueryParams): Promise { + return supertest + .agent(this.app) + .get('/search/MGRS/tiles') + .set('Content-Type', 'application/json') + .set('x-api-key', 'abc123') + .set('x-user-id', 'abc123') + .query(queryParams ?? {}); + } +} diff --git a/tests/integration/mgrs/mgrs.spec.ts b/tests/integration/mgrs/mgrs.spec.ts new file mode 100644 index 00000000..0431f4e2 --- /dev/null +++ b/tests/integration/mgrs/mgrs.spec.ts @@ -0,0 +1,113 @@ +/* eslint-disable @typescript-eslint/naming-convention */ +import config from 'config'; +import { DependencyContainer } from 'tsyringe'; +import { Application } from 'express'; +import { CleanupRegistry } from '@map-colonies/cleanup-registry'; +import jsLogger from '@map-colonies/js-logger'; +import { trace } from '@opentelemetry/api'; +import httpStatusCodes from 'http-status-codes'; +import { getApp } from '../../../src/app'; +import { SERVICES } from '../../../src/common/constants'; +import { S3_REPOSITORY_SYMBOL } from '../../../src/common/s3/s3Repository'; +import { cronLoadTileLatLonDataSymbol } from '../../../src/latLon/DAL/latLonDAL'; +import { GetGeotextSearchParams, QueryResult } from '../../../src/location/interfaces'; +import { GeoContextMode, IApplication } from '../../../src/common/interfaces'; +import { MgrsRequestSender } from './helpers/requestSender'; + +describe('/search/MGRS', function () { + let requestSender: MgrsRequestSender; + let app: { app: Application; container: DependencyContainer }; + + beforeEach(async function () { + app = await getApp({ + override: [ + { token: SERVICES.LOGGER, provider: { useValue: jsLogger({ enabled: false }) } }, + { token: SERVICES.TRACER, provider: { useValue: trace.getTracer('testTracer') } }, + { token: S3_REPOSITORY_SYMBOL, provider: { useValue: {} } }, + { token: SERVICES.S3_CLIENT, provider: { useValue: {} } }, + { token: cronLoadTileLatLonDataSymbol, provider: { useValue: {} } }, + { token: SERVICES.ELASTIC_CLIENTS, provider: { useValue: {} } }, + ], + useChild: true, + }); + + requestSender = new MgrsRequestSender(app.app); + }); + + afterAll(async function () { + const cleanupRegistry = app.container.resolve(SERVICES.CLEANUP_REGISTRY); + await cleanupRegistry.trigger(); + app.container.reset(); + + jest.clearAllTimers(); + }); + + describe('Happy Path', function () { + it('should return 200 status code and MGRS in geojson', async function () { + const response = await requestSender.getTile({ tile: '18SUJ2339007393' }); + + expect(response.status).toBe(httpStatusCodes.OK); + expect(response.body).toMatchObject({ + type: 'Feature', + geocoding: { + query: { + tile: '18SUJ2339007393', + }, + response: { + max_score: 1, + results_count: 1, + match_latency_ms: 0, + }, + }, + bbox: [-77.03654883669269, 38.89767541638445, -77.03653756947197, 38.897684623284015], + geometry: { + type: 'Polygon', + coordinates: [ + [ + [-77.03654883669269, 38.89767541638445], + [-77.03654883669269, 38.897684623284015], + [-77.03653756947197, 38.897684623284015], + [-77.03653756947197, 38.89767541638445], + [-77.03654883669269, 38.89767541638445], + ], + ], + }, + properties: { + score: 1, + }, + }); + }); + }); + + describe('Bad Path', function () { + it('should return 400 status code when MGRS is invalid', async function () { + const response = await requestSender.getTile({ tile: 'ABC{}' }); + + expect(response.body).toMatchObject({ + message: 'Invalid MGRS tile', + }); + expect(response.status).toBe(httpStatusCodes.BAD_REQUEST); + }); + + it('should return 400 status code when MGRS is missing', async function () { + const response = await requestSender.getTile(); + + expect(response.status).toBe(httpStatusCodes.BAD_REQUEST); + expect(response.body).toMatchObject({ + message: "request/query must have required property 'tile'", + }); + }); + }); + + describe('Sad Path', function () { + // All requests with status code 4XX-5XX + it('should return 500 status code when MGRSPoint zone letter A not handled', async function () { + const response = await requestSender.getTile({ tile: '{ABC}' }); + + expect(response.status).toBe(httpStatusCodes.INTERNAL_SERVER_ERROR); + expect(response.body).toMatchObject({ + message: 'MGRSPoint zone letter A not handled: {ABC}', + }); + }); + }); +}); From 4c46357aca03b295ce27d3528e1f9c291c959dfc Mon Sep 17 00:00:00 2001 From: Niv Greenstein Date: Sun, 8 Sep 2024 17:47:35 +0300 Subject: [PATCH 209/262] fix: changed bbox type from explicit to Bbox type from 'geojson' --- src/common/elastic/utils.ts | 3 ++- src/common/interfaces.ts | 4 ++-- src/location/utils.ts | 4 ++-- tests/integration/control/tile/tile.spec.ts | 3 ++- 4 files changed, 8 insertions(+), 6 deletions(-) diff --git a/src/common/elastic/utils.ts b/src/common/elastic/utils.ts index 5dcb413a..da798cba 100644 --- a/src/common/elastic/utils.ts +++ b/src/common/elastic/utils.ts @@ -1,4 +1,5 @@ import { estypes } from '@elastic/elasticsearch'; +import { BBox } from 'geojson'; import { WGS84Coordinate } from '../interfaces'; import { ServiceUnavailableError } from '../errors'; import { ElasticClient } from './index'; @@ -20,7 +21,7 @@ export const queryElastic = async (client: ElasticClient, body: estypes.Searc /* eslint-disable @typescript-eslint/naming-convention */ export const boundingBox = ( - bbox: number[] + bbox: BBox ): { geo_bounding_box: { geometry: { diff --git a/src/common/interfaces.ts b/src/common/interfaces.ts index ad25417c..f569c40a 100644 --- a/src/common/interfaces.ts +++ b/src/common/interfaces.ts @@ -1,4 +1,4 @@ -import { Feature, FeatureCollection as GeoJSONFeatureCollection } from 'geojson'; +import { BBox, Feature, FeatureCollection as GeoJSONFeatureCollection } from 'geojson'; export interface IConfig { get: (setting: string) => T; @@ -13,7 +13,7 @@ export interface OpenApiConfig { } export interface GeoContext { - bbox?: number[]; + bbox?: BBox; radius?: number; lon?: number; lat?: number; diff --git a/src/location/utils.ts b/src/location/utils.ts index 2c96645b..89e01779 100644 --- a/src/location/utils.ts +++ b/src/location/utils.ts @@ -1,5 +1,5 @@ import https from 'https'; -import { Geometry, Point } from 'geojson'; +import { BBox, Geometry, Point } from 'geojson'; import { SearchResponse } from '@elastic/elasticsearch/lib/api/types'; import { StatusCodes } from 'http-status-codes'; import axios, { AxiosError, AxiosResponse as Response } from 'axios'; @@ -23,7 +23,7 @@ const parsePoint = (split: string[] | number[]): Geometry => ({ coordinates: split.map(Number), }); -const parseBbox = (split: string[] | number[]): Geometry => { +const parseBbox = (split: [string, string, string, string] | BBox): Geometry => { const [xMin, yMin, xMax, yMax] = split.map(Number); return { type: 'Polygon', diff --git a/tests/integration/control/tile/tile.spec.ts b/tests/integration/control/tile/tile.spec.ts index 7ae212a8..a7cd3e62 100644 --- a/tests/integration/control/tile/tile.spec.ts +++ b/tests/integration/control/tile/tile.spec.ts @@ -5,6 +5,7 @@ import { DependencyContainer } from 'tsyringe'; import { Application } from 'express'; import { CleanupRegistry } from '@map-colonies/cleanup-registry'; import httpStatusCodes from 'http-status-codes'; +import { BBox } from 'geojson'; import { getApp } from '../../../../src/app'; import { SERVICES } from '../../../../src/common/constants'; import { GetTilesQueryParams } from '../../../../src/control/tile/controllers/tileController'; @@ -341,7 +342,7 @@ describe('/search/control/tiles', function () { tile: 'RIT', limit: 5, disable_fuzziness: false, - geo_context: { bbox }, + geo_context: { bbox: bbox as BBox }, geo_context_mode: GeoContextMode.FILTER, }); From 1a5f0a4fde021a52e6b852dba305bc214728094c Mon Sep 17 00:00:00 2001 From: Niv Greenstein Date: Sun, 8 Sep 2024 17:47:43 +0300 Subject: [PATCH 210/262] delete: removed unused file --- tests/integration/setupMockData.ts | 84 ------------------------------ 1 file changed, 84 deletions(-) delete mode 100644 tests/integration/setupMockData.ts diff --git a/tests/integration/setupMockData.ts b/tests/integration/setupMockData.ts deleted file mode 100644 index 464d2eea..00000000 --- a/tests/integration/setupMockData.ts +++ /dev/null @@ -1,84 +0,0 @@ -// /* eslint-disable @typescript-eslint/naming-convention */ -// import jsLogger from '@map-colonies/js-logger'; -// import { DeleteBucketCommand, DeleteObjectCommand, S3Client } from '@aws-sdk/client-s3'; -// import { trace } from '@opentelemetry/api'; -// import { getApp } from '../../src/app'; -// import { elasticConfigPath, SERVICES } from '../../src/common/constants'; -// import { S3Config, s3ConfigPath } from '../../src/common/s3'; -// import { IConfig } from '../../src/common/interfaces'; -// import importDataToS3 from '../../devScripts/importDataToS3'; -// import importDataToElastic from '../../devScripts/importDataToElastic'; -// import { ElasticClients } from '../../src/common/elastic'; -// import { ElasticDbClientsConfig } from '../../src/common/elastic/interfaces'; - -// let s3Client: S3Client; -// let s3Config: S3Config | undefined; -// let elasticClients: ElasticClients; -// let elasticClientsConfig: ElasticDbClientsConfig; - -// beforeAll(async function () { -// const app = await getApp({ -// override: [ -// { token: SERVICES.LOGGER, provider: { useValue: jsLogger({ enabled: false }) } }, -// { token: SERVICES.TRACER, provider: { useValue: trace.getTracer('testTracer') } }, -// ], -// useChild: true, -// }); - -// const config = app.container.resolve(SERVICES.CONFIG); - -// s3Client = app.container.resolve(SERVICES.S3_CLIENT); -// s3Config = config.get(s3ConfigPath); - -// elasticClients = app.container.resolve(SERVICES.ELASTIC_CLIENTS); -// elasticClientsConfig = config.get(elasticConfigPath); - -// await Promise.allSettled([await importDataToS3(config), await importDataToElastic(config)]).then((results) => { -// results.forEach((result) => { -// if (result.status === 'rejected') { -// throw result.reason; -// } -// }); -// }); -// }); - -// afterAll(async function () { -// const clearS3Data = new Promise((resolve, reject) => { -// void (async (): Promise => { -// try { -// if (s3Config?.files.latLonConvertionTable !== undefined) { -// await s3Client.send( -// new DeleteObjectCommand({ Bucket: s3Config.files.latLonConvertionTable.bucket, Key: s3Config.files.latLonConvertionTable.fileName }) -// ); -// await s3Client.send(new DeleteBucketCommand({ Bucket: s3Config.files.latLonConvertionTable.bucket })); -// } -// resolve(); -// } catch (error) { -// console.error(error); -// reject(error); -// } -// })(); -// }); - -// const clearElasticData = new Promise((resolve, reject) => { -// void (async (): Promise => { -// try { -// for (const [key, _] of Object.entries(elasticClientsConfig)) { -// await elasticClients[key as keyof ElasticDbClientsConfig].indices.delete({ index: '*' }); -// } -// resolve(); -// } catch (error) { -// console.error(error); -// reject(error); -// } -// })(); -// }); - -// await Promise.allSettled([clearS3Data, clearElasticData]).then((results) => { -// results.forEach((result) => { -// if (result.status === 'rejected') { -// throw result.reason; -// } -// }); -// }); -// }); From 9f21b64a607dceeed5ee3e8218928145381da72b Mon Sep 17 00:00:00 2001 From: NatalieShaked Date: Mon, 9 Sep 2024 14:34:14 +0300 Subject: [PATCH 211/262] feat: added request id to response header --- .../middlewares/feedbackApi.middleware.ts | 25 +++++++------------ src/serverBuilder.ts | 2 ++ 2 files changed, 11 insertions(+), 16 deletions(-) diff --git a/src/common/middlewares/feedbackApi.middleware.ts b/src/common/middlewares/feedbackApi.middleware.ts index 6064e86a..c5d382cb 100644 --- a/src/common/middlewares/feedbackApi.middleware.ts +++ b/src/common/middlewares/feedbackApi.middleware.ts @@ -12,7 +12,7 @@ export class FeedbackApiMiddlewareManager { // eslint-disable-next-line @typescript-eslint/explicit-function-return-type public saveResponses = (req: Request, res: Response, next: NextFunction) => { - const reqId = crypto.randomUUID(); + const reqId = res.getHeader('request-id'); const redisClient = this.redis; const logger = this.logger; @@ -29,28 +29,21 @@ export class FeedbackApiMiddlewareManager { geocodingResponseDetails.response = body; try { - await redisClient.setEx(reqId, REDIS_TTL, JSON.stringify(geocodingResponseDetails)); + await redisClient.setEx(reqId as string, REDIS_TTL, JSON.stringify(geocodingResponseDetails)); logger.info({ msg: 'saving response to redis' }); } catch (err) { logger.error('Error setting key:', err); } - - //await setKeyWithTTL(reqId, JSON.stringify(geocodingResponseDetails), redisClient); - return originalJson.call(this, body); }; res.json = logJson as unknown as Response['json']; next(); }; -} -// async function setKeyWithTTL(key: string, value: string, redis: RedisClient) { -// try { -// await redis.set(key, value, { -// EX: 300, // 5 minutes in seconds -// }); -// console.log(`Key '${key}' set with a TTL of 5 minutes.`); -// } catch (err) { -// console.error('Error setting key:', err); -// } -// } + // eslint-disable-next-line @typescript-eslint/explicit-function-return-type + public setRequestId = (req: Request, res: Response, next: NextFunction) => { + const reqId = crypto.randomUUID(); + res.append('request-id', reqId); + next(); + }; +} diff --git a/src/serverBuilder.ts b/src/serverBuilder.ts index a2450759..db0efea4 100644 --- a/src/serverBuilder.ts +++ b/src/serverBuilder.ts @@ -93,6 +93,8 @@ export class ServerBuilder { const apiSpecPath = this.config.get('openapiConfig.filePath'); this.serverInstance.use(OpenApiMiddleware({ apiSpec: apiSpecPath, validateRequests: true, ignorePaths: ignorePathRegex })); this.serverInstance.disable('x-powered-by'); + + this.serverInstance.use(this.feedbackApiMiddleware.setRequestId); } private registerPostRoutesMiddleware(): void { From 03d24d283ea317c1aea7c883d554ff23a05370e6 Mon Sep 17 00:00:00 2001 From: NatalieShaked Date: Mon, 9 Sep 2024 15:03:44 +0300 Subject: [PATCH 212/262] feat: sent x-api-key header to redis --- src/common/interfaces.ts | 1 + src/common/middlewares/feedbackApi.middleware.ts | 1 + 2 files changed, 2 insertions(+) diff --git a/src/common/interfaces.ts b/src/common/interfaces.ts index 3c0052f3..198fc188 100644 --- a/src/common/interfaces.ts +++ b/src/common/interfaces.ts @@ -74,6 +74,7 @@ export enum GeoContextMode { export interface GeocodingResponse { userId: string; + apiKey: string response: JSON; respondedAt: Date; } diff --git a/src/common/middlewares/feedbackApi.middleware.ts b/src/common/middlewares/feedbackApi.middleware.ts index c5d382cb..27f905cf 100644 --- a/src/common/middlewares/feedbackApi.middleware.ts +++ b/src/common/middlewares/feedbackApi.middleware.ts @@ -19,6 +19,7 @@ export class FeedbackApiMiddlewareManager { logger.info({ msg: 'saving response to redis' }); const geocodingResponseDetails: GeocodingResponse = { userId: req.headers['x-user-id'] as string, + apiKey: req.headers['x-api-key'] as string, response: JSON.parse('{}') as JSON, respondedAt: new Date(), }; From c02c9b73830df71a9e1d03e0f77ab6fa4ce8bef6 Mon Sep 17 00:00:00 2001 From: NatalieShaked Date: Mon, 9 Sep 2024 15:48:27 +0300 Subject: [PATCH 213/262] feat: sent the current site to redis, will adjust the s3 entrypoint once merged --- src/common/constants.ts | 1 + src/common/interfaces.ts | 3 ++- src/common/middlewares/feedbackApi.middleware.ts | 6 +++++- 3 files changed, 8 insertions(+), 2 deletions(-) diff --git a/src/common/constants.ts b/src/common/constants.ts index ac72a036..de2b81b9 100644 --- a/src/common/constants.ts +++ b/src/common/constants.ts @@ -26,3 +26,4 @@ export const REDIS_SYMBOL = Symbol('REDIS'); export const REDIS_TTL = 300; export const elasticConfigPath = 'db.elastic'; +export const siteIndex = 1; diff --git a/src/common/interfaces.ts b/src/common/interfaces.ts index 198fc188..2bfcbff5 100644 --- a/src/common/interfaces.ts +++ b/src/common/interfaces.ts @@ -74,7 +74,8 @@ export enum GeoContextMode { export interface GeocodingResponse { userId: string; - apiKey: string + apiKey: string; + site: string; response: JSON; respondedAt: Date; } diff --git a/src/common/middlewares/feedbackApi.middleware.ts b/src/common/middlewares/feedbackApi.middleware.ts index 27f905cf..e3774b2b 100644 --- a/src/common/middlewares/feedbackApi.middleware.ts +++ b/src/common/middlewares/feedbackApi.middleware.ts @@ -2,7 +2,7 @@ import * as crypto from 'node:crypto'; import { Request, Response, NextFunction } from 'express'; import { Logger } from '@map-colonies/js-logger'; import { inject, injectable } from 'tsyringe'; -import { REDIS_TTL, SERVICES } from '../constants'; +import { REDIS_TTL, SERVICES, siteIndex } from '../constants'; import { RedisClient } from '../redis'; import { GeocodingResponse } from '../interfaces'; @@ -16,10 +16,14 @@ export class FeedbackApiMiddlewareManager { const redisClient = this.redis; const logger = this.logger; + const s3Endpoint = ''; //s3 endpoint from default + const drSite = s3Endpoint.split('.'); + logger.info({ msg: 'saving response to redis' }); const geocodingResponseDetails: GeocodingResponse = { userId: req.headers['x-user-id'] as string, apiKey: req.headers['x-api-key'] as string, + site: drSite[siteIndex], response: JSON.parse('{}') as JSON, respondedAt: new Date(), }; From 5406ea1f303ae62cd3405d58876a348dc0018eab Mon Sep 17 00:00:00 2001 From: NatalieShaked Date: Thu, 12 Sep 2024 14:03:16 +0300 Subject: [PATCH 214/262] feat: added redis health check --- src/common/utils.ts | 5 +++++ 1 file changed, 5 insertions(+) diff --git a/src/common/utils.ts b/src/common/utils.ts index e99870fc..cc7b5b74 100644 --- a/src/common/utils.ts +++ b/src/common/utils.ts @@ -6,6 +6,7 @@ import { WGS84Coordinate } from './interfaces'; import { TimeoutError } from './errors'; import { SERVICES } from './constants'; import { ElasticClients } from './elastic'; +import { RedisClient } from './redis'; type SnakeToCamelCase = S extends `${infer T}_${infer U}` ? `${T}${Capitalize>}` : S; @@ -81,6 +82,8 @@ export const healthCheckFactory: FactoryFunction = (container: DependencyC const logger = container.resolve(SERVICES.LOGGER); const elasticClients = container.resolve(SERVICES.ELASTIC_CLIENTS); const s3Client = container.resolve(SERVICES.S3_CLIENT); + const redis = container.resolve(SERVICES.REDIS); + logger.info('Healthcheck is running'); try { @@ -91,6 +94,8 @@ export const healthCheckFactory: FactoryFunction = (container: DependencyC void s3Client.send(new ListBucketsCommand({})); + void redis.ping(); + logger.info('healthcheck passed'); } catch (error) { logger.error(`Healthcheck failed. Error: ${(error as Error).message}`); From 3541c48c7a01bbf143c19a31cc0690e7e61bca0d Mon Sep 17 00:00:00 2001 From: NatalieShaked Date: Thu, 12 Sep 2024 14:03:30 +0300 Subject: [PATCH 215/262] delete: removed redis healthcheck --- src/common/redis/index.ts | 15 ++------------- 1 file changed, 2 insertions(+), 13 deletions(-) diff --git a/src/common/redis/index.ts b/src/common/redis/index.ts index 6e4cf92f..357badc1 100644 --- a/src/common/redis/index.ts +++ b/src/common/redis/index.ts @@ -1,11 +1,9 @@ import { readFileSync } from 'fs'; -import { ILogger } from '@map-colonies/detiler-common'; -import { HealthCheck } from '@godaddy/terminus'; +import { Logger } from '@map-colonies/js-logger'; import { createClient, RedisClientOptions } from 'redis'; import { DependencyContainer, FactoryFunction } from 'tsyringe'; import { SERVICES } from '../constants'; import { RedisConfig, IConfig } from '../interfaces'; -import { promiseTimeout } from '../utils'; const DEFAULT_LIMIT_FROM = 0; const DEFAULT_LIMIT_SIZE = 1000; @@ -33,7 +31,7 @@ export const DEFAULT_LIMIT = { from: DEFAULT_LIMIT_FROM, size: DEFAULT_LIMIT_SIZ export type RedisClient = ReturnType; export const redisClientFactory: FactoryFunction = (container: DependencyContainer): RedisClient => { - const logger = container.resolve(SERVICES.LOGGER); + const logger = container.resolve(SERVICES.LOGGER); const config = container.resolve(SERVICES.CONFIG); const dbConfig = config.get('db.redis'); const connectionOptions = createConnectionOptions(dbConfig); @@ -47,12 +45,3 @@ export const redisClientFactory: FactoryFunction = (container: Depe return redisClient; }; - -export const healthCheckFunctionFactory = (redis: RedisClient): HealthCheck => { - return async (): Promise => { - const check = redis.ping().then(() => { - return; - }); - return promiseTimeout(CONNECTION_TIMEOUT, check); - }; -}; From ebc5efea770ab0615c9866ca5b6de06fd60cbe2f Mon Sep 17 00:00:00 2001 From: NatalieShaked Date: Thu, 12 Sep 2024 14:03:48 +0300 Subject: [PATCH 216/262] fix: added missing redis config --- config/custom-environment-variables.json | 23 ++++++++++++++++++++++- config/default.json | 2 +- 2 files changed, 23 insertions(+), 2 deletions(-) diff --git a/config/custom-environment-variables.json b/config/custom-environment-variables.json index 2be63e7b..b56daf27 100644 --- a/config/custom-environment-variables.json +++ b/config/custom-environment-variables.json @@ -78,7 +78,28 @@ "__name": "S3_FILES_DATA", "__format": "json" } - } + }, + "redis": { + "host": "REDIS_HOST", + "port": { + "__name": "REDIS_PORT", + "__format": "number" + }, + "username": "REDIS_USERNAME", + "password": "REDIS_PASSWORD", + "enableSslAuth": { + "__name": "REDIS_ENABLE_SSL_AUTH", + "__format": "boolean" + }, + "sslPaths": { + "ca": "REDIS_CA_PATH", + "key": "REDIS_KEY_PATH", + "cert": "REDIS_CERT_PATH" + }, + "database": { + "__name": "REDIS_DATABASE", + "__format": "number" + } }, "application": { "services": { diff --git a/config/default.json b/config/default.json index 8ece3b33..9c165b10 100644 --- a/config/default.json +++ b/config/default.json @@ -71,7 +71,7 @@ } }, "redis": { - "host": "localhost", + "host": "REDIS_HOST", "port": 6379, "username": "", "password": "", From 5b607a65fee6b42ccada44b99abbe43c66b14021 Mon Sep 17 00:00:00 2001 From: NatalieShaked Date: Thu, 12 Sep 2024 14:12:22 +0300 Subject: [PATCH 217/262] fix: added missing imports; removed pg --- dataSource.ts | 13 ------ package-lock.json | 92 ---------------------------------------- package.json | 7 +-- src/common/interfaces.ts | 6 --- src/index.ts | 2 +- 5 files changed, 2 insertions(+), 118 deletions(-) delete mode 100644 dataSource.ts diff --git a/dataSource.ts b/dataSource.ts deleted file mode 100644 index ad31f0f6..00000000 --- a/dataSource.ts +++ /dev/null @@ -1,13 +0,0 @@ -import config from 'config'; -import { DataSource } from 'typeorm'; -import { createConnectionOptions } from './src/common/postgresql'; -import { PostgresDbConfig } from './src/common/interfaces'; - -const connectionOptions = config.get('db.postgresql'); - -export const appDataSource = new DataSource({ - ...createConnectionOptions(connectionOptions), - entities: ['src/**/DAL/*.ts'], - migrationsTableName: 'migrations_table', - migrations: ['db/migrations/*.ts'], -}); diff --git a/package-lock.json b/package-lock.json index 3f707131..45416fbb 100644 --- a/package-lock.json +++ b/package-lock.json @@ -38,7 +38,6 @@ "mgrs": "^2.0.0", "node-cron": "^3.0.3", "node-fetch-commonjs": "^3.3.2", - "pg": "^8.11.5", "proj4": "^2.11.0", "redis": "^4.7.0", "reflect-metadata": "^0.1.13", @@ -18192,43 +18191,6 @@ "node": ">=8" } }, - "node_modules/pg": { - "version": "8.11.5", - "resolved": "https://registry.npmjs.org/pg/-/pg-8.11.5.tgz", - "integrity": "sha512-jqgNHSKL5cbDjFlHyYsCXmQDrfIX/3RsNwYqpd4N0Kt8niLuNoRNH+aazv6cOd43gPh9Y4DjQCtb+X0MH0Hvnw==", - "dependencies": { - "pg-connection-string": "^2.6.4", - "pg-pool": "^3.6.2", - "pg-protocol": "^1.6.1", - "pg-types": "^2.1.0", - "pgpass": "1.x" - }, - "engines": { - "node": ">= 8.0.0" - }, - "optionalDependencies": { - "pg-cloudflare": "^1.1.1" - }, - "peerDependencies": { - "pg-native": ">=3.0.1" - }, - "peerDependenciesMeta": { - "pg-native": { - "optional": true - } - } - }, - "node_modules/pg-cloudflare": { - "version": "1.1.1", - "resolved": "https://registry.npmjs.org/pg-cloudflare/-/pg-cloudflare-1.1.1.tgz", - "integrity": "sha512-xWPagP/4B6BgFO+EKz3JONXv3YDgvkbVrGw2mTo3D6tVDQRh1e7cqVGvyR3BE+eQgAvx1XhW/iEASj4/jCWl3Q==", - "optional": true - }, - "node_modules/pg-connection-string": { - "version": "2.6.4", - "resolved": "https://registry.npmjs.org/pg-connection-string/-/pg-connection-string-2.6.4.tgz", - "integrity": "sha512-v+Z7W/0EO707aNMaAEfiGnGL9sxxumwLl2fJvCQtMn9Fxsg+lPpPkdcyBSv/KFgpGdYkMfn+EI1Or2EHjpgLCA==" - }, "node_modules/pg-int8": { "version": "1.0.1", "resolved": "https://registry.npmjs.org/pg-int8/-/pg-int8-1.0.1.tgz", @@ -18237,14 +18199,6 @@ "node": ">=4.0.0" } }, - "node_modules/pg-pool": { - "version": "3.6.2", - "resolved": "https://registry.npmjs.org/pg-pool/-/pg-pool-3.6.2.tgz", - "integrity": "sha512-Htjbg8BlwXqSBQ9V8Vjtc+vzf/6fVUuak/3/XXKA9oxZprwW3IMDQTGHP+KDmVL7rtd+R1QjbnCFPuTHm3G4hg==", - "peerDependencies": { - "pg": ">=8.0" - } - }, "node_modules/pg-protocol": { "version": "1.6.1", "resolved": "https://registry.npmjs.org/pg-protocol/-/pg-protocol-1.6.1.tgz", @@ -18265,14 +18219,6 @@ "node": ">=4" } }, - "node_modules/pgpass": { - "version": "1.0.5", - "resolved": "https://registry.npmjs.org/pgpass/-/pgpass-1.0.5.tgz", - "integrity": "sha512-FdW9r/jQZhSeohs1Z3sI1yxFQNFvMcnmfuj4WBMUTxOrAyLMaTcE1aAMBiTlbMNaXvBCQuVi0R7hd8udDSP7ug==", - "dependencies": { - "split2": "^4.1.0" - } - }, "node_modules/picocolors": { "version": "1.0.0", "resolved": "https://registry.npmjs.org/picocolors/-/picocolors-1.0.0.tgz", @@ -35206,41 +35152,11 @@ "integrity": "sha512-gDKb8aZMDeD/tZWs9P6+q0J9Mwkdl6xMV8TjnGP3qJVJ06bdMgkbBlLU8IdfOsIsFz2BW1rNVT3XuNEl8zPAvw==", "dev": true }, - "pg": { - "version": "8.11.5", - "resolved": "https://registry.npmjs.org/pg/-/pg-8.11.5.tgz", - "integrity": "sha512-jqgNHSKL5cbDjFlHyYsCXmQDrfIX/3RsNwYqpd4N0Kt8niLuNoRNH+aazv6cOd43gPh9Y4DjQCtb+X0MH0Hvnw==", - "requires": { - "pg-cloudflare": "^1.1.1", - "pg-connection-string": "^2.6.4", - "pg-pool": "^3.6.2", - "pg-protocol": "^1.6.1", - "pg-types": "^2.1.0", - "pgpass": "1.x" - } - }, - "pg-cloudflare": { - "version": "1.1.1", - "resolved": "https://registry.npmjs.org/pg-cloudflare/-/pg-cloudflare-1.1.1.tgz", - "integrity": "sha512-xWPagP/4B6BgFO+EKz3JONXv3YDgvkbVrGw2mTo3D6tVDQRh1e7cqVGvyR3BE+eQgAvx1XhW/iEASj4/jCWl3Q==", - "optional": true - }, - "pg-connection-string": { - "version": "2.6.4", - "resolved": "https://registry.npmjs.org/pg-connection-string/-/pg-connection-string-2.6.4.tgz", - "integrity": "sha512-v+Z7W/0EO707aNMaAEfiGnGL9sxxumwLl2fJvCQtMn9Fxsg+lPpPkdcyBSv/KFgpGdYkMfn+EI1Or2EHjpgLCA==" - }, "pg-int8": { "version": "1.0.1", "resolved": "https://registry.npmjs.org/pg-int8/-/pg-int8-1.0.1.tgz", "integrity": "sha512-WCtabS6t3c8SkpDBUlb1kjOs7l66xsGdKpIPZsg4wR+B3+u9UAum2odSsF9tnvxg80h4ZxLWMy4pRjOsFIqQpw==" }, - "pg-pool": { - "version": "3.6.2", - "resolved": "https://registry.npmjs.org/pg-pool/-/pg-pool-3.6.2.tgz", - "integrity": "sha512-Htjbg8BlwXqSBQ9V8Vjtc+vzf/6fVUuak/3/XXKA9oxZprwW3IMDQTGHP+KDmVL7rtd+R1QjbnCFPuTHm3G4hg==", - "requires": {} - }, "pg-protocol": { "version": "1.6.1", "resolved": "https://registry.npmjs.org/pg-protocol/-/pg-protocol-1.6.1.tgz", @@ -35258,14 +35174,6 @@ "postgres-interval": "^1.1.0" } }, - "pgpass": { - "version": "1.0.5", - "resolved": "https://registry.npmjs.org/pgpass/-/pgpass-1.0.5.tgz", - "integrity": "sha512-FdW9r/jQZhSeohs1Z3sI1yxFQNFvMcnmfuj4WBMUTxOrAyLMaTcE1aAMBiTlbMNaXvBCQuVi0R7hd8udDSP7ug==", - "requires": { - "split2": "^4.1.0" - } - }, "picocolors": { "version": "1.0.0", "resolved": "https://registry.npmjs.org/picocolors/-/picocolors-1.0.0.tgz", diff --git a/package.json b/package.json index 11a9e5d1..b7148db5 100644 --- a/package.json +++ b/package.json @@ -23,11 +23,7 @@ "assets:copy": "copyfiles -f ./config/* ./dist/config && copyfiles -f ./openapi3.yaml ./dist/ && copyfiles ./package.json dist", "clean": "rimraf dist", "install": "npx husky install", - "dev:scripts": "npx ts-node ./devScripts/index.ts", - "typeorm": "node --require ts-node/register ./node_modules/typeorm/cli.js -d dataSource.ts", - "migration:create": "npm run typeorm migration:generate", - "migration:run": "npm run typeorm migration:run", - "migration:revert": "npm run typeorm migration:revert" + "dev:scripts": "npx ts-node ./devScripts/index.ts" }, "directories": { "test": "tests" @@ -74,7 +70,6 @@ "mgrs": "^2.0.0", "node-cron": "^3.0.3", "node-fetch-commonjs": "^3.3.2", - "pg": "^8.11.5", "proj4": "^2.11.0", "redis": "^4.7.0", "reflect-metadata": "^0.1.13", diff --git a/src/common/interfaces.ts b/src/common/interfaces.ts index 7dbaabb2..9b8138ed 100644 --- a/src/common/interfaces.ts +++ b/src/common/interfaces.ts @@ -1,4 +1,3 @@ -import { DataSourceOptions } from 'typeorm'; import { RedisClientOptions } from 'redis'; import { BBox, Feature, FeatureCollection as GeoJSONFeatureCollection } from 'geojson'; @@ -21,11 +20,6 @@ export type RedisConfig = { sslPaths: { ca: string; cert: string; key: string }; } & RedisClientOptions; -export type PostgresDbConfig = { - enableSslAuth: boolean; - sslPaths: { ca: string; cert: string; key: string }; -} & DataSourceOptions; - export interface GeoContext { bbox?: BBox; radius?: number; diff --git a/src/index.ts b/src/index.ts index 2ed8f243..a2f0f74b 100644 --- a/src/index.ts +++ b/src/index.ts @@ -7,7 +7,7 @@ import { DependencyContainer } from 'tsyringe'; import { createTerminus } from '@godaddy/terminus'; import { Logger } from '@map-colonies/js-logger'; import config from 'config'; -import { DEFAULT_SERVER_PORT, ON_SIGNAL, SERVICES } from './common/constants'; +import { DEFAULT_SERVER_PORT, HEALTHCHECK, ON_SIGNAL, SERVICES } from './common/constants'; import { getApp } from './app'; let depContainer: DependencyContainer | undefined; From 5e6ff685e9f9a11d1ad87f4c203af640e14e7624 Mon Sep 17 00:00:00 2001 From: NatalieShaked Date: Thu, 12 Sep 2024 14:55:22 +0300 Subject: [PATCH 218/262] fix: added missing closing bracket --- config/custom-environment-variables.json | 43 ++++++++++++------------ 1 file changed, 22 insertions(+), 21 deletions(-) diff --git a/config/custom-environment-variables.json b/config/custom-environment-variables.json index b56daf27..f4f7a4ff 100644 --- a/config/custom-environment-variables.json +++ b/config/custom-environment-variables.json @@ -100,27 +100,28 @@ "__name": "REDIS_DATABASE", "__format": "number" } - }, - "application": { - "services": { - "tokenTypesUrl": "TOKEN_TYPE_URL" - }, - "sources": { - "__name": "GEOTEXT_SOURCES", - "__format": "json" - }, - "regions": { - "__name": "REGIONS", - "__format": "json" - }, - "nameTranslationsKeys": { - "__name": "NAME_TRANSLATION_KEYS", - "__format": "json" }, - "controlObjectDisplayNamePrefixes": { - "__name": "CONTROL_DISPLAY_KEY_TRANSLATIONS", - "__format": "json" - }, - "mainLanguageRegex": "MAIN_LANGUAGE_REGEX" + "application": { + "services": { + "tokenTypesUrl": "TOKEN_TYPE_URL" + }, + "sources": { + "__name": "GEOTEXT_SOURCES", + "__format": "json" + }, + "regions": { + "__name": "REGIONS", + "__format": "json" + }, + "nameTranslationsKeys": { + "__name": "NAME_TRANSLATION_KEYS", + "__format": "json" + }, + "controlObjectDisplayNamePrefixes": { + "__name": "CONTROL_DISPLAY_KEY_TRANSLATIONS", + "__format": "json" + }, + "mainLanguageRegex": "MAIN_LANGUAGE_REGEX" + } } } From 259a029f01d8f088524b661a8bbb0637ac71747e Mon Sep 17 00:00:00 2001 From: NatalieShaked Date: Thu, 12 Sep 2024 14:57:25 +0300 Subject: [PATCH 219/262] fix: added missing closing bracket --- config/test.json | 59 ++++++++++++++++++++++++------------------------ 1 file changed, 30 insertions(+), 29 deletions(-) diff --git a/config/test.json b/config/test.json index 285bbc49..1018cd67 100644 --- a/config/test.json +++ b/config/test.json @@ -56,35 +56,36 @@ "cert": "" }, "database": 1 - }, - "application": { - "services": { - "tokenTypesUrl": "http://localhost:5001/NLP_ANALYSES" }, - "cronLoadTileLatLonDataPattern": "0 * * * *", - "elasticQueryBoosts": { - "name": 1.1, - "placeType": 1.1, - "subPlaceType": 1.1, - "hierarchy": 1.1, - "viewbox": 1.1 - }, - "sources": { - "OSM": "OSM", - "GOOGLE": "GOOGLE" - }, - "regions": { - "USA": ["New York", "Los Angeles"], - "FRANCE": ["Paris"] - }, - "controlObjectDisplayNamePrefixes": { - "TILE": "Tile", - "SUB_TILE": "Sub Tile", - "ROUTE": "Route", - "ITEM": "Item", - "CONTROL_POINT": "Control Point" - }, - "nameTranslationsKeys": ["en", "fr"], - "mainLanguageRegex": "[a-zA-Z]" + "application": { + "services": { + "tokenTypesUrl": "http://localhost:5001/NLP_ANALYSES" + }, + "cronLoadTileLatLonDataPattern": "0 * * * *", + "elasticQueryBoosts": { + "name": 1.1, + "placeType": 1.1, + "subPlaceType": 1.1, + "hierarchy": 1.1, + "viewbox": 1.1 + }, + "sources": { + "OSM": "OSM", + "GOOGLE": "GOOGLE" + }, + "regions": { + "USA": ["New York", "Los Angeles"], + "FRANCE": ["Paris"] + }, + "controlObjectDisplayNamePrefixes": { + "TILE": "Tile", + "SUB_TILE": "Sub Tile", + "ROUTE": "Route", + "ITEM": "Item", + "CONTROL_POINT": "Control Point" + }, + "nameTranslationsKeys": ["en", "fr"], + "mainLanguageRegex": "[a-zA-Z]" + } } } From 5084e3b7454b745288211527d591b45741a1b6a6 Mon Sep 17 00:00:00 2001 From: NatalieShaked Date: Thu, 12 Sep 2024 15:00:23 +0300 Subject: [PATCH 220/262] fix: application placement --- config/custom-environment-variables.json | 44 ++++++++--------- config/test.json | 60 ++++++++++++------------ 2 files changed, 52 insertions(+), 52 deletions(-) diff --git a/config/custom-environment-variables.json b/config/custom-environment-variables.json index f4f7a4ff..48e05ca7 100644 --- a/config/custom-environment-variables.json +++ b/config/custom-environment-variables.json @@ -100,28 +100,28 @@ "__name": "REDIS_DATABASE", "__format": "number" } - }, - "application": { - "services": { - "tokenTypesUrl": "TOKEN_TYPE_URL" - }, - "sources": { - "__name": "GEOTEXT_SOURCES", - "__format": "json" - }, - "regions": { - "__name": "REGIONS", - "__format": "json" - }, - "nameTranslationsKeys": { - "__name": "NAME_TRANSLATION_KEYS", - "__format": "json" - }, - "controlObjectDisplayNamePrefixes": { - "__name": "CONTROL_DISPLAY_KEY_TRANSLATIONS", - "__format": "json" - }, - "mainLanguageRegex": "MAIN_LANGUAGE_REGEX" } + }, + "application": { + "services": { + "tokenTypesUrl": "TOKEN_TYPE_URL" + }, + "sources": { + "__name": "GEOTEXT_SOURCES", + "__format": "json" + }, + "regions": { + "__name": "REGIONS", + "__format": "json" + }, + "nameTranslationsKeys": { + "__name": "NAME_TRANSLATION_KEYS", + "__format": "json" + }, + "controlObjectDisplayNamePrefixes": { + "__name": "CONTROL_DISPLAY_KEY_TRANSLATIONS", + "__format": "json" + }, + "mainLanguageRegex": "MAIN_LANGUAGE_REGEX" } } diff --git a/config/test.json b/config/test.json index 1018cd67..eb85c46e 100644 --- a/config/test.json +++ b/config/test.json @@ -56,36 +56,36 @@ "cert": "" }, "database": 1 - }, - "application": { - "services": { - "tokenTypesUrl": "http://localhost:5001/NLP_ANALYSES" - }, - "cronLoadTileLatLonDataPattern": "0 * * * *", - "elasticQueryBoosts": { - "name": 1.1, - "placeType": 1.1, - "subPlaceType": 1.1, - "hierarchy": 1.1, - "viewbox": 1.1 - }, - "sources": { - "OSM": "OSM", - "GOOGLE": "GOOGLE" - }, - "regions": { - "USA": ["New York", "Los Angeles"], - "FRANCE": ["Paris"] - }, - "controlObjectDisplayNamePrefixes": { - "TILE": "Tile", - "SUB_TILE": "Sub Tile", - "ROUTE": "Route", - "ITEM": "Item", - "CONTROL_POINT": "Control Point" - }, - "nameTranslationsKeys": ["en", "fr"], - "mainLanguageRegex": "[a-zA-Z]" } + }, + "application": { + "services": { + "tokenTypesUrl": "http://localhost:5001/NLP_ANALYSES" + }, + "cronLoadTileLatLonDataPattern": "0 * * * *", + "elasticQueryBoosts": { + "name": 1.1, + "placeType": 1.1, + "subPlaceType": 1.1, + "hierarchy": 1.1, + "viewbox": 1.1 + }, + "sources": { + "OSM": "OSM", + "GOOGLE": "GOOGLE" + }, + "regions": { + "USA": ["New York", "Los Angeles"], + "FRANCE": ["Paris"] + }, + "controlObjectDisplayNamePrefixes": { + "TILE": "Tile", + "SUB_TILE": "Sub Tile", + "ROUTE": "Route", + "ITEM": "Item", + "CONTROL_POINT": "Control Point" + }, + "nameTranslationsKeys": ["en", "fr"], + "mainLanguageRegex": "[a-zA-Z]" } } From 0d37f0b8cb94b6162f14b62e8c19b93328024195 Mon Sep 17 00:00:00 2001 From: Niv Greenstein Date: Thu, 12 Sep 2024 16:58:29 +0300 Subject: [PATCH 221/262] fix: revised openapi schema --- openapi3.yaml | 281 +++++++++++++++++++++++++++++++++----------------- 1 file changed, 185 insertions(+), 96 deletions(-) diff --git a/openapi3.yaml b/openapi3.yaml index d8f9cbb1..fe86d34e 100644 --- a/openapi3.yaml +++ b/openapi3.yaml @@ -111,7 +111,33 @@ paths: content: application/json: schema: - "$ref": "#/components/schemas/genericGeocodingResponse" + allOf: + - "$ref": "#/components/schemas/genericGeocodingResponse" + - type: "object" + required: + - "type" + - "features" + properties: + type: + type: "string" + enum: + - "FeatureCollection" + features: + type: "array" + items: + type: "object" + properties: + geometry: + oneOf: + - $ref: "#/components/schemas/Point" + - $ref: "#/components/schemas/Polygon" + properties: + type: "object" + properties: + placetype: + type: "string" + sub_placetype: + type: "string" 400: "$ref": "#/components/responses/BadRequest" 401: @@ -396,8 +422,42 @@ paths: schema: type: object properties: - geojson: - type: object # Make it geojson + type: + type: "string" + enum: + - "Feature" + geocoding: + type: "object" + properties: + query: + type: "object" + properties: + lat: + type: number + lon: + type: number + response: + type: "object" + properties: + max_score: + type: number + results_count: + type: number + match_latency_ms: + type: number + bbox: + $ref: "#/components/schemas/BoundingBox" + geometry: + oneOf: + - $ref: "#/components/schemas/Point" + - $ref: "#/components/schemas/Polygon" + properties: + type: object + properties: + name: + type: string + additionalProperty1: + type: object 400: "$ref": "#/components/responses/BadRequest" 401: @@ -581,99 +641,132 @@ components: properties: type: type: "string" - enum: ["Feature"] + enum: + - "Feature" + geocoding: + type: "object" + properties: + query: + type: "object" + properties: + tile: + type: string + example: "18SUJ2339007393" + response: + type: "object" + properties: + max_score: + type: number + results_count: + type: number + match_latency_ms: + type: number + + bbox: + $ref: "#/components/schemas/BoundingBox" geometry: - $ref: "#/components/schemas/Polygon" + oneOf: + - $ref: "#/components/schemas/Point" + - $ref: "#/components/schemas/Polygon" properties: - type: "object" + type: object + properties: + score: + type: number tilesSchema: - type: "object" - description: "GeoJson feature collection representing a tile (Polygon)" - required: - - "type" - - "features" - properties: - type: - type: "string" - enum: - - "FeatureCollection" - features: - type: "array" - items: - $ref: "#/components/schemas/tileSchema" + allOf: + - $ref: "#/components/schemas/genericGeocodingResponse" + - type: "object" + description: "GeoJson feature collection representing a tile (Polygon)" + required: + - "type" + - "features" + properties: + type: + type: "string" + enum: + - "FeatureCollection" + features: + type: "array" + items: + $ref: "#/components/schemas/tileSchema" itemsSchema: - type: "object" - description: "GeoJson feature collection representing an item" - required: - - "type" - - "features" - properties: - type: - type: "string" - enum: - - "FeatureCollection" - features: - type: "array" - items: - type: "object" - properties: - geometry: - oneOf: - - $ref: "#/components/schemas/Point" - - $ref: "#/components/schemas/Polygon" - properties: + allOf: + - $ref: "#/components/schemas/genericGeocodingResponse" + - type: "object" + description: "GeoJson feature collection representing an item" + required: + - "type" + - "features" + properties: + type: + type: "string" + enum: + - "FeatureCollection" + features: + type: "array" + items: type: "object" - required: - - "TYPE" - - "object_command_name" - - "entity_heb" - - "sub_tile_id" properties: - TYPE: - type: "string" - object_command_name: - type: "string" - entity_heb: - type: "string" - tile_name: - type: "string" - sub_tile_id: - type: "string" + geometry: + oneOf: + - $ref: "#/components/schemas/Point" + - $ref: "#/components/schemas/Polygon" + properties: + type: "object" + required: + - "TYPE" + - "object_command_name" + - "entity_heb" + - "sub_tile_id" + properties: + TYPE: + type: "string" + object_command_name: + type: "string" + entity_heb: + type: "string" + tile_name: + type: "string" + sub_tile_id: + type: "string" routesSchema: - type: "object" - description: "GeoJson feature collection representing a route (MultiLineString, LineString)" - required: - - "type" - - "features" - properties: - type: - type: "string" - enum: - - "FeatureCollection" - features: - type: "array" - items: - type: "object" - properties: - geometry: - oneOf: - - $ref: "#/components/schemas/LineString" - - $ref: "#/components/schemas/MultiLineString" - properties: + allOf: + - $ref: "#/components/schemas/genericGeocodingResponse" + - type: "object" + description: "GeoJson feature collection representing a route (MultiLineString, LineString)" + required: + - "type" + - "features" + properties: + type: + type: "string" + enum: + - "FeatureCollection" + features: + type: "array" + items: type: "object" - required: - - "TYPE" - - "object_command_name" - - "entity_heb" properties: - TYPE: - type: "string" - object_command_name: - type: "string" - entity_heb: - type: "string" - SECTION: - type: "string" + geometry: + oneOf: + - $ref: "#/components/schemas/LineString" + - $ref: "#/components/schemas/MultiLineString" + properties: + type: "object" + required: + - "TYPE" + - "object_command_name" + - "entity_heb" + properties: + TYPE: + type: "string" + object_command_name: + type: "string" + entity_heb: + type: "string" + SECTION: + type: "string" genericGeocodingResponse: # we need discriminator for Control and Location type: "object" description: "GeoJson feature collection representing an item" @@ -735,8 +828,6 @@ components: type: integer minimum: 0 maximum: 5000 - bbox: - $ref: "#/components/schemas/BoundingBox" features: type: "array" items: @@ -755,8 +846,6 @@ components: - display_name - names properties: - id: - type: string score: type: number minimum: 0 @@ -772,14 +861,14 @@ components: - source - layer properties: - id: + layer: type: string source: type: string - layer: - type: string - display_name: - type: string + source_id: + type: array + items: + type: string names: type: object required: From c7e87dd679a9ec7bc76cbe8b59b5065384cd2f33 Mon Sep 17 00:00:00 2001 From: Niv Greenstein Date: Thu, 12 Sep 2024 17:00:16 +0300 Subject: [PATCH 222/262] fix: fixed latLon routes and tests --- src/latLon/models/latLonManager.ts | 35 +++++++++++++----- tests/integration/latLon/latLon.spec.ts | 47 ++++++++++++++++--------- 2 files changed, 57 insertions(+), 25 deletions(-) diff --git a/src/latLon/models/latLonManager.ts b/src/latLon/models/latLonManager.ts index 26ff4991..4bfeea19 100644 --- a/src/latLon/models/latLonManager.ts +++ b/src/latLon/models/latLonManager.ts @@ -9,6 +9,7 @@ import { convertUTMToWgs84, convertWgs84ToUTM, validateWGS84Coordinate } from '. import { BadRequestError } from '../../common/errors'; import { WGS84Coordinate } from '../../common/interfaces'; import { parseGeo } from '../../location/utils'; +import { convertCamelToSnakeCase } from '../../control/utils'; @injectable() export class LatLonManager { @@ -62,11 +63,17 @@ export class LatLonManager { return { type: 'Feature', - query: { - lat, - lon, + geocoding: { + query: { + lat, + lon, + }, + response: convertCamelToSnakeCase({ + maxScore: 1, + resultsCount: 1, + matchLatencyMs: 0, + }), }, - response: {}, bbox, geometry: parseGeo({ bbox, @@ -75,6 +82,7 @@ export class LatLonManager { coordinates: [lon, lat], }, properties: { + name: tileCoordinateData.tile_name, tileName: tileCoordinateData.tile_name, subTileNumber: new Array(3).fill('').map(function (value, i) { return xNumber[i] + yNumber[i]; @@ -92,20 +100,29 @@ export class LatLonManager { [4]: '10m', [5]: '1m', }; + const mgrsStr = mgrs.forward([lon, lat], accuracy); return { type: 'Feature', - query: { - lat, - lon, + geocoding: { + query: { + lat, + lon, + }, + response: convertCamelToSnakeCase({ + maxScore: 1, + resultsCount: 1, + matchLatencyMs: 0, + }), }, - response: {}, + bbox: mgrs.inverse(mgrsStr), geometry: { type: 'Point', coordinates: [lon, lat], }, properties: { + name: mgrsStr, accuracy: accuracyString[accuracy], - mgrs: mgrs.forward([lon, lat], accuracy), + mgrs: mgrsStr, }, }; } diff --git a/tests/integration/latLon/latLon.spec.ts b/tests/integration/latLon/latLon.spec.ts index b91192e2..75d5a21c 100644 --- a/tests/integration/latLon/latLon.spec.ts +++ b/tests/integration/latLon/latLon.spec.ts @@ -43,18 +43,21 @@ describe('/lookup', function () { }); expect(response.status).toBe(httpStatusCodes.OK); - // expect(response).toSatisfyApiSpec(); + expect(response).toSatisfyApiSpec(); expect(response.body).toEqual({ type: 'Feature', - query: { - lat: 52.57326537485767, - lon: 12.948781146422107, - }, - response: {}, - properties: { - tileName: 'BRN', - subTileNumber: ['06', '97', '97'], + geocoding: { + query: { + lat: 52.57326537485767, + lon: 12.948781146422107, + }, + response: { + max_score: 1, + results_count: 1, + match_latency_ms: 0, + }, }, + bbox: [12.93694771534361, 52.51211561266182, 13.080296161196031, 52.60444267653175], geometry: { type: 'Polygon', coordinates: [ @@ -67,7 +70,11 @@ describe('/lookup', function () { ], ], }, - bbox: [12.93694771534361, 52.51211561266182, 13.080296161196031, 52.60444267653175], + properties: { + name: 'BRN', + tileName: 'BRN', + subTileNumber: ['06', '97', '97'], + }, }); }); @@ -79,19 +86,27 @@ describe('/lookup', function () { }); expect(response.status).toBe(httpStatusCodes.OK); - // expect(response).toSatisfyApiSpec(); + expect(response).toSatisfyApiSpec(); expect(response.body).toEqual({ type: 'Feature', - query: { - lat: 52.57326537485767, - lon: 12.948781146422107, + geocoding: { + query: { + lat: 52.57326537485767, + lon: 12.948781146422107, + }, + response: { + max_score: 1, + results_count: 1, + match_latency_ms: 0, + }, }, - response: {}, + bbox: [12.948777289238832, 52.57325754975297, 12.948791616108007, 52.57326678960368], geometry: { type: 'Point', coordinates: [12.948781146422107, 52.57326537485767], }, properties: { + name: '33UUU6099626777', accuracy: '1m', mgrs: '33UUU6099626777', }, @@ -108,7 +123,7 @@ describe('/lookup', function () { }); expect(response.status).toBe(httpStatusCodes.BAD_REQUEST); - // expect(response).toSatisfyApiSpec(); + // expect(response).toSatisfyApiSpec(); expect(response.body).toEqual({ message: "Invalid lat lon, check 'lat' and 'lon' keys exists and their values are legal", }); From 516d103cbb5be679391aca8ff262abdeebf8e47a Mon Sep 17 00:00:00 2001 From: Niv Greenstein Date: Thu, 12 Sep 2024 17:00:32 +0300 Subject: [PATCH 223/262] feat: added toSatisfyApiSpec(); to route --- tests/integration/mgrs/mgrs.spec.ts | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/tests/integration/mgrs/mgrs.spec.ts b/tests/integration/mgrs/mgrs.spec.ts index 0431f4e2..eb126aa8 100644 --- a/tests/integration/mgrs/mgrs.spec.ts +++ b/tests/integration/mgrs/mgrs.spec.ts @@ -10,8 +10,6 @@ import { getApp } from '../../../src/app'; import { SERVICES } from '../../../src/common/constants'; import { S3_REPOSITORY_SYMBOL } from '../../../src/common/s3/s3Repository'; import { cronLoadTileLatLonDataSymbol } from '../../../src/latLon/DAL/latLonDAL'; -import { GetGeotextSearchParams, QueryResult } from '../../../src/location/interfaces'; -import { GeoContextMode, IApplication } from '../../../src/common/interfaces'; import { MgrsRequestSender } from './helpers/requestSender'; describe('/search/MGRS', function () { @@ -47,6 +45,7 @@ describe('/search/MGRS', function () { const response = await requestSender.getTile({ tile: '18SUJ2339007393' }); expect(response.status).toBe(httpStatusCodes.OK); + expect(response).toSatisfyApiSpec(); expect(response.body).toMatchObject({ type: 'Feature', geocoding: { From 04de080c9bbf04c116c937b86bd49145801e5a6a Mon Sep 17 00:00:00 2001 From: Niv Greenstein Date: Thu, 12 Sep 2024 17:00:47 +0300 Subject: [PATCH 224/262] updated version --- package-lock.json | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/package-lock.json b/package-lock.json index b857e52e..37477bfe 100644 --- a/package-lock.json +++ b/package-lock.json @@ -1,12 +1,12 @@ { "name": "geocoding", - "version": "0.1.0-prealpha.2", + "version": "0.1.0-prealpha.3", "lockfileVersion": 2, "requires": true, "packages": { "": { "name": "geocoding", - "version": "0.1.0-prealpha.2", + "version": "0.1.0-prealpha.3", "hasInstallScript": true, "license": "ISC", "dependencies": { From 57361c35098fa6f15b1c9e3082fda64a2b635784 Mon Sep 17 00:00:00 2001 From: Niv Greenstein Date: Sun, 15 Sep 2024 11:27:51 +0300 Subject: [PATCH 225/262] fix: fixed healthCheckFactory so it won't cause server crush --- src/common/utils.ts | 30 ++++++++++++++++++------------ 1 file changed, 18 insertions(+), 12 deletions(-) diff --git a/src/common/utils.ts b/src/common/utils.ts index 8c45db35..a8ba210d 100644 --- a/src/common/utils.ts +++ b/src/common/utils.ts @@ -80,18 +80,24 @@ export const healthCheckFactory: FactoryFunction = (container: DependencyC const logger = container.resolve(SERVICES.LOGGER); const elasticClients = container.resolve(SERVICES.ELASTIC_CLIENTS); const s3Client = container.resolve(SERVICES.S3_CLIENT); - logger.info('Healthcheck is running'); - try { - for (const [key, client] of Object.entries(elasticClients)) { - logger.info(`Checking health of ${key}`); - void client.cluster.health({}); - } - - void s3Client.send(new ListBucketsCommand({})); - - logger.info('healthcheck passed'); - } catch (error) { - logger.error(`Healthcheck failed. Error: ${(error as Error).message}`); + for (const [key, client] of Object.entries(elasticClients)) { + client.cluster + .health({}) + .then(() => { + return; + }) + .catch((error) => { + logger.error(`Healthcheck failed for ${key}. Error: ${(error as Error).message}`); + }); } + + s3Client + .send(new ListBucketsCommand({})) + .then(() => { + return; + }) + .catch((error) => { + logger.error(`Healthcheck failed for S3. Error: ${(error as Error).message}`); + }); }; From 5955e131aa7e3d70ac6edca61b6ccf19582acaea Mon Sep 17 00:00:00 2001 From: NatalieShaked Date: Sun, 15 Sep 2024 15:07:26 +0300 Subject: [PATCH 226/262] fix: made minor fixes to the code in order for it to be more tidy --- package-lock.json | 11 ----------- package.json | 1 - src/common/errors.ts | 8 +++++++- src/common/interfaces.ts | 8 -------- src/common/middlewares/feedbackApi.middleware.ts | 1 - src/common/redis/index.ts | 3 ++- src/common/redis/interfaces.ts | 8 ++++++++ 7 files changed, 17 insertions(+), 23 deletions(-) create mode 100644 src/common/redis/interfaces.ts diff --git a/package-lock.json b/package-lock.json index 45416fbb..4632fa1b 100644 --- a/package-lock.json +++ b/package-lock.json @@ -14,7 +14,6 @@ "@elastic/elasticsearch": "^8.13.1", "@godaddy/terminus": "^4.12.1", "@map-colonies/cleanup-registry": "^1.1.0", - "@map-colonies/detiler-common": "^1.0.0", "@map-colonies/error-express-handler": "^2.1.0", "@map-colonies/express-access-log-middleware": "^2.0.1", "@map-colonies/js-logger": "^1.0.1", @@ -4855,11 +4854,6 @@ "tiny-typed-emitter": "^2.1.0" } }, - "node_modules/@map-colonies/detiler-common": { - "version": "1.0.0", - "resolved": "https://registry.npmjs.org/@map-colonies/detiler-common/-/detiler-common-1.0.0.tgz", - "integrity": "sha512-5gxowEjHBD9CcgqonYQGyarIXRElYnhO4qO/ffxN3IFieWyUWUuKFhc+U6rnvy53TN0UorPfTuOrCO85W9ZsKw==" - }, "node_modules/@map-colonies/error-express-handler": { "version": "2.1.0", "resolved": "https://registry.npmjs.org/@map-colonies/error-express-handler/-/error-express-handler-2.1.0.tgz", @@ -24876,11 +24870,6 @@ "tiny-typed-emitter": "^2.1.0" } }, - "@map-colonies/detiler-common": { - "version": "1.0.0", - "resolved": "https://registry.npmjs.org/@map-colonies/detiler-common/-/detiler-common-1.0.0.tgz", - "integrity": "sha512-5gxowEjHBD9CcgqonYQGyarIXRElYnhO4qO/ffxN3IFieWyUWUuKFhc+U6rnvy53TN0UorPfTuOrCO85W9ZsKw==" - }, "@map-colonies/error-express-handler": { "version": "2.1.0", "resolved": "https://registry.npmjs.org/@map-colonies/error-express-handler/-/error-express-handler-2.1.0.tgz", diff --git a/package.json b/package.json index b7148db5..9f55b225 100644 --- a/package.json +++ b/package.json @@ -46,7 +46,6 @@ "@elastic/elasticsearch": "^8.13.1", "@godaddy/terminus": "^4.12.1", "@map-colonies/cleanup-registry": "^1.1.0", - "@map-colonies/detiler-common": "^1.0.0", "@map-colonies/error-express-handler": "^2.1.0", "@map-colonies/express-access-log-middleware": "^2.0.1", "@map-colonies/js-logger": "^1.0.1", diff --git a/src/common/errors.ts b/src/common/errors.ts index 374b146c..f23ed0cd 100644 --- a/src/common/errors.ts +++ b/src/common/errors.ts @@ -50,4 +50,10 @@ export class NotImplementedError extends Error implements HttpError { } } -export class TimeoutError extends Error {} +export class TimeoutError extends Error implements HttpError { + public readonly status = httpStatus.REQUEST_TIMEOUT; + + public constructor(message: string) { + super(message); + } +} diff --git a/src/common/interfaces.ts b/src/common/interfaces.ts index 9b8138ed..a120b1ad 100644 --- a/src/common/interfaces.ts +++ b/src/common/interfaces.ts @@ -1,4 +1,3 @@ -import { RedisClientOptions } from 'redis'; import { BBox, Feature, FeatureCollection as GeoJSONFeatureCollection } from 'geojson'; export interface IConfig { @@ -13,13 +12,6 @@ export interface OpenApiConfig { uiPath: string; } -export type RedisConfig = { - host: string; - port: number; - enableSslAuth: boolean; - sslPaths: { ca: string; cert: string; key: string }; -} & RedisClientOptions; - export interface GeoContext { bbox?: BBox; radius?: number; diff --git a/src/common/middlewares/feedbackApi.middleware.ts b/src/common/middlewares/feedbackApi.middleware.ts index e3774b2b..193d0443 100644 --- a/src/common/middlewares/feedbackApi.middleware.ts +++ b/src/common/middlewares/feedbackApi.middleware.ts @@ -30,7 +30,6 @@ export class FeedbackApiMiddlewareManager { const originalJson = res.json; const logJson = async function (this: Response, body: JSON): Promise { - // console.log('Response:', body); geocodingResponseDetails.response = body; try { diff --git a/src/common/redis/index.ts b/src/common/redis/index.ts index 357badc1..514c3ed5 100644 --- a/src/common/redis/index.ts +++ b/src/common/redis/index.ts @@ -3,7 +3,8 @@ import { Logger } from '@map-colonies/js-logger'; import { createClient, RedisClientOptions } from 'redis'; import { DependencyContainer, FactoryFunction } from 'tsyringe'; import { SERVICES } from '../constants'; -import { RedisConfig, IConfig } from '../interfaces'; +import { IConfig } from '../interfaces'; +import { RedisConfig } from './interfaces'; const DEFAULT_LIMIT_FROM = 0; const DEFAULT_LIMIT_SIZE = 1000; diff --git a/src/common/redis/interfaces.ts b/src/common/redis/interfaces.ts new file mode 100644 index 00000000..d569f1eb --- /dev/null +++ b/src/common/redis/interfaces.ts @@ -0,0 +1,8 @@ +import { RedisClientOptions } from 'redis'; + +export type RedisConfig = { + host: string; + port: number; + enableSslAuth: boolean; + sslPaths: { ca: string; cert: string; key: string }; + } & RedisClientOptions; \ No newline at end of file From 7f55e46c178623508553bf0af2d7cf49b4c62ec0 Mon Sep 17 00:00:00 2001 From: Niv Greenstein Date: Sun, 15 Sep 2024 15:29:47 +0300 Subject: [PATCH 227/262] feat: added ajv and changed validateGeoContext to use it --- package-lock.json | 468 +++++++++++++++++++------------------------- package.json | 1 + src/common/utils.ts | 88 ++++++++- 3 files changed, 286 insertions(+), 271 deletions(-) diff --git a/package-lock.json b/package-lock.json index 37477bfe..bd0c0643 100644 --- a/package-lock.json +++ b/package-lock.json @@ -25,6 +25,7 @@ "@opentelemetry/instrumentation-express": "0.32.1", "@opentelemetry/instrumentation-http": "0.35.1", "@smithy/node-http-handler": "^3.1.4", + "ajv": "^8.17.1", "axios": "^0.21.1", "compression": "^1.7.4", "config": "^3.3.9", @@ -3155,28 +3156,6 @@ "node": ">=v14" } }, - "node_modules/@commitlint/config-validator/node_modules/ajv": { - "version": "8.12.0", - "resolved": "https://registry.npmjs.org/ajv/-/ajv-8.12.0.tgz", - "integrity": "sha512-sRu1kpcO9yLtYxBKvqfTeh9KzZEwO3STyX1HT+4CaDzC6HpTGYhIhPIzj9XuKU7KYDwnaeh5hcOwjy1QuJzBPA==", - "dev": true, - "dependencies": { - "fast-deep-equal": "^3.1.1", - "json-schema-traverse": "^1.0.0", - "require-from-string": "^2.0.2", - "uri-js": "^4.2.2" - }, - "funding": { - "type": "github", - "url": "https://github.com/sponsors/epoberezkin" - } - }, - "node_modules/@commitlint/config-validator/node_modules/json-schema-traverse": { - "version": "1.0.0", - "resolved": "https://registry.npmjs.org/json-schema-traverse/-/json-schema-traverse-1.0.0.tgz", - "integrity": "sha512-NM8/P9n3XjXhIZn1lLhkFaACTOURQXjWhV4BA/RnOv8xvgqtqpAX9IO4mRQxSx1Rlo4tqzeqb0sOlruaOy3dug==", - "dev": true - }, "node_modules/@commitlint/ensure": { "version": "17.4.4", "resolved": "https://registry.npmjs.org/@commitlint/ensure/-/ensure-17.4.4.tgz", @@ -3775,6 +3754,23 @@ "url": "https://opencollective.com/eslint" } }, + "node_modules/@eslint/eslintrc/node_modules/ajv": { + "version": "6.12.6", + "resolved": "https://registry.npmjs.org/ajv/-/ajv-6.12.6.tgz", + "integrity": "sha512-j3fVLgvTo527anyYyJOGTYJbG+vnnQYvE0m5mmkc1TK+nxAppkCLMIL0aZ4dblVCNoGShhm+kzE4ZUykBoMg4g==", + "dev": true, + "license": "MIT", + "dependencies": { + "fast-deep-equal": "^3.1.1", + "fast-json-stable-stringify": "^2.0.0", + "json-schema-traverse": "^0.4.1", + "uri-js": "^4.2.2" + }, + "funding": { + "type": "github", + "url": "https://github.com/sponsors/epoberezkin" + } + }, "node_modules/@eslint/eslintrc/node_modules/globals": { "version": "13.20.0", "resolved": "https://registry.npmjs.org/globals/-/globals-13.20.0.tgz", @@ -3790,6 +3786,13 @@ "url": "https://github.com/sponsors/sindresorhus" } }, + "node_modules/@eslint/eslintrc/node_modules/json-schema-traverse": { + "version": "0.4.1", + "resolved": "https://registry.npmjs.org/json-schema-traverse/-/json-schema-traverse-0.4.1.tgz", + "integrity": "sha512-xbbCH5dCYU5T8LcEhhuh7HJ88HXuW3qsI3Y0zOZFKfZEHcpWiHU/Jxzk629Brsab/mMiHQti9wMP+845RPe3Vg==", + "dev": true, + "license": "MIT" + }, "node_modules/@eslint/eslintrc/node_modules/type-fest": { "version": "0.20.2", "resolved": "https://registry.npmjs.org/type-fest/-/type-fest-0.20.2.tgz", @@ -3819,6 +3822,28 @@ "ajv": "^6.12.6" } }, + "node_modules/@fastify/ajv-compiler/node_modules/ajv": { + "version": "6.12.6", + "resolved": "https://registry.npmjs.org/ajv/-/ajv-6.12.6.tgz", + "integrity": "sha512-j3fVLgvTo527anyYyJOGTYJbG+vnnQYvE0m5mmkc1TK+nxAppkCLMIL0aZ4dblVCNoGShhm+kzE4ZUykBoMg4g==", + "license": "MIT", + "dependencies": { + "fast-deep-equal": "^3.1.1", + "fast-json-stable-stringify": "^2.0.0", + "json-schema-traverse": "^0.4.1", + "uri-js": "^4.2.2" + }, + "funding": { + "type": "github", + "url": "https://github.com/sponsors/epoberezkin" + } + }, + "node_modules/@fastify/ajv-compiler/node_modules/json-schema-traverse": { + "version": "0.4.1", + "resolved": "https://registry.npmjs.org/json-schema-traverse/-/json-schema-traverse-0.4.1.tgz", + "integrity": "sha512-xbbCH5dCYU5T8LcEhhuh7HJ88HXuW3qsI3Y0zOZFKfZEHcpWiHU/Jxzk629Brsab/mMiHQti9wMP+845RPe3Vg==", + "license": "MIT" + }, "node_modules/@fastify/busboy": { "version": "2.1.1", "resolved": "https://registry.npmjs.org/@fastify/busboy/-/busboy-2.1.1.tgz", @@ -7363,12 +7388,6 @@ "url": "https://github.com/sponsors/epoberezkin" } }, - "node_modules/@redocly/ajv/node_modules/json-schema-traverse": { - "version": "1.0.0", - "resolved": "https://registry.npmjs.org/json-schema-traverse/-/json-schema-traverse-1.0.0.tgz", - "integrity": "sha512-NM8/P9n3XjXhIZn1lLhkFaACTOURQXjWhV4BA/RnOv8xvgqtqpAX9IO4mRQxSx1Rlo4tqzeqb0sOlruaOy3dug==", - "dev": true - }, "node_modules/@redocly/openapi-cli": { "version": "1.0.0-beta.95", "resolved": "https://registry.npmjs.org/@redocly/openapi-cli/-/openapi-cli-1.0.0-beta.95.tgz", @@ -9371,14 +9390,15 @@ "dev": true }, "node_modules/ajv": { - "version": "6.12.6", - "resolved": "https://registry.npmjs.org/ajv/-/ajv-6.12.6.tgz", - "integrity": "sha512-j3fVLgvTo527anyYyJOGTYJbG+vnnQYvE0m5mmkc1TK+nxAppkCLMIL0aZ4dblVCNoGShhm+kzE4ZUykBoMg4g==", + "version": "8.17.1", + "resolved": "https://registry.npmjs.org/ajv/-/ajv-8.17.1.tgz", + "integrity": "sha512-B/gBuNg5SiMTrPkC+A2+cW0RszwxYmn6VYxB/inlBStS5nx6xHIt/ehKRhIMhqusl7a8LjQoZnjCs5vhwxOQ1g==", + "license": "MIT", "dependencies": { - "fast-deep-equal": "^3.1.1", - "fast-json-stable-stringify": "^2.0.0", - "json-schema-traverse": "^0.4.1", - "uri-js": "^4.2.2" + "fast-deep-equal": "^3.1.3", + "fast-uri": "^3.0.1", + "json-schema-traverse": "^1.0.0", + "require-from-string": "^2.0.2" }, "funding": { "type": "github", @@ -9401,26 +9421,6 @@ } } }, - "node_modules/ajv-formats/node_modules/ajv": { - "version": "8.11.0", - "resolved": "https://registry.npmjs.org/ajv/-/ajv-8.11.0.tgz", - "integrity": "sha512-wGgprdCvMalC0BztXvitD2hC04YffAvtsUn93JbGXYLAtCUO4xd17mCCZQxUOItiBwZvJScWo8NIvQMQ71rdpg==", - "dependencies": { - "fast-deep-equal": "^3.1.1", - "json-schema-traverse": "^1.0.0", - "require-from-string": "^2.0.2", - "uri-js": "^4.2.2" - }, - "funding": { - "type": "github", - "url": "https://github.com/sponsors/epoberezkin" - } - }, - "node_modules/ajv-formats/node_modules/json-schema-traverse": { - "version": "1.0.0", - "resolved": "https://registry.npmjs.org/json-schema-traverse/-/json-schema-traverse-1.0.0.tgz", - "integrity": "sha512-NM8/P9n3XjXhIZn1lLhkFaACTOURQXjWhV4BA/RnOv8xvgqtqpAX9IO4mRQxSx1Rlo4tqzeqb0sOlruaOy3dug==" - }, "node_modules/ansi-escapes": { "version": "4.3.2", "resolved": "https://registry.npmjs.org/ansi-escapes/-/ansi-escapes-4.3.2.tgz", @@ -12317,6 +12317,23 @@ "url": "https://opencollective.com/eslint" } }, + "node_modules/eslint/node_modules/ajv": { + "version": "6.12.6", + "resolved": "https://registry.npmjs.org/ajv/-/ajv-6.12.6.tgz", + "integrity": "sha512-j3fVLgvTo527anyYyJOGTYJbG+vnnQYvE0m5mmkc1TK+nxAppkCLMIL0aZ4dblVCNoGShhm+kzE4ZUykBoMg4g==", + "dev": true, + "license": "MIT", + "dependencies": { + "fast-deep-equal": "^3.1.1", + "fast-json-stable-stringify": "^2.0.0", + "json-schema-traverse": "^0.4.1", + "uri-js": "^4.2.2" + }, + "funding": { + "type": "github", + "url": "https://github.com/sponsors/epoberezkin" + } + }, "node_modules/eslint/node_modules/ansi-styles": { "version": "4.3.0", "resolved": "https://registry.npmjs.org/ansi-styles/-/ansi-styles-4.3.0.tgz", @@ -12467,6 +12484,13 @@ "node": ">=8" } }, + "node_modules/eslint/node_modules/json-schema-traverse": { + "version": "0.4.1", + "resolved": "https://registry.npmjs.org/json-schema-traverse/-/json-schema-traverse-0.4.1.tgz", + "integrity": "sha512-xbbCH5dCYU5T8LcEhhuh7HJ88HXuW3qsI3Y0zOZFKfZEHcpWiHU/Jxzk629Brsab/mMiHQti9wMP+845RPe3Vg==", + "dev": true, + "license": "MIT" + }, "node_modules/eslint/node_modules/locate-path": { "version": "6.0.0", "resolved": "https://registry.npmjs.org/locate-path/-/locate-path-6.0.0.tgz", @@ -12781,21 +12805,6 @@ "path-to-regexp": "^6.2.0" } }, - "node_modules/express-openapi-validator/node_modules/ajv": { - "version": "8.12.0", - "resolved": "https://registry.npmjs.org/ajv/-/ajv-8.12.0.tgz", - "integrity": "sha512-sRu1kpcO9yLtYxBKvqfTeh9KzZEwO3STyX1HT+4CaDzC6HpTGYhIhPIzj9XuKU7KYDwnaeh5hcOwjy1QuJzBPA==", - "dependencies": { - "fast-deep-equal": "^3.1.1", - "json-schema-traverse": "^1.0.0", - "require-from-string": "^2.0.2", - "uri-js": "^4.2.2" - }, - "funding": { - "type": "github", - "url": "https://github.com/sponsors/epoberezkin" - } - }, "node_modules/express-openapi-validator/node_modules/ajv-draft-04": { "version": "1.0.0", "resolved": "https://registry.npmjs.org/ajv-draft-04/-/ajv-draft-04-1.0.0.tgz", @@ -12809,11 +12818,6 @@ } } }, - "node_modules/express-openapi-validator/node_modules/json-schema-traverse": { - "version": "1.0.0", - "resolved": "https://registry.npmjs.org/json-schema-traverse/-/json-schema-traverse-1.0.0.tgz", - "integrity": "sha512-NM8/P9n3XjXhIZn1lLhkFaACTOURQXjWhV4BA/RnOv8xvgqtqpAX9IO4mRQxSx1Rlo4tqzeqb0sOlruaOy3dug==" - }, "node_modules/express-openapi-validator/node_modules/media-typer": { "version": "1.1.0", "resolved": "https://registry.npmjs.org/media-typer/-/media-typer-1.1.0.tgz", @@ -12917,6 +12921,28 @@ "node": ">= 10.0.0" } }, + "node_modules/fast-json-stringify/node_modules/ajv": { + "version": "6.12.6", + "resolved": "https://registry.npmjs.org/ajv/-/ajv-6.12.6.tgz", + "integrity": "sha512-j3fVLgvTo527anyYyJOGTYJbG+vnnQYvE0m5mmkc1TK+nxAppkCLMIL0aZ4dblVCNoGShhm+kzE4ZUykBoMg4g==", + "license": "MIT", + "dependencies": { + "fast-deep-equal": "^3.1.1", + "fast-json-stable-stringify": "^2.0.0", + "json-schema-traverse": "^0.4.1", + "uri-js": "^4.2.2" + }, + "funding": { + "type": "github", + "url": "https://github.com/sponsors/epoberezkin" + } + }, + "node_modules/fast-json-stringify/node_modules/json-schema-traverse": { + "version": "0.4.1", + "resolved": "https://registry.npmjs.org/json-schema-traverse/-/json-schema-traverse-0.4.1.tgz", + "integrity": "sha512-xbbCH5dCYU5T8LcEhhuh7HJ88HXuW3qsI3Y0zOZFKfZEHcpWiHU/Jxzk629Brsab/mMiHQti9wMP+845RPe3Vg==", + "license": "MIT" + }, "node_modules/fast-levenshtein": { "version": "2.0.6", "resolved": "https://registry.npmjs.org/fast-levenshtein/-/fast-levenshtein-2.0.6.tgz", @@ -12936,6 +12962,12 @@ "resolved": "https://registry.npmjs.org/fast-safe-stringify/-/fast-safe-stringify-2.1.1.tgz", "integrity": "sha512-W+KJc2dmILlPplD/H4K9l9LcAHAfPtP6BY84uVLXQ6Evcz9Lcg33Y2z1IVblT6xdY54PXYVHEv+0Wpq8Io6zkA==" }, + "node_modules/fast-uri": { + "version": "3.0.1", + "resolved": "https://registry.npmjs.org/fast-uri/-/fast-uri-3.0.1.tgz", + "integrity": "sha512-MWipKbbYiYI0UC7cl8m/i/IWTqfC8YXsqjzybjddLsFjStroQzsHXkc73JutMvBiXmOvapk+axIl79ig5t55Bw==", + "license": "MIT" + }, "node_modules/fast-xml-parser": { "version": "4.4.1", "resolved": "https://registry.npmjs.org/fast-xml-parser/-/fast-xml-parser-4.4.1.tgz", @@ -16565,9 +16597,10 @@ "dev": true }, "node_modules/json-schema-traverse": { - "version": "0.4.1", - "resolved": "https://registry.npmjs.org/json-schema-traverse/-/json-schema-traverse-0.4.1.tgz", - "integrity": "sha512-xbbCH5dCYU5T8LcEhhuh7HJ88HXuW3qsI3Y0zOZFKfZEHcpWiHU/Jxzk629Brsab/mMiHQti9wMP+845RPe3Vg==" + "version": "1.0.0", + "resolved": "https://registry.npmjs.org/json-schema-traverse/-/json-schema-traverse-1.0.0.tgz", + "integrity": "sha512-NM8/P9n3XjXhIZn1lLhkFaACTOURQXjWhV4BA/RnOv8xvgqtqpAX9IO4mRQxSx1Rlo4tqzeqb0sOlruaOy3dug==", + "license": "MIT" }, "node_modules/json-stable-stringify-without-jsonify": { "version": "1.0.1", @@ -16699,26 +16732,6 @@ "set-cookie-parser": "^2.4.1" } }, - "node_modules/light-my-request/node_modules/ajv": { - "version": "8.12.0", - "resolved": "https://registry.npmjs.org/ajv/-/ajv-8.12.0.tgz", - "integrity": "sha512-sRu1kpcO9yLtYxBKvqfTeh9KzZEwO3STyX1HT+4CaDzC6HpTGYhIhPIzj9XuKU7KYDwnaeh5hcOwjy1QuJzBPA==", - "dependencies": { - "fast-deep-equal": "^3.1.1", - "json-schema-traverse": "^1.0.0", - "require-from-string": "^2.0.2", - "uri-js": "^4.2.2" - }, - "funding": { - "type": "github", - "url": "https://github.com/sponsors/epoberezkin" - } - }, - "node_modules/light-my-request/node_modules/json-schema-traverse": { - "version": "1.0.0", - "resolved": "https://registry.npmjs.org/json-schema-traverse/-/json-schema-traverse-1.0.0.tgz", - "integrity": "sha512-NM8/P9n3XjXhIZn1lLhkFaACTOURQXjWhV4BA/RnOv8xvgqtqpAX9IO4mRQxSx1Rlo4tqzeqb0sOlruaOy3dug==" - }, "node_modules/light-my-request/node_modules/process-warning": { "version": "1.0.0", "resolved": "https://registry.npmjs.org/process-warning/-/process-warning-1.0.0.tgz", @@ -17699,28 +17712,6 @@ "openapi-types": "^9.3.1" } }, - "node_modules/openapi-response-validator/node_modules/ajv": { - "version": "8.11.0", - "resolved": "https://registry.npmjs.org/ajv/-/ajv-8.11.0.tgz", - "integrity": "sha512-wGgprdCvMalC0BztXvitD2hC04YffAvtsUn93JbGXYLAtCUO4xd17mCCZQxUOItiBwZvJScWo8NIvQMQ71rdpg==", - "dev": true, - "dependencies": { - "fast-deep-equal": "^3.1.1", - "json-schema-traverse": "^1.0.0", - "require-from-string": "^2.0.2", - "uri-js": "^4.2.2" - }, - "funding": { - "type": "github", - "url": "https://github.com/sponsors/epoberezkin" - } - }, - "node_modules/openapi-response-validator/node_modules/json-schema-traverse": { - "version": "1.0.0", - "resolved": "https://registry.npmjs.org/json-schema-traverse/-/json-schema-traverse-1.0.0.tgz", - "integrity": "sha512-NM8/P9n3XjXhIZn1lLhkFaACTOURQXjWhV4BA/RnOv8xvgqtqpAX9IO4mRQxSx1Rlo4tqzeqb0sOlruaOy3dug==", - "dev": true - }, "node_modules/openapi-schema-validator": { "version": "9.3.1", "resolved": "https://registry.npmjs.org/openapi-schema-validator/-/openapi-schema-validator-9.3.1.tgz", @@ -17733,28 +17724,6 @@ "openapi-types": "^9.3.1" } }, - "node_modules/openapi-schema-validator/node_modules/ajv": { - "version": "8.11.0", - "resolved": "https://registry.npmjs.org/ajv/-/ajv-8.11.0.tgz", - "integrity": "sha512-wGgprdCvMalC0BztXvitD2hC04YffAvtsUn93JbGXYLAtCUO4xd17mCCZQxUOItiBwZvJScWo8NIvQMQ71rdpg==", - "dev": true, - "dependencies": { - "fast-deep-equal": "^3.1.1", - "json-schema-traverse": "^1.0.0", - "require-from-string": "^2.0.2", - "uri-js": "^4.2.2" - }, - "funding": { - "type": "github", - "url": "https://github.com/sponsors/epoberezkin" - } - }, - "node_modules/openapi-schema-validator/node_modules/json-schema-traverse": { - "version": "1.0.0", - "resolved": "https://registry.npmjs.org/json-schema-traverse/-/json-schema-traverse-1.0.0.tgz", - "integrity": "sha512-NM8/P9n3XjXhIZn1lLhkFaACTOURQXjWhV4BA/RnOv8xvgqtqpAX9IO4mRQxSx1Rlo4tqzeqb0sOlruaOy3dug==", - "dev": true - }, "node_modules/openapi-types": { "version": "9.3.1", "resolved": "https://registry.npmjs.org/openapi-types/-/openapi-types-9.3.1.tgz", @@ -23470,26 +23439,6 @@ "requires": { "@commitlint/types": "^17.4.4", "ajv": "^8.11.0" - }, - "dependencies": { - "ajv": { - "version": "8.12.0", - "resolved": "https://registry.npmjs.org/ajv/-/ajv-8.12.0.tgz", - "integrity": "sha512-sRu1kpcO9yLtYxBKvqfTeh9KzZEwO3STyX1HT+4CaDzC6HpTGYhIhPIzj9XuKU7KYDwnaeh5hcOwjy1QuJzBPA==", - "dev": true, - "requires": { - "fast-deep-equal": "^3.1.1", - "json-schema-traverse": "^1.0.0", - "require-from-string": "^2.0.2", - "uri-js": "^4.2.2" - } - }, - "json-schema-traverse": { - "version": "1.0.0", - "resolved": "https://registry.npmjs.org/json-schema-traverse/-/json-schema-traverse-1.0.0.tgz", - "integrity": "sha512-NM8/P9n3XjXhIZn1lLhkFaACTOURQXjWhV4BA/RnOv8xvgqtqpAX9IO4mRQxSx1Rlo4tqzeqb0sOlruaOy3dug==", - "dev": true - } } }, "@commitlint/ensure": { @@ -23945,6 +23894,18 @@ "strip-json-comments": "^3.1.1" }, "dependencies": { + "ajv": { + "version": "6.12.6", + "resolved": "https://registry.npmjs.org/ajv/-/ajv-6.12.6.tgz", + "integrity": "sha512-j3fVLgvTo527anyYyJOGTYJbG+vnnQYvE0m5mmkc1TK+nxAppkCLMIL0aZ4dblVCNoGShhm+kzE4ZUykBoMg4g==", + "dev": true, + "requires": { + "fast-deep-equal": "^3.1.1", + "fast-json-stable-stringify": "^2.0.0", + "json-schema-traverse": "^0.4.1", + "uri-js": "^4.2.2" + } + }, "globals": { "version": "13.20.0", "resolved": "https://registry.npmjs.org/globals/-/globals-13.20.0.tgz", @@ -23954,6 +23915,12 @@ "type-fest": "^0.20.2" } }, + "json-schema-traverse": { + "version": "0.4.1", + "resolved": "https://registry.npmjs.org/json-schema-traverse/-/json-schema-traverse-0.4.1.tgz", + "integrity": "sha512-xbbCH5dCYU5T8LcEhhuh7HJ88HXuW3qsI3Y0zOZFKfZEHcpWiHU/Jxzk629Brsab/mMiHQti9wMP+845RPe3Vg==", + "dev": true + }, "type-fest": { "version": "0.20.2", "resolved": "https://registry.npmjs.org/type-fest/-/type-fest-0.20.2.tgz", @@ -23974,6 +23941,24 @@ "integrity": "sha512-gvCOUNpXsWrIQ3A4aXCLIdblL0tDq42BG/2Xw7oxbil9h11uow10ztS2GuFazNBfjbrsZ5nl+nPl5jDSjj5TSg==", "requires": { "ajv": "^6.12.6" + }, + "dependencies": { + "ajv": { + "version": "6.12.6", + "resolved": "https://registry.npmjs.org/ajv/-/ajv-6.12.6.tgz", + "integrity": "sha512-j3fVLgvTo527anyYyJOGTYJbG+vnnQYvE0m5mmkc1TK+nxAppkCLMIL0aZ4dblVCNoGShhm+kzE4ZUykBoMg4g==", + "requires": { + "fast-deep-equal": "^3.1.1", + "fast-json-stable-stringify": "^2.0.0", + "json-schema-traverse": "^0.4.1", + "uri-js": "^4.2.2" + } + }, + "json-schema-traverse": { + "version": "0.4.1", + "resolved": "https://registry.npmjs.org/json-schema-traverse/-/json-schema-traverse-0.4.1.tgz", + "integrity": "sha512-xbbCH5dCYU5T8LcEhhuh7HJ88HXuW3qsI3Y0zOZFKfZEHcpWiHU/Jxzk629Brsab/mMiHQti9wMP+845RPe3Vg==" + } } }, "@fastify/busboy": { @@ -26575,14 +26560,6 @@ "json-schema-traverse": "^1.0.0", "require-from-string": "^2.0.2", "uri-js": "^4.2.2" - }, - "dependencies": { - "json-schema-traverse": { - "version": "1.0.0", - "resolved": "https://registry.npmjs.org/json-schema-traverse/-/json-schema-traverse-1.0.0.tgz", - "integrity": "sha512-NM8/P9n3XjXhIZn1lLhkFaACTOURQXjWhV4BA/RnOv8xvgqtqpAX9IO4mRQxSx1Rlo4tqzeqb0sOlruaOy3dug==", - "dev": true - } } }, "@redocly/openapi-cli": { @@ -28383,14 +28360,14 @@ "dev": true }, "ajv": { - "version": "6.12.6", - "resolved": "https://registry.npmjs.org/ajv/-/ajv-6.12.6.tgz", - "integrity": "sha512-j3fVLgvTo527anyYyJOGTYJbG+vnnQYvE0m5mmkc1TK+nxAppkCLMIL0aZ4dblVCNoGShhm+kzE4ZUykBoMg4g==", + "version": "8.17.1", + "resolved": "https://registry.npmjs.org/ajv/-/ajv-8.17.1.tgz", + "integrity": "sha512-B/gBuNg5SiMTrPkC+A2+cW0RszwxYmn6VYxB/inlBStS5nx6xHIt/ehKRhIMhqusl7a8LjQoZnjCs5vhwxOQ1g==", "requires": { - "fast-deep-equal": "^3.1.1", - "fast-json-stable-stringify": "^2.0.0", - "json-schema-traverse": "^0.4.1", - "uri-js": "^4.2.2" + "fast-deep-equal": "^3.1.3", + "fast-uri": "^3.0.1", + "json-schema-traverse": "^1.0.0", + "require-from-string": "^2.0.2" } }, "ajv-formats": { @@ -28399,24 +28376,6 @@ "integrity": "sha512-Wx0Kx52hxE7C18hkMEggYlEifqWZtYaRgouJor+WMdPnQyEK13vgEWyVNup7SoeeoLMsr4kf5h6dOW11I15MUA==", "requires": { "ajv": "^8.0.0" - }, - "dependencies": { - "ajv": { - "version": "8.11.0", - "resolved": "https://registry.npmjs.org/ajv/-/ajv-8.11.0.tgz", - "integrity": "sha512-wGgprdCvMalC0BztXvitD2hC04YffAvtsUn93JbGXYLAtCUO4xd17mCCZQxUOItiBwZvJScWo8NIvQMQ71rdpg==", - "requires": { - "fast-deep-equal": "^3.1.1", - "json-schema-traverse": "^1.0.0", - "require-from-string": "^2.0.2", - "uri-js": "^4.2.2" - } - }, - "json-schema-traverse": { - "version": "1.0.0", - "resolved": "https://registry.npmjs.org/json-schema-traverse/-/json-schema-traverse-1.0.0.tgz", - "integrity": "sha512-NM8/P9n3XjXhIZn1lLhkFaACTOURQXjWhV4BA/RnOv8xvgqtqpAX9IO4mRQxSx1Rlo4tqzeqb0sOlruaOy3dug==" - } } }, "ansi-escapes": { @@ -30338,6 +30297,18 @@ "text-table": "^0.2.0" }, "dependencies": { + "ajv": { + "version": "6.12.6", + "resolved": "https://registry.npmjs.org/ajv/-/ajv-6.12.6.tgz", + "integrity": "sha512-j3fVLgvTo527anyYyJOGTYJbG+vnnQYvE0m5mmkc1TK+nxAppkCLMIL0aZ4dblVCNoGShhm+kzE4ZUykBoMg4g==", + "dev": true, + "requires": { + "fast-deep-equal": "^3.1.1", + "fast-json-stable-stringify": "^2.0.0", + "json-schema-traverse": "^0.4.1", + "uri-js": "^4.2.2" + } + }, "ansi-styles": { "version": "4.3.0", "resolved": "https://registry.npmjs.org/ansi-styles/-/ansi-styles-4.3.0.tgz", @@ -30437,6 +30408,12 @@ "integrity": "sha512-EykJT/Q1KjTWctppgIAgfSO0tKVuZUjhgMr17kqTumMl6Afv3EISleU7qZUzoXDFTAHTDC4NOoG/ZxU3EvlMPQ==", "dev": true }, + "json-schema-traverse": { + "version": "0.4.1", + "resolved": "https://registry.npmjs.org/json-schema-traverse/-/json-schema-traverse-0.4.1.tgz", + "integrity": "sha512-xbbCH5dCYU5T8LcEhhuh7HJ88HXuW3qsI3Y0zOZFKfZEHcpWiHU/Jxzk629Brsab/mMiHQti9wMP+845RPe3Vg==", + "dev": true + }, "locate-path": { "version": "6.0.0", "resolved": "https://registry.npmjs.org/locate-path/-/locate-path-6.0.0.tgz", @@ -31000,28 +30977,12 @@ "path-to-regexp": "^6.2.0" }, "dependencies": { - "ajv": { - "version": "8.12.0", - "resolved": "https://registry.npmjs.org/ajv/-/ajv-8.12.0.tgz", - "integrity": "sha512-sRu1kpcO9yLtYxBKvqfTeh9KzZEwO3STyX1HT+4CaDzC6HpTGYhIhPIzj9XuKU7KYDwnaeh5hcOwjy1QuJzBPA==", - "requires": { - "fast-deep-equal": "^3.1.1", - "json-schema-traverse": "^1.0.0", - "require-from-string": "^2.0.2", - "uri-js": "^4.2.2" - } - }, "ajv-draft-04": { "version": "1.0.0", "resolved": "https://registry.npmjs.org/ajv-draft-04/-/ajv-draft-04-1.0.0.tgz", "integrity": "sha512-mv00Te6nmYbRp5DCwclxtt7yV/joXJPGS7nM+97GdxvuttCOfgI3K4U25zboyeX0O+myI8ERluxQe5wljMmVIw==", "requires": {} }, - "json-schema-traverse": { - "version": "1.0.0", - "resolved": "https://registry.npmjs.org/json-schema-traverse/-/json-schema-traverse-1.0.0.tgz", - "integrity": "sha512-NM8/P9n3XjXhIZn1lLhkFaACTOURQXjWhV4BA/RnOv8xvgqtqpAX9IO4mRQxSx1Rlo4tqzeqb0sOlruaOy3dug==" - }, "media-typer": { "version": "1.1.0", "resolved": "https://registry.npmjs.org/media-typer/-/media-typer-1.1.0.tgz", @@ -31092,6 +31053,24 @@ "deepmerge": "^4.2.2", "rfdc": "^1.2.0", "string-similarity": "^4.0.1" + }, + "dependencies": { + "ajv": { + "version": "6.12.6", + "resolved": "https://registry.npmjs.org/ajv/-/ajv-6.12.6.tgz", + "integrity": "sha512-j3fVLgvTo527anyYyJOGTYJbG+vnnQYvE0m5mmkc1TK+nxAppkCLMIL0aZ4dblVCNoGShhm+kzE4ZUykBoMg4g==", + "requires": { + "fast-deep-equal": "^3.1.1", + "fast-json-stable-stringify": "^2.0.0", + "json-schema-traverse": "^0.4.1", + "uri-js": "^4.2.2" + } + }, + "json-schema-traverse": { + "version": "0.4.1", + "resolved": "https://registry.npmjs.org/json-schema-traverse/-/json-schema-traverse-0.4.1.tgz", + "integrity": "sha512-xbbCH5dCYU5T8LcEhhuh7HJ88HXuW3qsI3Y0zOZFKfZEHcpWiHU/Jxzk629Brsab/mMiHQti9wMP+845RPe3Vg==" + } } }, "fast-levenshtein": { @@ -31110,6 +31089,11 @@ "resolved": "https://registry.npmjs.org/fast-safe-stringify/-/fast-safe-stringify-2.1.1.tgz", "integrity": "sha512-W+KJc2dmILlPplD/H4K9l9LcAHAfPtP6BY84uVLXQ6Evcz9Lcg33Y2z1IVblT6xdY54PXYVHEv+0Wpq8Io6zkA==" }, + "fast-uri": { + "version": "3.0.1", + "resolved": "https://registry.npmjs.org/fast-uri/-/fast-uri-3.0.1.tgz", + "integrity": "sha512-MWipKbbYiYI0UC7cl8m/i/IWTqfC8YXsqjzybjddLsFjStroQzsHXkc73JutMvBiXmOvapk+axIl79ig5t55Bw==" + }, "fast-xml-parser": { "version": "4.4.1", "resolved": "https://registry.npmjs.org/fast-xml-parser/-/fast-xml-parser-4.4.1.tgz", @@ -33791,9 +33775,9 @@ "dev": true }, "json-schema-traverse": { - "version": "0.4.1", - "resolved": "https://registry.npmjs.org/json-schema-traverse/-/json-schema-traverse-0.4.1.tgz", - "integrity": "sha512-xbbCH5dCYU5T8LcEhhuh7HJ88HXuW3qsI3Y0zOZFKfZEHcpWiHU/Jxzk629Brsab/mMiHQti9wMP+845RPe3Vg==" + "version": "1.0.0", + "resolved": "https://registry.npmjs.org/json-schema-traverse/-/json-schema-traverse-1.0.0.tgz", + "integrity": "sha512-NM8/P9n3XjXhIZn1lLhkFaACTOURQXjWhV4BA/RnOv8xvgqtqpAX9IO4mRQxSx1Rlo4tqzeqb0sOlruaOy3dug==" }, "json-stable-stringify-without-jsonify": { "version": "1.0.1", @@ -33896,22 +33880,6 @@ "set-cookie-parser": "^2.4.1" }, "dependencies": { - "ajv": { - "version": "8.12.0", - "resolved": "https://registry.npmjs.org/ajv/-/ajv-8.12.0.tgz", - "integrity": "sha512-sRu1kpcO9yLtYxBKvqfTeh9KzZEwO3STyX1HT+4CaDzC6HpTGYhIhPIzj9XuKU7KYDwnaeh5hcOwjy1QuJzBPA==", - "requires": { - "fast-deep-equal": "^3.1.1", - "json-schema-traverse": "^1.0.0", - "require-from-string": "^2.0.2", - "uri-js": "^4.2.2" - } - }, - "json-schema-traverse": { - "version": "1.0.0", - "resolved": "https://registry.npmjs.org/json-schema-traverse/-/json-schema-traverse-1.0.0.tgz", - "integrity": "sha512-NM8/P9n3XjXhIZn1lLhkFaACTOURQXjWhV4BA/RnOv8xvgqtqpAX9IO4mRQxSx1Rlo4tqzeqb0sOlruaOy3dug==" - }, "process-warning": { "version": "1.0.0", "resolved": "https://registry.npmjs.org/process-warning/-/process-warning-1.0.0.tgz", @@ -34670,26 +34638,6 @@ "requires": { "ajv": "^8.4.0", "openapi-types": "^9.3.1" - }, - "dependencies": { - "ajv": { - "version": "8.11.0", - "resolved": "https://registry.npmjs.org/ajv/-/ajv-8.11.0.tgz", - "integrity": "sha512-wGgprdCvMalC0BztXvitD2hC04YffAvtsUn93JbGXYLAtCUO4xd17mCCZQxUOItiBwZvJScWo8NIvQMQ71rdpg==", - "dev": true, - "requires": { - "fast-deep-equal": "^3.1.1", - "json-schema-traverse": "^1.0.0", - "require-from-string": "^2.0.2", - "uri-js": "^4.2.2" - } - }, - "json-schema-traverse": { - "version": "1.0.0", - "resolved": "https://registry.npmjs.org/json-schema-traverse/-/json-schema-traverse-1.0.0.tgz", - "integrity": "sha512-NM8/P9n3XjXhIZn1lLhkFaACTOURQXjWhV4BA/RnOv8xvgqtqpAX9IO4mRQxSx1Rlo4tqzeqb0sOlruaOy3dug==", - "dev": true - } } }, "openapi-schema-validator": { @@ -34702,26 +34650,6 @@ "ajv-formats": "^2.0.2", "lodash.merge": "^4.6.1", "openapi-types": "^9.3.1" - }, - "dependencies": { - "ajv": { - "version": "8.11.0", - "resolved": "https://registry.npmjs.org/ajv/-/ajv-8.11.0.tgz", - "integrity": "sha512-wGgprdCvMalC0BztXvitD2hC04YffAvtsUn93JbGXYLAtCUO4xd17mCCZQxUOItiBwZvJScWo8NIvQMQ71rdpg==", - "dev": true, - "requires": { - "fast-deep-equal": "^3.1.1", - "json-schema-traverse": "^1.0.0", - "require-from-string": "^2.0.2", - "uri-js": "^4.2.2" - } - }, - "json-schema-traverse": { - "version": "1.0.0", - "resolved": "https://registry.npmjs.org/json-schema-traverse/-/json-schema-traverse-1.0.0.tgz", - "integrity": "sha512-NM8/P9n3XjXhIZn1lLhkFaACTOURQXjWhV4BA/RnOv8xvgqtqpAX9IO4mRQxSx1Rlo4tqzeqb0sOlruaOy3dug==", - "dev": true - } } }, "openapi-types": { diff --git a/package.json b/package.json index 538c30c1..0fd4a334 100644 --- a/package.json +++ b/package.json @@ -61,6 +61,7 @@ "@opentelemetry/instrumentation-express": "0.32.1", "@opentelemetry/instrumentation-http": "0.35.1", "@smithy/node-http-handler": "^3.1.4", + "ajv": "^8.17.1", "axios": "^0.21.1", "compression": "^1.7.4", "config": "^3.3.9", diff --git a/src/common/utils.ts b/src/common/utils.ts index a8ba210d..fc0e3c38 100644 --- a/src/common/utils.ts +++ b/src/common/utils.ts @@ -1,10 +1,12 @@ +import * as Ajv from 'ajv'; import utm from 'utm-latlng'; import { ListBucketsCommand, S3Client } from '@aws-sdk/client-s3'; import { DependencyContainer, FactoryFunction } from 'tsyringe'; import { Logger } from '@map-colonies/js-logger'; -import { WGS84Coordinate } from './interfaces'; +import { GeoContext, WGS84Coordinate } from './interfaces'; import { SERVICES } from './constants'; import { ElasticClients } from './elastic'; +import { BadRequestError } from './errors'; type SnakeToCamelCase = S extends `${infer T}_${infer U}` ? `${T}${Capitalize>}` : S; @@ -14,6 +16,8 @@ type CamelToSnakeCase = S extends `${infer T}${infer U}` : `${Lowercase}_${CamelToSnakeCase>}` : S; +const ajv = new Ajv.Ajv(); + export type ConvertSnakeToCamelCase = { [K in keyof T as SnakeToCamelCase]: T[K]; }; @@ -101,3 +105,85 @@ export const healthCheckFactory: FactoryFunction = (container: DependencyC logger.error(`Healthcheck failed for S3. Error: ${(error as Error).message}`); }); }; + +export const validateGeoContext = (geoContext: GeoContext): boolean => { + const geoCOntextSchema: Ajv.Schema = { + oneOf: [ + { + type: 'object', + properties: { + bbox: { + oneOf: [ + { + type: 'array', + minItems: 4, + maxItems: 4, + items: { + type: 'number', + }, + }, + { + type: 'array', + minItems: 6, + maxItems: 6, + items: { + type: 'number', + }, + }, + ], + }, + }, + required: ['bbox'], + additionalProperties: false, + }, + { + type: 'object', + properties: { + lat: { + type: 'number', + }, + lon: { + type: 'number', + }, + radius: { + type: 'number', + }, + }, + required: ['lat', 'lon', 'radius'], + additionalProperties: false, + }, + { + type: 'object', + properties: { + x: { + type: 'number', + }, + y: { + type: 'number', + }, + zone: { + type: 'number', + }, + radius: { + type: 'number', + }, + }, + required: ['x', 'y', 'zone', 'radius'], + additionalProperties: false, + }, + ], + }; + + const validate = ajv.compile(geoCOntextSchema); + const isValid = validate(geoContext); + const messagePrefix = 'geo_context validation: '; + + if (!isValid) { + throw new BadRequestError( + messagePrefix + + 'geo_context must contain one of the following: {"bbox": [number,number,number,number] | [number,number,number,number,number,number]}, {"lat": number, "lon": number, "radius": number}, or {"x": number, "y": number, "zone": number, "radius": number}' + ); + } + + return true; +}; From bfbeb9163831feeff0580813398a65d28c3f0f67 Mon Sep 17 00:00:00 2001 From: Niv Greenstein Date: Sun, 15 Sep 2024 15:38:37 +0300 Subject: [PATCH 228/262] feat: added parsePoint, parseBbox, parseGeo, geoContextQuery to /common/utils --- src/common/utils.ts | 71 ++++++++++++++++++++++++++++++++++++++++++++- 1 file changed, 70 insertions(+), 1 deletion(-) diff --git a/src/common/utils.ts b/src/common/utils.ts index fc0e3c38..b1640bd2 100644 --- a/src/common/utils.ts +++ b/src/common/utils.ts @@ -1,9 +1,12 @@ import * as Ajv from 'ajv'; import utm from 'utm-latlng'; +import { BBox, Geometry, Point } from 'geojson'; +import { estypes } from '@elastic/elasticsearch'; import { ListBucketsCommand, S3Client } from '@aws-sdk/client-s3'; import { DependencyContainer, FactoryFunction } from 'tsyringe'; import { Logger } from '@map-colonies/js-logger'; -import { GeoContext, WGS84Coordinate } from './interfaces'; +import { ELASTIC_KEYWORDS } from '../control/constants'; +import { GeoContext, GeoContextMode, WGS84Coordinate } from './interfaces'; import { SERVICES } from './constants'; import { ElasticClients } from './elastic'; import { BadRequestError } from './errors'; @@ -18,6 +21,27 @@ type CamelToSnakeCase = S extends `${infer T}${infer U}` const ajv = new Ajv.Ajv(); +const parsePoint = (split: string[] | number[]): Geometry => ({ + type: 'Point', + coordinates: split.map(Number), +}); + +const parseBbox = (split: [string, string, string, string] | BBox): Geometry => { + const [xMin, yMin, xMax, yMax] = split.map(Number); + return { + type: 'Polygon', + coordinates: [ + [ + [xMin, yMin], + [xMin, yMax], + [xMax, yMax], + [xMax, yMin], + [xMin, yMin], + ], + ], + }; +}; + export type ConvertSnakeToCamelCase = { [K in keyof T as SnakeToCamelCase]: T[K]; }; @@ -106,6 +130,22 @@ export const healthCheckFactory: FactoryFunction = (container: DependencyC }); }; +export const parseGeo = (input: GeoContext): Geometry | undefined => { + //TODO: Add geojson validation + //TODO: refactor this function + if (input.bbox !== undefined) { + return parseBbox(input.bbox); + } else if ( + (input.x !== undefined && input.y !== undefined && input.zone !== undefined && input.zone !== undefined) || + (input.lon !== undefined && input.lat !== undefined) + ) { + const { x, y, zone, radius } = input; + const { lon, lat } = x && y && zone ? convertUTMToWgs84(x, y, zone) : (input as Required>); + + return { type: 'Circle', coordinates: (parsePoint([lon, lat]) as Point).coordinates, radius: `${radius ?? ''}` } as unknown as Geometry; + } +}; + export const validateGeoContext = (geoContext: GeoContext): boolean => { const geoCOntextSchema: Ajv.Schema = { oneOf: [ @@ -187,3 +227,32 @@ export const validateGeoContext = (geoContext: GeoContext): boolean => { return true; }; + +export const geoContextQuery = ( + geoContext?: GeoContext, + geoContextMode?: GeoContextMode, + elasticGeoShapeField = ELASTIC_KEYWORDS.geometry +): { [key in 'filter' | 'should']?: estypes.QueryDslQueryContainer[] } => { + if (geoContext === undefined && geoContextMode === undefined) { + return {}; + } + if ((geoContext !== undefined && geoContextMode === undefined) || (geoContext === undefined && geoContextMode !== undefined)) { + throw new BadRequestError('/control/utils/geoContextQuery: geo_context and geo_context_mode must be both defined or both undefined'); + } + + validateGeoContext(geoContext!); + + return { + [geoContextMode === GeoContextMode.FILTER ? 'filter' : 'should']: [ + { + // eslint-disable-next-line @typescript-eslint/naming-convention + geo_shape: { + [elasticGeoShapeField]: { + shape: parseGeo(geoContext!), + }, + boost: geoContextMode === GeoContextMode.BIAS ? 1.1 : 1, //TODO: change magic number + }, + }, + ], + }; +}; From c51a011fe0d4bc028634e1fec22926c1ed99f613 Mon Sep 17 00:00:00 2001 From: Niv Greenstein Date: Sun, 15 Sep 2024 15:39:54 +0300 Subject: [PATCH 229/262] fix: changed function import location --- src/control/item/DAL/queries.ts | 3 +-- src/control/route/DAL/queries.ts | 3 +-- src/control/tile/DAL/queries.ts | 3 +-- src/latLon/models/latLonManager.ts | 3 +-- 4 files changed, 4 insertions(+), 8 deletions(-) diff --git a/src/control/item/DAL/queries.ts b/src/control/item/DAL/queries.ts index f1205691..9e612b70 100644 --- a/src/control/item/DAL/queries.ts +++ b/src/control/item/DAL/queries.ts @@ -1,8 +1,7 @@ import { estypes } from '@elastic/elasticsearch'; import { CommonRequestParameters } from '../../../common/interfaces'; import { ELASTIC_KEYWORDS } from '../../constants'; -import { geoContextQuery } from '../../utils'; -import { ConvertSnakeToCamelCase } from '../../../common/utils'; +import { ConvertSnakeToCamelCase, geoContextQuery } from '../../../common/utils'; export interface ItemQueryParams extends ConvertSnakeToCamelCase { commandName: string; diff --git a/src/control/route/DAL/queries.ts b/src/control/route/DAL/queries.ts index 9c70090f..b278ee23 100644 --- a/src/control/route/DAL/queries.ts +++ b/src/control/route/DAL/queries.ts @@ -1,8 +1,7 @@ import { estypes } from '@elastic/elasticsearch'; import { CommonRequestParameters } from '../../../common/interfaces'; -import { geoContextQuery } from '../../utils'; import { ELASTIC_KEYWORDS } from '../../constants'; -import { ConvertSnakeToCamelCase } from '../../../common/utils'; +import { ConvertSnakeToCamelCase, geoContextQuery } from '../../../common/utils'; export interface RouteQueryParams extends ConvertSnakeToCamelCase { commandName: string; diff --git a/src/control/tile/DAL/queries.ts b/src/control/tile/DAL/queries.ts index 85f2c221..3f5cd19f 100644 --- a/src/control/tile/DAL/queries.ts +++ b/src/control/tile/DAL/queries.ts @@ -1,8 +1,7 @@ import { estypes } from '@elastic/elasticsearch'; import { CommonRequestParameters } from '../../../common/interfaces'; import { ELASTIC_KEYWORDS } from '../../constants'; -import { geoContextQuery } from '../../utils'; -import { ConvertSnakeToCamelCase } from '../../../common/utils'; +import { ConvertSnakeToCamelCase, geoContextQuery } from '../../../common/utils'; export interface TileQueryParams extends ConvertSnakeToCamelCase { tile?: string; diff --git a/src/latLon/models/latLonManager.ts b/src/latLon/models/latLonManager.ts index 4bfeea19..d2ad9383 100644 --- a/src/latLon/models/latLonManager.ts +++ b/src/latLon/models/latLonManager.ts @@ -5,10 +5,9 @@ import { BBox, Feature } from 'geojson'; import * as mgrs from 'mgrs'; import { SERVICES } from '../../common/constants'; import { LatLonDAL } from '../DAL/latLonDAL'; -import { convertUTMToWgs84, convertWgs84ToUTM, validateWGS84Coordinate } from '../../common/utils'; +import { convertUTMToWgs84, convertWgs84ToUTM, parseGeo, validateWGS84Coordinate } from '../../common/utils'; import { BadRequestError } from '../../common/errors'; import { WGS84Coordinate } from '../../common/interfaces'; -import { parseGeo } from '../../location/utils'; import { convertCamelToSnakeCase } from '../../control/utils'; @injectable() From 196ab769eee6edb1dfbdc86c6ba18cf63f5e94f4 Mon Sep 17 00:00:00 2001 From: Niv Greenstein Date: Sun, 15 Sep 2024 15:40:54 +0300 Subject: [PATCH 230/262] feat: changed geotextQuery to use geoContextQuery --- src/location/DAL/queries.ts | 17 +++++------------ 1 file changed, 5 insertions(+), 12 deletions(-) diff --git a/src/location/DAL/queries.ts b/src/location/DAL/queries.ts index 22b51bf2..4d41b827 100644 --- a/src/location/DAL/queries.ts +++ b/src/location/DAL/queries.ts @@ -6,7 +6,7 @@ import { QueryDslQueryContainer } from '@elastic/elasticsearch/lib/api/types'; import { TextSearchParams } from '../interfaces'; import { GeoContextMode, IApplication } from '../../common/interfaces'; import { BadRequestError } from '../../common/errors'; -import { parseGeo } from '../utils'; +import { geoContextQuery } from '../../common/utils'; const TEXT_FIELD = 'text'; const PLACETYPE_FIELD = 'placetype.keyword'; @@ -57,21 +57,14 @@ export const geotextQuery = ( }; if (geoContext && geoContextMode) { - const geo_shape = { - [GEOJSON_FIELD]: { - shape: parseGeo(geoContext), - }, - }; + const geoContextQueryFilter = geoContextQuery(geoContext, GeoContextMode.FILTER, GEOJSON_FIELD).filter![0]; + if (geoContextMode === GeoContextMode.FILTER) { - (esQuery.query?.function_score?.query?.bool?.filter as QueryDslQueryContainer[]).push({ - geo_shape: geo_shape, - }); + (esQuery.query?.function_score?.query?.bool?.filter as QueryDslQueryContainer[]).push(geoContextQueryFilter); } else { esQuery.query?.function_score?.functions?.push({ weight: boosts.viewbox, - filter: { - geo_shape, - }, + filter: geoContextQueryFilter, }); } } From e6064118946857e2c45b488f369d4759d1a9f47e Mon Sep 17 00:00:00 2001 From: Niv Greenstein Date: Sun, 15 Sep 2024 15:47:47 +0300 Subject: [PATCH 231/262] fix: changed import --- src/mgrs/models/mgrsManager.ts | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/mgrs/models/mgrsManager.ts b/src/mgrs/models/mgrsManager.ts index 23490801..b340b5d6 100644 --- a/src/mgrs/models/mgrsManager.ts +++ b/src/mgrs/models/mgrsManager.ts @@ -7,8 +7,8 @@ import * as mgrs from 'mgrs'; import { SERVICES } from '../../common/constants'; import { IApplication } from '../../common/interfaces'; import { GetTileQueryParams } from '../controllers/mgrsController'; -import { parseGeo } from '../../location/utils'; import { BadRequestError } from '../../common/errors'; +import { parseGeo } from '../../common/utils'; @injectable() export class MgrsManager { From 7d21029b049000e2321adcea07104816fc4e0c1f Mon Sep 17 00:00:00 2001 From: Niv Greenstein Date: Sun, 15 Sep 2024 15:51:06 +0300 Subject: [PATCH 232/262] delete: deleted unused code --- src/control/utils.ts | 57 ++------------------------------------ src/location/interfaces.ts | 5 ---- src/location/utils.ts | 41 +-------------------------- 3 files changed, 3 insertions(+), 100 deletions(-) diff --git a/src/control/utils.ts b/src/control/utils.ts index 6e5a189f..011dacdd 100644 --- a/src/control/utils.ts +++ b/src/control/utils.ts @@ -1,17 +1,14 @@ /* eslint-disable @typescript-eslint/no-unused-expressions */ /* eslint-disable @typescript-eslint/naming-convention */ import { estypes } from '@elastic/elasticsearch'; -import { parseGeo } from '../location/utils'; -import { CommonRequestParameters, GeoContext, GeoContextMode, IApplication, IConfig } from '../common/interfaces'; -import { BadRequestError } from '../common/errors'; +import { CommonRequestParameters, IApplication, IConfig } from '../common/interfaces'; import { elasticConfigPath } from '../common/constants'; import { ElasticDbClientsConfig } from '../common/elastic/interfaces'; import { Item } from '../control/item/models/item'; import { Tile } from '../control/tile/models/tile'; import { Route } from '../control/route/models/route'; import { ConvertSnakeToCamelCase } from '../common/utils'; -import { BBOX_LENGTH } from '../location/interfaces'; -import { CONTROL_FIELDS, ELASTIC_KEYWORDS } from './constants'; +import { CONTROL_FIELDS } from './constants'; import { ControlResponse } from './interfaces'; const LAST_ELEMENT_INDEX = -1; @@ -98,56 +95,6 @@ export const formatResponse = ( ], }); -export const geoContextQuery = ( - geoContext?: GeoContext, - geoContextMode?: GeoContextMode -): { [key in 'filter' | 'should']?: estypes.QueryDslQueryContainer[] } => { - if (geoContext === undefined && geoContextMode === undefined) { - return {}; - } - if ((geoContext !== undefined && geoContextMode === undefined) || (geoContext === undefined && geoContextMode !== undefined)) { - throw new BadRequestError('/control/utils/geoContextQuery: geo_context and geo_context_mode must be both defined or both undefined'); - } - - validateGeoContext(geoContext!); - - return { - [geoContextMode === GeoContextMode.FILTER ? 'filter' : 'should']: [ - { - // eslint-disable-next-line @typescript-eslint/naming-convention - geo_shape: { - [ELASTIC_KEYWORDS.geometry]: { - shape: parseGeo(geoContext!), - }, - boost: geoContextMode === GeoContextMode.BIAS ? 1.1 : 1, //TODO: change magic number - }, - }, - ], - }; -}; - -export const validateGeoContext = (geoContext: GeoContext): boolean => { - //TODO: Add validation for possible values - - const messagePrefix = 'geo_context validation: '; - - const validPairs = [['bbox'], ['lat', 'lon', 'radius'], ['x', 'y', 'zone', 'radius']]; - - if (geoContext.bbox !== undefined && geoContext.bbox.length !== BBOX_LENGTH) { - throw new BadRequestError(messagePrefix + 'bbox must contain 4 values'); - } - - if ( - !validPairs.some( - (pair) => pair.every((key) => geoContext[key as keyof GeoContext] !== undefined) && Object.keys(geoContext).length === pair.length - ) - ) { - throw new BadRequestError(messagePrefix + 'geo_context must contain one of the following: {bbox}, {lat, lon, radius}, or {x, y, zone, radius}'); - } - - return true; -}; - // eslint-disable-next-line @typescript-eslint/naming-convention export const additionalControlSearchProperties = (config: IConfig, size: number): Pick => ({ size, diff --git a/src/location/interfaces.ts b/src/location/interfaces.ts index aff76c38..a510a7cd 100644 --- a/src/location/interfaces.ts +++ b/src/location/interfaces.ts @@ -59,8 +59,3 @@ export interface QueryResult { }[]; } /* eslint-enable @typescript-eslint/naming-convention */ - -//Defenitions -export const POINT_LENGTH = 2; -export const BBOX_LENGTH = 4; -export const INDEX_NOT_FOUND = -1; diff --git a/src/location/utils.ts b/src/location/utils.ts index 89e01779..a62e60fe 100644 --- a/src/location/utils.ts +++ b/src/location/utils.ts @@ -1,11 +1,9 @@ import https from 'https'; -import { BBox, Geometry, Point } from 'geojson'; import { SearchResponse } from '@elastic/elasticsearch/lib/api/types'; import { StatusCodes } from 'http-status-codes'; import axios, { AxiosError, AxiosResponse as Response } from 'axios'; import { InternalServerError } from '../common/errors'; -import { GeoContext, IApplication } from '../common/interfaces'; -import { convertUTMToWgs84 } from '../common/utils'; +import { IApplication } from '../common/interfaces'; import { QueryResult, TextSearchParams } from './interfaces'; import { TextSearchHit } from './models/elasticsearchHits'; import { generateDisplayName } from './parsing'; @@ -18,27 +16,6 @@ const axiosInstance = axios.create({ httpsAgent: new https.Agent({ rejectUnauthorized: false }), }); -const parsePoint = (split: string[] | number[]): Geometry => ({ - type: 'Point', - coordinates: split.map(Number), -}); - -const parseBbox = (split: [string, string, string, string] | BBox): Geometry => { - const [xMin, yMin, xMax, yMax] = split.map(Number); - return { - type: 'Polygon', - coordinates: [ - [ - [xMin, yMin], - [xMin, yMax], - [xMax, yMax], - [xMax, yMin], - [xMin, yMin], - ], - ], - }; -}; - export const fetchNLPService = async (endpoint: string, requestData: object): Promise => { let res: Response | null = null, data: T[] | undefined | null = null; @@ -58,22 +35,6 @@ export const fetchNLPService = async (endpoint: string, requestData: object): export const cleanQuery = (query: string): string[] => query.replace(FIND_QUOTES, '').split(FIND_SPECIAL); -export const parseGeo = (input: GeoContext): Geometry | undefined => { - //TODO: Add geojson validation - //TODO: refactor this function - if (input.bbox !== undefined) { - return parseBbox(input.bbox); - } else if ( - (input.x !== undefined && input.y !== undefined && input.zone !== undefined && input.zone !== undefined) || - (input.lon !== undefined && input.lat !== undefined) - ) { - const { x, y, zone, radius } = input; - const { lon, lat } = x && y && zone ? convertUTMToWgs84(x, y, zone) : (input as Required>); - - return { type: 'Circle', coordinates: (parsePoint([lon, lat]) as Point).coordinates, radius: `${radius ?? ''}` } as unknown as Geometry; - } -}; - /* eslint-disable @typescript-eslint/naming-convention */ export const convertResult = ( params: TextSearchParams, From 2da61815485cabac89d909ffea9026a37874dff4 Mon Sep 17 00:00:00 2001 From: Niv Greenstein Date: Sun, 15 Sep 2024 15:51:24 +0300 Subject: [PATCH 233/262] fix: fixed tests according to new changes --- tests/integration/control/item/item.spec.ts | 3 +- tests/integration/control/route/route.spec.ts | 3 +- tests/integration/control/tile/tile.spec.ts | 8 ++- tests/integration/location/location.spec.ts | 61 ++++++++++++++++++- 4 files changed, 69 insertions(+), 6 deletions(-) diff --git a/tests/integration/control/item/item.spec.ts b/tests/integration/control/item/item.spec.ts index 350353ea..48ebb11a 100644 --- a/tests/integration/control/item/item.spec.ts +++ b/tests/integration/control/item/item.spec.ts @@ -436,7 +436,8 @@ describe('/search/control/items', function () { expect(response.status).toBe(httpStatusCodes.BAD_REQUEST); expect(response.body).toMatchObject({ - message: 'geo_context validation: geo_context must contain one of the following: {bbox}, {lat, lon, radius}, or {x, y, zone, radius}', + message: + 'geo_context validation: geo_context must contain one of the following: {"bbox": [number,number,number,number] | [number,number,number,number,number,number]}, {"lat": number, "lon": number, "radius": number}, or {"x": number, "y": number, "zone": number, "radius": number}', }); } } diff --git a/tests/integration/control/route/route.spec.ts b/tests/integration/control/route/route.spec.ts index a7097bc3..ed69fd61 100644 --- a/tests/integration/control/route/route.spec.ts +++ b/tests/integration/control/route/route.spec.ts @@ -553,7 +553,8 @@ describe('/search/control/route', function () { expect(response.status).toBe(httpStatusCodes.BAD_REQUEST); expect(response.body).toMatchObject({ - message: 'geo_context validation: geo_context must contain one of the following: {bbox}, {lat, lon, radius}, or {x, y, zone, radius}', + message: + 'geo_context validation: geo_context must contain one of the following: {"bbox": [number,number,number,number] | [number,number,number,number,number,number]}, {"lat": number, "lon": number, "radius": number}, or {"x": number, "y": number, "zone": number, "radius": number}', }); } } diff --git a/tests/integration/control/tile/tile.spec.ts b/tests/integration/control/tile/tile.spec.ts index a7cd3e62..be1d426d 100644 --- a/tests/integration/control/tile/tile.spec.ts +++ b/tests/integration/control/tile/tile.spec.ts @@ -336,7 +336,7 @@ describe('/search/control/tiles', function () { }); test.each([[[1]], [[1, 1]], [[1, 1, 1]], [[1, 1, 1, 1, 1]]])( - 'should return 400 status code and error message when bbox not containing 4 values', + 'should return 400 status code and error message when bbox not containing 4 or 6 values', async function (bbox) { const response = await requestSender.getTiles({ tile: 'RIT', @@ -348,7 +348,8 @@ describe('/search/control/tiles', function () { expect(response.status).toBe(httpStatusCodes.BAD_REQUEST); expect(response.body).toMatchObject({ - message: 'geo_context validation: bbox must contain 4 values', + message: + 'geo_context validation: geo_context must contain one of the following: {"bbox": [number,number,number,number] | [number,number,number,number,number,number]}, {"lat": number, "lon": number, "radius": number}, or {"x": number, "y": number, "zone": number, "radius": number}', }); } ); @@ -482,7 +483,8 @@ describe('/search/control/tiles', function () { expect(response.status).toBe(httpStatusCodes.BAD_REQUEST); expect(response.body).toMatchObject({ - message: 'geo_context validation: geo_context must contain one of the following: {bbox}, {lat, lon, radius}, or {x, y, zone, radius}', + message: + 'geo_context validation: geo_context must contain one of the following: {"bbox": [number,number,number,number] | [number,number,number,number,number,number]}, {"lat": number, "lon": number, "radius": number}, or {"x": number, "y": number, "zone": number, "radius": number}', }); } } diff --git a/tests/integration/location/location.spec.ts b/tests/integration/location/location.spec.ts index 2beb3926..605b4b3d 100644 --- a/tests/integration/location/location.spec.ts +++ b/tests/integration/location/location.spec.ts @@ -12,7 +12,7 @@ import { SERVICES } from '../../../src/common/constants'; import { S3_REPOSITORY_SYMBOL } from '../../../src/common/s3/s3Repository'; import { cronLoadTileLatLonDataSymbol } from '../../../src/latLon/DAL/latLonDAL'; import { GetGeotextSearchParams, QueryResult } from '../../../src/location/interfaces'; -import { GeoContextMode, IApplication } from '../../../src/common/interfaces'; +import { GeoContext, GeoContextMode, IApplication } from '../../../src/common/interfaces'; import { LocationRequestSender } from './helpers/requestSender'; import { OSM_LA_PORT, @@ -520,5 +520,64 @@ describe('/search/location', function () { nockScope.done(); }); + + test.each<(keyof GeoContext)[][]>([[['lat', 'lon', 'radius']], [['x', 'y', 'zone', 'radius']]])( + 'should return 400 for all invalid geo_context object for the keys %s', + async function (keys) { + function generateCombinations(keys: (keyof GeoContext)[]): GeoContext[] { + const combinations: object[] = []; + + function backtrack(current: object, remainingKeys: string[]): void { + if (remainingKeys.length === 0) { + combinations.push(current); + return; + } + + const key = remainingKeys[0]; + const remaining = remainingKeys.slice(1); + + backtrack({ ...current, [key]: 1 }, remaining); + backtrack(current, remaining); + } + + backtrack({}, keys); + + return combinations; + } + + const geoContexts = generateCombinations(keys); + + for (const geo_context of geoContexts) { + if (Object.keys(geo_context).length === keys.length) { + continue; + } + const query = 'airport'; + const tokenTypesUrlScope = nock(config.get('application').services.tokenTypesUrl) + .post('', { tokens: query.split(' ') }) + .reply(httpStatusCodes.OK, [ + { + tokens: ['port'], + prediction: ['essence'], + }, + ]); + + const response = await requestSender.getQuery({ + query, + limit: 5, + disable_fuzziness: false, + geo_context: JSON.stringify(geo_context) as unknown as GetGeotextSearchParams['geo_context'], + geo_context_mode: GeoContextMode.BIAS, + }); + + expect(response.status).toBe(httpStatusCodes.BAD_REQUEST); + expect(response.body).toMatchObject({ + message: + 'geo_context validation: geo_context must contain one of the following: {"bbox": [number,number,number,number] | [number,number,number,number,number,number]}, {"lat": number, "lon": number, "radius": number}, or {"x": number, "y": number, "zone": number, "radius": number}', + }); + + tokenTypesUrlScope.done(); + } + } + ); }); }); From c4d3fc17683386b0ec7978a905f9fffa8f4bf51f Mon Sep 17 00:00:00 2001 From: Niv Greenstein Date: Wed, 18 Sep 2024 14:04:49 +0300 Subject: [PATCH 234/262] feat: added elasticsearch query for tile by bbox --- src/control/tile/DAL/queries.ts | 43 ++++++++++++++++++++++++++++++--- 1 file changed, 40 insertions(+), 3 deletions(-) diff --git a/src/control/tile/DAL/queries.ts b/src/control/tile/DAL/queries.ts index 3f5cd19f..47098a0e 100644 --- a/src/control/tile/DAL/queries.ts +++ b/src/control/tile/DAL/queries.ts @@ -1,7 +1,8 @@ import { estypes } from '@elastic/elasticsearch'; +import { BBox } from 'geojson'; import { CommonRequestParameters } from '../../../common/interfaces'; import { ELASTIC_KEYWORDS } from '../../constants'; -import { ConvertSnakeToCamelCase, geoContextQuery } from '../../../common/utils'; +import { ConvertSnakeToCamelCase, geoContextQuery, parseGeo } from '../../../common/utils'; export interface TileQueryParams extends ConvertSnakeToCamelCase { tile?: string; @@ -14,7 +15,7 @@ export const queryForTiles = ({ geoContext, geoContextMode, disableFuzziness, -}: Omit & Required>): estypes.SearchRequest => ({ +}: Omit & Required>): estypes.SearchRequest => ({ query: { bool: { must: [ @@ -45,7 +46,7 @@ export const queryForSubTiles = ({ geoContextMode, subTile, disableFuzziness, -}: Required): estypes.SearchRequest => ({ +}: Omit, 'mgrs'>): estypes.SearchRequest => ({ query: { bool: { must: [ @@ -74,3 +75,39 @@ export const queryForSubTiles = ({ }, }, }); + +export const queryForTilesByBbox = ({ + bbox, + geoContext, + geoContextMode, +}: { bbox: BBox } & ConvertSnakeToCamelCase): estypes.SearchRequest => { + const { filter: geoCOntextQueryFilter, should } = geoContextQuery(geoContext, geoContextMode); + + const query: estypes.SearchRequest = { + query: { + bool: { + must: [ + { + term: { + ['properties.TYPE.keyword']: 'TILE', + }, + }, + ], + filter: [ + { + // eslint-disable-next-line @typescript-eslint/naming-convention + geo_shape: { + [ELASTIC_KEYWORDS.geometry]: { + shape: parseGeo({ bbox }), + relation: 'intersects', + }, + }, + }, + ...(geoCOntextQueryFilter ?? []), + ], + should: should ?? {}, + }, + }, + }; + return query; +}; From 7e7d2ae9f40d6504847aaafc5b86e390d46faaa1 Mon Sep 17 00:00:00 2001 From: Niv Greenstein Date: Wed, 18 Sep 2024 14:05:56 +0300 Subject: [PATCH 235/262] feat(tileRepository): added getTilesByBbox --- src/control/tile/DAL/tileRepository.ts | 7 +++++++ 1 file changed, 7 insertions(+) diff --git a/src/control/tile/DAL/tileRepository.ts b/src/control/tile/DAL/tileRepository.ts index c7f764ab..a54b7b62 100644 --- a/src/control/tile/DAL/tileRepository.ts +++ b/src/control/tile/DAL/tileRepository.ts @@ -24,6 +24,13 @@ const createTileRepository = (client: ElasticClient, config: IConfig, logger: Lo return response; }, + async getTilesByBbox(searchParams: { bbox: BBox } & Omit): Promise> { + const response = await queryElastic(client, { + ...additionalControlSearchProperties(config, searchParams.limit), + ...queryForTilesByBbox(searchParams), + }); + return response; + }, }; }; From e0a70fc3c8844ae34a50d60a3a64a4c9401bfc19 Mon Sep 17 00:00:00 2001 From: Niv Greenstein Date: Wed, 18 Sep 2024 14:07:22 +0300 Subject: [PATCH 236/262] feat: removed no implementation error and added call to getTilesByBbox. --- src/control/tile/models/tileManager.ts | 13 +++++-------- 1 file changed, 5 insertions(+), 8 deletions(-) diff --git a/src/control/tile/models/tileManager.ts b/src/control/tile/models/tileManager.ts index 1ca48507..5dea8c41 100644 --- a/src/control/tile/models/tileManager.ts +++ b/src/control/tile/models/tileManager.ts @@ -2,12 +2,13 @@ import { IConfig } from 'config'; import { Logger } from '@map-colonies/js-logger'; import { inject, injectable } from 'tsyringe'; import { estypes } from '@elastic/elasticsearch'; +import * as mgrs from 'mgrs'; import { SERVICES } from '../../../common/constants'; import { TILE_REPOSITORY_SYMBOL, TileRepository } from '../DAL/tileRepository'; import { formatResponse } from '../../utils'; import { TileQueryParams } from '../DAL/queries'; import { FeatureCollection, IApplication } from '../../../common/interfaces'; -import { BadRequestError, NotImplementedError } from '../../../common/errors'; +import { BadRequestError } from '../../../common/errors'; import { Tile } from './tile'; @injectable() @@ -29,15 +30,11 @@ export class TileManager { throw new BadRequestError("/control/tiles: only one of 'tile' or 'mgrs' query parameter must be defined"); } - //TODO: Handle MGRS query - if (tileQueryParams.mgrs !== undefined) { - throw new NotImplementedError('MGRS query is not implemented yet'); - } - let elasticResponse: estypes.SearchResponse | undefined = undefined; - if (tileQueryParams.subTile ?? '') { - elasticResponse = await this.tileRepository.getSubTiles(tileQueryParams as Required, limit); + if (tileQueryParams.mgrs !== undefined) { + elasticResponse = await this.tileRepository.getTilesByBbox({ bbox: mgrs.inverse(tileQueryParams.mgrs), ...tileQueryParams }); + } else if (tileQueryParams.subTile ?? '') { } else { elasticResponse = await this.tileRepository.getTiles(tileQueryParams as TileQueryParams & Required>, limit); } From 9669e6919f4537be7cb9fcd5c64bf71dad47999d Mon Sep 17 00:00:00 2001 From: Niv Greenstein Date: Wed, 18 Sep 2024 14:08:04 +0300 Subject: [PATCH 237/262] fix: removed size param becasue limit it is part of TileQueryParams --- src/control/tile/DAL/tileRepository.ts | 17 ++++++++++++----- src/control/tile/models/tileManager.ts | 5 ++--- 2 files changed, 14 insertions(+), 8 deletions(-) diff --git a/src/control/tile/DAL/tileRepository.ts b/src/control/tile/DAL/tileRepository.ts index a54b7b62..dea6cac8 100644 --- a/src/control/tile/DAL/tileRepository.ts +++ b/src/control/tile/DAL/tileRepository.ts @@ -1,26 +1,33 @@ import { Logger } from '@map-colonies/js-logger'; import { estypes } from '@elastic/elasticsearch'; import { FactoryFunction } from 'tsyringe'; +import { BBox } from 'geojson'; import { ElasticClient, ElasticClients } from '../../../common/elastic'; import { Tile } from '../models/tile'; import { queryElastic } from '../../../common/elastic/utils'; import { IConfig } from '../../../common/interfaces'; import { SERVICES } from '../../../common/constants'; import { additionalControlSearchProperties } from '../../utils'; -import { queryForTiles, queryForSubTiles, TileQueryParams } from './queries'; +import { queryForTiles, queryForSubTiles, TileQueryParams, queryForTilesByBbox } from './queries'; // eslint-disable-next-line @typescript-eslint/explicit-function-return-type const createTileRepository = (client: ElasticClient, config: IConfig, logger: Logger) => { return { // eslint-disable-next-line @typescript-eslint/explicit-module-boundary-types - async getTiles(tileQueryParams: TileQueryParams & Required>, size: number): Promise> { - const response = await queryElastic(client, { ...additionalControlSearchProperties(config, size), ...queryForTiles(tileQueryParams) }); + async getTiles(tileQueryParams: TileQueryParams & Required>): Promise> { + const response = await queryElastic(client, { + ...additionalControlSearchProperties(config, tileQueryParams.limit), + ...queryForTiles(tileQueryParams), + }); return response; }, - async getSubTiles(tileQueryParams: Required, size: number): Promise> { - const response = await queryElastic(client, { ...additionalControlSearchProperties(config, size), ...queryForSubTiles(tileQueryParams) }); + async getSubTiles(tileQueryParams: Required): Promise> { + const response = await queryElastic(client, { + ...additionalControlSearchProperties(config, tileQueryParams.limit), + ...queryForSubTiles(tileQueryParams), + }); return response; }, diff --git a/src/control/tile/models/tileManager.ts b/src/control/tile/models/tileManager.ts index 5dea8c41..06819d6a 100644 --- a/src/control/tile/models/tileManager.ts +++ b/src/control/tile/models/tileManager.ts @@ -21,8 +21,6 @@ export class TileManager { ) {} public async getTiles(tileQueryParams: TileQueryParams): Promise> { - const { limit } = tileQueryParams; - if ( (tileQueryParams.tile === undefined && tileQueryParams.mgrs === undefined) || (tileQueryParams.tile !== undefined && tileQueryParams.mgrs !== undefined) @@ -35,8 +33,9 @@ export class TileManager { if (tileQueryParams.mgrs !== undefined) { elasticResponse = await this.tileRepository.getTilesByBbox({ bbox: mgrs.inverse(tileQueryParams.mgrs), ...tileQueryParams }); } else if (tileQueryParams.subTile ?? '') { + elasticResponse = await this.tileRepository.getSubTiles(tileQueryParams as Required); } else { - elasticResponse = await this.tileRepository.getTiles(tileQueryParams as TileQueryParams & Required>, limit); + elasticResponse = await this.tileRepository.getTiles(tileQueryParams as TileQueryParams & Required>); } return formatResponse(elasticResponse, tileQueryParams, this.application.controlObjectDisplayNamePrefixes); From f6a7f18b546d9e9eda09a51cb8036f6f76129dbc Mon Sep 17 00:00:00 2001 From: Niv Greenstein Date: Wed, 18 Sep 2024 14:16:38 +0300 Subject: [PATCH 238/262] fix: added error handling for invalid MGRS tile --- src/control/tile/models/tileManager.ts | 14 +++++++++++++- 1 file changed, 13 insertions(+), 1 deletion(-) diff --git a/src/control/tile/models/tileManager.ts b/src/control/tile/models/tileManager.ts index 06819d6a..0825d120 100644 --- a/src/control/tile/models/tileManager.ts +++ b/src/control/tile/models/tileManager.ts @@ -3,6 +3,7 @@ import { Logger } from '@map-colonies/js-logger'; import { inject, injectable } from 'tsyringe'; import { estypes } from '@elastic/elasticsearch'; import * as mgrs from 'mgrs'; +import { BBox } from 'geojson'; import { SERVICES } from '../../../common/constants'; import { TILE_REPOSITORY_SYMBOL, TileRepository } from '../DAL/tileRepository'; import { formatResponse } from '../../utils'; @@ -31,7 +32,18 @@ export class TileManager { let elasticResponse: estypes.SearchResponse | undefined = undefined; if (tileQueryParams.mgrs !== undefined) { - elasticResponse = await this.tileRepository.getTilesByBbox({ bbox: mgrs.inverse(tileQueryParams.mgrs), ...tileQueryParams }); + let bbox: BBox = [0, 0, 0, 0]; + try { + bbox = mgrs.inverse(tileQueryParams.mgrs); + bbox.forEach((coord) => { + if (isNaN(coord)) { + throw new Error('Invalid MGRS'); + } + }); + } catch (error) { + throw new BadRequestError(`Invalid MGRS: ${tileQueryParams.mgrs}`); + } + elasticResponse = await this.tileRepository.getTilesByBbox({ bbox, ...tileQueryParams }); } else if (tileQueryParams.subTile ?? '') { elasticResponse = await this.tileRepository.getSubTiles(tileQueryParams as Required); } else { From 28bf14551248e5511224434ee29ce32324e8a4db Mon Sep 17 00:00:00 2001 From: Niv Greenstein Date: Wed, 18 Sep 2024 14:18:19 +0300 Subject: [PATCH 239/262] fix: fixed should to be object or undefined. --- src/control/tile/DAL/queries.ts | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/control/tile/DAL/queries.ts b/src/control/tile/DAL/queries.ts index 47098a0e..35dadf7a 100644 --- a/src/control/tile/DAL/queries.ts +++ b/src/control/tile/DAL/queries.ts @@ -105,7 +105,7 @@ export const queryForTilesByBbox = ({ }, ...(geoCOntextQueryFilter ?? []), ], - should: should ?? {}, + should, }, }, }; From 7a13488d0c8feb51badde7ae143226f076e0953d Mon Sep 17 00:00:00 2001 From: Niv Greenstein Date: Wed, 18 Sep 2024 14:37:28 +0300 Subject: [PATCH 240/262] feat: added MGRS to control tile tests --- tests/integration/control/tile/tile.spec.ts | 85 +++++++++++++++++++++ 1 file changed, 85 insertions(+) diff --git a/tests/integration/control/tile/tile.spec.ts b/tests/integration/control/tile/tile.spec.ts index be1d426d..1c12661e 100644 --- a/tests/integration/control/tile/tile.spec.ts +++ b/tests/integration/control/tile/tile.spec.ts @@ -298,6 +298,63 @@ describe('/search/control/tiles', function () { expectedResponse(requestParams, [SUB_TILE_66], expect) ); }); + + it('should return 200 status code and control tiles by MGRS', async function () { + const requestParams: GetTilesQueryParams = { + mgrs: '33TTG958462', + limit: 5, + disable_fuzziness: false, + }; + const response = await requestSender.getTiles(requestParams); + + expect(response.status).toBe(httpStatusCodes.OK); + // expect(response).toSatisfyApiSpec(); + expect(response.body).toMatchObject>>( + expectedResponse(requestParams, [RIT_TILE, RIC_TILE], expect) + ); + }); + + it('should return 200 status code and control tiles by MGRS and geo_context filter (WGS84)', async function () { + const geo_context: { bbox: BBox } = { bbox: [12.593772987581218, 41.89988905812697, 12.593772987581218, 41.89988905812697] }; + const requestParams: GetTilesQueryParams = { + mgrs: '33TTG958462', + geo_context, + geo_context_mode: GeoContextMode.FILTER, + limit: 5, + disable_fuzziness: false, + }; + const response = await requestSender.getTiles({ + ...requestParams, + geo_context: JSON.stringify(geo_context) as unknown as GetTilesQueryParams['geo_context'], + }); + + expect(response.status).toBe(httpStatusCodes.OK); + // expect(response).toSatisfyApiSpec(); + expect(response.body).toMatchObject>>( + expectedResponse(requestParams, [RIC_TILE], expect) + ); + }); + + it('should return 200 status code and control tiles by MGRS and geo_context bias (WGS84)', async function () { + const geo_context: { bbox: BBox } = { bbox: [12.593772987581218, 41.89988905812697, 12.593772987581218, 41.89988905812697] }; + const requestParams: GetTilesQueryParams = { + mgrs: '33TTG958462', + geo_context, + geo_context_mode: GeoContextMode.BIAS, + limit: 5, + disable_fuzziness: false, + }; + const response = await requestSender.getTiles({ + ...requestParams, + geo_context: JSON.stringify(geo_context) as unknown as GetTilesQueryParams['geo_context'], + }); + + expect(response.status).toBe(httpStatusCodes.OK); + // expect(response).toSatisfyApiSpec(); + expect(response.body).toMatchObject>>( + expectedResponse(requestParams, [RIC_TILE, RIT_TILE], expect) + ); + }); }); describe('Bad Path', function () { // All requests with status code of 400 @@ -335,6 +392,34 @@ describe('/search/control/tiles', function () { }); }); + it('should return 400 status code and error message when mgrs value is invalid', async function () { + let response = await requestSender.getTiles({ mgrs: '', limit: 5, disable_fuzziness: false }); + + expect(response.status).toBe(httpStatusCodes.BAD_REQUEST); + expect(response.body).toMatchObject({ + message: "Empty value found for query parameter 'mgrs'", + }); + + response = await requestSender.getTiles({ tile: 'i', limit: 5, disable_fuzziness: false }); + + expect(response.status).toBe(httpStatusCodes.BAD_REQUEST); + expect(response.body).toMatchObject({ + message: 'request/query/tile must NOT have fewer than 2 characters', + }); + }); + + test.each(['{dsa}', '0', 'z', '000', '!321'])( + 'should return 400 status code and error message when mgrs tile is invalid', + async (mgrs) => { + const response = await requestSender.getTiles({ mgrs, limit: 5, disable_fuzziness: false }); + + expect(response.status).toBe(httpStatusCodes.BAD_REQUEST); + expect(response.body).toMatchObject({ + message: `Invalid MGRS: ${mgrs}`, + }); + } + ); + test.each([[[1]], [[1, 1]], [[1, 1, 1]], [[1, 1, 1, 1, 1]]])( 'should return 400 status code and error message when bbox not containing 4 or 6 values', async function (bbox) { From 7595b04920fff2a6903d44da77192f413c0a1e2d Mon Sep 17 00:00:00 2001 From: Niv Greenstein Date: Thu, 19 Sep 2024 11:19:35 +0300 Subject: [PATCH 241/262] Update pull_request.yaml --- .github/workflows/pull_request.yaml | 78 +++++++++++++++++++++++++++-- 1 file changed, 74 insertions(+), 4 deletions(-) diff --git a/.github/workflows/pull_request.yaml b/.github/workflows/pull_request.yaml index 8315f335..6fbc952e 100644 --- a/.github/workflows/pull_request.yaml +++ b/.github/workflows/pull_request.yaml @@ -34,8 +34,8 @@ jobs: github_token: ${{ secrets.GITHUB_TOKEN }} file: ./openapi3.yaml - tests: - name: Run Tests + integration-test: + name: Run Integration Tests runs-on: ubuntu-latest container: node:16 @@ -96,14 +96,84 @@ jobs: sleep 5; done - - name: Run tests - run: npm run test + - name: Run integration tests + run: npm run test:integration - uses: actions/upload-artifact@v2 with: name: Test Reporters path: reports/** + unit-test: + name: Run Unit Tests + runs-on: ubuntu-latest + container: node:16 + + services: + # Label used to access the service container + elasticsearch: + # Docker Hub image + image: elasticsearch:8.13.0 + env: + discovery.type: single-node + xpack.security.enabled: false + xpack.security.enrollment.enabled: false + ES_JAVA_OPTS: "-Xms512m -Xmx512m" + ports: + - 9200:9200 + # Set health checks to wait until elastic has started + options: >- + --health-cmd "curl -f http://localhost:9200/_cluster/health || exit 1" + --health-interval 10s + --health-timeout 5s + --health-retries 5 + + minio: + # Docker Hub image + image: minio/minio:edge-cicd + env: + MINIO_ROOT_USER: minio + MINIO_ROOT_PASSWORD: minio123 + ports: + - 9000:9000 + # Set health checks to wait until elastic has started + options: >- + --health-cmd "curl -f http://localhost:9000/minio/health/live || exit 1" + --health-interval 10s + --health-timeout 5s + --health-retries 5 + + strategy: + matrix: + node: [18.x, 20.x] + + steps: + - name: Check out Git repository + uses: actions/checkout@v2 + + - name: Set up Node.js + uses: actions/setup-node@v1 + with: + node-version: ${{ matrix.node }} + + - name: Install Node.js dependencies + run: npm ci + + - name: Wait for Elasticsearch + run: | + until curl -s http://elasticsearch:9200/_cluster/health | grep '"status":"green"'; do + echo "Waiting for Elasticsearch..."; + sleep 5; + done + + - name: Run unit tests + run: npm run test:unit + + - uses: actions/upload-artifact@v2 + with: + name: Test Reporters + path: reports/** + security: runs-on: ubuntu-latest steps: From eb63dff50c84f4eded449cefe03db81dfd5728a3 Mon Sep 17 00:00:00 2001 From: Niv Greenstein Date: Thu, 19 Sep 2024 13:09:22 +0300 Subject: [PATCH 242/262] Update pull_request.yaml --- .github/workflows/pull_request.yaml | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/.github/workflows/pull_request.yaml b/.github/workflows/pull_request.yaml index 6fbc952e..e248d2d5 100644 --- a/.github/workflows/pull_request.yaml +++ b/.github/workflows/pull_request.yaml @@ -99,7 +99,7 @@ jobs: - name: Run integration tests run: npm run test:integration - - uses: actions/upload-artifact@v2 + - uses: actions/upload-artifact@v4 with: name: Test Reporters path: reports/** @@ -169,7 +169,7 @@ jobs: - name: Run unit tests run: npm run test:unit - - uses: actions/upload-artifact@v2 + - uses: actions/upload-artifact@v4 with: name: Test Reporters path: reports/** From 5ab7784725c6619d992a1b65f5ec951a60a5c3ea Mon Sep 17 00:00:00 2001 From: Niv Greenstein Date: Thu, 19 Sep 2024 13:20:08 +0300 Subject: [PATCH 243/262] Update test.json --- config/test.json | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/config/test.json b/config/test.json index 8b97a838..c192120f 100644 --- a/config/test.json +++ b/config/test.json @@ -30,7 +30,7 @@ } }, "s3": { - "endpoint": "http://localhost:9000", + "endpoint": "http://minio:9000", "credentials": { "accessKeyId": "minio", "secretAccessKey": "minio123" From 308fe5e41decc72bbf1da730039029a640d2f07f Mon Sep 17 00:00:00 2001 From: Niv Greenstein Date: Thu, 19 Sep 2024 13:27:17 +0300 Subject: [PATCH 244/262] Update test.json --- config/test.json | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/config/test.json b/config/test.json index c192120f..8014b339 100644 --- a/config/test.json +++ b/config/test.json @@ -2,7 +2,7 @@ "db": { "elastic": { "control": { - "node": "http://localhost:9200", + "node": "http://elasticsearch:9200", "auth": { "username": "elastic", "password": "changeme" @@ -13,7 +13,7 @@ } }, "geotext": { - "node": "http://localhost:9200", + "node": "http://elasticsearch:9200", "auth": { "username": "elastic", "password": "changeme" From 97415fec119a8e6825a3e0a840aaf2edea494a5f Mon Sep 17 00:00:00 2001 From: Niv Greenstein Date: Thu, 19 Sep 2024 15:22:18 +0300 Subject: [PATCH 245/262] feat(latLon.spec.ts): added test should return error when cronPattern is not defined --- tests/integration/latLon/latLon.spec.ts | 18 ++++++++++++++++++ 1 file changed, 18 insertions(+) diff --git a/tests/integration/latLon/latLon.spec.ts b/tests/integration/latLon/latLon.spec.ts index 75d5a21c..60b09ec6 100644 --- a/tests/integration/latLon/latLon.spec.ts +++ b/tests/integration/latLon/latLon.spec.ts @@ -184,4 +184,22 @@ describe('/lookup', function () { // dataLoadErrorSpy.mockRestore(); // }); }); + + it('should return error when cronPattern is not defined', async () => { + try { + await getApp({ + override: [ + { token: SERVICES.LOGGER, provider: { useValue: jsLogger({ enabled: false }) } }, + { token: SERVICES.TRACER, provider: { useValue: trace.getTracer('testTracer') } }, + { token: SERVICES.APPLICATION, provider: { useValue: {} } }, + ], + useChild: true, + }); + // If no error is thrown, fail the test + throw new Error('Expected error was not thrown'); + } catch (error) { + // eslint-disable-next-line jest/no-conditional-expect + expect((error as Error).message).toBe('cron pattern is not defined'); + } + }); }); From 9916991ea49af11ec7a16e82ac496d06d38e003d Mon Sep 17 00:00:00 2001 From: Niv Greenstein Date: Thu, 19 Sep 2024 15:22:57 +0300 Subject: [PATCH 246/262] chore: changed version to 0.1.0-prealpha.4 --- package.json | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/package.json b/package.json index 0fd4a334..890dbf5f 100644 --- a/package.json +++ b/package.json @@ -1,6 +1,6 @@ { "name": "geocoding", - "version": "0.1.0-prealpha.3", + "version": "0.1.0-prealpha.4", "description": "Geocoding service for MapColonies", "main": "./src/index.ts", "scripts": { From f815de74cc1fdcbb5e3129d2210d0888a323f7c2 Mon Sep 17 00:00:00 2001 From: Niv Greenstein Date: Sun, 22 Sep 2024 15:11:46 +0300 Subject: [PATCH 247/262] fix: changed s3 files download location in order to mount emptyDir in the pod --- src/common/s3/s3Repository.ts | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/common/s3/s3Repository.ts b/src/common/s3/s3Repository.ts index c247379b..81920772 100644 --- a/src/common/s3/s3Repository.ts +++ b/src/common/s3/s3Repository.ts @@ -26,7 +26,7 @@ const createS3Repository = (s3Client: S3Client, config: IConfig, logger: Logger) const { Body } = await s3Client.send(command); - const filePath = __dirname + '/table.json'; + const filePath = __dirname + '/downloads/table.json'; await new Promise((resolve, reject) => { (Body as NodeJS.ReadableStream) From 117ce76cadda6867d39ab917a9a9c0db87ed53ab Mon Sep 17 00:00:00 2001 From: Niv Greenstein Date: Sun, 22 Sep 2024 15:12:45 +0300 Subject: [PATCH 248/262] fix: changed to node:crypto to specifically use the new nodejs crypto built in module --- devScripts/importDataToElastic.ts | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/devScripts/importDataToElastic.ts b/devScripts/importDataToElastic.ts index 3679b505..887936c6 100644 --- a/devScripts/importDataToElastic.ts +++ b/devScripts/importDataToElastic.ts @@ -1,7 +1,7 @@ /* eslint-disable @typescript-eslint/no-unsafe-assignment */ /* eslint-disable @typescript-eslint/naming-convention */ /* eslint-disable @typescript-eslint/no-unsafe-member-access */ -import crypto from 'crypto'; +import crypto from 'node:crypto'; import { Client } from '@elastic/elasticsearch'; import { ElasticDbClientsConfig } from '../src/common/elastic/interfaces'; import { elasticConfigPath } from '../src/common/constants'; From 4b2a521ce5420ff6c08d02c487d0fa10454756d4 Mon Sep 17 00:00:00 2001 From: Niv Greenstein Date: Mon, 23 Sep 2024 13:37:26 +0300 Subject: [PATCH 249/262] fix: added delete all indices before new init --- devScripts/importDataToElastic.ts | 25 ++++++++++++++++++++++++- 1 file changed, 24 insertions(+), 1 deletion(-) diff --git a/devScripts/importDataToElastic.ts b/devScripts/importDataToElastic.ts index 887936c6..84a5d85b 100644 --- a/devScripts/importDataToElastic.ts +++ b/devScripts/importDataToElastic.ts @@ -2,7 +2,8 @@ /* eslint-disable @typescript-eslint/naming-convention */ /* eslint-disable @typescript-eslint/no-unsafe-member-access */ import crypto from 'node:crypto'; -import { Client } from '@elastic/elasticsearch'; +import httpStatusCodes from 'http-status-codes'; +import { Client, estypes } from '@elastic/elasticsearch'; import { ElasticDbClientsConfig } from '../src/common/elastic/interfaces'; import { elasticConfigPath } from '../src/common/constants'; import { IConfig } from '../src/common/interfaces'; @@ -14,6 +15,28 @@ const main = async (config: IConfig): Promise => { const controlClient = new Client({ ...elasticConfig.control }); const geotextClient = new Client({ ...elasticConfig.geotext }); + for (const { client, indices } of [ + { + client: controlClient, + indices: [elasticConfig.control.properties.index] as string[], + }, + { + client: geotextClient, + indices: Object.values(elasticConfig.geotext.properties.index as { [key: string]: string }), + }, + ]) { + for (const index of indices) { + try { + await client.indices.delete({ index }); + } catch (error) { + if ((error as estypes.ErrorCause).meta.statusCode !== httpStatusCodes.NOT_FOUND) { + throw error; + } + console.error(error); + } + } + } + for (const { client, index, key } of [ { client: controlClient, From 82cd127e1faffef68fba370f0725e82ad7e298f9 Mon Sep 17 00:00:00 2001 From: Niv Greenstein Date: Mon, 23 Sep 2024 13:37:54 +0300 Subject: [PATCH 250/262] fix: added mkdir to s3 downloads file --- src/common/s3/s3Repository.ts | 9 ++++++++- 1 file changed, 8 insertions(+), 1 deletion(-) diff --git a/src/common/s3/s3Repository.ts b/src/common/s3/s3Repository.ts index 81920772..71e24ede 100644 --- a/src/common/s3/s3Repository.ts +++ b/src/common/s3/s3Repository.ts @@ -1,5 +1,6 @@ /* eslint-disable @typescript-eslint/naming-convention */ import fs from 'fs'; +import path from 'path'; import { Logger } from '@map-colonies/js-logger'; import { FactoryFunction } from 'tsyringe'; import { GetObjectCommand, S3Client } from '@aws-sdk/client-s3'; @@ -26,7 +27,13 @@ const createS3Repository = (s3Client: S3Client, config: IConfig, logger: Logger) const { Body } = await s3Client.send(command); - const filePath = __dirname + '/downloads/table.json'; + const filePath = path.join(__dirname, 'downloads', Key); + + try { + await fs.promises.mkdir(path.dirname(filePath), { recursive: true }); + } catch { + //folder already exists + } await new Promise((resolve, reject) => { (Body as NodeJS.ReadableStream) From 2577db438360655d435853c3326af2e56f1e2cea Mon Sep 17 00:00:00 2001 From: Niv Greenstein Date: Mon, 23 Sep 2024 13:39:03 +0300 Subject: [PATCH 251/262] fix: change 'name' to 'names' according to openapi --- package-lock.json | 4 ++-- src/control/interfaces.ts | 2 +- src/control/utils.ts | 2 +- src/latLon/DAL/latLonDAL.ts | 2 +- src/location/interfaces.ts | 2 +- src/location/utils.ts | 2 +- tests/integration/control/utils.ts | 2 +- tests/integration/location/location.spec.ts | 20 ++++++++++---------- tests/integration/location/mockObjects.ts | 14 +++++++------- 9 files changed, 25 insertions(+), 25 deletions(-) diff --git a/package-lock.json b/package-lock.json index bd0c0643..168166a4 100644 --- a/package-lock.json +++ b/package-lock.json @@ -1,12 +1,12 @@ { "name": "geocoding", - "version": "0.1.0-prealpha.3", + "version": "0.1.0-prealpha.4", "lockfileVersion": 2, "requires": true, "packages": { "": { "name": "geocoding", - "version": "0.1.0-prealpha.3", + "version": "0.1.0-prealpha.4", "hasInstallScript": true, "license": "ISC", "dependencies": { diff --git a/src/control/interfaces.ts b/src/control/interfaces.ts index 6ff17d97..0464e51c 100644 --- a/src/control/interfaces.ts +++ b/src/control/interfaces.ts @@ -23,7 +23,7 @@ export interface ControlResponse extends FeatureColl // eslint-disable-next-line @typescript-eslint/naming-convention source_id: string[]; }[]; - name: { + names: { [key: string]: string | string[] | undefined; display: string; default: string[]; diff --git a/src/control/utils.ts b/src/control/utils.ts index 011dacdd..7079f423 100644 --- a/src/control/utils.ts +++ b/src/control/utils.ts @@ -83,7 +83,7 @@ export const formatResponse = ( source_id: [], }, ], - name: { + names: { default: [generatedDisplayName.at(LAST_ELEMENT_INDEX)], display: generatedDisplayName.join(' '), }, diff --git a/src/latLon/DAL/latLonDAL.ts b/src/latLon/DAL/latLonDAL.ts index 5326a18f..3a7bda3a 100644 --- a/src/latLon/DAL/latLonDAL.ts +++ b/src/latLon/DAL/latLonDAL.ts @@ -69,7 +69,7 @@ export class LatLonDAL { this.logger.debug('latLonData initialized'); } catch (error) { - this.logger.error('Failed to initialize latLon data', error); + this.logger.error(`Failed to initialize latLon data. Error: ${(error as Error).message}`); this.dataLoadError = true; } finally { this.onGoingUpdate = false; diff --git a/src/location/interfaces.ts b/src/location/interfaces.ts index a510a7cd..13a56e3c 100644 --- a/src/location/interfaces.ts +++ b/src/location/interfaces.ts @@ -47,7 +47,7 @@ export interface QueryResult { source_id?: string[]; layer?: string; }[]; - name: { + names: { [key: string]: string | string[] | undefined; display: string; default: string[]; diff --git a/src/location/utils.ts b/src/location/utils.ts index a62e60fe..486de35d 100644 --- a/src/location/utils.ts +++ b/src/location/utils.ts @@ -87,7 +87,7 @@ export const convertResult = ( source_id: feature?.source_id.map((id) => id.replace(/(^\{)|(\}$)/g, '')), // TODO: check if to remove this }, ], - name: { + names: { [nameKeys[0]]: new RegExp(mainLanguageRegex).test(feature!.text[0]) ? allNames.shift() : allNames.pop(), [nameKeys[1]]: allNames.pop(), ['default']: [feature!.name], diff --git a/tests/integration/control/utils.ts b/tests/integration/control/utils.ts index e68d4a1e..eb0b7574 100644 --- a/tests/integration/control/utils.ts +++ b/tests/integration/control/utils.ts @@ -20,7 +20,7 @@ const expectedObjectWithScore = (source: T, expec source_id: [], }, ], - name: { + names: { default: [expect.any(String) as string], display: expect.any(String) as string, }, diff --git a/tests/integration/location/location.spec.ts b/tests/integration/location/location.spec.ts index 605b4b3d..03776cf8 100644 --- a/tests/integration/location/location.spec.ts +++ b/tests/integration/location/location.spec.ts @@ -89,8 +89,8 @@ describe('/search/location', function () { ...NY_JFK_AIRPORT, properties: { ...NY_JFK_AIRPORT.properties, - name: { - ...NY_JFK_AIRPORT.properties.name, + names: { + ...NY_JFK_AIRPORT.properties.names, display: expect.stringContaining('JFK') as string, }, }, @@ -140,8 +140,8 @@ describe('/search/location', function () { ...NY_JFK_AIRPORT, properties: { ...NY_JFK_AIRPORT.properties, - name: { - ...NY_JFK_AIRPORT.properties.name, + names: { + ...NY_JFK_AIRPORT.properties.names, display: expect.stringContaining('JFK') as string, }, }, @@ -192,8 +192,8 @@ describe('/search/location', function () { ...NY_JFK_AIRPORT, properties: { ...NY_JFK_AIRPORT.properties, - name: { - ...NY_JFK_AIRPORT.properties.name, + names: { + ...NY_JFK_AIRPORT.properties.names, display: expect.stringContaining('JFK') as string, }, }, @@ -222,8 +222,8 @@ describe('/search/location', function () { ...NY_JFK_AIRPORT, properties: { ...NY_JFK_AIRPORT.properties, - name: { - ...NY_JFK_AIRPORT.properties.name, + names: { + ...NY_JFK_AIRPORT.properties.names, display: expect.stringContaining('JFK') as string, }, }, @@ -241,8 +241,8 @@ describe('/search/location', function () { ...NY_JFK_AIRPORT, properties: { ...NY_JFK_AIRPORT.properties, - name: { - ...NY_JFK_AIRPORT.properties.name, + names: { + ...NY_JFK_AIRPORT.properties.names, display: expect.stringContaining('JFK') as string, }, }, diff --git a/tests/integration/location/mockObjects.ts b/tests/integration/location/mockObjects.ts index 9195e135..a610d28e 100644 --- a/tests/integration/location/mockObjects.ts +++ b/tests/integration/location/mockObjects.ts @@ -54,7 +54,7 @@ export const NY_JFK_AIRPORT: MockLocationQueryFeature = { }, properties: { matches: [{ source: 'OSM', layer: 'osm_airports', source_id: ['03ed6d97-fc81-4340-b68a-11993554eef1'] }], - name: { + names: { en: ['JFK Airport'], fr: ['Aeropuerto JFK'], default: ['JFK'], @@ -99,7 +99,7 @@ export const NY_POLICE_AIRPORT: MockLocationQueryFeature = { }, properties: { matches: [{ source: 'OSM', layer: 'osm_airports', source_id: ['009c6b65-3dcb-4c4f-9f02-d766ebb5d808'] }], - name: { + names: { en: ['Nassau County Police Airport'], fr: ['Aeropuerto de la Policía del Condado de Nassau'], default: ['Nassau County Police Airport'], @@ -146,7 +146,7 @@ export const LA_AIRPORT: MockLocationQueryFeature = { }, properties: { matches: [{ source: 'OSM', layer: 'osm_airports', source_id: ['a4f373ab-b824-41e2-b160-e7729c73bea6'] }], - name: { + names: { en: ['Los Angeles International Airport'], fr: ['Aeropuerto Internacional de Los Ángeles'], default: ['Los Angeles International Airport'], @@ -194,7 +194,7 @@ export const OSM_LA_PORT: MockLocationQueryFeature = { }, properties: { matches: [{ source: 'OSM', layer: 'osm_ports', source_id: ['0f36d985-cfbd-4aed-b0cb-ee56600c77f4'] }], - name: { + names: { en: ['Port of Los Angeles'], fr: ['Puerto de Los Ángeles'], default: ['Port of Los Angeles'], @@ -241,7 +241,7 @@ export const GOOGLE_LA_PORT: MockLocationQueryFeature = { }, properties: { matches: [{ source: 'GOOGLE', layer: 'google_ports', source_id: ['1bb11f54-939e-457b-bf68-a3920ccf629c'] }], - name: { + names: { en: ['Port of Los Angeles'], fr: ['Puerto de Los Ángeles'], default: ['Port of Los Angeles'], @@ -282,7 +282,7 @@ export const LA_WHITE_POINT_SCHOOL: MockLocationQueryFeature = { source_id: ['1a5b981b-bb0e-44dd-b9e2-424b92f2de49'], }, ], - name: { + names: { en: ['White Point Elementary School'], fr: ['Escuela Primaria White Point'], default: ['White Point Elementary School'], @@ -321,7 +321,7 @@ export const PARIS_WI_SCHOOL: MockLocationQueryFeature = { source_id: ['dc02a3f9-156a-4f61-85bd-fd040cd322a3'], }, ], - name: { + names: { en: ['Wi School Paris 9'], fr: ['Ecole Wi Paris 9'], default: ['Wi School Paris 9'], From e2ca1b131043658b9eab0ee2a5e38ade0cbcce22 Mon Sep 17 00:00:00 2001 From: NatalieShaked Date: Mon, 23 Sep 2024 14:13:18 +0300 Subject: [PATCH 252/262] fix: feedback sends to redis without adding time to the response --- src/common/constants.ts | 1 - .../middlewares/feedbackApi.middleware.ts | 24 ++++++++++--------- src/common/redis/interfaces.ts | 10 ++++---- 3 files changed, 18 insertions(+), 17 deletions(-) diff --git a/src/common/constants.ts b/src/common/constants.ts index ee1c5880..6573ef73 100644 --- a/src/common/constants.ts +++ b/src/common/constants.ts @@ -28,4 +28,3 @@ export const REDIS_SYMBOL = Symbol('REDIS'); export const REDIS_TTL = 300; export const elasticConfigPath = 'db.elastic'; -export const siteIndex = 1; diff --git a/src/common/middlewares/feedbackApi.middleware.ts b/src/common/middlewares/feedbackApi.middleware.ts index 193d0443..c8c6f87f 100644 --- a/src/common/middlewares/feedbackApi.middleware.ts +++ b/src/common/middlewares/feedbackApi.middleware.ts @@ -2,17 +2,19 @@ import * as crypto from 'node:crypto'; import { Request, Response, NextFunction } from 'express'; import { Logger } from '@map-colonies/js-logger'; import { inject, injectable } from 'tsyringe'; -import { REDIS_TTL, SERVICES, siteIndex } from '../constants'; +import { REDIS_TTL, SERVICES } from '../constants'; import { RedisClient } from '../redis'; import { GeocodingResponse } from '../interfaces'; +const siteIndex = 1; + @injectable() export class FeedbackApiMiddlewareManager { public constructor(@inject(SERVICES.LOGGER) private readonly logger: Logger, @inject(SERVICES.REDIS) private readonly redis: RedisClient) {} // eslint-disable-next-line @typescript-eslint/explicit-function-return-type public saveResponses = (req: Request, res: Response, next: NextFunction) => { - const reqId = res.getHeader('request-id'); + const reqId = res.getHeader('x-request-id'); const redisClient = this.redis; const logger = this.logger; @@ -29,25 +31,25 @@ export class FeedbackApiMiddlewareManager { }; const originalJson = res.json; - const logJson = async function (this: Response, body: JSON): Promise { + const logJson = function (this: Response, body: JSON): Response { geocodingResponseDetails.response = body; + redisClient + .setEx(reqId as string, REDIS_TTL, JSON.stringify(geocodingResponseDetails)) + .then(() => { + logger.info({ msg: `response ${reqId?.toString() ?? ''} saved to redis` }); + }) + .catch((error) => logger.error('Error setting key:', error)); - try { - await redisClient.setEx(reqId as string, REDIS_TTL, JSON.stringify(geocodingResponseDetails)); - logger.info({ msg: 'saving response to redis' }); - } catch (err) { - logger.error('Error setting key:', err); - } return originalJson.call(this, body); }; - res.json = logJson as unknown as Response['json']; + res.json = logJson; next(); }; // eslint-disable-next-line @typescript-eslint/explicit-function-return-type public setRequestId = (req: Request, res: Response, next: NextFunction) => { const reqId = crypto.randomUUID(); - res.append('request-id', reqId); + res.append('x-request-id', reqId); next(); }; } diff --git a/src/common/redis/interfaces.ts b/src/common/redis/interfaces.ts index d569f1eb..145a202b 100644 --- a/src/common/redis/interfaces.ts +++ b/src/common/redis/interfaces.ts @@ -1,8 +1,8 @@ import { RedisClientOptions } from 'redis'; export type RedisConfig = { - host: string; - port: number; - enableSslAuth: boolean; - sslPaths: { ca: string; cert: string; key: string }; - } & RedisClientOptions; \ No newline at end of file + host: string; + port: number; + enableSslAuth: boolean; + sslPaths: { ca: string; cert: string; key: string }; +} & RedisClientOptions; From 2efa01516f4340d2b84dcfcb448c65f80db75bcd Mon Sep 17 00:00:00 2001 From: Niv Greenstein Date: Mon, 23 Sep 2024 15:20:44 +0300 Subject: [PATCH 253/262] fix: added missing target_grid and version to the response --- src/latLon/controllers/latLonController.ts | 8 ++++---- src/latLon/models/latLonManager.ts | 18 ++++++++++++++++-- tests/integration/latLon/latLon.spec.ts | 4 ++++ 3 files changed, 24 insertions(+), 6 deletions(-) diff --git a/src/latLon/controllers/latLonController.ts b/src/latLon/controllers/latLonController.ts index cff3e100..c16b601f 100644 --- a/src/latLon/controllers/latLonController.ts +++ b/src/latLon/controllers/latLonController.ts @@ -28,7 +28,7 @@ export class LatLonController { public getCoordinates: GetCoordinatesHandler = async (req, res, next) => { try { - const { lat, lon, target_grid } = req.query; + const { target_grid: targetGrid } = req.query; let response: | ({ @@ -36,10 +36,10 @@ export class LatLonController { } & Feature) | undefined = undefined; - if (target_grid === 'control') { - response = await this.manager.latLonToTile({ lat, lon }); + if (targetGrid === 'control') { + response = await this.manager.latLonToTile({ ...req.query, targetGrid }); } else { - response = this.manager.latLonToMGRS({ lat, lon }); + response = this.manager.latLonToMGRS({ ...req.query, targetGrid }); } return res.status(httpStatus.OK).json(response); diff --git a/src/latLon/models/latLonManager.ts b/src/latLon/models/latLonManager.ts index d2ad9383..311e3c18 100644 --- a/src/latLon/models/latLonManager.ts +++ b/src/latLon/models/latLonManager.ts @@ -18,7 +18,7 @@ export class LatLonManager { @inject(SERVICES.CONFIG) private readonly config: IConfig ) {} - public async latLonToTile({ lat, lon }: WGS84Coordinate): Promise<{ [key: string]: unknown } & Feature> { + public async latLonToTile({ lat, lon, targetGrid }: WGS84Coordinate & { targetGrid: string }): Promise<{ [key: string]: unknown } & Feature> { if (!validateWGS84Coordinate({ lat, lon })) { this.logger.warn("LatLonManager.latLonToTile: Invalid lat lon, check 'lat' and 'lon' keys exists and their values are legal"); throw new BadRequestError("Invalid lat lon, check 'lat' and 'lon' keys exists and their values are legal"); @@ -63,9 +63,11 @@ export class LatLonManager { return { type: 'Feature', geocoding: { + version: process.env.npm_package_version, query: { lat, lon, + ...convertCamelToSnakeCase({ targetGrid }), }, response: convertCamelToSnakeCase({ maxScore: 1, @@ -90,7 +92,17 @@ export class LatLonManager { }; } - public latLonToMGRS({ lat, lon, accuracy = 5 }: { lat: number; lon: number; accuracy?: number }): { [key: string]: unknown } & Feature { + public latLonToMGRS({ + lat, + lon, + accuracy = 5, + targetGrid, + }: { + lat: number; + lon: number; + accuracy?: number; + targetGrid: string; + }): { [key: string]: unknown } & Feature { const accuracyString: Record = { [0]: '100km', [1]: '10km', @@ -103,9 +115,11 @@ export class LatLonManager { return { type: 'Feature', geocoding: { + version: process.env.npm_package_version, query: { lat, lon, + ...convertCamelToSnakeCase({ targetGrid }), }, response: convertCamelToSnakeCase({ maxScore: 1, diff --git a/tests/integration/latLon/latLon.spec.ts b/tests/integration/latLon/latLon.spec.ts index 60b09ec6..17e53bbe 100644 --- a/tests/integration/latLon/latLon.spec.ts +++ b/tests/integration/latLon/latLon.spec.ts @@ -47,9 +47,11 @@ describe('/lookup', function () { expect(response.body).toEqual({ type: 'Feature', geocoding: { + version: expect.any(String) as string, query: { lat: 52.57326537485767, lon: 12.948781146422107, + target_grid: 'control', }, response: { max_score: 1, @@ -90,9 +92,11 @@ describe('/lookup', function () { expect(response.body).toEqual({ type: 'Feature', geocoding: { + version: expect.any(String) as string, query: { lat: 52.57326537485767, lon: 12.948781146422107, + target_grid: 'MGRS', }, response: { max_score: 1, From e5dd67fe1855b2874c65abed04faea4f8fdec33a Mon Sep 17 00:00:00 2001 From: NatalieShaked Date: Mon, 23 Sep 2024 15:52:32 +0300 Subject: [PATCH 254/262] fix: minor fixes for code clearness --- src/common/constants.ts | 6 ++-- src/common/interfaces.ts | 2 +- .../middlewares/feedbackApi.middleware.ts | 28 +++++++++++-------- src/common/middlewares/utils.ts | 5 ++++ 4 files changed, 25 insertions(+), 16 deletions(-) create mode 100644 src/common/middlewares/utils.ts diff --git a/src/common/constants.ts b/src/common/constants.ts index 6573ef73..c1044083 100644 --- a/src/common/constants.ts +++ b/src/common/constants.ts @@ -1,9 +1,7 @@ -import { hostname } from 'os'; import { readPackageJsonSync } from '@map-colonies/read-pkg'; export const SERVICE_NAME = readPackageJsonSync().name ?? 'unknown_service'; export const DEFAULT_SERVER_PORT = 80; -export const HOSTNAME = hostname(); export const IGNORED_OUTGOING_TRACE_ROUTES = [/^.*\/v1\/metrics.*$/]; export const IGNORED_INCOMING_TRACE_ROUTES = [/^.*\/docs.*$/]; @@ -24,7 +22,7 @@ export const SERVICES: Record = { export const ON_SIGNAL = Symbol('onSignal'); export const HEALTHCHECK = Symbol('healthcheck'); -export const REDIS_SYMBOL = Symbol('REDIS'); -export const REDIS_TTL = 300; export const elasticConfigPath = 'db.elastic'; +export const s3EndpointConfig = 'db.s3.endpoint'; + diff --git a/src/common/interfaces.ts b/src/common/interfaces.ts index a120b1ad..14cd497b 100644 --- a/src/common/interfaces.ts +++ b/src/common/interfaces.ts @@ -61,7 +61,7 @@ export enum GeoContextMode { BIAS = 'bias', } -export interface GeocodingResponse { +export interface FeebackApiGeocodingResponse { userId: string; apiKey: string; site: string; diff --git a/src/common/middlewares/feedbackApi.middleware.ts b/src/common/middlewares/feedbackApi.middleware.ts index c8c6f87f..33b1b1d8 100644 --- a/src/common/middlewares/feedbackApi.middleware.ts +++ b/src/common/middlewares/feedbackApi.middleware.ts @@ -2,30 +2,36 @@ import * as crypto from 'node:crypto'; import { Request, Response, NextFunction } from 'express'; import { Logger } from '@map-colonies/js-logger'; import { inject, injectable } from 'tsyringe'; -import { REDIS_TTL, SERVICES } from '../constants'; +import { SERVICES, s3EndpointConfig } from '../constants'; import { RedisClient } from '../redis'; -import { GeocodingResponse } from '../interfaces'; +import { FeebackApiGeocodingResponse, IConfig } from '../interfaces'; +import { XApi } from './utils'; -const siteIndex = 1; +const SITE_INDEX = 1; +const REDIS_TTL = 300; @injectable() export class FeedbackApiMiddlewareManager { - public constructor(@inject(SERVICES.LOGGER) private readonly logger: Logger, @inject(SERVICES.REDIS) private readonly redis: RedisClient) {} + public constructor( + @inject(SERVICES.LOGGER) private readonly logger: Logger, + @inject(SERVICES.REDIS) private readonly redis: RedisClient, + @inject(SERVICES.CONFIG) private readonly config: IConfig + ) {} // eslint-disable-next-line @typescript-eslint/explicit-function-return-type public saveResponses = (req: Request, res: Response, next: NextFunction) => { - const reqId = res.getHeader('x-request-id'); + const reqId = res.getHeader(XApi.REQUEST); const redisClient = this.redis; const logger = this.logger; - const s3Endpoint = ''; //s3 endpoint from default + const s3Endpoint = this.config.get(s3EndpointConfig); const drSite = s3Endpoint.split('.'); logger.info({ msg: 'saving response to redis' }); - const geocodingResponseDetails: GeocodingResponse = { - userId: req.headers['x-user-id'] as string, - apiKey: req.headers['x-api-key'] as string, - site: drSite[siteIndex], + const geocodingResponseDetails: FeebackApiGeocodingResponse = { + userId: req.headers[XApi.USER] as string, + apiKey: req.headers[XApi.KEY] as string, + site: drSite[SITE_INDEX], response: JSON.parse('{}') as JSON, respondedAt: new Date(), }; @@ -49,7 +55,7 @@ export class FeedbackApiMiddlewareManager { // eslint-disable-next-line @typescript-eslint/explicit-function-return-type public setRequestId = (req: Request, res: Response, next: NextFunction) => { const reqId = crypto.randomUUID(); - res.append('x-request-id', reqId); + res.append(XApi.REQUEST, reqId); next(); }; } diff --git a/src/common/middlewares/utils.ts b/src/common/middlewares/utils.ts new file mode 100644 index 00000000..31b6288c --- /dev/null +++ b/src/common/middlewares/utils.ts @@ -0,0 +1,5 @@ +export enum XApi { + REQUEST = 'x-request-id', + USER = 'x-user-id', + KEY = 'x-api-key', +} From 84e71e9c3f854ce477a716abcbb20d7a55f19bd7 Mon Sep 17 00:00:00 2001 From: NatalieShaked Date: Sun, 29 Sep 2024 17:41:45 +0300 Subject: [PATCH 255/262] fix: app will work also if redis was not able to connect --- src/common/redis/index.ts | 22 ++++++++++++---------- src/containerConfig.ts | 10 +++++++--- 2 files changed, 19 insertions(+), 13 deletions(-) diff --git a/src/common/redis/index.ts b/src/common/redis/index.ts index 514c3ed5..2642959c 100644 --- a/src/common/redis/index.ts +++ b/src/common/redis/index.ts @@ -31,18 +31,20 @@ export const DEFAULT_LIMIT = { from: DEFAULT_LIMIT_FROM, size: DEFAULT_LIMIT_SIZ export type RedisClient = ReturnType; -export const redisClientFactory: FactoryFunction = (container: DependencyContainer): RedisClient => { +export const redisClientFactory: FactoryFunction = (container: DependencyContainer): RedisClient | undefined => { const logger = container.resolve(SERVICES.LOGGER); const config = container.resolve(SERVICES.CONFIG); const dbConfig = config.get('db.redis'); const connectionOptions = createConnectionOptions(dbConfig); - - const redisClient = createClient(connectionOptions) - .on('error', (error: Error) => logger.error({ msg: 'redis client errored', err: error })) - .on('reconnecting', (...args) => logger.warn({ msg: 'redis client reconnecting', ...args })) - .on('end', (...args) => logger.info({ msg: 'redis client end', ...args })) - .on('connect', (...args) => logger.debug({ msg: 'redis client connected', ...args })) - .on('ready', (...args) => logger.debug({ msg: 'redis client is ready', ...args })); - - return redisClient; + try { + const redisClient = createClient(connectionOptions) + // .on('error', (error: Error) => logger.error({ msg: 'redis client errored', err: error })) + // .on('reconnecting', (...args) => logger.warn({ msg: 'redis client reconnecting', ...args })) + .on('end', (...args) => logger.info({ msg: 'redis client end', ...args })) + .on('connect', (...args) => logger.debug({ msg: 'redis client connected', ...args })) + .on('ready', (...args) => logger.debug({ msg: 'redis client is ready', ...args })); + return redisClient; + } catch (error) { + logger.error('Connection to Redis was unsuccessful', error); + } }; diff --git a/src/containerConfig.ts b/src/containerConfig.ts index be3ba2a4..831c2f03 100644 --- a/src/containerConfig.ts +++ b/src/containerConfig.ts @@ -153,9 +153,13 @@ export const registerExternalValues = async (options?: RegisterOptions): Promise token: SERVICES.REDIS, provider: { useFactory: instancePerContainerCachingFactory(redisClientFactory) }, postInjectionHook: async (deps: DependencyContainer): Promise => { - const redis = deps.resolve(SERVICES.REDIS); - cleanupRegistry.register({ func: redis.disconnect.bind(redis), id: SERVICES.REDIS }); - await redis.connect(); + try { + const redis = deps.resolve(SERVICES.REDIS); + cleanupRegistry.register({ func: redis.disconnect.bind(redis), id: SERVICES.REDIS }); + await redis.connect(); + } catch (error) { + logger.error('Connection to redis failed', error); + } }, }, ]; From 9422a044a2ed59871b8b8a5bc5b02fa73b8c57d2 Mon Sep 17 00:00:00 2001 From: NatalieShaked Date: Sun, 29 Sep 2024 18:04:02 +0300 Subject: [PATCH 256/262] fix: added site to env vars --- config/custom-environment-variables.json | 3 ++- helm/templates/configmap.yaml | 1 + helm/values.yaml | 1 + src/common/constants.ts | 3 +-- src/common/middlewares/feedbackApi.middleware.ts | 8 +++----- 5 files changed, 8 insertions(+), 8 deletions(-) diff --git a/config/custom-environment-variables.json b/config/custom-environment-variables.json index 48e05ca7..4a8fdc81 100644 --- a/config/custom-environment-variables.json +++ b/config/custom-environment-variables.json @@ -122,6 +122,7 @@ "__name": "CONTROL_DISPLAY_KEY_TRANSLATIONS", "__format": "json" }, - "mainLanguageRegex": "MAIN_LANGUAGE_REGEX" + "mainLanguageRegex": "MAIN_LANGUAGE_REGEX", + "site": "SITE" } } diff --git a/helm/templates/configmap.yaml b/helm/templates/configmap.yaml index b83572d1..a43002d5 100644 --- a/helm/templates/configmap.yaml +++ b/helm/templates/configmap.yaml @@ -10,6 +10,7 @@ data: RESPONSE_COMPRESSION_ENABLED: {{ .Values.env.responseCompressionEnabled | quote }} LOG_LEVEL: {{ .Values.env.logLevel | quote }} LOG_PRETTY_PRINT_ENABLED: {{ .Values.env.logPrettyPrintEnabled | quote }} + SITE: {{ .Values.env.site | quote }} {{ if .Values.env.tracing.enabled }} TELEMETRY_TRACING_ENABLED: 'true' TELEMETRY_TRACING_URL: {{ $tracingUrl }} diff --git a/helm/values.yaml b/helm/values.yaml index 3adb3d5b..5fd8cc0e 100644 --- a/helm/values.yaml +++ b/helm/values.yaml @@ -62,6 +62,7 @@ env: protocol: TCP logLevel: info logPrettyPrintEnabled: false + site: main responseCompressionEnabled: true requestPayloadLimit: 1mb tracing: diff --git a/src/common/constants.ts b/src/common/constants.ts index c1044083..ab5493a1 100644 --- a/src/common/constants.ts +++ b/src/common/constants.ts @@ -23,6 +23,5 @@ export const SERVICES: Record = { export const ON_SIGNAL = Symbol('onSignal'); export const HEALTHCHECK = Symbol('healthcheck'); +export const siteConfig = 'application.site'; export const elasticConfigPath = 'db.elastic'; -export const s3EndpointConfig = 'db.s3.endpoint'; - diff --git a/src/common/middlewares/feedbackApi.middleware.ts b/src/common/middlewares/feedbackApi.middleware.ts index 33b1b1d8..c44db663 100644 --- a/src/common/middlewares/feedbackApi.middleware.ts +++ b/src/common/middlewares/feedbackApi.middleware.ts @@ -2,12 +2,11 @@ import * as crypto from 'node:crypto'; import { Request, Response, NextFunction } from 'express'; import { Logger } from '@map-colonies/js-logger'; import { inject, injectable } from 'tsyringe'; -import { SERVICES, s3EndpointConfig } from '../constants'; +import { SERVICES, siteConfig } from '../constants'; import { RedisClient } from '../redis'; import { FeebackApiGeocodingResponse, IConfig } from '../interfaces'; import { XApi } from './utils'; -const SITE_INDEX = 1; const REDIS_TTL = 300; @injectable() @@ -24,14 +23,13 @@ export class FeedbackApiMiddlewareManager { const redisClient = this.redis; const logger = this.logger; - const s3Endpoint = this.config.get(s3EndpointConfig); - const drSite = s3Endpoint.split('.'); + const drSite = this.config.get(siteConfig); logger.info({ msg: 'saving response to redis' }); const geocodingResponseDetails: FeebackApiGeocodingResponse = { userId: req.headers[XApi.USER] as string, apiKey: req.headers[XApi.KEY] as string, - site: drSite[SITE_INDEX], + site: drSite, response: JSON.parse('{}') as JSON, respondedAt: new Date(), }; From 4628395db2080a2e129afa459fd03d6bd842c0c9 Mon Sep 17 00:00:00 2001 From: Niv Greenstein Date: Sun, 29 Sep 2024 19:38:43 +0300 Subject: [PATCH 257/262] feat: created GenericGeocodingResponse --- src/common/interfaces.ts | 33 ++++++++++++++++++++++++++++++++- 1 file changed, 32 insertions(+), 1 deletion(-) diff --git a/src/common/interfaces.ts b/src/common/interfaces.ts index 14cd497b..861f2aec 100644 --- a/src/common/interfaces.ts +++ b/src/common/interfaces.ts @@ -1,4 +1,6 @@ -import { BBox, Feature, FeatureCollection as GeoJSONFeatureCollection } from 'geojson'; +import { estypes } from '@elastic/elasticsearch'; +import { BBox, Feature, FeatureCollection as GeoJSONFeatureCollection, GeoJsonProperties } from 'geojson'; +import { RemoveUnderscore } from './utils'; export interface IConfig { get: (setting: string) => T; @@ -77,3 +79,32 @@ export interface CommonRequestParameters { disable_fuzziness: boolean; } /* eslint-enable @typescript-eslint/naming-convention */ + +export interface GenericGeocodingResponse extends FeatureCollection { + geocoding: { + version?: string; + query: G & CommonRequestParameters; + response: Pick & { + /* eslint-disable @typescript-eslint/naming-convention */ + results_count?: estypes.SearchHitsMetadata['total']; + match_latency_ms?: estypes.SearchResponse['took']; + /* eslint-enable @typescript-eslint/naming-convention */ + } & { [key: string]: unknown }; + }; + features: (T & { + properties: RemoveUnderscore, '_score'>> & + GeoJsonProperties & { + matches: { + layer: string; + source: string; + // eslint-disable-next-line @typescript-eslint/naming-convention + source_id: string[]; + }[]; + names: { + [key: string]: string | string[] | undefined; + display: string; + default: string[]; + }; + }; + })[]; +} From 4454246ec1eb3f2861be6070f95c547cf0d7955a Mon Sep 17 00:00:00 2001 From: Niv Greenstein Date: Sun, 29 Sep 2024 19:40:14 +0300 Subject: [PATCH 258/262] fix: changed from ControlResponse to GenericGeocodingResponse --- src/control/utils.ts | 9 ++++----- src/location/controllers/locationController.ts | 6 ++++-- src/location/models/locationManager.ts | 7 ++++--- 3 files changed, 12 insertions(+), 10 deletions(-) diff --git a/src/control/utils.ts b/src/control/utils.ts index a1080f88..cabace51 100644 --- a/src/control/utils.ts +++ b/src/control/utils.ts @@ -1,7 +1,7 @@ /* eslint-disable @typescript-eslint/no-unused-expressions */ /* eslint-disable @typescript-eslint/naming-convention */ import { estypes } from '@elastic/elasticsearch'; -import { CommonRequestParameters, IApplication, IConfig } from '../common/interfaces'; +import { CommonRequestParameters, GenericGeocodingResponse, IApplication, IConfig } from '../common/interfaces'; import { elasticConfigPath } from '../common/constants'; import { ElasticDbClientsConfig } from '../common/elastic/interfaces'; import { Item } from '../control/item/models/item'; @@ -9,7 +9,6 @@ import { Tile } from '../control/tile/models/tile'; import { Route } from '../control/route/models/route'; import { ConvertSnakeToCamelCase } from '../common/utils'; import { CONTROL_FIELDS } from './constants'; -import { ControlResponse } from './interfaces'; const LAST_ELEMENT_INDEX = -1; @@ -57,7 +56,7 @@ export const formatResponse = ( elasticResponse: estypes.SearchResponse, requestParams: CommonRequestParameters | ConvertSnakeToCamelCase, displayNamePrefixes: IApplication['controlObjectDisplayNamePrefixes'] -): ControlResponse => ({ +): GenericGeocodingResponse => ({ type: 'FeatureCollection', geocoding: { version: process.env.npm_package_version, @@ -91,7 +90,7 @@ export const formatResponse = ( }, }; } - }) as ControlResponse['features']), + }) as GenericGeocodingResponse['features']), ], }); @@ -101,4 +100,4 @@ export const additionalControlSearchProperties = (config: IConfig, size: number) index: config.get(elasticConfigPath).control.properties.index as string, // eslint-disable-next-line @typescript-eslint/naming-convention _source: CONTROL_FIELDS, -}); \ No newline at end of file +}); diff --git a/src/location/controllers/locationController.ts b/src/location/controllers/locationController.ts index a8ec476e..491c06b5 100644 --- a/src/location/controllers/locationController.ts +++ b/src/location/controllers/locationController.ts @@ -2,14 +2,16 @@ import { Logger } from '@map-colonies/js-logger'; import { BoundCounter, Meter } from '@opentelemetry/api-metrics'; import { RequestHandler } from 'express'; import httpStatus from 'http-status-codes'; +import { Feature } from 'geojson'; import { injectable, inject } from 'tsyringe'; import { SERVICES } from '../../common/constants'; import { GeotextSearchManager } from '../models/locationManager'; -import { GetGeotextSearchParams, QueryResult } from '../interfaces'; +import { GetGeotextSearchParams } from '../interfaces'; +import { GenericGeocodingResponse } from '../../common/interfaces'; type GetGeotextSearchHandler = RequestHandler< unknown, - QueryResult | { message: string; error: string }, //response + GenericGeocodingResponse | { message: string; error: string }, //response undefined, GetGeotextSearchParams >; diff --git a/src/location/models/locationManager.ts b/src/location/models/locationManager.ts index 135cf2b4..867656a7 100644 --- a/src/location/models/locationManager.ts +++ b/src/location/models/locationManager.ts @@ -1,11 +1,12 @@ import { IConfig } from 'config'; import { Logger } from '@map-colonies/js-logger'; import { inject, injectable } from 'tsyringe'; +import { Feature } from 'geojson'; import { SERVICES, elasticConfigPath } from '../../common/constants'; import { GEOTEXT_REPOSITORY_SYMBOL, GeotextRepository } from '../DAL/locationRepository'; -import { GetGeotextSearchParams, QueryResult, TextSearchParams } from '../interfaces'; +import { GetGeotextSearchParams, TextSearchParams } from '../interfaces'; import { convertResult } from '../utils'; -import { IApplication } from '../../common/interfaces'; +import { GenericGeocodingResponse, IApplication } from '../../common/interfaces'; import { ElasticDbClientsConfig } from '../../common/elastic/interfaces'; import { ConvertSnakeToCamelCase } from '../../common/utils'; @@ -18,7 +19,7 @@ export class GeotextSearchManager { @inject(GEOTEXT_REPOSITORY_SYMBOL) private readonly geotextRepository: GeotextRepository ) {} - public async search(params: ConvertSnakeToCamelCase): Promise { + public async search(params: ConvertSnakeToCamelCase): Promise> { const extractNameEndpoint = this.appConfig.services.tokenTypesUrl; const { geotext: geotextIndex, From 73bb0421a37fa98f94a0cba4af9ebb99b5bff88c Mon Sep 17 00:00:00 2001 From: Niv Greenstein Date: Sun, 29 Sep 2024 19:40:28 +0300 Subject: [PATCH 259/262] delete: removed ControlResponse --- src/control/interfaces.ts | 33 --------------------------------- 1 file changed, 33 deletions(-) delete mode 100644 src/control/interfaces.ts diff --git a/src/control/interfaces.ts b/src/control/interfaces.ts deleted file mode 100644 index 0464e51c..00000000 --- a/src/control/interfaces.ts +++ /dev/null @@ -1,33 +0,0 @@ -import { Feature, GeoJsonProperties } from 'geojson'; -import { estypes } from '@elastic/elasticsearch'; -import { CommonRequestParameters, FeatureCollection } from '../common/interfaces'; -import { RemoveUnderscore } from '../common/utils'; - -export interface ControlResponse extends FeatureCollection { - geocoding?: { - version?: string; - query?: G & CommonRequestParameters; - response: Pick & { - /* eslint-disable @typescript-eslint/naming-convention */ - results_count?: estypes.SearchHitsMetadata['total']; - match_latency_ms?: estypes.SearchResponse['took']; - /* eslint-enable @typescript-eslint/naming-convention */ - }; - }; - features: (T & { - properties: RemoveUnderscore, '_score'>> & - GeoJsonProperties & { - matches: { - layer: string; - source: string; - // eslint-disable-next-line @typescript-eslint/naming-convention - source_id: string[]; - }[]; - names: { - [key: string]: string | string[] | undefined; - display: string; - default: string[]; - }; - }; - })[]; -} From d83f18b1c9d677ef46dcf31315237c40b13de600 Mon Sep 17 00:00:00 2001 From: Niv Greenstein Date: Sun, 29 Sep 2024 19:40:56 +0300 Subject: [PATCH 260/262] fix: changed type from QueryResult to GenericGeocodingResponse --- src/location/interfaces.ts | 33 --------------------------------- src/location/utils.ts | 15 ++++++++------- 2 files changed, 8 insertions(+), 40 deletions(-) diff --git a/src/location/interfaces.ts b/src/location/interfaces.ts index 13a56e3c..87d6f138 100644 --- a/src/location/interfaces.ts +++ b/src/location/interfaces.ts @@ -26,36 +26,3 @@ export interface GetGeotextSearchParams extends CommonRequestParameters { source?: string[]; region?: string[]; } - -/* eslint-disable @typescript-eslint/naming-convention */ - -export interface QueryResult { - type: string; - geocoding: { - version?: string; - query: ConvertCamelToSnakeCase; - response: { max_score: number; results_count: number; match_latency_ms: number } & Partial< - ConvertCamelToSnakeCase> - >; - }; - features: { - type: string; - geometry?: GeoJSON; - properties: { - matches: { - source?: string; - source_id?: string[]; - layer?: string; - }[]; - names: { - [key: string]: string | string[] | undefined; - display: string; - default: string[]; - }; - placetype?: string; - sub_placetype?: string; - regions?: { region: string; sub_region_names: string[] }[]; - } & RemoveUnderscore>; - }[]; -} -/* eslint-enable @typescript-eslint/naming-convention */ diff --git a/src/location/utils.ts b/src/location/utils.ts index 486de35d..559d7770 100644 --- a/src/location/utils.ts +++ b/src/location/utils.ts @@ -1,10 +1,11 @@ import https from 'https'; +import { Feature, Geometry } from 'geojson'; import { SearchResponse } from '@elastic/elasticsearch/lib/api/types'; import { StatusCodes } from 'http-status-codes'; import axios, { AxiosError, AxiosResponse as Response } from 'axios'; import { InternalServerError } from '../common/errors'; -import { IApplication } from '../common/interfaces'; -import { QueryResult, TextSearchParams } from './interfaces'; +import { GenericGeocodingResponse, IApplication } from '../common/interfaces'; +import { TextSearchParams } from './interfaces'; import { TextSearchHit } from './models/elasticsearchHits'; import { generateDisplayName } from './parsing'; @@ -50,7 +51,7 @@ export const convertResult = ( nameKeys: IApplication['nameTranslationsKeys']; mainLanguageRegex: IApplication['mainLanguageRegex']; } = { nameKeys: [], mainLanguageRegex: '' } -): QueryResult => ({ +): GenericGeocodingResponse => ({ type: 'FeatureCollection', geocoding: { version: process.env.npm_package_version, @@ -73,18 +74,18 @@ export const convertResult = ( hierarchies: params.hierarchies, }, }, - features: results.hits.hits.map(({ _source: feature, _score: score, highlight }, index): QueryResult['features'][number] => { + features: results.hits.hits.map(({ _source: feature, _score: score, highlight }, index): GenericGeocodingResponse['features'][number] => { const allNames = [feature!.text, feature!.translated_text || []]; return { type: 'Feature', - geometry: feature?.geo_json, + geometry: feature!.geo_json as Geometry, properties: { score, matches: [ { - layer: feature?.layer_name, + layer: feature!.layer_name, source: (sources ?? {})[feature?.source ?? ''] ?? feature?.source, - source_id: feature?.source_id.map((id) => id.replace(/(^\{)|(\}$)/g, '')), // TODO: check if to remove this + source_id: feature?.source_id.map((id) => id.replace(/(^\{)|(\}$)/g, '')) ?? [], }, ], names: { From e95c4e917eb46316522a9f91c142c279bafb6e7a Mon Sep 17 00:00:00 2001 From: Niv Greenstein Date: Sun, 29 Sep 2024 19:41:23 +0300 Subject: [PATCH 261/262] fix: fixed all tests to right types --- tests/integration/control/item/item.spec.ts | 27 +++++++------ tests/integration/control/route/route.spec.ts | 39 +++++++++---------- tests/integration/control/tile/tile.spec.ts | 37 +++++++++--------- tests/integration/control/utils.ts | 9 ++--- tests/integration/location/location.spec.ts | 25 ++++++------ tests/integration/location/mockObjects.ts | 12 ++---- tests/integration/location/utils.ts | 34 +++++++++------- 7 files changed, 89 insertions(+), 94 deletions(-) diff --git a/tests/integration/control/item/item.spec.ts b/tests/integration/control/item/item.spec.ts index 48ebb11a..0fad0366 100644 --- a/tests/integration/control/item/item.spec.ts +++ b/tests/integration/control/item/item.spec.ts @@ -9,8 +9,7 @@ import { getApp } from '../../../../src/app'; import { SERVICES } from '../../../../src/common/constants'; import { GetItemsQueryParams } from '../../../../src/control/item/controllers/itemController'; import { Item } from '../../../../src/control/item/models/item'; -import { ControlResponse } from '../../../../src/control/interfaces'; -import { CommonRequestParameters, GeoContext, GeoContextMode } from '../../../../src/common/interfaces'; +import { CommonRequestParameters, GenericGeocodingResponse, GeoContext, GeoContextMode } from '../../../../src/common/interfaces'; import { cronLoadTileLatLonDataSymbol } from '../../../../src/latLon/DAL/latLonDAL'; import { S3_REPOSITORY_SYMBOL } from '../../../../src/common/s3/s3Repository'; import { expectedResponse } from '../utils'; @@ -52,7 +51,7 @@ describe('/search/control/items', function () { expect(response.status).toBe(httpStatusCodes.OK); // expect(response).toSatisfyApiSpec(); - expect(response.body).toMatchObject>>( + expect(response.body).toMatchObject>>( expectedResponse(requestParams, [ITEM_1234, ITEM_1235, ITEM_1236], expect) ); }); @@ -64,7 +63,7 @@ describe('/search/control/items', function () { expect(response.status).toBe(httpStatusCodes.OK); // expect(response).toSatisfyApiSpec(); - expect(response.body).toMatchObject>>( + expect(response.body).toMatchObject>>( expectedResponse(requestParams, [ITEM_1234, ITEM_1235, ITEM_1236], expect) ); }); @@ -76,7 +75,7 @@ describe('/search/control/items', function () { expect(response.status).toBe(httpStatusCodes.OK); // expect(response).toSatisfyApiSpec(); - expect(response.body).toMatchObject>>( + expect(response.body).toMatchObject>>( expectedResponse(requestParams, [], expect) ); }); @@ -93,7 +92,7 @@ describe('/search/control/items', function () { expect(response.status).toBe(httpStatusCodes.OK); // expect(response).toSatisfyApiSpec(); - expect(response.body).toMatchObject>>( + expect(response.body).toMatchObject>>( expectedResponse(requestParams, [ITEM_1236, ITEM_1234, ITEM_1235], expect) ); }); @@ -110,7 +109,7 @@ describe('/search/control/items', function () { expect(response.status).toBe(httpStatusCodes.OK); // expect(response).toSatisfyApiSpec(); - expect(response.body).toMatchObject>>( + expect(response.body).toMatchObject>>( expectedResponse(requestParams, [ITEM_1236], expect) ); }); @@ -132,7 +131,7 @@ describe('/search/control/items', function () { expect(response.status).toBe(httpStatusCodes.OK); // expect(response).toSatisfyApiSpec(); - expect(response.body).toMatchObject>>( + expect(response.body).toMatchObject>>( expectedResponse(requestParams, [ITEM_1236, ITEM_1234, ITEM_1235], expect) ); }); @@ -153,7 +152,7 @@ describe('/search/control/items', function () { expect(response.status).toBe(httpStatusCodes.OK); // expect(response).toSatisfyApiSpec(); - expect(response.body).toMatchObject>>( + expect(response.body).toMatchObject>>( expectedResponse(requestParams, [ITEM_1236], expect) ); }); @@ -174,7 +173,7 @@ describe('/search/control/items', function () { expect(response.status).toBe(httpStatusCodes.OK); // expect(response).toSatisfyApiSpec(); - expect(response.body).toMatchObject>>( + expect(response.body).toMatchObject>>( expectedResponse(requestParams, [ITEM_1236, ITEM_1234, ITEM_1235], expect) ); }); @@ -195,7 +194,7 @@ describe('/search/control/items', function () { expect(response.status).toBe(httpStatusCodes.OK); // expect(response).toSatisfyApiSpec(); - expect(response.body).toMatchObject>>( + expect(response.body).toMatchObject>>( expectedResponse(requestParams, [ITEM_1236], expect) ); }); @@ -210,7 +209,7 @@ describe('/search/control/items', function () { expect(response.status).toBe(httpStatusCodes.OK); // expect(response).toSatisfyApiSpec(); - expect(response.body).toMatchObject>>( + expect(response.body).toMatchObject>>( expectedResponse(requestParams, [ITEM_1234], expect) ); }); @@ -225,7 +224,7 @@ describe('/search/control/items', function () { expect(response.status).toBe(httpStatusCodes.OK); // expect(response).toSatisfyApiSpec(); - expect(response.body).toMatchObject>>( + expect(response.body).toMatchObject>>( expectedResponse(requestParams, [], expect) ); }); @@ -241,7 +240,7 @@ describe('/search/control/items', function () { expect(response.status).toBe(httpStatusCodes.OK); // expect(response).toSatisfyApiSpec(); - expect(response.body).toMatchObject>>( + expect(response.body).toMatchObject>>( expectedResponse(requestParams, [ITEM_1234, ITEM_1235], expect) ); }); diff --git a/tests/integration/control/route/route.spec.ts b/tests/integration/control/route/route.spec.ts index ed69fd61..5b117c7a 100644 --- a/tests/integration/control/route/route.spec.ts +++ b/tests/integration/control/route/route.spec.ts @@ -9,8 +9,7 @@ import { getApp } from '../../../../src/app'; import { SERVICES } from '../../../../src/common/constants'; import { GetRoutesQueryParams } from '../../../../src/control/route/controllers/routeController'; import { Route } from '../../../../src/control/route/models/route'; -import { ControlResponse } from '../../../../src/control/interfaces'; -import { CommonRequestParameters, GeoContext, GeoContextMode } from '../../../../src/common/interfaces'; +import { CommonRequestParameters, GenericGeocodingResponse, GeoContext, GeoContextMode } from '../../../../src/common/interfaces'; import { S3_REPOSITORY_SYMBOL } from '../../../../src/common/s3/s3Repository'; import { cronLoadTileLatLonDataSymbol } from '../../../../src/latLon/DAL/latLonDAL'; import { expectedResponse } from '../utils'; @@ -52,7 +51,7 @@ describe('/search/control/route', function () { expect(response.status).toBe(httpStatusCodes.OK); // expect(response).toSatisfyApiSpec(); - expect(response.body).toMatchObject>>( + expect(response.body).toMatchObject>>( expectedResponse(requestParams, [ROUTE_VIA_CAMILLUCCIA_A, ROUTE_VIA_CAMILLUCCIA_B], expect) ); }); @@ -69,7 +68,7 @@ describe('/search/control/route', function () { expect(response.status).toBe(httpStatusCodes.OK); // expect(response).toSatisfyApiSpec(); - expect(response.body).toMatchObject>>( + expect(response.body).toMatchObject>>( expectedResponse(requestParams, [ROUTE_VIA_CAMILLUCCIA_B, ROUTE_VIA_CAMILLUCCIA_A], expect) ); }); @@ -86,7 +85,7 @@ describe('/search/control/route', function () { expect(response.status).toBe(httpStatusCodes.OK); // expect(response).toSatisfyApiSpec(); - expect(response.body).toMatchObject>>( + expect(response.body).toMatchObject>>( expectedResponse(requestParams, [ROUTE_VIA_CAMILLUCCIA_B], expect) ); }); @@ -107,7 +106,7 @@ describe('/search/control/route', function () { expect(response.status).toBe(httpStatusCodes.OK); // expect(response).toSatisfyApiSpec(); - expect(response.body).toMatchObject>>( + expect(response.body).toMatchObject>>( expectedResponse(requestParams, [ROUTE_VIA_CAMILLUCCIA_B, ROUTE_VIA_CAMILLUCCIA_A], expect) ); }); @@ -128,7 +127,7 @@ describe('/search/control/route', function () { expect(response.status).toBe(httpStatusCodes.OK); // expect(response).toSatisfyApiSpec(); - expect(response.body).toMatchObject>>( + expect(response.body).toMatchObject>>( expectedResponse(requestParams, [ROUTE_VIA_CAMILLUCCIA_B], expect) ); }); @@ -149,7 +148,7 @@ describe('/search/control/route', function () { expect(response.status).toBe(httpStatusCodes.OK); // expect(response).toSatisfyApiSpec(); - expect(response.body).toMatchObject>>( + expect(response.body).toMatchObject>>( expectedResponse(requestParams, [ROUTE_VIA_CAMILLUCCIA_B, ROUTE_VIA_CAMILLUCCIA_A], expect) ); }); @@ -170,7 +169,7 @@ describe('/search/control/route', function () { expect(response.status).toBe(httpStatusCodes.OK); // expect(response).toSatisfyApiSpec(); - expect(response.body).toMatchObject>>( + expect(response.body).toMatchObject>>( expectedResponse(requestParams, [ROUTE_VIA_CAMILLUCCIA_B], expect) ); }); @@ -185,7 +184,7 @@ describe('/search/control/route', function () { expect(response.status).toBe(httpStatusCodes.OK); // expect(response).toSatisfyApiSpec(); - expect(response.body).toMatchObject>>( + expect(response.body).toMatchObject>>( expectedResponse(requestParams, [ROUTE_VIA_CAMILLUCCIA_B], expect) ); }); @@ -200,7 +199,7 @@ describe('/search/control/route', function () { expect(response.status).toBe(httpStatusCodes.OK); // expect(response).toSatisfyApiSpec(); - expect(response.body).toMatchObject>>( + expect(response.body).toMatchObject>>( expectedResponse(requestParams, [], expect) ); }); @@ -218,7 +217,7 @@ describe('/search/control/route', function () { expect(response.status).toBe(httpStatusCodes.OK); // expect(response).toSatisfyApiSpec(); - expect(response.body).toMatchObject>>( + expect(response.body).toMatchObject>>( expectedResponse(requestParams, [CONTROL_POINT_OLIMPIADE_112, CONTROL_POINT_OLIMPIADE_111], expect) ); }); @@ -236,7 +235,7 @@ describe('/search/control/route', function () { expect(response.status).toBe(httpStatusCodes.OK); // expect(response).toSatisfyApiSpec(); - expect(response.body).toMatchObject>>( + expect(response.body).toMatchObject>>( expectedResponse(requestParams, [CONTROL_POINT_OLIMPIADE_112], expect) ); }); @@ -258,7 +257,7 @@ describe('/search/control/route', function () { expect(response.status).toBe(httpStatusCodes.OK); // expect(response).toSatisfyApiSpec(); - expect(response.body).toMatchObject>>( + expect(response.body).toMatchObject>>( expectedResponse(requestParams, [CONTROL_POINT_OLIMPIADE_112, CONTROL_POINT_OLIMPIADE_111], expect) ); }); @@ -280,7 +279,7 @@ describe('/search/control/route', function () { expect(response.status).toBe(httpStatusCodes.OK); // expect(response).toSatisfyApiSpec(); - expect(response.body).toMatchObject>>( + expect(response.body).toMatchObject>>( expectedResponse(requestParams, [CONTROL_POINT_OLIMPIADE_112], expect) ); }); @@ -302,7 +301,7 @@ describe('/search/control/route', function () { expect(response.status).toBe(httpStatusCodes.OK); // expect(response).toSatisfyApiSpec(); - expect(response.body).toMatchObject>>( + expect(response.body).toMatchObject>>( expectedResponse(requestParams, [CONTROL_POINT_OLIMPIADE_112, CONTROL_POINT_OLIMPIADE_111], expect) ); }); @@ -324,7 +323,7 @@ describe('/search/control/route', function () { expect(response.status).toBe(httpStatusCodes.OK); // expect(response).toSatisfyApiSpec(); - expect(response.body).toMatchObject>>( + expect(response.body).toMatchObject>>( expectedResponse(requestParams, [CONTROL_POINT_OLIMPIADE_112], expect) ); }); @@ -340,7 +339,7 @@ describe('/search/control/route', function () { expect(response.status).toBe(httpStatusCodes.OK); // expect(response).toSatisfyApiSpec(); - expect(response.body).toMatchObject>>( + expect(response.body).toMatchObject>>( expectedResponse(requestParams, [CONTROL_POINT_OLIMPIADE_111, CONTROL_POINT_OLIMPIADE_112], expect) ); }); @@ -356,7 +355,7 @@ describe('/search/control/route', function () { expect(response.status).toBe(httpStatusCodes.OK); // expect(response).toSatisfyApiSpec(); - expect(response.body).toMatchObject>>( + expect(response.body).toMatchObject>>( expectedResponse(requestParams, [CONTROL_POINT_OLIMPIADE_111], expect) ); }); @@ -372,7 +371,7 @@ describe('/search/control/route', function () { expect(response.status).toBe(httpStatusCodes.OK); // expect(response).toSatisfyApiSpec(); - expect(response.body).toMatchObject>>( + expect(response.body).toMatchObject>>( expectedResponse(requestParams, [], expect) ); }); diff --git a/tests/integration/control/tile/tile.spec.ts b/tests/integration/control/tile/tile.spec.ts index 1c12661e..c44d6967 100644 --- a/tests/integration/control/tile/tile.spec.ts +++ b/tests/integration/control/tile/tile.spec.ts @@ -10,8 +10,7 @@ import { getApp } from '../../../../src/app'; import { SERVICES } from '../../../../src/common/constants'; import { GetTilesQueryParams } from '../../../../src/control/tile/controllers/tileController'; import { Tile } from '../../../../src/control/tile/models/tile'; -import { ControlResponse } from '../../../../src/control/interfaces'; -import { CommonRequestParameters, GeoContext, GeoContextMode } from '../../../../src/common/interfaces'; +import { CommonRequestParameters, GenericGeocodingResponse, GeoContext, GeoContextMode } from '../../../../src/common/interfaces'; import { S3_REPOSITORY_SYMBOL } from '../../../../src/common/s3/s3Repository'; import { cronLoadTileLatLonDataSymbol } from '../../../../src/latLon/DAL/latLonDAL'; import { expectedResponse } from '../utils'; @@ -53,7 +52,7 @@ describe('/search/control/tiles', function () { expect(response.status).toBe(httpStatusCodes.OK); // expect(response).toSatisfyApiSpec(); - expect(response.body).toMatchObject>>( + expect(response.body).toMatchObject>>( expectedResponse(requestParams, [RIT_TILE, RIC_TILE], expect) ); }); @@ -70,7 +69,7 @@ describe('/search/control/tiles', function () { expect(response.status).toBe(httpStatusCodes.OK); // expect(response).toSatisfyApiSpec(); - expect(response.body).toMatchObject>>( + expect(response.body).toMatchObject>>( expectedResponse(requestParams, [RIC_TILE, RIT_TILE], expect) ); }); @@ -87,7 +86,7 @@ describe('/search/control/tiles', function () { expect(response.status).toBe(httpStatusCodes.OK); // expect(response).toSatisfyApiSpec(); - expect(response.body).toMatchObject>>( + expect(response.body).toMatchObject>>( expectedResponse(requestParams, [RIC_TILE], expect) ); }); @@ -108,7 +107,7 @@ describe('/search/control/tiles', function () { expect(response.status).toBe(httpStatusCodes.OK); // expect(response).toSatisfyApiSpec(); - expect(response.body).toMatchObject>>( + expect(response.body).toMatchObject>>( expectedResponse(requestParams, [RIC_TILE, RIT_TILE], expect) ); }); @@ -129,7 +128,7 @@ describe('/search/control/tiles', function () { expect(response.status).toBe(httpStatusCodes.OK); // expect(response).toSatisfyApiSpec(); - expect(response.body).toMatchObject>>( + expect(response.body).toMatchObject>>( expectedResponse(requestParams, [RIC_TILE], expect) ); }); @@ -150,7 +149,7 @@ describe('/search/control/tiles', function () { expect(response.status).toBe(httpStatusCodes.OK); // expect(response).toSatisfyApiSpec(); - expect(response.body).toMatchObject>>( + expect(response.body).toMatchObject>>( expectedResponse(requestParams, [RIC_TILE, RIT_TILE], expect) ); }); @@ -171,7 +170,7 @@ describe('/search/control/tiles', function () { expect(response.status).toBe(httpStatusCodes.OK); // expect(response).toSatisfyApiSpec(); - expect(response.body).toMatchObject>>( + expect(response.body).toMatchObject>>( expectedResponse(requestParams, [RIC_TILE], expect) ); }); @@ -186,7 +185,7 @@ describe('/search/control/tiles', function () { expect(response.status).toBe(httpStatusCodes.OK); // expect(response).toSatisfyApiSpec(); - expect(response.body).toMatchObject>>( + expect(response.body).toMatchObject>>( expectedResponse(requestParams, [RIC_TILE], expect) ); }); @@ -201,7 +200,7 @@ describe('/search/control/tiles', function () { expect(response.status).toBe(httpStatusCodes.OK); // expect(response).toSatisfyApiSpec(); - expect(response.body).toMatchObject>>( + expect(response.body).toMatchObject>>( expectedResponse(requestParams, [], expect) ); }); @@ -217,7 +216,7 @@ describe('/search/control/tiles', function () { expect(response.status).toBe(httpStatusCodes.OK); // expect(response).toSatisfyApiSpec(); - expect(response.body).toMatchObject>>( + expect(response.body).toMatchObject>>( expectedResponse(requestParams, [SUB_TILE_66, SUB_TILE_65], expect) ); }); @@ -233,7 +232,7 @@ describe('/search/control/tiles', function () { expect(response.status).toBe(httpStatusCodes.OK); // expect(response).toSatisfyApiSpec(); - expect(response.body).toMatchObject>>( + expect(response.body).toMatchObject>>( expectedResponse(requestParams, [], expect) ); }); @@ -255,7 +254,7 @@ describe('/search/control/tiles', function () { // expect(response.status).toBe(httpStatusCodes.OK); // expect(response).toSatisfyApiSpec(); - expect(response.body).toMatchObject>>( + expect(response.body).toMatchObject>>( expectedResponse(requestParams, [SUB_TILE_66], expect) ); }); @@ -277,7 +276,7 @@ describe('/search/control/tiles', function () { // expect(response.status).toBe(httpStatusCodes.OK); // expect(response).toSatisfyApiSpec(); - expect(response.body).toMatchObject>>( + expect(response.body).toMatchObject>>( expectedResponse(requestParams, [SUB_TILE_66], expect) ); }); @@ -294,7 +293,7 @@ describe('/search/control/tiles', function () { expect(response.status).toBe(httpStatusCodes.OK); // // expect(response).toSatisfyApiSpec(); - expect(response.body).toMatchObject>>( + expect(response.body).toMatchObject>>( expectedResponse(requestParams, [SUB_TILE_66], expect) ); }); @@ -309,7 +308,7 @@ describe('/search/control/tiles', function () { expect(response.status).toBe(httpStatusCodes.OK); // expect(response).toSatisfyApiSpec(); - expect(response.body).toMatchObject>>( + expect(response.body).toMatchObject>>( expectedResponse(requestParams, [RIT_TILE, RIC_TILE], expect) ); }); @@ -330,7 +329,7 @@ describe('/search/control/tiles', function () { expect(response.status).toBe(httpStatusCodes.OK); // expect(response).toSatisfyApiSpec(); - expect(response.body).toMatchObject>>( + expect(response.body).toMatchObject>>( expectedResponse(requestParams, [RIC_TILE], expect) ); }); @@ -351,7 +350,7 @@ describe('/search/control/tiles', function () { expect(response.status).toBe(httpStatusCodes.OK); // expect(response).toSatisfyApiSpec(); - expect(response.body).toMatchObject>>( + expect(response.body).toMatchObject>>( expectedResponse(requestParams, [RIC_TILE, RIT_TILE], expect) ); }); diff --git a/tests/integration/control/utils.ts b/tests/integration/control/utils.ts index eb0b7574..889b2f9e 100644 --- a/tests/integration/control/utils.ts +++ b/tests/integration/control/utils.ts @@ -1,6 +1,5 @@ /* eslint-disable @typescript-eslint/naming-convention */ -import { CommonRequestParameters } from '../../../src/common/interfaces'; -import { ControlResponse } from '../../../src/control/interfaces'; +import { CommonRequestParameters, GenericGeocodingResponse } from '../../../src/common/interfaces'; import { GetItemsQueryParams } from '../../../src/control/item/controllers/itemController'; import { Item } from '../../../src/control/item/models/item'; import { GetRoutesQueryParams } from '../../../src/control/route/controllers/routeController'; @@ -8,7 +7,7 @@ import { Route } from '../../../src/control/route/models/route'; import { GetTilesQueryParams } from '../../../src/control/tile/controllers/tileController'; import { Tile } from '../../../src/control/tile/models/tile'; -const expectedObjectWithScore = (source: T, expect: jest.Expect): ControlResponse['features'][number] => ({ +const expectedObjectWithScore = (source: T, expect: jest.Expect): GenericGeocodingResponse['features'][number] => ({ ...source, properties: { ...source.properties, @@ -30,7 +29,7 @@ const expectedObjectWithScore = (source: T, expec const expectedGeocodingElasticResponseMetrics = ( resultsCount: number, expect: jest.Expect -): NonNullable['geocoding']>['response'] => ({ +): NonNullable['geocoding']>['response'] => ({ results_count: resultsCount, max_score: expect.any(Number) as number, match_latency_ms: expect.any(Number) as number, @@ -40,7 +39,7 @@ export const expectedResponse = > => ({ +): GenericGeocodingResponse> => ({ type: 'FeatureCollection', geocoding: { version: process.env.npm_package_version, diff --git a/tests/integration/location/location.spec.ts b/tests/integration/location/location.spec.ts index faa8c15e..46eb28a2 100644 --- a/tests/integration/location/location.spec.ts +++ b/tests/integration/location/location.spec.ts @@ -2,6 +2,7 @@ import config from 'config'; import { DependencyContainer } from 'tsyringe'; import { Application } from 'express'; +import { Feature } from 'geojson'; import { CleanupRegistry } from '@map-colonies/cleanup-registry'; import jsLogger from '@map-colonies/js-logger'; import { trace } from '@opentelemetry/api'; @@ -11,8 +12,8 @@ import { getApp } from '../../../src/app'; import { SERVICES } from '../../../src/common/constants'; import { S3_REPOSITORY_SYMBOL } from '../../../src/common/s3/s3Repository'; import { cronLoadTileLatLonDataSymbol } from '../../../src/latLon/DAL/latLonDAL'; -import { GetGeotextSearchParams, QueryResult } from '../../../src/location/interfaces'; -import { GeoContext, GeoContextMode, IApplication } from '../../../src/common/interfaces'; +import { GetGeotextSearchParams } from '../../../src/location/interfaces'; +import { GenericGeocodingResponse, GeoContext, GeoContextMode, IApplication } from '../../../src/common/interfaces'; import { LocationRequestSender } from './helpers/requestSender'; import { OSM_LA_PORT, @@ -72,7 +73,7 @@ describe('/search/location', function () { expect(response.status).toBe(httpStatusCodes.OK); // expect(response).toSatisfyApiSpec(); - expect(response.body).toMatchObject( + expect(response.body).toMatchObject>( expectedResponse( { ...requestParams, @@ -125,7 +126,7 @@ describe('/search/location', function () { expect(response.status).toBe(httpStatusCodes.OK); // expect(response).toSatisfyApiSpec(); - expect(response.body).toMatchObject( + expect(response.body).toMatchObject>( expectedResponse( requestParams, { @@ -175,7 +176,7 @@ describe('/search/location', function () { expect(response.status).toBe(httpStatusCodes.OK); // expect(response).toSatisfyApiSpec(); - expect(response.body).toMatchObject( + expect(response.body).toMatchObject>( expectedResponse( { ...requestParams, @@ -208,7 +209,7 @@ describe('/search/location', function () { test.each< Pick & { - hierarchies: QueryResult['geocoding']['response']['hierarchies']; + hierarchies: GenericGeocodingResponse['geocoding']['response']['hierarchies']; returnedFeatures: MockLocationQueryFeature[]; } >([ @@ -265,7 +266,7 @@ describe('/search/location', function () { expect(response.status).toBe(httpStatusCodes.OK); // expect(response).toSatisfyApiSpec(); - expect(response.body).toMatchObject( + expect(response.body).toMatchObject>( expectedResponse( { ...requestParams, @@ -285,8 +286,8 @@ describe('/search/location', function () { test.each< Pick & { - place_types: QueryResult['geocoding']['response']['place_types']; - sub_place_types: QueryResult['geocoding']['response']['sub_place_types']; + place_types: GenericGeocodingResponse['geocoding']['response']['place_types']; + sub_place_types: GenericGeocodingResponse['geocoding']['response']['sub_place_types']; returnedFeatures: MockLocationQueryFeature[]; } >([ @@ -319,7 +320,7 @@ describe('/search/location', function () { expect(response.status).toBe(httpStatusCodes.OK); // expect(response).toSatisfyApiSpec(); - expect(response.body).toMatchObject( + expect(response.body).toMatchObject>( expectedResponse( { ...requestParams, @@ -370,7 +371,7 @@ describe('/search/location', function () { expect(response.status).toBe(httpStatusCodes.OK); // expect(response).toSatisfyApiSpec(); - expect(response.body).toMatchObject( + expect(response.body).toMatchObject>( expectedResponse( requestParams, { @@ -402,7 +403,7 @@ describe('/search/location', function () { expect(response.status).toBe(httpStatusCodes.OK); // expect(response).toSatisfyApiSpec(); - expect(response.body).toMatchObject( + expect(response.body).toMatchObject>( expectedResponse( requestParams, { diff --git a/tests/integration/location/mockObjects.ts b/tests/integration/location/mockObjects.ts index a610d28e..8fede417 100644 --- a/tests/integration/location/mockObjects.ts +++ b/tests/integration/location/mockObjects.ts @@ -1,16 +1,10 @@ /* eslint-disable @typescript-eslint/naming-convention */ /* eslint-disable @typescript-eslint/no-magic-numbers */ -import { QueryResult } from '../../../src/location/interfaces'; +import { Feature } from 'geojson'; +import { GenericGeocodingResponse } from '../../../src/common/interfaces'; import { HierarchySearchHit } from '../../../src/location/models/elasticsearchHits'; -type ChangeFields = Omit & R; - -export type MockLocationQueryFeature = ChangeFields< - QueryResult['features'][number], - { - properties: Omit; - } ->; +export type MockLocationQueryFeature = GenericGeocodingResponse['features'][number]; export const NY_JFK_AIRPORT: MockLocationQueryFeature = { type: 'Feature', diff --git a/tests/integration/location/utils.ts b/tests/integration/location/utils.ts index 4c5269da..ee83b5d0 100644 --- a/tests/integration/location/utils.ts +++ b/tests/integration/location/utils.ts @@ -1,20 +1,23 @@ /* eslint-disable @typescript-eslint/naming-convention */ -import { QueryResult } from '../../../src/location/interfaces'; +import { Feature } from 'geojson'; +import { GetGeotextSearchParams } from '../../../src/location/interfaces'; +import { GenericGeocodingResponse } from '../../../src/common/interfaces'; import { MockLocationQueryFeature } from './mockObjects'; -const expectedObjectWithScore = (obj: MockLocationQueryFeature, expect: jest.Expect): QueryResult['features'][number] => ({ - ...obj, - properties: { - ...obj.properties, - score: expect.any(Number) as number, - }, -}); +const expectedObjectWithScore = (obj: MockLocationQueryFeature, expect: jest.Expect): GenericGeocodingResponse['features'][number] => + ({ + ...obj, + properties: { + ...obj.properties, + score: expect.any(Number) as number, + }, + } as GenericGeocodingResponse['features'][number]); const expectedGeocodingElasticResponseMetrics = ( - responseParams: Partial, + responseParams: Partial['geocoding']['response']>, resultsCount: number, expect: jest.Expect -): NonNullable['response'] => ({ +): NonNullable['geocoding']>['response'] => ({ results_count: resultsCount, max_score: expect.any(Number) as number, match_latency_ms: expect.any(Number) as number, @@ -22,11 +25,11 @@ const expectedGeocodingElasticResponseMetrics = ( }); export const expectedResponse = ( - requestParams: QueryResult['geocoding']['query'], - responseParams: Partial, + requestParams: GetGeotextSearchParams, + responseParams: Partial['geocoding']['response']>, arr: MockLocationQueryFeature[], expect: jest.Expect -): QueryResult => ({ +): GenericGeocodingResponse => ({ type: 'FeatureCollection', geocoding: { version: process.env.npm_package_version, @@ -37,6 +40,7 @@ export const expectedResponse = ( }); export const hierarchiesWithAnyWieght = ( - hierarchies: QueryResult['geocoding']['response']['hierarchies'], + hierarchies: GenericGeocodingResponse['geocoding']['response']['hierarchies'], expect: jest.Expect -): QueryResult['geocoding']['response']['hierarchies'] => hierarchies?.map((hierarchy) => ({ ...hierarchy, weight: expect.any(Number) as number })); +): GenericGeocodingResponse['geocoding']['response']['hierarchies'] => + (hierarchies as { hierarchy: string }[] | undefined)?.map((hierarchy) => ({ ...hierarchy, weight: expect.any(Number) as number })); From c340fd4e3d669b7a58c3fa7ef76df8bc23e50efc Mon Sep 17 00:00:00 2001 From: Niv Greenstein Date: Sun, 29 Sep 2024 19:41:51 +0300 Subject: [PATCH 262/262] fix: fixed latLonDal init and prevent it from being created 3 times --- src/containerConfig.ts | 3 ++- src/latLon/DAL/latLonDAL.ts | 24 +++++++++++++++++++++--- src/latLon/models/latLonManager.ts | 4 ++-- src/serverBuilder.ts | 1 - 4 files changed, 25 insertions(+), 7 deletions(-) diff --git a/src/containerConfig.ts b/src/containerConfig.ts index 831c2f03..a365a4b5 100644 --- a/src/containerConfig.ts +++ b/src/containerConfig.ts @@ -21,7 +21,7 @@ import { ROUTE_ROUTER_SYMBOL, routeRouterFactory } from './control/route/routes/ import { LAT_LON_ROUTER_SYMBOL, latLonRouterFactory } from './latLon/routes/latLonRouter'; import { GEOTEXT_REPOSITORY_SYMBOL, geotextRepositoryFactory } from './location/DAL/locationRepository'; import { GEOTEXT_SEARCH_ROUTER_SYMBOL, geotextSearchRouterFactory } from './location/routes/locationRouter'; -import { cronLoadTileLatLonDataFactory, cronLoadTileLatLonDataSymbol } from './latLon/DAL/latLonDAL'; +import { cronLoadTileLatLonDataFactory, cronLoadTileLatLonDataSymbol, latLonDalSymbol, latLonSignletonFactory } from './latLon/DAL/latLonDAL'; import { ITEM_REPOSITORY_SYMBOL, itemRepositoryFactory } from './control/item/DAL/itemRepository'; import { RedisClient, redisClientFactory } from './common/redis'; import { s3ClientFactory } from './common/s3'; @@ -132,6 +132,7 @@ export const registerExternalValues = async (options?: RegisterOptions): Promise { token: GEOTEXT_REPOSITORY_SYMBOL, provider: { useFactory: geotextRepositoryFactory } }, { token: GEOTEXT_SEARCH_ROUTER_SYMBOL, provider: { useFactory: geotextSearchRouterFactory } }, { token: MGRS_ROUTER_SYMBOL, provider: { useFactory: mgrsRouterFactory } }, + { token: latLonDalSymbol, provider: { useFactory: latLonSignletonFactory } }, { token: cronLoadTileLatLonDataSymbol, provider: { diff --git a/src/latLon/DAL/latLonDAL.ts b/src/latLon/DAL/latLonDAL.ts index 3a7bda3a..fff62f1b 100644 --- a/src/latLon/DAL/latLonDAL.ts +++ b/src/latLon/DAL/latLonDAL.ts @@ -10,6 +10,10 @@ import { ConvertCamelToSnakeCase } from '../../common/utils'; import { S3_REPOSITORY_SYMBOL, S3Repository } from '../../common/s3/s3Repository'; type LatLon = ConvertCamelToSnakeCase; +let scheduledTask: cron.ScheduledTask | null = null; + +let latLonDALInstance: LatLonDAL | null = null; + @injectable() export class LatLonDAL { private readonly latLonMap: Map; @@ -31,6 +35,7 @@ export class LatLonDAL { this.onGoingUpdate = true; this.dataLoad = undefined; this.dataLoadError = false; + this.init().catch((error) => { this.logger.error('Failed to initialize lat-lon data', error); this.dataLoadError = true; @@ -108,14 +113,27 @@ export class LatLonDAL { } catch (error) { this.logger.error(`Failed to delete latLonData file ${latLonDataPath}. Error: ${(error as Error).message}`); } - this.logger.info('cronLoadTileLatLonData: update completed'); + this.logger.info('loadLatLonData: update completed'); } } export const cronLoadTileLatLonDataSymbol = Symbol('cronLoadTileLatLonDataSymbol'); +export const latLonDalSymbol = Symbol('latLonDalSymbol'); +export const latLonSignletonFactory: FactoryFunction = (dependencyContainer) => { + const logger = dependencyContainer.resolve(SERVICES.LOGGER); + const s3Repository = dependencyContainer.resolve(S3_REPOSITORY_SYMBOL); + + if (latLonDALInstance !== null) { + return latLonDALInstance; + } + + latLonDALInstance = new LatLonDAL(logger, s3Repository); + return latLonDALInstance; +}; + export const cronLoadTileLatLonDataFactory: FactoryFunction = (dependencyContainer) => { - const latLonDAL = dependencyContainer.resolve(LatLonDAL); + const latLonDAL = dependencyContainer.resolve(latLonDalSymbol); const logger = dependencyContainer.resolve(SERVICES.LOGGER); const cronPattern: string | undefined = dependencyContainer.resolve(SERVICES.APPLICATION).cronLoadTileLatLonDataPattern; @@ -125,7 +143,7 @@ export const cronLoadTileLatLonDataFactory: FactoryFunction } /* istanbul ignore next */ - const scheduledTask = cron.schedule(cronPattern, () => { + scheduledTask = cron.schedule(cronPattern, () => { if (!latLonDAL.getOnGoingUpdate()) { logger.info('cronLoadTileLatLonData: starting update'); latLonDAL.init().catch((error) => { diff --git a/src/latLon/models/latLonManager.ts b/src/latLon/models/latLonManager.ts index 311e3c18..c8db9eff 100644 --- a/src/latLon/models/latLonManager.ts +++ b/src/latLon/models/latLonManager.ts @@ -4,7 +4,7 @@ import { inject, injectable } from 'tsyringe'; import { BBox, Feature } from 'geojson'; import * as mgrs from 'mgrs'; import { SERVICES } from '../../common/constants'; -import { LatLonDAL } from '../DAL/latLonDAL'; +import { LatLonDAL, latLonDalSymbol } from '../DAL/latLonDAL'; import { convertUTMToWgs84, convertWgs84ToUTM, parseGeo, validateWGS84Coordinate } from '../../common/utils'; import { BadRequestError } from '../../common/errors'; import { WGS84Coordinate } from '../../common/interfaces'; @@ -14,7 +14,7 @@ import { convertCamelToSnakeCase } from '../../control/utils'; export class LatLonManager { public constructor( @inject(SERVICES.LOGGER) private readonly logger: Logger, - @inject(LatLonDAL) private readonly latLonDAL: LatLonDAL, + @inject(latLonDalSymbol) private readonly latLonDAL: LatLonDAL, @inject(SERVICES.CONFIG) private readonly config: IConfig ) {} diff --git a/src/serverBuilder.ts b/src/serverBuilder.ts index 1c267728..6a533d37 100644 --- a/src/serverBuilder.ts +++ b/src/serverBuilder.ts @@ -37,7 +37,6 @@ export class ServerBuilder { ) { this.serverInstance = express(); // eslint-disable-next-line @typescript-eslint/no-unused-expressions - this.cronLoadTileLatLonData; } public build(): express.Application {