From 7749d3f3f04b386a9624f8463171fbb04f5226dc Mon Sep 17 00:00:00 2001 From: NatalieShaked Date: Tue, 27 Aug 2024 14:46:25 +0300 Subject: [PATCH 01/20] feat: added redis configuration --- config/custom-environment-variables.json | 17 +++ config/default.json | 6 + config/test.json | 6 + package-lock.json | 138 +++++++++++++++++++++ package.json | 1 + src/common/constants.ts | 4 + src/common/interfaces.ts | 4 + src/common/redis/domainFieldsRepository.ts | 5 + src/common/redis/index.ts | 33 +++++ src/common/redis/keys.ts | 5 + src/common/redis/redisManager.ts | 31 +++++ src/containerConfig.ts | 52 +++++++- 12 files changed, 296 insertions(+), 6 deletions(-) create mode 100644 src/common/redis/domainFieldsRepository.ts create mode 100644 src/common/redis/index.ts create mode 100644 src/common/redis/keys.ts create mode 100644 src/common/redis/redisManager.ts diff --git a/config/custom-environment-variables.json b/config/custom-environment-variables.json index ed72d6e5..c6ef51dc 100644 --- a/config/custom-environment-variables.json +++ b/config/custom-environment-variables.json @@ -86,6 +86,23 @@ }, "database": "POSTGRES_DB_NAME", "schema": "POSTGRES_DB_SCHEMA" + }, + "redis": { + "host": "REDIS_HOST", + "port": { + "__name": "REDIS_PORT", + "__format": "number" + }, + "db": { + "__name": "REDIS_INDEX", + "__format": "number" + }, + "username": "REDIS_USERNAME", + "password": "REDIS_PASSWORD", + "maxRetriesPerRequest": { + "__name": "REDIS_MAX_RETRIES_PER_REQUEST", + "__format": "number" + } } }, "application": { diff --git a/config/default.json b/config/default.json index cf9186b0..4e0dd4ad 100644 --- a/config/default.json +++ b/config/default.json @@ -71,6 +71,12 @@ }, "database": "postgres", "schema": "geocoder" + }, + "redis": { + "host": "localhost", + "port": 6379, + "db": 0, + "maxRetriesPerRequest": 1 } }, "application": { diff --git a/config/test.json b/config/test.json index 55384c41..f267cd35 100644 --- a/config/test.json +++ b/config/test.json @@ -45,6 +45,12 @@ }, "database": "postgres", "schema": "geocoder" + }, + "redis": { + "host": "localhost", + "port": 6379, + "db": 0, + "maxRetriesPerRequest": 1 } }, "application": { diff --git a/package-lock.json b/package-lock.json index 0c1d7e64..527fc462 100644 --- a/package-lock.json +++ b/package-lock.json @@ -30,6 +30,7 @@ "fast-xml-parser": "^4.4.0", "geojson-validation": "^1.0.2", "http-status-codes": "^2.2.0", + "ioredis": "^5.4.1", "mgrs": "^2.0.0", "node-cron": "^3.0.3", "node-fetch-commonjs": "^3.3.2", @@ -2953,6 +2954,11 @@ "node": ">=6.9.0" } }, + "node_modules/@ioredis/commands": { + "version": "1.2.0", + "resolved": "https://registry.npmjs.org/@ioredis/commands/-/commands-1.2.0.tgz", + "integrity": "sha512-Sx1pU8EM64o2BrqNpEO1CNLtKQwyhuXuqyfH7oGKCk+1a33d2r5saW8zNwm3j6BTExtjrv2BxTgzzkMwts6vGg==" + }, "node_modules/@isaacs/cliui": { "version": "8.0.2", "resolved": "https://registry.npmjs.org/@isaacs/cliui/-/cliui-8.0.2.tgz", @@ -8576,6 +8582,14 @@ "node": ">=0.8" } }, + "node_modules/cluster-key-slot": { + "version": "1.1.2", + "resolved": "https://registry.npmjs.org/cluster-key-slot/-/cluster-key-slot-1.1.2.tgz", + "integrity": "sha512-RMr0FhtfXemyinomL4hrWcYJxmX6deFdCxpJzhDttxgO1+bcCnkk+9drydLVDmAMG7NE6aN/fl4F7ucU/90gAA==", + "engines": { + "node": ">=0.10.0" + } + }, "node_modules/co": { "version": "4.6.0", "resolved": "https://registry.npmjs.org/co/-/co-4.6.0.tgz", @@ -9671,6 +9685,14 @@ "node": ">=0.4.0" } }, + "node_modules/denque": { + "version": "2.1.0", + "resolved": "https://registry.npmjs.org/denque/-/denque-2.1.0.tgz", + "integrity": "sha512-HVQE3AAb/pxF8fQAoiqpvg9i3evqug3hoiwakOyZAwJm+6vZehbkYXZ0l4JxS+I3QxM97v5aaRNhj8v5oBhekw==", + "engines": { + "node": ">=0.10" + } + }, "node_modules/depd": { "version": "2.0.0", "resolved": "https://registry.npmjs.org/depd/-/depd-2.0.0.tgz", @@ -12389,6 +12411,29 @@ "node": ">= 0.4" } }, + "node_modules/ioredis": { + "version": "5.4.1", + "resolved": "https://registry.npmjs.org/ioredis/-/ioredis-5.4.1.tgz", + "integrity": "sha512-2YZsvl7jopIa1gaePkeMtd9rAcSjOOjPtpcLlOeusyO+XH2SK5ZcT+UCrElPP+WVIInh2TzeI4XW9ENaSLVVHA==", + "dependencies": { + "@ioredis/commands": "^1.1.1", + "cluster-key-slot": "^1.1.0", + "debug": "^4.3.4", + "denque": "^2.1.0", + "lodash.defaults": "^4.2.0", + "lodash.isarguments": "^3.1.0", + "redis-errors": "^1.2.0", + "redis-parser": "^3.0.0", + "standard-as-callback": "^2.1.0" + }, + "engines": { + "node": ">=12.22.0" + }, + "funding": { + "type": "opencollective", + "url": "https://opencollective.com/ioredis" + } + }, "node_modules/ipaddr.js": { "version": "1.9.1", "resolved": "https://registry.npmjs.org/ipaddr.js/-/ipaddr.js-1.9.1.tgz", @@ -14983,11 +15028,21 @@ "integrity": "sha512-FT1yDzDYEoYWhnSGnpE/4Kj1fLZkDFyqRb7fNt6FdYOSxlUWAtp42Eh6Wb0rGIv/m9Bgo7x4GhQbm5Ys4SG5ow==", "dev": true }, + "node_modules/lodash.defaults": { + "version": "4.2.0", + "resolved": "https://registry.npmjs.org/lodash.defaults/-/lodash.defaults-4.2.0.tgz", + "integrity": "sha512-qjxPLHd3r5DnsdGacqOMU6pb/avJzdh9tFX2ymgoZE27BmjXrNy/y4LoaiTeAb+O3gL8AfpJGtqfX/ae2leYYQ==" + }, "node_modules/lodash.get": { "version": "4.4.2", "resolved": "https://registry.npmjs.org/lodash.get/-/lodash.get-4.4.2.tgz", "integrity": "sha512-z+Uw/vLuy6gQe8cfaFWD7p0wVv8fJl3mbzXh33RS+0oW2wvUqiRXiQ69gLWSLpgB5/6sU+r6BlQR0MBILadqTQ==" }, + "node_modules/lodash.isarguments": { + "version": "3.1.0", + "resolved": "https://registry.npmjs.org/lodash.isarguments/-/lodash.isarguments-3.1.0.tgz", + "integrity": "sha512-chi4NHZlZqZD18a0imDHnZPrDeBbTtVN7GXMwuGdRH9qotxAjYs3aVLKc7zNOG9eddR5Ksd8rvFEBc9SsggPpg==" + }, "node_modules/lodash.isequal": { "version": "4.5.0", "resolved": "https://registry.npmjs.org/lodash.isequal/-/lodash.isequal-4.5.0.tgz", @@ -17231,6 +17286,25 @@ "node": ">=8" } }, + "node_modules/redis-errors": { + "version": "1.2.0", + "resolved": "https://registry.npmjs.org/redis-errors/-/redis-errors-1.2.0.tgz", + "integrity": "sha512-1qny3OExCf0UvUV/5wpYKf2YwPcOqXzkwKKSmKHiE6ZMQs5heeE/c8eXK+PNllPvmjgAbfnsbpkGZWy8cBpn9w==", + "engines": { + "node": ">=4" + } + }, + "node_modules/redis-parser": { + "version": "3.0.0", + "resolved": "https://registry.npmjs.org/redis-parser/-/redis-parser-3.0.0.tgz", + "integrity": "sha512-DJnGAeenTdpMEH6uAJRK/uiyEIH9WVsUmoLwzudwGJUwZPp80PDBWPHXSAGNPwNvIXAbe7MSUB1zQFugFml66A==", + "dependencies": { + "redis-errors": "^1.0.0" + }, + "engines": { + "node": ">=4" + } + }, "node_modules/reflect-metadata": { "version": "0.1.13", "resolved": "https://registry.npmjs.org/reflect-metadata/-/reflect-metadata-0.1.13.tgz", @@ -17926,6 +18000,11 @@ "node": ">=8" } }, + "node_modules/standard-as-callback": { + "version": "2.1.0", + "resolved": "https://registry.npmjs.org/standard-as-callback/-/standard-as-callback-2.1.0.tgz", + "integrity": "sha512-qoRRSyROncaz1z0mvYqIE4lCd9p2R90i6GxW3uZv5ucSu8tU7B5HXUP1gG8pVZsYNVaXjk8ClXHPttLyxAL48A==" + }, "node_modules/standard-version": { "version": "9.5.0", "resolved": "https://registry.npmjs.org/standard-version/-/standard-version-9.5.0.tgz", @@ -21658,6 +21737,11 @@ "integrity": "sha512-H9XAx3hc0BQHY6l+IFSWHDySypcXsvsuLhgYLUGywmJ5pswRVQJUHpOsobnLYp2ZUaUlKiKDrgWWhosOwAEM8Q==", "dev": true }, + "@ioredis/commands": { + "version": "1.2.0", + "resolved": "https://registry.npmjs.org/@ioredis/commands/-/commands-1.2.0.tgz", + "integrity": "sha512-Sx1pU8EM64o2BrqNpEO1CNLtKQwyhuXuqyfH7oGKCk+1a33d2r5saW8zNwm3j6BTExtjrv2BxTgzzkMwts6vGg==" + }, "@isaacs/cliui": { "version": "8.0.2", "resolved": "https://registry.npmjs.org/@isaacs/cliui/-/cliui-8.0.2.tgz", @@ -25891,6 +25975,11 @@ "integrity": "sha512-JQHZ2QMW6l3aH/j6xCqQThY/9OH4D/9ls34cgkUBiEeocRTU04tHfKPBsUK1PqZCUQM7GiA0IIXJSuXHI64Kbg==", "dev": true }, + "cluster-key-slot": { + "version": "1.1.2", + "resolved": "https://registry.npmjs.org/cluster-key-slot/-/cluster-key-slot-1.1.2.tgz", + "integrity": "sha512-RMr0FhtfXemyinomL4hrWcYJxmX6deFdCxpJzhDttxgO1+bcCnkk+9drydLVDmAMG7NE6aN/fl4F7ucU/90gAA==" + }, "co": { "version": "4.6.0", "resolved": "https://registry.npmjs.org/co/-/co-4.6.0.tgz", @@ -26773,6 +26862,11 @@ "integrity": "sha512-ZySD7Nf91aLB0RxL4KGrKHBXl7Eds1DAmEdcoVawXnLD7SDhpNgtuII2aAkg7a7QS41jxPSZ17p4VdGnMHk3MQ==", "dev": true }, + "denque": { + "version": "2.1.0", + "resolved": "https://registry.npmjs.org/denque/-/denque-2.1.0.tgz", + "integrity": "sha512-HVQE3AAb/pxF8fQAoiqpvg9i3evqug3hoiwakOyZAwJm+6vZehbkYXZ0l4JxS+I3QxM97v5aaRNhj8v5oBhekw==" + }, "depd": { "version": "2.0.0", "resolved": "https://registry.npmjs.org/depd/-/depd-2.0.0.tgz", @@ -28802,6 +28896,22 @@ "side-channel": "^1.0.4" } }, + "ioredis": { + "version": "5.4.1", + "resolved": "https://registry.npmjs.org/ioredis/-/ioredis-5.4.1.tgz", + "integrity": "sha512-2YZsvl7jopIa1gaePkeMtd9rAcSjOOjPtpcLlOeusyO+XH2SK5ZcT+UCrElPP+WVIInh2TzeI4XW9ENaSLVVHA==", + "requires": { + "@ioredis/commands": "^1.1.1", + "cluster-key-slot": "^1.1.0", + "debug": "^4.3.4", + "denque": "^2.1.0", + "lodash.defaults": "^4.2.0", + "lodash.isarguments": "^3.1.0", + "redis-errors": "^1.2.0", + "redis-parser": "^3.0.0", + "standard-as-callback": "^2.1.0" + } + }, "ipaddr.js": { "version": "1.9.1", "resolved": "https://registry.npmjs.org/ipaddr.js/-/ipaddr.js-1.9.1.tgz", @@ -30729,11 +30839,21 @@ "integrity": "sha512-FT1yDzDYEoYWhnSGnpE/4Kj1fLZkDFyqRb7fNt6FdYOSxlUWAtp42Eh6Wb0rGIv/m9Bgo7x4GhQbm5Ys4SG5ow==", "dev": true }, + "lodash.defaults": { + "version": "4.2.0", + "resolved": "https://registry.npmjs.org/lodash.defaults/-/lodash.defaults-4.2.0.tgz", + "integrity": "sha512-qjxPLHd3r5DnsdGacqOMU6pb/avJzdh9tFX2ymgoZE27BmjXrNy/y4LoaiTeAb+O3gL8AfpJGtqfX/ae2leYYQ==" + }, "lodash.get": { "version": "4.4.2", "resolved": "https://registry.npmjs.org/lodash.get/-/lodash.get-4.4.2.tgz", "integrity": "sha512-z+Uw/vLuy6gQe8cfaFWD7p0wVv8fJl3mbzXh33RS+0oW2wvUqiRXiQ69gLWSLpgB5/6sU+r6BlQR0MBILadqTQ==" }, + "lodash.isarguments": { + "version": "3.1.0", + "resolved": "https://registry.npmjs.org/lodash.isarguments/-/lodash.isarguments-3.1.0.tgz", + "integrity": "sha512-chi4NHZlZqZD18a0imDHnZPrDeBbTtVN7GXMwuGdRH9qotxAjYs3aVLKc7zNOG9eddR5Ksd8rvFEBc9SsggPpg==" + }, "lodash.isequal": { "version": "4.5.0", "resolved": "https://registry.npmjs.org/lodash.isequal/-/lodash.isequal-4.5.0.tgz", @@ -32456,6 +32576,19 @@ "strip-indent": "^3.0.0" } }, + "redis-errors": { + "version": "1.2.0", + "resolved": "https://registry.npmjs.org/redis-errors/-/redis-errors-1.2.0.tgz", + "integrity": "sha512-1qny3OExCf0UvUV/5wpYKf2YwPcOqXzkwKKSmKHiE6ZMQs5heeE/c8eXK+PNllPvmjgAbfnsbpkGZWy8cBpn9w==" + }, + "redis-parser": { + "version": "3.0.0", + "resolved": "https://registry.npmjs.org/redis-parser/-/redis-parser-3.0.0.tgz", + "integrity": "sha512-DJnGAeenTdpMEH6uAJRK/uiyEIH9WVsUmoLwzudwGJUwZPp80PDBWPHXSAGNPwNvIXAbe7MSUB1zQFugFml66A==", + "requires": { + "redis-errors": "^1.0.0" + } + }, "reflect-metadata": { "version": "0.1.13", "resolved": "https://registry.npmjs.org/reflect-metadata/-/reflect-metadata-0.1.13.tgz", @@ -32985,6 +33118,11 @@ } } }, + "standard-as-callback": { + "version": "2.1.0", + "resolved": "https://registry.npmjs.org/standard-as-callback/-/standard-as-callback-2.1.0.tgz", + "integrity": "sha512-qoRRSyROncaz1z0mvYqIE4lCd9p2R90i6GxW3uZv5ucSu8tU7B5HXUP1gG8pVZsYNVaXjk8ClXHPttLyxAL48A==" + }, "standard-version": { "version": "9.5.0", "resolved": "https://registry.npmjs.org/standard-version/-/standard-version-9.5.0.tgz", diff --git a/package.json b/package.json index eaf1c718..98795cf1 100644 --- a/package.json +++ b/package.json @@ -66,6 +66,7 @@ "fast-xml-parser": "^4.4.0", "geojson-validation": "^1.0.2", "http-status-codes": "^2.2.0", + "ioredis": "^5.4.1", "mgrs": "^2.0.0", "node-cron": "^3.0.3", "node-fetch-commonjs": "^3.3.2", diff --git a/src/common/constants.ts b/src/common/constants.ts index 5ab1d8b1..39f7b888 100644 --- a/src/common/constants.ts +++ b/src/common/constants.ts @@ -1,7 +1,9 @@ +import { hostname } from 'os'; import { readPackageJsonSync } from '@map-colonies/read-pkg'; export const SERVICE_NAME = readPackageJsonSync().name ?? 'unknown_service'; export const DEFAULT_SERVER_PORT = 80; +export const HOSTNAME = hostname(); export const IGNORED_OUTGOING_TRACE_ROUTES = [/^.*\/v1\/metrics.*$/]; export const IGNORED_INCOMING_TRACE_ROUTES = [/^.*\/docs.*$/]; @@ -19,5 +21,7 @@ export const SERVICES: Record = { export const ON_SIGNAL = Symbol('onSignal'); export const HEALTHCHECK = Symbol('healthcheck'); +export const REDIS_SYMBOL = Symbol('REDIS'); export const elasticConfigPath = 'db.elastic'; +export const REDIS_KEYS_SEPARATOR = ':'; diff --git a/src/common/interfaces.ts b/src/common/interfaces.ts index b8d65a2c..d140f1a0 100644 --- a/src/common/interfaces.ts +++ b/src/common/interfaces.ts @@ -49,6 +49,10 @@ export interface IApplication { hierarchy: number; viewbox: number; }; + hashKey: { + value?: string; + enabled: boolean; + }; sources?: { [key: string]: string; }; diff --git a/src/common/redis/domainFieldsRepository.ts b/src/common/redis/domainFieldsRepository.ts new file mode 100644 index 00000000..9cf9e3cc --- /dev/null +++ b/src/common/redis/domainFieldsRepository.ts @@ -0,0 +1,5 @@ +export const IDOMAIN_FIELDS_REPO_SYMBOL = Symbol('DOMAINFIELDSREPO'); + +export interface IDomainFieldsRepository { + getFields: (fields: string[]) => Promise<(string | null)[]>; +} diff --git a/src/common/redis/index.ts b/src/common/redis/index.ts new file mode 100644 index 00000000..fb3396dc --- /dev/null +++ b/src/common/redis/index.ts @@ -0,0 +1,33 @@ +import Redis, { RedisOptions } from 'ioredis'; +import { HOSTNAME } from '../constants'; + +const RETRY_DELAY_INCREASE = 50; +const RETRY_DELAY_TOP = 2000; +let redis: Redis; + +const retryFunction = (times: number): number => { + const delay = Math.min(times * RETRY_DELAY_INCREASE, RETRY_DELAY_TOP); + return delay; +}; + +export const createRedisConnection = async (redisOptions: RedisOptions): Promise => { + try { + redisOptions = { + ...redisOptions, + retryStrategy: retryFunction, + lazyConnect: true, + connectionName: HOSTNAME, + }; + + redis = new Redis(redisOptions); + await redis.connect(); + return redis; + } catch (err) { + redis.disconnect(); + let errorMessage = 'Redis connection failed'; + if (err instanceof Error) { + errorMessage += ` with the following error: ${err.message}`; + } + throw new Error(errorMessage); + } +}; diff --git a/src/common/redis/keys.ts b/src/common/redis/keys.ts new file mode 100644 index 00000000..e268631c --- /dev/null +++ b/src/common/redis/keys.ts @@ -0,0 +1,5 @@ +import { REDIS_KEYS_SEPARATOR } from '../../common/constants'; + +export const keyConstructor = (...vals: string[]): string => { + return vals.filter((v) => v).join(REDIS_KEYS_SEPARATOR); // drop undefined, null and empty string from a key +}; diff --git a/src/common/redis/redisManager.ts b/src/common/redis/redisManager.ts new file mode 100644 index 00000000..0ea22c07 --- /dev/null +++ b/src/common/redis/redisManager.ts @@ -0,0 +1,31 @@ +import Redis from 'ioredis'; +import { inject, injectable } from 'tsyringe'; +import { REDIS_SYMBOL, SERVICES } from '../constants'; +import { IApplication } from '../interfaces'; +import { IDomainFieldsRepository } from './domainFieldsRepository'; + +@injectable() +export class RedisManager implements IDomainFieldsRepository { + public getData: (fields: string[]) => Promise<(string | null)[]>; + + public constructor(@inject(REDIS_SYMBOL) private readonly redis: Redis, @inject(SERVICES.APPLICATION) appConfig: IApplication) { + const { value, enabled } = appConfig.hashKey; + if (enabled && value !== undefined) { + this.getData = async (fields: string[]): Promise<(string | null)[]> => { + return this.redis.hmget(value, ...fields); + }; + } else { + this.getData = async (fields: string[]): Promise<(string | null)[]> => { + return this.redis.mget(fields); + }; + } + } + + public async getFields(fields: string[]): Promise<(string | null)[]> { + try { + return await this.getData(fields); + } catch (e) { + throw new Error('redis: failed to fetch keys'); + } + } +} diff --git a/src/containerConfig.ts b/src/containerConfig.ts index e6e49d95..658dc442 100644 --- a/src/containerConfig.ts +++ b/src/containerConfig.ts @@ -1,4 +1,5 @@ import config from 'config'; +import Redis, { RedisOptions } from 'ioredis'; import { getOtelMixin } from '@map-colonies/telemetry'; import { DataSource } from 'typeorm'; import { instancePerContainerCachingFactory } from 'tsyringe'; @@ -6,10 +7,11 @@ import { trace, metrics as OtelMetrics } from '@opentelemetry/api'; import { DependencyContainer } from 'tsyringe/dist/typings/types'; import jsLogger, { LoggerOptions } from '@map-colonies/js-logger'; import { Metrics } from '@map-colonies/telemetry'; -import { SERVICES, SERVICE_NAME } from './common/constants'; +import { SERVICES, SERVICE_NAME, REDIS_SYMBOL } from './common/constants'; import { tracing } from './common/tracing'; import { InjectionObject, registerDependencies } from './common/dependencyRegistration'; import { elasticClientFactory, ElasticClients } from './common/elastic'; +import { RedisManager } from './common/redis/redisManager'; import { IApplication } from './common/interfaces'; import { TILE_REPOSITORY_SYMBOL, tileRepositoryFactory } from './control/tile/DAL/tileRepository'; import { TILE_ROUTER_SYMBOL, tileRouterFactory } from './control/tile/routes/tileRouter'; @@ -23,6 +25,8 @@ import { GEOTEXT_REPOSITORY_SYMBOL, geotextRepositoryFactory } from './location/ import { GEOTEXT_SEARCH_ROUTER_SYMBOL, geotextSearchRouterFactory } from './location/routes/locationRouter'; import { cronLoadTileLatLonDataFactory, cronLoadTileLatLonDataSymbol } from './latLon/DAL/latLonDAL'; import { ITEM_REPOSITORY_SYMBOL, itemRepositoryFactory } from './control/item/DAL/itemRepository'; +import { createRedisConnection } from './common/redis/index'; +import { IDOMAIN_FIELDS_REPO_SYMBOL } from './common/redis/domainFieldsRepository'; export interface RegisterOptions { override?: InjectionObject[]; @@ -41,6 +45,8 @@ export const registerExternalValues = async (options?: RegisterOptions): Promise const applicationConfig: IApplication = config.get('application'); + let redisConnection: Redis | undefined; + const dependencies: InjectionObject[] = [ { token: SERVICES.CONFIG, provider: { useValue: config } }, { token: SERVICES.LOGGER, provider: { useValue: logger } }, @@ -77,6 +83,29 @@ export const registerExternalValues = async (options?: RegisterOptions): Promise } }, }, + { + token: REDIS_SYMBOL, + provider: { + useFactory: instancePerContainerCachingFactory(async () => { + redisConnection = await createRedisConnection(config.get('db')); + + redisConnection.on('connect', () => { + logger.info(`redis client is connected.`); + }); + + redisConnection.on('error', (err: Error) => { + logger.error({ err: err, msg: 'redis client got an error' }); + }); + + redisConnection.on('reconnecting', (delay: number) => { + logger.info(`redis client reconnecting, next reconnection attemp in ${delay}ms`); + }); + container.register(SERVICES.LOGGER, { useValue: logger }); + container.register(IDOMAIN_FIELDS_REPO_SYMBOL, { useClass: RedisManager }); + return redisConnection; + }), + }, + }, { token: TILE_REPOSITORY_SYMBOL, provider: { useFactory: tileRepositoryFactory } }, { token: TILE_ROUTER_SYMBOL, provider: { useFactory: tileRouterFactory } }, { token: ITEM_REPOSITORY_SYMBOL, provider: { useFactory: itemRepositoryFactory } }, @@ -96,11 +125,22 @@ export const registerExternalValues = async (options?: RegisterOptions): Promise { token: 'onSignal', provider: { - useValue: { - useValue: async (): Promise => { - await Promise.all([tracing.stop(), metrics.stop()]); - }, - }, + useFactory: instancePerContainerCachingFactory(async (): Promise => { + const promises: Promise[] = [tracing.stop(), metrics.stop()]; + if (redisConnection !== undefined) { + redisConnection.disconnect(); + + const promisifyQuit = new Promise((resolve) => { + redisConnection = redisConnection as Redis; + redisConnection.once('end', () => { + resolve(); + }); + void redisConnection.quit(); + }); + promises.push(promisifyQuit); + } + await Promise.all(promises); + }), }, }, ]; From 5bb7d11999385f5a2cf63f8a3a4963961fdbdd1d Mon Sep 17 00:00:00 2001 From: NatalieShaked Date: Tue, 27 Aug 2024 14:46:59 +0300 Subject: [PATCH 02/20] test: added redis to tests --- tests/integration/control/item/item.spec.ts | 30 +++++++++++++++++-- tests/integration/control/route/route.spec.ts | 28 ++++++++++++++++- tests/integration/control/tile/tile.spec.ts | 28 ++++++++++++++++- tests/integration/location/location.spec.ts | 27 ++++++++++++++++- 4 files changed, 108 insertions(+), 5 deletions(-) diff --git a/tests/integration/control/item/item.spec.ts b/tests/integration/control/item/item.spec.ts index 39e3659b..01327f88 100644 --- a/tests/integration/control/item/item.spec.ts +++ b/tests/integration/control/item/item.spec.ts @@ -3,8 +3,12 @@ import jsLogger from '@map-colonies/js-logger'; import { trace } from '@opentelemetry/api'; import httpStatusCodes from 'http-status-codes'; import { DataSource } from 'typeorm'; +import config from 'config'; +import Redis, { RedisOptions } from 'ioredis'; +import { DependencyContainer } from 'tsyringe'; +import { Application } from 'express'; import { getApp } from '../../../../src/app'; -import { SERVICES } from '../../../../src/common/constants'; +import { REDIS_SYMBOL, SERVICES } from '../../../../src/common/constants'; import { GetItemsQueryParams } from '../../../../src/control/item/controllers/itemController'; import { Item } from '../../../../src/control/item/models/item'; import { ControlResponse } from '../../../../src/control/interfaces'; @@ -12,14 +16,30 @@ import { CommonRequestParameters, GeoContext, GeoContextMode } from '../../../.. import { LATLON_CUSTOM_REPOSITORY_SYMBOL } from '../../../../src/latLon/DAL/latLonRepository'; import { cronLoadTileLatLonDataSymbol } from '../../../../src/latLon/DAL/latLonDAL'; import { expectedResponse } from '../utils'; +import { RedisManager } from '../../../../src/common/redis/redisManager'; +import { IDOMAIN_FIELDS_REPO_SYMBOL } from '../../../../src/common/redis/domainFieldsRepository'; +import { createRedisConnection } from '../../../../src/common/redis'; import { ItemRequestSender } from './helpers/requestSender'; import { ITEM_1234, ITEM_1235, ITEM_1236 } from './mockObjects'; describe('/search/control/items', function () { let requestSender: ItemRequestSender; + let app: { app: Application; container?: DependencyContainer }; + let redisConnection: Redis; + + beforeAll(async function () { + redisConnection = await createRedisConnection(config.get('db')); + app = await getApp({ + override: [ + { token: REDIS_SYMBOL, provider: { useValue: redisConnection } }, + { token: IDOMAIN_FIELDS_REPO_SYMBOL, provider: { useClass: RedisManager } }, + ], + useChild: true, + }); + }); beforeEach(async function () { - const app = await getApp({ + app = await getApp({ override: [ { token: SERVICES.LOGGER, provider: { useValue: jsLogger({ enabled: false }) } }, { token: SERVICES.TRACER, provider: { useValue: trace.getTracer('testTracer') } }, @@ -34,6 +54,12 @@ describe('/search/control/items', function () { requestSender = new ItemRequestSender(app.app); }); + afterAll(async function () { + if (!['end'].includes(redisConnection.status)) { + await redisConnection.quit(); + } + }); + describe('Happy Path', function () { it('should return 200 status code and items', async function () { const requestParams: GetItemsQueryParams = { command_name: '123', limit: 5, disable_fuzziness: false }; diff --git a/tests/integration/control/route/route.spec.ts b/tests/integration/control/route/route.spec.ts index fe95f22e..1dd5b394 100644 --- a/tests/integration/control/route/route.spec.ts +++ b/tests/integration/control/route/route.spec.ts @@ -3,20 +3,40 @@ import jsLogger from '@map-colonies/js-logger'; import { trace } from '@opentelemetry/api'; import httpStatusCodes from 'http-status-codes'; import { DataSource } from 'typeorm'; +import config from 'config'; +import Redis, { RedisOptions } from 'ioredis'; +import { DependencyContainer } from 'tsyringe'; +import { Application } from 'express'; import { getApp } from '../../../../src/app'; -import { SERVICES } from '../../../../src/common/constants'; +import { REDIS_SYMBOL, SERVICES } from '../../../../src/common/constants'; import { GetRoutesQueryParams } from '../../../../src/control/route/controllers/routeController'; import { Route } from '../../../../src/control/route/models/route'; import { ControlResponse } from '../../../../src/control/interfaces'; import { CommonRequestParameters, GeoContext, GeoContextMode } from '../../../../src/common/interfaces'; import { LATLON_CUSTOM_REPOSITORY_SYMBOL } from '../../../../src/latLon/DAL/latLonRepository'; import { cronLoadTileLatLonDataSymbol } from '../../../../src/latLon/DAL/latLonDAL'; +import { RedisManager } from '../../../../src/common/redis/redisManager'; +import { IDOMAIN_FIELDS_REPO_SYMBOL } from '../../../../src/common/redis/domainFieldsRepository'; +import { createRedisConnection } from '../../../../src/common/redis'; import { expectedResponse } from '../utils'; import { RouteRequestSender } from './helpers/requestSender'; import { ROUTE_VIA_CAMILLUCCIA_A, ROUTE_VIA_CAMILLUCCIA_B, CONTROL_POINT_OLIMPIADE_111, CONTROL_POINT_OLIMPIADE_112 } from './mockObjects'; describe('/search/control/route', function () { let requestSender: RouteRequestSender; + let app: { app: Application; container?: DependencyContainer }; + let redisConnection: Redis; + + beforeAll(async function () { + redisConnection = await createRedisConnection(config.get('db')); + app = await getApp({ + override: [ + { token: REDIS_SYMBOL, provider: { useValue: redisConnection } }, + { token: IDOMAIN_FIELDS_REPO_SYMBOL, provider: { useClass: RedisManager } }, + ], + useChild: true, + }); + }); beforeEach(async function () { const app = await getApp({ @@ -34,6 +54,12 @@ describe('/search/control/route', function () { requestSender = new RouteRequestSender(app.app); }); + afterAll(async function () { + if (!['end'].includes(redisConnection.status)) { + await redisConnection.quit(); + } + }); + describe('Happy Path', function () { it('should return 200 status code and routes', async function () { const requestParams: GetRoutesQueryParams = { command_name: 'via camilluccia', limit: 5, disable_fuzziness: false }; diff --git a/tests/integration/control/tile/tile.spec.ts b/tests/integration/control/tile/tile.spec.ts index 44685fcc..6564fd99 100644 --- a/tests/integration/control/tile/tile.spec.ts +++ b/tests/integration/control/tile/tile.spec.ts @@ -3,13 +3,20 @@ import jsLogger from '@map-colonies/js-logger'; import { trace } from '@opentelemetry/api'; import httpStatusCodes from 'http-status-codes'; import { DataSource } from 'typeorm'; +import config from 'config'; +import Redis, { RedisOptions } from 'ioredis'; +import { DependencyContainer } from 'tsyringe'; +import { Application } from 'express'; import { getApp } from '../../../../src/app'; -import { SERVICES } from '../../../../src/common/constants'; +import { REDIS_SYMBOL, SERVICES } from '../../../../src/common/constants'; import { GetTilesQueryParams } from '../../../../src/control/tile/controllers/tileController'; import { Tile } from '../../../../src/control/tile/models/tile'; import { ControlResponse } from '../../../../src/control/interfaces'; import { CommonRequestParameters, GeoContext, GeoContextMode } from '../../../../src/common/interfaces'; import { LATLON_CUSTOM_REPOSITORY_SYMBOL } from '../../../../src/latLon/DAL/latLonRepository'; +import { RedisManager } from '../../../../src/common/redis/redisManager'; +import { IDOMAIN_FIELDS_REPO_SYMBOL } from '../../../../src/common/redis/domainFieldsRepository'; +import { createRedisConnection } from '../../../../src/common/redis'; import { cronLoadTileLatLonDataSymbol } from '../../../../src/latLon/DAL/latLonDAL'; import { expectedResponse } from '../utils'; import { TileRequestSender } from './helpers/requestSender'; @@ -17,6 +24,19 @@ import { RIC_TILE, RIT_TILE, SUB_TILE_65, SUB_TILE_66 } from './mockObjects'; describe('/search/control/tiles', function () { let requestSender: TileRequestSender; + let app: { app: Application; container?: DependencyContainer }; + let redisConnection: Redis; + + beforeAll(async function () { + redisConnection = await createRedisConnection(config.get('db')); + app = await getApp({ + override: [ + { token: REDIS_SYMBOL, provider: { useValue: redisConnection } }, + { token: IDOMAIN_FIELDS_REPO_SYMBOL, provider: { useClass: RedisManager } }, + ], + useChild: true, + }); + }); beforeEach(async function () { const app = await getApp({ @@ -34,6 +54,12 @@ describe('/search/control/tiles', function () { requestSender = new TileRequestSender(app.app); }); + afterAll(async function () { + if (!['end'].includes(redisConnection.status)) { + await redisConnection.quit(); + } + }); + describe('Happy Path', function () { it('should return 200 status code and tiles', async function () { const requestParams: GetTilesQueryParams = { tile: 'RIT', limit: 5, disable_fuzziness: false }; diff --git a/tests/integration/location/location.spec.ts b/tests/integration/location/location.spec.ts index e61cdc0b..385c4f59 100644 --- a/tests/integration/location/location.spec.ts +++ b/tests/integration/location/location.spec.ts @@ -3,12 +3,18 @@ import config from 'config'; import jsLogger from '@map-colonies/js-logger'; import { trace } from '@opentelemetry/api'; import httpStatusCodes from 'http-status-codes'; +import Redis, { RedisOptions } from 'ioredis'; +import { DependencyContainer } from 'tsyringe'; +import { Application } from 'express'; import { DataSource } from 'typeorm'; import nock, { Body } from 'nock'; import { getApp } from '../../../src/app'; -import { SERVICES } from '../../../src/common/constants'; +import { REDIS_SYMBOL, SERVICES } from '../../../src/common/constants'; import { LATLON_CUSTOM_REPOSITORY_SYMBOL } from '../../../src/latLon/DAL/latLonRepository'; import { cronLoadTileLatLonDataSymbol } from '../../../src/latLon/DAL/latLonDAL'; +import { RedisManager } from '../../../src/common/redis/redisManager'; +import { IDOMAIN_FIELDS_REPO_SYMBOL } from '../../../src/common/redis/domainFieldsRepository'; +import { createRedisConnection } from '../../../src/common/redis'; import { GetGeotextSearchParams, QueryResult } from '../../../src/location/interfaces'; import { GeoContextMode, IApplication } from '../../../src/common/interfaces'; import { LocationRequestSender } from './helpers/requestSender'; @@ -29,6 +35,19 @@ import { expectedResponse, hierarchiesWithAnyWieght } from './utils'; describe('/search/control/tiles', function () { let requestSender: LocationRequestSender; + let app: { app: Application; container?: DependencyContainer }; + let redisConnection: Redis; + + beforeAll(async function () { + redisConnection = await createRedisConnection(config.get('db')); + app = await getApp({ + override: [ + { token: REDIS_SYMBOL, provider: { useValue: redisConnection } }, + { token: IDOMAIN_FIELDS_REPO_SYMBOL, provider: { useClass: RedisManager } }, + ], + useChild: true, + }); + }); beforeEach(async function () { const app = await getApp({ @@ -46,6 +65,12 @@ describe('/search/control/tiles', function () { requestSender = new LocationRequestSender(app.app); }); + afterAll(async function () { + if (!['end'].includes(redisConnection.status)) { + await redisConnection.quit(); + } + }); + describe('Happy Path', function () { it('should return 200 status code and airports', async function () { const requestParams: GetGeotextSearchParams = { query: 'airport', limit: 5, disable_fuzziness: true }; From 3536e6e38e50029029aef6792056521ac3b9f4da Mon Sep 17 00:00:00 2001 From: NatalieShaked Date: Thu, 29 Aug 2024 16:13:03 +0300 Subject: [PATCH 03/20] fix: fixed redis configuration --- config/custom-environment-variables.json | 17 +- config/default.json | 11 +- config/test.json | 11 +- helm/templates/configmap.yaml | 13 ++ helm/templates/deployment.yaml | 10 ++ helm/values.yaml | 13 ++ package-lock.json | 182 +++++++++++++++++++++ package.json | 3 + src/common/constants.ts | 2 +- src/common/errors.ts | 2 + src/common/interfaces.ts | 8 + src/common/redis/domainFieldsRepository.ts | 5 - src/common/redis/index.ts | 79 ++++++--- src/common/redis/keys.ts | 5 - src/common/redis/redisManager.ts | 31 ---- src/common/utils.ts | 14 ++ src/containerConfig.ts | 60 +++---- src/serverBuilder.ts | 8 +- 18 files changed, 354 insertions(+), 120 deletions(-) delete mode 100644 src/common/redis/domainFieldsRepository.ts delete mode 100644 src/common/redis/keys.ts delete mode 100644 src/common/redis/redisManager.ts diff --git a/config/custom-environment-variables.json b/config/custom-environment-variables.json index c6ef51dc..d341e4ca 100644 --- a/config/custom-environment-variables.json +++ b/config/custom-environment-variables.json @@ -93,14 +93,19 @@ "__name": "REDIS_PORT", "__format": "number" }, - "db": { - "__name": "REDIS_INDEX", - "__format": "number" - }, "username": "REDIS_USERNAME", "password": "REDIS_PASSWORD", - "maxRetriesPerRequest": { - "__name": "REDIS_MAX_RETRIES_PER_REQUEST", + "enableSslAuth": { + "__name": "REDIS_ENABLE_SSL_AUTH", + "__format": "boolean" + }, + "sslPaths": { + "ca": "REDIS_CA_PATH", + "key": "REDIS_KEY_PATH", + "cert": "REDIS_CERT_PATH" + }, + "database": { + "__name": "REDIS_DATABASE", "__format": "number" } } diff --git a/config/default.json b/config/default.json index 4e0dd4ad..1a507876 100644 --- a/config/default.json +++ b/config/default.json @@ -75,8 +75,15 @@ "redis": { "host": "localhost", "port": 6379, - "db": 0, - "maxRetriesPerRequest": 1 + "username": "", + "password": "", + "enableSslAuth": false, + "sslPaths": { + "ca": "", + "key": "", + "cert": "" + }, + "database": 0 } }, "application": { diff --git a/config/test.json b/config/test.json index f267cd35..dc66d247 100644 --- a/config/test.json +++ b/config/test.json @@ -49,8 +49,15 @@ "redis": { "host": "localhost", "port": 6379, - "db": 0, - "maxRetriesPerRequest": 1 + "username": "", + "password": "", + "enableSslAuth": false, + "sslPaths": { + "ca": "", + "key": "", + "cert": "" + }, + "database": 0 } }, "application": { diff --git a/helm/templates/configmap.yaml b/helm/templates/configmap.yaml index 69202f7f..b83572d1 100644 --- a/helm/templates/configmap.yaml +++ b/helm/templates/configmap.yaml @@ -18,5 +18,18 @@ data: TELEMETRY_METRICS_ENABLED: 'true' TELEMETRY_METRICS_URL: {{ $metricsUrl }} {{ end }} + {{- with .Values.redisConfig }} + REDIS_HOST: {{ .host }} + REDIS_DATABASE: {{ .database | quote}} + REDIS_PORT: {{ .port | quote }} + {{- if .sslAuth.enabled }} + REDIS_ENABLE_SSL_AUTH: "true" + REDIS_CERT_PATH: /tmp/certs/{{ .sslAuth.certFileName }} + REDIS_KEY_PATH: /tmp/certs/{{ .sslAuth.keyFileName }} + REDIS_CA_PATH: /tmp/certs/{{ .sslAuth.caFileName }} + {{- else }} + REDIS_ENABLE_SSL_AUTH: "false" + {{- end }} + {{- end }} npm_config_cache: /tmp/ {{- end }} diff --git a/helm/templates/deployment.yaml b/helm/templates/deployment.yaml index 007efa2a..5484cfd7 100644 --- a/helm/templates/deployment.yaml +++ b/helm/templates/deployment.yaml @@ -53,6 +53,11 @@ spec: name: root-ca subPath: {{ quote .Values.caKey }} {{- end }} + {{- if .Values.redisConfig.sslAuth.enabled }} + - name: cert-conf + mountPath: /tmp/certs + readOnly: true + {{- end }} {{- if .Values.extraVolumeMounts -}} {{ toYaml .Values.extraVolumeMounts | nindent 12 }} {{- end }} @@ -109,6 +114,11 @@ spec: secret: secretName: {{ .Values.caSecretName }} {{- end }} + {{- if .Values.redisConfig.sslAuth.enabled }} + - name: cert-conf + secret: + secretName: {{ .Values.redisConfig.sslAuth.secretName }} + {{- end }} {{- if .Values.extraVolumes -}} {{ tpl (toYaml .Values.extraVolumes) . | nindent 8 }} {{- end }} diff --git a/helm/values.yaml b/helm/values.yaml index 25fd7688..3adb3d5b 100644 --- a/helm/values.yaml +++ b/helm/values.yaml @@ -71,6 +71,19 @@ env: enabled: false url: http://localhost:55681/v1/metrics +redisConfig: + host: localhost + username: "" + password: "" + database: 0 + port: 6379 + sslAuth: + enabled: false + secretName: secret-name + certFileName: postgresql.crt + keyFileName: postgresql.key + caFileName: root.crt + resources: enabled: true value: diff --git a/package-lock.json b/package-lock.json index 527fc462..1b6ec40d 100644 --- a/package-lock.json +++ b/package-lock.json @@ -12,6 +12,8 @@ "dependencies": { "@elastic/elasticsearch": "^8.13.1", "@godaddy/terminus": "^4.12.1", + "@map-colonies/cleanup-registry": "^1.1.0", + "@map-colonies/detiler-common": "^1.0.0", "@map-colonies/error-express-handler": "^2.1.0", "@map-colonies/express-access-log-middleware": "^2.0.1", "@map-colonies/js-logger": "^1.0.1", @@ -36,6 +38,7 @@ "node-fetch-commonjs": "^3.3.2", "pg": "^8.11.5", "proj4": "^2.11.0", + "redis": "^4.7.0", "reflect-metadata": "^0.1.13", "tsyringe": "^4.8.0", "typeorm": "^0.3.20", @@ -3790,6 +3793,20 @@ "resolved": "https://registry.npmjs.org/@jsdevtools/ono/-/ono-7.1.3.tgz", "integrity": "sha512-4JQNk+3mVzK3xh2rqd6RB4J46qUR19azEHBneZyTZM+c456qOrbbM/5xcR8huNCCcbVt7+UmizG6GuUvPvKUYg==" }, + "node_modules/@map-colonies/cleanup-registry": { + "version": "1.1.0", + "resolved": "https://registry.npmjs.org/@map-colonies/cleanup-registry/-/cleanup-registry-1.1.0.tgz", + "integrity": "sha512-/lhIGklWPZSY37JwzhFJEtBlqwXDRhHSeCBwpPaGMxpycpt5ZRIVQxUt6Og4mt6c5GoRoX9dZYHY0qV3UMGvtQ==", + "dependencies": { + "nanoid": "^3.3.4", + "tiny-typed-emitter": "^2.1.0" + } + }, + "node_modules/@map-colonies/detiler-common": { + "version": "1.0.0", + "resolved": "https://registry.npmjs.org/@map-colonies/detiler-common/-/detiler-common-1.0.0.tgz", + "integrity": "sha512-5gxowEjHBD9CcgqonYQGyarIXRElYnhO4qO/ffxN3IFieWyUWUuKFhc+U6rnvy53TN0UorPfTuOrCO85W9ZsKw==" + }, "node_modules/@map-colonies/error-express-handler": { "version": "2.1.0", "resolved": "https://registry.npmjs.org/@map-colonies/error-express-handler/-/error-express-handler-2.1.0.tgz", @@ -6288,6 +6305,59 @@ "resolved": "https://registry.npmjs.org/@protobufjs/utf8/-/utf8-1.1.0.tgz", "integrity": "sha512-Vvn3zZrhQZkkBE8LSuW3em98c0FwgO4nxzv6OdSxPKJIEKY2bGbHn+mhGIPerzI4twdxaP8/0+06HBpwf345Lw==" }, + "node_modules/@redis/bloom": { + "version": "1.2.0", + "resolved": "https://registry.npmjs.org/@redis/bloom/-/bloom-1.2.0.tgz", + "integrity": "sha512-HG2DFjYKbpNmVXsa0keLHp/3leGJz1mjh09f2RLGGLQZzSHpkmZWuwJbAvo3QcRY8p80m5+ZdXZdYOSBLlp7Cg==", + "peerDependencies": { + "@redis/client": "^1.0.0" + } + }, + "node_modules/@redis/client": { + "version": "1.6.0", + "resolved": "https://registry.npmjs.org/@redis/client/-/client-1.6.0.tgz", + "integrity": "sha512-aR0uffYI700OEEH4gYnitAnv3vzVGXCFvYfdpu/CJKvk4pHfLPEy/JSZyrpQ+15WhXe1yJRXLtfQ84s4mEXnPg==", + "dependencies": { + "cluster-key-slot": "1.1.2", + "generic-pool": "3.9.0", + "yallist": "4.0.0" + }, + "engines": { + "node": ">=14" + } + }, + "node_modules/@redis/graph": { + "version": "1.1.1", + "resolved": "https://registry.npmjs.org/@redis/graph/-/graph-1.1.1.tgz", + "integrity": "sha512-FEMTcTHZozZciLRl6GiiIB4zGm5z5F3F6a6FZCyrfxdKOhFlGkiAqlexWMBzCi4DcRoyiOsuLfW+cjlGWyExOw==", + "peerDependencies": { + "@redis/client": "^1.0.0" + } + }, + "node_modules/@redis/json": { + "version": "1.0.7", + "resolved": "https://registry.npmjs.org/@redis/json/-/json-1.0.7.tgz", + "integrity": "sha512-6UyXfjVaTBTJtKNG4/9Z8PSpKE6XgSyEb8iwaqDcy+uKrd/DGYHTWkUdnQDyzm727V7p21WUMhsqz5oy65kPcQ==", + "peerDependencies": { + "@redis/client": "^1.0.0" + } + }, + "node_modules/@redis/search": { + "version": "1.2.0", + "resolved": "https://registry.npmjs.org/@redis/search/-/search-1.2.0.tgz", + "integrity": "sha512-tYoDBbtqOVigEDMAcTGsRlMycIIjwMCgD8eR2t0NANeQmgK/lvxNAvYyb6bZDD4frHRhIHkJu2TBRvB0ERkOmw==", + "peerDependencies": { + "@redis/client": "^1.0.0" + } + }, + "node_modules/@redis/time-series": { + "version": "1.1.0", + "resolved": "https://registry.npmjs.org/@redis/time-series/-/time-series-1.1.0.tgz", + "integrity": "sha512-c1Q99M5ljsIuc4YdaCwfUEXsofakb9c8+Zse2qxTadu8TalLXuAESzLvFAvNVbkmSlvlzIQOLpBCmWI9wTOt+g==", + "peerDependencies": { + "@redis/client": "^1.0.0" + } + }, "node_modules/@redocly/ajv": { "version": "8.6.4", "resolved": "https://registry.npmjs.org/@redocly/ajv/-/ajv-8.6.4.tgz", @@ -15573,6 +15643,23 @@ "thenify-all": "^1.0.0" } }, + "node_modules/nanoid": { + "version": "3.3.7", + "resolved": "https://registry.npmjs.org/nanoid/-/nanoid-3.3.7.tgz", + "integrity": "sha512-eSRppjcPIatRIMC1U6UngP8XFcz8MQWGQdt1MTBQ7NaAmvXDfvNxbvWV3x2y6CdEUciCSsDHDQZbhYaB8QEo2g==", + "funding": [ + { + "type": "github", + "url": "https://github.com/sponsors/ai" + } + ], + "bin": { + "nanoid": "bin/nanoid.cjs" + }, + "engines": { + "node": "^10 || ^12 || ^13.7 || ^14 || >=15.0.1" + } + }, "node_modules/natural-compare": { "version": "1.4.0", "resolved": "https://registry.npmjs.org/natural-compare/-/natural-compare-1.4.0.tgz", @@ -17286,6 +17373,19 @@ "node": ">=8" } }, + "node_modules/redis": { + "version": "4.7.0", + "resolved": "https://registry.npmjs.org/redis/-/redis-4.7.0.tgz", + "integrity": "sha512-zvmkHEAdGMn+hMRXuMBtu4Vo5P6rHQjLoHftu+lBqq8ZTA3RCVC/WzD790bkKKiNFp7d5/9PcSD19fJyyRvOdQ==", + "dependencies": { + "@redis/bloom": "1.2.0", + "@redis/client": "1.6.0", + "@redis/graph": "1.1.1", + "@redis/json": "1.0.7", + "@redis/search": "1.2.0", + "@redis/time-series": "1.1.0" + } + }, "node_modules/redis-errors": { "version": "1.2.0", "resolved": "https://registry.npmjs.org/redis-errors/-/redis-errors-1.2.0.tgz", @@ -18512,6 +18612,11 @@ "node": ">=6" } }, + "node_modules/tiny-typed-emitter": { + "version": "2.1.0", + "resolved": "https://registry.npmjs.org/tiny-typed-emitter/-/tiny-typed-emitter-2.1.0.tgz", + "integrity": "sha512-qVtvMxeXbVej0cQWKqVSSAHmKZEHAvxdF8HEUBFWts8h+xEo5m/lEiPakuyZ3BnCBjOD8i24kzNOiOLLgsSxhA==" + }, "node_modules/tmp": { "version": "0.0.33", "resolved": "https://registry.npmjs.org/tmp/-/tmp-0.0.33.tgz", @@ -22370,6 +22475,20 @@ "resolved": "https://registry.npmjs.org/@jsdevtools/ono/-/ono-7.1.3.tgz", "integrity": "sha512-4JQNk+3mVzK3xh2rqd6RB4J46qUR19azEHBneZyTZM+c456qOrbbM/5xcR8huNCCcbVt7+UmizG6GuUvPvKUYg==" }, + "@map-colonies/cleanup-registry": { + "version": "1.1.0", + "resolved": "https://registry.npmjs.org/@map-colonies/cleanup-registry/-/cleanup-registry-1.1.0.tgz", + "integrity": "sha512-/lhIGklWPZSY37JwzhFJEtBlqwXDRhHSeCBwpPaGMxpycpt5ZRIVQxUt6Og4mt6c5GoRoX9dZYHY0qV3UMGvtQ==", + "requires": { + "nanoid": "^3.3.4", + "tiny-typed-emitter": "^2.1.0" + } + }, + "@map-colonies/detiler-common": { + "version": "1.0.0", + "resolved": "https://registry.npmjs.org/@map-colonies/detiler-common/-/detiler-common-1.0.0.tgz", + "integrity": "sha512-5gxowEjHBD9CcgqonYQGyarIXRElYnhO4qO/ffxN3IFieWyUWUuKFhc+U6rnvy53TN0UorPfTuOrCO85W9ZsKw==" + }, "@map-colonies/error-express-handler": { "version": "2.1.0", "resolved": "https://registry.npmjs.org/@map-colonies/error-express-handler/-/error-express-handler-2.1.0.tgz", @@ -24161,6 +24280,46 @@ "resolved": "https://registry.npmjs.org/@protobufjs/utf8/-/utf8-1.1.0.tgz", "integrity": "sha512-Vvn3zZrhQZkkBE8LSuW3em98c0FwgO4nxzv6OdSxPKJIEKY2bGbHn+mhGIPerzI4twdxaP8/0+06HBpwf345Lw==" }, + "@redis/bloom": { + "version": "1.2.0", + "resolved": "https://registry.npmjs.org/@redis/bloom/-/bloom-1.2.0.tgz", + "integrity": "sha512-HG2DFjYKbpNmVXsa0keLHp/3leGJz1mjh09f2RLGGLQZzSHpkmZWuwJbAvo3QcRY8p80m5+ZdXZdYOSBLlp7Cg==", + "requires": {} + }, + "@redis/client": { + "version": "1.6.0", + "resolved": "https://registry.npmjs.org/@redis/client/-/client-1.6.0.tgz", + "integrity": "sha512-aR0uffYI700OEEH4gYnitAnv3vzVGXCFvYfdpu/CJKvk4pHfLPEy/JSZyrpQ+15WhXe1yJRXLtfQ84s4mEXnPg==", + "requires": { + "cluster-key-slot": "1.1.2", + "generic-pool": "3.9.0", + "yallist": "4.0.0" + } + }, + "@redis/graph": { + "version": "1.1.1", + "resolved": "https://registry.npmjs.org/@redis/graph/-/graph-1.1.1.tgz", + "integrity": "sha512-FEMTcTHZozZciLRl6GiiIB4zGm5z5F3F6a6FZCyrfxdKOhFlGkiAqlexWMBzCi4DcRoyiOsuLfW+cjlGWyExOw==", + "requires": {} + }, + "@redis/json": { + "version": "1.0.7", + "resolved": "https://registry.npmjs.org/@redis/json/-/json-1.0.7.tgz", + "integrity": "sha512-6UyXfjVaTBTJtKNG4/9Z8PSpKE6XgSyEb8iwaqDcy+uKrd/DGYHTWkUdnQDyzm727V7p21WUMhsqz5oy65kPcQ==", + "requires": {} + }, + "@redis/search": { + "version": "1.2.0", + "resolved": "https://registry.npmjs.org/@redis/search/-/search-1.2.0.tgz", + "integrity": "sha512-tYoDBbtqOVigEDMAcTGsRlMycIIjwMCgD8eR2t0NANeQmgK/lvxNAvYyb6bZDD4frHRhIHkJu2TBRvB0ERkOmw==", + "requires": {} + }, + "@redis/time-series": { + "version": "1.1.0", + "resolved": "https://registry.npmjs.org/@redis/time-series/-/time-series-1.1.0.tgz", + "integrity": "sha512-c1Q99M5ljsIuc4YdaCwfUEXsofakb9c8+Zse2qxTadu8TalLXuAESzLvFAvNVbkmSlvlzIQOLpBCmWI9wTOt+g==", + "requires": {} + }, "@redocly/ajv": { "version": "8.6.4", "resolved": "https://registry.npmjs.org/@redocly/ajv/-/ajv-8.6.4.tgz", @@ -31272,6 +31431,11 @@ "thenify-all": "^1.0.0" } }, + "nanoid": { + "version": "3.3.7", + "resolved": "https://registry.npmjs.org/nanoid/-/nanoid-3.3.7.tgz", + "integrity": "sha512-eSRppjcPIatRIMC1U6UngP8XFcz8MQWGQdt1MTBQ7NaAmvXDfvNxbvWV3x2y6CdEUciCSsDHDQZbhYaB8QEo2g==" + }, "natural-compare": { "version": "1.4.0", "resolved": "https://registry.npmjs.org/natural-compare/-/natural-compare-1.4.0.tgz", @@ -32576,6 +32740,19 @@ "strip-indent": "^3.0.0" } }, + "redis": { + "version": "4.7.0", + "resolved": "https://registry.npmjs.org/redis/-/redis-4.7.0.tgz", + "integrity": "sha512-zvmkHEAdGMn+hMRXuMBtu4Vo5P6rHQjLoHftu+lBqq8ZTA3RCVC/WzD790bkKKiNFp7d5/9PcSD19fJyyRvOdQ==", + "requires": { + "@redis/bloom": "1.2.0", + "@redis/client": "1.6.0", + "@redis/graph": "1.1.1", + "@redis/json": "1.0.7", + "@redis/search": "1.2.0", + "@redis/time-series": "1.1.0" + } + }, "redis-errors": { "version": "1.2.0", "resolved": "https://registry.npmjs.org/redis-errors/-/redis-errors-1.2.0.tgz", @@ -33508,6 +33685,11 @@ "resolved": "https://registry.npmjs.org/tiny-lru/-/tiny-lru-8.0.2.tgz", "integrity": "sha512-ApGvZ6vVvTNdsmt676grvCkUCGwzG9IqXma5Z07xJgiC5L7akUMof5U8G2JTI9Rz/ovtVhJBlY6mNhEvtjzOIg==" }, + "tiny-typed-emitter": { + "version": "2.1.0", + "resolved": "https://registry.npmjs.org/tiny-typed-emitter/-/tiny-typed-emitter-2.1.0.tgz", + "integrity": "sha512-qVtvMxeXbVej0cQWKqVSSAHmKZEHAvxdF8HEUBFWts8h+xEo5m/lEiPakuyZ3BnCBjOD8i24kzNOiOLLgsSxhA==" + }, "tmp": { "version": "0.0.33", "resolved": "https://registry.npmjs.org/tmp/-/tmp-0.0.33.tgz", diff --git a/package.json b/package.json index 98795cf1..a9e21788 100644 --- a/package.json +++ b/package.json @@ -48,6 +48,8 @@ "dependencies": { "@elastic/elasticsearch": "^8.13.1", "@godaddy/terminus": "^4.12.1", + "@map-colonies/cleanup-registry": "^1.1.0", + "@map-colonies/detiler-common": "^1.0.0", "@map-colonies/error-express-handler": "^2.1.0", "@map-colonies/express-access-log-middleware": "^2.0.1", "@map-colonies/js-logger": "^1.0.1", @@ -72,6 +74,7 @@ "node-fetch-commonjs": "^3.3.2", "pg": "^8.11.5", "proj4": "^2.11.0", + "redis": "^4.7.0", "reflect-metadata": "^0.1.13", "tsyringe": "^4.8.0", "typeorm": "^0.3.20", diff --git a/src/common/constants.ts b/src/common/constants.ts index 39f7b888..83b3a228 100644 --- a/src/common/constants.ts +++ b/src/common/constants.ts @@ -16,6 +16,7 @@ export const SERVICES: Record = { METER: Symbol('Meter'), APPLICATION: Symbol('Application'), ELASTIC_CLIENTS: Symbol('ElasticClients'), + REDIS: Symbol('Redis'), }; /* eslint-enable @typescript-eslint/naming-convention */ @@ -24,4 +25,3 @@ export const HEALTHCHECK = Symbol('healthcheck'); export const REDIS_SYMBOL = Symbol('REDIS'); export const elasticConfigPath = 'db.elastic'; -export const REDIS_KEYS_SEPARATOR = ':'; diff --git a/src/common/errors.ts b/src/common/errors.ts index 244c0576..c56a5641 100644 --- a/src/common/errors.ts +++ b/src/common/errors.ts @@ -49,3 +49,5 @@ export class NotImplementedError extends Error implements HttpError { super(message); } } + +export class TimeoutError extends Error {} \ No newline at end of file diff --git a/src/common/interfaces.ts b/src/common/interfaces.ts index d140f1a0..3cc33eac 100644 --- a/src/common/interfaces.ts +++ b/src/common/interfaces.ts @@ -1,4 +1,5 @@ import { DataSourceOptions } from 'typeorm'; +import { RedisClientOptions } from 'redis'; import { Feature, FeatureCollection as GeoJSONFeatureCollection } from 'geojson'; export interface IConfig { @@ -13,6 +14,13 @@ export interface OpenApiConfig { uiPath: string; } +export type RedisConfig = { + host: string; + port: number; + enableSslAuth: boolean; + sslPaths: { ca: string; cert: string; key: string }; +} & RedisClientOptions; + export type PostgresDbConfig = { enableSslAuth: boolean; sslPaths: { ca: string; cert: string; key: string }; diff --git a/src/common/redis/domainFieldsRepository.ts b/src/common/redis/domainFieldsRepository.ts deleted file mode 100644 index 9cf9e3cc..00000000 --- a/src/common/redis/domainFieldsRepository.ts +++ /dev/null @@ -1,5 +0,0 @@ -export const IDOMAIN_FIELDS_REPO_SYMBOL = Symbol('DOMAINFIELDSREPO'); - -export interface IDomainFieldsRepository { - getFields: (fields: string[]) => Promise<(string | null)[]>; -} diff --git a/src/common/redis/index.ts b/src/common/redis/index.ts index fb3396dc..6e4cf92f 100644 --- a/src/common/redis/index.ts +++ b/src/common/redis/index.ts @@ -1,33 +1,58 @@ -import Redis, { RedisOptions } from 'ioredis'; -import { HOSTNAME } from '../constants'; +import { readFileSync } from 'fs'; +import { ILogger } from '@map-colonies/detiler-common'; +import { HealthCheck } from '@godaddy/terminus'; +import { createClient, RedisClientOptions } from 'redis'; +import { DependencyContainer, FactoryFunction } from 'tsyringe'; +import { SERVICES } from '../constants'; +import { RedisConfig, IConfig } from '../interfaces'; +import { promiseTimeout } from '../utils'; -const RETRY_DELAY_INCREASE = 50; -const RETRY_DELAY_TOP = 2000; -let redis: Redis; +const DEFAULT_LIMIT_FROM = 0; +const DEFAULT_LIMIT_SIZE = 1000; -const retryFunction = (times: number): number => { - const delay = Math.min(times * RETRY_DELAY_INCREASE, RETRY_DELAY_TOP); - return delay; +const createConnectionOptions = (redisConfig: RedisConfig): Partial => { + const { host, port, enableSslAuth, sslPaths, ...clientOptions } = redisConfig; + clientOptions.socket = { host, port }; + if (enableSslAuth) { + clientOptions.socket = { + ...clientOptions.socket, + tls: true, + key: sslPaths.key !== '' ? readFileSync(sslPaths.key) : undefined, + cert: sslPaths.cert !== '' ? readFileSync(sslPaths.cert) : undefined, + ca: sslPaths.ca !== '' ? readFileSync(sslPaths.ca) : undefined, + }; + } + + return clientOptions; }; -export const createRedisConnection = async (redisOptions: RedisOptions): Promise => { - try { - redisOptions = { - ...redisOptions, - retryStrategy: retryFunction, - lazyConnect: true, - connectionName: HOSTNAME, - }; +export const CONNECTION_TIMEOUT = 5000; - redis = new Redis(redisOptions); - await redis.connect(); - return redis; - } catch (err) { - redis.disconnect(); - let errorMessage = 'Redis connection failed'; - if (err instanceof Error) { - errorMessage += ` with the following error: ${err.message}`; - } - throw new Error(errorMessage); - } +export const DEFAULT_LIMIT = { from: DEFAULT_LIMIT_FROM, size: DEFAULT_LIMIT_SIZE }; + +export type RedisClient = ReturnType; + +export const redisClientFactory: FactoryFunction = (container: DependencyContainer): RedisClient => { + const logger = container.resolve(SERVICES.LOGGER); + const config = container.resolve(SERVICES.CONFIG); + const dbConfig = config.get('db.redis'); + const connectionOptions = createConnectionOptions(dbConfig); + + const redisClient = createClient(connectionOptions) + .on('error', (error: Error) => logger.error({ msg: 'redis client errored', err: error })) + .on('reconnecting', (...args) => logger.warn({ msg: 'redis client reconnecting', ...args })) + .on('end', (...args) => logger.info({ msg: 'redis client end', ...args })) + .on('connect', (...args) => logger.debug({ msg: 'redis client connected', ...args })) + .on('ready', (...args) => logger.debug({ msg: 'redis client is ready', ...args })); + + return redisClient; +}; + +export const healthCheckFunctionFactory = (redis: RedisClient): HealthCheck => { + return async (): Promise => { + const check = redis.ping().then(() => { + return; + }); + return promiseTimeout(CONNECTION_TIMEOUT, check); + }; }; diff --git a/src/common/redis/keys.ts b/src/common/redis/keys.ts deleted file mode 100644 index e268631c..00000000 --- a/src/common/redis/keys.ts +++ /dev/null @@ -1,5 +0,0 @@ -import { REDIS_KEYS_SEPARATOR } from '../../common/constants'; - -export const keyConstructor = (...vals: string[]): string => { - return vals.filter((v) => v).join(REDIS_KEYS_SEPARATOR); // drop undefined, null and empty string from a key -}; diff --git a/src/common/redis/redisManager.ts b/src/common/redis/redisManager.ts deleted file mode 100644 index 0ea22c07..00000000 --- a/src/common/redis/redisManager.ts +++ /dev/null @@ -1,31 +0,0 @@ -import Redis from 'ioredis'; -import { inject, injectable } from 'tsyringe'; -import { REDIS_SYMBOL, SERVICES } from '../constants'; -import { IApplication } from '../interfaces'; -import { IDomainFieldsRepository } from './domainFieldsRepository'; - -@injectable() -export class RedisManager implements IDomainFieldsRepository { - public getData: (fields: string[]) => Promise<(string | null)[]>; - - public constructor(@inject(REDIS_SYMBOL) private readonly redis: Redis, @inject(SERVICES.APPLICATION) appConfig: IApplication) { - const { value, enabled } = appConfig.hashKey; - if (enabled && value !== undefined) { - this.getData = async (fields: string[]): Promise<(string | null)[]> => { - return this.redis.hmget(value, ...fields); - }; - } else { - this.getData = async (fields: string[]): Promise<(string | null)[]> => { - return this.redis.mget(fields); - }; - } - } - - public async getFields(fields: string[]): Promise<(string | null)[]> { - try { - return await this.getData(fields); - } catch (e) { - throw new Error('redis: failed to fetch keys'); - } - } -} diff --git a/src/common/utils.ts b/src/common/utils.ts index 6f75dec6..0b64a1c1 100644 --- a/src/common/utils.ts +++ b/src/common/utils.ts @@ -1,6 +1,7 @@ import proj4 from 'proj4'; import { utmProjection, wgs84Projection } from './projections'; import { WGS84Coordinate } from './interfaces'; +import { TimeoutError } from './errors'; type SnakeToCamelCase = S extends `${infer T}_${infer U}` ? `${T}${Capitalize>}` : S; @@ -79,3 +80,16 @@ export const validateTile = (tile: { tileName: string; subTileNumber: number[] } } return true; }; + +export const promiseTimeout = async (ms: number, promise: Promise): Promise => { + // create a promise that rejects in milliseconds + const timeout = new Promise((_, reject) => { + const id = setTimeout(() => { + clearTimeout(id); + reject(new TimeoutError(`Timed out in + ${ms} + ms.`)); + }, ms); + }); + + // returns a race between our timeout and the passed in promise + return Promise.race([promise, timeout]); +}; \ No newline at end of file diff --git a/src/containerConfig.ts b/src/containerConfig.ts index 658dc442..4424fa4b 100644 --- a/src/containerConfig.ts +++ b/src/containerConfig.ts @@ -1,17 +1,16 @@ import config from 'config'; -import Redis, { RedisOptions } from 'ioredis'; import { getOtelMixin } from '@map-colonies/telemetry'; import { DataSource } from 'typeorm'; import { instancePerContainerCachingFactory } from 'tsyringe'; import { trace, metrics as OtelMetrics } from '@opentelemetry/api'; +import { CleanupRegistry } from '@map-colonies/cleanup-registry'; import { DependencyContainer } from 'tsyringe/dist/typings/types'; import jsLogger, { LoggerOptions } from '@map-colonies/js-logger'; import { Metrics } from '@map-colonies/telemetry'; -import { SERVICES, SERVICE_NAME, REDIS_SYMBOL } from './common/constants'; +import { HEALTHCHECK, SERVICES, SERVICE_NAME } from './common/constants'; import { tracing } from './common/tracing'; import { InjectionObject, registerDependencies } from './common/dependencyRegistration'; import { elasticClientFactory, ElasticClients } from './common/elastic'; -import { RedisManager } from './common/redis/redisManager'; import { IApplication } from './common/interfaces'; import { TILE_REPOSITORY_SYMBOL, tileRepositoryFactory } from './control/tile/DAL/tileRepository'; import { TILE_ROUTER_SYMBOL, tileRouterFactory } from './control/tile/routes/tileRouter'; @@ -25,8 +24,8 @@ import { GEOTEXT_REPOSITORY_SYMBOL, geotextRepositoryFactory } from './location/ import { GEOTEXT_SEARCH_ROUTER_SYMBOL, geotextSearchRouterFactory } from './location/routes/locationRouter'; import { cronLoadTileLatLonDataFactory, cronLoadTileLatLonDataSymbol } from './latLon/DAL/latLonDAL'; import { ITEM_REPOSITORY_SYMBOL, itemRepositoryFactory } from './control/item/DAL/itemRepository'; -import { createRedisConnection } from './common/redis/index'; -import { IDOMAIN_FIELDS_REPO_SYMBOL } from './common/redis/domainFieldsRepository'; +import { healthCheckFunctionFactory, RedisClient, redisClientFactory } from './common/redis'; +import { HealthCheck } from '@godaddy/terminus'; export interface RegisterOptions { override?: InjectionObject[]; @@ -34,6 +33,8 @@ export interface RegisterOptions { } export const registerExternalValues = async (options?: RegisterOptions): Promise => { + const cleanupRegistry = new CleanupRegistry(); + const loggerConfig = config.get('telemetry.logger'); const logger = jsLogger({ ...loggerConfig, prettyPrint: loggerConfig.prettyPrint, mixin: getOtelMixin() }); @@ -45,8 +46,6 @@ export const registerExternalValues = async (options?: RegisterOptions): Promise const applicationConfig: IApplication = config.get('application'); - let redisConnection: Redis | undefined; - const dependencies: InjectionObject[] = [ { token: SERVICES.CONFIG, provider: { useValue: config } }, { token: SERVICES.LOGGER, provider: { useValue: logger } }, @@ -84,26 +83,21 @@ export const registerExternalValues = async (options?: RegisterOptions): Promise }, }, { - token: REDIS_SYMBOL, + token: SERVICES.REDIS, + provider: { useFactory: instancePerContainerCachingFactory(redisClientFactory) }, + postInjectionHook: async (deps: DependencyContainer): Promise => { + const redis = deps.resolve(SERVICES.REDIS); + cleanupRegistry.register({ func: redis.disconnect.bind(redis), id: SERVICES.REDIS }); + await redis.connect(); + }, + }, + { + token: HEALTHCHECK, provider: { - useFactory: instancePerContainerCachingFactory(async () => { - redisConnection = await createRedisConnection(config.get('db')); - - redisConnection.on('connect', () => { - logger.info(`redis client is connected.`); - }); - - redisConnection.on('error', (err: Error) => { - logger.error({ err: err, msg: 'redis client got an error' }); - }); - - redisConnection.on('reconnecting', (delay: number) => { - logger.info(`redis client reconnecting, next reconnection attemp in ${delay}ms`); - }); - container.register(SERVICES.LOGGER, { useValue: logger }); - container.register(IDOMAIN_FIELDS_REPO_SYMBOL, { useClass: RedisManager }); - return redisConnection; - }), + useFactory: (container): HealthCheck => { + const redis = container.resolve(SERVICES.REDIS); + return healthCheckFunctionFactory(redis); + }, }, }, { token: TILE_REPOSITORY_SYMBOL, provider: { useFactory: tileRepositoryFactory } }, @@ -127,20 +121,8 @@ export const registerExternalValues = async (options?: RegisterOptions): Promise provider: { useFactory: instancePerContainerCachingFactory(async (): Promise => { const promises: Promise[] = [tracing.stop(), metrics.stop()]; - if (redisConnection !== undefined) { - redisConnection.disconnect(); - - const promisifyQuit = new Promise((resolve) => { - redisConnection = redisConnection as Redis; - redisConnection.once('end', () => { - resolve(); - }); - void redisConnection.quit(); - }); - promises.push(promisifyQuit); - } - await Promise.all(promises); }), + useValue: cleanupRegistry.trigger.bind(cleanupRegistry), }, }, ]; diff --git a/src/serverBuilder.ts b/src/serverBuilder.ts index c9c04c50..5d6740ae 100644 --- a/src/serverBuilder.ts +++ b/src/serverBuilder.ts @@ -16,6 +16,7 @@ import { ROUTE_ROUTER_SYMBOL } from './control/route/routes/routeRouter'; import { LAT_LON_ROUTER_SYMBOL } from './latLon/routes/latLonRouter'; import { GEOTEXT_SEARCH_ROUTER_SYMBOL } from './location/routes/locationRouter'; import { cronLoadTileLatLonDataSymbol } from './latLon/DAL/latLonDAL'; +import { feedbackApiMiddlewareManager } from './common/middlewares/feedbackApi.middleware'; @injectable() export class ServerBuilder { @@ -29,7 +30,8 @@ export class ServerBuilder { @inject(ROUTE_ROUTER_SYMBOL) private readonly routeRouter: Router, @inject(LAT_LON_ROUTER_SYMBOL) private readonly latLonRouter: Router, @inject(GEOTEXT_SEARCH_ROUTER_SYMBOL) private readonly geotextRouter: Router, - @inject(cronLoadTileLatLonDataSymbol) private readonly cronLoadTileLatLonData: void + @inject(cronLoadTileLatLonDataSymbol) private readonly cronLoadTileLatLonData: void, + @inject(feedbackApiMiddlewareManager) private readonly feedbackApiMiddleware: feedbackApiMiddlewareManager ) { this.serverInstance = express(); // eslint-disable-next-line @typescript-eslint/no-unused-expressions @@ -55,12 +57,14 @@ export class ServerBuilder { } private buildRoutes(): void { + this.serverInstance.use(this.feedbackApiMiddleware.saveResponses); const router = Router(); + router.use('/lookup', this.latLonRouter); router.use('/location', this.geotextRouter); router.use('/control', this.buildControlRoutes()); - this.serverInstance.use('/search/', router); + this.serverInstance.use('/search', router); } private buildControlRoutes(): Router { From a560876820d4fd78f448690d1a2ea1f05aff3736 Mon Sep 17 00:00:00 2001 From: NatalieShaked Date: Thu, 29 Aug 2024 17:41:19 +0300 Subject: [PATCH 04/20] feat: api responses get sent to redis --- src/common/interfaces.ts | 10 +++-- .../middlewares/feedbackApi.middleware.ts | 38 +++++++++++++++++++ 2 files changed, 44 insertions(+), 4 deletions(-) create mode 100644 src/common/middlewares/feedbackApi.middleware.ts diff --git a/src/common/interfaces.ts b/src/common/interfaces.ts index 3cc33eac..3c0052f3 100644 --- a/src/common/interfaces.ts +++ b/src/common/interfaces.ts @@ -57,10 +57,6 @@ export interface IApplication { hierarchy: number; viewbox: number; }; - hashKey: { - value?: string; - enabled: boolean; - }; sources?: { [key: string]: string; }; @@ -76,6 +72,12 @@ export enum GeoContextMode { BIAS = 'bias', } +export interface GeocodingResponse { + userId: string; + response: JSON; + respondedAt: Date; +} + /* eslint-disable @typescript-eslint/naming-convention */ export interface CommonRequestParameters { geo_context?: GeoContext; diff --git a/src/common/middlewares/feedbackApi.middleware.ts b/src/common/middlewares/feedbackApi.middleware.ts new file mode 100644 index 00000000..bcf7b9aa --- /dev/null +++ b/src/common/middlewares/feedbackApi.middleware.ts @@ -0,0 +1,38 @@ +import { Request, Response, NextFunction } from 'express'; +import { SERVICES } from '../constants'; +import { Logger } from '@map-colonies/js-logger'; +import { inject, injectable } from 'tsyringe'; +import { RedisClient } from '../redis'; +import * as crypto from 'node:crypto'; +import { GeocodingResponse } from '../interfaces'; + +@injectable() +export class feedbackApiMiddlewareManager { + public constructor(@inject(SERVICES.LOGGER) private readonly logger: Logger, @inject(SERVICES.REDIS) private readonly redis: RedisClient) {} + + saveResponses = async (req: Request, res: Response, next: NextFunction) => { + this.logger.info({ msg: 'saving response to redis' }); + const reqId = crypto.randomUUID(); + const redisClient = this.redis + + // let response: JSON = JSON.parse('{}'); + let geocodingResponseDetails: GeocodingResponse = { + userId: req.headers['x-user-id'] as string, + response: JSON.parse('{}'), + respondedAt: new Date(), + }; + + const originalJson = res.json; + const logJson = async function (this: Response, body: any): Promise>> { + console.log('Response:', body); + // response = await body; + geocodingResponseDetails.response = await body; + await redisClient.set(reqId, JSON.stringify(geocodingResponseDetails)); + + return originalJson.call(this, body); + }; + + res.json = logJson as unknown as Response['json']; + next(); + }; +} From 1f97c76e08294c2413a17fe8e4b0b6565e3643b0 Mon Sep 17 00:00:00 2001 From: NatalieShaked Date: Thu, 29 Aug 2024 18:04:05 +0300 Subject: [PATCH 05/20] feat: added ttl of 5 mins to redis set --- .../middlewares/feedbackApi.middleware.ts | 30 +++++++++++++++---- 1 file changed, 25 insertions(+), 5 deletions(-) diff --git a/src/common/middlewares/feedbackApi.middleware.ts b/src/common/middlewares/feedbackApi.middleware.ts index bcf7b9aa..f22bc77e 100644 --- a/src/common/middlewares/feedbackApi.middleware.ts +++ b/src/common/middlewares/feedbackApi.middleware.ts @@ -5,16 +5,19 @@ import { inject, injectable } from 'tsyringe'; import { RedisClient } from '../redis'; import * as crypto from 'node:crypto'; import { GeocodingResponse } from '../interfaces'; +import { RedisClientType } from '@redis/client'; +import { RedisModules, RedisFunctions, RedisScripts } from 'redis'; @injectable() export class feedbackApiMiddlewareManager { public constructor(@inject(SERVICES.LOGGER) private readonly logger: Logger, @inject(SERVICES.REDIS) private readonly redis: RedisClient) {} saveResponses = async (req: Request, res: Response, next: NextFunction) => { - this.logger.info({ msg: 'saving response to redis' }); const reqId = crypto.randomUUID(); - const redisClient = this.redis + const redisClient = this.redis; + const logger = this.logger; + logger.info({ msg: 'saving response to redis' }); // let response: JSON = JSON.parse('{}'); let geocodingResponseDetails: GeocodingResponse = { userId: req.headers['x-user-id'] as string, @@ -25,14 +28,31 @@ export class feedbackApiMiddlewareManager { const originalJson = res.json; const logJson = async function (this: Response, body: any): Promise>> { console.log('Response:', body); - // response = await body; geocodingResponseDetails.response = await body; - await redisClient.set(reqId, JSON.stringify(geocodingResponseDetails)); + + try { + await redisClient.setEx(reqId, 300, JSON.stringify(geocodingResponseDetails)); + logger.info({ msg: 'saving response to redis' }); + } catch (err) { + logger.error('Error setting key:', err); + } + + //await setKeyWithTTL(reqId, JSON.stringify(geocodingResponseDetails), redisClient); return originalJson.call(this, body); }; - res.json = logJson as unknown as Response['json']; next(); }; } + +async function setKeyWithTTL(key: string, value: string, redis: RedisClient) { + try { + await redis.set(key, value, { + EX: 300, // 5 minutes in seconds + }); + console.log(`Key '${key}' set with a TTL of 5 minutes.`); + } catch (err) { + console.error('Error setting key:', err); + } +} From b5a4031de4e43450af06b859d1de76277680cb39 Mon Sep 17 00:00:00 2001 From: NatalieShaked Date: Sun, 1 Sep 2024 12:13:50 +0300 Subject: [PATCH 06/20] fix: removed unnecessary redis configs --- .../middlewares/feedbackApi.middleware.ts | 5 +--- tests/configurations/unit/jest.config.js | 1 + tests/integration/control/item/item.spec.ts | 30 ++----------------- tests/integration/control/route/route.spec.ts | 28 +---------------- tests/integration/control/tile/tile.spec.ts | 28 +---------------- tests/integration/location/location.spec.ts | 29 +----------------- 6 files changed, 7 insertions(+), 114 deletions(-) diff --git a/src/common/middlewares/feedbackApi.middleware.ts b/src/common/middlewares/feedbackApi.middleware.ts index f22bc77e..c3a178dd 100644 --- a/src/common/middlewares/feedbackApi.middleware.ts +++ b/src/common/middlewares/feedbackApi.middleware.ts @@ -5,8 +5,6 @@ import { inject, injectable } from 'tsyringe'; import { RedisClient } from '../redis'; import * as crypto from 'node:crypto'; import { GeocodingResponse } from '../interfaces'; -import { RedisClientType } from '@redis/client'; -import { RedisModules, RedisFunctions, RedisScripts } from 'redis'; @injectable() export class feedbackApiMiddlewareManager { @@ -18,7 +16,6 @@ export class feedbackApiMiddlewareManager { const logger = this.logger; logger.info({ msg: 'saving response to redis' }); - // let response: JSON = JSON.parse('{}'); let geocodingResponseDetails: GeocodingResponse = { userId: req.headers['x-user-id'] as string, response: JSON.parse('{}'), @@ -27,7 +24,7 @@ export class feedbackApiMiddlewareManager { const originalJson = res.json; const logJson = async function (this: Response, body: any): Promise>> { - console.log('Response:', body); + // console.log('Response:', body); geocodingResponseDetails.response = await body; try { diff --git a/tests/configurations/unit/jest.config.js b/tests/configurations/unit/jest.config.js index 07af12e5..bfd03fc4 100644 --- a/tests/configurations/unit/jest.config.js +++ b/tests/configurations/unit/jest.config.js @@ -13,6 +13,7 @@ module.exports = { '!*/common/**', '!**/controllers/**', '!**/routes/**', + '!**/redis/**', '!/src/*', ], coverageDirectory: '/coverage', diff --git a/tests/integration/control/item/item.spec.ts b/tests/integration/control/item/item.spec.ts index 01327f88..39e3659b 100644 --- a/tests/integration/control/item/item.spec.ts +++ b/tests/integration/control/item/item.spec.ts @@ -3,12 +3,8 @@ import jsLogger from '@map-colonies/js-logger'; import { trace } from '@opentelemetry/api'; import httpStatusCodes from 'http-status-codes'; import { DataSource } from 'typeorm'; -import config from 'config'; -import Redis, { RedisOptions } from 'ioredis'; -import { DependencyContainer } from 'tsyringe'; -import { Application } from 'express'; import { getApp } from '../../../../src/app'; -import { REDIS_SYMBOL, SERVICES } from '../../../../src/common/constants'; +import { SERVICES } from '../../../../src/common/constants'; import { GetItemsQueryParams } from '../../../../src/control/item/controllers/itemController'; import { Item } from '../../../../src/control/item/models/item'; import { ControlResponse } from '../../../../src/control/interfaces'; @@ -16,30 +12,14 @@ import { CommonRequestParameters, GeoContext, GeoContextMode } from '../../../.. import { LATLON_CUSTOM_REPOSITORY_SYMBOL } from '../../../../src/latLon/DAL/latLonRepository'; import { cronLoadTileLatLonDataSymbol } from '../../../../src/latLon/DAL/latLonDAL'; import { expectedResponse } from '../utils'; -import { RedisManager } from '../../../../src/common/redis/redisManager'; -import { IDOMAIN_FIELDS_REPO_SYMBOL } from '../../../../src/common/redis/domainFieldsRepository'; -import { createRedisConnection } from '../../../../src/common/redis'; import { ItemRequestSender } from './helpers/requestSender'; import { ITEM_1234, ITEM_1235, ITEM_1236 } from './mockObjects'; describe('/search/control/items', function () { let requestSender: ItemRequestSender; - let app: { app: Application; container?: DependencyContainer }; - let redisConnection: Redis; - - beforeAll(async function () { - redisConnection = await createRedisConnection(config.get('db')); - app = await getApp({ - override: [ - { token: REDIS_SYMBOL, provider: { useValue: redisConnection } }, - { token: IDOMAIN_FIELDS_REPO_SYMBOL, provider: { useClass: RedisManager } }, - ], - useChild: true, - }); - }); beforeEach(async function () { - app = await getApp({ + const app = await getApp({ override: [ { token: SERVICES.LOGGER, provider: { useValue: jsLogger({ enabled: false }) } }, { token: SERVICES.TRACER, provider: { useValue: trace.getTracer('testTracer') } }, @@ -54,12 +34,6 @@ describe('/search/control/items', function () { requestSender = new ItemRequestSender(app.app); }); - afterAll(async function () { - if (!['end'].includes(redisConnection.status)) { - await redisConnection.quit(); - } - }); - describe('Happy Path', function () { it('should return 200 status code and items', async function () { const requestParams: GetItemsQueryParams = { command_name: '123', limit: 5, disable_fuzziness: false }; diff --git a/tests/integration/control/route/route.spec.ts b/tests/integration/control/route/route.spec.ts index 1dd5b394..fe95f22e 100644 --- a/tests/integration/control/route/route.spec.ts +++ b/tests/integration/control/route/route.spec.ts @@ -3,40 +3,20 @@ import jsLogger from '@map-colonies/js-logger'; import { trace } from '@opentelemetry/api'; import httpStatusCodes from 'http-status-codes'; import { DataSource } from 'typeorm'; -import config from 'config'; -import Redis, { RedisOptions } from 'ioredis'; -import { DependencyContainer } from 'tsyringe'; -import { Application } from 'express'; import { getApp } from '../../../../src/app'; -import { REDIS_SYMBOL, SERVICES } from '../../../../src/common/constants'; +import { SERVICES } from '../../../../src/common/constants'; import { GetRoutesQueryParams } from '../../../../src/control/route/controllers/routeController'; import { Route } from '../../../../src/control/route/models/route'; import { ControlResponse } from '../../../../src/control/interfaces'; import { CommonRequestParameters, GeoContext, GeoContextMode } from '../../../../src/common/interfaces'; import { LATLON_CUSTOM_REPOSITORY_SYMBOL } from '../../../../src/latLon/DAL/latLonRepository'; import { cronLoadTileLatLonDataSymbol } from '../../../../src/latLon/DAL/latLonDAL'; -import { RedisManager } from '../../../../src/common/redis/redisManager'; -import { IDOMAIN_FIELDS_REPO_SYMBOL } from '../../../../src/common/redis/domainFieldsRepository'; -import { createRedisConnection } from '../../../../src/common/redis'; import { expectedResponse } from '../utils'; import { RouteRequestSender } from './helpers/requestSender'; import { ROUTE_VIA_CAMILLUCCIA_A, ROUTE_VIA_CAMILLUCCIA_B, CONTROL_POINT_OLIMPIADE_111, CONTROL_POINT_OLIMPIADE_112 } from './mockObjects'; describe('/search/control/route', function () { let requestSender: RouteRequestSender; - let app: { app: Application; container?: DependencyContainer }; - let redisConnection: Redis; - - beforeAll(async function () { - redisConnection = await createRedisConnection(config.get('db')); - app = await getApp({ - override: [ - { token: REDIS_SYMBOL, provider: { useValue: redisConnection } }, - { token: IDOMAIN_FIELDS_REPO_SYMBOL, provider: { useClass: RedisManager } }, - ], - useChild: true, - }); - }); beforeEach(async function () { const app = await getApp({ @@ -54,12 +34,6 @@ describe('/search/control/route', function () { requestSender = new RouteRequestSender(app.app); }); - afterAll(async function () { - if (!['end'].includes(redisConnection.status)) { - await redisConnection.quit(); - } - }); - describe('Happy Path', function () { it('should return 200 status code and routes', async function () { const requestParams: GetRoutesQueryParams = { command_name: 'via camilluccia', limit: 5, disable_fuzziness: false }; diff --git a/tests/integration/control/tile/tile.spec.ts b/tests/integration/control/tile/tile.spec.ts index 6564fd99..44685fcc 100644 --- a/tests/integration/control/tile/tile.spec.ts +++ b/tests/integration/control/tile/tile.spec.ts @@ -3,20 +3,13 @@ import jsLogger from '@map-colonies/js-logger'; import { trace } from '@opentelemetry/api'; import httpStatusCodes from 'http-status-codes'; import { DataSource } from 'typeorm'; -import config from 'config'; -import Redis, { RedisOptions } from 'ioredis'; -import { DependencyContainer } from 'tsyringe'; -import { Application } from 'express'; import { getApp } from '../../../../src/app'; -import { REDIS_SYMBOL, SERVICES } from '../../../../src/common/constants'; +import { SERVICES } from '../../../../src/common/constants'; import { GetTilesQueryParams } from '../../../../src/control/tile/controllers/tileController'; import { Tile } from '../../../../src/control/tile/models/tile'; import { ControlResponse } from '../../../../src/control/interfaces'; import { CommonRequestParameters, GeoContext, GeoContextMode } from '../../../../src/common/interfaces'; import { LATLON_CUSTOM_REPOSITORY_SYMBOL } from '../../../../src/latLon/DAL/latLonRepository'; -import { RedisManager } from '../../../../src/common/redis/redisManager'; -import { IDOMAIN_FIELDS_REPO_SYMBOL } from '../../../../src/common/redis/domainFieldsRepository'; -import { createRedisConnection } from '../../../../src/common/redis'; import { cronLoadTileLatLonDataSymbol } from '../../../../src/latLon/DAL/latLonDAL'; import { expectedResponse } from '../utils'; import { TileRequestSender } from './helpers/requestSender'; @@ -24,19 +17,6 @@ import { RIC_TILE, RIT_TILE, SUB_TILE_65, SUB_TILE_66 } from './mockObjects'; describe('/search/control/tiles', function () { let requestSender: TileRequestSender; - let app: { app: Application; container?: DependencyContainer }; - let redisConnection: Redis; - - beforeAll(async function () { - redisConnection = await createRedisConnection(config.get('db')); - app = await getApp({ - override: [ - { token: REDIS_SYMBOL, provider: { useValue: redisConnection } }, - { token: IDOMAIN_FIELDS_REPO_SYMBOL, provider: { useClass: RedisManager } }, - ], - useChild: true, - }); - }); beforeEach(async function () { const app = await getApp({ @@ -54,12 +34,6 @@ describe('/search/control/tiles', function () { requestSender = new TileRequestSender(app.app); }); - afterAll(async function () { - if (!['end'].includes(redisConnection.status)) { - await redisConnection.quit(); - } - }); - describe('Happy Path', function () { it('should return 200 status code and tiles', async function () { const requestParams: GetTilesQueryParams = { tile: 'RIT', limit: 5, disable_fuzziness: false }; diff --git a/tests/integration/location/location.spec.ts b/tests/integration/location/location.spec.ts index 385c4f59..c561aa42 100644 --- a/tests/integration/location/location.spec.ts +++ b/tests/integration/location/location.spec.ts @@ -3,18 +3,12 @@ import config from 'config'; import jsLogger from '@map-colonies/js-logger'; import { trace } from '@opentelemetry/api'; import httpStatusCodes from 'http-status-codes'; -import Redis, { RedisOptions } from 'ioredis'; -import { DependencyContainer } from 'tsyringe'; -import { Application } from 'express'; import { DataSource } from 'typeorm'; import nock, { Body } from 'nock'; import { getApp } from '../../../src/app'; -import { REDIS_SYMBOL, SERVICES } from '../../../src/common/constants'; +import { SERVICES } from '../../../src/common/constants'; import { LATLON_CUSTOM_REPOSITORY_SYMBOL } from '../../../src/latLon/DAL/latLonRepository'; import { cronLoadTileLatLonDataSymbol } from '../../../src/latLon/DAL/latLonDAL'; -import { RedisManager } from '../../../src/common/redis/redisManager'; -import { IDOMAIN_FIELDS_REPO_SYMBOL } from '../../../src/common/redis/domainFieldsRepository'; -import { createRedisConnection } from '../../../src/common/redis'; import { GetGeotextSearchParams, QueryResult } from '../../../src/location/interfaces'; import { GeoContextMode, IApplication } from '../../../src/common/interfaces'; import { LocationRequestSender } from './helpers/requestSender'; @@ -22,32 +16,17 @@ import { OSM_LA_PORT, GOOGLE_LA_PORT, LA_AIRPORT, - LA_WHITE_POINT_SCHOOL, NY_JFK_AIRPORT, NY_POLICE_AIRPORT, NY_HIERRARCHY, LA_HIERRARCHY, MockLocationQueryFeature, PARIS_WI_SCHOOL, - PARIS_HIERRARCHY, } from './mockObjects'; import { expectedResponse, hierarchiesWithAnyWieght } from './utils'; describe('/search/control/tiles', function () { let requestSender: LocationRequestSender; - let app: { app: Application; container?: DependencyContainer }; - let redisConnection: Redis; - - beforeAll(async function () { - redisConnection = await createRedisConnection(config.get('db')); - app = await getApp({ - override: [ - { token: REDIS_SYMBOL, provider: { useValue: redisConnection } }, - { token: IDOMAIN_FIELDS_REPO_SYMBOL, provider: { useClass: RedisManager } }, - ], - useChild: true, - }); - }); beforeEach(async function () { const app = await getApp({ @@ -65,12 +44,6 @@ describe('/search/control/tiles', function () { requestSender = new LocationRequestSender(app.app); }); - afterAll(async function () { - if (!['end'].includes(redisConnection.status)) { - await redisConnection.quit(); - } - }); - describe('Happy Path', function () { it('should return 200 status code and airports', async function () { const requestParams: GetGeotextSearchParams = { query: 'airport', limit: 5, disable_fuzziness: true }; From e0c9f30cb7d0be38f0bcf3752e752cf76fffa55b Mon Sep 17 00:00:00 2001 From: NatalieShaked Date: Sun, 1 Sep 2024 13:49:57 +0300 Subject: [PATCH 07/20] fix: eslint fixes --- openapi3.yaml | 580 +++++++++--------- src/common/constants.ts | 1 + src/common/errors.ts | 2 +- .../middlewares/feedbackApi.middleware.ts | 41 +- src/common/utils.ts | 2 +- src/containerConfig.ts | 13 +- src/control/route/DAL/routeRepository.ts | 6 +- src/control/tile/DAL/tileRepository.ts | 9 +- src/control/utils.ts | 2 +- src/index.ts | 2 +- .../controllers/locationController.ts | 4 +- src/location/parsing.ts | 4 +- src/location/utils.ts | 2 +- src/serverBuilder.ts | 4 +- tests/integration/location/mockObjects.ts | 2 +- 15 files changed, 335 insertions(+), 339 deletions(-) diff --git a/openapi3.yaml b/openapi3.yaml index f3b5cc9e..4c0948e5 100644 --- a/openapi3.yaml +++ b/openapi3.yaml @@ -1,7 +1,7 @@ -openapi: "3.0.1" +openapi: '3.0.1' info: - version: "0.1.0" - title: "Geocoding" + version: '0.1.0' + title: 'Geocoding' description: |- MapColonies Vector Geocoding api provides custom geodata search engine uniting multiple sources of data, query tiles, routes, items; Convertion functions - tile to/from WGS84 lat lng, WGS84 to/from US Army MGRS, WGS84 to/from UTM. license: @@ -15,7 +15,7 @@ paths: /search/query: get: operationId: GetSmartQuery - summary: "Search anything" + summary: 'Search anything' description: |- This is for general queries. the services will make a sophisticated guess. parameters: @@ -26,16 +26,16 @@ paths: type: string minLength: 3 maxLength: 100 - title: "Query" + title: 'Query' description: Text to search allowReserved: true - - $ref: "#/components/parameters/geo_context" - - $ref: "#/components/parameters/geo_context_mode" - - $ref: "#/components/parameters/limit" - - $ref: "#/components/parameters/disable_fuzziness" + - $ref: '#/components/parameters/geo_context' + - $ref: '#/components/parameters/geo_context_mode' + - $ref: '#/components/parameters/limit' + - $ref: '#/components/parameters/disable_fuzziness' responses: 200: - description: "OK
Will return valid GeoJSON FeatureCollection" + description: 'OK
Will return valid GeoJSON FeatureCollection' headers: x-request-id: schema: @@ -45,15 +45,15 @@ paths: content: application/json: schema: - $ref: "#/components/schemas/genericGeocodingResponse" + $ref: '#/components/schemas/genericGeocodingResponse' 400: - "$ref": "#/components/responses/BadRequest" + '$ref': '#/components/responses/BadRequest' 401: - "$ref": "#/components/responses/Unauthorized" + '$ref': '#/components/responses/Unauthorized' 403: - "$ref": "#/components/responses/Forbidden" + '$ref': '#/components/responses/Forbidden' 500: - "$ref": "#/components/responses/InternalError" + '$ref': '#/components/responses/InternalError' security: - x-api-key: [] x-user-id: [] @@ -61,8 +61,8 @@ paths: /search/location/query: get: operationId: locationGetQuery - summary: "Search engine" - description: "Search for geo-data. It serves data from multiple diverse mapcolonies vector sources and partners." + summary: 'Search engine' + description: 'Search for geo-data. It serves data from multiple diverse mapcolonies vector sources and partners.' tags: - Location Name Based Search parameters: @@ -73,7 +73,7 @@ paths: type: string minLength: 3 maxLength: 100 - title: "Query" + title: 'Query' description: Text to search allowReserved: true - name: source @@ -82,7 +82,7 @@ paths: schema: type: array items: - $ref: "#/components/schemas/Source" + $ref: '#/components/schemas/Source' title: Source description: | Sources to include (if not specified, all sources will be queried) @@ -92,17 +92,17 @@ paths: schema: type: array items: - $ref: "#/components/schemas/Region" + $ref: '#/components/schemas/Region' description: Regions to include (if not specified, all Regions will be queried) title: Region description: Regions to include (if not specified, all Regions will be queried) - - $ref: "#/components/parameters/geo_context" - - $ref: "#/components/parameters/geo_context_mode" - - $ref: "#/components/parameters/limit" - - $ref: "#/components/parameters/disable_fuzziness" + - $ref: '#/components/parameters/geo_context' + - $ref: '#/components/parameters/geo_context_mode' + - $ref: '#/components/parameters/limit' + - $ref: '#/components/parameters/disable_fuzziness' responses: 200: - description: "OK" + description: 'OK' x-request-id: schema: type: string @@ -111,15 +111,15 @@ paths: content: application/json: schema: - "$ref": "#/components/schemas/genericGeocodingResponse" + '$ref': '#/components/schemas/genericGeocodingResponse' 400: - "$ref": "#/components/responses/BadRequest" + '$ref': '#/components/responses/BadRequest' 401: - "$ref": "#/components/responses/Unauthorized" + '$ref': '#/components/responses/Unauthorized' 403: - "$ref": "#/components/responses/Forbidden" + '$ref': '#/components/responses/Forbidden' 500: - "$ref": "#/components/responses/InternalError" + '$ref': '#/components/responses/InternalError' security: - x-api-key: [] x-user-id: [] @@ -129,10 +129,10 @@ paths: operationId: locationGetRegions tags: - Location Name Based Search - summary: "Get all available regions to filter on using location query" + summary: 'Get all available regions to filter on using location query' responses: 200: - description: "All regions" + description: 'All regions' content: application/json: schema: @@ -140,13 +140,13 @@ paths: items: type: string # return geojson with name property 400: - "$ref": "#/components/responses/BadRequest" + '$ref': '#/components/responses/BadRequest' 401: - "$ref": "#/components/responses/Unauthorized" + '$ref': '#/components/responses/Unauthorized' 403: - "$ref": "#/components/responses/Forbidden" + '$ref': '#/components/responses/Forbidden' 500: - "$ref": "#/components/responses/InternalError" + '$ref': '#/components/responses/InternalError' security: - x-api-key: [] x-user-id: [] @@ -156,10 +156,10 @@ paths: operationId: locationGetSources tags: - Location Name Based Search - summary: "Get all available sources to filter on using location query" + summary: 'Get all available sources to filter on using location query' responses: 200: - description: "All sources" + description: 'All sources' content: application/json: schema: @@ -167,13 +167,13 @@ paths: items: type: string 400: - "$ref": "#/components/responses/BadRequest" + '$ref': '#/components/responses/BadRequest' 401: - "$ref": "#/components/responses/Unauthorized" + '$ref': '#/components/responses/Unauthorized' 403: - "$ref": "#/components/responses/Forbidden" + '$ref': '#/components/responses/Forbidden' 500: - "$ref": "#/components/responses/InternalError" + '$ref': '#/components/responses/InternalError' security: - x-api-key: [] x-user-id: [] @@ -183,50 +183,50 @@ paths: operationId: controlGetTilesByQueryParams tags: - Control - summary: "Search tiles and sub tiles" + summary: 'Search tiles and sub tiles' description: |- Tiles are consisted of 3 characters.
You can query and get results from 2 characters.

If you define a tile (full name of it) you can query the sub tile associated with it.
You may also search tile based on 1 meter precision MGRS tile. parameters: - - name: "tile" - in: "query" - description: "Tile name" + - name: 'tile' + in: 'query' + description: 'Tile name' schema: - type: "string" + type: 'string' minLength: 2 maxLength: 3 - - name: "sub_tile" - in: "query" - description: "Sub tile number" + - name: 'sub_tile' + in: 'query' + description: 'Sub tile number' schema: - type: "string" - pattern: "^[1-9][0-9]*$" + type: 'string' + pattern: '^[1-9][0-9]*$' - name: mgrs - description: "1 meters MGRS Tile" - example: "18SUJ2338907395" + description: '1 meters MGRS Tile' + example: '18SUJ2338907395' in: query schema: type: string - - $ref: "#/components/parameters/geo_context" - - $ref: "#/components/parameters/geo_context_mode" - - $ref: "#/components/parameters/limit" - - $ref: "#/components/parameters/disable_fuzziness" + - $ref: '#/components/parameters/geo_context' + - $ref: '#/components/parameters/geo_context_mode' + - $ref: '#/components/parameters/limit' + - $ref: '#/components/parameters/disable_fuzziness' responses: 200: - description: "OK" + description: 'OK' content: application/json: schema: - $ref: "#/components/schemas/tilesSchema" + $ref: '#/components/schemas/tilesSchema' 400: - "$ref": "#/components/responses/BadRequest" + '$ref': '#/components/responses/BadRequest' 401: - "$ref": "#/components/responses/Unauthorized" + '$ref': '#/components/responses/Unauthorized' 403: - "$ref": "#/components/responses/Forbidden" + '$ref': '#/components/responses/Forbidden' 500: - "$ref": "#/components/responses/InternalError" + '$ref': '#/components/responses/InternalError' security: - x-api-key: [] x-user-id: [] @@ -235,51 +235,51 @@ paths: operationId: controlGetItemsByQueryParams tags: - Control - summary: "Search control items" - description: "" + summary: 'Search control items' + description: '' parameters: - - name: "command_name" - in: "query" - description: "Object command name of the item" + - name: 'command_name' + in: 'query' + description: 'Object command name of the item' schema: - type: "string" + type: 'string' required: true - - name: "tile" - in: "query" - description: "The tile the item in it (full name of it)" + - name: 'tile' + in: 'query' + description: 'The tile the item in it (full name of it)' schema: - type: "string" + type: 'string' minLength: 3 maxLength: 3 required: false - - name: "sub_tile" - in: "query" - description: "The sub tile the item in it (required if tile is defined)" + - name: 'sub_tile' + in: 'query' + description: 'The sub tile the item in it (required if tile is defined)' schema: - type: "string" - pattern: "^[1-9][0-9]*$" - example: "66" - - $ref: "#/components/parameters/geo_context" - - $ref: "#/components/parameters/geo_context_mode" - - $ref: "#/components/parameters/limit" - - $ref: "#/components/parameters/disable_fuzziness" + type: 'string' + pattern: '^[1-9][0-9]*$' + example: '66' + - $ref: '#/components/parameters/geo_context' + - $ref: '#/components/parameters/geo_context_mode' + - $ref: '#/components/parameters/limit' + - $ref: '#/components/parameters/disable_fuzziness' responses: 200: - description: "OK" + description: 'OK' content: application/json: schema: - $ref: "#/components/schemas/itemsSchema" + $ref: '#/components/schemas/itemsSchema' 400: - "$ref": "#/components/responses/BadRequest" + '$ref': '#/components/responses/BadRequest' 401: - "$ref": "#/components/responses/Unauthorized" + '$ref': '#/components/responses/Unauthorized' 403: - "$ref": "#/components/responses/Forbidden" + '$ref': '#/components/responses/Forbidden' 500: - "$ref": "#/components/responses/InternalError" + '$ref': '#/components/responses/InternalError' security: - x-api-key: [] x-user-id: [] @@ -288,42 +288,42 @@ paths: operationId: controlGetRoutesByQueryParams tags: - Control - summary: "Search routes and report control points" - description: "You can query and get control routes.

If you define a route (full name of it) you can query the control points associated with it. " + summary: 'Search routes and report control points' + description: 'You can query and get control routes.

If you define a route (full name of it) you can query the control points associated with it. ' parameters: - - name: "command_name" - in: "query" - description: "Object command name of the item" + - name: 'command_name' + in: 'query' + description: 'Object command name of the item' schema: - type: "string" + type: 'string' required: true - - name: "control_point" - in: "query" - description: "The associated report control point of the route" + - name: 'control_point' + in: 'query' + description: 'The associated report control point of the route' schema: - type: "string" - pattern: "^[1-9][0-9]*$" - - $ref: "#/components/parameters/geo_context" - - $ref: "#/components/parameters/geo_context_mode" - - $ref: "#/components/parameters/limit" - - $ref: "#/components/parameters/disable_fuzziness" + type: 'string' + pattern: '^[1-9][0-9]*$' + - $ref: '#/components/parameters/geo_context' + - $ref: '#/components/parameters/geo_context_mode' + - $ref: '#/components/parameters/limit' + - $ref: '#/components/parameters/disable_fuzziness' responses: 200: - description: "OK (can be routesSchema or itemsSchema)" + description: 'OK (can be routesSchema or itemsSchema)' content: application/json: schema: anyOf: - - $ref: "#/components/schemas/routesSchema" - - $ref: "#/components/schemas/itemsSchema" + - $ref: '#/components/schemas/routesSchema' + - $ref: '#/components/schemas/itemsSchema' 400: - "$ref": "#/components/responses/BadRequest" + '$ref': '#/components/responses/BadRequest' 401: - "$ref": "#/components/responses/Unauthorized" + '$ref': '#/components/responses/Unauthorized' 403: - "$ref": "#/components/responses/Forbidden" + '$ref': '#/components/responses/Forbidden' 500: - "$ref": "#/components/responses/InternalError" + '$ref': '#/components/responses/InternalError' security: - x-api-key: [] x-user-id: [] @@ -332,29 +332,29 @@ paths: operationId: getMGRSToGeom tags: - MGRS - summary: "Convert a MGRS string to Geometry in GeoJSON" + summary: 'Convert a MGRS string to Geometry in GeoJSON' parameters: - - name: "tile" - in: "query" - description: "MGRS tile string" + - name: 'tile' + in: 'query' + description: 'MGRS tile string' schema: - type: "string" + type: 'string' required: true responses: 200: - description: "OK" + description: 'OK' content: application/json: schema: - $ref: "#/components/schemas/mgrsTileSchema" + $ref: '#/components/schemas/mgrsTileSchema' 400: - "$ref": "#/components/responses/BadRequest" + '$ref': '#/components/responses/BadRequest' 401: - "$ref": "#/components/responses/Unauthorized" + '$ref': '#/components/responses/Unauthorized' 403: - "$ref": "#/components/responses/Forbidden" + '$ref': '#/components/responses/Forbidden' 500: - "$ref": "#/components/responses/InternalError" + '$ref': '#/components/responses/InternalError' security: - x-api-key: [] x-user-id: [] @@ -363,33 +363,33 @@ paths: operationId: convertionGetLatLonToMgrs tags: - Convertions - summary: "Convert a WGS84 coordinate to US Army MGRS / Control Grid" + summary: 'Convert a WGS84 coordinate to US Army MGRS / Control Grid' parameters: - - name: "lat" - in: "query" - description: "Latitude of the coordinate" + - name: 'lat' + in: 'query' + description: 'Latitude of the coordinate' schema: - type: "number" + type: 'number' minimum: -90 maximum: 90 required: true - - name: "lon" - in: "query" - description: "Longitude of the coordinate" + - name: 'lon' + in: 'query' + description: 'Longitude of the coordinate' schema: - type: "number" + type: 'number' minimum: -180 maximum: 180 required: true - - name: "target_grid" + - name: 'target_grid' in: query description: Choose target grid schema: type: string - enum: ["control", "MGRS"] + enum: ['control', 'MGRS'] responses: 200: - description: "OK" + description: 'OK' content: application/json: schema: @@ -398,13 +398,13 @@ paths: geojson: type: object # Make it geojson 400: - "$ref": "#/components/responses/BadRequest" + '$ref': '#/components/responses/BadRequest' 401: - "$ref": "#/components/responses/Unauthorized" + '$ref': '#/components/responses/Unauthorized' 403: - "$ref": "#/components/responses/Forbidden" + '$ref': '#/components/responses/Forbidden' 500: - "$ref": "#/components/responses/InternalError" + '$ref': '#/components/responses/InternalError' security: - x-api-key: [] x-user-id: [] @@ -414,11 +414,11 @@ paths: operationId: postSearchFeedback tags: - Feedback - summary: "Retrieve feedback from services about quality of results" + summary: 'Retrieve feedback from services about quality of results' parameters: - name: choosen_result_id - in: "query" - description: "The result ID of chosen search" + in: 'query' + description: 'The result ID of chosen search' schema: type: string minimum: 1 @@ -434,17 +434,17 @@ paths: required: true responses: 204: - description: "OK - No Content" + description: 'OK - No Content' 400: - "$ref": "#/components/responses/BadRequest" + '$ref': '#/components/responses/BadRequest' 404: - $ref: "#/components/responses/NotFound" + $ref: '#/components/responses/NotFound' 401: - "$ref": "#/components/responses/Unauthorized" + '$ref': '#/components/responses/Unauthorized' 403: - "$ref": "#/components/responses/Forbidden" + '$ref': '#/components/responses/Forbidden' 500: - "$ref": "#/components/responses/InternalError" + '$ref': '#/components/responses/InternalError' security: - x-api-key: [] x-user-id: [] @@ -452,55 +452,55 @@ paths: components: responses: BadRequest: - description: "Invalid Request" + description: 'Invalid Request' content: application/json: schema: - $ref: "#/components/schemas/errorSchema" + $ref: '#/components/schemas/errorSchema' Unauthorized: - description: "Please provide a valid token" + description: 'Please provide a valid token' content: application/json: schema: - $ref: "#/components/schemas/errorSchema" + $ref: '#/components/schemas/errorSchema' NotFound: - description: "Resource Not Found" + description: 'Resource Not Found' content: application/json: schema: - $ref: "#/components/schemas/errorSchema" + $ref: '#/components/schemas/errorSchema' Forbidden: - description: "Token is not valid" + description: 'Token is not valid' content: application/json: schema: - $ref: "#/components/schemas/errorSchema" + $ref: '#/components/schemas/errorSchema' InternalError: - description: "Invalid Request" + description: 'Invalid Request' content: application/json: schema: - $ref: "#/components/schemas/errorSchema" + $ref: '#/components/schemas/errorSchema' parameters: disable_fuzziness: - name: "disable_fuzziness" - in: "query" + name: 'disable_fuzziness' + in: 'query' description: |- If an accurate result is obtained, only it will be returned schema: - $ref: "#/components/schemas/disable_fuzziness" + $ref: '#/components/schemas/disable_fuzziness' limit: - name: "limit" - in: "query" - description: "Maximum number of results" + name: 'limit' + in: 'query' + description: 'Maximum number of results' schema: default: 5 minimum: 1 maximum: 15 - type: "number" + type: 'number' geo_context: - in: "query" - name: "geo_context" + in: 'query' + name: 'geo_context' description: |- Geo context of search. @@ -511,185 +511,185 @@ components: {"lon":value,"lat":value,"radius":value}
{"x":value,"y":value,"zone": number, "radius":value} schema: - $ref: "#/components/schemas/geo_context" + $ref: '#/components/schemas/geo_context' geo_context_mode: - name: "geo_context_mode" - in: "query" - description: "Choose whether geo_context query parameter will be a filter or a bias value" + name: 'geo_context_mode' + in: 'query' + description: 'Choose whether geo_context query parameter will be a filter or a bias value' schema: type: string - enum: ["filter", "bias"] + enum: ['filter', 'bias'] schemas: errorSchema: - type: "object" + type: 'object' required: - - "message" + - 'message' properties: message: - type: "string" + type: 'string' status: - type: "number" + type: 'number' subTileSchema: - type: "object" - description: "An object represnting a sub tile" + type: 'object' + description: 'An object represnting a sub tile' required: - tileName - subTileNumber properties: tileName: - type: "string" + type: 'string' subTileNumber: - type: "array" + type: 'array' items: - type: "number" + type: 'number' minimum: 0 maximum: 100 minItems: 3 maxItems: 3 tileSchema: - type: "object" + type: 'object' properties: type: - type: "string" - enum: ["Feature"] + type: 'string' + enum: ['Feature'] geometry: - $ref: "#/components/schemas/Polygon" + $ref: '#/components/schemas/Polygon' properties: - type: "object" + type: 'object' required: - - "TYPE" - - "tile_name" + - 'TYPE' + - 'tile_name' properties: TYPE: - type: "string" + type: 'string' tile_name: - type: "string" + type: 'string' sub_tile_id: - type: "string" + type: 'string' SUB_TILE_NUMBER: - type: "array" + type: 'array' items: - type: "number" + type: 'number' minimum: 0 maximum: 100 minItems: 3 maxItems: 3 mgrsTileSchema: - type: "object" + type: 'object' properties: type: - type: "string" - enum: ["Feature"] + type: 'string' + enum: ['Feature'] geometry: - $ref: "#/components/schemas/Polygon" + $ref: '#/components/schemas/Polygon' properties: - type: "object" + type: 'object' tilesSchema: - type: "object" - description: "GeoJson feature collection representing a tile (Polygon)" + type: 'object' + description: 'GeoJson feature collection representing a tile (Polygon)' required: - - "type" - - "features" + - 'type' + - 'features' properties: type: - type: "string" + type: 'string' enum: - - "FeatureCollection" + - 'FeatureCollection' features: - type: "array" + type: 'array' items: - $ref: "#/components/schemas/tileSchema" + $ref: '#/components/schemas/tileSchema' itemsSchema: - type: "object" - description: "GeoJson feature collection representing an item" + type: 'object' + description: 'GeoJson feature collection representing an item' required: - - "type" - - "features" + - 'type' + - 'features' properties: type: - type: "string" + type: 'string' enum: - - "FeatureCollection" + - 'FeatureCollection' features: - type: "array" + type: 'array' items: - type: "object" + type: 'object' properties: geometry: oneOf: - - $ref: "#/components/schemas/Point" - - $ref: "#/components/schemas/Polygon" + - $ref: '#/components/schemas/Point' + - $ref: '#/components/schemas/Polygon' properties: - type: "object" + type: 'object' required: - - "TYPE" - - "object_command_name" - - "entity_heb" - - "sub_tile_id" + - 'TYPE' + - 'object_command_name' + - 'entity_heb' + - 'sub_tile_id' properties: TYPE: - type: "string" + type: 'string' object_command_name: - type: "string" + type: 'string' entity_heb: - type: "string" + type: 'string' tile_name: - type: "string" + type: 'string' sub_tile_id: - type: "string" + type: 'string' routesSchema: - type: "object" - description: "GeoJson feature collection representing a route (MultiLineString, LineString)" + type: 'object' + description: 'GeoJson feature collection representing a route (MultiLineString, LineString)' required: - - "type" - - "features" + - 'type' + - 'features' properties: type: - type: "string" + type: 'string' enum: - - "FeatureCollection" + - 'FeatureCollection' features: - type: "array" + type: 'array' items: - type: "object" + type: 'object' properties: geometry: oneOf: - - $ref: "#/components/schemas/LineString" - - $ref: "#/components/schemas/MultiLineString" + - $ref: '#/components/schemas/LineString' + - $ref: '#/components/schemas/MultiLineString' properties: - type: "object" + type: 'object' required: - - "TYPE" - - "object_command_name" - - "entity_heb" + - 'TYPE' + - 'object_command_name' + - 'entity_heb' properties: TYPE: - type: "string" + type: 'string' object_command_name: - type: "string" + type: 'string' entity_heb: - type: "string" + type: 'string' SECTION: - type: "string" + type: 'string' genericGeocodingResponse: # we need discriminator for Control and Location - type: "object" - description: "GeoJson feature collection representing an item" + type: 'object' + description: 'GeoJson feature collection representing an item' required: - - "type" - - "features" - - "geocoding" + - 'type' + - 'features' + - 'geocoding' properties: type: - type: "string" + type: 'string' enum: - - "FeatureCollection" + - 'FeatureCollection' geocoding: type: object required: - - "query" - - "response" + - 'query' + - 'response' properties: version: type: string @@ -698,10 +698,10 @@ components: query: type: object required: - - "text" - - "limit" - - "geo_context" - - "disable_fuzziness" + - 'text' + - 'limit' + - 'geo_context' + - 'disable_fuzziness' properties: text: type: string @@ -712,15 +712,15 @@ components: minimum: 1 maximum: 10 geo_context: - $ref: "#/components/schemas/geo_context" + $ref: '#/components/schemas/geo_context' disable_fuzziness: - $ref: "#/components/schemas/disable_fuzziness" + $ref: '#/components/schemas/disable_fuzziness' response: type: object required: - - "max_score" - - "results_count" - - "match_latency" + - 'max_score' + - 'results_count' + - 'match_latency' properties: max_score: type: number @@ -735,16 +735,16 @@ components: minimum: 0 maximum: 5000 bbox: - $ref: "#/components/schemas/BoundingBox" + $ref: '#/components/schemas/BoundingBox' features: - type: "array" + type: 'array' items: - type: "object" + type: 'object' properties: geometry: - $ref: "#/components/schemas/Geometry" + $ref: '#/components/schemas/Geometry' properties: - type: "object" + type: 'object' additionalProperties: true required: - id @@ -854,69 +854,69 @@ components: externalDocs: url: http://geojson.org/geojson-spec.html#id2 allOf: - - $ref: "#/components/schemas/Geometry" + - $ref: '#/components/schemas/Geometry' - properties: coordinates: - $ref: "#/components/schemas/Point3D" + $ref: '#/components/schemas/Point3D' LineString: type: object description: Geojson geometry externalDocs: url: http://geojson.org/geojson-spec.html#id3 allOf: - - $ref: "#/components/schemas/Geometry" + - $ref: '#/components/schemas/Geometry' - properties: coordinates: type: array items: - $ref: "#/components/schemas/Point3D" + $ref: '#/components/schemas/Point3D' Polygon: type: object description: Geojson geometry externalDocs: url: http://geojson.org/geojson-spec.html#id4 allOf: - - $ref: "#/components/schemas/Geometry" + - $ref: '#/components/schemas/Geometry' - properties: coordinates: type: array items: type: array items: - $ref: "#/components/schemas/Point3D" + $ref: '#/components/schemas/Point3D' MultiPoint: type: object description: Geojson geometry externalDocs: url: http://geojson.org/geojson-spec.html#id5 allOf: - - $ref: "#/components/schemas/Geometry" + - $ref: '#/components/schemas/Geometry' - properties: coordinates: type: array items: - $ref: "#/components/schemas/Point3D" + $ref: '#/components/schemas/Point3D' MultiLineString: type: object description: Geojson geometry externalDocs: url: http://geojson.org/geojson-spec.html#id6 allOf: - - $ref: "#/components/schemas/Geometry" + - $ref: '#/components/schemas/Geometry' - properties: coordinates: type: array items: type: array items: - $ref: "#/components/schemas/Point3D" + $ref: '#/components/schemas/Point3D' MultiPolygon: type: object description: Geojson geometry externalDocs: url: http://geojson.org/geojson-spec.html#id6 allOf: - - $ref: "#/components/schemas/Geometry" + - $ref: '#/components/schemas/Geometry' - properties: coordinates: type: array @@ -925,7 +925,7 @@ components: items: type: array items: - $ref: "#/components/schemas/Point3D" + $ref: '#/components/schemas/Point3D' Source: type: string @@ -935,8 +935,8 @@ components: title: Region BoundingBox: type: array - description: "Bounding box array that contains [minX,minY,maxX,maxY]" - example: "[-74.382527,40.477003,-73.322346,40.916383]" + description: 'Bounding box array that contains [minX,minY,maxX,maxY]' + example: '[-74.382527,40.477003,-73.322346,40.916383]' items: type: number minLength: 4 @@ -968,18 +968,18 @@ components: - type: object properties: bbox: - $ref: "#/components/schemas/BoundingBox" - - $ref: "#/components/schemas/WGS84Circle" - - $ref: "#/components/schemas/UTMCircle" + $ref: '#/components/schemas/BoundingBox' + - $ref: '#/components/schemas/WGS84Circle' + - $ref: '#/components/schemas/UTMCircle' disable_fuzziness: type: boolean default: false securitySchemes: x-api-key: - type: "apiKey" - name: "x-api-key" - in: "header" + type: 'apiKey' + name: 'x-api-key' + in: 'header' x-user-id: - type: "apiKey" - name: "x-user-id" - in: "header" + type: 'apiKey' + name: 'x-user-id' + in: 'header' diff --git a/src/common/constants.ts b/src/common/constants.ts index 83b3a228..ac72a036 100644 --- a/src/common/constants.ts +++ b/src/common/constants.ts @@ -24,4 +24,5 @@ export const ON_SIGNAL = Symbol('onSignal'); export const HEALTHCHECK = Symbol('healthcheck'); export const REDIS_SYMBOL = Symbol('REDIS'); +export const REDIS_TTL = 300; export const elasticConfigPath = 'db.elastic'; diff --git a/src/common/errors.ts b/src/common/errors.ts index c56a5641..374b146c 100644 --- a/src/common/errors.ts +++ b/src/common/errors.ts @@ -50,4 +50,4 @@ export class NotImplementedError extends Error implements HttpError { } } -export class TimeoutError extends Error {} \ No newline at end of file +export class TimeoutError extends Error {} diff --git a/src/common/middlewares/feedbackApi.middleware.ts b/src/common/middlewares/feedbackApi.middleware.ts index c3a178dd..6064e86a 100644 --- a/src/common/middlewares/feedbackApi.middleware.ts +++ b/src/common/middlewares/feedbackApi.middleware.ts @@ -1,34 +1,35 @@ +import * as crypto from 'node:crypto'; import { Request, Response, NextFunction } from 'express'; -import { SERVICES } from '../constants'; import { Logger } from '@map-colonies/js-logger'; import { inject, injectable } from 'tsyringe'; +import { REDIS_TTL, SERVICES } from '../constants'; import { RedisClient } from '../redis'; -import * as crypto from 'node:crypto'; import { GeocodingResponse } from '../interfaces'; @injectable() -export class feedbackApiMiddlewareManager { +export class FeedbackApiMiddlewareManager { public constructor(@inject(SERVICES.LOGGER) private readonly logger: Logger, @inject(SERVICES.REDIS) private readonly redis: RedisClient) {} - saveResponses = async (req: Request, res: Response, next: NextFunction) => { + // eslint-disable-next-line @typescript-eslint/explicit-function-return-type + public saveResponses = (req: Request, res: Response, next: NextFunction) => { const reqId = crypto.randomUUID(); const redisClient = this.redis; const logger = this.logger; logger.info({ msg: 'saving response to redis' }); - let geocodingResponseDetails: GeocodingResponse = { + const geocodingResponseDetails: GeocodingResponse = { userId: req.headers['x-user-id'] as string, - response: JSON.parse('{}'), + response: JSON.parse('{}') as JSON, respondedAt: new Date(), }; const originalJson = res.json; - const logJson = async function (this: Response, body: any): Promise>> { - // console.log('Response:', body); - geocodingResponseDetails.response = await body; + const logJson = async function (this: Response, body: JSON): Promise { + // console.log('Response:', body); + geocodingResponseDetails.response = body; try { - await redisClient.setEx(reqId, 300, JSON.stringify(geocodingResponseDetails)); + await redisClient.setEx(reqId, REDIS_TTL, JSON.stringify(geocodingResponseDetails)); logger.info({ msg: 'saving response to redis' }); } catch (err) { logger.error('Error setting key:', err); @@ -43,13 +44,13 @@ export class feedbackApiMiddlewareManager { }; } -async function setKeyWithTTL(key: string, value: string, redis: RedisClient) { - try { - await redis.set(key, value, { - EX: 300, // 5 minutes in seconds - }); - console.log(`Key '${key}' set with a TTL of 5 minutes.`); - } catch (err) { - console.error('Error setting key:', err); - } -} +// async function setKeyWithTTL(key: string, value: string, redis: RedisClient) { +// try { +// await redis.set(key, value, { +// EX: 300, // 5 minutes in seconds +// }); +// console.log(`Key '${key}' set with a TTL of 5 minutes.`); +// } catch (err) { +// console.error('Error setting key:', err); +// } +// } diff --git a/src/common/utils.ts b/src/common/utils.ts index 0b64a1c1..8421389d 100644 --- a/src/common/utils.ts +++ b/src/common/utils.ts @@ -92,4 +92,4 @@ export const promiseTimeout = async (ms: number, promise: Promise): Promis // returns a race between our timeout and the passed in promise return Promise.race([promise, timeout]); -}; \ No newline at end of file +}; diff --git a/src/containerConfig.ts b/src/containerConfig.ts index 4424fa4b..14dbe465 100644 --- a/src/containerConfig.ts +++ b/src/containerConfig.ts @@ -7,6 +7,7 @@ import { CleanupRegistry } from '@map-colonies/cleanup-registry'; import { DependencyContainer } from 'tsyringe/dist/typings/types'; import jsLogger, { LoggerOptions } from '@map-colonies/js-logger'; import { Metrics } from '@map-colonies/telemetry'; +import { HealthCheck } from '@godaddy/terminus'; import { HEALTHCHECK, SERVICES, SERVICE_NAME } from './common/constants'; import { tracing } from './common/tracing'; import { InjectionObject, registerDependencies } from './common/dependencyRegistration'; @@ -25,7 +26,6 @@ import { GEOTEXT_SEARCH_ROUTER_SYMBOL, geotextSearchRouterFactory } from './loca import { cronLoadTileLatLonDataFactory, cronLoadTileLatLonDataSymbol } from './latLon/DAL/latLonDAL'; import { ITEM_REPOSITORY_SYMBOL, itemRepositoryFactory } from './control/item/DAL/itemRepository'; import { healthCheckFunctionFactory, RedisClient, redisClientFactory } from './common/redis'; -import { HealthCheck } from '@godaddy/terminus'; export interface RegisterOptions { override?: InjectionObject[]; @@ -58,7 +58,7 @@ export const registerExternalValues = async (options?: RegisterOptions): Promise postInjectionHook: async (deps: DependencyContainer): Promise => { const elasticClients = deps.resolve(SERVICES.ELASTIC_CLIENTS); try { - const response = await Promise.all([elasticClients.control?.ping(), elasticClients.geotext?.ping()]); + const response = await Promise.all([elasticClients.control.ping(), elasticClients.geotext.ping()]); response.forEach((res) => { if (!res) { logger.error('Failed to connect to Elasticsearch', res); @@ -119,10 +119,11 @@ export const registerExternalValues = async (options?: RegisterOptions): Promise { token: 'onSignal', provider: { - useFactory: instancePerContainerCachingFactory(async (): Promise => { - const promises: Promise[] = [tracing.stop(), metrics.stop()]; - }), - useValue: cleanupRegistry.trigger.bind(cleanupRegistry), + useValue: { + useValue: async (): Promise => { + await Promise.all([tracing.stop(), metrics.stop()]); + }, + }, }, }, ]; diff --git a/src/control/route/DAL/routeRepository.ts b/src/control/route/DAL/routeRepository.ts index d1bfd737..34da5b2e 100644 --- a/src/control/route/DAL/routeRepository.ts +++ b/src/control/route/DAL/routeRepository.ts @@ -1,4 +1,3 @@ -import { Logger } from '@map-colonies/js-logger'; import { estypes } from '@elastic/elasticsearch'; import { FactoryFunction } from 'tsyringe'; import { ElasticClient } from '../../../common/elastic'; @@ -11,7 +10,7 @@ import { additionalControlSearchProperties } from '../../utils'; import { RouteQueryParams, queryForControlPointInRoute, queryForRoute } from './queries'; // eslint-disable-next-line @typescript-eslint/explicit-function-return-type -const createRouteRepository = (client: ElasticClient, config: IConfig, logger: Logger) => { +const createRouteRepository = (client: ElasticClient, config: IConfig) => { return { async getRoutes(routeQueryParams: RouteQueryParams, size: number): Promise> { const response = await queryElastic(client, { ...additionalControlSearchProperties(config, size), ...queryForRoute(routeQueryParams) }); @@ -38,8 +37,7 @@ export type RouteRepository = ReturnType; export const routeRepositoryFactory: FactoryFunction = (depContainer) => { return createRouteRepository( depContainer.resolve(SERVICES.ELASTIC_CLIENTS).control, - depContainer.resolve(SERVICES.CONFIG), - depContainer.resolve(SERVICES.LOGGER) + depContainer.resolve(SERVICES.CONFIG) ); }; diff --git a/src/control/tile/DAL/tileRepository.ts b/src/control/tile/DAL/tileRepository.ts index c7f764ab..a3063b08 100644 --- a/src/control/tile/DAL/tileRepository.ts +++ b/src/control/tile/DAL/tileRepository.ts @@ -1,4 +1,3 @@ -import { Logger } from '@map-colonies/js-logger'; import { estypes } from '@elastic/elasticsearch'; import { FactoryFunction } from 'tsyringe'; import { ElasticClient, ElasticClients } from '../../../common/elastic'; @@ -10,7 +9,7 @@ import { additionalControlSearchProperties } from '../../utils'; import { queryForTiles, queryForSubTiles, TileQueryParams } from './queries'; // eslint-disable-next-line @typescript-eslint/explicit-function-return-type -const createTileRepository = (client: ElasticClient, config: IConfig, logger: Logger) => { +const createTileRepository = (client: ElasticClient, config: IConfig) => { return { // eslint-disable-next-line @typescript-eslint/explicit-module-boundary-types async getTiles(tileQueryParams: TileQueryParams & Required>, size: number): Promise> { @@ -30,11 +29,7 @@ const createTileRepository = (client: ElasticClient, config: IConfig, logger: Lo export type TileRepository = ReturnType; export const tileRepositoryFactory: FactoryFunction = (depContainer) => { - return createTileRepository( - depContainer.resolve(SERVICES.ELASTIC_CLIENTS).control, - depContainer.resolve(SERVICES.CONFIG), - depContainer.resolve(SERVICES.LOGGER) - ); + return createTileRepository(depContainer.resolve(SERVICES.ELASTIC_CLIENTS).control, depContainer.resolve(SERVICES.CONFIG)); }; export const TILE_REPOSITORY_SYMBOL = Symbol('TILE_REPOSITORY_SYMBOL'); diff --git a/src/control/utils.ts b/src/control/utils.ts index 31aea8f2..13c47fa6 100644 --- a/src/control/utils.ts +++ b/src/control/utils.ts @@ -104,4 +104,4 @@ export const additionalControlSearchProperties = (config: IConfig, size: number) index: config.get(elasticConfigPath).control.properties.index as string, // eslint-disable-next-line @typescript-eslint/naming-convention _source: CONTROL_FIELDS, -}); +}); \ No newline at end of file diff --git a/src/index.ts b/src/index.ts index 4ee6fbac..adad9e8d 100644 --- a/src/index.ts +++ b/src/index.ts @@ -7,7 +7,7 @@ import { DependencyContainer } from 'tsyringe'; import { createTerminus } from '@godaddy/terminus'; import { Logger } from '@map-colonies/js-logger'; import config from 'config'; -import { DEFAULT_SERVER_PORT, HEALTHCHECK, ON_SIGNAL, SERVICES } from './common/constants'; +import { DEFAULT_SERVER_PORT, ON_SIGNAL, SERVICES } from './common/constants'; import { getApp } from './app'; let depContainer: DependencyContainer | undefined; diff --git a/src/location/controllers/locationController.ts b/src/location/controllers/locationController.ts index d1efeecd..a8ec476e 100644 --- a/src/location/controllers/locationController.ts +++ b/src/location/controllers/locationController.ts @@ -50,11 +50,11 @@ export class GeotextSearchController { } }; - public getRegions: GetRegionshHandler = (req, res, next) => { + public getRegions: GetRegionshHandler = (req, res) => { return res.status(httpStatus.OK).json(this.manager.regions()); }; - public getSources: GetSourcesHandler = (req, res, next) => { + public getSources: GetSourcesHandler = (req, res) => { return res.status(httpStatus.OK).json(this.manager.sources()); }; } diff --git a/src/location/parsing.ts b/src/location/parsing.ts index 08e83acc..a84d4cd6 100644 --- a/src/location/parsing.ts +++ b/src/location/parsing.ts @@ -2,10 +2,10 @@ /* istanbul ignore file */ import { XMLParser } from 'fast-xml-parser'; -type Highlight = { +interface Highlight { em: string | string[]; '#text': string; -}; +} const HIERARCHY_OF_INTEREST = 3; diff --git a/src/location/utils.ts b/src/location/utils.ts index 6f0191b9..679245b9 100644 --- a/src/location/utils.ts +++ b/src/location/utils.ts @@ -144,7 +144,7 @@ export const convertResult = ( sub_placetype: feature?.sub_placetype, regions: feature?.region.map((region) => ({ region: region, - sub_regions: feature.sub_region.filter((sub_region) => (regionCollection ?? {})[region ?? '']?.includes(sub_region)), + sub_regions: feature.sub_region.filter((sub_region) => (regionCollection ?? {})[region ?? ''].includes(sub_region)), })), }, }; diff --git a/src/serverBuilder.ts b/src/serverBuilder.ts index 5d6740ae..a2450759 100644 --- a/src/serverBuilder.ts +++ b/src/serverBuilder.ts @@ -16,7 +16,7 @@ import { ROUTE_ROUTER_SYMBOL } from './control/route/routes/routeRouter'; import { LAT_LON_ROUTER_SYMBOL } from './latLon/routes/latLonRouter'; import { GEOTEXT_SEARCH_ROUTER_SYMBOL } from './location/routes/locationRouter'; import { cronLoadTileLatLonDataSymbol } from './latLon/DAL/latLonDAL'; -import { feedbackApiMiddlewareManager } from './common/middlewares/feedbackApi.middleware'; +import { FeedbackApiMiddlewareManager } from './common/middlewares/feedbackApi.middleware'; @injectable() export class ServerBuilder { @@ -31,7 +31,7 @@ export class ServerBuilder { @inject(LAT_LON_ROUTER_SYMBOL) private readonly latLonRouter: Router, @inject(GEOTEXT_SEARCH_ROUTER_SYMBOL) private readonly geotextRouter: Router, @inject(cronLoadTileLatLonDataSymbol) private readonly cronLoadTileLatLonData: void, - @inject(feedbackApiMiddlewareManager) private readonly feedbackApiMiddleware: feedbackApiMiddlewareManager + @inject(FeedbackApiMiddlewareManager) private readonly feedbackApiMiddleware: FeedbackApiMiddlewareManager ) { this.serverInstance = express(); // eslint-disable-next-line @typescript-eslint/no-unused-expressions diff --git a/tests/integration/location/mockObjects.ts b/tests/integration/location/mockObjects.ts index 39745b7e..9f6c4867 100644 --- a/tests/integration/location/mockObjects.ts +++ b/tests/integration/location/mockObjects.ts @@ -408,4 +408,4 @@ export const PARIS_HIERRARCHY: HierarchySearchHit = { region: 'FRANCE', text: 'Paris', weight: 1.1, -}; \ No newline at end of file +}; From 9f21b64a607dceeed5ee3e8218928145381da72b Mon Sep 17 00:00:00 2001 From: NatalieShaked Date: Mon, 9 Sep 2024 14:34:14 +0300 Subject: [PATCH 08/20] feat: added request id to response header --- .../middlewares/feedbackApi.middleware.ts | 25 +++++++------------ src/serverBuilder.ts | 2 ++ 2 files changed, 11 insertions(+), 16 deletions(-) diff --git a/src/common/middlewares/feedbackApi.middleware.ts b/src/common/middlewares/feedbackApi.middleware.ts index 6064e86a..c5d382cb 100644 --- a/src/common/middlewares/feedbackApi.middleware.ts +++ b/src/common/middlewares/feedbackApi.middleware.ts @@ -12,7 +12,7 @@ export class FeedbackApiMiddlewareManager { // eslint-disable-next-line @typescript-eslint/explicit-function-return-type public saveResponses = (req: Request, res: Response, next: NextFunction) => { - const reqId = crypto.randomUUID(); + const reqId = res.getHeader('request-id'); const redisClient = this.redis; const logger = this.logger; @@ -29,28 +29,21 @@ export class FeedbackApiMiddlewareManager { geocodingResponseDetails.response = body; try { - await redisClient.setEx(reqId, REDIS_TTL, JSON.stringify(geocodingResponseDetails)); + await redisClient.setEx(reqId as string, REDIS_TTL, JSON.stringify(geocodingResponseDetails)); logger.info({ msg: 'saving response to redis' }); } catch (err) { logger.error('Error setting key:', err); } - - //await setKeyWithTTL(reqId, JSON.stringify(geocodingResponseDetails), redisClient); - return originalJson.call(this, body); }; res.json = logJson as unknown as Response['json']; next(); }; -} -// async function setKeyWithTTL(key: string, value: string, redis: RedisClient) { -// try { -// await redis.set(key, value, { -// EX: 300, // 5 minutes in seconds -// }); -// console.log(`Key '${key}' set with a TTL of 5 minutes.`); -// } catch (err) { -// console.error('Error setting key:', err); -// } -// } + // eslint-disable-next-line @typescript-eslint/explicit-function-return-type + public setRequestId = (req: Request, res: Response, next: NextFunction) => { + const reqId = crypto.randomUUID(); + res.append('request-id', reqId); + next(); + }; +} diff --git a/src/serverBuilder.ts b/src/serverBuilder.ts index a2450759..db0efea4 100644 --- a/src/serverBuilder.ts +++ b/src/serverBuilder.ts @@ -93,6 +93,8 @@ export class ServerBuilder { const apiSpecPath = this.config.get('openapiConfig.filePath'); this.serverInstance.use(OpenApiMiddleware({ apiSpec: apiSpecPath, validateRequests: true, ignorePaths: ignorePathRegex })); this.serverInstance.disable('x-powered-by'); + + this.serverInstance.use(this.feedbackApiMiddleware.setRequestId); } private registerPostRoutesMiddleware(): void { From 03d24d283ea317c1aea7c883d554ff23a05370e6 Mon Sep 17 00:00:00 2001 From: NatalieShaked Date: Mon, 9 Sep 2024 15:03:44 +0300 Subject: [PATCH 09/20] feat: sent x-api-key header to redis --- src/common/interfaces.ts | 1 + src/common/middlewares/feedbackApi.middleware.ts | 1 + 2 files changed, 2 insertions(+) diff --git a/src/common/interfaces.ts b/src/common/interfaces.ts index 3c0052f3..198fc188 100644 --- a/src/common/interfaces.ts +++ b/src/common/interfaces.ts @@ -74,6 +74,7 @@ export enum GeoContextMode { export interface GeocodingResponse { userId: string; + apiKey: string response: JSON; respondedAt: Date; } diff --git a/src/common/middlewares/feedbackApi.middleware.ts b/src/common/middlewares/feedbackApi.middleware.ts index c5d382cb..27f905cf 100644 --- a/src/common/middlewares/feedbackApi.middleware.ts +++ b/src/common/middlewares/feedbackApi.middleware.ts @@ -19,6 +19,7 @@ export class FeedbackApiMiddlewareManager { logger.info({ msg: 'saving response to redis' }); const geocodingResponseDetails: GeocodingResponse = { userId: req.headers['x-user-id'] as string, + apiKey: req.headers['x-api-key'] as string, response: JSON.parse('{}') as JSON, respondedAt: new Date(), }; From c02c9b73830df71a9e1d03e0f77ab6fa4ce8bef6 Mon Sep 17 00:00:00 2001 From: NatalieShaked Date: Mon, 9 Sep 2024 15:48:27 +0300 Subject: [PATCH 10/20] feat: sent the current site to redis, will adjust the s3 entrypoint once merged --- src/common/constants.ts | 1 + src/common/interfaces.ts | 3 ++- src/common/middlewares/feedbackApi.middleware.ts | 6 +++++- 3 files changed, 8 insertions(+), 2 deletions(-) diff --git a/src/common/constants.ts b/src/common/constants.ts index ac72a036..de2b81b9 100644 --- a/src/common/constants.ts +++ b/src/common/constants.ts @@ -26,3 +26,4 @@ export const REDIS_SYMBOL = Symbol('REDIS'); export const REDIS_TTL = 300; export const elasticConfigPath = 'db.elastic'; +export const siteIndex = 1; diff --git a/src/common/interfaces.ts b/src/common/interfaces.ts index 198fc188..2bfcbff5 100644 --- a/src/common/interfaces.ts +++ b/src/common/interfaces.ts @@ -74,7 +74,8 @@ export enum GeoContextMode { export interface GeocodingResponse { userId: string; - apiKey: string + apiKey: string; + site: string; response: JSON; respondedAt: Date; } diff --git a/src/common/middlewares/feedbackApi.middleware.ts b/src/common/middlewares/feedbackApi.middleware.ts index 27f905cf..e3774b2b 100644 --- a/src/common/middlewares/feedbackApi.middleware.ts +++ b/src/common/middlewares/feedbackApi.middleware.ts @@ -2,7 +2,7 @@ import * as crypto from 'node:crypto'; import { Request, Response, NextFunction } from 'express'; import { Logger } from '@map-colonies/js-logger'; import { inject, injectable } from 'tsyringe'; -import { REDIS_TTL, SERVICES } from '../constants'; +import { REDIS_TTL, SERVICES, siteIndex } from '../constants'; import { RedisClient } from '../redis'; import { GeocodingResponse } from '../interfaces'; @@ -16,10 +16,14 @@ export class FeedbackApiMiddlewareManager { const redisClient = this.redis; const logger = this.logger; + const s3Endpoint = ''; //s3 endpoint from default + const drSite = s3Endpoint.split('.'); + logger.info({ msg: 'saving response to redis' }); const geocodingResponseDetails: GeocodingResponse = { userId: req.headers['x-user-id'] as string, apiKey: req.headers['x-api-key'] as string, + site: drSite[siteIndex], response: JSON.parse('{}') as JSON, respondedAt: new Date(), }; From 5406ea1f303ae62cd3405d58876a348dc0018eab Mon Sep 17 00:00:00 2001 From: NatalieShaked Date: Thu, 12 Sep 2024 14:03:16 +0300 Subject: [PATCH 11/20] feat: added redis health check --- src/common/utils.ts | 5 +++++ 1 file changed, 5 insertions(+) diff --git a/src/common/utils.ts b/src/common/utils.ts index e99870fc..cc7b5b74 100644 --- a/src/common/utils.ts +++ b/src/common/utils.ts @@ -6,6 +6,7 @@ import { WGS84Coordinate } from './interfaces'; import { TimeoutError } from './errors'; import { SERVICES } from './constants'; import { ElasticClients } from './elastic'; +import { RedisClient } from './redis'; type SnakeToCamelCase = S extends `${infer T}_${infer U}` ? `${T}${Capitalize>}` : S; @@ -81,6 +82,8 @@ export const healthCheckFactory: FactoryFunction = (container: DependencyC const logger = container.resolve(SERVICES.LOGGER); const elasticClients = container.resolve(SERVICES.ELASTIC_CLIENTS); const s3Client = container.resolve(SERVICES.S3_CLIENT); + const redis = container.resolve(SERVICES.REDIS); + logger.info('Healthcheck is running'); try { @@ -91,6 +94,8 @@ export const healthCheckFactory: FactoryFunction = (container: DependencyC void s3Client.send(new ListBucketsCommand({})); + void redis.ping(); + logger.info('healthcheck passed'); } catch (error) { logger.error(`Healthcheck failed. Error: ${(error as Error).message}`); From 3541c48c7a01bbf143c19a31cc0690e7e61bca0d Mon Sep 17 00:00:00 2001 From: NatalieShaked Date: Thu, 12 Sep 2024 14:03:30 +0300 Subject: [PATCH 12/20] delete: removed redis healthcheck --- src/common/redis/index.ts | 15 ++------------- 1 file changed, 2 insertions(+), 13 deletions(-) diff --git a/src/common/redis/index.ts b/src/common/redis/index.ts index 6e4cf92f..357badc1 100644 --- a/src/common/redis/index.ts +++ b/src/common/redis/index.ts @@ -1,11 +1,9 @@ import { readFileSync } from 'fs'; -import { ILogger } from '@map-colonies/detiler-common'; -import { HealthCheck } from '@godaddy/terminus'; +import { Logger } from '@map-colonies/js-logger'; import { createClient, RedisClientOptions } from 'redis'; import { DependencyContainer, FactoryFunction } from 'tsyringe'; import { SERVICES } from '../constants'; import { RedisConfig, IConfig } from '../interfaces'; -import { promiseTimeout } from '../utils'; const DEFAULT_LIMIT_FROM = 0; const DEFAULT_LIMIT_SIZE = 1000; @@ -33,7 +31,7 @@ export const DEFAULT_LIMIT = { from: DEFAULT_LIMIT_FROM, size: DEFAULT_LIMIT_SIZ export type RedisClient = ReturnType; export const redisClientFactory: FactoryFunction = (container: DependencyContainer): RedisClient => { - const logger = container.resolve(SERVICES.LOGGER); + const logger = container.resolve(SERVICES.LOGGER); const config = container.resolve(SERVICES.CONFIG); const dbConfig = config.get('db.redis'); const connectionOptions = createConnectionOptions(dbConfig); @@ -47,12 +45,3 @@ export const redisClientFactory: FactoryFunction = (container: Depe return redisClient; }; - -export const healthCheckFunctionFactory = (redis: RedisClient): HealthCheck => { - return async (): Promise => { - const check = redis.ping().then(() => { - return; - }); - return promiseTimeout(CONNECTION_TIMEOUT, check); - }; -}; From ebc5efea770ab0615c9866ca5b6de06fd60cbe2f Mon Sep 17 00:00:00 2001 From: NatalieShaked Date: Thu, 12 Sep 2024 14:03:48 +0300 Subject: [PATCH 13/20] fix: added missing redis config --- config/custom-environment-variables.json | 23 ++++++++++++++++++++++- config/default.json | 2 +- 2 files changed, 23 insertions(+), 2 deletions(-) diff --git a/config/custom-environment-variables.json b/config/custom-environment-variables.json index 2be63e7b..b56daf27 100644 --- a/config/custom-environment-variables.json +++ b/config/custom-environment-variables.json @@ -78,7 +78,28 @@ "__name": "S3_FILES_DATA", "__format": "json" } - } + }, + "redis": { + "host": "REDIS_HOST", + "port": { + "__name": "REDIS_PORT", + "__format": "number" + }, + "username": "REDIS_USERNAME", + "password": "REDIS_PASSWORD", + "enableSslAuth": { + "__name": "REDIS_ENABLE_SSL_AUTH", + "__format": "boolean" + }, + "sslPaths": { + "ca": "REDIS_CA_PATH", + "key": "REDIS_KEY_PATH", + "cert": "REDIS_CERT_PATH" + }, + "database": { + "__name": "REDIS_DATABASE", + "__format": "number" + } }, "application": { "services": { diff --git a/config/default.json b/config/default.json index 8ece3b33..9c165b10 100644 --- a/config/default.json +++ b/config/default.json @@ -71,7 +71,7 @@ } }, "redis": { - "host": "localhost", + "host": "REDIS_HOST", "port": 6379, "username": "", "password": "", From 5b607a65fee6b42ccada44b99abbe43c66b14021 Mon Sep 17 00:00:00 2001 From: NatalieShaked Date: Thu, 12 Sep 2024 14:12:22 +0300 Subject: [PATCH 14/20] fix: added missing imports; removed pg --- dataSource.ts | 13 ------ package-lock.json | 92 ---------------------------------------- package.json | 7 +-- src/common/interfaces.ts | 6 --- src/index.ts | 2 +- 5 files changed, 2 insertions(+), 118 deletions(-) delete mode 100644 dataSource.ts diff --git a/dataSource.ts b/dataSource.ts deleted file mode 100644 index ad31f0f6..00000000 --- a/dataSource.ts +++ /dev/null @@ -1,13 +0,0 @@ -import config from 'config'; -import { DataSource } from 'typeorm'; -import { createConnectionOptions } from './src/common/postgresql'; -import { PostgresDbConfig } from './src/common/interfaces'; - -const connectionOptions = config.get('db.postgresql'); - -export const appDataSource = new DataSource({ - ...createConnectionOptions(connectionOptions), - entities: ['src/**/DAL/*.ts'], - migrationsTableName: 'migrations_table', - migrations: ['db/migrations/*.ts'], -}); diff --git a/package-lock.json b/package-lock.json index 3f707131..45416fbb 100644 --- a/package-lock.json +++ b/package-lock.json @@ -38,7 +38,6 @@ "mgrs": "^2.0.0", "node-cron": "^3.0.3", "node-fetch-commonjs": "^3.3.2", - "pg": "^8.11.5", "proj4": "^2.11.0", "redis": "^4.7.0", "reflect-metadata": "^0.1.13", @@ -18192,43 +18191,6 @@ "node": ">=8" } }, - "node_modules/pg": { - "version": "8.11.5", - "resolved": "https://registry.npmjs.org/pg/-/pg-8.11.5.tgz", - "integrity": "sha512-jqgNHSKL5cbDjFlHyYsCXmQDrfIX/3RsNwYqpd4N0Kt8niLuNoRNH+aazv6cOd43gPh9Y4DjQCtb+X0MH0Hvnw==", - "dependencies": { - "pg-connection-string": "^2.6.4", - "pg-pool": "^3.6.2", - "pg-protocol": "^1.6.1", - "pg-types": "^2.1.0", - "pgpass": "1.x" - }, - "engines": { - "node": ">= 8.0.0" - }, - "optionalDependencies": { - "pg-cloudflare": "^1.1.1" - }, - "peerDependencies": { - "pg-native": ">=3.0.1" - }, - "peerDependenciesMeta": { - "pg-native": { - "optional": true - } - } - }, - "node_modules/pg-cloudflare": { - "version": "1.1.1", - "resolved": "https://registry.npmjs.org/pg-cloudflare/-/pg-cloudflare-1.1.1.tgz", - "integrity": "sha512-xWPagP/4B6BgFO+EKz3JONXv3YDgvkbVrGw2mTo3D6tVDQRh1e7cqVGvyR3BE+eQgAvx1XhW/iEASj4/jCWl3Q==", - "optional": true - }, - "node_modules/pg-connection-string": { - "version": "2.6.4", - "resolved": "https://registry.npmjs.org/pg-connection-string/-/pg-connection-string-2.6.4.tgz", - "integrity": "sha512-v+Z7W/0EO707aNMaAEfiGnGL9sxxumwLl2fJvCQtMn9Fxsg+lPpPkdcyBSv/KFgpGdYkMfn+EI1Or2EHjpgLCA==" - }, "node_modules/pg-int8": { "version": "1.0.1", "resolved": "https://registry.npmjs.org/pg-int8/-/pg-int8-1.0.1.tgz", @@ -18237,14 +18199,6 @@ "node": ">=4.0.0" } }, - "node_modules/pg-pool": { - "version": "3.6.2", - "resolved": "https://registry.npmjs.org/pg-pool/-/pg-pool-3.6.2.tgz", - "integrity": "sha512-Htjbg8BlwXqSBQ9V8Vjtc+vzf/6fVUuak/3/XXKA9oxZprwW3IMDQTGHP+KDmVL7rtd+R1QjbnCFPuTHm3G4hg==", - "peerDependencies": { - "pg": ">=8.0" - } - }, "node_modules/pg-protocol": { "version": "1.6.1", "resolved": "https://registry.npmjs.org/pg-protocol/-/pg-protocol-1.6.1.tgz", @@ -18265,14 +18219,6 @@ "node": ">=4" } }, - "node_modules/pgpass": { - "version": "1.0.5", - "resolved": "https://registry.npmjs.org/pgpass/-/pgpass-1.0.5.tgz", - "integrity": "sha512-FdW9r/jQZhSeohs1Z3sI1yxFQNFvMcnmfuj4WBMUTxOrAyLMaTcE1aAMBiTlbMNaXvBCQuVi0R7hd8udDSP7ug==", - "dependencies": { - "split2": "^4.1.0" - } - }, "node_modules/picocolors": { "version": "1.0.0", "resolved": "https://registry.npmjs.org/picocolors/-/picocolors-1.0.0.tgz", @@ -35206,41 +35152,11 @@ "integrity": "sha512-gDKb8aZMDeD/tZWs9P6+q0J9Mwkdl6xMV8TjnGP3qJVJ06bdMgkbBlLU8IdfOsIsFz2BW1rNVT3XuNEl8zPAvw==", "dev": true }, - "pg": { - "version": "8.11.5", - "resolved": "https://registry.npmjs.org/pg/-/pg-8.11.5.tgz", - "integrity": "sha512-jqgNHSKL5cbDjFlHyYsCXmQDrfIX/3RsNwYqpd4N0Kt8niLuNoRNH+aazv6cOd43gPh9Y4DjQCtb+X0MH0Hvnw==", - "requires": { - "pg-cloudflare": "^1.1.1", - "pg-connection-string": "^2.6.4", - "pg-pool": "^3.6.2", - "pg-protocol": "^1.6.1", - "pg-types": "^2.1.0", - "pgpass": "1.x" - } - }, - "pg-cloudflare": { - "version": "1.1.1", - "resolved": "https://registry.npmjs.org/pg-cloudflare/-/pg-cloudflare-1.1.1.tgz", - "integrity": "sha512-xWPagP/4B6BgFO+EKz3JONXv3YDgvkbVrGw2mTo3D6tVDQRh1e7cqVGvyR3BE+eQgAvx1XhW/iEASj4/jCWl3Q==", - "optional": true - }, - "pg-connection-string": { - "version": "2.6.4", - "resolved": "https://registry.npmjs.org/pg-connection-string/-/pg-connection-string-2.6.4.tgz", - "integrity": "sha512-v+Z7W/0EO707aNMaAEfiGnGL9sxxumwLl2fJvCQtMn9Fxsg+lPpPkdcyBSv/KFgpGdYkMfn+EI1Or2EHjpgLCA==" - }, "pg-int8": { "version": "1.0.1", "resolved": "https://registry.npmjs.org/pg-int8/-/pg-int8-1.0.1.tgz", "integrity": "sha512-WCtabS6t3c8SkpDBUlb1kjOs7l66xsGdKpIPZsg4wR+B3+u9UAum2odSsF9tnvxg80h4ZxLWMy4pRjOsFIqQpw==" }, - "pg-pool": { - "version": "3.6.2", - "resolved": "https://registry.npmjs.org/pg-pool/-/pg-pool-3.6.2.tgz", - "integrity": "sha512-Htjbg8BlwXqSBQ9V8Vjtc+vzf/6fVUuak/3/XXKA9oxZprwW3IMDQTGHP+KDmVL7rtd+R1QjbnCFPuTHm3G4hg==", - "requires": {} - }, "pg-protocol": { "version": "1.6.1", "resolved": "https://registry.npmjs.org/pg-protocol/-/pg-protocol-1.6.1.tgz", @@ -35258,14 +35174,6 @@ "postgres-interval": "^1.1.0" } }, - "pgpass": { - "version": "1.0.5", - "resolved": "https://registry.npmjs.org/pgpass/-/pgpass-1.0.5.tgz", - "integrity": "sha512-FdW9r/jQZhSeohs1Z3sI1yxFQNFvMcnmfuj4WBMUTxOrAyLMaTcE1aAMBiTlbMNaXvBCQuVi0R7hd8udDSP7ug==", - "requires": { - "split2": "^4.1.0" - } - }, "picocolors": { "version": "1.0.0", "resolved": "https://registry.npmjs.org/picocolors/-/picocolors-1.0.0.tgz", diff --git a/package.json b/package.json index 11a9e5d1..b7148db5 100644 --- a/package.json +++ b/package.json @@ -23,11 +23,7 @@ "assets:copy": "copyfiles -f ./config/* ./dist/config && copyfiles -f ./openapi3.yaml ./dist/ && copyfiles ./package.json dist", "clean": "rimraf dist", "install": "npx husky install", - "dev:scripts": "npx ts-node ./devScripts/index.ts", - "typeorm": "node --require ts-node/register ./node_modules/typeorm/cli.js -d dataSource.ts", - "migration:create": "npm run typeorm migration:generate", - "migration:run": "npm run typeorm migration:run", - "migration:revert": "npm run typeorm migration:revert" + "dev:scripts": "npx ts-node ./devScripts/index.ts" }, "directories": { "test": "tests" @@ -74,7 +70,6 @@ "mgrs": "^2.0.0", "node-cron": "^3.0.3", "node-fetch-commonjs": "^3.3.2", - "pg": "^8.11.5", "proj4": "^2.11.0", "redis": "^4.7.0", "reflect-metadata": "^0.1.13", diff --git a/src/common/interfaces.ts b/src/common/interfaces.ts index 7dbaabb2..9b8138ed 100644 --- a/src/common/interfaces.ts +++ b/src/common/interfaces.ts @@ -1,4 +1,3 @@ -import { DataSourceOptions } from 'typeorm'; import { RedisClientOptions } from 'redis'; import { BBox, Feature, FeatureCollection as GeoJSONFeatureCollection } from 'geojson'; @@ -21,11 +20,6 @@ export type RedisConfig = { sslPaths: { ca: string; cert: string; key: string }; } & RedisClientOptions; -export type PostgresDbConfig = { - enableSslAuth: boolean; - sslPaths: { ca: string; cert: string; key: string }; -} & DataSourceOptions; - export interface GeoContext { bbox?: BBox; radius?: number; diff --git a/src/index.ts b/src/index.ts index 2ed8f243..a2f0f74b 100644 --- a/src/index.ts +++ b/src/index.ts @@ -7,7 +7,7 @@ import { DependencyContainer } from 'tsyringe'; import { createTerminus } from '@godaddy/terminus'; import { Logger } from '@map-colonies/js-logger'; import config from 'config'; -import { DEFAULT_SERVER_PORT, ON_SIGNAL, SERVICES } from './common/constants'; +import { DEFAULT_SERVER_PORT, HEALTHCHECK, ON_SIGNAL, SERVICES } from './common/constants'; import { getApp } from './app'; let depContainer: DependencyContainer | undefined; From 5e6ff685e9f9a11d1ad87f4c203af640e14e7624 Mon Sep 17 00:00:00 2001 From: NatalieShaked Date: Thu, 12 Sep 2024 14:55:22 +0300 Subject: [PATCH 15/20] fix: added missing closing bracket --- config/custom-environment-variables.json | 43 ++++++++++++------------ 1 file changed, 22 insertions(+), 21 deletions(-) diff --git a/config/custom-environment-variables.json b/config/custom-environment-variables.json index b56daf27..f4f7a4ff 100644 --- a/config/custom-environment-variables.json +++ b/config/custom-environment-variables.json @@ -100,27 +100,28 @@ "__name": "REDIS_DATABASE", "__format": "number" } - }, - "application": { - "services": { - "tokenTypesUrl": "TOKEN_TYPE_URL" - }, - "sources": { - "__name": "GEOTEXT_SOURCES", - "__format": "json" - }, - "regions": { - "__name": "REGIONS", - "__format": "json" - }, - "nameTranslationsKeys": { - "__name": "NAME_TRANSLATION_KEYS", - "__format": "json" }, - "controlObjectDisplayNamePrefixes": { - "__name": "CONTROL_DISPLAY_KEY_TRANSLATIONS", - "__format": "json" - }, - "mainLanguageRegex": "MAIN_LANGUAGE_REGEX" + "application": { + "services": { + "tokenTypesUrl": "TOKEN_TYPE_URL" + }, + "sources": { + "__name": "GEOTEXT_SOURCES", + "__format": "json" + }, + "regions": { + "__name": "REGIONS", + "__format": "json" + }, + "nameTranslationsKeys": { + "__name": "NAME_TRANSLATION_KEYS", + "__format": "json" + }, + "controlObjectDisplayNamePrefixes": { + "__name": "CONTROL_DISPLAY_KEY_TRANSLATIONS", + "__format": "json" + }, + "mainLanguageRegex": "MAIN_LANGUAGE_REGEX" + } } } From 259a029f01d8f088524b661a8bbb0637ac71747e Mon Sep 17 00:00:00 2001 From: NatalieShaked Date: Thu, 12 Sep 2024 14:57:25 +0300 Subject: [PATCH 16/20] fix: added missing closing bracket --- config/test.json | 59 ++++++++++++++++++++++++------------------------ 1 file changed, 30 insertions(+), 29 deletions(-) diff --git a/config/test.json b/config/test.json index 285bbc49..1018cd67 100644 --- a/config/test.json +++ b/config/test.json @@ -56,35 +56,36 @@ "cert": "" }, "database": 1 - }, - "application": { - "services": { - "tokenTypesUrl": "http://localhost:5001/NLP_ANALYSES" }, - "cronLoadTileLatLonDataPattern": "0 * * * *", - "elasticQueryBoosts": { - "name": 1.1, - "placeType": 1.1, - "subPlaceType": 1.1, - "hierarchy": 1.1, - "viewbox": 1.1 - }, - "sources": { - "OSM": "OSM", - "GOOGLE": "GOOGLE" - }, - "regions": { - "USA": ["New York", "Los Angeles"], - "FRANCE": ["Paris"] - }, - "controlObjectDisplayNamePrefixes": { - "TILE": "Tile", - "SUB_TILE": "Sub Tile", - "ROUTE": "Route", - "ITEM": "Item", - "CONTROL_POINT": "Control Point" - }, - "nameTranslationsKeys": ["en", "fr"], - "mainLanguageRegex": "[a-zA-Z]" + "application": { + "services": { + "tokenTypesUrl": "http://localhost:5001/NLP_ANALYSES" + }, + "cronLoadTileLatLonDataPattern": "0 * * * *", + "elasticQueryBoosts": { + "name": 1.1, + "placeType": 1.1, + "subPlaceType": 1.1, + "hierarchy": 1.1, + "viewbox": 1.1 + }, + "sources": { + "OSM": "OSM", + "GOOGLE": "GOOGLE" + }, + "regions": { + "USA": ["New York", "Los Angeles"], + "FRANCE": ["Paris"] + }, + "controlObjectDisplayNamePrefixes": { + "TILE": "Tile", + "SUB_TILE": "Sub Tile", + "ROUTE": "Route", + "ITEM": "Item", + "CONTROL_POINT": "Control Point" + }, + "nameTranslationsKeys": ["en", "fr"], + "mainLanguageRegex": "[a-zA-Z]" + } } } From 5084e3b7454b745288211527d591b45741a1b6a6 Mon Sep 17 00:00:00 2001 From: NatalieShaked Date: Thu, 12 Sep 2024 15:00:23 +0300 Subject: [PATCH 17/20] fix: application placement --- config/custom-environment-variables.json | 44 ++++++++--------- config/test.json | 60 ++++++++++++------------ 2 files changed, 52 insertions(+), 52 deletions(-) diff --git a/config/custom-environment-variables.json b/config/custom-environment-variables.json index f4f7a4ff..48e05ca7 100644 --- a/config/custom-environment-variables.json +++ b/config/custom-environment-variables.json @@ -100,28 +100,28 @@ "__name": "REDIS_DATABASE", "__format": "number" } - }, - "application": { - "services": { - "tokenTypesUrl": "TOKEN_TYPE_URL" - }, - "sources": { - "__name": "GEOTEXT_SOURCES", - "__format": "json" - }, - "regions": { - "__name": "REGIONS", - "__format": "json" - }, - "nameTranslationsKeys": { - "__name": "NAME_TRANSLATION_KEYS", - "__format": "json" - }, - "controlObjectDisplayNamePrefixes": { - "__name": "CONTROL_DISPLAY_KEY_TRANSLATIONS", - "__format": "json" - }, - "mainLanguageRegex": "MAIN_LANGUAGE_REGEX" } + }, + "application": { + "services": { + "tokenTypesUrl": "TOKEN_TYPE_URL" + }, + "sources": { + "__name": "GEOTEXT_SOURCES", + "__format": "json" + }, + "regions": { + "__name": "REGIONS", + "__format": "json" + }, + "nameTranslationsKeys": { + "__name": "NAME_TRANSLATION_KEYS", + "__format": "json" + }, + "controlObjectDisplayNamePrefixes": { + "__name": "CONTROL_DISPLAY_KEY_TRANSLATIONS", + "__format": "json" + }, + "mainLanguageRegex": "MAIN_LANGUAGE_REGEX" } } diff --git a/config/test.json b/config/test.json index 1018cd67..eb85c46e 100644 --- a/config/test.json +++ b/config/test.json @@ -56,36 +56,36 @@ "cert": "" }, "database": 1 - }, - "application": { - "services": { - "tokenTypesUrl": "http://localhost:5001/NLP_ANALYSES" - }, - "cronLoadTileLatLonDataPattern": "0 * * * *", - "elasticQueryBoosts": { - "name": 1.1, - "placeType": 1.1, - "subPlaceType": 1.1, - "hierarchy": 1.1, - "viewbox": 1.1 - }, - "sources": { - "OSM": "OSM", - "GOOGLE": "GOOGLE" - }, - "regions": { - "USA": ["New York", "Los Angeles"], - "FRANCE": ["Paris"] - }, - "controlObjectDisplayNamePrefixes": { - "TILE": "Tile", - "SUB_TILE": "Sub Tile", - "ROUTE": "Route", - "ITEM": "Item", - "CONTROL_POINT": "Control Point" - }, - "nameTranslationsKeys": ["en", "fr"], - "mainLanguageRegex": "[a-zA-Z]" } + }, + "application": { + "services": { + "tokenTypesUrl": "http://localhost:5001/NLP_ANALYSES" + }, + "cronLoadTileLatLonDataPattern": "0 * * * *", + "elasticQueryBoosts": { + "name": 1.1, + "placeType": 1.1, + "subPlaceType": 1.1, + "hierarchy": 1.1, + "viewbox": 1.1 + }, + "sources": { + "OSM": "OSM", + "GOOGLE": "GOOGLE" + }, + "regions": { + "USA": ["New York", "Los Angeles"], + "FRANCE": ["Paris"] + }, + "controlObjectDisplayNamePrefixes": { + "TILE": "Tile", + "SUB_TILE": "Sub Tile", + "ROUTE": "Route", + "ITEM": "Item", + "CONTROL_POINT": "Control Point" + }, + "nameTranslationsKeys": ["en", "fr"], + "mainLanguageRegex": "[a-zA-Z]" } } From 5955e131aa7e3d70ac6edca61b6ccf19582acaea Mon Sep 17 00:00:00 2001 From: NatalieShaked Date: Sun, 15 Sep 2024 15:07:26 +0300 Subject: [PATCH 18/20] fix: made minor fixes to the code in order for it to be more tidy --- package-lock.json | 11 ----------- package.json | 1 - src/common/errors.ts | 8 +++++++- src/common/interfaces.ts | 8 -------- src/common/middlewares/feedbackApi.middleware.ts | 1 - src/common/redis/index.ts | 3 ++- src/common/redis/interfaces.ts | 8 ++++++++ 7 files changed, 17 insertions(+), 23 deletions(-) create mode 100644 src/common/redis/interfaces.ts diff --git a/package-lock.json b/package-lock.json index 45416fbb..4632fa1b 100644 --- a/package-lock.json +++ b/package-lock.json @@ -14,7 +14,6 @@ "@elastic/elasticsearch": "^8.13.1", "@godaddy/terminus": "^4.12.1", "@map-colonies/cleanup-registry": "^1.1.0", - "@map-colonies/detiler-common": "^1.0.0", "@map-colonies/error-express-handler": "^2.1.0", "@map-colonies/express-access-log-middleware": "^2.0.1", "@map-colonies/js-logger": "^1.0.1", @@ -4855,11 +4854,6 @@ "tiny-typed-emitter": "^2.1.0" } }, - "node_modules/@map-colonies/detiler-common": { - "version": "1.0.0", - "resolved": "https://registry.npmjs.org/@map-colonies/detiler-common/-/detiler-common-1.0.0.tgz", - "integrity": "sha512-5gxowEjHBD9CcgqonYQGyarIXRElYnhO4qO/ffxN3IFieWyUWUuKFhc+U6rnvy53TN0UorPfTuOrCO85W9ZsKw==" - }, "node_modules/@map-colonies/error-express-handler": { "version": "2.1.0", "resolved": "https://registry.npmjs.org/@map-colonies/error-express-handler/-/error-express-handler-2.1.0.tgz", @@ -24876,11 +24870,6 @@ "tiny-typed-emitter": "^2.1.0" } }, - "@map-colonies/detiler-common": { - "version": "1.0.0", - "resolved": "https://registry.npmjs.org/@map-colonies/detiler-common/-/detiler-common-1.0.0.tgz", - "integrity": "sha512-5gxowEjHBD9CcgqonYQGyarIXRElYnhO4qO/ffxN3IFieWyUWUuKFhc+U6rnvy53TN0UorPfTuOrCO85W9ZsKw==" - }, "@map-colonies/error-express-handler": { "version": "2.1.0", "resolved": "https://registry.npmjs.org/@map-colonies/error-express-handler/-/error-express-handler-2.1.0.tgz", diff --git a/package.json b/package.json index b7148db5..9f55b225 100644 --- a/package.json +++ b/package.json @@ -46,7 +46,6 @@ "@elastic/elasticsearch": "^8.13.1", "@godaddy/terminus": "^4.12.1", "@map-colonies/cleanup-registry": "^1.1.0", - "@map-colonies/detiler-common": "^1.0.0", "@map-colonies/error-express-handler": "^2.1.0", "@map-colonies/express-access-log-middleware": "^2.0.1", "@map-colonies/js-logger": "^1.0.1", diff --git a/src/common/errors.ts b/src/common/errors.ts index 374b146c..f23ed0cd 100644 --- a/src/common/errors.ts +++ b/src/common/errors.ts @@ -50,4 +50,10 @@ export class NotImplementedError extends Error implements HttpError { } } -export class TimeoutError extends Error {} +export class TimeoutError extends Error implements HttpError { + public readonly status = httpStatus.REQUEST_TIMEOUT; + + public constructor(message: string) { + super(message); + } +} diff --git a/src/common/interfaces.ts b/src/common/interfaces.ts index 9b8138ed..a120b1ad 100644 --- a/src/common/interfaces.ts +++ b/src/common/interfaces.ts @@ -1,4 +1,3 @@ -import { RedisClientOptions } from 'redis'; import { BBox, Feature, FeatureCollection as GeoJSONFeatureCollection } from 'geojson'; export interface IConfig { @@ -13,13 +12,6 @@ export interface OpenApiConfig { uiPath: string; } -export type RedisConfig = { - host: string; - port: number; - enableSslAuth: boolean; - sslPaths: { ca: string; cert: string; key: string }; -} & RedisClientOptions; - export interface GeoContext { bbox?: BBox; radius?: number; diff --git a/src/common/middlewares/feedbackApi.middleware.ts b/src/common/middlewares/feedbackApi.middleware.ts index e3774b2b..193d0443 100644 --- a/src/common/middlewares/feedbackApi.middleware.ts +++ b/src/common/middlewares/feedbackApi.middleware.ts @@ -30,7 +30,6 @@ export class FeedbackApiMiddlewareManager { const originalJson = res.json; const logJson = async function (this: Response, body: JSON): Promise { - // console.log('Response:', body); geocodingResponseDetails.response = body; try { diff --git a/src/common/redis/index.ts b/src/common/redis/index.ts index 357badc1..514c3ed5 100644 --- a/src/common/redis/index.ts +++ b/src/common/redis/index.ts @@ -3,7 +3,8 @@ import { Logger } from '@map-colonies/js-logger'; import { createClient, RedisClientOptions } from 'redis'; import { DependencyContainer, FactoryFunction } from 'tsyringe'; import { SERVICES } from '../constants'; -import { RedisConfig, IConfig } from '../interfaces'; +import { IConfig } from '../interfaces'; +import { RedisConfig } from './interfaces'; const DEFAULT_LIMIT_FROM = 0; const DEFAULT_LIMIT_SIZE = 1000; diff --git a/src/common/redis/interfaces.ts b/src/common/redis/interfaces.ts new file mode 100644 index 00000000..d569f1eb --- /dev/null +++ b/src/common/redis/interfaces.ts @@ -0,0 +1,8 @@ +import { RedisClientOptions } from 'redis'; + +export type RedisConfig = { + host: string; + port: number; + enableSslAuth: boolean; + sslPaths: { ca: string; cert: string; key: string }; + } & RedisClientOptions; \ No newline at end of file From e2ca1b131043658b9eab0ee2a5e38ade0cbcce22 Mon Sep 17 00:00:00 2001 From: NatalieShaked Date: Mon, 23 Sep 2024 14:13:18 +0300 Subject: [PATCH 19/20] fix: feedback sends to redis without adding time to the response --- src/common/constants.ts | 1 - .../middlewares/feedbackApi.middleware.ts | 24 ++++++++++--------- src/common/redis/interfaces.ts | 10 ++++---- 3 files changed, 18 insertions(+), 17 deletions(-) diff --git a/src/common/constants.ts b/src/common/constants.ts index ee1c5880..6573ef73 100644 --- a/src/common/constants.ts +++ b/src/common/constants.ts @@ -28,4 +28,3 @@ export const REDIS_SYMBOL = Symbol('REDIS'); export const REDIS_TTL = 300; export const elasticConfigPath = 'db.elastic'; -export const siteIndex = 1; diff --git a/src/common/middlewares/feedbackApi.middleware.ts b/src/common/middlewares/feedbackApi.middleware.ts index 193d0443..c8c6f87f 100644 --- a/src/common/middlewares/feedbackApi.middleware.ts +++ b/src/common/middlewares/feedbackApi.middleware.ts @@ -2,17 +2,19 @@ import * as crypto from 'node:crypto'; import { Request, Response, NextFunction } from 'express'; import { Logger } from '@map-colonies/js-logger'; import { inject, injectable } from 'tsyringe'; -import { REDIS_TTL, SERVICES, siteIndex } from '../constants'; +import { REDIS_TTL, SERVICES } from '../constants'; import { RedisClient } from '../redis'; import { GeocodingResponse } from '../interfaces'; +const siteIndex = 1; + @injectable() export class FeedbackApiMiddlewareManager { public constructor(@inject(SERVICES.LOGGER) private readonly logger: Logger, @inject(SERVICES.REDIS) private readonly redis: RedisClient) {} // eslint-disable-next-line @typescript-eslint/explicit-function-return-type public saveResponses = (req: Request, res: Response, next: NextFunction) => { - const reqId = res.getHeader('request-id'); + const reqId = res.getHeader('x-request-id'); const redisClient = this.redis; const logger = this.logger; @@ -29,25 +31,25 @@ export class FeedbackApiMiddlewareManager { }; const originalJson = res.json; - const logJson = async function (this: Response, body: JSON): Promise { + const logJson = function (this: Response, body: JSON): Response { geocodingResponseDetails.response = body; + redisClient + .setEx(reqId as string, REDIS_TTL, JSON.stringify(geocodingResponseDetails)) + .then(() => { + logger.info({ msg: `response ${reqId?.toString() ?? ''} saved to redis` }); + }) + .catch((error) => logger.error('Error setting key:', error)); - try { - await redisClient.setEx(reqId as string, REDIS_TTL, JSON.stringify(geocodingResponseDetails)); - logger.info({ msg: 'saving response to redis' }); - } catch (err) { - logger.error('Error setting key:', err); - } return originalJson.call(this, body); }; - res.json = logJson as unknown as Response['json']; + res.json = logJson; next(); }; // eslint-disable-next-line @typescript-eslint/explicit-function-return-type public setRequestId = (req: Request, res: Response, next: NextFunction) => { const reqId = crypto.randomUUID(); - res.append('request-id', reqId); + res.append('x-request-id', reqId); next(); }; } diff --git a/src/common/redis/interfaces.ts b/src/common/redis/interfaces.ts index d569f1eb..145a202b 100644 --- a/src/common/redis/interfaces.ts +++ b/src/common/redis/interfaces.ts @@ -1,8 +1,8 @@ import { RedisClientOptions } from 'redis'; export type RedisConfig = { - host: string; - port: number; - enableSslAuth: boolean; - sslPaths: { ca: string; cert: string; key: string }; - } & RedisClientOptions; \ No newline at end of file + host: string; + port: number; + enableSslAuth: boolean; + sslPaths: { ca: string; cert: string; key: string }; +} & RedisClientOptions; From e5dd67fe1855b2874c65abed04faea4f8fdec33a Mon Sep 17 00:00:00 2001 From: NatalieShaked Date: Mon, 23 Sep 2024 15:52:32 +0300 Subject: [PATCH 20/20] fix: minor fixes for code clearness --- src/common/constants.ts | 6 ++-- src/common/interfaces.ts | 2 +- .../middlewares/feedbackApi.middleware.ts | 28 +++++++++++-------- src/common/middlewares/utils.ts | 5 ++++ 4 files changed, 25 insertions(+), 16 deletions(-) create mode 100644 src/common/middlewares/utils.ts diff --git a/src/common/constants.ts b/src/common/constants.ts index 6573ef73..c1044083 100644 --- a/src/common/constants.ts +++ b/src/common/constants.ts @@ -1,9 +1,7 @@ -import { hostname } from 'os'; import { readPackageJsonSync } from '@map-colonies/read-pkg'; export const SERVICE_NAME = readPackageJsonSync().name ?? 'unknown_service'; export const DEFAULT_SERVER_PORT = 80; -export const HOSTNAME = hostname(); export const IGNORED_OUTGOING_TRACE_ROUTES = [/^.*\/v1\/metrics.*$/]; export const IGNORED_INCOMING_TRACE_ROUTES = [/^.*\/docs.*$/]; @@ -24,7 +22,7 @@ export const SERVICES: Record = { export const ON_SIGNAL = Symbol('onSignal'); export const HEALTHCHECK = Symbol('healthcheck'); -export const REDIS_SYMBOL = Symbol('REDIS'); -export const REDIS_TTL = 300; export const elasticConfigPath = 'db.elastic'; +export const s3EndpointConfig = 'db.s3.endpoint'; + diff --git a/src/common/interfaces.ts b/src/common/interfaces.ts index a120b1ad..14cd497b 100644 --- a/src/common/interfaces.ts +++ b/src/common/interfaces.ts @@ -61,7 +61,7 @@ export enum GeoContextMode { BIAS = 'bias', } -export interface GeocodingResponse { +export interface FeebackApiGeocodingResponse { userId: string; apiKey: string; site: string; diff --git a/src/common/middlewares/feedbackApi.middleware.ts b/src/common/middlewares/feedbackApi.middleware.ts index c8c6f87f..33b1b1d8 100644 --- a/src/common/middlewares/feedbackApi.middleware.ts +++ b/src/common/middlewares/feedbackApi.middleware.ts @@ -2,30 +2,36 @@ import * as crypto from 'node:crypto'; import { Request, Response, NextFunction } from 'express'; import { Logger } from '@map-colonies/js-logger'; import { inject, injectable } from 'tsyringe'; -import { REDIS_TTL, SERVICES } from '../constants'; +import { SERVICES, s3EndpointConfig } from '../constants'; import { RedisClient } from '../redis'; -import { GeocodingResponse } from '../interfaces'; +import { FeebackApiGeocodingResponse, IConfig } from '../interfaces'; +import { XApi } from './utils'; -const siteIndex = 1; +const SITE_INDEX = 1; +const REDIS_TTL = 300; @injectable() export class FeedbackApiMiddlewareManager { - public constructor(@inject(SERVICES.LOGGER) private readonly logger: Logger, @inject(SERVICES.REDIS) private readonly redis: RedisClient) {} + public constructor( + @inject(SERVICES.LOGGER) private readonly logger: Logger, + @inject(SERVICES.REDIS) private readonly redis: RedisClient, + @inject(SERVICES.CONFIG) private readonly config: IConfig + ) {} // eslint-disable-next-line @typescript-eslint/explicit-function-return-type public saveResponses = (req: Request, res: Response, next: NextFunction) => { - const reqId = res.getHeader('x-request-id'); + const reqId = res.getHeader(XApi.REQUEST); const redisClient = this.redis; const logger = this.logger; - const s3Endpoint = ''; //s3 endpoint from default + const s3Endpoint = this.config.get(s3EndpointConfig); const drSite = s3Endpoint.split('.'); logger.info({ msg: 'saving response to redis' }); - const geocodingResponseDetails: GeocodingResponse = { - userId: req.headers['x-user-id'] as string, - apiKey: req.headers['x-api-key'] as string, - site: drSite[siteIndex], + const geocodingResponseDetails: FeebackApiGeocodingResponse = { + userId: req.headers[XApi.USER] as string, + apiKey: req.headers[XApi.KEY] as string, + site: drSite[SITE_INDEX], response: JSON.parse('{}') as JSON, respondedAt: new Date(), }; @@ -49,7 +55,7 @@ export class FeedbackApiMiddlewareManager { // eslint-disable-next-line @typescript-eslint/explicit-function-return-type public setRequestId = (req: Request, res: Response, next: NextFunction) => { const reqId = crypto.randomUUID(); - res.append('x-request-id', reqId); + res.append(XApi.REQUEST, reqId); next(); }; } diff --git a/src/common/middlewares/utils.ts b/src/common/middlewares/utils.ts new file mode 100644 index 00000000..31b6288c --- /dev/null +++ b/src/common/middlewares/utils.ts @@ -0,0 +1,5 @@ +export enum XApi { + REQUEST = 'x-request-id', + USER = 'x-user-id', + KEY = 'x-api-key', +}