fix: support config from values (MAPCO-10525)

README.md

@@ -93,6 +93,99 @@ npm run start
 
 ```
 
+## Run Migrations
+Run migrations before you start the app
+
+## Migrations Development
+* Update metadata file or change DB details (fakeDB for example)
+* npm run migration:create
+
+### Shell
+Run the following command:
+
+```sh
+npm run migration:run
+```
+
+### Docker
+Build the migrations image:
+
+```sh
+docker build -t sync-layer-server-migration:latest -f migrations.Dockerfile .
+```
+Run image:
+```sh
+docker run -it --rm --network host sync-layer-server-migration:latest
+```
+
+If you want to change the connection properties you can do it via either:
+1. Env variables
+2. Inject a config file based on your environment
+
+
+Via env variables:
+```sh
+docker run -it -e DB_USERNAME=VALUE  -e DB_PASSWORD=VALUE -e DB_NAME=VALUE -e DB_TYPE=VALUE -e DB_HOST=VALUE -e DB_PORT=VALUE --rm --network host sync-layer-server-migration:latest
+```
+
+#### SSL/TLS Configuration
+
+For secure database connections using SSL certificates:
+
+**Environment Variables:**
+- `DB_ENABLE_SSL` - Set to `true` to enable SSL (default: `false`)
+- `DB_SSL_KEY_PATH` - Path to client private key file
+- `DB_SSL_CERT_PATH` - Path to client certificate file
+- `DB_SSL_CA_PATH` - Path to CA certificate file
+
+**Example with SSL:**
+```sh
+docker run -it \
+  -e DB_HOST=your-db-host \
+  -e DB_PORT=5432 \
+  -e DB_USERNAME=postgres \
+  -e DB_PASSWORD=postgres \
+  -e DB_NAME=sync-layer \
+  -e DB_ENABLE_SSL=true \
+  -e DB_SSL_KEY_PATH=/app/certs/client-key.pem \
+  -e DB_SSL_CERT_PATH=/app/certs/client-cert.pem \
+  -e DB_SSL_CA_PATH=/app/certs/ca.pem \
+  -v /path/to/certs:/app/certs:ro \
+  --rm --network host \
+  sync-layer-server-migration:latest
+```
+
+Via injecting a config file, assuming you want to run the migration on your production:
+
+production.json:
+```json
+{
+  "openapiConfig": {
+    "filePath": "./openapi3.yaml",
+    "basePath": "/docs",
+    "rawPath": "/api",
+    "uiPath": "/api"
+  },
+  "logger": {
+    "level": "info"
+  },
+  "server": {
+    "port": "8085"
+  },
+  "db": {
+    "type": "postgres",
+    "username": "postgres",
+    "password": "postgres",
+    "database": "catalog",
+    "port": 5432
+  }
+}
+```
+```sh
+docker run -it --rm -e NODE_ENV=production --network host -v /path/to/proudction.json:/usr/app/config/production.json sync-layer-server-migrations:latest
+```
+---
+
 ## Running Tests
 
 To run tests, run the following command

config/default.json

@@ -43,7 +43,7 @@
     "type": "postgres",
     "host": "localhost",
     "port": 5432,
-    "database": "postgres",
+    "database": "sync_layer_dev",
     "username": "postgres",
     "password": "postgres",
     "enableSslAuth": false,

config/test.json

@@ -1 +1,7 @@
-{}
+{
+  "db": {
+    "host": "127.0.0.1'",
+    "port": 55432,
+    "database": "sync_layer_test"
+  }
+}

helm/templates/_tplValues.tpl

@@ -0,0 +1,48 @@
+{{/*
+Copyright VMware, Inc.
+SPDX-License-Identifier: APACHE-2.0
+*/}}
+
+{{/* vim: set filetype=mustache: */}}
+{{/*
+Renders a value that contains template perhaps with scope if the scope is present.
+Usage:
+{{ include "common.tplvalues.render" ( dict "value" .Values.path.to.the.Value "context" $ ) }}
+{{ include "common.tplvalues.render" ( dict "value" .Values.path.to.the.Value "context" $ "scope" $app ) }}
+*/}}
+{{- define "common.tplvalues.render" -}}
+{{- $value := typeIs "string" .value | ternary .value (.value | toYaml) }}
+{{- if contains "{{" (toJson .value) }}
+  {{- if .scope }}
+      {{- tpl (cat "{{- with $.RelativeScope -}}" $value "{{- end }}") (merge (dict "RelativeScope" .scope) .context) }}
+  {{- else }}
+    {{- tpl $value .context }}
+  {{- end }}
+{{- else }}
+    {{- $value }}
+{{- end }}
+{{- end -}}
+
+{{/*
+Merge a list of values that contains template after rendering them.
+Merge precedence is consistent with http://masterminds.github.io/sprig/dicts.html#merge-mustmerge
+Usage:
+{{ include "common.tplvalues.merge" ( dict "values" (list .Values.path.to.the.Value1 .Values.path.to.the.Value2) "context" $ ) }}
+*/}}
+{{- define "common.tplvalues.merge" -}}
+{{- $dst := dict -}}
+{{- range .values -}}
+{{- $dst = include "common.tplvalues.render" (dict "value" . "context" $.context "scope" $.scope) | fromYaml | merge $dst -}}
+{{- end -}}
+{{ $dst | toYaml }}
+{{- end -}}
+{{/*
+End of usage example
+*/}}
+
+{{/*
+Custom definitions
+*/}}
+{{- define "merged.postgres" -}}
+{{- include "common.tplvalues.merge" ( dict "values" ( list .Values.postgres .Values.global.postgres ) "context" . ) }}
+{{- end -}}

helm/templates/configmap.yaml

@@ -1,5 +1,6 @@
 {{- $tracingUrl := include "sync-layer-server.tracingUrl" . -}}
 {{- $metricsUrl := include "sync-layer-server.metricsUrl" . -}}
+{{- $postgres := (include "merged.postgres" . ) | fromYaml }}
 {{- if .Values.enabled -}}
 apiVersion: v1
 kind: ConfigMap
@@ -19,13 +20,13 @@ data:
   TELEMETRY_METRICS_URL: {{ $metricsUrl }}
   {{ end }}
   npm_config_cache: /tmp/
-  {{- with .Values.configManagement }}
+  {{ with .Values.configManagement }}
   CONFIG_NAME: {{ .name | quote }}
   CONFIG_VERSION: {{ .version | quote }}
   CONFIG_OFFLINE_MODE: {{ .offlineMode | quote }}
   CONFIG_SERVER_URL: {{ .serverUrl | quote }}
-  {{- end -}}
-  {{- with .Values.env.sync }}
+  {{ end }}
+  {{ with .Values.env.sync }}
   LAYERS: {{ .layers | toJson | quote }}
   SYNC_INTERVAL_MS: {{ .syncIntervalMs | quote }}
   POLL_INTERVAL_MS: {{ .pollIntervalMs | quote }}
@@ -36,5 +37,13 @@ data:
   REQUESTING_SYSTEM_NAME: {{ .requestingSystemName | quote }}
   USE_DELETE_ENTITIES: {{ .useDeleteEntities | quote }}
   AUTH_TOKEN: {{ .authToken | quote }}
-  {{- end -}}
-{{- end }}
+  {{ end }}
+  DB_TYPE: "postgres"
+  DB_HOST: {{ quote $postgres.host }}
+  DB_PORT: {{ quote $postgres.port }}
+  DB_NAME: {{ $postgres.name | quote }}
+  DB_CERT_PATH: /tmp/certs/{{ $postgres.ssl.certFileName }}
+  DB_KEY_PATH: /tmp/certs/{{ $postgres.ssl.keyFileName }}
+  DB_CA_PATH: /tmp/certs/{{ $postgres.ssl.caFileName }}
+  DB_ENABLE_SSL_AUTH: {{ $postgres.ssl.enabled | quote }}
+{{ end }}

helm/templates/db-secrets.yaml

@@ -0,0 +1,23 @@
+{{- $releaseName := .Release.Name -}}
+{{- $chartName := include "sync-layer-server.name" . -}}
+{{- $postgres := include "merged.postgres" . | fromYaml }}
+{{- $secretName := default (printf "%s-db-secret" (include "sync-layer-server.fullname" .)) $postgres.user.secretName }}
+{{- if not $postgres.user.useExternal -}}
+apiVersion: v1
+kind: Secret
+metadata:
+  name: {{ $secretName }}
+  annotations:
+    "helm.sh/resource-policy": keep
+  labels:
+    app: {{ $chartName }}
+    component: {{ $chartName }}
+    release: {{ $releaseName }}
+    {{- include "sync-layer-server.labels" . | nindent 4 }}
+type: Opaque
+data:
+  username: {{ $postgres.user.username | b64enc }}
+  {{- if $postgres.user.requirePassword }}
+  password: {{ $postgres.user.password | b64enc }}
+  {{- end }}
+{{- end }}

helm/templates/deployment.yaml

@@ -1,9 +1,11 @@
 {{- $releaseName := .Release.Name -}}
 {{- $chartName := include "sync-layer-server.name" . -}}
+{{- $fullname := include "sync-layer-server.fullname" . -}}
 {{- $cloudProviderFlavor := include "sync-layer-server.cloudProviderFlavor" . -}}
 {{- $cloudProviderDockerRegistryUrl := include "sync-layer-server.cloudProviderDockerRegistryUrl" . -}}
 {{- $cloudProviderImagePullSecretName := include "sync-layer-server.cloudProviderImagePullSecretName" . -}}
 {{- $imageTag := include "sync-layer-server.tag" . -}}
+{{- $postgres := include "merged.postgres" . | fromYaml -}}
 {{- if .Values.enabled -}}
 apiVersion: apps/v1
 kind: Deployment
@@ -66,6 +68,11 @@ spec:
             {{- if .Values.extraVolumeMounts -}}
               {{ toYaml .Values.extraVolumeMounts | nindent 12 }}
             {{- end }}
+            {{- if $postgres.ssl.enabled }}
+            - name: cert-conf
+              mountPath: /tmp/certs
+              readOnly: true
+            {{- end }}
           env:
             - name: K8S_POD_UID
               valueFrom:
@@ -81,7 +88,19 @@ spec:
             {{- end }}
             {{- if .Values.extraEnvVars }}
             {{- toYaml .Values.extraEnvVars | nindent 12 }}
-            {{- end }}            
+            {{- end }}
+            - name: DB_USERNAME
+              valueFrom:
+                secretKeyRef:
+                  name: {{ default (printf "%s-db-secrets" $fullname) $postgres.user.secretName }}
+                  key: username
+            {{- if $postgres.user.requirePassword }}
+            - name: DB_PASSWORD
+              valueFrom:
+                secretKeyRef:
+                  name: {{ default (printf "%s-db-secrets" $fullname) $postgres.user.secretName }}
+                  key: password
+            {{- end }}
           envFrom:
             - configMapRef:
                 name: {{ include "sync-layer-server.fullname" . }}
@@ -117,13 +136,23 @@ spec:
       volumes:
         - name: nginx-config
           configMap:
-            name: 'nginx-extra-configmap'      
+            name: 'nginx-extra-configmap'
+        {{- if and $postgres.enabled $postgres.ssl.enabled }}
+        - name: cert-conf
+          secret:
+            secretName: {{ $postgres.ssl.secretName }}
+        {{- end }}
         {{- if .Values.caSecretName }}
         - name: root-ca
           secret:
             secretName: {{ .Values.caSecretName }}
         {{- end }}
+        {{- if $postgres.ssl.enabled }}
+        - name: postgres-cert
+          secret:
+            secretName: {{ $postgres.ssl.secretName }}
+        {{- end }}
         {{- if .Values.extraVolumes -}}
           {{ tpl (toYaml .Values.extraVolumes) . | nindent 8 }}
-        {{- end }}    
+        {{- end }}
 {{- end -}}

helm/values.yaml

@@ -2,11 +2,30 @@ global:
   cloudProvider: {}
   tracing: {}
   metrics: {}
+  postgres:
+    # host: 127.0.0.1
+    # port: 5432
+    # schema: public
+    # name: sync_layer
+    user:
+      # requirePassword: true
+      # rejectUnauthorized: true
+      useExternal: true
+      secretName: secret-name
+      # username: postgres
+      # password: postgres
+    ssl:
+      # enabled: false
+      useExternal: true
+      secretName: secret-name
+      # caFileName: caFile
+      # certFileName: certFile
+      # keyFileName: keyFile
 
 mclabels:
   component: backend
-  partOf: boilerplates
-  owner: common
+  partOf: yahalom
+  owner: 3d
   prometheus:
     enabled: true
 
@@ -20,8 +39,8 @@ nameOverride: ""
 fullnameOverride: ""
 
 configManagement:
-  offlineMode: false
-  name: 'service-name'
+  offlineMode: true
+  name: 'sync-layer-server'
   version: 'latest'
   serverUrl: 'http://localhost:8080/api'
 

src/dal/db.data-source.ts

@@ -1,39 +1,18 @@
 /* istanbul ignore file */
-import path from 'path';
+import path from 'node:path';
+
 import { DataSource, type DataSourceOptions } from 'typeorm';
-import type { DbConfig } from '../types';
+import { getDbConfig } from '../common/dbConfig';
+import { initConfig } from '../common/config';
 import { createConnectionOptions } from './connectionOptions';
 
-const requireEnv = (name: string): string => {
-  const value = process.env[name];
-  if (value === undefined || value === '') {
-    throw new Error(`Missing required environment variable: ${name}`);
-  }
-  return value;
-};
-
-const dbConfig: DbConfig = {
-  type: 'postgres',
-  host: requireEnv('DB_HOST'),
-  port: parseInt(requireEnv('DB_PORT'), 10),
-  username: requireEnv('DB_USERNAME'),
-  password: requireEnv('DB_PASSWORD'),
-  database: requireEnv('DB_NAME'),
-  enableSslAuth: process.env.DB_ENABLE_SSL?.toLowerCase() === 'true',
-  sslPaths: {
-    key: process.env.DB_SSL_KEY_PATH ?? '',
-    cert: process.env.DB_SSL_CERT_PATH ?? '',
-    ca: process.env.DB_SSL_CA_PATH ?? '',
-  },
-};
-
 const buildConnectionOptions = (): DataSourceOptions => {
-  const options = createConnectionOptions(dbConfig);
+  const options = createConnectionOptions(getDbConfig());
   return {
     ...options,
     migrations: [path.join(__dirname, 'migrations', '*.ts')],
   };
 };
 
 /* eslint-disable @typescript-eslint/naming-convention */
-export default new DataSource(buildConnectionOptions());
+export default initConfig(true).then(() => new DataSource(buildConnectionOptions()));