diff --git a/README.md b/README.md index 57b1cda..f8ffce8 100644 --- a/README.md +++ b/README.md @@ -93,6 +93,99 @@ npm run start ``` +## Run Migrations +Run migrations before you start the app + +## Migrations Development +* Update metadata file or change DB details (fakeDB for example) +* npm run migration:create + +### Shell +Run the following command: + +```sh +npm run migration:run +``` + +### Docker +Build the migrations image: + +```sh +docker build -t sync-layer-server-migration:latest -f migrations.Dockerfile . +``` +Run image: +```sh +docker run -it --rm --network host sync-layer-server-migration:latest +``` + +If you want to change the connection properties you can do it via either: +1. Env variables +2. Inject a config file based on your environment + + +Via env variables: +```sh +docker run -it -e DB_USERNAME=VALUE -e DB_PASSWORD=VALUE -e DB_NAME=VALUE -e DB_TYPE=VALUE -e DB_HOST=VALUE -e DB_PORT=VALUE --rm --network host sync-layer-server-migration:latest +``` + +#### SSL/TLS Configuration + +For secure database connections using SSL certificates: + +**Environment Variables:** +- `DB_ENABLE_SSL` - Set to `true` to enable SSL (default: `false`) +- `DB_SSL_KEY_PATH` - Path to client private key file +- `DB_SSL_CERT_PATH` - Path to client certificate file +- `DB_SSL_CA_PATH` - Path to CA certificate file + +**Example with SSL:** +```sh +docker run -it \ + -e DB_HOST=your-db-host \ + -e DB_PORT=5432 \ + -e DB_USERNAME=postgres \ + -e DB_PASSWORD=postgres \ + -e DB_NAME=sync-layer \ + -e DB_ENABLE_SSL=true \ + -e DB_SSL_KEY_PATH=/app/certs/client-key.pem \ + -e DB_SSL_CERT_PATH=/app/certs/client-cert.pem \ + -e DB_SSL_CA_PATH=/app/certs/ca.pem \ + -v /path/to/certs:/app/certs:ro \ + --rm --network host \ + sync-layer-server-migration:latest +``` + +Via injecting a config file, assuming you want to run the migration on your production: + +production.json: +```json +{ + "openapiConfig": { + "filePath": "./openapi3.yaml", + "basePath": "/docs", + "rawPath": "/api", + "uiPath": "/api" + }, + "logger": { + "level": "info" + }, + "server": { + "port": "8085" + }, + "db": { + "type": "postgres", + "username": "postgres", + "password": "postgres", + "database": "catalog", + "port": 5432 + } +} +``` +```sh +docker run -it --rm -e NODE_ENV=production --network host -v /path/to/proudction.json:/usr/app/config/production.json sync-layer-server-migrations:latest +``` +--- + ## Running Tests To run tests, run the following command diff --git a/config/default.json b/config/default.json index 2f860a1..14b276c 100644 --- a/config/default.json +++ b/config/default.json @@ -43,7 +43,7 @@ "type": "postgres", "host": "localhost", "port": 5432, - "database": "postgres", + "database": "sync_layer_dev", "username": "postgres", "password": "postgres", "enableSslAuth": false, diff --git a/config/test.json b/config/test.json index 0967ef4..e1f347e 100644 --- a/config/test.json +++ b/config/test.json @@ -1 +1,7 @@ -{} +{ + "db": { + "host": "127.0.0.1'", + "port": 55432, + "database": "sync_layer_test" + } +} diff --git a/helm/templates/_tplValues.tpl b/helm/templates/_tplValues.tpl new file mode 100644 index 0000000..369115a --- /dev/null +++ b/helm/templates/_tplValues.tpl @@ -0,0 +1,48 @@ +{{/* +Copyright VMware, Inc. +SPDX-License-Identifier: APACHE-2.0 +*/}} + +{{/* vim: set filetype=mustache: */}} +{{/* +Renders a value that contains template perhaps with scope if the scope is present. +Usage: +{{ include "common.tplvalues.render" ( dict "value" .Values.path.to.the.Value "context" $ ) }} +{{ include "common.tplvalues.render" ( dict "value" .Values.path.to.the.Value "context" $ "scope" $app ) }} +*/}} +{{- define "common.tplvalues.render" -}} +{{- $value := typeIs "string" .value | ternary .value (.value | toYaml) }} +{{- if contains "{{" (toJson .value) }} + {{- if .scope }} + {{- tpl (cat "{{- with $.RelativeScope -}}" $value "{{- end }}") (merge (dict "RelativeScope" .scope) .context) }} + {{- else }} + {{- tpl $value .context }} + {{- end }} +{{- else }} + {{- $value }} +{{- end }} +{{- end -}} + +{{/* +Merge a list of values that contains template after rendering them. +Merge precedence is consistent with http://masterminds.github.io/sprig/dicts.html#merge-mustmerge +Usage: +{{ include "common.tplvalues.merge" ( dict "values" (list .Values.path.to.the.Value1 .Values.path.to.the.Value2) "context" $ ) }} +*/}} +{{- define "common.tplvalues.merge" -}} +{{- $dst := dict -}} +{{- range .values -}} +{{- $dst = include "common.tplvalues.render" (dict "value" . "context" $.context "scope" $.scope) | fromYaml | merge $dst -}} +{{- end -}} +{{ $dst | toYaml }} +{{- end -}} +{{/* +End of usage example +*/}} + +{{/* +Custom definitions +*/}} +{{- define "merged.postgres" -}} +{{- include "common.tplvalues.merge" ( dict "values" ( list .Values.postgres .Values.global.postgres ) "context" . ) }} +{{- end -}} diff --git a/helm/templates/configmap.yaml b/helm/templates/configmap.yaml index 7b64a0f..b460fc7 100644 --- a/helm/templates/configmap.yaml +++ b/helm/templates/configmap.yaml @@ -1,5 +1,6 @@ {{- $tracingUrl := include "sync-layer-server.tracingUrl" . -}} {{- $metricsUrl := include "sync-layer-server.metricsUrl" . -}} +{{- $postgres := (include "merged.postgres" . ) | fromYaml }} {{- if .Values.enabled -}} apiVersion: v1 kind: ConfigMap @@ -19,13 +20,13 @@ data: TELEMETRY_METRICS_URL: {{ $metricsUrl }} {{ end }} npm_config_cache: /tmp/ - {{- with .Values.configManagement }} + {{ with .Values.configManagement }} CONFIG_NAME: {{ .name | quote }} CONFIG_VERSION: {{ .version | quote }} CONFIG_OFFLINE_MODE: {{ .offlineMode | quote }} CONFIG_SERVER_URL: {{ .serverUrl | quote }} - {{- end -}} - {{- with .Values.env.sync }} + {{ end }} + {{ with .Values.env.sync }} LAYERS: {{ .layers | toJson | quote }} SYNC_INTERVAL_MS: {{ .syncIntervalMs | quote }} POLL_INTERVAL_MS: {{ .pollIntervalMs | quote }} @@ -36,5 +37,13 @@ data: REQUESTING_SYSTEM_NAME: {{ .requestingSystemName | quote }} USE_DELETE_ENTITIES: {{ .useDeleteEntities | quote }} AUTH_TOKEN: {{ .authToken | quote }} - {{- end -}} -{{- end }} + {{ end }} + DB_TYPE: "postgres" + DB_HOST: {{ quote $postgres.host }} + DB_PORT: {{ quote $postgres.port }} + DB_NAME: {{ $postgres.name | quote }} + DB_CERT_PATH: /tmp/certs/{{ $postgres.ssl.certFileName }} + DB_KEY_PATH: /tmp/certs/{{ $postgres.ssl.keyFileName }} + DB_CA_PATH: /tmp/certs/{{ $postgres.ssl.caFileName }} + DB_ENABLE_SSL_AUTH: {{ $postgres.ssl.enabled | quote }} +{{ end }} diff --git a/helm/templates/db-secrets.yaml b/helm/templates/db-secrets.yaml new file mode 100644 index 0000000..9455943 --- /dev/null +++ b/helm/templates/db-secrets.yaml @@ -0,0 +1,23 @@ +{{- $releaseName := .Release.Name -}} +{{- $chartName := include "sync-layer-server.name" . -}} +{{- $postgres := include "merged.postgres" . | fromYaml }} +{{- $secretName := default (printf "%s-db-secret" (include "sync-layer-server.fullname" .)) $postgres.user.secretName }} +{{- if not $postgres.user.useExternal -}} +apiVersion: v1 +kind: Secret +metadata: + name: {{ $secretName }} + annotations: + "helm.sh/resource-policy": keep + labels: + app: {{ $chartName }} + component: {{ $chartName }} + release: {{ $releaseName }} + {{- include "sync-layer-server.labels" . | nindent 4 }} +type: Opaque +data: + username: {{ $postgres.user.username | b64enc }} + {{- if $postgres.user.requirePassword }} + password: {{ $postgres.user.password | b64enc }} + {{- end }} +{{- end }} diff --git a/helm/templates/deployment.yaml b/helm/templates/deployment.yaml index ed539c8..97e6abc 100644 --- a/helm/templates/deployment.yaml +++ b/helm/templates/deployment.yaml @@ -1,9 +1,11 @@ {{- $releaseName := .Release.Name -}} {{- $chartName := include "sync-layer-server.name" . -}} +{{- $fullname := include "sync-layer-server.fullname" . -}} {{- $cloudProviderFlavor := include "sync-layer-server.cloudProviderFlavor" . -}} {{- $cloudProviderDockerRegistryUrl := include "sync-layer-server.cloudProviderDockerRegistryUrl" . -}} {{- $cloudProviderImagePullSecretName := include "sync-layer-server.cloudProviderImagePullSecretName" . -}} {{- $imageTag := include "sync-layer-server.tag" . -}} +{{- $postgres := include "merged.postgres" . | fromYaml -}} {{- if .Values.enabled -}} apiVersion: apps/v1 kind: Deployment @@ -66,6 +68,11 @@ spec: {{- if .Values.extraVolumeMounts -}} {{ toYaml .Values.extraVolumeMounts | nindent 12 }} {{- end }} + {{- if $postgres.ssl.enabled }} + - name: cert-conf + mountPath: /tmp/certs + readOnly: true + {{- end }} env: - name: K8S_POD_UID valueFrom: @@ -81,7 +88,19 @@ spec: {{- end }} {{- if .Values.extraEnvVars }} {{- toYaml .Values.extraEnvVars | nindent 12 }} - {{- end }} + {{- end }} + - name: DB_USERNAME + valueFrom: + secretKeyRef: + name: {{ default (printf "%s-db-secrets" $fullname) $postgres.user.secretName }} + key: username + {{- if $postgres.user.requirePassword }} + - name: DB_PASSWORD + valueFrom: + secretKeyRef: + name: {{ default (printf "%s-db-secrets" $fullname) $postgres.user.secretName }} + key: password + {{- end }} envFrom: - configMapRef: name: {{ include "sync-layer-server.fullname" . }} @@ -117,13 +136,23 @@ spec: volumes: - name: nginx-config configMap: - name: 'nginx-extra-configmap' + name: 'nginx-extra-configmap' + {{- if and $postgres.enabled $postgres.ssl.enabled }} + - name: cert-conf + secret: + secretName: {{ $postgres.ssl.secretName }} + {{- end }} {{- if .Values.caSecretName }} - name: root-ca secret: secretName: {{ .Values.caSecretName }} {{- end }} + {{- if $postgres.ssl.enabled }} + - name: postgres-cert + secret: + secretName: {{ $postgres.ssl.secretName }} + {{- end }} {{- if .Values.extraVolumes -}} {{ tpl (toYaml .Values.extraVolumes) . | nindent 8 }} - {{- end }} + {{- end }} {{- end -}} diff --git a/helm/values.yaml b/helm/values.yaml index 6352312..591e447 100644 --- a/helm/values.yaml +++ b/helm/values.yaml @@ -2,11 +2,30 @@ global: cloudProvider: {} tracing: {} metrics: {} + postgres: + # host: 127.0.0.1 + # port: 5432 + # schema: public + # name: sync_layer + user: + # requirePassword: true + # rejectUnauthorized: true + useExternal: true + secretName: secret-name + # username: postgres + # password: postgres + ssl: + # enabled: false + useExternal: true + secretName: secret-name + # caFileName: caFile + # certFileName: certFile + # keyFileName: keyFile mclabels: component: backend - partOf: boilerplates - owner: common + partOf: yahalom + owner: 3d prometheus: enabled: true @@ -20,8 +39,8 @@ nameOverride: "" fullnameOverride: "" configManagement: - offlineMode: false - name: 'service-name' + offlineMode: true + name: 'sync-layer-server' version: 'latest' serverUrl: 'http://localhost:8080/api' diff --git a/src/dal/db.data-source.ts b/src/dal/db.data-source.ts index 5b549c5..9a727c6 100644 --- a/src/dal/db.data-source.ts +++ b/src/dal/db.data-source.ts @@ -1,34 +1,13 @@ /* istanbul ignore file */ -import path from 'path'; +import path from 'node:path'; + import { DataSource, type DataSourceOptions } from 'typeorm'; -import type { DbConfig } from '../types'; +import { getDbConfig } from '../common/dbConfig'; +import { initConfig } from '../common/config'; import { createConnectionOptions } from './connectionOptions'; -const requireEnv = (name: string): string => { - const value = process.env[name]; - if (value === undefined || value === '') { - throw new Error(`Missing required environment variable: ${name}`); - } - return value; -}; - -const dbConfig: DbConfig = { - type: 'postgres', - host: requireEnv('DB_HOST'), - port: parseInt(requireEnv('DB_PORT'), 10), - username: requireEnv('DB_USERNAME'), - password: requireEnv('DB_PASSWORD'), - database: requireEnv('DB_NAME'), - enableSslAuth: process.env.DB_ENABLE_SSL?.toLowerCase() === 'true', - sslPaths: { - key: process.env.DB_SSL_KEY_PATH ?? '', - cert: process.env.DB_SSL_CERT_PATH ?? '', - ca: process.env.DB_SSL_CA_PATH ?? '', - }, -}; - const buildConnectionOptions = (): DataSourceOptions => { - const options = createConnectionOptions(dbConfig); + const options = createConnectionOptions(getDbConfig()); return { ...options, migrations: [path.join(__dirname, 'migrations', '*.ts')], @@ -36,4 +15,4 @@ const buildConnectionOptions = (): DataSourceOptions => { }; /* eslint-disable @typescript-eslint/naming-convention */ -export default new DataSource(buildConnectionOptions()); +export default initConfig(true).then(() => new DataSource(buildConnectionOptions())); diff --git a/tests/configurations/global-setup.ts b/tests/configurations/global-setup.ts index 7834474..c8da163 100644 --- a/tests/configurations/global-setup.ts +++ b/tests/configurations/global-setup.ts @@ -21,15 +21,33 @@ export default async function globalSetup(): Promise<() => Promise> { const { default: SyncLayerDataSource } = await import('@src/dal/db.data-source.js'); try { - await SyncLayerDataSource.initialize(); - await SyncLayerDataSource.runMigrations(); - console.log('✅ Migrations completed'); + SyncLayerDataSource.then((dataSource) => { + dataSource + .initialize() + .then(() => { + console.log('✅ Database connection established'); + return dataSource.runMigrations(); + }) + .then(() => { + console.log('✅ Migrations completed'); + }) + .catch((err) => { + console.warn('⚠️ Migration initialization warning:', err instanceof Error ? err.message : String(err)); + }) + .finally(() => { + if (dataSource.isInitialized) { + dataSource + .destroy() + .then(() => console.log('✅ Database connection closed')) + .catch((err) => console.warn('⚠️ Error closing database connection:', err instanceof Error ? err.message : String(err))); + } + }); + }).catch((err) => { + console.warn('⚠️ Error initializing database connection:', err instanceof Error ? err.message : String(err)); + }); } catch (err) { console.warn('⚠️ Migration initialization warning:', err instanceof Error ? err.message : String(err)); } finally { - if (SyncLayerDataSource.isInitialized) { - await SyncLayerDataSource.destroy(); - } } console.log('🚀 Environment ready for tests'); diff --git a/vitest.config.mts b/vitest.config.mts index 343fcec..244fe3e 100644 --- a/vitest.config.mts +++ b/vitest.config.mts @@ -1,6 +1,6 @@ import { defineConfig, ViteUserConfig } from 'vitest/config'; import tsconfig from './tsconfig.json'; -import path from 'path'; +import path from 'node:path'; // Create an alias object from the paths in tsconfig.json const pathAlias = Object.fromEntries(