Read sensitive data from docker secrets

composefiles/swarm-uniconfig.yml

@@ -74,27 +74,35 @@ services:
       # Enable G1GC and GC logging 
       - _JAVA_OPTIONS=-Xmx${UC_XMX_LIMIT_MEM} -XX:+AlwaysPreTouch -Djna.nosys=true -XX:+CrashOnOutOfMemoryError -XX:MaxGCPauseMillis=200 -Xlog:gc:/opt/uniconfig-frinx/log/gc-{{.Task.Slot}}.log:tags,time,uptime,level:filecount=10,filesize=5M -XX:+UseG1GC -XX:+ExplicitGCInvokesConcurrent -XX:+ParallelRefProcEnabled -XX:+UseStringDeduplication -XX:+UnlockExperimentalVMOptions -XX:G1NewSizePercent=20 -XX:+UnlockDiagnosticVMOptions -XX:G1SummarizeRSetStatsPeriod=1 ${UC_PROXY_HTTP_ENV} ${UC_PROXY_HTTP_PORT_ENV} ${UC_PROXY_HTTPS_ENV} ${UC_PROXY_HTTPS_PORT_ENV} ${UC_PROXY_NOPROXY_ENV}
       - dbPersistence_enabled=true
-      - dbPersistence_connection_dbName=uniconfig
-      - dbPersistence_connection_username=postgres
-      - dbPersistence_connection_password=postgres
       - dbPersistence_connection_databaseLocations_host=uniconfig-postgres
       - dbPersistence_connection_databaseLocations_port=5432
+    entrypoint: ["/opt/uniconfig-frinx/run_uniconfig.sh"]
     volumes:
       - frinx_uniconfig-controller_logs:/opt/uniconfig-frinx/log
       - ${UC_CONFIG_PATH}/config/lighty-uniconfig-config.json:/opt/uniconfig-frinx/config/lighty-uniconfig-config.json:ro
       - ${UC_CONFIG_PATH}/config/logback.xml:/opt/uniconfig-frinx/config/logback.xml:ro
       - ${UC_CONFIG_PATH}/cache:/opt/uniconfig-frinx/cache:rw
-      - ${UC_CONFIG_PATH}/config/.keystore:/opt/uniconfig-frinx/config/.keystore:ro
+    secrets:
+      - uniconfig
+      - source: frinx_uniconfig_keystore
+        target: /opt/uniconfig-frinx/config/.keystore
+    configs:
+      - source: uniconfig_startup
+        target: /opt/uniconfig-frinx/run_uniconfig.sh
+        mode: 0777
+      - source: set_env_secrets
+        target: /set_env_secrets.sh
+        mode: 0777
     healthcheck:
-      test: curl --insecure -o /dev/null --user admin:admin --silent --write-out "%{http_code}" -X POST 'https://127.0.0.1:8181/rests/operations/uniconfig-manager:health' -H "Content-Type:application/json" | grep 200 || exit 1
+    # remove password from 
+      test: | 
+        curl --insecure -o /dev/null --silent --write-out "%{http_code}" \
+        -X POST 'https://127.0.0.1:8181/rests/operations/uniconfig-manager:health' \
+        -H "Content-Type:application/json" -H "Authorization:Basic YWRtaW46YWRtaW4=" | grep 200 || exit 1
       interval: 10s
       timeout: 35s
       retries: 5
       start_period: ${UC_HEALTH_START_TIME}
-    configs:
-      - source: uniconfig_startup
-        target: /opt/uniconfig-frinx/run_uniconfig.sh
-        mode: 0777
     entrypoint: ['/opt/uniconfig-frinx/run_uniconfig.sh']
     command: -l ${LICENSE} 
     ulimits:
@@ -130,12 +138,15 @@ services:
     logging: *logging_loki
     labels:
       - traefik.enable=false  
-    environment:
-      - POSTGRES_PASSWORD=postgres
-      - POSTGRES_USER=postgres
-      - POSTGRES_DB=uniconfig
+    entrypoint: ["/set_env_secrets.sh","docker-entrypoint.sh postgres"]
     volumes:
       - frinx_uniconfig-postgresql_data:/var/lib/postgresql/data
+    secrets:
+      - uniconfig
+    configs:
+      - source: set_env_secrets
+        target: /set_env_secrets.sh
+        mode: 0777
     healthcheck:
       test: ["CMD-SHELL", "pg_isready -U postgres"]
       interval: 10s
@@ -172,12 +183,8 @@ services:
 configs:
   uniconfig_startup:
     file: ${UF_CONFIG_PATH}/uniconfig/run_uniconfig.sh
-
-volumes:
-  frinx_uniconfig-controller_logs:
-    name: frinx_uniconfig-controller_logs
-  frinx_uniconfig-postgresql_data:
-    name: frinx_uniconfig-postgresql_data
+  set_env_secrets:
+    file: ${UF_CONFIG_PATH}/set_env_secrets.sh
 
 secrets:
   frinx_uniconfig_tls_cert.pem:
@@ -186,6 +193,16 @@ secrets:
     external: true
   frinx_uniconfig_X509.crt:
     external: true
+  frinx_uniconfig_keystore:
+    external: true
+  uniconfig:
+    external: true
+
+volumes:
+  frinx_uniconfig-controller_logs:
+    name: frinx_uniconfig-controller_logs
+  frinx_uniconfig-postgresql_data:
+    name: frinx_uniconfig-postgresql_data
 
 networks:
   default:

composefiles/swarm-uniflow.yml

@@ -24,8 +24,18 @@ services:
     environment:
       - CONFIG_PROP=config.properties
       - _JAVA_OPTIONS="-Xmx${CS_RES_LIMIT_MEM}"
+    entrypoint: ["/app/startup.sh"]
     volumes:
       - ${UF_CONFIG_PATH}/conductor/config.properties:/app/config/config.properties:ro
+    secrets:
+      - uniflow
+    configs:
+      - source: set_env_secrets
+        target: /set_env_secrets.sh
+        mode: 0777
+      - source: conductor_startup
+        target: /app/startup.sh
+        mode: 0777
     healthcheck:
       test: curl -I -XGET http://localhost:8080/health
       interval: 10s
@@ -144,13 +154,16 @@ services:
     logging: *logging_loki
     labels:
       - traefik.enable=false  
-    environment:
-      - POSTGRES_PASSWORD=postgres
-      - POSTGRES_USER=postgres
-      - POSTGRES_MULTIPLE_DATABASES=conductor,frinx,schellar
+    entrypoint: ["/set_env_secrets.sh","docker-entrypoint.sh postgres"]
     volumes:
       - frinx_uniflow_postgresql_data:/var/lib/postgresql/data
       - ${UF_CONFIG_PATH}/uniflow-postgres:/docker-entrypoint-initdb.d:ro
+    secrets:
+      - uniflow
+    configs:
+      - source: set_env_secrets
+        target: /set_env_secrets.sh
+        mode: 0777
     healthcheck:
       test: pg_isready -U postgres
       interval: 10s
@@ -192,7 +205,13 @@ services:
       - CONDUCTOR_API_URL=http://conductor-server:8080/api
       - BACKEND=postgres
       - POSTGRES_MIGRATIONS_DIR=migrations
-      - POSTGRES_DATABASE_URL=host=postgresql port=5432 user=postgres password=postgres database=schellar
+    entrypoint: ["/set_env_secrets.sh", "./schellar"]
+    secrets:
+      - uniflow
+    configs:
+      - source: set_env_secrets
+        target: /set_env_secrets.sh
+        mode: 0777
     healthcheck:
       test: wget --spider -q conductor-server:8080/health && wget --spider -q 127.0.0.1:3000/liveness && nc -z postgresql:5432
       interval: 10s
@@ -260,14 +279,18 @@ services:
           memory: ${WP_RES_LIMIT_MEM}
 
   uniresource:
-    image: frinx/resource-manager:1.0.2
+    image: frinx/resource-manager:latest
     logging: *logging_loki
     labels:
-      - traefik.enable=false  
+      - traefik.enable=false
+    entrypoint: ["/set_env_secrets.sh", "./run.sh"]
+    secrets:
+      - uniflow
+    configs:
+      - source: set_env_secrets
+        target: /set_env_secrets.sh
+        mode: 0777
     environment:
-      - RM_DB_CONNECTION_STRING=postgres://postgres:postgres@postgresql:5432/postgres?sslmode=disable
-      - RM_API_PORT=8884
-      - RM_ADMIN_ROLES=OWNER
       - RM_ADMIN_GROUPS=network-admin
       - RM_LOG_PATH=/var/log/rm.log
       - RM_LOG_LEVEL=warn
@@ -306,9 +329,15 @@ services:
     image: frinx/frinx-inventory-server:1.1.0
     logging: *logging_loki
     labels:
-      - traefik.enable=false  
+      - traefik.enable=false
+    entrypoint: ["/run_inventory.sh"]
+    secrets:
+      - inventory
+    configs:
+      - source: set_env_secrets
+        target: /set_env_secrets.sh
+        mode: 0777
     environment:
-      - DATABASE_URL=postgres://postgres:postgres@inventory-postgres:5432/inventory?sslmode=disable
       - UNICONFIG_API_PROTOCOL=https
       - UNICONFIG_API_PORT=8181
       - UNICONFIG_LIST_URL=http://krakend:8080/static/list/uniconfig
@@ -350,13 +379,16 @@ services:
     user: postgres
     logging: *logging_loki
     labels:
-      - traefik.enable=false  
-    environment:
-      - POSTGRES_PASSWORD=postgres
-      - POSTGRES_USER=postgres
-      - POSTGRES_DB=inventory
+      - traefik.enable=false
+    entrypoint: ["/set_env_secrets.sh", "docker-entrypoint.sh postgres"]
     volumes:
       - frinx_uniflow_inventory_postgresql_data:/var/lib/postgresql/data
+    secrets:
+      - inventory
+    configs:
+      - source: set_env_secrets
+        target: /set_env_secrets.sh
+        mode: 0777
     healthcheck:
       test: pg_isready -U postgres
       interval: 10s
@@ -387,6 +419,17 @@ services:
           cpus: ${IP_RES_LIMIT_CPUS}
           memory: ${IP_RES_LIMIT_MEM}
 
+secrets:
+  inventory:
+    external: true
+  uniflow:
+    external: true
+
+configs:
+  set_env_secrets:
+    file: ${UF_CONFIG_PATH}/set_env_secrets.sh
+  conductor_startup:
+    file: ${UF_CONFIG_PATH}/conductor/startup.sh
 
 volumes:
   frinx_uniflow_elastic_data:

config/conductor/config.properties

@@ -8,11 +8,6 @@
 conductor.grpc-server.enabled=false
 conductor.app.ownerEmailMandatory=false
 
-conductor.db.type=postgres
-spring.datasource.url=jdbc:postgresql://postgresql:5432/conductor?charset=utf8&parseTime=true&interpolateParams=true
-spring.datasource.username=postgres
-spring.datasource.password=postgres
-
 # Hikari pool sizes are -1 by default and prevent startup
 spring.datasource.hikari.maximum-pool-size=10
 spring.datasource.hikari.minimum-idle=2
@@ -54,9 +49,6 @@ conductor.app.max-task-output-payload-size-threshold=1024000
 # PostgreSQL External Payload Storage variables
 conductor.external-payload-storage.type=postgres
 conductor.external-payload-storage.postgres.conductor-url=http://workflow-proxy:8088/proxy
-conductor.external-payload-storage.postgres.url=jdbc:postgresql://postgresql:5432/conductor?charset=utf8&parseTime=true&interpolateParams=true
-conductor.external-payload-storage.postgres.username=postgres
-conductor.external-payload-storage.postgres.password=postgres
 conductor.external-payload-storage.postgres.max-data-rows=1000000
 conductor.external-payload-storage.postgres.max-data-days=0
 conductor.external-payload-storage.postgres.max-data-months=0

config/inventory/run_inventory.sh

@@ -1,5 +1,7 @@
 #!/bin/sh
 
+. /set_env_secrets.sh
+
 yarn prisma migrate deploy --schema=prisma/schema.prisma 
 MIGRATION_SUCCESS=$?
 if [ ${MIGRATION_SUCCESS} -ne 0 ]; then

config/secrets/inventory

@@ -0,0 +1,5 @@
+# inventory postgres database settings
+POSTGRES_PASSWORD=postgres
+POSTGRES_USER=postgres
+POSTGRES_DB=uniconfig
+DATABASE_URL="postgres://postgres:postgres@inventory-postgres:5432/inventory?sslmode=disable"

config/secrets/uniconfig

@@ -0,0 +1,7 @@
+POSTGRES_PASSWORD=postgres
+POSTGRES_USER=postgres
+POSTGRES_DB=uniconfig
+dbPersistence_connection_dbName=uniconfig
+dbPersistence_connection_username=postgres
+dbPersistence_connection_password=postgres
+tls_keystorePassword=password

config/secrets/uniflow

@@ -0,0 +1,21 @@
+# postgresql database settings
+POSTGRES_PASSWORD=postgres
+POSTGRES_USER=postgres
+POSTGRES_MULTIPLE_DATABASES=conductor,frinx,schellar
+
+# schellar database connection settings
+POSTGRES_DATABASE_URL="host=postgresql port=5432 user=postgres password=postgres database=schellar"
+
+# conductor database connection settings
+conductor_db_type=postgres
+spring_datasource_url="jdbc:postgresql://postgresql:5432/conductor?charset=utf8&parseTime=true&interpolateParams=true"
+spring_datasource_username=postgres
+spring_datasource_password=postgres
+conductor_externalPayloadStorage_postgres_url="jdbc:postgresql://postgresql:5432/conductor?charset=utf8&parseTime=true&interpolateParams=true"
+conductor_externalPayloadStorage_postgres_username=postgres
+conductor_externalPayloadStorage_postgres_password=postgres
+
+# uniresource database connection settings
+RM_DB_CONNECTION_STRING=postgres://postgres:postgres@postgresql:5432/postgres?sslmode=disable
+RM_API_PORT=8884
+RM_ADMIN_ROLES=OWNER

config/set_env_secrets.sh

@@ -0,0 +1,26 @@
+#!/bin/sh
+
+# set env variables from docker secret to service
+
+set -a
+
+FILES="/run/secrets/*"
+for __filePath in $FILES
+do
+  if [ -f ${__filePath} ]; then
+    . ${__filePath}
+    cat ${__filePath} | while read line || [ -n "$line" ]; 
+    do 
+      case $line in
+        '#'*) 
+              ;; # ignore comments
+        '')
+              ;; # ignore empty spaces
+        *)
+            export $(echo "${line}" | cut -d '=' -f1) >/dev/null
+            test="${test} $line";;
+      esac
+    done
+  fi
+done
+${1}

config/uniconfig/run_uniconfig.sh

@@ -7,6 +7,11 @@ JAVA_MAX_MEM=${JAVA_MAX_MEM:="4G"}
 DEBUG_PARAMETER="--debug"
 UNICONFIG_ID=${CONTAINER_ID:=1}
 
+# set sensitive env variables from docker secrets
+if [[ -f "/set_env_secrets.sh" ]]; then
+  . /set_env_secrets.sh ''
+fi
+
 display_usage() {
     echo -e "Usage: $(basename "$0") [-f] [-l LICENSE_TOKEN] [--debug]"
     echo -e "where: "