AIOSEC-41: Adding Keycloak DB and API DB password generation. (#4)

* Adding Keycloak DB and API DB password generation.

Co-authored-by: Michael Schmid <michael@amazee.com>

.lagoon.secrets.yaml

@@ -24,6 +24,14 @@ parameters:
     description: Password to for opendistro-security cookies
     generate: expression
     from: "[a-zA-Z0-9]{32}"
+  - name: API_DB_PASSWORD
+    description: Password used for connecting to the api-db
+    generate: expression
+    from: "[a-zA-Z0-9]{32}"
+  - name: KEYCLOAK_DB_PASSWORD
+    description: Password used for connecting to the keycloak-db
+    generate: expression
+    from: "[a-zA-Z0-9]{32}"
   - name: SAFE_BRANCH
     description: Which branch this belongs to, special chars replaced with dashes
     required: true
@@ -61,6 +69,18 @@ objects:
     name: rabbitmq-username
   stringData:
     RABBITMQ_USERNAME: lagoon
+- kind: Secret
+  apiVersion: v1
+  metadata:
+    name: api-db-password
+  stringData:
+    API_DB_PASSWORD: ${API_DB_PASSWORD}
+- kind: Secret
+  apiVersion: v1
+  metadata:
+    name: keycloak-db-password
+  stringData:
+    KEYCLOAK_DB_PASSWORD: ${KEYCLOAK_DB_PASSWORD}
 - kind: Secret
   apiVersion: v1
   metadata:

docker-compose.yaml

@@ -8,7 +8,8 @@ services:
     ports:
       - '3366:3306'
     labels:
-      lagoon.type: mariadb-single
+      lagoon.type: custom
+      lagoon.template: services/api-db/.lagoon.app.yml
       lagoon.image: amazeeiolagoon/api-db:v1-5-0
   webhook-handler:
     image: ${IMAGE_REPO:-lagoon}/webhook-handler
@@ -336,7 +337,8 @@ services:
     ports:
       - '3336:3306'
     labels:
-      lagoon.type: mariadb-single
+      lagoon.type: custom
+      lagoon.template: services/keycloak-db/.lagoon.app.yml
       lagoon.image: amazeeiolagoon/keycloak-db:v1-5-0
   tests-kubernetes:
     image: ${IMAGE_REPO:-lagoon}/tests

images/mariadb/Dockerfile

@@ -41,7 +41,8 @@ RUN \
     net-tools \
     pwgen \
     tzdata \
-    wget; \
+    wget \
+    gettext; \
     rm -rf /tmp/* /var/tmp/* /var/cache/apk/* /var/cache/distfiles/*; \
     rm -rf /var/lib/mysql/* /etc/mysql/; \
     curl -sSL http://mysqltuner.pl/ -o mysqltuner.pl

images/mariadb/entrypoints/9999-mariadb-init.bash

@@ -96,7 +96,7 @@ EOF
     for f in `ls /docker-entrypoint-initdb.d/*`; do
       case "$f" in
         *.sh)     echo "$0: running $f"; . "$f" ;;
-        *.sql)    echo "$0: running $f"; cat $f| tee | mysql -u root -p${MARIADB_ROOT_PASSWORD}; echo ;;
+        *.sql)    echo "$0: running $f"; cat $f| envsubst | tee | mysql -u root -p${MARIADB_ROOT_PASSWORD}; echo ;;
         *)        echo "$0: ignoring $f" ;;
       esac
     echo

services/api-db/.lagoon.app.yml

@@ -0,0 +1,151 @@
+---
+apiVersion: v1
+kind: Template
+metadata:
+  creationTimestamp: null
+  name: lagoon-openshift-template-mariadb
+parameters:
+  - name: SERVICE_NAME
+    description: Name of this service
+    required: true
+  - name: SAFE_BRANCH
+    description: Which branch this belongs to, special chars replaced with dashes
+    required: true
+  - name: SAFE_PROJECT
+    description: Which project this belongs to, special chars replaced with dashes
+    required: true
+  - name: BRANCH
+    description: Which branch this belongs to, original value
+    required: true
+  - name: PROJECT
+    description: Which project this belongs to, original value
+    required: true
+  - name: LAGOON_GIT_SHA
+    description: git hash sha of the current deployment
+    required: true
+  - name: SERVICE_ROUTER_URL
+    description: URL of the Router for this service
+    value: ""
+  - name: OPENSHIFT_PROJECT
+    description: Name of the Project that this service is in
+    required: true
+  - name: REGISTRY
+    description: Registry where Images are pushed to
+    required: true
+  - name: DEPLOYMENT_STRATEGY
+    description: Strategy of Deploymentconfig
+    value: "Recreate"
+  - name: SERVICE_IMAGE
+    description: Pullable image of service
+    required: true
+  - name: CRONJOBS
+    description: Oneline of Cronjobs
+    value: ""
+  - name: PERSISTENT_STORAGE_SIZE
+    description: Size of the Storage to request
+    value: "5Gi"
+  - name: ENVIRONMENT_TYPE
+    description: production level of this environment
+    value: 'production'
+objects:
+- apiVersion: v1
+  kind: PersistentVolumeClaim
+  metadata:
+    annotations:
+      appuio.ch/backup: "false"
+    name: ${SERVICE_NAME}-data
+  spec:
+    accessModes:
+    - ReadWriteOnce
+    resources:
+      requests:
+        storage: ${PERSISTENT_STORAGE_SIZE}
+- apiVersion: v1
+  kind: Service
+  metadata:
+    creationTimestamp: null
+    labels:
+      service: ${SERVICE_NAME}
+      branch: ${SAFE_BRANCH}
+      project: ${SAFE_PROJECT}
+    name: ${SERVICE_NAME}
+  spec:
+    ports:
+    - name: 3306-tcp
+      port: 3306
+      protocol: TCP
+      targetPort: 3306
+    selector:
+      service: ${SERVICE_NAME}
+  status:
+    loadBalancer: {}
+- apiVersion: v1
+  kind: DeploymentConfig
+  metadata:
+    creationTimestamp: null
+    labels:
+      service: ${SERVICE_NAME}
+      branch: ${SAFE_BRANCH}
+      project: ${SAFE_PROJECT}
+    name: ${SERVICE_NAME}
+  spec:
+    replicas: 1
+    selector:
+      service: ${SERVICE_NAME}
+    strategy:
+      type: ${DEPLOYMENT_STRATEGY}
+    template:
+      metadata:
+        annotations:
+          appuio.ch/backupcommand: /bin/sh -c "mysqldump --max-allowed-packet=500M --events --routines --quick --add-locks --no-autocommit --single-transaction --all-databases"
+          backup.appuio.ch/file-extension: .${SERVICE_NAME}.sql
+        creationTimestamp: null
+        labels:
+          service: ${SERVICE_NAME}
+          branch: ${SAFE_BRANCH}
+          project: ${SAFE_PROJECT}
+      spec:
+        volumes:
+          - name: ${SERVICE_NAME}-data
+            persistentVolumeClaim:
+              claimName: ${SERVICE_NAME}-data
+        priorityClassName: lagoon-priority-${ENVIRONMENT_TYPE}
+        containers:
+        - image: ${SERVICE_IMAGE}
+          name: ${SERVICE_NAME}
+          ports:
+          - containerPort: 3306
+            protocol: TCP
+          readinessProbe:
+            tcpSocket:
+              port: 3306
+            initialDelaySeconds: 15
+            timeoutSeconds: 1
+          livenessProbe:
+            tcpSocket:
+              port: 3306
+            initialDelaySeconds: 120
+            periodSeconds: 5
+          volumeMounts:
+            - name: ${SERVICE_NAME}-data
+              mountPath: /var/lib/mysql
+          env:
+          - name: SERVICE_NAME
+            value: ${SERVICE_NAME}
+          - name: CRONJOBS
+            value: ${CRONJOBS}
+          - name: MARIADB_PASSWORD
+            valueFrom:
+              secretKeyRef:
+                name: api-db-password
+                key: API_DB_PASSWORD
+          envFrom:
+          - configMapRef:
+              name: lagoon-env
+          resources:
+            requests:
+              cpu: 10m
+              memory: 10Mi
+    test: false
+    triggers:
+    - type: ConfigChange

services/api-db/docker-entrypoint-initdb.d/01-migrations.sql

@@ -1042,6 +1042,15 @@ CREATE OR REPLACE PROCEDURE
   END;
 $$
 
+CREATE OR REPLACE PROCEDURE
+  update_user_password()
+
+  BEGIN
+    SET PASSWORD FOR '$MARIADB_USER'@'%' = PASSWORD('$MARIADB_PASSWORD');
+    FLUSH PRIVILEGES;
+  END;
+$$
+
 DELIMITER ;
 
 -- If adding new procedures, add them to the bottom of this list
@@ -1090,6 +1099,7 @@ CALL add_production_alias_to_project();
 CALL add_active_systems_misc_to_project();
 CALL add_container_registry_scope_to_env_vars();
 CALL add_internal_container_registry_scope_to_env_vars();
+CALL update_user_password();
 
 -- Drop legacy SSH key procedures
 DROP PROCEDURE IF EXISTS CreateProjectSshKey;

services/api-db/rerun_initdb.sh

@@ -3,7 +3,7 @@
 for f in `ls /docker-entrypoint-initdb.d/*`; do
   case "$f" in
     *.sh)     echo "$0: running $f"; . "$f" ;;
-    *.sql)    echo "$0: running $f"; cat $f| tee | mysql --verbose; echo ;;
+    *.sql)    echo "$0: running $f"; cat $f| envsubst | tee | mysql --verbose; echo ;;
     *)        echo "$0: ignoring $f" ;;
   esac
 echo

services/api/.lagoon.app.yml

@@ -124,6 +124,11 @@ objects:
                 secretKeyRef:
                   name: keycloak-api-client-secret
                   key: KEYCLOAK_API_CLIENT_SECRET
+            - name: API_DB_PASSWORD
+              valueFrom:
+                secretKeyRef:
+                  name: api-db-password
+                  key: API_DB_PASSWORD
             - name: SERVICE_NAME
               value: ${SERVICE_NAME}
             - name: CRONJOBS

services/api/src/clients/sqlClient.ts

@@ -2,19 +2,21 @@ import Client from 'mariasql';
 
 import * as logger from '../logger';
 
+const { API_DB_PASSWORD, API_DB_HOST, API_DB_PORT, API_DB_USER, API_DB_DATABASE } = process.env;
+
 export const getSqlClient = () => {
   const sqlClient = new Client({
-    host: 'api-db',
-    port: 3306,
-    user: 'api',
-    password: 'api',
-    db: 'infrastructure',
+    host: API_DB_HOST || 'api-db',
+    port: API_DB_PORT || 3306,
+    user: API_DB_USER || 'api',
+    password: API_DB_PASSWORD || 'api',
+    db: API_DB_DATABASE || 'infrastructure',
   });
 
   sqlClient.on('error', error => {
     logger.error(error.message);
   });
-  
+
   return sqlClient;
 };