chore(dp): One postgresql instance in dev/CI

.github/workflows/dp-workflow.yml

@@ -411,11 +411,7 @@ jobs:
           mkdir ${TEST_DIR}/k8s-pods-logs;
           for pod in $(kubectl get pods -o custom-columns=NAME:.metadata.name --no-headers);
           do
-            if [[ "$pod" == "postgres-database-0" ]]; then
-              kubectl logs --timestamps=true $pod --container postgres-test > ${TEST_DIR}/k8s-pods-logs/${pod}.log;
-            else
-              kubectl logs --timestamps=true $pod > ${TEST_DIR}/k8s-pods-logs/${pod}.log;
-            fi;
+            kubectl logs --timestamps=true $pod > ${TEST_DIR}/k8s-pods-logs/${pod}.log;
           done
       - name: Collect Elasticsearch data
         if: always()

dp/Makefile

@@ -38,7 +38,7 @@ orc8r_integration_tests: init_orc8r _ci_init cleanup_test_db
 dev_integration_tests: init_orc8r _ci_init cleanup_test_db
 	kubectl delete jobs -l jobName=domain-proxy-db-setup --ignore-not-found=true
 	kubectl delete pods test-runner-dev --ignore-not-found=true
-	skaffold dev -p orc8r-deployment,integration-tests-no-orc8,integration-tests-dev --trigger=manual
+	skaffold dev -p orc8r-deployment,integration-tests-no-orc8,integration-tests-orc8r-only,integration-tests-dev --trigger=manual
 
 .PHONY: orc8r
 orc8r: init_orc8r dev_orc8r
@@ -186,6 +186,7 @@ _postgres_db_delete: _switch_k8s_context
 .PHONY: _ci_chart_smoke_tests
 _ci_chart_smoke_tests: _install_skaffold_ci
 	skaffold run
+	sleep 5
 	helm test --timeout 10m domain-proxy
 
 .PHONY: migration
@@ -216,13 +217,11 @@ build_db:
 
 .PHONY: cleanup_test_db
 cleanup_test_db: _switch_k8s_context
-	kubectl exec -it postgres-database-0 \
-		--container postgres-test -- \
-		psql -p 5433 -U postgres -c \
+	kubectl exec -it postgres-database-0 -- \
+		psql -U postgres -c \
 		"drop database dp_test with (force);" || true ;\
-	kubectl exec -it postgres-database-0 \
-		--container postgres-test -- \
-		psql -p 5433 -U postgres -c \
+	kubectl exec -it postgres-database-0 -- \
+		psql -U postgres -c \
 		"create database dp_test;" || true;
 
 LOGS_COUNT ?= 1000

dp/cloud/docker/postgresql/Dockerfile

@@ -0,0 +1,14 @@
+# Copyright 2022 The Magma Authors.
+
+# This source code is licensed under the BSD-style license found in the
+# LICENSE file in the root directory of this source tree.
+
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+FROM postgres:13.3
+
+COPY --chmod=755 create_magma_dbs.sh /docker-entrypoint-initdb.d/create_magma_dbs.sh

dp/cloud/docker/postgresql/create_magma_dbs.sh

@@ -0,0 +1,19 @@
+#!/bin/bash
+
+# Copyright 2022 The Magma Authors.
+
+# This source code is licensed under the BSD-style license found in the
+# LICENSE file in the root directory of this source tree.
+
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+set -e
+
+echo "Creating postgresql DB dp_test"
+psql -U "${POSTGRES_USER}" -c "CREATE DATABASE dp_test"
+echo "Grant privileges to ${POSTGRES_USER} on dp_test..."
+psql -U "${POSTGRES_USER}" -c "GRANT ALL PRIVILEGES ON DATABASE dp_test TO ${POSTGRES_USER}"

dp/cloud/helm/dp/charts/domain-proxy/examples/orc8r_minikube_values.yaml

@@ -28,9 +28,9 @@ controller:
     database:
       driver: postgres  # mysql/postgres
       sql_dialect: psql  # maria/psql
-      db: dp_test
+      db: dp
       host: db
-      port: 5433
+      port: 5432
       user: postgres
       pass: postgres
     service_registry:
@@ -135,11 +135,11 @@ nms:
       service: true
     env:
       api_host: orc8r-nginx-proxy:443
-      mysql_db: dp_test
+      mysql_db: dp
       mysql_dialect: postgres
       mysql_host: db
       mysql_pass: postgres
-      mysql_port: 5433
+      mysql_port: 5432
       mysql_user: postgres
     image:
       repository: magmalte

dp/cloud/python/magma/db_service/config.py

@@ -48,7 +48,7 @@ class TestConfig(Config):
     Configuration class for db service
     """
     SQLALCHEMY_DB_URI = os.environ.get(
-        'SQLALCHEMY_DB_URI', 'postgresql+psycopg2://postgres:postgres@db:5433/dp_test',
+        'SQLALCHEMY_DB_URI', 'postgresql+psycopg2://postgres:postgres@db:5432/dp_test',
     )
 
 

dp/skaffold.yaml

@@ -39,6 +39,10 @@ build:
       context: ..
       docker:
         dockerfile: orc8r/cloud/docker/fluentd_forward/Dockerfile
+    - image: dp-postgres
+      context: ../dp/cloud/docker/postgresql
+      docker:
+        dockerfile: ../dp/cloud/docker/postgresql/Dockerfile
 
 deploy:
   kubeContext: minikube
@@ -133,6 +137,20 @@ profiles:
         value:
           manifests:
             - "./tools/deployment/tests/test_runner_deployment_orc8r.yml"
+      - op: add
+        path: /deploy/helm/releases/1/overrides
+        value:
+          nms:
+            magmalte:
+              env:
+                mysql_db: dp_test
+      - op: add
+        path: /deploy/helm/releases/2/overrides
+        value:
+          controller:
+            spec:
+              database:
+                db: dp_test
   - name: integration-tests-dev
     patches:
       - op: add
@@ -267,3 +285,5 @@ profiles:
     patches:
       - op: remove
         path: /build/artifacts/4  # remove fake sas image
+      - op: remove
+        path: /build/artifacts/7  # remove postgresql image

dp/tools/deployment/vendor/postgresql.yml

@@ -15,7 +15,8 @@ spec:
     spec:
       containers:
         - name: postgres
-          image: postgres:13.3
+          image: dp-postgres:latest
+          imagePullPolicy: IfNotPresent
           volumeMounts:
             - name: db-disk
               mountPath: /var/lib/postgresql/data
@@ -52,49 +53,6 @@ spec:
             successThreshold: 1
             timeoutSeconds: 10
             failureThreshold: 3
-        - name: postgres-test
-          image: postgres:13.3
-          args:
-            - -p 
-            - "5433"
-          volumeMounts:
-            - name: db-disk2
-              mountPath: /var/lib/postgresql/data2
-          env:
-            - name: POSTGRES_DB
-              value: "dp_test"
-            - name: POSTGRES_PASSWORD
-              value: postgres
-            - name: PGDATA
-              value: /var/lib/postgresql/data2/pgdata
-          livenessProbe:
-            exec:
-              command:
-                - /usr/bin/pg_isready
-                - -U
-                - "postgres"
-                - "--port=5433"
-                - -h
-                - 127.0.0.1
-            failureThreshold: 3
-            initialDelaySeconds: 20
-            periodSeconds: 30
-            successThreshold: 1
-            timeoutSeconds: 10
-          readinessProbe:
-            exec:
-              command:
-                - /usr/bin/pg_isready
-                - -U
-                - "postgres"
-                - "--port=5433"
-                - -h
-                - 127.0.0.1
-            initialDelaySeconds: 30
-            periodSeconds: 30
-            successThreshold: 1
-            timeoutSeconds: 10
-            failureThreshold: 3
   volumeClaimTemplates:
     - metadata:
         name: db-disk
@@ -103,16 +61,8 @@ spec:
         resources:
           requests:
             storage: 1Gi
-    - metadata:
-        name: db-disk2
-      spec:
-        accessModes: ["ReadWriteOnce"]
-        resources:
-          requests:
-            storage: 1Gi
 
 ---
-
 apiVersion: v1
 kind: Service
 metadata:
@@ -125,6 +75,3 @@ spec:
     - name: db1
       port: 5432
       targetPort: 5432
-    - name: db2
-      port: 5433
-      targetPort: 5433