Enable container based database migration support

* Add support for container based database migration via CR
* Update operator role to enable command execution on a pod
* Add check to postgres role around migration
* Add migration of data tasks
* Add scale down tasks
* Update status role to record db migration
* Misc. fix for conflicting label selectors

fixes #8472
https://pulp.plan.io/issues/8472

CHANGES/8472.feature

@@ -0,0 +1 @@
+Enable container based database migration support

deploy/crds/pulpproject_v1beta1_pulp_crd.yaml

@@ -94,6 +94,12 @@ spec:
               postgres_data_path:
                 description: Path where the PostgreSQL data are located
                 type: string
+              postgres_migrant_configuration_secret:
+                description: Secret where the old database configuration can be found for data migration
+                type: string
+              postgres_label_selector:
+                description: Label selector used to identify postgres pod for executing migration
+                type: string
               pulp_storage_type:
                 description: Configuration for the storage type to be utilized
                 type: string
@@ -369,6 +375,9 @@ spec:
               deployedImage:
                 description: URL of the image used for the deployed instance
                 type: string
+              migrantDatabaseConfigurationSecret:
+                description: The configuration secret used for migrating an old deployment.
+                type: string
               conditions:
                 description: The resulting conditions when a Service Telemetry is instantiated
                 items:

deploy/olm-catalog/pulp-operator/manifests/pulp-operator.clusterserviceversion.yaml

@@ -81,6 +81,17 @@ spec:
         x-descriptors:
         - urn:alm:descriptor:com.tectonic.ui:advanced
         - urn:alm:descriptor:com.tectonic.ui:hidden
+      - displayName: Migrant database configuration secret
+        path: postgres_migrant_configuration_secret
+        x-descriptors:
+        - urn:alm:descriptor:com.tectonic.ui:advanced
+        - urn:alm:descriptor:com.tectonic.ui:hidden
+        - urn:alm:descriptor:io.kubernetes:Secret
+      - displayName: Database migration label selector
+        path: postgres_label_selector
+        x-descriptors:
+        - urn:alm:descriptor:com.tectonic.ui:advanced
+        - urn:alm:descriptor:com.tectonic.ui:hidden
       - displayName: Ingress Type
         path: ingress_type
         x-descriptors:
@@ -267,6 +278,11 @@ spec:
         path: deployedImage
         x-descriptors:
           - urn:alm:descriptor:com.tectonic.ui:text
+      - displayName: Migrated Database
+        description: Configuration secret for previously deployed database
+        path: migrantDatabaseConfigurationSecret
+        x-descriptors:
+          - urn:alm:descriptor:io.kubernetes:Secret
   description: Pulp operator
   displayName: Pulp
   icon:
@@ -288,6 +304,7 @@ spec:
           - rbac.authorization.k8s.io
           resources:
           - pods
+          - pods/log
           - services
           - services/finalizers
           - endpoints
@@ -350,6 +367,19 @@ spec:
           - deployments
           verbs:
           - get
+        - apiGroups:
+            - apps
+          resources:
+            - deployments/scale
+          verbs:
+            - patch
+        - apiGroups:
+            - ""
+          resources:
+            - pods/exec
+          verbs:
+            - create
+            - get
         - apiGroups:
           - pulp.pulpproject.org
           resources:

deploy/olm-catalog/pulp-operator/manifests/pulp.pulpproject.org_pulps_crd.yml

@@ -94,6 +94,12 @@ spec:
               postgres_data_path:
                 description: Path where the PostgreSQL data are located
                 type: string
+              postgres_migrant_configuration_secret:
+                description: Secret where the old database configuration can be found for data migration
+                type: string
+              postgres_label_selector:
+                description: Label selector used to identify postgres pod for executing migration
+                type: string
               pulp_storage_type:
                 description: Configuration for the storage type to be utilized
                 type: string
@@ -369,6 +375,9 @@ spec:
               deployedImage:
                 description: URL of the image used for the deployed instance
                 type: string
+              migrantDatabaseConfigurationSecret:
+                description: The configuration secret used for migrating an old deployment.
+                type: string
               conditions:
                 description: The resulting conditions when a Service Telemetry is instantiated
                 items:

deploy/role.yaml

@@ -15,6 +15,7 @@ rules:
   - ""
   resources:
   - pods
+  - pods/log
   - services
   - services/finalizers
   - endpoints
@@ -75,6 +76,19 @@ rules:
   - deployments
   verbs:
   - get
+- apiGroups:
+    - apps
+  resources:
+    - deployments/scale
+  verbs:
+    - patch
+- apiGroups:
+    - ""
+  resources:
+    - pods/exec
+  verbs:
+    - create
+    - get
 - apiGroups:
   - pulp.pulpproject.org
   resources:

roles/postgres/defaults/main.yml

@@ -11,5 +11,11 @@ postgres_data_path: '/var/lib/postgresql/data/pgdata'
 # Secret to lookup that provide the PostgreSQL configuration
 postgres_configuration_secret: ''
 
+# Secret to lookup that provide the PostgreSQL configuration for old database
+postgres_migrant_configuration_secret: ''
+
 postgres_initdb_args: '--auth-host=scram-sha-256'
 postgres_host_auth_method: 'scram-sha-256'
+
+custom_resource_key: '_pulp_pulpproject_org_pulp'
+database_status_present: false

roles/postgres/tasks/main.yml

@@ -1,4 +1,25 @@
 ---
+
+- name: Obtain custom resource information
+  set_fact:
+    custom_resource: "{{ hostvars[inventory_hostname][custom_resource_key] }}"
+    custom_resource_status: "{{ hostvars[inventory_hostname][custom_resource_key]['status'] }}"
+
+
+- name: Record migrant database secret
+  set_fact:
+    recorded_db_migration_secret: "{{ custom_resource_status['migrantDatabaseConfigurationSecret'] }}"
+  when:
+    - custom_resource_status['migrantDatabaseConfigurationSecret'] is defined
+
+- name: Check if data migration has been performed
+  set_fact:
+    database_status_present: true
+  when:
+    - recorded_db_migration_secret is defined
+    - postgres_migrant_configuration_secret is defined
+    - recorded_db_migration_secret == postgres_migrant_configuration_secret
+
 - name: Check for specified PostgreSQL configuration
   k8s_info:
     kind: Secret
@@ -14,6 +35,22 @@
     name: '{{ meta.name }}-postgres-configuration'
   register: _default_pg_config_resources
 
+- name: Check for old PostgreSQL configuration secret
+  k8s_info:
+    kind: Secret
+    namespace: '{{ meta.namespace }}'
+    name: '{{ postgres_migrant_configuration_secret }}'
+  register: old_pg_config
+  when: postgres_migrant_configuration_secret | length
+
+- name: Set proper database name when migrating from old deployment
+  set_fact:
+    database_name: "{{ old_pg_config['resources'][0]['data']['database'] | b64decode }}"
+    database_username: "{{ old_pg_config['resources'][0]['data']['username'] | b64decode }}"
+  when:
+    - old_pg_config['resources'] is defined
+    - old_pg_config['resources'] | length
+
 - name: Set PostgreSQL configuration
   set_fact:
     _pg_config: '{{ _custom_pg_config_resources["resources"] | default([]) | length | ternary(_custom_pg_config_resources, _default_pg_config_resources) }}'
@@ -51,3 +88,11 @@
     postgres_database: "{{ pg_config['resources'][0]['data']['database'] | b64decode }}"
     postgres_port: "{{ pg_config['resources'][0]['data']['port'] | b64decode }}"
     postgres_host: "{{ pg_config['resources'][0]['data']['host'] | b64decode }}"
+
+
+- name: Migrate data from old Openshift instance
+  import_tasks: migrate_data.yml
+  when:
+    - old_pg_config['resources'] is defined
+    - old_pg_config['resources'] | length
+    - not database_status_present

roles/postgres/tasks/migrate_data.yml

@@ -0,0 +1,74 @@
+---
+
+- name: Store Database Configuration
+  set_fact:
+    migrant_postgres_user: "{{ old_pg_config['resources'][0]['data']['username'] | b64decode }}"
+    migrant_postgres_pass: "{{ old_pg_config['resources'][0]['data']['password'] | b64decode }}"
+    migrant_postgres_database: "{{ old_pg_config['resources'][0]['data']['database'] | b64decode }}"
+    migrant_postgres_port: "{{ old_pg_config['resources'][0]['data']['port'] | b64decode }}"
+    migrant_postgres_host: "{{ old_pg_config['resources'][0]['data']['host'] | b64decode }}"
+
+- name: Default label selector to custom resource generated postgres
+  set_fact:
+    postgres_label_selector: "app={{ meta.name }}-{{ deployment_type }}-postgres"
+  when: postgres_label_selector is not defined
+
+- name: Get the postgres pod information
+  k8s_info:
+    kind: Pod
+    namespace: '{{ meta.namespace }}'
+    label_selectors:
+      - "{{ postgres_label_selector }}"
+  register: postgres_pod
+  until: "postgres_pod['resources'][0]['status']['phase'] == 'Running'"
+  delay: 5
+  retries: 60
+
+- name: Set the resource pod name as a variable.
+  set_fact:
+    postgres_pod_name: "{{ postgres_pod['resources'][0]['metadata']['name'] }}"
+
+- include: scale_down.yml deployment_name={{ item }}
+  with_items:
+    - "{{ meta.name }}-api"
+    - "{{ meta.name }}-content"
+    - "{{ meta.name }}-resource-manager"
+    - "{{ meta.name }}-worker"
+    - "{{ meta.name }}-web"
+
+- name: Set pg_dump command
+  set_fact:
+    pgdump: >-
+      pg_dump --clean --create
+      -h {{ migrant_postgres_host }}
+      -U {{ migrant_postgres_user }}
+      -d {{ migrant_postgres_database }}
+      -p {{ migrant_postgres_port }}
+
+- name: Set pg_restore command
+  set_fact:
+    psql_restore: >-
+      psql -U {{ postgres_user }}
+      -d {{ postgres_database }}
+      -p {{ postgres_port }}
+
+- name: Stream backup from pg_dump to the new postgresql container
+  community.kubernetes.k8s_exec:
+    namespace: "{{ meta.namespace }}"
+    pod: "{{ postgres_pod_name }}"
+    command: |
+      bash -c """
+      set -e -o pipefail
+      PGPASSWORD={{ migrant_postgres_pass }} {{ pgdump }} | PGPASSWORD={{ postgres_pass }} {{ psql_restore }}
+      echo 'Successful'
+      """
+  register: data_migration
+  failed_when: "'Successful' not in data_migration['stdout']"
+
+- name: Set flag signifying that this instance has been migrated
+  set_fact:
+    postgres_migrated_from_secret: "{{ postgres_migrant_configuration_secret }}"
+
+- name: Migration succesful from previous database
+  debug:
+    msg: "Database migrated from secret: {{ postgres_migrated_from_secret }}"

roles/postgres/tasks/scale_down.yml

@@ -0,0 +1,18 @@
+---
+
+- name: Check for presence of Deployment
+  k8s_info:
+    api_version: v1
+    kind: Deployment
+    name: "{{ deployment_name }}"
+    namespace: "{{ meta.namespace }}"
+  register: scale_deployment
+
+- name: Scale down Deployment for migration
+  k8s_scale:
+    api_version: v1
+    kind: Deployment
+    name: "{{ deployment_name }}"
+    namespace: "{{ meta.namespace }}"
+    replicas: 0
+  when: scale_deployment['resources'] | length

roles/postgres/templates/postgres.yaml.j2

@@ -6,19 +6,19 @@ metadata:
   name: '{{ meta.name }}-postgres'
   namespace: '{{ meta.namespace }}'
   labels:
-    app: '{{ deployment_type }}-postgres'
+    app: '{{ meta.name }}-{{ deployment_type }}-postgres'
 spec:
   selector:
     matchLabels:
-      app: '{{ deployment_type }}-postgres'
+      app: '{{ meta.name }}-{{ deployment_type }}-postgres'
   serviceName: '{{ meta.name }}'
   replicas: 1
   updateStrategy:
     type: RollingUpdate
   template:
     metadata:
       labels:
-        app: '{{ deployment_type }}-postgres'
+        app: '{{ meta.name }}-{{ deployment_type }}-postgres'
     spec:
       containers:
         - image: '{{ postgres_image }}'
@@ -71,10 +71,10 @@ metadata:
   name: '{{ meta.name }}-postgres'
   namespace: '{{ meta.namespace }}'
   labels:
-    app: '{{ deployment_type }}-postgres'
+    app: '{{ meta.name }}-{{ deployment_type }}-postgres'
 spec:
   ports:
     - port: 5432
   clusterIP: None
   selector:
-    app: '{{ deployment_type }}-postgres'
+    app: '{{ meta.name }}-{{ deployment_type }}-postgres'

roles/pulp-api/templates/pulp-api.deployment.yaml.j2

@@ -5,16 +5,16 @@ metadata:
   name: "{{ meta.name }}-api"
   namespace: "{{ project_name }}"
   labels:
-    app: "{{ deployment_type }}-api"
+    app: "{{ meta.name }}-{{ deployment_type }}-api"
 spec:
   replicas: {{ pulp_api.replicas }}
   selector:
     matchLabels:
-      app: "{{ deployment_type }}-api"
+      app: "{{ meta.name }}-{{ deployment_type }}-api"
   template:
     metadata:
       labels:
-        app: "{{ deployment_type }}-api"
+        app: "{{ meta.name }}-{{ deployment_type }}-api"
     spec:
 {% if is_k8s %}
       securityContext:
@@ -37,7 +37,7 @@ spec:
 {% if file_storage %}
         - name: pulp-file-storage
           persistentVolumeClaim:
-            claimName: pulp-file-storage
+            claimName: {{ meta.name }}-pulp-file-storage
 {% else %}
         - name: pulp-tmp-file-storage
           emptyDir: {}

roles/pulp-api/templates/pulp-api.service.yaml.j2

@@ -5,10 +5,10 @@ metadata:
   name: "{{ meta.name }}-api-svc"
   namespace: "{{ project_name }}"
   labels:
-    app: "{{ deployment_type }}-api"
+    app: "{{ meta.name }}-{{ deployment_type }}-api"
 spec:
   selector:
-    app: "{{ deployment_type }}-api"
+    app: "{{ meta.name }}-{{ deployment_type }}-api"
   ports:
     - protocol: TCP
       targetPort: 24817

roles/pulp-api/templates/pulp-file-storage.pvc.yaml.j2

@@ -2,7 +2,7 @@
 apiVersion: v1
 kind: PersistentVolumeClaim
 metadata:
-  name: pulp-file-storage
+  name: "{{ meta.name }}-pulp-file-storage"
   namespace: "{{ project_name }}"
 spec:
   resources: