Enable backup of database and secrets associated with Pulp custom res…

…ource * Create new CRD and update CSV bundler * Create new role to capture secrets and db data to PVC * Update backup CR with backup PVC and directory information fixes #8473 https://pulp.plan.io/issues/8473
pulp · Apr 6, 2021 · 615f9db · 615f9db
1 parent 27a19db
commit 615f9db
Show file tree

Hide file tree

Showing 32 changed files with 771 additions and 6 deletions.
diff --git a/CHANGES/8473.feature b/CHANGES/8473.feature
@@ -0,0 +1 @@
+Enable backup of database and secrets associated with Pulp custom resource
diff --git a/deploy/crds/pulpproject_v1beta1_pulp_crd.yaml b/deploy/crds/pulpproject_v1beta1_pulp_crd.yaml
@@ -18,6 +18,8 @@ spec:
       status: {}
     schema:
       openAPIV3Schema:
+        type: object
+        x-kubernetes-preserve-unknown-fields: true
         description: Schema validation for the Pulp CRD
         properties:
           spec:
@@ -369,6 +371,9 @@ spec:
               adminPasswordSecret:
                 description: Admin password of the deployed instance
                 type: string
+              databaseConfigruationSecret:
+                description: Database configuration secret used by the deployed instance
+                type: string
               deployedVersion:
                 description: Version of the deployed instance
                 type: string
@@ -393,5 +398,3 @@ spec:
                   type: object
                 type: array
             type: object
-        type: object
-        x-kubernetes-preserve-unknown-fields: true
diff --git a/deploy/crds/pulpproject_v1beta1_pulpbackup_cr.ci.yaml b/deploy/crds/pulpproject_v1beta1_pulpbackup_cr.ci.yaml
@@ -0,0 +1,7 @@
+---
+apiVersion: pulp.pulpproject.org/v1beta1
+kind: PulpBackup
+metadata:
+  name: example-pulpbackup
+spec:
+  pulp_name: example-pulp
diff --git a/deploy/crds/pulpproject_v1beta1_pulpbackup_cr.default.yaml b/deploy/crds/pulpproject_v1beta1_pulpbackup_cr.default.yaml
@@ -0,0 +1,11 @@
+---
+apiVersion: pulp.pulpproject.org/v1beta
+kind: PulpBackup
+metadata:
+  name: example-pulpbackup
+spec:
+  pulp_name: ''
+  pulp_backup_pvc: ''
+  pulp_backup_size: ''
+  pulp_backup_storage_class: ''
+  postgres_label_selector: ''
diff --git a/deploy/crds/pulpproject_v1beta1_pulpbackup_crd.yaml b/deploy/crds/pulpproject_v1beta1_pulpbackup_crd.yaml
@@ -0,0 +1,68 @@
+---
+apiVersion: apiextensions.k8s.io/v1
+kind: CustomResourceDefinition
+metadata:
+  name: pulpbackups.pulp.pulpproject.org
+spec:
+  group: pulp.pulpproject.org
+  names:
+    kind: PulpBackup
+    listKind: PulpBackupList
+    plural: pulpbackups
+    singular: pulpbackup
+  scope: Namespaced
+  versions:
+    - name: v1beta1
+      served: true
+      storage: true
+      subresources:
+        status: {}
+      schema:
+        openAPIV3Schema:
+          type: object
+          x-kubernetes-preserve-unknown-fields: true
+          description: Schema validation for the PulpBackup CRD
+          properties:
+            spec:
+              type: object
+              properties:
+                pulp_name:
+                  description: Name of the deployment to be backed up
+                  type: string
+                pulp_backup_pvc:
+                  description: Name of the PVC to be used for storing the backup
+                  type: string
+                pulp_backup_size:
+                  description: Size of PVC
+                  type: string
+                pulp_backup_storage_class:
+                  description: Storage class to use when creating PVC for backup
+                  type: string
+                postgres_label_selector:
+                  description: Label selector used to identify postgres pod for executing migration
+                  type: string
+              oneOf:
+                - required: ["pulp_name"]
+            status:
+              properties:
+                pulpBackupClaim:
+                  description: The PVC name used for the backup
+                  type: string
+                pulpBackupDirectory:
+                  description: The directory data is backed up to on the PVC
+                  type: string
+                conditions:
+                  description: The resulting conditions when a Service Telemetry is instantiated
+                  items:
+                    properties:
+                      status:
+                        type: string
+                      type:
+                        type: string
+                      reason:
+                        type: string
+                      lastTransitionTime:
+                        type: string
+                    type: object
+                  type: array
+              type: object
diff --git a/deploy/olm-catalog/pulp-operator/manifests/pulp-operator.clusterserviceversion.yaml b/deploy/olm-catalog/pulp-operator/manifests/pulp-operator.clusterserviceversion.yaml
@@ -268,6 +268,11 @@ spec:
         path: adminPasswordSecret
         x-descriptors:
           - urn:alm:descriptor:io.kubernetes:Secret
+      - displayName: Database configuration
+        description: Configuration secret for current deployed database
+        path: databaseConfigruationSecret
+        x-descriptors:
+          - urn:alm:descriptor:io.kubernetes:Secret
       - displayName: Version
         description: Version of the instance deployed
         path: deployedVersion
@@ -283,6 +288,47 @@ spec:
         path: migrantDatabaseConfigurationSecret
         x-descriptors:
           - urn:alm:descriptor:io.kubernetes:Secret
+    - description: A Pulp Backup Instance
+      kind: PulpBackup
+      name: pulpbackups.pulp.pulpproject.org
+      version: v1beta1
+      displayName: Pulp Backup
+      specDescriptors:
+      - displayName: Pulp Custom Resource Name
+        path: pulp_name
+        x-descriptors:
+        - urn:alm:descriptor:com.tectonic.ui:text
+      - displayName: Pulp Backup Persistent Volume Claim
+        path: pulp_backup_pvc
+        x-descriptors:
+        - urn:alm:descriptor:com.tectonic.ui:text
+        - urn:alm:descriptor:com.tectonic.ui:advanced
+      - displayName: Pulp Backup PVC Size
+        path: pulp_backup_size
+        x-descriptors:
+        - urn:alm:descriptor:com.tectonic.ui:text
+        - urn:alm:descriptor:com.tectonic.ui:advanced
+      - displayName: Pulp Backup PVC Storage Class
+        path: pulp_backup_storage_class
+        x-descriptors:
+        - urn:alm:descriptor:com.tectonic.ui:text
+        - urn:alm:descriptor:com.tectonic.ui:advanced
+      - displayName: Database Backup Label Selector
+        path: postgres_label_selector
+        x-descriptors:
+        - urn:alm:descriptor:com.tectonic.ui:advanced
+        - urn:alm:descriptor:com.tectonic.ui:hidden
+      statusDescriptors:
+      - displayName: Backup Claim
+        description: The persistent volume claim name used during backup
+        path: pulpBackupClaim
+        x-descriptors:
+        - urn:alm:descriptor:com.tectonic.ui:text
+      - displayName: Backup Directory
+        description: The directory data is backed up to on the PVC
+        path: pulpBackupDirectory
+        x-descriptors:
+        - urn:alm:descriptor:com.tectonic.ui:text
   description: Pulp operator
   displayName: Pulp
   icon:

diff --git a/deploy/olm-catalog/pulp-operator/manifests/pulp.pulpproject.org_pulpbackups_crd.yml b/deploy/olm-catalog/pulp-operator/manifests/pulp.pulpproject.org_pulpbackups_crd.yml
@@ -0,0 +1,67 @@
+apiVersion: apiextensions.k8s.io/v1
+kind: CustomResourceDefinition
+metadata:
+  name: pulpbackups.pulp.pulpproject.org
+spec:
+  group: pulp.pulpproject.org
+  names:
+    kind: PulpBackup
+    listKind: PulpBackupList
+    plural: pulpbackups
+    singular: pulpbackup
+  scope: Namespaced
+  versions:
+    - name: v1beta1
+      served: true
+      storage: true
+      subresources:
+        status: {}
+      schema:
+        openAPIV3Schema:
+          type: object
+          x-kubernetes-preserve-unknown-fields: true
+          description: Schema validation for the PulpBackup CRD
+          properties:
+            spec:
+              type: object
+              properties:
+                pulp_name:
+                  description: Name of the deployment to be backed up
+                  type: string
+                pulp_backup_pvc:
+                  description: Name of the PVC to be used for storing the backup
+                  type: string
+                pulp_backup_size:
+                  description: Size of PVC
+                  type: string
+                pulp_backup_storage_class:
+                  description: Storage class to use when creating PVC for backup
+                  type: string
+                postgres_label_selector:
+                  description: Label selector used to identify postgres pod for executing migration
+                  type: string
+              oneOf:
+                - required: ["pulp_name"]
+            status:
+              properties:
+                pulpBackupClaim:
+                  description: The PVC name used for the backup
+                  type: string
+                pulpBackupDirectory:
+                  description: The directory data is backed up to on the PVC
+                  type: string
+                conditions:
+                  description: The resulting conditions when a Service Telemetry is instantiated
+                  items:
+                    properties:
+                      status:
+                        type: string
+                      type:
+                        type: string
+                      reason:
+                        type: string
+                      lastTransitionTime:
+                        type: string
+                    type: object
+                  type: array
+              type: object
diff --git a/deploy/olm-catalog/pulp-operator/manifests/pulp.pulpproject.org_pulps_crd.yml b/deploy/olm-catalog/pulp-operator/manifests/pulp.pulpproject.org_pulps_crd.yml
@@ -18,6 +18,8 @@ spec:
       status: {}
     schema:
       openAPIV3Schema:
+        type: object
+        x-kubernetes-preserve-unknown-fields: true
         description: Schema validation for the Pulp CRD
         properties:
           spec:
@@ -369,6 +371,9 @@ spec:
               adminPasswordSecret:
                 description: Admin password of the deployed instance
                 type: string
+              databaseConfigruationSecret:
+                description: Database configuration secret used by the deployed instance
+                type: string
               deployedVersion:
                 description: Version of the deployed instance
                 type: string
@@ -393,5 +398,3 @@ spec:
                   type: object
                 type: array
             type: object
-        type: object
-        x-kubernetes-preserve-unknown-fields: true
diff --git a/docs/roles/backup.md b/docs/roles/backup.md
@@ -0,0 +1 @@
+../../roles/backup/README.md
diff --git a/down.sh b/down.sh
@@ -27,6 +27,7 @@ $KUBECTL delete -f deploy/cluster_role_binding.yaml
 # It doesn't matter which cr we specify; the metadata up top is the same.
 $KUBECTL delete -f deploy/crds/pulpproject_v1beta1_pulp_cr.default.yaml
 $KUBECTL delete -f deploy/crds/pulpproject_v1beta1_pulp_crd.yaml
+$KUBECTL delete -f deploy/crds/pulpproject_v1beta1_pulpbackup_crd.yaml
 
 if [[ "$CI_TEST" == "true" ]]; then
   $KUBECTL delete -f .ci/assets/kubernetes/pulp-admin-password.secret.yaml

diff --git a/playbook.yml b/playbook.yml
@@ -18,7 +18,7 @@
           CONN_MAX_AGE: 0
       debug: "True"
       redis_host: "{{ meta.name }}-redis"
-      redis_port:  6379
+      redis_port: 6379
       redis_password: ''
     deployment_state: present
     registry: quay.io

diff --git a/roles/backup/README.md b/roles/backup/README.md
@@ -0,0 +1,39 @@
+Backup
+========
+
+The purpose of this role is to create a backup of your Pulp deployment.  This includes:
+  - backup of the PostgreSQL database
+  - custom user config file
+
+Requirements
+------------
+
+Requires the `openshift` Python library to interact with Kubernetes: `pip install openshift`.
+
+Role Variables
+--------------
+
+* `pulp_name`: The name of the pulp custom resource to backup
+* `pulp_backup_pvc`: The name of the PVC to uses for backup
+* `pulp_backup_size`: The size of storage for the PVC created by operator if one is not supplied
+* `pulp_backup_storage_class`: The storage class to be used for the backup PVC
+* `postgres_configuration_secret`: The postgres_configuration_secret
+
+
+Dependencies
+------------
+
+collections:
+
+  - community.kubernetes
+  - operator_sdk.util
+
+License
+-------
+
+GPLv2+
+
+Author Information
+------------------
+
+[Pulp Team](https://pulpproject.org/)
diff --git a/roles/backup/defaults/main.yml b/roles/backup/defaults/main.yml
@@ -0,0 +1,18 @@
+---
+# Required: specify name of pulp deployment to backup from
+pulp_name: ''
+
+# Specify a pre-created PVC (name) to backup to
+pulp_backup_pvc: ''
+
+# Size of backup PVC if created dynamically
+pulp_backup_size: ''
+
+# Specify storage class to determine how to dynamically create PVC's with
+pulp_backup_storage_class: ''
+
+# Secret Names
+pulp_admin_password_secret: "{{ pulp_name }}-admin-password"
+postgres_configuration_secret: "{{ pulp_name }}-postgres-configuration"
+
+custom_resource_key: '_pulp_pulpproject_org_pulpbackup'
diff --git a/roles/backup/meta/main.yml b/roles/backup/meta/main.yml
@@ -0,0 +1,31 @@
+---
+galaxy_info:
+  author: Pulp Team
+  description: A role to backup a Pulp deployment
+  issue_tracker_url: https://pulp.plan.io/projects/pulp/issues/new
+  license: GPL-2.0-or-later
+  company: Red Hat
+  min_ansible_version: 2.9
+  platforms:
+    - name: Debian
+      versions:
+        - buster
+    - name: Fedora
+      versions:
+        - 30
+        - 31
+        - 32
+        - 33
+    - name: EL
+      versions:
+        - 7
+        - 8
+  galaxy_tags:
+    - pulp
+    - pulpcore
+dependencies: []
+  # List your role dependencies here, one per line. Be sure to remove the '[]' above,
+  # if you add dependencies to this list.
+collections:
+- operator_sdk.util
+- community.kubernetes
diff --git a/roles/backup/tasks/cleanup.yml b/roles/backup/tasks/cleanup.yml
@@ -0,0 +1,15 @@
+---
+
+# After copying secret files to the PVC, delete the local tmp copies
+- name: Clean up _secrets directory
+  ansible.builtin.file:
+    path: "{{ playbook_dir }}/_secrets"
+    state: absent
+
+- name: Delete any existing management pod
+  community.kubernetes.k8s:
+    name: "{{ meta.name }}-db-management"
+    kind: Pod
+    namespace: "{{ meta.namespace }}"
+    state: absent
+    force: true