From 615f9dbc844d98dac0df97fe183f4314cd70abb9 Mon Sep 17 00:00:00 2001 From: Chris Hambridge Date: Tue, 6 Apr 2021 10:21:09 -0400 Subject: [PATCH] Enable backup of database and secrets associated with Pulp custom resource * Create new CRD and update CSV bundler * Create new role to capture secrets and db data to PVC * Update backup CR with backup PVC and directory information fixes #8473 https://pulp.plan.io/issues/8473 --- CHANGES/8473.feature | 1 + deploy/crds/pulpproject_v1beta1_pulp_crd.yaml | 7 +- .../pulpproject_v1beta1_pulpbackup_cr.ci.yaml | 7 ++ ...project_v1beta1_pulpbackup_cr.default.yaml | 11 ++ .../pulpproject_v1beta1_pulpbackup_crd.yaml | 68 ++++++++++++ .../pulp-operator.clusterserviceversion.yaml | 46 ++++++++ .../pulp.pulpproject.org_pulpbackups_crd.yml | 67 ++++++++++++ .../pulp.pulpproject.org_pulps_crd.yml | 7 +- docs/roles/backup.md | 1 + down.sh | 1 + playbook.yml | 2 +- roles/backup/README.md | 39 +++++++ roles/backup/defaults/main.yml | 18 ++++ roles/backup/meta/main.yml | 31 ++++++ roles/backup/tasks/cleanup.yml | 15 +++ roles/backup/tasks/error_handling.yml | 16 +++ roles/backup/tasks/init.yml | 100 ++++++++++++++++++ roles/backup/tasks/main.yml | 25 +++++ roles/backup/tasks/postgres.yml | 94 ++++++++++++++++ roles/backup/tasks/secrets.yml | 85 +++++++++++++++ roles/backup/tasks/update_status.yml | 27 +++++ .../templates/admin_password_secret.yaml.j2 | 10 ++ roles/backup/templates/backup.pvc.yaml.j2 | 15 +++ roles/backup/templates/event.yaml.j2 | 17 +++ roles/backup/templates/management-pod.yaml.j2 | 22 ++++ .../backup/templates/postgres_secret.yaml.j2 | 16 +++ roles/backup/templates/pulp_object.yaml.j2 | 9 ++ roles/backup/vars/main.yml | 4 + roles/postgres/tasks/main.yml | 1 - roles/pulp-status/tasks/main.yml | 9 ++ up.sh | 1 + watches.yaml | 5 + 32 files changed, 771 insertions(+), 6 deletions(-) create mode 100644 CHANGES/8473.feature create mode 100644 deploy/crds/pulpproject_v1beta1_pulpbackup_cr.ci.yaml create mode 100644 deploy/crds/pulpproject_v1beta1_pulpbackup_cr.default.yaml create mode 100644 deploy/crds/pulpproject_v1beta1_pulpbackup_crd.yaml create mode 100644 deploy/olm-catalog/pulp-operator/manifests/pulp.pulpproject.org_pulpbackups_crd.yml create mode 120000 docs/roles/backup.md create mode 100644 roles/backup/README.md create mode 100644 roles/backup/defaults/main.yml create mode 100644 roles/backup/meta/main.yml create mode 100644 roles/backup/tasks/cleanup.yml create mode 100644 roles/backup/tasks/error_handling.yml create mode 100644 roles/backup/tasks/init.yml create mode 100644 roles/backup/tasks/main.yml create mode 100644 roles/backup/tasks/postgres.yml create mode 100644 roles/backup/tasks/secrets.yml create mode 100644 roles/backup/tasks/update_status.yml create mode 100644 roles/backup/templates/admin_password_secret.yaml.j2 create mode 100644 roles/backup/templates/backup.pvc.yaml.j2 create mode 100644 roles/backup/templates/event.yaml.j2 create mode 100644 roles/backup/templates/management-pod.yaml.j2 create mode 100644 roles/backup/templates/postgres_secret.yaml.j2 create mode 100644 roles/backup/templates/pulp_object.yaml.j2 create mode 100644 roles/backup/vars/main.yml diff --git a/CHANGES/8473.feature b/CHANGES/8473.feature new file mode 100644 index 000000000..94cc9b8b6 --- /dev/null +++ b/CHANGES/8473.feature @@ -0,0 +1 @@ +Enable backup of database and secrets associated with Pulp custom resource diff --git a/deploy/crds/pulpproject_v1beta1_pulp_crd.yaml b/deploy/crds/pulpproject_v1beta1_pulp_crd.yaml index a46d29fbc..880d22d4f 100644 --- a/deploy/crds/pulpproject_v1beta1_pulp_crd.yaml +++ b/deploy/crds/pulpproject_v1beta1_pulp_crd.yaml @@ -18,6 +18,8 @@ spec: status: {} schema: openAPIV3Schema: + type: object + x-kubernetes-preserve-unknown-fields: true description: Schema validation for the Pulp CRD properties: spec: @@ -369,6 +371,9 @@ spec: adminPasswordSecret: description: Admin password of the deployed instance type: string + databaseConfigruationSecret: + description: Database configuration secret used by the deployed instance + type: string deployedVersion: description: Version of the deployed instance type: string @@ -393,5 +398,3 @@ spec: type: object type: array type: object - type: object - x-kubernetes-preserve-unknown-fields: true diff --git a/deploy/crds/pulpproject_v1beta1_pulpbackup_cr.ci.yaml b/deploy/crds/pulpproject_v1beta1_pulpbackup_cr.ci.yaml new file mode 100644 index 000000000..a09efbe96 --- /dev/null +++ b/deploy/crds/pulpproject_v1beta1_pulpbackup_cr.ci.yaml @@ -0,0 +1,7 @@ +--- +apiVersion: pulp.pulpproject.org/v1beta1 +kind: PulpBackup +metadata: + name: example-pulpbackup +spec: + pulp_name: example-pulp diff --git a/deploy/crds/pulpproject_v1beta1_pulpbackup_cr.default.yaml b/deploy/crds/pulpproject_v1beta1_pulpbackup_cr.default.yaml new file mode 100644 index 000000000..09478d98f --- /dev/null +++ b/deploy/crds/pulpproject_v1beta1_pulpbackup_cr.default.yaml @@ -0,0 +1,11 @@ +--- +apiVersion: pulp.pulpproject.org/v1beta +kind: PulpBackup +metadata: + name: example-pulpbackup +spec: + pulp_name: '' + pulp_backup_pvc: '' + pulp_backup_size: '' + pulp_backup_storage_class: '' + postgres_label_selector: '' diff --git a/deploy/crds/pulpproject_v1beta1_pulpbackup_crd.yaml b/deploy/crds/pulpproject_v1beta1_pulpbackup_crd.yaml new file mode 100644 index 000000000..2f7521dd9 --- /dev/null +++ b/deploy/crds/pulpproject_v1beta1_pulpbackup_crd.yaml @@ -0,0 +1,68 @@ +--- +apiVersion: apiextensions.k8s.io/v1 +kind: CustomResourceDefinition +metadata: + name: pulpbackups.pulp.pulpproject.org +spec: + group: pulp.pulpproject.org + names: + kind: PulpBackup + listKind: PulpBackupList + plural: pulpbackups + singular: pulpbackup + scope: Namespaced + versions: + - name: v1beta1 + served: true + storage: true + subresources: + status: {} + schema: + openAPIV3Schema: + type: object + x-kubernetes-preserve-unknown-fields: true + description: Schema validation for the PulpBackup CRD + properties: + spec: + type: object + properties: + pulp_name: + description: Name of the deployment to be backed up + type: string + pulp_backup_pvc: + description: Name of the PVC to be used for storing the backup + type: string + pulp_backup_size: + description: Size of PVC + type: string + pulp_backup_storage_class: + description: Storage class to use when creating PVC for backup + type: string + postgres_label_selector: + description: Label selector used to identify postgres pod for executing migration + type: string + oneOf: + - required: ["pulp_name"] + status: + properties: + pulpBackupClaim: + description: The PVC name used for the backup + type: string + pulpBackupDirectory: + description: The directory data is backed up to on the PVC + type: string + conditions: + description: The resulting conditions when a Service Telemetry is instantiated + items: + properties: + status: + type: string + type: + type: string + reason: + type: string + lastTransitionTime: + type: string + type: object + type: array + type: object diff --git a/deploy/olm-catalog/pulp-operator/manifests/pulp-operator.clusterserviceversion.yaml b/deploy/olm-catalog/pulp-operator/manifests/pulp-operator.clusterserviceversion.yaml index 8d4237b78..57c5dc2bf 100644 --- a/deploy/olm-catalog/pulp-operator/manifests/pulp-operator.clusterserviceversion.yaml +++ b/deploy/olm-catalog/pulp-operator/manifests/pulp-operator.clusterserviceversion.yaml @@ -268,6 +268,11 @@ spec: path: adminPasswordSecret x-descriptors: - urn:alm:descriptor:io.kubernetes:Secret + - displayName: Database configuration + description: Configuration secret for current deployed database + path: databaseConfigruationSecret + x-descriptors: + - urn:alm:descriptor:io.kubernetes:Secret - displayName: Version description: Version of the instance deployed path: deployedVersion @@ -283,6 +288,47 @@ spec: path: migrantDatabaseConfigurationSecret x-descriptors: - urn:alm:descriptor:io.kubernetes:Secret + - description: A Pulp Backup Instance + kind: PulpBackup + name: pulpbackups.pulp.pulpproject.org + version: v1beta1 + displayName: Pulp Backup + specDescriptors: + - displayName: Pulp Custom Resource Name + path: pulp_name + x-descriptors: + - urn:alm:descriptor:com.tectonic.ui:text + - displayName: Pulp Backup Persistent Volume Claim + path: pulp_backup_pvc + x-descriptors: + - urn:alm:descriptor:com.tectonic.ui:text + - urn:alm:descriptor:com.tectonic.ui:advanced + - displayName: Pulp Backup PVC Size + path: pulp_backup_size + x-descriptors: + - urn:alm:descriptor:com.tectonic.ui:text + - urn:alm:descriptor:com.tectonic.ui:advanced + - displayName: Pulp Backup PVC Storage Class + path: pulp_backup_storage_class + x-descriptors: + - urn:alm:descriptor:com.tectonic.ui:text + - urn:alm:descriptor:com.tectonic.ui:advanced + - displayName: Database Backup Label Selector + path: postgres_label_selector + x-descriptors: + - urn:alm:descriptor:com.tectonic.ui:advanced + - urn:alm:descriptor:com.tectonic.ui:hidden + statusDescriptors: + - displayName: Backup Claim + description: The persistent volume claim name used during backup + path: pulpBackupClaim + x-descriptors: + - urn:alm:descriptor:com.tectonic.ui:text + - displayName: Backup Directory + description: The directory data is backed up to on the PVC + path: pulpBackupDirectory + x-descriptors: + - urn:alm:descriptor:com.tectonic.ui:text description: Pulp operator displayName: Pulp icon: diff --git a/deploy/olm-catalog/pulp-operator/manifests/pulp.pulpproject.org_pulpbackups_crd.yml b/deploy/olm-catalog/pulp-operator/manifests/pulp.pulpproject.org_pulpbackups_crd.yml new file mode 100644 index 000000000..d6bae238a --- /dev/null +++ b/deploy/olm-catalog/pulp-operator/manifests/pulp.pulpproject.org_pulpbackups_crd.yml @@ -0,0 +1,67 @@ +apiVersion: apiextensions.k8s.io/v1 +kind: CustomResourceDefinition +metadata: + name: pulpbackups.pulp.pulpproject.org +spec: + group: pulp.pulpproject.org + names: + kind: PulpBackup + listKind: PulpBackupList + plural: pulpbackups + singular: pulpbackup + scope: Namespaced + versions: + - name: v1beta1 + served: true + storage: true + subresources: + status: {} + schema: + openAPIV3Schema: + type: object + x-kubernetes-preserve-unknown-fields: true + description: Schema validation for the PulpBackup CRD + properties: + spec: + type: object + properties: + pulp_name: + description: Name of the deployment to be backed up + type: string + pulp_backup_pvc: + description: Name of the PVC to be used for storing the backup + type: string + pulp_backup_size: + description: Size of PVC + type: string + pulp_backup_storage_class: + description: Storage class to use when creating PVC for backup + type: string + postgres_label_selector: + description: Label selector used to identify postgres pod for executing migration + type: string + oneOf: + - required: ["pulp_name"] + status: + properties: + pulpBackupClaim: + description: The PVC name used for the backup + type: string + pulpBackupDirectory: + description: The directory data is backed up to on the PVC + type: string + conditions: + description: The resulting conditions when a Service Telemetry is instantiated + items: + properties: + status: + type: string + type: + type: string + reason: + type: string + lastTransitionTime: + type: string + type: object + type: array + type: object diff --git a/deploy/olm-catalog/pulp-operator/manifests/pulp.pulpproject.org_pulps_crd.yml b/deploy/olm-catalog/pulp-operator/manifests/pulp.pulpproject.org_pulps_crd.yml index a46d29fbc..880d22d4f 100644 --- a/deploy/olm-catalog/pulp-operator/manifests/pulp.pulpproject.org_pulps_crd.yml +++ b/deploy/olm-catalog/pulp-operator/manifests/pulp.pulpproject.org_pulps_crd.yml @@ -18,6 +18,8 @@ spec: status: {} schema: openAPIV3Schema: + type: object + x-kubernetes-preserve-unknown-fields: true description: Schema validation for the Pulp CRD properties: spec: @@ -369,6 +371,9 @@ spec: adminPasswordSecret: description: Admin password of the deployed instance type: string + databaseConfigruationSecret: + description: Database configuration secret used by the deployed instance + type: string deployedVersion: description: Version of the deployed instance type: string @@ -393,5 +398,3 @@ spec: type: object type: array type: object - type: object - x-kubernetes-preserve-unknown-fields: true diff --git a/docs/roles/backup.md b/docs/roles/backup.md new file mode 120000 index 000000000..d221700b2 --- /dev/null +++ b/docs/roles/backup.md @@ -0,0 +1 @@ +../../roles/backup/README.md \ No newline at end of file diff --git a/down.sh b/down.sh index f114dcebd..5c8637b0f 100755 --- a/down.sh +++ b/down.sh @@ -27,6 +27,7 @@ $KUBECTL delete -f deploy/cluster_role_binding.yaml # It doesn't matter which cr we specify; the metadata up top is the same. $KUBECTL delete -f deploy/crds/pulpproject_v1beta1_pulp_cr.default.yaml $KUBECTL delete -f deploy/crds/pulpproject_v1beta1_pulp_crd.yaml +$KUBECTL delete -f deploy/crds/pulpproject_v1beta1_pulpbackup_crd.yaml if [[ "$CI_TEST" == "true" ]]; then $KUBECTL delete -f .ci/assets/kubernetes/pulp-admin-password.secret.yaml diff --git a/playbook.yml b/playbook.yml index 828165398..29074ab39 100644 --- a/playbook.yml +++ b/playbook.yml @@ -18,7 +18,7 @@ CONN_MAX_AGE: 0 debug: "True" redis_host: "{{ meta.name }}-redis" - redis_port: 6379 + redis_port: 6379 redis_password: '' deployment_state: present registry: quay.io diff --git a/roles/backup/README.md b/roles/backup/README.md new file mode 100644 index 000000000..b63f31321 --- /dev/null +++ b/roles/backup/README.md @@ -0,0 +1,39 @@ +Backup +======== + +The purpose of this role is to create a backup of your Pulp deployment. This includes: + - backup of the PostgreSQL database + - custom user config file + +Requirements +------------ + +Requires the `openshift` Python library to interact with Kubernetes: `pip install openshift`. + +Role Variables +-------------- + +* `pulp_name`: The name of the pulp custom resource to backup +* `pulp_backup_pvc`: The name of the PVC to uses for backup +* `pulp_backup_size`: The size of storage for the PVC created by operator if one is not supplied +* `pulp_backup_storage_class`: The storage class to be used for the backup PVC +* `postgres_configuration_secret`: The postgres_configuration_secret + + +Dependencies +------------ + +collections: + + - community.kubernetes + - operator_sdk.util + +License +------- + +GPLv2+ + +Author Information +------------------ + +[Pulp Team](https://pulpproject.org/) diff --git a/roles/backup/defaults/main.yml b/roles/backup/defaults/main.yml new file mode 100644 index 000000000..5e20786dd --- /dev/null +++ b/roles/backup/defaults/main.yml @@ -0,0 +1,18 @@ +--- +# Required: specify name of pulp deployment to backup from +pulp_name: '' + +# Specify a pre-created PVC (name) to backup to +pulp_backup_pvc: '' + +# Size of backup PVC if created dynamically +pulp_backup_size: '' + +# Specify storage class to determine how to dynamically create PVC's with +pulp_backup_storage_class: '' + +# Secret Names +pulp_admin_password_secret: "{{ pulp_name }}-admin-password" +postgres_configuration_secret: "{{ pulp_name }}-postgres-configuration" + +custom_resource_key: '_pulp_pulpproject_org_pulpbackup' diff --git a/roles/backup/meta/main.yml b/roles/backup/meta/main.yml new file mode 100644 index 000000000..4f69fcf05 --- /dev/null +++ b/roles/backup/meta/main.yml @@ -0,0 +1,31 @@ +--- +galaxy_info: + author: Pulp Team + description: A role to backup a Pulp deployment + issue_tracker_url: https://pulp.plan.io/projects/pulp/issues/new + license: GPL-2.0-or-later + company: Red Hat + min_ansible_version: 2.9 + platforms: + - name: Debian + versions: + - buster + - name: Fedora + versions: + - 30 + - 31 + - 32 + - 33 + - name: EL + versions: + - 7 + - 8 + galaxy_tags: + - pulp + - pulpcore +dependencies: [] + # List your role dependencies here, one per line. Be sure to remove the '[]' above, + # if you add dependencies to this list. +collections: +- operator_sdk.util +- community.kubernetes diff --git a/roles/backup/tasks/cleanup.yml b/roles/backup/tasks/cleanup.yml new file mode 100644 index 000000000..c817cfb2f --- /dev/null +++ b/roles/backup/tasks/cleanup.yml @@ -0,0 +1,15 @@ +--- + +# After copying secret files to the PVC, delete the local tmp copies +- name: Clean up _secrets directory + ansible.builtin.file: + path: "{{ playbook_dir }}/_secrets" + state: absent + +- name: Delete any existing management pod + community.kubernetes.k8s: + name: "{{ meta.name }}-db-management" + kind: Pod + namespace: "{{ meta.namespace }}" + state: absent + force: true diff --git a/roles/backup/tasks/error_handling.yml b/roles/backup/tasks/error_handling.yml new file mode 100644 index 000000000..f3361d6c3 --- /dev/null +++ b/roles/backup/tasks/error_handling.yml @@ -0,0 +1,16 @@ +--- + +- name: Set apiVersion and kind variables + set_fact: + api_version: '{{ hostvars["localhost"]["inventory_file"].split("/")[4:6] | join("/") }}' + kind: '{{ hostvars["localhost"]["inventory_file"].split("/")[6] }}' + +- name: Determine the timestamp + set_fact: + now: '{{ lookup("pipe", "date +%FT%TZ") }}' + +- name: Emit ocp event with error + community.kubernetes.k8s: + kind: Event + namespace: "{{ meta.namespace }}" + template: "event.yml.j2" diff --git a/roles/backup/tasks/init.yml b/roles/backup/tasks/init.yml new file mode 100644 index 000000000..cc1e8593c --- /dev/null +++ b/roles/backup/tasks/init.yml @@ -0,0 +1,100 @@ +--- + +- name: Delete any existing management pod + community.kubernetes.k8s: + name: "{{ meta.name }}-db-management" + kind: Pod + namespace: "{{ meta.namespace }}" + state: absent + force: true + wait: true + +# Check to make sure provided pvc exists, error loudly if not. Otherwise, the management pod will just stay in pending state forever. +- name: Check provided PVC exists + k8s_info: + name: "{{ pulp_backup_pvc }}" + kind: PersistentVolumeClaim + namespace: "{{ meta.namespace }}" + register: provided_pvc + when: + - pulp_backup_pvc != '' + +- name: Surface error to user + block: + - name: Set error message + set_fact: + error_msg: "{{ pulp_backup_pvc }} does not exist, please create this pvc first." + + - name: Handle error + import_tasks: error_handling.yml + + - name: Fail early if pvc is defined but does not exist + fail: + msg: "{{ pulp_backup_pvc }} does not exist, please create this pvc first." + when: + - pulp_backup_pvc != '' + - provided_pvc.resources | length == 0 + +# If pulp_backup_pvc is defined, use in management-pod.yml.j2 +- name: Set default pvc name + set_fact: + _default_backup_pvc: "{{ meta.name }}-backup-claim" + +# by default, it will re-use the old pvc if already created (unless pvc is provided) +- name: Set PVC to use for backup + set_fact: + backup_pvc: "{{ pulp_backup_pvc | default(_default_backup_pvc, true) }}" + +- name: Create persistent volume claim for backup + community.kubernetes.k8s: + state: present + definition: "{{ lookup('template', 'templates/' + item + '.pvc.yaml.j2') | from_yaml }}" + with_items: + - backup + when: + - pulp_backup_pvc == '' or pulp_backup_pvc is not defined + +- name: Create management pod from templated deployment config + community.kubernetes.k8s: + state: present + definition: "{{ lookup('template', 'templates/management-pod.yaml.j2') | from_yaml }}" + wait: true + +- name: Get Pulp custom resource object + k8s_info: + version: v1beta1 + kind: Pulp + namespace: '{{ meta.namespace }}' + name: '{{ pulp_name }}' + register: _pulp_cro + +- name: Set Pulp object + set_fact: + _pulp: "{{ _pulp_cro['resources'][0] }}" + +- name: Set apiVersion + set_fact: + pulp_api_version: "{{ _pulp['apiVersion'] }}" + +- name: Set user specified spec + set_fact: + pulp_spec: "{{ _pulp['spec'] }}" + +- name: Set pulp status + set_fact: + pulp_status: "{{ _pulp['status'] }}" + +- name: Set pulp admin secret if found + set_fact: + pulp_admin_password_secret: "{{ pulp_status['adminPasswordSecret'] }}" + when: + - pulp_status['adminPasswordSecret'] is defined + - pulp_status['adminPasswordSecret'] | length + +- name: Set pulp database configuration secret if found + set_fact: + postgres_configuration_secret: "{{ pulp_status['databaseConfigruationSecret'] }}" + when: + - pulp_status['databaseConfigruationSecret'] is defined + - pulp_status['databaseConfigruationSecret'] | length + diff --git a/roles/backup/tasks/main.yml b/roles/backup/tasks/main.yml new file mode 100644 index 000000000..5b3ad401f --- /dev/null +++ b/roles/backup/tasks/main.yml @@ -0,0 +1,25 @@ +--- + +- name: Obtain custom resource information + set_fact: + custom_resource: "{{ hostvars[inventory_hostname][custom_resource_key] }}" + custom_resource_status: "{{ hostvars[inventory_hostname][custom_resource_key]['status'] }}" + +- block: + - include_tasks: init.yml + + - include_tasks: postgres.yml + + - include_tasks: secrets.yml + + - name: Set flag signifying this backup was successful + set_fact: + pulp_backup_complete: "{{ _backup_dir }}" + + - include_tasks: cleanup.yml + + when: + - custom_resource_status['pulpBackupDirectory'] is not defined + +- name: Update status variables + include_tasks: update_status.yml diff --git a/roles/backup/tasks/postgres.yml b/roles/backup/tasks/postgres.yml new file mode 100644 index 000000000..df9c4b242 --- /dev/null +++ b/roles/backup/tasks/postgres.yml @@ -0,0 +1,94 @@ +--- + +- name: Check for specified PostgreSQL configuration + k8s_info: + kind: Secret + namespace: '{{ meta.namespace }}' + name: '{{ postgres_configuration_secret }}' + register: _custom_pg_config_resources + when: postgres_configuration_secret | length + +- name: Check for default PostgreSQL configuration + k8s_info: + kind: Secret + namespace: '{{ meta.namespace }}' + name: '{{ pulp_name }}-postgres-configuration' + register: _default_pg_config_resources + +- name: Set PostgreSQL configuration + set_fact: + pg_config: '{{ _custom_pg_config_resources["resources"] | default([]) | length | ternary(_custom_pg_config_resources, _default_pg_config_resources) }}' + +- name: Store Database Configuration + set_fact: + postgres_user: "{{ pg_config['resources'][0]['data']['username'] | b64decode }}" + postgres_pass: "{{ pg_config['resources'][0]['data']['password'] | b64decode }}" + postgres_database: "{{ pg_config['resources'][0]['data']['database'] | b64decode }}" + postgres_port: "{{ pg_config['resources'][0]['data']['port'] | b64decode }}" + postgres_host: "{{ pg_config['resources'][0]['data']['host'] | b64decode }}" + +- name: Default label selector to custom resource generated postgres + set_fact: + postgres_label_selector: "app={{ pulp_name }}-{{ deployment_type }}-postgres" + when: postgres_label_selector is not defined + +- name: Get the postgres pod information + k8s_info: + kind: Pod + namespace: '{{ meta.namespace }}' + label_selectors: + - "{{ postgres_label_selector }}" + register: postgres_pod + until: "postgres_pod['resources'][0]['status']['phase'] == 'Running'" + delay: 5 + retries: 60 + +- name: Set the resource pod name as a variable. + set_fact: + postgres_pod_name: "{{ postgres_pod['resources'][0]['metadata']['name'] }}" + +- name: Determine the timestamp for the backup once for all nodes + set_fact: + now: '{{ lookup("pipe", "date +%F-%T") }}' + +- name: Set backup directory name + set_fact: + _backup_dir: "/backups/pulp-openshift-backup-{{ now }}" + +- name: Create directory for backup + community.kubernetes.k8s_exec: + namespace: "{{ meta.namespace }}" + pod: "{{ meta.name }}-db-management" + command: >- + mkdir -p {{ _backup_dir }} + +- name: Precreate file for database dump + community.kubernetes.k8s_exec: + namespace: "{{ meta.namespace }}" + pod: "{{ meta.name }}-db-management" + command: >- + touch {{ _backup_dir }}/pulp.db + +- name: Set permissions on file for database dump + community.kubernetes.k8s_exec: + namespace: "{{ meta.namespace }}" + pod: "{{ meta.name }}-db-management" + command: >- + chmod 0600 {{ _backup_dir }}/pulp.db + +- name: Set pg_dump command + set_fact: + pgdump: >- + pg_dump --clean --create + -h {{ postgres_host }} + -U {{ postgres_user }} + -d {{ postgres_database }} + -p {{ postgres_port }} + +- name: Write pg_dump to backup on PVC + community.kubernetes.k8s_exec: + namespace: "{{ meta.namespace }}" + pod: "{{ meta.name }}-db-management" + command: >- + bash -c "PGPASSWORD={{ postgres_pass }} {{ pgdump }} > {{ _backup_dir }}/pulp.db" + register: data_migration diff --git a/roles/backup/tasks/secrets.yml b/roles/backup/tasks/secrets.yml new file mode 100644 index 000000000..f03a55119 --- /dev/null +++ b/roles/backup/tasks/secrets.yml @@ -0,0 +1,85 @@ +--- + +- name: Make _secrets directory + file: + path: "_secrets" + state: directory + mode: '0700' + +- name: Template Pulp object definition + template: + src: pulp_object.yaml.j2 + dest: "_secrets/pulp_object.yaml" + mode: '0600' + +- name: Set Pulp object template file as var + set_fact: + pulp_object_template: "{{ lookup('file', '_secrets/pulp_object.yaml') }}" + +- name: Write pulp object to pvc + community.kubernetes.k8s_exec: + namespace: "{{ meta.namespace }}" + pod: "{{ meta.name }}-db-management" + command: >- + bash -c "echo '{{ pulp_object_template }}' > {{ _backup_dir }}/pulp_object.yaml" + +- name: Get admin_password + k8s_info: + kind: Secret + namespace: '{{ meta.namespace }}' + name: '{{ pulp_admin_password_secret }}' + register: _admin_password + +- name: Set admin_password + set_fact: + admin_password: "{{ _admin_password['resources'][0]['data']['password'] | b64decode }}" + +- name: Template admin_password definition + template: + src: admin_password_secret.yaml.j2 + dest: "_secrets/admin_password_secret.yaml" + mode: '0600' + +- name: Set admin_password template + set_fact: + admin_password_template: "{{ lookup('file', '_secrets/admin_password_secret.yaml') }}" + +- name: Write secret_key to pvc + community.kubernetes.k8s_exec: + namespace: "{{ meta.namespace }}" + pod: "{{ meta.name }}-db-management" + command: >- + bash -c "echo '{{ admin_password_template }}' > {{ _backup_dir }}/admin_password_secret.yaml" + +- name: Get postgres configuration + k8s_info: + kind: Secret + namespace: '{{ meta.namespace }}' + name: '{{ postgres_configuration_secret }}' + register: _postgres_configuration + +- name: Set postgres configuration + set_fact: + database_password: "{{ _postgres_configuration['resources'][0]['data']['password'] | b64decode }}" + database_username: "{{ _postgres_configuration['resources'][0]['data']['username'] | b64decode }}" + database_name: "{{ _postgres_configuration['resources'][0]['data']['database'] | b64decode }}" + database_port: "{{ _postgres_configuration['resources'][0]['data']['port'] | b64decode }}" + database_host: "{{ _postgres_configuration['resources'][0]['data']['host'] | b64decode }}" + database_type: "{{ _postgres_configuration['resources'][0]['data']['type'] | b64decode }}" + +- name: Template postgres configuration definition + template: + src: postgres_secret.yaml.j2 + dest: "_secrets/postgres_secret.yaml" + mode: '0600' + +- name: Set postgres configuration + set_fact: + postgres_secret_template: "{{ lookup('file', '_secrets/postgres_secret.yaml') }}" + +- name: Write secret_key to pvc + community.kubernetes.k8s_exec: + namespace: "{{ meta.namespace }}" + pod: "{{ meta.name }}-db-management" + command: >- + bash -c "echo '{{ postgres_secret_template }}' > {{ _backup_dir }}/postgres_secret.yaml" diff --git a/roles/backup/tasks/update_status.yml b/roles/backup/tasks/update_status.yml new file mode 100644 index 000000000..a14476c1d --- /dev/null +++ b/roles/backup/tasks/update_status.yml @@ -0,0 +1,27 @@ +--- +- name: Set apiVersion and kind variables + set_fact: + api_version: '{{ hostvars["localhost"]["inventory_file"].split("/")[4:6] | join("/") }}' + kind: '{{ hostvars["localhost"]["inventory_file"].split("/")[6] }}' + +# The backup directory in this status can be referenced when restoring +- name: Update Pulp Backup status + operator_sdk.util.k8s_status: + api_version: '{{ api_version }}' + kind: "{{ kind }}" + name: "{{ meta.name }}" + namespace: "{{ meta.namespace }}" + status: + pulpBackupDirectory: "{{ _backup_dir }}" + when: pulp_backup_complete is defined + +# The backup directory in this status can be referenced when restoring +- name: Update Pulp Backup status + operator_sdk.util.k8s_status: + api_version: '{{ api_version }}' + kind: "{{ kind }}" + name: "{{ meta.name }}" + namespace: "{{ meta.namespace }}" + status: + pulpBackupClaim: "{{ backup_pvc }}" + when: pulp_backup_complete is defined diff --git a/roles/backup/templates/admin_password_secret.yaml.j2 b/roles/backup/templates/admin_password_secret.yaml.j2 new file mode 100644 index 000000000..3d1df46e4 --- /dev/null +++ b/roles/backup/templates/admin_password_secret.yaml.j2 @@ -0,0 +1,10 @@ +--- +apiVersion: v1 +kind: Secret +metadata: +{% raw %} + name: '{{ meta.name }}' + namespace: '{{ meta.namespace }}' +{% endraw %} +stringData: + password: '{{ admin_password }}' diff --git a/roles/backup/templates/backup.pvc.yaml.j2 b/roles/backup/templates/backup.pvc.yaml.j2 new file mode 100644 index 000000000..dc0f4db15 --- /dev/null +++ b/roles/backup/templates/backup.pvc.yaml.j2 @@ -0,0 +1,15 @@ +--- +apiVersion: v1 +kind: PersistentVolumeClaim +metadata: + name: {{ meta.name }}-backup-claim + namespace: {{ meta.namespace }} +spec: + accessModes: + - ReadWriteOnce +{% if pulp_backup_storage_class != '' %} + storageClassName: {{ pulp_backup_storage_class }} +{% endif %} + resources: + requests: + storage: {{ pulp_backup_size | default('5Gi', true) }} diff --git a/roles/backup/templates/event.yaml.j2 b/roles/backup/templates/event.yaml.j2 new file mode 100644 index 000000000..d26b28403 --- /dev/null +++ b/roles/backup/templates/event.yaml.j2 @@ -0,0 +1,17 @@ +--- +apiVersion: v1 +kind: Event +metadata: + name: backup-error.{{ now }} + namespace: {{ meta.namespace }} +involvedObject: + apiVersion: pulp.pulpproject.org/v1beta1 + kind: {{ kind }} + name: {{ meta.name }} + namespace: {{ meta.namespace }} +message: {{ error_msg }} +reason: BackupFailed +type: Warning +firstTimestamp: {{ now }} +lastTimestamp: {{ now }} +count: 1 diff --git a/roles/backup/templates/management-pod.yaml.j2 b/roles/backup/templates/management-pod.yaml.j2 new file mode 100644 index 000000000..5e72ff1f3 --- /dev/null +++ b/roles/backup/templates/management-pod.yaml.j2 @@ -0,0 +1,22 @@ +--- +apiVersion: v1 +kind: Pod +metadata: + name: {{ meta.name }}-db-management + namespace: {{ meta.namespace }} +spec: + containers: + - name: {{ meta.name }}-db-management + image: "{{ postgres_image }}" + imagePullPolicy: Always + command: ["sleep", "infinity"] + volumeMounts: + - name: {{ meta.name }}-backup + mountPath: /backups + readOnly: false + volumes: + - name: {{ meta.name }}-backup + persistentVolumeClaim: + claimName: {{ backup_pvc }} + readOnly: false + restartPolicy: Never diff --git a/roles/backup/templates/postgres_secret.yaml.j2 b/roles/backup/templates/postgres_secret.yaml.j2 new file mode 100644 index 000000000..b1ad1f6bb --- /dev/null +++ b/roles/backup/templates/postgres_secret.yaml.j2 @@ -0,0 +1,16 @@ +# Postgres Secret. +--- +apiVersion: v1 +kind: Secret +metadata: +{% raw %} + name: '{{ meta.name }}-postgres-configuration' + namespace: '{{ meta.namespace }}' +{% endraw %} +stringData: + password: '{{ database_password }}' + username: '{{ database_username }}' + database: '{{ database_name }}' + port: '{{ database_port }}' + host: '{{ database_host }}' + type: '{{ database_type }}' diff --git a/roles/backup/templates/pulp_object.yaml.j2 b/roles/backup/templates/pulp_object.yaml.j2 new file mode 100644 index 000000000..e063655fa --- /dev/null +++ b/roles/backup/templates/pulp_object.yaml.j2 @@ -0,0 +1,9 @@ +--- +apiVersion: '{{ pulp_api_version }}' +kind: Pulp +metadata: +{% raw %} + name: '{{ meta.name }}' + namespace: '{{ meta.namespace }}' +{% endraw %} +spec: {{ pulp_spec }} diff --git a/roles/backup/vars/main.yml b/roles/backup/vars/main.yml new file mode 100644 index 000000000..1dd3986c9 --- /dev/null +++ b/roles/backup/vars/main.yml @@ -0,0 +1,4 @@ +--- + +deployment_type: "pulp" +postgres_image: postgres:12 diff --git a/roles/postgres/tasks/main.yml b/roles/postgres/tasks/main.yml index 3bf2a11b8..39e10748d 100644 --- a/roles/postgres/tasks/main.yml +++ b/roles/postgres/tasks/main.yml @@ -5,7 +5,6 @@ custom_resource: "{{ hostvars[inventory_hostname][custom_resource_key] }}" custom_resource_status: "{{ hostvars[inventory_hostname][custom_resource_key]['status'] }}" - - name: Record migrant database secret set_fact: recorded_db_migration_secret: "{{ custom_resource_status['migrantDatabaseConfigurationSecret'] }}" diff --git a/roles/pulp-status/tasks/main.yml b/roles/pulp-status/tasks/main.yml index 852fcf6f9..d6c58035f 100644 --- a/roles/pulp-status/tasks/main.yml +++ b/roles/pulp-status/tasks/main.yml @@ -47,6 +47,15 @@ status: adminPasswordSecret: "{{ pulp_admin_password_secret_obj['resources'][0]['metadata']['name'] }}" +- name: Update database configuration status + operator_sdk.util.k8s_status: + api_version: '{{ api_version }}' + kind: "{{ kind }}" + name: "{{ meta.name }}" + namespace: "{{ meta.namespace }}" + status: + databaseConfigruationSecret: "{{ _pg_config['resources'][0]['metadata']['name'] }}" + - name: Update version status operator_sdk.util.k8s_status: api_version: '{{ api_version }}' diff --git a/up.sh b/up.sh index ac6dc607c..aa7a397fe 100755 --- a/up.sh +++ b/up.sh @@ -22,6 +22,7 @@ KUBE_ASSETS_DIR=${KUBE_ASSETS_DIR:-".ci/assets/kubernetes"} # special logic to update # The Custom Resource (and any ConfigMaps) do not. $KUBECTL apply -f deploy/crds/pulpproject_v1beta1_pulp_crd.yaml +$KUBECTL apply -f deploy/crds/pulpproject_v1beta1_pulpbackup_crd.yaml if [[ -e deploy/crds/pulpproject_v1beta1_pulp_cr.yaml ]]; then CUSTOM_RESOURCE=pulpproject_v1beta1_pulp_cr.yaml elif [[ "$CI_TEST" == "true" ]]; then diff --git a/watches.yaml b/watches.yaml index 7c596ef92..6f2897535 100644 --- a/watches.yaml +++ b/watches.yaml @@ -3,3 +3,8 @@ group: pulp.pulpproject.org kind: Pulp playbook: playbook.yml + +- version: v1beta1 + group: pulp.pulpproject.org + kind: PulpBackup + role: backup