From 00948132d03420ef3a7e733a6f02d6258e59dc34 Mon Sep 17 00:00:00 2001 From: Steven Smith Date: Thu, 2 Jun 2022 16:28:26 -0400 Subject: [PATCH] Adds the backup and restore guide for the Operator --- manage_quay/master.adoc | 3 + modules/backing-up-and-restoring-intro.adoc | 4 + modules/backing-up-red-hat-quay.adoc | 212 ++++++++++++++++++++ modules/restoring-red-hat-quay.adoc | 200 ++++++++++++++++++ 4 files changed, 419 insertions(+) create mode 100644 modules/backing-up-and-restoring-intro.adoc create mode 100644 modules/backing-up-red-hat-quay.adoc create mode 100644 modules/restoring-red-hat-quay.adoc diff --git a/manage_quay/master.adoc b/manage_quay/master.adoc index 2951c88d4..cf043f89d 100644 --- a/manage_quay/master.adoc +++ b/manage_quay/master.adoc @@ -98,6 +98,9 @@ include::modules/mirroring-tag-patterns.adoc[leveloffset=+2] include::modules/mirroring-working-with.adoc[leveloffset=+2] include::modules/mirroring-recommend.adoc[leveloffset=+2] +include::modules/backing-up-and-restoring-intro.adoc[leveloffset=+1] +include::modules/backing-up-red-hat-quay.adoc[leveloffset=+2] +include::modules/restoring-red-hat-quay.adoc[leveloffset=+2] :context: manage_quay diff --git a/modules/backing-up-and-restoring-intro.adoc b/modules/backing-up-and-restoring-intro.adoc new file mode 100644 index 000000000..56358cda8 --- /dev/null +++ b/modules/backing-up-and-restoring-intro.adoc @@ -0,0 +1,4 @@ +[[backing-up-and-restoring-intro]] += Backing up and restoring Red Hat Quay on an OpenShift Container Platform deployment + +Use the content within this section to back up and restore {productname} on an OpenShift Container Platform deployment. diff --git a/modules/backing-up-red-hat-quay.adoc b/modules/backing-up-red-hat-quay.adoc new file mode 100644 index 000000000..4f616844d --- /dev/null +++ b/modules/backing-up-red-hat-quay.adoc @@ -0,0 +1,212 @@ +[[backing-up-red-hat-quay]] +== Backing up Red Hat Quay + +This procedure is exclusively for OpenShift Container Platform and NooBaa deployments. + +.Prerequisites + +* A {productname} deployment on OpenShift Container Platform. + + +.Procedure + + +. Backup the `QuayRegistry` custom resource by exporting it: ++ +[source,terminal] +---- +$ oc get quayregistry -n -o yaml > quay-registry.yaml +---- + +. Edit the resulting `quayregistry.yaml` and remove the status section and the following metadata fields: ++ +[source,yaml] +---- + metadata.creationTimestamp + metadata.finalizers + metadata.generation + metadata.resourceVersion + metadata.uid +---- + +. Backup the managed keys secret: ++ +[NOTE] +==== +If you are running a version older than Red Hat Quay 3.7.0, this step can be skipped. Some secrets are automatically generated while deploying Quay for the first time. These are stored in a secret called `-quay-registry-managed-secret-keys` in the QuayRegistry namespace. +==== ++ +[source,terminal] +---- +$ oc get secret -n -quay-registry-managed-secret-keys -o yaml > managed-secret-keys.yaml +---- + +. Edit the the resulting `managed-secret-keys.yaml` file and remove all owner references. Your `managed-secret-keys.yaml` file should look similar to the following: ++ +[source,yaml] +---- +apiVersion: v1 +kind: Secret +type: Opaque +metadata: + name: -quay-registry-managed-secret-keys + namespace: +data: + CONFIG_EDITOR_PW: + DATABASE_SECRET_KEY: + DB_ROOT_PW: + DB_URI: + SECRET_KEY: + SECURITY_SCANNER_V4_PSK: +---- ++ +All information under the `data` property should remain the same. + +. Backup the current Quay configuration: ++ +[source,terminal] +---- +$ oc get secret -n $(oc get quayregistry -n -o jsonpath='{.spec.configBundleSecret}') -o yaml > config-bundle.yaml +---- + +. Scale down the Quay the Quay Operator: ++ +[source,terminal] +---- +$ oc scale --replicas=0 deployment $(oc get deployment -n |awk '/^quay-operator/ {print $1}') -n +---- + +. Scale down the Quay namespace: ++ +[source,terminal] +---- +$ oc scale --replicas=0 deployment $(oc get deployment -n -l quay-component=quay -o jsonpath='{.items[0].metadata.name}') -n +---- + +. Wait for the `registry-quay-app` pods to disappear. You can check their status by running the following command: ++ +[source,terminal] +---- +$ oc get pods -n +---- ++ +Example output: ++ +[source,terminal] +---- +registry-quay-config-editor-77847fc4f5-nsbbv 1/1 Running 0 9m1s +registry-quay-database-66969cd859-n2ssm 1/1 Running 0 6d1h +registry-quay-mirror-758fc68ff7-5wxlp 1/1 Running 0 8m29s +registry-quay-mirror-758fc68ff7-lbl82 1/1 Running 0 8m29s +registry-quay-redis-7cc5f6c977-956g8 1/1 Running 0 5d21h +---- + +. Identify the Quay PostgreSQL pod name: ++ +[source,terminal] +---- +$ oc get pod -l quay-component=postgres -n -o jsonpath='{.items[0].metadata.name}' +---- ++ +Exampe output: +[source,terminal] +---- +quayregistry-quay-database-59f54bb7-58xs7 +---- + +. Obtain the Quay database name: ++ +[source,terminal] +---- +$ oc -n rsh $(oc get pod -l app=quay -o NAME -n |head -n 1) cat /conf/stack/config.yaml|awk -F"/" '/^DB_URI/ {print $4}' +quayregistry-quay-database +---- + +. Download a backup database: ++ +[source,terminal] +---- +$ oc exec quayregistry-quay-database-59f54bb7-58xs7 -- /usr/bin/pg_dump -C quayregistry-quay-database > backup.sql +---- + +. Decode and export the `AWS_ACCESS_KEY_ID`: ++ +[source,terminal] +---- +$ export AWS_ACCESS_KEY_ID=$(oc get secret -l app=noobaa -n -o jsonpath='{.items[0].data.AWS_ACCESS_KEY_ID}' |base64 -d) +---- + +. Decode and export the `AWS_SECRET_ACCESS_KEY_ID`: ++ +[source,terminal] +---- +$ export AWS_SECRET_ACCESS_KEY=$(oc get secret -l app=noobaa -n -o jsonpath='{.items[0].data.AWS_SECRET_ACCESS_KEY}' |base64 -d) +---- + +. Create a new directory and copy all blobs to it: ++ +[source,terminal] +---- +$ mkdir blobs +---- ++ +[source,terminal] ++ +---- +$ aws s3 sync --no-verify-ssl --endpoint https://$(oc get route s3 -n openshift-storage -o jsonpath='{.spec.host}') s3://$(oc get cm -l app=noobaa -n -o jsonpath='{.items[0].data.BUCKET_NAME}') ./blobs +---- + + +[NOTE] +==== +You can also use link:https://rclone.org/[rclone] or link:https://s3tools.org/s3cmd[sc3md] instead of the AWS command line utility. +==== + +. Scale up the Quay the Quay Operator: ++ +[source,terminal] +---- +$ oc scale --replicas=1 deployment $(oc get deployment -n |awk '/^quay-operator/ {print $1}') -n +---- + +. Scale up the Quay namespace: ++ +[source,terminal] +---- +$ oc scale --replicas=1 deployment $(oc get deployment -n -l quay-component=quay -o jsonpath='{.items[0].metadata.name}') -n +---- + +. Check the status of the Operator: ++ +[source,terminal] +---- +$ oc get quayregistry -n -o yaml +---- ++ +Example output: ++ +[source,yaml] +---- +apiVersion: quay.redhat.com/v1 +kind: QuayRegistry +metadata: + ... + name: example-registry + namespace: + ... +spec: + components: + - kind: quay + managed: true + ... + - kind: clairpostgres + managed: true + configBundleSecret: init-config-bundle-secret +status: + configEditorCredentialsSecret: example-registry-quay-config-editor-credentials-fg2gdgtm24 + configEditorEndpoint: https://example-registry-quay-config-editor-quay-enterprise.apps.docs.gcp.quaydev.org + currentVersion: 3.7.0 + lastUpdated: 2022-05-11 13:28:38.199476938 +0000 UTC + registryEndpoint: https://example-registry-quay-quay-enterprise.apps.docs.gcp.quaydev.org + 0 5d21h +---- diff --git a/modules/restoring-red-hat-quay.adoc b/modules/restoring-red-hat-quay.adoc new file mode 100644 index 000000000..3d8d9d2e0 --- /dev/null +++ b/modules/restoring-red-hat-quay.adoc @@ -0,0 +1,200 @@ +[[restoring-up-red-hat-quay]] +== Restoring Red Hat Quay + +This procedure is used to restore {productname} when the Red Hat Quay Operator manages the database. It should be performed after a backup of your Quay registry has been performed. + + +.Prerequisites + +* {productname} is deployed on OpenShift Container Platform using the Quay Operator. +* Your {productname} database has been backed up. + +.Procedure + + +. Restore the backed up Quay configuration and the randomly generated keys: ++ +[source,terminal] +---- +$ oc create -f ./config-bundle.yaml +---- ++ +[source,terminal] +---- +$ oc create -f ./managed-secret-keys.yaml +---- ++ +[NOTE] +==== +If you receive the error `Error from server (AlreadyExists): error when creating "./config-bundle.yaml": secrets "config-bundle-secret" already exists`, you must delete your exist resource with `$ oc delete Secret config-bundle-secret -n ` and recreate it with `$ oc create -f ./config-bundle.yaml`. +==== + +. Restore the QuayRegistry custom resource: ++ +[source,terminal] +---- +$ oc create -f ./quay-registry.yaml +---- + +. Scale down the Quay the Quay Operator: ++ +[source,terminal] +---- +$ oc scale --replicas=0 deployment $(oc get deployment -n |awk '/^quay-operator/ {print $1}') -n +---- + +. Scale down the Quay namespace: ++ +[source,terminal] +---- +$ oc scale --replicas=0 deployment $(oc get deployment -n -l quay-component=quay -o jsonpath='{.items[0].metadata.name}') -n +---- + +. Identify your Quay database pod: ++ +[source,terminal] +---- +$ oc get pod -l quay-component=postgres -n -o jsonpath='{.items[0].metadata.name}' +---- ++ +Example output: ++ +---- +quayregistry-quay-database-59f54bb7-58xs7 +---- + +. Upload the backup by copying it from the local environment and into the pod: ++ +---- +$ oc cp ./backup.sql -n registry-quay-database-66969cd859-n2ssm:/tmp/backup.sql +---- + +. Open a remote terminal to the database: ++ +[source,terminal] +---- +$ oc rsh -n registry-quay-database-66969cd859-n2ssm +---- + +. Enter psql: ++ +[source,terminal] +---- +bash-4.4$ psql +---- + +. You can list the database by running the following command: ++ +---- +postgres=# \l +---- ++ +Example output: ++ +[source,terminal] + List of databases + Name | Owner | Encoding | Collate | Ctype | Access privileges +----------------------------+----------------------------+----------+------------+------------+----------------------- +postgres | postgres | UTF8 | en_US.utf8 | en_US.utf8 | +quayregistry-quay-database | quayregistry-quay-database | UTF8 | en_US.utf8 | en_US.utf8 | + + +. Drop the database: ++ +---- +postgres=# DROP DATABASE "quayregistry-quay-database"; +---- ++ +Example output: ++ +---- +DROP DATABASE +---- + +. Exit the postgres CLI to re-enter bash-4.4: ++ +---- +\q +---- + +. Redirect your PostgreSQL database to your backup database: ++ +[source,terminal] +---- +sh-4.4$ psql < /tmp/backup.sql +---- + +. Exit bash: ++ +---- +sh-4.4$ exit +---- + +. Export the `AWS_ACCESS_KEY_ID`: ++ +[source,terminal] +---- +$ export AWS_ACCESS_KEY_ID=$(oc get secret -l app=noobaa -n -o jsonpath='{.items[0].data.AWS_ACCESS_KEY_ID}' |base64 -d) +---- + +. Export the `AWS_SECRET_ACCESS_KEY`: ++ +[source,terminal] +---- +$ export AWS_SECRET_ACCESS_KEY=$(oc get secret -l app=noobaa -n -o jsonpath='{.items[0].data.AWS_SECRET_ACCESS_KEY}' |base64 -d) +---- + +. Upload all blobs to the bucket by running the following command: ++ +[source,terminal] +---- +$ aws s3 sync --no-verify-ssl --endpoint https://$(oc get route s3 -n openshift-storage -o jsonpath='{.spec.host}') ./blobs s3://$(oc get cm -l app=noobaa -n -o jsonpath='{.items[0].data.BUCKET_NAME}') +---- + +. Scale up the Quay the Quay Operator: ++ +[source,terminal] +---- +$ oc scale --replicas=1 deployment $(oc get deployment -n |awk '/^quay-operator/ {print $1}') -n +---- + +. Scale up the Quay namespace: ++ +[source,terminal] +---- +$ oc scale --replicas=1 deployment $(oc get deployment -n -l quay-component=quay -o jsonpath='{.items[0].metadata.name}') -n +---- +. Check the status of the Operator and ensure it has come back online: ++ +[source,terminal] +---- +$ oc get quayregistry -n -o yaml +---- ++ +Example output: ++ +[source,yaml] +---- +apiVersion: quay.redhat.com/v1 +kind: QuayRegistry +metadata: + ... + name: example-registry + namespace: quay-enterprise + ... +spec: + components: + - kind: quay + managed: true + ... + - kind: clairpostgres + managed: true + configBundleSecret: init-config-bundle-secret +status: + configEditorCredentialsSecret: example-registry-quay-config-editor-credentials-fg2gdgtm24 + configEditorEndpoint: https://example-registry-quay-config-editor-quay-enterprise.apps.docs.gcp.quaydev.org + currentVersion: 3.7.0 + lastUpdated: 2022-05-11 13:28:38.199476938 +0000 UTC + registryEndpoint: https://example-registry-quay-quay-enterprise.apps.docs.gcp.quaydev.org + 0 5d21h +----