diff --git a/_topic_maps/_topic_map.yml b/_topic_maps/_topic_map.yml index 91f8b0322db8..576d3de11e02 100644 --- a/_topic_maps/_topic_map.yml +++ b/_topic_maps/_topic_map.yml @@ -3033,30 +3033,30 @@ Topics: File: ztp-sno-additional-worker-node - Name: Pre-caching images for single-node OpenShift deployments File: ztp-precaching-tool -#- Name: Image-based upgrade for single-node OpenShift clusters -# Dir: image_based_upgrade -# Distros: openshift-origin,openshift-enterprise -# Topics: -# - Name: Understanding the image-based upgrade for single-node OpenShift clusters -# File: cnf-understanding-image-based-upgrade -# - Name: Preparing for an image-based upgrade for single-node OpenShift clusters -# Dir: preparing_for_image_based_upgrade -# Distros: openshift-origin,openshift-enterprise -# Topics: -# - Name: Configuring a shared container directory for the image-based upgrade -# File: cnf-image-based-upgrade-shared-container-image -# - Name: Installing Operators for the image-based upgrade -# File: cnf-image-based-upgrade-install-operators -# - Name: Generating a seed image for the image-based upgrade with Lifecycle Agent -# File: cnf-image-based-upgrade-generate-seed -# - Name: Creating ConfigMap objects for the image-based upgrade with Lifecycle Agent -# File: cnf-image-based-upgrade-prep-resources -# - Name: Creating ConfigMap objects for the image-based upgrade with Lifecycle Agent using GitOps ZTP -# File: ztp-image-based-upgrade-prep-resources -# - Name: Performing an image-based upgrade for single-node OpenShift clusters -# File: cnf-image-based-upgrade-base -# - Name: Performing an image-based upgrade for single-node OpenShift clusters using GitOps ZTP -# File: ztp-image-based-upgrade +- Name: Image-based upgrade for single-node OpenShift clusters + Dir: image_based_upgrade + Distros: openshift-origin,openshift-enterprise + Topics: + - Name: Understanding the image-based upgrade for single-node OpenShift clusters + File: cnf-understanding-image-based-upgrade + - Name: Preparing for an image-based upgrade for single-node OpenShift clusters + Dir: preparing_for_image_based_upgrade + Distros: openshift-origin,openshift-enterprise + Topics: + - Name: Configuring a shared container directory for the image-based upgrade + File: cnf-image-based-upgrade-shared-container-image + - Name: Installing Operators for the image-based upgrade + File: cnf-image-based-upgrade-install-operators + - Name: Generating a seed image for the image-based upgrade with Lifecycle Agent + File: cnf-image-based-upgrade-generate-seed + - Name: Creating ConfigMap objects for the image-based upgrade with Lifecycle Agent + File: cnf-image-based-upgrade-prep-resources + - Name: Creating ConfigMap objects for the image-based upgrade with Lifecycle Agent using GitOps ZTP + File: ztp-image-based-upgrade-prep-resources + - Name: Performing an image-based upgrade for single-node OpenShift clusters + File: cnf-image-based-upgrade-base + - Name: Performing an image-based upgrade for single-node OpenShift clusters using GitOps ZTP + File: ztp-image-based-upgrade --- Name: Reference design specifications Dir: telco_ref_design_specs diff --git a/edge_computing/image_based_upgrade/cnf-image-based-upgrade-base.adoc b/edge_computing/image_based_upgrade/cnf-image-based-upgrade-base.adoc new file mode 100644 index 000000000000..2db81597fd44 --- /dev/null +++ b/edge_computing/image_based_upgrade/cnf-image-based-upgrade-base.adoc @@ -0,0 +1,27 @@ +:_mod-docs-content-type: ASSEMBLY +[id="cnf-image-based-upgrade-for-sno"] += Performing an image-based upgrade for {sno} clusters +include::_attributes/common-attributes.adoc[] +:context: cnf-image-based-upgrade + +toc::[] + +You can use the {lcao} to do a manual image-based upgrade of a {sno} cluster. + +When you deploy the {lcao} on a cluster, an `ImageBasedUpgrade` CR is automatically created. +You update this CR to specify the image repository of the seed image and to move through the different stages. + +:FeatureName: The Lifecycle Agent + +include::modules/cnf-image-based-upgrade-prep.adoc[leveloffset=+1] + +[role="_additional-resources"] +.Additional resources + +* xref:../../edge_computing/image_based_upgrade/preparing_for_image_based_upgrade/ztp-image-based-upgrade-prep-resources.adoc#ztp-image-based-upgrade-creating-backup-resources-with-ztp_ztp-gitops[Creating ConfigMap objects for the image-based upgrade with Lifecycle Agent] + +include::modules/cnf-image-based-upgrade-with-backup.adoc[leveloffset=+1] + +include::modules/cnf-image-based-upgrade-rollback.adoc[leveloffset=+1] + +include::modules/cnf-image-based-upgrade-troubleshooting.adoc[leveloffset=+1] \ No newline at end of file diff --git a/edge_computing/image_based_upgrade/cnf-understanding-image-based-upgrade.adoc b/edge_computing/image_based_upgrade/cnf-understanding-image-based-upgrade.adoc new file mode 100644 index 000000000000..6c40c7022a91 --- /dev/null +++ b/edge_computing/image_based_upgrade/cnf-understanding-image-based-upgrade.adoc @@ -0,0 +1,124 @@ +:_mod-docs-content-type: ASSEMBLY +[id="understanding-image-based-upgrade-for-sno"] += Understanding the image-based upgrade for {sno} clusters +include::_attributes/common-attributes.adoc[] +:context: understanding-image-based-upgrade + +toc::[] + +:FeatureName: The Lifecycle Agent + +From {product-title} 4.14.13, the {lcao} provides you with an alternative way to upgrade the platform version of a {sno} cluster. +The image-based upgrade is faster than the standard upgrade method and allows you to directly upgrade from {product-title} <4.y> to <4.y+2>, and <4.y.z> to <4.y.z+n>. + +// Lifecycle Agent (LCAO) + +This upgrade method utilizes a generated OCI image from a dedicated seed cluster that is installed on the target {sno} cluster as a new `ostree` stateroot. +A seed cluster is a {sno} cluster deployed with the target {product-title} version, Day 2 Operators, and configurations that are common to all target clusters. + +You can use the seed image, which is generated from the seed cluster, to upgrade the platform version on any {sno} cluster that has the same combination of hardware, Day 2 Operators, and cluster configuration as the seed cluster. + +[IMPORTANT] +==== +The image-based upgrade uses custom images that are specific to the hardware platform that the clusters are running on. +Each different hardware platform requires a separate seed image. +==== + +The {lcao} uses two custom resources (CRs) on the participating clusters to orchestrate the upgrade: + +* On the seed cluster, the `SeedGenerator` CR allows for the seed image generation. This CR specifies the repository to push the seed image to. +* On the target cluster, the `ImageBasedUpgrade` CR specifies the seed image for the upgrade of the target cluster and the backup configurations for your workloads. + +.Example SeedGenerator CR +[source,yaml] +---- +apiVersion: lca.openshift.io/v1 +kind: SeedGenerator +metadata: + name: seedimage +spec: + seedImage: +---- + +.Example ImageBasedUpgrade CR +[source,yaml] +---- +apiVersion: lca.openshift.io/v1 +kind: ImageBasedUpgrade +metadata: + name: upgrade +spec: + stage: Idle <1> + seedImageRef: <2> + version: + image: + pullSecretRef: + name: + autoRollbackOnFailure: {} +# initMonitorTimeoutSeconds: 1800 <3> + extraManifests: <4> + - name: example-extra-manifests + namespace: openshift-lifecycle-agent + oadpContent: <5> + - name: oadp-cm-example + namespace: openshift-adp +---- +<1> Defines the desired stage for the `ImageBasedUpgrade` CR. The value can be `Idle`, `Prep`, `Upgrade`, or `Rollback`. +<2> Defines the target platform version, the seed image to be used, and the secret required to access the image. +<3> (Optional) Specify the time frame in seconds to roll back when the upgrade does not complete within that time frame after the first reboot. If not defined or set to `0`, the default value of `1800` seconds (30 minutes) is used. +<4> (Optional) Specify the list of `ConfigMap` resources that contain your custom catalog sources to retain after the upgrade, and your extra manifests to apply to the target cluster that are not part of the seed image. +<5> Specify the list of `ConfigMap` resources that contain the OADP `Backup` and `Restore` CRs. + +include::modules/cnf-image-based-upgrade.adoc[leveloffset=+1] + +[role="_additional-resources"] +.Additional resources + +* xref:../../edge_computing/image_based_upgrade/cnf-image-based-upgrade-base.adoc#cnf-image-based-upgrade-for-sno[Performing an image-based upgrade with Lifecycle Agent] + +* xref:../../edge_computing/image_based_upgrade/ztp-image-based-upgrade.adoc#ztp-image-based-upgrade-for-sno[Performing an image-based upgrade with Lifecycle Agent and GitOps ZTP] + +* xref:../../edge_computing/image_based_upgrade/cnf-image-based-upgrade-base.adoc#ztp-image-based-upgrade-rollback_cnf-image-based-upgrade[(Optional) Moving to the Rollback stage of the image-based upgrade with Lifecycle Agent] + +* xref:../../edge_computing/image_based_upgrade/ztp-image-based-upgrade.adoc#ztp-image-based-upgrade-with-talm-rollback_ztp-image-based-upgrade[(Optional) Moving to the Rollback stage of the image-based upgrade with Lifecycle Agent and GitOps ZTP] + +include::modules/cnf-image-based-upgrade-guidelines.adoc[leveloffset=+1] + +[role="_additional-resources"] +.Additional resources + +* xref:../../installing/disconnected_install/installing-mirroring-installation-images.adoc#installing-mirroring-installation-images[Mirroring images for a disconnected installation] + +include::modules/ztp-image-based-upgrade-cluster-validated-software.adoc[leveloffset=+2] + +include::modules/ztp-image-based-upgrade-hub-cluster-guide.adoc[leveloffset=+2] + +include::modules/ztp-image-based-upgrade-seed-image-guide.adoc[leveloffset=+2] + +[role="_additional-resources"] +.Additional resources + +* xref:../../edge_computing/image_based_upgrade/preparing_for_image_based_upgrade/cnf-image-based-upgrade-shared-container-image.adoc#cnf-image-based-upgrade-shared-container-directory_shared-container-directory[Configuring a shared container directory between ostree stateroots] + +* xref:../../edge_computing/image_based_upgrade/preparing_for_image_based_upgrade/cnf-image-based-upgrade-shared-container-image.adoc#ztp-image-based-upgrade-shared-container-directory_shared-container-directory[Configuring a shared container directory between ostree stateroots when using GitOps ZTP] + +* xref:../../edge_computing/image_based_upgrade/preparing_for_image_based_upgrade/cnf-image-based-upgrade-generate-seed.adoc#ztp-image-based-seed-image-config_generate-seed[Seed image configuration] + +include::modules/ztp-image-based-upgrade-backup-guide.adoc[leveloffset=+2] + +include::modules/ztp-image-based-upgrade-extra-manifests-guide.adoc[leveloffset=+2] + +[role="_additional-resources"] +.Additional resources + +* xref:../../edge_computing/image_based_upgrade/cnf-image-based-upgrade-base.adoc#cnf-image-based-upgrade-for-sno[Performing an image-based upgrade with Lifecycle Agent] + +* xref:../../edge_computing/image_based_upgrade/ztp-image-based-upgrade.adoc#ztp-image-based-upgrade-for-sno[Performing an image-based upgrade with Lifecycle Agent and GitOps ZTP] + +* xref:../../edge_computing/ztp-preparing-the-hub-cluster.adoc#ztp-preparing-the-hub-cluster[Preparing the hub cluster for ZTP] + +* xref:../../edge_computing/image_based_upgrade/preparing_for_image_based_upgrade/cnf-image-based-upgrade-prep-resources.adoc#cnf-image-based-upgrade-creating-backup-oadp-resources_nongitops[Creating ConfigMap objects for the image-based upgrade with Lifecycle Agent] + +* xref:../../edge_computing/image_based_upgrade/preparing_for_image_based_upgrade/ztp-image-based-upgrade-prep-resources.adoc#ztp-image-based-upgrade-creating-backup-resources-with-ztp_ztp-gitops[Creating ConfigMap objects for the image-based upgrade with GitOps ZTP] + +* xref:../../backup_and_restore/application_backup_and_restore/installing/about-installing-oadp.adoc#about-installing-oadp[About installing OADP] \ No newline at end of file diff --git a/edge_computing/image_based_upgrade/preparing_for_ibu/_attributes b/edge_computing/image_based_upgrade/preparing_for_image_based_upgrade/_attributes similarity index 100% rename from edge_computing/image_based_upgrade/preparing_for_ibu/_attributes rename to edge_computing/image_based_upgrade/preparing_for_image_based_upgrade/_attributes diff --git a/edge_computing/image_based_upgrade/preparing_for_image_based_upgrade/cnf-image-based-upgrade-generate-seed.adoc b/edge_computing/image_based_upgrade/preparing_for_image_based_upgrade/cnf-image-based-upgrade-generate-seed.adoc new file mode 100644 index 000000000000..dfd6ed37e5e5 --- /dev/null +++ b/edge_computing/image_based_upgrade/preparing_for_image_based_upgrade/cnf-image-based-upgrade-generate-seed.adoc @@ -0,0 +1,23 @@ +:_mod-docs-content-type: ASSEMBLY +[id="cnf-image-based-upgrade-seed-image"] += Generating a seed image for the image-based upgrade with {lcao} +include::_attributes/common-attributes.adoc[] +:context: generate-seed + +toc::[] + +Use the {lcao} to generate the seed image with the `SeedGenerator` CR. + +:FeatureName: The Lifecycle Agent + + +include::modules/cnf-image-based-upgrade-seed-image-config.adoc[leveloffset=+1] + +include::modules/cnf-image-based-upgrade-generate-seed-image.adoc[leveloffset=+1] + +[role="_additional-resources"] +.Additional resources + +* xref:../../../edge_computing/image_based_upgrade/preparing_for_image_based_upgrade/cnf-image-based-upgrade-shared-container-image.adoc#cnf-image-based-upgrade-shared-container-directory_shared-container-directory[Configuring a shared container directory between ostree stateroots] + +* xref:../../../edge_computing/image_based_upgrade/preparing_for_image_based_upgrade/cnf-image-based-upgrade-shared-container-image#ztp-image-based-upgrade-shared-container-directory_shared-container-directory[Configuring a shared container directory between ostree stateroots when using GitOps ZTP] \ No newline at end of file diff --git a/edge_computing/image_based_upgrade/preparing_for_image_based_upgrade/cnf-image-based-upgrade-install-operators.adoc b/edge_computing/image_based_upgrade/preparing_for_image_based_upgrade/cnf-image-based-upgrade-install-operators.adoc new file mode 100644 index 000000000000..2a3118fc6c87 --- /dev/null +++ b/edge_computing/image_based_upgrade/preparing_for_image_based_upgrade/cnf-image-based-upgrade-install-operators.adoc @@ -0,0 +1,30 @@ +:_mod-docs-content-type: ASSEMBLY +[id="cnf-image-based-upgrade-install-operators"] += Installing Operators for the image-based upgrade +include::_attributes/common-attributes.adoc[] +:context: install-operators + +toc::[] + +You need to prepare your clusters for the upgrade by installing the {lcao} and the OADP Operator. + +To install the OADP Operator with the non-GitOps method, see _Additional resources_. + +:FeatureName: The Lifecycle Agent + +[role="_additional-resources"] +.Additional resources + +* xref:../../../backup_and_restore/application_backup_and_restore/installing/oadp-installing-operator.adoc[Installing the OADP Operator] + +* xref:../../../backup_and_restore/application_backup_and_restore/installing/installing-oadp-ocs.adoc#oadp-about-backup-snapshot-locations_installing-oadp-ocs[About backup and snapshot locations and their secrets] + +* xref:../../../backup_and_restore/application_backup_and_restore/backing_up_and_restoring/oadp-creating-backup-cr.adoc[Creating a Backup CR] + +* xref:../../../backup_and_restore/application_backup_and_restore/backing_up_and_restoring/restoring-applications.adoc#oadp-creating-restore-cr_restoring-applications[Creating a Restore CR] + +include::modules/cnf-image-based-upgrade-installing-lifecycle-agent-using-cli.adoc[leveloffset=+1] + +include::modules/cnf-image-based-upgrade-installing-lifecycle-agent-using-web-console.adoc[leveloffset=+1] + +include::modules/ztp-image-based-upgrade-installing-lifecycle-agent-using-gitops.adoc[leveloffset=+1] \ No newline at end of file diff --git a/edge_computing/image_based_upgrade/preparing_for_image_based_upgrade/cnf-image-based-upgrade-prep-resources.adoc b/edge_computing/image_based_upgrade/preparing_for_image_based_upgrade/cnf-image-based-upgrade-prep-resources.adoc new file mode 100644 index 000000000000..5bde2a0bcb84 --- /dev/null +++ b/edge_computing/image_based_upgrade/preparing_for_image_based_upgrade/cnf-image-based-upgrade-prep-resources.adoc @@ -0,0 +1,24 @@ +:_mod-docs-content-type: ASSEMBLY +[id="cnf-image-based-upgrade-prep-resources"] += Creating ConfigMap objects for the image-based upgrade with {lcao} +include::_attributes/common-attributes.adoc[] +:context: nongitops + +toc::[] + +The {lcao} needs all your OADP resources, extra manifests, and custom catalog sources wrapped in a `ConfigMap` object to process them for the image-based upgrade. + +:FeatureName: The Lifecycle Agent + +include::modules/cnf-image-based-upgrade-prep-oadp.adoc[leveloffset=+1] + +include::modules/cnf-image-based-upgrade-prep-extramanifests.adoc[leveloffset=+1] + +include::modules/cnf-image-based-upgrade-prep-catalogsource.adoc[leveloffset=+1] + +[role="_additional-resources"] +.Additional resources + +* xref:../../../backup_and_restore/application_backup_and_restore/installing/about-installing-oadp.adoc[About installing OADP] + +* xref:../../../edge_computing/image_based_upgrade/cnf-image-based-upgrade-base.adoc#cnf-image-based-upgrade-for-sno[Performing an image-based upgrade with Lifecycle Agent] \ No newline at end of file diff --git a/edge_computing/image_based_upgrade/preparing_for_image_based_upgrade/cnf-image-based-upgrade-shared-container-image.adoc b/edge_computing/image_based_upgrade/preparing_for_image_based_upgrade/cnf-image-based-upgrade-shared-container-image.adoc new file mode 100644 index 000000000000..33247cf38a6a --- /dev/null +++ b/edge_computing/image_based_upgrade/preparing_for_image_based_upgrade/cnf-image-based-upgrade-shared-container-image.adoc @@ -0,0 +1,16 @@ +:_mod-docs-content-type: ASSEMBLY +[id="cnf-image-based-upgrade-shared-varlibcontainers"] += Configuring a shared container directory for the image-based upgrade +include::_attributes/common-attributes.adoc[] +:context: shared-container-directory + +toc::[] + +Your {sno} clusters need to have a shared `var/lib/containers` partition for the image-based upgrade. +You can do this at install time. + +:FeatureName: The Lifecycle Agent + +include::modules/cnf-image-based-upgrade-share-container-directory.adoc[leveloffset=+1] + +include::modules/ztp-image-based-upgrade-share-container-directory.adoc[leveloffset=+1] \ No newline at end of file diff --git a/edge_computing/image_based_upgrade/preparing_for_ibu/images b/edge_computing/image_based_upgrade/preparing_for_image_based_upgrade/images similarity index 100% rename from edge_computing/image_based_upgrade/preparing_for_ibu/images rename to edge_computing/image_based_upgrade/preparing_for_image_based_upgrade/images diff --git a/edge_computing/image_based_upgrade/preparing_for_ibu/modules b/edge_computing/image_based_upgrade/preparing_for_image_based_upgrade/modules similarity index 100% rename from edge_computing/image_based_upgrade/preparing_for_ibu/modules rename to edge_computing/image_based_upgrade/preparing_for_image_based_upgrade/modules diff --git a/edge_computing/image_based_upgrade/preparing_for_ibu/snippets b/edge_computing/image_based_upgrade/preparing_for_image_based_upgrade/snippets similarity index 100% rename from edge_computing/image_based_upgrade/preparing_for_ibu/snippets rename to edge_computing/image_based_upgrade/preparing_for_image_based_upgrade/snippets diff --git a/edge_computing/image_based_upgrade/preparing_for_image_based_upgrade/ztp-image-based-upgrade-prep-resources.adoc b/edge_computing/image_based_upgrade/preparing_for_image_based_upgrade/ztp-image-based-upgrade-prep-resources.adoc new file mode 100644 index 000000000000..9ccfba9b9122 --- /dev/null +++ b/edge_computing/image_based_upgrade/preparing_for_image_based_upgrade/ztp-image-based-upgrade-prep-resources.adoc @@ -0,0 +1,23 @@ +:_mod-docs-content-type: ASSEMBLY +[id="ztp-image-based-upgrade-prep-resources"] += Creating ConfigMap objects for the image-based upgrade with {lcao} using {ztp} +include::_attributes/common-attributes.adoc[] +:context: ztp-gitops + +toc::[] + +The {lcao} needs all your OADP resources, extra manifests, and custom catalog sources wrapped in a `ConfigMap` object to process them for the image-based upgrade. + +:FeatureName: The Lifecycle Agent + +include::modules/ztp-image-based-upgrade-prep-oadp.adoc[leveloffset=+1] + + +include::modules/ztp-image-based-upgrade-prep-label-extramanifests.adoc[leveloffset=+1] + +[role="_additional-resources"] +.Additional resources + +* xref:../../../edge_computing/image_based_upgrade/preparing_for_image_based_upgrade/cnf-image-based-upgrade-shared-container-image.adoc#ztp-image-based-upgrade-shared-container-directory_shared-container-directory[Configuring a shared container directory between ostree stateroots when using GitOps ZTP] + +* xref:../../../edge_computing/image_based_upgrade/ztp-image-based-upgrade.adoc#ztp-image-based-upgrade-for-sno[Performing an image-based upgrade with Lifecycle Agent and GitOps ZTP] \ No newline at end of file diff --git a/edge_computing/image_based_upgrade/ztp-image-based-upgrade.adoc b/edge_computing/image_based_upgrade/ztp-image-based-upgrade.adoc new file mode 100644 index 000000000000..3a483a2bc5e6 --- /dev/null +++ b/edge_computing/image_based_upgrade/ztp-image-based-upgrade.adoc @@ -0,0 +1,39 @@ +:_mod-docs-content-type: ASSEMBLY +[id="ztp-image-based-upgrade-for-sno"] += Performing an image-based upgrade for {sno} clusters using GitOps ZTP +:context: ztp-image-based-upgrade +include::_attributes/common-attributes.adoc[] + +toc::[] + +You can upgrade your managed {sno} cluster with the image-based upgrade through GitOps ZTP. + +When you deploy the {lcao} on a cluster, an `ImageBasedUpgrade` CR is automatically created. +You update this CR to specify the image repository of the seed image and to move through the different stages. + +:FeatureName: The Lifecycle Agent + +// Lifecycle Agent (LCAO) + +include::modules/ztp-image-based-upgrade-talm-prep.adoc[leveloffset=+1] + +[role="_additional-resources"] +.Additional resources + +* xref:../../edge_computing/ztp-preparing-the-hub-cluster.adoc#ztp-preparing-the-ztp-git-repository-ver-ind_ztp-preparing-the-hub-cluster[Preparing the GitOps ZTP site configuration repository for version independence] + +* xref:../../edge_computing/image_based_upgrade/preparing_for_image_based_upgrade/ztp-image-based-upgrade-prep-resources.adoc#ztp-image-based-upgrade-prep-resources[Creating ConfigMap objects for the image-based upgrade with Lifecycle Agent using GitOps ZTP] + +* xref:../../edge_computing/image_based_upgrade/preparing_for_image_based_upgrade/cnf-image-based-upgrade-shared-container-image.adoc#ztp-image-based-upgrade-shared-container-directory_shared-container-directory[Configuring a shared container directory between ostree stateroots when using GitOps ZTP] + +* xref:../../backup_and_restore/application_backup_and_restore/installing/installing-oadp-ocs.adoc#oadp-about-backup-snapshot-locations_installing-oadp-ocs[About backup and snapshot locations and their secrets] + +* xref:../../backup_and_restore/application_backup_and_restore/backing_up_and_restoring/oadp-creating-backup-cr.adoc#oadp-creating-backup-cr-doc[Creating a Backup CR] + +* xref:../../backup_and_restore/application_backup_and_restore/backing_up_and_restoring/restoring-applications.adoc#oadp-creating-restore-cr_restoring-applications[Creating a Restore CR] + +include::modules/ztp-image-based-upgrade-talm-upgrade.adoc[leveloffset=+1] + +include::modules/ztp-image-based-upgrade-talm-rollback.adoc[leveloffset=+1] + +include::modules/cnf-image-based-upgrade-troubleshooting.adoc[leveloffset=+1] \ No newline at end of file diff --git a/images/696_OpenShift_Lifecycle_Agent_0624_0.png b/images/696_OpenShift_Lifecycle_Agent_0624_0.png new file mode 100644 index 000000000000..153ea58e597b Binary files /dev/null and b/images/696_OpenShift_Lifecycle_Agent_0624_0.png differ diff --git a/images/696_OpenShift_Lifecycle_Agent_0624_1.png b/images/696_OpenShift_Lifecycle_Agent_0624_1.png new file mode 100644 index 000000000000..72fb94cc30ef Binary files /dev/null and b/images/696_OpenShift_Lifecycle_Agent_0624_1.png differ diff --git a/images/696_OpenShift_Lifecycle_Agent_0624_2.png b/images/696_OpenShift_Lifecycle_Agent_0624_2.png new file mode 100644 index 000000000000..b6ae2e9b1c9d Binary files /dev/null and b/images/696_OpenShift_Lifecycle_Agent_0624_2.png differ diff --git a/images/696_OpenShift_Lifecycle_Agent_0624_3.png b/images/696_OpenShift_Lifecycle_Agent_0624_3.png new file mode 100644 index 000000000000..52db05e6726e Binary files /dev/null and b/images/696_OpenShift_Lifecycle_Agent_0624_3.png differ diff --git a/images/696_OpenShift_Lifecycle_Agent_0624_4.png b/images/696_OpenShift_Lifecycle_Agent_0624_4.png new file mode 100644 index 000000000000..fcadd8fae77c Binary files /dev/null and b/images/696_OpenShift_Lifecycle_Agent_0624_4.png differ diff --git a/images/696_OpenShift_Lifecycle_Agent_0624_5.png b/images/696_OpenShift_Lifecycle_Agent_0624_5.png new file mode 100644 index 000000000000..98fa95d63bbc Binary files /dev/null and b/images/696_OpenShift_Lifecycle_Agent_0624_5.png differ diff --git a/modules/cnf-image-based-upgrade-generate-seed-image.adoc b/modules/cnf-image-based-upgrade-generate-seed-image.adoc new file mode 100644 index 000000000000..aa1a21690e5c --- /dev/null +++ b/modules/cnf-image-based-upgrade-generate-seed-image.adoc @@ -0,0 +1,184 @@ +// Module included in the following assemblies: +// * edge_computing/image-based-upgrade/cnf-preparing-for-image-based-upgrade.adoc + +:_mod-docs-content-type: PROCEDURE +[id="ztp-image-based-upgrade-seed-generation_{context}"] += Generating a seed image with the {lcao} + +Use the {lcao} to generate the seed image with the `SeedGenerator` CR. The Operator checks for required system configurations, performs any necessary system cleanup before generating the seed image, and launches the image generation. The seed image generation includes the following tasks: + +* Stopping cluster operators +* Preparing the seed image configuration +* Generating and pushing the seed image to the image repository specified in the `SeedGenerator` CR +* Restoring cluster operators +* Expiring seed cluster certificates +* Generating new certificates for the seed cluster +* Restoring and updating the `SeedGenerator` CR on the seed cluster + +.Prerequisites + +* Configure a shared container directory on the seed cluster. +* Install the OADP Operator and the {lcao} on the seed cluster. + +.Procedure + +. Detach the cluster from the hub to delete any cluster-specific resources from the seed cluster that must not be in the seed image: + +.. If you are using {rh-rhacm}, manually detach the seed cluster by running the following command: ++ +[source,terminal] +---- +$ oc delete managedcluster sno-worker-example +---- + +... Wait until the `ManagedCluster` CR is removed. Once the CR is removed, create the proper `SeedGenerator` CR. The {lcao} cleans up the {rh-rhacm} artifacts. + +.. If you are using GitOps ZTP, detach your cluster by removing the seed cluster's `SiteConfig` CR from the `kustomization.yaml`: + +... Remove your seed cluster's `SiteConfig` CR from the `kustomization.yaml`. ++ +[source,yaml] +---- +apiVersion: kustomize.config.k8s.io/v1beta1 +kind: Kustomization + +generators: +#- example-seed-sno1.yaml +- example-target-sno2.yaml +- example-target-sno3.yaml +---- + +... Commit the `kustomization.yaml` changes in your Git repository and push the changes. + ++ +The ArgoCD pipeline detects the changes and removes the managed cluster. + +. Create the `Secret`. + +.. Create the authentication file by running the following command: ++ +-- +.Authentication file +[source,terminal] +---- +$ MY_USER=myuserid +$ AUTHFILE=/tmp/my-auth.json +$ podman login --authfile ${AUTHFILE} -u ${MY_USER} quay.io/${MY_USER} +---- + +[source,terminal] +---- +$ base64 -w 0 ${AUTHFILE} ; echo +---- +-- + +.. Copy the output into the `seedAuth` field in the `Secret` YAML file named `seedgen` in the `openshift-lifecycle-agent` namespace. ++ +-- +[source,yaml] +---- +apiVersion: v1 +kind: Secret +metadata: + name: seedgen <1> + namespace: openshift-lifecycle-agent +type: Opaque +data: + seedAuth: <2> +---- +<1> The `Secret` resource must have the `name: seedgen` and `namespace: openshift-lifecycle-agent` fields. +<2> Specifies a base64-encoded authfile for write-access to the registry for pushing the generated seed images. +-- + +.. Apply the `Secret`. ++ +[source,terminal] +---- +$ oc apply -f secretseedgenerator.yaml +---- + +. Create the `SeedGenerator` CR: ++ +-- +[source,yaml] +---- +apiVersion: lca.openshift.io/v1 +kind: SeedGenerator +metadata: + name: seedimage <1> +spec: + seedImage: <2> +---- +<1> The `SeedGenerator` CR must be named `seedimage`. +<2> Specify the container image URL, for example, `quay.io/example/seed-container-image:`. It is recommended to use the `:` format. +-- + +. Generate the seed image by running the following command: ++ +[source,terminal] +---- +$ oc apply -f seedgenerator.yaml +---- + ++ +[IMPORTANT] +==== +The cluster reboots and loses API capabilities while the {lcao} generates the seed image. +Applying the `SeedGenerator` CR stops the `kubelet` and the CRI-O operations, then it starts the image generation. +==== + +If you want to generate further seed images, you must provision a new seed cluster with the version you want to generate a seed image from. + +.Verification + +. Once the cluster recovers and it is available, you can check the status of the `SeedGenerator` CR: ++ +-- +[source,terminal] +---- +$ oc get seedgenerator -oyaml +---- + +.Example output +[source,yaml] +---- +status: + conditions: + - lastTransitionTime: "2024-02-13T21:24:26Z" + message: Seed Generation completed + observedGeneration: 1 + reason: Completed + status: "False" + type: SeedGenInProgress + - lastTransitionTime: "2024-02-13T21:24:26Z" + message: Seed Generation completed + observedGeneration: 1 + reason: Completed + status: "True" + type: SeedGenCompleted <1> + observedGeneration: 1 +---- +<1> The seed image generation is complete. +-- + +. Verify that the {sno} cluster is running and is attached to the {rh-rhacm} hub cluster: ++ +-- +[source,terminal] +---- +$ oc get managedclusters sno-worker-example +---- + +.Example output +[source,terminal] +---- +NAME HUB ACCEPTED MANAGED CLUSTER URLS JOINED AVAILABLE AGE +sno-worker-example true https://api.sno-worker-example.example.redhat.com True True 21h <1> +---- +<1> The cluster is attached if you see that the value is `True` for both `JOINED` and `AVAILABLE`. + +[NOTE] +==== +The cluster requires time to recover after restarting the `kubelet` operation. +==== +-- \ No newline at end of file diff --git a/modules/cnf-image-based-upgrade-guidelines.adoc b/modules/cnf-image-based-upgrade-guidelines.adoc new file mode 100644 index 000000000000..a794e0331bcc --- /dev/null +++ b/modules/cnf-image-based-upgrade-guidelines.adoc @@ -0,0 +1,15 @@ +// Module included in the following assemblies: +// * edge_computing/image-based-upgrade/cnf-understanding-image-based-upgrade.adoc + +:_mod-docs-content-type: CONCEPT +[id="ztp-image-based-upgrade-guide_{context}"] += Guidelines for the image-based upgrade + +For a successful image-based upgrade, your deployments need to consider certain requirements. + +There are different deployment methods in which you can perform the image-based upgrade: + +{ztp}:: You use the {ztp-first} to deploy and configure your clusters. +Non-GitOps:: You only use {rh-rhacm-first} to deploy and configure your clusters. + +You can perform an image-based upgrade in disconnected environments. For more information about how to mirror images for a disconnected environment, see _Additional resources_. \ No newline at end of file diff --git a/modules/cnf-image-based-upgrade-installing-lifecycle-agent-using-cli.adoc b/modules/cnf-image-based-upgrade-installing-lifecycle-agent-using-cli.adoc new file mode 100644 index 000000000000..94d54aee1844 --- /dev/null +++ b/modules/cnf-image-based-upgrade-installing-lifecycle-agent-using-cli.adoc @@ -0,0 +1,111 @@ +// Module included in the following assemblies: +// * edge_computing/image-based-upgrade/cnf-preparing-for-image-based-upgrade.adoc + +:_mod-docs-content-type: PROCEDURE +[id="installing-lcao-using-cli_{context}"] += Installing the {lcao} by using the CLI + +You can use the OpenShift CLI (`oc`) to install the {lcao}. + +.Prerequisites + +* Install the OpenShift CLI (`oc`). +* Log in as a user with `cluster-admin` privileges. + +.Procedure + +. Create a `Namespace` object YAML file for the {lcao}, for example `lcao-namespace.yaml`. ++ +[source,yaml] +---- +apiVersion: v1 +kind: Namespace +metadata: + name: openshift-lifecycle-agent + annotations: + workload.openshift.io/allowed: management +---- + +.. Create the `Namespace` CR: ++ +[source,terminal] +---- +$ oc create -f lcao-namespace.yaml +---- + +. Create an `OperatorGroup` object YAML file for the {lcao}, for example `lcao-operatorgroup.yaml`. ++ +[source,yaml] +---- +apiVersion: operators.coreos.com/v1 +kind: OperatorGroup +metadata: + name: openshift-lifecycle-agent + namespace: openshift-lifecycle-agent +spec: + targetNamespaces: + - openshift-lifecycle-agent +---- + +.. Create the `OperatorGroup` CR: ++ +[source,terminal] +---- +$ oc create -f lcao-operatorgroup.yaml +---- + +. Create a `Subscription` CR: + +.. Create the `Subscription` object YAML file for the {lcao}, for example, `lcao-subscription.yaml`: ++ +[source,yaml] +---- +apiVersion: operators.coreos.com/v1 +kind: Subscription +metadata: + name: openshift-lifecycle-agent-subscription + namespace: openshift-lifecycle-agent +spec: + channel: "stable" + name: lifecycle-agent + source: redhat-operators + sourceNamespace: openshift-marketplace +---- + +.. Create the `Subscription` CR by running the following command: ++ +[source,terminal] +---- +$ oc create -f lcao-subscription.yaml +---- + +.Verification + +. Verify that the installation succeeded by inspecting the CSV resource: ++ +[source,terminal] +---- +$ oc get csv -n openshift-lifecycle-agent +---- ++ +.Example output +[source,terminal,subs="attributes+"] +---- +NAME DISPLAY VERSION REPLACES PHASE +lifecycle-agent.v{product-version}.0 Openshift Lifecycle Agent {product-version}.0 Succeeded +---- + +. Verify that the {lcao} is up and running: ++ +[source,terminal] +---- +$ oc get deploy -n openshift-lifecycle-agent +---- + ++ +.Example output +[source,terminal] +---- +NAME READY UP-TO-DATE AVAILABLE AGE +lifecycle-agent-controller-manager 1/1 1 1 14s +---- \ No newline at end of file diff --git a/modules/cnf-image-based-upgrade-installing-lifecycle-agent-using-web-console.adoc b/modules/cnf-image-based-upgrade-installing-lifecycle-agent-using-web-console.adoc new file mode 100644 index 000000000000..b0827f9abf25 --- /dev/null +++ b/modules/cnf-image-based-upgrade-installing-lifecycle-agent-using-web-console.adoc @@ -0,0 +1,36 @@ +// Module included in the following assemblies: +// * edge_computing/image-based-upgrade/cnf-preparing-for-image-based-upgrade.adoc + +:_mod-docs-content-type: PROCEDURE +[id="installing-lifecycle-agent-using-web-console_{context}"] += Installing the {lcao} by using the web console + +You can use the {product-title} web console to install the {lcao}. + +.Prerequisites + +* Log in as a user with `cluster-admin` privileges. + +.Procedure + +. In the {product-title} web console, navigate to *Operators* → *OperatorHub*. +. Search for the *{lcao}* from the list of available Operators, and then click *Install*. +. On the *Install Operator* page, under *A specific namespace on the cluster* select *openshift-lifecycle-agent*. Then, click Install. +. Click *Install*. + +.Verification + +To confirm that the installation is successful: + +. Navigate to the *Operators* → *Installed Operators* page. +. Ensure that the {lcao} is listed in the *openshift-lifecycle-agent* project with a *Status* of *InstallSucceeded*. + +[NOTE] +==== +During installation an Operator might display a *Failed* status. If the installation later succeeds with an *InstallSucceeded* message, you can ignore the *Failed* message. +==== + +If the Operator is not installed successfully: + +. Go to the *Operators* → *Installed Operators* page and inspect the *Operator Subscriptions* and *Install Plans* tabs for any failure or errors under *Status*. +. Go to the *Workloads* → *Pods* page and check the logs for pods in the *openshift-lifecycle-agent* project. \ No newline at end of file diff --git a/modules/cnf-image-based-upgrade-prep-catalogsource.adoc b/modules/cnf-image-based-upgrade-prep-catalogsource.adoc new file mode 100644 index 000000000000..cfeafee5e8c5 --- /dev/null +++ b/modules/cnf-image-based-upgrade-prep-catalogsource.adoc @@ -0,0 +1,46 @@ +// Module included in the following assemblies: +// * edge_computing/image-based-upgrade/cnf-preparing-for-image-based-upgrade.adoc + +:_mod-docs-content-type: PROCEDURE +[id="cnf-image-based-upgrade-creating-backup-custom-catalog-sources_{context}"] += (Optional) Creating ConfigMap objects of custom catalog sources for the image-based upgrade with {lcao} + +You can keep your custom catalog sources after the upgrade by generating a `ConfigMap` object for your catalog sources and adding them to the `spec.extraManifest` field in the `ImageBasedUpgrade` CR. +For more information about catalog sources, see xref:https://access.redhat.com/documentation/en-us/openshift_container_platform/4.15/html-single/operators/index#olm-catalogsource_olm-understanding-olm[Catalog source]. + +.Procedure + +. Create a YAML file that contains the `CatalogSource` CR. ++ +-- +[source,yaml] +---- +apiVersion: operators.coreos.com/v1 +kind: CatalogSource +metadata: + name: example-catalogsources + namespace: openshift-marketplace +spec: + sourceType: grpc + displayName: disconnected-redhat-operators + image: quay.io/example-org/example-catalog:v1 +---- +-- + +. Create the `ConfigMap` object: ++ +[source,terminal] +---- +$ oc create configmap example-catalogsources-cm --from-file=example-catalogsources.yaml= -n openshift-lifecycle-agent +---- + +. Patch the `ImageBasedUpgrade` CR: ++ +[source,terminal] +---- +$ oc patch imagebasedupgrades.lca.openshift.io upgrade \ +-p='{"spec": {"extraManifests": [{"name": "example-catalogsources-cm", "namespace": "openshift-lifecycle-agent"}]}}' \ +--type=merge -n openshift-lifecycle-agent +---- + +To use the `ConfigMap` objects in the upgrade, see the _Performing an image-based upgrade with {lcao}_ section. \ No newline at end of file diff --git a/modules/cnf-image-based-upgrade-prep-extramanifests.adoc b/modules/cnf-image-based-upgrade-prep-extramanifests.adoc new file mode 100644 index 000000000000..1b6a05f5076f --- /dev/null +++ b/modules/cnf-image-based-upgrade-prep-extramanifests.adoc @@ -0,0 +1,64 @@ + +// Module included in the following assemblies: +// * edge_computing/image-based-upgrade/cnf-preparing-for-image-based-upgrade.adoc + +:_mod-docs-content-type: PROCEDURE +[id="cnf-image-based-upgrade-creating-backup-extra-manifests_{context}"] += (Optional) Creating ConfigMap objects of extra manifests for the image-based upgrade with {lcao} + +Create your additional manifests that you want to apply to the target cluster. + +.Procedure + +. Create a YAML file that contains your extra manifests. ++ +[source,yaml] +---- +apiVersion: sriovnetwork.openshift.io/v1 +kind: SriovNetworkNodePolicy +metadata: + name: "pci-sriov-net-e5l" + namespace: openshift-sriov-network-operator +spec: + deviceType: vfio-pci + isRdma: false + nicSelector: + pfNames: [ens1f0] + nodeSelector: + node-role.kubernetes.io/master: "" + mtu: 1500 + numVfs: 8 + priority: 99 + resourceName: pci_sriov_net_e5l +--- +apiVersion: sriovnetwork.openshift.io/v1 +kind: SriovNetwork +metadata: + name: "networking-e5l" + namespace: openshift-sriov-network-operator +spec: + ipam: |- + { + } + linkState: auto + networkNamespace: sriov-namespace + resourceName: pci_sriov_net_e5l + spoofChk: "on" + trust: "off" +---- + +. Create the `ConfigMap` object: ++ +[source,terminal] +---- +$ oc create configmap example-extra-manifests-cm --from-file=example-extra-manifests.yaml= -n openshift-lifecycle-agent +---- + +. Patch the `ImageBasedUpgrade` CR: ++ +[source,terminal] +---- +$ oc patch imagebasedupgrades.lca.openshift.io upgrade \ +-p='{"spec": {"extraManifests": [{"name": "example-extra-manifests-cm", "namespace": "openshift-lifecycle-agent"}]}}' \ +--type=merge -n openshift-lifecycle-agent +---- \ No newline at end of file diff --git a/modules/cnf-image-based-upgrade-prep-oadp.adoc b/modules/cnf-image-based-upgrade-prep-oadp.adoc new file mode 100644 index 000000000000..be29c1cb93ea --- /dev/null +++ b/modules/cnf-image-based-upgrade-prep-oadp.adoc @@ -0,0 +1,73 @@ +// Module included in the following assemblies: +// * edge_computing/image-based-upgrade/cnf-preparing-for-image-based-upgrade.adoc + +:_mod-docs-content-type: PROCEDURE +[id="cnf-image-based-upgrade-creating-backup-oadp-resources_{context}"] += Creating OADP ConfigMap objects for the image-based upgrade with {lcao} + +Create your OADP resources that are used to back up and restore your resources during the upgrade. + +.Prerequisites + +* Generate a seed image from a compatible seed cluster. +* Create backup and restore resources. +* Create a separate partition on the target cluster for the container images that is shared between stateroots. For more information about, see _Additional resources_. +* Deploy a version of {lcao} that is compatible with the version used with the seed image. +* Install the OADP Operator, the `DataProtectionApplication` CR and its secret on the target cluster. +* Create an S3-compatible storage solution and a ready-to-use bucket with proper credentials configured. For more information, see _Additional resources_. + +.Procedure + +. Create the OADP `Backup` and `Restore` CRs for platform artifacts in the same namespace where the OADP Operator is installed, which is `openshift-adp`. + +.. If the target cluster is managed by {rh-rhacm}, add the following YAML file for backing up and restoring {rh-rhacm} artifacts: ++ +-- +.PlatformBackupRestore.yaml for {rh-rhacm} +include::snippets/ibu-PlatformBackupRestore.adoc[] +-- + +.. If you created persistent volumes on your cluster through {lvms}, add the following YAML file for {lvms} artifacts: ++ +.PlatformBackupRestoreLvms.yaml for {lvms} +include::snippets/ibu-PlatformBackupRestoreLvms.adoc[] + +. (Optional) If you need to restore applications over the upgrade, create the OADP `Backup` and `Restore` CRs for your application in the `openshift-adp` namespace. + +.. Create the OADP CRs for cluster-scoped application artifacts in the `openshift-adp` namespace. ++ +.Example OADP CRs for cluster-scoped application artifacts for LSO and {LVMS} +include::snippets/ibu-ApplicationClusterScopedBackupRestore.adoc[] + +.. Create the OADP CRs for your namespace-scoped application artifacts. ++ +-- +.Example OADP CRs namespace-scoped application artifacts when LSO is used +include::snippets/ibu-ApplicationBackupRestoreLso.adoc[] + +.Example OADP CRs namespace-scoped application artifacts when {lvms} is used +include::snippets/ibu-ApplicationBackupRestoreLvms.adoc[] + +[IMPORTANT] +==== +The same version of the applications must function on both the current and the target release of {product-title}. +==== +-- + +. Generate a `ConfigMap` object for your OADP CRs. + +.. Create the `ConfigMap` object: ++ +[source,terminal] +---- +$ oc create configmap oadp-cm-example --from-file=example-oadp-resources.yaml= -n openshift-adp +---- + +. Patch the `ImageBasedUpgrade` CR: ++ +[source,terminal] +---- +$ oc patch imagebasedupgrades.lca.openshift.io upgrade \ +-p='{"spec": {"oadpContent": [{"name": "oadp-cm-example", "namespace": "openshift-adp"}]}}' \ +--type=merge -n openshift-lifecycle-agent +---- \ No newline at end of file diff --git a/modules/cnf-image-based-upgrade-prep.adoc b/modules/cnf-image-based-upgrade-prep.adoc new file mode 100644 index 000000000000..a3c570637981 --- /dev/null +++ b/modules/cnf-image-based-upgrade-prep.adoc @@ -0,0 +1,119 @@ +// Module included in the following assemblies: +// * edge_computing/image-based-upgrade/cnf-image-based-upgrade-base.adoc + +:_mod-docs-content-type: PROCEDURE +[id="ztp-image-based-upgrade-prep_{context}"] += Moving to the Prep stage of the image-based upgrade with {lcao} + +Once you have created all the resources that you need during the upgrade, you can move on to the `Prep` stage. +For more information, see the _Creating ConfigMap objects for the image-based upgrade with {lcao}_ section. + +.Prerequisites + +* Created resources to back up and restore your clusters. + +.Procedure + +. Check that you have patched your `ImageBasedUpgrade` CR: ++ +[source,yaml] +---- +apiVersion: lca.openshift.io/v1 +kind: ImageBasedUpgrade +metadata: + name: upgrade +spec: + stage: Idle + seedImageRef: + version: 4.15.2 <1> + image: <2> + pullSecretRef: <3> + autoRollbackOnFailure: {} +# initMonitorTimeoutSeconds: 1800 <4> + extraManifests: <5> + - name: example-extra-manifests-cm + namespace: openshift-lifecycle-agent + - name: example-catalogsources-cm + namespace: openshift-lifecycle-agent + oadpContent: <6> + - name: oadp-cm-example + namespace: openshift-adp +---- +<1> Specify the target platform version. The value must match the version of the seed image. +<2> Specify the URL of the repository where the target cluster can pull the seed image from. +<3> Specify the reference to a secret with credentials to pull container images if the images are in a private registry. +<4> (Optional) Specify the time frame in seconds to roll back if the upgrade does not complete within that time frame after the first reboot. If not defined or set to `0`, the default value of `1800` seconds (30 minutes) is used. +<5> (Optional) Specify the list of `ConfigMap` resources that contain your custom catalog sources to retain after the upgrade, and your extra manifests to apply to the target cluster that are not part of the seed image. +<6> Add the `oadpContent` section with the OADP `ConfigMap` information. + +. When you are ready to start the `Prep` stage, change the value of the `stage` field to `Prep` in the `ImageBasedUpgrade` CR: ++ +-- +[source,terminal] +---- +$ oc patch imagebasedupgrades.lca.openshift.io upgrade -p='{"spec": {"stage": "Prep"}}' --type=merge -n openshift-lifecycle-agent +---- + +If you provide `ConfigMap` objects for OADP resources and extra manifests, {lcao} validates the specified `ConfigMap` objects during the `Prep` stage. +You may encounter the following issues: + +* validation warnings or errors if the {lcao} detects any issues with `extraManifests` +* validation errors if the {lcao} detects any issues with with `oadpContent` + +Validation warnings do not block the `Upgrade` stage but you must decide if it is safe to proceed with the upgrade. +These warnings, for example missing CRDs, namespaces, or dry run failures, update the `status.conditions` for the `Prep` stage and `annotation` fields in the `ImageBasedUpgrade` CR with details about the warning. + +.Example validation warning +[source,yaml] +---- +[...] +metadata: +annotations: + extra-manifest.lca.openshift.io/validation-warning: '...' +[...] +---- + +However, validation errors, such as adding `MachineConfig` or Operator manifests to extra manifests, cause the `Prep` stage to fail and block the `Upgrade` stage. + +When the validations pass, the cluster creates a new `ostree` stateroot, which involves pulling and unpacking the seed image, and running host level commands. +Finally, all the required images are precached on the target cluster. +-- + +.Verification + +. Check the status of the `ImageBasedUpgrade` CR. ++ +-- +[source,terminal] +---- +$ oc get ibu -oyaml +---- + +.Example output +[source,yaml] +---- + conditions: + - lastTransitionTime: "2024-01-01T09:00:00Z" + message: In progress + observedGeneration: 13 + reason: InProgress + status: "False" + type: Idle + - lastTransitionTime: "2024-01-01T09:00:00Z" + message: Prep completed + observedGeneration: 13 + reason: Completed + status: "False" + type: PrepInProgress + - lastTransitionTime: "2024-01-01T09:00:00Z" + message: Prep stage completed successfully + observedGeneration: 13 + reason: Completed + status: "True" + type: PrepCompleted + observedGeneration: 13 + validNextStages: + - Idle + - Upgrade +---- +-- \ No newline at end of file diff --git a/modules/cnf-image-based-upgrade-rollback.adoc b/modules/cnf-image-based-upgrade-rollback.adoc new file mode 100644 index 000000000000..c6f8bf9bf6c3 --- /dev/null +++ b/modules/cnf-image-based-upgrade-rollback.adoc @@ -0,0 +1,55 @@ +// Module included in the following assemblies: +// * edge_computing/image-based-upgrade/cnf-image-based-upgrade-base.adoc + +:_mod-docs-content-type: PROCEDURE +[id="ztp-image-based-upgrade-rollback_{context}"] += (Optional) Moving to the Rollback stage of the image-based upgrade with {lcao} + +You can manually roll back the changes if you encounter unresolvable issues after an upgrade. + +An automatic rollback is initiated if the upgrade does not complete within the time frame specified in the `initMonitorTimeoutSeconds` field after rebooting. + +.Example ImageBasedUpgrade CR +[source,yaml] +---- +apiVersion: lca.openshift.io/v1 +kind: ImageBasedUpgrade +metadata: + name: upgrade +spec: + stage: Idle + seedImageRef: + version: 4.15.2 + image: + autoRollbackOnFailure: {} +# initMonitorTimeoutSeconds: 1800 <1> +[...] +---- +<1> (Optional) Specify the time frame in seconds to roll back if the upgrade does not complete within that time frame after the first reboot. If not defined or set to `0`, the default value of `1800` seconds (30 minutes) is used. + +.Prerequisites + +* Log in to the hub cluster as a user with `cluster-admin` privileges. + +.Procedure + +. Move to the rollback stage by changing the value of the `stage` field to `Rollback` in the `ImageBasedUpgrade` CR. ++ +[source,terminal] +---- +$ oc patch imagebasedupgrades.lca.openshift.io upgrade -p='{"spec": {"stage": "Rollback"}}' --type=merge +---- + +. The {lcao} reboots the cluster with the previously installed version of {product-title} and restores the applications. + +. When you are satisfied with the changes and ready to finalize the rollback, change the value of the `stage` field to `Idle` in the `ImageBasedUpgrade` CR: ++ +[source,terminal] +---- +$ oc patch imagebasedupgrades.lca.openshift.io upgrade -p='{"spec": {"stage": "Idle"}}' --type=merge -n openshift-lifecycle-agent +---- + +[WARNING] +==== +If you move to the `Idle` stage after a rollback, the {lcao} cleans up resources that can be used to troubleshoot a failed upgrade. +==== \ No newline at end of file diff --git a/modules/cnf-image-based-upgrade-seed-image-config.adoc b/modules/cnf-image-based-upgrade-seed-image-config.adoc new file mode 100644 index 000000000000..71c2584cbcbe --- /dev/null +++ b/modules/cnf-image-based-upgrade-seed-image-config.adoc @@ -0,0 +1,129 @@ +// Module included in the following assemblies: +// * edge_computing/image-based-upgrade/cnf-preparing-for-image-based-upgrade.adoc + +:_mod-docs-content-type: PROCEDURE +[id="ztp-image-based-seed-image-config_{context}"] += Seed image configuration + +The seed image targets a set of {sno} clusters with similar configuration. +This means that the seed image must have all the components and configuration that the seed cluster shares with the target clusters. +Therefore, the seed image generated from the seed cluster cannot contain any cluster-specific configuration. + +The following table lists the components, resources, and configurations that you must and must not include in your seed image: + +.Seed image configuration +[cols=2*, width="80%", options="header"] +|==== +|Cluster configuration +|Include in seed image + +|Performance profile +|Yes + +|`MachineConfig` resources for the target cluster +|Yes + +|IP version ^[1]^ +|Yes + +|Set of Day 2 Operators, including the {lcao} and the OADP Operator +|Yes + +|Disconnected registry configuration +|Yes + +|Valid proxy configuration ^[2]^ +|Yes + +|FIPS configuration +|Yes + +|Dedicated partition on the primary disk for container storage that matches the size of the target clusters +|Yes + +a|Local volumes + +* `StorageClass` used in `LocalVolume` for LSO +* `LocalVolume` for LSO +* `LVMCluster` CR for LVMS +|No + +|OADP `DataProtectionApplication` CR +|No +|==== +. Dual-stack networking is not supported in this release. +. The proxy configuration does not have to be the same. + +[id="ztp-image-based-upgrade-seed-image-config-ran_{context}"] +== Seed image configuration using the RAN DU profile + +The following table lists the components, resources, and configurations that you must and must not include in the seed image when using the RAN DU profile: + +.Seed image configuration with RAN DU profile +[cols=2*, width="80%", options="header"] +|==== +|Resource +|Include in seed image + +|All extra manifests that are applied as part of Day 0 installation +|Yes + +|All Day 2 Operator subscriptions +|Yes + +|`ClusterLogging.yaml` +|Yes + +|`DisableOLMPprof.yaml` +|Yes + +|`TunedPerformancePatch.yaml` +|Yes + +|`PerformanceProfile.yaml` +|Yes + +|`SriovOperatorConfig.yaml` +|Yes + +|`DisableSnoNetworkDiag.yaml` +|Yes + +|`StorageClass.yaml` +|No, if it is used in `StorageLV.yaml` + +|`StorageLV.yaml` +|No + +|`StorageLVMCluster.yaml` +|No +|==== + +.Seed image configuration with RAN DU profile for extra manifests +[cols=2*, width="80%", options="header"] +|==== +|Resource +|Apply as extra manifest + +|`ClusterLogForwarder.yaml` +|Yes + +|`ReduceMonitoringFootprint.yaml` +|Yes + +|`SriovFecClusterConfig.yaml` +|Yes + +|`PtpOperatorConfigForEvent.yaml` +|Yes + +|`DefaultCatsrc.yaml` +|Yes + +|`PtpConfig.yaml` +|If the interfaces of the target cluster are common with the seed cluster, you can include them in the seed image. Otherwise, apply it as extra manifests. + +a|`SriovNetwork.yaml` +`SriovNetworkNodePolicy.yaml` +|If the configuration, including namespaces, is exactly the same on both the seed and target cluster, you can include them in the seed image. Otherwise, apply them as extra manifests. +|==== \ No newline at end of file diff --git a/modules/cnf-image-based-upgrade-share-container-directory.adoc b/modules/cnf-image-based-upgrade-share-container-directory.adoc new file mode 100644 index 000000000000..5ef4d0b525b9 --- /dev/null +++ b/modules/cnf-image-based-upgrade-share-container-directory.adoc @@ -0,0 +1,68 @@ +// Module included in the following assemblies: +// * edge_computing/image-based-upgrade/cnf-preparing-for-image-based-upgrade.adoc + +:_mod-docs-content-type: PROCEDURE +[id="cnf-image-based-upgrade-shared-container-directory_{context}"] += Configuring a shared container directory between `ostree` stateroots + +You must apply a `MachineConfig` to both the seed and the target clusters during installation time to create a separate partition and share the `/var/lib/containers` directory between the two `ostree` stateroots that will be used during the upgrade process. + +[IMPORTANT] +==== +You must complete this procedure at installation time. +==== + +.Procedure + +. Apply a `MachineConfig` to create a separate partition. ++ +[source,yaml] +---- +apiVersion: machineconfiguration.openshift.io/v1 +kind: MachineConfig +metadata: + labels: + machineconfiguration.openshift.io/role: master + name: 98-var-lib-containers-partitioned +spec: + config: + ignition: + version: 3.2.0 + storage: + disks: + - device: /dev/disk/by-id/wwn- <1> + partitions: + - label: varlibcontainers + startMiB: <2> + sizeMiB: <3> + filesystems: + - device: /dev/disk/by-partlabel/varlibcontainers + format: xfs + mountOptions: + - defaults + - prjquota + path: /var/lib/containers + wipeFilesystem: true + systemd: + units: + - contents: |- + # Generated by Butane + [Unit] + Before=local-fs.target + Requires=systemd-fsck@dev-disk-by\x2dpartlabel-varlibcontainers.service + After=systemd-fsck@dev-disk-by\x2dpartlabel-varlibcontainers.service + + [Mount] + Where=/var/lib/containers + What=/dev/disk/by-partlabel/varlibcontainers + Type=xfs + Options=defaults,prjquota + + [Install] + RequiredBy=local-fs.target + enabled: true + name: var-lib-containers.mount +---- +<1> Specify the root disk. +<2> Specify the start of the partition in MiB. If the value is too small, the installation will fail. +<3> Specify a minimum size for the partition of 500GB to ensure adequate disk space for precached images. If the value is too small, the deployments after installation will fail. \ No newline at end of file diff --git a/modules/cnf-image-based-upgrade-troubleshooting.adoc b/modules/cnf-image-based-upgrade-troubleshooting.adoc new file mode 100644 index 000000000000..f1f1515575dd --- /dev/null +++ b/modules/cnf-image-based-upgrade-troubleshooting.adoc @@ -0,0 +1,289 @@ +// Module included in the following assemblies: +// * edge_computing/image-based-upgrade/cnf-image-based-upgrade-base.adoc +// * edge_computing/image-based-upgrade/ztp-image-based-upgrade.adoc + +:_mod-docs-content-type: PROCEDURE +[id="ztp-image-based-upgrade-troubleshooting_{context}"] += Troubleshooting image-based upgrades with {lcao} + +You can encounter issues during the image-based upgrade. + +[id="ztp-image-based-upgrade-troubleshooting-must-gather_{context}"] +== Collecting logs + +You can use the `oc adm must-gather` CLI to collect information for debugging and troubleshooting. + +.Procedure + +. Collect data about the Operators by running the following command: ++ +[source,terminal] +---- +$ oc adm must-gather \ +--dest-dir=must-gather/tmp \ +--image=$(oc -n openshift-lifecycle-agent get deployment.apps/lifecycle-agent-controller-manager -o jsonpath='{.spec.template.spec.containers[?(@.name == "manager")].image}') \ +--image=quay.io/konveyor/oadp-must-gather:latest \ <1> +--image=quay.io/openshift/origin-must-gather:latest <1> +---- +<1> (Optional) You can add these options if you need to gather more information from the OADP and the SR-IOV Operators. + +[id="ztp-image-based-upgrade-troubleshooting-manual-cleanup_{context}"] +== AbortFailed or FinalizeFailed error + +Issue:: +During the finalize stage or when you stop the process at the `Prep` stage, normally, {lcao} cleans up the following resources: + +* Stateroot that is no longer required +* Precaching resources +* OADP CRs +* `ImageBasedUpgrade` CR + +If the {lcao} fails to perform the above steps, it transitions to the `AbortFailed` or `FinalizeFailed` states. +The condition message and log show which steps failed. + +.Example error message +[source,yaml] +---- +message: failed to delete all the backup CRs. Perform cleanup manually then add 'lca.openshift.io/manual-cleanup-done' annotation to ibu CR to transition back to Idle + observedGeneration: 5 + reason: AbortFailed + status: "False" + type: Idle +---- + +Resolution:: + +. Inspect the logs to determine why the failure occurred. + +. To prompt {lcao} to retry the cleanup, add the `lca.openshift.io/manual-cleanup-done` annotation to the `ImageBasedUpgrade` CR. + +After observing this annotation, {lcao} retries the cleanup and if it is successful, the `ImageBasedUpgrade` stage transitions to `Idle`. + +If the cleanup fails again, you can manually clean up the resources. + +[id="ztp-image-based-upgrade-troubleshooting-stateroot_{context}"] +=== Cleaning up stateroot manually + +Issue:: + +Normally, the {lcao} cleans up the new stateroot when you stop at the `Prep` stage or the {lcao} cleans up the old stateroot when you finalize after a successful upgrade or a rollback. +If this step fails, it is recommended that you inspect the logs to determine why the failure occurred. + +Resolution:: + +. Check if there are any existing deployments in the stateroot by running the following command directly on the node being upgraded: ++ +[source,terminal] +---- +$ ostree admin status +---- + +. Clean up the existing deployment: ++ +[source,terminal] +---- +$ ostree admin undeploy +---- + +. After cleaning up all the deployments of the stateroot, wipe the stateroot directory: ++ +-- +[WARNING] +==== +Ensure that the booted deployment is not in this stateroot. +==== + +[source,terminal] +---- +$ stateroot="" +$ unshare -m /bin/sh -c "mount -o remount,rw /sysroot && rm -rf /sysroot/ostree/deploy/${stateroot}" +---- +-- + +[id="ztp-image-based-upgrade-troubleshooting-oadp-resources_{context}"] +=== Cleaning up OADP resources manually + +Issue:: + +This step can fail due to connection issues between {lcao} and the S3 backend. By restoring the connection and adding the `lca.openshift.io/manual-cleanup-done` annotation, the {lcao} can successfully cleanup backup resources. + +Resolution:: + +. Check the backend connectivity by running the following command: ++ +-- +[source,terminal] +---- +$ oc get backupstoragelocations.velero.io -n openshift-adp +---- + +.Example output +[source,terminal] +---- +NAME PHASE LAST VALIDATED AGE DEFAULT +dataprotectionapplication-1 Available 33s 8d true +---- +-- + +. Remove all backup resources and then add the `lca.openshift.io/manual-cleanup-done` annotation to the `ImageBasedUpgrade` CR. + +[id="ztp-image-based-upgrade-troubleshooting-lvms_{context}"] +== {lvms} volume contents not restored + +When {lvms} is used to provide dynamic persistent volume storage, {lvms} might not restore the persistent volume contents if it is configured incorrectly. + +[id="ztp-image-based-upgrade-troubleshooting-lvms-backup_{context}"] +=== Missing {lvms}-related fields in Backup CR + +Issue:: +Your `Backup` CRs might be missing fields that are needed to restore your persistent volumes. +You have the following output when you check for events in your application pod: ++ +-- +[source,terminal] +---- +$ oc describe pod +---- + +.Example +[source,terminal] +---- +Events: + Type Reason Age From Message + ---- ------ ---- ---- ------- + Warning FailedScheduling 58s (x2 over 66s) default-scheduler 0/1 nodes are available: pod has unbound immediate PersistentVolumeClaims. preemption: 0/1 nodes are available: 1 Preemption is not helpful for scheduling.. + Normal Scheduled 56s default-scheduler Successfully assigned default/db-1234 to sno1.example.lab + Warning FailedMount 24s (x7 over 55s) kubelet MountVolume.SetUp failed for volume "pvc-1234" : rpc error: code = Unknown desc = VolumeID is not found +---- +-- + +Resolution:: +You must include `logicalvolumes.topolvm.io` in the application `Backup` CR. +Without this resource, the application restores its persistent volume claims and persistent volume manifests correctly, however, the `logicalvolume` associated with this persistent volume is not restored properly after pivot. ++ +.Example Backup CR +[source,yaml] +---- +apiVersion: velero.io/v1 +kind: Backup +metadata: + labels: + velero.io/storage-location: default + name: small-app + namespace: openshift-adp +spec: + includedNamespaces: + - test + includedNamespaceScopedResources: + - secrets + - persistentvolumeclaims + - deployments + - statefulsets + includedClusterScopedResources: + - persistentVolumes <1> + - volumesnapshotcontents <1> + - logicalvolumes.topolvm.io <1> +---- +<1> Required to restore the persistent volumes for your application. + +[id="ztp-image-based-upgrade-troubleshooting-lvms-restore_{context}"] +=== Missing {lvms}-related fields in Restore CR + +Issue:: +The expected resources for the applications are restored but the persistent volume contents are not preserved after upgrading. ++ +-- +.Example output before pivot +[source,terminal] +---- +$ oc get pv,pvc,logicalvolumes.topolvm.io -A +---- + +[source,terminal] +---- +NAME CAPACITY ACCESS MODES RECLAIM POLICY STATUS CLAIM STORAGECLASS REASON AGE +persistentvolume/pvc-1234 1Gi RWO Retain Bound default/pvc-db lvms-vg1 4h45m + +NAMESPACE NAME STATUS VOLUME CAPACITY ACCESS MODES STORAGECLASS AGE +default persistentvolumeclaim/pvc-db Bound pvc-1234 1Gi RWO lvms-vg1 4h45m + +NAMESPACE NAME AGE + logicalvolume.topolvm.io/pvc-1234 4h45m +---- + +.Example output after pivot +[source,terminal] +---- +$ oc get pv,pvc,logicalvolumes.topolvm.io -A +---- + +[source,terminal] +---- +NAME CAPACITY ACCESS MODES RECLAIM POLICY STATUS CLAIM STORAGECLASS REASON AGE +persistentvolume/pvc-1234 1Gi RWO Delete Bound default/pvc-db lvms-vg1 19s + +NAMESPACE NAME STATUS VOLUME CAPACITY ACCESS MODES STORAGECLASS AGE +default persistentvolumeclaim/pvc-db Bound pvc-1234 1Gi RWO lvms-vg1 19s + +NAMESPACE NAME AGE + logicalvolume.topolvm.io/pvc-1234 18s +---- +-- + +Resolution:: +The reason for this issue is that the `logicalvolume` status is not preserved in the `Restore` CR. +This status is important because it is required for Velero to reference the volumes that must be preserved after pivoting. +You must include the following fields in the application `Restore` CR: ++ +.Example Restore CR +[source,yaml] +---- +apiVersion: velero.io/v1 +kind: Restore +metadata: + name: sample-vote-app + namespace: openshift-adp + labels: + velero.io/storage-location: default + annotations: + lca.openshift.io/apply-wave: "3" +spec: + backupName: + sample-vote-app + restorePVs: true <1> + restoreStatus: <1> + includedResources: <1> + - logicalvolumes <1> +---- +<1> Required to preserve the persistent volumes for your application. + +[id="ztp-image-based-upgrade-troubleshooting-debugging-oadp-crs_{context}"] +== Debugging failed Backup and Restore CRs + +Issue:: +The backup or restore of artifacts failed. + +Resolution:: +You can debug `Backup` and `Restore` CRs and retrieve logs with the Velero CLI tool. +The Velero CLI tool provides more detailed information than the OpenShift CLI tool. + +. Describe the `Backup` CR that contains errors: ++ +[source,terminal] +---- +$ oc exec -n openshift-adp velero-7c87d58c7b-sw6fc -c velero -- ./velero describe backup -n openshift-adp backup-acm-klusterlet --details +---- + +. Describe the `Restore` CR that contains errors: ++ +[source,terminal] +---- +$ oc exec -n openshift-adp velero-7c87d58c7b-sw6fc -c velero -- ./velero describe restore -n openshift-adp restore-acm-klusterlet --details +---- + +. Download the backed up resources to a local directory: ++ +[source,terminal] +---- +$ oc exec -n openshift-adp velero-7c87d58c7b-sw6fc -c velero -- ./velero backup download -n openshift-adp backup-acm-klusterlet -o ~/backup-acm-klusterlet.tar.gz +---- \ No newline at end of file diff --git a/modules/cnf-image-based-upgrade-with-backup.adoc b/modules/cnf-image-based-upgrade-with-backup.adoc new file mode 100644 index 000000000000..66e6988cf767 --- /dev/null +++ b/modules/cnf-image-based-upgrade-with-backup.adoc @@ -0,0 +1,175 @@ +// Module included in the following assemblies: +// * edge_computing/image-based-upgrade/cnf-image-based-upgrade-base.adoc + +:_mod-docs-content-type: PROCEDURE +[id="ztp-image-based-upgrading-with-backup_{context}"] += Moving to the Upgrade stage of the image-based upgrade with {lcao} + +Once you generated the seed image and completed the `Prep` stage, you can upgrade the target cluster. +During the upgrade process, the OADP Operator creates a backup of the artifacts specified in the OADP CRs, then the {lcao} upgrades the cluster. + +If the upgrade fails or stops, an automatic rollback is initiated. +If you have an issue after the upgrade, you can initiate a manual rollback. +For more information about manual rollback, see "(Optional) Initiating a rollback with {lcao}". + +.Prerequisites + +* Complete the `Prep` stage. + +.Procedure + +. When you are ready, move to the upgrade stage by changing the value of the `stage` field to `Upgrade` in the `ImageBasedUpgrade` CR. ++ +[source,terminal] +---- +$ oc patch imagebasedupgrades.lca.openshift.io upgrade -p='{"spec": {"stage": "Upgrade"}}' --type=merge +---- + +. Check the status of the `ImageBasedUpgrade` CR: ++ +-- +[source,terminal] +---- +$ oc get ibu -oyaml +---- + +.Example output +[source,yaml] +---- +status: + conditions: + - lastTransitionTime: "2024-01-01T09:00:00Z" + message: In progress + observedGeneration: 5 + reason: InProgress + status: "False" + type: Idle + - lastTransitionTime: "2024-01-01T09:00:00Z" + message: Prep completed + observedGeneration: 5 + reason: Completed + status: "False" + type: PrepInProgress + - lastTransitionTime: "2024-01-01T09:00:00Z" + message: Prep completed successfully + observedGeneration: 5 + reason: Completed + status: "True" + type: PrepCompleted + - lastTransitionTime: "2024-01-01T09:00:00Z" + message: |- + Waiting for system to stabilize: one or more health checks failed + - one or more ClusterOperators not yet ready: authentication + - one or more MachineConfigPools not yet ready: master + - one or more ClusterServiceVersions not yet ready: sriov-fec.v2.8.0 + observedGeneration: 1 + reason: InProgress + status: "True" + type: UpgradeInProgress + observedGeneration: 1 + rollbackAvailabilityExpiration: "2024-05-19T14:01:52Z" + validNextStages: + - Rollback +---- +-- + +. The OADP Operator creates a backup of the data specified in the OADP `Backup` and `Restore` CRs. + +. The target cluster reboots. + +. Monitor the status of the CR: ++ +[source,terminal] +---- +$ oc get ibu -oyaml +---- + +. The cluster reboots. + +. Once you are satisfied with the upgrade, commit to the changes by changing the value of the `stage` field to `Idle` in the `ImageBasedUpgrade` CR: ++ +-- +[source,terminal] +---- +$ oc patch imagebasedupgrades.lca.openshift.io upgrade -p='{"spec": {"stage": "Idle"}}' --type=merge +---- + +[IMPORTANT] +==== +You cannot roll back the changes once you move to the `Idle` stage after an upgrade. +==== + +The {lcao} deletes all resources created during the upgrade process. +-- + +.Verification + +. Check the status of the `ImageBasedUpgrade` CR: ++ +-- +[source,terminal] +---- +$ oc get ibu -oyaml +---- + +.Example output +[source,yaml] +---- +status: + conditions: + - lastTransitionTime: "2024-01-01T09:00:00Z" + message: In progress + observedGeneration: 5 + reason: InProgress + status: "False" + type: Idle + - lastTransitionTime: "2024-01-01T09:00:00Z" + message: Prep completed + observedGeneration: 5 + reason: Completed + status: "False" + type: PrepInProgress + - lastTransitionTime: "2024-01-01T09:00:00Z" + message: Prep completed successfully + observedGeneration: 5 + reason: Completed + status: "True" + type: PrepCompleted + - lastTransitionTime: "2024-01-01T09:00:00Z" + message: Upgrade completed + observedGeneration: 1 + reason: Completed + status: "False" + type: UpgradeInProgress + - lastTransitionTime: "2024-01-01T09:00:00Z" + message: Upgrade completed + observedGeneration: 1 + reason: Completed + status: "True" + type: UpgradeCompleted + observedGeneration: 1 + rollbackAvailabilityExpiration: "2024-01-01T09:00:00Z" + validNextStages: + - Idle + - Rollback +---- +-- + +. Check the status of the cluster restoration: ++ +-- +[source,terminal] +---- +$ oc get restores -n openshift-adp -o custom-columns=NAME:.metadata.name,Status:.status.phase,Reason:.status.failureReason +---- + +.Example output +[source,terminal] +---- +NAME Status Reason +acm-klusterlet Completed <1> +apache-app Completed +localvolume Completed +---- +<1> The `acm-klusterlet` is specific to {rh-rhacm} environments only. +-- \ No newline at end of file diff --git a/modules/cnf-image-based-upgrade.adoc b/modules/cnf-image-based-upgrade.adoc new file mode 100644 index 000000000000..fd5626e369de --- /dev/null +++ b/modules/cnf-image-based-upgrade.adoc @@ -0,0 +1,104 @@ +// Module included in the following assemblies: +// * edge_computing/image-based-upgrade/cnf-understanding-image-based-upgrade.adoc + +:_mod-docs-content-type: CONCEPT +[id="ztp-image-based-upgrade-concept-stages_{context}"] += Stages of the image-based upgrade + +After generating the seed image on the seed cluster, you can move through the stages on the target cluster by setting the `spec.stage` field to the following values in the `ImageBasedUpgrade` CR: + +* `Idle` +* `Prep` +* `Upgrade` +* `Rollback` (Optional) + +image::../images/696_OpenShift_Lifecycle_Agent_0624_0.png[Stages of the image-based upgrade] + +[id="ztp-image-based-upgrade-concept-idle_{context}"] +== Idle stage + +The {lcao} creates an `ImageBasedUpgrade` CR set to `stage: Idle` when the Operator is first deployed. +This is the default stage, there is no ongoing upgrade and the cluster is ready to move to the `Prep` stage. + +image::../images/696_OpenShift_Lifecycle_Agent_0624_1.png[Transition from Idle stage] + +You also move to the `Idle` stage to do one of the following steps: + +* Finalize a successful upgrade +* Finalize a rollback +* Cancel an ongoing upgrade until the pre-pivot phase in the `Upgrade` stage + +Moving to the `Idle` stage ensures that the {lcao} cleans up resources, so the cluster is ready for upgrades again. + +image::../images/696_OpenShift_Lifecycle_Agent_0624_2.png[Transitions to Idle stage] + +[IMPORTANT] +==== +If using {rh-acm} when you cancel an upgrade, you must remove the `import.open-cluster-management.io/disable-auto-import` annotation from the target managed cluster to re-enable the automatic import of the cluster. +==== + +[id="ztp-image-based-upgrade-concept-prep_{context}"] +== Prep stage + +[NOTE] +==== +You can complete this stage before a scheduled maintenance window. +==== + +For the `Prep` stage, you specify the following upgrade details in the `ImageBasedUpgrade` CR: + +* seed image to use +* resources to back up +* extra manifests to apply and custom catalog sources to retain after the upgrade, if any + +Then, based on what you specify, the {lcao} prepares for the upgrade without impacting the current running version. +During this stage, the {lcao} ensures that the target cluster is ready to proceed to the `Upgrade` stage by checking if it meets certain conditions and pulls the seed image to the target cluster with additional container images specified in the seed image. + +You also prepare backup resources with the OADP Operator's `Backup` and `Restore` CRs. +These CRs are used in the `Upgrade` stage to reconfigure the cluster, register the cluster with {rh-rhacm}, and restore application artifacts. + +Additionally to the OADP Operator, the {lcao} uses the `ostree` versioning system to create a backup, which allows complete cluster reconfiguration after both upgrade and rollback. + +Once the `Prep` stage finishes, you can cancel the upgrade process by moving to the `Idle` stage or you can start the upgrade by moving to the `Upgrade` stage in the `ImageBasedUpgrade` CR. +If you cancel the upgrade, the Operator performs cleanup operations. + +image::../images/696_OpenShift_Lifecycle_Agent_0624_3.png[Transition from Prep stage] + +[id="ztp-image-based-upgrade-concept-upgrade_{context}"] +== Upgrade stage + +The `Upgrade` stage consists of two phases: + +pre-pivot:: Just before pivoting to the new stateroot, the {lcao} collects the required cluster specific artifacts and stores them in the new stateroot. The backup of your cluster resources specified in the `Prep` stage are created on a compatible Object storage solution. The {lcao} exports CRs specified in the `extraManifests` field in the `ImageBasedUpgrade` CR or the CRs described in the ZTP policies that are bound to the target cluster. Once pre-pivot phase is completed, the {lcao} sets the new stateroot deployment as the default boot entry and reboots the node. +post-pivot:: After booting from the new stateroot, the {lcao} reconfigures the cluster by applying cluster-specific artifacts that were collected in the pre-pivot phase. The Operator applies all saved CRs, and restores the backups. +The Operator also regenerates the seed image's cluster cryptography. +This ensures that each {sno} cluster upgraded with the same seed image has unique and valid cryptographic objects. + +Once the upgrade is complete and you are satisfied with the changes, you can finalize the upgrade by moving to the `Idle` stage. + +[IMPORTANT] +==== +When you finalize the upgrade, you cannot roll back to the original release. +==== + +image::../images/696_OpenShift_Lifecycle_Agent_0624_4.png[Transitions from Upgrade stage] + +If you want to cancel the upgrade, you can do so until the pre-pivot phase of the `Upgrade` stage. +If you encounter issues after the upgrade, you can move to the `Rollback` stage for a manual rollback. + +[id="ztp-image-based-upgrade-concept-rollback_{context}"] +== (Optional) Rollback stage + +The rollback stage can be initiated manually or automatically upon failure. +During the `Rollback` stage, the {lcao} sets the original `ostree` stateroot deployment as default. +Then, the node reboots with the previous release of {product-title} and application configurations. + +[WARNING] +==== +If you move to the `Idle` stage after a rollback, the {lcao} cleans up resources that can be used to troubleshoot a failed upgrade. +==== + +The {lcao} initiates an automatic rollback if the upgrade does not complete within a specified time limit. +For more information about the automatic rollback, see the relevant _(Optional) Initiating a rollback with Lifecycle Agent_ sections. + +image::../images/696_OpenShift_Lifecycle_Agent_0624_4.png[Transition from Rollback stage] \ No newline at end of file diff --git a/modules/ztp-image-based-upgrade-backup-guide.adoc b/modules/ztp-image-based-upgrade-backup-guide.adoc new file mode 100644 index 000000000000..7d83e30e2674 --- /dev/null +++ b/modules/ztp-image-based-upgrade-backup-guide.adoc @@ -0,0 +1,81 @@ +// Module included in the following assemblies: +// * edge_computing/image-based-upgrade/cnf-understanding-image-based-upgrade.adoc + +[id="ztp-image-based-upgrade-backup-guide_{context}"] += OADP backup and restore guidelines + +With the OADP Operator, you can back up and restore your applications on your target clusters by using `Backup` and `Restore` CRs wrapped in `ConfigMap` objects. +The application must work on the current and the target {product-title} versions so they can be restored after the upgrade. +The backups must include resources that were initially created. + +The following resources must be excluded from the backup: + +* `pods` +* `endpoints` +* `controllerrevision` +* `podmetrics` +* `packagemanifest` +* `replicaset` +* `localvolume`, if using Local Storage Operator (LSO) + +There are two local storage implementations for {sno}: + +Local Storage Operator (LSO):: The {lcao} backs up and restores the required artifacts, including `LocalVolumes` and their associated `StorageClasses`. You must exclude the `persistentVolumes` resource in the application `Backup` CR. + +{lvms}:: You must create the `Backup` and `Restore` CRs for {lvms} artifacts. You must include the `persistentVolumes` resource in the application `Backup` CR. + +For the image-based upgrade, only one operator is supported on a given target cluster. + +[IMPORTANT] +==== +For both Operators, you must not apply the Operator CRs as extra manifests through the `ImageBasedUpgrade` CR. +==== + +The persistent volume contents are preserved and used after the pivot. +When you are configuring the `DataProtectionApplication` CR, you must ensure that the `.spec.configuration.restic.enable` is set to `false` for an image-based upgrade. +This disables Container Storage Interface integration. + +[id="ztp-image-based-upgrade-apply-wave-guide_{context}"] +== lca.openshift.io/apply-wave guidelines + + +The `lca.openshift.io/apply-wave` annotation determines the apply order of `Backup` or `Restore` CRs. +The value of the annotation must be a string number. +If you define the `lca.openshift.io/apply-wave` annotation in the `Backup` or `Restore` CRs, they will be applied in increasing order based on the annotation value. +If you do not define the annotation, they will be applied together. + +You must restore the platform artifacts before restoring your application, so the `lca.openshift.io/apply-wave` annotation must be numerically lower in your platform `Restore` CRs, for example {rh-rhacm} and {lvms} artifacts, than that of the application so that the platform artifacts are restored before your applications. + +If your application includes cluster-scoped resources, you must create separate `Backup` and `Restore` CRs to scope the backup to the specific cluster-scoped resources created by the application. +The `Restore` CR for the cluster-scoped resources must be restored before the remaining application `Restore` CR(s). + +[id="ztp-image-based-upgrade-apply-label-guide_{context}"] +== lca.openshift.io/apply-label guidelines + +You can back up specific resources exclusively with the `lca.openshift.io/apply-label` annotation. +Based on which resources you define in the annotation, the {lcao} applies the `lca.openshift.io/backup: ` label and adds the `labelSelector.matchLabels.lca.openshift.io/backup: ` label selector to the specified resources when creating the `Backup` CRs. + +To use the `lca.openshift.io/apply-label` annotation for backing up specific resources, the resources listed in the annotation must also be included in the `spec` section. +If the `lca.openshift.io/apply-label` annotation is used in the `Backup` CR, only the resources listed in the annotation will be backed up, even if other resource types are specified in the `spec` section or not. + +.Example CR +[source,yaml] +---- +apiVersion: velero.io/v1 +kind: Backup +metadata: + name: acm-klusterlet + namespace: openshift-adp + annotations: + lca.openshift.io/apply-label: rbac.authorization.k8s.io/v1/clusterroles/klusterlet,apps/v1/deployments/open-cluster-management-agent/klusterlet <1> + labels: + velero.io/storage-location: default +spec: + includedNamespaces: + - open-cluster-management-agent + includedClusterScopedResources: + - clusterroles + includedNamespaceScopedResources: + - deployments +---- +<1> The value must be a list of comma-separated objects in `group/version/resource/name` format for cluster-scoped resources or `group/version/resource/namespace/name` format for namespace-scoped resources, and it must be attached to the related `Backup` CR. \ No newline at end of file diff --git a/modules/ztp-image-based-upgrade-cluster-validated-software.adoc b/modules/ztp-image-based-upgrade-cluster-validated-software.adoc new file mode 100644 index 000000000000..da6782fb915d --- /dev/null +++ b/modules/ztp-image-based-upgrade-cluster-validated-software.adoc @@ -0,0 +1,56 @@ +// Module included in the following assemblies: +// * edge_computing/image-based-upgrade/cnf-understanding-image-based-upgrade.adoc + +[id="ztp-image-based-upgrade-cluster-validated-software_{context}"] += Minimum software version of components + +Depending on your deployment method, the image-based upgrade requires the following minimum software versions. + +.Minimum software version of components +[cols=3*, width="80%", options="header"] +|==== +|Component +|Software version +|Required + +|{lcao} +|4.16 +|Yes + +|OADP Operator +|1.3.1 +|Yes + +|Managed cluster version +|4.14.13 +|Yes + +|Hub cluster version +|4.16 +|Yes, if using {rh-rhacm} + +|{rh-rhacm} +|2.10.2 +|Yes, if using {rh-rhacm} + +|{ztp} plugin +|4.16 +|Only for {ztp} deployment method + +|{gitops-title} +|1.12 +|Only for {ztp} deployment method + +|{cgu-operator-first} +|4.16 +|Only for {ztp} deployment method + +|Local Storage Operator ^[1]^ +|4.14 +|Yes + +|{lvms-first} ^[1]^ +|4.14.2 +|Yes +|==== +. The persistent storage must be provided by either the {lvms} or the Local Storage Operator, not both. diff --git a/modules/ztp-image-based-upgrade-extra-manifests-guide.adoc b/modules/ztp-image-based-upgrade-extra-manifests-guide.adoc new file mode 100644 index 000000000000..4b65963981b5 --- /dev/null +++ b/modules/ztp-image-based-upgrade-extra-manifests-guide.adoc @@ -0,0 +1,36 @@ +// Module included in the following assemblies: +// * edge_computing/image-based-upgrade/cnf-understanding-image-based-upgrade.adoc + +[id="ztp-image-based-upgrade-extra-manifests-guide_{context}"] += Extra manifest guidelines + +The {lcao} uses extra manifests to restore your target clusters after rebooting with the new default stateroot deployment and before restoring application artifacts. + +Different deployment methods require a different way to apply the extra manifests: + +{ztp}:: You use the `lca.openshift.io/target-ocp-version: ` label to mark the extra manifests that the {lcao} must extract and apply after the pivot. +You can specify the the number of manifests labeled with `lca.openshift.io/target-ocp-version` by using the `lca.openshift.io/target-ocp-version-manifest-count` annotation in the `ImageBasedUpgrade` CR. +If specified, the {lcao} verifies that the number of manifests extracted from policies matches the number provided in the annotation during the prep and upgrade stages. ++ +.Example for the lca.openshift.io/target-ocp-version-manifest-count annotation +[source,yaml] +---- +apiVersion: lca.openshift.io/v1 +kind: ImageBasedUpgrade +metadata: + annotations: + lca.openshift.io/target-ocp-version-manifest-count: "5" + name: upgrade +---- + +Non-Gitops:: You mark your extra manifests with the `lca.openshift.io/apply-wave` annotation to determine the apply order. The labeled extra manifests are wrapped in `ConfigMap` objects and referenced in the `ImageBasedUpgrade` CR that the {lcao} uses after the pivot. + +If the target cluster uses custom catalog sources, you must include them as extra manifests that point to the correct release version. + +[IMPORTANT] +==== +You cannot apply the following items as extra manifests: + +* `MachineConfig` objects +* OLM Operator subscriptions +==== \ No newline at end of file diff --git a/modules/ztp-image-based-upgrade-hub-cluster-guide.adoc b/modules/ztp-image-based-upgrade-hub-cluster-guide.adoc new file mode 100644 index 000000000000..5a2b9447fc1f --- /dev/null +++ b/modules/ztp-image-based-upgrade-hub-cluster-guide.adoc @@ -0,0 +1,10 @@ +// Module included in the following assemblies: +// * edge_computing/image-based-upgrade/cnf-understanding-image-based-upgrade.adoc + +[id="ztp-image-based-upgrade-hub-cluster-guide_{context}"] += Hub cluster guidelines + +If you are using {rh-rhacm-first}, your hub cluster needs to meet the following conditions: + +* To avoid including any {rh-rhacm} resources in your seed image, you need to disable all optional {rh-rhacm} add-ons before generating the seed image. +* Your hub cluster must be upgraded at least to the target version before performing an image-based upgrade a target {sno} cluster. \ No newline at end of file diff --git a/modules/ztp-image-based-upgrade-installing-lifecycle-agent-using-gitops.adoc b/modules/ztp-image-based-upgrade-installing-lifecycle-agent-using-gitops.adoc new file mode 100644 index 000000000000..fe00d085cc60 --- /dev/null +++ b/modules/ztp-image-based-upgrade-installing-lifecycle-agent-using-gitops.adoc @@ -0,0 +1,377 @@ +// Module included in the following assemblies: +// * edge_computing/image-based-upgrade/cnf-preparing-for-image-based-upgrade.adoc + +:_mod-docs-content-type: PROCEDURE +[id="zp-image-based-upgrade-installing-operators-with-gitops_{context}"] += Installing {lcao} and the OADP Operator with ZTP GitOps + +You need both the {lcao} and the OADP Operator to do an image-based upgrade with ZTP GitOps. + +[id="zp-image-based-upgrade-installing-lcao-with-gitops_{context}"] +== Installing the {lcao} with GitOps ZTP + +Install the {lcao} with GitOps ZTP to do an image-based upgrade. + +.Prerequisites + +* Create a directory called `custom-crs` in the `source-crs` directory. The `source-crs` directory must be located in the same location as `kustomization.yaml` file. + +.Procedure + +. Create the following CRs in the `openshift-lifecycle-agent` namespace and push them to the `source-crs/custom-crs` directory. ++ +-- +.LcaSubscriptionNS.yaml +[source,yaml] +---- +apiVersion: v1 +kind: Namespace +metadata: + name: openshift-lifecycle-agent + annotations: + workload.openshift.io/allowed: management + ran.openshift.io/ztp-deploy-wave: "2" + labels: + kubernetes.io/metadata.name: openshift-lifecycle-agent +---- + +.LcaSubscriptionOperGroup.yaml +[source,yaml] +---- +apiVersion: operators.coreos.com/v1 +kind: OperatorGroup +metadata: + name: lifecycle-agent-operatorgroup + namespace: openshift-lifecycle-agent + annotations: + ran.openshift.io/ztp-deploy-wave: "2" +spec: + targetNamespaces: + - openshift-lifecycle-agent +---- + +.LcaSubscription.yaml +[source,yaml] +---- +apiVersion: operators.coreos.com/v1 +kind: Subscription +metadata: + name: lifecycle-agent + namespace: openshift-lifecycle-agent + annotations: + ran.openshift.io/ztp-deploy-wave: "2" +spec: + channel: "stable" + name: lifecycle-agent + source: redhat-operators + sourceNamespace: openshift-marketplace + installPlanApproval: Manual + status: + state: AtLatestKnown +---- + +.Example directory structure +[source,terminal] +---- +├── kustomization.yaml +├── sno +│ ├── example-cnf.yaml +│ ├── common-ranGen.yaml +│ ├── group-du-sno-ranGen.yaml +│ ├── group-du-sno-validator-ranGen.yaml +│ └── ns.yaml +├── source-crs +│ ├── custom-crs +│ │ ├── LcaSubscriptionNS.yaml +│ │ ├── LcaSubscriptionOperGroup.yaml +│ │ ├── LcaSubscription.yaml +---- +-- + +. Add the CRs to your common `PolicyGenTemplate`. ++ +.Example directory structure +[source,yaml] +---- +apiVersion: ran.openshift.io/v1 +kind: PolicyGenTemplate +metadata: + name: "example-common-latest" + namespace: "ztp-common" +spec: + bindingRules: + common: "true" + du-profile: "latest" + sourceFiles: + - fileName: custom-crs/LcaSubscriptionNS.yaml + policyName: "subscriptions-policy" + - fileName: custom-crs/LcaSubscriptionOperGroup.yaml + policyName: "subscriptions-policy" + - fileName: custom-crs/LcaSubscription.yaml + policyName: "subscriptions-policy" +[...] +---- + +[id="zp-image-based-upgrade-installing-oadp-with-gitops_{context}"] +== Installing and configuring the OADP Operator with GitOps ZTP + +You can install and configure the OADP Operator with GitOps ZTP well before you want to start the upgrade. + +.Prerequisites + +* Create a directory called `custom-crs` in the `source-crs` directory. The `source-crs` directory must be located in the same location as `kustomization.yaml` file. + +.Procedure + +. Create the following CRs in the `openshift-adp` namespace and push them to the `source-crs/custom-crs` directory. ++ +-- +.OadpSubscriptionNS.yaml +[source,yaml] +---- +apiVersion: v1 +kind: Namespace +metadata: + name: openshift-adp + annotations: + workload.openshift.io/allowed: management + ran.openshift.io/ztp-deploy-wave: "2" + labels: + kubernetes.io/metadata.name: openshift-adp +---- + +.OadpSubscriptionOperGroup.yaml +[source,yaml] +---- +apiVersion: operators.coreos.com/v1 +kind: OperatorGroup +metadata: + name: redhat-oadp-operator + namespace: openshift-adp + annotations: + ran.openshift.io/ztp-deploy-wave: "2" +spec: + targetNamespaces: + - openshift-adp +---- + +.OadpSubscription.yaml +[source,yaml] +---- +apiVersion: operators.coreos.com/v1 +kind: Subscription +metadata: + name: redhat-oadp-operator + namespace: openshift-adp + annotations: + ran.openshift.io/ztp-deploy-wave: "2" +spec: + channel: stable-1.3 + name: redhat-oadp-operator + source: redhat-operators + sourceNamespace: openshift-marketplace + installPlanApproval: Manual +status: + state: AtLatestKnown +---- + +.OadpOperatorStatus.yaml +[source,yaml] +---- +apiVersion: operators.coreos.com/v1 +kind: Operator +metadata: + name: redhat-oadp-operator.openshift-adp + annotations: + ran.openshift.io/ztp-deploy-wave: "2" +status: + components: + refs: + - kind: Subscription + namespace: openshift-adp + conditions: + - type: CatalogSourcesUnhealthy + status: "False" + - kind: InstallPlan + namespace: openshift-adp + conditions: + - type: Installed + status: "True" + - kind: ClusterServiceVersion + namespace: openshift-adp + conditions: + - type: Succeeded + status: "True" + reason: InstallSucceeded +---- + +.Example directory structure +[source,terminal] +---- +├── kustomization.yaml +├── sno +│ ├── example-cnf.yaml +│ ├── common-ranGen.yaml +│ ├── group-du-sno-ranGen.yaml +│ ├── group-du-sno-validator-ranGen.yaml +│ └── ns.yaml +├── source-crs +│ ├── custom-crs +│ │ ├── OadpSubscriptionNS.yaml +│ │ ├── OadpSubscriptionOperGroup.yaml +│ │ ├── OadpSubscription.yaml +│ │ ├── OadpOperatorStatus.yaml +---- +-- + +. Add the CRs to your common `PolicyGenTemplate`. ++ +.Example directory structure ++ +[source,yaml] +---- +apiVersion: ran.openshift.io/v1 +kind: PolicyGenTemplate +metadata: + name: "example-common-latest" + namespace: "ztp-common" +spec: + bindingRules: + common: "true" + du-profile: "latest" + sourceFiles: + - fileName: custom-crs/OadpSubscriptionNS.yaml + policyName: "subscriptions-policy" + - fileName: custom-crs/OadpSubscriptionOperGroup.yaml + policyName: "subscriptions-policy" + - fileName: custom-crs/OadpSubscription.yaml + policyName: "subscriptions-policy" + - fileName: custom-crs/OadpOperatorStatus.yaml + policyName: "subscriptions-policy" +[...] +---- + +. Create the `DataProtectionApplication` CR and the S3 secret. + +.. Create the following CRs in your `source-crs/custom-crs` directory: ++ +-- +.DataProtectionApplication.yaml +[source,yaml] +---- +apiVersion: oadp.openshift.io/v1 +kind: DataProtectionApplication +metadata: + name: dataprotectionapplication + namespace: openshift-adp + annotations: + ran.openshift.io/ztp-deploy-wave: "100" +spec: + configuration: + restic: + enable: false <1> + velero: + defaultPlugins: + - aws + - openshift + resourceTimeout: 10m + backupLocations: + - velero: + config: + profile: "default" + region: minio + s3Url: $url + insecureSkipTLSVerify: "true" + s3ForcePathStyle: "true" + provider: aws + default: true + credential: + key: cloud + name: cloud-credentials + objectStorage: + bucket: $bucketName <2> + prefix: $prefixName <2> +status: + conditions: + - reason: Complete + status: "True" + type: Reconciled +---- +<1> The `spec.configuration.restic.enable` field must be set to `false` for an image-based upgrade because persistent volume contents are retained and reused after the upgrade. +<2> The bucket defines the bucket name that is created in S3 backend. The prefix defines the name of the subdirectory that will be automatically created in the bucket. The combination of bucket and prefix must be unique for each target cluster to avoid interference between them. To ensure a unique storage directory for each target cluster, you can use the {rh-rhacm} hub template function, for example, prefix: {{hub .ManagedClusterName hub}}. + +.OadpSecret.yaml +[source,yaml] +---- +apiVersion: v1 +kind: Secret +metadata: + name: cloud-credentials + namespace: openshift-adp + annotations: + ran.openshift.io/ztp-deploy-wave: "100" +type: Opaque +---- + +.OadpBackupStorageLocationStatus.yaml +[source,yaml] +---- +apiVersion: velero.io/v1 +kind: BackupStorageLocation +metadata: + namespace: openshift-adp + annotations: + ran.openshift.io/ztp-deploy-wave: "100" +status: + phase: Available +---- + +The `OadpBackupStorageLocationStatus.yaml` CR verifies the availability of backup storage locations created by OADP. +-- + +.. Add the CRs to your site `PolicyGenTemplate` with overrides. ++ +[source,yaml] +---- +apiVersion: ran.openshift.io/v1 +kind: PolicyGenTemplate +metadata: + name: "example-cnf" + namespace: "ztp-site" +spec: + bindingRules: + sites: "example-cnf" + du-profile: "latest" + mcp: "master" + sourceFiles: + ... + - fileName: custom-crs/OadpSecret.yaml + policyName: "config-policy" + data: + cloud: <1> + - fileName: custom-crs/DataProtectionApplication.yaml + policyName: "config-policy" + spec: + backupLocations: + - velero: + config: + config: + region: minio + s3Url: <2> + profile: "default" + insecureSkipTLSVerify: "true" + s3ForcePathStyle: "true" + provider: aws + default: true + credential: + key: cloud + name: cloud-credentials + objectStorage: + bucket: <3> + prefix: <3> + - fileName: custom-crs/OadpBackupStorageLocationStatus.yaml + policyName: "config-policy" +---- +<1> Specify your credentials for your S3 storage backend. +<2> Specify the URL for your S3-compatible bucket. +<3> The `bucket` defines the bucket name that is created in S3 backend. The `prefix` defines the name of the subdirectory that will be automatically created in the `bucket`. The combination of `bucket` and `prefix` must be unique for each target cluster to avoid interference between them. To ensure a unique storage directory for each target cluster, you can use the {rh-rhacm} hub template function, for example, `prefix: {{hub .ManagedClusterName hub}}`. \ No newline at end of file diff --git a/modules/ztp-image-based-upgrade-prep-label-extramanifests.adoc b/modules/ztp-image-based-upgrade-prep-label-extramanifests.adoc new file mode 100644 index 000000000000..367dc651e4d7 --- /dev/null +++ b/modules/ztp-image-based-upgrade-prep-label-extramanifests.adoc @@ -0,0 +1,96 @@ +// Module included in the following assemblies: +// * edge_computing/image-based-upgrade/cnf-preparing-for-image-based-upgrade.adoc + +:_mod-docs-content-type: PROCEDURE +[id="ztp-image-based-upgrade-labeling-extramanifests-with-ztp_{context}"] += Labeling extra manifests for the image-based upgrade with GitOps ZTP + +The {lcao} only extracts extra manifests that are labeled with the `lca.openshift.io/target-ocp-version: ` label. + +.Prerequisites + +* Provision one or more managed clusters with GitOps ZTP. +* Log in as a user with `cluster-admin` privileges. +* Generate a seed image from a compatible seed cluster. +* Create a separate partition on the target cluster for the container images that is shared between stateroots. For more information about, see _Additional resources_. +* Deploy a version of {lcao} that is compatible with the version used with the seed image. + +.Procedure + +. If you have manifests that you want to be extracted and applied during the upgrade, label them with the `lca.openshift.io/target-ocp-version: ` label in your existing `PolicyGenTemplate` CR: ++ +[source,yaml] +---- +apiVersion: ran.openshift.io/v1 +kind: PolicyGenTemplate +metadata: + name: example-sno +spec: + bindingRules: + sites: "example-sno" + du-profile: "4.15" + mcp: "master" + sourceFiles: + - fileName: SriovNetwork.yaml + policyName: "config-policy" + metadata: + name: "sriov-nw-du-fh" + labels: + lca.openshift.io/target-ocp-version: "4.15" <1> + spec: + resourceName: du_fh + vlan: 140 + - fileName: SriovNetworkNodePolicy.yaml + policyName: "config-policy" + metadata: + name: "sriov-nnp-du-fh" + labels: + lca.openshift.io/target-ocp-version: "4.15" <1> + spec: + deviceType: netdevice + isRdma: false + nicSelector: + pfNames: ["ens5f0"] + numVfs: 8 + priority: 10 + resourceName: du_fh + - fileName: SriovNetwork.yaml + policyName: "config-policy" + metadata: + name: "sriov-nw-du-mh" + labels: + lca.openshift.io/target-ocp-version: "4.15" <1> + spec: + resourceName: du_mh + vlan: 150 + - fileName: SriovNetworkNodePolicy.yaml + policyName: "config-policy" + metadata: + name: "sriov-nnp-du-mh" + labels: + lca.openshift.io/target-ocp-version: "4.15" <1> + spec: + deviceType: vfio-pci + isRdma: false + nicSelector: + pfNames: ["ens7f0"] + numVfs: 8 + priority: 10 + resourceName: du_mh + - fileName: DefaultCatsrc.yaml <2> + policyName: "config-policy" + metadata: + name: default-cat-source + namespace: openshift-marketplace + labels: + lca.openshift.io/target-ocp-version: "4.15" <1> + spec: + displayName: default-cat-source + image: quay.io/example-org/example-catalog:v1 +---- +<1> Ensure that the `lca.openshift.io/target-ocp-version` label matches either the y-stream or the z-stream of the target {product-title} version that is specified in the `spec.seedImageRef.version` field of the `ImageBasedUpgrade` CR. The {lcao} only applies the CRs that match the specified version. +<2> (Optional) If you do not want to use custom catalog sources, remove this entry. + +. Push the changes to your Git repository. + +To start the upgrade process, see the _Performing an image-based upgrade with {lcao} and GitOps ZTP_ section. \ No newline at end of file diff --git a/modules/ztp-image-based-upgrade-prep-oadp.adoc b/modules/ztp-image-based-upgrade-prep-oadp.adoc new file mode 100644 index 000000000000..34ecaae1ba1c --- /dev/null +++ b/modules/ztp-image-based-upgrade-prep-oadp.adoc @@ -0,0 +1,136 @@ +// Module included in the following assemblies: +// * edge_computing/image-based-upgrade/cnf-preparing-for-image-based-upgrade.adoc + +:_mod-docs-content-type: PROCEDURE +[id="ztp-image-based-upgrade-creating-backup-resources-with-ztp_{context}"] += Creating OADP resources for the image-based upgrade with GitOps ZTP + +You need to prepare your OADP resources to restore your application after an upgrade. + +.Prerequisites + +* Provision one or more managed clusters with GitOps ZTP. +* Log in as a user with `cluster-admin` privileges. +* Generate a seed image from a compatible seed cluster. +* Create a separate partition on the target cluster for the container images that is shared between stateroots. For more information about, see _Additional resources_. +* Deploy a version of {lcao} that is compatible with the version used with the seed image. +* Install the OADP Operator, the `DataProtectionApplication` CR and its secret on the target cluster. +* Create an S3-compatible storage solution and a ready-to-use bucket with proper credentials configured. For more information, see Additional resources. + +.Procedure + +. Ensure that your Git repository that you use with the ArgoCD policies application contains the following directory structure: ++ +-- +[source,terminal] +---- +├── source-crs/ +│ ├── ibu/ +│ │ ├── ImageBasedUpgrade.yaml +│ │ ├── PlatformBackupRestore.yaml +│ │ ├── PlatformBackupRestoreLvms.yaml +├── ... +├── ibu-upgrade-ranGen.yaml +├── kustomization.yaml +---- + +[IMPORTANT] +==== +The `kustomization.yaml` file must be located in the same directory structure as shown above to reference the `PlatformBackupRestore.yaml` manifest. +==== + +The `source-crs/ibu/PlatformBackupRestore.yaml` file is provided in the ZTP container image. + +.PlatformBackupRestore.yaml +include::snippets/ibu-PlatformBackupRestore.adoc[] + +If you use {lvms} to create persistent volumes, you can use the `source-crs/ibu/PlatformBackupRestoreLvms.yaml` provided in the ZTP container image to back up your {lvms} resources. + +.PlatformBackupRestoreLvms.yaml +include::snippets/ibu-PlatformBackupRestoreLvms.adoc[] +-- + +. (Optional) If you need to restore applications over the upgrade, create the OADP `Backup` and `Restore` CRs for your application in the `openshift-adp` namespace. + +.. Create the OADP CRs for cluster-scoped application artifacts in the `openshift-adp` namespace. ++ +.Example OADP CRs for cluster-scoped application artifacts for LSO and {LVMS} +include::snippets/ibu-ApplicationClusterScopedBackupRestore.adoc[] + +.. Create the OADP CRs for your namespace-scoped application artifacts in the `source-crs/custom-crs` directory. ++ +-- +.Example OADP CRs namespace-scoped application artifacts when LSO is used +include::snippets/ibu-ApplicationBackupRestoreLso.adoc[] + +.Example OADP CRs namespace-scoped application artifacts when {lvms} is used +include::snippets/ibu-ApplicationBackupRestoreLvms.adoc[] + +[IMPORTANT] +==== +The same version of the applications must function on both the current and the target release of {product-title}. +==== +-- + +. Create the `oadp-cm` `ConfigMap` object through the `oadp-cm-policy` in a new `PolicyGenTemplate` called `ibu-upgrade-ranGen.yaml`. ++ +[source,yaml] +---- +apiVersion: ran.openshift.io/v1 +kind: PolicyGenTemplate +metadata: + name: example-group-ibu + namespace: "ztp-group" +spec: + bindingRules: + group-du-sno: "" + mcp: "master" + evaluationInterval: + compliant: 10s + noncompliant: 10s + sourceFiles: + - fileName: ConfigMapGeneric.yaml + complianceType: mustonlyhave + policyName: "oadp-cm-policy" + metadata: + name: oadp-cm + namespace: openshift-adp +---- + +. Create a `kustomization.yaml` with the following content: ++ +[source,yaml] +---- +apiVersion: kustomize.config.k8s.io/v1beta1 +kind: Kustomization + +generators: <1> +- ibu-upgrade-ranGen.yaml + +configMapGenerator: <2> +- files: + - source-crs/ibu/PlatformBackupRestore.yaml + #- source-crs/custom-crs/ApplicationClusterScopedBackupRestore.yaml + #- source-crs/custom-crs/ApplicationApplicationBackupRestoreLso.yaml + name: oadp-cm + namespace: ztp-group +generatorOptions: + disableNameSuffixHash: true + + +patches: <3> +- target: + group: policy.open-cluster-management.io + version: v1 + kind: Policy + name: group-ibu-oadp-cm-policy + patch: |- + - op: replace + path: /spec/policy-templates/0/objectDefinition/spec/object-templates/0/objectDefinition/data + value: '{{hub copyConfigMapData "ztp-group" "oadp-cm" hub}}' +---- +<1> Generates the `oadp-cm-policy`. +<2> Creates the `oadp-cm` `ConfigMap` object on the hub cluster with `Backup` and `Restore` CRs. +<3> Overrides the data field of `ConfigMap` added in `oadp-cm-policy`. A hub template is used to propagate the `oadp-cm` `ConfigMap` to all target clusters. + +. Push the changes to your Git repository. \ No newline at end of file diff --git a/modules/ztp-image-based-upgrade-seed-image-guide.adoc b/modules/ztp-image-based-upgrade-seed-image-guide.adoc new file mode 100644 index 000000000000..73a5a1eaeac6 --- /dev/null +++ b/modules/ztp-image-based-upgrade-seed-image-guide.adoc @@ -0,0 +1,30 @@ +// Module included in the following assemblies: +// * edge_computing/image-based-upgrade/cnf-understanding-image-based-upgrade.adoc + +[id="ztp-image-based-upgrade-seed-image-guide_{context}"] += Seed image guidelines + +The seed image targets a set of {sno} clusters with similar configuration. +This means that the seed cluster needs to have the same configuration as the target clusters for the following items: + +* Performance profile +* `MachineConfig` resources for the target cluster +* IP version ++ +[NOTE] +==== +Dual-stack networking is not supported in this release. +==== + +* Set of Day 2 Operators, including the {lcao} and the OADP Operator +* Disconnected registry +* FIPS configuration +* If the target cluster has multiple IPs and one of them belongs to the subnet that was used for creating the seed image, the upgrade fails if the target cluster's node IP does not belong to that subnet. + +The following configurations only have to partially match on the participating clusters: + +* If the target cluster has a proxy configuration, the seed cluster must have a proxy configuration too but the configuration does not have to be the same. +* A dedicated partition on the primary disk for container storage is required on all participating clusters. However, the size and start of the partition does not have to be the same. Only the `spec.config.storage.disks.partitions.label: varlibcontainers` label in the `MachineConfig` CR must match on both the seed and target clusters. +For more information about how to create the disk partition, see _Configuring a shared container directory between ostree stateroots_ or _Configuring a shared container directory between ostree stateroots when using GitOps ZTP_. + +For more information about what to include in the seed image, see _Seed image configuration_ and _Seed image configuration using the RAN DU profile_. \ No newline at end of file diff --git a/modules/ztp-image-based-upgrade-share-container-directory.adoc b/modules/ztp-image-based-upgrade-share-container-directory.adoc index 4c11c2b9b4cd..3ae080f2bd96 100644 --- a/modules/ztp-image-based-upgrade-share-container-directory.adoc +++ b/modules/ztp-image-based-upgrade-share-container-directory.adoc @@ -1,85 +1,11 @@ // Module included in the following assemblies: -// * scalability_and_performance/ztp-image-based-upgrade.adoc +// * edge_computing/image-based-upgrade/cnf-preparing-for-image-based-upgrade.adoc :_mod-docs-content-type: PROCEDURE [id="ztp-image-based-upgrade-shared-container-directory_{context}"] -= Sharing the container directory between `ostree` stateroots += Configuring a shared container directory between `ostree` stateroots when using GitOps ZTP -You must apply a `MachineConfig` to both the seed and the target clusters during installation time to create a separate partition and share the `/var/lib/containers` directory between the two `ostree` stateroots that will be used during the upgrade process. - -[id="ztp-image-based-upgrade-shared-container-directory-acm_{context}"] -== Sharing the container directory between `ostree` stateroots when using {rh-rhacm} - -When you are using {rh-rhacm}, you must apply a `MachineConfig` to both the seed and target clusters. - -[IMPORTANT] -==== -You must complete this procedure at installation time. -==== - -.Prerequisites - -* Log in as a user with `cluster-admin` privileges. - -.Procedure - -. Apply a `MachineConfig` to create a separate partition. -+ -[source,yaml] ----- -apiVersion: machineconfiguration.openshift.io/v1 -kind: MachineConfig -metadata: - labels: - machineconfiguration.openshift.io/role: master - name: 98-var-lib-containers-partitioned -spec: - config: - ignition: - version: 3.2.0 - storage: - disks: - - device: /dev/disk/by-id/wwn- <1> - partitions: - - label: varlibcontainers - startMiB: <2> - sizeMiB: <3> - filesystems: - - device: /dev/disk/by-partlabel/varlibcontainers - format: xfs - mountOptions: - - defaults - - prjquota - path: /var/lib/containers - wipeFilesystem: true - systemd: - units: - - contents: |- - # Generated by Butane - [Unit] - Before=local-fs.target - Requires=systemd-fsck@dev-disk-by\x2dpartlabel-varlibcontainers.service - After=systemd-fsck@dev-disk-by\x2dpartlabel-varlibcontainers.service - - [Mount] - Where=/var/lib/containers - What=/dev/disk/by-partlabel/varlibcontainers - Type=xfs - Options=defaults,prjquota - - [Install] - RequiredBy=local-fs.target - enabled: true - name: var-lib-containers.mount ----- -<1> Specify the root disk. -<2> Specify the start of the partition in MiB. If the value is too small, the installation will fail. -<3> Specify the size of the partition. If the value is too small, the deployments after installation will fail. - -[id="ztp-image-based-upgrade-shared-container-directory-ztp_{context}"] -== Sharing the container directory between `ostree` stateroots when using GitOps ZTP - -When you are using the GitOps ZTP workflow, you can do the following procedure to create a separate disk partition on both the seed and target cluster and to share the `/var/lib/containers` directory. +When you are using the GitOps ZTP workflow, you do the following procedure to create a separate disk partition on both the seed and target cluster and to share the `/var/lib/containers` directory. [IMPORTANT] ==== @@ -88,7 +14,6 @@ You must complete this procedure at installation time. .Prerequisites -* Log in as a user with `cluster-admin` privileges. * Install Butane. .Procedure @@ -105,8 +30,8 @@ storage: wipe_table: false partitions: - label: var-lib-containers - start_mib: <2> - size_mib: <3> + start_mib: <2> + size_mib: <3> filesystems: - path: /var/lib/containers device: /dev/disk/by-partlabel/var-lib-containers @@ -119,7 +44,7 @@ storage: ---- <1> Specify the root disk. <2> Specify the start of the partition in MiB. If the value is too small, the installation will fail. -<3> Specify the size of the partition. If the value is too small, the deployments after installation will fail. +<3> Specify a minimum size for the partition of 500GB to ensure adequate disk space for precached images. If the value is too small, the deployments after installation will fail. . Convert the `storage.bu` to an Ignition file. + diff --git a/modules/ztp-image-based-upgrade-talm-prep.adoc b/modules/ztp-image-based-upgrade-talm-prep.adoc new file mode 100644 index 000000000000..4c6c3405b9b5 --- /dev/null +++ b/modules/ztp-image-based-upgrade-talm-prep.adoc @@ -0,0 +1,194 @@ +// Module included in the following assemblies: +// * edge_computing/image-based-upgrade/ztp-image-based-upgrade.adoc + +:_mod-docs-content-type: PROCEDURE +[id="ztp-image-based-upgrade-prep-gitops_{context}"] += Moving to the Prep stage of the image-based upgrade with {lcao} and GitOps ZTP + +When you deploy the {lcao} on a cluster, an `ImageBasedUpgrade` CR is automatically created. You update this CR to specify the image repository of the seed image and to move through the different stages. + +.Prerequisites + +* Create policies and `ConfigMap` objects for resources used in the image-based upgrade. For more information, see _Creating ConfigMap objects for the image-based upgrade with GitOps ZTP_ + +.Procedure + +. Add policies for the `Prep`, `Upgrade`, and `Idle` stages to your existing group `PolicyGenTemplate` called `ibu-upgrade-ranGen.yaml`. ++ +[source,yaml] +---- +apiVersion: ran.openshift.io/v1 +kind: PolicyGenTemplate +metadata: + name: example-group-ibu + namespace: "ztp-group" +spec: + bindingRules: + group-du-sno: "" + mcp: "master" + evaluationInterval: <1> + compliant: 10s + noncompliant: 10s + sourceFiles: + - fileName: ConfigMapGeneric.yaml + complianceType: mustonlyhave + policyName: "oadp-cm-policy" + metadata: + name: oadp-cm + namespace: openshift-adp + - fileName: ibu/ImageBasedUpgrade.yaml + policyName: "prep-policy" + spec: + stage: Prep + seedImageRef: <2> + version: "4.15.0" + image: "quay.io/user/lca-seed:4.15.0" + pullSecretRef: + name: "" + oadpContent: <3> + - name: "oadp-cm" + namespace: "openshift-adp" + status: + conditions: + - reason: Completed + status: "True" + type: PrepCompleted + message: "Prep stage completed successfully" + - fileName: ibu/ImageBasedUpgrade.yaml + policyName: "upgrade-policy" + spec: + stage: Upgrade + status: + conditions: + - reason: Completed + status: "True" + type: UpgradeCompleted + - fileName: ibu/ImageBasedUpgrade.yaml + policyName: "finalize-stage-policy" + complianceType: mustonlyhave + spec: + stage: Idle + - fileName: ibu/ImageBasedUpgrade.yaml + policyName: "finalize-stage-policy" + status: + conditions: + - reason: Idle + status: "True" + type: Idle +---- +<1> The policy evaluation interval for compliant and non-compliant policies. Set them to `10s` to ensure that the policies status accurately reflects the current upgrade status. +<2> Define the seed image, OpenShift Container Platform version, and pull secret for the upgrade in the Prep stage. +<3> Define the OADP `ConfigMap` resources required for backup and restore. + +. Verify that the policies required for an image-based upgrade are created: ++ +-- +[source,terminal] +---- +$ oc get policies -n spoke1 | grep -E "example-group-ibu" +---- + +.Example output +[source,terminal] +---- +ztp-group.example-group-ibu-oadp-cm-policy inform NonCompliant 31h +ztp-group.example-group-ibu-prep-policy inform NonCompliant 31h +ztp-group.example-group-ibu-upgrade-policy inform NonCompliant 31h +ztp-group.example-group-ibu-finalize-policy inform NonCompliant 31h +ztp-group.example-group-ibu-rollback-policy inform NonCompliant 31h +---- +-- + +. Update the `du-profile` cluster label to the target platform version or the corresponding policy-binding label in the `SiteConfig` CR. ++ +-- +[source,yaml] +---- +apiVersion: ran.openshift.io/v1 +kind: SiteConfig +[...] +spec: + [...] + clusterLabels: + du-profile: "4.15.0" +---- + +[IMPORTANT] +==== +Updating the labels to the target platform version unbinds the existing set of policies. +==== +-- + +. Commit and push the updated `SiteConfig` CR to the Git repository. + +. When you are ready to move to the `Prep` stage, create the `ClusterGroupUpgrade` CR with the `Prep` and OADP `ConfigMap` policies: ++ +[source,yaml] +---- +apiVersion: ran.openshift.io/v1alpha1 +kind: ClusterGroupUpgrade +metadata: + name: cgu-ibu-prep + namespace: default +spec: + clusters: + - spoke1 + enable: true + managedPolicies: + - example-group-oadp-cm-policy + - example-group-ibu-prep-policy + remediationStrategy: + canaries: + - spoke1 + maxConcurrency: 1 + timeout: 240 +---- + +. Apply the `Prep` policy: ++ +-- +[source,terminal] +---- +$ oc apply -f cgu-ibu-prep.yml +---- + +If you provide `ConfigMap` objects for OADP resources and extra manifests, {lcao} validates the specified `ConfigMap` objects during the `Prep` stage. +You may encounter the following issues: + +* validation warnings or errors if the {lcao} detects any issues with `extraManifests` +* validation errors if the {lcao} detects any issues with with `oadpContent` + +Validation warnings do not block the `Upgrade` stage but you must decide if it is safe to proceed with the upgrade. +These warnings, for example missing CRDs, namespaces or dry run failures, update the `status.conditions` in the `Prep` stage and `annotation` fields in the `ImageBasedUpgrade` CR with details about the warning. + +.Example validation warning +[source,yaml] +---- +[...] +metadata: +annotations: + extra-manifest.lca.openshift.io/validation-warning: '...' +[...] +---- + +However, validation errors, such as adding `MachineConfig` or Operator manifests to extra manifests, cause the `Prep` stage to fail and block the `Upgrade` stage. + +When the validations pass, the cluster creates a new `ostree` stateroot, which involves pulling and unpacking the seed image, and running host level commands. +Finally, all the required images are precached on the target cluster. +-- + +. Monitor the status and wait for the `cgu-ibu-prep` `ClusterGroupUpgrade` to report `Completed`. ++ +-- +[source,terminal] +---- +$ oc get cgu -n default +---- + +.Example output +[source,terminal] +---- +NAME AGE STATE DETAILS +cgu-ibu-prep 31h Completed All clusters are compliant with all the managed policies +---- +-- \ No newline at end of file diff --git a/modules/ztp-image-based-upgrade-talm-rollback.adoc b/modules/ztp-image-based-upgrade-talm-rollback.adoc new file mode 100644 index 000000000000..1b2a54f5dd16 --- /dev/null +++ b/modules/ztp-image-based-upgrade-talm-rollback.adoc @@ -0,0 +1,106 @@ +// Module included in the following assemblies: +// * edge_computing/image-based-upgrade/ztp-image-based-upgrade.adoc + +:_mod-docs-content-type: PROCEDURE +[id="ztp-image-based-upgrade-with-talm-rollback_{context}"] += (Optional) Moving to the Rollback stage with {lcao} and GitOps ZTP + +If you encounter an issue after upgrade, you can start a manual rollback. + +.Procedure + +. Revert the `du-profile` or the corresponding policy-binding label to the original platform version in the `SiteConfig` CR: ++ +[source,yaml] +---- +apiVersion: ran.openshift.io/v1 +kind: SiteConfig +[...] +spec: + [...] + clusterLabels: + du-profile: "4.14.x" +---- + +. When you are ready to initiate the rollback, add the `Rollback` policy to your existing `PolicyGenTemplate` CR: ++ +[source,yaml] +---- +[...] +- fileName: ibu/ImageBasedUpgrade.yaml + policyName: "rollback-policy" + spec: + stage: Rollback + status: + conditions: + - message: Rollback completed + reason: Completed + status: "True" + type: RollbackCompleted +---- + +. Create a `ClusterGroupUpgrade` CR that references the `Rollback` policy: ++ +[source,yaml] +---- +apiVersion: ran.openshift.io/v1alpha1 +kind: ClusterGroupUpgrade +metadata: + name: cgu-ibu-rollback + namespace: default +spec: + actions: + beforeEnable: + removeClusterAnnotations: + - import.open-cluster-management.io/disable-auto-import + clusters: + - spoke1 + enable: true + managedPolicies: + - example-group-ibu-rollback-policy + remediationStrategy: + canaries: + - spoke1 + maxConcurrency: 1 + timeout: 240 +---- + +. Apply the `Rollback` policy: ++ +[source,terminal] +---- +$ oc apply -f cgu-ibu-rollback.yml +---- + +. When you are satisfied with the changes and you are ready to finalize the rollback, create a `ClusterGroupUpgrade` CR that references the policy that finalizes the upgrade: ++ +[source,yaml] +---- +apiVersion: ran.openshift.io/v1alpha1 +kind: ClusterGroupUpgrade +metadata: + name: cgu-ibu-finalize + namespace: default +spec: + actions: + beforeEnable: + removeClusterAnnotations: + - import.open-cluster-management.io/disable-auto-import + clusters: + - spoke1 + enable: true + managedPolicies: + - example-group-ibu-finalize-policy + remediationStrategy: + canaries: + - spoke1 + maxConcurrency: 1 + timeout: 240 +---- + +. Apply the policy: ++ +[source,terminal] +---- +$ oc apply -f cgu-ibu-finalize.yml +---- \ No newline at end of file diff --git a/modules/ztp-image-based-upgrade-talm-upgrade.adoc b/modules/ztp-image-based-upgrade-talm-upgrade.adoc new file mode 100644 index 000000000000..c2411ab658d7 --- /dev/null +++ b/modules/ztp-image-based-upgrade-talm-upgrade.adoc @@ -0,0 +1,125 @@ +// Module included in the following assemblies: +// * edge_computing/image-based-upgrade/ztp-image-based-upgrade.adoc + +:_mod-docs-content-type: PROCEDURE +[id="ztp-image-based-upgrade-upgrade-gitops_{context}"] += Moving to the Upgrade stage of the image-based upgrade with {lcao} and GitOps ZTP + +After you completed the `Prep` stage, you can upgrade the target cluster. During the upgrade process, the OADP Operator creates a backup of the artifacts specified in the OADP CRs, then the {lcao} upgrades the cluster. + +If the upgrade fails or stops, an automatic rollback is initiated. If you have an issue after the upgrade, you can initiate a manual rollback. For more information about manual rollback, see "(Optional) Initiating a rollback with Lifecycle Agent and GitOps ZTP". + +.Prerequisites + +* Complete the `Prep` stage. + +.Procedure + +. When you are ready to move to the `Upgrade` stage, create the `ClusterGroupUpgrade` CR that references the `Upgrade` policy: ++ +[source,yaml] +---- +apiVersion: ran.openshift.io/v1alpha1 +kind: ClusterGroupUpgrade +metadata: + name: cgu-ibu-upgrade + namespace: default +spec: + actions: + beforeEnable: + addClusterAnnotations: + import.open-cluster-management.io/disable-auto-import: "true" <1> + afterCompletion: + removeClusterAnnotations: + - import.open-cluster-management.io/disable-auto-import <2> + clusters: + - spoke1 + enable: true + managedPolicies: + - example-group-ibu-upgrade-policy + remediationStrategy: + canaries: + - spoke1 + maxConcurrency: 1 + timeout: 240 +---- +<1> Applies the `disable-auto-import` annotation to the managed cluster before starting the upgrade. This annotation ensures the automatic importing of managed cluster is disabled during the upgrade stage until the cluster is ready. +<2> Removes the `disable-auto-import` annotation after the upgrade is complete. + +. Apply the `Upgrade` policy: ++ +[source,terminal] +---- +$ oc apply -f cgu-ibu-upgrade.yml +---- + +.. Monitor the status and wait for the `cgu-ibu-upgrade` `ClusterGroupUpgrade` to report `Completed`. ++ +-- +[source,terminal] +---- +$ oc get cgu -n default +---- + +.Example output +[source,terminal] +---- +NAME AGE STATE DETAILS +example-group-ibu-prep 31h Completed All clusters are compliant with all the managed policies +example-group-ibu-upgrade 31h Completed All clusters are compliant with all the managed policies +---- +-- + +. When you are satisfied with the changes and ready to finalize the upgrade, create a `ClusterGroupUpgrade` CR that references the policy that finalizes the upgrade: ++ +-- +[source,yaml] +---- +apiVersion: ran.openshift.io/v1alpha1 +kind: ClusterGroupUpgrade +metadata: + name: cgu-ibu-finalize + namespace: default +spec: + clusters: + - spoke1 + enable: true + managedPolicies: + - example-group-ibu-finalize-policy + remediationStrategy: + canaries: + - spoke1 + maxConcurrency: 1 + timeout: 240 +---- + +[IMPORTANT] +==== +Ensure that no other `ClusterGroupUpgrade` CRs are in progress because this causes {cgu-operator} to continuously reconcile them. Delete all `"In-Progress"` `ClusterGroupUpgrade` CRs before applying the `cgu-ibu-finalize.yaml`. +==== +-- + +. Apply the policy: ++ +[source,terminal] +---- +$ oc apply -f cgu-ibu-finalize.yaml +---- + +.. Monitor the status and wait for the `cgu-ibu-upgrade` `ClusterGroupUpgrade` to report `Completed`. ++ +-- +[source,terminal] +---- +$ oc get cgu -n default +---- + +.Example output +[source,terminal] +---- +NAME AGE STATE DETAILS +cgu-ibu-finalize 30h Completed All clusters are compliant with all the managed policies +cgu-ibu-prep 31h Completed All clusters are compliant with all the managed policies +cgu-ibu-upgrade 31h Completed All clusters are compliant with all the managed policies +---- +-- \ No newline at end of file diff --git a/snippets/ibu-ApplicationBackupRestoreLso.adoc b/snippets/ibu-ApplicationBackupRestoreLso.adoc new file mode 100644 index 000000000000..17b5a67176ac --- /dev/null +++ b/snippets/ibu-ApplicationBackupRestoreLso.adoc @@ -0,0 +1,40 @@ +[source,yaml] +---- +apiVersion: velero.io/v1 +kind: Backup +metadata: + labels: + velero.io/storage-location: default + name: backup-app + namespace: openshift-adp +spec: + includedNamespaces: + - test + includedNamespaceScopedResources: + - secrets + - persistentvolumeclaims + - deployments + - statefulsets + - configmaps + - cronjobs + - services + - job + - poddisruptionbudgets + - <1> + excludedClusterScopedResources: + - persistentVolumes +--- +apiVersion: velero.io/v1 +kind: Restore +metadata: + name: test-app + namespace: openshift-adp + labels: + velero.io/storage-location: default + annotations: + lca.openshift.io/apply-wave: "4" +spec: + backupName: + backup-app +---- +<1> Define custom resources for your application. \ No newline at end of file diff --git a/snippets/ibu-ApplicationBackupRestoreLvms.adoc b/snippets/ibu-ApplicationBackupRestoreLvms.adoc new file mode 100644 index 000000000000..90a1a5b20d78 --- /dev/null +++ b/snippets/ibu-ApplicationBackupRestoreLvms.adoc @@ -0,0 +1,48 @@ +[source,yaml] +---- +apiVersion: velero.io/v1 +kind: Backup +metadata: + labels: + velero.io/storage-location: default + name: backup-app + namespace: openshift-adp +spec: + includedNamespaces: + - test + includedNamespaceScopedResources: + - secrets + - persistentvolumeclaims + - deployments + - statefulsets + - configmaps + - cronjobs + - services + - job + - poddisruptionbudgets + - <1> + includedClusterScopedResources: + - persistentVolumes <2> + - logicalvolumes.topolvm.io <2> + - volumesnapshotcontents <3> +--- +apiVersion: velero.io/v1 +kind: Restore +metadata: + name: test-app + namespace: openshift-adp + labels: + velero.io/storage-location: default + annotations: + lca.openshift.io/apply-wave: "4" +spec: + backupName: + backup-app + restorePVs: true + restoreStatus: <2> + includedResources: <2> + - logicalvolumes <2> +---- +<1> Define custom resources for your application. +<2> Required field. +<3> Optional if you use {lvms} volume snapshots. \ No newline at end of file diff --git a/snippets/ibu-ApplicationClusterScopedBackupRestore.adoc b/snippets/ibu-ApplicationClusterScopedBackupRestore.adoc new file mode 100644 index 000000000000..6a5e202d7be0 --- /dev/null +++ b/snippets/ibu-ApplicationClusterScopedBackupRestore.adoc @@ -0,0 +1,35 @@ +[source,yaml] +---- +apiVersion: velero.io/v1 +kind: Backup +metadata: + annotations: + lca.openshift.io/apply-label: "apiextensions.k8s.io/v1/customresourcedefinitions/test.example.com,security.openshift.io/v1/securitycontextconstraints/test,rbac.authorization.k8s.io/v1/clusterroles/test-role,rbac.authorization.k8s.io/v1/clusterrolebindings/system:openshift:scc:test" <1> + name: backup-app-cluster-resources + labels: + velero.io/storage-location: default + namespace: openshift-adp +spec: + includedClusterScopedResources: + - customresourcedefinitions + - securitycontextconstraints + - clusterrolebindings + - clusterroles + excludedClusterScopedResources: + - Namespace +--- +apiVersion: velero.io/v1 +kind: Restore +metadata: + name: test-app-cluster-resources + namespace: openshift-adp + labels: + velero.io/storage-location: default + annotations: + lca.openshift.io/apply-wave: "3" <2> +spec: + backupName: + backup-app-cluster-resources +---- +<1> Replace the example resource name with your actual resources. +<2> The `lca.openshift.io/apply-wave` value must be higher than what the value in the platform `Restore` CRs and lower than the value in the application namespace-scoped `Restore` CR. \ No newline at end of file diff --git a/snippets/ibu-PlatformBackupRestore.adoc b/snippets/ibu-PlatformBackupRestore.adoc new file mode 100644 index 000000000000..037c9fc00908 --- /dev/null +++ b/snippets/ibu-PlatformBackupRestore.adoc @@ -0,0 +1,39 @@ +[source,yaml] +---- +apiVersion: velero.io/v1 +kind: Backup +metadata: + name: acm-klusterlet + annotations: + lca.openshift.io/apply-label: "apps/v1/deployments/open-cluster-management-agent/klusterlet,v1/secrets/open-cluster-management-agent/bootstrap-hub-kubeconfig,rbac.authorization.k8s.io/v1/clusterroles/klusterlet,v1/serviceaccounts/open-cluster-management-agent/klusterlet,scheduling.k8s.io/v1/priorityclasses/klusterlet-critical,rbac.authorization.k8s.io/v1/clusterroles/open-cluster-management:klusterlet-admin-aggregate-clusterrole,rbac.authorization.k8s.io/v1/clusterrolebindings/klusterlet,operator.open-cluster-management.io/v1/klusterlets/klusterlet,apiextensions.k8s.io/v1/customresourcedefinitions/klusterlets.operator.open-cluster-management.io,v1/secrets/open-cluster-management-agent/open-cluster-management-image-pull-credentials" <1> + labels: + velero.io/storage-location: default + namespace: openshift-adp +spec: + includedNamespaces: + - open-cluster-management-agent + includedClusterScopedResources: + - klusterlets.operator.open-cluster-management.io + - clusterroles.rbac.authorization.k8s.io + - clusterrolebindings.rbac.authorization.k8s.io + - priorityclasses.scheduling.k8s.io + includedNamespaceScopedResources: + - deployments + - serviceaccounts + - secrets + excludedNamespaceScopedResources: [] +--- +apiVersion: velero.io/v1 +kind: Restore +metadata: + name: acm-klusterlet + namespace: openshift-adp + labels: + velero.io/storage-location: default + annotations: + lca.openshift.io/apply-wave: "1" +spec: + backupName: + acm-klusterlet +---- +<1> If your `multiclusterHub` CR does not have `.spec.imagePullSecret` defined and the secret does not exist on the `open-cluster-management-agent` namespace in your hub cluster, remove the `v1/secrets/open-cluster-management-agent/open-cluster-management-image-pull-credentials`. \ No newline at end of file diff --git a/snippets/ibu-PlatformBackupRestoreLvms.adoc b/snippets/ibu-PlatformBackupRestoreLvms.adoc new file mode 100644 index 000000000000..51ba1ec5b17a --- /dev/null +++ b/snippets/ibu-PlatformBackupRestoreLvms.adoc @@ -0,0 +1,31 @@ +[source,yaml] +---- +apiVersion: velero.io/v1 +kind: Backup +metadata: + labels: + velero.io/storage-location: default + name: lvmcluster + namespace: openshift-adp +spec: + includedNamespaces: + - openshift-storage + includedNamespaceScopedResources: + - lvmclusters + - lvmvolumegroups + - lvmvolumegroupnodestatuses +--- +apiVersion: velero.io/v1 +kind: Restore +metadata: + name: lvmcluster + namespace: openshift-adp + labels: + velero.io/storage-location: default + annotations: + lca.openshift.io/apply-wave: "2" <1> +spec: + backupName: + lvmcluster +---- +<1> The `lca.openshift.io/apply-wave` value must be lower than the values specified in the application `Restore` CRs. \ No newline at end of file