diff --git a/ci-operator/config/quay/quay/quay-quay-master__omr-v2-upgrade-v3.yaml b/ci-operator/config/quay/quay/quay-quay-master__omr-v2-upgrade-v3.yaml new file mode 100644 index 0000000000000..b2a09c5b73ea2 --- /dev/null +++ b/ci-operator/config/quay/quay/quay-quay-master__omr-v2-upgrade-v3.yaml @@ -0,0 +1,63 @@ +base_images: + cli: + name: "4.22" + namespace: ocp + tag: cli + upi-installer: + name: "4.22" + namespace: ocp + tag: upi-installer +build_root: + image_stream_tag: + name: release + namespace: openshift + tag: rhel-9-release-golang-1.25-openshift-4.22 +images: + items: + - dockerfile_path: Dockerfile.mirror + optional: true + to: quay-mirror +releases: + latest: + release: + architecture: amd64 + channel: stable + version: "4.22" +resources: + '*': + limits: + memory: 4Gi + requests: + cpu: 100m + memory: 200Mi +tests: +- as: disconnected-install + cron: 0 3 * * 0 + steps: + allow_best_effort_post_steps: true + cluster_profile: aws-quay-qe + dependencies: + OMR_IMAGE: pipeline:quay-mirror + env: + ADD_INGRESS_RECORDS_MANUALLY: "yes" + BASE_DOMAIN: quayqe.devcluster.openshift.com + COMPUTE_NODE_TYPE: m5.xlarge + CONTROL_PLANE_INSTANCE_TYPE: m6i.xlarge + PUBLISH: Internal + post: + - ref: quay-omr-v3-gather + - chain: ipi-aws-post-disconnected + pre: + - chain: quay-aws-ipi-disconnected-omr-v2-upgrade-v3-provision + test: + - ref: quay-omr-v2-validate + - ref: quay-omr-v2-seed-migration-sentinel + - ref: quay-omr-v3-upgrade-from-v2 + - ref: quay-omr-v3-validate + - ref: quay-omr-v3-validate-migration-sentinel + timeout: 10h0m0s +zz_generated_metadata: + branch: master + org: quay + repo: quay + variant: omr-v2-upgrade-v3 diff --git a/ci-operator/jobs/quay/quay/quay-quay-master-periodics.yaml b/ci-operator/jobs/quay/quay/quay-quay-master-periodics.yaml index 5ad7b50b5e294..7fe4b92ea60c1 100644 --- a/ci-operator/jobs/quay/quay/quay-quay-master-periodics.yaml +++ b/ci-operator/jobs/quay/quay/quay-quay-master-periodics.yaml @@ -86,3 +86,89 @@ periodics: - name: result-aggregator secret: secretName: result-aggregator +- agent: kubernetes + cluster: build11 + cron: 0 3 * * 0 + decorate: true + decoration_config: + sparse_checkout_files: + - Dockerfile.mirror + timeout: 10h0m0s + extra_refs: + - base_ref: master + org: quay + repo: quay + sparse_checkout_files: + - Dockerfile.mirror + labels: + ci-operator.openshift.io/cloud: aws + ci-operator.openshift.io/cloud-cluster-profile: aws-quay-qe + ci-operator.openshift.io/variant: omr-v2-upgrade-v3 + ci.openshift.io/generator: prowgen + pj-rehearse.openshift.io/can-be-rehearsed: "true" + name: periodic-ci-quay-quay-master-omr-v2-upgrade-v3-disconnected-install + spec: + containers: + - args: + - --gcs-upload-secret=/secrets/gcs/service-account.json + - --image-import-pull-secret=/etc/pull-secret/.dockerconfigjson + - --lease-server-credentials-file=/etc/boskos/credentials + - --report-credentials-file=/etc/report/credentials + - --secret-dir=/secrets/ci-pull-credentials + - --target=disconnected-install + - --variant=omr-v2-upgrade-v3 + command: + - ci-operator + env: + - name: HTTP_SERVER_IP + valueFrom: + fieldRef: + fieldPath: status.podIP + image: quay-proxy.ci.openshift.org/openshift/ci:ci_ci-operator_latest + imagePullPolicy: Always + name: "" + ports: + - containerPort: 8080 + name: http + resources: + requests: + cpu: 10m + volumeMounts: + - mountPath: /etc/boskos + name: boskos + readOnly: true + - mountPath: /secrets/ci-pull-credentials + name: ci-pull-credentials + readOnly: true + - mountPath: /secrets/gcs + name: gcs-credentials + readOnly: true + - mountPath: /secrets/manifest-tool + name: manifest-tool-local-pusher + readOnly: true + - mountPath: /etc/pull-secret + name: pull-secret + readOnly: true + - mountPath: /etc/report + name: result-aggregator + readOnly: true + serviceAccountName: ci-operator + volumes: + - name: boskos + secret: + items: + - key: credentials + path: credentials + secretName: boskos-credentials + - name: ci-pull-credentials + secret: + secretName: ci-pull-credentials + - name: manifest-tool-local-pusher + secret: + secretName: manifest-tool-local-pusher + - name: pull-secret + secret: + secretName: registry-pull-credentials + - name: result-aggregator + secret: + secretName: result-aggregator diff --git a/ci-operator/jobs/quay/quay/quay-quay-master-presubmits.yaml b/ci-operator/jobs/quay/quay/quay-quay-master-presubmits.yaml index e799db7d7f5e4..525c6d5c0a226 100644 --- a/ci-operator/jobs/quay/quay/quay-quay-master-presubmits.yaml +++ b/ci-operator/jobs/quay/quay/quay-quay-master-presubmits.yaml @@ -58,6 +58,65 @@ presubmits: secret: secretName: result-aggregator trigger: (?m)^/test( | .* )images,?($|\s.*) + - agent: kubernetes + always_run: true + branches: + - ^master$ + - ^master- + cluster: build03 + context: ci/prow/omr-v2-upgrade-v3-images + decorate: true + decoration_config: + sparse_checkout_files: + - Dockerfile.mirror + labels: + ci-operator.openshift.io/variant: omr-v2-upgrade-v3 + ci.openshift.io/generator: prowgen + pj-rehearse.openshift.io/can-be-rehearsed: "true" + name: pull-ci-quay-quay-master-omr-v2-upgrade-v3-images + optional: true + rerun_command: /test omr-v2-upgrade-v3-images + spec: + containers: + - args: + - --gcs-upload-secret=/secrets/gcs/service-account.json + - --image-import-pull-secret=/etc/pull-secret/.dockerconfigjson + - --report-credentials-file=/etc/report/credentials + - --target=[images] + - --variant=omr-v2-upgrade-v3 + command: + - ci-operator + image: quay-proxy.ci.openshift.org/openshift/ci:ci_ci-operator_latest + imagePullPolicy: Always + name: "" + resources: + requests: + cpu: 10m + volumeMounts: + - mountPath: /secrets/gcs + name: gcs-credentials + readOnly: true + - mountPath: /secrets/manifest-tool + name: manifest-tool-local-pusher + readOnly: true + - mountPath: /etc/pull-secret + name: pull-secret + readOnly: true + - mountPath: /etc/report + name: result-aggregator + readOnly: true + serviceAccountName: ci-operator + volumes: + - name: manifest-tool-local-pusher + secret: + secretName: manifest-tool-local-pusher + - name: pull-secret + secret: + secretName: registry-pull-credentials + - name: result-aggregator + secret: + secretName: result-aggregator + trigger: (?m)^/test( | .* )omr-v2-upgrade-v3-images,?($|\s.*) - agent: kubernetes always_run: false branches: diff --git a/ci-operator/step-registry/quay/aws-ipi-disconnected-omr-v2-upgrade-v3-provision/OWNERS b/ci-operator/step-registry/quay/aws-ipi-disconnected-omr-v2-upgrade-v3-provision/OWNERS new file mode 120000 index 0000000000000..ec405d65a79df --- /dev/null +++ b/ci-operator/step-registry/quay/aws-ipi-disconnected-omr-v2-upgrade-v3-provision/OWNERS @@ -0,0 +1 @@ +../OWNERS \ No newline at end of file diff --git a/ci-operator/step-registry/quay/aws-ipi-disconnected-omr-v2-upgrade-v3-provision/quay-aws-ipi-disconnected-omr-v2-upgrade-v3-provision-chain.metadata.json b/ci-operator/step-registry/quay/aws-ipi-disconnected-omr-v2-upgrade-v3-provision/quay-aws-ipi-disconnected-omr-v2-upgrade-v3-provision-chain.metadata.json new file mode 100644 index 0000000000000..07873bc008b4d --- /dev/null +++ b/ci-operator/step-registry/quay/aws-ipi-disconnected-omr-v2-upgrade-v3-provision/quay-aws-ipi-disconnected-omr-v2-upgrade-v3-provision-chain.metadata.json @@ -0,0 +1,11 @@ +{ + "path": "quay/aws-ipi-disconnected-omr-v2-upgrade-v3-provision/quay-aws-ipi-disconnected-omr-v2-upgrade-v3-provision-chain.yaml", + "owners": { + "approvers": [ + "quay-approvers" + ], + "reviewers": [ + "quay-approvers" + ] + } +} \ No newline at end of file diff --git a/ci-operator/step-registry/quay/aws-ipi-disconnected-omr-v2-upgrade-v3-provision/quay-aws-ipi-disconnected-omr-v2-upgrade-v3-provision-chain.yaml b/ci-operator/step-registry/quay/aws-ipi-disconnected-omr-v2-upgrade-v3-provision/quay-aws-ipi-disconnected-omr-v2-upgrade-v3-provision-chain.yaml new file mode 100644 index 0000000000000..b526475591a8f --- /dev/null +++ b/ci-operator/step-registry/quay/aws-ipi-disconnected-omr-v2-upgrade-v3-provision/quay-aws-ipi-disconnected-omr-v2-upgrade-v3-provision-chain.yaml @@ -0,0 +1,63 @@ +chain: + as: quay-aws-ipi-disconnected-omr-v2-upgrade-v3-provision + steps: + - ref: aws-provision-vpc-disconnected + - chain: aws-provision-bastionhost + - ref: quay-omr-v2-host-provision + - ref: quay-omr-v2-install + - ref: ipi-install-rbac + - ref: quay-omr-v3-mirror + - ref: ipi-conf + - ref: ipi-conf-telemetry + - ref: ipi-conf-aws-custom-vpc + - ref: ipi-conf-manual-creds + - ref: ipi-conf-aws + - ref: quay-omr-v3-ipi-conf-mirror + - ref: ipi-install-monitoringpvc + - ref: proxy-config-generate + - ref: aws-provision-cco-manual-users-static + - ref: ipi-conf-manual-creds-remove-unnecessary-creds + - ref: ipi-install-install-aws + - ref: ipi-install-times-collection + - ref: nodes-readiness + - ref: multiarch-validate-nodes + - as: disable-default-sources + from: cli + commands: |- + set -euo pipefail + if test -f "${SHARED_DIR}/proxy-conf.sh"; then + source "${SHARED_DIR}/proxy-conf.sh" + fi + oc patch OperatorHub cluster --type json \ + -p '[{"op": "add", "path": "/spec/disableAllDefaultSources", "value": true}]' + resources: + requests: + cpu: 100m + memory: 100Mi + timeout: 5m0s + grace_period: 5m0s + - ref: aws-provision-tags-for-byo-vpc + env: + - name: ZONES_COUNT + default: "2" + documentation: Use two availability zones so the workflow also supports regions such as us-west-1. + - name: CONTROL_PLANE_INSTANCE_TYPE + default: "m6i.xlarge" + documentation: Instance type for control plane nodes. + - name: COMPUTE_NODE_TYPE + default: "m5.xlarge" + documentation: Instance type for compute nodes. + - name: ADD_INGRESS_RECORDS_MANUALLY + default: "yes" + documentation: Create the private cluster ingress DNS record during installation. + - name: PUBLISH + default: "Internal" + documentation: Cluster publish strategy for the disconnected install. + - name: EXTRACT_MANIFEST_INCLUDED + default: "true" + documentation: Extract only CredentialsRequests included by the install config so disabled capabilities do not inject secrets into absent namespaces. + documentation: |- + Provision the standard disconnected AWS VPC and FCOS proxy bastion plus a + dedicated large RHEL host, install released OMR v2 on the RHEL host, mirror + the selected OpenShift release through it, and install the cluster with + manual CCO credentials. diff --git a/ci-operator/step-registry/quay/omr-v2/OWNERS b/ci-operator/step-registry/quay/omr-v2/OWNERS new file mode 120000 index 0000000000000..ec405d65a79df --- /dev/null +++ b/ci-operator/step-registry/quay/omr-v2/OWNERS @@ -0,0 +1 @@ +../OWNERS \ No newline at end of file diff --git a/ci-operator/step-registry/quay/omr-v2/host-provision/OWNERS b/ci-operator/step-registry/quay/omr-v2/host-provision/OWNERS new file mode 120000 index 0000000000000..8c272259fbba9 --- /dev/null +++ b/ci-operator/step-registry/quay/omr-v2/host-provision/OWNERS @@ -0,0 +1 @@ +../../OWNERS \ No newline at end of file diff --git a/ci-operator/step-registry/quay/omr-v2/host-provision/quay-omr-v2-host-provision-commands.sh b/ci-operator/step-registry/quay/omr-v2/host-provision/quay-omr-v2-host-provision-commands.sh new file mode 100644 index 0000000000000..075db9543ba8c --- /dev/null +++ b/ci-operator/step-registry/quay/omr-v2/host-provision/quay-omr-v2-host-provision-commands.sh @@ -0,0 +1,309 @@ +#!/bin/bash + +set -o nounset +set -o errexit +set -o pipefail + +EXIT_CODE=100 +stack_name="${NAMESPACE}-${UNIQUE_HASH}-omr2" +template_file="" +stack_started=false +published=false +declare -a output_files=( + "${SHARED_DIR}/omr_host_public_address" + "${SHARED_DIR}/omr_host_private_address" + "${SHARED_DIR}/omr_host_ssh_user" + "${SHARED_DIR}/omr_host_instance_id" +) + +cleanup() { + local status=$? + + trap - EXIT TERM + set +o errexit + if [[ "${status}" -eq 0 ]]; then + EXIT_CODE=0 + fi + printf '%s\n' "${EXIT_CODE}" > "${SHARED_DIR}/install-pre-config-status.txt" + + if [[ "${published}" != true ]]; then + rm -f -- "${output_files[@]}" + fi + if [[ -n "${template_file}" ]]; then + rm -f -- "${template_file}" + fi + if [[ "${stack_started}" == true ]]; then + aws --region "${region}" cloudformation describe-stack-events \ + --stack-name "${stack_name}" --output json \ + > "${ARTIFACT_DIR}/stack-events-${stack_name}.json" 2>/dev/null || true + fi + exit "${status}" +} + +terminate() { + exit 143 +} + +trap cleanup EXIT +trap terminate TERM + +mkdir -p "${ARTIFACT_DIR}" +rm -f -- "${output_files[@]}" + +for command in aws base64 ssh yq-go; do + if ! command -v "${command}" >/dev/null 2>&1; then + echo "Required command ${command} is unavailable in the host provision step image." >&2 + exit 1 + fi +done + +for required_file in \ + "${CLUSTER_PROFILE_DIR}/.awscred" \ + "${CLUSTER_PROFILE_DIR}/ssh-privatekey" \ + "${CLUSTER_PROFILE_DIR}/ssh-publickey" \ + "${SHARED_DIR}/public_subnet_ids" \ + "${SHARED_DIR}/vpc_id"; do + if [[ ! -s "${required_file}" ]]; then + echo "Required host provision input ${required_file} is missing or empty." >&2 + exit 1 + fi +done + +if [[ ! "${OMR_V2_INSTANCE_TYPE}" =~ ^[a-z0-9][a-z0-9.]*$ ]]; then + echo "OMR_V2_INSTANCE_TYPE is invalid." >&2 + exit 1 +fi +if [[ ! "${OMR_V2_ROOT_VOLUME_SIZE}" =~ ^[0-9]+$ ]] || + (( OMR_V2_ROOT_VOLUME_SIZE < 100 || OMR_V2_ROOT_VOLUME_SIZE > 16384 )); then + echo "OMR_V2_ROOT_VOLUME_SIZE must be between 100 and 16384 GiB." >&2 + exit 1 +fi + +export AWS_SHARED_CREDENTIALS_FILE="${CLUSTER_PROFILE_DIR}/.awscred" +region="${REGION:-$LEASED_RESOURCE}" +vpc_id=$(<"${SHARED_DIR}/vpc_id") +public_subnet=$(yq-go r "${SHARED_DIR}/public_subnet_ids" '[0]') + +if [[ ! "${vpc_id}" =~ ^vpc-[a-f0-9]+$ ]] || + [[ ! "${public_subnet}" =~ ^subnet-[a-f0-9]+$ ]]; then + echo "The disconnected VPC or public subnet identifier is invalid." >&2 + exit 1 +fi + +vpc_cidr=$(aws --region "${region}" ec2 describe-vpcs \ + --vpc-ids "${vpc_id}" --query 'Vpcs[0].CidrBlock' --output text) +if [[ -z "${vpc_cidr}" || "${vpc_cidr}" == "None" ]]; then + echo "Could not resolve the disconnected VPC CIDR." >&2 + exit 1 +fi + +ami_id="${OMR_V2_AMI_ID}" +if [[ -z "${ami_id}" ]]; then + ami_id=$(aws --region "${region}" ec2 describe-images \ + --owners 309956199498 \ + --filters \ + 'Name=name,Values=RHEL-9.*_HVM-*-x86_64-*-Hourly2-GP3' \ + 'Name=state,Values=available' \ + 'Name=architecture,Values=x86_64' \ + 'Name=root-device-type,Values=ebs' \ + 'Name=virtualization-type,Values=hvm' \ + --query 'reverse(sort_by(Images,&CreationDate))[0].ImageId' \ + --output text) +fi +if [[ ! "${ami_id}" =~ ^ami-[a-f0-9]+$ ]]; then + echo "Could not resolve an official RHEL 9 AMI in ${region}." >&2 + exit 1 +fi +root_device_name=$(aws --region "${region}" ec2 describe-images \ + --image-ids "${ami_id}" --query 'Images[0].RootDeviceName' --output text) +if [[ ! "${root_device_name}" =~ ^/dev/[A-Za-z0-9]+$ ]]; then + echo "Could not resolve the root device name for ${ami_id}." >&2 + exit 1 +fi + +public_key_base64=$(base64 -w 0 < "${CLUSTER_PROFILE_DIR}/ssh-publickey") +if [[ -z "${public_key_base64}" ]]; then + echo "The encoded SSH public key is empty." >&2 + exit 1 +fi + +template_file=$(mktemp /tmp/quay-omr-v2-host.XXXXXX.yaml) +cat > "${template_file}" <<'EOF' +AWSTemplateFormatVersion: 2010-09-09 +Description: Dedicated RHEL host for OMR v2 to v3 migration testing +Parameters: + AmiId: + Type: AWS::EC2::Image::Id + InstanceType: + Type: String + PublicKeyBase64: + Type: String + PublicSubnet: + Type: AWS::EC2::Subnet::Id + RootVolumeSize: + Type: Number + RootDeviceName: + Type: String + VpcCidr: + Type: String + VpcId: + Type: AWS::EC2::VPC::Id +Resources: + OMRSecurityGroup: + Type: AWS::EC2::SecurityGroup + Properties: + GroupDescription: Dedicated OMR migration host security group + VpcId: !Ref VpcId + SecurityGroupIngress: + - IpProtocol: tcp + FromPort: 22 + ToPort: 22 + CidrIp: 0.0.0.0/0 + - IpProtocol: tcp + FromPort: 8443 + ToPort: 8443 + CidrIp: !Ref VpcCidr + OMRHost: + Type: AWS::EC2::Instance + Properties: + ImageId: !Ref AmiId + InstanceType: !Ref InstanceType + NetworkInterfaces: + - AssociatePublicIpAddress: true + DeviceIndex: "0" + GroupSet: + - !GetAtt OMRSecurityGroup.GroupId + SubnetId: !Ref PublicSubnet + BlockDeviceMappings: + - DeviceName: !Ref RootDeviceName + Ebs: + DeleteOnTermination: true + Encrypted: true + VolumeSize: !Ref RootVolumeSize + VolumeType: gp3 + Tags: + - Key: Name + Value: !Sub "${AWS::StackName}-host" + UserData: + Fn::Base64: !Sub | + #!/bin/bash + set -o nounset + set -o errexit + set -o pipefail + install -d -m 0700 -o ec2-user -g ec2-user /home/ec2-user/.ssh + # OMR appends a generated local-install key, so preserve the line boundary. + authorized_key=$(printf '%s' '${PublicKeyBase64}' | base64 --decode) + printf '%s\n' "$authorized_key" > /home/ec2-user/.ssh/authorized_keys + unset authorized_key + chown ec2-user:ec2-user /home/ec2-user/.ssh/authorized_keys + chmod 0600 /home/ec2-user/.ssh/authorized_keys + restorecon -RF /home/ec2-user/.ssh || true + dnf install -y curl gzip openssl podman slirp4netns tar + install -d -m 0700 -o ec2-user -g ec2-user \ + /home/ec2-user/.config \ + /home/ec2-user/.config/containers + printf '[network]\ndefault_rootless_network_cmd = "slirp4netns"\n' > /home/ec2-user/.config/containers/containers.conf + chown ec2-user:ec2-user /home/ec2-user/.config/containers/containers.conf +Outputs: + InstanceId: + Value: !Ref OMRHost + PrivateDnsName: + Value: !GetAtt OMRHost.PrivateDnsName + PublicIp: + Value: !GetAtt OMRHost.PublicIp +EOF + +expiration_date=$(date -d '12 hours' --iso=minutes --utc) +printf '%s\n' "${stack_name}" >> "${SHARED_DIR}/to_be_removed_cf_stack_list" +stack_started=true +aws --region "${region}" cloudformation create-stack \ + --stack-name "${stack_name}" \ + --template-body "file://${template_file}" \ + --tags "Key=expirationDate,Value=${expiration_date}" \ + --parameters \ + "ParameterKey=AmiId,ParameterValue=${ami_id}" \ + "ParameterKey=InstanceType,ParameterValue=${OMR_V2_INSTANCE_TYPE}" \ + "ParameterKey=PublicKeyBase64,ParameterValue=${public_key_base64}" \ + "ParameterKey=PublicSubnet,ParameterValue=${public_subnet}" \ + "ParameterKey=RootDeviceName,ParameterValue=${root_device_name}" \ + "ParameterKey=RootVolumeSize,ParameterValue=${OMR_V2_ROOT_VOLUME_SIZE}" \ + "ParameterKey=VpcCidr,ParameterValue=${vpc_cidr}" \ + "ParameterKey=VpcId,ParameterValue=${vpc_id}" +aws --region "${region}" cloudformation wait stack-create-complete \ + --stack-name "${stack_name}" + +instance_id=$(aws --region "${region}" cloudformation describe-stacks \ + --stack-name "${stack_name}" \ + --query 'Stacks[0].Outputs[?OutputKey == `InstanceId`].OutputValue | [0]' \ + --output text) +private_address=$(aws --region "${region}" cloudformation describe-stacks \ + --stack-name "${stack_name}" \ + --query 'Stacks[0].Outputs[?OutputKey == `PrivateDnsName`].OutputValue | [0]' \ + --output text) +public_address=$(aws --region "${region}" cloudformation describe-stacks \ + --stack-name "${stack_name}" \ + --query 'Stacks[0].Outputs[?OutputKey == `PublicIp`].OutputValue | [0]' \ + --output text) + +if [[ ! "${instance_id}" =~ ^i-[a-f0-9]+$ ]] || + [[ ! "${private_address}" =~ ^[A-Za-z0-9][A-Za-z0-9.-]*$ ]] || + [[ ! "${public_address}" =~ ^[0-9.]+$ ]]; then + echo "The dedicated OMR host stack returned invalid connection details." >&2 + exit 1 +fi + +printf '%s\n' "${instance_id}" >> "${SHARED_DIR}/aws-instance-ids.txt" + +# Retain connection and instance identity if later bootstrap validation fails so +# the unconditional post step can still collect AWS and SSH diagnostics. +public_tmp=$(mktemp "${SHARED_DIR}/.omr_host_public_address.XXXXXX") +private_tmp=$(mktemp "${SHARED_DIR}/.omr_host_private_address.XXXXXX") +user_tmp=$(mktemp "${SHARED_DIR}/.omr_host_ssh_user.XXXXXX") +instance_tmp=$(mktemp "${SHARED_DIR}/.omr_host_instance_id.XXXXXX") +printf '%s\n' "${public_address}" > "${public_tmp}" +printf '%s\n' "${private_address}" > "${private_tmp}" +printf '%s\n' ec2-user > "${user_tmp}" +printf '%s\n' "${instance_id}" > "${instance_tmp}" +chmod 0644 "${public_tmp}" "${private_tmp}" "${user_tmp}" "${instance_tmp}" +mv -f -- "${public_tmp}" "${SHARED_DIR}/omr_host_public_address" +mv -f -- "${private_tmp}" "${SHARED_DIR}/omr_host_private_address" +mv -f -- "${user_tmp}" "${SHARED_DIR}/omr_host_ssh_user" +mv -f -- "${instance_tmp}" "${SHARED_DIR}/omr_host_instance_id" +published=true + +aws --region "${region}" ec2 wait instance-status-ok --instance-ids "${instance_id}" + +if ! whoami >/dev/null 2>&1; then + if [[ -w /etc/passwd ]]; then + echo "${USER_NAME:-default}:x:$(id -u):0:${USER_NAME:-default} user:${HOME}:/sbin/nologin" >> /etc/passwd + else + echo "/etc/passwd is not writable and the current UID has no passwd entry." >&2 + exit 1 + fi +fi + +ssh_options=( + -o UserKnownHostsFile=/dev/null + -o StrictHostKeyChecking=no + -o "IdentityFile=${CLUSTER_PROFILE_DIR}/ssh-privatekey" + -o ConnectTimeout=10 + -o ConnectionAttempts=3 +) +ssh_ready=false +for attempt in $(seq 1 30); do + if ssh "${ssh_options[@]}" "ec2-user@${public_address}" true >/dev/null 2>&1; then + ssh_ready=true + break + fi + echo "Waiting for dedicated OMR host SSH (${attempt}/30)." + sleep 10 +done +if [[ "${ssh_ready}" != true ]]; then + echo "The dedicated OMR host did not become reachable over SSH." >&2 + exit 1 +fi + +ssh "${ssh_options[@]}" "ec2-user@${public_address}" \ + "sudo cloud-init status --wait && podman system migrate && podman --version && openssl version && command -v slirp4netns" + +echo "Dedicated RHEL 9 OMR host ${instance_id} is ready." diff --git a/ci-operator/step-registry/quay/omr-v2/host-provision/quay-omr-v2-host-provision-ref.metadata.json b/ci-operator/step-registry/quay/omr-v2/host-provision/quay-omr-v2-host-provision-ref.metadata.json new file mode 100644 index 0000000000000..85501e997ada3 --- /dev/null +++ b/ci-operator/step-registry/quay/omr-v2/host-provision/quay-omr-v2-host-provision-ref.metadata.json @@ -0,0 +1,11 @@ +{ + "path": "quay/omr-v2/host-provision/quay-omr-v2-host-provision-ref.yaml", + "owners": { + "approvers": [ + "quay-approvers" + ], + "reviewers": [ + "quay-approvers" + ] + } +} \ No newline at end of file diff --git a/ci-operator/step-registry/quay/omr-v2/host-provision/quay-omr-v2-host-provision-ref.yaml b/ci-operator/step-registry/quay/omr-v2/host-provision/quay-omr-v2-host-provision-ref.yaml new file mode 100644 index 0000000000000..61a72f5440dc7 --- /dev/null +++ b/ci-operator/step-registry/quay/omr-v2/host-provision/quay-omr-v2-host-provision-ref.yaml @@ -0,0 +1,25 @@ +ref: + as: quay-omr-v2-host-provision + from: upi-installer + cli: latest + commands: quay-omr-v2-host-provision-commands.sh + resources: + requests: + cpu: 100m + memory: 256Mi + timeout: 30m0s + grace_period: 2m0s + env: + - name: OMR_V2_AMI_ID + default: "" + documentation: Optional explicit RHEL 9 AMI ID. The newest official Red Hat RHEL 9 x86_64 gp3 AMI is discovered when empty. + - name: OMR_V2_INSTANCE_TYPE + default: m6i.2xlarge + documentation: EC2 instance type for the dedicated OMR migration host. + - name: OMR_V2_ROOT_VOLUME_SIZE + default: "500" + documentation: Size in GiB of the dedicated host gp3 root volume. + documentation: |- + Provision a dedicated large RHEL 9 host in the disconnected cluster VPC, + authorize the cluster profile SSH key, install rootless Podman prerequisites, + and register its CloudFormation stack for the standard AWS teardown chain. diff --git a/ci-operator/step-registry/quay/omr-v2/install/OWNERS b/ci-operator/step-registry/quay/omr-v2/install/OWNERS new file mode 120000 index 0000000000000..8c272259fbba9 --- /dev/null +++ b/ci-operator/step-registry/quay/omr-v2/install/OWNERS @@ -0,0 +1 @@ +../../OWNERS \ No newline at end of file diff --git a/ci-operator/step-registry/quay/omr-v2/install/quay-omr-v2-install-commands.sh b/ci-operator/step-registry/quay/omr-v2/install/quay-omr-v2-install-commands.sh new file mode 100644 index 0000000000000..18dc4101e5488 --- /dev/null +++ b/ci-operator/step-registry/quay/omr-v2/install/quay-omr-v2-install-commands.sh @@ -0,0 +1,249 @@ +#!/bin/bash + +set -o nounset +set -o errexit +set -o pipefail + +umask 077 +EXIT_CODE=100 +work_dir="" +published=false +declare -a ssh_options=() +declare -a runtime_files=( + "${SHARED_DIR}/mirror_registry_url" + "${SHARED_DIR}/mirror_registry_creds" + "${SHARED_DIR}/mirror_registry_ca.crt" + "${SHARED_DIR}/omr_v2_version" + "${SHARED_DIR}/omr_v2_sha256" +) + +cleanup() { + local status=$? + + trap - EXIT TERM + set +o errexit + if [[ "${status}" -eq 0 ]]; then + EXIT_CODE=0 + fi + printf '%s\n' "${EXIT_CODE}" > "${SHARED_DIR}/install-pre-config-status.txt" + if [[ "${published}" != true ]]; then + rm -f -- "${runtime_files[@]}" + fi + if [[ -n "${work_dir}" && -d "${work_dir}" ]]; then + rm -rf -- "${work_dir}" + fi + exit "${status}" +} + +terminate() { + exit 143 +} + +trap cleanup EXIT +trap terminate TERM + +mkdir -p "${ARTIFACT_DIR}" +rm -f -- "${runtime_files[@]}" + +for command in scp ssh; do + if ! command -v "${command}" >/dev/null 2>&1; then + echo "Required command ${command} is unavailable in the OMR v2 install step image." >&2 + exit 1 + fi +done + +for required_file in \ + "${CLUSTER_PROFILE_DIR}/ssh-privatekey" \ + "${SHARED_DIR}/omr_host_private_address" \ + "${SHARED_DIR}/omr_host_public_address" \ + "${SHARED_DIR}/omr_host_ssh_user"; do + if [[ ! -s "${required_file}" ]]; then + echo "Required OMR v2 install input ${required_file} is missing or empty." >&2 + exit 1 + fi +done + +safe_download_url_re='^https://[A-Za-z0-9./_?&=%:+-]+$' +if [[ ! "${OMR_V2_DOWNLOAD_URL}" =~ ${safe_download_url_re} ]]; then + echo "OMR_V2_DOWNLOAD_URL must be a safely quoted HTTPS URL." >&2 + exit 1 +fi + +host_public=$(<"${SHARED_DIR}/omr_host_public_address") +host_private=$(<"${SHARED_DIR}/omr_host_private_address") +host_user=$(<"${SHARED_DIR}/omr_host_ssh_user") +if [[ ! "${host_public}" =~ ^[A-Za-z0-9][A-Za-z0-9.-]*$ ]] || + [[ ! "${host_private}" =~ ^[A-Za-z0-9][A-Za-z0-9.-]*$ ]] || + [[ ! "${host_user}" =~ ^[A-Za-z_][A-Za-z0-9_-]*$ ]]; then + echo "A dedicated OMR host connection value is invalid." >&2 + exit 1 +fi + +if ! whoami >/dev/null 2>&1; then + if [[ -w /etc/passwd ]]; then + echo "${USER_NAME:-default}:x:$(id -u):0:${USER_NAME:-default} user:${HOME}:/sbin/nologin" >> /etc/passwd + else + echo "/etc/passwd is not writable and the current UID has no passwd entry." >&2 + exit 1 + fi +fi + +ssh_options=( + -o UserKnownHostsFile=/dev/null + -o StrictHostKeyChecking=no + -o "IdentityFile=${CLUSTER_PROFILE_DIR}/ssh-privatekey" + -o ConnectTimeout=10 + -o ConnectionAttempts=3 +) +remote="${host_user}@${host_public}" +remote_work_dir="/home/${host_user}/omr-v2-artifact" +work_dir=$(mktemp -d /tmp/quay-omr-v2-install.XXXXXX) +chmod 0700 "${work_dir}" + +cat > "${work_dir}/install-omr-v2" <<'EOF' +#!/bin/bash +set -o nounset +set -o errexit +set -o pipefail +umask 077 + +download_url="$1" +registry_hostname="$2" +work_dir="$3" +quay_root="/home/$(id -un)/quay-install" +quay_storage="${quay_root}/quay-storage" +sqlite_storage="${quay_root}/sqlite-storage" +archive="${work_dir}/mirror-registry-amd64.tar.gz" + +rm -rf -- "${work_dir}" +install -d -m 0700 "${work_dir}" +curl --fail --location --retry 5 --connect-timeout 30 \ + --output "${archive}" "${download_url}" +sha256sum "${archive}" | awk '{print $1}' > "${work_dir}/sha256" +tar -xzf "${archive}" -C "${work_dir}" +if [[ ! -x "${work_dir}/mirror-registry" ]]; then + echo "The extracted mirror-registry executable is missing." >&2 + exit 1 +fi + +version_output=$("${work_dir}/mirror-registry" --version 2>&1) +printf '%s\n' "${version_output}" > "${work_dir}/version-output" +version=$(printf '%s\n' "${version_output}" | grep -Eo 'v?2\.[0-9]+\.[0-9]+' | \ + head -n 1 | sed 's/^v//' || true) +if [[ -z "${version}" || "${version%%.*}" != 2 ]]; then + echo "The floating mirror-registry artifact is not an identifiable v2 release." >&2 + exit 1 +fi +printf '%s\n' "${version}" > "${work_dir}/version" + +install -d -m 0700 "${quay_root}" "${quay_storage}" "${sqlite_storage}" +openssl rand -hex 24 > "/home/$(id -un)/.omr-v2-admin-password" +chmod 0600 "/home/$(id -un)/.omr-v2-admin-password" +admin_password=$(<"/home/$(id -un)/.omr-v2-admin-password") +install_status=0 +set +o errexit +"${work_dir}/mirror-registry" install \ + --initUser admin \ + --initPassword "${admin_password}" \ + --no-color \ + --quayHostname "${registry_hostname}" \ + --quayRoot "${quay_root}" \ + --quayStorage "${quay_storage}" \ + --sqliteStorage "${sqlite_storage}" \ + -v 2>&1 | \ + tee "${work_dir}/install.log" | \ + sed "s/${admin_password}//g" | \ + tee "${work_dir}/install-sanitized.log" +pipeline_status=("${PIPESTATUS[@]}") +set -o errexit +install_status="${pipeline_status[0]}" +if [[ "${pipeline_status[1]}" -ne 0 || + "${pipeline_status[2]}" -ne 0 || + "${pipeline_status[3]}" -ne 0 ]]; then + echo "Failed to capture the OMR v2 installer diagnostics." >&2 + install_status=1 +fi +chmod 0600 "${work_dir}/install-sanitized.log" +unset admin_password +if [[ "${install_status}" -ne 0 ]]; then + echo "OMR v2 installation failed; the credential-bearing installer log remains on the host." >&2 + exit 1 +fi + +export XDG_RUNTIME_DIR="/run/user/$(id -u)" +for service in quay-app.service quay-redis.service quay-pod.service; do + systemctl --user is-active --quiet "${service}" +done +echo "Checking OMR v2 health at https://${registry_hostname}:8443/health/instance." +curl --retry 20 --retry-delay 3 --retry-all-errors \ + --silent --show-error --fail \ + --cacert "${quay_root}/quay-rootCA/rootCA.pem" \ + "https://${registry_hostname}:8443/health/instance" >/dev/null +rm -f -- "${archive}" +EOF +chmod 0755 "${work_dir}/install-omr-v2" + +scp "${ssh_options[@]}" "${work_dir}/install-omr-v2" \ + "${remote}:/home/${host_user}/install-omr-v2" +install_status=0 +ssh "${ssh_options[@]}" "${remote}" \ + "chmod 0755 '/home/${host_user}/install-omr-v2' && + '/home/${host_user}/install-omr-v2' '${OMR_V2_DOWNLOAD_URL}' '${host_private}' '${remote_work_dir}'" || install_status=$? +scp "${ssh_options[@]}" "${remote}:${remote_work_dir}/install-sanitized.log" \ + "${ARTIFACT_DIR}/mirror-registry-install.log" 2>/dev/null || true +scp "${ssh_options[@]}" "${remote}:${remote_work_dir}/version-output" \ + "${ARTIFACT_DIR}/mirror-registry-version.txt" 2>/dev/null || true +scp "${ssh_options[@]}" "${remote}:${remote_work_dir}/sha256" \ + "${ARTIFACT_DIR}/mirror-registry-sha256.txt" 2>/dev/null || true +if [[ "${install_status}" -ne 0 ]]; then + echo "OMR v2 installation failed with status ${install_status}; available redacted diagnostics were copied to artifacts." >&2 + exit "${install_status}" +fi + +scp "${ssh_options[@]}" "${remote}:${remote_work_dir}/version" "${work_dir}/version" +scp "${ssh_options[@]}" "${remote}:${remote_work_dir}/sha256" "${work_dir}/sha256" +scp "${ssh_options[@]}" \ + "${remote}:/home/${host_user}/quay-install/quay-rootCA/rootCA.pem" \ + "${work_dir}/rootCA.pem" +scp "${ssh_options[@]}" "${remote}:/home/${host_user}/.omr-v2-admin-password" \ + "${work_dir}/admin-password" + +version=$(tr -d '\r\n' < "${work_dir}/version") +archive_sha=$(tr -d '\r\n' < "${work_dir}/sha256") +admin_password=$(tr -d '\r\n' < "${work_dir}/admin-password") +if [[ ! "${version}" =~ ^2\.[0-9]+\.[0-9]+$ ]] || + [[ ! "${archive_sha}" =~ ^[a-f0-9]{64}$ ]] || + [[ -z "${admin_password}" ]]; then + echo "OMR v2 runtime metadata failed validation." >&2 + exit 1 +fi +if ! grep -q '^-----BEGIN CERTIFICATE-----$' "${work_dir}/rootCA.pem" || + ! grep -q '^-----END CERTIFICATE-----$' "${work_dir}/rootCA.pem"; then + echo "The OMR v2 root CA is invalid." >&2 + exit 1 +fi + +ca_tmp=$(mktemp "${SHARED_DIR}/.mirror_registry_ca.crt.XXXXXX") +creds_tmp=$(mktemp "${SHARED_DIR}/.mirror_registry_creds.XXXXXX") +endpoint_tmp=$(mktemp "${SHARED_DIR}/.mirror_registry_url.XXXXXX") +version_tmp=$(mktemp "${SHARED_DIR}/.omr_v2_version.XXXXXX") +sha_tmp=$(mktemp "${SHARED_DIR}/.omr_v2_sha256.XXXXXX") +cp "${work_dir}/rootCA.pem" "${ca_tmp}" +printf 'admin:%s\n' "${admin_password}" > "${creds_tmp}" +printf '%s:8443\n' "${host_private}" > "${endpoint_tmp}" +printf '%s\n' "${version}" > "${version_tmp}" +printf '%s\n' "${archive_sha}" > "${sha_tmp}" +unset admin_password +chmod 0644 "${ca_tmp}" "${endpoint_tmp}" "${version_tmp}" "${sha_tmp}" +chmod 0600 "${creds_tmp}" + +mv -f -- "${ca_tmp}" "${SHARED_DIR}/mirror_registry_ca.crt" +mv -f -- "${creds_tmp}" "${SHARED_DIR}/mirror_registry_creds" +mv -f -- "${version_tmp}" "${SHARED_DIR}/omr_v2_version" +mv -f -- "${sha_tmp}" "${SHARED_DIR}/omr_v2_sha256" +# Publish the endpoint last; downstream steps treat it as the ready marker. +mv -f -- "${endpoint_tmp}" "${SHARED_DIR}/mirror_registry_url" +published=true + +ssh "${ssh_options[@]}" "${remote}" "rm -f -- '/home/${host_user}/install-omr-v2'" +echo "OMR v${version} is healthy on the dedicated RHEL host (archive SHA-256 ${archive_sha})." diff --git a/ci-operator/step-registry/quay/omr-v2/install/quay-omr-v2-install-ref.metadata.json b/ci-operator/step-registry/quay/omr-v2/install/quay-omr-v2-install-ref.metadata.json new file mode 100644 index 0000000000000..0d8a6b2fc298e --- /dev/null +++ b/ci-operator/step-registry/quay/omr-v2/install/quay-omr-v2-install-ref.metadata.json @@ -0,0 +1,11 @@ +{ + "path": "quay/omr-v2/install/quay-omr-v2-install-ref.yaml", + "owners": { + "approvers": [ + "quay-approvers" + ], + "reviewers": [ + "quay-approvers" + ] + } +} \ No newline at end of file diff --git a/ci-operator/step-registry/quay/omr-v2/install/quay-omr-v2-install-ref.yaml b/ci-operator/step-registry/quay/omr-v2/install/quay-omr-v2-install-ref.yaml new file mode 100644 index 0000000000000..dd02b29da35c1 --- /dev/null +++ b/ci-operator/step-registry/quay/omr-v2/install/quay-omr-v2-install-ref.yaml @@ -0,0 +1,22 @@ +ref: + as: quay-omr-v2-install + from_image: + namespace: ci + name: tests-private-base + tag: "4.22" + cli: latest + commands: quay-omr-v2-install-commands.sh + resources: + requests: + cpu: 100m + memory: 256Mi + timeout: 60m0s + grace_period: 5m0s + env: + - name: OMR_V2_DOWNLOAD_URL + default: https://mirror.openshift.com/pub/cgw/mirror-registry/latest/mirror-registry-amd64.tar.gz + documentation: Floating official OMR v2 archive URL. The step rejects any extracted artifact whose semantic major version is not 2. + documentation: |- + Download the official floating OMR v2 artifact on the dedicated RHEL host, + verify that it is still major version 2, install it rootlessly with explicit + storage paths, and atomically publish its endpoint, credentials, and CA. diff --git a/ci-operator/step-registry/quay/omr-v2/seed-migration-sentinel/OWNERS b/ci-operator/step-registry/quay/omr-v2/seed-migration-sentinel/OWNERS new file mode 100644 index 0000000000000..bd932d737f687 --- /dev/null +++ b/ci-operator/step-registry/quay/omr-v2/seed-migration-sentinel/OWNERS @@ -0,0 +1,4 @@ +approvers: +- quay-approvers +reviewers: +- quay-approvers diff --git a/ci-operator/step-registry/quay/omr-v2/seed-migration-sentinel/quay-omr-v2-seed-migration-sentinel-commands.sh b/ci-operator/step-registry/quay/omr-v2/seed-migration-sentinel/quay-omr-v2-seed-migration-sentinel-commands.sh new file mode 100644 index 0000000000000..01597fe927722 --- /dev/null +++ b/ci-operator/step-registry/quay/omr-v2/seed-migration-sentinel/quay-omr-v2-seed-migration-sentinel-commands.sh @@ -0,0 +1,151 @@ +#!/bin/bash + +set -o nounset +set -o errexit +set -o pipefail + +umask 077 +work_dir="" +shared_tmp_dir="" + +cleanup() { + local status=$? + + trap - EXIT TERM + set +o errexit + if [[ -n "${shared_tmp_dir}" && -d "${shared_tmp_dir}" ]]; then + rm -rf -- "${shared_tmp_dir}" + fi + if [[ -n "${work_dir}" && -d "${work_dir}" ]]; then + rm -rf -- "${work_dir}" + fi + exit "${status}" +} + +terminate() { + exit 143 +} + +trap cleanup EXIT +trap terminate TERM + +mkdir -p "${ARTIFACT_DIR}" +exec > >(tee "${ARTIFACT_DIR}/omr-v2-migration-sentinel.log") 2>&1 + +if [[ -f "${SHARED_DIR}/proxy-conf.sh" ]]; then + # shellcheck disable=SC1091 + source "${SHARED_DIR}/proxy-conf.sh" +fi + +for command in base64 jq oc tar; do + if ! command -v "${command}" >/dev/null 2>&1; then + echo "Required command ${command} is unavailable in the sentinel seed image." >&2 + exit 1 + fi +done + +for required_file in \ + "${SHARED_DIR}/mirror_registry_ca.crt" \ + "${SHARED_DIR}/mirror_registry_creds" \ + "${SHARED_DIR}/mirror_registry_url" \ + "${SHARED_DIR}/omr_mirrored_cli_image"; do + if [[ ! -s "${required_file}" ]]; then + echo "Required sentinel seed input ${required_file} is missing or empty." >&2 + exit 1 + fi +done + +: "${UNIQUE_HASH:?UNIQUE_HASH is required}" +if [[ ! "${UNIQUE_HASH}" =~ ^[A-Za-z0-9]+$ ]]; then + echo "UNIQUE_HASH contains unexpected characters." >&2 + exit 1 +fi + +if ! whoami >/dev/null 2>&1; then + if [[ -w /etc/passwd ]]; then + echo "${USER_NAME:-default}:x:$(id -u):0:${USER_NAME:-default} user:${HOME}:/sbin/nologin" >> /etc/passwd + else + echo "/etc/passwd is not writable and the current UID has no passwd entry." >&2 + exit 1 + fi +fi + +registry_host=$(tr -d '\r\n' < "${SHARED_DIR}/mirror_registry_url") +base_image=$(tr -d '\r\n' < "${SHARED_DIR}/omr_mirrored_cli_image") +if [[ -z "${registry_host}" || "${registry_host}" == */* || + "${registry_host}" =~ [[:space:]] ]]; then + echo "The runtime OMR endpoint is invalid." >&2 + exit 1 +fi +if [[ "${base_image}" != "${registry_host}/"* || + ! "${base_image}" =~ @sha256:[a-f0-9]{64}$ ]]; then + echo "The mirrored CLI image is not an OMR digest pullspec: ${base_image}" >&2 + exit 1 +fi + +work_dir=$(mktemp -d /tmp/quay-omr-v2-sentinel.XXXXXX) +chmod 0700 "${work_dir}" +auth_file="${work_dir}/registry-auth.json" + +registry_credentials=$(tr -d '\r\n' < "${SHARED_DIR}/mirror_registry_creds") +if [[ -z "${registry_credentials}" || "${registry_credentials}" != *:* ]]; then + echo "The runtime OMR credential is invalid." >&2 + exit 1 +fi +registry_auth=$(printf '%s' "${registry_credentials}" | base64 -w 0) +unset registry_credentials +jq -n --arg host "${registry_host}" --arg auth "${registry_auth}" ' + {auths: {($host): {auth: $auth}}} +' > "${auth_file}" +unset registry_auth +chmod 0600 "${auth_file}" + +sentinel_repository="${registry_host}/admin/omr-migration-sentinel" +sentinel_tag="${sentinel_repository}:pre-${UNIQUE_HASH}" +sentinel_marker="omr-v2-${UNIQUE_HASH}" +rootfs="${work_dir}/rootfs" +layer_archive="${work_dir}/sentinel-layer.tar.gz" +install -d -m 0755 "${rootfs}/omr-sentinel" +printf '%s\n' "${sentinel_marker}" > "${rootfs}/omr-sentinel/pre-migration-marker" +chmod 0644 "${rootfs}/omr-sentinel/pre-migration-marker" +tar --create --gzip --file "${layer_archive}" --directory "${rootfs}" . + +oc image append \ + --from="${base_image}" \ + --to="${sentinel_tag}" \ + --registry-config="${auth_file}" \ + --certificate-authority="${SHARED_DIR}/mirror_registry_ca.crt" \ + "${layer_archive}" + +sentinel_info="${ARTIFACT_DIR}/omr-v2-migration-sentinel-image.json" +oc image info "${sentinel_tag}" \ + --registry-config="${auth_file}" \ + --certificate-authority="${SHARED_DIR}/mirror_registry_ca.crt" \ + -o json > "${sentinel_info}" +sentinel_digest=$(jq -er ' + .digest | select(test("^sha256:[a-f0-9]{64}$")) +' "${sentinel_info}") +sentinel_image="${sentinel_repository}@${sentinel_digest}" + +rm -f -- \ + "${SHARED_DIR}/omr_migration_sentinel_digest" \ + "${SHARED_DIR}/omr_migration_sentinel_image" \ + "${SHARED_DIR}/omr_migration_sentinel_marker" \ + "${SHARED_DIR}/omr_migration_sentinel_tag" +shared_tmp_dir=$(mktemp -d "${SHARED_DIR}/.omr-migration-sentinel.XXXXXX") +printf '%s\n' "${sentinel_digest}" > "${shared_tmp_dir}/digest" +printf '%s\n' "${sentinel_image}" > "${shared_tmp_dir}/image" +printf '%s\n' "${sentinel_marker}" > "${shared_tmp_dir}/marker" +printf '%s\n' "${sentinel_tag}" > "${shared_tmp_dir}/tag" +chmod 0644 "${shared_tmp_dir}/digest" "${shared_tmp_dir}/image" \ + "${shared_tmp_dir}/marker" "${shared_tmp_dir}/tag" +mv -f -- "${shared_tmp_dir}/digest" "${SHARED_DIR}/omr_migration_sentinel_digest" +mv -f -- "${shared_tmp_dir}/marker" "${SHARED_DIR}/omr_migration_sentinel_marker" +mv -f -- "${shared_tmp_dir}/tag" "${SHARED_DIR}/omr_migration_sentinel_tag" +# Publish the immutable pullspec last so the post-migration step cannot consume +# partial sentinel metadata. +mv -f -- "${shared_tmp_dir}/image" "${SHARED_DIR}/omr_migration_sentinel_image" +rmdir "${shared_tmp_dir}" +shared_tmp_dir="" + +echo "Seeded OMR v2 migration sentinel ${sentinel_image}." diff --git a/ci-operator/step-registry/quay/omr-v2/seed-migration-sentinel/quay-omr-v2-seed-migration-sentinel-ref.metadata.json b/ci-operator/step-registry/quay/omr-v2/seed-migration-sentinel/quay-omr-v2-seed-migration-sentinel-ref.metadata.json new file mode 100644 index 0000000000000..bf9bc65bbfb27 --- /dev/null +++ b/ci-operator/step-registry/quay/omr-v2/seed-migration-sentinel/quay-omr-v2-seed-migration-sentinel-ref.metadata.json @@ -0,0 +1,11 @@ +{ + "path": "quay/omr-v2/seed-migration-sentinel/quay-omr-v2-seed-migration-sentinel-ref.yaml", + "owners": { + "approvers": [ + "quay-approvers" + ], + "reviewers": [ + "quay-approvers" + ] + } +} \ No newline at end of file diff --git a/ci-operator/step-registry/quay/omr-v2/seed-migration-sentinel/quay-omr-v2-seed-migration-sentinel-ref.yaml b/ci-operator/step-registry/quay/omr-v2/seed-migration-sentinel/quay-omr-v2-seed-migration-sentinel-ref.yaml new file mode 100644 index 0000000000000..64921e3903243 --- /dev/null +++ b/ci-operator/step-registry/quay/omr-v2/seed-migration-sentinel/quay-omr-v2-seed-migration-sentinel-ref.yaml @@ -0,0 +1,14 @@ +ref: + as: quay-omr-v2-seed-migration-sentinel + from: upi-installer + commands: quay-omr-v2-seed-migration-sentinel-commands.sh + resources: + requests: + cpu: 100m + memory: 256Mi + timeout: 20m0s + grace_period: 2m0s + documentation: |- + Append a unique marker layer to the mirrored CLI image, push it into a + dedicated repository through OMR v2, and publish its tag and digest for + post-migration persistence validation. diff --git a/ci-operator/step-registry/quay/omr-v2/validate/OWNERS b/ci-operator/step-registry/quay/omr-v2/validate/OWNERS new file mode 120000 index 0000000000000..8c272259fbba9 --- /dev/null +++ b/ci-operator/step-registry/quay/omr-v2/validate/OWNERS @@ -0,0 +1 @@ +../../OWNERS \ No newline at end of file diff --git a/ci-operator/step-registry/quay/omr-v2/validate/quay-omr-v2-validate-commands.sh b/ci-operator/step-registry/quay/omr-v2/validate/quay-omr-v2-validate-commands.sh new file mode 100644 index 0000000000000..4e3ff71d7a6fa --- /dev/null +++ b/ci-operator/step-registry/quay/omr-v2/validate/quay-omr-v2-validate-commands.sh @@ -0,0 +1,154 @@ +#!/bin/bash + +set -o nounset +set -o errexit +set -o pipefail + +mkdir -p "${ARTIFACT_DIR}" +exec > >(tee "${ARTIFACT_DIR}/omr-v2-validation.log") 2>&1 + +if [[ -f "${SHARED_DIR}/proxy-conf.sh" ]]; then + # shellcheck disable=SC1091 + source "${SHARED_DIR}/proxy-conf.sh" +fi + +validate_idms_mappings() { + local idms_json="$1" + local mirror_host="$2" + local release_image="$3" + local release_source="${release_image%@*}" + local source + local -a sources=( + "${release_source}" + quay.io/openshift-release-dev/ocp-v4.0-art-dev + ) + + if [[ "${release_source}" == "${release_image}" ]]; then + echo "Release image does not contain a digest: ${release_image}" + return 1 + fi + + for source in "${sources[@]}"; do + if ! jq -e --arg source "${source}" --arg mirror "${mirror_host}/" ' + any(.items[].spec.imageDigestMirrors[]?; + .source == $source and any(.mirrors[]?; startswith($mirror))) + ' "${idms_json}" >/dev/null; then + echo "IDMS does not map ${source} to ${mirror_host}." + return 1 + fi + done +} + +required_files=( + mirror_registry_url + oc-mirror-signature-configmap.json + omr_host_public_address + omr_host_ssh_user + omr_mirror_completed_at + omr_mirror_repository + omr_mirrored_cli_image + omr_v2_version +) +for name in "${required_files[@]}"; do + if [[ ! -s "${SHARED_DIR}/${name}" ]]; then + echo "Required shared file is missing or empty: ${name}" + exit 1 + fi +done + +omr_v2_version=$(<"${SHARED_DIR}/omr_v2_version") +if [[ ! "${omr_v2_version}" =~ ^2\.[0-9]+\.[0-9]+$ ]]; then + echo "The installed mirror-registry artifact is not a v2 release: ${omr_v2_version}" + exit 1 +fi + +signature_configmap="${SHARED_DIR}/oc-mirror-signature-configmap.json" +if ! jq -e ' + .apiVersion == "v1" + and .kind == "ConfigMap" + and .metadata.name == "mirrored-release-signatures" + and .metadata.namespace == "openshift-config-managed" + and (.binaryData | type == "object" and length > 0) +' "${signature_configmap}" >/dev/null; then + echo "The generated oc-mirror release signature ConfigMap is invalid." + exit 1 +fi +oc apply -f "${signature_configmap}" + +oc wait clusterversion/version --for=condition=Available --timeout=10m +oc get clusterversion version -o wide +oc get clusteroperators +degraded=$(oc get clusteroperators -o json | jq -r ' + [.items[] | select(any(.status.conditions[]?; .type == "Degraded" and .status == "True")) | .metadata.name] + | join(" ") +') +if [[ -n "${degraded}" ]]; then + echo "Degraded ClusterOperators: ${degraded}" + exit 1 +fi +oc wait nodes --all --for=condition=Ready --timeout=10m +oc get nodes -o wide + +idms_json="${ARTIFACT_DIR}/image-digest-mirror-sets-v2.json" +oc get imagedigestmirrorsets -o json > "${idms_json}" +mirror_host=$(<"${SHARED_DIR}/mirror_registry_url") +validate_idms_mappings \ + "${idms_json}" \ + "${mirror_host}" \ + "${OPENSHIFT_INSTALL_RELEASE_IMAGE_OVERRIDE}" + +if ! whoami >/dev/null 2>&1; then + if [[ -w /etc/passwd ]]; then + echo "${USER_NAME:-default}:x:$(id -u):0:${USER_NAME:-default} user:${HOME}:/sbin/nologin" >> /etc/passwd + else + echo "/etc/passwd is not writable and the current UID has no passwd entry." + exit 1 + fi +fi + +host_address=$(<"${SHARED_DIR}/omr_host_public_address") +host_user=$(<"${SHARED_DIR}/omr_host_ssh_user") +ssh_options=( + -o UserKnownHostsFile=/dev/null + -o StrictHostKeyChecking=no + -o "IdentityFile=${CLUSTER_PROFILE_DIR}/ssh-privatekey" +) +remote="${host_user}@${host_address}" +ssh "${ssh_options[@]}" "${remote}" \ + 'export XDG_RUNTIME_DIR="/run/user/$(id -u)"; systemctl --user is-active --quiet quay-app.service quay-redis.service quay-pod.service' + +smoke_namespace=quay-omr-v2-smoke +smoke_pod=omr-v2-mirrored-cli-smoke +smoke_image=$(<"${SHARED_DIR}/omr_mirrored_cli_image") +oc create namespace "${smoke_namespace}" \ + --dry-run=client -o yaml | oc apply -f - +trap 'oc delete namespace "${smoke_namespace}" --ignore-not-found --wait=false >/dev/null 2>&1 || true' EXIT +oc -n "${smoke_namespace}" delete pod "${smoke_pod}" --ignore-not-found --wait=true +oc -n "${smoke_namespace}" run "${smoke_pod}" --image="${smoke_image}" \ + --image-pull-policy=Always --restart=Never --command -- oc version --client=true +if ! oc -n "${smoke_namespace}" wait pod/"${smoke_pod}" \ + --for=jsonpath='{.status.phase}'=Succeeded --timeout=5m; then + oc -n "${smoke_namespace}" describe pod "${smoke_pod}" + oc -n "${smoke_namespace}" logs "${smoke_pod}" || true + exit 1 +fi +oc -n "${smoke_namespace}" get pod "${smoke_pod}" -o wide +oc -n "${smoke_namespace}" logs "${smoke_pod}" | tee "${ARTIFACT_DIR}/mirrored-cli-smoke-v2.log" + +mirror_completed_at=$(<"${SHARED_DIR}/omr_mirror_completed_at") +mirror_repository=$(<"${SHARED_DIR}/omr_mirror_repository") +repository_path="${mirror_repository#*/}" +journal_file="${ARTIFACT_DIR}/quay-v2-journal-after-mirror.log" +if ssh "${ssh_options[@]}" "${remote}" \ + "export XDG_RUNTIME_DIR=/run/user/\$(id -u); journalctl --user -u quay-app.service --since '${mirror_completed_at}' --no-pager --output=short-iso" \ + > "${journal_file}"; then + if grep -F "/v2/${repository_path}/" "${journal_file}" >/dev/null; then + echo "The OMR v2 journal recorded a request for /v2/${repository_path}/ after mirror completion." + else + echo "The OMR v2 journal did not record a request for /v2/${repository_path}/; the mirrored image pull succeeded." + fi +else + echo "Unable to collect the OMR v2 journal; the mirrored image pull succeeded." +fi + +echo "OMR v${omr_v2_version} disconnected installation validation passed." diff --git a/ci-operator/step-registry/quay/omr-v2/validate/quay-omr-v2-validate-ref.metadata.json b/ci-operator/step-registry/quay/omr-v2/validate/quay-omr-v2-validate-ref.metadata.json new file mode 100644 index 0000000000000..bf5ea6f763221 --- /dev/null +++ b/ci-operator/step-registry/quay/omr-v2/validate/quay-omr-v2-validate-ref.metadata.json @@ -0,0 +1,11 @@ +{ + "path": "quay/omr-v2/validate/quay-omr-v2-validate-ref.yaml", + "owners": { + "approvers": [ + "quay-approvers" + ], + "reviewers": [ + "quay-approvers" + ] + } +} \ No newline at end of file diff --git a/ci-operator/step-registry/quay/omr-v2/validate/quay-omr-v2-validate-ref.yaml b/ci-operator/step-registry/quay/omr-v2/validate/quay-omr-v2-validate-ref.yaml new file mode 100644 index 0000000000000..da9e6d1bdf953 --- /dev/null +++ b/ci-operator/step-registry/quay/omr-v2/validate/quay-omr-v2-validate-ref.yaml @@ -0,0 +1,17 @@ +ref: + as: quay-omr-v2-validate + from: upi-installer + commands: quay-omr-v2-validate-commands.sh + resources: + requests: + cpu: 100m + memory: 256Mi + timeout: 20m0s + grace_period: 2m0s + dependencies: + - name: "release:latest" + env: OPENSHIFT_INSTALL_RELEASE_IMAGE_OVERRIDE + documentation: |- + Prove that the disconnected cluster installed through OMR v2 is healthy, + uses the expected digest mirror mappings, can pull the mirrored CLI digest, + and recorded that pull while all rootless v2 services are active. diff --git a/ci-operator/step-registry/quay/omr-v3/gather/quay-omr-v3-gather-commands.sh b/ci-operator/step-registry/quay/omr-v3/gather/quay-omr-v3-gather-commands.sh index 95ebba3a26774..69c603ff38c28 100644 --- a/ci-operator/step-registry/quay/omr-v3/gather/quay-omr-v3-gather-commands.sh +++ b/ci-operator/step-registry/quay/omr-v3/gather/quay-omr-v3-gather-commands.sh @@ -5,25 +5,69 @@ set -o pipefail mkdir -p "${ARTIFACT_DIR}" -if [[ ! -s "${SHARED_DIR}/bastion_public_address" || ! -s "${SHARED_DIR}/bastion_ssh_user" ]]; then - echo "Bastion connection details are unavailable; skipping OMR diagnostics." +# Provisioning diagnostics do not require a reachable host. This also captures +# CloudFormation failures that happen before connection files can be published. +if [[ -s "${CLUSTER_PROFILE_DIR}/.awscred" ]] && command -v aws >/dev/null 2>&1; then + export AWS_SHARED_CREDENTIALS_FILE="${CLUSTER_PROFILE_DIR}/.awscred" + region="${REGION:-$LEASED_RESOURCE}" + stack_name="${NAMESPACE}-${UNIQUE_HASH}-omr2" + if aws --region "${region}" cloudformation describe-stacks \ + --stack-name "${stack_name}" \ + > "${ARTIFACT_DIR}/omr-host-stack.json" 2>/dev/null; then + aws --region "${region}" cloudformation describe-stack-events \ + --stack-name "${stack_name}" \ + > "${ARTIFACT_DIR}/omr-host-stack-events.json" 2>&1 || true + aws --region "${region}" cloudformation describe-stack-resources \ + --stack-name "${stack_name}" \ + > "${ARTIFACT_DIR}/omr-host-stack-resources.json" 2>&1 || true + fi + if [[ -s "${SHARED_DIR}/omr_host_instance_id" ]]; then + instance_id=$(<"${SHARED_DIR}/omr_host_instance_id") + if [[ "${instance_id}" =~ ^i-[a-f0-9]+$ ]]; then + aws --region "${region}" ec2 describe-instances \ + --instance-ids "${instance_id}" \ + > "${ARTIFACT_DIR}/omr-host-instance.json" 2>&1 || true + aws --region "${region}" ec2 describe-instance-status \ + --include-all-instances --instance-ids "${instance_id}" \ + > "${ARTIFACT_DIR}/omr-host-instance-status.json" 2>&1 || true + aws --region "${region}" ec2 get-console-output --latest \ + --instance-id "${instance_id}" \ + > "${ARTIFACT_DIR}/omr-host-console-output.json" 2>&1 || true + fi + fi +fi + +rootless=false +host_address_file="${SHARED_DIR}/bastion_public_address" +host_private_file="${SHARED_DIR}/bastion_private_address" +host_user_file="${SHARED_DIR}/bastion_ssh_user" +if [[ -s "${SHARED_DIR}/omr_host_public_address" || + -s "${SHARED_DIR}/omr_host_ssh_user" ]]; then + rootless=true + host_address_file="${SHARED_DIR}/omr_host_public_address" + host_private_file="${SHARED_DIR}/omr_host_private_address" + host_user_file="${SHARED_DIR}/omr_host_ssh_user" +fi + +if [[ ! -s "${host_address_file}" || ! -s "${host_user_file}" ]]; then + echo "OMR host connection details are unavailable; skipping OMR diagnostics." exit 0 fi -if ! whoami &> /dev/null; then +if ! whoami >/dev/null 2>&1; then if [[ -w /etc/passwd ]]; then echo "${USER_NAME:-default}:x:$(id -u):0:${USER_NAME:-default} user:${HOME}:/sbin/nologin" >> /etc/passwd else - echo "Current uid has no passwd entry; skipping OMR diagnostics." + echo "Current UID has no passwd entry; skipping OMR diagnostics." exit 0 fi fi -bastion_address=$(<"${SHARED_DIR}/bastion_public_address") -bastion_user=$(<"${SHARED_DIR}/bastion_ssh_user") -bastion_private_dns="" -if [[ -s "${SHARED_DIR}/bastion_private_address" ]]; then - bastion_private_dns=$(<"${SHARED_DIR}/bastion_private_address") +host_address=$(<"${host_address_file}") +host_user=$(<"${host_user_file}") +host_private="" +if [[ -s "${host_private_file}" ]]; then + host_private=$(<"${host_private_file}") fi ssh_options=( -o ConnectTimeout=30 @@ -31,31 +75,61 @@ ssh_options=( -o StrictHostKeyChecking=no -o "IdentityFile=${CLUSTER_PROFILE_DIR}/ssh-privatekey" ) +remote="${host_user}@${host_address}" -remote="${bastion_user}@${bastion_address}" -ssh "${ssh_options[@]}" "${remote}" \ - "sudo systemctl status quay.service --no-pager --full" \ - > "${ARTIFACT_DIR}/quay-service-status.log" 2>&1 || true -ssh "${ssh_options[@]}" "${remote}" \ - "sudo journalctl -u quay.service --no-pager --output=short-iso" \ - > "${ARTIFACT_DIR}/quay-service-journal.log" 2>&1 || true -ssh "${ssh_options[@]}" "${remote}" \ - "sudo podman ps --all --no-trunc" \ - > "${ARTIFACT_DIR}/quay-podman-ps.log" 2>&1 || true -ssh "${ssh_options[@]}" "${remote}" \ - "sudo podman inspect quay systemd-quay" \ - > "${ARTIFACT_DIR}/quay-podman-inspect.log" 2>&1 || true -ssh "${ssh_options[@]}" "${remote}" \ - "sudo podman logs quay 2>/dev/null || sudo podman logs systemd-quay 2>/dev/null" \ - > "${ARTIFACT_DIR}/quay-container.log" 2>&1 || true -ssh "${ssh_options[@]}" "${remote}" \ - "sudo ss -lntp; echo; df -h; echo; sudo du -sh /var/lib/quay" \ - > "${ARTIFACT_DIR}/quay-host-resources.log" 2>&1 || true - -if [[ -n "${bastion_private_dns}" ]]; then - ssh "${ssh_options[@]}" "${remote}" \ - "sudo curl --silent --show-error --fail --cacert /var/lib/quay/ssl.cert 'https://${bastion_private_dns}:8443/healthz'" \ - > "${ARTIFACT_DIR}/quay-health.log" 2>&1 || true +if [[ "${rootless}" == true ]]; then + ssh "${ssh_options[@]}" "${remote}" \ + 'sudo cloud-init status --long; echo; sudo tail -n 500 /var/log/cloud-init-output.log' \ + > "${ARTIFACT_DIR}/omr-host-cloud-init.log" 2>&1 || true + ssh "${ssh_options[@]}" "${remote}" \ + 'export XDG_RUNTIME_DIR="/run/user/$(id -u)"; systemctl --user status quay.service quay-app.service quay-redis.service quay-pod.service --no-pager --full' \ + > "${ARTIFACT_DIR}/quay-user-service-status.log" 2>&1 || true + ssh "${ssh_options[@]}" "${remote}" \ + 'export XDG_RUNTIME_DIR="/run/user/$(id -u)"; journalctl --user -u quay.service -u quay-app.service -u quay-redis.service -u quay-pod.service --no-pager --output=short-iso' \ + > "${ARTIFACT_DIR}/quay-user-service-journal.log" 2>&1 || true + ssh "${ssh_options[@]}" "${remote}" \ + 'export XDG_RUNTIME_DIR="/run/user/$(id -u)"; podman ps --all --no-trunc; echo; podman volume ls; echo; podman images --digests --no-trunc' \ + > "${ARTIFACT_DIR}/quay-podman-state.log" 2>&1 || true + ssh "${ssh_options[@]}" "${remote}" \ + 'export XDG_RUNTIME_DIR="/run/user/$(id -u)"; for container in quay systemd-quay quay-app quay-redis; do podman inspect --format "Name={{.Name}} Image={{.ImageName}} State={{json .State}} Mounts={{json .Mounts}}" "${container}" 2>/dev/null || true; done' \ + > "${ARTIFACT_DIR}/quay-podman-inspect.log" 2>&1 || true + ssh "${ssh_options[@]}" "${remote}" \ + 'export XDG_RUNTIME_DIR="/run/user/$(id -u)"; for container in quay systemd-quay quay-app quay-redis; do echo "### ${container}"; podman logs "${container}" 2>/dev/null || true; done' \ + > "${ARTIFACT_DIR}/quay-container.log" 2>&1 || true + ssh "${ssh_options[@]}" "${remote}" \ + 'sudo ss -lntp; echo; df -h; echo; du -sh "$HOME/quay-install" "$HOME/quay-v3" 2>/dev/null || true' \ + > "${ARTIFACT_DIR}/quay-host-resources.log" 2>&1 || true + + if [[ -n "${host_private}" ]]; then + ssh "${ssh_options[@]}" "${remote}" \ + "curl --silent --show-error --fail --cacert '/home/${host_user}/quay-install/quay-rootCA/rootCA.pem' 'https://${host_private}:8443/healthz'" \ + > "${ARTIFACT_DIR}/quay-health.log" 2>&1 || true + fi +else + ssh "${ssh_options[@]}" "${remote}" \ + "sudo systemctl status quay.service --no-pager --full" \ + > "${ARTIFACT_DIR}/quay-service-status.log" 2>&1 || true + ssh "${ssh_options[@]}" "${remote}" \ + "sudo journalctl -u quay.service --no-pager --output=short-iso" \ + > "${ARTIFACT_DIR}/quay-service-journal.log" 2>&1 || true + ssh "${ssh_options[@]}" "${remote}" \ + "sudo podman ps --all --no-trunc" \ + > "${ARTIFACT_DIR}/quay-podman-state.log" 2>&1 || true + ssh "${ssh_options[@]}" "${remote}" \ + "sudo podman inspect --format 'Name={{.Name}} Image={{.ImageName}} State={{json .State}} Mounts={{json .Mounts}}' quay systemd-quay" \ + > "${ARTIFACT_DIR}/quay-podman-inspect.log" 2>&1 || true + ssh "${ssh_options[@]}" "${remote}" \ + "sudo podman logs quay 2>/dev/null || sudo podman logs systemd-quay 2>/dev/null" \ + > "${ARTIFACT_DIR}/quay-container.log" 2>&1 || true + ssh "${ssh_options[@]}" "${remote}" \ + "sudo ss -lntp; echo; df -h; echo; sudo du -sh /var/lib/quay" \ + > "${ARTIFACT_DIR}/quay-host-resources.log" 2>&1 || true + + if [[ -n "${host_private}" ]]; then + ssh "${ssh_options[@]}" "${remote}" \ + "sudo curl --silent --show-error --fail --cacert /var/lib/quay/ssl.cert 'https://${host_private}:8443/healthz'" \ + > "${ARTIFACT_DIR}/quay-health.log" 2>&1 || true + fi fi echo "Best-effort OMR diagnostics collection finished." diff --git a/ci-operator/step-registry/quay/omr-v3/gather/quay-omr-v3-gather-ref.yaml b/ci-operator/step-registry/quay/omr-v3/gather/quay-omr-v3-gather-ref.yaml index bc656d0d47464..430300e968ec5 100644 --- a/ci-operator/step-registry/quay/omr-v3/gather/quay-omr-v3-gather-ref.yaml +++ b/ci-operator/step-registry/quay/omr-v3/gather/quay-omr-v3-gather-ref.yaml @@ -13,6 +13,8 @@ ref: timeout: 10m0s grace_period: 1m0s documentation: |- - Best-effort collection of OMR service, container, socket, disk, and health - diagnostics from the standard bastion. The step does not read credentials, - pull secrets, SSH keys, kubeconfig, or the complete shared directory. + Best-effort collection of OMR AWS provisioning, cloud-init, service, + container, socket, disk, and health diagnostics. A dedicated rootless host + is preferred when present, including both v2 and v3 user services; the + standard bastion remains supported. The step does not read credentials, + sensitive config, or the complete shared directory. diff --git a/ci-operator/step-registry/quay/omr-v3/mirror/quay-omr-v3-mirror-commands.sh b/ci-operator/step-registry/quay/omr-v3/mirror/quay-omr-v3-mirror-commands.sh index f7d051e642d43..a1130fb4aacbb 100755 --- a/ci-operator/step-registry/quay/omr-v3/mirror/quay-omr-v3-mirror-commands.sh +++ b/ci-operator/step-registry/quay/omr-v3/mirror/quay-omr-v3-mirror-commands.sh @@ -288,12 +288,20 @@ for command in base64 curl jq oc scp sha256sum ssh tar tee yq-v4; do fi done +remote_address_file="${SHARED_DIR}/bastion_private_address" +remote_user_file="${SHARED_DIR}/bastion_ssh_user" +if [[ -s "${SHARED_DIR}/omr_host_public_address" || + -s "${SHARED_DIR}/omr_host_ssh_user" ]]; then + remote_address_file="${SHARED_DIR}/omr_host_public_address" + remote_user_file="${SHARED_DIR}/omr_host_ssh_user" +fi + for required_file in \ "${SHARED_DIR}/mirror_registry_url" \ "${SHARED_DIR}/mirror_registry_creds" \ "${SHARED_DIR}/mirror_registry_ca.crt" \ - "${SHARED_DIR}/bastion_private_address" \ - "${SHARED_DIR}/bastion_ssh_user" \ + "${remote_address_file}" \ + "${remote_user_file}" \ "${CLUSTER_PROFILE_DIR}/pull-secret" \ "${CLUSTER_PROFILE_DIR}/ssh-privatekey"; do if [[ ! -s "${required_file}" ]]; then @@ -381,11 +389,12 @@ if ! whoami > /dev/null 2>&1; then fi fi -BASTION_IP=$(<"${SHARED_DIR}/bastion_private_address") -if [[ -s "${SHARED_DIR}/bastion_public_address" ]]; then +BASTION_IP=$(<"${remote_address_file}") +if [[ "${remote_address_file}" == "${SHARED_DIR}/bastion_private_address" && + -s "${SHARED_DIR}/bastion_public_address" ]]; then BASTION_IP=$(<"${SHARED_DIR}/bastion_public_address") fi -BASTION_SSH_USER=$(<"${SHARED_DIR}/bastion_ssh_user") +BASTION_SSH_USER=$(<"${remote_user_file}") if [[ ! "${BASTION_IP}" =~ ^[A-Za-z0-9][A-Za-z0-9.-]*$ ]] || [[ ! "${BASTION_SSH_USER}" =~ ^[A-Za-z_][A-Za-z0-9_-]*$ ]] || [[ ! "${UNIQUE_HASH}" =~ ^[A-Za-z0-9]+$ ]]; then @@ -528,7 +537,8 @@ omr_cli_tmp=$(mktemp "${SHARED_DIR}/.omr_mirrored_cli_image.XXXXXX") omr_completed_tmp=$(mktemp "${SHARED_DIR}/.omr_mirror_completed_at.XXXXXX") printf '%s\n' "${component_mirror}" > "${omr_repository_tmp}" printf '%s@%s\n' "${component_mirror}" "${cli_digest}" > "${omr_cli_tmp}" -date -u +%Y-%m-%dT%H:%M:%SZ > "${omr_completed_tmp}" +# Older systemd versions reject RFC 3339's T/Z form in journalctl --since. +date -u '+%Y-%m-%d %H:%M:%S UTC' > "${omr_completed_tmp}" chmod 0644 \ "${install_patch_tmp}" \ "${signature_json_tmp}" \ diff --git a/ci-operator/step-registry/quay/omr-v3/mirror/quay-omr-v3-mirror-ref.yaml b/ci-operator/step-registry/quay/omr-v3/mirror/quay-omr-v3-mirror-ref.yaml index 27d10beda8859..987027d31302d 100644 --- a/ci-operator/step-registry/quay/omr-v3/mirror/quay-omr-v3-mirror-ref.yaml +++ b/ci-operator/step-registry/quay/omr-v3/mirror/quay-omr-v3-mirror-ref.yaml @@ -13,6 +13,7 @@ ref: env: OPENSHIFT_INSTALL_RELEASE_IMAGE_OVERRIDE documentation: |- Download and verify the latest oc-mirror client, mirror the selected - OpenShift release into runtime OMR v3 from the AWS bastion with oc-mirror - v2, and publish generated installer mappings with the ci-operator release - source alias, release signatures, and atomic validation proofs. + OpenShift release into the runtime OMR host with oc-mirror v2, preferring a + dedicated OMR host when one is provisioned and otherwise using the AWS + bastion. Publish installer mappings, release signatures, and validation + proofs atomically. diff --git a/ci-operator/step-registry/quay/omr-v3/upgrade-from-v2/OWNERS b/ci-operator/step-registry/quay/omr-v3/upgrade-from-v2/OWNERS new file mode 120000 index 0000000000000..8c272259fbba9 --- /dev/null +++ b/ci-operator/step-registry/quay/omr-v3/upgrade-from-v2/OWNERS @@ -0,0 +1 @@ +../../OWNERS \ No newline at end of file diff --git a/ci-operator/step-registry/quay/omr-v3/upgrade-from-v2/quay-omr-v3-upgrade-from-v2-commands.sh b/ci-operator/step-registry/quay/omr-v3/upgrade-from-v2/quay-omr-v3-upgrade-from-v2-commands.sh new file mode 100644 index 0000000000000..28c8cc5aaa6f5 --- /dev/null +++ b/ci-operator/step-registry/quay/omr-v3/upgrade-from-v2/quay-omr-v3-upgrade-from-v2-commands.sh @@ -0,0 +1,204 @@ +#!/bin/bash + +set -o nounset +set -o errexit +set -o pipefail + +umask 077 +work_dir="" +remote_work_dir="" +remote_ready=false +declare -a ssh_options=() + +cleanup() { + local status=$? + + trap - EXIT TERM + set +o errexit + if [[ "${remote_ready}" == true ]]; then + ssh "${ssh_options[@]}" "${host_user}@${host_address}" \ + "rm -rf -- '${remote_work_dir}'" >/dev/null 2>&1 || true + fi + if [[ -n "${work_dir}" && -d "${work_dir}" ]]; then + rm -rf -- "${work_dir}" + fi + exit "${status}" +} + +terminate() { + exit 143 +} + +trap cleanup EXIT +trap terminate TERM + +mkdir -p "${ARTIFACT_DIR}" +: "${OMR_IMAGE:?OMR_IMAGE must identify the PR-built quay-mirror image}" + +for command in oc scp sha256sum skopeo ssh; do + if ! command -v "${command}" >/dev/null 2>&1; then + echo "Required command ${command} is unavailable in the migration step image." >&2 + exit 1 + fi +done + +for required_file in \ + "${CLUSTER_PROFILE_DIR}/ssh-privatekey" \ + "${SHARED_DIR}/mirror_registry_ca.crt" \ + "${SHARED_DIR}/mirror_registry_creds" \ + "${SHARED_DIR}/mirror_registry_url" \ + "${SHARED_DIR}/omr_host_public_address" \ + "${SHARED_DIR}/omr_host_ssh_user"; do + if [[ ! -s "${required_file}" ]]; then + echo "Required OMR migration input ${required_file} is missing or empty." >&2 + exit 1 + fi +done + +host_address=$(<"${SHARED_DIR}/omr_host_public_address") +host_user=$(<"${SHARED_DIR}/omr_host_ssh_user") +if [[ ! "${host_address}" =~ ^[A-Za-z0-9][A-Za-z0-9.-]*$ ]] || + [[ ! "${host_user}" =~ ^[A-Za-z_][A-Za-z0-9_-]*$ ]] || + [[ ! "${UNIQUE_HASH}" =~ ^[A-Za-z0-9]+$ ]]; then + echo "A dedicated OMR host connection value or UNIQUE_HASH is invalid." >&2 + exit 1 +fi + +if ! whoami >/dev/null 2>&1; then + if [[ -w /etc/passwd ]]; then + echo "${USER_NAME:-default}:x:$(id -u):0:${USER_NAME:-default} user:${HOME}:/sbin/nologin" >> /etc/passwd + else + echo "/etc/passwd is not writable and the current UID has no passwd entry." >&2 + exit 1 + fi +fi + +work_dir=$(mktemp -d /tmp/quay-omr-v3-migrate.XXXXXX) +chmod 0700 "${work_dir}" +auth_file="${work_dir}/registry-auth.json" +image_archive="${work_dir}/quay-mirror.tar" +extract_dir="${work_dir}/extract" +mkdir -p "${extract_dir}" + +unset KUBECONFIG +oc registry login --to="${auth_file}" +chmod 0600 "${auth_file}" +skopeo copy --retry-times=3 --src-authfile="${auth_file}" \ + "docker://${OMR_IMAGE}" \ + "docker-archive:${image_archive}:quay-mirror:ci" +[[ -s "${image_archive}" ]] +oc image extract "${OMR_IMAGE}" \ + --registry-config="${auth_file}" \ + --path="/quay:${extract_dir}" +if [[ ! -s "${extract_dir}/quay" ]]; then + echo "The extracted OMR v3 installer binary is missing or empty." >&2 + exit 1 +fi +chmod 0755 "${extract_dir}/quay" + +sha256sum \ + "${SHARED_DIR}/mirror_registry_ca.crt" \ + "${SHARED_DIR}/mirror_registry_creds" \ + "${SHARED_DIR}/mirror_registry_url" \ + > "${work_dir}/runtime-material.sha256" + +ssh_options=( + -o UserKnownHostsFile=/dev/null + -o StrictHostKeyChecking=no + -o "IdentityFile=${CLUSTER_PROFILE_DIR}/ssh-privatekey" + -o ConnectTimeout=10 + -o ConnectionAttempts=3 +) +remote="${host_user}@${host_address}" +remote_work_dir="/home/${host_user}/omr-v3-upgrade-${UNIQUE_HASH}" +ssh "${ssh_options[@]}" "${remote}" \ + "rm -rf -- '${remote_work_dir}' && install -d -m 0700 '${remote_work_dir}'" +remote_ready=true + +cat > "${work_dir}/migrate-to-v3" <<'EOF' +#!/bin/bash +set -o nounset +set -o errexit +set -o pipefail +umask 077 + +work_dir="$1" +data_dir="/home/$(id -un)/quay-v3" +registry_hostname="$2" +registry_endpoint="${registry_hostname}:8443" +export XDG_RUNTIME_DIR="/run/user/$(id -u)" + +if ! "${work_dir}/quay" migrate \ + -data-dir "${data_dir}" \ + -image-archive "${work_dir}/quay-mirror.tar" \ + -cleanup > "${work_dir}/migration.log" 2>&1; then + echo "OMR v3 migration failed; see the sanitized migration artifact." >&2 + exit 1 +fi + +systemctl --user is-active --quiet quay.service +for service in quay-app.service quay-redis.service quay-pod.service; do + if systemctl --user is-active --quiet "${service}"; then + echo "Old OMR v2 service ${service} is still active." >&2 + exit 1 + fi + if [[ -e "/home/$(id -un)/.config/systemd/user/${service}" ]]; then + echo "Old OMR v2 unit ${service} was not removed." >&2 + exit 1 + fi +done + +curl --retry 20 --retry-delay 3 --retry-all-errors \ + --silent --show-error --fail \ + --cacert "/home/$(id -un)/quay-install/quay-rootCA/rootCA.pem" \ + "https://${registry_endpoint}/healthz" >/dev/null +cert_dir="${work_dir}/certs" +auth_file="${work_dir}/registry-auth.json" +install -d -m 0700 "${cert_dir}" +install -m 0600 \ + "/home/$(id -un)/quay-install/quay-rootCA/rootCA.pem" \ + "${cert_dir}/ca.crt" +admin_password=$(<"/home/$(id -un)/.omr-v2-admin-password") +if ! printf '%s' "${admin_password}" | podman login \ + --authfile "${auth_file}" \ + --cert-dir "${cert_dir}" \ + --tls-verify=true \ + --username admin \ + --password-stdin \ + "${registry_endpoint}" >/dev/null; then + unset admin_password + echo "The migrated OMR v3 registry rejected the preserved administrator credential." >&2 + exit 1 +fi +unset admin_password +EOF +chmod 0755 "${work_dir}/migrate-to-v3" + +scp "${ssh_options[@]}" "${image_archive}" \ + "${remote}:${remote_work_dir}/quay-mirror.tar" +scp "${ssh_options[@]}" "${extract_dir}/quay" \ + "${remote}:${remote_work_dir}/quay" +scp "${ssh_options[@]}" "${work_dir}/migrate-to-v3" \ + "${remote}:${remote_work_dir}/migrate-to-v3" +ssh "${ssh_options[@]}" "${remote}" \ + "chmod 0755 '${remote_work_dir}/quay' '${remote_work_dir}/migrate-to-v3'" + +registry_endpoint=$(<"${SHARED_DIR}/mirror_registry_url") +registry_hostname="${registry_endpoint%:8443}" +if [[ ! "${registry_hostname}" =~ ^[A-Za-z0-9][A-Za-z0-9.-]*$ ]]; then + echo "The runtime OMR hostname is invalid." >&2 + exit 1 +fi + +migration_status=0 +ssh "${ssh_options[@]}" "${remote}" \ + "'${remote_work_dir}/migrate-to-v3' '${remote_work_dir}' '${registry_hostname}'" || migration_status=$? +scp "${ssh_options[@]}" "${remote}:${remote_work_dir}/migration.log" \ + "${ARTIFACT_DIR}/omr-v3-migration.log" 2>/dev/null || true +if [[ "${migration_status}" -ne 0 ]]; then + echo "OMR v3 migration failed with status ${migration_status}." + exit "${migration_status}" +fi + +sha256sum --check "${work_dir}/runtime-material.sha256" +echo "OMR v2 content and runtime identity migrated successfully to rootless OMR v3." diff --git a/ci-operator/step-registry/quay/omr-v3/upgrade-from-v2/quay-omr-v3-upgrade-from-v2-ref.metadata.json b/ci-operator/step-registry/quay/omr-v3/upgrade-from-v2/quay-omr-v3-upgrade-from-v2-ref.metadata.json new file mode 100644 index 0000000000000..1639ebdedf348 --- /dev/null +++ b/ci-operator/step-registry/quay/omr-v3/upgrade-from-v2/quay-omr-v3-upgrade-from-v2-ref.metadata.json @@ -0,0 +1,11 @@ +{ + "path": "quay/omr-v3/upgrade-from-v2/quay-omr-v3-upgrade-from-v2-ref.yaml", + "owners": { + "approvers": [ + "quay-approvers" + ], + "reviewers": [ + "quay-approvers" + ] + } +} \ No newline at end of file diff --git a/ci-operator/step-registry/quay/omr-v3/upgrade-from-v2/quay-omr-v3-upgrade-from-v2-ref.yaml b/ci-operator/step-registry/quay/omr-v3/upgrade-from-v2/quay-omr-v3-upgrade-from-v2-ref.yaml new file mode 100644 index 0000000000000..fab25bd7910ed --- /dev/null +++ b/ci-operator/step-registry/quay/omr-v3/upgrade-from-v2/quay-omr-v3-upgrade-from-v2-ref.yaml @@ -0,0 +1,21 @@ +ref: + as: quay-omr-v3-upgrade-from-v2 + from_image: + namespace: ci + name: tests-private-base + tag: "4.22" + cli: latest + commands: quay-omr-v3-upgrade-from-v2-commands.sh + dependencies: + - name: "pipeline:quay-mirror" + env: OMR_IMAGE + resources: + requests: + cpu: 100m + memory: 512Mi + timeout: 90m0s + grace_period: 5m0s + documentation: |- + Export the PR-built OMR v3 image and binary, migrate the populated rootless + OMR v2 installation on the dedicated RHEL host, clean up the v2 services, + and verify that the endpoint, CA, credentials, and registry health survive. diff --git a/ci-operator/step-registry/quay/omr-v3/validate-migration-sentinel/OWNERS b/ci-operator/step-registry/quay/omr-v3/validate-migration-sentinel/OWNERS new file mode 100644 index 0000000000000..bd932d737f687 --- /dev/null +++ b/ci-operator/step-registry/quay/omr-v3/validate-migration-sentinel/OWNERS @@ -0,0 +1,4 @@ +approvers: +- quay-approvers +reviewers: +- quay-approvers diff --git a/ci-operator/step-registry/quay/omr-v3/validate-migration-sentinel/quay-omr-v3-validate-migration-sentinel-commands.sh b/ci-operator/step-registry/quay/omr-v3/validate-migration-sentinel/quay-omr-v3-validate-migration-sentinel-commands.sh new file mode 100644 index 0000000000000..576d215e0e11b --- /dev/null +++ b/ci-operator/step-registry/quay/omr-v3/validate-migration-sentinel/quay-omr-v3-validate-migration-sentinel-commands.sh @@ -0,0 +1,302 @@ +#!/bin/bash + +set -o nounset +set -o errexit +set -o pipefail + +umask 077 +work_dir="" +sentinel_namespace=quay-omr-migration-sentinel +namespace_created=false + +dump_sentinel_workloads() { + local pod + + oc -n "${sentinel_namespace}" get daemonset,pods -o wide || true + oc -n "${sentinel_namespace}" describe daemonset/omr-migration-sentinel || true + while read -r pod; do + [[ -n "${pod}" ]] || continue + oc -n "${sentinel_namespace}" describe "${pod}" || true + oc -n "${sentinel_namespace}" logs "${pod}" || true + done < <(oc -n "${sentinel_namespace}" get pods -o name 2>/dev/null || true) +} + +cleanup() { + local status=$? + + trap - EXIT TERM + set +o errexit + if [[ "${status}" -ne 0 && "${namespace_created}" == true ]]; then + dump_sentinel_workloads + fi + if [[ "${namespace_created}" == true ]]; then + oc delete namespace "${sentinel_namespace}" \ + --ignore-not-found --wait=false >/dev/null 2>&1 || true + fi + if [[ -n "${work_dir}" && -d "${work_dir}" ]]; then + rm -rf -- "${work_dir}" + fi + exit "${status}" +} + +terminate() { + exit 143 +} + +trap cleanup EXIT +trap terminate TERM + +mkdir -p "${ARTIFACT_DIR}" +exec > >(tee "${ARTIFACT_DIR}/omr-v3-migration-sentinel.log") 2>&1 + +if [[ -f "${SHARED_DIR}/proxy-conf.sh" ]]; then + # shellcheck disable=SC1091 + source "${SHARED_DIR}/proxy-conf.sh" +fi + +for command in base64 jq oc tar; do + if ! command -v "${command}" >/dev/null 2>&1; then + echo "Required command ${command} is unavailable in the sentinel validation image." >&2 + exit 1 + fi +done + +for required_file in \ + "${SHARED_DIR}/mirror_registry_ca.crt" \ + "${SHARED_DIR}/mirror_registry_creds" \ + "${SHARED_DIR}/mirror_registry_url" \ + "${SHARED_DIR}/omr_migration_sentinel_digest" \ + "${SHARED_DIR}/omr_migration_sentinel_image" \ + "${SHARED_DIR}/omr_migration_sentinel_marker" \ + "${SHARED_DIR}/omr_migration_sentinel_tag"; do + if [[ ! -s "${required_file}" ]]; then + echo "Required sentinel validation input ${required_file} is missing or empty." >&2 + exit 1 + fi +done + +: "${UNIQUE_HASH:?UNIQUE_HASH is required}" +if [[ ! "${UNIQUE_HASH}" =~ ^[A-Za-z0-9]+$ ]]; then + echo "UNIQUE_HASH contains unexpected characters." >&2 + exit 1 +fi + +if ! whoami >/dev/null 2>&1; then + if [[ -w /etc/passwd ]]; then + echo "${USER_NAME:-default}:x:$(id -u):0:${USER_NAME:-default} user:${HOME}:/sbin/nologin" >> /etc/passwd + else + echo "/etc/passwd is not writable and the current UID has no passwd entry." >&2 + exit 1 + fi +fi + +registry_host=$(tr -d '\r\n' < "${SHARED_DIR}/mirror_registry_url") +expected_digest=$(tr -d '\r\n' < "${SHARED_DIR}/omr_migration_sentinel_digest") +expected_image=$(tr -d '\r\n' < "${SHARED_DIR}/omr_migration_sentinel_image") +expected_marker=$(tr -d '\r\n' < "${SHARED_DIR}/omr_migration_sentinel_marker") +expected_tag=$(tr -d '\r\n' < "${SHARED_DIR}/omr_migration_sentinel_tag") +expected_repository="${registry_host}/admin/omr-migration-sentinel" +if [[ -z "${registry_host}" || "${registry_host}" == */* || + "${registry_host}" =~ [[:space:]] ]]; then + echo "The runtime OMR endpoint is invalid." >&2 + exit 1 +fi +if [[ ! "${expected_digest}" =~ ^sha256:[a-f0-9]{64}$ ]] || + [[ "${expected_tag}" != "${expected_repository}:pre-${UNIQUE_HASH}" ]] || + [[ "${expected_image}" != "${expected_repository}@${expected_digest}" ]] || + [[ "${expected_marker}" != "omr-v2-${UNIQUE_HASH}" ]]; then + echo "The published OMR v2 sentinel metadata is inconsistent." >&2 + exit 1 +fi + +work_dir=$(mktemp -d /tmp/quay-omr-v3-sentinel.XXXXXX) +chmod 0700 "${work_dir}" +auth_file="${work_dir}/registry-auth.json" + +registry_credentials=$(tr -d '\r\n' < "${SHARED_DIR}/mirror_registry_creds") +if [[ -z "${registry_credentials}" || "${registry_credentials}" != *:* ]]; then + echo "The runtime OMR credential is invalid." >&2 + exit 1 +fi +registry_auth=$(printf '%s' "${registry_credentials}" | base64 -w 0) +unset registry_credentials +jq -n --arg host "${registry_host}" --arg auth "${registry_auth}" ' + {auths: {($host): {auth: $auth}}} +' > "${auth_file}" +unset registry_auth +chmod 0600 "${auth_file}" + +pre_info="${ARTIFACT_DIR}/omr-v3-migrated-sentinel-image.json" +oc image info "${expected_tag}" \ + --registry-config="${auth_file}" \ + --certificate-authority="${SHARED_DIR}/mirror_registry_ca.crt" \ + -o json > "${pre_info}" +actual_digest=$(jq -er ' + .digest | select(test("^sha256:[a-f0-9]{64}$")) +' "${pre_info}") +if [[ "${actual_digest}" != "${expected_digest}" ]]; then + echo "The migrated sentinel tag changed digest: expected ${expected_digest}, got ${actual_digest}." >&2 + exit 1 +fi + +oc create namespace "${sentinel_namespace}" --dry-run=client -o yaml | oc apply -f - +namespace_created=true + +daemonset_file="${work_dir}/sentinel-daemonset.json" +jq -n --arg image "${expected_image}" --arg marker "${expected_marker}" ' + { + apiVersion: "apps/v1", + kind: "DaemonSet", + metadata: { + name: "omr-migration-sentinel", + namespace: "quay-omr-migration-sentinel" + }, + spec: { + selector: {matchLabels: {app: "omr-migration-sentinel"}}, + template: { + metadata: {labels: {app: "omr-migration-sentinel"}}, + spec: { + tolerations: [{operator: "Exists"}], + terminationGracePeriodSeconds: 1, + containers: [{ + name: "sentinel", + image: $image, + imagePullPolicy: "Always", + command: [ + "/bin/sh", + "-ec", + "test \"$(cat /omr-sentinel/pre-migration-marker)\" = \"${EXPECTED_MARKER}\"\nprintf \"validated %s on %s\\n\" \"${EXPECTED_MARKER}\" \"${NODE_NAME}\"\nwhile true; do sleep 3600; done" + ], + env: [ + {name: "EXPECTED_MARKER", value: $marker}, + { + name: "NODE_NAME", + valueFrom: {fieldRef: {fieldPath: "spec.nodeName"}} + } + ], + resources: { + requests: {cpu: "10m", memory: "32Mi"}, + limits: {memory: "128Mi"} + }, + securityContext: { + allowPrivilegeEscalation: false, + capabilities: {drop: ["ALL"]}, + runAsNonRoot: true, + seccompProfile: {type: "RuntimeDefault"} + } + }] + } + } + } + } +' > "${daemonset_file}" +oc apply -f "${daemonset_file}" +if ! oc -n "${sentinel_namespace}" rollout status \ + daemonset/omr-migration-sentinel --timeout=10m; then + echo "The migrated sentinel did not start on every node." >&2 + exit 1 +fi + +nodes_file="${work_dir}/nodes.json" +daemonset_status_file="${ARTIFACT_DIR}/omr-migration-sentinel-daemonset.json" +oc get nodes -o json > "${nodes_file}" +oc -n "${sentinel_namespace}" get daemonset/omr-migration-sentinel \ + -o json > "${daemonset_status_file}" +node_count=$(jq -er '.items | length | select(. > 0)' "${nodes_file}") +if ! jq -e --argjson nodes "${node_count}" ' + (.status.desiredNumberScheduled // 0) == $nodes + and (.status.currentNumberScheduled // 0) == $nodes + and (.status.updatedNumberScheduled // 0) == $nodes + and (.status.numberReady // 0) == $nodes + and (.status.numberAvailable // 0) == $nodes + and (.status.numberMisscheduled // 0) == 0 +' "${daemonset_status_file}" >/dev/null; then + echo "The sentinel DaemonSet did not validate exactly one ready pod on each node." >&2 + exit 1 +fi +oc -n "${sentinel_namespace}" get pods -l app=omr-migration-sentinel \ + -o wide | tee "${ARTIFACT_DIR}/omr-migration-sentinel-pods.txt" + +post_marker="omr-v3-${UNIQUE_HASH}" +post_tag="${expected_repository}:post-${UNIQUE_HASH}" +post_rootfs="${work_dir}/post-rootfs" +post_layer="${work_dir}/post-migration-layer.tar.gz" +install -d -m 0755 "${post_rootfs}/omr-sentinel" +printf '%s\n' "${post_marker}" > "${post_rootfs}/omr-sentinel/post-migration-marker" +chmod 0644 "${post_rootfs}/omr-sentinel/post-migration-marker" +tar --create --gzip --file "${post_layer}" --directory "${post_rootfs}" . + +oc image append \ + --from="${expected_image}" \ + --to="${post_tag}" \ + --registry-config="${auth_file}" \ + --certificate-authority="${SHARED_DIR}/mirror_registry_ca.crt" \ + "${post_layer}" + +post_info="${ARTIFACT_DIR}/omr-v3-post-migration-sentinel-image.json" +oc image info "${post_tag}" \ + --registry-config="${auth_file}" \ + --certificate-authority="${SHARED_DIR}/mirror_registry_ca.crt" \ + -o json > "${post_info}" +post_digest=$(jq -er ' + .digest | select(test("^sha256:[a-f0-9]{64}$")) +' "${post_info}") +if [[ "${post_digest}" == "${expected_digest}" ]]; then + echo "The post-migration write did not produce a new image digest." >&2 + exit 1 +fi +post_image="${expected_repository}@${post_digest}" + +post_pod_file="${work_dir}/post-migration-pod.json" +jq -n \ + --arg image "${post_image}" \ + --arg pre_marker "${expected_marker}" \ + --arg post_marker "${post_marker}" ' + { + apiVersion: "v1", + kind: "Pod", + metadata: { + name: "omr-v3-post-migration-write", + namespace: "quay-omr-migration-sentinel" + }, + spec: { + restartPolicy: "Never", + tolerations: [{operator: "Exists"}], + containers: [{ + name: "sentinel", + image: $image, + imagePullPolicy: "Always", + command: [ + "/bin/sh", + "-ec", + "test \"$(cat /omr-sentinel/pre-migration-marker)\" = \"${PRE_MARKER}\"\ntest \"$(cat /omr-sentinel/post-migration-marker)\" = \"${POST_MARKER}\"\nprintf \"validated post-migration write %s\\n\" \"${POST_MARKER}\"" + ], + env: [ + {name: "PRE_MARKER", value: $pre_marker}, + {name: "POST_MARKER", value: $post_marker} + ], + resources: { + requests: {cpu: "10m", memory: "32Mi"}, + limits: {memory: "128Mi"} + }, + securityContext: { + allowPrivilegeEscalation: false, + capabilities: {drop: ["ALL"]}, + runAsNonRoot: true, + seccompProfile: {type: "RuntimeDefault"} + } + }] + } + } +' > "${post_pod_file}" +oc apply -f "${post_pod_file}" +if ! oc -n "${sentinel_namespace}" wait pod/omr-v3-post-migration-write \ + --for=jsonpath='{.status.phase}'=Succeeded --timeout=5m; then + echo "OpenShift could not pull and validate the image written through OMR v3." >&2 + exit 1 +fi +oc -n "${sentinel_namespace}" get pod/omr-v3-post-migration-write -o wide +oc -n "${sentinel_namespace}" logs pod/omr-v3-post-migration-write | + tee "${ARTIFACT_DIR}/omr-v3-post-migration-write.log" + +echo "OMR v2 sentinel content survived migration, every OpenShift node pulled it, and OMR v3 accepted and served a new write." diff --git a/ci-operator/step-registry/quay/omr-v3/validate-migration-sentinel/quay-omr-v3-validate-migration-sentinel-ref.metadata.json b/ci-operator/step-registry/quay/omr-v3/validate-migration-sentinel/quay-omr-v3-validate-migration-sentinel-ref.metadata.json new file mode 100644 index 0000000000000..a7c6ebf89f73b --- /dev/null +++ b/ci-operator/step-registry/quay/omr-v3/validate-migration-sentinel/quay-omr-v3-validate-migration-sentinel-ref.metadata.json @@ -0,0 +1,11 @@ +{ + "path": "quay/omr-v3/validate-migration-sentinel/quay-omr-v3-validate-migration-sentinel-ref.yaml", + "owners": { + "approvers": [ + "quay-approvers" + ], + "reviewers": [ + "quay-approvers" + ] + } +} \ No newline at end of file diff --git a/ci-operator/step-registry/quay/omr-v3/validate-migration-sentinel/quay-omr-v3-validate-migration-sentinel-ref.yaml b/ci-operator/step-registry/quay/omr-v3/validate-migration-sentinel/quay-omr-v3-validate-migration-sentinel-ref.yaml new file mode 100644 index 0000000000000..16f62908afeca --- /dev/null +++ b/ci-operator/step-registry/quay/omr-v3/validate-migration-sentinel/quay-omr-v3-validate-migration-sentinel-ref.yaml @@ -0,0 +1,14 @@ +ref: + as: quay-omr-v3-validate-migration-sentinel + from: upi-installer + commands: quay-omr-v3-validate-migration-sentinel-commands.sh + resources: + requests: + cpu: 100m + memory: 256Mi + timeout: 30m0s + grace_period: 2m0s + documentation: |- + Verify that the OMR v2 sentinel tag and digest survived migration, pull its + unique marker layer on every OpenShift node, then append and pull a second + marker layer to prove that migrated OMR v3 is writable and readable. diff --git a/ci-operator/step-registry/quay/omr-v3/validate/quay-omr-v3-validate-commands.sh b/ci-operator/step-registry/quay/omr-v3/validate/quay-omr-v3-validate-commands.sh index 7b8f38f206100..66356f5777cb9 100644 --- a/ci-operator/step-registry/quay/omr-v3/validate/quay-omr-v3-validate-commands.sh +++ b/ci-operator/step-registry/quay/omr-v3/validate/quay-omr-v3-validate-commands.sh @@ -45,8 +45,6 @@ required_files=( omr_mirror_completed_at omr_mirror_repository omr_mirrored_cli_image - bastion_public_address - bastion_ssh_user ) for name in "${required_files[@]}"; do if [[ ! -s "${SHARED_DIR}/${name}" ]]; then @@ -55,6 +53,20 @@ for name in "${required_files[@]}"; do fi done +host_address_file="${SHARED_DIR}/bastion_public_address" +host_user_file="${SHARED_DIR}/bastion_ssh_user" +if [[ -s "${SHARED_DIR}/omr_host_public_address" || + -s "${SHARED_DIR}/omr_host_ssh_user" ]]; then + host_address_file="${SHARED_DIR}/omr_host_public_address" + host_user_file="${SHARED_DIR}/omr_host_ssh_user" +fi +for required_file in "${host_address_file}" "${host_user_file}"; do + if [[ ! -s "${required_file}" ]]; then + echo "Required OMR host connection file is missing or empty: ${required_file}" + exit 1 + fi +done + signature_configmap="${SHARED_DIR}/oc-mirror-signature-configmap.json" if ! jq -e ' .apiVersion == "v1" @@ -101,24 +113,13 @@ if ! whoami &> /dev/null; then fi fi -bastion_address=$(<"${SHARED_DIR}/bastion_public_address") -bastion_user=$(<"${SHARED_DIR}/bastion_ssh_user") -mirror_completed_at=$(<"${SHARED_DIR}/omr_mirror_completed_at") -mirror_repository=$(<"${SHARED_DIR}/omr_mirror_repository") -repository_path="${mirror_repository#*/}" +host_address=$(<"${host_address_file}") +host_user=$(<"${host_user_file}") ssh_options=( -o UserKnownHostsFile=/dev/null -o StrictHostKeyChecking=no -o "IdentityFile=${CLUSTER_PROFILE_DIR}/ssh-privatekey" ) -journal_file="${ARTIFACT_DIR}/quay-journal-after-mirror.log" -ssh "${ssh_options[@]}" "${bastion_user}@${bastion_address}" \ - "sudo journalctl -u quay.service --since '${mirror_completed_at}' --no-pager --output=short-iso" \ - > "${journal_file}" -if ! grep -F "/v2/${repository_path}/" "${journal_file}" > /dev/null; then - echo "No OMR request for /v2/${repository_path}/ was recorded after mirror completion." - exit 1 -fi smoke_namespace=quay-omr-v3-smoke smoke_pod=omr-v3-mirrored-cli-smoke @@ -127,8 +128,8 @@ oc create namespace "${smoke_namespace}" \ --dry-run=client -o yaml | oc apply -f - trap 'oc delete namespace "${smoke_namespace}" --ignore-not-found --wait=false >/dev/null 2>&1 || true' EXIT oc -n "${smoke_namespace}" delete pod "${smoke_pod}" --ignore-not-found --wait=true -oc -n "${smoke_namespace}" run "${smoke_pod}" --image="${smoke_image}" --restart=Never \ - --command -- oc version --client=true +oc -n "${smoke_namespace}" run "${smoke_pod}" --image="${smoke_image}" \ + --image-pull-policy=Always --restart=Never --command -- oc version --client=true if ! oc -n "${smoke_namespace}" wait pod/"${smoke_pod}" --for=jsonpath='{.status.phase}'=Succeeded --timeout=5m; then oc -n "${smoke_namespace}" describe pod "${smoke_pod}" oc -n "${smoke_namespace}" logs "${smoke_pod}" || true @@ -137,4 +138,32 @@ fi oc -n "${smoke_namespace}" get pod "${smoke_pod}" -o wide oc -n "${smoke_namespace}" logs "${smoke_pod}" | tee "${ARTIFACT_DIR}/mirrored-cli-smoke.log" +# Run the smoke pull before reading the journal so a newly migrated v3 service +# always has a post-migration registry request to validate. +mirror_completed_at=$(<"${SHARED_DIR}/omr_mirror_completed_at") +mirror_repository=$(<"${SHARED_DIR}/omr_mirror_repository") +repository_path="${mirror_repository#*/}" +journal_file="${ARTIFACT_DIR}/quay-journal-after-mirror.log" +journal_collected=false +if [[ "${host_user_file}" == "${SHARED_DIR}/omr_host_ssh_user" ]]; then + if ssh "${ssh_options[@]}" "${host_user}@${host_address}" \ + "export XDG_RUNTIME_DIR=/run/user/\$(id -u); journalctl --user -u quay.service --since '${mirror_completed_at}' --no-pager --output=short-iso" \ + > "${journal_file}"; then + journal_collected=true + fi +else + if ssh "${ssh_options[@]}" "${host_user}@${host_address}" \ + "sudo journalctl -u quay.service --since '${mirror_completed_at}' --no-pager --output=short-iso" \ + > "${journal_file}"; then + journal_collected=true + fi +fi +if [[ "${journal_collected}" == false ]]; then + echo "Unable to collect the OMR v3 journal; the mirrored image pull succeeded." +elif grep -F "/v2/${repository_path}/" "${journal_file}" >/dev/null; then + echo "The OMR v3 journal recorded a request for /v2/${repository_path}/ after mirror completion." +else + echo "The OMR v3 journal did not record a request for /v2/${repository_path}/; the mirrored image pull succeeded." +fi + echo "OMR v3 disconnected validation passed." diff --git a/ci-operator/step-registry/quay/omr-v3/validate/quay-omr-v3-validate-ref.yaml b/ci-operator/step-registry/quay/omr-v3/validate/quay-omr-v3-validate-ref.yaml index 2e38a765a0222..6b5ebb6c2dc64 100644 --- a/ci-operator/step-registry/quay/omr-v3/validate/quay-omr-v3-validate-ref.yaml +++ b/ci-operator/step-registry/quay/omr-v3/validate/quay-omr-v3-validate-ref.yaml @@ -14,5 +14,5 @@ ref: documentation: |- Applies the generated oc-mirror release signatures, then validates cluster availability, non-degraded operators, Ready nodes, both OpenShift IDMS - mappings, OMR requests after release mirroring completed, and a pod started - from the mirrored CLI digest. + mappings, a pod started from the mirrored CLI digest, and the resulting OMR + request. A dedicated rootless OMR host is preferred when present.