Bug 1853220: [release-4.5] Added E2E ES upgrade test 4.4->4.5

hack/cr.yaml

@@ -12,7 +12,7 @@ spec:
       limits:
         memory: 1Gi
       requests:
-        cpu: 500m
+        cpu: 100m
         memory: 1Gi
   nodes:
   - nodeCount: 1

hack/testing-olm-upgrade/resources/cr.yaml

@@ -0,0 +1,43 @@
+---
+apiVersion: "logging.openshift.io/v1"
+kind: "Elasticsearch"
+metadata:
+  name: "elasticsearch"
+  annotations:
+      elasticsearch.openshift.io/loglevel: trace
+spec:
+  managementState: "Managed"
+  nodeSpec:
+    resources:
+      limits:
+        memory: 1Gi
+      requests:
+        cpu: 100m
+        memory: 1Gi
+  nodes:
+  - nodeCount: 3
+    roles:
+    - client
+    - data
+    - master
+    storage:
+      storageClassName: gp2
+      size: 10G
+  redundancyPolicy: SingleRedundancy
+  indexManagement:
+    policies:
+    - name: infra-policy
+      pollInterval: 30m
+      phases:
+        hot:
+          actions:
+            rollover:
+              maxAge: 2m
+        delete:
+          minAge: 5m
+    mappings:
+    - name:  infra
+      policyRef: infra-policy
+      aliases:
+      - infra
+      - logs.infra

hack/testing-olm-upgrade/test-020-olm-upgrade-4.4-4.5.sh

@@ -0,0 +1,194 @@
+#!/bin/bash
+# Given an OLM manifest, verify a green field deployment
+# of cluster logging by asserting CLO creates the resources
+# that begets the operands that make up logging.
+
+set -euo pipefail
+
+repo_dir="$( cd "$(dirname "$0")/../.." ; pwd -P )"
+source "$repo_dir/hack/testing-olm/utils"
+source "$repo_dir/hack/testing-olm/assertions"
+
+start_seconds=$(date +%s)
+
+ARTIFACT_DIR=${ARTIFACT_DIR:-"$repo_dir/_output/"}
+ARTIFACT_DIR="${ARTIFACT_DIR}/$(basename ${BASH_SOURCE[0]})"
+if [ ! -d $ARTIFACT_DIR ] ; then
+  mkdir -p $ARTIFACT_DIR
+fi
+
+KUBECONFIG=${KUBECONFIG:-$HOME/.kube/config}
+TIMEOUT_MIN=$((2 * $minute))
+ES_POD_TIMEOUT=$((5 * $minute))
+export CLUSTER_LOGGING_OPERATOR_NAMESPACE="openshift-logging"
+manifest_dir=${repo_dir}/manifests
+version="4.5"
+previous_version="4.4"
+
+cleanup(){
+  local return_code="$?"
+  set +e
+  log::info "Running cleanup"
+  end_seconds=$(date +%s)
+  runtime="$(($end_seconds - $start_seconds))s"
+  oc -n openshift-operators-redhat -o yaml get subscription elasticsearch-operator > $ARTIFACT_DIR/subscription-eo.yml 2>&1 ||:
+  oc -n openshift-operators-redhat -o yaml get deployment/elasticsearch-operator > $ARTIFACT_DIR/elasticsearch-operator-deployment.yml 2>&1 ||:
+  oc -n openshift-operators-redhat -o yaml get pods > $ARTIFACT_DIR/openshift-operators-redhat-pods.yml 2>&1 ||:
+  oc -n openshift-operators-redhat -o yaml get configmaps > $ARTIFACT_DIR/openshift-operators-redhat-configmaps.yml 2>&1 ||:
+  oc -n openshift-operators-redhat describe deployment/elasticsearch-operator > $ARTIFACT_DIR/elasticsearch-operator.describe 2>&1 ||:
+
+  oc logs -n "openshift-operators-redhat" deployment/elasticsearch-operator > $ARTIFACT_DIR/elasticsearch-operator.log 2>&1 ||:
+  oc  -n openshift-operator-lifecycle-manager logs --since=$runtime deployment/catalog-operator > $ARTIFACT_DIR/catalog-operator.logs 2>&1 ||:
+  oc  -n openshift-operator-lifecycle-manager logs --since=$runtime deployment/olm-operator > $ARTIFACT_DIR/olm-operator.logs 2>&1 ||:
+
+  ${repo_dir}/olm_deploy/scripts/operator-uninstall.sh
+  ${repo_dir}/olm_deploy/scripts/catalog-uninstall.sh
+
+  set -e
+  exit ${return_code}
+}
+trap cleanup exit
+
+get_es_pods_count() {
+  	oc -n openshift-operators-redhat get pods -l component=elasticsearch --no-headers=true --ignore-not-found | wc -l
+}
+
+get_es_cluster_status() {
+  oc -n openshift-operators-redhat get pods -l component=elasticsearch --no-headers=true --ignore-not-found \
+   | awk 'NR==1{print $1}' \
+   | xargs -I '{}' oc -n openshift-operators-redhat exec '{}' -c elasticsearch -- es_util --query=_cluster/health \
+   | jq .status
+}
+
+get_es_indices() {
+  oc -n openshift-operators-redhat get pods -l component=elasticsearch --no-headers=true --ignore-not-found \
+    | awk 'NR==1{print $1}' \
+    | xargs -I '{}' oc -n openshift-operators-redhat exec '{}' -c elasticsearch -- es_util --query=_cat/indices
+}
+
+read_es_indices() {
+	local -n map=$1
+	while IFS= read -r line; do
+		id="$(echo "$line" | awk '{print $3}')"
+		echo "Index: '$id' stored"
+		map[$id]=$line
+	done <<< "$(get_es_indices)"
+}
+
+compare_indices_names(){
+	local failed=false
+	local -n old_m=$1
+	local -n new_m=$2
+	for k in "${!old_m[@]}"; do
+		echo "Checking index $k..."
+		if [ ! ${new_m[$k]+_} ]; then
+			failed=true
+			printf "\t'%s' is missing in the set of indices\n" "$k"
+		else
+			printf "\t'%s' found\n" "$k"
+		fi
+	done
+	if [ "$failed" = true ]; then
+		echo "Compare indices names failed!!!"
+		return 1
+	fi
+	echo "Indices names are fully matched"
+}
+
+es_cluster_ready() {
+	oc -n openshift-operators-redhat get pods -l component=elasticsearch \
+		--no-headers=true --ignore-not-found \
+		| awk '{print $2}' | grep '2/2'
+}
+
+# deploy elasticsearch-operator
+log::info "Deploying elasticsearch-operator ${previous_version} from marketplace..."
+deploy_marketplace_operator "openshift-operators-redhat" "elasticsearch-operator" "$previous_version" "elasticsearch-operator"
+
+# check if the operator is running
+log::info "Verifying if elasticsearch-operator deployment is ready..."
+try_until_text "oc -n openshift-operators-redhat get deployment elasticsearch-operator -o jsonpath={.status.updatedReplicas} --ignore-not-found" "1" ${TIMEOUT_MIN}
+
+log::info "Deploying ES secrets..."
+rm -rf /tmp/example-secrets ||: \
+	mkdir /tmp/example-secrets && \
+	"$repo_dir"/hack/cert_generation.sh /tmp/example-secrets "openshift-operators-redhat" elasticsearch
+"$repo_dir"/hack/deploy-example-secrets.sh "openshift-operators-redhat"
+
+# deploy elasticsearch CR
+log::info "Deploying ES CR..."
+oc -n "openshift-operators-redhat" create -f ${repo_dir}/hack/testing-olm-upgrade/resources/cr.yaml
+
+# check if there are elasticsearch pod
+log::info "Checking if ES pod is ready in the namespace openshift-operators-redhat..."
+try_until_text "oc -n openshift-operators-redhat get pods -l component=elasticsearch -o jsonpath={.items[0].status.phase} --ignore-not-found" "Running" ${ES_POD_TIMEOUT}
+
+log::info "Getting the previous statate of elasticsearch-operator deployment"
+# get the previous status of the elasticsearch-operator
+oc describe -n "openshift-operators-redhat" deployment/elasticsearch-operator > "$ARTIFACT_DIR"/elasticsearch-operator.describe.before_update 2>&1
+
+# check if ES cluster has 3 running pods
+log::info "Checking if the ES has 3 nodes"
+try_func_until_text get_es_pods_count "3" ${ES_POD_TIMEOUT}
+
+log::info "Check if at least 1 node is ready (2/2 containers running)"
+try_until_success es_cluster_ready ${ES_POD_TIMEOUT}
+
+# check if ES cluster is in green state
+log::info "Checking if the ES cluster is all yellow/green"
+try_func_until_text_alt get_es_cluster_status "\"green\"" "\"yellow\"" ${ES_POD_TIMEOUT}
+
+# read OLD 4.4 indices into and map them by their names
+log::info "Reading old ES indices"
+try_func_until_result_is_not_empty get_es_indices ${ES_POD_TIMEOUT}
+declare -A old_indices
+read_es_indices old_indices
+
+#### INSTALLING 4.5
+log::info "Deploying the ES operator from the catalog..."
+# deploy cluster logging catalog from local code
+"${repo_dir}"/olm_deploy/scripts/catalog-deploy.sh
+
+log::info "Patching subscription..."
+# patch subscription
+payload="{\"op\":\"replace\",\"path\":\"/spec/source\",\"value\":\"elasticsearch-catalog\"}"
+payload="$payload,{\"op\":\"replace\",\"path\":\"/spec/sourceNamespace\",\"value\":\"openshift-operators-redhat\"}"
+payload="$payload,{\"op\":\"replace\",\"path\":\"/spec/channel\",\"value\":\"$version\"}"
+oc -n "openshift-operators-redhat" patch subscription elasticsearch-operator --type json -p "[$payload]"
+
+# patch the minikube version
+log::info "Patching minKubeVersion to 1.16.1..."
+pl="{\"op\":\"replace\",\"path\":\"/spec/minKubeVersion\",\"value\":\"1.16.1\"}"
+try_until_success "oc -n openshift-operators-redhat patch clusterserviceversion elasticsearch-operator.v${version}.0 --type json -p [$pl]" ${TIMEOUT_MIN}
+
+#verify deployment is rolled out
+log::info "Checking if deployment successfully updated..."
+OPENSHIFT_VERSION=${OPENSHIFT_VERSION:-4.5}
+IMAGE_ELASTICSEARCH_OPERATOR=${IMAGE_ELASTICSEARCH_OPERATOR:-registry.svc.ci.openshift.org/ocp/${OPENSHIFT_VERSION}:elasticsearch-operator}
+if [ -n "${IMAGE_FORMAT:-}" ] ; then
+  IMAGE_ELASTICSEARCH_OPERATOR=$(echo $IMAGE_FORMAT | sed -e "s,\${component},elasticsearch-operator,")
+fi
+log::info "Checking if deployment version is ${IMAGE_ELASTICSEARCH_OPERATOR}..."
+try_until_text "oc -n openshift-operators-redhat get deployment elasticsearch-operator -o jsonpath={.spec.template.spec.containers[0].image}" "${IMAGE_ELASTICSEARCH_OPERATOR}" ${TIMEOUT_MIN}
+
+log::info "Checking if the ES pod is running..."
+try_until_text "oc -n openshift-operators-redhat get deployment -l component=elasticsearch -o jsonpath={.items[0].status.updatedReplicas} --ignore-not-found" "1" ${ES_POD_TIMEOUT}
+
+# check if ES cluster has 3 running pods
+log::info "Checking if the ES cluster still has 3 nodes..."
+try_func_until_text get_es_pods_count "3" ${ES_POD_TIMEOUT}
+
+log::info "Check if at least 1 node is ready (2/2 containers running)"
+try_until_success es_cluster_ready ${ES_POD_TIMEOUT}
+
+# check if ES cluster is in green state
+log::info "Checking if the ES cluster is all yellow/green"
+try_func_until_text_alt get_es_cluster_status "\"green\"" "\"yellow\"" ${ES_POD_TIMEOUT}
+
+# read new 4.5 indices and map them by their names
+log::info "Reading new ES indices"
+try_func_until_result_is_not_empty get_es_indices ${ES_POD_TIMEOUT}
+declare -A new_indices
+read_es_indices new_indices
+
+compare_indices_names old_indices new_indices

hack/testing-olm/assertions

@@ -0,0 +1,6 @@
+#!/bin/bash
+source "$( cd "$( dirname "${BASH_SOURCE[0]}" )" >/dev/null 2>&1 && pwd )/utils"
+assert_eo_exist(){
+	# verify ER
+	try_until_success "oc -n $NAMESPACE get elasticsearch elasticsearch" ${TIMEOUT_MIN}
+}

hack/testing-olm/utils

@@ -97,6 +97,61 @@ try_until_text() {
   return 1
 }
 
+try_func_until_text() {
+  local func=$1
+  local expected=$2
+  local timeout=$3
+  local now=$(date +%s%3N)
+  local expire=$(($now + $timeout))
+  while [ $now -lt $expire ]; do
+	if [[ $($func) == "${expected}" ]] ; then
+      		echo "Passed"
+      		return 0
+    	fi
+    	now=$(date +%s%3N)
+  done
+  echo "Fail"
+  return 1
+}
+
+try_func_until_text_alt() {
+  local func=$1
+  local expected=$2
+  local alt_expected=$3
+  local timeout=$4
+  local now=$(date +%s%3N)
+  local expire=$(($now + $timeout))
+  while [ $now -lt $expire ]; do
+    f_result=$($func)
+	  if [[ "${f_result}" == "${expected}" ]] ; then
+      		echo "Passed"
+      		return 0
+    elif [[ "${f_result}" == "${alt_expected}" ]]; then
+          echo "Passed"
+          return 0
+    fi
+    now=$(date +%s%3N)
+  done
+  echo "Fail"
+  return 1
+}
+
+try_func_until_result_is_not_empty() {
+  local func=$1
+  local timeout=$2
+  local now=$(date +%s%3N)
+  local expire=$(($now + $timeout))
+  while [ $now -lt $expire ]; do
+	if [[ $($func) != "" ]] ; then
+      		echo "Passed"
+      		return 0
+    	fi
+    	now=$(date +%s%3N)
+  done
+  echo "Fail"
+  return 1
+}
+
 gather_logging_resources() {
   set +e
   local LOGGING_NS=$1
@@ -166,3 +221,79 @@ function deploy_elasticsearch_operator() {
   # install the elasticsearch operator from that catalog
   ${repo_dir}/olm_deploy/scripts/operator-install.sh
 }
+
+deploy_marketplace_operator(){
+  local ns=$1
+  local name=$2
+  local channel=$3
+  local package=${4:-$name}
+  local global=${5:-false}
+  if [ "${global}" = "false" ] ; then
+    cat <<EOL | oc create -f -
+apiVersion: v1
+kind: List
+items:
+- apiVersion: v1
+  kind: Namespace
+  metadata:
+    name: "$ns"
+- apiVersion: operators.coreos.com/v1
+  kind: OperatorGroup
+  metadata:
+    name: "$ns"
+    namespace: "$ns"
+  spec:
+    targetNamespaces:
+    - "$ns"
+    packages: "$name"
+- apiVersion: operators.coreos.com/v1alpha1
+  kind: Subscription
+  metadata:
+    name: "$name"
+    namespace: "$ns"
+  spec:
+    channel: "$channel"
+    installPlanApproval: Automatic
+    name: "$package"
+    source: redhat-operators
+    sourceNamespace: openshift-marketplace
+EOL
+  else
+    cat <<EOL | oc create -f -
+apiVersion: v1
+kind: List
+items:
+- apiVersion: v1
+  kind: Namespace
+  metadata:
+    name: "$ns"
+- apiVersion: operators.coreos.com/v1
+  kind: OperatorGroup
+  metadata:
+    name: "$name"
+    namespace: "$ns"
+  spec:
+    packages: "$name"
+- apiVersion: operators.coreos.com/v1alpha1
+  kind: Subscription
+  metadata:
+    name: "$name"
+    namespace: "$ns"
+  spec:
+    channel: "$channel"
+    installPlanApproval: Automatic
+    name: "$package"
+    source: redhat-operators
+    sourceNamespace: openshift-marketplace
+EOL
+
+fi
+  wait_for_deployment_to_be_ready "$ns" "$name" $((2 * "$minute"))
+}
+
+wait_for_deployment_to_be_ready(){
+  local namespace=$1
+  local name=$2
+  local timeout=$3
+  try_until_text "oc -n $namespace get deployment $name -o jsonpath={.status.availableReplicas} --ignore-not-found" "1" $timeout
+}