diff --git a/hack/cr.yaml b/hack/cr.yaml index 9d549d4db..1b695f980 100644 --- a/hack/cr.yaml +++ b/hack/cr.yaml @@ -12,7 +12,7 @@ spec: limits: memory: 1Gi requests: - cpu: 500m + cpu: 100m memory: 1Gi nodes: - nodeCount: 1 diff --git a/hack/testing-olm-upgrade/resources/cr.yaml b/hack/testing-olm-upgrade/resources/cr.yaml new file mode 100644 index 000000000..f8402c1e3 --- /dev/null +++ b/hack/testing-olm-upgrade/resources/cr.yaml @@ -0,0 +1,43 @@ +--- +apiVersion: "logging.openshift.io/v1" +kind: "Elasticsearch" +metadata: + name: "elasticsearch" + annotations: + elasticsearch.openshift.io/loglevel: trace +spec: + managementState: "Managed" + nodeSpec: + resources: + limits: + memory: 1Gi + requests: + cpu: 100m + memory: 1Gi + nodes: + - nodeCount: 3 + roles: + - client + - data + - master + storage: + storageClassName: gp2 + size: 10G + redundancyPolicy: SingleRedundancy + indexManagement: + policies: + - name: infra-policy + pollInterval: 30m + phases: + hot: + actions: + rollover: + maxAge: 2m + delete: + minAge: 5m + mappings: + - name: infra + policyRef: infra-policy + aliases: + - infra + - logs.infra diff --git a/hack/testing-olm-upgrade/test-020-olm-upgrade-4.4-4.5.sh b/hack/testing-olm-upgrade/test-020-olm-upgrade-4.4-4.5.sh new file mode 100755 index 000000000..35df20070 --- /dev/null +++ b/hack/testing-olm-upgrade/test-020-olm-upgrade-4.4-4.5.sh @@ -0,0 +1,194 @@ +#!/bin/bash +# Given an OLM manifest, verify a green field deployment +# of cluster logging by asserting CLO creates the resources +# that begets the operands that make up logging. + +set -euo pipefail + +repo_dir="$( cd "$(dirname "$0")/../.." ; pwd -P )" +source "$repo_dir/hack/testing-olm/utils" +source "$repo_dir/hack/testing-olm/assertions" + +start_seconds=$(date +%s) + +ARTIFACT_DIR=${ARTIFACT_DIR:-"$repo_dir/_output/"} +ARTIFACT_DIR="${ARTIFACT_DIR}/$(basename ${BASH_SOURCE[0]})" +if [ ! -d $ARTIFACT_DIR ] ; then + mkdir -p $ARTIFACT_DIR +fi + +KUBECONFIG=${KUBECONFIG:-$HOME/.kube/config} +TIMEOUT_MIN=$((2 * $minute)) +ES_POD_TIMEOUT=$((5 * $minute)) +export CLUSTER_LOGGING_OPERATOR_NAMESPACE="openshift-logging" +manifest_dir=${repo_dir}/manifests +version="4.5" +previous_version="4.4" + +cleanup(){ + local return_code="$?" + set +e + log::info "Running cleanup" + end_seconds=$(date +%s) + runtime="$(($end_seconds - $start_seconds))s" + oc -n openshift-operators-redhat -o yaml get subscription elasticsearch-operator > $ARTIFACT_DIR/subscription-eo.yml 2>&1 ||: + oc -n openshift-operators-redhat -o yaml get deployment/elasticsearch-operator > $ARTIFACT_DIR/elasticsearch-operator-deployment.yml 2>&1 ||: + oc -n openshift-operators-redhat -o yaml get pods > $ARTIFACT_DIR/openshift-operators-redhat-pods.yml 2>&1 ||: + oc -n openshift-operators-redhat -o yaml get configmaps > $ARTIFACT_DIR/openshift-operators-redhat-configmaps.yml 2>&1 ||: + oc -n openshift-operators-redhat describe deployment/elasticsearch-operator > $ARTIFACT_DIR/elasticsearch-operator.describe 2>&1 ||: + + oc logs -n "openshift-operators-redhat" deployment/elasticsearch-operator > $ARTIFACT_DIR/elasticsearch-operator.log 2>&1 ||: + oc -n openshift-operator-lifecycle-manager logs --since=$runtime deployment/catalog-operator > $ARTIFACT_DIR/catalog-operator.logs 2>&1 ||: + oc -n openshift-operator-lifecycle-manager logs --since=$runtime deployment/olm-operator > $ARTIFACT_DIR/olm-operator.logs 2>&1 ||: + + ${repo_dir}/olm_deploy/scripts/operator-uninstall.sh + ${repo_dir}/olm_deploy/scripts/catalog-uninstall.sh + + set -e + exit ${return_code} +} +trap cleanup exit + +get_es_pods_count() { + oc -n openshift-operators-redhat get pods -l component=elasticsearch --no-headers=true --ignore-not-found | wc -l +} + +get_es_cluster_status() { + oc -n openshift-operators-redhat get pods -l component=elasticsearch --no-headers=true --ignore-not-found \ + | awk 'NR==1{print $1}' \ + | xargs -I '{}' oc -n openshift-operators-redhat exec '{}' -c elasticsearch -- es_util --query=_cluster/health \ + | jq .status +} + +get_es_indices() { + oc -n openshift-operators-redhat get pods -l component=elasticsearch --no-headers=true --ignore-not-found \ + | awk 'NR==1{print $1}' \ + | xargs -I '{}' oc -n openshift-operators-redhat exec '{}' -c elasticsearch -- es_util --query=_cat/indices +} + +read_es_indices() { + local -n map=$1 + while IFS= read -r line; do + id="$(echo "$line" | awk '{print $3}')" + echo "Index: '$id' stored" + map[$id]=$line + done <<< "$(get_es_indices)" +} + +compare_indices_names(){ + local failed=false + local -n old_m=$1 + local -n new_m=$2 + for k in "${!old_m[@]}"; do + echo "Checking index $k..." + if [ ! ${new_m[$k]+_} ]; then + failed=true + printf "\t'%s' is missing in the set of indices\n" "$k" + else + printf "\t'%s' found\n" "$k" + fi + done + if [ "$failed" = true ]; then + echo "Compare indices names failed!!!" + return 1 + fi + echo "Indices names are fully matched" +} + +es_cluster_ready() { + oc -n openshift-operators-redhat get pods -l component=elasticsearch \ + --no-headers=true --ignore-not-found \ + | awk '{print $2}' | grep '2/2' +} + +# deploy elasticsearch-operator +log::info "Deploying elasticsearch-operator ${previous_version} from marketplace..." +deploy_marketplace_operator "openshift-operators-redhat" "elasticsearch-operator" "$previous_version" "elasticsearch-operator" + +# check if the operator is running +log::info "Verifying if elasticsearch-operator deployment is ready..." +try_until_text "oc -n openshift-operators-redhat get deployment elasticsearch-operator -o jsonpath={.status.updatedReplicas} --ignore-not-found" "1" ${TIMEOUT_MIN} + +log::info "Deploying ES secrets..." +rm -rf /tmp/example-secrets ||: \ + mkdir /tmp/example-secrets && \ + "$repo_dir"/hack/cert_generation.sh /tmp/example-secrets "openshift-operators-redhat" elasticsearch +"$repo_dir"/hack/deploy-example-secrets.sh "openshift-operators-redhat" + +# deploy elasticsearch CR +log::info "Deploying ES CR..." +oc -n "openshift-operators-redhat" create -f ${repo_dir}/hack/testing-olm-upgrade/resources/cr.yaml + +# check if there are elasticsearch pod +log::info "Checking if ES pod is ready in the namespace openshift-operators-redhat..." +try_until_text "oc -n openshift-operators-redhat get pods -l component=elasticsearch -o jsonpath={.items[0].status.phase} --ignore-not-found" "Running" ${ES_POD_TIMEOUT} + +log::info "Getting the previous statate of elasticsearch-operator deployment" +# get the previous status of the elasticsearch-operator +oc describe -n "openshift-operators-redhat" deployment/elasticsearch-operator > "$ARTIFACT_DIR"/elasticsearch-operator.describe.before_update 2>&1 + +# check if ES cluster has 3 running pods +log::info "Checking if the ES has 3 nodes" +try_func_until_text get_es_pods_count "3" ${ES_POD_TIMEOUT} + +log::info "Check if at least 1 node is ready (2/2 containers running)" +try_until_success es_cluster_ready ${ES_POD_TIMEOUT} + +# check if ES cluster is in green state +log::info "Checking if the ES cluster is all yellow/green" +try_func_until_text_alt get_es_cluster_status "\"green\"" "\"yellow\"" ${ES_POD_TIMEOUT} + +# read OLD 4.4 indices into and map them by their names +log::info "Reading old ES indices" +try_func_until_result_is_not_empty get_es_indices ${ES_POD_TIMEOUT} +declare -A old_indices +read_es_indices old_indices + +#### INSTALLING 4.5 +log::info "Deploying the ES operator from the catalog..." +# deploy cluster logging catalog from local code +"${repo_dir}"/olm_deploy/scripts/catalog-deploy.sh + +log::info "Patching subscription..." +# patch subscription +payload="{\"op\":\"replace\",\"path\":\"/spec/source\",\"value\":\"elasticsearch-catalog\"}" +payload="$payload,{\"op\":\"replace\",\"path\":\"/spec/sourceNamespace\",\"value\":\"openshift-operators-redhat\"}" +payload="$payload,{\"op\":\"replace\",\"path\":\"/spec/channel\",\"value\":\"$version\"}" +oc -n "openshift-operators-redhat" patch subscription elasticsearch-operator --type json -p "[$payload]" + +# patch the minikube version +log::info "Patching minKubeVersion to 1.16.1..." +pl="{\"op\":\"replace\",\"path\":\"/spec/minKubeVersion\",\"value\":\"1.16.1\"}" +try_until_success "oc -n openshift-operators-redhat patch clusterserviceversion elasticsearch-operator.v${version}.0 --type json -p [$pl]" ${TIMEOUT_MIN} + +#verify deployment is rolled out +log::info "Checking if deployment successfully updated..." +OPENSHIFT_VERSION=${OPENSHIFT_VERSION:-4.5} +IMAGE_ELASTICSEARCH_OPERATOR=${IMAGE_ELASTICSEARCH_OPERATOR:-registry.svc.ci.openshift.org/ocp/${OPENSHIFT_VERSION}:elasticsearch-operator} +if [ -n "${IMAGE_FORMAT:-}" ] ; then + IMAGE_ELASTICSEARCH_OPERATOR=$(echo $IMAGE_FORMAT | sed -e "s,\${component},elasticsearch-operator,") +fi +log::info "Checking if deployment version is ${IMAGE_ELASTICSEARCH_OPERATOR}..." +try_until_text "oc -n openshift-operators-redhat get deployment elasticsearch-operator -o jsonpath={.spec.template.spec.containers[0].image}" "${IMAGE_ELASTICSEARCH_OPERATOR}" ${TIMEOUT_MIN} + +log::info "Checking if the ES pod is running..." +try_until_text "oc -n openshift-operators-redhat get deployment -l component=elasticsearch -o jsonpath={.items[0].status.updatedReplicas} --ignore-not-found" "1" ${ES_POD_TIMEOUT} + +# check if ES cluster has 3 running pods +log::info "Checking if the ES cluster still has 3 nodes..." +try_func_until_text get_es_pods_count "3" ${ES_POD_TIMEOUT} + +log::info "Check if at least 1 node is ready (2/2 containers running)" +try_until_success es_cluster_ready ${ES_POD_TIMEOUT} + +# check if ES cluster is in green state +log::info "Checking if the ES cluster is all yellow/green" +try_func_until_text_alt get_es_cluster_status "\"green\"" "\"yellow\"" ${ES_POD_TIMEOUT} + +# read new 4.5 indices and map them by their names +log::info "Reading new ES indices" +try_func_until_result_is_not_empty get_es_indices ${ES_POD_TIMEOUT} +declare -A new_indices +read_es_indices new_indices + +compare_indices_names old_indices new_indices \ No newline at end of file diff --git a/hack/testing-olm/assertions b/hack/testing-olm/assertions new file mode 100644 index 000000000..a24336e10 --- /dev/null +++ b/hack/testing-olm/assertions @@ -0,0 +1,6 @@ +#!/bin/bash +source "$( cd "$( dirname "${BASH_SOURCE[0]}" )" >/dev/null 2>&1 && pwd )/utils" +assert_eo_exist(){ + # verify ER + try_until_success "oc -n $NAMESPACE get elasticsearch elasticsearch" ${TIMEOUT_MIN} +} \ No newline at end of file diff --git a/hack/testing-olm/utils b/hack/testing-olm/utils index bd54a4f8c..2b212bfea 100644 --- a/hack/testing-olm/utils +++ b/hack/testing-olm/utils @@ -97,6 +97,61 @@ try_until_text() { return 1 } +try_func_until_text() { + local func=$1 + local expected=$2 + local timeout=$3 + local now=$(date +%s%3N) + local expire=$(($now + $timeout)) + while [ $now -lt $expire ]; do + if [[ $($func) == "${expected}" ]] ; then + echo "Passed" + return 0 + fi + now=$(date +%s%3N) + done + echo "Fail" + return 1 +} + +try_func_until_text_alt() { + local func=$1 + local expected=$2 + local alt_expected=$3 + local timeout=$4 + local now=$(date +%s%3N) + local expire=$(($now + $timeout)) + while [ $now -lt $expire ]; do + f_result=$($func) + if [[ "${f_result}" == "${expected}" ]] ; then + echo "Passed" + return 0 + elif [[ "${f_result}" == "${alt_expected}" ]]; then + echo "Passed" + return 0 + fi + now=$(date +%s%3N) + done + echo "Fail" + return 1 +} + +try_func_until_result_is_not_empty() { + local func=$1 + local timeout=$2 + local now=$(date +%s%3N) + local expire=$(($now + $timeout)) + while [ $now -lt $expire ]; do + if [[ $($func) != "" ]] ; then + echo "Passed" + return 0 + fi + now=$(date +%s%3N) + done + echo "Fail" + return 1 +} + gather_logging_resources() { set +e local LOGGING_NS=$1 @@ -166,3 +221,79 @@ function deploy_elasticsearch_operator() { # install the elasticsearch operator from that catalog ${repo_dir}/olm_deploy/scripts/operator-install.sh } + +deploy_marketplace_operator(){ + local ns=$1 + local name=$2 + local channel=$3 + local package=${4:-$name} + local global=${5:-false} + if [ "${global}" = "false" ] ; then + cat <