Bug 1395170 - Logging upgrade mode: Upgrade pod log states "No matchi…

…ng indexes found - skipping update_for_common_data_model"

https://bugzilla.redhat.com/show_bug.cgi?id=1395170
If the admin-cert exists, upgrade will skip the uuid_migrate step
which sets up the cert/key needed to use curl for the common data
model upgrade code.  The fix is to call those shell functions as
needed if the variables and files do not exist.
This also allows test-upgrade.sh to be run standalone, outside of
the context of logging.sh, and tests specifically for the existing
admin-cert case by skipping the removeAdminCert step in the test.
Also changes the tests so that they clean up old indices created
for testing.

deployer/scripts/upgrade.sh

@@ -655,10 +655,14 @@ function update_es_for_235() {
 # daterx - the date regex that matches the .%Y.%m.%d at the end of the indices
 # we are interested in - the awk will strip that part off
 function get_list_of_proj_uuid_indices() {
+    set -o pipefail
     curl -s --cacert $CA --key $KEY --cert $CERT https://$es_host:$es_port/_cat/indices | \
         awk -v daterx='[.]20[0-9]{2}[.][0-1]?[0-9][.][0-9]{1,2}$' \
             '$3 !~ "^[.]" && $3 !~ "^project." && $3 ~ daterx {print gensub(daterx, "", 1, $3)}' | \
-        sort -u
+        sort -u || { rc=$?; set +o pipefail; >&2 echo Error $rc getting list of indices; return $rc; }
+    rc=$?
+    set +o pipefail
+    return $rc
 }
 
 function update_for_common_data_model() {
@@ -766,13 +770,16 @@ function upgrade_logging() {
   if [[ $installedVersion -ne $LOGGING_VERSION ]]; then
     if [[ -n "$migrate" ]]; then
       uuid_migrate
-    elif [[ -n "$common_data_model" ]] ; then
-      # set these in case uuid_migrate did not
-      initialize_es_vars
     fi
     if [[ -n "$common_data_model" ]] ; then
       # make sure these env. vars. are exported inside the function
       # to be available to all pipes, subshells, etc.
+      if [ -z "${CERT:-}" ] ; then
+        initialize_es_vars
+      fi
+      if [ ! -f $CERT ] ; then
+        recreate_admin_certs
+      fi
       PROJ_PREFIX=project. CA=$CA KEY=$KEY CERT=$CERT es_host=$es_host es_port=$es_port update_for_common_data_model
     fi
 

deployer/scripts/util.sh

@@ -190,3 +190,131 @@ function extract_nodeselector() {
     echo nodeSelector: "{" $(join , "${selectors[@]}") "}"
   fi
 }
+
+function wait_for_latest_build_complete() {
+
+  interval=30
+  waittime=120
+
+  local bc=$1
+  local lastVersion=$(oc get bc $bc -o jsonpath='{.status.lastVersion}')
+  local status
+
+  for (( i = 1; i <= $waittime; i++ )); do
+    status=$(oc get build/$bc-$lastVersion -o jsonpath='{.status.phase}')
+    case $status in
+      "Complete")
+        return 0
+        ;;
+      "Failed")
+        return 1
+        ;;
+      "Pending"|"Running")
+        sleep $interval
+        ;;
+    esac
+  done
+
+  return 1
+}
+
+function wait_for_new_builds_complete() {
+
+  retries=30
+  for bc in $(oc get bc -l logging-infra -o jsonpath='{.items[*].metadata.name}'); do
+
+    for (( i = 1; i <= retries; i++ )); do
+
+      wait_for_latest_build_complete "$bc" && break
+
+      [[ $i -eq $retries ]] && return 1
+
+      oc delete builds -l buildconfig=$bc
+
+      if [ "$USE_LOCAL_SOURCE" = false ] ; then
+          oc start-build $bc
+      else
+          oc start-build --from-dir $OS_O_A_L_DIR $bc
+      fi
+    done
+
+  done
+
+  return 0
+}
+
+function wait_for_builds_complete()
+{
+    waittime=3600 # seconds - 1 hour
+    interval=60
+    complete=0
+    while [ $waittime -gt 0 -a $complete = 0 ] ; do
+        # all lines must have $4 == "Complete"
+        complete=`oc get builds | awk '$4 == "STATUS" || $4 == "Complete" {complete++}; END {print NR == complete}'`
+        if [ $complete = 1 ] ; then
+            echo Builds are complete
+            break
+        fi
+        # restart failed builds
+        # get a list of the new failures
+        curfailedbuilds=`oc get builds | awk '$4 == "Failed" {print $1}'`
+        for build in $curfailedbuilds ; do
+            # get the bc
+            bc=`oc get build $build --template='{{.metadata.labels.buildconfig}}'`
+            # see if there is a build in progress for this bc
+            statuses=`oc describe bc $bc | awk -v pre=$bc '$1 ~ pre {print $2}'`
+            needbuild=0
+            for status in $statuses ; do
+                case $status in
+                "running"|"complete"|"pending")
+                    echo build in progress for $bc - delete failed build $build status $status
+                    # delete the failed build - otherwise it will show up in the list and
+                    # the main loop will never Complete
+                    oc logs build/$build > $LOG_DIR/build-$build.log 2>&1
+                    oc delete build $build
+                    needbuild=0
+                    break
+                    ;;
+                "failed")
+                    # if the build failed, there will be at least 1 failed status
+                    # if there is another build running or complete, it will be
+                    # detected above
+                    needbuild=1
+                    ;;
+                esac
+            done
+            # if we are here and needbuild=1, there were no running or complete builds
+            if [ $needbuild = "1" ] ; then
+                # start a new build
+                if [ "$USE_LOCAL_SOURCE" = false ] ; then
+                    oc start-build $bc
+                else
+                    oc start-build --from-dir $OS_O_A_L_DIR $bc
+                fi
+            fi
+        done
+        sleep $interval
+        waittime=`expr $waittime - $interval`
+    done
+    if [ $complete = 0 ] ; then
+        echo error builds are not complete
+        oc get builds
+        return 1
+    fi
+    return 0
+}
+
+function get_running_pod() {
+    # $1 is component for selector
+    oc get pods -l component=$1 | awk -v sel=$1 '$1 ~ sel && $3 == "Running" {print $1}'
+}
+
+function get_latest_pod() {
+
+  label=$1
+
+  local times=(`oc get pods -l $label -o jsonpath='{.items[*].metadata.creationTimestamp}' | xargs -n1 | sort -r | xargs`)
+  local pod=$(oc get pods -l $label -o jsonpath="{.items[?(@.metadata.creationTimestamp==\"${times[0]}\")].metadata.name}")
+
+  echo $pod
+}

hack/testing/logging.sh

@@ -107,134 +107,6 @@ function cleanup()
     return $out
 }
 
-function wait_for_latest_build_complete() {
-
-  interval=30
-  waittime=120
-
-  local bc=$1
-  local lastVersion=$(oc get bc $bc -o jsonpath='{.status.lastVersion}')
-  local status
-
-  for (( i = 1; i <= $waittime; i++ )); do
-    status=$(oc get build/$bc-$lastVersion -o jsonpath='{.status.phase}')
-    case $status in
-      "Complete")
-        return 0
-        ;;
-      "Failed")
-        return 1
-        ;;
-      "Pending"|"Running")
-        sleep $interval
-        ;;
-    esac
-  done
-
-  return 1
-}
-
-function wait_for_new_builds_complete() {
-
-  retries=30
-  for bc in $(oc get bc -l logging-infra -o jsonpath='{.items[*].metadata.name}'); do
-
-    for (( i = 1; i <= retries; i++ )); do
-
-      wait_for_latest_build_complete "$bc" && break
-
-      [[ $i -eq $retries ]] && return 1
-
-      oc delete builds -l buildconfig=$bc
-
-      if [ "$USE_LOCAL_SOURCE" = false ] ; then
-          oc start-build $bc
-      else
-          oc start-build --from-dir $OS_O_A_L_DIR $bc
-      fi
-    done
-
-  done
-
-  return 0
-}
-
-function wait_for_builds_complete()
-{
-    waittime=3600 # seconds - 1 hour
-    interval=60
-    complete=0
-    while [ $waittime -gt 0 -a $complete = 0 ] ; do
-        # all lines must have $4 == "Complete"
-        complete=`oc get builds | awk '$4 == "STATUS" || $4 == "Complete" {complete++}; END {print NR == complete}'`
-        if [ $complete = 1 ] ; then
-            echo Builds are complete
-            break
-        fi
-        # restart failed builds
-        # get a list of the new failures
-        curfailedbuilds=`oc get builds | awk '$4 == "Failed" {print $1}'`
-        for build in $curfailedbuilds ; do
-            # get the bc
-            bc=`oc get build $build --template='{{.metadata.labels.buildconfig}}'`
-            # see if there is a build in progress for this bc
-            statuses=`oc describe bc $bc | awk -v pre=$bc '$1 ~ pre {print $2}'`
-            needbuild=0
-            for status in $statuses ; do
-                case $status in
-                "running"|"complete"|"pending")
-                    echo build in progress for $bc - delete failed build $build status $status
-                    # delete the failed build - otherwise it will show up in the list and
-                    # the main loop will never Complete
-                    oc logs build/$build > $LOG_DIR/build-$build.log 2>&1
-                    oc delete build $build
-                    needbuild=0
-                    break
-                    ;;
-                "failed")
-                    # if the build failed, there will be at least 1 failed status
-                    # if there is another build running or complete, it will be
-                    # detected above
-                    needbuild=1
-                    ;;
-                esac
-            done
-            # if we are here and needbuild=1, there were no running or complete builds
-            if [ $needbuild = "1" ] ; then
-                # start a new build
-                if [ "$USE_LOCAL_SOURCE" = false ] ; then
-                    oc start-build $bc
-                else
-                    oc start-build --from-dir $OS_O_A_L_DIR $bc
-                fi
-            fi
-        done
-        sleep $interval
-        waittime=`expr $waittime - $interval`
-    done
-    if [ $complete = 0 ] ; then
-        echo error builds are not complete
-        oc get builds
-        return 1
-    fi
-    return 0
-}
-
-function get_running_pod() {
-    # $1 is component for selector
-    oc get pods -l component=$1 | awk -v sel=$1 '$1 ~ sel && $3 == "Running" {print $1}'
-}
-
-function get_latest_pod() {
-
-  label=$1
-
-  local times=(`oc get pods -l $label -o jsonpath='{.items[*].metadata.creationTimestamp}' | xargs -n1 | sort -r | xargs`)
-  local pod=$(oc get pods -l $label -o jsonpath="{.items[?(@.metadata.creationTimestamp==\"${times[0]}\")].metadata.name}")
-
-  echo $pod
-}
-
 trap "exit" INT TERM
 trap "cleanup" EXIT
 

hack/testing/test-curator.sh

@@ -38,6 +38,16 @@ add_message_to_index() {
 }' > /dev/null
 }
 
+delete_indices() {
+    # ops is $1
+    espod=`get_running_pod es${1:-""}`
+    host=logging-es${1:-""}
+    cert=/etc/elasticsearch/secret/admin-cert
+    key=/etc/elasticsearch/secret/admin-key
+    url="https://${host}:9200/*.curatortest.*"
+    oc exec $espod -- curl -s -k --cert $cert --key $key -XDELETE "$url"
+}
+
 get_running_pod() {
     # $1 is component for selector
     oc get pods -l component=$1 | awk -v sel=$1 '$1 ~ sel && $3 == "Running" {print $1}'
@@ -138,7 +148,7 @@ create_indices() {
                 break
             fi
         done
-        add_message_to_index "${this_proj}.$1" "$this_proj $1 message" $myops
+        add_message_to_index "${this_proj}.curatortest.$1" "$this_proj $1 message" $myops
         shift
     done
 }
@@ -154,10 +164,10 @@ verify_indices() {
     rc=0
     while [ -n "${1:-}" ] ; do
         proj="$1" ; shift
-        this_idx="project.${proj}.$1"
+        this_idx="project.${proj}.curatortest.$1"
         for skip in ${skip_list[*]} ; do
             if [ `expr ${proj} : "$skip"` -gt 0 ]; then
-                this_idx="${proj}.$1"
+                this_idx="${proj}.curatortest.$1"
                 break
             fi
         done
@@ -235,6 +245,11 @@ if uses_config_maps ; then
 fi
 
 cleanup() {
+    # delete indices
+    delete_indices
+    if [ "$CLUSTER" = "true" ] ; then
+        delete_indices "$OPS"
+    fi
     if [ -n "${origconfig:-}" -a -f $origconfig ] ; then
         oc delete configmap logging-curator || :
         sleep 1