From d0732c43f1c57eb21587187c531b5445422c1456 Mon Sep 17 00:00:00 2001
From: Eric Wolinetz <ewolinet@redhat.com>
Date: Wed, 17 Mar 2021 17:34:03 -0500
Subject: [PATCH] Adding logic to perform rollover and delete for json write
 indices

---
 internal/indexmanagement/scripts.go | 380 ++++++++++++++++------------
 1 file changed, 218 insertions(+), 162 deletions(-)

diff --git a/internal/indexmanagement/scripts.go b/internal/indexmanagement/scripts.go
index 114bbbea9..d65e78385 100644
--- a/internal/indexmanagement/scripts.go
+++ b/internal/indexmanagement/scripts.go
@@ -132,24 +132,18 @@ function getWriteIndex() {
   local policy="$1"
 
   # find out the current write index for ${POLICY_MAPPING}-write and check if there is the next generation of it
-  aliasResponse=$(curl -s $ES_SERVICE/_alias/${policy} \
-    --cacert /etc/indexmanagement/keys/admin-ca \
-    --connect-timeout ${CONNECT_TIMEOUT} \
-    -H"Authorization: Bearer $(cat /var/run/secrets/kubernetes.io/serviceaccount/token)" \
-    -HContent-Type:application/json \
-    --retry 5 \
-    --retry-delay 5)
+  aliasResponse="$(getAlias "${policy}")"
 
   if [ -z "$aliasResponse" ]; then
     echo "Received an empty response from elasticsearch -- server may not be ready"
-    exit 1
+    return 1
   fi
 
   jsonResponse="$(echo [$aliasResponse] | sed 's/}{/},{/g')"
 
   if ! writeIndices="$(python /tmp/scripts/getWriteIndex.py "${policy}" "$jsonResponse")" ; then
     echo $writeIndices
-    exit 1
+    return 1
   fi
 
   writeIndex="$(ensureOneWriteIndex "$policy" "$writeIndices")"
@@ -169,196 +163,258 @@ function ensureOneWriteIndex() {
       writeIndex="$index"
     else
       # extra write index -- mark it as not a write index
-      curl -s "$ES_SERVICE/_aliases" \
-        --connect-timeout ${CONNECT_TIMEOUT} \
-        --cacert /etc/indexmanagement/keys/admin-ca \
-        -HContent-Type:application/json \
-        -XPOST \
-        -H"Authorization: Bearer $(cat /var/run/secrets/kubernetes.io/serviceaccount/token)" \
-        -o /dev/null \
-        -d '{"actions":[{"add":{"index": "'$index'", "alias": "'${POLICY_MAPPING}-write'", "is_write_index": false}}]}' \
-        --retry 5 \
-        --retry-delay 5
+      removeAsWriteIndexForAlias "$index" "$policy"
     fi
   done
 
   echo $writeIndex
 }
-`
 
-const rolloverScript = `
-set -euo pipefail
-source /tmp/scripts/indexManagement
+function getWriteAliases() {
+  local policy="$1"
 
-decoded=$(echo $PAYLOAD | base64 -d)
+  # "_cat/aliases/${policy}*-write?h=alias" | uniq
+  aliasResponse="$(catWriteAliases "$policy")"
 
-echo "Index management rollover process starting"
+  if [ -z "$aliasResponse" ]; then
+    echo "Received an empty response from elasticsearch -- server may not be ready"
+    return 1
+  fi
 
-# get current write index
-if ! writeIndex="$(getWriteIndex "${POLICY_MAPPING}-write")" ; then
-  echo $writeIndex
-  exit 1
-fi
+  echo $aliasResponse
+}
 
-echo "Current write index for ${POLICY_MAPPING}-write: $writeIndex"
+function rollover() {
 
-# try to rollover
-code=$(curl -s "$ES_SERVICE/${POLICY_MAPPING}-write/_rollover?pretty" \
-  -w "%{response_code}" \
-  --connect-timeout ${CONNECT_TIMEOUT} \
-  --cacert /etc/indexmanagement/keys/admin-ca \
-  -HContent-Type:application/json \
-  -XPOST \
-  -H"Authorization: Bearer $(cat /var/run/secrets/kubernetes.io/serviceaccount/token)" \
-  -o /tmp/response.txt \
-  -d $decoded \
-  --retry 5 \
-  --retry-delay 5)
-
-echo "Checking results from _rollover call"
-
-if [ "$code" != "200" ] ; then
-  # already in bad state
-  echo "Calculating next write index based on current write index..."
-
-  indexGeneration="$(echo $writeIndex | cut -d'-' -f2)"
-  writeBase="$(echo $writeIndex | cut -d'-' -f1)"
-
-  # if we don't strip off the leading 0s it does math wrong...
-  generation=$(echo $indexGeneration | sed 's/^0*//')
-  # pad the index name again with 0s
-  nextGeneration="$(printf '%06g' $(($generation + 1)))"
-  nextIndex="$writeBase-$nextGeneration"
-else
-  # check response to see if it did roll over (field in response)
-  if ! nextIndex="$(python /tmp/scripts/checkRollover.py "/tmp/response.txt" "$writeIndex")" ; then
-    echo $nextIndex
-    exit 1
+  local policy="$1"
+  local decoded="$2"
+
+  echo "========================"
+  echo "Index management rollover process starting for $policy"
+  echo ""
+
+  # get current write index
+  if ! writeIndex="$(getWriteIndex "${policy}-write")" ; then
+    echo $writeIndex
+    return 1
   fi
-fi
 
-echo "Next write index for ${POLICY_MAPPING}-write: $nextIndex"
-echo "Checking if $nextIndex exists"
+  echo "Current write index for ${policy}-write: $writeIndex"
 
-# if true, ensure next index was created
-code=$(curl -s "$ES_SERVICE/$nextIndex/" \
-  -w "%{response_code}" \
-  --connect-timeout ${CONNECT_TIMEOUT} \
-  --cacert /etc/indexmanagement/keys/admin-ca \
-  -H"Authorization: Bearer $(cat /var/run/secrets/kubernetes.io/serviceaccount/token)" \
-  -o /dev/null \
-  --retry 5 \
-  --retry-delay 5)
-if [ "$code" == "404" ] ; then
+  # try to rollover
+  code="$(rolloverForPolicy "${policy}-write" "$decoded")"
+
+  echo "Checking results from _rollover call"
+
+  if [ "$code" != "200" ] ; then
+    # already in bad state
+    echo "Calculating next write index based on current write index..."
+
+    indexGeneration="$(echo $writeIndex | cut -d'-' -f2)"
+    writeBase="$(echo $writeIndex | cut -d'-' -f1)"
+
+    # if we don't strip off the leading 0s it does math wrong...
+    generation="$(echo $indexGeneration | sed 's/^0*//')"
+    # pad the index name again with 0s
+    nextGeneration="$(printf '%06g' $(($generation + 1)))"
+    nextIndex="$writeBase-$nextGeneration"
+  else
+    # check response to see if it did roll over (field in response)
+    if ! nextIndex="$(python /tmp/scripts/checkRollover.py "/tmp/response.txt" "$writeIndex")" ; then
+      echo $nextIndex
+      return 1
+    fi
+  fi
+
+  echo "Next write index for ${policy}-write: $nextIndex"
+  echo "Checking if $nextIndex exists"
+
+  # if true, ensure next index was created
+  code="$(checkIndexExists "$nextIndex")"
+  if [ "$code" == "404" ] ; then
+    cat /tmp/response.txt
+    return 1
+  fi
+
+  echo "Checking if $nextIndex is the write index for ${policy}-write"
+
+  ## if true, ensure write-alias points to next index
+  if ! writeIndex="$(getWriteIndex "${policy}-write")" ; then
+    echo $writeIndex
+    return 1
+  fi
+
+  if [ "$nextIndex" == "$writeIndex" ] ; then
+    echo "Done!"
+    return 0
+  fi
+
+  echo "Updating alias for ${policy}-write"
+
+  # else - try to update alias to be correct
+  code="$(updateWriteIndex "$writeIndex" "$nextIndex" "${policy}-write")"
+
+  if [ "$code" == 200 ] ; then
+    echo "Done!"
+    return 0
+  fi
   cat /tmp/response.txt
-  exit 1
-fi
+  return 1
+}
 
-echo "Checking if $nextIndex is the write index for ${POLICY_MAPPING}-write"
+function delete() {
 
-## if true, ensure write-alias points to next index
-if ! writeIndex="$(getWriteIndex "${POLICY_MAPPING}-write")" ; then
-  echo $writeIndex
-  exit 1
-fi
+  local policy="$1"
+  ERRORS="$(mktemp /tmp/delete-XXXXXX)"
 
-if [ "$nextIndex" == "$writeIndex" ] ; then
-  echo "Done!"
-  exit 0
-fi
+  echo "========================"
+  echo "Index management delete process starting for $policy"
+  echo ""
+
+  if ! writeIndex="$(getWriteIndex "${policy}-write")" ; then
+    echo $writeIndex
+    return 1
+  fi
+
+  indices="$(getIndicesAgeForAlias "${policy}-write")"
 
-echo "Updating alias for ${POLICY_MAPPING}-write"
+  if [ -z "$indices" ]; then
+    echo "Received an empty response from elasticsearch -- server may not be ready"
+    return 1
+  fi
 
-# else - try to update alias to be correct
-code=$(curl -s "$ES_SERVICE/_aliases" \
-  -w "%{response_code}" \
-  --connect-timeout ${CONNECT_TIMEOUT} \
+  jsonResponse="$(echo [$indices] | sed 's/}{/},{/g')"
+
+  # Delete in batches of 25 for cases where there are a large number of indices to remove
+  nowInMillis=$(date +%s%3N)
+  minAgeFromEpoc=$(($nowInMillis - $MIN_AGE))
+  if ! indices=$(python /tmp/scripts/getNext25Indices.py "$minAgeFromEpoc" "$writeIndex" "$jsonResponse" 2>>$ERRORS) ; then
+    cat $ERRORS
+    rm $ERRORS
+    return 1
+  fi
+  # Dump any findings to stdout but don't error
+  if [ -s $ERRORS ]; then
+    cat $ERRORS
+    rm $ERRORS
+  fi
+
+  if [ "${indices}" == "" ] ; then
+      echo No indices to delete
+      return 0
+  else
+      echo deleting indices: "${indices}"
+  fi
+
+  for sets in ${indices}; do
+    code="$(deleteIndices "${sets}")"
+
+    if [ "$code" != 200 ] ; then
+      cat /tmp/response.txt
+      return 1
+    fi
+  done
+
+  echo "Done!"
+}
+
+function curlES() {
+  curl -s \
+  --connect-timeout "${CONNECT_TIMEOUT}" \
   --cacert /etc/indexmanagement/keys/admin-ca \
   -HContent-Type:application/json \
-  -XPOST \
   -H"Authorization: Bearer $(cat /var/run/secrets/kubernetes.io/serviceaccount/token)" \
-  -o /tmp/response.txt \
-  -d '{"actions":[{"add":{"index": "'$writeIndex'", "alias": "'${POLICY_MAPPING}-write'", "is_write_index": false}},{"add":{"index": "'$nextIndex'", "alias": "'${POLICY_MAPPING}-write'", "is_write_index": true}}]}' \
   --retry 5 \
-  --retry-delay 5)
+  --retry-delay 5 \
+  "$@"
+}
 
-if [ "$code" == 200 ] ; then
-  echo "Done!"
-  exit 0
-fi
-cat /tmp/response.txt
-exit 1
+function getAlias() {
+  local alias="$1"
+
+  curlES "$ES_SERVICE/_alias/${alias}"
+}
+
+function getIndicesAgeForAlias() {
+  local alias="$1"
+
+  curlES "$ES_SERVICE/${alias}/_settings/index.creation_date"
+}
+
+function deleteIndices() {
+  local index="$1"
+
+  curlES "$ES_SERVICE/${index}?pretty" -w "%{response_code}" -o /tmp/response.txt -XDELETE
+}
+
+function removeAsWriteIndexForAlias() {
+  local index="$1"
+  local alias="$2"
+
+  curlES "$ES_SERVICE/_aliases" -o /dev/null -d '{"actions":[{"add":{"index": "'$index'", "alias": "'$alias'", "is_write_index": false}}]}'
+}
+
+function catWriteAliases() {
+  local policy="$1"
+
+  curlES "$ES_SERVICE/_cat/aliases/${policy}*-write?h=alias" | sort | uniq
+}
+
+function rolloverForPolicy() {
+  local policy="$1"
+  local decoded="$2"
+
+  curlES "$ES_SERVICE/${policy}/_rollover?pretty" -w "%{response_code}" -XPOST -o /tmp/response.txt -d $decoded
+}
+
+function checkIndexExists() {
+  local index="$1"
+
+  curlES "$ES_SERVICE/${index}" -w "%{response_code}" -o /dev/null
+}
+
+function updateWriteIndex() {
+  currentIndex="$1"
+  nextIndex="2"
+  alias="$3"
+
+  curlES "$ES_SERVICE/_aliases" -w "%{response_code}" -XPOST -o /tmp/response.txt -d '{"actions":[{"add":{"index": "'$currentIndex'", "alias": "'$alias'", "is_write_index": false}},{"add":{"index": "'$nextIndex'", "alias": "'$alias'", "is_write_index": true}}]}'
+}
+`
+
+const rolloverScript = `
+set -euo pipefail
+source /tmp/scripts/indexManagement
+
+decoded=$(echo $PAYLOAD | base64 -d)
+
+# need to get a list of all mappings under ${POLICY_MAPPING}, drop suffix '-write' iterate over
+writeAliases="$(getWriteAliases "$POLICY_MAPPING")"
+
+for aliasBase in $writeAliases; do
+
+  alias="$(echo $aliasBase | sed 's/-write$//g')"
+  if ! rollover "$alias" "$decoded" ; then
+    exit 1
+  fi
+done
 `
 
 const deleteScript = `
 set -uo pipefail
-ERRORS=/tmp/errors.txt
 
 source /tmp/scripts/indexManagement
 
-echo "" > $ERRORS
-
-echo "Index management delete process starting"
+# need to get a list of all mappings under ${POLICY_MAPPING}, drop suffix '-write' iterate over
+writeAliases="$(getWriteAliases "$POLICY_MAPPING")"
 
-if ! writeIndex="$(getWriteIndex "${POLICY_MAPPING}-write")" ; then
-  echo $writeIndex
-  exit 1
-fi
+for aliasBase in $writeAliases; do
 
-indices=$(curl -s $ES_SERVICE/${POLICY_MAPPING}/_settings/index.creation_date \
-  --cacert /etc/indexmanagement/keys/admin-ca \
-  --connect-timeout ${CONNECT_TIMEOUT} \
-  -H"Authorization: Bearer $(cat /var/run/secrets/kubernetes.io/serviceaccount/token)" \
-  -HContent-Type:application/json \
-  --retry 5 \
-  --retry-delay 5)
-
-if [ -z "$indices" ]; then
-  echo "Received an empty response from elasticsearch -- server may not be ready"
-  exit 1
-fi
-
-jsonResponse="$(echo [$indices] | sed 's/}{/},{/g')"
-
-# Delete in batches of 25 for cases where there are a large number of indices to remove
-nowInMillis=$(date +%s%3N)
-minAgeFromEpoc=$(($nowInMillis - $MIN_AGE))
-
-if ! indices=$(python /tmp/scripts/getNext25Indices.py "$minAgeFromEpoc" "$writeIndex" "$jsonResponse" 2>>$ERRORS) ; then
-  cat $ERRORS
-  exit 1
-fi
-# Dump any findings to stdout but don't error
-if [ -s $ERRORS ]; then
-  cat $ERRORS
-fi
-
-if [ "${indices}" == "" ] ; then
-    echo No indices to delete
-    exit 0
-else
-    echo deleting indices: "${indices}"
-fi
-
-for sets in ${indices}; do
-  code=$(curl -s $ES_SERVICE/${sets}?pretty \
-    -w "%{response_code}" \
-    --connect-timeout ${CONNECT_TIMEOUT} \
-    --cacert /etc/indexmanagement/keys/admin-ca \
-    -HContent-Type:application/json \
-    -H"Authorization: Bearer $(cat /var/run/secrets/kubernetes.io/serviceaccount/token)" \
-    -o /tmp/response.txt \
-    -XDELETE \
-    --retry 5 \
-    --retry-delay 5)
-
-  if [ "$code" != 200 ] ; then
-    cat /tmp/response.txt
+  alias="$(echo $aliasBase | sed 's/-write$//g')"
+  if ! delete "$alias" ; then
     exit 1
   fi
 done
-
-echo "Done!"
 `
 
 var scriptMap = map[string]string{