Update the shared cleanup script to support multiple ASB configs, also deal with the read replica support

helpers/shared-cleanup.sh

@@ -28,48 +28,103 @@ for util in oc jq mysql; do
 	fi
 done
 
-# services with a port are not servicebrokers.
-echo "getting a list of services for cluster $(oc whoami --show-server)..."
-oc get service --all-namespaces -o=jsonpath='{range .items[*]}{.metadata.namespace}{"\t"}{.metadata.name}{"\t"}{.spec.externalName}{"\n"}{end}' | awk '$2 ~ /^mariadb-/ { print }' > mariadb-services
+# Colours.
+shw_grey () {
+  echo $(tput bold)$(tput setaf 0) $@ $(tput sgr 0)
+}
+shw_norm () {
+  echo $(tput bold)$(tput setaf 9) $@ $(tput sgr 0)
+}
+shw_info () {
+  echo $(tput bold)$(tput setaf 4) $@ $(tput sgr 0)
+}
+shw_warn () {
+  echo $(tput bold)$(tput setaf 2) $@ $(tput sgr 0)
+}
+shw_err ()  {
+  echo $(tput bold)$(tput setaf 1) $@ $(tput sgr 0)
+}
 
-# get a list of database servers
-# ignore the dedicated servers
-SERVERS=$(awk '{print $3}' mariadb-services | sort -u | grep -v "^dedicated")
+# Services with a port are not servicebrokers.
+shw_grey "Getting a list of services for cluster $(oc whoami --show-server)."
+oc get service --all-namespaces -o=jsonpath='{range .items[*]}{.metadata.namespace}{"\t"}{.metadata.name}{"\t"}{.spec.externalName}{"\n"}{end}' \
+ | awk '$2 ~ /^mariadb-/ {print}' > /tmp/mariadb-services
+# Remove read replica services.
+sed -i.bak '/mariadb-readreplica-/d' /tmp/mariadb-services
+# Remove random database pods.
+sed -i.bak '/mariadb-d7[[:space:]]*$/d' /tmp/mariadb-services
 
+# Get a list of database clusters:
+# - Ignore the dedicated clusters.
+# - Ignore the read replicas.
+SERVERS=$(awk '{print $3}' /tmp/mariadb-services | sort -u | grep -v "^dedicated" | grep -v ".cluster-ro-")
+
+# Ensure you can connect to all database clusters, once you do that, list every
+# database that you can that belongs to the Ansible Service Broker.
 for SERVER in $SERVERS; do
-  CONFFILE=${HOME}/.my.cnf-${SERVER}
+  CONFFILE="${HOME}/.my.cnf-${SERVER}"
   if [ -f "$CONFFILE" ]; then
-    echo "getting database list for server ${SERVER}..."
-    mysql --defaults-file="$CONFFILE" -se 'show databases;' | grep -Ev "mysql$|_schema$" > "${SERVER}-databases"
+    shw_info "Getting current database list for cluster ${SERVER}..."
+    # The ASB will never create a database smaller than 5 characters.
+    mysql --defaults-file="$CONFFILE" -se 'show databases;' | grep -Ev "mysql$|_schema$" | grep -E '^.{5,}$' > "/tmp/${SERVER}-databases"
   else
-    echo "ERROR: please create $CONFFILE so I can know how to connect to $SERVER"
+    shw_err "ERROR: please create $CONFFILE so I can know how to connect to $SERVER"
     exit 2
   fi
 done
 
-errors=()
-for PROJECT in $(awk '$3 ~ /^dedicated/ {next} {print $1}' mariadb-services); do
-  echo "checking project $PROJECT"
-  DBHOST=$(grep "^${PROJECT}\s" mariadb-services | awk '{print $3}')
-  DATABASE=$(oc -n "$PROJECT" get configmap lagoon-env -o json | jq -r '.data | with_entries(select(.key|match("_DATABASE";"i")))[]' || :)
+# For every active project, find out it's database name, and remove this the
+# database cluster file (to indicate it has been found).
+ERRORS=()
+for PROJECT in $(awk '$3 ~ /^dedicated/ {next} {print $1}' /tmp/mariadb-services); do
+  shw_info "Checking namespace '${PROJECT}'."
+
+  # In the case that there are multiple ASB configs for the 1 project, this will
+  # return an array with each database in it.
+  DATABASES=($(oc -n "${PROJECT}" get configmap lagoon-env -o json | jq -r '.data | with_entries(select(.key|match("_DATABASE";"i")))[]' || :))
 
-  if [ -z "$DATABASE" ]; then
-    echo "some problem with $PROJECT"
-    errors+=("$PROJECT")
+  if [ ${#DATABASES[@]} -eq 0 ]; then
+    shw_err " > Some problem with ${PROJECT}"
+    ERRORS+=("${PROJECT}")
   else
-    echo "found database $DATABASE on host $DBHOST"
-    sed -i.bak -e "/${DATABASE}/d" "${DBHOST}-databases"
+    # Iterate over the potential many database names.
+    for (( i=0; i<${#DATABASES[@]}; i++ )) ; do
+      # @TODO it would be technically possible to have the 2 databases spread
+      # across multiple database clusters, this code assumes a single project
+      # uses a single database cluster.
+      DBHOST=$(grep --max-count=1 "^${PROJECT}[[:space:]]" /tmp/mariadb-services | awk '{print $3}')
+      shw_warn " > Found database '${DATABASES[$i]}' on host '${DBHOST}'."
+      sed -i.bak -e "/${DATABASES[$i]}/d" "/tmp/${DBHOST}-databases"
+    done
   fi
 done
 
-echo; echo 
-echo These projects could not adaquately checked:
-printf "%s\\n" "${errors[@]}"
+echo; echo
+shw_info "These projects could not adaquately checked:"
+printf "%s\\n" "${ERRORS[@]}"
 echo
 
-
 for SERVER in $SERVERS; do
-  echo "Orphaned databases for: ${SERVER}..."
-  cat "${SERVER}-databases"
+  CONFFILE="${HOME}/.my.cnf-${SERVER}"
   echo
+  shw_info "Orphaned databases for '${SERVER}'"
+
+  # List servcer uptime.
+  shw_grey "MySQL uptime (last_update can only ever be this old)"
+  mysql --defaults-file="${CONFFILE}" -e "SELECT TIME_FORMAT(SEC_TO_TIME(VARIABLE_VALUE ),'%Hh %im') as Uptime from performance_schema.global_status where VARIABLE_NAME='Uptime';"
+
+  rm -f /tmp/${SERVER}-databases-drop
+  while IFS= read -r line || [[ -n "$line" ]]; do
+    shw_info " $line"
+    echo -n " - Last updated: "
+    mysql --defaults-file="${CONFFILE}" -se "SELECT from_unixtime(UNIX_TIMESTAMP(MAX(UPDATE_TIME))) as last_update FROM information_schema.tables WHERE TABLE_SCHEMA IN ('$line');"
+    echo -n " - Table count: "
+    mysql --defaults-file="${CONFFILE}" -se "SELECT COUNT(1) AS TableCount FROM information_schema.tables WHERE table_schema = '$line';"
+    echo "DROP DATABASE \`$line\`;" >>  /tmp/${SERVER}-databases-drop
+  done < "/tmp/${SERVER}-databases"
+
+  if [ -f "/tmp/${SERVER}-databases-drop" ]; then
+    shw_grey "To remove these databases:"
+    cat /tmp/${SERVER}-databases-drop
+  fi
 done