Updates to Koios and setup process based on previous feedbacks

- Add config sanity checks to dbsync.sh and dbsync.sh itself to prereqs
- Remove sections for dbsync from setup-grest.sh
- Move common functions from setup-grest to env
- Make Postgres DB name configurable
- Parameterise deployment of postgREST, HAProxy, Monitoring agents, overwrite of configs and systemd deployments

docs/Build/dbsync.md

@@ -63,8 +63,6 @@ In order to verify that you can run dbsync, before making a start - you'd want t
 
 ``` bash
 cd $CNODE_HOME/scripts
-curl -s -o dbsync.sh https://raw.githubusercontent.com/cardano-community/guild-operators/master/scripts/grest-helper-scripts/dbsync.sh
-chmod 755 dbsync.sh
 export PGPASSFILE=$CNODE_HOME/priv/.pgpass
 ./dbsync.sh
 ```

scripts/cnode-helper-scripts/env

@@ -88,6 +88,7 @@ OFFLINE_MODE='N'
 
 [[ -f "${PARENT}"/.env_branch ]] && BRANCH=$(cat "${PARENT}"/.env_branch) || BRANCH=master
 
+URL_RAW="https://raw.githubusercontent.com/cardano-community/guild-operators/${BRANCH}"
 export LC_ALL=C.UTF-8
 
 # special mapping of coreutils gdate to date for MacOS
@@ -97,7 +98,6 @@ fi
 
 [[ -z ${CURL_TIMEOUT} ]] && CURL_TIMEOUT=10
 
-#
 telegramSend() {
   if [[ -z "${TG_BOT_TOKEN}" ]] || [[ -z "${TG_CHAT_ID}" ]]; then
     echo "Warn: to use the telegramSend function you must first set the bot and chat id in the env file"
@@ -340,6 +340,29 @@ STANDOUT='\e[7m'
 BOLD='\e[1m'
 NC='\e[0m'
 
+# Description : Exit with error message
+#             : $1 = Error message we'd like to display before exiting (function will pre-fix 'ERROR: ' to the argument)
+err_exit() {
+  printf "${FG_RED}ERROR${NC}: $1" >&2
+  echo -e "Exiting...\n" >&2
+  pushd -0 >/dev/null && dirs -c
+  exit 1
+}
+
+# Description : Check if provided file exists
+#             : $1 = File (with path) to check
+is_file() {
+  local file=$1
+  [[ -f $file ]]
+}
+
+# Description : Check if provided directory exists
+#             : $1 = Directory (with path) to check
+is_dir() {
+  local dir=$1
+  [[ -d $dir ]]
+}
+
 # Description : Query user for yes or no answer
 getAnswer() {
   getAnswerAny answer "$* (yes/no)"
@@ -363,7 +386,7 @@ getAnswerAny() {
   printf "${NC}"
 }
 
-# Description : Query user for any question
+# Description : Check and apply delta updates using different combinations to retain custom config
 #             : $1 = name of script to update
 #             : $2 = [Y|N] ignore question and auto update
 #             : $3 = [Y|N] do a complete file comparision
@@ -377,15 +400,13 @@ checkUpdate() {
   if [[ ${OFFLINE_MODE} = N && ${BRANCH} != master && ${BRANCH} != alpha ]]; then
     if ! curl -s -f -m ${CURL_TIMEOUT} "https://api.github.com/repos/cardano-community/guild-operators/branches" | jq -e ".[] | select(.name == \"${BRANCH}\")" &>/dev/null ; then
       echo -e "WARN!! The folder was configured against ${BRANCH} branch - which does not exist anymore, falling back to alpha branch"
+      # alpha because if someone was testing something against a custom branch, it would likely be merged to alpha first. Production systems should not be using custom branch anyways
       BRANCH=alpha
       echo "${BRANCH}" > "${CNODE_HOME}"/scripts/.env_branch
     fi
   fi
-
-  URL_RAW="https://raw.githubusercontent.com/cardano-community/guild-operators/${BRANCH}"
   URL="${URL_RAW}/scripts/cnode-helper-scripts"
   URL_DOCS="${URL_RAW}/docs/Scripts"
-
   if curl -s -f -m ${CURL_TIMEOUT} -o "${PARENT}/${1}".tmp "${URL}/${1}" 2>/dev/null; then
     # get vname from env file
     if [[ $(grep "_HOME=" "${PARENT}"/env) =~ ^#?([^[:space:]]+)_HOME ]]; then

scripts/cnode-helper-scripts/prereqs.sh

@@ -460,6 +460,7 @@ pushd "${CNODE_HOME}"/scripts >/dev/null || err_exit
 curl -s -f -m ${CURL_TIMEOUT} -o env.tmp ${URL_RAW}/scripts/cnode-helper-scripts/env
 curl -s -f -m ${CURL_TIMEOUT} -o rotatePoolKeys.sh ${URL_RAW}/scripts/cnode-helper-scripts/rotatePoolKeys.sh
 curl -s -f -m ${CURL_TIMEOUT} -o cnode.sh.tmp ${URL_RAW}/scripts/cnode-helper-scripts/cnode.sh
+curl -s -f -m ${CURL_TIMEOUT} -o dbsync.sh.tmp ${URL_RAW}/scripts/grest-helper-scripts/dbsync.sh
 curl -s -f -m ${CURL_TIMEOUT} -o cntools.sh ${URL_RAW}/scripts/cnode-helper-scripts/cntools.sh
 curl -s -f -m ${CURL_TIMEOUT} -o cntools.config.tmp ${URL_RAW}/scripts/cnode-helper-scripts/cntools.config
 curl -s -f -m ${CURL_TIMEOUT} -o cntools.library ${URL_RAW}/scripts/cnode-helper-scripts/cntools.library
@@ -500,10 +501,11 @@ updateWithCustomConfig() {
   mv -f ${file}.tmp ${file}
 }
 
-[[ ${FORCE_OVERWRITE} = 'Y' ]] && echo "Forced full upgrade! Please edit scripts/env, scripts/cnode.sh, scripts/gLiveView.sh and scripts/topologyUpdater.sh (alongwith files/topology.json, files/config.json, files/dbsync.json) as required/"
+[[ ${FORCE_OVERWRITE} = 'Y' ]] && echo "Forced full upgrade! Please edit scripts/env, scripts/cnode.sh, scripts/dbsync.sh, scripts/gLiveView.sh and scripts/topologyUpdater.sh (alongwith files/topology.json, files/config.json, files/dbsync.json) as required/"
 
 updateWithCustomConfig "env"
 updateWithCustomConfig "cnode.sh"
+updateWithCustomConfig "dbsync.sh"
 updateWithCustomConfig "gLiveView.sh"
 updateWithCustomConfig "topologyUpdater.sh"
 updateWithCustomConfig "cntools.config"

scripts/grest-helper-scripts/dbsync.sh

@@ -25,28 +25,52 @@
 
 usage() {
   cat <<-EOF
+		
 		Usage: $(basename "$0") [-d]
+		
 		Cardano DB Sync wrapper script !!
 		-d    Deploy dbsync as a systemd service
+		
 		EOF
   exit 1
 }
 
 set_defaults() {
-  [[ -z "${PGPASSFILE}" ]] && PGPASSFILE="${CNODE_HOME}/priv/.pgpass"
   [[ -z "${DBSYNCBIN}" ]] && DBSYNCBIN="${HOME}/.cabal/bin/cardano-db-sync-extended"
-  [[ -z "${DBSYNC_STATE_DIR}" ]] && DBSYNC_STATE_DIR="${CNODE_HOME}/guild-db/ledger-state"
-  [[ -z "${DBSYNC_SCHEMA_DIR}" ]] && DBSYNC_SCHEMA_DIR="${CNODE_HOME}/guild-db/schema"
+  [[ -z "${PGPASSFILE}" ]] && PGPASSFILE="${CNODE_HOME}/priv/.pgpass"
   [[ -z "${DBSYNC_CONFIG}" ]] && DBSYNC_CONFIG="${CNODE_HOME}/files/dbsync.json"
+  [[ -z "${DBSYNC_SCHEMA_DIR}" ]] && DBSYNC_SCHEMA_DIR="${CNODE_HOME}/guild-db/schema"
+  [[ -z "${DBSYNC_STATE_DIR}" ]] && DBSYNC_STATE_DIR="${CNODE_HOME}/guild-db/ledger-state"
+  [[ -z "${SYSTEMD_PGNAME}" ]] && SYSTEMD_PGNAME="postgresql"
 }
 
 check_defaults() {
   if [[ ! -f "${DBSYNCBIN}" ]] && [[ ! $(command -v cardano-db-sync-extended &>/dev/null) ]]; then
-    echo "ERROR: cardano-db-sync-extended seems to be absent in PATH, please investigate \$PATH environment variable!"
-    exit 1
+    echo "ERROR: cardano-db-sync-extended seems to be absent in PATH, please investigate \$PATH environment variable!" && exit 1
   elif [[ ! -f "${PGPASSFILE}" ]]; then
-    echo "ERROR: The PGPASSFILE (${PGPASSFILE}) not found, please ensure you've followed the instructions on guild-operators website!"
+    echo "ERROR: The PGPASSFILE (${PGPASSFILE}) not found, please ensure you've followed the instructions on guild-operators website!" && exit 1
     exit 1
+  elif [[ ! -f "${DBSYNC_CONFIG}" ]]; then
+    echo "ERROR: Could not find the dbsync config file: ${DBSYNC_CONFIG} . Please ensure you've run prereqs.sh and/or edit the DBSYNC_CONFIG variable if using a custom file." && exit 1
+  elif [[ ! -d "${DBSYNC_SCHEMA_DIR}" ]]; then
+    echo "ERROR: The schema directory (${DBSYNC_SCHEMA_DIR}) does not exist. Please ensure you've follow the instructions on guild-operators website" && exit 1
+  fi
+}
+
+check_config_sanity() {
+  genfiles=$(jq -r '[ .ByronGenesisFile, .ShelleyGenesisFile, .AlonzoGenesisFile] | @tsv' "${CONFIG}")
+  [[ -z "${genfiles[1]}" ]] || [[ -z "${genfiles[2]}" ]] && err_exit "ERROR!! Could not find Shelley/Alonzo Genesis Files in ${CONFIG}! Please re-run prereqs.sh with right arguments!" && exit 1
+  BYGENHASH=$(cardano-cli byron genesis print-genesis-hash --genesis-json "${genfiles[0]}" 2>/dev/null)
+  BYGENHASHCFG=$(jq '.ByronGenesisHash' <"${CONFIG}" 2>/dev/null)
+  SHGENHASH=$(cardano-cli genesis hash --genesis "${genfiles[1]}" 2>/dev/null)
+  SHGENHASHCFG=$(jq '.ShelleyGenesisHash' <"${CONFIG}" 2>/dev/null)
+  ALGENHASH=$(cardano-cli genesis hash --genesis "${genfiles[2]}" 2>/dev/null)
+  ALGENHASHCFG=$(jq '.AlonzoGenesisHash' <"${CONFIG}" 2>/dev/null)
+  # If hash are missing/do not match, add that to the end of config. We could have sorted it based on logic, but that would mess up sdiff comparison outputs
+  if [[ "${BYGENHASH}" != "${BYGENHASHCFG}" ]] || [[ "${SHGENHASH}" != "${SHGENHASHCFG}" ]] || [[ "${ALGENHASH}" != "${ALGENHASHCFG}" ]]; then
+    cp "${CONFIG}" "${CONFIG}".tmp
+    jq --arg BYGENHASH ${BYGENHASH} --arg SHGENHASH ${SHGENHASH} --arg ALGENHASH ${ALGENHASH} '.ByronGenesisHash = $BYGENHASH | .ShelleyGenesisHash = $SHGENHASH | .AlonzoGenesisHash = $ALGENHASH' <"${CONFIG}" >"${CONFIG}".tmp
+    mv -f "${CONFIG}".tmp "${CONFIG}"
   fi
 }
 
@@ -63,29 +87,25 @@ while getopts :d opt; do
 done
 
 # Check if env file is missing in current folder (no update checks as will mostly run as daemon), source env if present
-[[ ${0} != '-bash' ]] && PARENT=$(dirname $0) || PARENT="$(pwd)" # If sourcing at terminal, $0 would be "-bash" , which is invalid. Thus, fallback to present working directory
-
-if [[ ! -f "${PARENT}"/env ]]; then
-  echo -e "\nCommon env file missing: ${PARENT}/env"
-  echo -e "This is a mandatory prerequisite, please install with prereqs.sh or manually download from GitHub\n"
-  exit 1
-fi
+[[ ! -f "./env" ]] && echo -e "\nCommon env file missing, please ensure latest prereqs.sh was run and this script is being run from ${CNODE_HOME}/scripts folder! \n" && exit 1
 . "${PARENT}"/env
 case $? in
-  1) echo -e "ERROR: dbsync failed to load common env file\nPlease verify set values in 'User Variables' section in env file or log an issue on GitHub" && exit 1;;
+  1) echo -e "ERROR: Failed to load common env file\nPlease verify set values in 'User Variables' section in env file or log an issue on GitHub" && exit 1;;
   2) clear ;;
 esac
 
+# Set defaults and do basic sanity tests
 set_defaults
 check_defaults
+check_config_sanity
 
 if [[ "${DEPLOY_SYSTEMD}" == "Y" ]]; then
   echo "Deploying systemd service.."
   sudo bash -c "cat <<-EOF > /etc/systemd/system/${CNODE_NAME}-dbsync.service
 	[Unit]
 	Description=Cardano DB Sync
-	After=${CNODE_NAME}.service postgresql.service
-	Requires=postgresql.service
+	After=${CNODE_NAME}.service ${SYSTEMD_PGNAME}.service
+	Requires=${SYSTEMD_PGNAME}.service
 	
 	[Service]
 	Type=simple