genCompose

#!/bin/bash



# --------------------------------------------------------------------------------------------
# Name:		genCompose
# Language:	bash
# Author:	Jorge Torralba
# Version:	1 
# Date:		May 9, 2024
# Location:	https://github.com/jtorral/DockerPgHa
#
# This script will generate a docker-compose file based on the arguments passed to it.
# Since this is yaml based file creation, do not adjust the spacing you may see in the script.
# Very sensitive to spacing.
# It's a pain, but it works for now.
# --------------------------------------------------------------------------------------------



# -- show help info
function usage() {
        echo -e
        echo -e "Usage:"
        echo -e "       $0 [OPTION]"
        echo -e
        echo -e "       -d Number of data centers to simulate. (default = 1)"
        echo -e "       -c number of nodes per data center. (default = 2)"
        echo -e "       -n Prefix name to use for db containers (lower case no special characters)."
        echo -e "       -v Postgres Major version number. i.e 16"
        echo -e "       -b Start patroni in background and keep container running even if it stops. "
        echo -e "          Good for upgrades and maintenance tasks." 
        echo -e
        echo -e "       Number of db nodes is capped at 9. So (-d * -c) should be <= 9 "
        echo -e "       Number of etcd nodes are calculated on (-d * -c ) / 2 + 1 "
        echo -e
        if [ ! -z "$usageMessage" ]; then
           echo -e "${usageMessage}"
           echo -e
        fi
        exit
}


# -- build depends_on list for start order of containers
# -- Not a reliable way to control starting but it's a start.
# -- Consider using scripts that check

buildDependList() {

   # -- Which service block (nodename) are we building this depends on list for?
   buildingListFor=$1
 
   if [ -z $2 ]; then
      addEtcd="etcd"
   else
      addEtcd=$2
   fi

   local etcdNode

   pgDependList="depends_on: "

   # -- lets add etcd nodes first
   # -- Also add capabaility of not adding the etcd nodes with a value other than "etcd" passed

   if [ "${addEtcd}" == "etcd" ]; then
      for etcdNode in ${etcdNameList};
      do
         # -- Preserve spaces below
         pgDependList="${pgDependList}\n      - ${etcdNode}"
      done
   fi

   # -- We build the depends_on list of postgres nodes. However, the first node does not  depend on any other node
   # -- So, we skip this for the first node 
   if [ "$buildingListFor" == "$firstNode" ]; then
      return
   fi

   # -- For now lets just depend on first node. If we want others, we can build a loop instead
   # -- Preserve spaces below
   pgDependList="${pgDependList}\n      - ${firstNode}"

}






# -- Lets check running containers so we don't duplicatethem with the same name and ports

function checkUsed() {
   # -- Check for postgres ports already mapped
   usedPgPorts=$(docker ps --format "{{.Ports}}" | grep ":" | grep 5432 | awk -F ":" '{print $2}' | awk -F "-" '{print $1}' | sort | tail -1)

   # -- Lets grab a container number. We focus on etcd1- since we know that will always be running if we spin up pgha clusters
   # -- using these scripts. Otherwise, the awk might fail and grab another name to check that does not match the naming convention used
   # -- remember the first number after the first hyphen in the name represents the container number
   usedContainers=$(docker ps --format "{{.Names}}" | grep "etcd1-" | awk -F "-" '{print $2}' | sort | tail -1)
}





# -- Determin how many etcd nodes we need
function etcdHostsNeeded() {
   etcdNeeded=$(( ($totalNodes / 2) + 1))
   if [ $(($etcdNeeded%2)) -eq 0 ];
   then
       etcdNeeded=$(( $etcdNeeded + 1 ))
   fi
   if [ $etcdNeeded -lt 3 ]; then
      etcdNeeded=3
   fi
}





# -- Build string of etcd node names to use
function buildEtcdNodeNames() {
   etcdHostsNeeded
   local x
   etcdNameList=""
   for (( x=1; x<=$etcdNeeded; x++ ))
   do
      etcdNameList="${etcdNameList} etcd${x}-${containerNumber}"
   done
}





# -- Build string of postgres node names to use
function buildPgNodeNames() {
   local d
   local c
   local nodeCount
   nodeCount=0
   pgNameList=""
    
   # --- nodeName is provided at command line as argument
   for (( d=1; d<=$numDatacenters; d++ ))
   do
      for (( c=1; c<=$nodesPerDataCenter; c++ ))
      do
         nodeCount=$(( $nodeCount + 1 ))
         if [ $d -eq 1 ] && [ $c -eq 1 ]; then
            firstNode="${nodeName}${d}-${containerNumber}-node${c}"
         fi
         pgNameList="${pgNameList} ${nodeName}${d}-${containerNumber}-node${c}"
      done
   done
}





# -- Build string of pgbackrest node names to use. In reality just 1. But you never know
function builPgBackrestNodeNames() {
   # -- Only one bgbackrest server is actually needed. 
   # -- So, just hardcoding the name here
   pgBackrestNeeded=1
   pgBackrestNameList="pgbackrest1-${containerNumber}"
   pgBackrestNodeName="pgbackrest1-${containerNumber}"

   # -- If you want to use the loop to build the server name, just uncomment the code below
   # -- Which is what is used for building multiple server names for etcd and pg servers 

   #pgBackrestNeeded=1
   #pgBackrestNameList=""
   #for (( x=1; x<=$pgBackrestNeeded; x++ ))
   #do
      #pgBackrestNameList="${pgBackrestNameList} pgbackrest${x}-${containerNumber}"
   #done
}





function buildEtcdEnv() {
   local x
   endPoints=""
   initialCluster=""
   etcdNodeList=""
   for etcdNode in ${etcdNameList}
   do
      endPoints="${endPoints}${etcdNode}:2380,"
      initialCluster="${initialCluster}${etcdNode}=http://${etcdNode}:2380,"
      etcdNodeList="${etcdNodeList}${etcdNode}:2379,"
   done

   # - remove trailing commas
   endPoints=$(echo $endPoints | sed 's/,$//g')
   initialCluster=$(echo $initialCluster | sed 's/,$//g')
   etcdNodeList=$(echo $etcdNodeList | sed 's/,$//g')
}





# -- Build the etcd service block s=for the yaml file. One for each etcd node
buildEtcdService() {

buildEtcdEnv

for etcdNode in ${etcdNameList};
do
echo "
  ${etcdNode}:
    image: pgha-etcd-3.5
    restart: unless-stopped
    environment:
      ENDPOINTS: $endPoints
    command:
      - '/usr/bin/etcd'
      - '--name=${etcdNode}'
      - '--initial-advertise-peer-urls=http://${etcdNode}:2380'
      - '--listen-peer-urls=http://0.0.0.0:2380'
      - '--listen-client-urls=http://0.0.0.0:2379'
      - '--advertise-client-urls=http://${etcdNode}:2379'
      - '--heartbeat-interval=250'
      - '--election-timeout=1250'
      - '--initial-cluster=$initialCluster'
      - '--initial-cluster-state=new'
      - '--initial-cluster-token=pgha'
    networks:
      net1-${containerNumber}:
        priority: 1000
      net2-${containerNumber}:
        priority: 10
    ports:
      - 2379
    volumes:
      - ${etcdNode}:/etcd_data
    hostname: ${etcdNode}
    container_name: ${etcdNode}
   
   " >> $composeFile
done

}




function getNodeInfo() {
   local NodeName
   nodeName=$1
   dataCenter=$(echo $nodeName | awk -F "-" '{print $1}' | grep -o -E '[0-9]+')
   nodeNumber=$(echo $nodeName | awk -F "-" '{print $3}' | grep -o -E '[0-9]+')
}




buildPgBackrestService() {

   local nodeCount
   local pgNode
   local backrestNode
   nodeCount=1
   pgBackrestEnvList="environment:"

   for backrestNode in ${pgBackrestNameList};
   do
      buildDependList $backrestNode "noetcd"

      # -- Lets build the pgBackrest node lists that will be used for the config
      for pgNode in ${pgNameList};
      do
         pgBackrestEnvList="${pgBackrestEnvList}\n      NODE${nodeCount}: ${pgNode}"
         nodeCount=$(( $nodeCount + 1 ))
      done


echo -e "
  ${backrestNode}:
    ${pgDependList}
    ${pgBackrestEnvList}
      CFG_DIR: /pgha/config
      REPO_PATH: /pgha/pgbackrest
      DATADIR: /pgdata/${pgMajor}
      STANZA_NAME: pgha_db
      PGPORT: 5432
  
    image: pgha-pgbackrest
    restart: unless-stopped
    networks:
      net1-${containerNumber}:
        priority: 1000
      net2-${containerNumber}:
        priority: 10
      net3-${containerNumber}:
        priority: 10
    volumes:
      - ${backrestNode}-data:/pgha/pgbackrest
      - ${backrestNode}-home:/var/lib/postgresql
    hostname: ${backrestNode}
    container_name: ${backrestNode}
    cap_add:
      - SYS_ADMIN
    entrypoint: /entrypoint.sh
  
" >> $composeFile

  done
}







buildNetworks() {

   networkList="networks: "

   # --  Network creation variables for custom networks
   networkList="${networkList}\n  net1-${containerNumber}:"
   networkList="${networkList}\n  net2-${containerNumber}:"
   networkList="${networkList}\n  net3-${containerNumber}:"

   echo -e "
$networkList
" >> $composeFile

}





buildVolumes() {

   volumeList=""
   etcdVolumeList=""
   pgbackrestVolumeList=""

   # -- Build volumes used by pg containers
   for pgNode in ${pgNameList};
   do
      v1="${pgNode}-data:"
      v2="${pgNode}-home:"
      v3="${pgNode}-conf:"
      volumeList="${volumeList}\n  ${v1}\n  ${v2}\n  ${v3}"
   done


   # -- Build volumes used by etcd containers
   etcdVolumes=""
   for etcdNode in ${etcdNameList};
   do
      etcdVolumeList="${etcdVolumeList}\n  ${etcdNode}:"
   done

   # -- Build volumes used by pgbackrest containers
   pgbackrestVolumeList=""
   for pgBackrestNode in ${pgBackrestNameList};
   do
      pgbackrestVolumeList="${pgbackrestVolumeList}\n  ${pgBackrestNode}-data:";
      pgbackrestVolumeList="${pgbackrestVolumeList}\n  ${pgBackrestNode}-home:";
   done

   echo -e "volumes: $etcdVolumeList $volumeList $pgbackrestVolumeList " >> $composeFile

}






buildPgService() {

   buildEtcdEnv

   local nodeCount
   local dataCenter

   nodeCount=0
   for pgNode in ${pgNameList};
   do
      nodeCount=$(( $nodeCount + 1 ))
   
      # -- We know the nodename has the datacenter number at the end of it. So lets pull that out since we need 
      # -- that data center to assign network priority
      # -- So the call sets dataCenter and NodeNumber. We only use dataCenter here
      getNodeInfo $pgNode
   
      # - Assign network priority based on dataCenter
      priority1=10;
      priority2=10;
      priority3=10;
   
      case $dataCenter in
         1 | 4 | 7 | 10 | 13 | 16 | 19)
            priority1=1000
            ;;
         2 | 5 | 8 | 11 | 14 | 17 | 20)
            priority2=1000
            ;;
         3 | 6 | 9 | 12 | 15 | 18 | 21)
            priority3=1000
            ;;
      esac
   
   # --  lets generate unique ports to map
   portMap=$(( startingPort + $nodeCount ))
   
   # - Lets build the depend_on list. We let it know which service we are builing it for by passing the node
   # - We also pass a 2nd argument "etcd" so that the function adss the etcd nodes to the depends on list
   buildDependList $pgNode "etcd"

echo -e "
  ${pgNode}:
    ${pgDependList}
    environment:
      ETCD_NODES: $etcdNodeList
      NODE_NAME: ${pgNode}
      CFG_DIR: /pgha/config
      PATRONI_CFG: /pgha/config/patroni.conf
      DATADIR: /pgdata/${pgMajor}
      PG_BIN_DIR: /usr/lib/postgresql/${pgMajor}/bin
      NAMESPACE: pgha
      SCOPE: pgha_cluster
      STANZA_NAME: pgha_db
      PGBACKREST_SERVER: ${pgBackrestNodeName}
      BACKGROUND: $backGround

    image: pgha-pg16-patroni
    restart: unless-stopped
    cap_add:
      - SYS_ADMIN
    entrypoint: /entrypoint.sh
    networks:
      net1-${containerNumber}:
        priority: ${priority1}
      net2-${containerNumber}:
        priority: ${priority2}
      net3-${containerNumber}:
        priority: ${priority3}
    ports:
      - "${portMap}:5432"
    volumes:
      - ${pgNode}-data:/pgdata
      - ${pgNode}-home:/var/lib/postgresql
      - ${pgNode}-conf:/pgha/config
    hostname: ${pgNode}
    container_name: ${pgNode}

" >> $composeFile

   done

}










# ---  Validate its a number
function checkNumber() {
        num=$1
        var=$2
        valid=0
        regexp='^[0-9]+$'
        if ! [[ $num =~ $regexp ]] ; then
           echo -e
           echo -e "Invalid number format of \"$num\" entered for \"$var\". Please correct and try again"
           echo -e
           exit
        fi
}


function checkAlpha() {
        str=$1
        var=$2
        valid=0
        regexp='^[a-z]+$'
        if ! [[ $str =~ $regexp ]] ; then
           echo -e
           echo -e "Only lower case letters are allowed. No special characters. Invalid format of \"$str\" entered for \"$var\". Please correct and try again"
           echo -e
           exit
        fi
}




composeFile='docker-compose.yaml'
nodeName=""
numDatacenters=1
nodesPerDataCenter=2
backGround=0


while getopts bn:d:c:v: name
do
   case $name in
      n) nodeName="$OPTARG";;
      v) pgMajor="$OPTARG";;
      b) backGround="1";;
      d) numDatacenters="$OPTARG";;
      c) nodesPerDataCenter="$OPTARG";;
      *) usage;;
      ?) usage;;
   esac
done
shift $(($OPTIND - 1))

if [ -z "$nodeName" ] || [ -z "$pgMajor" ]; then
   usage
fi

checkAlpha  $nodeName "-n"
checkNumber $pgMajor "-v"
checkNumber $numDatacenters "-d"
checkNumber $nodesPerDataCenter "-c"


maxNodes=9
totalNodes=$(($numDatacenters * $nodesPerDataCenter))

if [ $totalNodes -gt $maxNodes ]; then
   usageMessage="You have exceeded the max number of nodes (${maxNodes}) allowed. "
   usage
fi

if [ $totalNodes -lt 2 ]; then
   usageMessage="You must have at least 2 nodes for Patroni. You have specified $numDatacenters data center and $nodesPerDataCenter node per data center."
   usage
fi


# -- Lets find used ports and container numbers
checkUsed

if [ -z $usedPgPorts ]; then
   startingPort=5432
else
   startingPort=$usedPgPorts
fi

if [ -z $usedContainers ]; then
   containerNumber=1
else
   containerNumber=$(( $usedContainers + 1 ))
fi

# -- Lets start populating the compose file
echo "services:"  > $composeFile

buildEtcdNodeNames
buildPgNodeNames
builPgBackrestNodeNames

buildEtcdService
buildPgService
buildPgBackrestService

buildVolumes
buildNetworks

echo -e "\n--- Finished. File $composeFile generated\n"