Improve clickhouse-keeper manifests

deploy/builder/templates-config/config.d/01-clickhouse-05-trace_log.xml

@@ -1,8 +1,8 @@
-<yandex>
-    <trace_log replace="1">
-        <database>system</database>
-        <table>trace_log</table>
-        <engine>Engine = MergeTree PARTITION BY event_date ORDER BY event_time TTL event_date + interval 30 day</engine>
-        <flush_interval_milliseconds>7500</flush_interval_milliseconds>
-    </trace_log>
-</yandex>
+<yandex>
+    <trace_log replace="1">
+        <database>system</database>
+        <table>trace_log</table>
+        <engine>Engine = MergeTree PARTITION BY event_date ORDER BY event_time TTL event_date + interval 30 day</engine>
+        <flush_interval_milliseconds>7500</flush_interval_milliseconds>
+    </trace_log>
+</yandex>

deploy/clickhouse-keeper/clickhouse-keeper-manually/clickhouse-keeper-1-node-256M-for-test-only.yaml

@@ -78,6 +78,7 @@ data:
         </logger>
         <listen_host>0.0.0.0</listen_host>
         <keeper_server incl="keeper_server">
+            <enable_reconfiguration>true</enable_reconfiguration>
             <path>/var/lib/clickhouse-keeper</path>
             <tcp_port>2181</tcp_port>
             <four_letter_word_white_list>*</four_letter_word_white_list>
@@ -92,10 +93,227 @@ data:
             <metrics>true</metrics>
             <events>true</events>
             <asynchronous_metrics>true</asynchronous_metrics>
-            <!-- https://github.com/ClickHouse/ClickHouse/issues/46136 -->
-            <status_info>false</status_info>
+            <status_info>true</status_info>
         </prometheus>
     </clickhouse>
+---
+# Setup ClickHouse Keeper scripts
+apiVersion: v1
+kind: ConfigMap
+metadata:
+  name: clickhouse-keeper-scripts
+  labels:
+    app: clickhouse-keeper
+data:
+  env.sh: |
+    #!/usr/bin/env bash
+    export DOMAIN=`hostname -d`
+    export CLIENT_HOST=clickhouse-keeper
+    export CLIENT_PORT=2181
+    export RAFT_PORT=9444
+  keeperFunctions.sh: |
+    #!/usr/bin/env bash
+    set -ex
+    function keeperConfig() {
+      echo "$HOST.$DOMAIN:$RAFT_PORT;$ROLE;$WEIGHT"
+    }
+    function keeperConnectionString() {
+      # If the client service address is not yet available, then return localhost
+      set +e
+      getent hosts "${CLIENT_HOST}" 2>/dev/null 1>/dev/null
+      if [[ $? -ne 0 ]]; then
+        set -e
+        echo "-h localhost -p ${CLIENT_PORT}"
+      else
+        set -e
+        echo "-h ${CLIENT_HOST} -p ${CLIENT_PORT}"
+      fi
+    }
+
+  keeperStart.sh: |
+    #!/usr/bin/env bash
+    set -ex
+    source /conf/env.sh
+    source /conf/keeperFunctions.sh
+
+    HOST=`hostname -s`
+    if [[ $HOST =~ (.*)-([0-9]+)$ ]]; then
+      NAME=${BASH_REMATCH[1]}
+      ORD=${BASH_REMATCH[2]}
+    else
+      echo Failed to parse name and ordinal of Pod
+      exit 1
+    fi
+    export MY_ID=$((ORD+1))
+    set +e
+    getent hosts $DOMAIN  
+    if [[ $? -eq 0 ]]; then
+      ACTIVE_ENSEMBLE=true
+    else
+      ACTIVE_ENSEMBLE=false
+    fi
+    set -e
+    mkdir -p /tmp/clickhouse-keeper/config.d/
+    if [[ "true" == "${ACTIVE_ENSEMBLE}" ]]; then
+      # get current config from clickhouse-keeper
+      CURRENT_KEEPER_CONFIG=$(clickhouse-keeper client -h ${CLIENT_HOST} -p ${CLIENT_PORT} -q "get /keeper/config" || true) 
+      # generate dynamic config, add current server to xml
+      {
+        echo "<yandex><keeper_server>"
+        echo "<server_id>${MY_ID}</server_id>"
+        echo "<raft_configuration>"
+        if [[ "0" == $(echo "${CURRENT_KEEPER_CONFIG}" | grep -c "${HOST}.${DOMAIN}") ]]; then
+          echo "<server><id>${MY_ID}</id><hostname>${HOST}.${DOMAIN}</hostname><port>${RAFT_PORT}</port><priority>1</priority><start_as_follower>true</start_as_follower></server>"
+        fi
+        while IFS= read -r line; do
+          id=$(echo "$line" | cut -d '=' -f 1 | cut -d '.' -f 2)
+          if [[ "" != "${id}" ]]; then
+            hostname=$(echo "$line" | cut -d '=' -f 2 | cut -d ';' -f 1 | cut -d ':' -f 1)
+            port=$(echo "$line" | cut -d '=' -f 2 | cut -d ';' -f 1 | cut -d ':' -f 2)
+            priority=$(echo "$line" | cut -d ';' -f 3)
+            priority=${priority:-1}
+            port=${port:-$RAFT_PORT}
+            echo "<server><id>$id</id><hostname>$hostname</hostname><port>$port</port><priority>$priority</priority></server>"
+          fi
+        done <<< "$CURRENT_KEEPER_CONFIG"      
+        echo "</raft_configuration>"
+        echo "</keeper_server></yandex>"
+      } > /tmp/clickhouse-keeper/config.d/generated-keeper-settings.xml
+    else
+      # generate dynamic config, add current server to xml
+      {
+        echo "<yandex><keeper_server>"
+        echo "<server_id>${MY_ID}</server_id>"
+        echo "<raft_configuration>"
+        echo "<server><id>${MY_ID}</id><hostname>${HOST}.${DOMAIN}</hostname><port>${RAFT_PORT}</port><priority>1</priority></server>"
+        echo "</raft_configuration>"
+        echo "</keeper_server></yandex>"
+      } > /tmp/clickhouse-keeper/config.d/generated-keeper-settings.xml
+    fi
+
+    # run clickhouse-keeper    
+    cat /tmp/clickhouse-keeper/config.d/generated-keeper-settings.xml
+    rm -rfv /var/lib/clickhouse-keeper/terminated
+    clickhouse-keeper --config-file=/etc/clickhouse-keeper/keeper_config.xml
+
+  keeperTeardown.sh: |
+    #!/usr/bin/env bash
+    set -ex
+    exec > /proc/1/fd/1
+    exec 2> /proc/1/fd/2
+    source /conf/env.sh
+    source /conf/keeperFunctions.sh
+    set +e
+    KEEPER_URL=$(keeperConnectionString)
+    set -e
+    HOST=`hostname -s`
+    if [[ $HOST =~ (.*)-([0-9]+)$ ]]; then
+        NAME=${BASH_REMATCH[1]}
+        ORD=${BASH_REMATCH[2]}
+    else
+        echo Failed to parse name and ordinal of Pod
+        exit 1
+    fi
+    export MY_ID=$((ORD+1))
+
+    CURRENT_KEEPER_CONFIG=$(clickhouse-keeper client -h localhost -p ${CLIENT_PORT} -q "get /keeper/config")
+    CLUSTER_SIZE=$(echo -e "${CURRENT_KEEPER_CONFIG}" | grep -c -E '^server\.[0-9]+=') 
+    echo "CLUSTER_SIZE=$CLUSTER_SIZE, MyId=$MY_ID"
+    # If CLUSTER_SIZE > 1, this server is being permanently removed from raft_configuration.
+    if [[ "$CLUSTER_SIZE" -gt "1" ]]; then
+      clickhouse-keeper-client -q "reconfig remove $MY_ID" ${KEEPER_URL}
+    fi
+
+    # Wait to remove $MY_ID from quorum
+    # for (( i = 0; i < 6; i++ )); do
+    #    CURRENT_KEEPER_CONFIG=$(clickhouse-keeper client -h localhost -p ${CLIENT_PORT} -q "get /keeper/config")
+    #    if [[ "0" == $(echo -e "${CURRENT_KEEPER_CONFIG}" | grep -c -E "^server.${MY_ID}=$HOST.+participant;[0-1]$") ]]; then
+    #      echo "$MY_ID removed from quorum"
+    #      break
+    #    else
+    #      echo "$MY_ID still present in quorum"
+    #    fi
+    #    sleep 1
+    # done
+
+    # Wait for client connections to drain. Kubernetes will wait until the configured
+    # "terminationGracePeriodSeconds" before forcibly killing the container
+    for (( i = 0; i < 3; i++ )); do
+      CONN_COUNT=`echo $(exec 3<>/dev/tcp/127.0.0.1/2181 ; printf "cons" >&3 ; IFS=; tee <&3; exec 3<&- ;) | grep -v "^$" | grep -v "127.0.0.1" | wc -l`
+      if [[ "$CONN_COUNT" -gt "0" ]]; then
+        echo "$CONN_COUNT non-local connections still connected."
+        sleep 1
+      else
+        echo "$CONN_COUNT non-local connections"
+        break
+      fi
+    done
+
+    touch /var/lib/clickhouse-keeper/terminated
+    # Kill the primary process ourselves to circumvent the terminationGracePeriodSeconds
+    ps -ef | grep clickhouse-keeper | grep -v grep | awk '{print $1}' | xargs kill
+
+
+  keeperLive.sh: |
+    #!/usr/bin/env bash
+    set -ex
+    source /conf/env.sh
+    OK=$(exec 3<>/dev/tcp/127.0.0.1/${CLIENT_PORT} ; printf "ruok" >&3 ; IFS=; tee <&3; exec 3<&- ;)
+    # Check to see if keeper service answers
+    if [[ "$OK" == "imok" ]]; then
+      exit 0
+    else
+      exit 1
+    fi
+
+  keeperReady.sh: |
+    #!/usr/bin/env bash
+    set -ex
+    exec > /proc/1/fd/1
+    exec 2> /proc/1/fd/2
+    source /conf/env.sh
+    source /conf/keeperFunctions.sh
+
+    HOST=`hostname -s`
+
+    # Check to see if clickhouse-keeper service answers
+    set +e
+    getent hosts $DOMAIN
+    if [[ $? -ne 0 ]]; then
+      echo "no active DNS records in service, first running pod"
+      exit 0
+    elif [[ -f /var/lib/clickhouse-keeper/terminated ]]; then
+      echo "termination in progress"
+      exit 0
+    else
+      set -e
+      # An ensemble exists, check to see if this node is already a member.
+      # Extract resource name and this members' ordinal value from pod hostname
+      if [[ $HOST =~ (.*)-([0-9]+)$ ]]; then
+        NAME=${BASH_REMATCH[1]}
+        ORD=${BASH_REMATCH[2]}
+      else
+        echo "Failed to parse name and ordinal of Pod"
+        exit 1
+      fi
+      MY_ID=$((ORD+1))
+
+      CURRENT_KEEPER_CONFIG=$(clickhouse-keeper client -h ${CLIENT_HOST} -p ${CLIENT_PORT} -q "get /keeper/config" || exit 0)
+      # Check to see if clickhouse-keeper for this node is a participant in raft cluster
+      if [[ "1" == $(echo -e "${CURRENT_KEEPER_CONFIG}" | grep -c -E "^server.${MY_ID}=${HOST}.+participant;1$") ]]; then
+        echo "clickhouse-keeper instance is available and an active participant"
+        exit 0
+      else
+        echo "clickhouse-keeper instance is ready to add as participant with 1 weight."
+
+        ROLE=participant
+        WEIGHT=1
+        KEEPER_URL=$(keeperConnectionString)
+        NEW_KEEPER_CONFIG=$(keeperConfig)
+        clickhouse-keeper-client -q "reconfig add 'server.$MY_ID=$NEW_KEEPER_CONFIG'" ${KEEPER_URL}
+        exit 0        
+      fi
+    fi
 
 ---
 # Setup ClickHouse Keeper StatefulSet
@@ -112,8 +330,6 @@ spec:
       app: clickhouse-keeper
   serviceName: clickhouse-keepers
   replicas: 1
-  updateStrategy:
-    type: RollingUpdate
   podManagementPolicy: OrderedReady
   template:
     metadata:
@@ -142,10 +358,14 @@ spec:
             items:
               - key: keeper_config.xml
                 path: keeper_config.xml
+        - name: clickhouse-keeper-scripts
+          configMap:
+            name: clickhouse-keeper-scripts
+            defaultMode: 0755
       containers:
         - name: clickhouse-keeper
           imagePullPolicy: Always
-          image: "clickhouse/clickhouse-keeper:head-alpine"
+          image: "clickhouse/clickhouse-keeper:latest-alpine"
           resources:
             requests:
               memory: "256M"
@@ -158,50 +378,46 @@ spec:
               mountPath: /etc/clickhouse-keeper/
             - name: clickhouse-keeper-datadir-volume
               mountPath: /var/lib/clickhouse-keeper
-          env:
-            - name: SERVERS
-              value: "1"
-            - name: RAFT_PORT
-              value: "9444"
+            - name: clickhouse-keeper-scripts
+              mountPath: /conf/
           command:
-            - bash
-            - -x
-            - -c
-            - |
-              HOST=`hostname -s` &&
-              DOMAIN=`hostname -d` &&
-              if [[ $HOST =~ (.*)-([0-9]+)$ ]]; then
-                  NAME=${BASH_REMATCH[1]}
-                  ORD=${BASH_REMATCH[2]}
-              else
-                  echo "Failed to parse name and ordinal of Pod"
-                  exit 1
-              fi &&
-              export MY_ID=$((ORD+1)) &&
-              mkdir -p /tmp/clickhouse-keeper/config.d/ &&
-              {
-                echo "<yandex><keeper_server>"
-                echo "<server_id>${MY_ID}</server_id>"
-                echo "<raft_configuration>"
-                for (( i=1; i<=$SERVERS; i++ )); do
-                    echo "<server><id>${i}</id><hostname>$NAME-$((i-1)).${DOMAIN}</hostname><port>${RAFT_PORT}</port></server>"
-                done
-                echo "</raft_configuration>"
-                echo "</keeper_server></yandex>"
-              } > /tmp/clickhouse-keeper/config.d/generated-keeper-settings.xml &&
-              cat /tmp/clickhouse-keeper/config.d/generated-keeper-settings.xml && 
-              clickhouse-keeper --config-file=/etc/clickhouse-keeper/keeper_config.xml --force-recovery
+            - /conf/keeperStart.sh
+          lifecycle:
+            preStop:
+              exec:
+                command:
+                  - /conf/keeperTeardown.sh
           livenessProbe:
             exec:
               command:
-                - bash
-                - -xc
-                - 'date && OK=$(exec 3<>/dev/tcp/127.0.0.1/2181 ; printf "ruok" >&3 ; IFS=; tee <&3; exec 3<&- ;); if [[ "$OK" == "imok" ]]; then exit 0; else exit 1; fi'
-            initialDelaySeconds: 20
-            timeoutSeconds: 15
+                - /conf/keeperLive.sh
+            failureThreshold: 3
+            initialDelaySeconds: 60
+            periodSeconds: 10
+            successThreshold: 1
+            timeoutSeconds: 10
+          readinessProbe:
+            exec:
+              command:
+                - /conf/keeperReady.sh
+            failureThreshold: 3
+            initialDelaySeconds: 10
+            periodSeconds: 10
+            successThreshold: 1
+            timeoutSeconds: 20
           ports:
+            - containerPort: 2181
+              name: client
+              protocol: TCP
+            - containerPort: 9444
+              name: quorum
+              protocol: TCP
             - containerPort: 7000
-              name: prometheus
+              name: metrics
+              protocol: TCP
+      restartPolicy: Always
+      schedulerName: default-scheduler
+      terminationGracePeriodSeconds: 40
   volumeClaimTemplates:
     - metadata:
         name: clickhouse-keeper-datadir-volume