JIRA PROD-1027 : templates/master/00-master: add disaster recovery tools

templates/master/00-master/_base/files/usr-local-bin-etcd-member-recover-sh.yaml

@@ -0,0 +1,79 @@
+filesystem: "root"
+mode: 0755
+path: "/usr/local/bin/etcd-member-recover.sh"
+contents:
+  inline: |
+    #!/usr/bin/env bash
+
+    # example
+    # export SETUP_ETCD_ENVIRONMENT=$(oc adm release info --image-for setup-etcd-environment --registry-config=./config.json)
+    # export KUBE_CLIENT_AGENT=$(oc adm release info --image-for kube-client-agent --registry-config=./config.json)
+    # sudo -E ./etcd-member-recover.sh 192.168.1.100
+
+    if [[ $EUID -ne 0 ]]; then
+      echo "This script must be run as root"
+      exit 1
+    fi
+
+    : ${SETUP_ETCD_ENVIRONMENT:?"Need to set SETUP_ETCD_ENVIRONMENT"}
+    : ${KUBE_CLIENT_AGENT:?"Need to set KUBE_CLIENT_AGENT"}
+
+    usage () {
+        echo 'Recovery server IP address required: ./etcd-member-recover.sh 192.168.1.100'
+        exit
+    }
+
+    if [ "$1" == "" ]; then
+        usage
+    fi
+
+    RECOVERY_SERVER_IP=$1
+
+    ASSET_DIR=./assets
+    ASSET_DIR_TMP="$ASSET_DIR/tmp"
+    CONFIG_FILE_DIR=/etc/kubernetes
+    MANIFEST_DIR="${CONFIG_FILE_DIR}/manifests"
+    MANIFEST_STOPPED_DIR=/etc/kubernetes/manifests-stopped
+
+    ETCD_MANIFEST="${MANIFEST_DIR}/etcd-member.yaml"
+    ETCD_CONFIG=/etc/etcd/etcd.conf
+    ETCDCTL=$ASSET_DIR/bin/etcdctl
+    ETCD_VERSION=v3.3.10
+    ETCD_DATA_DIR=/var/lib/etcd
+    ETCD_STATIC_RESOURCES="${CONFIG_FILE_DIR}/static-pod-resources/etcd-member"
+
+    SHARED=/usr/local/share/openshift-recovery
+    TEMPLATE="$SHARED/template/etcd-generate-certs.yaml.template"
+
+    source "/usr/local/bin/openshift-recovery-tools"
+
+    function run {
+      init
+      dl_etcdctl
+      backup_manifest
+      backup_etcd_conf
+      backup_etcd_client_certs
+      stop_etcd
+      backup_data_dir
+      backup_certs
+      remove_certs
+      gen_config
+      download_cert_recover_template
+      DISCOVERY_DOMAIN=$(grep -oP '(?<=discovery-srv=).*[^"]' $ASSET_DIR/backup/etcd-member.yaml )
+      if [ -z "$DISCOVERY_DOMAIN" ]; then
+        echo "Discovery domain can not be extracted from $ASSET_DIR/backup/etcd-member.yaml"
+        exit 1
+      fi
+      CLUSTER_NAME=$(echo ${DISCOVERY_DOMAIN} | grep -oP '^.*?(?=\.)')
+      populate_template '__ETCD_DISCOVERY_DOMAIN__' "$DISCOVERY_DOMAIN" "$TEMPLATE" "$ASSET_DIR/tmp/etcd-generate-certs.stage1"
+      populate_template '__SETUP_ETCD_ENVIRONMENT__' "$SETUP_ETCD_ENVIRONMENT" "$ASSET_DIR/tmp/etcd-generate-certs.stage1" "$ASSET_DIR/tmp/etcd-generate-certs.stage2"
+      populate_template '__KUBE_CLIENT_AGENT__' "$KUBE_CLIENT_AGENT" "$ASSET_DIR/tmp/etcd-generate-certs.stage2" "$MANIFEST_STOPPED_DIR/etcd-generate-certs.yaml"
+      start_cert_recover
+      verify_certs
+      stop_cert_recover
+      patch_manifest
+      etcd_member_add
+      start_etcd
+    }
+
+    run

templates/master/00-master/_base/files/usr-local-bin-etcd-snapshot-restore-sh.yaml

@@ -0,0 +1,53 @@
+filesystem: "root"
+mode: 0755
+path: "/usr/local/bin/etcd-snapshot-restore.sh"
+contents:
+  inline: |
+    #!/usr/bin/env bash
+
+    set -o errexit
+    set -o pipefail
+
+    # example
+    # etcd-snapshot-restore.sh $path-to-snapshot
+
+    if [[ $EUID -ne 0 ]]; then
+      echo "This script must be run as root"
+      exit 1
+    fi
+
+    ASSET_DIR=./assets
+    SNAPSHOT_FILE="${ASSET_DIR}/backup/etcd/member/snap/db"
+
+    if [ "$1" != "" ]; then
+      SNAPSHOT_FILE="$1"
+    fi
+
+    CONFIG_FILE_DIR=/etc/kubernetes
+    MANIFEST_DIR="${CONFIG_FILE_DIR}/manifests"
+    MANIFEST_STOPPED_DIR="${CONFIG_FILE_DIR}/manifests-stopped"
+    ETCD_VERSION=v3.3.10
+    ETCDCTL="${ASSET_DIR}/bin/etcdctl"
+    ETCD_DATA_DIR=/var/lib/etcd
+    ETCD_MANIFEST="${MANIFEST_DIR}/etcd-member.yaml"
+    ETCD_STATIC_RESOURCES="${CONFIG_FILE_DIR}/static-pod-resources/etcd-member"
+    STOPPED_STATIC_PODS="${ASSET_DIR}/tmp/stopped-static-pods"
+
+    source "/usr/local/bin/openshift-recovery-tools"
+
+    function run {
+      init
+      dl_etcdctl
+      backup_manifest
+      stop_static_pods
+      stop_etcd
+      stop_kubelet
+      stop_all_containers
+      backup_data_dir
+      remove_data_dir
+      restore_snapshot
+      start_static_pods
+      start_kubelet
+    }
+
+    run