From 711c1102f185a78ef0b50c7c430bed236d3b1d4d Mon Sep 17 00:00:00 2001
From: Lis <listai@tencent.com>
Date: Thu, 22 Dec 2022 15:13:58 +0800
Subject: [PATCH] feat(installer): add oidc scripts (#2194)

---
 hack/oidc-scripts/README.md   |  24 ++
 hack/oidc-scripts/oidc.conf   |   9 +
 hack/oidc-scripts/oidc.sh     | 405 ++++++++++++++++++++++++++++++++++
 hack/oidc-scripts/rollback.sh | 208 +++++++++++++++++
 4 files changed, 646 insertions(+)
 create mode 100644 hack/oidc-scripts/README.md
 create mode 100644 hack/oidc-scripts/oidc.conf
 create mode 100755 hack/oidc-scripts/oidc.sh
 create mode 100644 hack/oidc-scripts/rollback.sh

diff --git a/hack/oidc-scripts/README.md b/hack/oidc-scripts/README.md
new file mode 100644
index 000000000..fe0940f90
--- /dev/null
+++ b/hack/oidc-scripts/README.md
@@ -0,0 +1,24 @@
+# OIDC config and rollback scripts
+
+## How to use oidc.sh script
+`oidc.sh` script use to config the OIDC login.
+When the first tile use the `oidc.sh`.You should fill in the configuration file `oidc.conf`then execute the `oidc.sh`with the `oidc.conf`:
+```
+chmod +x oidc.sh
+./oidc.sh oidc.conf
+```
+After that,it will store the `oidc.conf` in configmap, you can check:
+```
+kubectl get cm -n tke oidc-config
+```
+Except for the first time，you can execute the script without the script:
+```
+./oidc.sh
+```
+## How to use rollback.sh script
+`rollback.sh` use to rollback OIDC login, after rollback you can use tkeanywhere to login.
+```
+chmod +x rollback.sh
+./rollback.sh
+```
+
diff --git a/hack/oidc-scripts/oidc.conf b/hack/oidc-scripts/oidc.conf
new file mode 100644
index 000000000..5b5879bf9
--- /dev/null
+++ b/hack/oidc-scripts/oidc.conf
@@ -0,0 +1,9 @@
+ca_crt="/root/oidc/ca.crt"
+secret_id="123456"
+secret_key="123456"
+endpoint="oidc.cn"
+master_id="123456"
+username="oidc"
+issuer_url="https://oidc.cn/iamapp/api/v1/OIDC"
+hostnames="oidc.cn,iam-api.oidc.cn"
+tke_domain_name="tke.domain.name"
diff --git a/hack/oidc-scripts/oidc.sh b/hack/oidc-scripts/oidc.sh
new file mode 100755
index 000000000..c4ee56555
--- /dev/null
+++ b/hack/oidc-scripts/oidc.sh
@@ -0,0 +1,405 @@
+#!/usr/bin/env bash
+#set -x
+
+function setConfigEnvs() {
+  echo "==== 0. Execute OIDC config setConfigEnvs doing ===="
+  kubectl get cm oidc-config -n tke
+  if [ $? -ne 0 ]; then
+    echo "Ignore ERROR: configmap oidc-config not found, creating oidc-config configmap for persistent oidc value in tkeanywhere ..."
+    if [ "$1" != "" ]; then
+      cp $1 oidc.conf
+    fi
+    sed -i s/[[:space:]]//g oidc.conf
+    kubectl create cm oidc-config -n tke --from-file=./oidc.conf
+  else
+    kubectl get cm oidc-config -n tke -ojson | python -c "import json; import sys; obj=json.load(sys.stdin); print obj['data']['oidc.conf']" >oidc.conf
+  fi
+
+  . oidc.conf
+  vip=$(kubectl get cls global -o=jsonpath='{.spec.features.ha.tke.vip}')
+  region="ap-guangzhou"
+  rm oidc.conf
+  echo "==== 0. Execute OIDC config setConfigEnvs success ===="
+}
+
+function validateInput() {
+  echo "==== 1. Execute OIDC config validateInput doing ===="
+  if [ "$ca_crt" == "" ]; then
+    curl -v $issuer_url/.well-known/openid-configuration
+    if [ $? -ne 0 ]; then
+      echo "Validate ERROR: validate issuer URL $issuer_url failed."
+      echo "Please check use cmd: curl -v $issuer_url/.well-known/openid-configuration"
+      exit 1
+    fi
+  else
+    curl --cacert $ca_crt $issuer_url/.well-known/openid-configuration
+    if [ $? -ne 0 ]; then
+      echo "Validate ERROR: validate $ca_crt and $issuer_url failed."
+      echo "Please check use cmd: curl --cacert $ca_crt $issuer_url/.well-known/openid-configuration"
+      exit 1
+    fi
+  fi
+  echo "==== 1. Execute OIDC config validateInput success ===="
+}
+
+function backup() {
+  echo "==== 2. Execute OIDC config backup doing ===="
+  api=$(kubectl -n tke get -o=name deployment | grep api)
+  cm=$(kubectl -n tke get -o=name configmap | grep api)
+  gw=$(kubectl -n tke get -o=name configmap | grep gateway)
+  ds=$(kubectl -n tke get -o=name daemonset | grep gateway)
+  r=("$api $cm $ds $gw")
+  echo ======= backup current resources =======
+  dir=/opt/oidcbackup
+  mkdir -p ${dir}/deployment.apps ${dir}/daemonset.apps ${dir}/configmap
+  for n in $r; do
+    kubectl -n tke get -o=yaml $n >${dir}/$n.yaml
+  done
+  echo "==== 2. Execute OIDC config backup success ===="
+}
+
+function createcms() {
+  echo "==== 3. Execute OIDC config createcms doing ===="
+  # create oidc-ca configmap
+  kubectl get cm oidc-ca -n tke
+  if [ $? -ne 0 ] && [ "$ca_crt" != "" ]; then
+    echo "Ignore ERROR: configmap oidc-ca not found, creating oidc-ca configmap ..."
+    kubectl create cm oidc-ca -n tke --from-file=$ca_crt
+  fi
+
+  # create cloudindustry-config configmap
+  cat <<EOF | kubectl apply -f -
+kind: ConfigMap
+metadata:
+  name: cloudindustry-config
+  namespace: tke
+apiVersion: v1
+data:
+  config: |
+    {
+        "secret_id": "$secret_id",
+        "secret_key": "$secret_key",
+        "endpoint": "$endpoint",
+        "region": "$region",
+        "master_id": "$master_id"
+    }
+EOF
+  echo "==== 3. Execute OIDC config createcms success ===="
+}
+
+function createOIDCAuthCMTmpFile() {
+  content=$(
+    cat <<EOF
+\n
+[authentication.oidc]\n
+client_secret = "$secret_key"\n
+client_id = "$secret_id"\n
+issuer_url = "$issuer_url"\n
+username_prefix ="-"\n
+username_claim = "name"\n
+groups_claim = "groups"\n
+tenantid_claim = "federated_claims"
+EOF
+  )
+  kubectl get cm oidc-ca -n tke
+  if [ $? -eq 0 ]; then
+    content=$(
+      cat <<EOF
+${content}\n
+ca_file = "/app/oidc/ca.crt"
+EOF
+    )
+
+  fi
+  echo -e $content >oidc_auth_tmp.txt
+  sed -i 's/^/     /g' oidc_auth_tmp.txt
+  sed -i '1d' oidc_auth_tmp.txt
+  sed -i '1{x;p;x;}' oidc_auth_tmp.txt
+}
+
+function removeOIDCAuthCMTmpFile() {
+  rm oidc_auth_tmp.txt
+}
+
+function modifyConfigMap() {
+  echo ======= modify configmap $1 =======
+  file=./$1-cm.yaml
+  kubectl -n tke get cm $1 -o yaml >$file
+
+  start=$(sed -n '/last-applied-configuration/=' $file)
+  if [ "$start" != "" ]; then
+    end=$(($start + 1))
+    sed -i "$start, $end d" $file
+  fi
+
+  if [ "$1" = "tke-auth-api" ]; then
+    line=$(sed -n '/authentication.oidc/=' $file)
+    if [ "$line" = "" ]; then
+      sed -i "/privileged_username/r oidc_auth_tmp.txt" $file
+    fi
+    line=$(sed -n '/init_tenant_type/=' $file)
+    if [ "$line" = "" ]; then
+      sed -i '/assets_path/a\    init_tenant_type = "cloudindustry"' $file
+    fi
+    line=$(sed -n '/init_tenant_id/=' $file)
+    if [ "$line" = "" ]; then
+      sed -i '/assets_path/a\    init_tenant_id = "default"' $file
+    fi
+    line=$(sed -n '/cloudindustry_config_file/=' $file)
+    if [ "$line" = "" ]; then
+      sed -i '/assets_path/a\    cloudindustry_config_file = "/app/cloudindustry/config"' $file
+    fi
+
+    line=$(sed -n '/init_client_id/=' $file)
+    if [ "$line" != "" ]; then
+      sed -i "${line}c \ \ \ \ init_client_id = \"${secret_id}\"" $file
+    fi
+    line=$(sed -n '/init_client_secret/=' $file)
+    if [ "$line" != "" ]; then
+      sed -i "${line}c \ \ \ \ init_client_secret = \"${secret_key}\"" $file
+    fi
+  else
+    kubectl get cm oidc-ca -n tke
+    if [ $? -ne 0 ]; then
+      echo "Ignore ERROR: configmap oidc-ca not found, ignore ca.crt ..."
+      sed -i '/^      ca_file = "\/app\/certs\/ca.crt"$/d' $file
+    else
+      line=$(sed -n '/^      ca_file/=' $file)
+      if [ "$line" != "" ]; then
+        sed -i "${line}c \ \ \ \ \ \ ca_file = \"\/app\/oidc\/ca.crt\"" $file
+      fi
+    fi
+
+    if [ "$1" = "tke-gateway" ]; then
+      sed -i '/disableOIDCProxy/d' $file
+      sed -i "/kind: GatewayConfiguration/a\    disableOIDCProxy: true" $file
+      sed -i '/generic/d' $file
+      sed -i "/port = 80/a\    [generic]" $file
+      sed -i '/external_hostname/d' $file
+      sed -i "/generic/a\    external_hostname = \"${tke_domain_name}\"" $file
+    fi
+
+    sed -i '/external_issuer_url/d' $file
+
+    sed -i '/client_secret/d' $file
+    sed -i "/authentication.oidc/a\      client_secret = \"${secret_key}\"" $file
+
+    sed -i '/client_id/d' $file
+    sed -i "/authentication.oidc/a\      client_id = \"${secret_id}\"" $file
+
+    sed -i '/issuer_url/d' $file
+    sed -i "/authentication.oidc/a\      issuer_url = \"${issuer_url}\"" $file
+  fi
+
+  sed -i '/resourceVersion/d' $file
+  sed -i '/uid/d' $file
+  kubectl apply -f $file
+  rm $file
+}
+
+function createOIDCVolumeTmpFiles() {
+  kubectl get cm oidc-ca -n tke
+  if [ $? -ne 0 ]; then
+    cat >>./oidc_volumeMounts_tmp.txt <<EOF
+        - mountPath: /app/cloudindustry
+          name: cloudindustry-config-volume
+EOF
+
+    cat >>./oidc_volumes_tmp.txt <<EOF
+      - configMap:
+          defaultMode: 420
+          name: cloudindustry-config
+        name: cloudindustry-config-volume
+EOF
+
+  else
+    cat >>./oidc_volumeMounts_tmp.txt <<EOF
+        - mountPath: /app/oidc
+          name: oidc-ca-volume
+        - mountPath: /app/cloudindustry
+          name: cloudindustry-config-volume
+EOF
+
+    cat >>./oidc_volumes_tmp.txt <<EOF
+      - configMap:
+          defaultMode: 420
+          name: oidc-ca
+        name: oidc-ca-volume
+      - configMap:
+          defaultMode: 420
+          name: cloudindustry-config
+        name: cloudindustry-config-volume
+EOF
+
+  fi
+}
+
+function removeOIDCVolumeTmpFiles() {
+  rm oidc_volumeMounts_tmp.txt
+  rm oidc_volumes_tmp.txt
+}
+
+function createOIDCHostAliasTmpFile() {
+  if [ "$hostnames" != "" ] && [ "$vip" != "" ]; then
+    file=oidc_hostAlias_tmp.txt
+    rm -rf $file
+    echo "      hostAliases:" >>$file
+    echo "      - hostnames:" >>$file
+    arr=(${hostnames//,/ })
+    for name in "${arr[@]}"; do
+      #    remove ','
+      name=$(echo "$name" | sed -e "s/,$//")
+      #    remove '"'
+      name=$(echo "$name" | sed -e "s/^\"//" -e "s/\"$//")
+      echo "****** ${name}"
+      echo "        - ${name}" >>$file
+    done
+    echo "        ip: ${vip}" >>$file
+  fi
+}
+
+function rmOIDCHostAliasTmpFile() {
+  rm -f oidc_hostAlias_tmp.txt
+}
+
+function modifyResource() {
+  echo ======= modify $1 $2 =======
+  file=./$2-$1.yaml
+
+  kubectl -n tke annotate $1 $2 tkeanywhere/oidc="true" --overwrite=true
+  kubectl -n tke get $1 $2 -o yaml >$file
+
+  start=$(sed -n '/last-applied-configuration/=' $file)
+  if [ "$start" != "" ]; then
+    end=$(($start + 1))
+    sed -i "$start, $end d" $file
+  fi
+
+  line=$(sed -n '/hostAliases/=' $file)
+  if [ "$line" = "" ] && [ "$hostnames" != "" ] && [ "$vip" != "" ]; then
+    sed -i "/dnsPolicy/r oidc_hostAlias_tmp.txt" $file
+  fi
+
+  line=$(sed -n '/oidc-ca-volume/=' $file)
+  if [ "$line" = "" ]; then
+    sed -i "/volumes/r oidc_volumes_tmp.txt" $file
+    sed -i "/volumeMounts/r oidc_volumeMounts_tmp.txt" $file
+  fi
+
+  kubectl apply -f $file
+  rm $file
+}
+
+function adduser() {
+  echo "==== 3. Execute OIDC config adduser doing ===="
+  kubectl get platforms.business.tkestack.io platform-default -oyaml >default.yaml
+  start=$(sed -n '/last-applied-configuration/=' default.yaml)
+  if [ "$start" != "" ]; then
+    end=$(($start + 1))
+    sed -i "$start, $end d" default.yaml
+  fi
+  cat default.yaml | grep $username
+  if [ $? -ne 0 ]; then
+    sed -i "/- admin/a\  - ${username}" default.yaml
+    kubectl apply -f default.yaml
+  fi
+  rm default.yaml
+  echo "==== 3. Execute OIDC config adduser success ===="
+}
+
+function configAll() {
+  echo "==== 4. Execute OIDC config modifyConfigMap doing ===="
+  createOIDCAuthCMTmpFile
+  anno=$(kubectl -n tke get ds tke-gateway -o=jsonpath='{.metadata.annotations.tkeanywhere/oidc}')
+  if [ "$anno" == "" ]; then
+    modifyConfigMap tke-gateway
+  fi
+  for cm in $(kubectl get cm -n tke -o=name | grep api | awk '{print $1}' | awk -F '/' '{print $2}'); do
+    anno=$(kubectl -n tke get deploy $cm -o=jsonpath='{.metadata.annotations.tkeanywhere/oidc}')
+    if [ "$anno" == "" ]; then
+      modifyConfigMap $cm
+    fi
+  done
+  echo "==== 4. Execute OIDC config modifyConfigMap success ===="
+  kubectl delete idp default
+  removeOIDCAuthCMTmpFile
+
+  createOIDCVolumeTmpFiles
+  createOIDCHostAliasTmpFile
+  echo "==== 5. Execute OIDC config modifyResource doing ===="
+  for deploy in $(kubectl get deployment -n tke -o=name | grep api | awk '{print $1}' | awk -F '/' '{print $2}'); do
+    anno=$(kubectl -n tke get deploy $cm -o=jsonpath='{.metadata.annotations.tkeanywhere/oidc}')
+    if [ "$anno" == "" ]; then
+      modifyResource deploy $deploy
+    fi
+  done
+
+  anno=$(kubectl -n tke get ds tke-gateway -o=jsonpath='{.metadata.annotations.tkeanywhere/oidc}')
+  if [ "$anno" == "" ]; then
+    modifyResource daemonset tke-gateway
+  fi
+  removeOIDCVolumeTmpFiles
+  rmOIDCHostAliasTmpFile
+  echo "==== 5. Execute OIDC config modifyResource success ===="
+}
+
+function checkall() {
+  echo "==== 6. Execute OIDC config checkall doing ===="
+  check_daemonset tke-gateway
+  for deploy in $(kubectl get deployment -n tke -o=name | grep api | awk '{print $1}' | awk -F '/' '{print $2}'); do
+    check_deployment $deploy
+  done
+  echo "==== 6. Execute OIDC config checkall success ===="
+}
+
+function check_deployment {
+  check_workload_status_args deployment $1 tke .status.readyReplicas
+  check=true
+  while [ $check = true ]
+  do
+  check=false
+  result=$(kubectl get deployment $1 -n tke --ignore-not-found -o go-template --template='{{if or (ne (.status.replicas) (.status.readyReplicas)) (ne (.status.replicas) (.status.updatedReplicas))}}false{{else}}true{{end}}')
+  if [ $check = false ] && ([ -z $result ] || [ $result != true ]); then
+    check=true
+    echo "waiting $1"
+    sleep 1
+  fi
+  done
+}
+
+function check_daemonset {
+  check_workload_status_args daemonset $1 tke .status.numberReady
+  check=true
+  while [ $check = true ]
+  do
+  check=false
+  result=$(kubectl get daemonset $1 -n tke --ignore-not-found -o go-template --template='{{if or (ne (.status.desiredNumberScheduled) (.status.numberReady)) (ne (.status.desiredNumberScheduled) (.status.updatedNumberScheduled))}}false{{else}}true{{end}}')
+  if [ $check = false ] && ([ -z $result ] || [ $result != true ]); then
+    check=true
+    echo "waiting $1"
+    sleep 1
+  fi
+  done
+}
+
+function check_workload_status_args {
+    check=true
+    while [ $check = true ]
+    do
+    check=false
+    result=$(kubectl get $1 $2 -n $3 --ignore-not-found -o go-template --template='{{if not ('$4')}}false{{else}}true{{end}}')
+    if [ $check = false ] && ([ -z $result ] || [ $result != true ]); then
+        check=true
+        echo "waiting $2 val $4"
+        sleep 1
+    fi
+    done
+}
+
+setConfigEnvs $1
+validateInput
+backup
+createcms
+configAll
+adduser
+checkall
diff --git a/hack/oidc-scripts/rollback.sh b/hack/oidc-scripts/rollback.sh
new file mode 100644
index 000000000..7d7b07ab3
--- /dev/null
+++ b/hack/oidc-scripts/rollback.sh
@@ -0,0 +1,208 @@
+#!/usr/bin/env bash
+
+function backup() {
+  echo "==== 0. Execute OIDC rollback backup doing ===="
+  api=$(kubectl -n tke get -o=name deployment | grep api)
+  cm=$(kubectl -n tke get -o=name configmap | grep api)
+  ds=$(kubectl -n tke get -o=name daemonset | grep gateway)
+  gw=$(kubectl -n tke get -o=name configmap | grep gateway)
+  r=("$api $cm $ds $gw")
+  echo ======= backup current resources =======
+  dir=/opt/oidcbackup_beforrollback
+  mkdir -p ${dir}/deployment.apps ${dir}/daemonset.apps ${dir}/configmap
+  for n in $r; do
+    kubectl -n tke get -o=yaml $n >${dir}/$n.yaml
+  done
+  echo "==== 0. Execute OIDC rollback backup success ===="
+}
+
+function rollbackConfigMap() {
+  echo ======= rollback configmap $1 =======
+
+  file=$1.yaml
+
+  kubectl -n tke get cm $1 -o yaml >$file
+
+  start=$(sed -n '/last-applied-configuration/=' $file)
+  if [ "$start" != "" ]; then
+    end=$(($start + 1))
+    sed -i "$start, $end d" $file
+  fi
+
+  if [ "$file" == "tke-auth-api.yaml" ]; then
+
+    start=$(sed -n '/authentication.oidc/=' $file)
+    if [ "$start" != "" ]; then
+      end=$(($start + 8))
+      sed -i "$start, $end d" $file
+    fi
+
+    sed -i '/init_tenant_type/d' $file
+    sed -i '/init_tenant_id/d' $file
+    sed -i '/cloudindustry_config_file/d' $file
+    line=$(sed -n '/init_client_id/=' $file)
+    if [ "$line" != "" ]; then
+      sed -i "${line}c \ \ \ \ init_client_id = \"default\"" $file
+    fi
+    line=$(sed -n '/init_client_secret/=' $file)
+    if [ "$line" != "" ]; then
+      sed -i "${line}c \ \ \ \ init_client_secret = \"2HWJXNnGagpGvnSBQ6Y2P8xJylu\"" $file
+    fi
+  else
+    #    sed -i '/client_secret/d' $file
+    line=$(sed -n '/client_secret/=' $file)
+    if [ "$line" != "" ]; then
+      sed -i "${line}c \ \ \ \ \ \ client_secret = \"2HWJXNnGagpGvnSBQ6Y2P8xJylu\"" $file
+    fi
+    line=$(sed -n '/client_id/=' $file)
+    if [ "$line" != "" ]; then
+      sed -i "${line}c \ \ \ \ \ \ client_id = \"default\"" $file
+    fi
+    line=$(sed -n '/\<issuer_url/=' $file)
+    if [ "$line" != "" ]; then
+      sed -i "${line}c \ \ \ \ \ \ issuer_url = \"https://tke-auth-api/oidc\"" $file
+      cat $file | grep external_issuer_url
+      if [ $? != 0 ]; then
+        sed -i "/issuer_url/a\      external_issuer_url = \"https://tke-auth-api/oidc\"" $file
+      fi
+    fi
+    line=$(sed -n '/^      ca_file/=' $file)
+    if [ "$line" == "" ]; then
+      sed -i '/username_prefix/i\      ca_file = "\/app\/certs\/ca.crt"' $file
+    else
+      sed -i "${line}c \ \ \ \ \ \ ca_file = \"\/app\/certs\/ca.crt\"" $file
+    fi
+    if [ "$file" == "tke-gateway.yaml" ]; then
+      sed -i '/disableOIDCProxy: true/d' $file
+      sed -i '/generic/d' $file
+      sed -i '/external_hostname/d' $file
+    fi
+  fi
+
+  sed -i '/resourceVersion/d' $file
+  sed -i '/uid/d' $file
+
+  kubectl apply -f $file
+  rm $file
+
+}
+
+function rollbackWorkload() {
+  resourceType=$1
+  resource=$2
+  echo ======= rollback $resourceType $resource =======
+  kubectl -n tke annotate $1 $2 tkeanywhere/oidc-
+
+  script="
+import json; import sys
+
+obj=json.load(sys.stdin)
+
+obj['spec']['template']['spec'].pop('hostAliases', None)
+
+for i, v in enumerate(obj['spec']['template']['spec']['containers'][0]['volumeMounts']):
+    if v['name'] == 'oidc-ca-volume' :
+      obj['spec']['template']['spec']['containers'][0]['volumeMounts'].pop(i)
+
+for i, v in enumerate(obj['spec']['template']['spec']['containers'][0]['volumeMounts']):
+    if v['name'] == 'cloudindustry-config-volume':
+      obj['spec']['template']['spec']['containers'][0]['volumeMounts'].pop(i)
+
+for i, v in enumerate(obj['spec']['template']['spec']['volumes']):
+    if v['name'] == 'oidc-ca-volume':
+      obj['spec']['template']['spec']['volumes'].pop(i)
+
+for i, v in enumerate(obj['spec']['template']['spec']['volumes']):
+    if v['name'] == 'cloudindustry-config-volume':
+      obj['spec']['template']['spec']['volumes'].pop(i)
+
+print json.dumps(obj, indent=2)
+"
+  kubectl -n tke get $resourceType $resource -o json | python -c """$script""" >$resource-$resourceType.yaml
+
+  kubectl apply -f $resource-$resourceType.yaml
+  rm $resource-$resourceType.yaml
+}
+
+function removeuser() {
+  echo "==== 3. Execute OIDC rollback removeuser doing ===="
+  kubectl get  platforms.business.tkestack.io platform-default -oyaml > default.yaml
+  sed -i '/- admin/{n;d}' default.yaml
+  kubectl apply -f default.yaml
+  echo "==== 3. Execute OIDC rollback removeuser success ===="
+}
+
+function rollbackall() {
+  echo "==== 1. Execute OIDC rollback rollbackConfigMap doing ===="
+  rollbackConfigMap tke-gateway
+  for cm in $(kubectl get cm -n tke -o=name | grep api | awk '{print $1}' | awk -F '/' '{print $2}'); do
+    rollbackConfigMap $cm
+  done
+  echo "==== 1. Execute OIDC rollback rollbackConfigMap success ===="
+
+  kubectl delete idp default
+
+  echo "==== 2. Execute OIDC rollback rollback doing ===="
+  for deploy in $(kubectl get deployment -n tke -o=name | grep api | awk '{print $1}' | awk -F '/' '{print $2}'); do
+    rollbackWorkload deploy $deploy
+  done
+  rollbackWorkload daemonSet tke-gateway
+  echo "==== 2. Execute OIDC rollback rollback success ===="
+}
+function checkall() {
+  echo "==== 4. Execute OIDC rollback checkall doing ===="
+  check_daemonset tke-gateway
+  for deploy in $(kubectl get deployment -n tke -o=name | grep api | awk '{print $1}' | awk -F '/' '{print $2}'); do
+    check_deployment $deploy
+  done
+  echo "==== 4. Execute OIDC rollback checkall success ===="
+}
+
+function check_deployment {
+  check_workload_status_args deployment $1 tke .status.readyReplicas
+  check=true
+  while [ $check = true ]
+  do
+  check=false
+  result=$(kubectl get deployment $1 -n tke --ignore-not-found -o go-template --template='{{if or (ne (.status.replicas) (.status.readyReplicas)) (ne (.status.replicas) (.status.updatedReplicas))}}false{{else}}true{{end}}')
+  if [ $check = false ] && ([ -z $result ] || [ $result != true ]); then
+    check=true
+    echo "waiting $1"
+    sleep 1
+  fi
+  done
+}
+
+function check_daemonset {
+  check_workload_status_args daemonset $1 tke .status.numberReady
+  check=true
+  while [ $check = true ]
+  do
+  check=false
+  result=$(kubectl get daemonset $1 -n tke --ignore-not-found -o go-template --template='{{if or (ne (.status.desiredNumberScheduled) (.status.numberReady)) (ne (.status.desiredNumberScheduled) (.status.updatedNumberScheduled))}}false{{else}}true{{end}}')
+  if [ $check = false ] && ([ -z $result ] || [ $result != true ]); then
+    check=true
+    echo "waiting $1"
+    sleep 1
+  fi
+  done
+}
+
+function check_workload_status_args {
+    check=true
+    while [ $check = true ]
+    do
+    check=false
+    result=$(kubectl get $1 $2 -n $3 --ignore-not-found -o go-template --template='{{if not ('$4')}}false{{else}}true{{end}}')
+    if [ $check = false ] && ([ -z $result ] || [ $result != true ]); then
+        check=true
+        echo "waiting $2 val $4"
+        sleep 1
+    fi
+    done
+}
+
+backup
+rollbackall
+removeuser
+checkall