From 73ad89f9df5322ca4d78ac003c129320d6c44555 Mon Sep 17 00:00:00 2001 From: Robert Choi Date: Fri, 25 Feb 2022 15:06:09 +0900 Subject: [PATCH 1/2] remove prometheus CRs before deleting LMA apps --- templates/argo-additional-rbac.yaml | 10 ++ .../argo-cd/delete-apps-by-label-wftpl.yaml | 18 ++-- .../decapod-apps/lma-uniformed-wftpl.yaml | 12 +-- .../remove-lma-uniformed-wftpl.yaml | 99 +++++++++++++++++-- 4 files changed, 116 insertions(+), 23 deletions(-) diff --git a/templates/argo-additional-rbac.yaml b/templates/argo-additional-rbac.yaml index 1c43d1e..00de6ab 100644 --- a/templates/argo-additional-rbac.yaml +++ b/templates/argo-additional-rbac.yaml @@ -65,3 +65,13 @@ rules: - delete - list - patch +- apiGroups: + - monitoring.coreos.com + resources: + - prometheuses + - alertmanagers + verbs: + - create + - get + - delete + - list diff --git a/templates/argo-cd/delete-apps-by-label-wftpl.yaml b/templates/argo-cd/delete-apps-by-label-wftpl.yaml index 84614dc..f790e01 100644 --- a/templates/argo-cd/delete-apps-by-label-wftpl.yaml +++ b/templates/argo-cd/delete-apps-by-label-wftpl.yaml @@ -4,11 +4,13 @@ metadata: name: delete-apps namespace: argo spec: + arguments: + parameters: + # Filter to filter argocd apps (E.g., "app=lma") + - name: filter + value: "" templates: - name: DeleteAppsByLabel - inputs: - parameters: - - name: filter activeDeadlineSeconds: 900 container: name: 'delete-apps' @@ -33,10 +35,10 @@ spec: deleted=False - # Delete normal apps including CRs + # Delete normal apps except operators and CRs app_list=$(./argocd app list -l $FILTER --output name | grep -v operator) if [[ $? -eq 0 && -n $app_list ]]; then - echo "[1st phase] Deleting apps: $app_list" + echo -e "[1st phase] Deleting apps:\n$app_list" echo "$app_list" | xargs ./argocd app delete --cascade -y until [ $(./argocd app list -l $FILTER --output name | grep -v operator | wc -l) == 0 ] @@ -51,10 +53,10 @@ spec: echo "No apps found except operators. Skipping 1st phase.." fi - # Delete operators + # Delete operators and CRs app_list=$(./argocd app list -l $FILTER --output name) if [[ $? -eq 0 && -n $app_list ]]; then - echo "[2nd phase] Deleting operators: $app_list" + echo -e "[2nd phase] Deleting operators:\n$app_list" echo "$app_list" | xargs ./argocd app delete --cascade -y until [ $(./argocd app list -l $FILTER --output name | wc -l) == 0 ] @@ -78,4 +80,4 @@ spec: name: "decapod-argocd-config" env: - name: FILTER - value: "{{inputs.parameters.filter}}" + value: "{{workflow.parameters.filter}}" diff --git a/templates/decapod-apps/lma-uniformed-wftpl.yaml b/templates/decapod-apps/lma-uniformed-wftpl.yaml index 7a572a2..869dae6 100644 --- a/templates/decapod-apps/lma-uniformed-wftpl.yaml +++ b/templates/decapod-apps/lma-uniformed-wftpl.yaml @@ -22,9 +22,6 @@ spec: value: "" templates: - name: prepare - inputs: {} - outputs: {} - metadata: {} steps: - - name: configuration template: configuration @@ -32,10 +29,12 @@ spec: - - name: deploy template: deploy arguments: {} + + ####################### + # Template Definition # + ####################### + - name: configuration - inputs: {} - outputs: {} - metadata: {} container: name: config image: 'k8s.gcr.io/hyperkube:v1.18.8' @@ -52,6 +51,7 @@ spec: resources: {} imagePullPolicy: IfNotPresent activeDeadlineSeconds: 120 + - name: deploy dag: tasks: diff --git a/templates/decapod-apps/remove-lma-uniformed-wftpl.yaml b/templates/decapod-apps/remove-lma-uniformed-wftpl.yaml index 9318a5a..c012776 100644 --- a/templates/decapod-apps/remove-lma-uniformed-wftpl.yaml +++ b/templates/decapod-apps/remove-lma-uniformed-wftpl.yaml @@ -1,4 +1,3 @@ -## Example workflow that calls delete-project template ## apiVersion: argoproj.io/v1alpha1 kind: WorkflowTemplate metadata: @@ -8,22 +7,37 @@ spec: entrypoint: process arguments: parameters: - # Filter to filter argocd apps (E.g., "app=lma") + # 'cluster_id' is used for multi-cluster case + # TODO: better to rename this to 'site_name' later + - name: cluster_id + value: "" + - name: app_prefix + value: "" + # Filter to filter argocd apps - name: filter value: "app=lma" templates: - name: process steps: + - - name: disableAutoSyncOfPrometheus + template: disableAutoSync + + # Delete prometheus CR first so that it doesn't hang + # due to earlier service-account deletion # + - - name: removePrometheusCR + template: removePrometheusCR + arguments: + parameters: + - name: namespace + value: "lma" + + # Remove argo CD apps - - name: removeLMA templateRef: name: delete-apps template: DeleteAppsByLabel - arguments: - parameters: - - name: filter - value: "{{ workflow.parameters.filter }}" - # LMA appGroup specific task # + # Cleanup secrets and PVCs - - name: cleanupLeftoverResources template: removeLeftoverResources arguments: @@ -31,6 +45,63 @@ spec: - name: namespace value: "lma" + ####################### + # Template Definition # + ####################### + + - name: disableAutoSync + container: + name: disable-auto-sync + image: docker.io/sktcloud/argocd-cli:v2.2.5 + command: + - /bin/bash + - '-c' + - | + PROM_APP_NAME="prometheus" + if [[ -n "{{workflow.parameters.app_prefix}}" ]]; then + PROM_APP_NAME="{{workflow.parameters.app_prefix}}-prometheus" + fi + echo "[temp] ARGO_SERVER: $ARGO_SERVER" + + # Login to Argo CD server + ./argocd login $ARGO_SERVER --plaintext --insecure --username $ARGO_USERNAME \ + --password $ARGO_PASSWORD + + echo "Disabling auto-sync of [$PROM_APP_NAME]' app.." + + # Disable auto-sync + ./argocd app set $PROM_APP_NAME --sync-policy none + envFrom: + - secretRef: + name: "decapod-argocd-config" + + - name: removePrometheusCR + inputs: + parameters: + - name: namespace + container: + name: remove-prometheus-cr + image: k8s.gcr.io/hyperkube:v1.18.6 + command: + - /bin/bash + - '-c' + - | + echo "Deleting prometheus CRs..." + + kube_params="" + + if [[ -n "{{workflow.parameters.cluster_id}}" ]]; then + kube_secret=$(kubectl get secret -n argo {{workflow.parameters.cluster_id}}-kubeconfig -o jsonpath="{.data.value}" | base64 -d) + echo -e "kube_secret:\n$kube_secret" | head -n 5 + cat <<< "$kube_secret" > /etc/kubeconfig + + kube_params+="--kubeconfig=/etc/kubeconfig" + fi + + kubectl $kube_params delete prometheus --all -n {{inputs.parameters.namespace}} + kubectl $kube_params delete alertmanager --all -n {{inputs.parameters.namespace}} + + - name: removeLeftoverResources inputs: parameters: @@ -42,5 +113,15 @@ spec: - /bin/bash - '-c' - | - kubectl delete secret etcd-client-cert prometheus-operator-admission -n {{inputs.parameters.namespace}} || true - kubectl delete pvc --all -n {{inputs.parameters.namespace}} + echo "Deleting secrets and PVCs..." + + kube_params="" + if [[ -n "{{workflow.parameters.cluster_id}}" ]]; then + kube_secret=$(kubectl get secret -n argo {{workflow.parameters.cluster_id}}-kubeconfig -o jsonpath="{.data.value}" | base64 -d) + cat <<< "$kube_secret" > /etc/kubeconfig + + kube_params+="--kubeconfig=/etc/kubeconfig" + fi + + kubectl $kube_params delete secret etcd-client-cert prometheus-operator-admission -n {{inputs.parameters.namespace}} || true + kubectl $kube_params delete pvc --all -n {{inputs.parameters.namespace}} From 70fdac0d8fa11f1166ae6325b5a716d60559ab15 Mon Sep 17 00:00:00 2001 From: Robert Choi Date: Wed, 2 Mar 2022 13:38:28 +0900 Subject: [PATCH 2/2] read kubesecret from cluster namespace --- templates/decapod-apps/remove-lma-uniformed-wftpl.yaml | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/templates/decapod-apps/remove-lma-uniformed-wftpl.yaml b/templates/decapod-apps/remove-lma-uniformed-wftpl.yaml index c012776..0fd3f56 100644 --- a/templates/decapod-apps/remove-lma-uniformed-wftpl.yaml +++ b/templates/decapod-apps/remove-lma-uniformed-wftpl.yaml @@ -91,7 +91,7 @@ spec: kube_params="" if [[ -n "{{workflow.parameters.cluster_id}}" ]]; then - kube_secret=$(kubectl get secret -n argo {{workflow.parameters.cluster_id}}-kubeconfig -o jsonpath="{.data.value}" | base64 -d) + kube_secret=$(kubectl get secret -n {{workflow.parameters.cluster_id}} {{workflow.parameters.cluster_id}}-kubeconfig -o jsonpath="{.data.value}" | base64 -d) echo -e "kube_secret:\n$kube_secret" | head -n 5 cat <<< "$kube_secret" > /etc/kubeconfig @@ -117,7 +117,7 @@ spec: kube_params="" if [[ -n "{{workflow.parameters.cluster_id}}" ]]; then - kube_secret=$(kubectl get secret -n argo {{workflow.parameters.cluster_id}}-kubeconfig -o jsonpath="{.data.value}" | base64 -d) + kube_secret=$(kubectl get secret -n {{workflow.parameters.cluster_id}} {{workflow.parameters.cluster_id}}-kubeconfig -o jsonpath="{.data.value}" | base64 -d) cat <<< "$kube_secret" > /etc/kubeconfig kube_params+="--kubeconfig=/etc/kubeconfig"