diff --git a/deploy_apps/tks-lma-federation-wftpl.yaml b/deploy_apps/tks-lma-federation-wftpl.yaml index cf9618b5..d101d331 100644 --- a/deploy_apps/tks-lma-federation-wftpl.yaml +++ b/deploy_apps/tks-lma-federation-wftpl.yaml @@ -269,48 +269,6 @@ spec: default: "Something wrong" path: /mnt/out/managed_cluster.txt - - name: GetMyThanosScEndpoint - inputs: - parameters: - - name: target_namespace - outputs: - parameters: - - name: my_thanos_sc_ep - valueFrom: - path: /mnt/out/thanos_sc_ep.txt - volumes: - - name: out - emptyDir: {} - container: - name: 'get-thanos-ep' - image: harbor.taco-cat.xyz/tks/hyperkube:v1.18.6 - envFrom: - - secretRef: - name: "git-svc-token" - env: - - name: TARGET_NAMESPACE - value: '{{inputs.parameters.target_namespace}}' - volumeMounts: - - name: out - mountPath: /mnt/out - command: - - /bin/bash - - '-c' - - | - kube_secret=$(kubectl get secret -n {{workflow.parameters.cluster_id}} {{workflow.parameters.cluster_id}}-tks-kubeconfig -o jsonpath="{.data.value}" | base64 -d) - echo -e "kube_secret:\n$kube_secret" | head -n 5 - cat <<< "$kube_secret" > /etc/kubeconfig - - THANOS_SC_PORT='10901' - THANOS_SC_SVC='lma-thanos-external' - thanos_sc_ep=$(kubectl --kubeconfig=/etc/kubeconfig get svc ${THANOS_SC_SVC} -n ${TARGET_NAMESPACE} -o jsonpath='{.status.loadBalancer.ingress[0].hostname}') - if [[ -z "$thanos_sc_ep" ]]; then - echo "Error: could not retrieve thanos sidecar endpoint from service resource." - exit 1 - else - echo "$thanos_sc_ep:$THANOS_SC_PORT" > /mnt/out/thanos_sc_ep.txt - fi - - name: create-keycloak-client activeDeadlineSeconds: 600 inputs: @@ -358,7 +316,8 @@ spec: sleep 5 done - grafana_ep_secret=${kubectl --kubeconfig=kubeconfig get secret -n taco-system tks-endpoint-secret -o jsonpath='{.data.grafana}'| base64 -d } + grafana_ep_secret=$(kubectl get secret -n ${cluster_id} tks-endpoint-secret -o jsonpath='{.data.grafana}'| base64 -d ) + if [ grafana_ep_secret == "" ]; then while [ -z $(kubectl --kubeconfig=kubeconfig get svc -n lma grafana -o jsonpath="{.status.loadBalancer.ingress[*].hostname}") ]; do if [ "$(kubectl --kubeconfig=kubeconfig get svc -n lma grafana -o jsonpath='{.spec.type}')" != "LoadBalancer" ]; then diff --git a/deploy_apps/tks-primary-cluster.yaml b/deploy_apps/tks-primary-cluster.yaml index ecafe826..c54c6e6e 100644 --- a/deploy_apps/tks-primary-cluster.yaml +++ b/deploy_apps/tks-primary-cluster.yaml @@ -66,7 +66,7 @@ spec: # template: sub-get-cluster - - name: set-primary-cluster-on-tks-info - template: sub-set-primay-cluster-on-tks-info + template: sub-set-primary-cluster-on-tks-info arguments: parameters: - name: cluster_id @@ -111,40 +111,18 @@ spec: ] when: "{{workflow.parameters.object_store}} == minio" - - - name: pre-change-target - template: sub-pre-change-logging-target + - - name: loki-use-s3 + template: loki-use-s3 arguments: parameters: - name: primary_cluster value: '{{inputs.parameters.primary_cluster}}' - name: member_clusters value: '{{inputs.parameters.member_clusters}}' - when: "{{workflow.parameters.object_store}} == s3" - - - - name: render-pre-modified-clusters - templateRef: - name: event-gitea-render-manifests - template: main - arguments: - parameters: - - name: decapod_site_repo - value: "{{ workflow.parameters.github_account }}/{{item}}" + - name: github_account + value: "{{ workflow.parameters.github_account }}" - name: base_repo_branch value: "{{ workflow.parameters.base_repo_branch }}" - withParam: "{{ steps.pre-change-target.outputs.parameters.modified_cluster_list}}" - - - - name: federation-components-preinstall-for-s3 - templateRef: - name: create-application - template: installApps - arguments: - parameters: - - name: list - value: | - [ - { "app_group": "lma", "path": "lma-bucket", "namespace": "taco-system", "target_cluster": "" }, - { "app_group": "lma", "path": "loki", "namespace": "lma", "target_cluster": "" } - ] when: "{{workflow.parameters.object_store}} == s3" - - name: change-target @@ -204,13 +182,55 @@ spec: value: "{{ workflow.parameters.base_repo_branch }}" withParam: "{{ steps.change-target.outputs.parameters.modified_cluster_list}}" + - name: loki-use-s3 + inputs: + parameters: + - name: primary_cluster + - name: member_clusters + - name: github_account + - name: base_repo_branch + + steps: + - - name: pre-change-target + template: sub-pre-change-logging-target + arguments: + parameters: + - name: primary_cluster + value: '{{inputs.parameters.primary_cluster}}' + - name: member_clusters + value: '{{inputs.parameters.member_clusters}}' + + - - name: render-pre-modified-clusters + templateRef: + name: event-gitea-render-manifests + template: main + arguments: + parameters: + - name: decapod_site_repo + value: "{{ workflow.parameters.github_account }}/{{item}}" + - name: base_repo_branch + value: "{{ workflow.parameters.base_repo_branch }}" + withParam: "{{ steps.pre-change-target.outputs.parameters.modified_cluster_list}}" + + - - name: federation-components-preinstall-for-s3 + templateRef: + name: create-application + template: installApps + arguments: + parameters: + - name: list + value: | + [ + { "app_group": "lma", "path": "lma-bucket", "namespace": "taco-system", "target_cluster": "" }, + { "app_group": "lma", "path": "loki", "namespace": "lma", "target_cluster": "" } + ] + - name: update-eps-for-thanos inputs: parameters: - name: primary_cluster - name: member_clusters steps: - - - name: change-thanos-sidecar template: sub-change-thanos-sidecar arguments: @@ -309,7 +329,7 @@ spec: primary_cluster=${current_cluster} fi - S3_Service="s3://ap-northeast-2" + S3_SERVICE="s3://ap-northeast-2" cp /kube/value kubeconfig_adm export KUBECONFIG=kubeconfig_adm @@ -334,7 +354,7 @@ spec: for member in $member_clusters do # 1. endpoint of fb on eachcluster - log "INFO" "##### change the loki target to $LOKI_HOST:$LOKI_PORT and $S3_Service (the current target is ${member})" + log "INFO" "##### change the loki target to $LOKI_HOST:$LOKI_PORT and $S3_SERVICE (the current target is ${member})" [ -d ${member} ] || git clone ${repository_base}${member} cd ${member} @@ -347,9 +367,9 @@ spec: yq -i e ".global.clusterName=\"${member}\"" ${member}/lma/site-values.yaml yq -i e "del(.charts[] | select(.name == \"loki\").override.loki.storageConfig.aws)" ${member}/lma/site-values.yaml - yq -i e ".charts |= map(select(.name == \"loki\").override.loki.storageConfig.aws.s3=\"s3://ap-northeast-2\")" ${member}/lma/site-values.yaml - yq -i e ".charts |= map(select(.name == \"loki\").override.loki.storageConfig.aws.dynamodb.dynamodb_url=\"dynamodb://ap-northeast-2\")" ${member}/lma/site-values.yaml - yq -i e ".charts |= map(select(.name == \"loki\").override.loki.storageConfig.aws.bucketnames=\"${primary_cluster}-tks-loki\")" ${member}/lma/site-values.yaml + yq -i e ".charts |= map(select(.name == \"loki\").override.loki.storageConfig.aws.s3=\"s3://ap-northeast-2\")" ${member}/lma/site-values.yaml + yq -i e ".charts |= map(select(.name == \"loki\").override.loki.storageConfig.aws.dynamodb.dynamodb_url=\"dynamodb://ap-northeast-2\")" ${member}/lma/site-values.yaml + yq -i e ".charts |= map(select(.name == \"loki\").override.loki.storageConfig.aws.bucketnames=\"${primary_cluster}-tks-loki\")" ${member}/lma/site-values.yaml if [ `kubectl get AWSManagedMachinePool -n ${member} ${member}-mp-taco --ignore-not-found=true | grep -v NAME | wc -l ` -eq 1 ]; then if [ -z $iamRoles ]; then @@ -363,7 +383,7 @@ spec: cd - done - yq -i e ".global.iamRoles=[${iamRoles}]" ${primary_cluster}/${primary_cluster}/lma/site-values.yaml + yq -i e ".global.tksIamRoles=[${iamRoles}]" ${primary_cluster}/${primary_cluster}/lma/site-values.yaml git config --global user.name "tks" git config --global user.email "tks@sktelecom.com" @@ -446,25 +466,37 @@ spec: primary_kube_secret=$(kubectl get secret -n ${primary_cluster} ${primary_cluster}-tks-kubeconfig -o jsonpath="{.data.value}" | base64 -d) # echo -e "primary_kube_secret:\n$primary_kube_secret" | head -n 5 cat <<< "$primary_kube_secret" > kubeconfig + LOKI_SERVICE=$(kubectl get secret -n ${primary_cluster} tks-endpoint-secret -o jsonpath='{.data.loki}'| base64 -d ) - while [ -z $(kubectl --kubeconfig=kubeconfig get svc -n lma loki-loki-distributed-gateway -o jsonpath="{.status.loadBalancer.ingress[*].hostname}") ] - do - if [ "$(kubectl --kubeconfig=kubeconfig get svc -n lma loki-loki-distributed-gateway -o jsonpath="{.spec.type}")" -neq "LoadBalancer" ]; then - log "FAIL" "The infras on primary are not cofigured properly.(No LoadBalancer)" - exit -1 - fi + if [ "$LOKI_SERVICE" == "" ]; then + while [ -z $(kubectl --kubeconfig=kubeconfig get svc -n lma loki-loki-distributed-gateway -o jsonpath="{.status.loadBalancer.ingress[*].hostname}") ] + do + if [ "$(kubectl --kubeconfig=kubeconfig get svc -n lma loki-loki-distributed-gateway -o jsonpath="{.spec.type}")" -neq "LoadBalancer" ]; then + log "FAIL" "The infras on primary are not cofigured properly.(No LoadBalancer)" + exit -1 + fi - echo "Waiting for generating the loadbalancer of LOKI(3s)" - sleep 3 - done + echo "Waiting for generating the loadbalancer of LOKI(3s)" + sleep 3 + done - LOKI_HOST=$(kubectl --kubeconfig=kubeconfig get svc -n lma loki-loki-distributed-gateway -o jsonpath="{.status.loadBalancer.ingress[0].hostname}") - LOKI_PORT=$(kubectl --kubeconfig=kubeconfig get svc -n lma loki-loki-distributed-gateway -o jsonpath="{.spec.ports[0].port}") + LOKI_HOST=$(kubectl --kubeconfig=kubeconfig get svc -n lma loki-loki-distributed-gateway -o jsonpath="{.status.loadBalancer.ingress[0].hostname}") + LOKI_PORT=$(kubectl --kubeconfig=kubeconfig get svc -n lma loki-loki-distributed-gateway -o jsonpath="{.spec.ports[0].port}") + else + LOKI_HOST=$(echo $LOKI_SERVICE | awk -F : '{print $1}') + LOKI_PORT=$(echo $LOKI_SERVICE | awk -F : '{print $2}') + if [ "$LOKI_PORT" == "" ]; then + LOKI_PORT=80 + fi + fi if [ "$OBJECT_SOTRE" == "minio" ]; then - S3_HOST=$(kubectl --kubeconfig=kubeconfig get svc -n lma minio -o jsonpath="{.status.loadBalancer.ingress[0].hostname}") - S3_PORT=$(kubectl --kubeconfig=kubeconfig get svc -n lma minio -o jsonpath="{.spec.ports[0].port}") - S3_Service=${S3_HOST}:${S3_PORT} + S3_SERVICE=$(kubectl get secret -n ${primary_cluster} tks-endpoint-secret -o jsonpath='{.data.minio}'| base64 -d ) + if [ "$S3_SERVICE" == "" ]; then + S3_HOST=$(kubectl --kubeconfig=kubeconfig get svc -n lma minio -o jsonpath="{.status.loadBalancer.ingress[0].hostname}") + S3_PORT=$(kubectl --kubeconfig=kubeconfig get svc -n lma minio -o jsonpath="{.spec.ports[0].port}") + S3_SERVICE=${S3_HOST}:${S3_PORT} + fi fi ################# @@ -480,19 +512,19 @@ spec: for member in $member_clusters do # 1. endpoint of fb on eachcluster - log "INFO" "##### change the loki target to $LOKI_HOST:$LOKI_PORT and $S3_Service (the current target is ${member})" + log "INFO" "##### change the loki target to $LOKI_HOST:$LOKI_PORT and $S3_SERVICE (the current target is ${member})" [ -d ${member} ] || git clone ${repository_base}${member} cd ${member} yq -i e ".global.lokiHost=\"${LOKI_HOST}\"" ${member}/lma/site-values.yaml yq -i e ".global.lokiPort=\"${LOKI_PORT}\"" ${member}/lma/site-values.yaml if [ "$OBJECT_SOTRE" == "minio" ]; then - yq -i e ".global.s3Service=\"${S3_Service}\"" ${member}/lma/site-values.yaml + yq -i e ".global.s3Service=\"${S3_SERVICE}\"" ${member}/lma/site-values.yaml fi yq -i e ".global.clusterName=\"${member}\"" ${member}/lma/site-values.yaml - # 2. grafana datasource on primay_cluster + # 2. grafana datasource on primary_cluster if [ ${member} = ${primary_cluster} ]; then yq -i e ".global.grafanaDatasourceMetric=\"thanos-query.lma:9090\"" ${member}/lma/site-values.yaml yq -i e ".global.TksWebhookUrl=\"{{workflow.parameters.alert_tks}}\"" ${member}/lma/site-values.yaml @@ -511,9 +543,9 @@ spec: do cd ${member} if [[ `git status --porcelain` ]]; then - log "INFO" "##### commit changes on ${member} to $LOKI_HOST:$LOKI_PORT and $S3_Service" + log "INFO" "##### commit changes on ${member} to $LOKI_HOST:$LOKI_PORT and $S3_SERVICE" if [ "$OBJECT_SOTRE" == "minio" ]; then - cmessage="the loki to $LOKI_HOST:$LOKI_PORT and grafana to $S3_Service (cluster ${member})" + cmessage="the loki to $LOKI_HOST:$LOKI_PORT and grafana to $S3_SERVICE (cluster ${member})" else cmessage="the loki to $LOKI_HOST:$LOKI_PORT (cluster ${member})" fi @@ -581,50 +613,60 @@ spec: primary_cluster=${current_cluster} fi - S3_Service="s3://ap-northeast-2" - cp /kube/value kubeconfig_adm - export KUBECONFIG=kubeconfig_adm + primary_kube_secret=$(kubectl get secret -n ${primary_cluster} ${primary_cluster}-tks-kubeconfig -o jsonpath="{.data.value}" | base64 -d) + # echo -e "primary_kube_secret:\n$primary_kube_secret" | head -n 5 + cat <<< "$primary_kube_secret" > kubeconfig + S3_SERVICE=$(kubectl get secret -n ${primary_cluster} tks-endpoint-secret -o jsonpath='{.data.minio}'| base64 -d ) + if [ "$S3_SERVICE" == "" ]; then - ################# - # updates - ################# - GIT_ACCOUNT={{workflow.parameters.github_account}} - if [[ $GIT_SVC_URL == https://* ]]; then - repository_base=https://${TOKEN//[$'\t\r\n ']}@${GIT_SVC_URL/http:\/\//}/${GIT_ACCOUNT}/ - else - repository_base=http://${TOKEN//[$'\t\r\n ']}@${GIT_SVC_URL/http:\/\//}/${GIT_ACCOUNT}/ - fi + S3_SERVICE="s3://ap-northeast-2" + cp /kube/value kubeconfig_adm + export KUBECONFIG=kubeconfig_adm - log "INFO" "##### change the loki target to $LOKI_HOST:$LOKI_PORT and $S3_Service (the current target is ${current_cluster})" - [ -d ${current_cluster} ] || git clone ${repository_base}${current_cluster} - cd ${current_cluster} + ################# + # updates + ################# + GIT_ACCOUNT={{workflow.parameters.github_account}} + if [[ $GIT_SVC_URL == https://* ]]; then + repository_base=https://${TOKEN//[$'\t\r\n ']}@${GIT_SVC_URL/http:\/\//}/${GIT_ACCOUNT}/ + else + repository_base=http://${TOKEN//[$'\t\r\n ']}@${GIT_SVC_URL/http:\/\//}/${GIT_ACCOUNT}/ + fi - yq -i e "del(.charts[] | select(.name == \"thanos-config\").override.objectStorage)" ${current_cluster}/lma/site-values.yaml - yq -i e ".charts |= map(select(.name == \"thanos-config\").override.objectStorage.type=\"s3\")" ${current_cluster}/lma/site-values.yaml - yq -i e ".charts |= map(select(.name == \"thanos-config\").override.objectStorage.rawConfig.endpoint=\"s3.ap-northeast-2.amazonaws.com\")" ${current_cluster}/lma/site-values.yaml - yq -i e ".charts |= map(select(.name == \"thanos-config\").override.objectStorage.rawConfig.region=\"ap-northeast-2\")" ${current_cluster}/lma/site-values.yaml - yq -i e ".charts |= map(select(.name == \"thanos-config\").override.objectStorage.rawConfig.bucket=\"${primary_cluster}-tks-thanos\")" ${current_cluster}/lma/site-values.yaml - yq -i e ".charts |= map(select(.name == \"thanos-config\").override.objectStorage.rawConfig.signature_version2=false)" ${current_cluster}/lma/site-values.yaml + log "INFO" "##### change the loki target to $LOKI_HOST:$LOKI_PORT and $S3_SERVICE (the current target is ${current_cluster})" + [ -d ${current_cluster} ] || git clone ${repository_base}${current_cluster} + cd ${current_cluster} - git config --global user.name "tks" - git config --global user.email "tks@sktelecom.com" + yq -i e "del(.charts[] | select(.name == \"thanos-config\").override.objectStorage)" ${current_cluster}/lma/site-values.yaml + yq -i e ".charts |= map(select(.name == \"thanos-config\").override.objectStorage.type=\"s3\")" ${current_cluster}/lma/site-values.yaml + yq -i e ".charts |= map(select(.name == \"thanos-config\").override.objectStorage.rawConfig.endpoint=\"s3.ap-northeast-2.amazonaws.com\")" ${current_cluster}/lma/site-values.yaml + yq -i e ".charts |= map(select(.name == \"thanos-config\").override.objectStorage.rawConfig.region=\"ap-northeast-2\")" ${current_cluster}/lma/site-values.yaml + yq -i e ".charts |= map(select(.name == \"thanos-config\").override.objectStorage.rawConfig.bucket=\"${primary_cluster}-tks-thanos\")" ${current_cluster}/lma/site-values.yaml + yq -i e ".charts |= map(select(.name == \"thanos-config\").override.objectStorage.rawConfig.signature_version2=false)" ${current_cluster}/lma/site-values.yaml - if [[ `git status --porcelain` ]]; then - log "INFO" "##### commit changes on ${current_cluster} to use s3" - cmessage="changes on ${current_cluster} to use s3" - git add ${current_cluster}/lma/site-values.yaml - git commit -m "change loki and thanos endpoints. (by set-primary workflow)" -m "$cmessage" - git push - modified_clusters=${current_cluster} - # echo -n "${current_cluster} " >> /mnt/out/modified_cluster_list.txt + git config --global user.name "tks" + git config --global user.email "tks@sktelecom.com" + + if [[ `git status --porcelain` ]]; then + log "INFO" "##### commit changes on ${current_cluster} to use s3" + cmessage="changes on ${current_cluster} to use s3" + git add ${current_cluster}/lma/site-values.yaml + git commit -m "change loki and thanos endpoints. (by set-primary workflow)" -m "$cmessage" + git push + modified_clusters=${current_cluster} + # echo -n "${current_cluster} " >> /mnt/out/modified_cluster_list.txt + else + log "INFO" "No change on the cluster ${current_cluster}" + echo NO_CHANGE_HERE > /mnt/out/modified_cluster_list.txt + fi + cd - + rm -rf ${current_cluster} + + jq -n '$ARGS.positional' --args $modified_clusters > /mnt/out/modified_cluster_list.txt else - log "INFO" "No change on the cluster ${current_cluster}" - echo NO_CHANGE_HERE > /mnt/out/modified_cluster_list.txt + echo "This site uses the predefined loki and static object stores." + echo NO_CHANGE_HERE > /mnt/out/modified_cluster_list.txt fi - cd - - rm -rf ${current_cluster} - - jq -n '$ARGS.positional' --args $modified_clusters > /mnt/out/modified_cluster_list.txt env: - name: OBJECT_SOTRE @@ -689,7 +731,10 @@ spec: # Thanos Endpoints kube_secret=$(kubectl get secret -n ${member} ${member}-tks-kubeconfig -o jsonpath="{.data.value}" | base64 -d) cat <<< "$kube_secret" > kubeconfig - if [ `kubectl --kubeconfig=kubeconfig get svc -n lma lma-thanos-external --ignore-not-found=true | grep -v NAME | wc -l ` -eq 1 ]; then + PROMETHEUS_URL=$(kubectl get secret -n ${member} tks-endpoint-secret -o jsonpath='{.data.prometheus}'| base64 -d ) + if [ "$PROMETHEUS_URL" != "" ]; then + eplist="${eplist}, \"${PROMETHEUS_URL}\"" + elif [ `kubectl --kubeconfig=kubeconfig get svc -n lma lma-thanos-external --ignore-not-found=true | grep -v NAME | wc -l ` -eq 1 ]; then while [ -z $(kubectl --kubeconfig=kubeconfig get svc -n lma lma-thanos-external -o jsonpath="{.status.loadBalancer.ingress[*].hostname}") ] do if [ "$(kubectl --kubeconfig=kubeconfig get svc -n lma lma-thanos-external -o jsonpath="{.spec.type}")" -neq "LoadBalancer" ]; then @@ -754,29 +799,29 @@ spec: yq -i e ".global.thanosQueryStores=[${eplist}]" ${primary_cluster}/lma/site-values.yaml if [[ `git status --porcelain` ]]; then - log "INFO" "##### commit changes endpoints for the thanos query on ${primay_cluster} " - cmessage="changes endpoints for the thanos query on ${primay_cluster}" + log "INFO" "##### commit changes endpoints for the thanos query on ${primary_cluster} " + cmessage="changes endpoints for the thanos query on ${primary_cluster}" git add ${primary_cluster}/lma/site-values.yaml git commit -m "change thanos-query stores. (by set-primary workflow)" -m "$cmessage" git push echo ${primary_cluster} > /mnt/out/changed.txt else - log "INFO" "No change on the cluster ${member}" + log "INFO" "No change on the cluster ${primary_cluster}" echo NO_CHANGE_HERE > /mnt/out/changed.txt fi if [ "$OBJECT_SOTRE" != "s3" ]; then - yq -i e ".global.iamRoles=[${iamRoles}]" ${primary_cluster}/lma/site-values.yaml + yq -i e ".global.tksIamRoles=[${iamRoles}]" ${primary_cluster}/lma/site-values.yaml if [[ `git status --porcelain` ]]; then - log "INFO" "##### commit changes iamRoles for the s3 on ${primay_cluster} " - cmessage="changes iamRoles for the s3 on ${primay_cluster}" + log "INFO" "##### commit changes iamRoles for the s3 on ${primary_cluster} " + cmessage="changes iamRoles for the s3 on ${primary_cluster}" git add ${primary_cluster}/lma/site-values.yaml git commit -m "change iamRoles(s3). (by set-primary workflow)" -m "$cmessage" git push echo ${primary_cluster} > /mnt/out/changed.txt else - log "INFO" "(iamRoles) No change on the cluster ${member}" + log "INFO" "(iamRoles) No change on the cluster ${primary_cluster}" fi fi @@ -808,7 +853,6 @@ spec: path: /mnt/out/changed.txt activeDeadlineSeconds: 900 - - name: sub-remove-individual-loki-and-grafana inputs: parameters: @@ -946,7 +990,7 @@ spec: retryStrategy: limit: 2 - - name: sub-set-primay-cluster-on-tks-info + - name: sub-set-primary-cluster-on-tks-info inputs: parameters: - name: cluster_id diff --git a/deploy_apps/tks-remove-lma-federation-wftpl.yaml b/deploy_apps/tks-remove-lma-federation-wftpl.yaml index 90fe35ac..b35f2dc2 100644 --- a/deploy_apps/tks-remove-lma-federation-wftpl.yaml +++ b/deploy_apps/tks-remove-lma-federation-wftpl.yaml @@ -116,7 +116,7 @@ spec: - - name: set-primary-cluster-on-tks-info templateRef: name: tks-primary-cluster - template: sub-set-primay-cluster-on-tks-info + template: sub-set-primary-cluster-on-tks-info arguments: parameters: - name: cluster_id @@ -201,7 +201,6 @@ spec: TKS_API_URL = "{{workflow.parameters.tks_api_url}}" CLUSTER_ID = "{{workflow.parameters.cluster_id}}" - def getToken() : data = { 'organizationId' : os.environ['ORGANIZATION_ID'], @@ -336,7 +335,7 @@ spec: sleep 5 done - grafana_ep_secret=${kubectl --kubeconfig=kubeconfig get secret -n taco-system tks-endpoint-secret -o jsonpath='{.data.grafana}'| base64 -d } + grafana_ep_secret=${kubectl get secret -n ${cluster_id} tks-endpoint-secret -o jsonpath='{.data.grafana}'| base64 -d } if [ grafana_ep_secret == "" ]; then while [ -z $(kubectl --kubeconfig=kubeconfig get svc -n lma grafana -o jsonpath="{.status.loadBalancer.ingress[*].hostname}") ]; do if [ "$(kubectl --kubeconfig=kubeconfig get svc -n lma grafana -o jsonpath='{.spec.type}')" != "LoadBalancer" ]; then diff --git a/git-repo/create-cluster-repo.yaml b/git-repo/create-cluster-repo.yaml index dfca470e..fdfed90a 100644 --- a/git-repo/create-cluster-repo.yaml +++ b/git-repo/create-cluster-repo.yaml @@ -91,6 +91,14 @@ spec: ## Replace site-values with fetched params ## yq -i e ".global.clusterName=\"$CLUSTER_ID\"" $CLUSTER_ID/$CLUSTER_ID/tks-cluster/site-values.yaml + + val_tksCpNode=$(echo $CLUSTER_INFO | jq -r '.tksCpNode') + val_tksInfraNode=$(echo $CLUSTER_INFO | jq -r '.tksInfraNode') + val_tksUserNode=$(echo $CLUSTER_INFO | jq -r '.tksUserNode') + yq -i e ".global.tksCpNode=$val_tksCpNode" $CLUSTER_ID/$CLUSTER_ID/tks-cluster/site-values.yaml + yq -i e ".global.tksInfraNode=$val_tksInfraNode" $CLUSTER_ID/$CLUSTER_ID/tks-cluster/site-values.yaml + yq -i e ".global.tksUserNode=$val_tksUserNode" $CLUSTER_ID/$CLUSTER_ID/tks-cluster/site-values.yaml + echo $INFRA_PROVIDER case $INFRA_PROVIDER in aws) @@ -98,22 +106,16 @@ spec: val_ssh_key=$(echo $CLUSTER_INFO | jq -r '.sshKeyName') val_region=$(echo $CLUSTER_INFO | jq -r '.clusterRegion') - val_tksCpNode=$(echo $CLUSTER_INFO | jq -r '.tksCpNode') - val_tksInfraNode=$(echo $CLUSTER_INFO | jq -r '.tksInfraNode') val_tksInfraNodeMax=$(echo $CLUSTER_INFO | jq -r '.tksInfraNodeMax') val_tksInfraNodeType=$(echo $CLUSTER_INFO | jq -r '.tksInfraNodeType') - val_tksUserNode=$(echo $CLUSTER_INFO | jq -r '.tksUserNode') val_tksUserNodeMax=$(echo $CLUSTER_INFO | jq -r '.tksUserNodeMax') val_tksUserNodeType=$(echo $CLUSTER_INFO | jq -r '.tksUserNodeType') yq -i e ".global.sshKeyName=\"$val_ssh_key\"" $CLUSTER_ID/$CLUSTER_ID/tks-cluster/site-values.yaml yq -i e ".global.clusterRegion=\"$val_region\"" $CLUSTER_ID/$CLUSTER_ID/tks-cluster/site-values.yaml - yq -i e ".global.tksCpNode=$val_tksCpNode" $CLUSTER_ID/$CLUSTER_ID/tks-cluster/site-values.yaml - yq -i e ".global.tksInfraNode=$val_tksInfraNode" $CLUSTER_ID/$CLUSTER_ID/tks-cluster/site-values.yaml yq -i e ".global.tksInfraNodeMax=$val_tksInfraNodeMax" $CLUSTER_ID/$CLUSTER_ID/tks-cluster/site-values.yaml yq -i e ".global.tksInfraNodeType=\"$val_tksInfraNodeType\"" $CLUSTER_ID/$CLUSTER_ID/tks-cluster/site-values.yaml - yq -i e ".global.tksUserNode=$val_tksUserNode" $CLUSTER_ID/$CLUSTER_ID/tks-cluster/site-values.yaml yq -i e ".global.tksUserNodeMax=$val_tksUserNodeMax" $CLUSTER_ID/$CLUSTER_ID/tks-cluster/site-values.yaml yq -i e ".global.tksUserNodeType=\"$val_tksUserNodeType\"" $CLUSTER_ID/$CLUSTER_ID/tks-cluster/site-values.yaml @@ -126,7 +128,11 @@ spec: ;; byoh) - echo "BYOH" + cluster_endpoint_host=$(echo $CLUSTER_INFO | jq -r '.byoClusterEndpointHost') + cluster_endpoint_port=$(echo $CLUSTER_INFO | jq -r '.byoClusterEndpointPort') + + yq -i e ".global.clusterEndpointHost=\"$cluster_endpoint_host\"" $CLUSTER_ID/$CLUSTER_ID/tks-cluster/site-values.yaml + yq -i e ".global.clusterEndpointPort=$cluster_endpoint_port" $CLUSTER_ID/$CLUSTER_ID/tks-cluster/site-values.yaml ;; *) @@ -195,7 +201,7 @@ spec: path: /mnt/out/infra_provider.txt - name: managed_cluster valueFrom: - default: "Something wrong" + default: "false" path: /mnt/out/managed_cluster.txt - name: createRepoCredential @@ -249,4 +255,4 @@ spec: parameter: "{{steps.createClusterRepo.outputs.parameters.infra_provider}}" - name: managed_cluster valueFrom: - parameter: "{{steps.createClusterRepo.outputs.parameters.managed_cluster}}" \ No newline at end of file + parameter: "{{steps.createClusterRepo.outputs.parameters.managed_cluster}}" diff --git a/tks-cli/tks-cli.yaml b/tks-cli/tks-cli.yaml index 50a3817d..164f2514 100644 --- a/tks-cli/tks-cli.yaml +++ b/tks-cli/tks-cli.yaml @@ -12,7 +12,7 @@ spec: - name: login-tks-api container: name: login-tks-api - image: harbor.taco-cat.xyz/tks/tks-e2e-test:v3.1.3 + image: harbor.taco-cat.xyz/tks/tks-e2e-test:v3.1.4 envFrom: - secretRef: name: "tks-api-secret" @@ -31,7 +31,7 @@ spec: - name: description container: name: create-organization - image: harbor.taco-cat.xyz/tks/tks-e2e-test:v3.1.3 + image: harbor.taco-cat.xyz/tks/tks-e2e-test:v3.1.4 envFrom: - secretRef: name: "tks-api-secret" @@ -90,9 +90,10 @@ spec: - name: description - name: infra_conf - name: cloud_service + - name: cluster_endpoint container: name: create-usercluster - image: harbor.taco-cat.xyz/tks/tks-e2e-test:v3.1.3 + image: harbor.taco-cat.xyz/tks/tks-e2e-test:v3.1.4 envFrom: - secretRef: name: "tks-api-secret" @@ -109,6 +110,7 @@ spec: TKS_USER_NODE=$(echo "{{inputs.parameters.infra_conf}}" | jq -r '.tksUserNode') CL_NAME="{{inputs.parameters.cluster_name}}" + echo "* Create $CL_NAME cluster" tks cluster create ${CL_NAME} \ --stack-template-id "{{inputs.parameters.stack_template_id}}" \ @@ -116,6 +118,7 @@ spec: --cloud-account-id "{{inputs.parameters.cloud_account_id}}" \ --description "{{inputs.parameters.description}}" \ --cloud-service "{{inputs.parameters.cloud_service}}" \ + --cluster-endpoint "{{inputs.parameters.cluster_endpoint}}" \ --stack 1 \ --tks-cp-node $TKS_CP_NODE \ --tks-infra-node $TKS_INFRA_NODE \ @@ -159,7 +162,7 @@ spec: - name: organization_id container: name: install-usercluster - image: harbor.taco-cat.xyz/tks/tks-e2e-test:v3.1.3 + image: harbor.taco-cat.xyz/tks/tks-e2e-test:v3.1.4 envFrom: - secretRef: name: "tks-api-secret" @@ -212,7 +215,7 @@ spec: - name: cluster_id container: name: delete-usercluster - image: harbor.taco-cat.xyz/tks/tks-e2e-test:v3.1.3 + image: harbor.taco-cat.xyz/tks/tks-e2e-test:v3.1.4 envFrom: - secretRef: name: "tks-api-secret" @@ -255,7 +258,7 @@ spec: - name: description container: name: create-appgroup - image: harbor.taco-cat.xyz/tks/tks-e2e-test:v3.1.3 + image: harbor.taco-cat.xyz/tks/tks-e2e-test:v3.1.4 envFrom: - secretRef: name: "tks-api-secret" @@ -305,7 +308,7 @@ spec: - name: appgroup_id container: name: delete-appgroup - image: harbor.taco-cat.xyz/tks/tks-e2e-test:v3.1.3 + image: harbor.taco-cat.xyz/tks/tks-e2e-test:v3.1.4 envFrom: - secretRef: name: "tks-api-secret" @@ -345,7 +348,7 @@ spec: - name: name container: name: get-appgroup-id - image: harbor.taco-cat.xyz/tks/tks-e2e-test:v3.1.3 + image: harbor.taco-cat.xyz/tks/tks-e2e-test:v3.1.4 envFrom: - secretRef: name: "tks-api-secret" diff --git a/tks-cluster/create-usercluster-wftpl.yaml b/tks-cluster/create-usercluster-wftpl.yaml index b228d1f8..d70daa0c 100644 --- a/tks-cluster/create-usercluster-wftpl.yaml +++ b/tks-cluster/create-usercluster-wftpl.yaml @@ -29,6 +29,8 @@ spec: value: "http://tks-api.tks.svc:9110" - name: base_repo_branch value: "main" + - name: keycloak_url + value: 'https://keycloak.yourdomain.org/auth' volumes: - name: kubeconfig-adm @@ -112,6 +114,16 @@ spec: ] when: "{{steps.tks-create-cluster-repo.outputs.parameters.infra_provider}} == byoh" + - - name: tks-create-config-secret + template: create-endpoint-secret + arguments: + parameters: + - name: cluster_info + value: "{{steps.tks-get-cluster-info.outputs.parameters.cluster_info}}" + - name: cluster_id + value: "{{ workflow.parameters.cluster_id }}" + when: "{{steps.tks-create-cluster-repo.outputs.parameters.infra_provider}} == byoh" + - - name: init-cluster-for-tks template: init-cluster-for-tks arguments: @@ -122,6 +134,10 @@ spec: value: "{{steps.tks-create-cluster-repo.outputs.parameters.infra_provider}}" - name: cloud_account_id value: "{{ workflow.parameters.cloud_account_id }}" + - name: keycloak_url + value: "{{ workflow.parameters.keycloak_url }}" + - name: contract_id + value: "{{ workflow.parameters.contract_id }}" - - name: create-aws-cluster-autoscaler-iam templateRef: @@ -216,9 +232,7 @@ spec: "target_cluster": "" } ] - when: >- - {{steps.tks-create-cluster-repo.outputs.parameters.infra_provider}} != byoh && - {{steps.tks-create-cluster-repo.outputs.parameters.managed_cluster}} == false + when: "{{steps.tks-create-cluster-repo.outputs.parameters.infra_provider}} != byoh" - - name: install-addons-common templateRef: @@ -235,18 +249,6 @@ spec: "namespace": "taco-system", "target_cluster": "" }, - { - "app_group": "tks-cluster", - "path": "metrics-server", - "namespace": "kube-system", - "target_cluster": "" - }, - { - "app_group": "tks-cluster", - "path": "cluster-autoscaler", - "namespace": "kube-system", - "target_cluster": "" - }, { "app_group": "tks-cluster", "path": "argo-rollouts", @@ -270,6 +272,12 @@ spec: "namespace": "taco-system", "target_cluster": "" }, + { + "app_group": "tks-cluster", + "path": "metrics-server", + "namespace": "kube-system", + "target_cluster": "" + }, { "app_group": "tks-cluster", "path": "s3-chart", @@ -318,14 +326,74 @@ spec: value: | [ { - "app_group": "tks-cluster-byoh", + "app_group": "tks-cluster", "path": "local-path-provisioner", "namespace": "taco-system", "target_cluster": "" - } + }, + { + "app_group": "tks-cluster", + "path": "metrics-server", + "namespace": "kube-system", + "target_cluster": "" + } ] when: "{{steps.tks-create-cluster-repo.outputs.parameters.infra_provider}} == byoh" + - - name: set-keycloak-config + templateRef: + name: set-user-cluster + template: main + arguments: + parameters: + - name: cluster_id + value: "{{ workflow.parameters.cluster_id }}" + - name: server_url + value: "{{ workflow.parameters.keycloak_url }}" + - name: target_realm_name + value: "{{ workflow.parameters.contract_id }}" + - name: target_client_id + value: "{{ workflow.parameters.cluster_id}}-k8s-api" + - name: keycloak_credential_secret_name + value: "keycloak" + - name: keycloak_credential_secret_namespace + value: "keycloak" + + - - name: set-cluster-role-binding-cluster-admin + templateRef: + name: k8s-client + template: create-cluster-role-binding + arguments: + parameters: + - name: target_cluster_id + value: "{{workflow.parameters.cluster_id}}" + - name: is_self_target + value: "false" + - name: rolebinding_name + value: "{{workflow.parameters.cluster_id}}-cluster-admin" + - name: role_name + value: "admin" + - name: group_list + value: '["{{workflow.parameters.cluster_id}}-cluster-admin", "cluster-admin"]' + + - - name: set-cluster-role-binding-cluster-view + templateRef: + name: k8s-client + template: create-cluster-role-binding + arguments: + parameters: + - name: target_cluster_id + value: "{{workflow.parameters.cluster_id}}" + - name: is_self_target + value: "false" + - name: rolebinding_name + value: "{{workflow.parameters.cluster_id}}-cluster-view" + - name: role_name + value: "view" + - name: group_list + value: '["{{workflow.parameters.cluster_id}}-cluster-view", "cluster-view"]' + + ####################### # Template Definition # ####################### @@ -335,6 +403,8 @@ spec: - name: cluster_id - name: infra_provider - name: cloud_account_id + - name: keycloak_url + - name: contract_id container: name: cluster-init image: harbor.taco-cat.xyz/tks/tks-cluster-init:v1.0.0 @@ -405,10 +475,21 @@ spec: ;; byoh) + kcp_count=$(kubectl get kcp -n $CLUSTER_ID $CLUSTER_ID | grep -v NAME | wc -l) kubectl wait --for=condition=Available --timeout=600s kcp -n $CLUSTER_ID $CLUSTER_ID KUBECONFIG_WORKLOAD=$(kubectl get secret -n $CLUSTER_ID $CLUSTER_ID-kubeconfig -o jsonpath="{.data.value}" | base64 -d) cat <<< "$KUBECONFIG_WORKLOAD" > kubeconfig_workload + + for machine in $(kubectl get machine -n $CLUSTER_ID -l cluster.x-k8s.io/deployment-name=$CLUSTER_ID-md-$TKS_NODE_NAME -oname) + do + kubectl wait --for=condition=ready --timeout=3600s -n $CLUSTER_ID $machine + done + + for node in $(kubectl get machine -n $CLUSTER_ID -l cluster.x-k8s.io/deployment-name=$CLUSTER_ID-md-$TKS_NODE_NAME -o=jsonpath='{.items[*].status.nodeRef.name}') + do + kubectl --kubeconfig=kubeconfig_workload label node $node taco-lma=enabled tks-ingressgateway=enabled tks-egressgateway=enabled tks-msa=enabled --overwrite + done ;; *) @@ -448,7 +529,29 @@ spec: export KUBECONFIG=kubeconfig_adm if [ $kcp_count = 1 ]; then TKS_KUBECONFIG_WORKLOAD=$(kubectl get secret -n $CLUSTER_ID $CLUSTER_ID-kubeconfig -o jsonpath="{.data.value}" | base64 -d) - TKS_USER_KUBECONFIG_WORKLOAD=$(kubectl get secret -n $CLUSTER_ID $CLUSTER_ID-kubeconfig -o jsonpath="{.data.value}" | base64 -d) + if [ "$INFRA_PROVIDER" == "byoh" ]; then + ISSUER_URL=$KEYCLOAK_URL/realms/$CONTRACT_ID + CLIENT_ID=$CLUSTER_ID-k8s-api + OIDC_USER_NAME="oidc-user" + EXISTING_USER_NAME=$CLUSTER_ID-admin + + kubectl get secret -n $CLUSTER_ID $CLUSTER_ID-kubeconfig -o jsonpath="{.data.value}" | base64 -d > tmp_user_kubeconfig + kubectl --kubeconfig=tmp_user_kubeconfig config unset users.$EXISTING_USER_NAME + kubectl --kubeconfig=tmp_user_kubeconfig config set-credentials $OIDC_USER_NAME \ + --exec-api-version=client.authentication.k8s.io/v1beta1 \ + --exec-command=kubectl \ + --exec-arg=oidc-login \ + --exec-arg=get-token \ + --exec-arg=--oidc-issuer-url=$ISSUER_URL \ + --exec-arg=--oidc-client-id=$CLIENT_ID \ + --exec-arg=--grant-type=password + + CONTEXT_NAME=$(kubectl --kubeconfig=tmp_user_kubeconfig config current-context) + kubectl --kubeconfig=tmp_user_kubeconfig config set-context $CONTEXT_NAME --user $OIDC_USER_NAME + TKS_USER_KUBECONFIG_WORKLOAD=$(cat tmp_user_kubeconfig) + else + TKS_USER_KUBECONFIG_WORKLOAD=$(kubectl get secret -n $CLUSTER_ID $CLUSTER_ID-kubeconfig -o jsonpath="{.data.value}" | base64 -d) + fi elif [ $awsmcp_count = 1 ]; then CAPA_USER_KUBECONFIG_WORKLOAD=$(kubectl get secret -n $CLUSTER_ID $CLUSTER_ID-user-kubeconfig -o jsonpath="{.data.value}" | base64 -d) @@ -482,7 +585,7 @@ spec: EOF kubectl --kubeconfig kubeconfig_workload apply -f sc-taco-storage.yaml else - echo "Wrong AWS Cluster type!" + echo "Wrong Cluster type!" exit 1 fi cat <<< $TKS_KUBECONFIG_WORKLOAD > tks_kubeconfig_workload @@ -507,6 +610,10 @@ spec: value: "{{ inputs.parameters.cloud_account_id }}" - name: TKS_NODE_NAME value: "taco" + - name: KEYCLOAK_URL + value: "{{ inputs.parameters.keycloak_url }}" + - name: CONTRACT_ID + value: "{{ inputs.parameters.contract_id }}" - name: prepare-cluster-autoscaler container: @@ -551,3 +658,46 @@ spec: kubectl --kubeconfig kubeconfig_temp apply -f https://raw.githubusercontent.com/kubernetes-csi/external-snapshotter/master/client/config/crd/snapshot.storage.k8s.io_volumesnapshotclasses.yaml -n kube-system kubectl --kubeconfig kubeconfig_temp apply -f https://raw.githubusercontent.com/kubernetes-csi/external-snapshotter/master/client/config/crd/snapshot.storage.k8s.io_volumesnapshotcontents.yaml -n kube-system kubectl --kubeconfig kubeconfig_temp apply -f https://raw.githubusercontent.com/kubernetes-csi/external-snapshotter/master/client/config/crd/snapshot.storage.k8s.io_volumesnapshots.yaml -n kube-system + + - name: create-endpoint-secret + inputs: + parameters: + - name: cluster_info + - name: cluster_id + container: + name: create-namespace + image: harbor.taco-cat.xyz/tks/hyperkube:v1.18.6 + command: + - /bin/bash + - '-c' + - | + CLUSTER_ID={{inputs.parameters.cluster_id}} + CLUSTER_IP=$(echo $CLUSTER_INFO | jq -r '.byoClusterEndpointHost') + + if [ "$CLUSTER_IP" = "" ]; then + echo "ERROR::Endpoint for the cluster is not given!!!" + exit -1 + fi + + cat < tks-endpoint-secret.yaml + --- + apiVersion: v1 + kind: Secret + metadata: + name: tks-endpoint-secret + namespace: ${CLUSTER_ID} + data: + grafana: $(echo ${CLUSTER_IP}:30001 | base64) # 30001 + loki: $(echo ${CLUSTER_IP}:30002 | base64) # 30002 + minio: $(echo ${CLUSTER_IP}:30003 | base64) # 30003 + prometheus: $(echo ${CLUSTER_IP}:30004 | base64) # 30004 + thanos: $(echo ${CLUSTER_IP}:30005 | base64) # 30005 (queryfrontend만 합시다...) + kiali: $(echo ${CLUSTER_IP}:30011 | base64) # 30011 + jaeger: $(echo ${CLUSTER_IP}:30012 | base64) # 30012 + EOF + kubectl apply -f tks-endpoint-secret.yaml + env: + - name: CLUSTER_INFO + value: "{{inputs.parameters.cluster_info}}" + + activeDeadlineSeconds: 30 diff --git a/tks-cluster/remove-usercluster-wftpl.yaml b/tks-cluster/remove-usercluster-wftpl.yaml index e6685f0d..1b1cca7a 100644 --- a/tks-cluster/remove-usercluster-wftpl.yaml +++ b/tks-cluster/remove-usercluster-wftpl.yaml @@ -21,6 +21,11 @@ spec: value: "{{workflow.parameters.cluster_id}}" - name: filter value: "app={{workflow.parameters.cluster_id}}-{{workflow.parameters.app_group_prefix}}" + - name: keycloak_url + value: '{{workflow.parameters.keycloak_url}}' + - name: contract_id + value: "P0010010a" + volumes: - name: tks-proto-vol configMap: @@ -62,6 +67,9 @@ spec: parameters: - name: app_name value: "{{workflow.parameters.app_prefix}}-cluster-autoscaler" + when: >- + {{steps.findInfraProvider.outputs.parameters.infra_provider}} != byoh && + {{steps.findInfraProvider.outputs.parameters.managed_cluster}} == false - - name: deleteClusterAutoscalerRbacApp templateRef: @@ -190,6 +198,24 @@ spec: name: tks-delete-cluster-repo template: deleteClusterRepo + - - name: unset-keycloak-config + templateRef: + name: keycloak-client + template: delete-client + arguments: + parameters: + - name: server_url + value: "{{ workflow.parameters.keycloak_url }}" + - name: target_realm_name + value: "{{ workflow.parameters.contract_id }}" + - name: target_client_id + value: "{{ workflow.parameters.cluster_id}}-k8s-api" + - name: keycloak_credential_secret_name + value: "keycloak" + - name: keycloak_credential_secret_namespace + value: "keycloak" + + ####################### # Template Definition # @@ -240,7 +266,7 @@ spec: path: /mnt/out/infra_provider.txt - name: managed_cluster valueFrom: - default: "Something wrong" + default: "false" path: /mnt/out/managed_cluster.txt - name: disableAutoSync diff --git a/tks-stack/tks-stack-create.yaml b/tks-stack/tks-stack-create.yaml index 0743262b..f0acdc30 100644 --- a/tks-stack/tks-stack-create.yaml +++ b/tks-stack/tks-stack-create.yaml @@ -29,6 +29,8 @@ spec: value: "s3" - name: base_repo_branch value: develop + - name: cluster_endpoint + value: "" templates: - name: main @@ -60,6 +62,8 @@ spec: value: "{{workflow.parameters.infra_conf}}" - name: cloud_service value: "{{workflow.parameters.cloud_service}}" + - name: cluster_endpoint + value: "{{workflow.parameters.cluster_endpoint}}" - - name: call-create-appgroup-for-LMA templateRef: