From 5ea9ff5c9c8eace0a207823b75dc9164f7c085fa Mon Sep 17 00:00:00 2001 From: Ryan Lo Date: Mon, 14 Sep 2020 17:58:32 +0800 Subject: [PATCH] SUBMARINE-624. Replace ingress-nginx with Traefik --- dev-support/k8s/deploy-traefik.sh | 59 ++++ .../k8s/traefik/cluster-role-binding.yaml | 29 ++ dev-support/k8s/traefik/cluster-role.yaml | 61 ++++ dev-support/k8s/traefik/crd.yaml | 120 +++++++ dev-support/k8s/traefik/deployment.yaml | 50 +++ dev-support/k8s/traefik/kustomization.yaml | 34 ++ dev-support/k8s/traefik/service-account.yaml | 21 ++ .../k8s/traefik/service.yaml | 22 +- submarine-cloud/hack/ingress/mandatory.yaml | 307 ------------------ submarine-cloud/hack/integration-test.sh | 1 + submarine-cloud/hack/kind-cluster-build.sh | 17 +- 11 files changed, 386 insertions(+), 335 deletions(-) create mode 100755 dev-support/k8s/deploy-traefik.sh create mode 100644 dev-support/k8s/traefik/cluster-role-binding.yaml create mode 100644 dev-support/k8s/traefik/cluster-role.yaml create mode 100644 dev-support/k8s/traefik/crd.yaml create mode 100644 dev-support/k8s/traefik/deployment.yaml create mode 100644 dev-support/k8s/traefik/kustomization.yaml create mode 100644 dev-support/k8s/traefik/service-account.yaml rename submarine-cloud/hack/ingress/service-nodeport.yaml => dev-support/k8s/traefik/service.yaml (71%) delete mode 100644 submarine-cloud/hack/ingress/mandatory.yaml diff --git a/dev-support/k8s/deploy-traefik.sh b/dev-support/k8s/deploy-traefik.sh new file mode 100755 index 0000000000..3fb8400346 --- /dev/null +++ b/dev-support/k8s/deploy-traefik.sh @@ -0,0 +1,59 @@ +#!/usr/bin/env bash +# +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# +set -euo pipefail + +readonly TRAEFIK_IMAGE="traefik:v2.2" + +if [ -L "${BASH_SOURCE-$0}" ]; then + PWD=$(dirname "$(readlink "${BASH_SOURCE-$0}")") +else + PWD=$(dirname "${BASH_SOURCE-$0}") +fi +CURRENT_PATH=$(cd "${PWD}">/dev/null; pwd) +export CURRENT_PATH +export SUBMARINE_HOME=${CURRENT_PATH}/../.. +# lib.sh use the ROOT variable +export ROOT="${SUBMARINE_HOME}/submarine-cloud/" +export KUBECONFIG="${HOME}/.kube/kind-config-${clusterName:-kind}" + +# shellcheck source=./../../submarine-cloud/hack/lib.sh +source "${SUBMARINE_HOME}/submarine-cloud/hack/lib.sh" + +########################################### +# Load local docker image into registry +# Globals: +# KIND_BIN +# Arguments: +# image +########################################### +function load_image_to_registry() { + if [[ ! $(docker inspect "$1" > /dev/null) ]] ; then + docker pull "$1" + fi + ${KIND_BIN} load docker-image "$1" +} + +function main() { + hack::ensure_kubectl + echo "Setting up ingress on a kind cluster." + load_image_to_registry "${TRAEFIK_IMAGE}" + ${KUBECTL_BIN} apply -k "${CURRENT_PATH}/traefik" +} + +main "@" diff --git a/dev-support/k8s/traefik/cluster-role-binding.yaml b/dev-support/k8s/traefik/cluster-role-binding.yaml new file mode 100644 index 0000000000..fe862602be --- /dev/null +++ b/dev-support/k8s/traefik/cluster-role-binding.yaml @@ -0,0 +1,29 @@ +# +# Licensed to the Apache Software Foundation (ASF) under one or more +# contributor license agreements. See the NOTICE file distributed with +# this work for additional information regarding copyright ownership. +# The ASF licenses this file to You under the Apache License, Version 2.0 +# (the "License"); you may not use this file except in compliance with +# the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# + +kind: ClusterRoleBinding +apiVersion: rbac.authorization.k8s.io/v1beta1 +metadata: + name: traefik-ingress-controller +roleRef: + apiGroup: rbac.authorization.k8s.io + kind: ClusterRole + name: traefik-ingress-controller +subjects: +- kind: ServiceAccount + name: traefik-ingress-controller + namespace: default diff --git a/dev-support/k8s/traefik/cluster-role.yaml b/dev-support/k8s/traefik/cluster-role.yaml new file mode 100644 index 0000000000..74537deaa0 --- /dev/null +++ b/dev-support/k8s/traefik/cluster-role.yaml @@ -0,0 +1,61 @@ +# +# Licensed to the Apache Software Foundation (ASF) under one or more +# contributor license agreements. See the NOTICE file distributed with +# this work for additional information regarding copyright ownership. +# The ASF licenses this file to You under the Apache License, Version 2.0 +# (the "License"); you may not use this file except in compliance with +# the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# + +kind: ClusterRole +apiVersion: rbac.authorization.k8s.io/v1beta1 +metadata: + name: traefik-ingress-controller + +rules: +- apiGroups: + - "" + resources: + - services + - endpoints + - secrets + verbs: + - get + - list + - watch +- apiGroups: + - extensions + resources: + - ingresses + verbs: + - get + - list + - watch +- apiGroups: + - extensions + resources: + - ingresses/status + verbs: + - update +- apiGroups: + - traefik.containo.us + resources: + - middlewares + - ingressroutes + - traefikservices + - ingressroutetcps + - ingressrouteudps + - tlsoptions + - tlsstores + verbs: + - get + - list + - watch diff --git a/dev-support/k8s/traefik/crd.yaml b/dev-support/k8s/traefik/crd.yaml new file mode 100644 index 0000000000..76776bdec1 --- /dev/null +++ b/dev-support/k8s/traefik/crd.yaml @@ -0,0 +1,120 @@ +# +# Licensed to the Apache Software Foundation (ASF) under one or more +# contributor license agreements. See the NOTICE file distributed with +# this work for additional information regarding copyright ownership. +# The ASF licenses this file to You under the Apache License, Version 2.0 +# (the "License"); you may not use this file except in compliance with +# the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# + +apiVersion: apiextensions.k8s.io/v1beta1 +kind: CustomResourceDefinition +metadata: + name: ingressroutes.traefik.containo.us + +spec: + group: traefik.containo.us + version: v1alpha1 + names: + kind: IngressRoute + plural: ingressroutes + singular: ingressroute + scope: Namespaced + +--- +apiVersion: apiextensions.k8s.io/v1beta1 +kind: CustomResourceDefinition +metadata: + name: middlewares.traefik.containo.us + +spec: + group: traefik.containo.us + version: v1alpha1 + names: + kind: Middleware + plural: middlewares + singular: middleware + scope: Namespaced + +--- +apiVersion: apiextensions.k8s.io/v1beta1 +kind: CustomResourceDefinition +metadata: + name: ingressroutetcps.traefik.containo.us + +spec: + group: traefik.containo.us + version: v1alpha1 + names: + kind: IngressRouteTCP + plural: ingressroutetcps + singular: ingressroutetcp + scope: Namespaced + +--- +apiVersion: apiextensions.k8s.io/v1beta1 +kind: CustomResourceDefinition +metadata: + name: ingressrouteudps.traefik.containo.us + +spec: + group: traefik.containo.us + version: v1alpha1 + names: + kind: IngressRouteUDP + plural: ingressrouteudps + singular: ingressrouteudp + scope: Namespaced + +--- +apiVersion: apiextensions.k8s.io/v1beta1 +kind: CustomResourceDefinition +metadata: + name: tlsoptions.traefik.containo.us + +spec: + group: traefik.containo.us + version: v1alpha1 + names: + kind: TLSOption + plural: tlsoptions + singular: tlsoption + scope: Namespaced + +--- +apiVersion: apiextensions.k8s.io/v1beta1 +kind: CustomResourceDefinition +metadata: + name: tlsstores.traefik.containo.us + +spec: + group: traefik.containo.us + version: v1alpha1 + names: + kind: TLSStore + plural: tlsstores + singular: tlsstore + scope: Namespaced + +--- +apiVersion: apiextensions.k8s.io/v1beta1 +kind: CustomResourceDefinition +metadata: + name: traefikservices.traefik.containo.us + +spec: + group: traefik.containo.us + version: v1alpha1 + names: + kind: TraefikService + plural: traefikservices + singular: traefikservice + scope: Namespaced diff --git a/dev-support/k8s/traefik/deployment.yaml b/dev-support/k8s/traefik/deployment.yaml new file mode 100644 index 0000000000..be670cba1b --- /dev/null +++ b/dev-support/k8s/traefik/deployment.yaml @@ -0,0 +1,50 @@ +# +# Licensed to the Apache Software Foundation (ASF) under one or more +# contributor license agreements. See the NOTICE file distributed with +# this work for additional information regarding copyright ownership. +# The ASF licenses this file to You under the Apache License, Version 2.0 +# (the "License"); you may not use this file except in compliance with +# the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# + +kind: Deployment +apiVersion: apps/v1 +metadata: + namespace: default + name: traefik + labels: + app: traefik +spec: + replicas: 1 + selector: + matchLabels: + app: traefik + template: + metadata: + labels: + app: traefik + spec: + serviceAccountName: traefik-ingress-controller + containers: + - name: traefik + image: traefik:v2.2 + args: + - --accesslog + - --entrypoints.web.Address=:80 + - --entrypoints.websecure.Address=:443 + - --ping=true + - --providers.kubernetescrd + - --providers.kubernetesingress + ports: + - name: web + containerPort: 80 + - name: websecure + containerPort: 443 diff --git a/dev-support/k8s/traefik/kustomization.yaml b/dev-support/k8s/traefik/kustomization.yaml new file mode 100644 index 0000000000..e7f05d060e --- /dev/null +++ b/dev-support/k8s/traefik/kustomization.yaml @@ -0,0 +1,34 @@ +# +# Licensed to the Apache Software Foundation (ASF) under one or more +# contributor license agreements. See the NOTICE file distributed with +# this work for additional information regarding copyright ownership. +# The ASF licenses this file to You under the Apache License, Version 2.0 +# (the "License"); you may not use this file except in compliance with +# the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# + +apiVersion: kustomize.config.k8s.io/v1beta1 +kind: Kustomization +resources: +- cluster-role.yaml +- cluster-role-binding.yaml +- crd.yaml +- deployment.yaml +- service-account.yaml +- service.yaml +namespace: default +commonLabels: + kustomize.component: traefik + +images: +- name: traefik + newName: traefik + newTag: v2.2 diff --git a/dev-support/k8s/traefik/service-account.yaml b/dev-support/k8s/traefik/service-account.yaml new file mode 100644 index 0000000000..fee03de623 --- /dev/null +++ b/dev-support/k8s/traefik/service-account.yaml @@ -0,0 +1,21 @@ +# +# Licensed to the Apache Software Foundation (ASF) under one or more +# contributor license agreements. See the NOTICE file distributed with +# this work for additional information regarding copyright ownership. +# The ASF licenses this file to You under the Apache License, Version 2.0 +# (the "License"); you may not use this file except in compliance with +# the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# + +apiVersion: v1 +kind: ServiceAccount +metadata: + name: traefik-ingress-controller diff --git a/submarine-cloud/hack/ingress/service-nodeport.yaml b/dev-support/k8s/traefik/service.yaml similarity index 71% rename from submarine-cloud/hack/ingress/service-nodeport.yaml rename to dev-support/k8s/traefik/service.yaml index 84331e9b47..d00005ae43 100644 --- a/submarine-cloud/hack/ingress/service-nodeport.yaml +++ b/dev-support/k8s/traefik/service.yaml @@ -14,25 +14,21 @@ # See the License for the specific language governing permissions and # limitations under the License. # + apiVersion: v1 kind: Service metadata: - name: ingress-nginx - namespace: ingress-nginx - labels: - app.kubernetes.io/name: ingress-nginx - app.kubernetes.io/part-of: ingress-nginx + name: traefik spec: type: NodePort ports: - - name: http + - protocol: TCP + name: web port: 80 - targetPort: 80 - protocol: TCP - - name: https + nodePort: 32080 + - protocol: TCP + name: websecure port: 443 - targetPort: 443 - protocol: TCP + nodePort: 32443 selector: - app.kubernetes.io/name: ingress-nginx - app.kubernetes.io/part-of: ingress-nginx + app: traefik diff --git a/submarine-cloud/hack/ingress/mandatory.yaml b/submarine-cloud/hack/ingress/mandatory.yaml deleted file mode 100644 index 699d13a93a..0000000000 --- a/submarine-cloud/hack/ingress/mandatory.yaml +++ /dev/null @@ -1,307 +0,0 @@ -# -# Licensed to the Apache Software Foundation (ASF) under one or more -# contributor license agreements. See the NOTICE file distributed with -# this work for additional information regarding copyright ownership. -# The ASF licenses this file to You under the Apache License, Version 2.0 -# (the "License"); you may not use this file except in compliance with -# the License. You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. -# -apiVersion: v1 -kind: Namespace -metadata: - name: ingress-nginx - labels: - app.kubernetes.io/name: ingress-nginx - app.kubernetes.io/part-of: ingress-nginx - ---- -kind: ConfigMap -apiVersion: v1 -metadata: - name: nginx-configuration - namespace: ingress-nginx - labels: - app.kubernetes.io/name: ingress-nginx - app.kubernetes.io/part-of: ingress-nginx - ---- -kind: ConfigMap -apiVersion: v1 -metadata: - name: tcp-services - namespace: ingress-nginx - labels: - app.kubernetes.io/name: ingress-nginx - app.kubernetes.io/part-of: ingress-nginx - ---- -kind: ConfigMap -apiVersion: v1 -metadata: - name: udp-services - namespace: ingress-nginx - labels: - app.kubernetes.io/name: ingress-nginx - app.kubernetes.io/part-of: ingress-nginx - ---- -apiVersion: v1 -kind: ServiceAccount -metadata: - name: nginx-ingress-serviceaccount - namespace: ingress-nginx - labels: - app.kubernetes.io/name: ingress-nginx - app.kubernetes.io/part-of: ingress-nginx - ---- -apiVersion: rbac.authorization.k8s.io/v1beta1 -kind: ClusterRole -metadata: - name: nginx-ingress-clusterrole - labels: - app.kubernetes.io/name: ingress-nginx - app.kubernetes.io/part-of: ingress-nginx -rules: - - apiGroups: - - "" - resources: - - configmaps - - endpoints - - nodes - - pods - - secrets - verbs: - - list - - watch - - apiGroups: - - "" - resources: - - nodes - verbs: - - get - - apiGroups: - - "" - resources: - - services - verbs: - - get - - list - - watch - - apiGroups: - - "" - resources: - - events - verbs: - - create - - patch - - apiGroups: - - "extensions" - - "networking.k8s.io" - resources: - - ingresses - verbs: - - get - - list - - watch - - apiGroups: - - "extensions" - - "networking.k8s.io" - resources: - - ingresses/status - verbs: - - update - ---- -apiVersion: rbac.authorization.k8s.io/v1beta1 -kind: Role -metadata: - name: nginx-ingress-role - namespace: ingress-nginx - labels: - app.kubernetes.io/name: ingress-nginx - app.kubernetes.io/part-of: ingress-nginx -rules: - - apiGroups: - - "" - resources: - - configmaps - - pods - - secrets - - namespaces - verbs: - - get - - apiGroups: - - "" - resources: - - configmaps - resourceNames: - # Defaults to "-" - # Here: "-" - # This has to be adapted if you change either parameter - # when launching the nginx-ingress-controller. - - "ingress-controller-leader-nginx" - verbs: - - get - - update - - apiGroups: - - "" - resources: - - configmaps - verbs: - - create - - apiGroups: - - "" - resources: - - endpoints - verbs: - - get - ---- -apiVersion: rbac.authorization.k8s.io/v1beta1 -kind: RoleBinding -metadata: - name: nginx-ingress-role-nisa-binding - namespace: ingress-nginx - labels: - app.kubernetes.io/name: ingress-nginx - app.kubernetes.io/part-of: ingress-nginx -roleRef: - apiGroup: rbac.authorization.k8s.io - kind: Role - name: nginx-ingress-role -subjects: - - kind: ServiceAccount - name: nginx-ingress-serviceaccount - namespace: ingress-nginx - ---- -apiVersion: rbac.authorization.k8s.io/v1beta1 -kind: ClusterRoleBinding -metadata: - name: nginx-ingress-clusterrole-nisa-binding - labels: - app.kubernetes.io/name: ingress-nginx - app.kubernetes.io/part-of: ingress-nginx -roleRef: - apiGroup: rbac.authorization.k8s.io - kind: ClusterRole - name: nginx-ingress-clusterrole -subjects: - - kind: ServiceAccount - name: nginx-ingress-serviceaccount - namespace: ingress-nginx - ---- -apiVersion: apps/v1 -kind: Deployment -metadata: - name: nginx-ingress-controller - namespace: ingress-nginx - labels: - app.kubernetes.io/name: ingress-nginx - app.kubernetes.io/part-of: ingress-nginx -spec: - replicas: 1 - selector: - matchLabels: - app.kubernetes.io/name: ingress-nginx - app.kubernetes.io/part-of: ingress-nginx - template: - metadata: - labels: - app.kubernetes.io/name: ingress-nginx - app.kubernetes.io/part-of: ingress-nginx - annotations: - prometheus.io/port: "10254" - prometheus.io/scrape: "true" - spec: - # wait up to five minutes for the drain of connections - terminationGracePeriodSeconds: 300 - serviceAccountName: nginx-ingress-serviceaccount - nodeSelector: - beta.kubernetes.io/os: linux - containers: - - name: nginx-ingress-controller - image: quay.io/kubernetes-ingress-controller/nginx-ingress-controller:master - args: - - /nginx-ingress-controller - - --configmap=$(POD_NAMESPACE)/nginx-configuration - - --tcp-services-configmap=$(POD_NAMESPACE)/tcp-services - - --udp-services-configmap=$(POD_NAMESPACE)/udp-services - - --publish-service=$(POD_NAMESPACE)/ingress-nginx - - --annotations-prefix=nginx.ingress.kubernetes.io - securityContext: - allowPrivilegeEscalation: true - capabilities: - drop: - - ALL - add: - - NET_BIND_SERVICE - # www-data -> 101 - runAsUser: 101 - env: - - name: POD_NAME - valueFrom: - fieldRef: - fieldPath: metadata.name - - name: POD_NAMESPACE - valueFrom: - fieldRef: - fieldPath: metadata.namespace - ports: - - name: http - containerPort: 80 - protocol: TCP - - name: https - containerPort: 443 - protocol: TCP - livenessProbe: - failureThreshold: 3 - httpGet: - path: /healthz - port: 10254 - scheme: HTTP - initialDelaySeconds: 10 - periodSeconds: 10 - successThreshold: 1 - timeoutSeconds: 10 - readinessProbe: - failureThreshold: 3 - httpGet: - path: /healthz - port: 10254 - scheme: HTTP - periodSeconds: 10 - successThreshold: 1 - timeoutSeconds: 10 - lifecycle: - preStop: - exec: - command: - - /wait-shutdown - ---- -apiVersion: v1 -kind: LimitRange -metadata: - name: ingress-nginx - namespace: ingress-nginx - labels: - app.kubernetes.io/name: ingress-nginx - app.kubernetes.io/part-of: ingress-nginx -spec: - limits: - - default: - min: - memory: 90Mi - cpu: 100m - type: Container diff --git a/submarine-cloud/hack/integration-test.sh b/submarine-cloud/hack/integration-test.sh index b0aff23d5f..7a9dc810d5 100755 --- a/submarine-cloud/hack/integration-test.sh +++ b/submarine-cloud/hack/integration-test.sh @@ -29,6 +29,7 @@ export KUBECONFIG=~/.kube/kind-config-${clusterName:-kind} function start() { $ROOT/hack/kind-cluster-build.sh + $SUBMARINE_HOME/dev-support/k8s/deploy-traefik.sh $SUBMARINE_HOME/dev-support/k8s/deploy-kubeflow-operators.sh -a $SUBMARINE_HOME/dev-support/k8s/deploy-notebook-controller.sh $ROOT/hack/deploy-submarine.sh --test diff --git a/submarine-cloud/hack/kind-cluster-build.sh b/submarine-cloud/hack/kind-cluster-build.sh index 4129387b50..ab371e4bc0 100755 --- a/submarine-cloud/hack/kind-cluster-build.sh +++ b/submarine-cloud/hack/kind-cluster-build.sh @@ -118,10 +118,10 @@ nodes: hostPort: 5000 listenAddress: 127.0.0.1 protocol: TCP - - containerPort: 80 + - containerPort: 32080 hostPort: 80 protocol: TCP - - containerPort: 443 + - containerPort: 32443 hostPort: 443 protocol: TCP EOF @@ -230,19 +230,6 @@ if ! docker inspect registry:2 >/dev/null ; then fi $KIND_BIN load docker-image registry:2 -# https://kind.sigs.k8s.io/docs/user/ingress/#ingress-nginx -echo "setting up ingress on a kind cluster." - -# load ingress dependence docker-image into kind -if ! docker inspect quay.io/kubernetes-ingress-controller/nginx-ingress-controller:master >/dev/null ; then - docker pull quay.io/kubernetes-ingress-controller/nginx-ingress-controller:master -fi -$KIND_BIN load docker-image quay.io/kubernetes-ingress-controller/nginx-ingress-controller:master - -$KUBECTL_BIN apply -f $ROOT/hack/ingress/mandatory.yaml -$KUBECTL_BIN apply -f $ROOT/hack/ingress/service-nodeport.yaml -$KUBECTL_BIN patch deployments -n ingress-nginx nginx-ingress-controller -p '{"spec":{"template":{"spec":{"containers":[{"name":"nginx-ingress-controller","ports":[{"containerPort":80,"hostPort":80},{"containerPort":443,"hostPort":443}]}],"nodeSelector":{"ingress-ready":"true"},"tolerations":[{"key":"node-role.kubernetes.io/master","operator":"Equal","effect":"NoSchedule"}]}}}}' - $KUBECTL_BIN get pod -A echo "############# success create cluster:[${clusterName}] #############"