Merge pull request #16 from TykTechnologies/feat/TT-9697/release-name…

…-discovery

[TT-9697] - Add label-based discovery

README.md

@@ -9,7 +9,7 @@ via tyk-helm-charts.
 
 ## What it does?
 
-### 1. Tyk post delpoyment bootstrapping
+### 1. Tyk post deployment bootstrapping
 a. Creates a basic organization with the values specified in the env vars
 via the tyk-helm charts
 <br>

constants/constants.go

@@ -5,7 +5,6 @@ const (
 	EnterprisePortalSecretEnabledEnvVar = "ENTERPRISE_PORTAL_SECRET_ENABLED"
 	BootstrapPortalEnvVar               = "BOOTSTRAP_PORTAL"
 	TykDashboardDeployEnvVar            = "TYK_DASHBOARD_DEPLOY"
-	GatewayAddressEnvVar                = "GATEWAY_ADDRESS"
 	OperatorSecretNameEnvVar            = "OPERATOR_SECRET_NAME"
 	EnterprisePortalSecretNameEnvVar    = "ENTERPRISE_PORTAL_SECRET_NAME"
 	TykAdminFirstNameEnvVar             = "TYK_ADMIN_FIRST_NAME"
@@ -14,13 +13,18 @@ const (
 	TykAdminPasswordEnvVar              = "TYK_ADMIN_PASSWORD"
 	TykPodNamespaceEnvVar               = "TYK_POD_NAMESPACE"
 	TykDashboardProtoEnvVar             = "TYK_DASHBOARD_PROTO"
-	TykDashboardSvcEnvVar               = "TYK_DASHBOARD_SVC"
 	TykDashboardInsecureSkipVerify      = "TYK_DASHBOARD_INSECURE_SKIP_VERIFY"
 	TykDashboardLicenseEnvVarName       = "TYK_DB_LICENSEKEY"
-	TykDbListenport                     = "TYK_DB_LISTENPORT"
 	TykDbLicensekeyEnvVar               = "TYK_DB_LICENSEKEY"
 	TykAdminSecretEnvVar                = "TYK_ADMIN_SECRET"
 	DashboardEnabledEnvVar              = "DASHBOARD_ENABLED"
 	TykOrgNameEnvVar                    = "TYK_ORG_NAME"
 	TykOrgCnameEnvVar                   = "TYK_ORG_CNAME"
+	ReleaseNameEnvVar                   = "RELEASE_NAME"
+
+	TykBootstrapLabel                = "tyk.tyk.io/k8s-bootstrap"
+	TykBootstrapPreDeleteLabel       = "tyk-k8s-bootstrap-pre-delete"
+	TykBootstrapDashboardDeployLabel = "tyk-dashboard"
+	TykBootstrapDashboardSvcLabel    = "tyk-dashboard"
+	TykBootstrapReleaseLabel         = "release"
 )

data/app.go

@@ -1,15 +1,20 @@
 package data
 
 import (
+	"context"
 	"fmt"
+	metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
+	"k8s.io/apimachinery/pkg/labels"
+	"k8s.io/client-go/kubernetes"
+	"k8s.io/client-go/rest"
 	"os"
 	"strconv"
 	"tyk/tyk/bootstrap/constants"
 )
 
 type AppArguments struct {
 	DashboardHost                 string
-	DashboardPort                 int
+	DashboardPort                 int32
 	DashBoardLicense              string
 	TykAdminSecret                string
 	CurrentOrgName                string
@@ -31,38 +36,20 @@ type AppArguments struct {
 	OperatorSecretName            string
 	EnterprisePortalSecretEnabled bool
 	EnterprisePortalSecretName    string
-	GatewayAdress                 string
 	BootstrapPortal               bool
 	DashboardDeploymentName       string
+	ReleaseName                   string
 }
 
 var AppConfig = AppArguments{
-	IsDashboardEnabled:            false,
-	OperatorSecretEnabled:         false,
-	EnterprisePortalSecretEnabled: false,
-	BootstrapPortal:               false,
-	DashboardProto:                "",
-	DashboardHost:                 "",
-	DashboardPort:                 3000,
-	DashBoardLicense:              "",
-	TykAdminSecret:                "12345",
-	CurrentOrgName:                "TYKTYK",
-	Cname:                         "tykCName",
-	TykAdminPassword:              "123456",
-	TykAdminFirstName:             "firstName",
-	TykAdminEmailAddress:          "tyk@tyk.io",
-	TykAdminLastName:              "lastName",
-	UserAuth:                      "",
-	OrgId:                         "",
-	CatalogId:                     "",
-	DashboardUrl:                  "",
-	TykPodNamespace:               "",
-	DashboardSvc:                  "",
-	DashboardInsecureSkipVerify:   false,
-	OperatorSecretName:            "",
-	EnterprisePortalSecretName:    "",
-	GatewayAdress:                 "",
-	DashboardDeploymentName:       "",
+	DashboardPort:        3000,
+	TykAdminSecret:       "12345",
+	CurrentOrgName:       "TYKTYK",
+	Cname:                "tykCName",
+	TykAdminPassword:     "123456",
+	TykAdminFirstName:    "firstName",
+	TykAdminEmailAddress: "tyk@tyk.io",
+	TykAdminLastName:     "lastName",
 }
 
 func InitAppDataPreDelete() error {
@@ -79,17 +66,15 @@ func InitAppDataPostInstall() error {
 	AppConfig.TykAdminPassword = os.Getenv(constants.TykAdminPasswordEnvVar)
 	AppConfig.TykPodNamespace = os.Getenv(constants.TykPodNamespaceEnvVar)
 	AppConfig.DashboardProto = os.Getenv(constants.TykDashboardProtoEnvVar)
-	AppConfig.DashboardSvc = os.Getenv(constants.TykDashboardSvcEnvVar)
-	dbPort, err := strconv.ParseInt(os.Getenv(constants.TykDbListenport), 10, 64)
-	if err != nil {
-		return err
-	}
-	AppConfig.DashboardPort = int(dbPort)
+
 	AppConfig.DashBoardLicense = os.Getenv(constants.TykDbLicensekeyEnvVar)
 	AppConfig.TykAdminSecret = os.Getenv(constants.TykAdminSecretEnvVar)
 	AppConfig.CurrentOrgName = os.Getenv(constants.TykOrgNameEnvVar)
 	AppConfig.Cname = os.Getenv(constants.TykOrgCnameEnvVar)
-	AppConfig.DashboardUrl = GetDashboardUrl()
+	AppConfig.ReleaseName = os.Getenv(constants.ReleaseNameEnvVar)
+
+	var err error
+
 	dashEnabledRaw := os.Getenv(constants.DashboardEnabledEnvVar)
 	if dashEnabledRaw != "" {
 		AppConfig.IsDashboardEnabled, err = strconv.ParseBool(os.Getenv(constants.DashboardEnabledEnvVar))
@@ -98,6 +83,18 @@ func InitAppDataPostInstall() error {
 		}
 	}
 
+	if AppConfig.IsDashboardEnabled {
+		if err := discoverDashboardSvc(); err != nil {
+			return err
+		}
+		AppConfig.DashboardUrl = fmt.Sprintf("%s://%s.%s.svc.cluster.local:%d",
+			AppConfig.DashboardProto,
+			AppConfig.DashboardSvc,
+			AppConfig.TykPodNamespace,
+			AppConfig.DashboardPort,
+		)
+	}
+
 	operatorSecretEnabledRaw := os.Getenv(constants.OperatorSecretEnabledEnvVar)
 	if operatorSecretEnabledRaw != "" {
 		AppConfig.OperatorSecretEnabled, err = strconv.ParseBool(operatorSecretEnabledRaw)
@@ -116,7 +113,6 @@ func InitAppDataPostInstall() error {
 	}
 	AppConfig.EnterprisePortalSecretName = os.Getenv(constants.EnterprisePortalSecretNameEnvVar)
 
-	AppConfig.GatewayAdress = os.Getenv(constants.GatewayAddressEnvVar)
 	bootstrapPortalBoolRaw := os.Getenv(constants.BootstrapPortalEnvVar)
 	if bootstrapPortalBoolRaw != "" {
 		AppConfig.BootstrapPortal, err = strconv.ParseBool(bootstrapPortalBoolRaw)
@@ -137,10 +133,55 @@ func InitAppDataPostInstall() error {
 	return nil
 }
 
-func GetDashboardUrl() string {
-	return fmt.Sprintf("%s://%s.%s.svc.cluster.local:%d",
-		AppConfig.DashboardProto,
-		AppConfig.DashboardSvc,
-		AppConfig.TykPodNamespace,
-		AppConfig.DashboardPort)
+// discoverDashboardSvc lists Service objects with constants.TykBootstrapReleaseLabel label that has
+// constants.TykBootstrapDashboardSvcLabel value and gets this Service's metadata name, and port and
+// updates DashboardSvc and DashboardPort fields.
+func discoverDashboardSvc() error {
+	config, err := rest.InClusterConfig()
+	if err != nil {
+		return err
+	}
+
+	c, err := kubernetes.NewForConfig(config)
+	if err != nil {
+		return err
+	}
+
+	ls := metav1.LabelSelector{MatchLabels: map[string]string{
+		constants.TykBootstrapLabel: constants.TykBootstrapDashboardSvcLabel,
+	}}
+	if AppConfig.ReleaseName != "" {
+		ls.MatchLabels[constants.TykBootstrapReleaseLabel] = AppConfig.ReleaseName
+	}
+
+	l := labels.Set(ls.MatchLabels).String()
+
+	services, err := c.
+		CoreV1().
+		Services(AppConfig.TykPodNamespace).
+		List(context.TODO(), metav1.ListOptions{LabelSelector: l})
+	if err != nil {
+		return err
+	}
+
+	if len(services.Items) == 0 {
+		return fmt.Errorf("failed to find services with label %v\n", l)
+	}
+
+	if len(services.Items) > 1 {
+		fmt.Printf("[WARNING] Found multiple services with label %v\n", l)
+	}
+
+	service := services.Items[0]
+	if len(service.Spec.Ports) == 0 {
+		return fmt.Errorf("svc/%v/%v has no open ports\n", service.Name, service.Namespace)
+	}
+	if len(service.Spec.Ports) > 1 {
+		fmt.Printf("[WARNING] Found multiple open ports in svc/%v/%v\n", service.Name, service.Namespace)
+	}
+
+	AppConfig.DashboardPort = service.Spec.Ports[0].Port
+	AppConfig.DashboardSvc = service.Name
+
+	return nil
 }

hack/load_images.sh

@@ -0,0 +1,12 @@
+#!/bin/sh
+
+make build-all
+
+docker build -t tykio/tyk-k8s-bootstrap-pre-install:testing -f ./.container/image/bootstrap-pre-install/Dockerfile ./bin &&
+  kind load docker-image tykio/tyk-k8s-bootstrap-pre-install:testing
+
+docker build -t tykio/tyk-k8s-bootstrap-post:testing -f ./.container/image/bootstrap-post/Dockerfile ./bin &&
+  kind load docker-image tykio/tyk-k8s-bootstrap-post:testing
+
+docker build -t tykio/tyk-k8s-bootstrap-pre-delete:testing -f ./.container/image/bootstrap-pre-delete/Dockerfile ./bin &&
+  kind load docker-image tykio/tyk-k8s-bootstrap-pre-delete:testing

helpers/portal.go

@@ -6,15 +6,17 @@ import (
 	"errors"
 	"fmt"
 	"io"
-	"net/http"
-	"time"
-	"tyk/tyk/bootstrap/data"
 
 	metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
+	"k8s.io/apimachinery/pkg/labels"
 	"k8s.io/apimachinery/pkg/types"
 	"k8s.io/apimachinery/pkg/util/json"
 	"k8s.io/client-go/kubernetes"
 	"k8s.io/client-go/rest"
+	"net/http"
+	"time"
+	"tyk/tyk/bootstrap/constants"
+	"tyk/tyk/bootstrap/data"
 )
 
 func BoostrapPortal(client http.Client) error {
@@ -239,12 +241,45 @@ func RestartDashboard() error {
 		return err
 	}
 
-	deploymentsClient := clientset.AppsV1().Deployments(data.AppConfig.TykPodNamespace)
+	if data.AppConfig.DashboardDeploymentName == "" {
+		ls := metav1.LabelSelector{MatchLabels: map[string]string{
+			constants.TykBootstrapLabel: constants.TykBootstrapDashboardDeployLabel,
+		}}
+
+		if data.AppConfig.ReleaseName != "" {
+			ls.MatchLabels[constants.TykBootstrapReleaseLabel] = data.AppConfig.ReleaseName
+		}
+
+		deployments, err := clientset.
+			AppsV1().
+			Deployments(data.AppConfig.TykPodNamespace).
+			List(
+				context.TODO(),
+				metav1.ListOptions{
+					LabelSelector: labels.Set(ls.MatchLabels).String(),
+				},
+			)
+		if err != nil {
+			return errors.New(fmt.Sprintf("failed to list Tyk Dashboard Deployment, err: %v", err))
+		}
+
+		for _, deployment := range deployments.Items {
+			data.AppConfig.DashboardDeploymentName = deployment.ObjectMeta.Name
+		}
+	}
+
 	timeStamp := fmt.Sprintf(`{"spec": {"template": {"metadata": {"annotations": {"kubectl.kubernetes.io/restartedAt": "%s"}}}}}`,
 		time.Now().Format("20060102150405"))
 
-	_, err = deploymentsClient.Patch(context.TODO(), data.AppConfig.DashboardDeploymentName,
-		types.StrategicMergePatchType, []byte(timeStamp), metav1.PatchOptions{})
-
+	_, err = clientset.
+		AppsV1().
+		Deployments(data.AppConfig.TykPodNamespace).
+		Patch(
+			context.TODO(),
+			data.AppConfig.DashboardDeploymentName,
+			types.StrategicMergePatchType,
+			[]byte(timeStamp),
+			metav1.PatchOptions{},
+		)
 	return err
 }

predelete/predelete.go

@@ -4,6 +4,7 @@ import (
 	"context"
 	"fmt"
 	"os"
+	"tyk/tyk/bootstrap/constants"
 	"tyk/tyk/bootstrap/data"
 
 	metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
@@ -100,31 +101,38 @@ func PreDeleteEnterprisePortalSecret(clientset *kubernetes.Clientset) error {
 	return nil
 }
 
+// PreDeleteBootstrappingJobs deletes all jobs within the release namespace, that has specific label.
 func PreDeleteBootstrappingJobs(clientset *kubernetes.Clientset) error {
-	jobs, err := clientset.BatchV1().Jobs(data.AppConfig.TykPodNamespace).
-		List(context.TODO(), metav1.ListOptions{})
+	// Usually, the raw strings in label selectors are not recommended.
+	jobs, err := clientset.
+		BatchV1().
+		Jobs(data.AppConfig.TykPodNamespace).
+		List(
+			context.TODO(),
+			metav1.ListOptions{
+				LabelSelector: fmt.Sprintf("%s", constants.TykBootstrapLabel),
+			},
+		)
 	if err != nil {
 		return err
 	}
 
-	found := false
-	for _, value := range jobs.Items {
-		if value.Name == os.Getenv("BOOTSTRAP_JOB_NAME") {
+	var errCascading error
+	for _, job := range jobs.Items {
+		// Do not need to delete pre-delete job. It will be deleted by Helm.
+		jobLabel := job.ObjectMeta.Labels[constants.TykBootstrapLabel]
+		if jobLabel != constants.TykBootstrapPreDeleteLabel {
 			deletePropagationType := metav1.DeletePropagationBackground
-			err = clientset.BatchV1().Jobs(data.AppConfig.TykPodNamespace).
-				Delete(context.TODO(), value.Name, metav1.DeleteOptions{PropagationPolicy: &deletePropagationType})
-			if err != nil {
-				return err
+
+			err2 := clientset.
+				BatchV1().
+				Jobs(data.AppConfig.TykPodNamespace).
+				Delete(context.TODO(), job.Name, metav1.DeleteOptions{PropagationPolicy: &deletePropagationType})
+			if err2 != nil {
+				errCascading = err2
 			}
-			found = true
-			break
 		}
 	}
 
-	if !found {
-		fmt.Println("A previously created bootstrapping job has not been identified")
-	} else {
-		fmt.Println("A previously created bootstrapping job was identified and deleted")
-	}
-	return nil
+	return errCascading
 }

readiness/readiness.go

@@ -57,17 +57,16 @@ func CheckIfRequiredDeploymentsAreReady() error {
 				}
 			}
 		}
-		fmt.Printf("The following pods have containers that are not ready: ")
+
+		if len(notReadyPods) == 0 {
+			return nil
+		}
+
+		fmt.Printf("The following pods have containers that are NOT ready: ")
 		for pod, _ := range notReadyPods {
 			fmt.Println(pod)
 		}
 
-		if len(notReadyPods) != 0 {
-			fmt.Println("TYK PRO IS NOT READY")
-		} else {
-			fmt.Println("TYK PRO IS READY TO BE BOOTSTRAPPED")
-			return nil
-		}
 		time.Sleep(2 * time.Second)
 	}
 }