Add insecure-skip-tls-verify option in cluster's secret

cmd/create-cluster-secret/main.go

@@ -1,19 +1,20 @@
 package main
 
 import (
-	"encoding/base64"
 	"flag"
-	shipperv1 "github.com/bookingcom/shipper/pkg/apis/shipper/v1"
-	"github.com/bookingcom/shipper/pkg/client/clientset/versioned"
+	"os/user"
+	"path"
+	"strconv"
+
 	"github.com/golang/glog"
 	corev1 "k8s.io/api/core/v1"
 	"k8s.io/apimachinery/pkg/api/errors"
 	metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
 	"k8s.io/client-go/kubernetes"
 	"k8s.io/client-go/tools/clientcmd"
-	"os/user"
-	"path"
-	"strconv"
+
+	shipperv1 "github.com/bookingcom/shipper/pkg/apis/shipper/v1"
+	"github.com/bookingcom/shipper/pkg/client/clientset/versioned"
 )
 
 var (
@@ -51,7 +52,6 @@ func main() {
 	kubeClient := kubernetes.NewForConfigOrDie(restCfg)
 
 	secretData := make(map[string][]byte)
-	secretData["tls.insecure-skip-tls-verify"] = []byte(base64.StdEncoding.EncodeToString([]byte(strconv.FormatBool(restCfg.Insecure))))
 	secretData["tls.ca"] = restCfg.CAData
 	secretData["tls.crt"] = restCfg.CertData
 	secretData["tls.key"] = restCfg.KeyData
@@ -64,12 +64,20 @@ func main() {
 				ObjectMeta: metav1.ObjectMeta{
 					Name: *clusterName,
 					Annotations: map[string]string{
-						shipperv1.SecretChecksumAnnotation: "some-checksum",
+						shipperv1.SecretChecksumAnnotation:             "some-checksum",
+						shipperv1.SecretClusterSkipTlsVerifyAnnotation: strconv.FormatBool(restCfg.Insecure),
 					},
 				},
 				Type: corev1.SecretTypeOpaque,
 				Data: secretData,
 			}
+
+			// Only add shipperv1.SecretClusterSkipTlsVerifyAnnotation if the
+			// configuration specifies an insecure connection.
+			if restCfg.Insecure == true {
+				clusterSecret.Annotations[shipperv1.SecretClusterSkipTlsVerifyAnnotation] = strconv.FormatBool(restCfg.Insecure)
+			}
+
 			if _, err := kubeClient.CoreV1().Secrets(*shipperNamespace).Create(clusterSecret); err != nil {
 				glog.Fatal(err)
 			}
@@ -79,6 +87,19 @@ func main() {
 		}
 	} else if *replaceSecret {
 		existingSecret.Data = secretData
+		if existingSecret.Annotations == nil {
+			existingSecret.Annotations = map[string]string{}
+		}
+
+		// Delete the shipperv1.SecretClusterSkipTlsVerifyAnnotation if
+		// configuration specifies a secure connection, add the annotation
+		// it otherwise.
+		if restCfg.Insecure == false {
+			delete(existingSecret.Annotations, shipperv1.SecretClusterSkipTlsVerifyAnnotation)
+		} else {
+			existingSecret.Annotations[shipperv1.SecretClusterSkipTlsVerifyAnnotation] = strconv.FormatBool(restCfg.Insecure)
+		}
+
 		if _, err := nsSecrets.Update(existingSecret); err != nil {
 			glog.Fatal(err)
 		}

pkg/apis/shipper/v1/types.go

@@ -36,8 +36,9 @@ const (
 	ReleaseTemplateIterationAnnotation = "shipper.booking.com/release.template.iteration"
 	ReleaseClustersAnnotation          = "shipper.booking.com/release.clusters"
 
-	SecretChecksumAnnotation    = "shipper.booking.com/cluster-secret.checksum"
-	SecretClusterNameAnnotation = "shipper.booking.com/cluster-secret.clusterName"
+	SecretChecksumAnnotation             = "shipper.booking.com/cluster-secret.checksum"
+	SecretClusterNameAnnotation          = "shipper.booking.com/cluster-secret.clusterName"
+	SecretClusterSkipTlsVerifyAnnotation = "shipper.booking.com/cluster-secret.insecure-tls-skip-verify"
 
 	LBLabel         = "shipper-lb"
 	LBForProduction = "production"

pkg/clusterclientstore/store.go

@@ -2,6 +2,7 @@ package clusterclientstore
 
 import (
 	"fmt"
+	"strconv"
 	"time"
 
 	"github.com/golang/glog"
@@ -11,7 +12,7 @@ import (
 	"k8s.io/apimachinery/pkg/util/wait"
 	kubeinformers "k8s.io/client-go/informers"
 	corev1informer "k8s.io/client-go/informers/core/v1"
-	kubernetes "k8s.io/client-go/kubernetes"
+	"k8s.io/client-go/kubernetes"
 	"k8s.io/client-go/rest"
 	kubecache "k8s.io/client-go/tools/cache"
 	"k8s.io/client-go/util/workqueue"
@@ -253,7 +254,11 @@ func (s *Store) create(cluster *shipperv1.Cluster, secret *corev1.Secret) error
 		panic(fmt.Sprintf("Secret %q doesn't have a checksum annotation. this should be checked before calling 'create'", secret.Name))
 	}
 
-	config := buildConfig(cluster.Spec.APIMaster, secret, s.restTimeout)
+	config, err := buildConfig(cluster.Spec.APIMaster, secret, s.restTimeout)
+	if err != nil {
+		return fmt.Errorf("create client configuration for Cluster %q: %s", cluster.Name, err)
+	}
+
 	client, err := s.buildClient(cluster.Name, config)
 	if err != nil {
 		return fmt.Errorf("create client for Cluster %q: %s", cluster.Name, err)
@@ -262,7 +267,11 @@ func (s *Store) create(cluster *shipperv1.Cluster, secret *corev1.Secret) error
 	// These are only used in shared informers. Setting HTTP timeout here would
 	// affect watches which is undesirable. Instead, we leave it to client-go (see
 	// k8s.io/client-go/tools/cache) to govern watch durations.
-	informerConfig := buildConfig(cluster.Spec.APIMaster, secret, nil)
+	informerConfig, err := buildConfig(cluster.Spec.APIMaster, secret, nil)
+	if err != nil {
+		return fmt.Errorf("build informer client configuration for Cluster %q: %s", cluster.Name, err)
+	}
+
 	informerClient, err := s.buildClient(cluster.Name, informerConfig)
 	if err != nil {
 		return fmt.Errorf("create informer client for Cluster %q: %s", cluster.Name, err)
@@ -292,7 +301,7 @@ func (s *Store) create(cluster *shipperv1.Cluster, secret *corev1.Secret) error
 
 // TODO(btyler): error here or let any invalid data get picked up by errors from
 // kube.NewForConfig or auth problems at connection time?
-func buildConfig(host string, secret *corev1.Secret, restTimeout *time.Duration) *rest.Config {
+func buildConfig(host string, secret *corev1.Secret, restTimeout *time.Duration) (*rest.Config, error) {
 	config := &rest.Config{
 		Host: host,
 	}
@@ -310,7 +319,7 @@ func buildConfig(host string, secret *corev1.Secret, restTimeout *time.Duration)
 
 		token := secret.Data["token"]
 		config.BearerToken = string(token)
-		return config
+		return config, nil
 	}
 
 	// Let's figure it's either a TLS secret or an opaque thing formatted like a
@@ -332,5 +341,14 @@ func buildConfig(host string, secret *corev1.Secret, restTimeout *time.Duration)
 		config.KeyData = key
 	}
 
-	return config
+	if encodedInsecureSkipTlsVerify, ok := secret.Annotations[shipperv1.SecretClusterSkipTlsVerifyAnnotation]; ok {
+		if insecureSkipTlsVerify, err := strconv.ParseBool(encodedInsecureSkipTlsVerify); err == nil {
+			glog.Infof("found %q annotation with value %q for host %q", shipperv1.SecretClusterSkipTlsVerifyAnnotation, encodedInsecureSkipTlsVerify, host)
+			config.Insecure = insecureSkipTlsVerify
+		} else {
+			glog.Infof("found %q annotation with value %q for host %q but failed to decode a bool from it, ignoring it", shipperv1.SecretClusterSkipTlsVerifyAnnotation, encodedInsecureSkipTlsVerify, host)
+		}
+	}
+
+	return config, nil
 }

test/e2e/e2e_test.go

@@ -4,6 +4,7 @@ import (
 	"flag"
 	"fmt"
 	"os"
+	"strconv"
 	"testing"
 	"time"
 
@@ -647,6 +648,15 @@ func buildTargetClient(clusterName string) kubernetes.Interface {
 	config.CertData = secret.Data["tls.crt"]
 	config.KeyData = secret.Data["tls.key"]
 
+	if encodedInsecureSkipTlsVerify, ok := secret.Annotations[shipperv1.SecretClusterSkipTlsVerifyAnnotation]; ok {
+		if insecureSkipTlsVerify, err := strconv.ParseBool(encodedInsecureSkipTlsVerify); err == nil {
+			glog.Infof("found %q annotation with value %q", shipperv1.SecretClusterSkipTlsVerifyAnnotation, encodedInsecureSkipTlsVerify)
+			config.Insecure = insecureSkipTlsVerify
+		} else {
+			glog.Infof("found %q annotation with value %q, failed to decode a bool from it, ignoring it", shipperv1.SecretClusterSkipTlsVerifyAnnotation, encodedInsecureSkipTlsVerify)
+		}
+	}
+
 	client, err := kubernetes.NewForConfig(config)
 	if err != nil {
 		glog.Fatalf("could not build target kubeclient for cluster %q: problem fetching cluster: %q", clusterName, err)