Catalog Controller should listen on https and serve metrics over TLS …

…secured channel (openshift#1851)

* Controller should be listening on Secure port with TLS

* Secure scrape configuration specific to the Catalog Controller

* corrected default port which isnt used anymore

* Move SecureServingOptions into the ControllerManagerConfiguration

* Use SecureServingOptions.BindAddress instead of address

charts/catalog/templates/controller-manager-deployment.yaml

@@ -41,8 +41,8 @@ spec:
               fieldPath: metadata.namespace
         args:
         - controller-manager
-        - --port
-        - "8080"
+        - --secure-port
+        - "8444"
         {{ if .Values.controllerManager.leaderElection.activated -}}
         - "--leader-election-namespace={{ .Release.Namespace }}"
         - "--leader-elect-resource-lock=configmaps"
@@ -83,24 +83,26 @@ spec:
         - NamespacedServiceBroker=true
         {{- end }}
         ports:
-        - containerPort: 8080
+        - containerPort: 8444
         volumeMounts:
         - name: service-catalog-cert
-          mountPath: /etc/service-catalog-ssl
+          mountPath: /var/run/kubernetes-service-catalog
           readOnly: true
         readinessProbe:
           httpGet:
-            port: 8080
+            port: 8444
             path: /healthz
+            scheme: HTTPS
           failureThreshold: 1
           initialDelaySeconds: 20
           periodSeconds: 10
           successThreshold: 1
           timeoutSeconds: 2
         livenessProbe:
           httpGet:
-            port: 8080
+            port: 8444
             path: /healthz
+            scheme: HTTPS
           failureThreshold: 3
           initialDelaySeconds: 20
           periodSeconds: 10
@@ -113,3 +115,9 @@ spec:
           items:
           - key: tls.crt
             path: apiserver.crt
+          - key: tls.key
+            path: apiserver.key
+          {{- if .Values.apiserver.tls.requestHeaderCA }}
+          - key: requestheader-ca.crt
+            path: requestheader-ca.crt
+          {{- end }}

cmd/controller-manager/app/controller_manager.go

@@ -93,6 +93,10 @@ func Run(controllerManagerOptions *options.ControllerManagerServer) error {
 	// 	glog.Errorf("unable to register configz: %s", err)
 	// }
 
+	if controllerManagerOptions.Port > 0 {
+		glog.Warning("program option --port is obsolete and ignored, specify --secure-port instead")
+	}
+
 	// Build the K8s kubeconfig / client / clientBuilder
 	glog.V(4).Info("Building k8s kubeconfig")
 
@@ -142,6 +146,14 @@ func Run(controllerManagerOptions *options.ControllerManagerServer) error {
 	}
 	serviceCatalogKubeconfig.Insecure = controllerManagerOptions.ServiceCatalogInsecureSkipVerify
 
+	// Initialize SSL/TLS configuration.  Ensures we have a certificate and key to use.
+	// This is the same code as what is done in the API Server.  By default, Helm created
+	// cert and key for us, this just ensures the files are found and are readable and
+	// creates self signed versions if not.
+	if err := controllerManagerOptions.SecureServingOptions.MaybeDefaultWithSelfSignedCerts("" /*AdvertiseAddress*/, nil /*alternateDNS*/, []net.IP{net.ParseIP("127.0.0.1")}); err != nil {
+		return fmt.Errorf("failed to establish SecureServingOptions %v", err)
+	}
+
 	glog.V(4).Info("Starting http server and mux")
 	// Start http server and handlers
 	go func() {
@@ -165,10 +177,12 @@ func Run(controllerManagerOptions *options.ControllerManagerServer) error {
 			}
 		}
 		server := &http.Server{
-			Addr:    net.JoinHostPort(controllerManagerOptions.Address, strconv.Itoa(int(controllerManagerOptions.Port))),
+			Addr: net.JoinHostPort(controllerManagerOptions.SecureServingOptions.BindAddress.String(),
+				strconv.Itoa(int(controllerManagerOptions.SecureServingOptions.BindPort))),
 			Handler: mux,
 		}
-		glog.Fatal(server.ListenAndServe())
+		glog.Fatal(server.ListenAndServeTLS(controllerManagerOptions.SecureServingOptions.ServerCert.CertKey.CertFile,
+			controllerManagerOptions.SecureServingOptions.ServerCert.CertKey.KeyFile))
 	}()
 
 	// Create event broadcaster

cmd/controller-manager/app/options/options.go

@@ -30,6 +30,14 @@ import (
 	k8scomponentconfig "github.com/kubernetes-incubator/service-catalog/pkg/kubernetes/pkg/apis/componentconfig"
 	"github.com/kubernetes-incubator/service-catalog/pkg/kubernetes/pkg/client/leaderelectionconfig"
 	osb "github.com/pmorie/go-open-service-broker-client/v2"
+	genericoptions "k8s.io/apiserver/pkg/server/options"
+)
+
+const (
+	// Use the same SSL configuration as we use in Catalog API Server.
+	// Store generated SSL certificates in a place that won't collide with the
+	// k8s core API server.
+	certDirectory = "/var/run/kubernetes-service-catalog"
 )
 
 // ControllerManagerServer is the main context object for the controller
@@ -43,7 +51,7 @@ const (
 	defaultServiceBrokerRelistInterval            = 24 * time.Hour
 	defaultContentType                            = "application/json"
 	defaultBindAddress                            = "0.0.0.0"
-	defaultPort                                   = 10000
+	defaultPort                                   = 8444
 	defaultK8sKubeconfigPath                      = "./kubeconfig"
 	defaultServiceCatalogKubeconfigPath           = "./service-catalog-kubeconfig"
 	defaultOSBAPIContextProfile                   = true
@@ -61,7 +69,7 @@ func NewControllerManagerServer() *ControllerManagerServer {
 	s := ControllerManagerServer{
 		ControllerManagerConfiguration: componentconfig.ControllerManagerConfiguration{
 			Address:                                defaultBindAddress,
-			Port:                                   defaultPort,
+			Port:                                   0,
 			ContentType:                            defaultContentType,
 			K8sKubeconfigPath:                      defaultK8sKubeconfigPath,
 			ServiceCatalogKubeconfigPath:           defaultServiceCatalogKubeconfigPath,
@@ -76,16 +84,22 @@ func NewControllerManagerServer() *ControllerManagerServer {
 			EnableContentionProfiling:              false,
 			ReconciliationRetryDuration:            defaultReconciliationRetryDuration,
 			OperationPollingMaximumBackoffDuration: defaultOperationPollingMaximumBackoffDuration,
+			SecureServingOptions:                   genericoptions.NewSecureServingOptions(),
 		},
 	}
+	// set defaults, these will be overriden by user specified flags
+	s.SecureServingOptions.BindPort = defaultPort
+	s.SecureServingOptions.ServerCert.CertDirectory = certDirectory
 	s.LeaderElection.LeaderElect = true
 	return &s
 }
 
 // AddFlags adds flags for a ControllerManagerServer to the specified FlagSet.
 func (s *ControllerManagerServer) AddFlags(fs *pflag.FlagSet) {
-	fs.Var(k8scomponentconfig.IPVar{Val: &s.Address}, "address", "The IP address to serve on (set to 0.0.0.0 for all interfaces)")
-	fs.Int32Var(&s.Port, "port", s.Port, "The port that the controller-manager's http service runs on")
+	fs.Var(k8scomponentconfig.IPVar{Val: &s.Address}, "address", "DEPRECATED: see --bind-address instead")
+	fs.MarkDeprecated("address", "see --bind-address instead")
+	fs.Int32Var(&s.Port, "port", 0, "DEPRECATED: see --secure-port instead")
+	fs.MarkDeprecated("port", "see --secure-port instead")
 	fs.StringVar(&s.ContentType, "api-content-type", s.ContentType, "Content type of requests sent to API servers")
 	fs.StringVar(&s.K8sAPIServerURL, "k8s-api-server-url", "", "The URL for the k8s API server")
 	fs.StringVar(&s.K8sKubeconfigPath, "k8s-kubeconfig", "", "Path to k8s core kubeconfig")
@@ -103,6 +117,6 @@ func (s *ControllerManagerServer) AddFlags(fs *pflag.FlagSet) {
 	fs.StringVar(&s.LeaderElectionNamespace, "leader-election-namespace", s.LeaderElectionNamespace, "Namespace to use for leader election lock")
 	fs.DurationVar(&s.ReconciliationRetryDuration, "reconciliation-retry-duration", s.ReconciliationRetryDuration, "The maximum amount of time to retry reconciliations on a resource before failing")
 	fs.DurationVar(&s.OperationPollingMaximumBackoffDuration, "operation-polling-maximum-backoff-duration", s.OperationPollingMaximumBackoffDuration, "The maximum amount of time to back-off while polling an OSB API operation")
-
+	s.SecureServingOptions.AddFlags(fs)
 	utilfeature.DefaultFeatureGate.AddFlag(fs)
 }

contrib/examples/prometheus/prometheus.yml

@@ -301,38 +301,25 @@ data:
       - source_labels: [__meta_kubernetes_service_name]
         target_label: kubernetes_name
 
-    # Example scrape config for pods
-    #
-    # The relabeling allows the actual pod scrape endpoint to be configured via the
-    # following annotations:
-    #
-    # * `prometheus.io/scrape`: Only scrape pods that have a value of `true`
-    # * `prometheus.io/path`: If the metrics path is not `/metrics` override this.
-    # * `prometheus.io/port`: Scrape the pod on the indicated port instead of the
-    # pod's declared ports (default is a port-free target if none are declared).
-    - job_name: 'kubernetes-pods'
+    # Scrape config for Service Catalog
+    - job_name: 'service-catalog'
+      scheme: https
+      # This TLS & bearer token file config is used to connect to the actual scrape
+      # endpoints for cluster components.
+      tls_config:
+        ca_file: /var/run/secrets/kubernetes.io/serviceaccount/ca.crt
+        # If your node certificates are self-signed or use a different CA to the
+        # master CA, then disable certificate verification below.
+        insecure_skip_verify: true
+      bearer_token_file: /var/run/secrets/kubernetes.io/serviceaccount/token
 
       kubernetes_sd_configs:
       - role: pod
+        namespaces:
+          names:
+          - catalog
 
       relabel_configs:
-      - source_labels: [__meta_kubernetes_pod_annotation_prometheus_io_scrape]
-        action: keep
-        regex: true
-      - source_labels: [__meta_kubernetes_pod_annotation_prometheus_io_path]
-        action: replace
-        target_label: __metrics_path__
-        regex: (.+)
-      - source_labels: [__address__, __meta_kubernetes_pod_annotation_prometheus_io_port]
-        action: replace
-        regex: ([^:]+)(?::\d+)?;(\d+)
-        replacement: $1:$2
-        target_label: __address__
-      - action: labelmap
-        regex: __meta_kubernetes_pod_label_(.+)
-      - source_labels: [__meta_kubernetes_namespace]
-        action: replace
-        target_label: kubernetes_namespace
       - source_labels: [__meta_kubernetes_pod_name]
-        action: replace
-        target_label: kubernetes_pod_name
+        action: keep
+        regex: (.+)controller-manager-(.+)

pkg/apis/componentconfig/types.go

@@ -24,14 +24,16 @@ import (
 	"time"
 
 	"github.com/kubernetes-incubator/service-catalog/pkg/kubernetes/pkg/apis/componentconfig"
+	genericoptions "k8s.io/apiserver/pkg/server/options"
 )
 
 // ControllerManagerConfiguration encapsulates configuration for the
 // controller manager.
 type ControllerManagerConfiguration struct {
-	// Address is the IP address to serve on (set to 0.0.0.0 for all interfaces).
+	// DEPRECATED/Ignored, use SecureServingOptions.BindAddress instead.
 	Address string
-	// Port is the port that the controller's http service runs on.
+
+	// DEPRECATED/Ignored, use SecureServingOptions.SecurePort instead.
 	Port int32
 
 	// ContentType is the content type for requests sent to API servers.
@@ -100,4 +102,6 @@ type ControllerManagerConfiguration struct {
 	// OperationPollingMaximumBackoffDuration is the maximum duration that exponential
 	// backoff for polling OSB API operations will use.
 	OperationPollingMaximumBackoffDuration time.Duration
+
+	SecureServingOptions *genericoptions.SecureServingOptions
 }