Add support for cluster using http forward proxy #2481

docs/http_proxy.md

@@ -0,0 +1,40 @@
+
+HTTP Forward Proxy Support
+==========================
+
+It is possible to launch a kubernetes cluster from behind an http forward proxy ("corporate proxy").  To do so, you will need to configure the `egressProxy` for the cluster.
+
+It is assumed the proxy is already existing.  If you want a private topology on AWS, for example, with an proxy instead of a NAT instance, you'll need to create the proxy yourself.  See [Running in a shared VPC](run_in_existing_vpc.md).
+
+This configuration only manages proxy configurations for Kops and the Kubernetes cluster.  We can not handle proxy configuration for application containers and pods.
+
+## Configuration
+
+Add `spec.egressProxy` port and url as follows
+
+``` yaml
+spec:
+  egressProxy:
+    httpProxy:
+      host: proxy.corp.local
+      port: 3128
+```
+
+Currently we assume the same configuration for http and https traffic.
+
+## Proxy Excludes
+
+Most clients will blindly try to use the proxy to make all calls, even to localhost and the local subnet, unless configured otherwise.  Some basic exclusions necessary for successful launch and operation are added for you at initial cluster creation.  If you wish to add additional exclusions, add or edit `egressProxy.excludes` with a comma separated list of hostnames.  Matching is based on suffix, ie, `corp.local` will match `images.corp.local`, and `.corp.local` will match `corp.local` and `images.corp.local`, following typical `no_proxy` environment variable conventions.
+
+``` yaml
+spec:
+  egressProxy:
+    httpProxy:
+      host: proxy.corp.local
+      port: 3128
+    excludes: corp.local,internal.corp.com
+```
+
+## AWS VPC Endpoints and S3 access
+
+If you are hosting on AWS have configured VPC "Endpoints" for S3 or other services, you may want to add these to the `spec.egressProxy.excludes`.  Keep in mind that the S3 bucket must be in the same region as the VPC for it to be accessible via the endpoint.

docs/run_in_existing_vpc.md

@@ -146,3 +146,7 @@ Please note:
   in their route table.  Private subnets should not have public IPs, and will typically have a NAT gateway
   configured as their default route.
 * kops won't create a route-table at all if we're not creating subnets.
+
+### Proxy VPC Egress
+
+See [HTTP Forward Proxy Support](http_proxy.md)

nodeup/pkg/model/convenience.go

@@ -16,7 +16,14 @@ limitations under the License.
 
 package model
 
-import "k8s.io/kops/upup/pkg/fi"
+import (
+	"strconv"
+
+	"github.com/golang/glog"
+	"k8s.io/client-go/pkg/api/v1"
+	"k8s.io/kops/pkg/apis/kops"
+	"k8s.io/kops/upup/pkg/fi"
+)
 
 // s is a helper that builds a *string from a string value
 func s(v string) *string {
@@ -28,6 +35,33 @@ func i64(v int64) *int64 {
 	return fi.Int64(v)
 }
 
+func getProxyEnvVars(proxies *kops.EgressProxySpec) []v1.EnvVar {
+	if proxies == nil {
+		glog.V(8).Info("proxies is == nil, returning empty list")
+		return []v1.EnvVar{}
+	}
+
+	if proxies.HTTPProxy.Host == "" {
+		glog.Warning("EgressProxy set but no proxy host provided")
+	}
+
+	var httpProxyURL string
+	if proxies.HTTPProxy.Port == 0 {
+		httpProxyURL = "http://" + proxies.HTTPProxy.Host
+	} else {
+		httpProxyURL = "http://" + proxies.HTTPProxy.Host + ":" + strconv.Itoa(proxies.HTTPProxy.Port)
+	}
+
+	noProxy := proxies.ProxyExcludes
+
+	return []v1.EnvVar{
+		{Name: "http_proxy", Value: httpProxyURL},
+		{Name: "https_proxy", Value: httpProxyURL},
+		{Name: "NO_PROXY", Value: noProxy},
+		{Name: "no_proxy", Value: noProxy},
+	}
+}
+
 // b returns a pointer to a boolean
 func b(v bool) *bool {
 	return fi.Bool(v)

nodeup/pkg/model/kubeapiserver.go

@@ -223,6 +223,7 @@ func (b *KubeAPIServerBuilder) buildPod() (*v1.Pod, error) {
 				HostPort:      8080,
 			},
 		},
+		Env: getProxyEnvVars(b.Cluster.Spec.EgressProxy),
 	}
 
 	for _, path := range b.SSLHostPaths() {

nodeup/pkg/model/kubecontrollermanager.go

@@ -186,6 +186,7 @@ func (b *KubeControllerManagerBuilder) buildPod() (*v1.Pod, error) {
 			InitialDelaySeconds: 15,
 			TimeoutSeconds:      15,
 		},
+		Env: getProxyEnvVars(b.Cluster.Spec.EgressProxy),
 	}
 
 	for _, path := range b.SSLHostPaths() {

nodeup/pkg/model/kubescheduler.go

@@ -140,6 +140,7 @@ func (b *KubeSchedulerBuilder) buildPod() (*v1.Pod, error) {
 			InitialDelaySeconds: 15,
 			TimeoutSeconds:      15,
 		},
+		Env: getProxyEnvVars(b.Cluster.Spec.EgressProxy),
 	}
 
 	addHostPathMapping(pod, container, "varlibkubescheduler", "/var/lib/kube-scheduler")

nodeup/pkg/model/protokube.go

@@ -276,6 +276,9 @@ func (t *ProtokubeBuilder) ProtokubeFlags(k8sVersion semver.Version) *ProtokubeF
 func (t *ProtokubeBuilder) ProtokubeEnvironmentVariables() string {
 	var buffer bytes.Buffer
 
+	// TODO write out an environments file for this.  This is getting a tad long.
+
+	// Pass in required credentials when using user-defined s3 endpoint
 	if os.Getenv("AWS_REGION") != "" {
 		buffer.WriteString(" ")
 		buffer.WriteString("-e 'AWS_REGION=")
@@ -284,7 +287,6 @@ func (t *ProtokubeBuilder) ProtokubeEnvironmentVariables() string {
 		buffer.WriteString(" ")
 	}
 
-	// Pass in required credentials when using user-defined s3 endpoint
 	if os.Getenv("S3_ENDPOINT") != "" {
 		buffer.WriteString(" ")
 		buffer.WriteString("-e S3_ENDPOINT=")
@@ -306,9 +308,21 @@ func (t *ProtokubeBuilder) ProtokubeEnvironmentVariables() string {
 		buffer.WriteString(" ")
 	}
 
+	t.writeProxyEnvVars(&buffer)
+
 	return buffer.String()
 }
 
+func (t *ProtokubeBuilder) writeProxyEnvVars(buffer *bytes.Buffer) {
+	for _, envVar := range getProxyEnvVars(t.Cluster.Spec.EgressProxy) {
+		buffer.WriteString(" -e ")
+		buffer.WriteString(envVar.Name)
+		buffer.WriteString("=")
+		buffer.WriteString(envVar.Value)
+		buffer.WriteString(" ")
+	}
+}
+
 // buildCertificateTask is responsible for build a certificate request task
 func (t *ProtokubeBuilder) buildCeritificateTask(c *fi.ModelBuilderContext, name, filename string) error {
 	cert, err := t.KeyStore.Cert(name)

pkg/apis/kops/cluster.go

@@ -93,6 +93,9 @@ type ClusterSpec struct {
 	NonMasqueradeCIDR string `json:"nonMasqueradeCIDR,omitempty"`
 	// SSHAccess is a list of the CIDRs that can access SSH.
 	SSHAccess []string `json:"sshAccess,omitempty"`
+	// HTTPProxy defines connection information to support use of a private cluster behind an forward HTTP Proxy
+	EgressProxy *EgressProxySpec `json:"egressProxy,omitempty"`
+
 	// KubernetesAPIAccess is a list of the CIDRs that can access the Kubernetes API endpoint (master HTTPS)
 	KubernetesAPIAccess []string `json:"kubernetesApiAccess,omitempty"`
 	// IsolatesMasters determines whether we should lock down masters so that they are not on the pod network.
@@ -285,6 +288,19 @@ type ClusterSubnetSpec struct {
 	Type SubnetType `json:"type,omitempty"`
 }
 
+type EgressProxySpec struct {
+	HTTPProxy     HTTPProxy `json:"httpProxy,omitempty"`
+	ProxyExcludes string    `json:"excludes,omitempty"`
+}
+
+type HTTPProxy struct {
+	Host string `json:"host,omitempty"`
+	Port int    `json:"port,omitempty"`
+	// TODO #3070
+	// User     string `json:"user,omitempty"`
+	// Password string `json:"password,omitempty"`
+}
+
 // FillDefaults populates default values.
 // This is different from PerformAssignments, because these values are changeable, and thus we don't need to
 // store them (i.e. we don't need to 'lock them')

pkg/apis/kops/v1alpha1/cluster.go

@@ -212,6 +212,9 @@ type ClusterSpec struct {
 	//KubeProxyTestArgs             string `json:",omitempty"`
 	//KubeProxyTestLogLevel         string `json:",omitempty"`
 
+	// HTTPProxy defines connection information to support use of a private cluster behind an forward HTTP Proxy
+	EgressProxy *EgressProxySpec `json:"egressProxy,omitempty"`
+
 	// EtcdClusters stores the configuration for each cluster
 	EtcdClusters []*EtcdClusterSpec `json:"etcdClusters,omitempty"`
 
@@ -368,3 +371,16 @@ type ClusterZoneSpec struct {
 
 	Egress string `json:"egress,omitempty"`
 }
+
+type EgressProxySpec struct {
+	HTTPProxy     HTTPProxy `json:"httpProxy,omitempty"`
+	ProxyExcludes string    `json:"excludes,omitempty"`
+}
+
+type HTTPProxy struct {
+	Host string `json:"host,omitempty"`
+	Port int    `json:"port,omitempty"`
+	// TODO #3070
+	// User     string `json:"user,omitempty"`
+	// Password string `json:"password,omitempty"`
+}

pkg/apis/kops/v1alpha1/zz_generated.conversion.go

@@ -67,6 +67,8 @@ func RegisterConversions(scheme *runtime.Scheme) error {
 		Convert_kops_DNSSpec_To_v1alpha1_DNSSpec,
 		Convert_v1alpha1_DockerConfig_To_kops_DockerConfig,
 		Convert_kops_DockerConfig_To_v1alpha1_DockerConfig,
+		Convert_v1alpha1_EgressProxySpec_To_kops_EgressProxySpec,
+		Convert_kops_EgressProxySpec_To_v1alpha1_EgressProxySpec,
 		Convert_v1alpha1_EtcdClusterSpec_To_kops_EtcdClusterSpec,
 		Convert_kops_EtcdClusterSpec_To_v1alpha1_EtcdClusterSpec,
 		Convert_v1alpha1_EtcdMemberSpec_To_kops_EtcdMemberSpec,
@@ -83,6 +85,8 @@ func RegisterConversions(scheme *runtime.Scheme) error {
 		Convert_kops_FederationSpec_To_v1alpha1_FederationSpec,
 		Convert_v1alpha1_FlannelNetworkingSpec_To_kops_FlannelNetworkingSpec,
 		Convert_kops_FlannelNetworkingSpec_To_v1alpha1_FlannelNetworkingSpec,
+		Convert_v1alpha1_HTTPProxy_To_kops_HTTPProxy,
+		Convert_kops_HTTPProxy_To_v1alpha1_HTTPProxy,
 		Convert_v1alpha1_HookSpec_To_kops_HookSpec,
 		Convert_kops_HookSpec_To_v1alpha1_HookSpec,
 		Convert_v1alpha1_InstanceGroup_To_kops_InstanceGroup,
@@ -524,6 +528,15 @@ func autoConvert_v1alpha1_ClusterSpec_To_kops_ClusterSpec(in *ClusterSpec, out *
 	out.IsolateMasters = in.IsolateMasters
 	out.UpdatePolicy = in.UpdatePolicy
 	out.AdditionalPolicies = in.AdditionalPolicies
+	if in.EgressProxy != nil {
+		in, out := &in.EgressProxy, &out.EgressProxy
+		*out = new(kops.EgressProxySpec)
+		if err := Convert_v1alpha1_EgressProxySpec_To_kops_EgressProxySpec(*in, *out, s); err != nil {
+			return err
+		}
+	} else {
+		out.EgressProxy = nil
+	}
 	if in.EtcdClusters != nil {
 		in, out := &in.EtcdClusters, &out.EtcdClusters
 		*out = make([]*kops.EtcdClusterSpec, len(*in))
@@ -705,6 +718,15 @@ func autoConvert_kops_ClusterSpec_To_v1alpha1_ClusterSpec(in *kops.ClusterSpec,
 	out.ServiceClusterIPRange = in.ServiceClusterIPRange
 	out.NonMasqueradeCIDR = in.NonMasqueradeCIDR
 	// WARNING: in.SSHAccess requires manual conversion: does not exist in peer-type
+	if in.EgressProxy != nil {
+		in, out := &in.EgressProxy, &out.EgressProxy
+		*out = new(EgressProxySpec)
+		if err := Convert_kops_EgressProxySpec_To_v1alpha1_EgressProxySpec(*in, *out, s); err != nil {
+			return err
+		}
+	} else {
+		out.EgressProxy = nil
+	}
 	// WARNING: in.KubernetesAPIAccess requires manual conversion: does not exist in peer-type
 	out.IsolateMasters = in.IsolateMasters
 	out.UpdatePolicy = in.UpdatePolicy
@@ -946,6 +968,32 @@ func Convert_kops_DockerConfig_To_v1alpha1_DockerConfig(in *kops.DockerConfig, o
 	return autoConvert_kops_DockerConfig_To_v1alpha1_DockerConfig(in, out, s)
 }
 
+func autoConvert_v1alpha1_EgressProxySpec_To_kops_EgressProxySpec(in *EgressProxySpec, out *kops.EgressProxySpec, s conversion.Scope) error {
+	if err := Convert_v1alpha1_HTTPProxy_To_kops_HTTPProxy(&in.HTTPProxy, &out.HTTPProxy, s); err != nil {
+		return err
+	}
+	out.ProxyExcludes = in.ProxyExcludes
+	return nil
+}
+
+// Convert_v1alpha1_EgressProxySpec_To_kops_EgressProxySpec is an autogenerated conversion function.
+func Convert_v1alpha1_EgressProxySpec_To_kops_EgressProxySpec(in *EgressProxySpec, out *kops.EgressProxySpec, s conversion.Scope) error {
+	return autoConvert_v1alpha1_EgressProxySpec_To_kops_EgressProxySpec(in, out, s)
+}
+
+func autoConvert_kops_EgressProxySpec_To_v1alpha1_EgressProxySpec(in *kops.EgressProxySpec, out *EgressProxySpec, s conversion.Scope) error {
+	if err := Convert_kops_HTTPProxy_To_v1alpha1_HTTPProxy(&in.HTTPProxy, &out.HTTPProxy, s); err != nil {
+		return err
+	}
+	out.ProxyExcludes = in.ProxyExcludes
+	return nil
+}
+
+// Convert_kops_EgressProxySpec_To_v1alpha1_EgressProxySpec is an autogenerated conversion function.
+func Convert_kops_EgressProxySpec_To_v1alpha1_EgressProxySpec(in *kops.EgressProxySpec, out *EgressProxySpec, s conversion.Scope) error {
+	return autoConvert_kops_EgressProxySpec_To_v1alpha1_EgressProxySpec(in, out, s)
+}
+
 func autoConvert_v1alpha1_EtcdClusterSpec_To_kops_EtcdClusterSpec(in *EtcdClusterSpec, out *kops.EtcdClusterSpec, s conversion.Scope) error {
 	out.Name = in.Name
 	out.EnableEtcdTLS = in.EnableEtcdTLS
@@ -1162,6 +1210,28 @@ func Convert_kops_FlannelNetworkingSpec_To_v1alpha1_FlannelNetworkingSpec(in *ko
 	return autoConvert_kops_FlannelNetworkingSpec_To_v1alpha1_FlannelNetworkingSpec(in, out, s)
 }
 
+func autoConvert_v1alpha1_HTTPProxy_To_kops_HTTPProxy(in *HTTPProxy, out *kops.HTTPProxy, s conversion.Scope) error {
+	out.Host = in.Host
+	out.Port = in.Port
+	return nil
+}
+
+// Convert_v1alpha1_HTTPProxy_To_kops_HTTPProxy is an autogenerated conversion function.
+func Convert_v1alpha1_HTTPProxy_To_kops_HTTPProxy(in *HTTPProxy, out *kops.HTTPProxy, s conversion.Scope) error {
+	return autoConvert_v1alpha1_HTTPProxy_To_kops_HTTPProxy(in, out, s)
+}
+
+func autoConvert_kops_HTTPProxy_To_v1alpha1_HTTPProxy(in *kops.HTTPProxy, out *HTTPProxy, s conversion.Scope) error {
+	out.Host = in.Host
+	out.Port = in.Port
+	return nil
+}
+
+// Convert_kops_HTTPProxy_To_v1alpha1_HTTPProxy is an autogenerated conversion function.
+func Convert_kops_HTTPProxy_To_v1alpha1_HTTPProxy(in *kops.HTTPProxy, out *HTTPProxy, s conversion.Scope) error {
+	return autoConvert_kops_HTTPProxy_To_v1alpha1_HTTPProxy(in, out, s)
+}
+
 func autoConvert_v1alpha1_HookSpec_To_kops_HookSpec(in *HookSpec, out *kops.HookSpec, s conversion.Scope) error {
 	if in.ExecContainer != nil {
 		in, out := &in.ExecContainer, &out.ExecContainer

pkg/apis/kops/v1alpha2/cluster.go

@@ -116,6 +116,9 @@ type ClusterSpec struct {
 	// Currently only a single CIDR is supported (though a richer grammar could be added in future)
 	SSHAccess []string `json:"sshAccess,omitempty"`
 
+	// HTTPProxy defines connection information to support use of a private cluster behind an forward HTTP Proxy
+	EgressProxy *EgressProxySpec `json:"egressProxy,omitempty"`
+
 	// KubernetesAPIAccess determines the permitted access to the API endpoints (master HTTPS)
 	// Currently only a single CIDR is supported (though a richer grammar could be added in future)
 	KubernetesAPIAccess []string `json:"kubernetesApiAccess,omitempty"`
@@ -293,3 +296,17 @@ type ClusterSubnetSpec struct {
 
 	Type SubnetType `json:"type,omitempty"`
 }
+
+type EgressProxySpec struct {
+	HTTPProxy     HTTPProxy `json:"httpProxy,omitempty"`
+	ProxyExcludes string    `json:"excludes,omitempty"`
+}
+
+type HTTPProxy struct {
+	Host string `json:"host,omitempty"`
+	Port int    `json:"port,omitempty"`
+
+	// TODO #3070
+	// User     string `json:"user,omitempty"`
+	// Password string `json:"password,omitempty"`
+}