WIP: Allow kops cluster behind a proxy

Allow a kops cluster to operate behind a proxy by passing proxy configuration to addons and nodes. The API will change to be consistent with the rest of the project, but currently uses the environment variables `CLUSTER_HTTP_PROXY`, `CLUSTER_HTTPS_PROXY`, and `CLUSTER_NO_PROXY` available when and where kops is invoked. Relates to kubernetes#2481
johnzeringue · Jun 15, 2017 · c9dab72 · c9dab72
1 parent 0c1fd6b
commit c9dab72
Show file tree

Hide file tree

Showing 10 changed files with 118 additions and 17 deletions.
diff --git a/nodeup/pkg/bootstrap/install.go b/nodeup/pkg/bootstrap/install.go
@@ -110,21 +110,33 @@ func (i *Installation) buildSystemdJob() *nodetasks.Service {
 	manifest.Set("Unit", "Description", "Run kops bootstrap (nodeup)")
 	manifest.Set("Unit", "Documentation", "https://github.com/kubernetes/kops")
 
+	environment := make(map[string]string)
+
 	// Pass in required credentials when using user-defined s3 endpoint
 	if os.Getenv("S3_ENDPOINT") != "" {
+		environment["S3_ENDPOINT"] = os.Getenv("S3_ENDPOINT")
+		environment["S3_REGION"] = os.Getenv("S3_REGION")
+		environment["S3_ACCESS_KEY_ID"] = os.Getenv("S3_ACCESS_KEY_ID")
+		environment["S3_SECRET_ACCESS_KEY"] = os.Getenv("S3_SECRET_ACCESS_KEY")
+	}
+
+	// Pass in proxy settings
+	if os.Getenv("HTTP_PROXY") != "" || os.Getenv("HTTPS_PROXY") != "" {
+		environment["HTTP_PROXY"] = os.Getenv("HTTP_PROXY")
+		environment["HTTPS_PROXY"] = os.Getenv("HTTPS_PROXY")
+		environment["NO_PROXY"] = os.Getenv("NO_PROXY")
+	}
+
+	if len(environment) != 0 {
 		var buffer bytes.Buffer
-		buffer.WriteString("\"S3_ENDPOINT=")
-		buffer.WriteString(os.Getenv("S3_ENDPOINT"))
-		buffer.WriteString("\" ")
-		buffer.WriteString("\"S3_REGION=")
-		buffer.WriteString(os.Getenv("S3_REGION"))
-		buffer.WriteString("\" ")
-		buffer.WriteString("\"S3_ACCESS_KEY_ID=")
-		buffer.WriteString(os.Getenv("S3_ACCESS_KEY_ID"))
-		buffer.WriteString("\" ")
-		buffer.WriteString("\"S3_SECRET_ACCESS_KEY=")
-		buffer.WriteString(os.Getenv("S3_SECRET_ACCESS_KEY"))
-		buffer.WriteString("\" ")
+
+		for name, value := range environment {
+			buffer.WriteString("\"")
+			buffer.WriteString(name)
+			buffer.WriteString("=")
+			buffer.WriteString(value)
+			buffer.WriteString("\" ")
+		}
 
 		manifest.Set("Service", "Environment", buffer.String())
 	}

diff --git a/nodeup/pkg/model/kubeapiserver.go b/nodeup/pkg/model/kubeapiserver.go
@@ -28,6 +28,7 @@ import (
 	"k8s.io/kops/pkg/flagbuilder"
 	"k8s.io/kops/upup/pkg/fi"
 	"k8s.io/kops/upup/pkg/fi/nodeup/nodetasks"
+	"os"
 )
 
 // KubeAPIServerBuilder install kube-apiserver (just the manifest at the moment)
@@ -160,6 +161,14 @@ func (b *KubeAPIServerBuilder) buildPod() (*v1.Pod, error) {
 		},
 	}
 
+	if os.Getenv("HTTP_PROXY") != "" || os.Getenv("HTTPS_PROXY") != "" {
+		container.Env = []v1.EnvVar{
+			{Name: "HTTP_PROXY", Value: os.Getenv("HTTP_PROXY")},
+			{Name: "HTTPS_PROXY", Value: os.Getenv("HTTPS_PROXY")},
+			{Name: "NO_PROXY", Value: os.Getenv("NO_PROXY")},
+		}
+	}
+
 	for _, path := range b.SSLHostPaths() {
 		name := strings.Replace(path, "/", "", -1)
 

diff --git a/nodeup/pkg/model/kubecontrollermanager.go b/nodeup/pkg/model/kubecontrollermanager.go
@@ -28,6 +28,7 @@ import (
 	"k8s.io/kops/pkg/flagbuilder"
 	"k8s.io/kops/upup/pkg/fi"
 	"k8s.io/kops/upup/pkg/fi/nodeup/nodetasks"
+	"os"
 )
 
 // KubeControllerManagerBuilder install kube-controller-manager (just the manifest at the moment)
@@ -188,6 +189,14 @@ func (b *KubeControllerManagerBuilder) buildPod() (*v1.Pod, error) {
 		},
 	}
 
+	if os.Getenv("HTTP_PROXY") != "" || os.Getenv("HTTPS_PROXY") != "" {
+		container.Env = []v1.EnvVar{
+			{Name: "HTTP_PROXY", Value: os.Getenv("HTTP_PROXY")},
+			{Name: "HTTPS_PROXY", Value: os.Getenv("HTTPS_PROXY")},
+			{Name: "NO_PROXY", Value: os.Getenv("NO_PROXY")},
+		}
+	}
+
 	for _, path := range b.SSLHostPaths() {
 		name := strings.Replace(path, "/", "", -1)
 

diff --git a/nodeup/pkg/model/kubelet.go b/nodeup/pkg/model/kubelet.go
@@ -29,6 +29,7 @@ import (
 	"k8s.io/kops/upup/pkg/fi"
 	"k8s.io/kops/upup/pkg/fi/nodeup/nodetasks"
 	"k8s.io/kops/upup/pkg/fi/utils"
+	"os"
 )
 
 // KubeletBuilder install kubelet
@@ -147,6 +148,12 @@ func (b *KubeletBuilder) buildSystemdEnvironmentFile(kubeletConfig *kops.Kubelet
 
 	sysconfig := "DAEMON_ARGS=\"" + flags + "\"\n"
 
+	if os.Getenv("HTTP_PROXY") != "" || os.Getenv("HTTPS_PROXY") != "" {
+		sysconfig += "HTTP_PROXY=" + os.Getenv("HTTP_PROXY") + "\n"
+		sysconfig += "HTTPS_PROXY=" + os.Getenv("HTTPS_PROXY") + "\n"
+		sysconfig += "NO_PROXY=" + os.Getenv("NO_PROXY") + "\n"
+	}
+
 	t := &nodetasks.File{
 		Path:     "/etc/sysconfig/kubelet",
 		Contents: fi.NewStringResource(sysconfig),

diff --git a/nodeup/pkg/model/protokube.go b/nodeup/pkg/model/protokube.go
@@ -95,7 +95,7 @@ func (b *ProtokubeBuilder) buildSystemdService() (*nodetasks.Service, error) {
 	manifest.Set("Unit", "Description", "Kubernetes Protokube Service")
 	manifest.Set("Unit", "Documentation", "https://github.com/kubernetes/kops")
 
-	//manifest.Set("Service", "EnvironmentFile", "/etc/sysconfig/protokube")
+	manifest.Set("Service", "EnvironmentFile", "/etc/environment")
 	manifest.Set("Service", "ExecStartPre", b.ProtokubeImagePullCommand())
 	manifest.Set("Service", "ExecStart", protokubeCommand)
 	manifest.Set("Service", "Restart", "always")
@@ -246,9 +246,9 @@ func (t *ProtokubeBuilder) ProtokubeFlags(k8sVersion semver.Version) *ProtokubeF
 }
 
 func (t *ProtokubeBuilder) ProtokubeEnvironmentVariables() string {
+	var buffer bytes.Buffer
 	// Pass in required credentials when using user-defined s3 endpoint
 	if os.Getenv("S3_ENDPOINT") != "" {
-		var buffer bytes.Buffer
 		buffer.WriteString(" ")
 		buffer.WriteString("-e S3_ENDPOINT=")
 		buffer.WriteString("'")
@@ -267,9 +267,9 @@ func (t *ProtokubeBuilder) ProtokubeEnvironmentVariables() string {
 		buffer.WriteString(os.Getenv("S3_SECRET_ACCESS_KEY"))
 		buffer.WriteString("'")
 		buffer.WriteString(" ")
-
-		return buffer.String()
 	}
 
-	return ""
+	buffer.WriteString(" --env-file=/etc/environment ")
+
+	return buffer.String()
 }
diff --git a/pkg/model/bootstrapscript.go b/pkg/model/bootstrapscript.go
@@ -71,6 +71,17 @@ func (b *BootstrapScript) ResourceNodeUp(ig *kops.InstanceGroup) (*fi.ResourceHo
 			}
 			return ""
 		},
+
+		// Pass in configuration for the cluster proxy
+		"ProxyEnv": func() string {
+			if os.Getenv("CLUSTER_HTTP_PROXY") != "" || os.Getenv("CLUSTER_HTTPS_PROXY") != "" {
+				return fmt.Sprintf("export HTTP_PROXY=%s\nexport HTTPS_PROXY=%s\nexport NO_PROXY=%s\n",
+					os.Getenv("CLUSTER_HTTP_PROXY"),
+					os.Getenv("CLUSTER_HTTPS_PROXY"),
+					os.Getenv("CLUSTER_NO_PROXY"))
+			}
+			return ""
+		},
 	}
 
 	templateResource, err := NewTemplateResource("nodeup", resources.AWSNodeUpTemplate, functions, nil)

diff --git a/pkg/model/resources/nodeup.go b/pkg/model/resources/nodeup.go
@@ -39,6 +39,31 @@ NODEUP_URL={{ NodeUpSource }}
 NODEUP_HASH={{ NodeUpSourceHash }}
 
 {{ S3Env }}
+{{ ProxyEnv }}
+
+if [[ ! -z $HTTP_PROXY ]] || [[ ! -z $HTTPS_PROXY ]]; then
+  printf "\nAcquire::http::Proxy \\\"$HTTP_PROXY\\\";" >> /etc/apt/apt.conf.d/01proxy
+
+  printf "\nhttp_proxy=$HTTP_PROXY" >> /etc/wgetrc
+  printf "\nhttps_proxy=$HTTPS_PROXY" >> /etc/wgetrc
+  printf "\nuse_proxy=on" >> /etc/wgetrc
+
+  echo HTTP_PROXY=$HTTP_PROXY >> /etc/environment
+  echo HTTPS_PROXY=$HTTPS_PROXY >> /etc/environment
+  echo NO_PROXY=$NO_PROXY >> /etc/environment
+
+  mkdir -p /etc/systemd/system/docker.service.d
+  printf "[Service]\n" >> /etc/systemd/system/docker.service.d/http-proxy.conf
+  printf "Environment=\"HTTP_PROXY=$HTTP_PROXY\" \"HTTPS_PROXY=$HTTPS_PROXY\" \"NO_PROXY=$NO_PROXY\"\n" >> /etc/systemd/system/docker.service.d/http-proxy.conf
+
+  printf "DOCKER_OPTS=\"--config=/root/.docker\"\n" >> /etc/default/docker
+  printf "HTTP_PROXY=$HTTP_PROXY\n" >> /etc/default/docker
+  printf "HTTPS_PROXY=$HTTPS_PROXY\n" >> /etc/default/docker
+  printf "NO_PROXY=$NO_PROXY\n" >> /etc/default/docker
+
+  systemctl daemon-reload
+  systemctl restart docker
+fi
 
 function ensure-install-dir() {
   INSTALL_DIR="/var/cache/kubernetes-install"

diff --git a/upup/models/cloudup/resources/addons/dns-controller.addons.k8s.io/k8s-1.6.yaml.template b/upup/models/cloudup/resources/addons/dns-controller.addons.k8s.io/k8s-1.6.yaml.template
@@ -34,6 +34,13 @@ spec:
       containers:
       - name: dns-controller
         image: kope/dns-controller:1.6.1
+{{ if ProxyEnv }}
+        env:
+{{ end }}
+{{ range $name, $value := ProxyEnv }}
+        - name: {{ $name }}
+          value: "{{ $value }}"
+{{ end }}
         command:
 {{ range $arg := DnsControllerArgv }}
         - "{{ $arg }}"

diff --git a/upup/models/cloudup/resources/addons/dns-controller.addons.k8s.io/pre-k8s-1.6.yaml.template b/upup/models/cloudup/resources/addons/dns-controller.addons.k8s.io/pre-k8s-1.6.yaml.template
@@ -29,6 +29,13 @@ spec:
       containers:
       - name: dns-controller
         image: {{ DnsControllerImage }}:1.6.1
+{{ if ProxyEnv }}
+        env:
+{{ end }}
+{{ range $name, $value := ProxyEnv }}
+        - name: {{ $name }}
+          value: "{{ $value }}"
+{{ end }}
         command:
 {{ range $arg := DnsControllerArgv }}
         - "{{ $arg }}"

diff --git a/upup/pkg/fi/cloudup/template_functions.go b/upup/pkg/fi/cloudup/template_functions.go
@@ -101,6 +101,8 @@ func (tf *TemplateFunctions) AddTo(dest template.FuncMap) {
 	dest["EncodeGCELabel"] = gce.EncodeGCELabel
 
 	dest["DnsControllerImage"] = tf.DnsControllerImage
+
+	dest["ProxyEnv"] = tf.ProxyEnv
 }
 
 // SharedVPC is a simple helper function which makes the templates for a shared VPC clearer
@@ -205,3 +207,15 @@ func (tf *TemplateFunctions) ExternalDnsArgv() ([]string, error) {
 
 	return argv, nil
 }
+
+func (tf *TemplateFunctions) ProxyEnv() map[string]string {
+	if os.Getenv("CLUSTER_HTTP_PROXY") != "" || os.Getenv("CLUSTER_HTTPS_PROXY") != "" {
+		return map[string]string {
+			"HTTP_PROXY": os.Getenv("CLUSTER_HTTP_PROXY"),
+			"HTTPS_PROXY": os.Getenv("CLUSTER_HTTPS_PROXY"),
+			"NO_PROXY": os.Getenv("CLUSTER_NO_PROXY"),
+		}
+	}
+
+	return nil
+}