From 30add92bdb8039932a21755c00faf003ff2cf904 Mon Sep 17 00:00:00 2001
From: Ole Markus With <o.with@sportradar.com>
Date: Sun, 29 Dec 2019 11:05:29 +0100
Subject: [PATCH] Fixes some issues with running Cilium nodeport

* Cilium need to talk to the internal cluster API on  public IPs instead of the internal service
* Tell people explicitly they have to disable kubeproxy so it won't conflict with nodeport
---
 pkg/apis/kops/validation/legacy.go                       | 4 ++++
 .../addons/networking.cilium.io/k8s-1.12.yaml.template   | 9 +++++++++
 .../addons/networking.cilium.io/k8s-1.7.yaml.template    | 9 +++++++++
 upup/pkg/fi/cloudup/bootstrapchannelbuilder.go           | 2 +-
 .../tests/bootstrapchannelbuilder/cilium/manifest.yaml   | 8 ++++----
 5 files changed, 27 insertions(+), 5 deletions(-)

diff --git a/pkg/apis/kops/validation/legacy.go b/pkg/apis/kops/validation/legacy.go
index 53e05e3b010e3..2e1ea2abb035d 100644
--- a/pkg/apis/kops/validation/legacy.go
+++ b/pkg/apis/kops/validation/legacy.go
@@ -609,6 +609,10 @@ func ValidateCluster(c *kops.Cluster, strict bool) *field.Error {
 		return field.Invalid(fieldSpec.Child("Networking"), "amazon-vpc-routed-eni", "amazon-vpc-routed-eni networking is supported only in AWS")
 	}
 
+	if c.Spec.Networking.Cilium != nil && c.Spec.Networking.Cilium.EnableNodePort && *c.Spec.KubeProxy.Enabled {
+		return field.Invalid(fieldSpec.Child("KubeProxy"), "enabled", "When Cilium NodePort is enabled, KubeProxy must be disabled")
+	}
+
 	if errs := newValidateCluster(c); len(errs) != 0 {
 		return errs[0]
 	}
diff --git a/upup/models/cloudup/resources/addons/networking.cilium.io/k8s-1.12.yaml.template b/upup/models/cloudup/resources/addons/networking.cilium.io/k8s-1.12.yaml.template
index c0d3d3667e1fb..fc18c6349ad78 100644
--- a/upup/models/cloudup/resources/addons/networking.cilium.io/k8s-1.12.yaml.template
+++ b/upup/models/cloudup/resources/addons/networking.cilium.io/k8s-1.12.yaml.template
@@ -363,6 +363,10 @@ spec:
               key: custom-cni-conf
               name: cilium-config
               optional: true
+        - name: KUBERNETES_SERVICE_HOST
+          value: "{{.MasterInternalName}}"
+        - name: KUBERNETES_SERVICE_PORT
+          value: "443"
         {{ with .Networking.Cilium.EnablePolicy }}
         - name: CILIUM_ENABLE_POLICY
           value: {{ . }}
@@ -642,6 +646,11 @@ spec:
               key: AWS_DEFAULT_REGION
               name: cilium-aws
               optional: true
+        - name: KUBERNETES_SERVICE_HOST
+          value: "{{.MasterInternalName}}"
+        - name: KUBERNETES_SERVICE_PORT
+          value: "443"
+{{ with .Networking.Cilium }}
         image: "docker.io/cilium/operator:{{ .Version }}"
         imagePullPolicy: IfNotPresent
         name: cilium-operator
diff --git a/upup/models/cloudup/resources/addons/networking.cilium.io/k8s-1.7.yaml.template b/upup/models/cloudup/resources/addons/networking.cilium.io/k8s-1.7.yaml.template
index aa8b382961509..4f0278e57be54 100644
--- a/upup/models/cloudup/resources/addons/networking.cilium.io/k8s-1.7.yaml.template
+++ b/upup/models/cloudup/resources/addons/networking.cilium.io/k8s-1.7.yaml.template
@@ -363,6 +363,10 @@ spec:
               key: custom-cni-conf
               name: cilium-config
               optional: true
+        - name: KUBERNETES_SERVICE_HOST
+          value: "{{ .MasterInternalName }}"
+        - name: KUBERNETES_SERVICE_PORT
+          value: "443"
         {{ with .Networking.Cilium.EnablePolicy }}
         - name: CILIUM_ENABLE_POLICY
           value: {{ . }}
@@ -634,6 +638,11 @@ spec:
               key: AWS_DEFAULT_REGION
               name: cilium-aws
               optional: true
+        - name: KUBERNETES_SERVICE_HOST
+          value: "{{ .MasterInternalName }}"
+        - name: KUBERNETES_SERVICE_PORT
+          value: "443"
+{{ with .Networking.Cilium }}
         image: "docker.io/cilium/operator:{{ .Version }}"
         imagePullPolicy: IfNotPresent
         name: cilium-operator
diff --git a/upup/pkg/fi/cloudup/bootstrapchannelbuilder.go b/upup/pkg/fi/cloudup/bootstrapchannelbuilder.go
index 3bcd0eafbd186..cddd74c333570 100644
--- a/upup/pkg/fi/cloudup/bootstrapchannelbuilder.go
+++ b/upup/pkg/fi/cloudup/bootstrapchannelbuilder.go
@@ -909,7 +909,7 @@ func (b *BootstrapChannelBuilder) buildAddons() *channelsapi.Addons {
 
 	if b.cluster.Spec.Networking.Cilium != nil {
 		key := "networking.cilium.io"
-		version := "1.6.4-kops.3"
+		version := "1.6.6-kops.0"
 
 		{
 			id := "k8s-1.7"
diff --git a/upup/pkg/fi/cloudup/tests/bootstrapchannelbuilder/cilium/manifest.yaml b/upup/pkg/fi/cloudup/tests/bootstrapchannelbuilder/cilium/manifest.yaml
index 73c63fb2025de..430dc0a2c91df 100644
--- a/upup/pkg/fi/cloudup/tests/bootstrapchannelbuilder/cilium/manifest.yaml
+++ b/upup/pkg/fi/cloudup/tests/bootstrapchannelbuilder/cilium/manifest.yaml
@@ -89,16 +89,16 @@ spec:
   - id: k8s-1.7
     kubernetesVersion: <1.12.0
     manifest: networking.cilium.io/k8s-1.7.yaml
-    manifestHash: 6928e95ec4b8359075e3dfb069f74e290e2e6eb2
+   manifestHash: 870d0a940ece8e98b38b3e8a20c062fc247e9b23
     name: networking.cilium.io
     selector:
       role.kubernetes.io/networking: "1"
-    version: 1.6.4-kops.3
+    version: 1.6.5-kops.1
   - id: k8s-1.12
     kubernetesVersion: '>=1.12.0'
     manifest: networking.cilium.io/k8s-1.12.yaml
-    manifestHash: 84295d293c8a461f7d510721c48b969cd1d99e54
+    manifestHash: 870d0a940ece8e98b38b3e8a20c062fc247e9b23
     name: networking.cilium.io
     selector:
       role.kubernetes.io/networking: "1"
-    version: 1.6.4-kops.3
+    version: 1.6.5-kops.1