Merge pull request #2504 from reactiveops/flag-watch-ingress

Automatic merge from submit-queue

Adds DNSControllerSpec and WatchIngress flag

This PR is in reference to #2496, #2468 and the issues referenced in there relating to use of the watch-ingress flag. 

This PR attempts to rectify this situation and gives users who want it, the option to turn on watch-ingress without forcing it on them. Also spits out a warning to the logs about potential side effects.

Includes notes in `docs/cluster_spec.md` to explain.

docs/cluster_spec.md

@@ -130,6 +130,19 @@ spec:
 
 Will result in the flag `--runtime-config=batch/v2alpha1=true,apps/v1alpha1=true`. Note that `kube-apiserver` accepts `true` as a value for switch-like flags.
 
+### externalDns
+
+This block contains configuration options for your `external-DNS` provider.
+The current external-DNS provider is the kops `dns-controller`, which can set up DNS records for Kubernetes resources.
+`dns-controller` is scheduled to be phased out and replaced with `external-dns`.
+
+```yaml
+spec:
+  watchIngress: true
+```
+
+Default _kops_ behavior is false. `watchIngress: true` uses the default _dns-controller_ behavior which is to watch the ingress controller for changes. Set this option at risk of interrupting Service updates in some cases.
+
 ### kubelet
 
 This block contains configurations for `kubelet`.  See https://kubernetes.io/docs/admin/kubelet/

pkg/apis/kops/cluster.go

@@ -126,6 +126,7 @@ type ClusterSpec struct {
 	Kubelet               *KubeletConfigSpec           `json:"kubelet,omitempty"`
 	MasterKubelet         *KubeletConfigSpec           `json:"masterKubelet,omitempty"`
 	CloudConfig           *CloudConfiguration          `json:"cloudConfig,omitempty"`
+	ExternalDNS           *ExternalDNSConfig           `json:"externalDns,omitempty"`
 
 	// Networking configuration
 	Networking *NetworkingSpec `json:"networking,omitempty"`
@@ -263,6 +264,10 @@ type KubeDNSConfig struct {
 	ServerIP string `json:"serverIP,omitempty"`
 }
 
+type ExternalDNSConfig struct {
+	WatchIngress *bool `json:"watchIngress,omitempty"`
+}
+
 // EtcdClusterSpec is the etcd cluster specification
 type EtcdClusterSpec struct {
 	// Name is the name of the etcd cluster (main, events etc)

pkg/apis/kops/v1alpha1/cluster.go

@@ -231,6 +231,7 @@ type ClusterSpec struct {
 	Kubelet               *KubeletConfigSpec           `json:"kubelet,omitempty"`
 	MasterKubelet         *KubeletConfigSpec           `json:"masterKubelet,omitempty"`
 	CloudConfig           *CloudConfiguration          `json:"cloudConfig,omitempty"`
+	ExternalDNS           *ExternalDNSConfig           `json:"externalDns,omitempty"`
 
 	// Networking configuration
 	Networking *NetworkingSpec `json:"networking,omitempty"`
@@ -369,6 +370,10 @@ type KubeDNSConfig struct {
 	ServerIP string `json:"serverIP,omitempty"`
 }
 
+type ExternalDNSConfig struct {
+	WatchIngress *bool `json:"watchIngress,omitempty"`
+}
+
 // EtcdClusterSpec is the etcd cluster specification
 type EtcdClusterSpec struct {
 	// Name is the name of the etcd cluster (main, events etc)

pkg/apis/kops/v1alpha1/zz_generated.conversion.go

@@ -75,6 +75,8 @@ func RegisterConversions(scheme *runtime.Scheme) error {
 		Convert_kops_EtcdMemberSpec_To_v1alpha1_EtcdMemberSpec,
 		Convert_v1alpha1_ExecContainerAction_To_kops_ExecContainerAction,
 		Convert_kops_ExecContainerAction_To_v1alpha1_ExecContainerAction,
+		Convert_v1alpha1_ExternalDNSConfig_To_kops_ExternalDNSConfig,
+		Convert_kops_ExternalDNSConfig_To_v1alpha1_ExternalDNSConfig,
 		Convert_v1alpha1_ExternalNetworkingSpec_To_kops_ExternalNetworkingSpec,
 		Convert_kops_ExternalNetworkingSpec_To_v1alpha1_ExternalNetworkingSpec,
 		Convert_v1alpha1_Federation_To_kops_Federation,
@@ -653,6 +655,15 @@ func autoConvert_v1alpha1_ClusterSpec_To_kops_ClusterSpec(in *ClusterSpec, out *
 	} else {
 		out.CloudConfig = nil
 	}
+	if in.ExternalDNS != nil {
+		in, out := &in.ExternalDNS, &out.ExternalDNS
+		*out = new(kops.ExternalDNSConfig)
+		if err := Convert_v1alpha1_ExternalDNSConfig_To_kops_ExternalDNSConfig(*in, *out, s); err != nil {
+			return err
+		}
+	} else {
+		out.ExternalDNS = nil
+	}
 	if in.Networking != nil {
 		in, out := &in.Networking, &out.Networking
 		*out = new(kops.NetworkingSpec)
@@ -867,6 +878,15 @@ func autoConvert_kops_ClusterSpec_To_v1alpha1_ClusterSpec(in *kops.ClusterSpec,
 	} else {
 		out.CloudConfig = nil
 	}
+	if in.ExternalDNS != nil {
+		in, out := &in.ExternalDNS, &out.ExternalDNS
+		*out = new(ExternalDNSConfig)
+		if err := Convert_kops_ExternalDNSConfig_To_v1alpha1_ExternalDNSConfig(*in, *out, s); err != nil {
+			return err
+		}
+	} else {
+		out.ExternalDNS = nil
+	}
 	if in.Networking != nil {
 		in, out := &in.Networking, &out.Networking
 		*out = new(NetworkingSpec)
@@ -1144,6 +1164,24 @@ func Convert_kops_ExecContainerAction_To_v1alpha1_ExecContainerAction(in *kops.E
 	return autoConvert_kops_ExecContainerAction_To_v1alpha1_ExecContainerAction(in, out, s)
 }
 
+func autoConvert_v1alpha1_ExternalDNSConfig_To_kops_ExternalDNSConfig(in *ExternalDNSConfig, out *kops.ExternalDNSConfig, s conversion.Scope) error {
+	out.WatchIngress = in.WatchIngress
+	return nil
+}
+
+func Convert_v1alpha1_ExternalDNSConfig_To_kops_ExternalDNSConfig(in *ExternalDNSConfig, out *kops.ExternalDNSConfig, s conversion.Scope) error {
+	return autoConvert_v1alpha1_ExternalDNSConfig_To_kops_ExternalDNSConfig(in, out, s)
+}
+
+func autoConvert_kops_ExternalDNSConfig_To_v1alpha1_ExternalDNSConfig(in *kops.ExternalDNSConfig, out *ExternalDNSConfig, s conversion.Scope) error {
+	out.WatchIngress = in.WatchIngress
+	return nil
+}
+
+func Convert_kops_ExternalDNSConfig_To_v1alpha1_ExternalDNSConfig(in *kops.ExternalDNSConfig, out *ExternalDNSConfig, s conversion.Scope) error {
+	return autoConvert_kops_ExternalDNSConfig_To_v1alpha1_ExternalDNSConfig(in, out, s)
+}
+
 func autoConvert_v1alpha1_ExternalNetworkingSpec_To_kops_ExternalNetworkingSpec(in *ExternalNetworkingSpec, out *kops.ExternalNetworkingSpec, s conversion.Scope) error {
 	return nil
 }

pkg/apis/kops/v1alpha2/cluster.go

@@ -153,7 +153,7 @@ type ClusterSpec struct {
 	Kubelet               *KubeletConfigSpec           `json:"kubelet,omitempty"`
 	MasterKubelet         *KubeletConfigSpec           `json:"masterKubelet,omitempty"`
 	CloudConfig           *CloudConfiguration          `json:"cloudConfig,omitempty"`
-
+	ExternalDNS           *ExternalDNSConfig           `json:"externalDns,omitempty"`
 	// Networking configuration
 	Networking *NetworkingSpec `json:"networking,omitempty"`
 	// API field controls how the API is exposed outside the cluster
@@ -285,6 +285,10 @@ type KubeDNSConfig struct {
 	ServerIP string `json:"serverIP,omitempty"`
 }
 
+type ExternalDNSConfig struct {
+	WatchIngress *bool `json:"watchIngress,omitempty"`
+}
+
 // EtcdClusterSpec is the etcd cluster specification
 type EtcdClusterSpec struct {
 	// Name is the name of the etcd cluster (main, events etc)

pkg/apis/kops/v1alpha2/zz_generated.conversion.go

@@ -79,6 +79,8 @@ func RegisterConversions(scheme *runtime.Scheme) error {
 		Convert_kops_EtcdMemberSpec_To_v1alpha2_EtcdMemberSpec,
 		Convert_v1alpha2_ExecContainerAction_To_kops_ExecContainerAction,
 		Convert_kops_ExecContainerAction_To_v1alpha2_ExecContainerAction,
+		Convert_v1alpha2_ExternalDNSConfig_To_kops_ExternalDNSConfig,
+		Convert_kops_ExternalDNSConfig_To_v1alpha2_ExternalDNSConfig,
 		Convert_v1alpha2_ExternalNetworkingSpec_To_kops_ExternalNetworkingSpec,
 		Convert_kops_ExternalNetworkingSpec_To_v1alpha2_ExternalNetworkingSpec,
 		Convert_v1alpha2_Federation_To_kops_Federation,
@@ -691,6 +693,15 @@ func autoConvert_v1alpha2_ClusterSpec_To_kops_ClusterSpec(in *ClusterSpec, out *
 	} else {
 		out.CloudConfig = nil
 	}
+	if in.ExternalDNS != nil {
+		in, out := &in.ExternalDNS, &out.ExternalDNS
+		*out = new(kops.ExternalDNSConfig)
+		if err := Convert_v1alpha2_ExternalDNSConfig_To_kops_ExternalDNSConfig(*in, *out, s); err != nil {
+			return err
+		}
+	} else {
+		out.ExternalDNS = nil
+	}
 	if in.Networking != nil {
 		in, out := &in.Networking, &out.Networking
 		*out = new(kops.NetworkingSpec)
@@ -920,6 +931,15 @@ func autoConvert_kops_ClusterSpec_To_v1alpha2_ClusterSpec(in *kops.ClusterSpec,
 	} else {
 		out.CloudConfig = nil
 	}
+	if in.ExternalDNS != nil {
+		in, out := &in.ExternalDNS, &out.ExternalDNS
+		*out = new(ExternalDNSConfig)
+		if err := Convert_kops_ExternalDNSConfig_To_v1alpha2_ExternalDNSConfig(*in, *out, s); err != nil {
+			return err
+		}
+	} else {
+		out.ExternalDNS = nil
+	}
 	if in.Networking != nil {
 		in, out := &in.Networking, &out.Networking
 		*out = new(NetworkingSpec)
@@ -1242,6 +1262,24 @@ func Convert_kops_ExecContainerAction_To_v1alpha2_ExecContainerAction(in *kops.E
 	return autoConvert_kops_ExecContainerAction_To_v1alpha2_ExecContainerAction(in, out, s)
 }
 
+func autoConvert_v1alpha2_ExternalDNSConfig_To_kops_ExternalDNSConfig(in *ExternalDNSConfig, out *kops.ExternalDNSConfig, s conversion.Scope) error {
+	out.WatchIngress = in.WatchIngress
+	return nil
+}
+
+func Convert_v1alpha2_ExternalDNSConfig_To_kops_ExternalDNSConfig(in *ExternalDNSConfig, out *kops.ExternalDNSConfig, s conversion.Scope) error {
+	return autoConvert_v1alpha2_ExternalDNSConfig_To_kops_ExternalDNSConfig(in, out, s)
+}
+
+func autoConvert_kops_ExternalDNSConfig_To_v1alpha2_ExternalDNSConfig(in *kops.ExternalDNSConfig, out *ExternalDNSConfig, s conversion.Scope) error {
+	out.WatchIngress = in.WatchIngress
+	return nil
+}
+
+func Convert_kops_ExternalDNSConfig_To_v1alpha2_ExternalDNSConfig(in *kops.ExternalDNSConfig, out *ExternalDNSConfig, s conversion.Scope) error {
+	return autoConvert_kops_ExternalDNSConfig_To_v1alpha2_ExternalDNSConfig(in, out, s)
+}
+
 func autoConvert_v1alpha2_ExternalNetworkingSpec_To_kops_ExternalNetworkingSpec(in *ExternalNetworkingSpec, out *kops.ExternalNetworkingSpec, s conversion.Scope) error {
 	return nil
 }

upup/pkg/fi/cloudup/template_functions.go

@@ -30,15 +30,22 @@ package cloudup
 import (
 	"encoding/base64"
 	"fmt"
+
+	"github.com/golang/glog"
+
 	"os"
 	"strconv"
+
 	"strings"
 	"text/template"
 
 	"k8s.io/apimachinery/pkg/util/sets"
 	"k8s.io/kops/pkg/apis/kops"
 	"k8s.io/kops/pkg/dns"
 	"k8s.io/kops/pkg/model"
+
+	"k8s.io/kops/upup/pkg/fi"
+
 	"k8s.io/kops/upup/pkg/fi/cloudup/gce"
 )
 
@@ -125,12 +132,27 @@ func (tf *TemplateFunctions) GetInstanceGroup(name string) (*kops.InstanceGroup,
 	return nil, fmt.Errorf("InstanceGroup %q not found", name)
 }
 
+// DnsControllerArgv returns the args to the DNS controller
 func (tf *TemplateFunctions) DnsControllerArgv() ([]string, error) {
 	var argv []string
 
 	argv = append(argv, "/usr/bin/dns-controller")
 
-	argv = append(argv, "--watch-ingress=false")
+	externalDns := tf.cluster.Spec.ExternalDNS
+	if externalDns == nil {
+		externalDns = &kops.ExternalDNSConfig{}
+		argv = append(argv, "--watch-ingress=false")
+		glog.Infoln("watch-ingress=false set on DNSController")
+	} else {
+		watchIngress := fi.BoolValue(externalDns.WatchIngress)
+		if watchIngress {
+			glog.Warningln("--watch-ingress=true set on DNSController. ")
+			glog.Warningln("this may cause problems with previously defined services: https://github.com/kubernetes/kops/issues/2496")
+		} else {
+			argv = append(argv, "--watch-ingress=false")
+		}
+	}
+	// argv = append(argv, "--watch-ingress=false")
 
 	switch kops.CloudProviderID(tf.cluster.Spec.CloudProvider) {
 	case kops.CloudProviderAWS: