diff --git a/pkg/dnshelpers/util.go b/pkg/dnshelpers/util.go new file mode 100644 index 000000000..dbfac9e71 --- /dev/null +++ b/pkg/dnshelpers/util.go @@ -0,0 +1,185 @@ +package dnshelpers + +import ( + "fmt" + "net" + "strings" + + configv1 "github.com/openshift/api/config/v1" + + "k8s.io/apimachinery/pkg/util/sets" + + corev1 "k8s.io/api/core/v1" + + utilerrors "k8s.io/apimachinery/pkg/util/errors" + + "k8s.io/klog" +) + +// GetEscapedPreferredInternalIPAddressForNodeName returns the first internal ip address of the correct family with escaping +// for ipv6. +func GetEscapedPreferredInternalIPAddressForNodeName(network *configv1.Network, node *corev1.Node) (string, error) { + address, family, err := GetPreferredInternalIPAddressForNodeName(network, node) + if err != nil { + return "", err + } + if family == "tcp6" { + return "[" + address + "]", nil + } else { + return address, nil + } +} + +// GetPreferredInternalIPAddressForNodeName returns the first internal ip address of the correct family and the family +func GetPreferredInternalIPAddressForNodeName(network *configv1.Network, node *corev1.Node) (string, string, error) { + ipFamily, err := GetPreferredIPFamily(network) + if err != nil { + return "", "", err + } + + for _, currAddress := range node.Status.Addresses { + if currAddress.Type == corev1.NodeInternalIP { + switch ipFamily { + case "tcp4": + isIPv4, err := IsIPv4(currAddress.Address) + if err != nil { + return "", "", err + } + if isIPv4 { + return currAddress.Address, ipFamily, nil + } + case "tcp6": + isIPv4, err := IsIPv4(currAddress.Address) + if err != nil { + return "", "", err + } + if !isIPv4 { + return currAddress.Address, ipFamily, nil + } + default: + return "", "", fmt.Errorf("unexpected ip family: %q", ipFamily) + } + } + } + + return "", "", fmt.Errorf("no matches found for ip family %q for node %q", ipFamily, node.Name) +} + +func GetPreferredIPFamily(network *configv1.Network) (string, error) { + if len(network.Status.ServiceNetwork) == 0 || len(network.Status.ServiceNetwork[0]) == 0 { + return "", fmt.Errorf("networks.%s/cluster: status.serviceNetwork not found", configv1.GroupName) + } + + serviceCIDR := network.Status.ServiceNetwork[0] + if len(serviceCIDR) == 0 { + return "", fmt.Errorf("networks.%s/cluster: status.serviceNetwork[0] is empty", configv1.GroupName) + } + ip, _, err := net.ParseCIDR(serviceCIDR) + + switch { + case err != nil: + return "", err + case ip.To4() == nil: + return "tcp6", nil + default: + return "tcp4", nil + } +} + +func IsIPv4(ipString string) (bool, error) { + ip := net.ParseIP(ipString) + + switch { + case ip == nil: + return false, fmt.Errorf("not an IP") + case ip.To4() == nil: + return false, nil + default: + return true, nil + } +} + +func GetInternalIPAddressesForNodeName(node *corev1.Node) ([]string, error) { + addresses := []string{} + for _, currAddress := range node.Status.Addresses { + if currAddress.Type == corev1.NodeInternalIP { + addresses = append(addresses, currAddress.Address) + } + } + if len(addresses) == 0 { + return nil, fmt.Errorf("node/%s missing %s", node.Name, corev1.NodeInternalIP) + } + + return addresses, nil +} + +func ReverseLookupFirstHit(discoveryDomain string, ips ...string) (string, error) { + errs := []error{} + for _, ip := range ips { + ret, err := reverseLookupForOneIP(discoveryDomain, ip) + if err == nil { + return ret, nil + } + errs = append(errs, err) + } + + if len(errs) == 0 { + return "", fmt.Errorf("something weird happened for %q, %#v", discoveryDomain, ips) + } + return "", utilerrors.NewAggregate(errs) +} + +func ReverseLookupAllHits(discoveryDomain string, ips ...string) ([]string, error) { + ret := []string{} + found := sets.NewString() + errs := []error{} + for _, ip := range ips { + curr, err := reverseLookupForOneIP(discoveryDomain, ip) + if err != nil { + errs = append(errs, err) + } else if !found.Has(curr) { + ret = append(ret, curr) + found.Insert(curr) + } + } + + switch { + case len(ret) > 0: + //ignore errors + return ret, nil + case len(errs) == 0: + // we got no result and no error + return nil, fmt.Errorf("something weird happened for %q, %#v", discoveryDomain, ips) + default: + // we got errors + return nil, utilerrors.NewAggregate(errs) + } +} +func reverseLookupForOneIP(discoveryDomain, ipAddress string) (string, error) { + service := "etcd-server-ssl" + proto := "tcp" + + _, srvs, err := net.LookupSRV(service, proto, discoveryDomain) + if err != nil { + return "", err + } + selfTarget := "" + for _, srv := range srvs { + klog.V(4).Infof("checking against %s", srv.Target) + addrs, err := net.LookupHost(srv.Target) + if err != nil { + return "", fmt.Errorf("could not resolve member %q", srv.Target) + } + + for _, addr := range addrs { + if addr == ipAddress { + selfTarget = strings.Trim(srv.Target, ".") + break + } + } + } + if selfTarget == "" { + return "", fmt.Errorf("could not find self") + } + return selfTarget, nil +} diff --git a/pkg/etcdcli/etcdcli.go b/pkg/etcdcli/etcdcli.go index 798364b89..88daf5465 100644 --- a/pkg/etcdcli/etcdcli.go +++ b/pkg/etcdcli/etcdcli.go @@ -5,13 +5,15 @@ import ( "fmt" "time" + configv1informers "github.com/openshift/client-go/config/informers/externalversions/config/v1" + configv1listers "github.com/openshift/client-go/config/listers/config/v1" + "github.com/openshift/cluster-etcd-operator/pkg/dnshelpers" "github.com/openshift/cluster-etcd-operator/pkg/operator/operatorclient" "github.com/openshift/library-go/pkg/operator/v1helpers" "go.etcd.io/etcd/clientv3" "go.etcd.io/etcd/etcdserver/etcdserverpb" "go.etcd.io/etcd/pkg/transport" "google.golang.org/grpc" - corev1 "k8s.io/api/core/v1" apierrors "k8s.io/apimachinery/pkg/api/errors" "k8s.io/apimachinery/pkg/labels" "k8s.io/apimachinery/pkg/runtime/schema" @@ -23,17 +25,21 @@ import ( type etcdClientGetter struct { nodeLister corev1listers.NodeLister endpointsLister corev1listers.EndpointsLister + networkLister configv1listers.NetworkLister nodeListerSynced cache.InformerSynced endpointsListerSynced cache.InformerSynced + networkListerSynced cache.InformerSynced } -func NewEtcdClient(kubeInformers v1helpers.KubeInformersForNamespaces) EtcdClient { +func NewEtcdClient(kubeInformers v1helpers.KubeInformersForNamespaces, networkInformer configv1informers.NetworkInformer) EtcdClient { return &etcdClientGetter{ nodeLister: kubeInformers.InformersFor("").Core().V1().Nodes().Lister(), endpointsLister: kubeInformers.InformersFor(operatorclient.TargetNamespace).Core().V1().Endpoints().Lister(), + networkLister: networkInformer.Lister(), nodeListerSynced: kubeInformers.InformersFor("").Core().V1().Nodes().Informer().HasSynced, endpointsListerSynced: kubeInformers.InformersFor(operatorclient.TargetNamespace).Core().V1().Endpoints().Informer().HasSynced, + networkListerSynced: networkInformer.Informer().HasSynced, } } @@ -44,11 +50,19 @@ func (g *etcdClientGetter) getEtcdClient() (*clientv3.Client, error) { if !g.endpointsListerSynced() { return nil, fmt.Errorf("node lister not synced") } + if !g.networkListerSynced() { + return nil, fmt.Errorf("network lister not synced") + } + + network, err := g.networkLister.Get("cluster") + if err != nil { + return nil, err + } etcdEndpoints := []string{} nodes, err := g.nodeLister.List(labels.Set{"node-role.kubernetes.io/master": ""}.AsSelector()) for _, node := range nodes { - internalIP, err := getInternalIPAddressForNodeName(node) + internalIP, err := dnshelpers.GetEscapedPreferredInternalIPAddressForNodeName(network, node) if err != nil { return nil, err } @@ -100,15 +114,6 @@ func getEtcdClient(endpoints []string) (*clientv3.Client, error) { return cli, err } -func getInternalIPAddressForNodeName(node *corev1.Node) (string, error) { - for _, currAddress := range node.Status.Addresses { - if currAddress.Type == corev1.NodeInternalIP { - return currAddress.Address, nil - } - } - return "", fmt.Errorf("node/%s missing %s", node.Name, corev1.NodeInternalIP) -} - func (g *etcdClientGetter) MemberAdd(peerURL string) error { cli, err := g.getEtcdClient() if err != nil { diff --git a/pkg/operator/clustermembercontroller/clustermembercontroller.go b/pkg/operator/clustermembercontroller/clustermembercontroller.go index 6c7a167e4..e392559c5 100644 --- a/pkg/operator/clustermembercontroller/clustermembercontroller.go +++ b/pkg/operator/clustermembercontroller/clustermembercontroller.go @@ -9,6 +9,7 @@ import ( operatorv1 "github.com/openshift/api/operator/v1" configv1informers "github.com/openshift/client-go/config/informers/externalversions/config/v1" configv1listers "github.com/openshift/client-go/config/listers/config/v1" + "github.com/openshift/cluster-etcd-operator/pkg/dnshelpers" "github.com/openshift/cluster-etcd-operator/pkg/etcdcli" "github.com/openshift/library-go/pkg/operator/events" "github.com/openshift/library-go/pkg/operator/v1helpers" @@ -263,11 +264,11 @@ func (c *ClusterMemberController) getValidPodFQDNToScale(podToAdd *corev1.Pod) ( if podToAdd.Spec.NodeName == "" { return "", fmt.Errorf("node name empty for %s", podToAdd.Name) } - nodeInternalIP, err := c.getNodeInternalIP(podToAdd.Spec.NodeName) + nodeInternalIPs, err := c.getNodeInternalIPs(podToAdd.Spec.NodeName) if err != nil { return "", err } - podFQDN, err := ReverseLookupSelf("etcd-server-ssl", "tcp", etcdDiscoveryDomain, nodeInternalIP) + podFQDN, err := dnshelpers.ReverseLookupFirstHit(etcdDiscoveryDomain, nodeInternalIPs...) if err != nil { return "", err } @@ -275,19 +276,10 @@ func (c *ClusterMemberController) getValidPodFQDNToScale(podToAdd *corev1.Pod) ( return podFQDN, nil } -func (c *ClusterMemberController) getNodeInternalIP(nodeName string) (string, error) { +func (c *ClusterMemberController) getNodeInternalIPs(nodeName string) ([]string, error) { node, err := c.nodeLister.Get(nodeName) if err != nil { - return "", err - } - if node.Status.Addresses == nil { - return "", fmt.Errorf("cannot get node IP address, addresses for node %s is nil", nodeName) - } - - for _, addr := range node.Status.Addresses { - if addr.Type == corev1.NodeInternalIP { - return addr.Address, nil - } + return nil, err } - return "", fmt.Errorf("unable to get internal IP address for node %s", nodeName) + return dnshelpers.GetInternalIPAddressesForNodeName(node) } diff --git a/pkg/operator/clustermembercontroller/util.go b/pkg/operator/clustermembercontroller/util.go deleted file mode 100644 index 0f5f0feb2..000000000 --- a/pkg/operator/clustermembercontroller/util.go +++ /dev/null @@ -1,35 +0,0 @@ -package clustermembercontroller - -import ( - "fmt" - "net" - "strings" - - "k8s.io/klog" -) - -func ReverseLookupSelf(service, proto, name, self string) (string, error) { - _, srvs, err := net.LookupSRV(service, proto, name) - if err != nil { - return "", err - } - selfTarget := "" - for _, srv := range srvs { - klog.V(4).Infof("checking against %s", srv.Target) - addrs, err := net.LookupHost(srv.Target) - if err != nil { - return "", fmt.Errorf("could not resolve member %q", srv.Target) - } - - for _, addr := range addrs { - if addr == self { - selfTarget = strings.Trim(srv.Target, ".") - break - } - } - } - if selfTarget == "" { - return "", fmt.Errorf("could not find self") - } - return selfTarget, nil -} diff --git a/pkg/operator/etcdcertsigner/etcdcertsignercontroller.go b/pkg/operator/etcdcertsigner/etcdcertsignercontroller.go index b8fcdce85..3e128fde5 100644 --- a/pkg/operator/etcdcertsigner/etcdcertsignercontroller.go +++ b/pkg/operator/etcdcertsigner/etcdcertsignercontroller.go @@ -6,13 +6,13 @@ import ( "crypto/x509/pkix" "errors" "fmt" - "net" "strings" "time" operatorv1 "github.com/openshift/api/operator/v1" configv1informers "github.com/openshift/client-go/config/informers/externalversions/config/v1" configv1listers "github.com/openshift/client-go/config/listers/config/v1" + "github.com/openshift/cluster-etcd-operator/pkg/dnshelpers" "github.com/openshift/cluster-etcd-operator/pkg/operator/operatorclient" "github.com/openshift/library-go/pkg/crypto" "github.com/openshift/library-go/pkg/operator/events" @@ -292,26 +292,25 @@ func (c *EtcdCertSignerController) createSecretForNode(node *corev1.Node) error if err != nil { return err } - nodeInternalIP, err := getInternalIPAddressForNodeName(node) + nodeInternalIPs, err := dnshelpers.GetInternalIPAddressesForNodeName(node) if err != nil { return err } - // since we use hostNetwork, this matches - podFQDN, err := reverseLookup("etcd-server-ssl", "tcp", etcdDiscoveryDomain, nodeInternalIP) + // the nodeInternalIPs should never have conflicting values. + podFQDN, err := dnshelpers.ReverseLookupFirstHit(etcdDiscoveryDomain, nodeInternalIPs...) if err != nil { return err } - peerHostNames := []string{"localhost", podFQDN, etcdDiscoveryDomain, nodeInternalIP} - serverHostNames := []string{ + peerHostNames := append([]string{"localhost", podFQDN, etcdDiscoveryDomain}, nodeInternalIPs...) + serverHostNames := append([]string{ "localhost", "etcd.kube-system.svc", "etcd.kube-system.svc.cluster.local", "etcd.openshift-etcd.svc", "etcd.openshift-etcd.svc.cluster.local", "*." + etcdDiscoveryDomain, - nodeInternalIP, "127.0.0.1", - } + }, nodeInternalIPs...) // create the certificates and update them in the API pCert, pKey, err := createNewCombinedClientAndServingCerts(etcdCASecret.Data["tls.crt"], etcdCASecret.Data["tls.key"], podFQDN, peerOrg, peerHostNames) @@ -333,15 +332,6 @@ func (c *EtcdCertSignerController) createSecretForNode(node *corev1.Node) error return nil } -func getInternalIPAddressForNodeName(node *corev1.Node) (string, error) { - for _, currAddress := range node.Status.Addresses { - if currAddress.Type == corev1.NodeInternalIP { - return currAddress.Address, nil - } - } - return "", fmt.Errorf("node/%s missing %s", node.Name, corev1.NodeInternalIP) -} - func createNewCombinedClientAndServingCerts(caCert, caKey []byte, podFQDN, org string, peerHostNames []string) (*bytes.Buffer, *bytes.Buffer, error) { cn, err := getCommonNameFromOrg(org) etcdCAKeyPair, err := crypto.GetCAFromBytes(caCert, caKey) @@ -410,33 +400,6 @@ func getCommonNameFromOrg(org string) (string, error) { return "", errors.New("unable to recognise secret name") } -// returns the target from the SRV record that resolves to ip. -func reverseLookup(service, proto, name, ip string) (string, error) { - _, srvs, err := net.LookupSRV(service, proto, name) - if err != nil { - return "", err - } - selfTarget := "" - for _, srv := range srvs { - klog.V(4).Infof("checking against %s", srv.Target) - addrs, err := net.LookupHost(srv.Target) - if err != nil { - return "", fmt.Errorf("could not resolve member %q", srv.Target) - } - - for _, addr := range addrs { - if addr == ip { - selfTarget = strings.Trim(srv.Target, ".") - break - } - } - } - if selfTarget == "" { - return "", fmt.Errorf("could not find self") - } - return selfTarget, nil -} - func (c *EtcdCertSignerController) getEtcdDiscoveryDomain() (string, error) { infrastructure, err := c.infrastructureLister.Get("cluster") if err != nil { diff --git a/pkg/operator/hostendpointscontroller/host_endpoints_controller.go b/pkg/operator/hostendpointscontroller/host_endpoints_controller.go index 28abfcf08..c8f06383d 100644 --- a/pkg/operator/hostendpointscontroller/host_endpoints_controller.go +++ b/pkg/operator/hostendpointscontroller/host_endpoints_controller.go @@ -9,6 +9,8 @@ import ( "strings" "time" + "github.com/openshift/cluster-etcd-operator/pkg/dnshelpers" + operatorv1 "github.com/openshift/api/operator/v1" configv1informers "github.com/openshift/client-go/config/informers/externalversions/config/v1" configv1listers "github.com/openshift/client-go/config/listers/config/v1" @@ -44,6 +46,7 @@ const ( type HostEndpointsController struct { operatorClient v1helpers.OperatorClient infrastructureLister configv1listers.InfrastructureLister + networkLister configv1listers.NetworkLister nodeLister corev1listers.NodeLister endpointsLister corev1listers.EndpointsLister endpointsClient corev1client.EndpointsGetter @@ -59,6 +62,7 @@ func NewHostEndpointsController( kubeClient kubernetes.Interface, kubeInformers operatorv1helpers.KubeInformersForNamespaces, infrastructureInformer configv1informers.InfrastructureInformer, + networkInformer configv1informers.NetworkInformer, ) *HostEndpointsController { kubeInformersForTargetNamespace := kubeInformers.InformersFor(operatorclient.TargetNamespace) endpointsInformer := kubeInformersForTargetNamespace.Core().V1().Endpoints() @@ -73,9 +77,11 @@ func NewHostEndpointsController( endpointsInformer.Informer().HasSynced, nodeInformer.Informer().HasSynced, infrastructureInformer.Informer().HasSynced, + networkInformer.Informer().HasSynced, }, operatorClient: operatorClient, infrastructureLister: infrastructureInformer.Lister(), + networkLister: networkInformer.Lister(), nodeLister: nodeInformer.Lister(), endpointsLister: endpointsInformer.Lister(), endpointsClient: kubeClient.CoreV1(), @@ -83,6 +89,7 @@ func NewHostEndpointsController( operatorClient.Informer().AddEventHandler(c.eventHandler()) endpointsInformer.Informer().AddEventHandler(c.eventHandler()) infrastructureInformer.Informer().AddEventHandler(c.eventHandler()) + networkInformer.Informer().AddEventHandler(c.eventHandler()) nodeInformer.Informer().AddEventHandler(c.eventHandler()) return c } @@ -135,18 +142,20 @@ func (c *HostEndpointsController) syncHostEndpoints() error { required.Annotations["alpha.installer.openshift.io/dns-suffix"] = discoveryDomain // create endpoint addresses for each node + network, err := c.networkLister.Get("cluster") + if err != nil { + return err + } + nodes, err := c.nodeLister.List(labels.Set{"node-role.kubernetes.io/master": ""}.AsSelector()) if err != nil { return fmt.Errorf("unable to list expected etcd member nodes: %v", err) } endpointAddresses := []corev1.EndpointAddress{} for _, node := range nodes { - var nodeInternalIP string - for _, nodeAddress := range node.Status.Addresses { - if nodeAddress.Type == corev1.NodeInternalIP { - nodeInternalIP = nodeAddress.Address - break - } + nodeInternalIP, _, err := dnshelpers.GetPreferredInternalIPAddressForNodeName(network, node) + if err != nil { + return err } if len(nodeInternalIP) == 0 { return fmt.Errorf("unable to determine internal ip address for node %s", node.Name) diff --git a/pkg/operator/starter.go b/pkg/operator/starter.go index 737b854c9..ad1a59273 100644 --- a/pkg/operator/starter.go +++ b/pkg/operator/starter.go @@ -73,7 +73,7 @@ func RunOperator(ctx context.Context, controllerContext *controllercmd.Controlle if err != nil { return err } - etcdClient := etcdcli.NewEtcdClient(kubeInformersForNamespaces) + etcdClient := etcdcli.NewEtcdClient(kubeInformersForNamespaces, configInformers.Config().V1().Networks()) resourceSyncController, err := resourcesynccontroller.NewResourceSyncController( operatorClient, @@ -113,6 +113,7 @@ func RunOperator(ctx context.Context, controllerContext *controllercmd.Controlle kubeInformersForNamespaces.InformersFor("openshift-etcd"), kubeInformersForNamespaces, configInformers.Config().V1().Infrastructures(), + configInformers.Config().V1().Networks(), kubeClient, controllerContext.EventRecorder, ) @@ -170,6 +171,7 @@ func RunOperator(ctx context.Context, controllerContext *controllercmd.Controlle coreClient, kubeInformersForNamespaces, configInformers.Config().V1().Infrastructures(), + configInformers.Config().V1().Networks(), ) clusterMemberController := clustermembercontroller.NewClusterMemberController( diff --git a/pkg/operator/targetconfigcontroller/etcd_env.go b/pkg/operator/targetconfigcontroller/etcd_env.go index 4977b6273..7dacea6c3 100644 --- a/pkg/operator/targetconfigcontroller/etcd_env.go +++ b/pkg/operator/targetconfigcontroller/etcd_env.go @@ -2,16 +2,15 @@ package targetconfigcontroller import ( "fmt" - "net" "strings" + "github.com/openshift/cluster-etcd-operator/pkg/dnshelpers" + "github.com/openshift/cluster-etcd-operator/pkg/operator/operatorclient" operatorv1 "github.com/openshift/api/operator/v1" configv1listers "github.com/openshift/client-go/config/listers/config/v1" - corev1 "k8s.io/api/core/v1" corev1listers "k8s.io/client-go/listers/core/v1" - "k8s.io/klog" ) type envVarContext struct { @@ -20,6 +19,7 @@ type envVarContext struct { nodeLister corev1listers.NodeLister infrastructureLister configv1listers.InfrastructureLister + networkLister configv1listers.NetworkLister endpointLister corev1listers.EndpointsLister } @@ -78,15 +78,25 @@ func getFixedEtcdEnvVars(envVarContext envVarContext) (map[string]string, error) } func getAllClusterMembers(envVarContext envVarContext) (map[string]string, error) { + network, err := envVarContext.networkLister.Get("cluster") + if err != nil { + return nil, err + } + ret := map[string]string{} endpoints := []string{} for _, nodeInfo := range envVarContext.status.NodeStatuses { - endpoint, err := getInternalIPAddressForNodeName(envVarContext, nodeInfo.NodeName) + node, err := envVarContext.nodeLister.Get(nodeInfo.NodeName) if err != nil { return nil, err } - endpoints = append(endpoints, fmt.Sprintf("https://%s:2379", endpoint)) + + endpointIP, err := dnshelpers.GetEscapedPreferredInternalIPAddressForNodeName(network, node) + if err != nil { + return nil, err + } + endpoints = append(endpoints, fmt.Sprintf("https://%s:2379", endpointIP)) } hostEtcdEndpoints, err := envVarContext.endpointLister.Endpoints(operatorclient.TargetNamespace).Get("host-etcd") @@ -95,7 +105,17 @@ func getAllClusterMembers(envVarContext envVarContext) (map[string]string, error } for _, endpointAddress := range hostEtcdEndpoints.Subsets[0].Addresses { if endpointAddress.Hostname == "etcd-bootstrap" { - endpoints = append(endpoints, "https://"+endpointAddress.IP+":2379") + + isIPV4, err := dnshelpers.IsIPv4(endpointAddress.IP) + if err != nil { + return nil, err + } + if isIPV4 { + endpoints = append(endpoints, "https://"+endpointAddress.IP+":2379") + } else { + endpoints = append(endpoints, "https://["+endpointAddress.IP+"]:2379") + } + break } } @@ -115,30 +135,26 @@ func getEtcdName(envVarContext envVarContext) (map[string]string, error) { } func getEscapedIPAddress(envVarContext envVarContext) (map[string]string, error) { + network, err := envVarContext.networkLister.Get("cluster") + if err != nil { + return nil, err + } + ret := map[string]string{} for _, nodeInfo := range envVarContext.status.NodeStatuses { - address, err := getInternalIPAddressForNodeName(envVarContext, nodeInfo.NodeName) + node, err := envVarContext.nodeLister.Get(nodeInfo.NodeName) if err != nil { return nil, err } - ret[fmt.Sprintf("NODE_%s_IP", envVarSafe(nodeInfo.NodeName))] = address - } - return ret, nil -} - -func getInternalIPAddressForNodeName(envVarContext envVarContext, nodeName string) (string, error) { - node, err := envVarContext.nodeLister.Get(nodeName) - if err != nil { - return "", err - } - - for _, currAddress := range node.Status.Addresses { - if currAddress.Type == corev1.NodeInternalIP { - return currAddress.Address, nil + escapedIPAddress, err := dnshelpers.GetEscapedPreferredInternalIPAddressForNodeName(network, node) + if err != nil { + return nil, err } + ret[fmt.Sprintf("NODE_%s_IP", envVarSafe(nodeInfo.NodeName))] = "[" + escapedIPAddress + "]" } - return "", fmt.Errorf("node/%s missing %s", node.Name, corev1.NodeInternalIP) + + return ret, nil } func getDNSName(envVarContext envVarContext) (map[string]string, error) { @@ -155,46 +171,24 @@ func getDNSName(envVarContext envVarContext) (map[string]string, error) { } for _, nodeInfo := range envVarContext.status.NodeStatuses { - ip, err := getInternalIPAddressForNodeName(envVarContext, nodeInfo.NodeName) + node, err := envVarContext.nodeLister.Get(nodeInfo.NodeName) if err != nil { return nil, err } - dnsName, err := reverseLookup("etcd-server-ssl", "tcp", etcdDiscoveryDomain, ip) + ips, err := dnshelpers.GetInternalIPAddressesForNodeName(node) if err != nil { return nil, err } - ret[fmt.Sprintf("NODE_%s_ETCD_DNS_NAME", envVarSafe(nodeInfo.NodeName))] = dnsName - } - - return ret, nil -} -// returns the target from the SRV record that resolves to ip. -func reverseLookup(service, proto, name, ip string) (string, error) { - _, srvs, err := net.LookupSRV(service, proto, name) - if err != nil { - return "", err - } - selfTarget := "" - for _, srv := range srvs { - klog.V(4).Infof("checking against %s", srv.Target) - addrs, err := net.LookupHost(srv.Target) + dnsName, err := dnshelpers.ReverseLookupFirstHit(etcdDiscoveryDomain, ips...) if err != nil { - return "", fmt.Errorf("could not resolve member %q", srv.Target) - } - - for _, addr := range addrs { - if addr == ip { - selfTarget = strings.Trim(srv.Target, ".") - break - } + return nil, err } + ret[fmt.Sprintf("NODE_%s_ETCD_DNS_NAME", envVarSafe(nodeInfo.NodeName))] = dnsName } - if selfTarget == "" { - return "", fmt.Errorf("could not find self") - } - return selfTarget, nil + + return ret, nil } func envVarSafe(nodeName string) string { diff --git a/pkg/operator/targetconfigcontroller/targetconfigcontroller.go b/pkg/operator/targetconfigcontroller/targetconfigcontroller.go index a7c65edd6..92fe088df 100644 --- a/pkg/operator/targetconfigcontroller/targetconfigcontroller.go +++ b/pkg/operator/targetconfigcontroller/targetconfigcontroller.go @@ -5,13 +5,12 @@ import ( "strings" "time" - "github.com/openshift/cluster-etcd-operator/pkg/operator/operatorclient" - - "k8s.io/apimachinery/pkg/util/sets" - operatorv1 "github.com/openshift/api/operator/v1" configv1informers "github.com/openshift/client-go/config/informers/externalversions/config/v1" configv1listers "github.com/openshift/client-go/config/listers/config/v1" + "github.com/openshift/cluster-etcd-operator/pkg/operator/etcd_assets" + "github.com/openshift/cluster-etcd-operator/pkg/operator/operatorclient" + "github.com/openshift/cluster-etcd-operator/pkg/version" "github.com/openshift/library-go/pkg/operator/events" "github.com/openshift/library-go/pkg/operator/resource/resourceapply" "github.com/openshift/library-go/pkg/operator/resource/resourcemerge" @@ -19,6 +18,7 @@ import ( "github.com/openshift/library-go/pkg/operator/v1helpers" corev1 "k8s.io/api/core/v1" utilruntime "k8s.io/apimachinery/pkg/util/runtime" + "k8s.io/apimachinery/pkg/util/sets" "k8s.io/apimachinery/pkg/util/wait" "k8s.io/client-go/informers" "k8s.io/client-go/kubernetes" @@ -27,9 +27,6 @@ import ( "k8s.io/client-go/tools/cache" "k8s.io/client-go/util/workqueue" "k8s.io/klog" - - "github.com/openshift/cluster-etcd-operator/pkg/operator/etcd_assets" - "github.com/openshift/cluster-etcd-operator/pkg/version" ) const workQueueKey = "key" @@ -42,6 +39,7 @@ type TargetConfigController struct { kubeClient kubernetes.Interface infrastructureLister configv1listers.InfrastructureLister + networkLister configv1listers.NetworkLister configMapLister corev1listers.ConfigMapLister endpointLister corev1listers.EndpointsLister nodeLister corev1listers.NodeLister @@ -58,6 +56,7 @@ func NewTargetConfigController( kubeInformersForOpenshiftEtcdNamespace informers.SharedInformerFactory, kubeInformersForNamespaces v1helpers.KubeInformersForNamespaces, infrastructureInformer configv1informers.InfrastructureInformer, + networkInformer configv1informers.NetworkInformer, kubeClient kubernetes.Interface, eventRecorder events.Recorder, ) *TargetConfigController { @@ -68,6 +67,7 @@ func NewTargetConfigController( operatorClient: operatorClient, kubeClient: kubeClient, infrastructureLister: infrastructureInformer.Lister(), + networkLister: networkInformer.Lister(), configMapLister: kubeInformersForNamespaces.ConfigMapLister(), endpointLister: kubeInformersForNamespaces.InformersFor(operatorclient.TargetNamespace).Core().V1().Endpoints().Lister(), nodeLister: kubeInformersForNamespaces.InformersFor("").Core().V1().Nodes().Lister(), @@ -81,6 +81,7 @@ func NewTargetConfigController( kubeInformersForOpenshiftEtcdNamespace.Core().V1().Secrets().Informer().HasSynced, kubeInformersForNamespaces.InformersFor("").Core().V1().Nodes().Informer().HasSynced, infrastructureInformer.Informer().HasSynced, + networkInformer.Informer().HasSynced, }, } @@ -88,6 +89,7 @@ func NewTargetConfigController( kubeInformersForOpenshiftEtcdNamespace.Core().V1().ConfigMaps().Informer().AddEventHandler(c.eventHandler()) kubeInformersForOpenshiftEtcdNamespace.Core().V1().Secrets().Informer().AddEventHandler(c.eventHandler()) infrastructureInformer.Informer().AddEventHandler(c.eventHandler()) + networkInformer.Informer().AddEventHandler(c.eventHandler()) kubeInformersForNamespaces.InformersFor(operatorclient.TargetNamespace).Core().V1().Endpoints().Informer().AddEventHandler(c.eventHandler()) // TODO only trigger on master nodes @@ -190,6 +192,7 @@ func (c *TargetConfigController) managePod(client coreclientv1.ConfigMapsGetter, endpointLister: c.endpointLister, nodeLister: c.nodeLister, infrastructureLister: c.infrastructureLister, + networkLister: c.networkLister, }) if err != nil { return nil, false, err