Skip to content
New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Jump to bottom

bug 1804913: make ipv6 support a little better #173

Merged
merged 2 commits into from
Feb 21, 2020
Merged

bug 1804913: make ipv6 support a little better #173

Show file tree
Hide file tree
Changes from 1 commit
Commits
Show all changes
2 commits
Select commit Hold shift + click to select a range
1ca6b5b
update the cert signer to including all node IP addresses
deads2k Feb 19, 2020
64c0b0a
tolerate ipv6 in all URLs and env vars
deads2k Feb 19, 2020
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
85 changes: 85 additions & 0 deletions pkg/dnshelpers/util.go
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -5,6 +5,8 @@ import (
"net"
"strings"

configv1 "github.com/openshift/api/config/v1"

"k8s.io/apimachinery/pkg/util/sets"

corev1 "k8s.io/api/core/v1"
Expand All @@ -14,6 +16,89 @@ import (
"k8s.io/klog"
)

// GetEscapedPreferredInternalIPAddressForNodeName returns the first internal ip address of the correct family with escaping
// for ipv6.
func GetEscapedPreferredInternalIPAddressForNodeName(network *configv1.Network, node *corev1.Node) (string, error) {
address, family, err := GetPreferredInternalIPAddressForNodeName(network, node)
if err != nil {
return "", err
}
if family == "tcp6" {
return "[" + address + "]", nil
} else {
return address, nil
}
}

// GetPreferredInternalIPAddressForNodeName returns the first internal ip address of the correct family and the family
func GetPreferredInternalIPAddressForNodeName(network *configv1.Network, node *corev1.Node) (string, string, error) {
ipFamily, err := GetPreferredIPFamily(network)
if err != nil {
return "", "", err
}

for _, currAddress := range node.Status.Addresses {
if currAddress.Type == corev1.NodeInternalIP {
switch ipFamily {
case "tcp4":
isIPv4, err := IsIPv4(currAddress.Address)
if err != nil {
return "", "", err
}
if isIPv4 {
return currAddress.Address, ipFamily, nil
Copy link
Contributor

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

if an IPv4 address in IPv6 format, isIPv4 will be true, but you will need the brackets.
net.IP's String() will return a canonical IP address string that will match the ipFamily type.

Copy link
Contributor

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

node.Status.Addresses should never have IPv4 addresses in IPv6 format (other things will almost definitely break if it does)

}
case "tcp6":
isIPv4, err := IsIPv4(currAddress.Address)
if err != nil {
return "", "", err
}
if !isIPv4 {
return currAddress.Address, ipFamily, nil
}
default:
return "", "", fmt.Errorf("unexpected ip family: %q", ipFamily)
}
}
}

return "", "", fmt.Errorf("no matches found for ip family %q for node %q", ipFamily, node.Name)
}

func GetPreferredIPFamily(network *configv1.Network) (string, error) {
if len(network.Status.ServiceNetwork) == 0 || len(network.Status.ServiceNetwork[0]) == 0 {
return "", fmt.Errorf("networks.%s/cluster: status.serviceNetwork not found", configv1.GroupName)
}

serviceCIDR := network.Status.ServiceNetwork[0]
if len(serviceCIDR) == 0 {
return "", fmt.Errorf("networks.%s/cluster: status.serviceNetwork[0] is empty", configv1.GroupName)
}
ip, _, err := net.ParseCIDR(serviceCIDR)

switch {
case err != nil:
return "", err
case ip.To4() == nil:
return "tcp6", nil
default:
return "tcp4", nil
}
}

func IsIPv4(ipString string) (bool, error) {
ip := net.ParseIP(ipString)

switch {
case ip == nil:
return false, fmt.Errorf("not an IP")
case ip.To4() == nil:
return false, nil
default:
return true, nil
}
}

func GetInternalIPAddressesForNodeName(node *corev1.Node) ([]string, error) {
addresses := []string{}
for _, currAddress := range node.Status.Addresses {
Expand Down
29 changes: 17 additions & 12 deletions pkg/etcdcli/etcdcli.go
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -5,13 +5,15 @@ import (
"fmt"
"time"

configv1informers "github.com/openshift/client-go/config/informers/externalversions/config/v1"
configv1listers "github.com/openshift/client-go/config/listers/config/v1"
"github.com/openshift/cluster-etcd-operator/pkg/dnshelpers"
"github.com/openshift/cluster-etcd-operator/pkg/operator/operatorclient"
"github.com/openshift/library-go/pkg/operator/v1helpers"
"go.etcd.io/etcd/clientv3"
"go.etcd.io/etcd/etcdserver/etcdserverpb"
"go.etcd.io/etcd/pkg/transport"
"google.golang.org/grpc"
corev1 "k8s.io/api/core/v1"
apierrors "k8s.io/apimachinery/pkg/api/errors"
"k8s.io/apimachinery/pkg/labels"
"k8s.io/apimachinery/pkg/runtime/schema"
Expand All @@ -23,17 +25,21 @@ import (
type etcdClientGetter struct {
nodeLister corev1listers.NodeLister
endpointsLister corev1listers.EndpointsLister
networkLister configv1listers.NetworkLister

nodeListerSynced cache.InformerSynced
endpointsListerSynced cache.InformerSynced
networkListerSynced cache.InformerSynced
}

func NewEtcdClient(kubeInformers v1helpers.KubeInformersForNamespaces) EtcdClient {
func NewEtcdClient(kubeInformers v1helpers.KubeInformersForNamespaces, networkInformer configv1informers.NetworkInformer) EtcdClient {
return &etcdClientGetter{
nodeLister: kubeInformers.InformersFor("").Core().V1().Nodes().Lister(),
endpointsLister: kubeInformers.InformersFor(operatorclient.TargetNamespace).Core().V1().Endpoints().Lister(),
networkLister: networkInformer.Lister(),
nodeListerSynced: kubeInformers.InformersFor("").Core().V1().Nodes().Informer().HasSynced,
endpointsListerSynced: kubeInformers.InformersFor(operatorclient.TargetNamespace).Core().V1().Endpoints().Informer().HasSynced,
networkListerSynced: networkInformer.Informer().HasSynced,
}
}

Expand All @@ -44,11 +50,19 @@ func (g *etcdClientGetter) getEtcdClient() (*clientv3.Client, error) {
if !g.endpointsListerSynced() {
return nil, fmt.Errorf("node lister not synced")
}
if !g.networkListerSynced() {
return nil, fmt.Errorf("network lister not synced")
}

network, err := g.networkLister.Get("cluster")
if err != nil {
return nil, err
}

etcdEndpoints := []string{}
nodes, err := g.nodeLister.List(labels.Set{"node-role.kubernetes.io/master": ""}.AsSelector())
for _, node := range nodes {
internalIP, err := getInternalIPAddressForNodeName(node)
internalIP, err := dnshelpers.GetEscapedPreferredInternalIPAddressForNodeName(network, node)
if err != nil {
return nil, err
}
Expand Down Expand Up @@ -100,15 +114,6 @@ func getEtcdClient(endpoints []string) (*clientv3.Client, error) {
return cli, err
}

func getInternalIPAddressForNodeName(node *corev1.Node) (string, error) {
for _, currAddress := range node.Status.Addresses {
if currAddress.Type == corev1.NodeInternalIP {
return currAddress.Address, nil
}
}
return "", fmt.Errorf("node/%s missing %s", node.Name, corev1.NodeInternalIP)
}

func (g *etcdClientGetter) MemberAdd(peerURL string) error {
cli, err := g.getEtcdClient()
if err != nil {
Expand Down
21 changes: 15 additions & 6 deletions pkg/operator/hostendpointscontroller/host_endpoints_controller.go
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -9,6 +9,8 @@ import (
"strings"
"time"

"github.com/openshift/cluster-etcd-operator/pkg/dnshelpers"

operatorv1 "github.com/openshift/api/operator/v1"
configv1informers "github.com/openshift/client-go/config/informers/externalversions/config/v1"
configv1listers "github.com/openshift/client-go/config/listers/config/v1"
Expand Down Expand Up @@ -44,6 +46,7 @@ const (
type HostEndpointsController struct {
operatorClient v1helpers.OperatorClient
infrastructureLister configv1listers.InfrastructureLister
networkLister configv1listers.NetworkLister
nodeLister corev1listers.NodeLister
endpointsLister corev1listers.EndpointsLister
endpointsClient corev1client.EndpointsGetter
Expand All @@ -59,6 +62,7 @@ func NewHostEndpointsController(
kubeClient kubernetes.Interface,
kubeInformers operatorv1helpers.KubeInformersForNamespaces,
infrastructureInformer configv1informers.InfrastructureInformer,
networkInformer configv1informers.NetworkInformer,
) *HostEndpointsController {
kubeInformersForTargetNamespace := kubeInformers.InformersFor(operatorclient.TargetNamespace)
endpointsInformer := kubeInformersForTargetNamespace.Core().V1().Endpoints()
Expand All @@ -73,16 +77,19 @@ func NewHostEndpointsController(
endpointsInformer.Informer().HasSynced,
nodeInformer.Informer().HasSynced,
infrastructureInformer.Informer().HasSynced,
networkInformer.Informer().HasSynced,
},
operatorClient: operatorClient,
infrastructureLister: infrastructureInformer.Lister(),
networkLister: networkInformer.Lister(),
nodeLister: nodeInformer.Lister(),
endpointsLister: endpointsInformer.Lister(),
endpointsClient: kubeClient.CoreV1(),
}
operatorClient.Informer().AddEventHandler(c.eventHandler())
endpointsInformer.Informer().AddEventHandler(c.eventHandler())
infrastructureInformer.Informer().AddEventHandler(c.eventHandler())
networkInformer.Informer().AddEventHandler(c.eventHandler())
nodeInformer.Informer().AddEventHandler(c.eventHandler())
return c
}
Expand Down Expand Up @@ -135,18 +142,20 @@ func (c *HostEndpointsController) syncHostEndpoints() error {
required.Annotations["alpha.installer.openshift.io/dns-suffix"] = discoveryDomain

// create endpoint addresses for each node
network, err := c.networkLister.Get("cluster")
if err != nil {
return err
}

nodes, err := c.nodeLister.List(labels.Set{"node-role.kubernetes.io/master": ""}.AsSelector())
if err != nil {
return fmt.Errorf("unable to list expected etcd member nodes: %v", err)
}
endpointAddresses := []corev1.EndpointAddress{}
for _, node := range nodes {
var nodeInternalIP string
for _, nodeAddress := range node.Status.Addresses {
if nodeAddress.Type == corev1.NodeInternalIP {
nodeInternalIP = nodeAddress.Address
break
}
nodeInternalIP, _, err := dnshelpers.GetPreferredInternalIPAddressForNodeName(network, node)
if err != nil {
return err
}
if len(nodeInternalIP) == 0 {
return fmt.Errorf("unable to determine internal ip address for node %s", node.Name)
Expand Down
4 changes: 3 additions & 1 deletion pkg/operator/starter.go
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -73,7 +73,7 @@ func RunOperator(ctx context.Context, controllerContext *controllercmd.Controlle
if err != nil {
return err
}
etcdClient := etcdcli.NewEtcdClient(kubeInformersForNamespaces)
etcdClient := etcdcli.NewEtcdClient(kubeInformersForNamespaces, configInformers.Config().V1().Networks())

resourceSyncController, err := resourcesynccontroller.NewResourceSyncController(
operatorClient,
Expand Down Expand Up @@ -113,6 +113,7 @@ func RunOperator(ctx context.Context, controllerContext *controllercmd.Controlle
kubeInformersForNamespaces.InformersFor("openshift-etcd"),
kubeInformersForNamespaces,
configInformers.Config().V1().Infrastructures(),
configInformers.Config().V1().Networks(),
kubeClient,
controllerContext.EventRecorder,
)
Expand Down Expand Up @@ -170,6 +171,7 @@ func RunOperator(ctx context.Context, controllerContext *controllercmd.Controlle
coreClient,
kubeInformersForNamespaces,
configInformers.Config().V1().Infrastructures(),
configInformers.Config().V1().Networks(),
)

clusterMemberController := clustermembercontroller.NewClusterMemberController(
Expand Down
65 changes: 43 additions & 22 deletions pkg/operator/targetconfigcontroller/etcd_env.go
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -10,7 +10,6 @@ import (

operatorv1 "github.com/openshift/api/operator/v1"
configv1listers "github.com/openshift/client-go/config/listers/config/v1"
corev1 "k8s.io/api/core/v1"
corev1listers "k8s.io/client-go/listers/core/v1"
)

Expand All @@ -20,6 +19,7 @@ type envVarContext struct {

nodeLister corev1listers.NodeLister
infrastructureLister configv1listers.InfrastructureLister
networkLister configv1listers.NetworkLister
endpointLister corev1listers.EndpointsLister
}

Expand Down Expand Up @@ -78,15 +78,25 @@ func getFixedEtcdEnvVars(envVarContext envVarContext) (map[string]string, error)
}

func getAllClusterMembers(envVarContext envVarContext) (map[string]string, error) {
network, err := envVarContext.networkLister.Get("cluster")
if err != nil {
return nil, err
}

ret := map[string]string{}

endpoints := []string{}
for _, nodeInfo := range envVarContext.status.NodeStatuses {
endpoint, err := getInternalIPAddressForNodeName(envVarContext, nodeInfo.NodeName)
node, err := envVarContext.nodeLister.Get(nodeInfo.NodeName)
if err != nil {
return nil, err
}

endpointIP, err := dnshelpers.GetEscapedPreferredInternalIPAddressForNodeName(network, node)
if err != nil {
return nil, err
}
endpoints = append(endpoints, fmt.Sprintf("https://%s:2379", endpoint))
endpoints = append(endpoints, fmt.Sprintf("https://%s:2379", endpointIP))
}

hostEtcdEndpoints, err := envVarContext.endpointLister.Endpoints(operatorclient.TargetNamespace).Get("host-etcd")
Expand All @@ -95,7 +105,17 @@ func getAllClusterMembers(envVarContext envVarContext) (map[string]string, error
}
for _, endpointAddress := range hostEtcdEndpoints.Subsets[0].Addresses {
if endpointAddress.Hostname == "etcd-bootstrap" {
endpoints = append(endpoints, "https://"+endpointAddress.IP+":2379")

isIPV4, err := dnshelpers.IsIPv4(endpointAddress.IP)
if err != nil {
return nil, err
}
if isIPV4 {
endpoints = append(endpoints, "https://"+endpointAddress.IP+":2379")
} else {
endpoints = append(endpoints, "https://["+endpointAddress.IP+"]:2379")
}

break
}
}
Expand All @@ -115,30 +135,26 @@ func getEtcdName(envVarContext envVarContext) (map[string]string, error) {
}

func getEscapedIPAddress(envVarContext envVarContext) (map[string]string, error) {
network, err := envVarContext.networkLister.Get("cluster")
if err != nil {
return nil, err
}

ret := map[string]string{}
for _, nodeInfo := range envVarContext.status.NodeStatuses {
address, err := getInternalIPAddressForNodeName(envVarContext, nodeInfo.NodeName)
node, err := envVarContext.nodeLister.Get(nodeInfo.NodeName)
if err != nil {
return nil, err
}
ret[fmt.Sprintf("NODE_%s_IP", envVarSafe(nodeInfo.NodeName))] = address
}

return ret, nil
}

func getInternalIPAddressForNodeName(envVarContext envVarContext, nodeName string) (string, error) {
node, err := envVarContext.nodeLister.Get(nodeName)
if err != nil {
return "", err
}

for _, currAddress := range node.Status.Addresses {
if currAddress.Type == corev1.NodeInternalIP {
return currAddress.Address, nil
escapedIPAddress, err := dnshelpers.GetEscapedPreferredInternalIPAddressForNodeName(network, node)
if err != nil {
return nil, err
}
ret[fmt.Sprintf("NODE_%s_IP", envVarSafe(nodeInfo.NodeName))] = "[" + escapedIPAddress + "]"
}
return "", fmt.Errorf("node/%s missing %s", node.Name, corev1.NodeInternalIP)

return ret, nil
}

func getDNSName(envVarContext envVarContext) (map[string]string, error) {
Expand All @@ -155,12 +171,17 @@ func getDNSName(envVarContext envVarContext) (map[string]string, error) {
}

for _, nodeInfo := range envVarContext.status.NodeStatuses {
ip, err := getInternalIPAddressForNodeName(envVarContext, nodeInfo.NodeName)
node, err := envVarContext.nodeLister.Get(nodeInfo.NodeName)
if err != nil {
return nil, err
}

ips, err := dnshelpers.GetInternalIPAddressesForNodeName(node)
if err != nil {
return nil, err
}

dnsName, err := dnshelpers.ReverseLookupFirstHit(etcdDiscoveryDomain, ip)
dnsName, err := dnshelpers.ReverseLookupFirstHit(etcdDiscoveryDomain, ips...)
if err != nil {
return nil, err
}
Expand Down
Loading