Skip to content
New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Jump to bottom

Changes to support IPv6 addresses on nodes #268

Merged
merged 1 commit into from
Sep 1, 2021
Merged

Changes to support IPv6 addresses on nodes #268

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
1 commit
Select commit
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
16 changes: 16 additions & 0 deletions providers/gce/gce.go
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -94,6 +94,12 @@ var _ cloudprovider.Zones = (*Cloud)(nil)
var _ cloudprovider.PVLabeler = (*Cloud)(nil)
var _ cloudprovider.Clusters = (*Cloud)(nil)

type StackType string

const NetworkStackDualStack StackType = "IPV4_IPV6"
sdmodi marked this conversation as resolved.
Show resolved Hide resolved
Copy link
Member

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

is there a reference from where this constant is coming from

Copy link
Member

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

These are in the comments of the go sdk.
https://pkg.go.dev/google.golang.org/api@v0.56.0/compute/v0.alpha
(search for "IPV4_IPV6")
Unfortunately the generated code did not have the constants directly avaliable.

Copy link
Author

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

This is actually being set by the GKE cluster server. This value is going to be written to gce.conf.

const NetworkStackIPV4 StackType = "IPV4"
const NetworkStackIPV6 StackType = "IPV6"

// Cloud is an implementation of Interface, LoadBalancer and Instances for Google Compute Engine.
type Cloud struct {
// ClusterID contains functionality for getting (and initializing) the ingress-uid. Call Cloud.Initialize()
Expand Down Expand Up @@ -167,6 +173,9 @@ type Cloud struct {
s *cloud.Service

metricsCollector loadbalancerMetricsCollector
// stackType indicates whether the cluster is a single stack IPv4, single
// stack IPv6 or a dual stack cluster
stackType StackType
}

// ConfigGlobal is the in memory representation of the gce.conf config data
Expand All @@ -181,6 +190,7 @@ type ConfigGlobal struct {
NetworkProjectID string `gcfg:"network-project-id"`
NetworkName string `gcfg:"network-name"`
SubnetworkName string `gcfg:"subnetwork-name"`
StackType string `gcfg:"stack-type"`
// DEPRECATED: Do not rely on this value as it may be incorrect.
// SecondaryRangeName is the name of the secondary range to allocate IP
// aliases. The secondary range must be present on the subnetwork the
Expand Down Expand Up @@ -236,6 +246,7 @@ type CloudConfig struct {
TokenSource oauth2.TokenSource
UseMetadataServer bool
AlphaFeatureGate *AlphaFeatureGate
StackType string
}

func init() {
Expand Down Expand Up @@ -393,6 +404,10 @@ func generateCloudConfig(configFile *ConfigFile) (cloudConfig *CloudConfig, err
cloudConfig.SecondaryRangeName = configFile.Global.SecondaryRangeName
}

if configFile != nil {
cloudConfig.StackType = configFile.Global.StackType
Copy link
Member

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

What would be the default value for this StackType field? Is empty the same as IPV4?

Copy link
Author

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

We should treat empty the same as IPV4

Copy link
Member

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

Yep seems good. Was thinking what would happen when using this with old clusters.

}

return cloudConfig, err
}

Expand Down Expand Up @@ -525,6 +540,7 @@ func CreateGCECloud(config *CloudConfig) (*Cloud, error) {
AlphaFeatureGate: config.AlphaFeatureGate,
nodeZones: map[string]sets.String{},
metricsCollector: newLoadBalancerMetrics(),
stackType: StackType(config.StackType),
}

gce.manager = &gceServiceManager{gce}
Expand Down
69 changes: 59 additions & 10 deletions providers/gce/gce_instances.go
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -28,6 +28,7 @@ import (
"time"

"cloud.google.com/go/compute/metadata"
computealpha "google.golang.org/api/compute/v0.alpha"
computebeta "google.golang.org/api/compute/v0.beta"
compute "google.golang.org/api/compute/v1"
"k8s.io/klog/v2"
Expand All @@ -45,6 +46,7 @@ import (
const (
defaultZone = ""
networkInterfaceIP = "instance/network-interfaces/%s/ip"
networkInterfaceIPV6 = "instance/network-interfaces/%s/ipv6s"
networkInterfaceAccessConfigs = "instance/network-interfaces/%s/access-configs"
networkInterfaceExternalIP = "instance/network-interfaces/%s/access-configs/%s/external-ip"
)
Expand Down Expand Up @@ -117,6 +119,27 @@ func (g *Cloud) NodeAddresses(ctx context.Context, nodeName types.NodeName) ([]v
}
nodeAddresses = append(nodeAddresses, v1.NodeAddress{Type: v1.NodeInternalIP, Address: internalIP})

if g.stackType == NetworkStackDualStack {
// Both internal and external IPv6 addresses are written to this array
ipv6s, err := metadata.Get(fmt.Sprintf(networkInterfaceIPV6, nic))
if err != nil {
return nil, fmt.Errorf("couldn't get internal IPV6 addresses for node %v: %v", nodeName, err)
}
ipv6Arr := strings.Split(ipv6s, "/\n")
var internalIPV6 string
for _, ip := range ipv6Arr {
if ip == "" {
continue
}
internalIPV6 = ip
break
}
if internalIPV6 != "" {
nodeAddresses = append(nodeAddresses, v1.NodeAddress{Type: v1.NodeInternalIP, Address: internalIPV6})
} else {
klog.Warningf("internal IPV6 range is empty for node %v", nodeName)
}
}
acs, err := metadata.Get(fmt.Sprintf(networkInterfaceAccessConfigs, nic))
if err != nil {
return nil, fmt.Errorf("couldn't get access configs: %v", err)
Expand Down Expand Up @@ -160,12 +183,12 @@ func (g *Cloud) NodeAddresses(ctx context.Context, nodeName types.NodeName) ([]v
return nil, fmt.Errorf("couldn't get instance details: %v", err)
}

instance, err := g.c.Instances().Get(timeoutCtx, meta.ZonalKey(canonicalizeInstanceName(instanceObj.Name), instanceObj.Zone))
instance, err := g.c.AlphaInstances().Get(timeoutCtx, meta.ZonalKey(canonicalizeInstanceName(instanceObj.Name), instanceObj.Zone))
if err != nil {
return []v1.NodeAddress{}, fmt.Errorf("error while querying for instance: %v", err)
return nil, fmt.Errorf("error while querying for instance: %v", err)
}

return nodeAddressesFromInstance(instance)
return g.nodeAddressesFromInstance(instance)
}

// NodeAddressesByProviderID will not be called from the node that is requesting this ID.
Expand All @@ -179,12 +202,12 @@ func (g *Cloud) NodeAddressesByProviderID(ctx context.Context, providerID string
return []v1.NodeAddress{}, err
}

instance, err := g.c.Instances().Get(timeoutCtx, meta.ZonalKey(canonicalizeInstanceName(name), zone))
instance, err := g.c.AlphaInstances().Get(timeoutCtx, meta.ZonalKey(canonicalizeInstanceName(name), zone))
if err != nil {
return []v1.NodeAddress{}, fmt.Errorf("error while querying for providerID %q: %v", providerID, err)
}

return nodeAddressesFromInstance(instance)
return g.nodeAddressesFromInstance(instance)
}

// instanceByProviderID returns the cloudprovider instance of the node
Expand Down Expand Up @@ -216,7 +239,7 @@ func (g *Cloud) InstanceShutdown(ctx context.Context, node *v1.Node) (bool, erro
return false, cloudprovider.NotImplemented
}

func nodeAddressesFromInstance(instance *compute.Instance) ([]v1.NodeAddress, error) {
func (g *Cloud) nodeAddressesFromInstance(instance *computealpha.Instance) ([]v1.NodeAddress, error) {
if len(instance.NetworkInterfaces) < 1 {
return nil, fmt.Errorf("could not find network interfaces for instanceID %q", instance.Id)
}
Expand All @@ -227,11 +250,27 @@ func nodeAddressesFromInstance(instance *compute.Instance) ([]v1.NodeAddress, er
for _, config := range nic.AccessConfigs {
nodeAddresses = append(nodeAddresses, v1.NodeAddress{Type: v1.NodeExternalIP, Address: config.NatIP})
}
if g.stackType == NetworkStackDualStack {
ipv6Addr := getIPV6AddressFromInterface(nic)
if ipv6Addr != "" {
nodeAddresses = append(nodeAddresses, v1.NodeAddress{Type: v1.NodeInternalIP, Address: ipv6Addr})
}
}
}

return nodeAddresses, nil
}

func getIPV6AddressFromInterface(nic *computealpha.NetworkInterface) string {
ipv6Addr := nic.Ipv6Address
if ipv6Addr == "" && nic.Ipv6AccessType == "EXTERNAL" {
Copy link
Contributor

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

There was a discussion by gce team on supporting both public and private v6 IPs on the same interface. We may have to revisit this if that is the case.

Copy link
Author

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

Yes. For now, VMs just get one IPv6 address.

Copy link
Member

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

Question about the access type - does "INTERNAL" correspond to directpath? What happen if the customer enables both dualstack and directpath?

Copy link
Author

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

INTERNAL does not directly correspond to directpath. INTERNAL means that the subnet has only private IPv6 addresses. Today directpath uses INTERNAL addresses. When dual stack is enabled with directpath on a subnet with INTERNAL addresses, everything just works. Directpath and dual stack use the same IPs. Directpath on subnets with EXTERNAL addresses is not working currently. The GCE team is trying to figure something out.

Copy link
Member

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

Got it, thanks for the clarification. Make sense, seems like we will worry about the directpath use case later then.

for _, r := range nic.Ipv6AccessConfigs {
ipv6Addr = r.ExternalIpv6
}
}
return ipv6Addr
}

// InstanceTypeByProviderID returns the cloudprovider instance type of the node
// with the specified unique providerID This method will not be called from the
// node that is requesting this ID. i.e. metadata service and other local
Expand Down Expand Up @@ -298,12 +337,12 @@ func (g *Cloud) InstanceMetadata(ctx context.Context, node *v1.Node) (*cloudprov
return nil, err
}

instance, err := g.c.Instances().Get(timeoutCtx, meta.ZonalKey(canonicalizeInstanceName(name), zone))
instance, err := g.c.AlphaInstances().Get(timeoutCtx, meta.ZonalKey(canonicalizeInstanceName(name), zone))
if err != nil {
return nil, fmt.Errorf("error while querying for providerID %q: %v", providerID, err)
}

addresses, err := nodeAddressesFromInstance(instance)
addresses, err := g.nodeAddressesFromInstance(instance)
if err != nil {
return nil, err
}
Expand Down Expand Up @@ -502,8 +541,8 @@ func (g *Cloud) AliasRangesByProviderID(providerID string) (cidrs []string, err
return nil, err
}

var res *computebeta.Instance
res, err = g.c.BetaInstances().Get(ctx, meta.ZonalKey(canonicalizeInstanceName(name), zone))
var res *computealpha.Instance
res, err = g.c.AlphaInstances().Get(ctx, meta.ZonalKey(canonicalizeInstanceName(name), zone))
if err != nil {
return
}
Expand All @@ -512,6 +551,16 @@ func (g *Cloud) AliasRangesByProviderID(providerID string) (cidrs []string, err
for _, r := range networkInterface.AliasIpRanges {
cidrs = append(cidrs, r.IpCidrRange)
}
if g.stackType == NetworkStackDualStack {
ipv6Addr := getIPV6AddressFromInterface(networkInterface)
if ipv6Addr == "" {
return nil, fmt.Errorf("IPV6 address not found for %s", providerID)
}
// The podCIDR range is the first /112 subrange from the /96 assigned to
// the node
ipv6PodCIDR := fmt.Sprintf("%s/112", ipv6Addr)
cidrs = append(cidrs, ipv6PodCIDR)
}
}
return
}
Expand Down
Loading