Skip to content

Commit

Permalink
Use Alpha Compute API only in case of dual stack
Browse files Browse the repository at this point in the history 
#268 introduced IPv6 support, but it also began to
require Google Cloud Compute Aplha API access. Unfortunately by default
projects don't have this access and therefore, when the CCM fails with:

googleapi: Error 403: Required 'Alpha Access' permission for 'Compute API', forbidden

This commit starts using Alpha API only for dual stack deployments,
where it's really required. For all other cases we will continue to
use GA API.

Fixes: #281
  • Loading branch information
@Fedosin
Fedosin committed Oct 14, 2021
1 parent b0261e5 commit 965c9ea
Show file tree
Hide file tree
Showing 2 changed files with 176 additions and 64 deletions.
104 changes: 83 additions & 21 deletions providers/gce/gce_instances.go
View file
Original file line number Diff line number Diff line change
Expand Up @@ -183,7 +183,16 @@ func (g *Cloud) NodeAddresses(ctx context.Context, nodeName types.NodeName) ([]v
return nil, fmt.Errorf("couldn't get instance details: %v", err)
}

instance, err := g.c.AlphaInstances().Get(timeoutCtx, meta.ZonalKey(canonicalizeInstanceName(instanceObj.Name), instanceObj.Zone))
if g.stackType == NetworkStackDualStack {
alphaInstance, err := g.c.AlphaInstances().Get(timeoutCtx, meta.ZonalKey(canonicalizeInstanceName(instanceObj.Name), instanceObj.Zone))
if err != nil {
return nil, fmt.Errorf("error while querying for instance: %v", err)
}

return g.nodeAddressesFromAlphaInstance(alphaInstance)
}

instance, err := g.c.Instances().Get(timeoutCtx, meta.ZonalKey(canonicalizeInstanceName(instanceObj.Name), instanceObj.Zone))
if err != nil {
return nil, fmt.Errorf("error while querying for instance: %v", err)
}
Expand All @@ -202,7 +211,16 @@ func (g *Cloud) NodeAddressesByProviderID(ctx context.Context, providerID string
return []v1.NodeAddress{}, err
}

instance, err := g.c.AlphaInstances().Get(timeoutCtx, meta.ZonalKey(canonicalizeInstanceName(name), zone))
if g.stackType == NetworkStackDualStack {
alphaInstance, err := g.c.AlphaInstances().Get(timeoutCtx, meta.ZonalKey(canonicalizeInstanceName(name), zone))
if err != nil {
return []v1.NodeAddress{}, fmt.Errorf("error while querying for providerID %q: %v", providerID, err)
}

return g.nodeAddressesFromAlphaInstance(alphaInstance)
}

instance, err := g.c.Instances().Get(timeoutCtx, meta.ZonalKey(canonicalizeInstanceName(name), zone))
if err != nil {
return []v1.NodeAddress{}, fmt.Errorf("error while querying for providerID %q: %v", providerID, err)
}
Expand Down Expand Up @@ -239,7 +257,22 @@ func (g *Cloud) InstanceShutdown(ctx context.Context, node *v1.Node) (bool, erro
return false, cloudprovider.NotImplemented
}

func (g *Cloud) nodeAddressesFromInstance(instance *computealpha.Instance) ([]v1.NodeAddress, error) {
func (g *Cloud) nodeAddressesFromInstance(instance *compute.Instance) ([]v1.NodeAddress, error) {
if len(instance.NetworkInterfaces) < 1 {
return nil, fmt.Errorf("could not find network interfaces for instanceID %q", instance.Id)
}
nodeAddresses := []v1.NodeAddress{}
for _, nic := range instance.NetworkInterfaces {
nodeAddresses = append(nodeAddresses, v1.NodeAddress{Type: v1.NodeInternalIP, Address: nic.NetworkIP})
for _, config := range nic.AccessConfigs {
nodeAddresses = append(nodeAddresses, v1.NodeAddress{Type: v1.NodeExternalIP, Address: config.NatIP})
}
}

return nodeAddresses, nil
}

func (g *Cloud) nodeAddressesFromAlphaInstance(instance *computealpha.Instance) ([]v1.NodeAddress, error) {
if len(instance.NetworkInterfaces) < 1 {
return nil, fmt.Errorf("could not find network interfaces for instanceID %q", instance.Id)
}
Expand Down Expand Up @@ -337,19 +370,37 @@ func (g *Cloud) InstanceMetadata(ctx context.Context, node *v1.Node) (*cloudprov
return nil, err
}

instance, err := g.c.AlphaInstances().Get(timeoutCtx, meta.ZonalKey(canonicalizeInstanceName(name), zone))
if err != nil {
return nil, fmt.Errorf("error while querying for providerID %q: %v", providerID, err)
}
var addresses []v1.NodeAddress
var instanceType string
if g.stackType == NetworkStackDualStack {
alphaInstance, err := g.c.AlphaInstances().Get(timeoutCtx, meta.ZonalKey(canonicalizeInstanceName(name), zone))
if err != nil {
return nil, fmt.Errorf("error while querying for providerID %q: %v", providerID, err)
}

addresses, err := g.nodeAddressesFromInstance(instance)
if err != nil {
return nil, err
addresses, err = g.nodeAddressesFromAlphaInstance(alphaInstance)
if err != nil {
return nil, err
}

instanceType = lastComponent(alphaInstance.MachineType)
} else {
instance, err := g.c.Instances().Get(timeoutCtx, meta.ZonalKey(canonicalizeInstanceName(name), zone))
if err != nil {
return nil, fmt.Errorf("error while querying for providerID %q: %v", providerID, err)
}

addresses, err = g.nodeAddressesFromInstance(instance)
if err != nil {
return nil, err
}

instanceType = lastComponent(instance.MachineType)
}

return &cloudprovider.InstanceMetadata{
ProviderID: providerID,
InstanceType: lastComponent(instance.MachineType),
InstanceType: instanceType,
NodeAddresses: addresses,
Zone: zone,
Region: region,
Expand Down Expand Up @@ -541,17 +592,17 @@ func (g *Cloud) AliasRangesByProviderID(providerID string) (cidrs []string, err
return nil, err
}

var res *computealpha.Instance
res, err = g.c.AlphaInstances().Get(ctx, meta.ZonalKey(canonicalizeInstanceName(name), zone))
if err != nil {
return
}

for _, networkInterface := range res.NetworkInterfaces {
for _, r := range networkInterface.AliasIpRanges {
cidrs = append(cidrs, r.IpCidrRange)
if g.stackType == NetworkStackDualStack {
var res *computealpha.Instance
res, err = g.c.AlphaInstances().Get(ctx, meta.ZonalKey(canonicalizeInstanceName(name), zone))
if err != nil {
return
}
if g.stackType == NetworkStackDualStack {

for _, networkInterface := range res.NetworkInterfaces {
for _, r := range networkInterface.AliasIpRanges {
cidrs = append(cidrs, r.IpCidrRange)
}
ipv6Addr := getIPV6AddressFromInterface(networkInterface)
if ipv6Addr == "" {
return nil, fmt.Errorf("IPV6 address not found for %s", providerID)
Expand All @@ -561,6 +612,17 @@ func (g *Cloud) AliasRangesByProviderID(providerID string) (cidrs []string, err
ipv6PodCIDR := fmt.Sprintf("%s/112", ipv6Addr)
cidrs = append(cidrs, ipv6PodCIDR)
}
} else {
var res *computebeta.Instance
res, err = g.c.BetaInstances().Get(ctx, meta.ZonalKey(canonicalizeInstanceName(name), zone))
if err != nil {
return
}
for _, networkInterface := range res.NetworkInterfaces {
for _, r := range networkInterface.AliasIpRanges {
cidrs = append(cidrs, r.IpCidrRange)
}
}
}
return
}
Expand Down
136 changes: 93 additions & 43 deletions providers/gce/gce_instances_test.go
View file
Original file line number Diff line number Diff line change
Expand Up @@ -78,6 +78,94 @@ func TestNodeAddresses(t *testing.T) {
gce, err := fakeGCECloud(DefaultTestClusterValues())
require.NoError(t, err)

instanceMap := make(map[string]*ga.Instance)
instance := &ga.Instance{
Name: "n1",
Zone: "us-central1-b",
}
instanceMap["n1"] = instance

instance = &ga.Instance{
Name: "n2",
Zone: "us-central1-b",
}
instanceMap["n2"] = instance

// n3 is instance not present in the alphaInstanceMap
instance = &ga.Instance{
Name: "n3",
Zone: "us-central1-b",
}
instanceMap["n3"] = instance

instance = &ga.Instance{
Name: "n5",
Zone: "us-central1-b",
}
instanceMap["n5"] = instance

mockGCE := gce.c.(*cloud.MockGCE)
mi := mockGCE.Instances().(*cloud.MockInstances)
mi.GetHook = func(ctx context.Context, key *meta.Key, m *cloud.MockInstances) (bool, *ga.Instance, error) {
ret, ok := instanceMap[key.Name]
if !ok {
return true, nil, fmt.Errorf("instance not found")
}
return true, ret, nil
}

testcases := []struct {
name string
nodeName string
wantErr string
wantAddrs []v1.NodeAddress
}{
{
name: "internal single stack instance",
nodeName: "n1",
wantAddrs: []v1.NodeAddress{
{Type: v1.NodeInternalIP, Address: "10.1.1.1"},
},
},
{
name: "instance not found",
nodeName: "x1",
wantErr: "instance not found",
},
{
name: "external single stack instance",
nodeName: "n2",
wantAddrs: []v1.NodeAddress{
{Type: v1.NodeInternalIP, Address: "10.1.1.2"},
{Type: v1.NodeExternalIP, Address: "20.1.1.2"},
},
},
{
name: "network interface not found",
nodeName: "n4",
wantErr: "could not find network interface",
},
}

for _, test := range testcases {
t.Run(test.name, func(t *testing.T) {
gce.stackType = NetworkStackIPV4
gotAddrs, err := gce.NodeAddresses(context.TODO(), types.NodeName(test.nodeName))
if err != nil && (test.wantErr == "" || !strings.Contains(err.Error(), test.wantErr)) {
t.Errorf("gce.NodeAddresses. Want err: %v, got: %v", test.wantErr, err)
return
} else if err == nil && test.wantErr != "" {
t.Errorf("gce.NodeAddresses. Want err: %v, got: %v", test.wantErr, err)
}
assert.Equal(t, test.wantAddrs, gotAddrs)
})
}
}

func TestAlphaNodeAddresses(t *testing.T) {
gce, err := fakeGCECloud(DefaultTestClusterValues())
require.NoError(t, err)

instanceMap := make(map[string]*ga.Instance)
alphaInstanceMap := make(map[string]*alpha.Instance)
// n1 is instance with internal IPv6 address
Expand Down Expand Up @@ -185,63 +273,29 @@ func TestNodeAddresses(t *testing.T) {
testcases := []struct {
name string
nodeName string
dualStack bool
wantErr string
wantAddrs []v1.NodeAddress
}{
{
name: "internal single stack instance",
name: "internal dual stack instance",
nodeName: "n1",
wantAddrs: []v1.NodeAddress{
{Type: v1.NodeInternalIP, Address: "10.1.1.1"},
},
},
{
name: "internal dual stack instance",
nodeName: "n1",
dualStack: true,
wantAddrs: []v1.NodeAddress{
{Type: v1.NodeInternalIP, Address: "10.1.1.1"},
{Type: v1.NodeInternalIP, Address: "2001:2d00::0:1"},
},
},
{
name: "instance not found",
nodeName: "x1",
wantErr: "instance not found",
},
{
name: "alpha instance not found",
nodeName: "n3",
wantErr: "alpha instance not found",
},
{
name: "external single stack instance",
name: "external dual stack instance",
nodeName: "n2",
wantAddrs: []v1.NodeAddress{
{Type: v1.NodeInternalIP, Address: "10.1.1.2"},
{Type: v1.NodeExternalIP, Address: "20.1.1.2"},
},
},
{
name: "external dual stack instance",
nodeName: "n2",
dualStack: true,
wantAddrs: []v1.NodeAddress{
{Type: v1.NodeInternalIP, Address: "10.1.1.2"},
{Type: v1.NodeExternalIP, Address: "20.1.1.2"},
{Type: v1.NodeInternalIP, Address: "2001:1900::0:2"},
},
},
{
name: "network interface not found",
nodeName: "n4",
wantErr: "could not find network interface",
},
{
name: "single stack instance",
nodeName: "n5",
dualStack: true,
name: "single stack instance",
nodeName: "n5",
wantAddrs: []v1.NodeAddress{
{Type: v1.NodeInternalIP, Address: "10.1.1.5"},
{Type: v1.NodeExternalIP, Address: "20.1.1.5"},
Expand All @@ -251,11 +305,7 @@ func TestNodeAddresses(t *testing.T) {

for _, test := range testcases {
t.Run(test.name, func(t *testing.T) {
if test.dualStack {
gce.stackType = NetworkStackDualStack
} else {
gce.stackType = NetworkStackIPV4
}
gce.stackType = NetworkStackDualStack
gotAddrs, err := gce.NodeAddresses(context.TODO(), types.NodeName(test.nodeName))
if err != nil && (test.wantErr == "" || !strings.Contains(err.Error(), test.wantErr)) {
t.Errorf("gce.NodeAddresses. Want err: %v, got: %v", test.wantErr, err)
Expand Down

0 comments on commit 965c9ea

Please sign in to comment.