kops-controller: create IPAM controller for GCE

cmd/kops-controller/controllers/awsipam.go

@@ -146,8 +146,10 @@ func (r *AWSIPAMReconciler) Reconcile(ctx context.Context, req ctrl.Request) (ct
 			return ctrl.Result{}, fmt.Errorf("unexpected amount of ipv6 prefixes on interface %q: %v", *eni.NetworkInterfaces[0].NetworkInterfaceId, len(eni.NetworkInterfaces[0].Ipv6Prefixes))
 		}
 
-		patchNodePodCIDRs(r.coreV1Client, ctx, node, *eni.NetworkInterfaces[0].Ipv6Prefixes[0].Ipv6Prefix)
-
+		ipv6Address := aws.StringValue(eni.NetworkInterfaces[0].Ipv6Prefixes[0].Ipv6Prefix)
+		if err := patchNodePodCIDRs(r.coreV1Client, ctx, node, ipv6Address); err != nil {
+			return ctrl.Result{}, err
+		}
 	}
 
 	return ctrl.Result{}, nil

cmd/kops-controller/controllers/gceipam.go

@@ -0,0 +1,149 @@
+/*
+Copyright 2023 The Kubernetes Authors.
+
+Licensed under the Apache License, Version 2.0 (the "License");
+you may not use this file except in compliance with the License.
+You may obtain a copy of the License at
+
+    http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License.
+*/
+
+package controllers
+
+import (
+	"context"
+	"fmt"
+	"net/url"
+	"strings"
+
+	"github.com/go-logr/logr"
+	"google.golang.org/api/compute/v1"
+	corev1 "k8s.io/api/core/v1"
+	apierrors "k8s.io/apimachinery/pkg/api/errors"
+	corev1client "k8s.io/client-go/kubernetes/typed/core/v1"
+	"k8s.io/klog/v2"
+	ctrl "sigs.k8s.io/controller-runtime"
+	"sigs.k8s.io/controller-runtime/pkg/client"
+	"sigs.k8s.io/controller-runtime/pkg/manager"
+)
+
+// NewGCEIPAMReconciler is the constructor for a GCEIPAMReconciler
+func NewGCEIPAMReconciler(mgr manager.Manager) (*GCEIPAMReconciler, error) {
+	klog.Info("starting gce ipam controller")
+	r := &GCEIPAMReconciler{
+		client: mgr.GetClient(),
+		log:    ctrl.Log.WithName("controllers").WithName("gce-ipam"),
+	}
+
+	coreClient, err := corev1client.NewForConfig(mgr.GetConfig())
+	if err != nil {
+		return nil, fmt.Errorf("building corev1 client: %w", err)
+	}
+	r.coreV1Client = coreClient
+
+	gceClient, err := compute.NewService(context.Background())
+	if err != nil {
+		return nil, fmt.Errorf("building compute API client: %w", err)
+	}
+	r.gceClient = gceClient
+
+	return r, nil
+}
+
+// GCEIPAMReconciler observes Node objects, assigning their`PodCIDRs` from the instance's `ExternalIpv6`.
+type GCEIPAMReconciler struct {
+	// client is the controller-runtime client
+	client client.Client
+
+	// log is a logr
+	log logr.Logger
+
+	// coreV1Client is a client-go client for patching nodes
+	coreV1Client *corev1client.CoreV1Client
+
+	// gceClient is a client for GCE
+	gceClient *compute.Service
+}
+
+// +kubebuilder:rbac:groups=,resources=nodes,verbs=get;list;watch;patch
+// Reconcile is the main reconciler function that observes node changes.
+func (r *GCEIPAMReconciler) Reconcile(ctx context.Context, req ctrl.Request) (ctrl.Result, error) {
+	_ = r.log.WithValues("node", req.NamespacedName)
+
+	node := &corev1.Node{}
+	if err := r.client.Get(ctx, req.NamespacedName, node); err != nil {
+		klog.Warningf("unable to fetch node %s: %w", node.Name, err)
+		if apierrors.IsNotFound(err) {
+			// we'll ignore not-found errors, since they can't be fixed by an immediate
+			// requeue (we'll need to wait for a new notification), and we can get them
+			// on deleted requests.
+			return ctrl.Result{}, nil
+		}
+		return ctrl.Result{}, err
+	}
+
+	if len(node.Spec.PodCIDRs) == 0 {
+		// CCM Node Controller has not done its thing yet
+		if node.Spec.ProviderID == "" {
+			klog.Infof("node %q has empty provider ID", node.Name)
+			return ctrl.Result{}, nil
+		}
+
+		// e.g. providerID: gce://example-project-id/us-west2-a/instance-id
+		providerURL, err := url.Parse(node.Spec.ProviderID)
+		if err != nil {
+			return ctrl.Result{}, fmt.Errorf("parsing providerID %q: %w", node.Spec.ProviderID, err)
+		}
+		tokens := strings.Split(strings.Trim(providerURL.Path, "/"), "/")
+		if len(tokens) != 2 {
+			return ctrl.Result{}, fmt.Errorf("unexpected format for providerID %q", node.Spec.ProviderID)
+		}
+		project := providerURL.Host
+		zone := tokens[0]
+		instanceID := tokens[1]
+		if project == "" || zone == "" || instanceID == "" {
+			return ctrl.Result{}, fmt.Errorf("unexpected format for providerID %q", node.Spec.ProviderID)
+		}
+
+		instance, err := r.gceClient.Instances.Get(project, zone, instanceID).Context(ctx).Do()
+		if err != nil {
+			return ctrl.Result{}, fmt.Errorf("getting instance %s/%s/%s: %w", project, zone, instanceID, err)
+		}
+
+		var ipv6Addresses []string
+		for _, nic := range instance.NetworkInterfaces {
+			for _, ipv6AccessConfig := range nic.Ipv6AccessConfigs {
+				if ipv6AccessConfig.ExternalIpv6 != "" {
+					ipv6Address := fmt.Sprintf("%s/%d", ipv6AccessConfig.ExternalIpv6, ipv6AccessConfig.ExternalIpv6PrefixLength)
+					ipv6Addresses = append(ipv6Addresses, ipv6Address)
+				}
+			}
+		}
+
+		if len(ipv6Addresses) == 0 {
+			return ctrl.Result{}, fmt.Errorf("no ipv6 address found on interface %q", instance.NetworkInterfaces[0].Name)
+		}
+		if len(ipv6Addresses) != 1 {
+			return ctrl.Result{}, fmt.Errorf("multiple ipv6 addresses found on interface %q: %v", instance.NetworkInterfaces[0].Name, ipv6Addresses)
+		}
+
+		ipv6Address := ipv6Addresses[0]
+		if err := patchNodePodCIDRs(r.coreV1Client, ctx, node, ipv6Address); err != nil {
+			return ctrl.Result{}, err
+		}
+	}
+
+	return ctrl.Result{}, nil
+}
+
+func (r *GCEIPAMReconciler) SetupWithManager(mgr ctrl.Manager) error {
+	return ctrl.NewControllerManagedBy(mgr).
+		For(&corev1.Node{}).
+		Complete(r)
+}

cmd/kops-controller/main.go

@@ -175,19 +175,10 @@ func main() {
 	}
 
 	if opt.EnableCloudIPAM {
-		setupLog.Info("enabling IPAM controller")
-		if opt.Cloud != "aws" {
-			klog.Error("IPAM controller only supported by aws")
-			os.Exit(1)
-		}
-		ipamController, err := controllers.NewAWSIPAMReconciler(mgr)
-		if err != nil {
-			setupLog.Error(err, "unable to create controller", "controller", "IPAMController")
-			os.Exit(1)
-		}
-		if err := ipamController.SetupWithManager(mgr); err != nil {
-			setupLog.Error(err, "unable to create controller", "controller", "IPAMController")
+		if err := setupCloudIPAM(mgr, &opt); err != nil {
+			setupLog.Error(err, "unable to setup cloud IPAM")
 			os.Exit(1)
+
 		}
 	}
 
@@ -324,3 +315,35 @@ func addGossipController(mgr manager.Manager, opt *config.Options) error {
 
 	return nil
 }
+
+// Reconciler is the interface for a standard Reconciler.
+type Reconciler interface {
+	SetupWithManager(mgr manager.Manager) error
+}
+
+func setupCloudIPAM(mgr manager.Manager, opt *config.Options) error {
+	setupLog.Info("enabling IPAM controller")
+	var controller Reconciler
+	switch opt.Cloud {
+	case "aws":
+		ipamController, err := controllers.NewAWSIPAMReconciler(mgr)
+		if err != nil {
+			return fmt.Errorf("creating aws IPAM controller: %w", err)
+		}
+		controller = ipamController
+	case "gce":
+		ipamController, err := controllers.NewGCEIPAMReconciler(mgr)
+		if err != nil {
+			return fmt.Errorf("creating gce IPAM controller: %w", err)
+		}
+		controller = ipamController
+	default:
+		return fmt.Errorf("kOps IPAM controller is not supported on cloud %q", opt.Cloud)
+	}
+
+	if err := controller.SetupWithManager(mgr); err != nil {
+		return fmt.Errorf("registering IPAM controller: %w", err)
+	}
+
+	return nil
+}

pkg/model/components/gcpcloudcontrollermanager.go

@@ -65,13 +65,20 @@ func (b *GCPCloudControllerManagerOptionsBuilder) BuildOptions(options interface
 	}
 
 	if ccmConfig.Controllers == nil {
-		ccmConfig.Controllers = []string{
-			"*",
+		var changes []string
 
-			// Don't run gkenetworkparamset controller, looks for some CRDs (GKENetworkParamSet and Network) which are only installed on GKE
-			// However, the version we're current running doesn't support this controller anyway, so we need to introduce this later,
-			// possibly based on the image version.
-			// "-gkenetworkparams",
+		// Don't run gkenetworkparamset controller, looks for some CRDs (GKENetworkParamSet and Network) which are only installed on GKE
+		// However, the version we're current running doesn't support this controller anyway, so we need to introduce this later,
+		// possibly based on the image version.
+		// changes = append(ccmConfig.Controllers, "-gkenetworkparams")
+
+		// Turn off some controllers if kops-controller is running them
+		if clusterSpec.IsKopsControllerIPAM() {
+			changes = append(ccmConfig.Controllers, "-nodeipam", "-route")
+		}
+
+		if len(changes) != 0 {
+			ccmConfig.Controllers = append([]string{"*"}, changes...)
 		}
 	}
 

tests/integration/update_cluster/ha_gce/data/aws_s3_object_cluster-completed.spec_content

@@ -20,8 +20,6 @@ spec:
     cidrAllocatorType: CloudAllocator
     clusterCIDR: 100.96.0.0/11
     clusterName: ha-gce-example-com
-    controllers:
-    - '*'
     image: k8scloudprovidergcp/cloud-controller-manager:latest
     leaderElection:
       leaderElect: true

tests/integration/update_cluster/ha_gce/data/aws_s3_object_ha-gce.example.com-addons-bootstrap_content

@@ -62,7 +62,7 @@ spec:
     version: 9.99.0
   - id: k8s-1.23
     manifest: gcp-cloud-controller.addons.k8s.io/k8s-1.23.yaml
-    manifestHash: 865e79337a96353842b6f5c359701cdfe3d1da805d235ea49c91d4ab79a35050
+    manifestHash: 39ecc974e248a7e0f0c6c0813e8e27be9713af775939bd09c83dfc2c1946bf2f
     name: gcp-cloud-controller.addons.k8s.io
     prune:
       kinds:

tests/integration/update_cluster/ha_gce/data/aws_s3_object_ha-gce.example.com-addons-gcp-cloud-controller.addons.k8s.io-k8s-1.23_content

@@ -37,7 +37,6 @@ spec:
         - --cidr-allocator-type=CloudAllocator
         - --cluster-cidr=100.96.0.0/11
         - --cluster-name=ha-gce-example-com
-        - --controllers=*
         - --leader-elect=true
         - --v=2
         - --cloud-provider=gce

tests/integration/update_cluster/many-addons-gce/data/aws_s3_object_cluster-completed.spec_content

@@ -22,8 +22,6 @@ spec:
     cidrAllocatorType: CloudAllocator
     clusterCIDR: 100.96.0.0/11
     clusterName: minimal-example-com
-    controllers:
-    - '*'
     image: k8scloudprovidergcp/cloud-controller-manager:latest
     leaderElection:
       leaderElect: true

tests/integration/update_cluster/many-addons-gce/data/aws_s3_object_minimal.example.com-addons-bootstrap_content

@@ -133,7 +133,7 @@ spec:
     version: 9.99.0
   - id: k8s-1.23
     manifest: gcp-cloud-controller.addons.k8s.io/k8s-1.23.yaml
-    manifestHash: 5b2863c906b3bd49e625b3ef83f27f3bd5840c52118ed57b38bd20e12330d37f
+    manifestHash: 49730987c3f5dba8914954740cc96dbb08c090c0ac9c943a160a32cb94b53764
     name: gcp-cloud-controller.addons.k8s.io
     prune:
       kinds:

tests/integration/update_cluster/many-addons-gce/data/aws_s3_object_minimal.example.com-addons-gcp-cloud-controller.addons.k8s.io-k8s-1.23_content

@@ -37,7 +37,6 @@ spec:
         - --cidr-allocator-type=CloudAllocator
         - --cluster-cidr=100.96.0.0/11
         - --cluster-name=minimal-example-com
-        - --controllers=*
         - --leader-elect=true
         - --v=2
         - --cloud-provider=gce

tests/integration/update_cluster/minimal_gce/data/aws_s3_object_cluster-completed.spec_content

@@ -20,8 +20,6 @@ spec:
     cidrAllocatorType: CloudAllocator
     clusterCIDR: 100.96.0.0/11
     clusterName: minimal-gce-example-com
-    controllers:
-    - '*'
     image: k8scloudprovidergcp/cloud-controller-manager:latest
     leaderElection:
       leaderElect: true

tests/integration/update_cluster/minimal_gce/data/aws_s3_object_minimal-gce.example.com-addons-bootstrap_content

@@ -62,7 +62,7 @@ spec:
     version: 9.99.0
   - id: k8s-1.23
     manifest: gcp-cloud-controller.addons.k8s.io/k8s-1.23.yaml
-    manifestHash: 79edeb95bce8a2771bd84b7ecd55639b2fc895e3cdd533015727368996cab6ce
+    manifestHash: 1e7b574d8a629441fef3f17dec13a0c73ab48b2c40160c30a8c78a99652db702
     name: gcp-cloud-controller.addons.k8s.io
     prune:
       kinds:

tests/integration/update_cluster/minimal_gce/data/aws_s3_object_minimal-gce.example.com-addons-gcp-cloud-controller.addons.k8s.io-k8s-1.23_content

@@ -37,7 +37,6 @@ spec:
         - --cidr-allocator-type=CloudAllocator
         - --cluster-cidr=100.96.0.0/11
         - --cluster-name=minimal-gce-example-com
-        - --controllers=*
         - --leader-elect=true
         - --v=2
         - --cloud-provider=gce

tests/integration/update_cluster/minimal_gce_dns-none/data/aws_s3_object_cluster-completed.spec_content

@@ -24,8 +24,6 @@ spec:
     cidrAllocatorType: CloudAllocator
     clusterCIDR: 100.96.0.0/11
     clusterName: minimal-gce-example-com
-    controllers:
-    - '*'
     image: k8scloudprovidergcp/cloud-controller-manager:latest
     leaderElection:
       leaderElect: true

tests/integration/update_cluster/minimal_gce_dns-none/data/aws_s3_object_minimal-gce.example.com-addons-bootstrap_content

@@ -55,7 +55,7 @@ spec:
     version: 9.99.0
   - id: k8s-1.23
     manifest: gcp-cloud-controller.addons.k8s.io/k8s-1.23.yaml
-    manifestHash: 79edeb95bce8a2771bd84b7ecd55639b2fc895e3cdd533015727368996cab6ce
+    manifestHash: 1e7b574d8a629441fef3f17dec13a0c73ab48b2c40160c30a8c78a99652db702
     name: gcp-cloud-controller.addons.k8s.io
     prune:
       kinds:

tests/integration/update_cluster/minimal_gce_dns-none/data/aws_s3_object_minimal-gce.example.com-addons-gcp-cloud-controller.addons.k8s.io-k8s-1.23_content

@@ -37,7 +37,6 @@ spec:
         - --cidr-allocator-type=CloudAllocator
         - --cluster-cidr=100.96.0.0/11
         - --cluster-name=minimal-gce-example-com
-        - --controllers=*
         - --leader-elect=true
         - --v=2
         - --cloud-provider=gce

tests/integration/update_cluster/minimal_gce_ilb/data/aws_s3_object_cluster-completed.spec_content

@@ -24,8 +24,6 @@ spec:
     cidrAllocatorType: CloudAllocator
     clusterCIDR: 100.96.0.0/11
     clusterName: minimal-gce-ilb-example-com
-    controllers:
-    - '*'
     image: k8scloudprovidergcp/cloud-controller-manager:latest
     leaderElection:
       leaderElect: true

tests/integration/update_cluster/minimal_gce_ilb/data/aws_s3_object_minimal-gce-ilb.example.com-addons-bootstrap_content

@@ -62,7 +62,7 @@ spec:
     version: 9.99.0
   - id: k8s-1.23
     manifest: gcp-cloud-controller.addons.k8s.io/k8s-1.23.yaml
-    manifestHash: 9311c506f8a1674f99ad4959e584fe982a33a2a36692477ca5d9d8e5318fb135
+    manifestHash: 8e9e414b952c3c6298c27cdc50866ab48e18ac2689c70ce926a8639fc54ff786
     name: gcp-cloud-controller.addons.k8s.io
     prune:
       kinds:

tests/integration/update_cluster/minimal_gce_ilb/data/aws_s3_object_minimal-gce-ilb.example.com-addons-gcp-cloud-controller.addons.k8s.io-k8s-1.23_content

@@ -37,7 +37,6 @@ spec:
         - --cidr-allocator-type=CloudAllocator
         - --cluster-cidr=100.96.0.0/11
         - --cluster-name=minimal-gce-ilb-example-com
-        - --controllers=*
         - --leader-elect=true
         - --v=2
         - --cloud-provider=gce

tests/integration/update_cluster/minimal_gce_ilb_longclustername/data/aws_s3_object_cluster-completed.spec_content

@@ -24,8 +24,6 @@ spec:
     cidrAllocatorType: CloudAllocator
     clusterCIDR: 100.96.0.0/11
     clusterName: minimal-gce-with-a-very-very-very-very-very-long-name-example-com
-    controllers:
-    - '*'
     image: k8scloudprovidergcp/cloud-controller-manager:latest
     leaderElection:
       leaderElect: true

tests/integration/update_cluster/minimal_gce_ilb_longclustername/data/aws_s3_object_minimal-gce-with-a-very-very-very-very-very-long-name.example.com-addons-bootstrap_content

@@ -62,7 +62,7 @@ spec:
     version: 9.99.0
   - id: k8s-1.23
     manifest: gcp-cloud-controller.addons.k8s.io/k8s-1.23.yaml
-    manifestHash: 2b00060fec2abeb81de79e436f13dfe1f9a88c5f3b1e449043a24d85de3016a1
+    manifestHash: 24c22638647064142f86c461088862d4de22ca298d5f018bf2801c20f5df21d4
     name: gcp-cloud-controller.addons.k8s.io
     prune:
       kinds: