Merge pull request #15474 from scaleway/scw_use_kops_controller_for_b…

…ootstrap

scaleway: use kops controller for bootstrap

cmd/kops-controller/main.go

@@ -46,6 +46,7 @@ import (
 	"k8s.io/kops/upup/pkg/fi/cloudup/gce/tpm/gcetpmverifier"
 	"k8s.io/kops/upup/pkg/fi/cloudup/hetzner"
 	"k8s.io/kops/upup/pkg/fi/cloudup/openstack"
+	"k8s.io/kops/upup/pkg/fi/cloudup/scaleway"
 	ctrl "sigs.k8s.io/controller-runtime"
 	"sigs.k8s.io/controller-runtime/pkg/client"
 	"sigs.k8s.io/controller-runtime/pkg/manager"
@@ -146,6 +147,12 @@ func main() {
 				setupLog.Error(err, "unable to create verifier")
 				os.Exit(1)
 			}
+		} else if opt.Server.Provider.Scaleway != nil {
+			verifier, err = scaleway.NewScalewayVerifier(ctx, opt.Server.Provider.Scaleway)
+			if err != nil {
+				setupLog.Error(err, "unable to create verifier")
+				os.Exit(1)
+			}
 		} else {
 			klog.Fatalf("server cloud provider config not provided")
 		}

cmd/kops-controller/pkg/config/options.go

@@ -22,6 +22,7 @@ import (
 	gcetpm "k8s.io/kops/upup/pkg/fi/cloudup/gce/tpm"
 	"k8s.io/kops/upup/pkg/fi/cloudup/hetzner"
 	"k8s.io/kops/upup/pkg/fi/cloudup/openstack"
+	"k8s.io/kops/upup/pkg/fi/cloudup/scaleway"
 )
 
 type Options struct {
@@ -71,6 +72,7 @@ type ServerProviderOptions struct {
 	Hetzner      *hetzner.HetznerVerifierOptions     `json:"hetzner,omitempty"`
 	OpenStack    *openstack.OpenStackVerifierOptions `json:"openstack,omitempty"`
 	DigitalOcean *do.DigitalOceanVerifierOptions     `json:"do,omitempty"`
+	Scaleway     *scaleway.ScalewayVerifierOptions   `json:"scaleway,omitempty"`
 }
 
 // DiscoveryOptions configures our support for discovery, particularly gossip DNS (i.e. k8s.local)

cmd/kops/integration_test.go

@@ -1810,7 +1810,6 @@ func (i *integrationTest) runTestTerraformScaleway(t *testing.T) {
 		"aws_s3_object_"+i.clusterName+"-addons-kubelet-api.rbac.addons.k8s.io-k8s-1.9_content",
 		"aws_s3_object_"+i.clusterName+"-addons-limit-range.addons.k8s.io_content",
 		"aws_s3_object_"+i.clusterName+"-addons-networking.cilium.io-k8s-1.16_content",
-		"aws_s3_object_"+i.clusterName+"-addons-rbac.addons.k8s.io-k8s-1.8_content",
 		"scaleway_instance_server_control-plane-fr-par-1_user_data",
 		"scaleway_instance_server_nodes-fr-par-1_user_data",
 	)

nodeup/pkg/model/bootstrap_client.go

@@ -34,6 +34,7 @@ import (
 	"k8s.io/kops/upup/pkg/fi/cloudup/gce/tpm/gcetpmsigner"
 	"k8s.io/kops/upup/pkg/fi/cloudup/hetzner"
 	"k8s.io/kops/upup/pkg/fi/cloudup/openstack"
+	"k8s.io/kops/upup/pkg/fi/cloudup/scaleway"
 	"k8s.io/kops/upup/pkg/fi/nodeup/nodetasks"
 )
 
@@ -80,13 +81,18 @@ func (b BootstrapClientBuilder) Build(c *fi.NodeupModelBuilderContext) error {
 			return err
 		}
 		authenticator = a
-
 	case kops.CloudProviderDO:
 		a, err := do.NewAuthenticator()
 		if err != nil {
 			return err
 		}
 		authenticator = a
+	case kops.CloudProviderScaleway:
+		a, err := scaleway.NewScalewayAuthenticator()
+		if err != nil {
+			return err
+		}
+		authenticator = a
 
 	default:
 		return fmt.Errorf("unsupported cloud provider for authenticator %q", b.CloudProvider())

pkg/apis/kops/model/features.go

@@ -33,6 +33,8 @@ func UseKopsControllerForNodeBootstrap(cluster *kops.Cluster) bool {
 		return true
 	case kops.CloudProviderDO:
 		return true
+	case kops.CloudProviderScaleway:
+		return true
 	default:
 		return false
 	}
@@ -45,6 +47,8 @@ func UseChallengeCallback(cloudProvider kops.CloudProviderID) bool {
 		return true
 	case kops.CloudProviderDO:
 		return true
+	case kops.CloudProviderScaleway:
+		return true
 	default:
 		return false
 	}
@@ -56,7 +60,7 @@ func UseKopsControllerForNodeConfig(cluster *kops.Cluster) bool {
 		switch cluster.Spec.GetCloudProvider() {
 		case kops.CloudProviderGCE:
 			// We can use cloud-discovery here.
-		case kops.CloudProviderHetzner:
+		case kops.CloudProviderHetzner, kops.CloudProviderScaleway:
 			// We don't have a cloud-discovery mechanism implemented in nodeup for hetzner,
 			// but we assume that we're using a load balancer with a fixed IP address
 		default:

pkg/model/bootstrapscript.go

@@ -231,7 +231,7 @@ func (b *BootstrapScript) buildEnvironmentVariables() (map[string]string, error)
 		}
 	}
 
-	if cluster.Spec.GetCloudProvider() == kops.CloudProviderScaleway {
+	if cluster.Spec.GetCloudProvider() == kops.CloudProviderScaleway && (b.ig.IsControlPlane() || cluster.UsesLegacyGossip()) {
 		profile, err := scaleway.CreateValidScalewayProfile()
 		if err != nil {
 			return nil, err

pkg/model/scalewaymodel/api_loadbalancer.go

@@ -20,9 +20,11 @@ import (
 	"fmt"
 
 	"github.com/scaleway/scaleway-sdk-go/api/lb/v1"
+	"github.com/scaleway/scaleway-sdk-go/scw"
 	"k8s.io/klog/v2"
 	"k8s.io/kops/pkg/apis/kops"
 	"k8s.io/kops/pkg/dns"
+	"k8s.io/kops/pkg/wellknownports"
 	"k8s.io/kops/upup/pkg/fi"
 	"k8s.io/kops/upup/pkg/fi/cloudup/scaleway"
 	"k8s.io/kops/upup/pkg/fi/cloudup/scalewaytasks"
@@ -78,36 +80,48 @@ func (b *APILoadBalancerModelBuilder) Build(c *fi.CloudupModelBuilderContext) er
 
 	c.AddTask(loadBalancer)
 
-	lbBackend := &scalewaytasks.LBBackend{
-		Name:                 fi.PtrTo("lb-backend"),
-		Lifecycle:            b.Lifecycle,
+	lbBackendHttps, lbFrontendHttps := createLbBackendAndFrontend("https", wellknownports.KubeAPIServer, zone, loadBalancer)
+	lbBackendHttps.Lifecycle = b.Lifecycle
+	c.AddTask(lbBackendHttps)
+	lbFrontendHttps.Lifecycle = b.Lifecycle
+	c.AddTask(lbFrontendHttps)
+
+	if dns.IsGossipClusterName(b.Cluster.Name) || b.Cluster.UsesPrivateDNS() || b.Cluster.UsesNoneDNS() {
+		// Ensure the LB hostname is included in the TLS certificate,
+		// if we're not going to use an alias for it
+		loadBalancer.ForAPIServer = true
+
+		if b.Cluster.UsesNoneDNS() || b.UseKopsControllerForNodeBootstrap() {
+			lbBackendKopsController, lbFrontendKopsController := createLbBackendAndFrontend("kops-controller", wellknownports.KopsControllerPort, zone, loadBalancer)
+			lbBackendKopsController.Lifecycle = b.Lifecycle
+			c.AddTask(lbBackendKopsController)
+			lbFrontendKopsController.Lifecycle = b.Lifecycle
+			c.AddTask(lbFrontendKopsController)
+		}
+	}
+
+	return nil
+}
+
+func createLbBackendAndFrontend(name string, port int, zone scw.Zone, loadBalancer *scalewaytasks.LoadBalancer) (*scalewaytasks.LBBackend, *scalewaytasks.LBFrontend) {
+	lbBackendKopsController := &scalewaytasks.LBBackend{
+		Name:                 fi.PtrTo("lb-backend-" + name),
 		Zone:                 fi.PtrTo(string(zone)),
 		ForwardProtocol:      fi.PtrTo(string(lb.ProtocolTCP)),
-		ForwardPort:          fi.PtrTo(int32(443)),
+		ForwardPort:          fi.PtrTo(int32(port)),
 		ForwardPortAlgorithm: fi.PtrTo(string(lb.ForwardPortAlgorithmRoundrobin)),
 		StickySessions:       fi.PtrTo(string(lb.StickySessionsTypeNone)),
 		ProxyProtocol:        fi.PtrTo(string(lb.ProxyProtocolProxyProtocolUnknown)),
 		LoadBalancer:         loadBalancer,
 	}
 
-	c.AddTask(lbBackend)
-
-	lbFrontend := &scalewaytasks.LBFrontend{
-		Name:         fi.PtrTo("lb-frontend"),
-		Lifecycle:    b.Lifecycle,
+	lbFrontendKopsController := &scalewaytasks.LBFrontend{
+		Name:         fi.PtrTo("lb-frontend-" + name),
 		Zone:         fi.PtrTo(string(zone)),
-		InboundPort:  fi.PtrTo(int32(443)),
+		InboundPort:  fi.PtrTo(int32(port)),
 		LoadBalancer: loadBalancer,
-		LBBackend:    lbBackend,
-	}
-
-	c.AddTask(lbFrontend)
-
-	if dns.IsGossipClusterName(b.Cluster.Name) || b.Cluster.UsesPrivateDNS() || b.Cluster.UsesNoneDNS() {
-		// Ensure the LB hostname is included in the TLS certificate,
-		// if we're not going to use an alias for it
-		loadBalancer.ForAPIServer = true
+		LBBackend:    lbBackendKopsController,
 	}
 
-	return nil
+	return lbBackendKopsController, lbFrontendKopsController
 }

pkg/nodeidentity/scaleway/identify.go

@@ -48,12 +48,16 @@ type nodeIdentifier struct {
 
 // New creates and returns a nodeidentity.Identifier for Nodes running on Scaleway
 func New(CacheNodeidentityInfo bool) (nodeidentity.Identifier, error) {
+	profile, err := scaleway.CreateValidScalewayProfile()
+	if err != nil {
+		return nil, err
+	}
 	scwClient, err := scw.NewClient(
-		scw.WithUserAgent("kubernetes-kops/"+kopsv.Version),
-		scw.WithEnv(),
+		scw.WithProfile(profile),
+		scw.WithUserAgent(scaleway.KopsUserAgentPrefix+kopsv.Version),
 	)
 	if err != nil {
-		return nil, err
+		return nil, fmt.Errorf("creating client for Scaleway NodeIdentifier: %w", err)
 	}
 
 	return &nodeIdentifier{

protokube/pkg/protokube/scaleway_volumes.go

@@ -66,14 +66,18 @@ func NewScwCloudProvider() (*ScwCloudProvider, error) {
 	privateIP := metadata.PrivateIP
 	klog.V(4).Infof("Found first private net IP of the running server: %q", privateIP)
 
+	profile, err := scaleway.CreateValidScalewayProfile()
+	if err != nil {
+		return nil, err
+	}
 	scwClient, err := scw.NewClient(
+		scw.WithProfile(profile),
 		scw.WithUserAgent(scaleway.KopsUserAgentPrefix+kopsv.Version),
-		scw.WithEnv(),
 		scw.WithDefaultZone(zone),
 		scw.WithDefaultRegion(region),
 	)
 	if err != nil {
-		return nil, fmt.Errorf("error creating client: %w", err)
+		return nil, fmt.Errorf("error creating client for Protokube: %w", err)
 	}
 
 	instanceAPI := instance.NewAPI(scwClient)

tests/integration/update_cluster/minimal_scaleway/data/aws_s3_object_nodeupconfig-nodes-fr-par-1_content

@@ -15,36 +15,13 @@ Assets:
   - 54e79e4d48b9e191767e4abc08be1a8476a1c757e9a9f8c45c6ded001226867f@https://github.com/opencontainers/runc/releases/download/v1.1.5/runc.arm64
   - 2f599c3d54f4c4bdbcc95aaf0c7b513a845d8f9503ec5b34c9f86aa1bc34fc0c@https://artifacts.k8s.io/binaries/kops/1.21.0-alpha.1/linux/arm64/protokube,https://github.com/kubernetes/kops/releases/download/v1.21.0-alpha.1/protokube-linux-arm64
   - 9d842e3636a95de2315cdea2be7a282355aac0658ef0b86d5dc2449066538f13@https://artifacts.k8s.io/binaries/kops/1.21.0-alpha.1/linux/arm64/channels,https://github.com/kubernetes/kops/releases/download/v1.21.0-alpha.1/channels-linux-arm64
-CAs:
-  kubernetes-ca: |
-    -----BEGIN CERTIFICATE-----
-    MIIBbjCCARigAwIBAgIMFpANqBD8NSD82AUSMA0GCSqGSIb3DQEBCwUAMBgxFjAU
-    BgNVBAMTDWt1YmVybmV0ZXMtY2EwHhcNMjEwNzA3MDcwODAwWhcNMzEwNzA3MDcw
-    ODAwWjAYMRYwFAYDVQQDEw1rdWJlcm5ldGVzLWNhMFwwDQYJKoZIhvcNAQEBBQAD
-    SwAwSAJBANFI3zr0Tk8krsW8vwjfMpzJOlWQ8616vG3YPa2qAgI7V4oKwfV0yIg1
-    jt+H6f4P/wkPAPTPTfRp9Iy8oHEEFw0CAwEAAaNCMEAwDgYDVR0PAQH/BAQDAgEG
-    MA8GA1UdEwEB/wQFMAMBAf8wHQYDVR0OBBYEFNG3zVjTcLlJwDsJ4/K9DV7KohUA
-    MA0GCSqGSIb3DQEBCwUAA0EAB8d03fY2w7WKpfO29qI295pu2C4ca9AiVGOpgSc8
-    tmQsq6rcxt3T+rb589PVtz0mw/cKTxOk6gH2CCC+yHfy2w==
-    -----END CERTIFICATE-----
-    -----BEGIN CERTIFICATE-----
-    MIIBbjCCARigAwIBAgIMFpANvmSa0OAlYmXKMA0GCSqGSIb3DQEBCwUAMBgxFjAU
-    BgNVBAMTDWt1YmVybmV0ZXMtY2EwHhcNMjEwNzA3MDcwOTM2WhcNMzEwNzA3MDcw
-    OTM2WjAYMRYwFAYDVQQDEw1rdWJlcm5ldGVzLWNhMFwwDQYJKoZIhvcNAQEBBQAD
-    SwAwSAJBAMF6F4aZdpe0RUpyykaBpWwZCnwbffhYGOw+fs6RdLuUq7QCNmJm/Eq7
-    WWOziMYDiI9SbclpD+6QiJ0N3EqppVUCAwEAAaNCMEAwDgYDVR0PAQH/BAQDAgEG
-    MA8GA1UdEwEB/wQFMAMBAf8wHQYDVR0OBBYEFLImp6ARjPDAH6nhI+scWVt3Q9bn
-    MA0GCSqGSIb3DQEBCwUAA0EAVQVx5MUtuAIeePuP9o51xtpT2S6Fvfi8J4ICxnlA
-    9B7UD2ushcVFPtaeoL9Gfu8aY4KJBeqqg5ojl4qmRnThjw==
-    -----END CERTIFICATE-----
+CAs: {}
 ClusterName: scw-minimal.k8s.local
 ContainerRuntime: containerd
 Hooks:
 - null
 - null
 KeypairIDs:
-  kube-proxy: "6986354184403674830529235586"
-  kubelet: "6986354184404014133128804066"
   kubernetes-ca: "6982820025135291416230495506"
 KubeProxy: null
 KubeletConfig:

tests/integration/update_cluster/minimal_scaleway/data/aws_s3_object_scw-minimal.k8s.local-addons-bootstrap_content

@@ -6,7 +6,7 @@ spec:
   addons:
   - id: k8s-1.16
     manifest: kops-controller.addons.k8s.io/k8s-1.16.yaml
-    manifestHash: bffd10e2291b38f02725cd04366b8029878272a79e68825e88d98a681068cd6c
+    manifestHash: 0652ed8a25e088a043a9f162f3b73dba1b9b5dcbb8efcb2b6550826d81f32a2b
     name: kops-controller.addons.k8s.io
     needsRollingUpdate: control-plane
     selector:
@@ -19,13 +19,6 @@ spec:
     selector:
       k8s-addon: coredns.addons.k8s.io
     version: 9.99.0
-  - id: k8s-1.8
-    manifest: rbac.addons.k8s.io/k8s-1.8.yaml
-    manifestHash: f81bd7c57bc1902ca342635d7ad7d01b82dfeaff01a1192b076e66907d87871e
-    name: rbac.addons.k8s.io
-    selector:
-      k8s-addon: rbac.addons.k8s.io
-    version: 9.99.0
   - id: k8s-1.9
     manifest: kubelet-api.rbac.addons.k8s.io/k8s-1.9.yaml
     manifestHash: 01c120e887bd98d82ef57983ad58a0b22bc85efb48108092a24c4b82e4c9ea81

tests/integration/update_cluster/minimal_scaleway/data/aws_s3_object_scw-minimal.k8s.local-addons-kops-controller.addons.k8s.io-k8s-1.16_content

@@ -1,7 +1,7 @@
 apiVersion: v1
 data:
   config.yaml: |
-    {"clusterName":"scw-minimal.k8s.local","cloud":"scaleway","configBase":"memfs://tests/scw-minimal.k8s.local","secretStore":"memfs://tests/scw-minimal.k8s.local/secrets","discovery":{"enabled":true}}
+    {"clusterName":"scw-minimal.k8s.local","cloud":"scaleway","configBase":"memfs://tests/scw-minimal.k8s.local","secretStore":"memfs://tests/scw-minimal.k8s.local/secrets","server":{"Listen":":3988","provider":{"scaleway":{}},"serverKeyPath":"/etc/kubernetes/kops-controller/pki/kops-controller.key","serverCertificatePath":"/etc/kubernetes/kops-controller/pki/kops-controller.crt","caBasePath":"/etc/kubernetes/kops-controller/pki","signingCAs":["kubernetes-ca"],"certNames":["kubelet","kubelet-server"]},"discovery":{"enabled":true}}
 kind: ConfigMap
 metadata:
   creationTimestamp: null
@@ -32,6 +32,8 @@ spec:
       k8s-app: kops-controller
   template:
     metadata:
+      annotations:
+        dns.alpha.kubernetes.io/internal: kops-controller.internal.scw-minimal.k8s.local
       creationTimestamp: null
       labels:
         k8s-addon: kops-controller.addons.k8s.io

tests/integration/update_cluster/minimal_scaleway/data/scaleway_instance_server_nodes-fr-par-1_user_data

@@ -150,10 +150,33 @@ __EOF_CLUSTER_SPEC
 cat > conf/kube_env.yaml << '__EOF_KUBE_ENV'
 CloudProvider: scaleway
 ClusterName: scw-minimal.k8s.local
-ConfigBase: memfs://tests/scw-minimal.k8s.local
+ConfigServer:
+  CACertificates: |
+    -----BEGIN CERTIFICATE-----
+    MIIBbjCCARigAwIBAgIMFpANqBD8NSD82AUSMA0GCSqGSIb3DQEBCwUAMBgxFjAU
+    BgNVBAMTDWt1YmVybmV0ZXMtY2EwHhcNMjEwNzA3MDcwODAwWhcNMzEwNzA3MDcw
+    ODAwWjAYMRYwFAYDVQQDEw1rdWJlcm5ldGVzLWNhMFwwDQYJKoZIhvcNAQEBBQAD
+    SwAwSAJBANFI3zr0Tk8krsW8vwjfMpzJOlWQ8616vG3YPa2qAgI7V4oKwfV0yIg1
+    jt+H6f4P/wkPAPTPTfRp9Iy8oHEEFw0CAwEAAaNCMEAwDgYDVR0PAQH/BAQDAgEG
+    MA8GA1UdEwEB/wQFMAMBAf8wHQYDVR0OBBYEFNG3zVjTcLlJwDsJ4/K9DV7KohUA
+    MA0GCSqGSIb3DQEBCwUAA0EAB8d03fY2w7WKpfO29qI295pu2C4ca9AiVGOpgSc8
+    tmQsq6rcxt3T+rb589PVtz0mw/cKTxOk6gH2CCC+yHfy2w==
+    -----END CERTIFICATE-----
+    -----BEGIN CERTIFICATE-----
+    MIIBbjCCARigAwIBAgIMFpANvmSa0OAlYmXKMA0GCSqGSIb3DQEBCwUAMBgxFjAU
+    BgNVBAMTDWt1YmVybmV0ZXMtY2EwHhcNMjEwNzA3MDcwOTM2WhcNMzEwNzA3MDcw
+    OTM2WjAYMRYwFAYDVQQDEw1rdWJlcm5ldGVzLWNhMFwwDQYJKoZIhvcNAQEBBQAD
+    SwAwSAJBAMF6F4aZdpe0RUpyykaBpWwZCnwbffhYGOw+fs6RdLuUq7QCNmJm/Eq7
+    WWOziMYDiI9SbclpD+6QiJ0N3EqppVUCAwEAAaNCMEAwDgYDVR0PAQH/BAQDAgEG
+    MA8GA1UdEwEB/wQFMAMBAf8wHQYDVR0OBBYEFLImp6ARjPDAH6nhI+scWVt3Q9bn
+    MA0GCSqGSIb3DQEBCwUAA0EAVQVx5MUtuAIeePuP9o51xtpT2S6Fvfi8J4ICxnlA
+    9B7UD2ushcVFPtaeoL9Gfu8aY4KJBeqqg5ojl4qmRnThjw==
+    -----END CERTIFICATE-----
+  servers:
+  - https://kops-controller.internal.scw-minimal.k8s.local:3988/
 InstanceGroupName: nodes-fr-par-1
 InstanceGroupRole: Node
-NodeupConfigHash: ThbM3OQQUCmecnKq4GQW0fdWp6sjwEiAjfmqga3QcXY=
+NodeupConfigHash: l6ITXtzPONIgO+uyEAe0rYXGYgBVjPkJ9/Ov2xKxX/U=
 
 __EOF_KUBE_ENV