From 49465a62c764b31c857e054bce9a1dfcda5a2c77 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Le=C3=AFla=20MARABESE?= Date: Tue, 6 Jun 2023 16:56:43 +0200 Subject: [PATCH 1/5] add backend for kops controller port --- pkg/model/scalewaymodel/api_loadbalancer.go | 54 ++++++++++------- upup/pkg/fi/cloudup/scalewaytasks/instance.go | 59 +++++++++---------- .../fi/cloudup/scalewaytasks/lb_backend.go | 2 +- 3 files changed, 62 insertions(+), 53 deletions(-) diff --git a/pkg/model/scalewaymodel/api_loadbalancer.go b/pkg/model/scalewaymodel/api_loadbalancer.go index 8ece77d2e7d47..51c87e3abee10 100644 --- a/pkg/model/scalewaymodel/api_loadbalancer.go +++ b/pkg/model/scalewaymodel/api_loadbalancer.go @@ -20,9 +20,11 @@ import ( "fmt" "github.com/scaleway/scaleway-sdk-go/api/lb/v1" + "github.com/scaleway/scaleway-sdk-go/scw" "k8s.io/klog/v2" "k8s.io/kops/pkg/apis/kops" "k8s.io/kops/pkg/dns" + "k8s.io/kops/pkg/wellknownports" "k8s.io/kops/upup/pkg/fi" "k8s.io/kops/upup/pkg/fi/cloudup/scaleway" "k8s.io/kops/upup/pkg/fi/cloudup/scalewaytasks" @@ -78,36 +80,48 @@ func (b *APILoadBalancerModelBuilder) Build(c *fi.CloudupModelBuilderContext) er c.AddTask(loadBalancer) - lbBackend := &scalewaytasks.LBBackend{ - Name: fi.PtrTo("lb-backend"), - Lifecycle: b.Lifecycle, + lbBackendHttps, lbFrontendHttps := createLbBackendAndFrontend("https", wellknownports.KubeAPIServer, zone, loadBalancer) + lbBackendHttps.Lifecycle = b.Lifecycle + c.AddTask(lbBackendHttps) + lbFrontendHttps.Lifecycle = b.Lifecycle + c.AddTask(lbFrontendHttps) + + if dns.IsGossipClusterName(b.Cluster.Name) || b.Cluster.UsesPrivateDNS() || b.Cluster.UsesNoneDNS() { + // Ensure the LB hostname is included in the TLS certificate, + // if we're not going to use an alias for it + loadBalancer.ForAPIServer = true + + if b.Cluster.UsesNoneDNS() { + lbBackendKopsController, lbFrontendKopsController := createLbBackendAndFrontend("kops-controller", wellknownports.KopsControllerPort, zone, loadBalancer) + lbBackendKopsController.Lifecycle = b.Lifecycle + c.AddTask(lbBackendKopsController) + lbFrontendKopsController.Lifecycle = b.Lifecycle + c.AddTask(lbFrontendKopsController) + } + } + + return nil +} + +func createLbBackendAndFrontend(name string, port int, zone scw.Zone, loadBalancer *scalewaytasks.LoadBalancer) (*scalewaytasks.LBBackend, *scalewaytasks.LBFrontend) { + lbBackendKopsController := &scalewaytasks.LBBackend{ + Name: fi.PtrTo("lb-backend-" + name), Zone: fi.PtrTo(string(zone)), ForwardProtocol: fi.PtrTo(string(lb.ProtocolTCP)), - ForwardPort: fi.PtrTo(int32(443)), + ForwardPort: fi.PtrTo(int32(port)), ForwardPortAlgorithm: fi.PtrTo(string(lb.ForwardPortAlgorithmRoundrobin)), StickySessions: fi.PtrTo(string(lb.StickySessionsTypeNone)), ProxyProtocol: fi.PtrTo(string(lb.ProxyProtocolProxyProtocolUnknown)), LoadBalancer: loadBalancer, } - c.AddTask(lbBackend) - - lbFrontend := &scalewaytasks.LBFrontend{ - Name: fi.PtrTo("lb-frontend"), - Lifecycle: b.Lifecycle, + lbFrontendKopsController := &scalewaytasks.LBFrontend{ + Name: fi.PtrTo("lb-frontend-" + name), Zone: fi.PtrTo(string(zone)), - InboundPort: fi.PtrTo(int32(443)), + InboundPort: fi.PtrTo(int32(port)), LoadBalancer: loadBalancer, - LBBackend: lbBackend, - } - - c.AddTask(lbFrontend) - - if dns.IsGossipClusterName(b.Cluster.Name) || b.Cluster.UsesPrivateDNS() || b.Cluster.UsesNoneDNS() { - // Ensure the LB hostname is included in the TLS certificate, - // if we're not going to use an alias for it - loadBalancer.ForAPIServer = true + LBBackend: lbBackendKopsController, } - return nil + return lbBackendKopsController, lbFrontendKopsController } diff --git a/upup/pkg/fi/cloudup/scalewaytasks/instance.go b/upup/pkg/fi/cloudup/scalewaytasks/instance.go index 34e96e0405d69..c9cf15d2049a6 100644 --- a/upup/pkg/fi/cloudup/scalewaytasks/instance.go +++ b/upup/pkg/fi/cloudup/scalewaytasks/instance.go @@ -278,43 +278,38 @@ func (_ *Instance) RenderScw(t *scaleway.ScwAPITarget, actual, expected, changes if err != nil { return fmt.Errorf("listing load-balancer's back-ends for instance creation: %w", err) } - if backEnds.TotalCount > 1 { - return fmt.Errorf("cannot have multiple back-ends for load-balancer %s", loadBalancer.Name) - } else if backEnds.TotalCount < 1 { - return fmt.Errorf("load-balancer %s should have 1 back-end, got 0", loadBalancer.Name) - } - backEnd := backEnds.Backends[0] - - // If we are adding instances, we also need to add them to the load-balancer's backend - if newInstanceCount > 0 { - _, err = lbService.AddBackendServers(&lb.ZonedAPIAddBackendServersRequest{ - Zone: zone, - BackendID: backEnd.ID, - ServerIP: controlPlanePrivateIPs, - }) - if err != nil { - return fmt.Errorf("adding servers' IPs to load-balancer's back-end: %w", err) - } - } else { - // If we are deleting instances, we also need to delete them from the load-balancer's backend - _, err = lbService.RemoveBackendServers(&lb.ZonedAPIRemoveBackendServersRequest{ - Zone: zone, - BackendID: backEnd.ID, - ServerIP: controlPlanePrivateIPs, + for _, backEnd := range backEnds.Backends { + // If we are adding instances, we also need to add them to the load-balancer's backend + if newInstanceCount > 0 { + _, err = lbService.AddBackendServers(&lb.ZonedAPIAddBackendServersRequest{ + Zone: zone, + BackendID: backEnd.ID, + ServerIP: controlPlanePrivateIPs, + }) + if err != nil { + return fmt.Errorf("adding servers' IPs to load-balancer's back-end: %w", err) + } + + } else { + // If we are deleting instances, we also need to delete them from the load-balancer's backend + _, err = lbService.RemoveBackendServers(&lb.ZonedAPIRemoveBackendServersRequest{ + Zone: zone, + BackendID: backEnd.ID, + ServerIP: controlPlanePrivateIPs, + }) + if err != nil { + return fmt.Errorf("removing servers' IPs from load-balancer's back-end: %w", err) + } + } + _, err = lbService.WaitForLb(&lb.ZonedAPIWaitForLBRequest{ + LBID: loadBalancer.ID, + Zone: zone, }) if err != nil { - return fmt.Errorf("removing servers' IPs from load-balancer's back-end: %w", err) + return fmt.Errorf("waiting for load-balancer %s: %w", loadBalancer.ID, err) } } - - _, err = lbService.WaitForLb(&lb.ZonedAPIWaitForLBRequest{ - LBID: loadBalancer.ID, - Zone: zone, - }) - if err != nil { - return fmt.Errorf("waiting for load-balancer %s: %w", loadBalancer.ID, err) - } } } diff --git a/upup/pkg/fi/cloudup/scalewaytasks/lb_backend.go b/upup/pkg/fi/cloudup/scalewaytasks/lb_backend.go index e32d48759295e..32432e15d3f49 100644 --- a/upup/pkg/fi/cloudup/scalewaytasks/lb_backend.go +++ b/upup/pkg/fi/cloudup/scalewaytasks/lb_backend.go @@ -136,7 +136,7 @@ func (l *LBBackend) RenderScw(t *scaleway.ScwAPITarget, actual, expected, change backendCreated, err := lbService.CreateBackend(&lb.ZonedAPICreateBackendRequest{ Zone: scw.Zone(fi.ValueOf(expected.Zone)), - LBID: fi.ValueOf(expected.LoadBalancer.LBID), // try expected instead of l + LBID: fi.ValueOf(expected.LoadBalancer.LBID), Name: fi.ValueOf(expected.Name), ForwardProtocol: lb.Protocol(fi.ValueOf(expected.ForwardProtocol)), ForwardPort: fi.ValueOf(expected.ForwardPort), From dab001c3e904c26c8f56250a6b26c9f92f1e86f8 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Le=C3=AFla=20MARABESE?= Date: Tue, 6 Jun 2023 16:59:29 +0200 Subject: [PATCH 2/5] scaleway authenticator and verifier --- cmd/kops-controller/main.go | 7 ++ cmd/kops-controller/pkg/config/options.go | 2 + nodeup/pkg/model/bootstrap_client.go | 8 +- pkg/apis/kops/model/features.go | 4 + pkg/model/bootstrapscript.go | 2 +- upup/pkg/fi/cloudup/scaleway/authenticator.go | 27 +++++ upup/pkg/fi/cloudup/scaleway/verifier.go | 107 ++++++++++++++++++ upup/pkg/fi/cloudup/template_functions.go | 3 + upup/pkg/fi/nodeup/command.go | 9 +- 9 files changed, 165 insertions(+), 4 deletions(-) create mode 100644 upup/pkg/fi/cloudup/scaleway/authenticator.go create mode 100644 upup/pkg/fi/cloudup/scaleway/verifier.go diff --git a/cmd/kops-controller/main.go b/cmd/kops-controller/main.go index 18d7b77509861..278676639b99f 100644 --- a/cmd/kops-controller/main.go +++ b/cmd/kops-controller/main.go @@ -46,6 +46,7 @@ import ( "k8s.io/kops/upup/pkg/fi/cloudup/gce/tpm/gcetpmverifier" "k8s.io/kops/upup/pkg/fi/cloudup/hetzner" "k8s.io/kops/upup/pkg/fi/cloudup/openstack" + "k8s.io/kops/upup/pkg/fi/cloudup/scaleway" ctrl "sigs.k8s.io/controller-runtime" "sigs.k8s.io/controller-runtime/pkg/client" "sigs.k8s.io/controller-runtime/pkg/manager" @@ -146,6 +147,12 @@ func main() { setupLog.Error(err, "unable to create verifier") os.Exit(1) } + } else if opt.Server.Provider.Scaleway != nil { + verifier, err = scaleway.NewScalewayVerifier(ctx, opt.Server.Provider.Scaleway) + if err != nil { + setupLog.Error(err, "unable to create verifier") + os.Exit(1) + } } else { klog.Fatalf("server cloud provider config not provided") } diff --git a/cmd/kops-controller/pkg/config/options.go b/cmd/kops-controller/pkg/config/options.go index 721d24e86189d..defc08124d248 100644 --- a/cmd/kops-controller/pkg/config/options.go +++ b/cmd/kops-controller/pkg/config/options.go @@ -22,6 +22,7 @@ import ( gcetpm "k8s.io/kops/upup/pkg/fi/cloudup/gce/tpm" "k8s.io/kops/upup/pkg/fi/cloudup/hetzner" "k8s.io/kops/upup/pkg/fi/cloudup/openstack" + "k8s.io/kops/upup/pkg/fi/cloudup/scaleway" ) type Options struct { @@ -71,6 +72,7 @@ type ServerProviderOptions struct { Hetzner *hetzner.HetznerVerifierOptions `json:"hetzner,omitempty"` OpenStack *openstack.OpenStackVerifierOptions `json:"openstack,omitempty"` DigitalOcean *do.DigitalOceanVerifierOptions `json:"do,omitempty"` + Scaleway *scaleway.ScalewayVerifierOptions `json:"scaleway,omitempty"` } // DiscoveryOptions configures our support for discovery, particularly gossip DNS (i.e. k8s.local) diff --git a/nodeup/pkg/model/bootstrap_client.go b/nodeup/pkg/model/bootstrap_client.go index 3b96b88044750..abc678277724c 100644 --- a/nodeup/pkg/model/bootstrap_client.go +++ b/nodeup/pkg/model/bootstrap_client.go @@ -34,6 +34,7 @@ import ( "k8s.io/kops/upup/pkg/fi/cloudup/gce/tpm/gcetpmsigner" "k8s.io/kops/upup/pkg/fi/cloudup/hetzner" "k8s.io/kops/upup/pkg/fi/cloudup/openstack" + "k8s.io/kops/upup/pkg/fi/cloudup/scaleway" "k8s.io/kops/upup/pkg/fi/nodeup/nodetasks" ) @@ -80,13 +81,18 @@ func (b BootstrapClientBuilder) Build(c *fi.NodeupModelBuilderContext) error { return err } authenticator = a - case kops.CloudProviderDO: a, err := do.NewAuthenticator() if err != nil { return err } authenticator = a + case kops.CloudProviderScaleway: + a, err := scaleway.NewScalewayAuthenticator() + if err != nil { + return err + } + authenticator = a default: return fmt.Errorf("unsupported cloud provider for authenticator %q", b.CloudProvider()) diff --git a/pkg/apis/kops/model/features.go b/pkg/apis/kops/model/features.go index ca42e88cb6ae8..eb5886a47ad3c 100644 --- a/pkg/apis/kops/model/features.go +++ b/pkg/apis/kops/model/features.go @@ -33,6 +33,8 @@ func UseKopsControllerForNodeBootstrap(cluster *kops.Cluster) bool { return true case kops.CloudProviderDO: return true + case kops.CloudProviderScaleway: + return true default: return false } @@ -45,6 +47,8 @@ func UseChallengeCallback(cloudProvider kops.CloudProviderID) bool { return true case kops.CloudProviderDO: return true + case kops.CloudProviderScaleway: + return true default: return false } diff --git a/pkg/model/bootstrapscript.go b/pkg/model/bootstrapscript.go index c13a7cdcd0eca..2efc9f962b67e 100644 --- a/pkg/model/bootstrapscript.go +++ b/pkg/model/bootstrapscript.go @@ -231,7 +231,7 @@ func (b *BootstrapScript) buildEnvironmentVariables() (map[string]string, error) } } - if cluster.Spec.GetCloudProvider() == kops.CloudProviderScaleway { + if cluster.Spec.GetCloudProvider() == kops.CloudProviderScaleway && (b.ig.IsControlPlane() || cluster.UsesLegacyGossip()) { profile, err := scaleway.CreateValidScalewayProfile() if err != nil { return nil, err diff --git a/upup/pkg/fi/cloudup/scaleway/authenticator.go b/upup/pkg/fi/cloudup/scaleway/authenticator.go new file mode 100644 index 0000000000000..168008f435668 --- /dev/null +++ b/upup/pkg/fi/cloudup/scaleway/authenticator.go @@ -0,0 +1,27 @@ +package scaleway + +import ( + "fmt" + + "github.com/scaleway/scaleway-sdk-go/api/instance/v1" + "k8s.io/kops/pkg/bootstrap" +) + +const ScalewayAuthenticationTokenPrefix = "x-scaleway-instance-server-id " + +type scalewayAuthenticator struct{} + +var _ bootstrap.Authenticator = &scalewayAuthenticator{} + +func NewScalewayAuthenticator() (bootstrap.Authenticator, error) { + return &scalewayAuthenticator{}, nil +} + +func (a *scalewayAuthenticator) CreateToken(body []byte) (string, error) { + metadataAPI := instance.NewMetadataAPI() + metadata, err := metadataAPI.GetMetadata() + if err != nil { + return "", fmt.Errorf("failed to retrieve server metadata: %w", err) + } + return ScalewayAuthenticationTokenPrefix + metadata.ID, nil +} diff --git a/upup/pkg/fi/cloudup/scaleway/verifier.go b/upup/pkg/fi/cloudup/scaleway/verifier.go new file mode 100644 index 0000000000000..504efeee06853 --- /dev/null +++ b/upup/pkg/fi/cloudup/scaleway/verifier.go @@ -0,0 +1,107 @@ +package scaleway + +import ( + "context" + "fmt" + "net" + "net/http" + "strconv" + "strings" + + "github.com/scaleway/scaleway-sdk-go/api/instance/v1" + "github.com/scaleway/scaleway-sdk-go/scw" + kopsv "k8s.io/kops" + "k8s.io/kops/pkg/bootstrap" + "k8s.io/kops/pkg/wellknownports" +) + +type ScalewayVerifierOptions struct{} + +type scalewayVerifier struct { + scwClient *scw.Client +} + +var _ bootstrap.Verifier = &scalewayVerifier{} + +func NewScalewayVerifier(ctx context.Context, opt *ScalewayVerifierOptions) (bootstrap.Verifier, error) { + profile, err := CreateValidScalewayProfile() + if err != nil { + return nil, fmt.Errorf("creating client for Scaleway Verifier: %w", err) + } + scwClient, err := scw.NewClient( + scw.WithProfile(profile), + scw.WithUserAgent(KopsUserAgentPrefix+kopsv.Version), + ) + if err != nil { + return nil, err + } + return &scalewayVerifier{ + scwClient: scwClient, + }, nil +} + +func (v scalewayVerifier) VerifyToken(ctx context.Context, rawRequest *http.Request, token string, body []byte, useInstanceIDForNodeName bool) (*bootstrap.VerifyResult, error) { + if !strings.HasPrefix(token, ScalewayAuthenticationTokenPrefix) { + return nil, fmt.Errorf("incorrect authorization type") + } + serverID := strings.TrimPrefix(token, ScalewayAuthenticationTokenPrefix) + + metadataAPI := instance.NewMetadataAPI() + metadata, err := metadataAPI.GetMetadata() + if err != nil { + return nil, fmt.Errorf("failed to retrieve server metadata: %w", err) + } + zone, err := scw.ParseZone(metadata.Location.ZoneID) + if err != nil { + return nil, fmt.Errorf("unable to parse Scaleway zone %q: %w", metadata.Location.ZoneID, err) + } + + profile, err := CreateValidScalewayProfile() + if err != nil { + return nil, err + } + scwClient, err := scw.NewClient( + scw.WithProfile(profile), + scw.WithUserAgent(KopsUserAgentPrefix+kopsv.Version), + ) + if err != nil { + return nil, fmt.Errorf("creating client for Scaleway Verifier: %w", err) + } + + instanceAPI := instance.NewAPI(scwClient) + serverResponse, err := instanceAPI.GetServer(&instance.GetServerRequest{ + ServerID: serverID, + Zone: zone, + }, scw.WithContext(ctx)) + if err != nil || serverResponse == nil { + return nil, fmt.Errorf("failed to get server %s: %w", serverID, err) + } + server := serverResponse.Server + + addresses := []string(nil) + challengeEndPoints := []string(nil) + if server.PrivateIP != nil { + addresses = append(addresses, *server.PrivateIP) + challengeEndPoints = append(challengeEndPoints, net.JoinHostPort(*server.PrivateIP, strconv.Itoa(wellknownports.NodeupChallenge))) + } + if server.IPv6 != nil { + addresses = append(addresses, server.IPv6.Address.String()) + challengeEndPoints = append(challengeEndPoints, net.JoinHostPort(server.IPv6.Address.String(), strconv.Itoa(wellknownports.NodeupChallenge))) + } + + igName := "" + for _, tag := range server.Tags { + if strings.HasPrefix(tag, TagInstanceGroup) { + igName = strings.TrimPrefix(tag, TagInstanceGroup+"=") + } + } + + result := &bootstrap.VerifyResult{ + NodeName: server.Name, + InstanceGroupName: igName, + CertificateNames: addresses, + ChallengeEndpoint: challengeEndPoints[0], + } + + return result, nil +} diff --git a/upup/pkg/fi/cloudup/template_functions.go b/upup/pkg/fi/cloudup/template_functions.go index 1a0524dcce1f8..2bf4056f93763 100644 --- a/upup/pkg/fi/cloudup/template_functions.go +++ b/upup/pkg/fi/cloudup/template_functions.go @@ -738,6 +738,9 @@ func (tf *TemplateFunctions) KopsControllerConfig() (string, error) { case kops.CloudProviderDO: config.Server.Provider.DigitalOcean = &do.DigitalOceanVerifierOptions{} + case kops.CloudProviderScaleway: + config.Server.Provider.Scaleway = &scaleway.ScalewayVerifierOptions{} + default: return "", fmt.Errorf("unsupported cloud provider %s", cluster.Spec.GetCloudProvider()) } diff --git a/upup/pkg/fi/nodeup/command.go b/upup/pkg/fi/nodeup/command.go index 4973a5808e035..31fe13c83ece2 100644 --- a/upup/pkg/fi/nodeup/command.go +++ b/upup/pkg/fi/nodeup/command.go @@ -59,6 +59,7 @@ import ( "k8s.io/kops/upup/pkg/fi/cloudup/gce/tpm/gcetpmsigner" "k8s.io/kops/upup/pkg/fi/cloudup/hetzner" "k8s.io/kops/upup/pkg/fi/cloudup/openstack" + "k8s.io/kops/upup/pkg/fi/cloudup/scaleway" "k8s.io/kops/upup/pkg/fi/nodeup/local" "k8s.io/kops/upup/pkg/fi/nodeup/nodetasks" "k8s.io/kops/upup/pkg/fi/secrets" @@ -763,14 +764,18 @@ func getNodeConfigFromServers(ctx context.Context, bootConfig *nodeup.BootConfig return nil, err } authenticator = a - case api.CloudProviderDO: a, err := do.NewAuthenticator() if err != nil { return nil, err } authenticator = a - + case api.CloudProviderScaleway: + a, err := scaleway.NewScalewayAuthenticator() + if err != nil { + return nil, err + } + authenticator = a default: return nil, fmt.Errorf("unsupported cloud provider for node configuration %s", bootConfig.CloudProvider) } From 4a943d82353916e32b0e37a1eaba0f86601cfcca Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Le=C3=AFla=20MARABESE?= Date: Tue, 6 Jun 2023 17:00:17 +0200 Subject: [PATCH 3/5] refactoring client --- pkg/nodeidentity/scaleway/identify.go | 10 +++++++--- protokube/pkg/protokube/scaleway_volumes.go | 8 ++++++-- 2 files changed, 13 insertions(+), 5 deletions(-) diff --git a/pkg/nodeidentity/scaleway/identify.go b/pkg/nodeidentity/scaleway/identify.go index f9af6d2aa4bcb..f7cbbac26e190 100644 --- a/pkg/nodeidentity/scaleway/identify.go +++ b/pkg/nodeidentity/scaleway/identify.go @@ -48,12 +48,16 @@ type nodeIdentifier struct { // New creates and returns a nodeidentity.Identifier for Nodes running on Scaleway func New(CacheNodeidentityInfo bool) (nodeidentity.Identifier, error) { + profile, err := scaleway.CreateValidScalewayProfile() + if err != nil { + return nil, err + } scwClient, err := scw.NewClient( - scw.WithUserAgent("kubernetes-kops/"+kopsv.Version), - scw.WithEnv(), + scw.WithProfile(profile), + scw.WithUserAgent(scaleway.KopsUserAgentPrefix+kopsv.Version), ) if err != nil { - return nil, err + return nil, fmt.Errorf("creating client for Scaleway NodeIdentifier: %w", err) } return &nodeIdentifier{ diff --git a/protokube/pkg/protokube/scaleway_volumes.go b/protokube/pkg/protokube/scaleway_volumes.go index f33a9b9cbba69..63899aa94bf19 100644 --- a/protokube/pkg/protokube/scaleway_volumes.go +++ b/protokube/pkg/protokube/scaleway_volumes.go @@ -66,14 +66,18 @@ func NewScwCloudProvider() (*ScwCloudProvider, error) { privateIP := metadata.PrivateIP klog.V(4).Infof("Found first private net IP of the running server: %q", privateIP) + profile, err := scaleway.CreateValidScalewayProfile() + if err != nil { + return nil, err + } scwClient, err := scw.NewClient( + scw.WithProfile(profile), scw.WithUserAgent(scaleway.KopsUserAgentPrefix+kopsv.Version), - scw.WithEnv(), scw.WithDefaultZone(zone), scw.WithDefaultRegion(region), ) if err != nil { - return nil, fmt.Errorf("error creating client: %w", err) + return nil, fmt.Errorf("error creating client for Protokube: %w", err) } instanceAPI := instance.NewAPI(scwClient) From 39ed84601f9275627881bfece61b66156159e989 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Le=C3=AFla=20MARABESE?= Date: Wed, 14 Jun 2023 15:09:49 +0200 Subject: [PATCH 4/5] keep support for gossip clusters --- pkg/apis/kops/model/features.go | 2 +- pkg/model/scalewaymodel/api_loadbalancer.go | 2 +- upup/pkg/fi/cloudup/apply_cluster.go | 2 +- 3 files changed, 3 insertions(+), 3 deletions(-) diff --git a/pkg/apis/kops/model/features.go b/pkg/apis/kops/model/features.go index eb5886a47ad3c..902e9ef337d4e 100644 --- a/pkg/apis/kops/model/features.go +++ b/pkg/apis/kops/model/features.go @@ -60,7 +60,7 @@ func UseKopsControllerForNodeConfig(cluster *kops.Cluster) bool { switch cluster.Spec.GetCloudProvider() { case kops.CloudProviderGCE: // We can use cloud-discovery here. - case kops.CloudProviderHetzner: + case kops.CloudProviderHetzner, kops.CloudProviderScaleway: // We don't have a cloud-discovery mechanism implemented in nodeup for hetzner, // but we assume that we're using a load balancer with a fixed IP address default: diff --git a/pkg/model/scalewaymodel/api_loadbalancer.go b/pkg/model/scalewaymodel/api_loadbalancer.go index 51c87e3abee10..f6ec257be88f6 100644 --- a/pkg/model/scalewaymodel/api_loadbalancer.go +++ b/pkg/model/scalewaymodel/api_loadbalancer.go @@ -91,7 +91,7 @@ func (b *APILoadBalancerModelBuilder) Build(c *fi.CloudupModelBuilderContext) er // if we're not going to use an alias for it loadBalancer.ForAPIServer = true - if b.Cluster.UsesNoneDNS() { + if b.Cluster.UsesNoneDNS() || b.UseKopsControllerForNodeBootstrap() { lbBackendKopsController, lbFrontendKopsController := createLbBackendAndFrontend("kops-controller", wellknownports.KopsControllerPort, zone, loadBalancer) lbBackendKopsController.Lifecycle = b.Lifecycle c.AddTask(lbBackendKopsController) diff --git a/upup/pkg/fi/cloudup/apply_cluster.go b/upup/pkg/fi/cloudup/apply_cluster.go index fead5b806fbb0..37c834d2f72be 100644 --- a/upup/pkg/fi/cloudup/apply_cluster.go +++ b/upup/pkg/fi/cloudup/apply_cluster.go @@ -1465,7 +1465,7 @@ func (n *nodeUpConfigBuilder) BuildConfig(ig *kops.InstanceGroup, apiserverAddit } else { // If we do have a fixed IP, we use it (on some clouds, initially) switch cluster.Spec.GetCloudProvider() { - case kops.CloudProviderHetzner: + case kops.CloudProviderHetzner, kops.CloudProviderScaleway: bootConfig.APIServerIPs = controlPlaneIPs } } From b9807d49e8d7ae4130384b4c8782e4694e915588 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Le=C3=AFla=20MARABESE?= Date: Wed, 14 Jun 2023 15:41:30 +0200 Subject: [PATCH 5/5] update expected + headers --- cmd/kops/integration_test.go | 1 - ...object_nodeupconfig-nodes-fr-par-1_content | 25 +-------------- ...minimal.k8s.local-addons-bootstrap_content | 9 +----- ...-controller.addons.k8s.io-k8s-1.16_content | 4 ++- ...-addons-rbac.addons.k8s.io-k8s-1.8_content | 19 ----------- ...y_instance_server_nodes-fr-par-1_user_data | 27 ++++++++++++++-- .../minimal_scaleway/kubernetes.tf | 32 +++++++++++-------- upup/pkg/fi/cloudup/scaleway/authenticator.go | 16 ++++++++++ upup/pkg/fi/cloudup/scaleway/verifier.go | 16 ++++++++++ 9 files changed, 81 insertions(+), 68 deletions(-) delete mode 100644 tests/integration/update_cluster/minimal_scaleway/data/aws_s3_object_scw-minimal.k8s.local-addons-rbac.addons.k8s.io-k8s-1.8_content diff --git a/cmd/kops/integration_test.go b/cmd/kops/integration_test.go index 75aa39a69eb24..e0991a855349d 100644 --- a/cmd/kops/integration_test.go +++ b/cmd/kops/integration_test.go @@ -1810,7 +1810,6 @@ func (i *integrationTest) runTestTerraformScaleway(t *testing.T) { "aws_s3_object_"+i.clusterName+"-addons-kubelet-api.rbac.addons.k8s.io-k8s-1.9_content", "aws_s3_object_"+i.clusterName+"-addons-limit-range.addons.k8s.io_content", "aws_s3_object_"+i.clusterName+"-addons-networking.cilium.io-k8s-1.16_content", - "aws_s3_object_"+i.clusterName+"-addons-rbac.addons.k8s.io-k8s-1.8_content", "scaleway_instance_server_control-plane-fr-par-1_user_data", "scaleway_instance_server_nodes-fr-par-1_user_data", ) diff --git a/tests/integration/update_cluster/minimal_scaleway/data/aws_s3_object_nodeupconfig-nodes-fr-par-1_content b/tests/integration/update_cluster/minimal_scaleway/data/aws_s3_object_nodeupconfig-nodes-fr-par-1_content index 85321bc3f93f4..da014335ffc2b 100644 --- a/tests/integration/update_cluster/minimal_scaleway/data/aws_s3_object_nodeupconfig-nodes-fr-par-1_content +++ b/tests/integration/update_cluster/minimal_scaleway/data/aws_s3_object_nodeupconfig-nodes-fr-par-1_content @@ -15,36 +15,13 @@ Assets: - 54e79e4d48b9e191767e4abc08be1a8476a1c757e9a9f8c45c6ded001226867f@https://github.com/opencontainers/runc/releases/download/v1.1.5/runc.arm64 - 2f599c3d54f4c4bdbcc95aaf0c7b513a845d8f9503ec5b34c9f86aa1bc34fc0c@https://artifacts.k8s.io/binaries/kops/1.21.0-alpha.1/linux/arm64/protokube,https://github.com/kubernetes/kops/releases/download/v1.21.0-alpha.1/protokube-linux-arm64 - 9d842e3636a95de2315cdea2be7a282355aac0658ef0b86d5dc2449066538f13@https://artifacts.k8s.io/binaries/kops/1.21.0-alpha.1/linux/arm64/channels,https://github.com/kubernetes/kops/releases/download/v1.21.0-alpha.1/channels-linux-arm64 -CAs: - kubernetes-ca: | - -----BEGIN CERTIFICATE----- - MIIBbjCCARigAwIBAgIMFpANqBD8NSD82AUSMA0GCSqGSIb3DQEBCwUAMBgxFjAU - BgNVBAMTDWt1YmVybmV0ZXMtY2EwHhcNMjEwNzA3MDcwODAwWhcNMzEwNzA3MDcw - ODAwWjAYMRYwFAYDVQQDEw1rdWJlcm5ldGVzLWNhMFwwDQYJKoZIhvcNAQEBBQAD - SwAwSAJBANFI3zr0Tk8krsW8vwjfMpzJOlWQ8616vG3YPa2qAgI7V4oKwfV0yIg1 - jt+H6f4P/wkPAPTPTfRp9Iy8oHEEFw0CAwEAAaNCMEAwDgYDVR0PAQH/BAQDAgEG - MA8GA1UdEwEB/wQFMAMBAf8wHQYDVR0OBBYEFNG3zVjTcLlJwDsJ4/K9DV7KohUA - MA0GCSqGSIb3DQEBCwUAA0EAB8d03fY2w7WKpfO29qI295pu2C4ca9AiVGOpgSc8 - tmQsq6rcxt3T+rb589PVtz0mw/cKTxOk6gH2CCC+yHfy2w== - -----END CERTIFICATE----- - -----BEGIN CERTIFICATE----- - MIIBbjCCARigAwIBAgIMFpANvmSa0OAlYmXKMA0GCSqGSIb3DQEBCwUAMBgxFjAU - BgNVBAMTDWt1YmVybmV0ZXMtY2EwHhcNMjEwNzA3MDcwOTM2WhcNMzEwNzA3MDcw - OTM2WjAYMRYwFAYDVQQDEw1rdWJlcm5ldGVzLWNhMFwwDQYJKoZIhvcNAQEBBQAD - SwAwSAJBAMF6F4aZdpe0RUpyykaBpWwZCnwbffhYGOw+fs6RdLuUq7QCNmJm/Eq7 - WWOziMYDiI9SbclpD+6QiJ0N3EqppVUCAwEAAaNCMEAwDgYDVR0PAQH/BAQDAgEG - MA8GA1UdEwEB/wQFMAMBAf8wHQYDVR0OBBYEFLImp6ARjPDAH6nhI+scWVt3Q9bn - MA0GCSqGSIb3DQEBCwUAA0EAVQVx5MUtuAIeePuP9o51xtpT2S6Fvfi8J4ICxnlA - 9B7UD2ushcVFPtaeoL9Gfu8aY4KJBeqqg5ojl4qmRnThjw== - -----END CERTIFICATE----- +CAs: {} ClusterName: scw-minimal.k8s.local ContainerRuntime: containerd Hooks: - null - null KeypairIDs: - kube-proxy: "6986354184403674830529235586" - kubelet: "6986354184404014133128804066" kubernetes-ca: "6982820025135291416230495506" KubeProxy: null KubeletConfig: diff --git a/tests/integration/update_cluster/minimal_scaleway/data/aws_s3_object_scw-minimal.k8s.local-addons-bootstrap_content b/tests/integration/update_cluster/minimal_scaleway/data/aws_s3_object_scw-minimal.k8s.local-addons-bootstrap_content index be600789bb6dc..58357b82c27cf 100644 --- a/tests/integration/update_cluster/minimal_scaleway/data/aws_s3_object_scw-minimal.k8s.local-addons-bootstrap_content +++ b/tests/integration/update_cluster/minimal_scaleway/data/aws_s3_object_scw-minimal.k8s.local-addons-bootstrap_content @@ -6,7 +6,7 @@ spec: addons: - id: k8s-1.16 manifest: kops-controller.addons.k8s.io/k8s-1.16.yaml - manifestHash: bffd10e2291b38f02725cd04366b8029878272a79e68825e88d98a681068cd6c + manifestHash: 0652ed8a25e088a043a9f162f3b73dba1b9b5dcbb8efcb2b6550826d81f32a2b name: kops-controller.addons.k8s.io needsRollingUpdate: control-plane selector: @@ -19,13 +19,6 @@ spec: selector: k8s-addon: coredns.addons.k8s.io version: 9.99.0 - - id: k8s-1.8 - manifest: rbac.addons.k8s.io/k8s-1.8.yaml - manifestHash: f81bd7c57bc1902ca342635d7ad7d01b82dfeaff01a1192b076e66907d87871e - name: rbac.addons.k8s.io - selector: - k8s-addon: rbac.addons.k8s.io - version: 9.99.0 - id: k8s-1.9 manifest: kubelet-api.rbac.addons.k8s.io/k8s-1.9.yaml manifestHash: 01c120e887bd98d82ef57983ad58a0b22bc85efb48108092a24c4b82e4c9ea81 diff --git a/tests/integration/update_cluster/minimal_scaleway/data/aws_s3_object_scw-minimal.k8s.local-addons-kops-controller.addons.k8s.io-k8s-1.16_content b/tests/integration/update_cluster/minimal_scaleway/data/aws_s3_object_scw-minimal.k8s.local-addons-kops-controller.addons.k8s.io-k8s-1.16_content index 03d5bba7a48fe..6f60fd70112e3 100644 --- a/tests/integration/update_cluster/minimal_scaleway/data/aws_s3_object_scw-minimal.k8s.local-addons-kops-controller.addons.k8s.io-k8s-1.16_content +++ b/tests/integration/update_cluster/minimal_scaleway/data/aws_s3_object_scw-minimal.k8s.local-addons-kops-controller.addons.k8s.io-k8s-1.16_content @@ -1,7 +1,7 @@ apiVersion: v1 data: config.yaml: | - {"clusterName":"scw-minimal.k8s.local","cloud":"scaleway","configBase":"memfs://tests/scw-minimal.k8s.local","secretStore":"memfs://tests/scw-minimal.k8s.local/secrets","discovery":{"enabled":true}} + {"clusterName":"scw-minimal.k8s.local","cloud":"scaleway","configBase":"memfs://tests/scw-minimal.k8s.local","secretStore":"memfs://tests/scw-minimal.k8s.local/secrets","server":{"Listen":":3988","provider":{"scaleway":{}},"serverKeyPath":"/etc/kubernetes/kops-controller/pki/kops-controller.key","serverCertificatePath":"/etc/kubernetes/kops-controller/pki/kops-controller.crt","caBasePath":"/etc/kubernetes/kops-controller/pki","signingCAs":["kubernetes-ca"],"certNames":["kubelet","kubelet-server"]},"discovery":{"enabled":true}} kind: ConfigMap metadata: creationTimestamp: null @@ -32,6 +32,8 @@ spec: k8s-app: kops-controller template: metadata: + annotations: + dns.alpha.kubernetes.io/internal: kops-controller.internal.scw-minimal.k8s.local creationTimestamp: null labels: k8s-addon: kops-controller.addons.k8s.io diff --git a/tests/integration/update_cluster/minimal_scaleway/data/aws_s3_object_scw-minimal.k8s.local-addons-rbac.addons.k8s.io-k8s-1.8_content b/tests/integration/update_cluster/minimal_scaleway/data/aws_s3_object_scw-minimal.k8s.local-addons-rbac.addons.k8s.io-k8s-1.8_content deleted file mode 100644 index ba115283a8fbd..0000000000000 --- a/tests/integration/update_cluster/minimal_scaleway/data/aws_s3_object_scw-minimal.k8s.local-addons-rbac.addons.k8s.io-k8s-1.8_content +++ /dev/null @@ -1,19 +0,0 @@ -apiVersion: rbac.authorization.k8s.io/v1 -kind: ClusterRoleBinding -metadata: - creationTimestamp: null - labels: - addon.kops.k8s.io/name: rbac.addons.k8s.io - addonmanager.kubernetes.io/mode: Reconcile - app.kubernetes.io/managed-by: kops - k8s-addon: rbac.addons.k8s.io - kubernetes.io/cluster-service: "true" - name: kubelet-cluster-admin -roleRef: - apiGroup: rbac.authorization.k8s.io - kind: ClusterRole - name: system:node -subjects: -- apiGroup: rbac.authorization.k8s.io - kind: User - name: kubelet diff --git a/tests/integration/update_cluster/minimal_scaleway/data/scaleway_instance_server_nodes-fr-par-1_user_data b/tests/integration/update_cluster/minimal_scaleway/data/scaleway_instance_server_nodes-fr-par-1_user_data index 6fee07127d0b9..17a67b8c056ee 100644 --- a/tests/integration/update_cluster/minimal_scaleway/data/scaleway_instance_server_nodes-fr-par-1_user_data +++ b/tests/integration/update_cluster/minimal_scaleway/data/scaleway_instance_server_nodes-fr-par-1_user_data @@ -150,10 +150,33 @@ __EOF_CLUSTER_SPEC cat > conf/kube_env.yaml << '__EOF_KUBE_ENV' CloudProvider: scaleway ClusterName: scw-minimal.k8s.local -ConfigBase: memfs://tests/scw-minimal.k8s.local +ConfigServer: + CACertificates: | + -----BEGIN CERTIFICATE----- + MIIBbjCCARigAwIBAgIMFpANqBD8NSD82AUSMA0GCSqGSIb3DQEBCwUAMBgxFjAU + BgNVBAMTDWt1YmVybmV0ZXMtY2EwHhcNMjEwNzA3MDcwODAwWhcNMzEwNzA3MDcw + ODAwWjAYMRYwFAYDVQQDEw1rdWJlcm5ldGVzLWNhMFwwDQYJKoZIhvcNAQEBBQAD + SwAwSAJBANFI3zr0Tk8krsW8vwjfMpzJOlWQ8616vG3YPa2qAgI7V4oKwfV0yIg1 + jt+H6f4P/wkPAPTPTfRp9Iy8oHEEFw0CAwEAAaNCMEAwDgYDVR0PAQH/BAQDAgEG + MA8GA1UdEwEB/wQFMAMBAf8wHQYDVR0OBBYEFNG3zVjTcLlJwDsJ4/K9DV7KohUA + MA0GCSqGSIb3DQEBCwUAA0EAB8d03fY2w7WKpfO29qI295pu2C4ca9AiVGOpgSc8 + tmQsq6rcxt3T+rb589PVtz0mw/cKTxOk6gH2CCC+yHfy2w== + -----END CERTIFICATE----- + -----BEGIN CERTIFICATE----- + MIIBbjCCARigAwIBAgIMFpANvmSa0OAlYmXKMA0GCSqGSIb3DQEBCwUAMBgxFjAU + BgNVBAMTDWt1YmVybmV0ZXMtY2EwHhcNMjEwNzA3MDcwOTM2WhcNMzEwNzA3MDcw + OTM2WjAYMRYwFAYDVQQDEw1rdWJlcm5ldGVzLWNhMFwwDQYJKoZIhvcNAQEBBQAD + SwAwSAJBAMF6F4aZdpe0RUpyykaBpWwZCnwbffhYGOw+fs6RdLuUq7QCNmJm/Eq7 + WWOziMYDiI9SbclpD+6QiJ0N3EqppVUCAwEAAaNCMEAwDgYDVR0PAQH/BAQDAgEG + MA8GA1UdEwEB/wQFMAMBAf8wHQYDVR0OBBYEFLImp6ARjPDAH6nhI+scWVt3Q9bn + MA0GCSqGSIb3DQEBCwUAA0EAVQVx5MUtuAIeePuP9o51xtpT2S6Fvfi8J4ICxnlA + 9B7UD2ushcVFPtaeoL9Gfu8aY4KJBeqqg5ojl4qmRnThjw== + -----END CERTIFICATE----- + servers: + - https://kops-controller.internal.scw-minimal.k8s.local:3988/ InstanceGroupName: nodes-fr-par-1 InstanceGroupRole: Node -NodeupConfigHash: ThbM3OQQUCmecnKq4GQW0fdWp6sjwEiAjfmqga3QcXY= +NodeupConfigHash: l6ITXtzPONIgO+uyEAe0rYXGYgBVjPkJ9/Ov2xKxX/U= __EOF_KUBE_ENV diff --git a/tests/integration/update_cluster/minimal_scaleway/kubernetes.tf b/tests/integration/update_cluster/minimal_scaleway/kubernetes.tf index 88dfc22509086..3cc78a9cae0f6 100644 --- a/tests/integration/update_cluster/minimal_scaleway/kubernetes.tf +++ b/tests/integration/update_cluster/minimal_scaleway/kubernetes.tf @@ -148,14 +148,6 @@ resource "aws_s3_object" "scw-minimal-k8s-local-addons-networking-cilium-io-k8s- server_side_encryption = "AES256" } -resource "aws_s3_object" "scw-minimal-k8s-local-addons-rbac-addons-k8s-io-k8s-1-8" { - bucket = "testingBucket" - content = file("${path.module}/data/aws_s3_object_scw-minimal.k8s.local-addons-rbac.addons.k8s.io-k8s-1.8_content") - key = "tests/scw-minimal.k8s.local/addons/rbac.addons.k8s.io/k8s-1.8.yaml" - provider = aws.files - server_side_encryption = "AES256" -} - resource "aws_s3_object" "scw-minimal-k8s-local-addons-scaleway-cloud-controller-addons-k8s-io-k8s-1-24" { bucket = "testingBucket" content = file("${path.module}/data/aws_s3_object_scw-minimal.k8s.local-addons-scaleway-cloud-controller.addons.k8s.io-k8s-1.24_content") @@ -226,18 +218,32 @@ resource "scaleway_lb" "api-scw-minimal-k8s-local" { type = "LB-S" } -resource "scaleway_lb_backend" "lb-backend" { +resource "scaleway_lb_backend" "lb-backend-https" { forward_port = 443 forward_protocol = "tcp" lb_id = scaleway_lb.api-scw-minimal-k8s-local.id - name = "lb-backend" + name = "lb-backend-https" +} + +resource "scaleway_lb_backend" "lb-backend-kops-controller" { + forward_port = 3988 + forward_protocol = "tcp" + lb_id = scaleway_lb.api-scw-minimal-k8s-local.id + name = "lb-backend-kops-controller" } -resource "scaleway_lb_frontend" "lb-frontend" { - backend_id = scaleway_lb_backend.lb-backend.id +resource "scaleway_lb_frontend" "lb-frontend-https" { + backend_id = scaleway_lb_backend.lb-backend-https.id inbound_port = 443 lb_id = scaleway_lb.api-scw-minimal-k8s-local.id - name = "lb-frontend" + name = "lb-frontend-https" +} + +resource "scaleway_lb_frontend" "lb-frontend-kops-controller" { + backend_id = scaleway_lb_backend.lb-backend-kops-controller.id + inbound_port = 3988 + lb_id = scaleway_lb.api-scw-minimal-k8s-local.id + name = "lb-frontend-kops-controller" } resource "scaleway_lb_ip" "api-scw-minimal-k8s-local" { diff --git a/upup/pkg/fi/cloudup/scaleway/authenticator.go b/upup/pkg/fi/cloudup/scaleway/authenticator.go index 168008f435668..b07118ec7a71f 100644 --- a/upup/pkg/fi/cloudup/scaleway/authenticator.go +++ b/upup/pkg/fi/cloudup/scaleway/authenticator.go @@ -1,3 +1,19 @@ +/* +Copyright 2023 The Kubernetes Authors. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + package scaleway import ( diff --git a/upup/pkg/fi/cloudup/scaleway/verifier.go b/upup/pkg/fi/cloudup/scaleway/verifier.go index 504efeee06853..fa64320911dc0 100644 --- a/upup/pkg/fi/cloudup/scaleway/verifier.go +++ b/upup/pkg/fi/cloudup/scaleway/verifier.go @@ -1,3 +1,19 @@ +/* +Copyright 2023 The Kubernetes Authors. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + package scaleway import (