Merge pull request #172 from weaveworks/update-to-v1alpha3

Update to v1alpha3 - Since upstream CAPI has changed so much this is an extremely breaking change: there is no automated upgrade from v1alpha1. - The v1alpha1 types are left in place for anyone who wants to run programs containing both. - We now run two controllers - upstream CAPI controller for Machine and Cluster, and wks controller for BareMetalMachine and BareMetalCluster. - Minimum Kubernetes version is now 1.16.0, as required by upstream CAPI. - Examples are not updated yet
weaveworks · Jun 24, 2020 · 0a81eff · 0a81eff
2 parents 86a5fdd + 2b644aa
commit 0a81eff
Show file tree

Hide file tree

Showing 65 changed files with 4,935 additions and 2,571 deletions.
diff --git a/README.md b/README.md
@@ -1,5 +1,9 @@
 # Weave Kubernetes System Control - `wksctl`
 
+**Please note that the code has recently updated from ClusterAPI v1alpha1 to v1alpha3 and as a result Everything Has Changed**
+While this note is in the README you may find inconsistencies in the code, and between the code, examples and documentation.
+Sorry about that.  Feel free to still open issues and/or ask questions [as below](#getting-help).
+
 `wksctl` allows simple creation of a Kubernetes cluster given a **set of IP addresses** and an **SSH key**. It can be run in a standalone environment but is best used via a [GitOps approach](https://www.weave.works/technologies/gitops/) in which cluster and machine descriptions are stored in Git and the state of the cluster tracks changes to the descriptions.
 
 Its features include:

diff --git a/cmd/controller/main.go b/cmd/controller/main.go
@@ -1,20 +1,20 @@
 package main
 
 import (
+	"context"
 	"fmt"
 	"os"
 
 	log "github.com/sirupsen/logrus"
 	"github.com/spf13/cobra"
-	"github.com/weaveworks/wksctl/pkg/apis"
-	"github.com/weaveworks/wksctl/pkg/apis/wksprovider/controller/wksctl"
+	wks "github.com/weaveworks/wksctl/pkg/apis/wksprovider/controller/wksctl"
+	baremetalv1 "github.com/weaveworks/wksctl/pkg/baremetal/v1alpha3"
 	machineutil "github.com/weaveworks/wksctl/pkg/cluster/machine"
 	"k8s.io/client-go/kubernetes"
-	clusterapis "sigs.k8s.io/cluster-api/pkg/apis"
-	clustercommon "sigs.k8s.io/cluster-api/pkg/apis/cluster/common"
-	capicluster "sigs.k8s.io/cluster-api/pkg/controller/cluster"
-	capimachine "sigs.k8s.io/cluster-api/pkg/controller/machine"
+	clusterv1 "sigs.k8s.io/cluster-api/api/v1alpha3"
+	"sigs.k8s.io/controller-runtime/pkg/client"
 	"sigs.k8s.io/controller-runtime/pkg/client/config"
+	"sigs.k8s.io/controller-runtime/pkg/controller"
 	"sigs.k8s.io/controller-runtime/pkg/manager"
 	"sigs.k8s.io/controller-runtime/pkg/runtime/signals"
 )
@@ -28,8 +28,8 @@ func main() {
 	Execute()
 }
 
-func initializeControllerNamespace() (string, error) {
-	controllerNamespace, err := machineutil.GetKubernetesNamespaceFromMachines()
+func initializeControllerNamespace(c client.Client) (string, error) {
+	controllerNamespace, err := machineutil.GetKubernetesNamespaceFromMachines(context.Background(), c)
 	if err != nil {
 		return "", err
 	}
@@ -75,6 +75,8 @@ func preRun(cmd *cobra.Command, args []string) {
 }
 
 func run(cmd *cobra.Command, args []string) {
+	log.Infof("Starting wks controller version %s", version)
+
 	cfg, err := config.GetConfig()
 	if err != nil {
 		log.Fatalf("failed to get the coordinates of the API server: %v", err)
@@ -87,48 +89,51 @@ func run(cmd *cobra.Command, args []string) {
 	if err != nil {
 		log.Fatalf("failed to create the cluster manager: %v", err)
 	}
-	ctlrNamespace, err := initializeControllerNamespace()
+
+	log.Info("registering scheme for all resources")
+	if err := baremetalv1.AddToScheme(mgr.GetScheme()); err != nil {
+		log.Fatal(err)
+	}
+	if err := clusterv1.AddToScheme(mgr.GetScheme()); err != nil {
+		log.Fatal(err)
+	}
+
+	log.Info("registering controllers to the cluster manager")
+	clusterReconciler, err := wks.NewClusterReconciler(mgr.GetClient(), mgr.GetEventRecorderFor(wks.ProviderName+"-controller"))
 	if err != nil {
-		log.Fatalf("failed to get controller namespace: %s", err)
+		log.Fatal(err)
+	}
+	if err = clusterReconciler.SetupWithManager(mgr, controller.Options{MaxConcurrentReconciles: 1}); err != nil {
+		log.Fatal(err)
+	}
+
+	var ctlrNamespace string
+	{
+		// Create another client as we can't use the manager's one until it is started
+		client, err := client.New(mgr.GetConfig(), client.Options{Scheme: mgr.GetScheme()})
+		if err != nil {
+			log.Fatalf("failed to create client: %s", err)
+		}
+		ctlrNamespace, err = initializeControllerNamespace(client)
+		if err != nil {
+			log.Fatalf("failed to get controller namespace x: %s", err)
+		}
 	}
-	log.Info("initializing machine actuator")
-	machineActuator, err := wks.NewMachineActuator(wks.MachineActuatorParams{
-		EventRecorder:       mgr.GetRecorder(wks.ProviderName + "-controller"),
+
+	machineController, err := wks.NewMachineController(wks.MachineControllerParams{
+		EventRecorder:       mgr.GetEventRecorderFor(wks.ProviderName + "-controller"),
 		Client:              mgr.GetClient(),
 		ClientSet:           clientSet,
 		ControllerNamespace: ctlrNamespace,
-		Scheme:              mgr.GetScheme(),
 		Verbose:             options.verbose,
 	})
 	if err != nil {
 		log.Fatalf("failed to create the machine actuator: %v", err)
 	}
-
-	log.Info("initializing cluster actuator")
-	clusterActuator, err := wks.NewClusterActuator(wks.ClusterActuatorParams{
-		EventRecorder: mgr.GetRecorder(wks.ProviderName + "-controller"),
-		Client:        mgr.GetClient(),
-		ClientSet:     clientSet,
-		Scheme:        mgr.GetScheme(),
-	})
-	if err != nil {
-		log.Fatalf("failed to create the cluster actuator: %v", err)
-	}
-
-	clustercommon.RegisterClusterProvisioner(wks.ProviderName, clusterActuator)
-
-	log.Info("registering scheme for all resources")
-	if err := apis.AddToScheme(mgr.GetScheme()); err != nil {
-		log.Fatal(err)
-	}
-	if err := clusterapis.AddToScheme(mgr.GetScheme()); err != nil {
+	if err = machineController.SetupWithManager(mgr, controller.Options{MaxConcurrentReconciles: 1}); err != nil {
 		log.Fatal(err)
 	}
 
-	log.Info("registering controllers to the cluster manager")
-	capimachine.AddWithActuator(mgr, machineActuator)
-	capicluster.AddWithActuator(mgr, clusterActuator)
-
 	log.Info("starting the cluster manager")
 	log.Fatal(mgr.Start(signals.SetupSignalHandler()))
 }
diff --git a/cmd/wksctl/apply/apply.go b/cmd/wksctl/apply/apply.go
@@ -147,9 +147,12 @@ func (a *Applier) initiateCluster(clusterManifestPath, machinesManifestPath stri
 	}
 
 	// TODO(damien): Transform the controller image into an addon.
-	controllerImage, err := addons.UpdateImage(a.Params.controllerImage, sp.ClusterSpec.ImageRepository)
-	if err != nil {
-		return errors.Wrap(err, "failed to apply the cluster's image repository to the WKS controller's image")
+	controllerImage := a.Params.controllerImage
+	if controllerImage != "" {
+		controllerImage, err = addons.UpdateImage(a.Params.controllerImage, sp.ClusterSpec.ImageRepository)
+		if err != nil {
+			return errors.Wrap(err, "failed to apply the cluster's image repository to the WKS controller's image")
+		}
 	}
 
 	if err := installer.SetupSeedNode(wksos.SeedNodeParams{
@@ -180,7 +183,7 @@ func (a *Applier) initiateCluster(clusterManifestPath, machinesManifestPath stri
 		SealedSecretCertPath: a.Params.sealedSecretCertPath,
 		ConfigDirectory:      configDir,
 		ImageRepository:      sp.ClusterSpec.ImageRepository,
-		ExternalLoadBalancer: sp.ClusterSpec.APIServer.ExternalLoadBalancer,
+		ControlPlaneEndpoint: sp.ClusterSpec.ControlPlaneEndpoint,
 		AdditionalSANs:       sp.ClusterSpec.APIServer.AdditionalSANs,
 		Namespace:            ns,
 		AddonNamespaces:      addonNamespaces,

diff --git a/cmd/wksctl/applyaddons/applyaddons.go b/cmd/wksctl/applyaddons/applyaddons.go
@@ -11,11 +11,9 @@ import (
 	"github.com/spf13/cobra"
 	"github.com/weaveworks/launcher/pkg/kubectl"
 	"github.com/weaveworks/wksctl/pkg/addons"
-	"github.com/weaveworks/wksctl/pkg/kubernetes/config"
 	"github.com/weaveworks/wksctl/pkg/specs"
 	"github.com/weaveworks/wksctl/pkg/utilities/manifest"
 	"github.com/weaveworks/wksctl/pkg/utilities/path"
-	clusterv1 "sigs.k8s.io/cluster-api/pkg/apis/cluster/v1alpha1"
 )
 
 var Cmd = &cobra.Command{
@@ -42,17 +40,6 @@ func init() {
 		&applyAddonsOptions.namespace, "namespace", manifest.DefaultNamespace, "namespace portion of kubeconfig path")
 }
 
-func applyAddons(cluster *clusterv1.Cluster, machines []*clusterv1.Machine, basePath string) error {
-	opts := &applyAddonsOptions
-	sp := specs.New(cluster, machines)
-	kubeconfig, err := config.NewKubeConfig(opts.artifactDirectory, machines)
-	if err != nil {
-		log.Fatal("Error generating kubeconf", err)
-	}
-
-	return applyAddonsUsingConfig(sp, basePath, kubeconfig)
-}
-
 func applyAddonsUsingConfig(sp *specs.Specs, basePath, kubeconfig string) error {
 	fmt.Println("==> Applying addons (2)")
 

diff --git a/cmd/wksctl/main.go b/cmd/wksctl/main.go
@@ -6,6 +6,9 @@ import (
 	log "github.com/sirupsen/logrus"
 	"github.com/spf13/cobra"
 	"github.com/weaveworks/go-checkpoint"
+	"k8s.io/client-go/kubernetes/scheme"
+	clusterv1 "sigs.k8s.io/cluster-api/api/v1alpha3"
+
 	"github.com/weaveworks/wksctl/cmd/wksctl/addon"
 	"github.com/weaveworks/wksctl/cmd/wksctl/apply"
 	"github.com/weaveworks/wksctl/cmd/wksctl/applyaddons"
@@ -17,6 +20,7 @@ import (
 	"github.com/weaveworks/wksctl/cmd/wksctl/registrysynccommands"
 	"github.com/weaveworks/wksctl/cmd/wksctl/version"
 	"github.com/weaveworks/wksctl/cmd/wksctl/zshcompletions"
+	baremetalv1 "github.com/weaveworks/wksctl/pkg/baremetal/v1alpha3"
 	v "github.com/weaveworks/wksctl/pkg/version"
 )
 
@@ -41,6 +45,9 @@ func configureLogger(cmd *cobra.Command, args []string) {
 }
 
 func main() {
+	clusterv1.AddToScheme(scheme.Scheme)
+	baremetalv1.AddToScheme(scheme.Scheme)
+
 	rootCmd.PersistentFlags().BoolVarP(&options.verbose, "verbose", "v", false, "Enable verbose output")
 
 	rootCmd.AddCommand(addon.Cmd)

diff --git a/cmd/wksctl/registrysynccommands/registrysynccommands.go b/cmd/wksctl/registrysynccommands/registrysynccommands.go
@@ -10,11 +10,7 @@ import (
 	"github.com/weaveworks/wksctl/pkg/cluster/machine"
 	"github.com/weaveworks/wksctl/pkg/kubernetes"
 	"github.com/weaveworks/wksctl/pkg/registry"
-	"github.com/weaveworks/wksctl/pkg/utilities"
 	v "github.com/weaveworks/wksctl/pkg/utilities/version"
-	"k8s.io/apimachinery/pkg/util/validation/field"
-	clusterv1 "sigs.k8s.io/cluster-api/pkg/apis/cluster/v1alpha1"
-	apierrors "sigs.k8s.io/cluster-api/pkg/errors"
 )
 
 var Cmd = &cobra.Command{
@@ -78,7 +74,7 @@ func registrySyncRun(cmd *cobra.Command, args []string) {
 
 func kubernetesVersionsRange() string {
 	if registrySyncOptions.machinesManifestPath != "" {
-		version, err := extractKubernetesVersionFromMachines(registrySyncOptions.machinesManifestPath)
+		version, _, err := machine.GetKubernetesVersionFromManifest(registrySyncOptions.machinesManifestPath)
 		if err != nil {
 			log.Fatalf("Failed to extract Kubernetes version from machines manifest: %s", err)
 		}
@@ -89,18 +85,3 @@ func kubernetesVersionsRange() string {
 	}
 	return v.AnyRange
 }
-
-func extractKubernetesVersionFromMachines(machinesManifestPath string) (string, error) {
-	errorsHandler := func(machines []*clusterv1.Machine, errors field.ErrorList) ([]*clusterv1.Machine, error) {
-		if len(errors) > 0 {
-			utilities.PrintErrors(errors)
-			return nil, apierrors.InvalidMachineConfiguration("%s failed validation", machinesManifestPath)
-		}
-		return machines, nil
-	}
-	machines, err := machine.ParseAndDefaultAndValidate(machinesManifestPath, errorsHandler)
-	if err != nil {
-		return "", err
-	}
-	return machines[0].Spec.Versions.Kubelet, nil
-}
diff --git a/docs/cluster.md b/docs/cluster.md
@@ -1,9 +1,13 @@
 # Cluster Definition
 
-Here's an example of the cluster definition.
+Here's an example of the cluster definition. It is spread across two
+objects: a `Cluster` which is defined by Kubernetes ClusterAPI and a
+`BareMetalCluster` defined by Weaveworks.  Here, both objects have the
+same name `example`, and the field `infrastructureRef` points from one
+to the other.
 
 ```
-apiVersion: cluster.k8s.io/v1alpha1
+apiVersion: cluster.x-k8s.io/v1alpha3
 kind: Cluster
 metadata:
   name: example
@@ -14,41 +18,47 @@ spec:
     pods:
       cidrBlocks: [192.168.0.0/16]
     serviceDomain: cluster.local
-  providerSpec:
-    value:
-      apiVersion: baremetalproviderspec/v1alpha1
-      kind: BareMetalClusterProviderSpec
-      sshKeyPath: cluster-key
-      user: root
-      os:
-        files:
-        - source:
-            configmap: repo
-            key: kubernetes.repo
-          destination: /etc/yum.repos.d/kubernetes.repo
-        - source:
-            configmap: repo
-            key: docker-ce.repo
-          destination: /etc/yum.repos.d/docker-ce.repo
-        - source:
-            configmap: docker
-            key: daemon.json
-          destination: /etc/docker/daemon.json
-      cri:
-        kind: docker
-        package: docker-ce
-        version: 19.03.8
-      apiServer:
-        extraArguments:
-        - name: alsologtostderr
-          value: "true"
-        - name: audit-log-maxsize
-          value: "10000"
-      kubeletArguments:
-      - name: alsologtostderr
-        value: "true"
-      - name: container-runtime
-        value: docker
+  infrastructureRef:
+      apiVersion: cluster.weave.works/v1alpha3
+      kind: BareMetalCluster
+      name: example
+---
+apiVersion: cluster.weave.works/v1alpha3
+kind: BareMetalCluster
+metadata:
+  name: example
+spec:
+  sshKeyPath: cluster-key
+  user: root
+  os:
+    files:
+    - source:
+        configmap: repo
+        key: kubernetes.repo
+      destination: /etc/yum.repos.d/kubernetes.repo
+    - source:
+        configmap: repo
+        key: docker-ce.repo
+      destination: /etc/yum.repos.d/docker-ce.repo
+    - source:
+        configmap: docker
+        key: daemon.json
+      destination: /etc/docker/daemon.json
+  cri:
+    kind: docker
+    package: docker-ce
+    version: 19.03.8
+  apiServer:
+    extraArguments:
+    - name: alsologtostderr
+      value: "true"
+    - name: audit-log-maxsize
+      value: "10000"
+  kubeletArguments:
+  - name: alsologtostderr
+    value: "true"
+  - name: container-runtime
+    value: docker
 ```
 
 ## Passing extra arguments to the API Server