✨ Add custerctl support for v1alpha2

Signed-off-by: Vince Prignano <vincepri@vmware.com>

cmd/clusterctl/clientcmd/configutil.go

@@ -23,6 +23,7 @@ import (
 	"k8s.io/client-go/rest"
 	"k8s.io/client-go/tools/clientcmd"
 	"k8s.io/client-go/tools/clientcmd/api"
+	"sigs.k8s.io/cluster-api/util/restmapper"
 	"sigs.k8s.io/controller-runtime/pkg/client"
 )
 
@@ -55,7 +56,9 @@ func NewControllerRuntimeClient(kubeconfigPath string, overrides clientcmd.Confi
 		return nil, err
 	}
 
-	return client.New(config, client.Options{})
+	return client.New(config, client.Options{
+		Mapper: restmapper.NewCached(config),
+	})
 }
 
 // newRestConfig creates a rest.Config for the given apiConfig

cmd/clusterctl/clusterdeployer/clusterclient/clusterclient.go

@@ -31,8 +31,10 @@ import (
 	"github.com/pkg/errors"
 	autoscalingv1 "k8s.io/api/autoscaling/v1"
 	apiv1 "k8s.io/api/core/v1"
+	corev1 "k8s.io/api/core/v1"
 	apierrors "k8s.io/apimachinery/pkg/api/errors"
 	metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
+	"k8s.io/apimachinery/pkg/apis/meta/v1/unstructured"
 	_ "k8s.io/client-go/plugin/pkg/client/auth" // nolint
 	tcmd "k8s.io/client-go/tools/clientcmd"
 	"k8s.io/klog"
@@ -67,22 +69,28 @@ var (
 type Client interface {
 	Apply(string) error
 	Close() error
+	CreateSecret(*corev1.Secret) error
 	CreateClusterObject(*clusterv1.Cluster) error
 	CreateMachineDeployments([]*clusterv1.MachineDeployment, string) error
 	CreateMachineSets([]*clusterv1.MachineSet, string) error
 	CreateMachines([]*clusterv1.Machine, string) error
+	CreateUnstructuredObject(*unstructured.Unstructured) error
 	Delete(string) error
 	DeleteClusters(string) error
 	DeleteNamespace(string) error
 	DeleteMachineDeployments(string) error
 	DeleteMachineSets(string) error
 	DeleteMachines(string) error
+	DeleteUnstructuredObjects(string, *unstructured.Unstructured) error
+	ForceDeleteSecret(namespace, name string) error
 	ForceDeleteCluster(namespace, name string) error
 	ForceDeleteMachine(namespace, name string) error
 	ForceDeleteMachineSet(namespace, name string) error
 	ForceDeleteMachineDeployment(namespace, name string) error
+	ForceDeleteUnstructuredObject(*unstructured.Unstructured) error
 	EnsureNamespace(string) error
 	GetKubeconfigFromSecret(namespace, clusterName string) (string, error)
+	GetClusterSecrets(*clusterv1.Cluster) ([]*corev1.Secret, error)
 	GetClusters(string) ([]*clusterv1.Cluster, error)
 	GetCluster(string, string) (*clusterv1.Cluster, error)
 	GetContextNamespace() string
@@ -96,7 +104,8 @@ type Client interface {
 	GetMachines(namespace string) ([]*clusterv1.Machine, error)
 	GetMachinesForCluster(*clusterv1.Cluster) ([]*clusterv1.Machine, error)
 	GetMachinesForMachineSet(*clusterv1.MachineSet) ([]*clusterv1.Machine, error)
-	ScaleStatefulSet(namespace, name string, scale int32) error
+	GetUnstructuredObject(*unstructured.Unstructured) error
+	ScaleDeployment(namespace, name string, scale int32) error
 	WaitForClusterV1alpha2Ready() error
 	WaitForResourceStatuses() error
 }
@@ -178,24 +187,37 @@ func (c *client) GetKubeconfigFromSecret(namespace, clusterName string) (string,
 	return string(data), nil
 }
 
-func (c *client) ScaleStatefulSet(ns string, name string, scale int32) error {
+func (c *client) ScaleDeployment(ns string, name string, scale int32) error {
 	clientset, err := clientcmd.NewCoreClientSetForDefaultSearchPath(c.kubeconfigFile, clientcmd.NewConfigOverrides())
 	if err != nil {
 		return errors.Wrap(err, "error creating core clientset")
 	}
 
-	_, err = clientset.AppsV1().StatefulSets(ns).UpdateScale(name, &autoscalingv1.Scale{
-		ObjectMeta: metav1.ObjectMeta{
-			Namespace: ns,
-			Name:      name,
-		},
-		Spec: autoscalingv1.ScaleSpec{
-			Replicas: scale,
-		},
-	})
-	if err != nil && !apierrors.IsNotFound(err) {
-		return err
+	deploymentClient := clientset.AppsV1().Deployments(ns)
+
+	if err := util.PollImmediate(retryAcquireClient, timeoutAcquireClient, func() (bool, error) {
+		_, err := deploymentClient.UpdateScale(name, &autoscalingv1.Scale{
+			ObjectMeta: metav1.ObjectMeta{
+				Namespace: ns,
+				Name:      name,
+			},
+			Spec: autoscalingv1.ScaleSpec{
+				Replicas: scale,
+			},
+		})
+		if err != nil && !apierrors.IsNotFound(err) {
+			return false, err
+		}
+
+		d, err := deploymentClient.Get(name, metav1.GetOptions{})
+		if err != nil {
+			return false, err
+		}
+		return d.Status.Replicas == 0 && d.Status.AvailableReplicas == 0 && d.Status.ReadyReplicas == 0, nil
+	}); err != nil {
+		return errors.Wrapf(err, "failed to scale deployment")
 	}
+
 	return nil
 }
 
@@ -230,11 +252,6 @@ func NewFromDefaultSearchPath(kubeconfigFile string, overrides tcmd.ConfigOverri
 		return nil, errors.Wrapf(err, "failed to acquire new client")
 	}
 
-	c, err := clientcmd.NewControllerRuntimeClient(kubeconfigFile, overrides)
-	if err != nil {
-		return nil, err
-	}
-
 	return &client{
 		kubeconfigFile:  kubeconfigFile,
 		clientSet:       c,
@@ -662,16 +679,91 @@ func (c *client) WaitForResourceStatuses() error {
 				klog.V(10).Info("retrying: machine status is empty")
 				return false, nil
 			}
-			// if m.Status.ProviderStatus == nil {
-			// 	klog.V(10).Info("retrying: machine.Status.ProviderStatus is not set")
-			// 	return false, nil
-			// }
 		}
 
 		return true, nil
 	})
 }
 
+func (c *client) GetClusterSecrets(cluster *clusterv1.Cluster) ([]*corev1.Secret, error) {
+	list := &corev1.SecretList{}
+	if err := c.clientSet.List(ctx, list, ctrlclient.InNamespace(cluster.Namespace)); err != nil {
+		return nil, errors.Wrapf(err, "error listing Secrets for Cluster %s/%s", cluster.Namespace, cluster.Name)
+	}
+
+	res := []*corev1.Secret{}
+	for i, secret := range list.Items {
+		if strings.HasPrefix(secret.Name, cluster.Name) {
+			res = append(res, &list.Items[i])
+		}
+	}
+	return res, nil
+}
+
+func (c *client) CreateSecret(s *corev1.Secret) error {
+	if err := c.clientSet.Create(context.Background(), s); err != nil {
+		return errors.Wrapf(err, "error creating Secret %s/%s", s.Namespace, s.Name)
+	}
+	return nil
+}
+
+func (c *client) ForceDeleteSecret(namespace, name string) error {
+	secret := &corev1.Secret{}
+	if err := c.clientSet.Get(ctx, ctrlclient.ObjectKey{Namespace: namespace, Name: name}, secret); err != nil {
+		return errors.Wrapf(err, "error getting Secret %s/%s", namespace, name)
+	}
+
+	if len(secret.Finalizers) > 0 {
+		secret.SetFinalizers([]string{})
+		if err := c.clientSet.Update(ctx, secret); err != nil {
+			return errors.Wrapf(err, "error removing finalizer for Secret %s/%s", secret.Namespace, secret.Name)
+		}
+	}
+
+	if err := c.clientSet.Delete(context.Background(), secret); err != nil {
+		return errors.Wrapf(err, "error deleting Secret %s/%s", secret.Namespace, secret.Name)
+	}
+	return nil
+}
+
+func (c *client) GetUnstructuredObject(u *unstructured.Unstructured) error {
+	key := ctrlclient.ObjectKey{Namespace: u.GetNamespace(), Name: u.GetName()}
+	if err := c.clientSet.Get(context.Background(), key, u); err != nil {
+		return errors.Wrapf(err, "error fetching unstructured object %q %v", u.GroupVersionKind(), key)
+	}
+	return nil
+}
+
+func (c *client) CreateUnstructuredObject(u *unstructured.Unstructured) error {
+	if err := c.clientSet.Create(context.Background(), u); err != nil {
+		return errors.Wrapf(err, "error creating unstructured object %q %s/%s",
+			u.GroupVersionKind(), u.GetNamespace(), u.GetName())
+	}
+	return nil
+}
+
+// DeleteUnstructuredObjects deletes all the unstructured objects of the same type in a namespace.
+// If the namespace is empty then all Clusters in all namespaces are deleted.
+func (c *client) DeleteUnstructuredObjects(namespace string, u *unstructured.Unstructured) error {
+	if err := c.clientSet.DeleteAllOf(ctx, u, ctrlclient.InNamespace(namespace)); err != nil {
+		return errors.Wrapf(err, "error deleting unstructured objects %q in namespace %q", u.GroupVersionKind(), namespace)
+	}
+	return nil
+}
+
+func (c *client) ForceDeleteUnstructuredObject(u *unstructured.Unstructured) error {
+	u.SetFinalizers([]string{})
+	if err := c.clientSet.Update(ctx, u); err != nil {
+		return errors.Wrapf(err, "error removing finalizer for unstructured object %q %s/%s",
+			u.GroupVersionKind(), u.GetNamespace(), u.GetName())
+	}
+	if err := c.clientSet.Delete(ctx, u); err != nil {
+		return errors.Wrapf(err, "error deleting unstructured object %q %s/%s",
+			u.GroupVersionKind(), u.GetNamespace(), u.GetName())
+	}
+	return nil
+}
+
 func (c *client) kubectlDelete(manifest string) error {
 	return c.kubectlManifestCmd("delete", manifest)
 }
@@ -765,11 +857,8 @@ func waitForMachineReady(cs ctrlclient.Client, machine *clusterv1.Machine) error
 			return false, nil
 		}
 
-		// TODO: update once machine controllers have a way to indicate a machine has been provisoned. https://github.com/kubernetes-sigs/cluster-api/issues/253
-		// Seeing a node cannot be purely relied upon because the provisioned control plane will not be registering with
-		// the stack that provisions it.
-		ready := machine.Status.NodeRef != nil || len(machine.Annotations) > 0
-		return ready, nil
+		// Return true if the Machine has a reference to a Node.
+		return machine.Status.NodeRef != nil, nil
 	})
 
 	return err

cmd/clusterctl/clusterdeployer/clusterdeployer.go

@@ -21,12 +21,14 @@ import (
 	"strings"
 
 	"github.com/pkg/errors"
+	"k8s.io/apimachinery/pkg/apis/meta/v1/unstructured"
 	"k8s.io/klog"
 	clusterv1 "sigs.k8s.io/cluster-api/api/v1alpha2"
 	"sigs.k8s.io/cluster-api/cmd/clusterctl/clusterdeployer/bootstrap"
 	"sigs.k8s.io/cluster-api/cmd/clusterctl/clusterdeployer/clusterclient"
 	"sigs.k8s.io/cluster-api/cmd/clusterctl/clusterdeployer/provider"
 	"sigs.k8s.io/cluster-api/cmd/clusterctl/phases"
+	"sigs.k8s.io/cluster-api/util/yaml"
 )
 
 type ClusterDeployer struct {
@@ -56,7 +58,10 @@ func New(
 }
 
 // Create the cluster from the provided cluster definition and machine list.
-func (d *ClusterDeployer) Create(cluster *clusterv1.Cluster, machines []*clusterv1.Machine, kubeconfigOutput string, providerComponentsStoreFactory provider.ComponentsStoreFactory) error {
+func (d *ClusterDeployer) Create(resources *yaml.ParseOutput, kubeconfigOutput string, providerComponentsStoreFactory provider.ComponentsStoreFactory) error {
+	cluster := resources.Clusters[0]
+	machines := resources.Machines
+
 	controlPlaneMachines, nodes, err := clusterclient.ExtractControlPlaneMachines(machines)
 	if err != nil {
 		return errors.Wrap(err, "unable to separate control plane machines from node machines")
@@ -81,16 +86,24 @@ func (d *ClusterDeployer) Create(cluster *clusterv1.Cluster, machines []*cluster
 	}
 
 	klog.Info("Provisioning target cluster via bootstrap cluster")
-	if err := phases.ApplyCluster(bootstrapClient, cluster); err != nil {
+	if err := phases.ApplyCluster(
+		bootstrapClient,
+		cluster,
+		yaml.ExtractClusterReferences(resources, cluster)...); err != nil {
 		return errors.Wrapf(err, "unable to create cluster %q in bootstrap cluster", cluster.Name)
 	}
 
 	if cluster.Namespace == "" {
 		cluster.Namespace = bootstrapClient.GetContextNamespace()
 	}
 
-	klog.Infof("Creating control plane machine in namespace %q", cluster.Namespace)
-	if err := phases.ApplyMachines(bootstrapClient, cluster.Namespace, []*clusterv1.Machine{controlPlaneMachines[0]}); err != nil {
+	firstControlPlane := controlPlaneMachines[0]
+	klog.Infof("Creating control plane machine %q in namespace %q", firstControlPlane.Name, cluster.Namespace)
+	if err := phases.ApplyMachines(
+		bootstrapClient,
+		cluster.Namespace,
+		[]*clusterv1.Machine{firstControlPlane},
+		yaml.ExtractMachineReferences(resources, firstControlPlane)...); err != nil {
 		return errors.Wrap(err, "unable to create control plane machine")
 	}
 
@@ -128,19 +141,32 @@ func (d *ClusterDeployer) Create(cluster *clusterv1.Cluster, machines []*cluster
 		// supported versions of k8s we are deploying (using kubeadm) have the fix.
 		klog.Info("Creating additional control plane machines in target cluster.")
 		for _, controlPlaneMachine := range controlPlaneMachines[1:] {
-			if err := phases.ApplyMachines(targetClient, cluster.Namespace, []*clusterv1.Machine{controlPlaneMachine}); err != nil {
+			if err := phases.ApplyMachines(
+				targetClient,
+				cluster.Namespace,
+				[]*clusterv1.Machine{controlPlaneMachine},
+				yaml.ExtractMachineReferences(resources, controlPlaneMachine)...,
+			); err != nil {
 				return errors.Wrap(err, "unable to create additional control plane machines")
 			}
 		}
 	}
 
 	klog.Info("Creating node machines in target cluster.")
-	if err := phases.ApplyMachines(targetClient, cluster.Namespace, nodes); err != nil {
+	extraMachineResources := []*unstructured.Unstructured{}
+	for _, m := range nodes {
+		extraMachineResources = append(extraMachineResources, yaml.ExtractMachineReferences(resources, m)...)
+	}
+	if err := phases.ApplyMachines(
+		targetClient,
+		cluster.Namespace,
+		nodes,
+		extraMachineResources...,
+	); err != nil {
 		return errors.Wrap(err, "unable to create node machines")
 	}
 
 	klog.Infof("Done provisioning cluster. You can now access your cluster with kubectl --kubeconfig %v", kubeconfigOutput)
-
 	return nil
 }