WIP curated packages full cluster lifecycle adaptations

Makefile

@@ -578,7 +578,7 @@ mocks: ## Generate mocks
 	${MOCKGEN} -destination=pkg/providers/docker/reconciler/mocks/reconciler.go -package=mocks -source "pkg/providers/docker/reconciler/reconciler.go"
 	${MOCKGEN} -destination=pkg/providers/tinkerbell/reconciler/mocks/reconciler.go -package=mocks -source "pkg/providers/tinkerbell/reconciler/reconciler.go"
 	${MOCKGEN} -destination=pkg/awsiamauth/reconciler/mocks/reconciler.go -package=mocks -source "pkg/awsiamauth/reconciler/reconciler.go"
-	${MOCKGEN} -destination=controllers/mocks/cluster_controller.go -package=mocks -source "controllers/cluster_controller.go" AWSIamConfigReconciler ClusterValidator
+	${MOCKGEN} -destination=controllers/mocks/cluster_controller.go -package=mocks -source "controllers/cluster_controller.go" AWSIamConfigReconciler ClusterValidator PackagesControllerClient
 	${MOCKGEN} -destination=pkg/workflow/task_mock_test.go -package=workflow_test -source "pkg/workflow/task.go"
 	${MOCKGEN} -destination=pkg/validations/createcluster/mocks/createcluster.go -package=mocks -source "pkg/validations/createcluster/createcluster.go"
 	${MOCKGEN} -destination=pkg/awsiamauth/mock_test.go -package=awsiamauth_test -source "pkg/awsiamauth/installer.go"

controllers/cluster_controller.go

@@ -3,10 +3,13 @@ package controllers
 import (
 	"context"
 	"fmt"
+	"os"
 	"time"
 
 	"github.com/go-logr/logr"
 	"github.com/pkg/errors"
+	"gopkg.in/yaml.v2"
+	corev1 "k8s.io/api/core/v1"
 	apierrors "k8s.io/apimachinery/pkg/api/errors"
 	metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
 	"k8s.io/apimachinery/pkg/types"
@@ -27,7 +30,10 @@ import (
 	"github.com/aws/eks-anywhere/pkg/controller/clientutil"
 	"github.com/aws/eks-anywhere/pkg/controller/clusters"
 	"github.com/aws/eks-anywhere/pkg/controller/handlers"
+	"github.com/aws/eks-anywhere/pkg/curatedpackages"
+	"github.com/aws/eks-anywhere/pkg/registrymirror"
 	"github.com/aws/eks-anywhere/pkg/utils/ptr"
+	"github.com/aws/eks-anywhere/release/api/v1alpha1"
 )
 
 const (
@@ -42,6 +48,7 @@ type ClusterReconciler struct {
 	providerReconcilerRegistry ProviderClusterReconcilerRegistry
 	awsIamAuth                 AWSIamConfigReconciler
 	clusterValidator           ClusterValidator
+	packageControllerClient    PackageControllerClient
 }
 
 type ProviderClusterReconcilerRegistry interface {
@@ -61,12 +68,13 @@ type ClusterValidator interface {
 }
 
 // NewClusterReconciler constructs a new ClusterReconciler.
-func NewClusterReconciler(client client.Client, registry ProviderClusterReconcilerRegistry, awsIamAuth AWSIamConfigReconciler, clusterValidator ClusterValidator) *ClusterReconciler {
+func NewClusterReconciler(client client.Client, registry ProviderClusterReconcilerRegistry, awsIamAuth AWSIamConfigReconciler, clusterValidator ClusterValidator, packageControllerClient PackageControllerClient) *ClusterReconciler {
 	return &ClusterReconciler{
 		client:                     client,
 		providerReconcilerRegistry: registry,
 		awsIamAuth:                 awsIamAuth,
 		clusterValidator:           clusterValidator,
+		packageControllerClient:    packageControllerClient,
 	}
 }
 
@@ -253,9 +261,91 @@ func (r *ClusterReconciler) postClusterProviderReconcile(ctx context.Context, lo
 		}
 	}
 
+	if !cluster.IsSelfManaged() {
+		if err := r.enablePackagesForWorkloadCluster(ctx, log, cluster); err != nil {
+			return controller.Result{}, err
+		}
+	}
+
 	return controller.Result{}, nil
 }
 
+type PackageControllerClient interface {
+	EnableCuratedPackagesFullLifecycle(context.Context, string, string, *v1alpha1.Image, *registrymirror.RegistryMirror, ...curatedpackages.PackageControllerClientOpt) error
+}
+
+var _ PackageControllerClient = (*curatedpackages.PackageControllerClient)(nil)
+
+func (r *ClusterReconciler) enablePackagesForWorkloadCluster(ctx context.Context, log logr.Logger, cluster *anywherev1.Cluster) error {
+
+	// TODO: This is set previously, but some tests don't set it, I should fix the tests.
+	if cluster.Spec.BundlesRef == nil {
+		return fmt.Errorf("no bundle ref found")
+	}
+	bundles := &v1alpha1.Bundles{}
+	nn := types.NamespacedName{
+		Name:      cluster.Spec.BundlesRef.Name,
+		Namespace: cluster.Spec.BundlesRef.Namespace,
+	}
+	if err := r.client.Get(ctx, nn, bundles); err != nil {
+		return err
+	}
+
+	verBundle := &v1alpha1.VersionsBundle{}
+	for _, b := range bundles.Spec.VersionsBundles {
+		if b.KubeVersion == string(cluster.Spec.KubernetesVersion) {
+			verBundle = &b
+			break
+		}
+	}
+	if verBundle == nil {
+		return fmt.Errorf("no bundle for k8s version %q", cluster.Spec.KubernetesVersion)
+	}
+	image, ok := verBundle.Charts()["eks-anywhere-packages"]
+	if !ok {
+		return fmt.Errorf("no chart image")
+	}
+
+	// Use the secret to get the kubeconfig for the workload cluster, and write it to a file I guess...
+	// Should I wait for it in fact?
+
+	kubeConfigNN := types.NamespacedName{
+		Namespace: constants.EksaSystemNamespace,
+		Name:      cluster.Name + "-kubeconfig",
+	}
+	kubeConfigSecret := &corev1.Secret{}
+	if err := r.client.Get(ctx, kubeConfigNN, kubeConfigSecret); err != nil {
+		return fmt.Errorf("getting kubeconfig secret: %w", err)
+	}
+	secretBytes, err := yaml.Marshal(kubeConfigSecret)
+	if err != nil {
+		return fmt.Errorf("marshaling secret %w", err)
+	}
+	f, err := os.CreateTemp("", "kubeconfig-*.yaml")
+	if err != nil {
+		return fmt.Errorf("opening kubeconfig file %w", err)
+	}
+	defer f.Close()
+	// TODO unlink
+
+	if _, err := f.Write(secretBytes); err != nil {
+		return fmt.Errorf("writing kubeconfig file %w", err)
+	}
+	f.Close()
+
+	rm := registrymirror.FromCluster(cluster)
+	var options []curatedpackages.PackageControllerClientOpt
+	r.packageControllerClient.EnableCuratedPackagesFullLifecycle(ctx,
+		cluster.Name,
+		f.Name(),
+		image,
+		rm,
+		options...,
+	)
+
+	return nil
+}
+
 func (r *ClusterReconciler) reconcileDelete(ctx context.Context, log logr.Logger, cluster *anywherev1.Cluster) (ctrl.Result, error) {
 	if cluster.IsSelfManaged() {
 		return ctrl.Result{}, errors.New("deleting self-managed clusters is not supported")

controllers/cluster_controller_test.go

@@ -68,7 +68,7 @@ func newVsphereClusterReconcilerTest(t *testing.T, objs ...runtime.Object) *vsph
 		Add(anywherev1.VSphereDatacenterKind, reconciler).
 		Build()
 
-	r := controllers.NewClusterReconciler(cl, &registry, iam, clusterValidator)
+	r := controllers.NewClusterReconciler(cl, &registry, iam, clusterValidator, nil)
 
 	return &vsphereClusterReconcilerTest{
 		govcClient: govcClient,
@@ -101,9 +101,9 @@ func TestClusterReconcilerReconcileSelfManagedCluster(t *testing.T) {
 
 	providerReconciler.EXPECT().ReconcileWorkerNodes(ctx, gomock.AssignableToTypeOf(logr.Logger{}), sameName(selfManagedCluster))
 
-	r := controllers.NewClusterReconciler(c, registry, iam, clusterValidator)
+	r := controllers.NewClusterReconciler(c, registry, iam, clusterValidator, nil)
 	result, err := r.Reconcile(ctx, clusterRequest(selfManagedCluster))
-	g.Expect(err).ToNot(HaveOccurred())
+	g.Expect(err).NotTo(HaveOccurred())
 	g.Expect(result).To(Equal(ctrl.Result{}))
 }
 
@@ -128,7 +128,7 @@ func TestClusterReconcilerReconcilePausedCluster(t *testing.T) {
 	iam := mocks.NewMockAWSIamConfigReconciler(ctrl)
 	clusterValidator := mocks.NewMockClusterValidator(ctrl)
 	registry := newRegistryMock(providerReconciler)
-	r := controllers.NewClusterReconciler(c, registry, iam, clusterValidator)
+	r := controllers.NewClusterReconciler(c, registry, iam, clusterValidator, nil)
 	g.Expect(r.Reconcile(ctx, clusterRequest(cluster))).To(Equal(reconcile.Result{}))
 	api := envtest.NewAPIExpecter(t, c)
 
@@ -164,7 +164,7 @@ func TestClusterReconcilerReconcileDeletedSelfManagedCluster(t *testing.T) {
 	registry := newRegistryMock(providerReconciler)
 	c := fake.NewClientBuilder().WithRuntimeObjects(selfManagedCluster).Build()
 
-	r := controllers.NewClusterReconciler(c, registry, iam, clusterValidator)
+	r := controllers.NewClusterReconciler(c, registry, iam, clusterValidator, nil)
 	_, err := r.Reconcile(ctx, clusterRequest(selfManagedCluster))
 	g.Expect(err).To(MatchError(ContainSubstring("deleting self-managed clusters is not supported")))
 }
@@ -233,7 +233,7 @@ func TestClusterReconcilerReconcileDeletePausedCluster(t *testing.T) {
 		managementCluster, cluster, capiCluster,
 	).Build()
 
-	r := controllers.NewClusterReconciler(c, newRegistryForDummyProviderReconciler(), iam, clusterValidator)
+	r := controllers.NewClusterReconciler(c, newRegistryForDummyProviderReconciler(), iam, clusterValidator, nil)
 	g.Expect(r.Reconcile(ctx, clusterRequest(cluster))).To(Equal(reconcile.Result{}))
 	api := envtest.NewAPIExpecter(t, c)
 
@@ -276,7 +276,7 @@ func TestClusterReconcilerReconcileDeleteClusterManagedByCLI(t *testing.T) {
 	iam := mocks.NewMockAWSIamConfigReconciler(controller)
 	clusterValidator := mocks.NewMockClusterValidator(controller)
 
-	r := controllers.NewClusterReconciler(c, newRegistryForDummyProviderReconciler(), iam, clusterValidator)
+	r := controllers.NewClusterReconciler(c, newRegistryForDummyProviderReconciler(), iam, clusterValidator, nil)
 	g.Expect(r.Reconcile(ctx, clusterRequest(cluster))).To(Equal(reconcile.Result{}))
 	api := envtest.NewAPIExpecter(t, c)