Skip to content

Commit

Permalink
Make sure that we are using the operator's client
Browse files Browse the repository at this point in the history
  • Loading branch information
asalkeld committed Sep 2, 2021
1 parent 9f54f7b commit 7640599
Show file tree
Hide file tree
Showing 11 changed files with 295 additions and 58 deletions.
2 changes: 1 addition & 1 deletion exp/operator/controllers/consts.go
Original file line number Diff line number Diff line change
Expand Up @@ -21,5 +21,5 @@ import "time"
const (
// preflightFailedRequeueAfter is how long to wait before trying to reconcile
// if some preflight check has failed.
preflightFailedRequeueAfter = 30 * time.Second
preflightFailedRequeueAfter = 5 * time.Second
)
115 changes: 115 additions & 0 deletions exp/operator/controllers/controllerclientproxy.go
Original file line number Diff line number Diff line change
@@ -0,0 +1,115 @@
package controllers

import (
"context"

"github.com/pkg/errors"
apierrors "k8s.io/apimachinery/pkg/api/errors"
metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
"k8s.io/apimachinery/pkg/apis/meta/v1/unstructured"
"k8s.io/client-go/rest"
"sigs.k8s.io/cluster-api/cmd/clusterctl/client/cluster"
"sigs.k8s.io/controller-runtime/pkg/client"
)

type controllerProxy struct {
ctrlClient client.Client
ctrlConfig *rest.Config
}

var _ cluster.Proxy = &controllerProxy{}

func (k *controllerProxy) CurrentNamespace() (string, error) { return "default", nil }
func (k *controllerProxy) ValidateKubernetesVersion() error { return nil }
func (k *controllerProxy) GetConfig() (*rest.Config, error) { return k.ctrlConfig, nil }
func (k *controllerProxy) NewClient() (client.Client, error) { return k.ctrlClient, nil }

// ListResources return only RBAC and Deployoments as this is used by:
// - from Delete return just RBAC, we don't delete CRDs, Namespaces and resources in the namespace.
// - from certmanager just return the resource that has a version annontation.
func (k *controllerProxy) ListResources(labels map[string]string, namespaces ...string) ([]unstructured.Unstructured, error) {
resourceList := []*metav1.APIResourceList{
{
GroupVersion: "v1",
APIResources: []metav1.APIResource{
{Name: "secrets", Namespaced: true},
{Name: "configmaps", Namespaced: true},
{Name: "services", Namespaced: true},
},
},
{
GroupVersion: "extensions/v1",
APIResources: []metav1.APIResource{
{Name: "daemonsets", Namespaced: true},
{Name: "deployments", Namespaced: true},
},
},
{
GroupVersion: "rbac.authorization.k8s.io/v1",
APIResources: []metav1.APIResource{
{Name: "clusterrolebindings"},
{Name: "clusterroles"},
{Name: "rolebindings", Namespaced: true},
{Name: "roles", Namespaced: true},
},
},
{
GroupVersion: "admissionregistration.k8s.io/v1",
APIResources: []metav1.APIResource{
{Name: "validatingwebhookconfiguration", Namespaced: true},
{Name: "mutatingwebhookconfiguration", Namespaced: true},
},
},
{
GroupVersion: "admissionregistration.k8s.io/v1beta1",
APIResources: []metav1.APIResource{
{Name: "validatingwebhookconfiguration", Namespaced: true},
{Name: "mutatingwebhookconfiguration", Namespaced: true},
},
},
{
GroupVersion: "cert-manager.io/v1",
APIResources: []metav1.APIResource{
{Name: "certificates", Namespaced: true},
{Name: "certificaterequests", Namespaced: true},
{Name: "issuer", Namespaced: true},
},
},
}

var ret []unstructured.Unstructured
for _, resourceGroup := range resourceList {
for _, resourceKind := range resourceGroup.APIResources {
if resourceKind.Namespaced {
for _, namespace := range namespaces {
objList, err := listObjByGVK(k.ctrlClient, resourceGroup.GroupVersion, resourceKind.Kind, []client.ListOption{client.MatchingLabels(labels), client.InNamespace(namespace)})
if err != nil {
return nil, err
}
ret = append(ret, objList.Items...)
}
} else {
objList, err := listObjByGVK(k.ctrlClient, resourceGroup.GroupVersion, resourceKind.Kind, []client.ListOption{client.MatchingLabels(labels)})
if err != nil {
return nil, err
}
ret = append(ret, objList.Items...)
}
}
}
return ret, nil
}

func listObjByGVK(c client.Client, groupVersion, kind string, options []client.ListOption) (*unstructured.UnstructuredList, error) {
ctx := context.TODO()
objList := new(unstructured.UnstructuredList)
objList.SetAPIVersion(groupVersion)
objList.SetKind(kind)

if err := c.List(ctx, objList, options...); err != nil {
if !apierrors.IsNotFound(err) {
return nil, errors.Wrapf(err, "failed to list objects for the %q GroupVersionKind", objList.GroupVersionKind())
}
}
return objList, nil
}
6 changes: 4 additions & 2 deletions exp/operator/controllers/genericprovider_controller.go
Original file line number Diff line number Diff line change
Expand Up @@ -23,6 +23,7 @@ import (

apierrors "k8s.io/apimachinery/pkg/api/errors"
utilerrors "k8s.io/apimachinery/pkg/util/errors"
"k8s.io/client-go/rest"
ctrl "sigs.k8s.io/controller-runtime"
"sigs.k8s.io/controller-runtime/pkg/client"
"sigs.k8s.io/controller-runtime/pkg/controller"
Expand All @@ -44,6 +45,7 @@ type GenericProviderReconciler struct {
Provider client.Object
ProviderList client.ObjectList
Client client.Client
Config *rest.Config
CertManagerInstaller SingletonInstaller
}

Expand Down Expand Up @@ -124,7 +126,7 @@ func (r *GenericProviderReconciler) reconcile(ctx context.Context, provider gene
"Generation", provider.GetGeneration(),
"ObservedGeneration", provider.GetStatus().ObservedGeneration)

reconciler := newReconcilePhases(r.Client, r.CertManagerInstaller)
reconciler := newReconcilePhases(r.Client, r.Config, r.CertManagerInstaller)
phases := []reconcilePhaseFn{
func(ctx context.Context, provider genericprovider.GenericProvider) (reconcile.Result, error) {
return preflightChecks(ctx, reconciler.ctrlClient, provider, genericProviderList)
Expand Down Expand Up @@ -158,7 +160,7 @@ func (r *GenericProviderReconciler) reconcileDelete(ctx context.Context, provide
log := ctrl.LoggerFrom(ctx)
log.V(2).Info("deleting provider resources")

reconciler := newReconcilePhases(r.Client, r.CertManagerInstaller)
reconciler := newReconcilePhases(r.Client, r.Config, r.CertManagerInstaller)
phases := []reconcilePhaseFn{
reconciler.load,
reconciler.delete,
Expand Down
100 changes: 80 additions & 20 deletions exp/operator/controllers/genericprovider_controller_test.go
Original file line number Diff line number Diff line change
Expand Up @@ -29,17 +29,81 @@ import (
"sigs.k8s.io/controller-runtime/pkg/client"
"sigs.k8s.io/controller-runtime/pkg/client/fake"

"sigs.k8s.io/cluster-api/api/v1alpha4"
clusterv1 "sigs.k8s.io/cluster-api/api/v1alpha4"
clusterctlv1 "sigs.k8s.io/cluster-api/cmd/clusterctl/api/v1alpha3"
"sigs.k8s.io/cluster-api/cmd/clusterctl/client/repository"
operatorv1 "sigs.k8s.io/cluster-api/exp/operator/api/v1alpha1"
"sigs.k8s.io/cluster-api/exp/operator/controllers/genericprovider"
)

func TestReconcilerPreflightConditions(t *testing.T) {
g := NewWithT(t)
const (
testMetadata = `
apiVersion: clusterctl.cluster.x-k8s.io/v1alpha3
releaseSeries:
- major: 0
minor: 4
contract: v1alpha4
`
testComponents = `
apiVersion: apps/v1
kind: Deployment
metadata:
labels:
cluster.x-k8s.io/provider: infrastructure-docker
control-plane: controller-manager
name: capd-controller-manager
namespace: capd-system
spec:
replicas: 1
selector:
matchLabels:
cluster.x-k8s.io/provider: infrastructure-docker
control-plane: controller-manager
template:
metadata:
labels:
cluster.x-k8s.io/provider: infrastructure-docker
control-plane: controller-manager
spec:
containers:
- image: gcr.io/google-samples/hello-app:1.0
name: manager
ports:
- containerPort: 8080
resources:
requests:
cpu: 200m
`
)

func insertDummyConfig(provider genericprovider.GenericProvider) {
spec := provider.GetSpec()
spec.FetchConfig = &operatorv1.FetchConfiguration{
Selector: &metav1.LabelSelector{
MatchLabels: map[string]string{
"test": "dummy-config",
},
},
}
provider.SetSpec(spec)
}

func dummyConfigMap(ns string) *corev1.ConfigMap {
return &corev1.ConfigMap{
ObjectMeta: metav1.ObjectMeta{
Name: "v0.4.2",
Namespace: ns,
Labels: map[string]string{
"test": "dummy-config",
},
},
Data: map[string]string{
"metadata": testMetadata,
"components": testComponents,
},
}
}

func TestReconcilerPreflightConditions(t *testing.T) {
testCases := []struct {
name string
namespace string
Expand Down Expand Up @@ -71,9 +135,12 @@ func TestReconcilerPreflightConditions(t *testing.T) {

for _, tc := range testCases {
t.Run(tc.name, func(t *testing.T) {
gs := NewWithT(t)
g := NewWithT(t)

t.Log("creating namespace", tc.namespace)
namespace := &corev1.Namespace{ObjectMeta: metav1.ObjectMeta{Name: tc.namespace}}
g.Expect(env.Create(ctx, namespace)).To(Succeed())
g.Expect(env.CreateAndWait(ctx, namespace)).To(Succeed())
g.Expect(env.CreateAndWait(ctx, dummyConfigMap(tc.namespace))).To(Succeed())
tc.provider.SetNamespace(tc.namespace)

if tc.provider.GetName() != "cluster-api" {
Expand All @@ -82,22 +149,15 @@ func TestReconcilerPreflightConditions(t *testing.T) {
Name: "cluster-api",
Namespace: tc.namespace,
},
Status: operatorv1.CoreProviderStatus{
ProviderStatus: operatorv1.ProviderStatus{
Conditions: []clusterv1.Condition{
{
Type: v1alpha4.ReadyCondition,
Status: corev1.ConditionTrue,
LastTransitionTime: metav1.Now(),
},
},
},
},
}
gs.Expect(env.Create(ctx, core)).To(Succeed())
gs.Expect(env.Status().Update(ctx, core)).To(Succeed())

insertDummyConfig(&genericprovider.CoreProviderWrapper{CoreProvider: core})
t.Log("creating core provider", core.Name)
g.Expect(env.CreateAndWait(ctx, core)).To(Succeed())
}
gs.Expect(env.Create(ctx, tc.provider.GetObject())).To(Succeed())
t.Log("creating test provider", tc.provider.GetName())
insertDummyConfig(tc.provider)
g.Expect(env.CreateAndWait(ctx, tc.provider.GetObject())).To(Succeed())

g.Eventually(func() bool {
if err := env.Get(ctx, client.ObjectKeyFromObject(tc.provider.GetObject()), tc.provider.GetObject()); err != nil {
Expand Down
34 changes: 23 additions & 11 deletions exp/operator/controllers/phases.go
Original file line number Diff line number Diff line change
Expand Up @@ -15,6 +15,7 @@ import (
"k8s.io/apimachinery/pkg/types"
versionutil "k8s.io/apimachinery/pkg/util/version"
"k8s.io/apimachinery/pkg/util/wait"
"k8s.io/client-go/rest"
"k8s.io/kubectl/pkg/scheme"
"k8s.io/utils/pointer"
ctrl "sigs.k8s.io/controller-runtime"
Expand All @@ -41,6 +42,7 @@ const (

type reconciler struct {
ctrlClient client.Client
ctrlConfig *rest.Config
certManagerInstaller SingletonInstaller

repo repository.Repository
Expand Down Expand Up @@ -77,15 +79,18 @@ func ifErrorWrapPhaseError(err error, reason string, ctype clusterv1.ConditionTy
}
}

func newReconcilePhases(c client.Client, certManagerInstaller SingletonInstaller) *reconciler {
func newReconcilePhases(c client.Client, config *rest.Config, certManagerInstaller SingletonInstaller) *reconciler {
return &reconciler{
ctrlClient: c,
ctrlConfig: config,
certManagerInstaller: certManagerInstaller,
clusterctlProvider: &clusterctlv1.Provider{},
}
}

func (s *reconciler) load(ctx context.Context, provider genericprovider.GenericProvider) (reconcile.Result, error) {
log := ctrl.LoggerFrom(ctx)
log.V(2).Info("loading provider", "name", provider.GetName())
reader, err := s.secretReader(ctx, provider)
if err != nil {
return reconcile.Result{}, ifErrorWrapPhaseError(err, "failed to load the secret reader", v1alpha1.PreflightCheckCondition)
Expand Down Expand Up @@ -126,6 +131,8 @@ func (s *reconciler) load(ctx context.Context, provider genericprovider.GenericP
}

func (s *reconciler) fetch(ctx context.Context, provider genericprovider.GenericProvider) (reconcile.Result, error) {
log := ctrl.LoggerFrom(ctx)
log.V(2).Info("fetching provider", "name", provider.GetName())
componentsFile, err := s.repo.GetFile(s.options.Version, s.repo.ComponentsPath())
if err != nil {
err = errors.Wrapf(err, "failed to read %q from provider's repository %q", s.repo.ComponentsPath(), s.providerConfig.ManifestLabel())
Expand All @@ -146,11 +153,18 @@ func (s *reconciler) fetch(ctx context.Context, provider genericprovider.Generic
return reconcile.Result{}, nil
}

func (s *reconciler) newClusterClient() cluster.Client {
return cluster.New(cluster.Kubeconfig{}, s.configClient, cluster.InjectProxy(&controllerProxy{
ctrlClient: s.ctrlClient,
ctrlConfig: s.ctrlConfig,
}))
}

// preInstall will
// 1. ensure basic CRDs.
// 2. delete existing components if required.
func (s *reconciler) preInstall(ctx context.Context, provider genericprovider.GenericProvider) (reconcile.Result, error) {
clusterClient := cluster.New(cluster.Kubeconfig{}, s.configClient)
clusterClient := s.newClusterClient()

err := clusterClient.ProviderInventory().EnsureCustomResourceDefinitions()
if err != nil {
Expand All @@ -164,8 +178,10 @@ func (s *reconciler) preInstall(ctx context.Context, provider genericprovider.Ge
return s.delete(ctx, provider)
}

func (s *reconciler) delete(_ context.Context, provider genericprovider.GenericProvider) (reconcile.Result, error) {
clusterClient := cluster.New(cluster.Kubeconfig{}, s.configClient)
func (s *reconciler) delete(ctx context.Context, provider genericprovider.GenericProvider) (reconcile.Result, error) {
log := ctrl.LoggerFrom(ctx)
log.V(2).Info("deleting provider", "name", provider.GetName())
clusterClient := s.newClusterClient()

s.clusterctlProvider.Name = clusterctlProviderName(provider).Name
s.clusterctlProvider.Namespace = provider.GetNamespace()
Expand Down Expand Up @@ -225,9 +241,8 @@ func clusterctlProviderName(provider genericprovider.GenericProvider) client.Obj

func (s *reconciler) installCertManager(ctx context.Context, provider genericprovider.GenericProvider) (reconcile.Result, error) {
if s.isCertManagerRequired(ctx) {
clusterClient := cluster.New(cluster.Kubeconfig{}, s.configClient)
status, err := s.certManagerInstaller.Install(func() error {
return clusterClient.CertManager().EnsureInstalled()
return s.newClusterClient().CertManager().EnsureInstalled()
})
if err != nil || status == InstallStatusUnknown {
return reconcile.Result{RequeueAfter: time.Minute}, ifErrorWrapPhaseError(err, string(status), v1alpha1.ProviderInstalledCondition)
Expand Down Expand Up @@ -270,14 +285,11 @@ func errorToReason(err error) string {
}

func (s *reconciler) install(ctx context.Context, provider genericprovider.GenericProvider) (reconcile.Result, error) {
clusterClient := cluster.New(cluster.Kubeconfig{}, s.configClient)
clusterClient := s.newClusterClient()
installer := clusterClient.ProviderInstaller()
installer.Add(s.components)

_, err := installer.Install(cluster.InstallOptions{
WaitProviders: true,
WaitProviderTimeout: 5 * time.Minute,
})
_, err := installer.Install(cluster.InstallOptions{})
if err != nil {
return reconcile.Result{}, ifErrorWrapPhaseError(err, errorToReason(err), v1alpha1.ProviderInstalledCondition)
}
Expand Down
Loading

0 comments on commit 7640599

Please sign in to comment.