Skip to content

Commit

Permalink
STOR-1874: Use real HyperShift API
Browse files Browse the repository at this point in the history
Use HyperShift HostedControlPlane type instead of Unstructured in hooks.go
  • Loading branch information
mpatlasov committed Jul 25, 2024
1 parent 0f6e810 commit 6ae0603
Show file tree
Hide file tree
Showing 5 changed files with 51 additions and 22 deletions.
5 changes: 5 additions & 0 deletions pkg/clients/builder.go
Original file line number Diff line number Diff line change
Expand Up @@ -10,6 +10,8 @@ import (
cfginformers "github.com/openshift/client-go/config/informers/externalversions"
opclient "github.com/openshift/client-go/operator/clientset/versioned"
opinformers "github.com/openshift/client-go/operator/informers/externalversions"
hypextclient "github.com/openshift/hypershift/client/clientset/clientset"
hypextinformers "github.com/openshift/hypershift/client/informers/externalversions"
"github.com/openshift/library-go/pkg/config/client"
"github.com/openshift/library-go/pkg/controller/controllercmd"
"github.com/openshift/library-go/pkg/operator/events"
Expand Down Expand Up @@ -106,6 +108,9 @@ func (b *Builder) BuildOrDie(ctx context.Context) *Clients {
klog.Warningf("unable to get owner reference (falling back to namespace): %v", err)
}
b.client.EventRecorder = events.NewKubeRecorder(guestKubeClient.CoreV1().Events(CSIDriverNamespace), b.userAgwent, controllerRef)

b.client.ControlPlaneHypeClient = hypextclient.NewForConfigOrDie(controlPlaneRestConfig)
b.client.ControlPlaneHypeInformer = hypextinformers.NewFilteredSharedInformerFactory(b.client.ControlPlaneHypeClient, b.resync, b.controllerConfig.OperatorNamespace, nil)
}
// store guestKubeConfig in case we need it later for running
b.guestKubeConfig = guestKubeConfig
Expand Down
13 changes: 13 additions & 0 deletions pkg/clients/clients.go
Original file line number Diff line number Diff line change
Expand Up @@ -9,6 +9,8 @@ import (
cfgv1informers "github.com/openshift/client-go/config/informers/externalversions/config/v1"
opclient "github.com/openshift/client-go/operator/clientset/versioned"
opinformers "github.com/openshift/client-go/operator/informers/externalversions"
hypextclient "github.com/openshift/hypershift/client/clientset/clientset"
hypextinformers "github.com/openshift/hypershift/client/informers/externalversions"
"github.com/openshift/library-go/pkg/operator/events"
"github.com/openshift/library-go/pkg/operator/v1helpers"
apiextclient "k8s.io/apiextensions-apiserver/pkg/client/clientset/clientset"
Expand Down Expand Up @@ -49,6 +51,11 @@ type Clients struct {
// Informer in HyperShift or standalone control plane. E.g. for HyperShift's HostedControlPlane and Prometheus CRs.
ControlPlaneDynamicInformer dynamicinformer.DynamicSharedInformerFactory

// HyperShift Client for HyperShift control plane
ControlPlaneHypeClient hypextclient.Interface
// HyperShift Informer in HyperShift control plane
ControlPlaneHypeInformer hypextinformers.SharedInformerFactory

// Kubernetes API client for guest or standalone.
KubeClient kubernetes.Interface
// Kubernetes API client for guest or standalone. Per namespace.
Expand Down Expand Up @@ -114,6 +121,9 @@ func (c *Clients) Start(ctx context.Context) {
}
c.ControlPlaneKubeInformers.Start(ctx.Done())
c.ControlPlaneDynamicInformer.Start(ctx.Done())
if c.ControlPlaneHypeInformer != nil {
c.ControlPlaneHypeInformer.Start(ctx.Done())
}
c.KubeInformers.Start(ctx.Done())
c.APIExtInformer.Start(ctx.Done())
c.DynamicInformer.Start(ctx.Done())
Expand All @@ -131,6 +141,9 @@ func (c *Clients) WaitForCacheSync(ctx context.Context) {
c.ControlPlaneKubeInformers.InformersFor(ns).WaitForCacheSync(ctx.Done())
}
c.ControlPlaneDynamicInformer.WaitForCacheSync(ctx.Done())
if c.ControlPlaneHypeInformer != nil {
c.ControlPlaneHypeInformer.WaitForCacheSync(ctx.Done())
}
for ns := range c.KubeInformers.Namespaces() {
c.KubeInformers.InformersFor(ns).WaitForCacheSync(ctx.Done())
}
Expand Down
7 changes: 7 additions & 0 deletions pkg/clients/fake.go
Original file line number Diff line number Diff line change
Expand Up @@ -9,6 +9,8 @@ import (
cfginformers "github.com/openshift/client-go/config/informers/externalversions"
fakeop "github.com/openshift/client-go/operator/clientset/versioned/fake"
opinformers "github.com/openshift/client-go/operator/informers/externalversions"
fakehype "github.com/openshift/hypershift/client/clientset/clientset/fake"
hypextinformers "github.com/openshift/hypershift/client/informers/externalversions"
"github.com/openshift/library-go/pkg/operator/events"
"github.com/openshift/library-go/pkg/operator/v1helpers"
corev1 "k8s.io/api/core/v1"
Expand Down Expand Up @@ -38,6 +40,9 @@ func NewFakeClients(controllerNamespace string, cr *opv1.ClusterCSIDriver) *Clie
controlPlaneDynamicClient := fake.NewSimpleDynamicClientWithCustomListKinds(scheme, gvrToListKind)
controlPlaneDynamicInformer := dynamicinformer.NewDynamicSharedInformerFactory(controlPlaneDynamicClient, 0)

controlPlaneHypeClient := fakehype.NewSimpleClientset()
controlPlaneHypeInformers := hypextinformers.NewSharedInformerFactory(controlPlaneHypeClient, 0)

guestKubeClient := fakecore.NewSimpleClientset()
guestKubeInformers := v1helpers.NewKubeInformersForNamespaces(guestKubeClient, controllerNamespace, "", CSIDriverNamespace)

Expand Down Expand Up @@ -75,6 +80,8 @@ func NewFakeClients(controllerNamespace string, cr *opv1.ClusterCSIDriver) *Clie
ControlPlaneKubeInformers: controlPlaneKubeInformers,
ControlPlaneDynamicClient: controlPlaneDynamicClient,
ControlPlaneDynamicInformer: controlPlaneDynamicInformer,
ControlPlaneHypeClient: controlPlaneHypeClient,
ControlPlaneHypeInformer: controlPlaneHypeInformers,

KubeClient: guestKubeClient,
KubeInformers: guestKubeInformers,
Expand Down
26 changes: 10 additions & 16 deletions pkg/driver/common/operator/hooks.go
Original file line number Diff line number Diff line change
Expand Up @@ -8,14 +8,14 @@ import (
"github.com/openshift/csi-operator/pkg/clients"
"github.com/openshift/csi-operator/pkg/generator"
"github.com/openshift/csi-operator/pkg/operator/config"
hypev1beta1api "github.com/openshift/hypershift/api/hypershift/v1beta1"
hypev1beta1listers "github.com/openshift/hypershift/client/listers/hypershift/v1beta1"
"github.com/openshift/library-go/pkg/controller/factory"
"github.com/openshift/library-go/pkg/operator/csi/csidrivercontrollerservicecontroller"
dc "github.com/openshift/library-go/pkg/operator/deploymentcontroller"
appsv1 "k8s.io/api/apps/v1"
"k8s.io/apimachinery/pkg/apis/meta/v1/unstructured"
"k8s.io/apimachinery/pkg/labels"
"k8s.io/apimachinery/pkg/runtime/schema"
"k8s.io/client-go/tools/cache"
"k8s.io/klog/v2"
)

Expand Down Expand Up @@ -76,7 +76,7 @@ func withHyperShiftNodeSelector(c *clients.Clients) (dc.DeploymentHookFunc, []fa
podSpec := &deployment.Spec.Template.Spec
// Add nodeSelector
nodeSelector, err := getHostedControlPlaneNodeSelector(
c.ControlPlaneDynamicInformer.ForResource(hostedControlPlaneGVR).Lister(),
c.ControlPlaneHypeInformer.Hypershift().V1beta1().HostedControlPlanes().Lister(),
c.ControlPlaneNamespace)
if err != nil {
return err
Expand All @@ -86,31 +86,28 @@ func withHyperShiftNodeSelector(c *clients.Clients) (dc.DeploymentHookFunc, []fa
return nil
}
informers := []factory.Informer{
c.ControlPlaneDynamicInformer.ForResource(hostedControlPlaneGVR).Informer(),
c.ControlPlaneHypeInformer.Hypershift().V1beta1().HostedControlPlanes().Informer(),
}
return hook, informers
}

// getHostedControlPlaneNodeSelector returns the node selector from the HostedControlPlane CR.
func getHostedControlPlaneNodeSelector(hostedControlPlaneLister cache.GenericLister, namespace string) (map[string]string, error) {
func getHostedControlPlaneNodeSelector(hostedControlPlaneLister hypev1beta1listers.HostedControlPlaneLister, namespace string) (map[string]string, error) {
hcp, err := getHostedControlPlane(hostedControlPlaneLister, namespace)
if err != nil {
return nil, err
}
nodeSelector, exists, err := unstructured.NestedStringMap(hcp.UnstructuredContent(), "spec", "nodeSelector")
if !exists {
nodeSelector := hcp.Spec.NodeSelector
if len(nodeSelector) == 0 {
return nil, nil
}
if err != nil {
return nil, err
}
klog.V(4).Infof("Using node selector %v", nodeSelector)
return nodeSelector, nil
}

// getHostedControlPlane returns the HostedControlPlane CR.
func getHostedControlPlane(hostedControlPlaneLister cache.GenericLister, namespace string) (*unstructured.Unstructured, error) {
list, err := hostedControlPlaneLister.ByNamespace(namespace).List(labels.Everything())
func getHostedControlPlane(hostedControlPlaneLister hypev1beta1listers.HostedControlPlaneLister, namespace string) (*hypev1beta1api.HostedControlPlane, error) {
list, err := hostedControlPlaneLister.List(labels.Everything())
if err != nil {
return nil, err
}
Expand All @@ -121,10 +118,7 @@ func getHostedControlPlane(hostedControlPlaneLister cache.GenericLister, namespa
return nil, fmt.Errorf("more than one HostedControlPlane found in namespace %s", namespace)
}

hcp := list[0].(*unstructured.Unstructured)
if hcp == nil {
return nil, fmt.Errorf("unknown type of HostedControlPlane found in namespace %s", namespace)
}
hcp := list[0]
return hcp, nil
}

Expand Down
22 changes: 16 additions & 6 deletions pkg/driver/common/operator/hooks_test.go
Original file line number Diff line number Diff line change
Expand Up @@ -7,12 +7,14 @@ import (

"github.com/openshift/csi-operator/pkg/clients"
"github.com/openshift/csi-operator/pkg/driver/common/operator/test_manifests"
hypev1beta1api "github.com/openshift/hypershift/api/hypershift/v1beta1"
fakehype "github.com/openshift/hypershift/client/clientset/clientset/fake"
hypescheme "github.com/openshift/hypershift/client/clientset/clientset/scheme"
"github.com/openshift/library-go/pkg/operator/resource/resourceread"
appsv1 "k8s.io/api/apps/v1"
v1 "k8s.io/api/core/v1"
metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
"k8s.io/apimachinery/pkg/apis/meta/v1/unstructured"
fakedynamic "k8s.io/client-go/dynamic/fake"
"k8s.io/apimachinery/pkg/runtime"
"k8s.io/client-go/kubernetes/fake"
)

Expand Down Expand Up @@ -95,14 +97,22 @@ func Test_WithStandaloneReplicas(t *testing.T) {
}
}

func getTestHostedControlPlane(assetName string) *unstructured.Unstructured {
return resourceread.ReadUnstructuredOrDie(test_manifests.ReadFileOrDie(assetName))
func readHcpOrDie(objBytes []byte) *hypev1beta1api.HostedControlPlane {
requiredObj, err := runtime.Decode(hypescheme.Codecs.UniversalDecoder(hypev1beta1api.SchemeGroupVersion), objBytes)
if err != nil {
panic(err)
}
return requiredObj.(*hypev1beta1api.HostedControlPlane)
}

func getTestHostedControlPlane(assetName string) *hypev1beta1api.HostedControlPlane {
return readHcpOrDie(test_manifests.ReadFileOrDie(assetName))
}

func Test_WithHyperShiftNodeSelector(t *testing.T) {
tests := []struct {
name string
hcp *unstructured.Unstructured
hcp *hypev1beta1api.HostedControlPlane
expectedNodeSelector map[string]string
}{
{
Expand All @@ -124,7 +134,7 @@ func Test_WithHyperShiftNodeSelector(t *testing.T) {
cr := clients.GetFakeOperatorCR()
c := clients.NewFakeClients("clusters-test", cr)
// Arrange: inject HostedControlPlane to the clients
c.ControlPlaneDynamicClient.(*fakedynamic.FakeDynamicClient).Tracker().Add(tt.hcp)
c.ControlPlaneHypeClient.(*fakehype.Clientset).Tracker().Add(tt.hcp)

hook, _ := withHyperShiftNodeSelector(c)
deployment := getTestDeployment()
Expand Down

0 comments on commit 6ae0603

Please sign in to comment.