Skip to content

Commit

Permalink
Merge pull request #74 from bharath-b-rh/cfe-878
Browse files Browse the repository at this point in the history
CFE-878: Add userLabels in Infrastructure to driver args list
  • Loading branch information
openshift-merge-robot committed Aug 15, 2023
2 parents eb7b7b7 + 6cc8991 commit 999fb4c
Show file tree
Hide file tree
Showing 147 changed files with 16,513 additions and 677 deletions.
1 change: 0 additions & 1 deletion assets/controller.yaml
Expand Up @@ -50,7 +50,6 @@ spec:
- --endpoint=$(CSI_ENDPOINT)
- --logtostderr
- --v=${LOG_LEVEL}
- --extra-labels=kubernetes-io-cluster-${CLUSTER_ID}=owned
env:
- name: GOOGLE_APPLICATION_CREDENTIALS
value: "/etc/cloud-sa/service_account.json"
Expand Down
2 changes: 1 addition & 1 deletion go.mod
Expand Up @@ -4,7 +4,7 @@ go 1.19

require (
github.com/google/go-cmp v0.5.9
github.com/openshift/api v0.0.0-20230613151523-ba04973d3ed1
github.com/openshift/api v0.0.0-20230804173756-26b8597c4de2
github.com/openshift/build-machinery-go v0.0.0-20230306181456-d321ffa04533
github.com/openshift/client-go v0.0.0-20230503144108-75015d2347cb
github.com/openshift/library-go v0.0.0-20230724150037-c515269de16e
Expand Down
4 changes: 2 additions & 2 deletions go.sum
Expand Up @@ -267,8 +267,8 @@ github.com/mwitkow/go-conntrack v0.0.0-20161129095857-cc309e4a2223/go.mod h1:qRW
github.com/mwitkow/go-conntrack v0.0.0-20190716064945-2f068394615f/go.mod h1:qRWi+5nqEBWmkhHvq77mSJWrCKwh8bxhgT7d/eI7P4U=
github.com/onsi/ginkgo/v2 v2.9.2 h1:BA2GMJOtfGAfagzYtrAlufIP0lq6QERkFmHLMLPwFSU=
github.com/onsi/gomega v1.27.4 h1:Z2AnStgsdSayCMDiCU42qIz+HLqEPcgiOCXjAU/w+8E=
github.com/openshift/api v0.0.0-20230613151523-ba04973d3ed1 h1:sgr89m3ejIIKhSbTtHq7HEZ80et4IAXDrJlk+u+rYX8=
github.com/openshift/api v0.0.0-20230613151523-ba04973d3ed1/go.mod h1:4VWG+W22wrB4HfBL88P40DxLEpSOaiBVxUnfalfJo9k=
github.com/openshift/api v0.0.0-20230804173756-26b8597c4de2 h1:K7rBUJvIEa9Ei7tyAv4wDwDLpOFKa6nP84JnqxrY73o=
github.com/openshift/api v0.0.0-20230804173756-26b8597c4de2/go.mod h1:yimSGmjsI+XF1mr+AKBs2//fSXIOhhetHGbMlBEfXbs=
github.com/openshift/build-machinery-go v0.0.0-20230306181456-d321ffa04533 h1:mh3ZYs7kPIIe3UUY6tJcTExmtjnXXUu0MrBuK2W/Qvw=
github.com/openshift/build-machinery-go v0.0.0-20230306181456-d321ffa04533/go.mod h1:b1BuldmJlbA/xYtdZvKi+7j5YGB44qJUJDZ9zwiNCfE=
github.com/openshift/client-go v0.0.0-20230503144108-75015d2347cb h1:Nij5OnaECrkmcRQMAE9LMbQXPo95aqFnf+12B7SyFVI=
Expand Down
47 changes: 47 additions & 0 deletions pkg/operator/starter.go
Expand Up @@ -3,8 +3,10 @@ package operator
import (
"context"
"fmt"
"strings"
"time"

appsv1 "k8s.io/api/apps/v1"
apiextclient "k8s.io/apiextensions-apiserver/pkg/client/clientset/clientset"
metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
"k8s.io/client-go/dynamic"
Expand All @@ -15,13 +17,15 @@ import (
opv1 "github.com/openshift/api/operator/v1"
configclient "github.com/openshift/client-go/config/clientset/versioned"
configinformers "github.com/openshift/client-go/config/informers/externalversions"
configlisters "github.com/openshift/client-go/config/listers/config/v1"
opclient "github.com/openshift/client-go/operator/clientset/versioned"
opinformers "github.com/openshift/client-go/operator/informers/externalversions"
"github.com/openshift/library-go/pkg/controller/controllercmd"
"github.com/openshift/library-go/pkg/controller/factory"
"github.com/openshift/library-go/pkg/operator/csi/csicontrollerset"
"github.com/openshift/library-go/pkg/operator/csi/csidrivercontrollerservicecontroller"
"github.com/openshift/library-go/pkg/operator/csi/csidrivernodeservicecontroller"
dc "github.com/openshift/library-go/pkg/operator/deploymentcontroller"
goc "github.com/openshift/library-go/pkg/operator/genericoperatorclient"
"github.com/openshift/library-go/pkg/operator/v1helpers"

Expand All @@ -41,6 +45,13 @@ const (

diskEncryptionKMSKey = "disk-encryption-kms-key"
defaultKMSKeyLocation = "global"

// globalInfrastructureName is the default name of the Infrastructure object
globalInfrastructureName = "cluster"

// ocpDefaultLabelFmt is the format string for the default label
// added to the OpenShift created GCP resources.
ocpDefaultLabelFmt = "kubernetes-io-cluster-%s=owned"
)

func RunOperator(ctx context.Context, controllerConfig *controllercmd.ControllerContext) error {
Expand Down Expand Up @@ -167,6 +178,7 @@ func RunOperator(ctx context.Context, controllerConfig *controllercmd.Controller
secretInformer,
),
csidrivercontrollerservicecontroller.WithReplicasHook(nodeInformer.Lister()),
withCustomLabels(infraInformer.Lister()),
).WithCSIDriverNodeService(
"GCPPDDriverNodeServiceController",
assets.ReadFile,
Expand Down Expand Up @@ -215,3 +227,38 @@ func RunOperator(ctx context.Context, controllerConfig *controllercmd.Controller

return fmt.Errorf("stopped")
}

// withCustomLabels adds labels from Infrastructure.Status.PlatformStatus.GCP.ResourceLabels to the
// driver command line as --extra-labels=<key1>=<value1>,<key2>=<value2>,...
func withCustomLabels(infraLister configlisters.InfrastructureLister) dc.DeploymentHookFunc {
return func(spec *opv1.OperatorSpec, deployment *appsv1.Deployment) error {
infra, err := infraLister.Get(globalInfrastructureName)
if err != nil {
return fmt.Errorf("custom labels: failed to fetch global Infrastructure object: %w", err)
}

var labels []string
if infra.Status.PlatformStatus != nil &&
infra.Status.PlatformStatus.GCP != nil &&
infra.Status.PlatformStatus.GCP.ResourceLabels != nil {
labels = make([]string, len(infra.Status.PlatformStatus.GCP.ResourceLabels))
for i, label := range infra.Status.PlatformStatus.GCP.ResourceLabels {
labels[i] = fmt.Sprintf("%s=%s", label.Key, label.Value)
}
}

labels = append(labels, fmt.Sprintf(ocpDefaultLabelFmt, infra.Status.InfrastructureName))
labelsStr := strings.Join(labels, ",")
labelsArg := fmt.Sprintf("--extra-labels=%s", labelsStr)
klog.V(1).Infof("withCustomLabels: adding extra-labels arg to driver with value %s", labelsStr)

for i := range deployment.Spec.Template.Spec.Containers {
container := &deployment.Spec.Template.Spec.Containers[i]
if container.Name != "csi-driver" {
continue
}
container.Args = append(container.Args, labelsArg)
}
return nil
}
}
144 changes: 144 additions & 0 deletions pkg/operator/starter_test.go
@@ -0,0 +1,144 @@
package operator

import (
"fmt"
"testing"

appsv1 "k8s.io/api/apps/v1"
corev1 "k8s.io/api/core/v1"
"k8s.io/apimachinery/pkg/api/equality"
metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
"k8s.io/apimachinery/pkg/runtime"

v1 "github.com/openshift/api/config/v1"
fakeconfig "github.com/openshift/client-go/config/clientset/versioned/fake"
configinformers "github.com/openshift/client-go/config/informers/externalversions"
)

func TestWithCustomLabels(t *testing.T) {

infraObj := &v1.Infrastructure{
ObjectMeta: metav1.ObjectMeta{
Name: "cluster",
},
Status: v1.InfrastructureStatus{
InfrastructureName: "test-vbc3g",
PlatformStatus: &v1.PlatformStatus{
GCP: &v1.GCPPlatformStatus{
ProjectID: "test",
Region: "test",
},
},
},
}

tmplDeployObj := &appsv1.Deployment{
Spec: appsv1.DeploymentSpec{
Template: corev1.PodTemplateSpec{
Spec: corev1.PodSpec{
Containers: []corev1.Container{
{
Name: "csi-driver",
Image: "example.io/example-csi-driver",
Args: []string{
"--endpoint=$(CSI_ENDPOINT)",
"--logtostderr",
"--v=2",
},
Env: []corev1.EnvVar{
{
Name: "GOOGLE_APPLICATION_CREDENTIALS",
Value: "/etc/cloud-sa/service_account.json",
},
{
Name: "CSI_ENDPOINT",
Value: "unix:///var/lib/csi/sockets/pluginproxy/csi.sock",
},
},
},
{
Name: "test-driver",
Image: "example.io/example-test-driver",
},
},
},
},
},
}

tests := []struct {
name string
labels []v1.GCPResourceLabel
expArgList string
createInfraCR bool
wantErr bool
}{
{
name: "labels not configured",
labels: []v1.GCPResourceLabel{},
expArgList: fmt.Sprintf("--extra-labels=%s", fmt.Sprintf(ocpDefaultLabelFmt, infraObj.Status.InfrastructureName)),
createInfraCR: true,
wantErr: false,
},
{
name: "labels configured",
labels: []v1.GCPResourceLabel{
{
Key: "key1",
Value: "value1",
},
{
Key: "key2",
Value: "value2",
},
{
Key: "key3",
Value: "value3",
},
},
expArgList: fmt.Sprintf("--extra-labels=key1=value1,key2=value2,"+
"key3=value3,%s", fmt.Sprintf(ocpDefaultLabelFmt, infraObj.Status.InfrastructureName)),
createInfraCR: true,
wantErr: false,
},
{
name: "Infrastructure CR does not exist",
labels: []v1.GCPResourceLabel{},
expArgList: "",
createInfraCR: false,
wantErr: true,
},
}

for _, test := range tests {
t.Run(test.name, func(t *testing.T) {
objs := make([]runtime.Object, 0)
if test.createInfraCR {
infraObj.Status.PlatformStatus.GCP.ResourceLabels = test.labels
objs = append(objs, infraObj)
}
configClient := fakeconfig.NewSimpleClientset(objs...)
configInformerFactory := configinformers.NewSharedInformerFactory(configClient, 0)
if test.createInfraCR {
configInformerFactory.Config().V1().Infrastructures().Informer().GetIndexer().Add(infraObj)
}

deployment := tmplDeployObj.DeepCopy()
updDeployment := tmplDeployObj.DeepCopy()
if test.expArgList != "" {
updDeployment.Spec.Template.Spec.Containers[0].Args = append(
updDeployment.Spec.Template.Spec.Containers[0].Args,
test.expArgList,
)
}

err := withCustomLabels(configInformerFactory.Config().V1().Infrastructures().Lister())(nil, deployment)
if (err != nil) != test.wantErr {
t.Errorf("unexpected error: %v", err)
}
if !equality.Semantic.DeepEqual(deployment, updDeployment) {
t.Errorf("unexpected deployment want: %+v got: %+v", updDeployment, deployment)
}
})
}
}
1 change: 1 addition & 0 deletions vendor/github.com/openshift/api/config/.codegen.yaml

Some generated files are not rendered by default. Learn more about how customized files appear on GitHub.

Some generated files are not rendered by default. Learn more about how customized files appear on GitHub.

0 comments on commit 999fb4c

Please sign in to comment.