From c261521b57e3fa7ac91b48d1a69f6c1bba5620fe Mon Sep 17 00:00:00 2001 From: Andrej Krejcir Date: Tue, 8 Aug 2023 14:26:53 +0200 Subject: [PATCH] feat: Use the latest image of vm-console-proxy - Updated data/vm-console-proxy-bundle/vm-console-proxy.yaml - Updated release script to change vm-console-proxy image tag. - csv-generator uses VM_CONSOLE_PROXY_IMAGE env variable to set the image. - Operator deploys new resources needed by vm-console-proxy. - Operator removes Route resource, that is no longer needed. Signed-off-by: Andrej Krejcir --- .../workflows/release-vm-console-proxy.yaml | 1 + config/manager/manager.template.yaml | 1 + config/rbac/role.yaml | 57 +++++- .../ssp-operator.clusterserviceversion.yaml | 58 +++++- .../vm-console-proxy.yaml | 76 ++++++- hack/csv-generator.go | 14 ++ hack/csv-generator_test.go | 4 + internal/common/environment.go | 1 + .../operands/vm-console-proxy/defaults.go | 8 + .../operands/vm-console-proxy/reconcile.go | 115 +++++++---- .../vm-console-proxy/reconcile_test.go | 191 +++++++++++++++--- .../bundle-test-duplicate.yaml | 76 ++++++- .../bundle-test-incorrect.yaml | 2 +- .../vm-console-proxy-bundle/bundle-test.yaml | 76 ++++++- internal/vm-console-proxy-bundle/bundle.go | 17 +- .../vm-console-proxy-bundle/bundle_test.go | 2 + tests/e2e-test-csv-generator.sh | 7 +- tests/tests_suite_test.go | 4 + tests/vm_console_proxy_test.go | 150 +++++++++++--- 19 files changed, 729 insertions(+), 131 deletions(-) create mode 100644 internal/operands/vm-console-proxy/defaults.go diff --git a/.github/workflows/release-vm-console-proxy.yaml b/.github/workflows/release-vm-console-proxy.yaml index 41e1bf82d..340bd12ee 100644 --- a/.github/workflows/release-vm-console-proxy.yaml +++ b/.github/workflows/release-vm-console-proxy.yaml @@ -17,6 +17,7 @@ jobs: OUTPUT_FILE=./data/vm-console-proxy-bundle/vm-console-proxy.yaml mkdir -p ./data/vm-console-proxy-bundle curl -L https://github.com/kubevirt/vm-console-proxy/releases/download/${RELEASE_VERSION}/vm-console-proxy.yaml > ${OUTPUT_FILE} + sed -i "s/defaultVmConsoleProxyImageTag = .*$/defaultVmConsoleProxyImageTag = \"${RELEASE_VERSION}\"/" ./internal/operands/vm-console-proxy/defaults.go - name: Create pull request if: ${{ github.event.client_payload.release_version }} != '' diff --git a/config/manager/manager.template.yaml b/config/manager/manager.template.yaml index 2a4ed61ed..72dbb672c 100644 --- a/config/manager/manager.template.yaml +++ b/config/manager/manager.template.yaml @@ -37,6 +37,7 @@ spec: - name: OPERATOR_VERSION - name: TEKTON_TASKS_IMAGE - name: TEKTON_TASKS_DISK_VIRT_IMAGE + - name: VM_CONSOLE_PROXY_IMAGE image: controller:latest name: manager resources: diff --git a/config/rbac/role.yaml b/config/rbac/role.yaml index 782420451..2a16d22ca 100644 --- a/config/rbac/role.yaml +++ b/config/rbac/role.yaml @@ -69,6 +69,17 @@ rules: verbs: - list - watch +- apiGroups: + - apiregistration.k8s.io + resources: + - apiservices + verbs: + - create + - delete + - get + - list + - update + - watch - apiGroups: - apps resources: @@ -184,6 +195,14 @@ rules: - get - list - watch +- apiGroups: + - "" + resources: + - pods + verbs: + - get + - list + - watch - apiGroups: - "" resources: @@ -191,9 +210,17 @@ rules: verbs: - create - delete + - get - list + - patch - update - watch +- apiGroups: + - "" + resources: + - serviceaccounts/token + verbs: + - create - apiGroups: - "" resources: @@ -225,14 +252,6 @@ rules: - list - update - watch -- apiGroups: - - kubevirt.io - resources: - - virtualmachineinstances - verbs: - - get - - list - - watch - apiGroups: - kubevirt.io resources: @@ -279,6 +298,18 @@ rules: - list - update - watch +- apiGroups: + - rbac.authorization.k8s.io + resources: + - clusterrolebindings + - clusterroles + - rolebindings + verbs: + - create + - delete + - list + - update + - watch - apiGroups: - rbac.authorization.k8s.io resources: @@ -320,7 +351,9 @@ rules: verbs: - create - delete + - get - list + - patch - update - watch - apiGroups: @@ -328,10 +361,8 @@ rules: resources: - routes verbs: - - create - delete - list - - update - watch - apiGroups: - ssp.kubevirt.io @@ -353,6 +384,12 @@ rules: - ssps/status verbs: - update +- apiGroups: + - subresources.kubevirt.io + resources: + - virtualmachineinstances/vnc + verbs: + - get - apiGroups: - subresources.kubevirt.io resources: diff --git a/data/olm-catalog/ssp-operator.clusterserviceversion.yaml b/data/olm-catalog/ssp-operator.clusterserviceversion.yaml index 72b0f2495..884a9a320 100644 --- a/data/olm-catalog/ssp-operator.clusterserviceversion.yaml +++ b/data/olm-catalog/ssp-operator.clusterserviceversion.yaml @@ -130,6 +130,17 @@ spec: verbs: - list - watch + - apiGroups: + - apiregistration.k8s.io + resources: + - apiservices + verbs: + - create + - delete + - get + - list + - update + - watch - apiGroups: - apps resources: @@ -245,6 +256,14 @@ spec: - get - list - watch + - apiGroups: + - "" + resources: + - pods + verbs: + - get + - list + - watch - apiGroups: - "" resources: @@ -252,9 +271,17 @@ spec: verbs: - create - delete + - get - list + - patch - update - watch + - apiGroups: + - "" + resources: + - serviceaccounts/token + verbs: + - create - apiGroups: - "" resources: @@ -286,14 +313,6 @@ spec: - list - update - watch - - apiGroups: - - kubevirt.io - resources: - - virtualmachineinstances - verbs: - - get - - list - - watch - apiGroups: - kubevirt.io resources: @@ -340,6 +359,18 @@ spec: - list - update - watch + - apiGroups: + - rbac.authorization.k8s.io + resources: + - clusterrolebindings + - clusterroles + - rolebindings + verbs: + - create + - delete + - list + - update + - watch - apiGroups: - rbac.authorization.k8s.io resources: @@ -381,7 +412,9 @@ spec: verbs: - create - delete + - get - list + - patch - update - watch - apiGroups: @@ -389,10 +422,8 @@ spec: resources: - routes verbs: - - create - delete - list - - update - watch - apiGroups: - ssp.kubevirt.io @@ -414,6 +445,12 @@ spec: - ssps/status verbs: - update + - apiGroups: + - subresources.kubevirt.io + resources: + - virtualmachineinstances/vnc + verbs: + - get - apiGroups: - subresources.kubevirt.io resources: @@ -494,6 +531,7 @@ spec: value: 0.14.0 - name: TEKTON_TASKS_IMAGE - name: TEKTON_TASKS_DISK_VIRT_IMAGE + - name: VM_CONSOLE_PROXY_IMAGE image: quay.io/kubevirt/ssp-operator:latest livenessProbe: httpGet: diff --git a/data/vm-console-proxy-bundle/vm-console-proxy.yaml b/data/vm-console-proxy-bundle/vm-console-proxy.yaml index 9468c8ac3..ba0b5a84e 100644 --- a/data/vm-console-proxy-bundle/vm-console-proxy.yaml +++ b/data/vm-console-proxy-bundle/vm-console-proxy.yaml @@ -23,10 +23,48 @@ rules: - kubevirt.io resources: - virtualmachineinstances + - virtualmachines verbs: - get - list - watch +- apiGroups: + - subresources.kubevirt.io + resources: + - virtualmachineinstances/vnc + verbs: + - get +- apiGroups: + - "" + resources: + - serviceaccounts + verbs: + - get + - list + - watch + - create + - update + - delete + - patch +- apiGroups: + - "" + resources: + - serviceaccounts/token + verbs: + - create +- apiGroups: + - rbac.authorization.k8s.io + resources: + - roles + - rolebindings + verbs: + - get + - list + - watch + - create + - update + - delete + - patch - apiGroups: - authentication.k8s.io resources: @@ -41,6 +79,20 @@ rules: - create --- apiVersion: rbac.authorization.k8s.io/v1 +kind: RoleBinding +metadata: + name: vm-console-proxy + namespace: kube-system +roleRef: + apiGroup: rbac.authorization.k8s.io + kind: Role + name: extension-apiserver-authentication-reader +subjects: +- kind: ServiceAccount + name: vm-console-proxy + namespace: kubevirt +--- +apiVersion: rbac.authorization.k8s.io/v1 kind: ClusterRoleBinding metadata: name: vm-console-proxy @@ -100,7 +152,7 @@ spec: - args: [] command: - /console - image: quay.io/kubevirt/vm-console-proxy:v0.2.0 + image: quay.io/kubevirt/vm-console-proxy:v0.3.1 imagePullPolicy: Always name: console ports: @@ -119,9 +171,6 @@ spec: - mountPath: /tmp/vm-console-proxy-cert name: vm-console-proxy-cert readOnly: true - - mountPath: /etc/virt-handler/clientcertificates - name: kubevirt-virt-handler-certs - readOnly: true securityContext: runAsNonRoot: true seccompProfile: @@ -135,6 +184,19 @@ spec: - name: vm-console-proxy-cert secret: secretName: vm-console-proxy-cert - - name: kubevirt-virt-handler-certs - secret: - secretName: kubevirt-virt-handler-certs +--- +apiVersion: apiregistration.k8s.io/v1 +kind: APIService +metadata: + annotations: + service.beta.openshift.io/inject-cabundle: "true" + name: v1alpha1.token.kubevirt.io +spec: + group: token.kubevirt.io + groupPriorityMinimum: 2000 + service: + name: vm-console-proxy + namespace: kubevirt + port: 443 + version: v1alpha1 + versionPriority: 10 diff --git a/hack/csv-generator.go b/hack/csv-generator.go index 62dad6a21..64da9ef8e 100644 --- a/hack/csv-generator.go +++ b/hack/csv-generator.go @@ -48,6 +48,7 @@ type generatorFlags struct { tektonTasksImage string tektonTasksDiskVirtImage string virtioImage string + vmConsoleProxyImage string } var ( @@ -83,6 +84,7 @@ func init() { rootCmd.Flags().StringVar(&f.tektonTasksImage, "tekton-tasks-image", "", "Link to tekton tasks image") rootCmd.Flags().StringVar(&f.tektonTasksDiskVirtImage, "tekton-tasks-disk-virt-image", "", "Link to tekton tasks disk virt image") rootCmd.Flags().StringVar(&f.virtioImage, "virtio-image", "", "Link to virtio image") + rootCmd.Flags().StringVar(&f.vmConsoleProxyImage, "vm-console-proxy-image", "", "Link to VM console proxy image") rootCmd.Flags().Int32Var(&f.webhookPort, "webhook-port", 0, "Container port for the admission webhook") rootCmd.Flags().BoolVar(&f.removeCerts, "webhook-remove-certs", false, "Remove the webhook certificate volume and mount") rootCmd.Flags().BoolVar(&f.dumpCRDs, "dump-crds", false, "Dump crds to stdout") @@ -216,6 +218,14 @@ func buildRelatedImages(flags generatorFlags) ([]interface{}, error) { relatedImages = append(relatedImages, relatedImage) } + if flags.vmConsoleProxyImage != "" { + relatedImage, err := buildRelatedImage(flags.vmConsoleProxyImage, "vm-console-proxy") + if err != nil { + return nil, err + } + relatedImages = append(relatedImages, relatedImage) + } + return relatedImages, nil } @@ -271,6 +281,10 @@ func updateContainerEnvVars(flags generatorFlags, container v1.Container) []v1.E if flags.virtioImage != "" { envVariable.Value = flags.virtioImage } + case common.VmConsoleProxyImageKey: + if flags.vmConsoleProxyImage != "" { + envVariable.Value = flags.vmConsoleProxyImage + } } updatedVariables = append(updatedVariables, envVariable) diff --git a/hack/csv-generator_test.go b/hack/csv-generator_test.go index 1afb0d110..3e81c6f2e 100644 --- a/hack/csv-generator_test.go +++ b/hack/csv-generator_test.go @@ -27,6 +27,7 @@ var _ = Describe("csv generator", func() { tektonTasksImage: "testTektonTasksImage", tektonTasksDiskVirtImage: "testTektonTasksDiskVirtImage", virtioImage: "testVirtioImage", + vmConsoleProxyImage: "testVmConsoleProxyImage", } envValues := []v1.EnvVar{ {Name: common.TemplateValidatorImageKey}, @@ -90,6 +91,9 @@ var _ = Describe("csv generator", func() { if envVariable.Name == common.VirtioImageKey { Expect(envVariable.Value).To(Equal(flags.virtioImage)) } + if envVariable.Name == common.VmConsoleProxyImageKey { + Expect(envVariable.Value).To(Equal(flags.vmConsoleProxyImage)) + } } break } diff --git a/internal/common/environment.go b/internal/common/environment.go index 6dcf5c460..cea95d45f 100644 --- a/internal/common/environment.go +++ b/internal/common/environment.go @@ -21,6 +21,7 @@ const ( TektonTasksImageKey = "TEKTON_TASKS_IMAGE" TektonTasksDiskVirtImageKey = "TEKTON_TASKS_DISK_VIRT_IMAGE" VirtioImageKey = "VIRTIO_IMG" + VmConsoleProxyImageKey = "VM_CONSOLE_PROXY_IMAGE" DefaultTektonTasksIMG = "quay.io/kubevirt/tekton-tasks:" + TektonTasksVersion DeafultTektonTasksDiskVirtIMG = "quay.io/kubevirt/tekton-tasks-disk-virt:" + TektonTasksVersion diff --git a/internal/operands/vm-console-proxy/defaults.go b/internal/operands/vm-console-proxy/defaults.go new file mode 100644 index 000000000..746e8629d --- /dev/null +++ b/internal/operands/vm-console-proxy/defaults.go @@ -0,0 +1,8 @@ +package vm_console_proxy + +// This file is updated by GitHub action defined in .github/workflows/release-vm-console-proxy.yaml + +const ( + defaultVmConsoleProxyImageTag = "v0.3.1" + defaultVmConsoleProxyImage = "quay.io/kubevirt/vm-console-proxy:" + defaultVmConsoleProxyImageTag +) diff --git a/internal/operands/vm-console-proxy/reconcile.go b/internal/operands/vm-console-proxy/reconcile.go index 75e88c0e6..6e380ac1c 100644 --- a/internal/operands/vm-console-proxy/reconcile.go +++ b/internal/operands/vm-console-proxy/reconcile.go @@ -8,8 +8,8 @@ import ( apps "k8s.io/api/apps/v1" core "k8s.io/api/core/v1" rbac "k8s.io/api/rbac/v1" - metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" utilruntime "k8s.io/apimachinery/pkg/util/runtime" + apiregv1 "k8s.io/kube-aggregator/pkg/apis/apiregistration/v1" "kubevirt.io/ssp-operator/internal/common" "kubevirt.io/ssp-operator/internal/operands" "sigs.k8s.io/controller-runtime/pkg/client" @@ -30,16 +30,25 @@ const ( // +kubebuilder:rbac:groups=core,resources=services,verbs=get;list;watch;create;update;delete // +kubebuilder:rbac:groups=core,resources=configmaps;serviceaccounts,verbs=list;watch;create;update;delete // +kubebuilder:rbac:groups=apps,resources=deployments,verbs=get;list;watch;create;update;delete -// +kubebuilder:rbac:groups=rbac.authorization.k8s.io,resources=clusterroles;clusterrolebindings,verbs=list;watch;create;update;delete -// +kubebuilder:rbac:groups=route.openshift.io,resources=routes,verbs=list;watch;create;update;delete +// +kubebuilder:rbac:groups=rbac.authorization.k8s.io,resources=clusterroles;clusterrolebindings;rolebindings,verbs=list;watch;create;update;delete +// +kubebuilder:rbac:groups=apiregistration.k8s.io,resources=apiservices,verbs=get;list;watch;create;update;delete + +// Deprecated: +// +kubebuilder:rbac:groups=route.openshift.io,resources=routes,verbs=list;watch;delete // RBAC for created roles -// +kubebuilder:rbac:groups=kubevirt.io,resources=virtualmachineinstances,verbs=get;list;watch +// +kubebuilder:rbac:groups=core,resources=pods,verbs=get;list;watch +// +kubebuilder:rbac:groups=core,resources=serviceaccounts,verbs=get;list;watch;create;update;delete;patch +// +kubebuilder:rbac:groups=core,resources=serviceaccounts/token,verbs=create +// +kubebuilder:rbac:groups=kubevirt.io,resources=virtualmachineinstances;virtualmachines,verbs=get;list;watch +// +kubebuilder:rbac:groups=subresources.kubevirt.io,resources=virtualmachineinstances/vnc,verbs=get +// +kubebuilder:rbac:groups=rbac.authorization.k8s.io,resources=roles;rolebindings,verbs=get;list;watch;create;update;delete;patch // +kubebuilder:rbac:groups=authentication.k8s.io,resources=tokenreviews,verbs=create // +kubebuilder:rbac:groups=authorization.k8s.io,resources=subjectaccessreviews,verbs=create func init() { utilruntime.Must(routev1.Install(common.Scheme)) + utilruntime.Must(apiregv1.AddToScheme(common.Scheme)) } func WatchClusterTypes() []operands.WatchType { @@ -50,6 +59,7 @@ func WatchClusterTypes() []operands.WatchType { {Object: &core.Service{}}, {Object: &apps.Deployment{}, WatchFullObject: true}, {Object: &core.ConfigMap{}}, + {Object: &apiregv1.APIService{}}, {Object: &routev1.Route{}}, } } @@ -58,9 +68,11 @@ type vmConsoleProxy struct { serviceAccount *core.ServiceAccount clusterRole *rbac.ClusterRole clusterRoleBinding *rbac.ClusterRoleBinding + roleBinding *rbac.RoleBinding service *core.Service deployment *apps.Deployment configMap *core.ConfigMap + apiService *apiregv1.APIService } var _ operands.Operand = &vmConsoleProxy{} @@ -70,9 +82,11 @@ func New(bundle *vm_console_proxy_bundle.Bundle) *vmConsoleProxy { serviceAccount: bundle.ServiceAccount, clusterRole: bundle.ClusterRole, clusterRoleBinding: bundle.ClusterRoleBinding, + roleBinding: bundle.RoleBinding, service: bundle.Service, deployment: bundle.Deployment, configMap: bundle.ConfigMap, + apiService: bundle.ApiService, } } @@ -103,20 +117,42 @@ func (v *vmConsoleProxy) Reconcile(request *common.Request) ([]common.ReconcileR return results, nil } - return common.CollectResourceStatus(request, + reconcileResults, err := common.CollectResourceStatus(request, reconcileServiceAccount(*v.serviceAccount.DeepCopy()), reconcileClusterRole(*v.clusterRole.DeepCopy()), reconcileClusterRoleBinding(*v.clusterRoleBinding.DeepCopy()), + reconcileRoleBinding(v.roleBinding.DeepCopy()), reconcileConfigMap(*v.configMap.DeepCopy()), reconcileService(*v.service.DeepCopy()), reconcileDeployment(*v.deployment.DeepCopy()), - reconcileRoute(v.service.GetName())) + reconcileApiService(v.apiService.DeepCopy())) + if err != nil { + return nil, err + } + + // Route is no longer needed. + routeCleanupResults, err := v.deleteRoute(request) + if err != nil { + return nil, err + } + for _, cleanupResult := range routeCleanupResults { + if !cleanupResult.Deleted { + reconcileResults = append(reconcileResults, common.ResourceDeletedResult(cleanupResult.Resource, common.OperationResultDeleted)) + } + } + + return reconcileResults, nil } func (v *vmConsoleProxy) Cleanup(request *common.Request) ([]common.CleanupResult, error) { // We need to use labels to find resources that were deployed by this operand, // because namespace annotation may not be present. + routeCleanupResults, err := v.deleteRoute(request) + if err != nil { + return nil, err + } + var objectsToDelete []client.Object serviceAccounts, err := findResourcesUsingLabels( @@ -163,6 +199,21 @@ func (v *vmConsoleProxy) Cleanup(request *common.Request) ([]common.CleanupResul } objectsToDelete = append(objectsToDelete, deployments...) + objectsToDelete = append(objectsToDelete, + v.clusterRole.DeepCopy(), + v.clusterRoleBinding.DeepCopy(), + v.roleBinding.DeepCopy(), + v.apiService.DeepCopy()) + + cleanupResults, err := common.DeleteAll(request, objectsToDelete...) + if err != nil { + return nil, err + } + + return append(cleanupResults, routeCleanupResults...), nil +} + +func (v *vmConsoleProxy) deleteRoute(request *common.Request) ([]common.CleanupResult, error) { routes, err := findResourcesUsingLabels( request.Context, routeName, @@ -172,13 +223,8 @@ func (v *vmConsoleProxy) Cleanup(request *common.Request) ([]common.CleanupResul if err != nil { return nil, err } - objectsToDelete = append(objectsToDelete, routes...) - - objectsToDelete = append(objectsToDelete, - v.clusterRole.DeepCopy(), - v.clusterRoleBinding.DeepCopy()) - return common.DeleteAll(request, objectsToDelete...) + return common.DeleteAll(request, routes...) } func reconcileServiceAccount(serviceAccount core.ServiceAccount) common.ReconcileFunc { @@ -209,6 +255,15 @@ func reconcileClusterRoleBinding(clusterRoleBinding rbac.ClusterRoleBinding) com } } +func reconcileRoleBinding(roleBinding *rbac.RoleBinding) common.ReconcileFunc { + return func(request *common.Request) (common.ReconcileResult, error) { + return common.CreateOrUpdate(request). + ClusterResource(roleBinding). + WithAppLabels(operandName, operandComponent). + Reconcile() + } +} + func reconcileConfigMap(configMap core.ConfigMap) common.ReconcileFunc { return func(request *common.Request) (common.ReconcileResult, error) { configMap.Namespace = getVmConsoleProxyNamespace(request) @@ -240,11 +295,21 @@ func reconcileDeployment(deployment apps.Deployment) common.ReconcileFunc { } } -func reconcileRoute(serviceName string) common.ReconcileFunc { +func reconcileApiService(apiService *apiregv1.APIService) common.ReconcileFunc { return func(request *common.Request) (common.ReconcileResult, error) { + apiService.Spec.Service.Namespace = getVmConsoleProxyNamespace(request) return common.CreateOrUpdate(request). - ClusterResource(newRoute(getVmConsoleProxyNamespace(request), serviceName)). + ClusterResource(apiService). WithAppLabels(operandName, operandComponent). + UpdateFunc(func(expected, found client.Object) { + foundApiService := found.(*apiregv1.APIService) + expectedApiService := expected.(*apiregv1.APIService) + + // Keep CA bundle the same in the found object + expectedApiService.Spec.CABundle = foundApiService.Spec.CABundle + + foundApiService.Spec = expectedApiService.Spec + }). Reconcile() } } @@ -261,7 +326,7 @@ func getVmConsoleProxyNamespace(request *common.Request) string { } func getVmConsoleProxyImage() string { - return common.EnvOrDefault("VM_CONSOLE_PROXY_IMAGE", "quay.io/kubevirt/vm-console-proxy:v0.1.0") + return common.EnvOrDefault(common.VmConsoleProxyImageKey, defaultVmConsoleProxyImage) } func findResourcesUsingLabels[PtrL interface { @@ -296,23 +361,3 @@ func findResourcesUsingLabels[PtrL interface { } return filteredItems, nil } - -func newRoute(namespace string, serviceName string) *routev1.Route { - return &routev1.Route{ - ObjectMeta: metav1.ObjectMeta{ - Name: routeName, - Namespace: namespace, - }, - Spec: routev1.RouteSpec{ - To: routev1.RouteTargetReference{ - Kind: "Service", - Name: serviceName, - }, - TLS: &routev1.TLSConfig{ - Termination: routev1.TLSTerminationReencrypt, - InsecureEdgeTerminationPolicy: routev1.InsecureEdgeTerminationPolicyRedirect, - }, - WildcardPolicy: routev1.WildcardPolicyNone, - }, - } -} diff --git a/internal/operands/vm-console-proxy/reconcile_test.go b/internal/operands/vm-console-proxy/reconcile_test.go index e451aab05..47f70c1f2 100644 --- a/internal/operands/vm-console-proxy/reconcile_test.go +++ b/internal/operands/vm-console-proxy/reconcile_test.go @@ -2,12 +2,14 @@ package vm_console_proxy import ( "context" + "os" "testing" . "github.com/onsi/ginkgo/v2" . "github.com/onsi/gomega" routev1 "github.com/openshift/api/route/v1" + libhandler "github.com/operator-framework/operator-lib/handler" apps "k8s.io/api/apps/v1" authenticationv1 "k8s.io/api/authentication/v1" core "k8s.io/api/core/v1" @@ -16,15 +18,18 @@ import ( metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" "k8s.io/apimachinery/pkg/types" "k8s.io/apimachinery/pkg/util/intstr" + apiregv1 "k8s.io/kube-aggregator/pkg/apis/apiregistration/v1" + "k8s.io/utils/pointer" kubevirt "kubevirt.io/api/core" - ssp "kubevirt.io/ssp-operator/api/v1beta2" - "kubevirt.io/ssp-operator/internal/common" - . "kubevirt.io/ssp-operator/internal/test-utils" - vm_console_proxy_bundle "kubevirt.io/ssp-operator/internal/vm-console-proxy-bundle" "sigs.k8s.io/controller-runtime/pkg/client" "sigs.k8s.io/controller-runtime/pkg/client/fake" logf "sigs.k8s.io/controller-runtime/pkg/log" "sigs.k8s.io/controller-runtime/pkg/reconcile" + + ssp "kubevirt.io/ssp-operator/api/v1beta2" + "kubevirt.io/ssp-operator/internal/common" + . "kubevirt.io/ssp-operator/internal/test-utils" + vm_console_proxy_bundle "kubevirt.io/ssp-operator/internal/vm-console-proxy-bundle" ) const ( @@ -33,6 +38,7 @@ const ( vmConsoleProxyName = "vm-console-proxy" clusterRoleName = "vm-console-proxy" clusterRoleBindingName = "vm-console-proxy" + roleBindingName = "vm-console-proxy" serviceAccountName = "vm-console-proxy" configMapName = "vm-console-proxy" serviceName = "vm-console-proxy" @@ -61,12 +67,6 @@ var _ = Describe("VM Console Proxy Operand", func() { Expect(name).To(Equal(operandName), "should return correct name") }) - It("should return functions from reconcile correctly", func() { - functions, err := operand.Reconcile(&request) - Expect(err).ToNot(HaveOccurred(), "should not throw err") - Expect(functions).To(HaveLen(7), "should return correct number of reconcile functions") - }) - It("should create vm-console-proxy resources", func() { _, err := operand.Reconcile(&request) Expect(err).ToNot(HaveOccurred()) @@ -74,10 +74,42 @@ var _ = Describe("VM Console Proxy Operand", func() { ExpectResourceExists(bundle.ServiceAccount, request) ExpectResourceExists(bundle.ClusterRole, request) ExpectResourceExists(bundle.ClusterRoleBinding, request) + ExpectResourceExists(bundle.RoleBinding, request) ExpectResourceExists(bundle.ConfigMap, request) ExpectResourceExists(bundle.Service, request) ExpectResourceExists(bundle.Deployment, request) - ExpectResourceExists(newRoute(namespace, serviceName), request) + ExpectResourceExists(bundle.ApiService, request) + }) + + It("should read deployment image the environment variable", func() { + originalImage := os.Getenv(common.VmConsoleProxyImageKey) + + newImage := "www.example.org/images/vm-console-proxy:latest" + Expect(os.Setenv(common.VmConsoleProxyImageKey, newImage)).To(Succeed()) + DeferCleanup(func() { + Expect(os.Setenv(common.VmConsoleProxyImageKey, originalImage)).To(Succeed()) + }) + + _, err := operand.Reconcile(&request) + Expect(err).ToNot(HaveOccurred()) + + deployment := &apps.Deployment{} + key := client.ObjectKeyFromObject(bundle.Deployment) + Expect(request.Client.Get(request.Context, key, deployment)).To(Succeed()) + + Expect(deployment.Spec.Template.Spec.Containers).To(HaveLen(1)) + Expect(deployment.Spec.Template.Spec.Containers[0].Image).To(Equal(newImage)) + }) + + It("should deploy APIService with ServiceReference pointing to the right namespace", func() { + _, err := operand.Reconcile(&request) + Expect(err).ToNot(HaveOccurred()) + + apiService := &apiregv1.APIService{} + key := client.ObjectKeyFromObject(bundle.ApiService) + Expect(request.Client.Get(request.Context, key, apiService)).To(Succeed()) + + Expect(apiService.Spec.Service.Namespace).To(Equal(namespace)) }) It("should remove cluster resources on cleanup", func() { @@ -87,10 +119,11 @@ var _ = Describe("VM Console Proxy Operand", func() { ExpectResourceExists(bundle.ServiceAccount, request) ExpectResourceExists(bundle.ClusterRole, request) ExpectResourceExists(bundle.ClusterRoleBinding, request) + ExpectResourceExists(bundle.RoleBinding, request) ExpectResourceExists(bundle.ConfigMap, request) ExpectResourceExists(bundle.Service, request) ExpectResourceExists(bundle.Deployment, request) - ExpectResourceExists(newRoute(namespace, serviceName), request) + ExpectResourceExists(bundle.ApiService, request) _, err = operand.Cleanup(&request) Expect(err).ToNot(HaveOccurred()) @@ -98,12 +131,45 @@ var _ = Describe("VM Console Proxy Operand", func() { ExpectResourceNotExists(bundle.ServiceAccount, request) ExpectResourceNotExists(bundle.ClusterRole, request) ExpectResourceNotExists(bundle.ClusterRoleBinding, request) + ExpectResourceNotExists(bundle.RoleBinding, request) ExpectResourceNotExists(bundle.ConfigMap, request) ExpectResourceNotExists(bundle.Service, request) ExpectResourceNotExists(bundle.Deployment, request) - ExpectResourceNotExists(newRoute(namespace, serviceName), request) + ExpectResourceNotExists(bundle.ApiService, request) }) + DescribeTable("should delete Route leftover from previous version", func(op func() error) { + route := &routev1.Route{ + ObjectMeta: metav1.ObjectMeta{ + Name: routeName, + Namespace: namespace, + Annotations: map[string]string{ + libhandler.TypeAnnotation: "SSP.ssp.kubevirt.io", + libhandler.NamespacedNameAnnotation: namespace + "/" + name, + }, + Labels: map[string]string{ + common.AppKubernetesNameLabel: operandName, + common.AppKubernetesComponentLabel: operandComponent, + common.AppKubernetesManagedByLabel: common.AppKubernetesManagedByValue, + }, + }, + Spec: routev1.RouteSpec{}, + } + + Expect(request.Client.Create(request.Context, route)).To(Succeed()) + Expect(op()).To(Succeed()) + ExpectResourceNotExists(route.DeepCopy(), request) + }, + Entry("on reconcile", func() error { + _, err := operand.Reconcile(&request) + return err + }), + Entry("on cleanup", func() error { + _, err := operand.Cleanup(&request) + return err + }), + ) + It("should not update service cluster IP", func() { _, err := operand.Reconcile(&request) Expect(err).ToNot(HaveOccurred()) @@ -182,10 +248,11 @@ var _ = Describe("VM Console Proxy Operand", func() { ExpectResourceExists(bundle.ServiceAccount, request) ExpectResourceExists(bundle.ClusterRole, request) ExpectResourceExists(bundle.ClusterRoleBinding, request) + ExpectResourceExists(bundle.RoleBinding, request) ExpectResourceExists(bundle.ConfigMap, request) ExpectResourceExists(bundle.Service, request) ExpectResourceExists(bundle.Deployment, request) - ExpectResourceExists(newRoute(namespace, serviceName), request) + ExpectResourceExists(bundle.ApiService, request) request.Instance.Spec.FeatureGates.DeployVmConsoleProxy = false @@ -195,10 +262,11 @@ var _ = Describe("VM Console Proxy Operand", func() { ExpectResourceNotExists(bundle.ServiceAccount, request) ExpectResourceNotExists(bundle.ClusterRole, request) ExpectResourceNotExists(bundle.ClusterRoleBinding, request) + ExpectResourceNotExists(bundle.RoleBinding, request) ExpectResourceNotExists(bundle.ConfigMap, request) ExpectResourceNotExists(bundle.Service, request) ExpectResourceNotExists(bundle.Deployment, request) - ExpectResourceNotExists(newRoute(namespace, serviceName), request) + ExpectResourceNotExists(bundle.ApiService, request) }) Context("with namespace annotation", func() { @@ -209,7 +277,6 @@ var _ = Describe("VM Console Proxy Operand", func() { configMap *core.ConfigMap service *core.Service deployment *apps.Deployment - route *routev1.Route ) BeforeEach(func() { @@ -225,8 +292,6 @@ var _ = Describe("VM Console Proxy Operand", func() { deployment = bundle.Deployment.DeepCopy() deployment.Namespace = otherNamespace - route = newRoute(otherNamespace, serviceName) - request.Instance.GetAnnotations()[VmConsoleProxyNamespaceAnnotation] = otherNamespace }) @@ -237,10 +302,11 @@ var _ = Describe("VM Console Proxy Operand", func() { ExpectResourceExists(serviceAccount, request) ExpectResourceExists(bundle.ClusterRole, request) ExpectResourceExists(bundle.ClusterRoleBinding, request) + ExpectResourceExists(bundle.RoleBinding, request) ExpectResourceExists(configMap, request) ExpectResourceExists(service, request) ExpectResourceExists(deployment, request) - ExpectResourceExists(route, request) + ExpectResourceExists(bundle.ApiService, request) }) It("should cleanup resources from namespace provided by annotation", func() { @@ -250,10 +316,11 @@ var _ = Describe("VM Console Proxy Operand", func() { ExpectResourceExists(serviceAccount, request) ExpectResourceExists(bundle.ClusterRole, request) ExpectResourceExists(bundle.ClusterRoleBinding, request) + ExpectResourceExists(bundle.RoleBinding, request) ExpectResourceExists(configMap, request) ExpectResourceExists(service, request) ExpectResourceExists(deployment, request) - ExpectResourceExists(route, request) + ExpectResourceExists(bundle.ApiService, request) _, err = operand.Cleanup(&request) Expect(err).ToNot(HaveOccurred()) @@ -261,10 +328,22 @@ var _ = Describe("VM Console Proxy Operand", func() { ExpectResourceNotExists(serviceAccount, request) ExpectResourceNotExists(bundle.ClusterRole, request) ExpectResourceNotExists(bundle.ClusterRoleBinding, request) + ExpectResourceNotExists(bundle.RoleBinding, request) ExpectResourceNotExists(configMap, request) ExpectResourceNotExists(service, request) ExpectResourceNotExists(deployment, request) - ExpectResourceNotExists(route, request) + ExpectResourceNotExists(bundle.ApiService, request) + }) + + It("should deploy APIService with ServiceReference pointing to the right namespace", func() { + _, err := operand.Reconcile(&request) + Expect(err).ToNot(HaveOccurred()) + + apiService := &apiregv1.APIService{} + key := client.ObjectKeyFromObject(bundle.ApiService) + Expect(request.Client.Get(request.Context, key, apiService)).To(Succeed()) + + Expect(apiService.Spec.Service.Namespace).To(Equal(otherNamespace)) }) It("should remove resources from the namespace", func() { @@ -274,10 +353,11 @@ var _ = Describe("VM Console Proxy Operand", func() { ExpectResourceExists(serviceAccount, request) ExpectResourceExists(bundle.ClusterRole, request) ExpectResourceExists(bundle.ClusterRoleBinding, request) + ExpectResourceExists(bundle.RoleBinding, request) ExpectResourceExists(configMap, request) ExpectResourceExists(service, request) ExpectResourceExists(deployment, request) - ExpectResourceExists(route, request) + ExpectResourceExists(bundle.ApiService, request) request.Instance.Spec.FeatureGates.DeployVmConsoleProxy = false @@ -289,11 +369,44 @@ var _ = Describe("VM Console Proxy Operand", func() { ExpectResourceNotExists(serviceAccount, request) ExpectResourceNotExists(bundle.ClusterRole, request) ExpectResourceNotExists(bundle.ClusterRoleBinding, request) + ExpectResourceNotExists(bundle.RoleBinding, request) ExpectResourceNotExists(configMap, request) ExpectResourceNotExists(service, request) ExpectResourceNotExists(deployment, request) - ExpectResourceNotExists(route, request) + ExpectResourceNotExists(bundle.ApiService, request) }) + + DescribeTable("should delete Route leftover from previous version", func(op func() error) { + route := &routev1.Route{ + ObjectMeta: metav1.ObjectMeta{ + Name: routeName, + Namespace: otherNamespace, + Annotations: map[string]string{ + libhandler.TypeAnnotation: "SSP.ssp.kubevirt.io", + libhandler.NamespacedNameAnnotation: namespace + "/" + name, + }, + Labels: map[string]string{ + common.AppKubernetesNameLabel: operandName, + common.AppKubernetesComponentLabel: operandComponent, + common.AppKubernetesManagedByLabel: common.AppKubernetesManagedByValue, + }, + }, + Spec: routev1.RouteSpec{}, + } + + Expect(request.Client.Create(request.Context, route)).To(Succeed()) + Expect(op()).To(Succeed()) + ExpectResourceNotExists(route, request) + }, + Entry("on reconcile", func() error { + _, err := operand.Reconcile(&request) + return err + }), + Entry("on cleanup", func() error { + _, err := operand.Cleanup(&request) + return err + }), + ) }) }) @@ -394,6 +507,22 @@ func getMockedTestBundle() *vm_console_proxy_bundle.Bundle { Namespace: namespace, }}, }, + RoleBinding: &rbac.RoleBinding{ + ObjectMeta: metav1.ObjectMeta{ + Name: roleBindingName, + Namespace: "kube-system", + }, + RoleRef: rbac.RoleRef{ + Kind: "Role", + Name: "extension-apiserver-authentication-reader", + APIGroup: rbac.GroupName, + }, + Subjects: []rbac.Subject{{ + Kind: "ServiceAccount", + Name: serviceAccountName, + Namespace: namespace, + }}, + }, ConfigMap: &core.ConfigMap{ ObjectMeta: metav1.ObjectMeta{ Name: configMapName, @@ -525,6 +654,22 @@ func getMockedTestBundle() *vm_console_proxy_bundle.Bundle { }, }, }, + ApiService: &apiregv1.APIService{ + ObjectMeta: metav1.ObjectMeta{ + Name: "v1alpha1.token.kubevirt.io", + }, + Spec: apiregv1.APIServiceSpec{ + Group: "token.kubevirt.io", + GroupPriorityMinimum: 2000, + Version: "v1alpha1", + VersionPriority: 10, + Service: &apiregv1.ServiceReference{ + Name: serviceName, + Namespace: namespace, + Port: pointer.Int32(443), + }, + }, + }, } } diff --git a/internal/vm-console-proxy-bundle/bundle-test-duplicate.yaml b/internal/vm-console-proxy-bundle/bundle-test-duplicate.yaml index cc5b48fb5..7f2a321c7 100644 --- a/internal/vm-console-proxy-bundle/bundle-test-duplicate.yaml +++ b/internal/vm-console-proxy-bundle/bundle-test-duplicate.yaml @@ -32,10 +32,48 @@ rules: - kubevirt.io resources: - virtualmachineinstances + - virtualmachines verbs: - get - list - watch +- apiGroups: + - subresources.kubevirt.io + resources: + - virtualmachineinstances/vnc + verbs: + - get +- apiGroups: + - "" + resources: + - serviceaccounts + verbs: + - get + - list + - watch + - create + - update + - delete + - patch +- apiGroups: + - "" + resources: + - serviceaccounts/token + verbs: + - create +- apiGroups: + - rbac.authorization.k8s.io + resources: + - roles + - rolebindings + verbs: + - get + - list + - watch + - create + - update + - delete + - patch - apiGroups: - authentication.k8s.io resources: @@ -50,6 +88,20 @@ rules: - create --- apiVersion: rbac.authorization.k8s.io/v1 +kind: RoleBinding +metadata: + name: vm-console-proxy + namespace: kube-system +roleRef: + apiGroup: rbac.authorization.k8s.io + kind: Role + name: extension-apiserver-authentication-reader +subjects: + - kind: ServiceAccount + name: vm-console-proxy + namespace: kubevirt +--- +apiVersion: rbac.authorization.k8s.io/v1 kind: ClusterRoleBinding metadata: name: vm-console-proxy @@ -109,7 +161,7 @@ spec: - args: [] command: - /console - image: quay.io/kubevirt/vm-console-proxy:latest + image: quay.io/kubevirt/vm-console-proxy:v0.3.1 imagePullPolicy: Always name: console ports: @@ -128,9 +180,6 @@ spec: - mountPath: /tmp/vm-console-proxy-cert name: vm-console-proxy-cert readOnly: true - - mountPath: /etc/virt-handler/clientcertificates - name: kubevirt-virt-handler-certs - readOnly: true securityContext: runAsNonRoot: true seccompProfile: @@ -144,6 +193,19 @@ spec: - name: vm-console-proxy-cert secret: secretName: vm-console-proxy-cert - - name: kubevirt-virt-handler-certs - secret: - secretName: kubevirt-virt-handler-certs +--- +apiVersion: apiregistration.k8s.io/v1 +kind: APIService +metadata: + annotations: + service.beta.openshift.io/inject-cabundle: "true" + name: v1alpha1.token.kubevirt.io +spec: + group: token.kubevirt.io + groupPriorityMinimum: 2000 + service: + name: vm-console-proxy + namespace: kubevirt + port: 443 + version: v1alpha1 + versionPriority: 10 diff --git a/internal/vm-console-proxy-bundle/bundle-test-incorrect.yaml b/internal/vm-console-proxy-bundle/bundle-test-incorrect.yaml index 48e245eaf..18126af10 100644 --- a/internal/vm-console-proxy-bundle/bundle-test-incorrect.yaml +++ b/internal/vm-console-proxy-bundle/bundle-test-incorrect.yaml @@ -1,4 +1,4 @@ -# Incorrect because ConfigMap is missing. +# Incorrect because resources are missing. apiVersion: v1 kind: ServiceAccount metadata: diff --git a/internal/vm-console-proxy-bundle/bundle-test.yaml b/internal/vm-console-proxy-bundle/bundle-test.yaml index a1160c42d..ba4bb762c 100644 --- a/internal/vm-console-proxy-bundle/bundle-test.yaml +++ b/internal/vm-console-proxy-bundle/bundle-test.yaml @@ -23,10 +23,48 @@ rules: - kubevirt.io resources: - virtualmachineinstances + - virtualmachines verbs: - get - list - watch +- apiGroups: + - subresources.kubevirt.io + resources: + - virtualmachineinstances/vnc + verbs: + - get +- apiGroups: + - "" + resources: + - serviceaccounts + verbs: + - get + - list + - watch + - create + - update + - delete + - patch +- apiGroups: + - "" + resources: + - serviceaccounts/token + verbs: + - create +- apiGroups: + - rbac.authorization.k8s.io + resources: + - roles + - rolebindings + verbs: + - get + - list + - watch + - create + - update + - delete + - patch - apiGroups: - authentication.k8s.io resources: @@ -41,6 +79,20 @@ rules: - create --- apiVersion: rbac.authorization.k8s.io/v1 +kind: RoleBinding +metadata: + name: vm-console-proxy + namespace: kube-system +roleRef: + apiGroup: rbac.authorization.k8s.io + kind: Role + name: extension-apiserver-authentication-reader +subjects: + - kind: ServiceAccount + name: vm-console-proxy + namespace: kubevirt +--- +apiVersion: rbac.authorization.k8s.io/v1 kind: ClusterRoleBinding metadata: name: vm-console-proxy @@ -100,7 +152,7 @@ spec: - args: [] command: - /console - image: quay.io/kubevirt/vm-console-proxy:latest + image: quay.io/kubevirt/vm-console-proxy:v0.3.1 imagePullPolicy: Always name: console ports: @@ -119,9 +171,6 @@ spec: - mountPath: /tmp/vm-console-proxy-cert name: vm-console-proxy-cert readOnly: true - - mountPath: /etc/virt-handler/clientcertificates - name: kubevirt-virt-handler-certs - readOnly: true securityContext: runAsNonRoot: true seccompProfile: @@ -135,6 +184,19 @@ spec: - name: vm-console-proxy-cert secret: secretName: vm-console-proxy-cert - - name: kubevirt-virt-handler-certs - secret: - secretName: kubevirt-virt-handler-certs +--- +apiVersion: apiregistration.k8s.io/v1 +kind: APIService +metadata: + annotations: + service.beta.openshift.io/inject-cabundle: "true" + name: v1alpha1.token.kubevirt.io +spec: + group: token.kubevirt.io + groupPriorityMinimum: 2000 + service: + name: vm-console-proxy + namespace: kubevirt + port: 443 + version: v1alpha1 + versionPriority: 10 diff --git a/internal/vm-console-proxy-bundle/bundle.go b/internal/vm-console-proxy-bundle/bundle.go index 3f8aab6c0..38d2965c6 100644 --- a/internal/vm-console-proxy-bundle/bundle.go +++ b/internal/vm-console-proxy-bundle/bundle.go @@ -14,6 +14,7 @@ import ( "k8s.io/apimachinery/pkg/apis/meta/v1/unstructured" "k8s.io/apimachinery/pkg/runtime" "k8s.io/apimachinery/pkg/util/yaml" + apiregv1 "k8s.io/kube-aggregator/pkg/apis/apiregistration/v1" ) const ( @@ -23,18 +24,22 @@ const ( const ( clusterRoleKind = "ClusterRole" clusterRoleBindingsKind = "ClusterRoleBinding" + roleBindingKind = "RoleBinding" serviceKind = "Service" deploymentKind = "Deployment" configMapKind = "ConfigMap" + apiServiceKind = "APIService" ) type Bundle struct { ServiceAccount *core.ServiceAccount ClusterRole *rbac.ClusterRole ClusterRoleBinding *rbac.ClusterRoleBinding + RoleBinding *rbac.RoleBinding Service *core.Service Deployment *apps.Deployment ConfigMap *core.ConfigMap + ApiService *apiregv1.APIService } func ReadBundle(path string) (*Bundle, error) { @@ -81,12 +86,16 @@ func loadBundleFromBytes(data []byte) (*Bundle, error) { destObj = &(bundle.ClusterRole) case clusterRoleBindingsKind: destObj = &(bundle.ClusterRoleBinding) + case roleBindingKind: + destObj = &(bundle.RoleBinding) case serviceKind: destObj = &(bundle.Service) case deploymentKind: destObj = &(bundle.Deployment) case configMapKind: destObj = &(bundle.ConfigMap) + case apiServiceKind: + destObj = &(bundle.ApiService) case "": return nil, fmt.Errorf("empty Kind found in vm-console-proxy bundle") default: @@ -105,7 +114,7 @@ func loadBundleFromBytes(data []byte) (*Bundle, error) { } func validateBundle(bundle *Bundle) error { - missingFields := make([]string, 0, 6) + missingFields := make([]string, 0, 8) if bundle.ServiceAccount == nil { missingFields = append(missingFields, "ServiceAccount") } @@ -115,6 +124,9 @@ func validateBundle(bundle *Bundle) error { if bundle.ClusterRoleBinding == nil { missingFields = append(missingFields, "ClusterRoleBinding") } + if bundle.RoleBinding == nil { + missingFields = append(missingFields, "RoleBinding") + } if bundle.Service == nil { missingFields = append(missingFields, "Service") } @@ -124,6 +136,9 @@ func validateBundle(bundle *Bundle) error { if bundle.ConfigMap == nil { missingFields = append(missingFields, "ConfigMap") } + if bundle.ApiService == nil { + missingFields = append(missingFields, "ApiService") + } if len(missingFields) > 0 { return fmt.Errorf("bundle is missing these objects: %v", missingFields) } diff --git a/internal/vm-console-proxy-bundle/bundle_test.go b/internal/vm-console-proxy-bundle/bundle_test.go index 35cea3208..449d88691 100644 --- a/internal/vm-console-proxy-bundle/bundle_test.go +++ b/internal/vm-console-proxy-bundle/bundle_test.go @@ -19,9 +19,11 @@ var _ = Describe("VM Console Proxy Bundle", func() { Expect(bundle.ServiceAccount).ToNot(BeNil(), "service account should not be nil") Expect(bundle.ClusterRole).ToNot(BeNil(), "cluster role should not be nil") Expect(bundle.ClusterRoleBinding).ToNot(BeNil(), "cluster role binding should not be nil") + Expect(bundle.RoleBinding).ToNot(BeNil(), "role binding should not be nil") Expect(bundle.Service).ToNot(BeNil(), "service should not be nil") Expect(bundle.Deployment).ToNot(BeNil(), "deployment should not be nil") Expect(bundle.ConfigMap).ToNot(BeNil(), "config map should not be nil") + Expect(bundle.ApiService).ToNot(BeNil(), "api service should not be nil") }) It("should fail if bundle does not contain needed resources", func() { diff --git a/tests/e2e-test-csv-generator.sh b/tests/e2e-test-csv-generator.sh index d3afeccfa..921fd19fd 100755 --- a/tests/e2e-test-csv-generator.sh +++ b/tests/e2e-test-csv-generator.sh @@ -1,7 +1,7 @@ #!/bin/bash olm_output=$(podman run --rm --entrypoint=/csv-generator quay.io/kubevirt/ssp-operator:latest \ ---csv-version=9.9.9 --namespace=namespace-test --operator-version=8.8.8 --validator-image=validator-test --dump-crds \ +--csv-version=9.9.9 --namespace=namespace-test --operator-version=8.8.8 --validator-image=validator-test --vm-console-proxy-image=proxy-test --dump-crds \ --operator-image=operator-test) if [ $(echo $olm_output | grep 'ClusterServiceVersion' | wc -l) -eq 0 ]; then @@ -19,6 +19,11 @@ if [ $(echo $olm_output | grep 'value: validator-test'| wc -l) -eq 0 ]; then exit 1 fi +if [ $(echo $olm_output | grep 'value: proxy-test'| wc -l) -eq 0 ]; then + echo "output doesn't contain correct proxy-image" + exit 1 +fi + if [ $(echo $olm_output | grep 'value: 8.8.8'| wc -l) -eq 0 ]; then echo "output doesn't contain correct operator-version" exit 1 diff --git a/tests/tests_suite_test.go b/tests/tests_suite_test.go index 467216e81..a65db3688 100644 --- a/tests/tests_suite_test.go +++ b/tests/tests_suite_test.go @@ -392,6 +392,7 @@ func (s *existingSspStrategy) SkipIfUpgradeLane() { var ( apiClient client.Client coreClient *kubernetes.Clientset + apiServerHostname string ctx context.Context strategy TestSuiteStrategy sspListerWatcher cache.ListerWatcher @@ -468,6 +469,9 @@ func setupApiClient() { cfg, err := config.GetConfig() Expect(err).ToNot(HaveOccurred()) + + apiServerHostname = cfg.Host + apiClient, err = client.New(cfg, client.Options{Scheme: testScheme}) Expect(err).ToNot(HaveOccurred()) coreClient, err = kubernetes.NewForConfig(cfg) diff --git a/tests/vm_console_proxy_test.go b/tests/vm_console_proxy_test.go index 374ae7dd9..9a79107f9 100644 --- a/tests/vm_console_proxy_test.go +++ b/tests/vm_console_proxy_test.go @@ -4,19 +4,22 @@ import ( "crypto/tls" "io" "net/http" - "net/url" "reflect" "time" . "github.com/onsi/ginkgo/v2" . "github.com/onsi/gomega" - routev1 "github.com/openshift/api/route/v1" apps "k8s.io/api/apps/v1" + authnv1 "k8s.io/api/authentication/v1" core "k8s.io/api/core/v1" rbac "k8s.io/api/rbac/v1" + "k8s.io/apimachinery/pkg/api/errors" + metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" "k8s.io/apimachinery/pkg/util/intstr" + apiregv1 "k8s.io/kube-aggregator/pkg/apis/apiregistration/v1" "k8s.io/utils/pointer" + kubevirtcorev1 "kubevirt.io/api/core/v1" ssp "kubevirt.io/ssp-operator/api/v1beta2" vm_console_proxy "kubevirt.io/ssp-operator/internal/operands/vm-console-proxy" @@ -27,11 +30,12 @@ var _ = Describe("VM Console Proxy Operand", func() { var ( clusterRoleResource testResource clusterRoleBindingResource testResource + roleBindingResource testResource serviceAccountResource testResource serviceResource testResource deploymentResource testResource configMapResource testResource - routeResource testResource + apiServiceResource testResource ) BeforeEach(OncePerOrdered, func() { @@ -73,6 +77,19 @@ var _ = Describe("VM Console Proxy Operand", func() { reflect.DeepEqual(old.Subjects, new.Subjects) }, } + roleBindingResource = testResource{ + Name: "vm-console-proxy", + Namespace: "kube-system", + Resource: &rbac.RoleBinding{}, + ExpectedLabels: expectedLabels, + UpdateFunc: func(roleBinding *rbac.RoleBinding) { + roleBinding.Subjects = nil + }, + EqualsFunc: func(old *rbac.RoleBinding, new *rbac.RoleBinding) bool { + return reflect.DeepEqual(old.RoleRef, new.RoleRef) && + reflect.DeepEqual(old.Subjects, new.Subjects) + }, + } serviceAccountResource = testResource{ Name: "vm-console-proxy", Namespace: strategy.GetVmConsoleProxyNamespace(), @@ -118,15 +135,14 @@ var _ = Describe("VM Console Proxy Operand", func() { reflect.DeepEqual(old.BinaryData, new.BinaryData) }, } - routeResource = testResource{ - Name: "vm-console-proxy", - Namespace: strategy.GetVmConsoleProxyNamespace(), - Resource: &routev1.Route{}, + apiServiceResource = testResource{ + Name: "v1alpha1.token.kubevirt.io", + Resource: &apiregv1.APIService{}, ExpectedLabels: expectedLabels, - UpdateFunc: func(route *routev1.Route) { - route.Spec.TLS = nil + UpdateFunc: func(apiService *apiregv1.APIService) { + apiService.Spec.VersionPriority = apiService.Spec.VersionPriority + 10 }, - EqualsFunc: func(old, new *routev1.Route) bool { + EqualsFunc: func(old *apiregv1.APIService, new *apiregv1.APIService) bool { return reflect.DeepEqual(old.Spec, new.Spec) }, } @@ -147,21 +163,23 @@ var _ = Describe("VM Console Proxy Operand", func() { }, Entry("[test_id:9888] cluster role", &clusterRoleResource), Entry("[test_id:9847] cluster role binding", &clusterRoleBindingResource), + Entry("[test_id:TODO] role binding", &roleBindingResource), Entry("[test_id:9848] service account", &serviceAccountResource), Entry("[test_id:9849] service", &serviceResource), Entry("[test_id:9850] deployment", &deploymentResource), Entry("[test_id:9852] config map", &configMapResource), - Entry("[test_id:9854] route", &routeResource), + Entry("[test_id:TODO] API service", &apiServiceResource), ) DescribeTable("should set app labels", expectAppLabels, Entry("[test_id:9887] cluster role", &clusterRoleResource), Entry("[test_id:9851] cluster role binding", &clusterRoleBindingResource), + Entry("[test_id:TODO] role binding", &roleBindingResource), Entry("[test_id:9853] service account", &serviceAccountResource), Entry("[test_id:9855] service", &serviceResource), Entry("[test_id:9856] deployment", &deploymentResource), Entry("[test_id:9857] config map", &configMapResource), - Entry("[test_id:9859] route", &routeResource), + Entry("[test_id:TODO] API service", &apiServiceResource), ) }) @@ -169,11 +187,12 @@ var _ = Describe("VM Console Proxy Operand", func() { DescribeTable("recreate after delete", expectRecreateAfterDelete, Entry("[test_id:9858] cluster role", &clusterRoleResource), Entry("[test_id:9860] cluster role binding", &clusterRoleBindingResource), + Entry("[test_id:TODO] role binding", &roleBindingResource), Entry("[test_id:9861] service account", &serviceAccountResource), Entry("[test_id:9862] service", &serviceResource), Entry("[test_id:9864] deployment", &deploymentResource), Entry("[test_id:9866] config map", &configMapResource), - Entry("[test_id:9867] route", &routeResource), + Entry("[test_id:TODO] API service", &apiServiceResource), ) }) @@ -181,10 +200,11 @@ var _ = Describe("VM Console Proxy Operand", func() { DescribeTable("should restore modified resource", expectRestoreAfterUpdate, Entry("[test_id:9863] cluster role", &clusterRoleResource), Entry("[test_id:9865] cluster role binding", &clusterRoleBindingResource), + Entry("[test_id:TODO] role binding", &roleBindingResource), Entry("[test_id:9869] service", &serviceResource), Entry("[test_id:9870] deployment", &deploymentResource), Entry("[test_id:9871] config map", &configMapResource), - Entry("[test_id:9872] route", &routeResource), + Entry("[test_id:TODO] API service", &apiServiceResource), ) Context("With pause", func() { @@ -195,28 +215,30 @@ var _ = Describe("VM Console Proxy Operand", func() { DescribeTable("should restore modified resource with pause", expectRestoreAfterUpdateWithPause, Entry("[test_id:9873] cluster role", &clusterRoleResource), Entry("[test_id:9874] cluster role binding", &clusterRoleBindingResource), + Entry("[test_id:TODO] role binding", &roleBindingResource), Entry("[test_id:9876] service", &serviceResource), Entry("[test_id:9877] deployment", &deploymentResource), Entry("[test_id:9878] config map", &configMapResource), - Entry("[test_id:9879] route", &routeResource), + Entry("[test_id:TODO] API service", &apiServiceResource), ) }) DescribeTable("should restore modified app labels", expectAppLabelsRestoreAfterUpdate, Entry("[test_id:9880] cluster role", &clusterRoleResource), Entry("[test_id:9881] cluster role binding", &clusterRoleBindingResource), + Entry("[test_id:TODO] role binding", &roleBindingResource), Entry("[test_id:9882] service account", &serviceAccountResource), Entry("[test_id:9886] service", &serviceResource), Entry("[test_id:9883] deployment", &deploymentResource), Entry("[test_id:9884] config map", &configMapResource), - Entry("[test_id:9885] route", &routeResource), + Entry("[test_id:TODO] API service", &apiServiceResource), ) }) - Context("Route to access proxy", func() { + Context("Accessing proxy", func() { var ( - routeApiUrl string - httpClient *http.Client + httpClient *http.Client + saToken string ) BeforeEach(func() { @@ -226,27 +248,97 @@ var _ = Describe("VM Console Proxy Operand", func() { Transport: transport, } - route := &routev1.Route{} - Expect(apiClient.Get(ctx, routeResource.GetKey(), route)).To(Succeed()) - routeApiUrl = "https://" + route.Spec.Host + "/api/v1alpha1" - }) + serviceAccount := &core.ServiceAccount{ + ObjectMeta: metav1.ObjectMeta{ + GenerateName: "proxy-test-account-", + Namespace: strategy.GetNamespace(), + }, + } + Expect(apiClient.Create(ctx, serviceAccount)).To(Succeed()) + DeferCleanup(func() { + err := apiClient.Delete(ctx, serviceAccount) + if err != nil && !errors.IsNotFound(err) { + Expect(err).ToNot(HaveOccurred()) + } + }) - It("[test_id:9889] should be able to access /token endpoint", func() { - url, err := url.JoinPath(routeApiUrl, strategy.GetNamespace(), "non-existing-vm", "token") + role := &rbac.Role{ + ObjectMeta: metav1.ObjectMeta{ + GenerateName: "proxy-test-role-", + Namespace: strategy.GetNamespace(), + }, + Rules: []rbac.PolicyRule{{ + APIGroups: []string{"token.kubevirt.io"}, + Resources: []string{"virtualmachines/vnc"}, + Verbs: []string{"get"}, + }, { + APIGroups: []string{kubevirtcorev1.SubresourceGroupName}, + Resources: []string{"virtualmachineinstances/vnc"}, + Verbs: []string{"get"}, + }}, + } + Expect(apiClient.Create(ctx, role)).To(Succeed()) + DeferCleanup(func() { + err := apiClient.Delete(ctx, role) + if err != nil && !errors.IsNotFound(err) { + Expect(err).ToNot(HaveOccurred()) + } + }) + + roleBinding := &rbac.RoleBinding{ + ObjectMeta: metav1.ObjectMeta{ + GenerateName: "proxy-role-binding-", + Namespace: strategy.GetNamespace(), + }, + Subjects: []rbac.Subject{{ + Kind: "ServiceAccount", + Name: serviceAccount.Name, + Namespace: serviceAccount.Namespace, + }}, + RoleRef: rbac.RoleRef{ + APIGroup: rbac.GroupName, + Kind: "Role", + Name: role.Name, + }, + } + Expect(apiClient.Create(ctx, roleBinding)).To(Succeed()) + DeferCleanup(func() { + err := apiClient.Delete(ctx, roleBinding) + if err != nil && !errors.IsNotFound(err) { + Expect(err).ToNot(HaveOccurred()) + } + }) + + tokenRequest, err := coreClient.CoreV1().ServiceAccounts(serviceAccount.Namespace).CreateToken(ctx, serviceAccount.Name, &authnv1.TokenRequest{}, metav1.CreateOptions{}) Expect(err).ToNot(HaveOccurred()) - // It may take a moment for the service to be reachable through route + saToken = tokenRequest.Status.Token + }) + + It("[test_id:TODO] should be able to access /vnc endpoint", func() { + vmNamespace := strategy.GetNamespace() + vmName := "non-existing-vm" + + url := apiServerHostname + "/apis/token.kubevirt.io/v1alpha1/namespaces/" + vmNamespace + "/virtualmachines/" + vmName + "/vnc" + + // It may take a moment for the service to be reachable Eventually(func(g Gomega) { - response, err := httpClient.Get(url) + request, err := http.NewRequest("GET", url, nil) + g.Expect(err).ToNot(HaveOccurred()) + + request.Header.Set("Authorization", "Bearer "+saToken) + + response, err := httpClient.Do(request) + g.Expect(err).ToNot(HaveOccurred()) defer func() { _ = response.Body.Close() }() - g.Expect(response.StatusCode).To(Equal(http.StatusUnauthorized)) + g.Expect(response.StatusCode).To(Equal(http.StatusNotFound)) body, err := io.ReadAll(response.Body) g.Expect(err).ToNot(HaveOccurred()) - g.Expect(body).To(ContainSubstring("authenticating token cannot be empty")) + g.Expect(body).To(ContainSubstring("VirtualMachine does not exist:")) }, env.ShortTimeout(), time.Second).Should(Succeed()) }) })