From d139354642fcbd593e1caf208612d7a1fa305c39 Mon Sep 17 00:00:00 2001 From: nikita-bhatia <107468860+nikita-bhatia@users.noreply.github.com> Date: Thu, 10 Aug 2023 00:04:49 +0530 Subject: [PATCH] Cherrypick PWX-32732 : use SCC based on IsPrivileged flag (#1194) * PWX-32732 : use SCC based on IsPrivileged flag (#1191) * use portworx-restricted based on IsPrivileged flag * Add UTs for install with non-privileged annotation * Add UTs for install with non-privileged annotation * remove debug logs * resolve conflict * fix failing test --- drivers/storage/portworx/component/csi.go | 7 +- .../storage/portworx/component/lighthouse.go | 136 ++++++------- .../portworx/component/pvccontroller.go | 186 +++++++++--------- drivers/storage/portworx/components_test.go | 173 ++++++++++++++-- .../testspec/csiClusterRole_k8s_1.11.yaml | 2 +- .../testspec/csiClusterRole_k8s_1.13.yaml | 2 +- .../testspec/csiClusterRole_k8s_1.14.yaml | 2 +- .../csiClusterRole_nonPrivileged.yaml | 61 ++++++ .../testspec/lighthouseClusterRole.yaml | 2 +- .../lighthouseClusterRole_nonPrivileged.yaml | 38 ++++ .../testspec/pvcControllerClusterRole.yaml | 2 +- ...vcControllerClusterRole_nonPrivileged.yaml | 55 ++++++ 12 files changed, 483 insertions(+), 183 deletions(-) create mode 100644 drivers/storage/portworx/testspec/csiClusterRole_nonPrivileged.yaml create mode 100644 drivers/storage/portworx/testspec/lighthouseClusterRole_nonPrivileged.yaml create mode 100644 drivers/storage/portworx/testspec/pvcControllerClusterRole_nonPrivileged.yaml diff --git a/drivers/storage/portworx/component/csi.go b/drivers/storage/portworx/component/csi.go index 42f704102..e3936d5a8 100644 --- a/drivers/storage/portworx/component/csi.go +++ b/drivers/storage/portworx/component/csi.go @@ -226,6 +226,11 @@ func (c *csi) createClusterRole( cluster *corev1.StorageCluster, csiConfig *pxutil.CSIConfiguration, ) error { + sccName := PxSCCName + if !pxutil.IsPrivileged(cluster) { + sccName = PxRestrictedSCCName + } + clusterRole := &rbacv1.ClusterRole{ ObjectMeta: metav1.ObjectMeta{ Name: CSIClusterRoleName, @@ -320,7 +325,7 @@ func (c *csi) createClusterRole( { APIGroups: []string{"security.openshift.io"}, Resources: []string{"securitycontextconstraints"}, - ResourceNames: []string{PxRestrictedSCCName}, + ResourceNames: []string{sccName}, Verbs: []string{"use"}, }, { diff --git a/drivers/storage/portworx/component/lighthouse.go b/drivers/storage/portworx/component/lighthouse.go index c36f6cf83..b72b06433 100644 --- a/drivers/storage/portworx/component/lighthouse.go +++ b/drivers/storage/portworx/component/lighthouse.go @@ -90,7 +90,7 @@ func (c *lighthouse) Reconcile(cluster *corev1.StorageCluster) error { if err := c.createServiceAccount(cluster.Namespace, ownerRef); err != nil { return err } - if err := c.createClusterRole(); err != nil { + if err := c.createClusterRole(cluster); err != nil { return err } if err := c.createClusterRoleBinding(cluster.Namespace); err != nil { @@ -147,75 +147,79 @@ func (c *lighthouse) createServiceAccount( ) } -func (c *lighthouse) createClusterRole() error { - return k8sutil.CreateOrUpdateClusterRole( - c.k8sClient, - &rbacv1.ClusterRole{ - ObjectMeta: metav1.ObjectMeta{ - Name: LhClusterRoleName, +func (c *lighthouse) createClusterRole(cluster *corev1.StorageCluster) error { + sccName := PxSCCName + if !pxutil.IsPrivileged(cluster) { + sccName = PxRestrictedSCCName + } + + clusterRole := &rbacv1.ClusterRole{ + ObjectMeta: metav1.ObjectMeta{ + Name: LhClusterRoleName, + }, + Rules: []rbacv1.PolicyRule{ + { + APIGroups: []string{""}, + Resources: []string{"pods"}, + Verbs: []string{"get", "list"}, }, - Rules: []rbacv1.PolicyRule{ - { - APIGroups: []string{""}, - Resources: []string{"pods"}, - Verbs: []string{"get", "list"}, - }, - { - APIGroups: []string{"extensions", "apps"}, - Resources: []string{"deployments"}, - Verbs: []string{"get", "list"}, - }, - { - APIGroups: []string{""}, - Resources: []string{"secrets"}, - Verbs: []string{"get", "create", "update"}, - }, - { - APIGroups: []string{""}, - Resources: []string{"configmaps"}, - Verbs: []string{"get", "create", "update"}, - }, - { - APIGroups: []string{""}, - Resources: []string{"nodes"}, - Verbs: []string{"get", "list", "watch"}, - }, - { - APIGroups: []string{""}, - Resources: []string{"services"}, - Verbs: []string{"get", "list", "watch", "create"}, - }, - { - APIGroups: []string{"stork.libopenstorage.org"}, - Resources: []string{"*"}, - Verbs: []string{"get", "list", "create", "delete", "update"}, - }, - { - APIGroups: []string{"monitoring.coreos.com"}, - Resources: []string{ - "alertmanagers", - "prometheuses", - "prometheuses/finalizers", - "servicemonitors", - "prometheusrules", - }, - Verbs: []string{"*"}, - }, - { - APIGroups: []string{"security.openshift.io"}, - Resources: []string{"securitycontextconstraints"}, - ResourceNames: []string{PxRestrictedSCCName, "anyuid"}, - Verbs: []string{"use"}, - }, - { - APIGroups: []string{"policy"}, - Resources: []string{"podsecuritypolicies"}, - ResourceNames: []string{constants.PrivilegedPSPName}, - Verbs: []string{"use"}, + { + APIGroups: []string{"extensions", "apps"}, + Resources: []string{"deployments"}, + Verbs: []string{"get", "list"}, + }, + { + APIGroups: []string{""}, + Resources: []string{"secrets"}, + Verbs: []string{"get", "create", "update"}, + }, + { + APIGroups: []string{""}, + Resources: []string{"configmaps"}, + Verbs: []string{"get", "create", "update"}, + }, + { + APIGroups: []string{""}, + Resources: []string{"nodes"}, + Verbs: []string{"get", "list", "watch"}, + }, + { + APIGroups: []string{""}, + Resources: []string{"services"}, + Verbs: []string{"get", "list", "watch", "create"}, + }, + { + APIGroups: []string{"stork.libopenstorage.org"}, + Resources: []string{"*"}, + Verbs: []string{"get", "list", "create", "delete", "update"}, + }, + { + APIGroups: []string{"monitoring.coreos.com"}, + Resources: []string{ + "alertmanagers", + "prometheuses", + "prometheuses/finalizers", + "servicemonitors", + "prometheusrules", }, + Verbs: []string{"*"}, + }, + { + APIGroups: []string{"security.openshift.io"}, + Resources: []string{"securitycontextconstraints"}, + ResourceNames: []string{sccName, "anyuid"}, + Verbs: []string{"use"}, + }, + { + APIGroups: []string{"policy"}, + Resources: []string{"podsecuritypolicies"}, + ResourceNames: []string{constants.PrivilegedPSPName}, + Verbs: []string{"use"}, }, }, - ) + } + + return k8sutil.CreateOrUpdateClusterRole(c.k8sClient, clusterRole) } func (c *lighthouse) createClusterRoleBinding( diff --git a/drivers/storage/portworx/component/pvccontroller.go b/drivers/storage/portworx/component/pvccontroller.go index c2a352a15..8fe7b80ef 100644 --- a/drivers/storage/portworx/component/pvccontroller.go +++ b/drivers/storage/portworx/component/pvccontroller.go @@ -111,7 +111,7 @@ func (c *pvcController) Reconcile(cluster *corev1.StorageCluster) error { if err := c.createServiceAccount(cluster.Namespace, ownerRef); err != nil { return err } - if err := c.createClusterRole(); err != nil { + if err := c.createClusterRole(cluster); err != nil { return err } if err := c.createClusterRoleBinding(cluster.Namespace); err != nil { @@ -162,99 +162,103 @@ func (c *pvcController) createServiceAccount( ) } -func (c *pvcController) createClusterRole() error { - return k8sutil.CreateOrUpdateClusterRole( - c.k8sClient, - &rbacv1.ClusterRole{ - ObjectMeta: metav1.ObjectMeta{ - Name: PVCClusterRoleName, +func (c *pvcController) createClusterRole(cluster *corev1.StorageCluster) error { + sccName := PxSCCName + if !pxutil.IsPrivileged(cluster) { + sccName = PxRestrictedSCCName + } + + clusterRole := &rbacv1.ClusterRole{ + ObjectMeta: metav1.ObjectMeta{ + Name: PVCClusterRoleName, + }, + Rules: []rbacv1.PolicyRule{ + { + APIGroups: []string{""}, + Resources: []string{"persistentvolumes"}, + Verbs: []string{"get", "list", "watch", "create", "delete", "update"}, }, - Rules: []rbacv1.PolicyRule{ - { - APIGroups: []string{""}, - Resources: []string{"persistentvolumes"}, - Verbs: []string{"get", "list", "watch", "create", "delete", "update"}, - }, - { - APIGroups: []string{""}, - Resources: []string{"persistentvolumes/status"}, - Verbs: []string{"update"}, - }, - { - APIGroups: []string{""}, - Resources: []string{"persistentvolumeclaims"}, - Verbs: []string{"get", "list", "watch", "update"}, - }, - { - APIGroups: []string{""}, - Resources: []string{"persistentvolumeclaims/status"}, - Verbs: []string{"update"}, - }, - { - APIGroups: []string{""}, - Resources: []string{"pods"}, - Verbs: []string{"get", "list", "watch", "create", "delete"}, - }, - { - APIGroups: []string{"storage.k8s.io"}, - Resources: []string{"storageclasses"}, - Verbs: []string{"get", "list", "watch"}, - }, - { - APIGroups: []string{""}, - Resources: []string{"endpoints", "services"}, - Verbs: []string{"get", "create", "delete", "update"}, - }, - { - APIGroups: []string{""}, - Resources: []string{"secrets"}, - Verbs: []string{"get", "list"}, - }, - { - APIGroups: []string{""}, - Resources: []string{"nodes"}, - Verbs: []string{"get", "list", "watch"}, - }, - { - APIGroups: []string{"", "events.k8s.io"}, - Resources: []string{"events"}, - Verbs: []string{"get", "list", "watch", "create", "update", "patch"}, - }, - { - APIGroups: []string{""}, - Resources: []string{"serviceaccounts"}, - Verbs: []string{"get", "create"}, - }, - { - APIGroups: []string{""}, - Resources: []string{"serviceaccounts/token"}, - Verbs: []string{"create"}, - }, - { - APIGroups: []string{""}, - Resources: []string{"configmaps"}, - Verbs: []string{"get", "list", "watch", "create", "update"}, - }, - { - APIGroups: []string{"security.openshift.io"}, - Resources: []string{"securitycontextconstraints"}, - ResourceNames: []string{PxRestrictedSCCName}, - Verbs: []string{"use"}, - }, - { - APIGroups: []string{"policy"}, - Resources: []string{"podsecuritypolicies"}, - ResourceNames: []string{constants.PrivilegedPSPName}, - Verbs: []string{"use"}, - }, - { - APIGroups: []string{"coordination.k8s.io"}, - Resources: []string{"leases"}, - Verbs: []string{"*"}, - }, + { + APIGroups: []string{""}, + Resources: []string{"persistentvolumes/status"}, + Verbs: []string{"update"}, + }, + { + APIGroups: []string{""}, + Resources: []string{"persistentvolumeclaims"}, + Verbs: []string{"get", "list", "watch", "update"}, + }, + { + APIGroups: []string{""}, + Resources: []string{"persistentvolumeclaims/status"}, + Verbs: []string{"update"}, + }, + { + APIGroups: []string{""}, + Resources: []string{"pods"}, + Verbs: []string{"get", "list", "watch", "create", "delete"}, + }, + { + APIGroups: []string{"storage.k8s.io"}, + Resources: []string{"storageclasses"}, + Verbs: []string{"get", "list", "watch"}, + }, + { + APIGroups: []string{""}, + Resources: []string{"endpoints", "services"}, + Verbs: []string{"get", "create", "delete", "update"}, + }, + { + APIGroups: []string{""}, + Resources: []string{"secrets"}, + Verbs: []string{"get", "list"}, + }, + { + APIGroups: []string{""}, + Resources: []string{"nodes"}, + Verbs: []string{"get", "list", "watch"}, + }, + { + APIGroups: []string{"", "events.k8s.io"}, + Resources: []string{"events"}, + Verbs: []string{"get", "list", "watch", "create", "update", "patch"}, + }, + { + APIGroups: []string{""}, + Resources: []string{"serviceaccounts"}, + Verbs: []string{"get", "create"}, + }, + { + APIGroups: []string{""}, + Resources: []string{"serviceaccounts/token"}, + Verbs: []string{"create"}, + }, + { + APIGroups: []string{""}, + Resources: []string{"configmaps"}, + Verbs: []string{"get", "list", "watch", "create", "update"}, + }, + { + APIGroups: []string{"security.openshift.io"}, + Resources: []string{"securitycontextconstraints"}, + ResourceNames: []string{sccName}, + Verbs: []string{"use"}, + }, + { + APIGroups: []string{"policy"}, + Resources: []string{"podsecuritypolicies"}, + ResourceNames: []string{constants.PrivilegedPSPName}, + Verbs: []string{"use"}, + }, + { + APIGroups: []string{"coordination.k8s.io"}, + Resources: []string{"leases"}, + Verbs: []string{"*"}, }, }, - ) + } + + return k8sutil.CreateOrUpdateClusterRole(c.k8sClient, clusterRole) } func (c *pvcController) createClusterRoleBinding( diff --git a/drivers/storage/portworx/components_test.go b/drivers/storage/portworx/components_test.go index 4dc4f25ad..f985ff36d 100644 --- a/drivers/storage/portworx/components_test.go +++ b/drivers/storage/portworx/components_test.go @@ -1554,7 +1554,32 @@ func TestPVCControllerInstall(t *testing.T) { err = driver.PreInstall(cluster) require.NoError(t, err) - verifyPVCControllerInstall(t, cluster, k8sClient) + verifyPVCControllerInstall(t, cluster, k8sClient, "pvcControllerClusterRole.yaml") + verifyPVCControllerDeployment(t, cluster, k8sClient, "pvcControllerDeployment.yaml") +} +func TestPVCControllerInstallWithNonPriviliged(t *testing.T) { + coreops.SetInstance(coreops.New(fakek8sclient.NewSimpleClientset())) + reregisterComponents() + k8sClient := testutil.FakeK8sClient() + driver := portworx{} + err := driver.Init(k8sClient, runtime.NewScheme(), record.NewFakeRecorder(0)) + require.NoError(t, err) + + cluster := &corev1.StorageCluster{ + ObjectMeta: metav1.ObjectMeta{ + Name: "px-cluster", + Namespace: "kube-system", + Annotations: map[string]string{ + pxutil.AnnotationPVCController: "true", + pxutil.AnnotationIsPrivileged: "false", + }, + }, + } + + err = driver.PreInstall(cluster) + require.NoError(t, err) + + verifyPVCControllerInstall(t, cluster, k8sClient, "pvcControllerClusterRole_nonPrivileged.yaml") verifyPVCControllerDeployment(t, cluster, k8sClient, "pvcControllerDeployment.yaml") } @@ -1591,7 +1616,7 @@ func TestPVCControllerInstallWithK8s1_24(t *testing.T) { expectedDeployment.Spec.Template.Spec.Containers[0] = expectedContainer err = driver.PreInstall(cluster) require.NoError(t, err) - verifyPVCControllerInstall(t, cluster, k8sClient) + verifyPVCControllerInstall(t, cluster, k8sClient, "pvcControllerClusterRole.yaml") verifyPVCControllerDeploymentObject(t, cluster, k8sClient, expectedDeployment) } @@ -1629,7 +1654,7 @@ func TestPVCControllerInstallWithK8s1_22(t *testing.T) { expectedDeployment.Spec.Template.Spec.Containers[0] = expectedContainer err = driver.PreInstall(cluster) require.NoError(t, err) - verifyPVCControllerInstall(t, cluster, k8sClient) + verifyPVCControllerInstall(t, cluster, k8sClient, "pvcControllerClusterRole.yaml") verifyPVCControllerDeploymentObject(t, cluster, k8sClient, expectedDeployment) // TestCase: Add both port and secure port annotations @@ -1640,7 +1665,7 @@ func TestPVCControllerInstallWithK8s1_22(t *testing.T) { expectedDeployment.Spec.Template.Spec.Containers[0] = expectedContainer err = driver.PreInstall(cluster) require.NoError(t, err) - verifyPVCControllerInstall(t, cluster, k8sClient) + verifyPVCControllerInstall(t, cluster, k8sClient, "pvcControllerClusterRole.yaml") verifyPVCControllerDeploymentObject(t, cluster, k8sClient, expectedDeployment) // TestCase: Simple install on AKS @@ -1653,7 +1678,7 @@ func TestPVCControllerInstallWithK8s1_22(t *testing.T) { expectedDeployment.Spec.Template.Spec.Containers[0] = expectedContainer err = driver.PreInstall(cluster) require.NoError(t, err) - verifyPVCControllerInstall(t, cluster, k8sClient) + verifyPVCControllerInstall(t, cluster, k8sClient, "pvcControllerClusterRole.yaml") verifyPVCControllerDeploymentObject(t, cluster, k8sClient, expectedDeployment) } @@ -1815,7 +1840,7 @@ func TestPVCControllerInstallForPKS(t *testing.T) { err = driver.PreInstall(cluster) require.NoError(t, err) - verifyPVCControllerInstall(t, cluster, k8sClient) + verifyPVCControllerInstall(t, cluster, k8sClient, "pvcControllerClusterRole.yaml") verifyPVCControllerDeployment(t, cluster, k8sClient, "pvcControllerDeployment.yaml") // Despite invalid pvc controller annotation, install for PKS @@ -1824,7 +1849,7 @@ func TestPVCControllerInstallForPKS(t *testing.T) { err = driver.PreInstall(cluster) require.NoError(t, err) - verifyPVCControllerInstall(t, cluster, k8sClient) + verifyPVCControllerInstall(t, cluster, k8sClient, "pvcControllerClusterRole.yaml") verifyPVCControllerDeployment(t, cluster, k8sClient, "pvcControllerDeployment.yaml") } @@ -1849,7 +1874,7 @@ func TestPVCControllerInstallForEKS(t *testing.T) { err = driver.PreInstall(cluster) require.NoError(t, err) - verifyPVCControllerInstall(t, cluster, k8sClient) + verifyPVCControllerInstall(t, cluster, k8sClient, "pvcControllerClusterRole.yaml") verifyPVCControllerDeployment(t, cluster, k8sClient, "pvcControllerDeployment.yaml") // Despite invalid pvc controller annotation, install for EKS @@ -1858,7 +1883,7 @@ func TestPVCControllerInstallForEKS(t *testing.T) { err = driver.PreInstall(cluster) require.NoError(t, err) - verifyPVCControllerInstall(t, cluster, k8sClient) + verifyPVCControllerInstall(t, cluster, k8sClient, "pvcControllerClusterRole.yaml") verifyPVCControllerDeployment(t, cluster, k8sClient, "pvcControllerDeployment.yaml") } @@ -1883,7 +1908,7 @@ func TestPVCControllerInstallForGKE(t *testing.T) { err = driver.PreInstall(cluster) require.NoError(t, err) - verifyPVCControllerInstall(t, cluster, k8sClient) + verifyPVCControllerInstall(t, cluster, k8sClient, "pvcControllerClusterRole.yaml") verifyPVCControllerDeployment(t, cluster, k8sClient, "pvcControllerDeployment.yaml") // Despite invalid pvc controller annotation, install for GKE @@ -1892,7 +1917,7 @@ func TestPVCControllerInstallForGKE(t *testing.T) { err = driver.PreInstall(cluster) require.NoError(t, err) - verifyPVCControllerInstall(t, cluster, k8sClient) + verifyPVCControllerInstall(t, cluster, k8sClient, "pvcControllerClusterRole.yaml") verifyPVCControllerDeployment(t, cluster, k8sClient, "pvcControllerDeployment.yaml") } @@ -1917,7 +1942,7 @@ func TestPVCControllerInstallForOKE(t *testing.T) { err = driver.PreInstall(cluster) require.NoError(t, err) - verifyPVCControllerInstall(t, cluster, k8sClient) + verifyPVCControllerInstall(t, cluster, k8sClient, "pvcControllerClusterRole.yaml") verifyPVCControllerDeployment(t, cluster, k8sClient, "pvcControllerDeployment.yaml") // Despite invalid pvc controller annotation, install for OKE @@ -1926,7 +1951,7 @@ func TestPVCControllerInstallForOKE(t *testing.T) { err = driver.PreInstall(cluster) require.NoError(t, err) - verifyPVCControllerInstall(t, cluster, k8sClient) + verifyPVCControllerInstall(t, cluster, k8sClient, "pvcControllerClusterRole.yaml") verifyPVCControllerDeployment(t, cluster, k8sClient, "pvcControllerDeployment.yaml") } @@ -1951,7 +1976,7 @@ func TestPVCControllerInstallForAKS(t *testing.T) { err = driver.PreInstall(cluster) require.NoError(t, err) - verifyPVCControllerInstall(t, cluster, k8sClient) + verifyPVCControllerInstall(t, cluster, k8sClient, "pvcControllerClusterRole.yaml") specFileName := "pvcControllerDeployment.yaml" pvcControllerDeployment := testutil.GetExpectedDeployment(t, specFileName) @@ -1969,7 +1994,7 @@ func TestPVCControllerInstallForAKS(t *testing.T) { err = driver.PreInstall(cluster) require.NoError(t, err) - verifyPVCControllerInstall(t, cluster, k8sClient) + verifyPVCControllerInstall(t, cluster, k8sClient, "pvcControllerClusterRole.yaml") verifyPVCControllerDeploymentObject(t, cluster, k8sClient, pvcControllerDeployment) } @@ -2065,6 +2090,7 @@ func verifyPVCControllerInstall( t *testing.T, cluster *corev1.StorageCluster, k8sClient client.Client, + clusterRoleFileName string, ) { // PVC Controller ServiceAccount serviceAccountList := &v1.ServiceAccountList{} @@ -2086,7 +2112,7 @@ func verifyPVCControllerInstall( require.NoError(t, err) require.Len(t, clusterRoleList.Items, 2) - expectedCR := testutil.GetExpectedClusterRole(t, "pvcControllerClusterRole.yaml") + expectedCR := testutil.GetExpectedClusterRole(t, clusterRoleFileName) actualCR := &rbacv1.ClusterRole{} err = testutil.Get(k8sClient, actualCR, component.PVCClusterRoleName, "") require.NoError(t, err) @@ -2528,11 +2554,52 @@ func TestLighthouseInstall(t *testing.T) { err = driver.PreInstall(cluster) + require.NoError(t, err) + verifyLightHouseInstall(t, cluster, k8sClient, "lighthouseClusterRole.yaml") + +} + +func TestLighthouseInstallWithNonPrivileged(t *testing.T) { + coreops.SetInstance(coreops.New(fakek8sclient.NewSimpleClientset())) + reregisterComponents() + k8sClient := testutil.FakeK8sClient() + driver := portworx{} + err := driver.Init(k8sClient, runtime.NewScheme(), record.NewFakeRecorder(0)) require.NoError(t, err) + cluster := &corev1.StorageCluster{ + ObjectMeta: metav1.ObjectMeta{ + Name: "px-cluster", + Namespace: "kube-test", + Annotations: map[string]string{ + pxutil.AnnotationPVCController: "true", + pxutil.AnnotationIsPrivileged: "false", + }, + }, + Spec: corev1.StorageClusterSpec{ + UserInterface: &corev1.UserInterfaceSpec{ + Enabled: true, + Image: "docker.io/portworx/px-lighthouse:2.1.1", + }, + }, + } + + err = driver.PreInstall(cluster) + + require.NoError(t, err) + verifyLightHouseInstall(t, cluster, k8sClient, "lighthouseClusterRole_nonPrivileged.yaml") + +} + +func verifyLightHouseInstall( + t *testing.T, + cluster *corev1.StorageCluster, + k8sClient client.Client, + clusterRoleFileName string, +) { // Lighthouse ServiceAccount serviceAccountList := &v1.ServiceAccountList{} - err = testutil.List(k8sClient, serviceAccountList) + err := testutil.List(k8sClient, serviceAccountList) require.NoError(t, err) require.Len(t, serviceAccountList.Items, 3) @@ -2550,7 +2617,7 @@ func TestLighthouseInstall(t *testing.T) { require.NoError(t, err) require.Len(t, clusterRoleList.Items, 3) - expectedCR := testutil.GetExpectedClusterRole(t, "lighthouseClusterRole.yaml") + expectedCR := testutil.GetExpectedClusterRole(t, clusterRoleFileName) actualCR := &rbacv1.ClusterRole{} err = testutil.Get(k8sClient, actualCR, component.LhClusterRoleName, "") require.NoError(t, err) @@ -5352,9 +5419,74 @@ func TestCSIInstall(t *testing.T) { err = driver.PreInstall(cluster) require.NoError(t, err) + verifyCsiInstall(t, cluster, k8sClient, "csiClusterRole_k8s_1.11.yaml") +} + +func TestCSIInstallNonPrivileged(t *testing.T) { + versionClient := fakek8sclient.NewSimpleClientset() + coreops.SetInstance(coreops.New(versionClient)) + versionClient.Discovery().(*fakediscovery.FakeDiscovery).FakedServerVersion = &version.Info{ + GitVersion: "v1.11.4", + } + reregisterComponents() + k8sClient := testutil.FakeK8sClient() + driver := portworx{} + err := driver.Init(k8sClient, runtime.NewScheme(), record.NewFakeRecorder(10)) + require.NoError(t, err) + + cluster := &corev1.StorageCluster{ + ObjectMeta: metav1.ObjectMeta{ + Name: "px-cluster", + Namespace: "kube-test", + Annotations: map[string]string{ + pxutil.AnnotationPVCController: "false", + pxutil.AnnotationIsPrivileged: "false", + }, + }, + Spec: corev1.StorageClusterSpec{ + Image: "portworx/image:2.1.2", + Placement: &corev1.PlacementSpec{ + NodeAffinity: &v1.NodeAffinity{ + RequiredDuringSchedulingIgnoredDuringExecution: &v1.NodeSelector{ + NodeSelectorTerms: []v1.NodeSelectorTerm{ + { + MatchExpressions: []v1.NodeSelectorRequirement{ + { + Key: "px/enabled", + Operator: v1.NodeSelectorOpNotIn, + Values: []string{"false"}, + }, + { + Key: "node-role.kubernetes.io/master", + Operator: v1.NodeSelectorOpDoesNotExist, + }, + }, + }, + }, + }, + }, + }, + Monitoring: &corev1.MonitoringSpec{Telemetry: &corev1.TelemetrySpec{}}, + }, + } + err = driver.SetDefaultsOnStorageCluster(cluster) + require.NoError(t, err) + + err = driver.PreInstall(cluster) + require.NoError(t, err) + + verifyCsiInstall(t, cluster, k8sClient, "csiClusterRole_nonPrivileged.yaml") +} + +func verifyCsiInstall( + t *testing.T, + cluster *corev1.StorageCluster, + k8sClient client.Client, + clusterRoleFileName string, +) { // CSI ServiceAccount serviceAccountList := &v1.ServiceAccountList{} - err = testutil.List(k8sClient, serviceAccountList) + err := testutil.List(k8sClient, serviceAccountList) require.NoError(t, err) require.Len(t, serviceAccountList.Items, 2) @@ -5372,7 +5504,7 @@ func TestCSIInstall(t *testing.T) { require.NoError(t, err) require.Len(t, clusterRoleList.Items, 2) - expectedCR := testutil.GetExpectedClusterRole(t, "csiClusterRole_k8s_1.11.yaml") + expectedCR := testutil.GetExpectedClusterRole(t, clusterRoleFileName) actualCR := &rbacv1.ClusterRole{} err = testutil.Get(k8sClient, actualCR, component.CSIClusterRoleName, "") require.NoError(t, err) @@ -5434,6 +5566,7 @@ func TestCSIInstall(t *testing.T) { require.True(t, errors.IsNotFound(err)) err = testutil.Get(k8sClient, csiDriver, pxutil.DeprecatedCSIDriverName, "") require.True(t, errors.IsNotFound(err)) + } func TestCSIInstallWithk8s1_13(t *testing.T) { diff --git a/drivers/storage/portworx/testspec/csiClusterRole_k8s_1.11.yaml b/drivers/storage/portworx/testspec/csiClusterRole_k8s_1.11.yaml index bd399a9bd..0df06075b 100644 --- a/drivers/storage/portworx/testspec/csiClusterRole_k8s_1.11.yaml +++ b/drivers/storage/portworx/testspec/csiClusterRole_k8s_1.11.yaml @@ -53,7 +53,7 @@ rules: verbs: ["*"] - apiGroups: ["security.openshift.io"] resources: ["securitycontextconstraints"] - resourceNames: ["portworx-restricted"] + resourceNames: ["portworx"] verbs: ["use"] - apiGroups: ["policy"] resources: ["podsecuritypolicies"] diff --git a/drivers/storage/portworx/testspec/csiClusterRole_k8s_1.13.yaml b/drivers/storage/portworx/testspec/csiClusterRole_k8s_1.13.yaml index a811316a2..4296b0666 100644 --- a/drivers/storage/portworx/testspec/csiClusterRole_k8s_1.13.yaml +++ b/drivers/storage/portworx/testspec/csiClusterRole_k8s_1.13.yaml @@ -60,7 +60,7 @@ rules: verbs: ["*"] - apiGroups: ["security.openshift.io"] resources: ["securitycontextconstraints"] - resourceNames: ["portworx-restricted"] + resourceNames: ["portworx"] verbs: ["use"] - apiGroups: ["policy"] resources: ["podsecuritypolicies"] diff --git a/drivers/storage/portworx/testspec/csiClusterRole_k8s_1.14.yaml b/drivers/storage/portworx/testspec/csiClusterRole_k8s_1.14.yaml index f8df52ddc..0d0b0ef6b 100644 --- a/drivers/storage/portworx/testspec/csiClusterRole_k8s_1.14.yaml +++ b/drivers/storage/portworx/testspec/csiClusterRole_k8s_1.14.yaml @@ -56,7 +56,7 @@ rules: verbs: ["*"] - apiGroups: ["security.openshift.io"] resources: ["securitycontextconstraints"] - resourceNames: ["portworx-restricted"] + resourceNames: ["portworx"] verbs: ["use"] - apiGroups: ["policy"] resources: ["podsecuritypolicies"] diff --git a/drivers/storage/portworx/testspec/csiClusterRole_nonPrivileged.yaml b/drivers/storage/portworx/testspec/csiClusterRole_nonPrivileged.yaml new file mode 100644 index 000000000..faf9caec3 --- /dev/null +++ b/drivers/storage/portworx/testspec/csiClusterRole_nonPrivileged.yaml @@ -0,0 +1,61 @@ +kind: ClusterRole +apiVersion: rbac.authorization.k8s.io/v1 +metadata: + name: px-csi +rules: + - apiGroups: ["apiextensions.k8s.io"] + resources: ["customresourcedefinitions"] + verbs: ["*"] + - apiGroups: [""] + resources: ["nodes"] + verbs: ["get", "list", "watch"] + - apiGroups: [""] + resources: ["persistentvolumes"] + verbs: ["get", "list", "watch", "create", "delete", "update", "patch"] + - apiGroups: [""] + resources: ["persistentvolumeclaims"] + verbs: ["get", "list", "watch", "update"] + - apiGroups: [""] + resources: ["persistentvolumeclaims/status"] + verbs: ["update", "patch"] + - apiGroups: [""] + resources: ["pods"] + verbs: ["get", "list", "watch"] + - apiGroups: ["storage.k8s.io"] + resources: ["storageclasses"] + verbs: ["get", "list", "watch"] + - apiGroups: ["storage.k8s.io"] + resources: ["volumeattachments"] + verbs: ["get", "list", "watch", "update", "patch"] + - apiGroups: ["storage.k8s.io"] + resources: ["csistoragecapacities"] + verbs: ["get", "list", "watch", "create", "update", "patch", "delete"] + - apiGroups: ["apps"] + resources: ["replicasets"] + verbs: ["get"] + - apiGroups: ["", "events.k8s.io"] + resources: ["events"] + verbs: ["get", "list", "watch", "create", "update", "patch"] + - apiGroups: [""] + resources: ["secrets"] + verbs: ["get", "list"] + - apiGroups: ["snapshot.storage.k8s.io"] + resources: ["volumesnapshots", "volumesnapshotcontents", "volumesnapshotclasses", "volumesnapshots/status", "volumesnapshotcontents/status"] + verbs: ["get", "list", "watch", "create", "delete", "update", "patch"] + - apiGroups: ["csi.storage.k8s.io"] + resources: ["csidrivers"] + verbs: ["create", "delete"] + - apiGroups: [""] + resources: ["endpoints"] + verbs: ["get", "list", "watch", "create", "delete", "update"] + - apiGroups: ["coordination.k8s.io"] + resources: ["leases"] + verbs: ["*"] + - apiGroups: ["security.openshift.io"] + resources: ["securitycontextconstraints"] + resourceNames: ["portworx-restricted"] + verbs: ["use"] + - apiGroups: ["policy"] + resources: ["podsecuritypolicies"] + resourceNames: ["px-privileged"] + verbs: ["use"] diff --git a/drivers/storage/portworx/testspec/lighthouseClusterRole.yaml b/drivers/storage/portworx/testspec/lighthouseClusterRole.yaml index 671069e12..3763a88b2 100644 --- a/drivers/storage/portworx/testspec/lighthouseClusterRole.yaml +++ b/drivers/storage/portworx/testspec/lighthouseClusterRole.yaml @@ -30,7 +30,7 @@ rules: verbs: ["*"] - apiGroups: ["security.openshift.io"] resources: ["securitycontextconstraints"] - resourceNames: ["portworx-restricted", "anyuid"] + resourceNames: ["portworx", "anyuid"] verbs: ["use"] - apiGroups: ["policy"] resources: ["podsecuritypolicies"] diff --git a/drivers/storage/portworx/testspec/lighthouseClusterRole_nonPrivileged.yaml b/drivers/storage/portworx/testspec/lighthouseClusterRole_nonPrivileged.yaml new file mode 100644 index 000000000..671069e12 --- /dev/null +++ b/drivers/storage/portworx/testspec/lighthouseClusterRole_nonPrivileged.yaml @@ -0,0 +1,38 @@ +kind: ClusterRole +apiVersion: rbac.authorization.k8s.io/v1 +metadata: + name: px-lighthouse + namespace: kube-test +rules: + - apiGroups: [""] + resources: ["pods"] + verbs: ["get", "list"] + - apiGroups: ["extensions", "apps"] + resources: ["deployments"] + verbs: ["get", "list"] + - apiGroups: [""] + resources: ["secrets"] + verbs: ["get", "create", "update"] + - apiGroups: [""] + resources: ["configmaps"] + verbs: ["get", "create", "update"] + - apiGroups: [""] + resources: ["nodes"] + verbs: ["get", "list", "watch"] + - apiGroups: [""] + resources: ["services"] + verbs: ["get", "list", "watch", "create"] + - apiGroups: ["stork.libopenstorage.org"] + resources: ["*"] + verbs: ["get", "list", "create", "delete", "update"] + - apiGroups: ["monitoring.coreos.com"] + resources: ["alertmanagers", "prometheuses", "prometheuses/finalizers", "servicemonitors", "prometheusrules"] + verbs: ["*"] + - apiGroups: ["security.openshift.io"] + resources: ["securitycontextconstraints"] + resourceNames: ["portworx-restricted", "anyuid"] + verbs: ["use"] + - apiGroups: ["policy"] + resources: ["podsecuritypolicies"] + resourceNames: ["px-privileged"] + verbs: ["use"] \ No newline at end of file diff --git a/drivers/storage/portworx/testspec/pvcControllerClusterRole.yaml b/drivers/storage/portworx/testspec/pvcControllerClusterRole.yaml index 50ba22d31..6f74f9fdd 100644 --- a/drivers/storage/portworx/testspec/pvcControllerClusterRole.yaml +++ b/drivers/storage/portworx/testspec/pvcControllerClusterRole.yaml @@ -44,7 +44,7 @@ rules: verbs: ["get", "list", "watch", "create", "update"] - apiGroups: ["security.openshift.io"] resources: ["securitycontextconstraints"] - resourceNames: ["portworx-restricted"] + resourceNames: ["portworx"] verbs: ["use"] - apiGroups: ["policy"] resources: ["podsecuritypolicies"] diff --git a/drivers/storage/portworx/testspec/pvcControllerClusterRole_nonPrivileged.yaml b/drivers/storage/portworx/testspec/pvcControllerClusterRole_nonPrivileged.yaml new file mode 100644 index 000000000..3de94dccf --- /dev/null +++ b/drivers/storage/portworx/testspec/pvcControllerClusterRole_nonPrivileged.yaml @@ -0,0 +1,55 @@ +kind: ClusterRole +apiVersion: rbac.authorization.k8s.io/v1 +metadata: + name: portworx-pvc-controller +rules: + - apiGroups: [""] + resources: ["persistentvolumes"] + verbs: ["get", "list", "watch", "create", "delete", "update"] + - apiGroups: [""] + resources: ["persistentvolumes/status"] + verbs: ["update"] + - apiGroups: [""] + resources: ["persistentvolumeclaims"] + verbs: ["get", "list", "watch", "update"] + - apiGroups: [""] + resources: ["persistentvolumeclaims/status"] + verbs: ["update"] + - apiGroups: [""] + resources: ["pods"] + verbs: ["get", "list", "watch", "create", "delete"] + - apiGroups: ["storage.k8s.io"] + resources: ["storageclasses"] + verbs: ["get", "list", "watch"] + - apiGroups: [""] + resources: ["endpoints", "services"] + verbs: ["get", "create", "delete", "update"] + - apiGroups: [""] + resources: ["secrets"] + verbs: ["get", "list"] + - apiGroups: [""] + resources: ["nodes"] + verbs: ["get", "list", "watch"] + - apiGroups: ["", "events.k8s.io"] + resources: ["events"] + verbs: ["get", "list", "watch", "create", "update", "patch"] + - apiGroups: [""] + resources: ["serviceaccounts"] + verbs: ["get", "create"] + - apiGroups: [""] + resources: ["serviceaccounts/token"] + verbs: ["create"] + - apiGroups: [""] + resources: ["configmaps"] + verbs: ["get", "list", "watch", "create", "update"] + - apiGroups: ["security.openshift.io"] + resources: ["securitycontextconstraints"] + resourceNames: ["portworx-restricted"] + verbs: ["use"] + - apiGroups: ["policy"] + resources: ["podsecuritypolicies"] + resourceNames: ["px-privileged"] + verbs: ["use"] + - apiGroups: ["coordination.k8s.io"] + resources: ["leases"] + verbs: ["*"]