feat: use multiple groups as capsule-user-group

Makefile

@@ -64,7 +64,7 @@ deploy: manifests kustomize
 remove: manifests kustomize
 	$(KUSTOMIZE) build config/default | kubectl delete -f -
 	kubectl delete clusterroles.rbac.authorization.k8s.io capsule-namespace-deleter capsule-namespace-provisioner --ignore-not-found
-	kubectl delete clusterrolebindings.rbac.authorization.k8s.io capsule-namespace-provisioner --ignore-not-found
+	kubectl delete clusterrolebindings.rbac.authorization.k8s.io -l capsule.clastix.io/rbac --ignore-not-found
 
 # Generate manifests e.g. CRD, RBAC etc.
 manifests: controller-gen

controllers/rbac/manager.go

@@ -20,6 +20,8 @@ import (
 	"context"
 	"fmt"
 
+	b64 "encoding/base64"
+
 	"github.com/go-logr/logr"
 	"github.com/hashicorp/go-multierror"
 	rbacv1 "k8s.io/api/rbac/v1"
@@ -34,10 +36,14 @@ import (
 	"sigs.k8s.io/controller-runtime/pkg/reconcile"
 )
 
+const (
+	rbacLabel = "capsule.clastix.io/rbac"
+)
+
 type Manager struct {
-	CapsuleGroup string
-	Log          logr.Logger
-	Client       client.Client
+	CapsuleGroups []string
+	Log           logr.Logger
+	Client        client.Client
 }
 
 // Using the Client interface, required by the Runnable interface
@@ -115,36 +121,42 @@ func (r *Manager) Reconcile(ctx context.Context, request reconcile.Request) (res
 }
 
 func (r *Manager) EnsureClusterRoleBinding() (err error) {
-	crb := &rbacv1.ClusterRoleBinding{
-		ObjectMeta: v1.ObjectMeta{
-			Name: ProvisionerRoleName,
-		},
-	}
-
-	_, err = controllerutil.CreateOrUpdate(context.TODO(), r.Client, crb, func() error {
-		// RoleRef is immutable, so we need to delete and recreate ClusterRoleBinding if it changed
-		if crb.ResourceVersion != "" && !equality.Semantic.DeepDerivative(provisionerClusterRoleBinding.RoleRef, crb.RoleRef) {
-			return ImmutableClusterRoleBindingError{}
-		}
-		crb.RoleRef = provisionerClusterRoleBinding.RoleRef
-		crb.Subjects = []rbacv1.Subject{
-			{
-				Kind: "Group",
-				Name: r.CapsuleGroup,
+	for _, group := range r.CapsuleGroups {
+		name := b64.RawStdEncoding.EncodeToString([]byte(group))
+		crb := &rbacv1.ClusterRoleBinding{
+			ObjectMeta: v1.ObjectMeta{
+				Name: fmt.Sprintf("%s-%v", ProvisionerRoleName, name),
+				Labels: map[string]string{
+					rbacLabel: fmt.Sprintf("%s-%v", ProvisionerRoleName, name),
+				},
 			},
 		}
-		return nil
-	})
-	if err != nil {
-		if _, ok := err.(ImmutableClusterRoleBindingError); ok {
-			if err = r.Client.Delete(context.TODO(), crb); err != nil {
-				r.Log.Error(err, "Cannot delete CRB during reset due to RoleRef change")
-				return
+
+		_, err = controllerutil.CreateOrUpdate(context.TODO(), r.Client, crb, func() error {
+			// RoleRef is immutable, so we need to delete and recreate ClusterRoleBinding if it changed
+			if crb.ResourceVersion != "" && !equality.Semantic.DeepDerivative(provisionerClusterRoleBinding.RoleRef, crb.RoleRef) {
+				return ImmutableClusterRoleBindingError{}
 			}
-			return r.Client.Create(context.TODO(), provisionerClusterRoleBinding, &client.CreateOptions{})
+			crb.RoleRef = provisionerClusterRoleBinding.RoleRef
+			crb.Subjects = []rbacv1.Subject{
+				{
+					Kind: "Group",
+					Name: group,
+				},
+			}
+			return nil
+		})
+		if err != nil {
+			if _, ok := err.(ImmutableClusterRoleBindingError); ok {
+				if err = r.Client.Delete(context.TODO(), crb); err != nil {
+					r.Log.Error(err, "Cannot delete CRB during reset due to RoleRef change")
+					return
+				}
+				return r.Client.Create(context.TODO(), provisionerClusterRoleBinding, &client.CreateOptions{})
+			}
+			r.Log.Error(err, "Cannot CreateOrUpdate CRB")
+			return
 		}
-		r.Log.Error(err, "Cannot CreateOrUpdate CRB")
-		return
 	}
 	return
 }

e2e/custom_capsule_group_test.go

@@ -20,6 +20,7 @@ package e2e
 
 import (
 	"context"
+	"fmt"
 
 	. "github.com/onsi/ginkgo"
 	. "github.com/onsi/gomega"
@@ -52,18 +53,41 @@ var _ = Describe("creating a Namespace as Tenant owner with custom --capsule-gro
 	})
 
 	It("should fail using a User non matching the capsule-user-group flag", func() {
-		args := append(defaulManagerPodArgs, []string{"--capsule-user-group=test"}...)
+		var managerPodArgs []string
+		capsuleGroups := []string{"test"}
+		for _, group := range capsuleGroups {
+			managerPodArgs = append(managerPodArgs, fmt.Sprintf("--capsule-user-group=%s", group))
+		}
+
+		args := append(defaulManagerPodArgs, managerPodArgs...)
 		ModifyCapsuleManagerPodArgs(args)
-		CapsuleClusterGroupParam(podRecreationTimeoutInterval).Should(BeIdenticalTo("test"))
+		CapsuleClusterGroupParam(podRecreationTimeoutInterval, capsuleGroups).Should(ContainElements(capsuleGroups))
 		ns := NewNamespace("cg-namespace-fail")
 		NamespaceCreation(ns, tnt, podRecreationTimeoutInterval).ShouldNot(Succeed())
 	})
 
-	It("should succeed and be available in Tenant namespaces list", func() {
+	It("should succeed and be available in Tenant namespaces list with multiple groups", func() {
+		var managerPodArgs []string
+		capsuleGroups := []string{"test", "alice"}
+		for _, group := range capsuleGroups {
+			managerPodArgs = append(managerPodArgs, fmt.Sprintf("--capsule-user-group=%s", group))
+		}
+
+		args := append(defaulManagerPodArgs, managerPodArgs...)
+		ModifyCapsuleManagerPodArgs(args)
+		CapsuleClusterGroupParam(podRecreationTimeoutInterval, capsuleGroups).Should(ContainElements(capsuleGroups))
+		ns := NewNamespace("cg-namespace-1")
+		NamespaceCreation(ns, tnt, podRecreationTimeoutInterval).Should(Succeed())
+		TenantNamespaceList(tnt, podRecreationTimeoutInterval).Should(ContainElement(ns.GetName()))
+	})
+
+	It("should succeed and be available in Tenant namespaces list with default single group", func() {
+		capsuleGroups := []string{"capsule.clastix.io"}
 		ModifyCapsuleManagerPodArgs(defaulManagerPodArgs)
-		CapsuleClusterGroupParam(podRecreationTimeoutInterval).Should(BeIdenticalTo("capsule.clastix.io"))
-		ns := NewNamespace("cg-namespace")
+		CapsuleClusterGroupParam(podRecreationTimeoutInterval, capsuleGroups).Should(ContainElements("capsule.clastix.io"))
+		ns := NewNamespace("cg-namespace-2")
 		NamespaceCreation(ns, tnt, podRecreationTimeoutInterval).Should(Succeed())
 		TenantNamespaceList(tnt, podRecreationTimeoutInterval).Should(ContainElement(ns.GetName()))
 	})
+
 })

e2e/utils_test.go

@@ -20,6 +20,8 @@ package e2e
 
 import (
 	"context"
+	b64 "encoding/base64"
+	"fmt"
 	"strconv"
 	"time"
 
@@ -70,21 +72,29 @@ func EventuallyCreation(f interface{}) AsyncAssertion {
 	return Eventually(f, defaultTimeoutInterval, defaultPollInterval)
 }
 
-func CapsuleClusterGroupParam(timeout time.Duration) AsyncAssertion {
+func CapsuleClusterGroupParam(timeout time.Duration, groups []string) AsyncAssertion {
 	capsuleCRB := &rbacv1.ClusterRoleBinding{}
 
-	return Eventually(func() string {
-		Expect(k8sClient.Get(context.TODO(), types.NamespacedName{Name: rbac.ProvisionerRoleName}, capsuleCRB)).Should(Succeed())
-		return capsuleCRB.Subjects[0].Name
+	return Eventually(func() []string {
+		var subjectNames []string
+		for _, group := range groups {
+			name := b64.RawStdEncoding.EncodeToString([]byte(group))
+			Expect(k8sClient.Get(context.TODO(), types.NamespacedName{Name: fmt.Sprintf("%s-%v", rbac.ProvisionerRoleName, name)}, capsuleCRB)).Should(Succeed())
+			subjectNames = append(subjectNames, capsuleCRB.Subjects[0].Name)
+		}
+		return subjectNames
 	}, timeout, defaultPollInterval)
 }
 
 func ModifyCapsuleManagerPodArgs(args []string) {
 	capsuleDeployment := &appsv1.Deployment{}
 	k8sClient.Get(context.TODO(), types.NamespacedName{Name: capsuleDeploymentName, Namespace: capsuleNamespace}, capsuleDeployment)
+
 	for i, container := range capsuleDeployment.Spec.Template.Spec.Containers {
 		if container.Name == capsuleManagerContainerName {
 			capsuleDeployment.Spec.Template.Spec.Containers[i].Args = args
+			capsuleDeployment.Spec.Template.Spec.Containers[i].LivenessProbe.FailureThreshold = 4
+			capsuleDeployment.Spec.Template.Spec.Containers[i].LivenessProbe.PeriodSeconds = 3
 		}
 	}
 	capsuleDeployment.ResourceVersion = ""

go.mod

@@ -8,6 +8,7 @@ require (
 	github.com/onsi/ginkgo v1.14.1
 	github.com/onsi/gomega v1.10.2
 	github.com/pkg/errors v0.9.1
+	github.com/spf13/pflag v1.0.5
 	github.com/stretchr/testify v1.6.1
 	go.uber.org/zap v1.15.0
 	golang.org/x/sync v0.0.0-20190911185100-cd5d95a43a6e

main.go

@@ -17,12 +17,14 @@ limitations under the License.
 package main
 
 import (
-	"flag"
+	goflag "flag"
 	"fmt"
 	"os"
 	"regexp"
 	goRuntime "runtime"
 
+	flag "github.com/spf13/pflag"
+
 	"go.uber.org/zap/zapcore"
 	"k8s.io/apimachinery/pkg/runtime"
 	utilruntime "k8s.io/apimachinery/pkg/util/runtime"
@@ -77,15 +79,16 @@ func main() {
 	var enableLeaderElection bool
 	var forceTenantPrefix bool
 	var version bool
-	var capsuleGroup string
+	var capsuleGroups []string
 	var protectedNamespaceRegexpString string
 	var protectedNamespaceRegexp *regexp.Regexp
 	var allowTenantIngressHostnamesCollision bool
 	var allowIngressHostnamesCollision bool
 	var namespace string
+	var goFlagSet goflag.FlagSet
 
 	flag.StringVar(&metricsAddr, "metrics-addr", ":8080", "The address the metric endpoint binds to.")
-	flag.StringVar(&capsuleGroup, "capsule-user-group", capsulev1alpha1.GroupVersion.Group, "Name of the group for capsule users")
+	flag.StringSliceVar(&capsuleGroups, "capsule-user-group", []string{capsulev1alpha1.GroupVersion.Group}, "Names of the groups for capsule users")
 	flag.BoolVar(&enableLeaderElection, "enable-leader-election", false,
 		"Enable leader election for controller manager. "+
 			"Enabling this will ensure there is only one active controller manager.")
@@ -109,7 +112,9 @@ func main() {
 			config.EncodeTime = zapcore.ISO8601TimeEncoder
 		}),
 	}
-	opts.BindFlags(flag.CommandLine)
+
+	opts.BindFlags(&goFlagSet)
+	flag.CommandLine.AddGoFlagSet(&goFlagSet)
 	flag.Parse()
 
 	ctrl.SetLogger(zap.New(zap.UseFlagOptions(&opts)))
@@ -172,10 +177,10 @@ func main() {
 		pvc.Webhook(pvc.Handler()),
 		registry.Webhook(registry.Handler()),
 		services.Webhook(services.Handler()),
-		ownerreference.Webhook(utils.InCapsuleGroup(capsuleGroup, ownerreference.Handler(forceTenantPrefix))),
-		namespacequota.Webhook(utils.InCapsuleGroup(capsuleGroup, namespacequota.Handler())),
-		networkpolicies.Webhook(utils.InCapsuleGroup(capsuleGroup, networkpolicies.Handler())),
-		tenantprefix.Webhook(utils.InCapsuleGroup(capsuleGroup, tenantprefix.Handler(forceTenantPrefix, protectedNamespaceRegexp))),
+		ownerreference.Webhook(utils.InCapsuleGroups(capsuleGroups, ownerreference.Handler(forceTenantPrefix))),
+		namespacequota.Webhook(utils.InCapsuleGroups(capsuleGroups, namespacequota.Handler())),
+		networkpolicies.Webhook(utils.InCapsuleGroups(capsuleGroups, networkpolicies.Handler())),
+		tenantprefix.Webhook(utils.InCapsuleGroups(capsuleGroups, tenantprefix.Handler(forceTenantPrefix, protectedNamespaceRegexp))),
 		tenant.Webhook(tenant.Handler(allowTenantIngressHostnamesCollision)),
 	)
 	if err = webhook.Register(manager, webhooksList...); err != nil {
@@ -184,8 +189,8 @@ func main() {
 	}
 
 	rbacManager := &rbac.Manager{
-		Log:          ctrl.Log.WithName("controllers").WithName("Rbac"),
-		CapsuleGroup: capsuleGroup,
+		Log:           ctrl.Log.WithName("controllers").WithName("Rbac"),
+		CapsuleGroups: capsuleGroups,
 	}
 	if err = manager.Add(rbacManager); err != nil {
 		setupLog.Error(err, "unable to create cluster roles")

pkg/utils/user_group.go

@@ -39,9 +39,6 @@ func NewUserGroupList(groups []string) UserGroupList {
 
 // Find sorts itself using the SliceStable and perform a binary-search for the given string.
 func (u userGroupList) Find(needle string) (found bool) {
-	sort.SliceStable(u, func(i, j int) bool {
-		return i < j
-	})
 	i := sort.SearchStrings(u, needle)
 	found = i < len(u) && u[i] == needle
 	return

pkg/utils/user_group_test.go

@@ -6,7 +6,7 @@ import (
 	"github.com/stretchr/testify/assert"
 )
 
-func TestIsInCapsuleGroup(t *testing.T) {
+func TestIsInCapsuleGroups(t *testing.T) {
 	groups := []string{
 		"DPS-QQ-DeparEE-Upload_RW",
 		"vsphere - glonqqqq-devopsq",

pkg/webhook/utils/in_capsule_group.go → pkg/webhook/utils/in_capsule_groups.go

@@ -26,16 +26,16 @@ import (
 	"github.com/clastix/capsule/pkg/webhook"
 )
 
-func InCapsuleGroup(capsuleGroup string, webhookHandler webhook.Handler) webhook.Handler {
+func InCapsuleGroups(capsuleGroups []string, webhookHandler webhook.Handler) webhook.Handler {
 	return &handler{
-		handler:      webhookHandler,
-		capsuleGroup: capsuleGroup,
+		handler:       webhookHandler,
+		capsuleGroups: capsuleGroups,
 	}
 }
 
 type handler struct {
-	capsuleGroup string
-	handler      webhook.Handler
+	capsuleGroups []string
+	handler       webhook.Handler
 }
 
 // If the user performing action is not a Capsule user, can be skipped
@@ -47,7 +47,12 @@ func (h handler) isCapsuleUser(req admission.Request) bool {
 	if groupList.Find("system:serviceaccounts:kube-system") {
 		return false
 	}
-	return groupList.Find(h.capsuleGroup)
+	for _, group := range h.capsuleGroups {
+		if groupList.Find(group) {
+			return true
+		}
+	}
+	return false
 }
 
 func (h *handler) OnCreate(client client.Client, decoder *admission.Decoder) webhook.Func {