Skip to content

Commit

Permalink
Convert clusterrole creation over to use RBAC helpers
Browse files Browse the repository at this point in the history
Signed-off-by: Brad Davidson <brad.davidson@rancher.com>
  • Loading branch information
brandond committed Sep 24, 2021
1 parent 6974e46 commit 6143ffd
Show file tree
Hide file tree
Showing 4 changed files with 198 additions and 533 deletions.
152 changes: 19 additions & 133 deletions pkg/rke2/clusterrole.go
Original file line number Diff line number Diff line change
Expand Up @@ -2,14 +2,13 @@ package rke2

import (
"context"
"fmt"
"sync"

"github.com/rancher/k3s/pkg/cli/cmds"
"github.com/sirupsen/logrus"
apierrors "k8s.io/apimachinery/pkg/api/errors"
metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
"k8s.io/client-go/kubernetes"
genericapiserver "k8s.io/apiserver/pkg/server"
"k8s.io/client-go/tools/clientcmd"
rbacrest "k8s.io/kubernetes/pkg/registry/rbac/rest"
)

// setClusterRoles applies common clusterroles and clusterrolebindings that are critical
Expand All @@ -21,146 +20,33 @@ func setClusterRoles() cmds.StartupHook {
<-args.APIServerReady
logrus.Info("Applying Cluster Role Bindings")

cs, err := newClient(args.KubeConfigAdmin, nil)
config, err := clientcmd.BuildConfigFromFlags("", args.KubeConfigAdmin)
if err != nil {
logrus.Fatalf("clusterrole: new k8s client: %s", err.Error())
logrus.Fatalf("clusterrole: new k8s client: %v", err)
}

if err := setKubeletAPIServerRoleBinding(ctx, cs); err != nil {
logrus.Fatalf("psp: set kubeletAPIServerRoleBinding: %s", err.Error())
}
stopCh := make(chan struct{})
defer close(stopCh)

if err := setKubeProxyServerRoleBinding(ctx, cs); err != nil {
logrus.Fatalf("psp: set kubeProxyServerRoleBinding: %s", err.Error())
// kube-apiserver has a post-start hook that reconciles the built-in cluster RBAC on every startup.
// We're reusing that here to bootstrap our own roles and bindings.
hookContext := genericapiserver.PostStartHookContext{
LoopbackClientConfig: config,
StopCh: stopCh,
}

if err := setTunnelControllerRoleBinding(ctx, cs); err != nil {
logrus.Fatalf("psp: set tunnelControllerRoleBinding: %s", err.Error())
policy := rbacrest.PolicyData{
ClusterRoles: clusterRoles(),
ClusterRoleBindings: clusterRoleBindings(),
Roles: roles(),
RoleBindings: roleBindings(),
}

if err := setCloudControllerManagerRoleBinding(ctx, cs); err != nil {
logrus.Fatalf("ccm: set cloudControllerManagerRoleBinding: %s", err.Error())
if err := policy.EnsureRBACPolicy()(hookContext); err != nil {
logrus.Fatalf("clusterrole: EnsureRBACPolicy failed: %v", err)
}

logrus.Info("Cluster Role Bindings applied successfully")
}()
return nil
}
}

// setKubeletAPIServerRoleBinding creates the clusterrolebinding that grants the apiserver full access to the kubelet API
func setKubeletAPIServerRoleBinding(ctx context.Context, cs *kubernetes.Clientset) error {
// check if clusterrolebinding exists
if _, err := cs.RbacV1().ClusterRoleBindings().Get(ctx, kubeletAPIServerRoleBindingName, metav1.GetOptions{}); err != nil {
if apierrors.IsNotFound(err) {
logrus.Infof("Setting Cluster RoleBinding: %s", kubeletAPIServerRoleBindingName)

tmpl := fmt.Sprintf(kubeletAPIServerRoleBindingTemplate, kubeletAPIServerRoleBindingName)
if err := deployClusterRoleBindingFromYaml(ctx, cs, tmpl); err != nil {
return err
}
} else {
return err
}
}
return nil
}

// setKubeProxyServerRoleBinding creates the clusterrole and clusterrolebinding that grants the kube-proxy access to the kubelet API
func setKubeProxyServerRoleBinding(ctx context.Context, cs *kubernetes.Clientset) error {
// check if clusterrole exists
if _, err := cs.RbacV1().ClusterRoles().Get(ctx, kubeProxyRoleName, metav1.GetOptions{}); err != nil {
if apierrors.IsNotFound(err) {
logrus.Infof("Setting Cluster Role: %s", kubeProxyRoleName)

tmpl := fmt.Sprintf(kubeProxyRoleTemplate, kubeProxyRoleName)
if err := deployClusterRoleFromYaml(ctx, cs, tmpl); err != nil {
return err
}
} else {
return err
}
}

// check if clusterrolebinding exists
if _, err := cs.RbacV1().ClusterRoleBindings().Get(ctx, kubeProxyRoleName, metav1.GetOptions{}); err != nil {
if apierrors.IsNotFound(err) {
logrus.Infof("Setting Cluster RoleBinding: %s", kubeProxyRoleName)

tmpl := fmt.Sprintf(kubeProxyServerRoleBindingTemplate, kubeProxyRoleName)
if err := deployClusterRoleBindingFromYaml(ctx, cs, tmpl); err != nil {
return err
}
} else {
return err
}
}

return nil
}

// setTunnelControllerRoleBinding creates the clusterrole and clusterrolebinding used by internal controllers
// such as the agent tunnel controller
func setTunnelControllerRoleBinding(ctx context.Context, cs *kubernetes.Clientset) error {
// check if clusterrole exists
if _, err := cs.RbacV1().ClusterRoles().Get(ctx, tunnelControllerRoleName, metav1.GetOptions{}); err != nil {
if apierrors.IsNotFound(err) {
logrus.Infof("Setting Cluster Role: %s", tunnelControllerRoleName)

tmpl := fmt.Sprintf(tunnelControllerRoleTemplate, tunnelControllerRoleName)
if err := deployClusterRoleFromYaml(ctx, cs, tmpl); err != nil {
return err
}
} else {
return err
}
}

// check if clusterrolebinding exists
if _, err := cs.RbacV1().ClusterRoleBindings().Get(ctx, tunnelControllerRoleName, metav1.GetOptions{}); err != nil {
if apierrors.IsNotFound(err) {
logrus.Infof("Setting Cluster RoleBinding: %s", tunnelControllerRoleName)

tmpl := fmt.Sprintf(tunnelControllerRoleBindingTemplate, tunnelControllerRoleName)
if err := deployClusterRoleBindingFromYaml(ctx, cs, tmpl); err != nil {
return err
}
} else {
return err
}
}

return nil
}

func setCloudControllerManagerRoleBinding(ctx context.Context, cs *kubernetes.Clientset) error {
// check if clusterrole exists
if _, err := cs.RbacV1().ClusterRoles().Get(ctx, cloudControllerManagerRoleName, metav1.GetOptions{}); err != nil {
if apierrors.IsNotFound(err) {
logrus.Infof("Setting Cluster Role: %s", cloudControllerManagerRoleName)

tmpl := fmt.Sprintf(cloudControllerManagerRoleTemplate, cloudControllerManagerRoleName)
if err := deployClusterRoleFromYaml(ctx, cs, tmpl); err != nil {
return err
}
} else {
return err
}
}

// check if clusterrolebinding exists
if _, err := cs.RbacV1().ClusterRoleBindings().Get(ctx, cloudControllerManagerRoleName, metav1.GetOptions{}); err != nil {
if apierrors.IsNotFound(err) {
logrus.Infof("Setting Cluster RoleBinding: %s", cloudControllerManagerRoleName)

tmpl := fmt.Sprintf(cloudControllerManagerRoleBindingTemplate, cloudControllerManagerRoleName)
if err := deployClusterRoleBindingFromYaml(ctx, cs, tmpl); err != nil {
return err
}
} else {
return err
}
}

return nil
}
179 changes: 179 additions & 0 deletions pkg/rke2/clusterrole_bootstrap.go
Original file line number Diff line number Diff line change
@@ -0,0 +1,179 @@
package rke2

import (
rbacapiv1 "k8s.io/api/rbac/v1"
rbacv1 "k8s.io/api/rbac/v1"
"k8s.io/apimachinery/pkg/api/meta"
metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
"k8s.io/apimachinery/pkg/runtime"
rbacv1helpers "k8s.io/kubernetes/pkg/apis/rbac/v1"
)

const (
kubeletAPIServerRoleBindingName = "kube-apiserver-kubelet-admin"
kubeProxyName = "system:kube-proxy"
tunnelControllerName = "system:rke2-controller"
cloudControllerManagerName = "rke2-cloud-controller-manager"
legacyGroup = ""
coordinationGroup = "coordination.k8s.io"
networkingGroup = "networking.k8s.io"
helmGroup = "helm.cattle.io"
)

var (
label = map[string]string{"rke2.io/bootstrapping": "rbac-defaults"}
annotation = map[string]string{rbacv1.AutoUpdateAnnotationKey: "true"}
)

// clusterRoles returns a list of clusterrolebindings to bootstrap
func clusterRoles() []rbacapiv1.ClusterRole {
roles := []rbacapiv1.ClusterRole{
{
ObjectMeta: metav1.ObjectMeta{Name: kubeProxyName},
Rules: []rbacv1.PolicyRule{
rbacv1helpers.NewRule("get", "list").Groups(legacyGroup).Resources("nodes").RuleOrDie(),
},
},
{
ObjectMeta: metav1.ObjectMeta{Name: tunnelControllerName},
Rules: []rbacv1.PolicyRule{
rbacv1helpers.NewRule("get").Groups(legacyGroup).Resources("nodes").RuleOrDie(),
rbacv1helpers.NewRule("list", "watch").Groups(legacyGroup).Resources("namespaces").RuleOrDie(),
rbacv1helpers.NewRule("list", "watch").Groups(networkingGroup).Resources("networkpolicies").RuleOrDie(),
rbacv1helpers.NewRule("list", "watch", "get").Groups(legacyGroup).Resources("endpoints", "pods").RuleOrDie(),
rbacv1helpers.NewRule("list", "get").Groups(legacyGroup).Resources("secrets").RuleOrDie(),
rbacv1helpers.NewRule("list", "get").Groups(helmGroup).Resources("helmcharts", "helmchartconfigs").RuleOrDie(),
},
},
{
ObjectMeta: metav1.ObjectMeta{Name: cloudControllerManagerName},
Rules: []rbacv1.PolicyRule{
rbacv1helpers.NewRule("get", "create", "update").Groups(coordinationGroup).Resources("leases").RuleOrDie(),
rbacv1helpers.NewRule("create", "patch", "update").Groups(legacyGroup).Resources("events").RuleOrDie(),
rbacv1helpers.NewRule("*").Groups(legacyGroup).Resources("nodes").RuleOrDie(),
rbacv1helpers.NewRule("patch").Groups(legacyGroup).Resources("nodes/status").RuleOrDie(),
rbacv1helpers.NewRule("list", "watch", "patch", "update").Groups(legacyGroup).Resources("services").RuleOrDie(),
rbacv1helpers.NewRule("create").Groups(legacyGroup).Resources("serviceaccounts").RuleOrDie(),
rbacv1helpers.NewRule("get", "list", "watch", "update").Groups("").Resources("persistentvolumes").RuleOrDie(),
rbacv1helpers.NewRule("create", "get", "list", "watch", "update").Groups(legacyGroup).Resources("endpoints").RuleOrDie(),
},
},
}
addClusterRoleLabel(roles)
return roles
}

// clusterRoleBindings returns a list of clusterrolebindings to bootstrap
func clusterRoleBindings() []rbacapiv1.ClusterRoleBinding {
rolebindings := []rbacapiv1.ClusterRoleBinding{
// https://github.com/kubernetes/kubernetes/issues/65939#issuecomment-403218465
ClusterRoleBindingName(rbacv1helpers.NewClusterBinding("system:kubelet-api-admin").Users("kube-apiserver"), kubeletAPIServerRoleBindingName).BindingOrDie(),
ClusterRoleBindingNamespacedUsers(ClusterRoleBindingName(rbacv1helpers.NewClusterBinding("system:auth-delegator"), cloudControllerManagerName+"-auth-delegator"), metav1.NamespaceSystem, cloudControllerManagerName).BindingOrDie(),
ClusterRoleBindingNamespacedUsers(rbacv1helpers.NewClusterBinding(cloudControllerManagerName), metav1.NamespaceSystem, cloudControllerManagerName).BindingOrDie(),
rbacv1helpers.NewClusterBinding(kubeProxyName).Users(kubeProxyName).BindingOrDie(),
rbacv1helpers.NewClusterBinding(tunnelControllerName).Users(tunnelControllerName).BindingOrDie(),
}
addClusterRoleBindingLabel(rolebindings)
return rolebindings
}

// roles returns a map of namespace to roles to bootstrap into that namespace
func roles() map[string][]rbacapiv1.Role {
return nil
}

// roleBindings returns a map of namespace to rolebindings to bootstrap into that namespace
func roleBindings() map[string][]rbacapiv1.RoleBinding {
ccmAuthReader := RoleBindingNamespacedUsers(RoleBindingName(rbacv1helpers.NewRoleBinding("extension-apiserver-authentication-reader", metav1.NamespaceSystem), cloudControllerManagerName+"-authentication-reader"), metav1.NamespaceSystem, cloudControllerManagerName).BindingOrDie()
addDefaultMetadata(&ccmAuthReader)
return map[string][]rbacapiv1.RoleBinding{
metav1.NamespaceSystem: {
ccmAuthReader,
},
}
}

// RoleBindingNamespacedUsers adds namespaced users to a RoleBindingBuilder's Subjects list.
// For some reason the core helpers don't have any methods for adding namespaced users, only namespaced service accounts.
func RoleBindingNamespacedUsers(r *rbacv1helpers.RoleBindingBuilder, namespace string, users ...string) *rbacv1helpers.RoleBindingBuilder {
for _, user := range users {
r.RoleBinding.Subjects = append(r.RoleBinding.Subjects, rbacv1.Subject{Kind: rbacv1.UserKind, Namespace: namespace, Name: user})
}
return r
}

// RoleBindingName sets the name on a RoleBindingBuilder's policy.
// The ClusterRoleBindingBuilder sets the ClusterRoleBinding name to the same as ClusterRole it's binding to without
// providing any way to override it.
func RoleBindingName(r *rbacv1helpers.RoleBindingBuilder, name string) *rbacv1helpers.RoleBindingBuilder {
r.RoleBinding.ObjectMeta.Name = name
return r
}

// ClusterRoleBindingNamespacedUsers adds namespaced users to a ClusterRoleBindingBuilder's Subjects list.
// For some reason the core helpers don't have any methods for adding namespaced users, only namespaced service accounts.
func ClusterRoleBindingNamespacedUsers(r *rbacv1helpers.ClusterRoleBindingBuilder, namespace string, users ...string) *rbacv1helpers.ClusterRoleBindingBuilder {
for _, user := range users {
r.ClusterRoleBinding.Subjects = append(r.ClusterRoleBinding.Subjects, rbacv1.Subject{Kind: rbacv1.UserKind, Namespace: namespace, Name: user})
}
return r
}

// ClusterRoleBindingName sets the name on a ClusterRoleBindingBuilder's policy.
// The ClusterRoleBindingBuilder sets the ClusterRoleBinding name to the same as ClusterRole it's binding to without
// providing any way to override it.
func ClusterRoleBindingName(r *rbacv1helpers.ClusterRoleBindingBuilder, name string) *rbacv1helpers.ClusterRoleBindingBuilder {
r.ClusterRoleBinding.ObjectMeta.Name = name
return r
}

// cribbed from k8s.io/kubernetes/plugin/pkg/auth/authorizer/rbac/bootstrappolicy/policy.go
func addDefaultMetadata(obj runtime.Object) {
metadata, err := meta.Accessor(obj)
if err != nil {
// if this happens, then some static code is broken
panic(err)
}

labels := metadata.GetLabels()
if labels == nil {
labels = map[string]string{}
}
for k, v := range label {
labels[k] = v
}
metadata.SetLabels(labels)

annotations := metadata.GetAnnotations()
if annotations == nil {
annotations = map[string]string{}
}
for k, v := range annotation {
annotations[k] = v
}
metadata.SetAnnotations(annotations)
}

// cribbed from k8s.io/kubernetes/plugin/pkg/auth/authorizer/rbac/bootstrappolicy/policy.go
func addClusterRoleLabel(roles []rbacv1.ClusterRole) {
for i := range roles {
addDefaultMetadata(&roles[i])
}
return
}

// cribbed from k8s.io/kubernetes/plugin/pkg/auth/authorizer/rbac/bootstrappolicy/policy.go
func addClusterRoleBindingLabel(rolebindings []rbacv1.ClusterRoleBinding) {
for i := range rolebindings {
addDefaultMetadata(&rolebindings[i])
}
return
}

// cribbed from k8s.io/kubernetes/plugin/pkg/auth/authorizer/rbac/bootstrappolicy/policy.go
func addRoleLabel(roles []rbacv1.Role) {
for i := range roles {
addDefaultMetadata(&roles[i])
}
return
}

0 comments on commit 6143ffd

Please sign in to comment.