-
Notifications
You must be signed in to change notification settings - Fork 38.7k
New issue
Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.
By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.
Already on GitHub? Sign in to your account
[Federation][kubefed] Create a dedicated service account for federation controller manager in the host cluster and give it appropriate permissions. #40392
Changes from all commits
4f969bd
1bb80bc
f521963
42ff4e3
0dde763
4aeef0c
05a0f64
File filter
Filter by extension
Conversations
Jump to
Diff view
Diff view
There are no files selected for viewing
Original file line number | Diff line number | Diff line change |
---|---|---|
|
@@ -47,6 +47,7 @@ import ( | |
"k8s.io/kubernetes/pkg/api" | ||
"k8s.io/kubernetes/pkg/api/resource" | ||
"k8s.io/kubernetes/pkg/apis/extensions" | ||
"k8s.io/kubernetes/pkg/apis/rbac" | ||
client "k8s.io/kubernetes/pkg/client/clientset_generated/internalclientset" | ||
"k8s.io/kubernetes/pkg/kubectl/cmd/templates" | ||
cmdutil "k8s.io/kubernetes/pkg/kubectl/cmd/util" | ||
|
@@ -62,6 +63,17 @@ const ( | |
AdminCN = "admin" | ||
HostClusterLocalDNSZoneName = "cluster.local." | ||
|
||
// User name used by federation controller manager to make | ||
// calls to federation API server. | ||
ControllerManagerUser = "federation-controller-manager" | ||
|
||
// Name of the ServiceAccount used by the federation controller manager | ||
// to access the secrets in the host cluster. | ||
ControllerManagerSA = "federation-controller-manager" | ||
|
||
// Group name of the legacy/core API group | ||
legacyAPIGroup = "" | ||
|
||
lbAddrRetryInterval = 5 * time.Second | ||
podWaitInterval = 2 * time.Second | ||
) | ||
|
@@ -220,7 +232,22 @@ func initFederation(cmdOut io.Writer, config util.AdminConfig, cmd *cobra.Comman | |
} | ||
|
||
// 7. Create federation controller manager | ||
_, err = createControllerManager(hostClientset, initFlags.FederationSystemNamespace, initFlags.Name, svc.Name, cmName, image, cmKubeconfigName, dnsZoneName, dnsProvider, dryRun) | ||
// 7a. Create a service account in the host cluster for federation | ||
// controller manager. | ||
sa, err := createControllerManagerSA(hostClientset, initFlags.FederationSystemNamespace, dryRun) | ||
if err != nil { | ||
return err | ||
} | ||
|
||
// 7b. Create RBAC role and role binding for federation controller | ||
// manager service account. | ||
_, _, err = createRoleBindings(hostClientset, initFlags.FederationSystemNamespace, sa.Name, dryRun) | ||
if err != nil { | ||
return err | ||
} | ||
|
||
// 7c. Create federation controller manager deployment. | ||
_, err = createControllerManager(hostClientset, initFlags.FederationSystemNamespace, initFlags.Name, svc.Name, cmName, image, cmKubeconfigName, dnsZoneName, dnsProvider, sa.Name, dryRun) | ||
if err != nil { | ||
return err | ||
} | ||
|
@@ -375,7 +402,7 @@ func createControllerManagerKubeconfigSecret(clientset *client.Clientset, namesp | |
config := kubeadmkubeconfigphase.MakeClientConfigWithCerts( | ||
fmt.Sprintf("https://%s", svcName), | ||
name, | ||
"federation-controller-manager", | ||
ControllerManagerUser, | ||
certutil.EncodeCertPEM(entKeyPairs.ca.Cert), | ||
certutil.EncodePrivateKeyPEM(entKeyPairs.controllerManager.Key), | ||
certutil.EncodeCertPEM(entKeyPairs.controllerManager.Cert), | ||
|
@@ -520,7 +547,55 @@ func createAPIServer(clientset *client.Clientset, namespace, name, image, creden | |
return clientset.Extensions().Deployments(namespace).Create(dep) | ||
} | ||
|
||
func createControllerManager(clientset *client.Clientset, namespace, name, svcName, cmName, image, kubeconfigName, dnsZoneName, dnsProvider string, dryRun bool) (*extensions.Deployment, error) { | ||
func createControllerManagerSA(clientset *client.Clientset, namespace string, dryRun bool) (*api.ServiceAccount, error) { | ||
sa := &api.ServiceAccount{ | ||
ObjectMeta: metav1.ObjectMeta{ | ||
Name: ControllerManagerSA, | ||
Namespace: namespace, | ||
Labels: componentLabel, | ||
}, | ||
} | ||
if dryRun { | ||
return sa, nil | ||
} | ||
return clientset.Core().ServiceAccounts(namespace).Create(sa) | ||
} | ||
|
||
func createRoleBindings(clientset *client.Clientset, namespace, saName string, dryRun bool) (*rbac.Role, *rbac.RoleBinding, error) { | ||
roleName := "federation-system:federation-controller-manager" | ||
There was a problem hiding this comment. Choose a reason for hiding this commentThe reason will be displayed to describe this comment to others. Learn more. Use the constants here? There was a problem hiding this comment. Choose a reason for hiding this commentThe reason will be displayed to describe this comment to others. Learn more. This is that constant. I intentionally made it a function-level constant as opposed to a top-level/package-level constant. This isn't supposed to be reused outside. There was a problem hiding this comment. Choose a reason for hiding this commentThe reason will be displayed to describe this comment to others. Learn more. nit: if it's constant, declare it as a constant. There was a problem hiding this comment. Choose a reason for hiding this commentThe reason will be displayed to describe this comment to others. Learn more. Will do that in a follow up. Thanks for the approval. |
||
role := &rbac.Role{ | ||
There was a problem hiding this comment. Choose a reason for hiding this commentThe reason will be displayed to describe this comment to others. Learn more. is the intent to have these permissions within a single namespace or across all namespaces? to grant this across all namespaces, this needs to be a clusterrole. if just inside There was a problem hiding this comment. Choose a reason for hiding this commentThe reason will be displayed to describe this comment to others. Learn more. This RoleBinding is very specifically for the |
||
// a role to use for bootstrapping the federation-controller-manager so it can access | ||
There was a problem hiding this comment. Choose a reason for hiding this commentThe reason will be displayed to describe this comment to others. Learn more. just to clarify, this is for watching secrets stored in the federation server, where it gets credentials for other servers? There was a problem hiding this comment. Choose a reason for hiding this commentThe reason will be displayed to describe this comment to others. Learn more. This is for watching secrets stored in the host cluster where federation control plane components are hosted. And yes, to get get the credentials for all the clusters. |
||
// secrets in the host cluster to access other clusters. | ||
ObjectMeta: metav1.ObjectMeta{ | ||
Name: roleName, | ||
Namespace: namespace, | ||
Labels: componentLabel, | ||
}, | ||
Rules: []rbac.PolicyRule{ | ||
rbac.NewRule("get", "list", "watch").Groups(legacyAPIGroup).Resources("secrets").RuleOrDie(), | ||
}, | ||
} | ||
|
||
rolebinding, err := rbac.NewRoleBinding(roleName, namespace).SAs(namespace, saName).Binding() | ||
if err != nil { | ||
return nil, nil, err | ||
} | ||
rolebinding.Labels = componentLabel | ||
|
||
if dryRun { | ||
There was a problem hiding this comment. Choose a reason for hiding this commentThe reason will be displayed to describe this comment to others. Learn more. Did you mean if !dryRun? |
||
return role, &rolebinding, nil | ||
} | ||
|
||
newRole, err := clientset.Rbac().Roles(namespace).Create(role) | ||
if err != nil { | ||
return nil, nil, err | ||
} | ||
|
||
newRolebinding, err := clientset.Rbac().RoleBindings(namespace).Create(&rolebinding) | ||
return newRole, newRolebinding, err | ||
} | ||
|
||
func createControllerManager(clientset *client.Clientset, namespace, name, svcName, cmName, image, kubeconfigName, dnsZoneName, dnsProvider, saName string, dryRun bool) (*extensions.Deployment, error) { | ||
dep := &extensions.Deployment{ | ||
ObjectMeta: metav1.ObjectMeta{ | ||
Name: cmName, | ||
|
@@ -578,6 +653,7 @@ func createControllerManager(clientset *client.Clientset, namespace, name, svcNa | |
}, | ||
}, | ||
}, | ||
ServiceAccountName: saName, | ||
}, | ||
}, | ||
}, | ||
|
Original file line number | Diff line number | Diff line change |
---|---|---|
|
@@ -254,3 +254,74 @@ func (r *ClusterRoleBindingBuilder) Binding() (ClusterRoleBinding, error) { | |
|
||
return r.ClusterRoleBinding, nil | ||
} | ||
|
||
// +k8s:deepcopy-gen=false | ||
There was a problem hiding this comment. Choose a reason for hiding this commentThe reason will be displayed to describe this comment to others. Learn more. I'm not a huge fan of having helpers in the internal types api packages. @deads2k I guess you're not making it worse, so I'm not asking for you to solve this problem. There was a problem hiding this comment. Choose a reason for hiding this commentThe reason will be displayed to describe this comment to others. Learn more.
RBAC really demonstrates the issue with using external versions. The RBAC authorizer doesn't care which version of the API you use, but you run into crazy problems with caches, informers, conversions, and helpers if you don't use the internal types. I suspect that if you saw the code duplication involved in having v1beta1 and v1alpha1 support, you'd be displeased. |
||
// RoleBindingBuilder let's us attach methods. It is similar to | ||
// ClusterRoleBindingBuilder above. | ||
type RoleBindingBuilder struct { | ||
RoleBinding RoleBinding | ||
} | ||
|
||
// NewRoleBinding creates a RoleBinding builder that can be used | ||
// to define the subjects of a role binding. At least one of | ||
// the `Groups`, `Users` or `SAs` method must be called before | ||
// calling the `Binding*` methods. | ||
func NewRoleBinding(roleName, namespace string) *RoleBindingBuilder { | ||
There was a problem hiding this comment. Choose a reason for hiding this commentThe reason will be displayed to describe this comment to others. Learn more. Document that at least one of the functions below must be called. There was a problem hiding this comment. Choose a reason for hiding this commentThe reason will be displayed to describe this comment to others. Learn more. Done. |
||
return &RoleBindingBuilder{ | ||
RoleBinding: RoleBinding{ | ||
ObjectMeta: metav1.ObjectMeta{ | ||
Name: roleName, | ||
Namespace: namespace, | ||
}, | ||
RoleRef: RoleRef{ | ||
APIGroup: GroupName, | ||
Kind: "Role", | ||
Name: roleName, | ||
}, | ||
}, | ||
} | ||
} | ||
|
||
// Groups adds the specified groups as the subjects of the RoleBinding. | ||
func (r *RoleBindingBuilder) Groups(groups ...string) *RoleBindingBuilder { | ||
There was a problem hiding this comment. Choose a reason for hiding this commentThe reason will be displayed to describe this comment to others. Learn more. Public methods should have a godoc string. There was a problem hiding this comment. Choose a reason for hiding this commentThe reason will be displayed to describe this comment to others. Learn more. Done. |
||
for _, group := range groups { | ||
r.RoleBinding.Subjects = append(r.RoleBinding.Subjects, Subject{Kind: GroupKind, Name: group}) | ||
} | ||
return r | ||
} | ||
|
||
// Users adds the specified users as the subjects of the RoleBinding. | ||
func (r *RoleBindingBuilder) Users(users ...string) *RoleBindingBuilder { | ||
for _, user := range users { | ||
r.RoleBinding.Subjects = append(r.RoleBinding.Subjects, Subject{Kind: UserKind, Name: user}) | ||
} | ||
return r | ||
} | ||
|
||
// SAs adds the specified service accounts as the subjects of the | ||
// RoleBinding. | ||
func (r *RoleBindingBuilder) SAs(namespace string, serviceAccountNames ...string) *RoleBindingBuilder { | ||
for _, saName := range serviceAccountNames { | ||
r.RoleBinding.Subjects = append(r.RoleBinding.Subjects, Subject{Kind: ServiceAccountKind, Namespace: namespace, Name: saName}) | ||
} | ||
return r | ||
} | ||
|
||
// BindingOrDie calls the binding method and panics if there is an error. | ||
func (r *RoleBindingBuilder) BindingOrDie() RoleBinding { | ||
ret, err := r.Binding() | ||
if err != nil { | ||
panic(err) | ||
} | ||
return ret | ||
} | ||
|
||
// Binding builds and returns the RoleBinding API object from the builder | ||
// object. | ||
func (r *RoleBindingBuilder) Binding() (RoleBinding, error) { | ||
if len(r.RoleBinding.Subjects) == 0 { | ||
return RoleBinding{}, fmt.Errorf("subjects are required: %#v", r.RoleBinding) | ||
} | ||
|
||
return r.RoleBinding, nil | ||
} |
There was a problem hiding this comment.
Choose a reason for hiding this comment
The reason will be displayed to describe this comment to others. Learn more.
Dont we need similar constant for federation controller manager to talk to host cluster?
There was a problem hiding this comment.
Choose a reason for hiding this comment
The reason will be displayed to describe this comment to others. Learn more.
That's the service account below.