Skip to content

Commit 05fab40

Browse files
cmurphyjakefhyde
cmurphy
authored and
jakefhyde
committed
Migrate cluster service account tokens to secrets
1 parent a8ae635 commit 05fab40

9 files changed

Lines changed: 151 additions & 83 deletions

File tree

‎pkg/api/norman/customization/project/project_store.go‎

Lines changed: 4 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -14,6 +14,7 @@ import (
1414
v32 "github.com/rancher/rancher/pkg/apis/management.cattle.io/v3"
1515
mgmtclient "github.com/rancher/rancher/pkg/client/generated/management/v3"
1616
"github.com/rancher/rancher/pkg/clustermanager"
17+
v1 "github.com/rancher/rancher/pkg/generated/norman/core/v1"
1718
v3 "github.com/rancher/rancher/pkg/generated/norman/management.cattle.io/v3"
1819
"github.com/rancher/rancher/pkg/resourcequota"
1920
mgmtschema "github.com/rancher/rancher/pkg/schemas/management.cattle.io/v3"
@@ -33,6 +34,7 @@ type projectStore struct {
3334
roleTemplateLister v3.RoleTemplateLister
3435
scaledContext *config.ScaledContext
3536
clusterLister v3.ClusterLister
37+
secretLister v1.SecretLister
3638
}
3739

3840
func SetProjectStore(schema *types.Schema, mgmt *config.ScaledContext) {
@@ -42,6 +44,7 @@ func SetProjectStore(schema *types.Schema, mgmt *config.ScaledContext) {
4244
roleTemplateLister: mgmt.Management.RoleTemplates("").Controller().Lister(),
4345
scaledContext: mgmt,
4446
clusterLister: mgmt.Management.Clusters("").Controller().Lister(),
47+
secretLister: mgmt.Core.Secrets("").Controller().Lister(),
4548
}
4649
schema.Store = store
4750
}
@@ -281,7 +284,7 @@ func (s *projectStore) getNamespacesCount(apiContext *types.APIContext, project
281284
return 0, err
282285
}
283286

284-
kubeConfig, err := clustermanager.ToRESTConfig(cluster, s.scaledContext)
287+
kubeConfig, err := clustermanager.ToRESTConfig(cluster, s.scaledContext, s.secretLister)
285288
if kubeConfig == nil || err != nil {
286289
return 0, err
287290
}

‎pkg/apis/management.cattle.io/v3/cluster_types.go‎

Lines changed: 13 additions & 11 deletions
Original file line numberDiff line numberDiff line change
@@ -61,17 +61,18 @@ const (
6161
ClusterConditionDefaultNamespaceAssigned condition.Cond = "DefaultNamespaceAssigned"
6262
// Deprecated: ClusterConditionSystemNamespacesAssigned true when cluster's system namespaces has been initially assigned to
6363
// a system project
64-
ClusterConditionSystemNamespacesAssigned condition.Cond = "SystemNamespacesAssigned"
65-
ClusterConditionAddonDeploy condition.Cond = "AddonDeploy"
66-
ClusterConditionSystemAccountCreated condition.Cond = "SystemAccountCreated"
67-
ClusterConditionAgentDeployed condition.Cond = "AgentDeployed"
68-
ClusterConditionGlobalAdminsSynced condition.Cond = "GlobalAdminsSynced"
69-
ClusterConditionInitialRolesPopulated condition.Cond = "InitialRolesPopulated"
70-
ClusterConditionServiceAccountMigrated condition.Cond = "ServiceAccountMigrated"
71-
ClusterConditionPrometheusOperatorDeployed condition.Cond = "PrometheusOperatorDeployed"
72-
ClusterConditionMonitoringEnabled condition.Cond = "MonitoringEnabled"
73-
ClusterConditionAlertingEnabled condition.Cond = "AlertingEnabled"
74-
ClusterConditionSecretsMigrated condition.Cond = "SecretsMigrated"
64+
ClusterConditionSystemNamespacesAssigned condition.Cond = "SystemNamespacesAssigned"
65+
ClusterConditionAddonDeploy condition.Cond = "AddonDeploy"
66+
ClusterConditionSystemAccountCreated condition.Cond = "SystemAccountCreated"
67+
ClusterConditionAgentDeployed condition.Cond = "AgentDeployed"
68+
ClusterConditionGlobalAdminsSynced condition.Cond = "GlobalAdminsSynced"
69+
ClusterConditionInitialRolesPopulated condition.Cond = "InitialRolesPopulated"
70+
ClusterConditionServiceAccountMigrated condition.Cond = "ServiceAccountMigrated"
71+
ClusterConditionPrometheusOperatorDeployed condition.Cond = "PrometheusOperatorDeployed"
72+
ClusterConditionMonitoringEnabled condition.Cond = "MonitoringEnabled"
73+
ClusterConditionAlertingEnabled condition.Cond = "AlertingEnabled"
74+
ClusterConditionSecretsMigrated condition.Cond = "SecretsMigrated"
75+
ClusterConditionServiceAccountSecretsMigrated condition.Cond = "ServiceAccountSecretsMigrated"
7576

7677
ClusterDriverImported = "imported"
7778
ClusterDriverLocal = "local"
@@ -160,6 +161,7 @@ type ClusterStatus struct {
160161
ComponentStatuses []ClusterComponentStatus `json:"componentStatuses,omitempty"`
161162
APIEndpoint string `json:"apiEndpoint,omitempty"`
162163
ServiceAccountToken string `json:"serviceAccountToken,omitempty"`
164+
ServiceAccountTokenSecret string `json:"serviceAccountTokenSecret,omitempty"`
163165
CACert string `json:"caCert,omitempty"`
164166
Capacity v1.ResourceList `json:"capacity,omitempty"`
165167
Allocatable v1.ResourceList `json:"allocatable,omitempty"`

‎pkg/auth/providers/publicapi/login.go‎

Lines changed: 4 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -29,6 +29,7 @@ import (
2929
client "github.com/rancher/rancher/pkg/client/generated/management/v3public"
3030
"github.com/rancher/rancher/pkg/clustermanager"
3131
"github.com/rancher/rancher/pkg/controllers/managementuser/clusterauthtoken/common"
32+
v1 "github.com/rancher/rancher/pkg/generated/norman/core/v1"
3233
v3 "github.com/rancher/rancher/pkg/generated/norman/management.cattle.io/v3"
3334
schema "github.com/rancher/rancher/pkg/schemas/management.cattle.io/v3public"
3435
"github.com/rancher/rancher/pkg/types/config"
@@ -49,6 +50,7 @@ func newLoginHandler(ctx context.Context, mgmt *config.ScaledContext) *loginHand
4950
userMGR: mgmt.UserManager,
5051
tokenMGR: tokens.NewManager(ctx, mgmt),
5152
clusterLister: mgmt.Management.Clusters("").Controller().Lister(),
53+
secretLister: mgmt.Core.Secrets("").Controller().Lister(),
5254
}
5355
}
5456

@@ -57,6 +59,7 @@ type loginHandler struct {
5759
userMGR user.Manager
5860
tokenMGR *tokens.Manager
5961
clusterLister v3.ClusterLister
62+
secretLister v1.SecretLister
6063
}
6164

6265
func (h *loginHandler) login(actionName string, action *types.Action, request *types.APIContext) error {
@@ -247,7 +250,7 @@ func (h *loginHandler) createClusterAuthTokenIfNeeded(token *v3.Token, tokenValu
247250
if !cluster.Spec.LocalClusterAuthEndpoint.Enabled {
248251
return nil
249252
}
250-
clusterConfig, err := clustermanager.ToRESTConfig(cluster, h.scaledContext)
253+
clusterConfig, err := clustermanager.ToRESTConfig(cluster, h.scaledContext, h.secretLister)
251254
if err != nil {
252255
return err
253256
}

‎pkg/clustermanager/manager.go‎

Lines changed: 17 additions & 9 deletions
Original file line numberDiff line numberDiff line change
@@ -18,7 +18,9 @@ import (
1818
"github.com/rancher/norman/types"
1919
v32 "github.com/rancher/rancher/pkg/apis/management.cattle.io/v3"
2020
"github.com/rancher/rancher/pkg/clusterrouter"
21+
"github.com/rancher/rancher/pkg/controllers/management/secretmigrator"
2122
clusterController "github.com/rancher/rancher/pkg/controllers/managementuser"
23+
v1 "github.com/rancher/rancher/pkg/generated/norman/core/v1"
2224
v3 "github.com/rancher/rancher/pkg/generated/norman/management.cattle.io/v3"
2325
"github.com/rancher/rancher/pkg/kontainer-engine/drivers/gke"
2426
"github.com/rancher/rancher/pkg/rbac"
@@ -33,7 +35,7 @@ import (
3335
"golang.org/x/sync/semaphore"
3436
"k8s.io/apiextensions-apiserver/pkg/client/clientset/clientset"
3537
apierrors "k8s.io/apimachinery/pkg/api/errors"
36-
v1 "k8s.io/apimachinery/pkg/apis/meta/v1"
38+
metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
3739
authv1 "k8s.io/client-go/kubernetes/typed/authorization/v1"
3840
"k8s.io/client-go/rest"
3941
clientcmdapi "k8s.io/client-go/tools/clientcmd/api"
@@ -44,6 +46,7 @@ type Manager struct {
4446
ScaledContext *config.ScaledContext
4547
clusterLister v3.ClusterLister
4648
clusters v3.ClusterInterface
49+
secretLister v1.SecretLister
4750
controllers sync.Map
4851
accessControl types.AccessControl
4952
rbac rbacv1.Interface
@@ -69,6 +72,7 @@ func NewManager(httpsPort int, context *config.ScaledContext, asl accesscontrol.
6972
accessControl: rbac.NewAccessControlWithASL("", asl),
7073
clusterLister: context.Management.Clusters("").Controller().Lister(),
7174
clusters: context.Management.Clusters(""),
75+
secretLister: context.Core.Secrets("").Controller().Lister(),
7276
startSem: semaphore.NewWeighted(int64(settings.ClusterControllerStartCount.GetInt())),
7377
}
7478
}
@@ -107,7 +111,7 @@ func (m *Manager) RESTConfig(cluster *v3.Cluster) (rest.Config, error) {
107111
}
108112

109113
func (m *Manager) markUnavailable(clusterName string) {
110-
if cluster, err := m.clusters.Get(clusterName, v1.GetOptions{}); err == nil {
114+
if cluster, err := m.clusters.Get(clusterName, metav1.GetOptions{}); err == nil {
111115
if !v32.ClusterConditionReady.IsFalse(cluster) {
112116
v32.ClusterConditionReady.False(cluster)
113117
m.clusters.Update(cluster)
@@ -170,7 +174,7 @@ func (m *Manager) startController(r *record, controllers, clusterOwner bool) err
170174
func (m *Manager) changed(r *record, cluster *v3.Cluster, controllers, clusterOwner bool) bool {
171175
existing := r.clusterRec
172176
if existing.Status.APIEndpoint != cluster.Status.APIEndpoint ||
173-
existing.Status.ServiceAccountToken != cluster.Status.ServiceAccountToken ||
177+
existing.Status.ServiceAccountTokenSecret != cluster.Status.ServiceAccountTokenSecret ||
174178
existing.Status.CACert != cluster.Status.CACert ||
175179
existing.Status.AppliedSpec.LocalClusterAuthEndpoint.Enabled != cluster.Status.AppliedSpec.LocalClusterAuthEndpoint.Enabled {
176180
return true
@@ -194,7 +198,7 @@ func (m *Manager) doStart(rec *record, clusterOwner bool) (exit error) {
194198
// Prior to k8s v1.14, we simply did a DiscoveryClient.Version() check to see if the user cluster is alive
195199
// As of k8s v1.14, kubeapi returns a successful version response even if etcd is not available.
196200
// To work around this, now we try to get a namespace from the API, even if not found, it means the API is up.
197-
if _, err := rec.cluster.K8sClient.CoreV1().Namespaces().Get(rec.ctx, "kube-system", v1.GetOptions{}); err != nil && !apierrors.IsNotFound(err) {
201+
if _, err := rec.cluster.K8sClient.CoreV1().Namespaces().Get(rec.ctx, "kube-system", metav1.GetOptions{}); err != nil && !apierrors.IsNotFound(err) {
198202
if i == 2 {
199203
m.markUnavailable(rec.cluster.ClusterName)
200204
}
@@ -252,7 +256,7 @@ func (m *Manager) doStart(rec *record, clusterOwner bool) (exit error) {
252256
}
253257
}
254258

255-
func ToRESTConfig(cluster *v3.Cluster, context *config.ScaledContext) (*rest.Config, error) {
259+
func ToRESTConfig(cluster *v3.Cluster, context *config.ScaledContext, secretLister v1.SecretLister) (*rest.Config, error) {
256260
if cluster == nil {
257261
return nil, nil
258262
}
@@ -261,7 +265,7 @@ func ToRESTConfig(cluster *v3.Cluster, context *config.ScaledContext) (*rest.Con
261265
return &context.RESTConfig, nil
262266
}
263267

264-
if cluster.Status.APIEndpoint == "" || cluster.Status.CACert == "" || cluster.Status.ServiceAccountToken == "" {
268+
if cluster.Status.APIEndpoint == "" || cluster.Status.CACert == "" || cluster.Status.ServiceAccountTokenSecret == "" {
265269
return nil, nil
266270
}
267271

@@ -292,11 +296,15 @@ func ToRESTConfig(cluster *v3.Cluster, context *config.ScaledContext) (*rest.Con
292296
}
293297
}
294298

299+
secret, err := secretLister.Get(secretmigrator.SecretNamespace, cluster.Status.ServiceAccountTokenSecret)
300+
if err != nil {
301+
return nil, err
302+
}
295303
// adding suffix to make tlsConfig hashkey unique
296304
suffix := []byte("\n" + cluster.Name)
297305
rc := &rest.Config{
298306
Host: u.String(),
299-
BearerToken: cluster.Status.ServiceAccountToken,
307+
BearerToken: string(secret.Data[secretmigrator.SecretKey]),
300308
TLSClientConfig: rest.TLSClientConfig{
301309
CAData: append(caBytes, suffix...),
302310
NextProtos: []string{"http/1.1"},
@@ -397,7 +405,7 @@ func VerifyIgnoreDNSName(caCertsPEM []byte) (func(rawCerts [][]byte, verifiedCha
397405
}
398406

399407
func (m *Manager) toRecord(ctx context.Context, cluster *v3.Cluster) (*record, error) {
400-
kubeConfig, err := ToRESTConfig(cluster, m.ScaledContext)
408+
kubeConfig, err := ToRESTConfig(cluster, m.ScaledContext, m.secretLister)
401409
if kubeConfig == nil || err != nil {
402410
return nil, err
403411
}
@@ -484,7 +492,7 @@ func (m *Manager) UserContext(clusterName string) (*config.UserContext, error) {
484492
// UserContextFromCluster accepts a pointer to a Cluster and returns a client
485493
// for that cluster. It does not start any controllers.
486494
func (m *Manager) UserContextFromCluster(cluster *v3.Cluster) (*config.UserContext, error) {
487-
kubeConfig, err := ToRESTConfig(cluster, m.ScaledContext)
495+
kubeConfig, err := ToRESTConfig(cluster, m.ScaledContext, m.secretLister)
488496
if err != nil {
489497
return nil, err
490498
}

‎pkg/controllers/management/clusterprovisioner/provisioner.go‎

Lines changed: 14 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -20,6 +20,7 @@ import (
2020
util "github.com/rancher/rancher/pkg/cluster"
2121
"github.com/rancher/rancher/pkg/controllers/management/imported"
2222
kd "github.com/rancher/rancher/pkg/controllers/management/kontainerdrivermetadata"
23+
"github.com/rancher/rancher/pkg/controllers/management/secretmigrator"
2324
v1 "github.com/rancher/rancher/pkg/generated/norman/apps/v1"
2425
corev1 "github.com/rancher/rancher/pkg/generated/norman/core/v1"
2526
v3 "github.com/rancher/rancher/pkg/generated/norman/management.cattle.io/v3"
@@ -62,6 +63,7 @@ type Provisioner struct {
6263
RKESystemImages v3.RkeK8sSystemImageInterface
6364
RKESystemImagesLister v3.RkeK8sSystemImageLister
6465
SecretLister corev1.SecretLister
66+
Secrets corev1.SecretInterface
6567
}
6668

6769
func Register(ctx context.Context, management *config.ManagementContext) {
@@ -80,6 +82,7 @@ func Register(ctx context.Context, management *config.ManagementContext) {
8082
RKESystemImages: management.Management.RkeK8sSystemImages(""),
8183
DaemonsetLister: management.Apps.DaemonSets("").Controller().Lister(),
8284
SecretLister: management.Core.Secrets("").Controller().Lister(),
85+
Secrets: management.Core.Secrets(""),
8386
}
8487
// Add handlers
8588
p.Clusters.AddLifecycle(ctx, "cluster-provisioner-controller", p)
@@ -494,7 +497,11 @@ func (p *Provisioner) reconcileCluster(cluster *v3.Cluster, create bool) (*v3.Cl
494497
return nil, err
495498
}
496499

497-
cluster.Status.ServiceAccountToken = serviceAccountToken
500+
secret, err := secretmigrator.NewMigrator(p.SecretLister, p.Secrets).CreateOrUpdateServiceAccountTokenSecret(cluster.Status.ServiceAccountTokenSecret, serviceAccountToken, cluster)
501+
if err != nil {
502+
return nil, err
503+
}
504+
cluster.Status.ServiceAccountTokenSecret = secret.Name
498505
apimgmtv3.ClusterConditionServiceAccountMigrated.True(cluster)
499506

500507
// Update the cluster in k8s
@@ -585,6 +592,11 @@ func (p *Provisioner) reconcileCluster(cluster *v3.Cluster, create bool) (*v3.Cl
585592

586593
apimgmtv3.ClusterConditionServiceAccountMigrated.True(cluster)
587594

595+
secret, err := secretmigrator.NewMigrator(p.SecretLister, p.Secrets).CreateOrUpdateServiceAccountTokenSecret(cluster.Status.ServiceAccountTokenSecret, serviceAccountToken, cluster)
596+
if err != nil {
597+
return nil, err
598+
}
599+
588600
saved := false
589601
for i := 0; i < 20; i++ {
590602
cluster, err = p.Clusters.Get(cluster.Name, metav1.GetOptions{})
@@ -599,7 +611,7 @@ func (p *Provisioner) reconcileCluster(cluster *v3.Cluster, create bool) (*v3.Cl
599611

600612
cluster.Status.AppliedSpec = censoredSpec
601613
cluster.Status.APIEndpoint = apiEndpoint
602-
cluster.Status.ServiceAccountToken = serviceAccountToken
614+
cluster.Status.ServiceAccountTokenSecret = secret.Name
603615
cluster.Status.CACert = caCert
604616
resetRkeConfigFlags(cluster, updateTriggered)
605617

0 commit comments

Comments
 (0)