-
Notifications
You must be signed in to change notification settings - Fork 858
/
cluster_management.go
263 lines (242 loc) · 9.66 KB
/
cluster_management.go
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
125
126
127
128
129
130
131
132
133
134
135
136
137
138
139
140
141
142
143
144
145
146
147
148
149
150
151
152
153
154
155
156
157
158
159
160
161
162
163
164
165
166
167
168
169
170
171
172
173
174
175
176
177
178
179
180
181
182
183
184
185
186
187
188
189
190
191
192
193
194
195
196
197
198
199
200
201
202
203
204
205
206
207
208
209
210
211
212
213
214
215
216
217
218
219
220
221
222
223
224
225
226
227
228
229
230
231
232
233
234
235
236
237
238
239
240
241
242
243
244
245
246
247
248
249
250
251
252
253
254
255
256
257
258
259
260
261
262
263
/*
Copyright 2021 The KubeVela Authors.
Licensed under the Apache License, Version 2.0 (the "License");
you may not use this file except in compliance with the License.
You may obtain a copy of the License at
http://www.apache.org/licenses/LICENSE-2.0
Unless required by applicable law or agreed to in writing, software
distributed under the License is distributed on an "AS IS" BASIS,
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
See the License for the specific language governing permissions and
limitations under the License.
*/
package multicluster
import (
"context"
"fmt"
v1alpha12 "github.com/oam-dev/cluster-gateway/pkg/apis/cluster/v1alpha1"
"github.com/pkg/errors"
v1 "k8s.io/api/core/v1"
v14 "k8s.io/api/storage/v1"
errors2 "k8s.io/apimachinery/pkg/api/errors"
"k8s.io/apimachinery/pkg/api/resource"
v12 "k8s.io/apimachinery/pkg/apis/meta/v1"
types2 "k8s.io/apimachinery/pkg/types"
"k8s.io/client-go/tools/clientcmd"
"k8s.io/client-go/tools/clientcmd/api"
"sigs.k8s.io/controller-runtime/pkg/client"
"github.com/oam-dev/kubevela/apis/core.oam.dev/v1beta1"
"github.com/oam-dev/kubevela/apis/types"
"github.com/oam-dev/kubevela/pkg/policy/envbinding"
errors3 "github.com/oam-dev/kubevela/pkg/utils/errors"
)
// ensureVelaSystemNamespaceInstalled ensures vela namespace to be installed in child cluster
func ensureVelaSystemNamespaceInstalled(ctx context.Context, c client.Client, clusterName string, createNamespace string) error {
remoteCtx := ContextWithClusterName(ctx, clusterName)
if err := c.Get(remoteCtx, types2.NamespacedName{Name: createNamespace}, &v1.Namespace{}); err != nil {
if !errors2.IsNotFound(err) {
return errors.Wrapf(err, "failed to check vela-system ")
}
if err = c.Create(remoteCtx, &v1.Namespace{ObjectMeta: v12.ObjectMeta{Name: createNamespace}}); err != nil {
return errors.Wrapf(err, "failed to create vela-system namespace")
}
}
return nil
}
// ensureClusterNotExists checks if child cluster has already been joined, if joined, error is returned
func ensureClusterNotExists(ctx context.Context, c client.Client, clusterName string) error {
secret := &v1.Secret{}
err := c.Get(ctx, types2.NamespacedName{Name: clusterName, Namespace: ClusterGatewaySecretNamespace}, secret)
if err == nil {
return ErrClusterExists
}
if !errors2.IsNotFound(err) {
return errors.Wrapf(err, "failed to check duplicate cluster secret")
}
return nil
}
// GetMutableClusterSecret retrieves the cluster secret and check if any application is using the cluster
func GetMutableClusterSecret(ctx context.Context, c client.Client, clusterName string) (*v1.Secret, error) {
clusterSecret := &v1.Secret{}
if err := c.Get(ctx, types2.NamespacedName{Namespace: ClusterGatewaySecretNamespace, Name: clusterName}, clusterSecret); err != nil {
return nil, errors.Wrapf(err, "failed to find target cluster secret %s", clusterName)
}
labels := clusterSecret.GetLabels()
if labels == nil || labels[v1alpha12.LabelKeyClusterCredentialType] == "" {
return nil, fmt.Errorf("invalid cluster secret %s: cluster credential type label %s is not set", clusterName, v1alpha12.LabelKeyClusterCredentialType)
}
apps := &v1beta1.ApplicationList{}
if err := c.List(ctx, apps); err != nil {
return nil, errors.Wrap(err, "failed to find applications to check clusters")
}
errs := errors3.ErrorList{}
for _, app := range apps.Items {
status, err := envbinding.GetEnvBindingPolicyStatus(app.DeepCopy(), "")
if err == nil && status != nil {
for _, env := range status.Envs {
for _, placement := range env.Placements {
if placement.Cluster == clusterName {
errs = append(errs, fmt.Errorf("application %s/%s (env: %s) is currently using cluster %s", app.Namespace, app.Name, env.Env, clusterName))
}
}
}
}
}
if errs.HasError() {
return nil, errors.Wrapf(errs, "cluster %s is in use now", clusterName)
}
return clusterSecret, nil
}
// JoinClusterByKubeConfig add child cluster by kubeconfig path, return cluster info and error
func JoinClusterByKubeConfig(_ctx context.Context, k8sClient client.Client, kubeconfigPath string, clusterName string) (*api.Cluster, error) {
config, err := clientcmd.LoadFromFile(kubeconfigPath)
if err != nil {
return nil, errors.Wrapf(err, "failed to get kubeconfig")
}
if len(config.CurrentContext) == 0 {
return nil, fmt.Errorf("current-context is not set")
}
ctx, ok := config.Contexts[config.CurrentContext]
if !ok {
return nil, fmt.Errorf("current-context %s not found", config.CurrentContext)
}
cluster, ok := config.Clusters[ctx.Cluster]
if !ok {
return nil, fmt.Errorf("cluster %s not found", ctx.Cluster)
}
authInfo, ok := config.AuthInfos[ctx.AuthInfo]
if !ok {
return nil, fmt.Errorf("authInfo %s not found", ctx.AuthInfo)
}
if clusterName == "" {
clusterName = ctx.Cluster
}
if clusterName == ClusterLocalName {
return cluster, fmt.Errorf("cannot use `%s` as cluster name, it is reserved as the local cluster", ClusterLocalName)
}
if err := ensureClusterNotExists(_ctx, k8sClient, clusterName); err != nil {
return cluster, errors.Wrapf(err, "cannot use cluster name %s", clusterName)
}
var credentialType v1alpha12.CredentialType
data := map[string][]byte{
"endpoint": []byte(cluster.Server),
"ca.crt": cluster.CertificateAuthorityData,
}
if len(authInfo.Token) > 0 {
credentialType = v1alpha12.CredentialTypeServiceAccountToken
data["token"] = []byte(authInfo.Token)
} else {
credentialType = v1alpha12.CredentialTypeX509Certificate
data["tls.crt"] = authInfo.ClientCertificateData
data["tls.key"] = authInfo.ClientKeyData
}
secret := &v1.Secret{
ObjectMeta: v12.ObjectMeta{
Name: clusterName,
Namespace: ClusterGatewaySecretNamespace,
Labels: map[string]string{
v1alpha12.LabelKeyClusterCredentialType: string(credentialType),
},
},
Type: v1.SecretTypeOpaque,
Data: data,
}
if err := k8sClient.Create(_ctx, secret); err != nil {
return cluster, errors.Wrapf(err, "failed to add cluster to kubernetes")
}
if err := ensureVelaSystemNamespaceInstalled(_ctx, k8sClient, clusterName, types.DefaultKubeVelaNS); err != nil {
return nil, errors.Wrapf(err, "failed to create vela namespace in cluster %s", clusterName)
}
return cluster, nil
}
// DetachCluster detach cluster by name, if cluster is using by application, it will return error
func DetachCluster(ctx context.Context, k8sClient client.Client, clusterName string) error {
if clusterName == ClusterLocalName {
return ErrReservedLocalClusterName
}
clusterSecret, err := GetMutableClusterSecret(ctx, k8sClient, clusterName)
if err != nil {
return errors.Wrapf(err, "cluster %s is not mutable now", clusterName)
}
return k8sClient.Delete(ctx, clusterSecret)
}
// RenameCluster rename cluster
func RenameCluster(ctx context.Context, k8sClient client.Client, oldClusterName string, newClusterName string) error {
if newClusterName == ClusterLocalName {
return ErrReservedLocalClusterName
}
clusterSecret, err := GetMutableClusterSecret(ctx, k8sClient, oldClusterName)
if err != nil {
return errors.Wrapf(err, "cluster %s is not mutable now", oldClusterName)
}
if err := ensureClusterNotExists(ctx, k8sClient, newClusterName); err != nil {
return errors.Wrapf(err, "cannot set cluster name to %s", newClusterName)
}
if err := k8sClient.Delete(ctx, clusterSecret); err != nil {
return errors.Wrapf(err, "failed to rename cluster from %s to %s", oldClusterName, newClusterName)
}
clusterSecret.ObjectMeta = v12.ObjectMeta{
Name: newClusterName,
Namespace: ClusterGatewaySecretNamespace,
Labels: clusterSecret.Labels,
Annotations: clusterSecret.Annotations,
}
if err := k8sClient.Create(ctx, clusterSecret); err != nil {
return errors.Wrapf(err, "failed to rename cluster from %s to %s", oldClusterName, newClusterName)
}
return nil
}
// ClusterInfo describes the basic information of a cluster
type ClusterInfo struct {
Nodes *v1.NodeList
WorkerNumber int
MasterNumber int
MemoryCapacity resource.Quantity
CPUCapacity resource.Quantity
PodCapacity resource.Quantity
MemoryAllocatable resource.Quantity
CPUAllocatable resource.Quantity
PodAllocatable resource.Quantity
StorageClasses *v14.StorageClassList
}
// GetClusterInfo retrieves current cluster info from cluster
func GetClusterInfo(_ctx context.Context, k8sClient client.Client, clusterName string) (*ClusterInfo, error) {
ctx := ContextWithClusterName(_ctx, clusterName)
nodes := &v1.NodeList{}
if err := k8sClient.List(ctx, nodes); err != nil {
return nil, errors.Wrapf(err, "failed to list cluster nodes")
}
var workerNumber, masterNumber int
var memoryCapacity, cpuCapacity, podCapacity, memoryAllocatable, cpuAllocatable, podAllcatable resource.Quantity
for _, node := range nodes.Items {
if _, ok := node.Labels["node-role.kubernetes.io/master"]; ok {
masterNumber++
} else {
workerNumber++
}
capacity := node.Status.Capacity
memoryCapacity.Add(*capacity.Memory())
cpuCapacity.Add(*capacity.Cpu())
podCapacity.Add(*capacity.Pods())
allocatable := node.Status.Allocatable
memoryAllocatable.Add(*allocatable.Memory())
cpuAllocatable.Add(*allocatable.Cpu())
podAllcatable.Add(*allocatable.Pods())
}
storageClasses := &v14.StorageClassList{}
if err := k8sClient.List(ctx, storageClasses); err != nil {
return nil, errors.Wrapf(err, "failed to list storage classes")
}
return &ClusterInfo{
Nodes: nodes,
WorkerNumber: workerNumber,
MasterNumber: masterNumber,
MemoryCapacity: memoryCapacity,
CPUCapacity: cpuCapacity,
PodCapacity: podCapacity,
MemoryAllocatable: memoryAllocatable,
CPUAllocatable: cpuAllocatable,
PodAllocatable: podAllcatable,
StorageClasses: storageClasses,
}, nil
}