-
Notifications
You must be signed in to change notification settings - Fork 1.3k
/
cluster.go
204 lines (176 loc) · 7.75 KB
/
cluster.go
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
125
126
127
128
129
130
131
132
133
134
135
136
137
138
139
140
141
142
143
144
145
146
147
148
149
150
151
152
153
154
155
156
157
158
159
160
161
162
163
164
165
166
167
168
169
170
171
172
173
174
175
176
177
178
179
180
181
182
183
184
185
186
187
188
189
190
191
192
193
194
195
196
197
198
199
200
201
202
203
204
/*
Copyright 2020 The Kubernetes Authors.
Licensed under the Apache License, Version 2.0 (the "License");
you may not use this file except in compliance with the License.
You may obtain a copy of the License at
http://www.apache.org/licenses/LICENSE-2.0
Unless required by applicable law or agreed to in writing, software
distributed under the License is distributed on an "AS IS" BASIS,
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
See the License for the specific language governing permissions and
limitations under the License.
*/
package internal
import (
"context"
"crypto/tls"
"crypto/x509"
"fmt"
"time"
"github.com/pkg/errors"
corev1 "k8s.io/api/core/v1"
"sigs.k8s.io/controller-runtime/pkg/client"
clusterv1 "sigs.k8s.io/cluster-api/api/v1beta1"
"sigs.k8s.io/cluster-api/controllers/remote"
expv1 "sigs.k8s.io/cluster-api/exp/api/v1beta1"
"sigs.k8s.io/cluster-api/util/collections"
"sigs.k8s.io/cluster-api/util/secret"
)
const (
// KubeadmControlPlaneControllerName defines the controller used when creating clients.
KubeadmControlPlaneControllerName = "kubeadm-controlplane-controller"
)
// ManagementCluster defines all behaviors necessary for something to function as a management cluster.
type ManagementCluster interface {
client.Reader
GetMachinesForCluster(ctx context.Context, cluster *clusterv1.Cluster, filters ...collections.Func) (collections.Machines, error)
GetMachinePoolsForCluster(ctx context.Context, cluster *clusterv1.Cluster) (*expv1.MachinePoolList, error)
GetWorkloadCluster(ctx context.Context, clusterKey client.ObjectKey) (WorkloadCluster, error)
}
// Management holds operations on the management cluster.
type Management struct {
Client client.Reader
Tracker *remote.ClusterCacheTracker
EtcdDialTimeout time.Duration
}
// RemoteClusterConnectionError represents a failure to connect to a remote cluster.
type RemoteClusterConnectionError struct {
Name string
Err error
}
// Error satisfies the error interface.
func (e *RemoteClusterConnectionError) Error() string { return e.Name + ": " + e.Err.Error() }
// Unwrap satisfies the unwrap error inteface.
func (e *RemoteClusterConnectionError) Unwrap() error { return e.Err }
// Get implements client.Reader.
func (m *Management) Get(ctx context.Context, key client.ObjectKey, obj client.Object) error {
return m.Client.Get(ctx, key, obj)
}
// List implements client.Reader.
func (m *Management) List(ctx context.Context, list client.ObjectList, opts ...client.ListOption) error {
return m.Client.List(ctx, list, opts...)
}
// GetMachinesForCluster returns a list of machines that can be filtered or not.
// If no filter is supplied then all machines associated with the target cluster are returned.
func (m *Management) GetMachinesForCluster(ctx context.Context, cluster *clusterv1.Cluster, filters ...collections.Func) (collections.Machines, error) {
return collections.GetFilteredMachinesForCluster(ctx, m.Client, cluster, filters...)
}
// GetMachinePoolsForCluster returns a list of machine pools owned by the cluster.
func (m *Management) GetMachinePoolsForCluster(ctx context.Context, cluster *clusterv1.Cluster) (*expv1.MachinePoolList, error) {
selectors := []client.ListOption{
client.InNamespace(cluster.GetNamespace()),
client.MatchingLabels{
clusterv1.ClusterLabelName: cluster.GetName(),
},
}
machinePoolList := &expv1.MachinePoolList{}
err := m.Client.List(ctx, machinePoolList, selectors...)
return machinePoolList, err
}
// GetWorkloadCluster builds a cluster object.
// The cluster comes with an etcd client generator to connect to any etcd pod living on a managed machine.
func (m *Management) GetWorkloadCluster(ctx context.Context, clusterKey client.ObjectKey) (WorkloadCluster, error) {
// TODO(chuckha): Inject this dependency.
// TODO(chuckha): memoize this function. The workload client only exists as long as a reconciliation loop.
restConfig, err := remote.RESTConfig(ctx, KubeadmControlPlaneControllerName, m.Client, clusterKey)
if err != nil {
return nil, err
}
restConfig.Timeout = 30 * time.Second
if m.Tracker == nil {
return nil, errors.New("Cannot get WorkloadCluster: No remote Cluster Cache")
}
c, err := m.Tracker.GetClient(ctx, clusterKey)
if err != nil {
return nil, err
}
clientConfig, err := m.Tracker.GetRESTConfig(ctx, clusterKey)
if err != nil {
return nil, err
}
// Make sure we use the same CA and Host as the client.
// Note: This has to be done to be able to communicate directly on self-hosted clusters.
restConfig.CAData = clientConfig.CAData
restConfig.CAFile = clientConfig.CAFile
restConfig.Host = clientConfig.Host
// Retrieves the etcd CA key Pair
crtData, keyData, err := m.getEtcdCAKeyPair(ctx, clusterKey)
if err != nil {
return nil, err
}
// If the CA key is defined, the cluster is using a managed etcd, and so we can generate a new
// etcd client certificate for the controllers.
// Otherwise the cluster is using an external etcd; in this case the only option to connect to etcd is to re-use
// the apiserver-etcd-client certificate.
// TODO: consider if we can detect if we are using external etcd in a more explicit way (e.g. looking at the config instead of deriving from the existing certificates)
var clientCert tls.Certificate
if keyData != nil {
clientCert, err = generateClientCert(crtData, keyData)
if err != nil {
return nil, err
}
} else {
clientCert, err = m.getAPIServerEtcdClientCert(ctx, clusterKey)
if err != nil {
return nil, err
}
}
caPool := x509.NewCertPool()
caPool.AppendCertsFromPEM(crtData)
tlsConfig := &tls.Config{
RootCAs: caPool,
Certificates: []tls.Certificate{clientCert},
MinVersion: tls.VersionTLS12,
}
tlsConfig.InsecureSkipVerify = true
return &Workload{
Client: c,
CoreDNSMigrator: &CoreDNSMigrator{},
etcdClientGenerator: NewEtcdClientGenerator(restConfig, tlsConfig, m.EtcdDialTimeout),
}, nil
}
func (m *Management) getEtcdCAKeyPair(ctx context.Context, clusterKey client.ObjectKey) ([]byte, []byte, error) {
etcdCASecret := &corev1.Secret{}
etcdCAObjectKey := client.ObjectKey{
Namespace: clusterKey.Namespace,
Name: fmt.Sprintf("%s-etcd", clusterKey.Name),
}
if err := m.Client.Get(ctx, etcdCAObjectKey, etcdCASecret); err != nil {
return nil, nil, errors.Wrapf(err, "failed to get secret; etcd CA bundle %s/%s", etcdCAObjectKey.Namespace, etcdCAObjectKey.Name)
}
crtData, ok := etcdCASecret.Data[secret.TLSCrtDataName]
if !ok {
return nil, nil, errors.Errorf("etcd tls crt does not exist for cluster %s/%s", clusterKey.Namespace, clusterKey.Name)
}
keyData := etcdCASecret.Data[secret.TLSKeyDataName]
return crtData, keyData, nil
}
func (m *Management) getAPIServerEtcdClientCert(ctx context.Context, clusterKey client.ObjectKey) (tls.Certificate, error) {
apiServerEtcdClientCertificateSecret := &corev1.Secret{}
apiServerEtcdClientCertificateObjectKey := client.ObjectKey{
Namespace: clusterKey.Namespace,
Name: fmt.Sprintf("%s-apiserver-etcd-client", clusterKey.Name),
}
if err := m.Client.Get(ctx, apiServerEtcdClientCertificateObjectKey, apiServerEtcdClientCertificateSecret); err != nil {
return tls.Certificate{}, errors.Wrapf(err, "failed to get secret; etcd apiserver-etcd-client %s/%s", apiServerEtcdClientCertificateObjectKey.Namespace, apiServerEtcdClientCertificateObjectKey.Name)
}
crtData, ok := apiServerEtcdClientCertificateSecret.Data[secret.TLSCrtDataName]
if !ok {
return tls.Certificate{}, errors.Errorf("etcd tls crt does not exist for cluster %s/%s", clusterKey.Namespace, clusterKey.Name)
}
keyData, ok := apiServerEtcdClientCertificateSecret.Data[secret.TLSKeyDataName]
if !ok {
return tls.Certificate{}, errors.Errorf("etcd tls key does not exist for cluster %s/%s", clusterKey.Namespace, clusterKey.Name)
}
return tls.X509KeyPair(crtData, keyData)
}