diff --git a/README.md b/README.md index 57e99f3..28b5942 100644 --- a/README.md +++ b/README.md @@ -2,7 +2,14 @@ This is a controller that tracks [GitopsCluster] objects. -It provides a CR for a `ClusterBootstrapConfig` which provides a [Job](https://kubernetes.io/docs/concepts/workloads/controllers/job/) template. +It provides the following CRs: + +- [ClusterBootstrapConfig](#clusterBootstrapConfig) +- [SecretSync](#secretsync) + +## ClusterBootstrapConfig + +`ClusterBootstrapConfig` CR provides a [Job](https://kubernetes.io/docs/concepts/workloads/controllers/job/) template. When a GitopsCluster is "Ready" a Job is created from the template, the template can access multiple fields. @@ -36,7 +43,7 @@ spec: This is using Go [templating](https://pkg.go.dev/text/template) and the `GitopsCluster` object is provided as the context, this means that expressions like `{{ .ObjectMeta.Name }}` will get the _name_ of the GitopsCluster that has transitioned to "Ready". -## Annotations +### Annotations Go templating doesn't easily support access to strings that have "/" in them, which is a common annotation [naming strategy](https://kubernetes.io/docs/concepts/overview/working-with-objects/annotations/#syntax-and-character-set). @@ -53,6 +60,35 @@ e.g. ``` +## SecretSync + +`SecretSync` provides a way to sync secrets from management cluster to leaf cluster. + +The CR references the secret on management cluster to be synced to matched leaf clusters. + +SecretSync has a selector to select group of clusters based on their labels + +Secrets will be re-synced to leaf clusters when updated + +### Example + +```yaml +apiVersion: capi.weave.works/v1alpha2 +kind: SecretSync +metadata: + name: my-dev-secret-syncer + namespace: default +spec: + clusterSelector: + matchLabels: + environment: dev + secretRef: + name: my-dev-secret + targetNamespace: my-namespace +``` + + + ## Installation Release files are available https://github.com/weaveworks/cluster-bootstrap-controller/releases diff --git a/api/v1alpha2/secretsync_types.go b/api/v1alpha2/secretsync_types.go new file mode 100644 index 0000000..789b2d6 --- /dev/null +++ b/api/v1alpha2/secretsync_types.go @@ -0,0 +1,66 @@ +package v1alpha2 + +import ( + v1 "k8s.io/api/core/v1" + metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" +) + +// SecretSyncSpec +type SecretSyncSpec struct { + // Label selector for Clusters. The Clusters that are + // selected by this will be the ones affected by this SecretSync. + // It must match the Cluster labels. This field is immutable. + // Label selector cannot be empty. + ClusterSelector metav1.LabelSelector `json:"clusterSelector"` + // SecretRef specifies the Secret to be bootstrapped to the matched clusters + // Secret must be in the same namespace of the SecretSync object + SecretRef v1.LocalObjectReference `json:"secretRef"` + // TargetNamespace specifies the namespace which the secret should be bootstrapped in + // The default value is the namespace of the referenced secret + //+optional + TargetNamespace string `json:"targetNamespace,omitempty"` +} + +// SecretSyncStatus secretsync object status +type SecretSyncStatus struct { + // SecretVersions a map contains the ResourceVersion of the secret of each cluster + // Cluster name is the key and secret's ResourceVersion is the value + SecretVersions map[string]string `json:"versions"` +} + +// SetClusterSecretVersion sets the latest secret version of the given cluster +func (s *SecretSyncStatus) SetClusterSecretVersion(cluster, version string) { + if s.SecretVersions == nil { + s.SecretVersions = make(map[string]string) + } + s.SecretVersions[cluster] = version +} + +// GetClusterSecretVersion returns secret's ResourceVersion of the given cluster +func (s *SecretSyncStatus) GetClusterSecretVersion(cluster string) string { + return s.SecretVersions[cluster] +} + +//+kubebuilder:object:root=true +//+kubebuilder:subresource:status +//+kubebuilder:storageversion +//+kubebuilder:scope:namespaced + +type SecretSync struct { + metav1.TypeMeta `json:",inline"` + metav1.ObjectMeta `json:"metadata,omitempty"` + Spec SecretSyncSpec `json:"spec,omitempty"` + Status SecretSyncStatus `json:"status,omitempty"` +} + +//+kubebuilder:object:root=true + +type SecretSyncList struct { + metav1.TypeMeta `json:",inline"` + metav1.ListMeta `json:"metadata,omitempty"` + Items []SecretSync `json:"items"` +} + +func init() { + SchemeBuilder.Register(&SecretSync{}, &SecretSyncList{}) +} diff --git a/api/v1alpha2/zz_generated.deepcopy.go b/api/v1alpha2/zz_generated.deepcopy.go index 3606db3..b735a36 100644 --- a/api/v1alpha2/zz_generated.deepcopy.go +++ b/api/v1alpha2/zz_generated.deepcopy.go @@ -147,3 +147,101 @@ func (in *JobTemplate) DeepCopy() *JobTemplate { in.DeepCopyInto(out) return out } + +// DeepCopyInto is an autogenerated deepcopy function, copying the receiver, writing into out. in must be non-nil. +func (in *SecretSync) DeepCopyInto(out *SecretSync) { + *out = *in + out.TypeMeta = in.TypeMeta + in.ObjectMeta.DeepCopyInto(&out.ObjectMeta) + in.Spec.DeepCopyInto(&out.Spec) + in.Status.DeepCopyInto(&out.Status) +} + +// DeepCopy is an autogenerated deepcopy function, copying the receiver, creating a new SecretSync. +func (in *SecretSync) DeepCopy() *SecretSync { + if in == nil { + return nil + } + out := new(SecretSync) + in.DeepCopyInto(out) + return out +} + +// DeepCopyObject is an autogenerated deepcopy function, copying the receiver, creating a new runtime.Object. +func (in *SecretSync) DeepCopyObject() runtime.Object { + if c := in.DeepCopy(); c != nil { + return c + } + return nil +} + +// DeepCopyInto is an autogenerated deepcopy function, copying the receiver, writing into out. in must be non-nil. +func (in *SecretSyncList) DeepCopyInto(out *SecretSyncList) { + *out = *in + out.TypeMeta = in.TypeMeta + in.ListMeta.DeepCopyInto(&out.ListMeta) + if in.Items != nil { + in, out := &in.Items, &out.Items + *out = make([]SecretSync, len(*in)) + for i := range *in { + (*in)[i].DeepCopyInto(&(*out)[i]) + } + } +} + +// DeepCopy is an autogenerated deepcopy function, copying the receiver, creating a new SecretSyncList. +func (in *SecretSyncList) DeepCopy() *SecretSyncList { + if in == nil { + return nil + } + out := new(SecretSyncList) + in.DeepCopyInto(out) + return out +} + +// DeepCopyObject is an autogenerated deepcopy function, copying the receiver, creating a new runtime.Object. +func (in *SecretSyncList) DeepCopyObject() runtime.Object { + if c := in.DeepCopy(); c != nil { + return c + } + return nil +} + +// DeepCopyInto is an autogenerated deepcopy function, copying the receiver, writing into out. in must be non-nil. +func (in *SecretSyncSpec) DeepCopyInto(out *SecretSyncSpec) { + *out = *in + in.ClusterSelector.DeepCopyInto(&out.ClusterSelector) + out.SecretRef = in.SecretRef +} + +// DeepCopy is an autogenerated deepcopy function, copying the receiver, creating a new SecretSyncSpec. +func (in *SecretSyncSpec) DeepCopy() *SecretSyncSpec { + if in == nil { + return nil + } + out := new(SecretSyncSpec) + in.DeepCopyInto(out) + return out +} + +// DeepCopyInto is an autogenerated deepcopy function, copying the receiver, writing into out. in must be non-nil. +func (in *SecretSyncStatus) DeepCopyInto(out *SecretSyncStatus) { + *out = *in + if in.SecretVersions != nil { + in, out := &in.SecretVersions, &out.SecretVersions + *out = make(map[string]string, len(*in)) + for key, val := range *in { + (*out)[key] = val + } + } +} + +// DeepCopy is an autogenerated deepcopy function, copying the receiver, creating a new SecretSyncStatus. +func (in *SecretSyncStatus) DeepCopy() *SecretSyncStatus { + if in == nil { + return nil + } + out := new(SecretSyncStatus) + in.DeepCopyInto(out) + return out +} diff --git a/config/crd/bases/capi.weave.works_secretsyncs.yaml b/config/crd/bases/capi.weave.works_secretsyncs.yaml new file mode 100644 index 0000000..d4a236f --- /dev/null +++ b/config/crd/bases/capi.weave.works_secretsyncs.yaml @@ -0,0 +1,120 @@ +--- +apiVersion: apiextensions.k8s.io/v1 +kind: CustomResourceDefinition +metadata: + annotations: + controller-gen.kubebuilder.io/version: v0.9.2 + creationTimestamp: null + name: secretsyncs.capi.weave.works +spec: + group: capi.weave.works + names: + kind: SecretSync + listKind: SecretSyncList + plural: secretsyncs + singular: secretsync + scope: Namespaced + versions: + - name: v1alpha2 + schema: + openAPIV3Schema: + properties: + apiVersion: + description: 'APIVersion defines the versioned schema of this representation + of an object. Servers should convert recognized schemas to the latest + internal value, and may reject unrecognized values. More info: https://git.k8s.io/community/contributors/devel/sig-architecture/api-conventions.md#resources' + type: string + kind: + description: 'Kind is a string value representing the REST resource this + object represents. Servers may infer this from the endpoint the client + submits requests to. Cannot be updated. In CamelCase. More info: https://git.k8s.io/community/contributors/devel/sig-architecture/api-conventions.md#types-kinds' + type: string + metadata: + type: object + spec: + description: SecretSyncSpec + properties: + clusterSelector: + description: ClusterSelector specifies the label selector to match + clusters with + properties: + matchExpressions: + description: matchExpressions is a list of label selector requirements. + The requirements are ANDed. + items: + description: A label selector requirement is a selector that + contains values, a key, and an operator that relates the key + and values. + properties: + key: + description: key is the label key that the selector applies + to. + type: string + operator: + description: operator represents a key's relationship to + a set of values. Valid operators are In, NotIn, Exists + and DoesNotExist. + type: string + values: + description: values is an array of string values. If the + operator is In or NotIn, the values array must be non-empty. + If the operator is Exists or DoesNotExist, the values + array must be empty. This array is replaced during a strategic + merge patch. + items: + type: string + type: array + required: + - key + - operator + type: object + type: array + matchLabels: + additionalProperties: + type: string + description: matchLabels is a map of {key,value} pairs. A single + {key,value} in the matchLabels map is equivalent to an element + of matchExpressions, whose key field is "key", the operator + is "In", and the values array contains only "value". The requirements + are ANDed. + type: object + type: object + x-kubernetes-map-type: atomic + secretRef: + description: SecretRef specifies the Secret to be bootstrapped to + the matched clusters Secret must be in the same namespace of the + SecretSync object + properties: + name: + description: 'Name of the referent. More info: https://kubernetes.io/docs/concepts/overview/working-with-objects/names/#names + TODO: Add other useful fields. apiVersion, kind, uid?' + type: string + type: object + x-kubernetes-map-type: atomic + targetNamespace: + description: TargetNamespace specifies the namespace which the secret + should be bootstrapped in The default value is the namespace of + the referenced secret + type: string + required: + - clusterSelector + - secretRef + type: object + status: + description: SecretSyncStatus secretsync object status + properties: + versions: + additionalProperties: + type: string + description: SecretVersions a map contains the ResourceVersion of + the secret of each cluster Cluster name is the key and secret's + ResourceVersion is the value + type: object + required: + - versions + type: object + type: object + served: true + storage: true + subresources: + status: {} diff --git a/config/rbac/role.yaml b/config/rbac/role.yaml index 55c18a4..f7c0acd 100644 --- a/config/rbac/role.yaml +++ b/config/rbac/role.yaml @@ -51,6 +51,32 @@ rules: - get - patch - update +- apiGroups: + - capi.weave.works + resources: + - secretsyncs + verbs: + - create + - delete + - get + - list + - patch + - update + - watch +- apiGroups: + - capi.weave.works + resources: + - secretsyncs/finalizers + verbs: + - update +- apiGroups: + - capi.weave.works + resources: + - secretsyncs/status + verbs: + - get + - patch + - update - apiGroups: - cluster.x-k8s.io resources: diff --git a/controllers/bootstrap_test.go b/controllers/bootstrap_test.go index d0ad70e..693043a 100644 --- a/controllers/bootstrap_test.go +++ b/controllers/bootstrap_test.go @@ -12,7 +12,6 @@ import ( corev1 "k8s.io/api/core/v1" metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" "k8s.io/apimachinery/pkg/runtime" - clientgoscheme "k8s.io/client-go/kubernetes/scheme" ptrutils "k8s.io/utils/pointer" "sigs.k8s.io/controller-runtime/pkg/client" "sigs.k8s.io/controller-runtime/pkg/client/fake" @@ -21,12 +20,6 @@ import ( "github.com/weaveworks/cluster-bootstrap-controller/test" ) -const ( - testConfigName = "test-config" - testClusterName = "test-cluster" - testNamespace = "testing" -) - func Test_bootstrapClusterWithConfig(t *testing.T) { bc := makeTestClusterBootstrapConfig() cl := makeTestCluster() @@ -141,20 +134,6 @@ func Test_bootstrapClusterWithConfig_sets_job_ttl(t *testing.T) { } } -func makeTestCluster(opts ...func(*gitopsv1alpha1.GitopsCluster)) *gitopsv1alpha1.GitopsCluster { - c := &gitopsv1alpha1.GitopsCluster{ - ObjectMeta: metav1.ObjectMeta{ - Name: testClusterName, - Namespace: testNamespace, - }, - Spec: gitopsv1alpha1.GitopsClusterSpec{}, - } - for _, o := range opts { - o(c) - } - return c -} - func makeTestClusterBootstrapConfig(opts ...func(*capiv1alpha2.ClusterBootstrapConfig)) *capiv1alpha2.ClusterBootstrapConfig { bc := &capiv1alpha2.ClusterBootstrapConfig{ ObjectMeta: metav1.ObjectMeta{ @@ -198,29 +177,3 @@ func makeTestClusterBootstrapConfig(opts ...func(*capiv1alpha2.ClusterBootstrapC } return bc } - -func makeTestClient(t *testing.T, objs ...runtime.Object) client.Client { - _, client := makeTestClientAndScheme(t, objs...) - return client -} - -func makeTestClientAndScheme(t *testing.T, objs ...runtime.Object) (*runtime.Scheme, client.Client) { - t.Helper() - s := runtime.NewScheme() - test.AssertNoError(t, clientgoscheme.AddToScheme(s)) - test.AssertNoError(t, capiv1alpha2.AddToScheme(s)) - test.AssertNoError(t, batchv1.AddToScheme(s)) - test.AssertNoError(t, gitopsv1alpha1.AddToScheme(s)) - return s, fake.NewClientBuilder().WithScheme(s).WithRuntimeObjects(objs...).Build() -} - -func makeTestVolume(name, secretName string) corev1.Volume { - return corev1.Volume{ - Name: name, - VolumeSource: corev1.VolumeSource{ - Secret: &corev1.SecretVolumeSource{ - SecretName: secretName, - }, - }, - } -} diff --git a/controllers/cluster.go b/controllers/cluster.go index 54dd85f..84e211e 100644 --- a/controllers/cluster.go +++ b/controllers/cluster.go @@ -5,8 +5,14 @@ import ( "fmt" "github.com/go-logr/logr" + gitopsv1alpha1 "github.com/weaveworks/cluster-controller/api/v1alpha1" corev1 "k8s.io/api/core/v1" + metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" + "k8s.io/apimachinery/pkg/labels" + "k8s.io/apimachinery/pkg/types" + "k8s.io/client-go/tools/clientcmd" "sigs.k8s.io/controller-runtime/pkg/client" + "sigs.k8s.io/controller-runtime/pkg/client/apiutil" "sigs.k8s.io/controller-runtime/pkg/log" ) @@ -68,3 +74,71 @@ func listReadyNodesWithLabel(ctx context.Context, logger logr.Logger, cl client. } return readiness, nil } + +func matchCluster(cluster *gitopsv1alpha1.GitopsCluster, labelSelector metav1.LabelSelector) bool { + selector, err := metav1.LabelSelectorAsSelector(&labelSelector) + if err != nil { + return false + } + labelSet := labels.Set(cluster.GetLabels()) + if selector.Empty() { + return false + } + return selector.Matches(labelSet) +} + +func clientForCluster(ctx context.Context, cl client.Client, configParser ConfigParser, name types.NamespacedName) (client.Client, error) { + kubeConfigBytes, err := getKubeConfig(ctx, cl, name) + if err != nil { + return nil, err + } + + client, err := configParser(kubeConfigBytes) + if err != nil { + return nil, fmt.Errorf("getting client for cluster %s: %w", name, err) + } + return client, nil +} + +func getKubeConfig(ctx context.Context, cl client.Client, cluster types.NamespacedName) ([]byte, error) { + secretName := types.NamespacedName{ + Namespace: cluster.Namespace, + Name: cluster.Name + "-kubeconfig", + } + + var secret corev1.Secret + if err := cl.Get(ctx, secretName, &secret); err != nil { + return nil, fmt.Errorf("unable to read KubeConfig secret %q error: %w", secretName, err) + } + + var kubeConfig []byte + for k := range secret.Data { + if k == "value" || k == "value.yaml" { + kubeConfig = secret.Data[k] + break + } + } + + if len(kubeConfig) == 0 { + return nil, fmt.Errorf("KubeConfig secret %q doesn't contain a 'value' key ", secretName) + } + + return kubeConfig, nil +} + +func kubeConfigBytesToClient(b []byte) (client.Client, error) { + restConfig, err := clientcmd.RESTConfigFromKubeConfig(b) + if err != nil { + return nil, fmt.Errorf("failed to parse KubeConfig from secret: %w", err) + } + restMapper, err := apiutil.NewDynamicRESTMapper(restConfig) + if err != nil { + return nil, fmt.Errorf("failed to create RESTMapper from config: %w", err) + } + + client, err := client.New(restConfig, client.Options{Mapper: restMapper}) + if err != nil { + return nil, fmt.Errorf("failed to create a client from config: %w", err) + } + return client, nil +} diff --git a/controllers/clusterbootstrapconfig_controller.go b/controllers/clusterbootstrapconfig_controller.go index f4e44ae..14d7465 100644 --- a/controllers/clusterbootstrapconfig_controller.go +++ b/controllers/clusterbootstrapconfig_controller.go @@ -24,17 +24,13 @@ import ( "github.com/fluxcd/pkg/runtime/conditions" gitopsv1alpha1 "github.com/weaveworks/cluster-controller/api/v1alpha1" - corev1 "k8s.io/api/core/v1" apierrors "k8s.io/apimachinery/pkg/api/errors" metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" - "k8s.io/apimachinery/pkg/labels" "k8s.io/apimachinery/pkg/runtime" "k8s.io/apimachinery/pkg/types" "k8s.io/apimachinery/pkg/util/sets" - "k8s.io/client-go/tools/clientcmd" ctrl "sigs.k8s.io/controller-runtime" "sigs.k8s.io/controller-runtime/pkg/client" - "sigs.k8s.io/controller-runtime/pkg/client/apiutil" "sigs.k8s.io/controller-runtime/pkg/handler" "sigs.k8s.io/controller-runtime/pkg/log" "sigs.k8s.io/controller-runtime/pkg/source" @@ -46,7 +42,7 @@ import ( type ClusterBootstrapConfigReconciler struct { client.Client Scheme *runtime.Scheme - configParser func(b []byte) (client.Client, error) + configParser ConfigParser } // NewClusterBootstrapConfigReconcielr creates and returns a configured @@ -89,7 +85,7 @@ func (r *ClusterBootstrapConfigReconciler) Reconcile(ctx context.Context, req ct for _, cluster := range clusters { if clusterBootstrapConfig.Spec.RequireClusterReady { clusterName := types.NamespacedName{Name: cluster.GetName(), Namespace: cluster.GetNamespace()} - clusterClient, err := r.clientForCluster(ctx, clusterName) + clusterClient, err := clientForCluster(ctx, r.Client, r.configParser, clusterName) if err != nil { if apierrors.IsNotFound(err) { logger.Info("waiting for cluster access secret to be available") @@ -215,85 +211,17 @@ func (r *ClusterBootstrapConfigReconciler) clusterToClusterBootstrapConfig(o cli return nil } - labels := labels.Set(cluster.GetLabels()) for i := range resourceList.Items { rs := &resourceList.Items[i] - selector, err := metav1.LabelSelectorAsSelector(&rs.Spec.ClusterSelector) - if err != nil { - return nil - } - - // If a ClusterResourceSet has a nil or empty selector, it should match nothing, not everything. - if selector.Empty() { - return nil - } - - if !selector.Matches(labels) { + if !matchCluster(cluster, rs.Spec.ClusterSelector) { continue } - name := client.ObjectKey{Namespace: rs.Namespace, Name: rs.Name} result = append(result, ctrl.Request{NamespacedName: name}) } return result } -func (r *ClusterBootstrapConfigReconciler) clientForCluster(ctx context.Context, name types.NamespacedName) (client.Client, error) { - kubeConfigBytes, err := r.getKubeConfig(ctx, name) - if err != nil { - return nil, err - } - - client, err := r.configParser(kubeConfigBytes) - if err != nil { - return nil, fmt.Errorf("getting client for cluster %s: %w", name, err) - } - return client, nil -} - -func (r *ClusterBootstrapConfigReconciler) getKubeConfig(ctx context.Context, cluster types.NamespacedName) ([]byte, error) { - secretName := types.NamespacedName{ - Namespace: cluster.Namespace, - Name: cluster.Name + "-kubeconfig", - } - - var secret corev1.Secret - if err := r.Client.Get(ctx, secretName, &secret); err != nil { - return nil, fmt.Errorf("unable to read KubeConfig secret %q error: %w", secretName, err) - } - - var kubeConfig []byte - for k := range secret.Data { - if k == "value" || k == "value.yaml" { - kubeConfig = secret.Data[k] - break - } - } - - if len(kubeConfig) == 0 { - return nil, fmt.Errorf("KubeConfig secret %q doesn't contain a 'value' key ", secretName) - } - - return kubeConfig, nil -} - -func kubeConfigBytesToClient(b []byte) (client.Client, error) { - restConfig, err := clientcmd.RESTConfigFromKubeConfig(b) - if err != nil { - return nil, fmt.Errorf("failed to parse KubeConfig from secret: %w", err) - } - restMapper, err := apiutil.NewDynamicRESTMapper(restConfig) - if err != nil { - return nil, fmt.Errorf("failed to create RESTMapper from config: %w", err) - } - - client, err := client.New(restConfig, client.Options{Mapper: restMapper}) - if err != nil { - return nil, fmt.Errorf("failed to create a client from config: %w", err) - } - return client, nil -} - func isProvisioned(from conditions.Getter) bool { return conditions.IsTrue(from, gitopsv1alpha1.ClusterProvisionedCondition) } diff --git a/controllers/clusterbootstrapconfig_controller_test.go b/controllers/clusterbootstrapconfig_controller_test.go index 05d3f50..956d28b 100644 --- a/controllers/clusterbootstrapconfig_controller_test.go +++ b/controllers/clusterbootstrapconfig_controller_test.go @@ -11,7 +11,6 @@ import ( batchv1 "k8s.io/api/batch/v1" corev1 "k8s.io/api/core/v1" metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" - "k8s.io/apimachinery/pkg/runtime" "k8s.io/apimachinery/pkg/types" "k8s.io/client-go/tools/clientcmd" clientcmdapi "k8s.io/client-go/tools/clientcmd/api" @@ -534,42 +533,6 @@ func Test_kubeConfigBytesToClient_with_invalidkubeconfig(t *testing.T) { } } -func makeReadyCondition() metav1.Condition { - return metav1.Condition{ - Type: "Ready", - Status: metav1.ConditionTrue, - } -} - -func makeClusterProvisionedCondition() metav1.Condition { - return metav1.Condition{ - Type: gitopsv1alpha1.ClusterProvisionedCondition, - Status: metav1.ConditionTrue, - } -} - -func makeNotReadyCondition() metav1.Condition { - return metav1.Condition{ - Type: "Ready", - Status: metav1.ConditionFalse, - } -} - -func makeTestReconciler(t *testing.T, objs ...runtime.Object) *ClusterBootstrapConfigReconciler { - s, tc := makeTestClientAndScheme(t, objs...) - return NewClusterBootstrapConfigReconciler(tc, s) -} - -func makeTestSecret(name types.NamespacedName, data map[string][]byte) *corev1.Secret { - return &corev1.Secret{ - ObjectMeta: metav1.ObjectMeta{ - Namespace: name.Namespace, - Name: name.Name, - }, - Data: data, - } -} - func assertNoJobsCreated(t *testing.T, cl client.Client) { var jobs batchv1.JobList if err := cl.List(context.TODO(), &jobs, client.InNamespace(testNamespace)); err != nil { diff --git a/controllers/common.go b/controllers/common.go new file mode 100644 index 0000000..2921345 --- /dev/null +++ b/controllers/common.go @@ -0,0 +1,7 @@ +package controllers + +import ( + "sigs.k8s.io/controller-runtime/pkg/client" +) + +type ConfigParser func(b []byte) (client.Client, error) diff --git a/controllers/secretsync_controller.go b/controllers/secretsync_controller.go new file mode 100644 index 0000000..da7bce6 --- /dev/null +++ b/controllers/secretsync_controller.go @@ -0,0 +1,313 @@ +/* +Copyright 2021. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package controllers + +import ( + "context" + "fmt" + "time" + + "github.com/fluxcd/pkg/runtime/conditions" + "github.com/fluxcd/pkg/runtime/patch" + gitopsv1alpha1 "github.com/weaveworks/cluster-controller/api/v1alpha1" + corev1 "k8s.io/api/core/v1" + v1 "k8s.io/api/core/v1" + apierrors "k8s.io/apimachinery/pkg/api/errors" + metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" + "k8s.io/apimachinery/pkg/fields" + "k8s.io/apimachinery/pkg/runtime" + "k8s.io/apimachinery/pkg/types" + ctrl "sigs.k8s.io/controller-runtime" + "sigs.k8s.io/controller-runtime/pkg/builder" + "sigs.k8s.io/controller-runtime/pkg/client" + "sigs.k8s.io/controller-runtime/pkg/handler" + "sigs.k8s.io/controller-runtime/pkg/log" + "sigs.k8s.io/controller-runtime/pkg/predicate" + "sigs.k8s.io/controller-runtime/pkg/reconcile" + "sigs.k8s.io/controller-runtime/pkg/source" + + "github.com/weaveworks/cluster-bootstrap-controller/api/v1alpha2" + capiv1alpha2 "github.com/weaveworks/cluster-bootstrap-controller/api/v1alpha2" +) + +const ( + secretRefIndexKey = "spec.secretRef" + clusterReadinessRequeue = time.Duration(1 * time.Minute) +) + +// SecretSyncReconciler reconciles a SecretSync object +type SecretSyncReconciler struct { + client.Client + Scheme *runtime.Scheme + configParser ConfigParser +} + +// NewSecretSyncReconciler creates and returns a configured +// reconciler ready for use. +func NewSecretSyncReconciler(c client.Client, s *runtime.Scheme) *SecretSyncReconciler { + return &SecretSyncReconciler{ + Client: c, + Scheme: s, + configParser: kubeConfigBytesToClient, + } +} + +//+kubebuilder:rbac:groups=capi.weave.works,resources=secretsyncs,verbs=get;list;watch;create;update;patch;delete +//+kubebuilder:rbac:groups=capi.weave.works,resources=secretsyncs/status,verbs=get;update;patch +//+kubebuilder:rbac:groups=capi.weave.works,resources=secretsyncs/finalizers,verbs=update +//+kubebuilder:rbac:groups=cluster.x-k8s.io,resources=clusters,verbs=get;list;watch;update;patch +//+kubebuilder:rbac:groups="",resources=secrets,verbs=get;list;watch +//+kubebuilder:rbac:groups="gitops.weave.works",resources=gitopsclusters,verbs=get;watch;list;patch + +func (r *SecretSyncReconciler) Reconcile(ctx context.Context, req ctrl.Request) (ctrl.Result, error) { + logger := log.FromContext(ctx) + + var secretSync capiv1alpha2.SecretSync + if err := r.Client.Get(ctx, req.NamespacedName, &secretSync); err != nil { + return ctrl.Result{}, client.IgnoreNotFound(err) + } + + secretName := types.NamespacedName{ + Name: secretSync.Spec.SecretRef.Name, + Namespace: req.Namespace, + } + var secret corev1.Secret + if err := r.Get(ctx, secretName, &secret); err != nil { + return ctrl.Result{}, client.IgnoreNotFound(err) + } + + if !secret.DeletionTimestamp.IsZero() { + logger.Info("skipping secret", "secret", secret.Name, "namespace", secret.Namespace, "reason", "Deleted") + return ctrl.Result{}, nil + } + + selector, err := metav1.LabelSelectorAsSelector(&secretSync.Spec.ClusterSelector) + if err != nil { + return ctrl.Result{}, fmt.Errorf("failed to convert selector: %w", err) + } + + if selector.Empty() { + logger.Info("empty cluster selector: no clusters are selected") + return ctrl.Result{}, nil + } + + clusters := &gitopsv1alpha1.GitopsClusterList{} + if err := r.Client.List(ctx, clusters, client.InNamespace(req.Namespace), client.MatchingLabelsSelector{Selector: selector}); err != nil { + return ctrl.Result{}, fmt.Errorf("failed to list clusters: %w", err) + } + + var requeue bool + + patchHelper, err := patch.NewHelper(&secretSync, r.Client) + if err != nil { + return ctrl.Result{}, err + } + + for i := range clusters.Items { + cluster := clusters.Items[i] + + if secretSync.Status.GetClusterSecretVersion(cluster.Name) == secret.ResourceVersion { + logger.Info("skipping cluster", "cluster", cluster.Name, "reason", "Synced") + continue + } + + if !cluster.DeletionTimestamp.IsZero() { + logger.Info("skipping cluster", "cluster", cluster.Name, "reason", "Deleted") + continue + } + + if !conditions.IsReady(&cluster) { + logger.Info("skipping cluster", "cluster", cluster.Name, "reason", "NotReady") + requeue = true + continue + } + + clusterName := client.ObjectKeyFromObject(&cluster) + clusterClient, err := clientForCluster(ctx, r.Client, r.configParser, clusterName) + if err != nil { + if apierrors.IsNotFound(err) { + logger.Info("waiting for cluster access secret to be available", "cluster", cluster.Name) + requeue = true + continue + } + return ctrl.Result{}, fmt.Errorf("failed to create client for cluster %s: %w", clusterName, err) + } + + ready, err := IsControlPlaneReady(ctx, clusterClient) + if err != nil { + logger.Error(err, "failed to check readiness of cluster", "cluster", cluster.Name) + continue + } + + if !ready { + logger.Info("waiting for control plane to be ready", "cluster", cluster.Name) + requeue = true + continue + } + + if err := r.syncSecret(ctx, secret, clusterClient, secretSync.Spec.TargetNamespace); err != nil { + logger.Error(err, "failed to sync secret", "cluster", cluster.Name, "secret", secret.Name) + continue + } + + secretSync.Status.SetClusterSecretVersion(cluster.Name, secret.ResourceVersion) + } + + if err := patchHelper.Patch(ctx, &secretSync); err != nil { + return ctrl.Result{}, fmt.Errorf("failed to patch object status: %w", err) + } + + if requeue { + return ctrl.Result{RequeueAfter: clusterReadinessRequeue}, nil + } + + return ctrl.Result{}, nil +} + +// syncSecret sync secret from management cluster to leaf cluster +func (r *SecretSyncReconciler) syncSecret(ctx context.Context, secret v1.Secret, cl client.Client, targetNamespace string) error { + namespace := secret.Namespace + if targetNamespace != "" { + namespace = targetNamespace + } + + if err := r.createNamespace(ctx, cl, namespace); err != nil { + return err + } + + newSecret := corev1.Secret{ + ObjectMeta: metav1.ObjectMeta{ + Name: secret.Name, + GenerateName: secret.GenerateName, + Namespace: namespace, + Labels: secret.Labels, + Annotations: secret.Annotations, + }, + Type: secret.Type, + Immutable: secret.Immutable, + Data: secret.Data, + } + + if err := cl.Create(ctx, &newSecret); err != nil { + if apierrors.IsAlreadyExists(err) { + if err := cl.Update(ctx, &newSecret); err != nil { + return fmt.Errorf("failed to update secret: %w", err) + } + } else { + return fmt.Errorf("failed to create secret: %w", err) + } + } + + return nil +} + +// createNamespace create secret's namespace if it doesn't exists +func (r *SecretSyncReconciler) createNamespace(ctx context.Context, cl client.Client, name string) error { + namespace := v1.Namespace{} + namespace.SetName(name) + + if err := cl.Create(ctx, &namespace); err != nil { + if !apierrors.IsAlreadyExists(err) { + return fmt.Errorf("failed to create namespace %s: %w", name, err) + } + } + + return nil +} + +// SetupWithManager sets up the controller with the Manager. +func (r *SecretSyncReconciler) SetupWithManager(mgr ctrl.Manager) error { + err := mgr.GetFieldIndexer().IndexField(context.Background(), &v1alpha2.SecretSync{}, secretRefIndexKey, func(obj client.Object) []string { + secretSync, ok := obj.(*v1alpha2.SecretSync) + if !ok { + return nil + } + return []string{secretSync.Spec.SecretRef.Name} + }) + + if err != nil { + return err + } + + return ctrl.NewControllerManagedBy(mgr). + For(&capiv1alpha2.SecretSync{}). + Watches( + &source.Kind{Type: &gitopsv1alpha1.GitopsCluster{}}, + handler.EnqueueRequestsFromMapFunc(r.clusterHandler), + ). + Watches( + &source.Kind{Type: &v1.Secret{}}, + handler.EnqueueRequestsFromMapFunc(r.secretHandler), + builder.WithPredicates(predicate.ResourceVersionChangedPredicate{}), + ). + Complete(r) +} + +// clusterHandler handler for GitOpsCluster objects +func (r *SecretSyncReconciler) clusterHandler(obj client.Object) []ctrl.Request { + cluster, ok := obj.(*gitopsv1alpha1.GitopsCluster) + if !ok { + return nil + } + + var resources capiv1alpha2.SecretSyncList + if err := r.Client.List(context.Background(), &resources, client.InNamespace(cluster.Namespace)); err != nil { + log.Log.Error(err, "failed to list secret syncs") + return nil + } + + result := []ctrl.Request{} + for _, resource := range resources.Items { + if !matchCluster(cluster, resource.Spec.ClusterSelector) { + continue + } + result = append(result, ctrl.Request{ + NamespacedName: types.NamespacedName{ + Name: resource.Name, + Namespace: resource.Namespace, + }, + }) + } + return result +} + +// clusterHandler handler for Secret objects +func (r *SecretSyncReconciler) secretHandler(obj client.Object) []ctrl.Request { + opts := client.ListOptions{ + FieldSelector: fields.OneTermEqualSelector(secretRefIndexKey, obj.GetName()), + Namespace: obj.GetNamespace(), + } + + var resources v1alpha2.SecretSyncList + ctx := context.Background() + if err := r.List(ctx, &resources, &opts); err != nil { + log.Log.Error(err, "failed to list secret syncs") + return nil + } + + var requests []reconcile.Request + for _, item := range resources.Items { + requests = append(requests, reconcile.Request{ + NamespacedName: types.NamespacedName{ + Name: item.Name, + Namespace: item.Namespace, + }, + }) + } + + return requests +} diff --git a/controllers/secretsync_controller_test.go b/controllers/secretsync_controller_test.go new file mode 100644 index 0000000..c3afa8e --- /dev/null +++ b/controllers/secretsync_controller_test.go @@ -0,0 +1,189 @@ +package controllers + +import ( + "bytes" + "context" + "fmt" + "testing" + + capiv1alpha2 "github.com/weaveworks/cluster-bootstrap-controller/api/v1alpha2" + gitopsv1alpha1 "github.com/weaveworks/cluster-controller/api/v1alpha1" + corev1 "k8s.io/api/core/v1" + v1 "k8s.io/api/core/v1" + metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" + "k8s.io/apimachinery/pkg/types" + ctrl "sigs.k8s.io/controller-runtime" + "sigs.k8s.io/controller-runtime/pkg/client" +) + +func TestSecretSync(t *testing.T) { + clusterA, clusterASecret, clusterAClient := makeReadyTestCluster(t, "a") + clusterB, clusterBSecret, clusterBClient := makeReadyTestCluster(t, "b") + + secretA := makeTestSecret(types.NamespacedName{ + Name: "secret-a", + Namespace: "default", + }, map[string][]byte{"value": []byte("a")}) + + secretB := makeTestSecret(types.NamespacedName{ + Name: "secret-b", + Namespace: "default", + }, map[string][]byte{"value": []byte("b")}) + + secretSyncA := makeSecretSync( + "secretsync-a", + secretA.GetNamespace(), + secretA.GetName(), + "ns-a", + map[string]string{"environment": "a"}, + ) + + secretSyncB := makeSecretSync( + "secretsync-b", + secretB.GetNamespace(), + secretB.GetName(), + "ns-b", + map[string]string{"environment": "b"}, + ) + + sc, cl := makeTestClientAndScheme( + t, clusterA, clusterB, + clusterASecret, clusterBSecret, + secretA, secretB, + secretSyncA, secretSyncB, + ) + + reconciler := NewSecretSyncReconciler(cl, sc) + reconciler.configParser = func(b []byte) (client.Client, error) { + clusters := map[string]client.Client{ + "a": clusterAClient, + "b": clusterBClient, + } + return clusters[string(b)], nil + } + + t.Run("create SecretSync a to sync secret a to leaf cluster a in namespace ns-a", func(t *testing.T) { + if _, err := reconciler.Reconcile(context.TODO(), ctrl.Request{NamespacedName: types.NamespacedName{ + Name: secretSyncA.GetName(), + Namespace: secretSyncA.GetNamespace(), + }}); err != nil { + t.Fatal(err) + } + + var secret v1.Secret + if err := clusterAClient.Get(context.TODO(), client.ObjectKey{Name: "secret-a", Namespace: "ns-a"}, &secret); err != nil { + t.Fatal(err) + } + + var secretSync capiv1alpha2.SecretSync + if err := cl.Get(context.TODO(), client.ObjectKeyFromObject(secretSyncA), &secretSync); err != nil { + t.Fatal(err) + } + + if _, ok := secretSync.Status.SecretVersions[clusterA.Name]; !ok { + t.Fatalf("secretsync a status is not updated") + } + + var secretb v1.Secret + if err := clusterAClient.Get(context.TODO(), client.ObjectKey{Name: "secret-b", Namespace: "ns-b"}, &secretb); err == nil { + t.Fatal("secret b found in cluster a") + } + }) + + t.Run("create SecretSync a to sync secret a to leaf cluster a in namespace ns-a", func(t *testing.T) { + if _, err := reconciler.Reconcile(context.TODO(), ctrl.Request{NamespacedName: types.NamespacedName{ + Name: secretSyncB.GetName(), + Namespace: secretSyncB.GetNamespace(), + }}); err != nil { + t.Fatal(err) + } + + var secret v1.Secret + if err := clusterBClient.Get(context.TODO(), client.ObjectKey{Name: "secret-b", Namespace: "ns-b"}, &secret); err != nil { + t.Fatal(err) + } + + var secretSync capiv1alpha2.SecretSync + if err := cl.Get(context.TODO(), client.ObjectKeyFromObject(secretSyncB), &secretSync); err != nil { + t.Fatal(err) + } + + if _, ok := secretSync.Status.SecretVersions[clusterB.Name]; !ok { + t.Fatalf("secretsync a status is not updated") + } + + var secreta v1.Secret + if err := clusterBClient.Get(context.TODO(), client.ObjectKey{Name: "secret-a", Namespace: "ns-a"}, &secreta); err == nil { + t.Fatal("secret a found in cluster b") + } + }) + + t.Run("update secret a. Secret a on cluster a should be updated too", func(t *testing.T) { + secretA.Data["value"] = []byte("aaaa") + if err := cl.Update(context.TODO(), secretA); err != nil { + t.Fatal(err) + } + + if _, err := reconciler.Reconcile(context.TODO(), ctrl.Request{NamespacedName: types.NamespacedName{ + Name: secretSyncA.GetName(), + Namespace: secretSyncA.GetNamespace(), + }}); err != nil { + t.Fatal(err) + } + + var secret v1.Secret + if err := clusterAClient.Get(context.TODO(), client.ObjectKey{Name: "secret-a", Namespace: "ns-a"}, &secret); err != nil { + t.Fatal(err) + } + if bytes.Compare(secret.Data["value"], []byte("aaaa")) != 0 { + t.Fatal("secret a is not update") + } + }) +} + +func makeSecretSync(name, namespace, secretName, targetNamespace string, selector map[string]string) *capiv1alpha2.SecretSync { + return &capiv1alpha2.SecretSync{ + ObjectMeta: metav1.ObjectMeta{ + Name: name, + Namespace: namespace, + }, + Spec: capiv1alpha2.SecretSyncSpec{ + ClusterSelector: metav1.LabelSelector{ + MatchLabels: selector, + }, + SecretRef: v1.LocalObjectReference{ + Name: secretName, + }, + TargetNamespace: targetNamespace, + }, + } +} + +func makeReadyTestCluster(t *testing.T, key string) (*gitopsv1alpha1.GitopsCluster, *v1.Secret, client.Client) { + cluster := makeTestCluster(func(c *gitopsv1alpha1.GitopsCluster) { + c.Name = fmt.Sprintf("cluster-%s", key) + c.Namespace = corev1.NamespaceDefault + c.SetLabels(map[string]string{ + "environment": key, + }) + c.Status.Conditions = append(c.Status.Conditions, makeReadyCondition()) + }) + + nodeCondition := corev1.NodeCondition{ + Type: "Ready", + Status: "True", + LastHeartbeatTime: metav1.Now(), + LastTransitionTime: metav1.Now(), Reason: "KubeletReady", + Message: "kubelet is posting ready status"} + + readyNode := makeNode(map[string]string{"node-role.kubernetes.io/master": ""}, nodeCondition) + + secret := makeTestSecret(types.NamespacedName{ + Name: cluster.GetName() + "-kubeconfig", + Namespace: cluster.GetNamespace(), + }, map[string][]byte{"value": []byte(key)}) + + _, cl := makeTestClientAndScheme(t, readyNode) + + return cluster, secret, cl +} diff --git a/controllers/utils_test.go b/controllers/utils_test.go new file mode 100644 index 0000000..1240728 --- /dev/null +++ b/controllers/utils_test.go @@ -0,0 +1,100 @@ +package controllers + +import ( + "testing" + + gitopsv1alpha1 "github.com/weaveworks/cluster-controller/api/v1alpha1" + batchv1 "k8s.io/api/batch/v1" + corev1 "k8s.io/api/core/v1" + metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" + "k8s.io/apimachinery/pkg/runtime" + "k8s.io/apimachinery/pkg/types" + clientgoscheme "k8s.io/client-go/kubernetes/scheme" + "sigs.k8s.io/controller-runtime/pkg/client" + "sigs.k8s.io/controller-runtime/pkg/client/fake" + + capiv1alpha2 "github.com/weaveworks/cluster-bootstrap-controller/api/v1alpha2" + "github.com/weaveworks/cluster-bootstrap-controller/test" +) + +const ( + testConfigName = "test-config" + testClusterName = "test-cluster" + testNamespace = "testing" +) + +func makeTestCluster(opts ...func(*gitopsv1alpha1.GitopsCluster)) *gitopsv1alpha1.GitopsCluster { + c := &gitopsv1alpha1.GitopsCluster{ + ObjectMeta: metav1.ObjectMeta{ + Name: testClusterName, + Namespace: testNamespace, + }, + Spec: gitopsv1alpha1.GitopsClusterSpec{}, + } + for _, o := range opts { + o(c) + } + return c +} + +func makeReadyCondition() metav1.Condition { + return metav1.Condition{ + Type: "Ready", + Status: metav1.ConditionTrue, + } +} + +func makeClusterProvisionedCondition() metav1.Condition { + return metav1.Condition{ + Type: gitopsv1alpha1.ClusterProvisionedCondition, + Status: metav1.ConditionTrue, + } +} + +func makeNotReadyCondition() metav1.Condition { + return metav1.Condition{ + Type: "Ready", + Status: metav1.ConditionFalse, + } +} + +func makeTestReconciler(t *testing.T, objs ...runtime.Object) *ClusterBootstrapConfigReconciler { + s, tc := makeTestClientAndScheme(t, objs...) + return NewClusterBootstrapConfigReconciler(tc, s) +} + +func makeTestSecret(name types.NamespacedName, data map[string][]byte) *corev1.Secret { + return &corev1.Secret{ + ObjectMeta: metav1.ObjectMeta{ + Namespace: name.Namespace, + Name: name.Name, + }, + Data: data, + } +} + +func makeTestClient(t *testing.T, objs ...runtime.Object) client.Client { + _, client := makeTestClientAndScheme(t, objs...) + return client +} + +func makeTestClientAndScheme(t *testing.T, objs ...runtime.Object) (*runtime.Scheme, client.Client) { + t.Helper() + s := runtime.NewScheme() + test.AssertNoError(t, clientgoscheme.AddToScheme(s)) + test.AssertNoError(t, capiv1alpha2.AddToScheme(s)) + test.AssertNoError(t, batchv1.AddToScheme(s)) + test.AssertNoError(t, gitopsv1alpha1.AddToScheme(s)) + return s, fake.NewClientBuilder().WithScheme(s).WithRuntimeObjects(objs...).Build() +} + +func makeTestVolume(name, secretName string) corev1.Volume { + return corev1.Volume{ + Name: name, + VolumeSource: corev1.VolumeSource{ + Secret: &corev1.SecretVolumeSource{ + SecretName: secretName, + }, + }, + } +} diff --git a/main.go b/main.go index e924cc0..b1d6b96 100644 --- a/main.go +++ b/main.go @@ -87,6 +87,14 @@ func main() { setupLog.Error(err, "unable to create controller", "controller", "ClusterBootstrapConfig") os.Exit(1) } + + if err := controllers.NewSecretSyncReconciler( + mgr.GetClient(), + mgr.GetScheme(), + ).SetupWithManager(mgr); err != nil { + setupLog.Error(err, "unable to create controller", "controller", "SecretSync") + os.Exit(1) + } //+kubebuilder:scaffold:builder if err := mgr.AddHealthzCheck("healthz", healthz.Ping); err != nil {