From a0c6563226ef9f6c97672d778edf50dc15ac1a8c Mon Sep 17 00:00:00 2001 From: Steakley Date: Thu, 22 Jul 2021 16:24:51 -0700 Subject: [PATCH] add endpoint url to namespace annotations and tests --- apis/core/v1alpha1/annotations.go | 8 +++++++ pkg/runtime/adoption_reconciler.go | 21 +++++++++++++++++- pkg/runtime/cache/namespace.go | 34 ++++++++++++++++++++++++----- pkg/runtime/cache/namespace_test.go | 10 +++++++++ pkg/runtime/reconciler.go | 27 +++++++++++++++++++---- 5 files changed, 90 insertions(+), 10 deletions(-) diff --git a/apis/core/v1alpha1/annotations.go b/apis/core/v1alpha1/annotations.go index 5a7f5ef..2314844 100644 --- a/apis/core/v1alpha1/annotations.go +++ b/apis/core/v1alpha1/annotations.go @@ -51,4 +51,12 @@ const ( // injected by POD IRSA, to decide in which region the resources should be // created. AnnotationDefaultRegion = AnnotationPrefix + "default-region" + // AnnotationEndpointURL is an annotation whose value is the identifier + // for the AWS endpoint in which the service controller will use to create + // its resources. If this annotation is set on a namespace, the Kubernetes user + // is indicating that the ACK service controller should create its resources using + // that specific endpoint. If this annotation is not set, ACK service controller + // will either use the default behavior of aws-sdk-go to create endpoints or + // aws-endpoint-url if it is set in controller binary flags and environment variables. + AnnotationEndpointURL = AnnotationPrefix + "endpoint-url" ) diff --git a/pkg/runtime/adoption_reconciler.go b/pkg/runtime/adoption_reconciler.go index fca7b7e..21428d2 100644 --- a/pkg/runtime/adoption_reconciler.go +++ b/pkg/runtime/adoption_reconciler.go @@ -115,9 +115,10 @@ func (r *adoptionReconciler) reconcile(req ctrlrt.Request) error { acctID := r.getOwnerAccountID(res) region := r.getRegion(res) roleARN := r.getRoleARN(acctID) + endpointURL := r.getEndpointURL(res) sess, err := r.sc.NewSession( - region, &r.cfg.EndpointURL, roleARN, + region, &endpointURL, roleARN, targetDescriptor.EmptyRuntimeObject().GetObjectKind().GroupVersionKind(), ) if err != nil { @@ -427,6 +428,24 @@ func (r *adoptionReconciler) getOwnerAccountID( return ackv1alpha1.AWSAccountID(r.cfg.AccountID) } +// getEndpointURL returns the AWS account that owns the supplied resource. +// We look for the namespace associated endpoint url, if that is set we use it. +// Otherwise if none of these annotations are set we use the endpoint url specified +// in the configuration +func (r *adoptionReconciler) getEndpointURL( + res *ackv1alpha1.AdoptedResource, +) string { + // look for endpoint url in the namespace annotations + namespace := res.GetNamespace() + endpointURL, ok := r.cache.Namespaces.GetEndpointURL(namespace) + if ok { + return endpointURL + } + + // use controller configuration + return r.cfg.EndpointURL +} + // getRoleARN return the Role ARN that should be assumed in order to manage // the resources. func (r *adoptionReconciler) getRoleARN( diff --git a/pkg/runtime/cache/namespace.go b/pkg/runtime/cache/namespace.go index 56a64dd..459d6f6 100644 --- a/pkg/runtime/cache/namespace.go +++ b/pkg/runtime/cache/namespace.go @@ -31,6 +31,8 @@ type namespaceInfo struct { defaultRegion string // services.k8s.aws/owner-account-id Annotation ownerAccountID string + // services.k8s.aws/endpoint-url Annotation + endpointURL string } // getDefaultRegion returns the default region value @@ -49,15 +51,23 @@ func (n *namespaceInfo) getOwnerAccountID() string { return n.ownerAccountID } +// getEndpointURL returns the namespace Endpoint URL +func (n *namespaceInfo) getEndpointURL() string { + if n == nil { + return "" + } + return n.endpointURL +} + // NamespaceCache is responsible of keeping track of namespaces // annotations, and caching those related to the ACK controller. type NamespaceCache struct { sync.RWMutex log logr.Logger - // Provide a namespace specifically to listen to. + // Provide a namespace specifically to listen to. // Provide empty string to listen to all namespaces except kube-system and kube-public. - watchNamespace string + watchNamespace string // Namespace informer informer k8scache.SharedInformer @@ -81,12 +91,12 @@ func NewNamespaceCache(clientset kubernetes.Interface, log logr.Logger, watchNam } } -// Check if the provided namespace should be listened to or not +// Check if the provided namespace should be listened to or not func isWatchNamespace(raw interface{}, watchNamespace string) bool { object, ok := raw.(*corev1.Namespace) if !ok { return false - } + } if watchNamespace != "" { return watchNamespace == object.ObjectMeta.Name @@ -143,8 +153,18 @@ func (c *NamespaceCache) GetOwnerAccountID(namespace string) (string, bool) { return "", false } +// GetEndpointURL returns the endpoint URL if it exists +func (c *NamespaceCache) GetEndpointURL(namespace string) (string, bool) { + info, ok := c.getNamespaceInfo(namespace) + if ok { + e := info.getEndpointURL() + return e, e != "" + } + return "", false +} + // getNamespaceInfo reads a namespace cached annotations and -// return a given namespace default aws region and owner account id. +// return a given namespace default aws region, owner account id and endpoint url. // This function is thread safe. func (c *NamespaceCache) getNamespaceInfo(ns string) (*namespaceInfo, bool) { c.RLock() @@ -166,6 +186,10 @@ func (c *NamespaceCache) setNamespaceInfoFromK8sObject(ns *corev1.Namespace) { if ok { nsInfo.ownerAccountID = OwnerAccountID } + EndpointURL, ok := nsa[ackv1alpha1.AnnotationEndpointURL] + if ok { + nsInfo.endpointURL = EndpointURL + } c.Lock() defer c.Unlock() c.namespaceInfos[ns.ObjectMeta.Name] = nsInfo diff --git a/pkg/runtime/cache/namespace_test.go b/pkg/runtime/cache/namespace_test.go index 61d8ec3..f48334b 100644 --- a/pkg/runtime/cache/namespace_test.go +++ b/pkg/runtime/cache/namespace_test.go @@ -63,6 +63,7 @@ func TestNamespaceCache(t *testing.T) { Annotations: map[string]string{ ackv1alpha1.AnnotationDefaultRegion: "us-west-2", ackv1alpha1.AnnotationOwnerAccountID: "012345678912", + ackv1alpha1.AnnotationEndpointURL: "https://amazon-service.region.amazonaws.com", }, }, }, @@ -79,6 +80,10 @@ func TestNamespaceCache(t *testing.T) { require.True(t, ok) require.Equal(t, "012345678912", ownerAccountID) + endpointURL, ok := namespaceCache.GetEndpointURL("production") + require.True(t, ok) + require.Equal(t, "https://amazon-service.region.amazonaws.com", endpointURL) + // Test update events k8sClient.CoreV1().Namespaces().Update( context.Background(), @@ -88,6 +93,7 @@ func TestNamespaceCache(t *testing.T) { Annotations: map[string]string{ ackv1alpha1.AnnotationDefaultRegion: "us-est-1", ackv1alpha1.AnnotationOwnerAccountID: "21987654321", + ackv1alpha1.AnnotationEndpointURL: "https://amazon-other-service.region.amazonaws.com", }, }, }, @@ -104,6 +110,10 @@ func TestNamespaceCache(t *testing.T) { require.True(t, ok) require.Equal(t, "21987654321", ownerAccountID) + endpointURL, ok = namespaceCache.GetEndpointURL("production") + require.True(t, ok) + require.Equal(t, "https://amazon-other-service.region.amazonaws.com", endpointURL) + // Test delete events k8sClient.CoreV1().Namespaces().Delete( context.Background(), diff --git a/pkg/runtime/reconciler.go b/pkg/runtime/reconciler.go index 3f5477e..f159724 100644 --- a/pkg/runtime/reconciler.go +++ b/pkg/runtime/reconciler.go @@ -27,6 +27,7 @@ import ( "sigs.k8s.io/controller-runtime/pkg/client" ackv1alpha1 "github.com/aws-controllers-k8s/runtime/apis/core/v1alpha1" + ackcompare "github.com/aws-controllers-k8s/runtime/pkg/compare" ackcfg "github.com/aws-controllers-k8s/runtime/pkg/config" ackerr "github.com/aws-controllers-k8s/runtime/pkg/errors" ackmetrics "github.com/aws-controllers-k8s/runtime/pkg/metrics" @@ -34,7 +35,6 @@ import ( ackrtcache "github.com/aws-controllers-k8s/runtime/pkg/runtime/cache" ackrtlog "github.com/aws-controllers-k8s/runtime/pkg/runtime/log" acktypes "github.com/aws-controllers-k8s/runtime/pkg/types" - ackcompare "github.com/aws-controllers-k8s/runtime/pkg/compare" ) // reconciler describes a generic reconciler within ACK. @@ -145,8 +145,9 @@ func (r *resourceReconciler) Reconcile(req ctrlrt.Request) (ctrlrt.Result, error acctID := r.getOwnerAccountID(res) region := r.getRegion(res) roleARN := r.getRoleARN(acctID) + endpointURL := r.getEndpointURL(res) sess, err := r.sc.NewSession( - region, &r.cfg.EndpointURL, roleARN, + region, &endpointURL, roleARN, res.RuntimeObject().GetObjectKind().GroupVersionKind(), ) if err != nil { @@ -178,7 +179,6 @@ func (r *resourceReconciler) reconcile( if res.IsBeingDeleted() { return r.cleanup(ctx, rm, res) } - return r.Sync(ctx, rm, res) } @@ -294,7 +294,7 @@ func (r *resourceReconciler) Sync( "requeueing resource after finding resource synced condition false", ) return requeue.NeededAfter( - ackerr.TemporaryOutOfSync, requeue.DefaultRequeueAfterDuration) + ackerr.TemporaryOutOfSync, requeue.DefaultRequeueAfterDuration) } } } @@ -552,6 +552,25 @@ func (r *resourceReconciler) getRegion( return ackv1alpha1.AWSRegion(r.cfg.Region) } +// getEndpointURL returns the AWS account that owns the supplied resource. +// We look for the namespace associated endpoint url, if that is set we use it. +// Otherwise if none of these annotations are set we use the endpoint url specified +// in the configuration +func (r *resourceReconciler) getEndpointURL( + res acktypes.AWSResource, +) string { + + // look for endpoint url in the namespace annotations + namespace := res.MetaObject().GetNamespace() + endpointURL, ok := r.cache.Namespaces.GetEndpointURL(namespace) + if ok { + return endpointURL + } + + // use controller configuration EndpointURL + return r.cfg.EndpointURL +} + // NewReconciler returns a new reconciler object that func NewReconciler( sc acktypes.ServiceController,