From 13a7a8711c33a40bf5f8454615708f0d2afa85b0 Mon Sep 17 00:00:00 2001 From: Amine Hilaly Date: Tue, 1 Sep 2020 20:25:16 +0200 Subject: [PATCH] CARM Caches implementation - reconciler.getRegion now try lookup for CRs region annotation, namespace default region annotation before finally trying to use flag/env values. - Add pkg/runtime/cache.NamespacesCache object to simplify access to namespace annotations and keep the cache up to date whenever new changes are made. - Add pkg/runtime/cache.AccountsCache object to simplify access to CARM configmap and keep the cache up to date whenever new changes are made. - Add pkg/runtime/cache.Caches object to quickly create NamespacesCache and AccountsCache - Inject NAMESPACE environment variable to controller pods using k8s downward api --- apis/core/v1alpha1/annotations.go | 18 +- pkg/runtime/cache/account.go | 115 ++++++++++++ pkg/runtime/cache/account_test.go | 143 +++++++++++++++ pkg/runtime/cache/cache.go | 84 +++++++++ pkg/runtime/cache/namespace.go | 172 ++++++++++++++++++ pkg/runtime/cache/namespace_test.go | 118 ++++++++++++ pkg/runtime/reconciler.go | 41 ++++- pkg/runtime/registry.go | 2 +- pkg/runtime/service_controller_test.go | 10 +- .../config/controller/deployment.yaml.tpl | 5 + 10 files changed, 697 insertions(+), 11 deletions(-) create mode 100644 pkg/runtime/cache/account.go create mode 100644 pkg/runtime/cache/account_test.go create mode 100644 pkg/runtime/cache/cache.go create mode 100644 pkg/runtime/cache/namespace.go create mode 100644 pkg/runtime/cache/namespace_test.go diff --git a/apis/core/v1alpha1/annotations.go b/apis/core/v1alpha1/annotations.go index eef8f0e7eb..b26edfb56c 100644 --- a/apis/core/v1alpha1/annotations.go +++ b/apis/core/v1alpha1/annotations.go @@ -25,7 +25,7 @@ const ( // CR, that means the user expects the ACK service controller to create the // backend AWS service API resource. AnnotationARN = AnnotationPrefix + "arn" - // AnnotationOwnerAccountID is an annotation whoe value is the identifier + // AnnotationOwnerAccountID is an annotation whose value is the identifier // for the AWS account to which the resource belongs. If this annotation // is set on a CR, the Kubernetes user is indicating that the ACK service // controller should create/patch/delete the resource in the specified AWS @@ -36,4 +36,20 @@ const ( // TODO(jaypipes): Link to documentation on cross-account resource // management AnnotationOwnerAccountID = AnnotationPrefix + "owner-account-id" + // AnnotationRegion is an annotation whose value is the identifier for the + // the AWS region in which the resources should be created. If this annotation + // is set on a CR metadata, that means the user is indicating to the ACK service + // controller that the CR should be created on specific region. ACK service + // controller will not override the resource region if this annotation is set. + AnnotationRegion = AnnotationPrefix + "region" + // AnnotationDefaultRegion is an annotation whose value is the identifier + // for the default AWS region in which resources should be created. If this + // annotation is set on a namespace, the Kubernetes user is indicating that + // the ACK service controller should set the regions in which the resource + // should be created, if a region annotation is not set on the CR metadata. + // If this annotation - and AnnotationRegion - are not set, ACK service + // controllers look for controller binary flags and environment variables + // injected by POD IRSA, to decide in which region the resources should be + // created. + AnnotationDefaultRegion = AnnotationPrefix + "default-region" ) diff --git a/pkg/runtime/cache/account.go b/pkg/runtime/cache/account.go new file mode 100644 index 0000000000..4a2fcf2a71 --- /dev/null +++ b/pkg/runtime/cache/account.go @@ -0,0 +1,115 @@ +// Copyright Amazon.com Inc. or its affiliates. All Rights Reserved. +// +// Licensed under the Apache License, Version 2.0 (the "License"). You may +// not use this file except in compliance with the License. A copy of the +// License is located at +// +// http://aws.amazon.com/apache2.0/ +// +// or in the "license" file accompanying this file. This file is distributed +// on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either +// express or implied. See the License for the specific language governing +// permissions and limitations under the License. + +package cache + +import ( + "sync" + + "github.com/go-logr/logr" + corev1 "k8s.io/api/core/v1" + informersv1 "k8s.io/client-go/informers/core/v1" + kubernetes "k8s.io/client-go/kubernetes" + k8scache "k8s.io/client-go/tools/cache" +) + +const ( + // ACKRoleAccountMap is the name of the configmap map object storing + // all the AWS Account IDs associated with their AWS Role ARNs. + ACKRoleAccountMap = "ack-role-account-map" +) + +// AccountCache is responsible for caching the CARM configmap +// data. It is listening to all the events related to the CARM map and +// make the changes accordingly. +type AccountCache struct { + sync.RWMutex + + log logr.Logger + + // ConfigMap informer + informer k8scache.SharedInformer + roleARNs map[string]string +} + +// NewAccountCache makes a new AccountCache from a client.Interface +// and a logr.Logger +func NewAccountCache(clientset kubernetes.Interface, log logr.Logger) *AccountCache { + sharedInformer := informersv1.NewConfigMapInformer( + clientset, + currentNamespace, + informerResyncPeriod, + k8scache.Indexers{}, + ) + return &AccountCache{ + informer: sharedInformer, + log: log.WithName("AccountCache"), + roleARNs: make(map[string]string), + } +} + +// resourceMatchACKRoleAccountConfigMap verifies if a resource is +// the CARM configmap. It verifies the name, namespace and object type. +func resourceMatchACKRoleAccountsConfigMap(raw interface{}) bool { + object, ok := raw.(*corev1.ConfigMap) + return ok && object.ObjectMeta.Name == ACKRoleAccountMap +} + +// Run adds the default event handler functions to the SharedInformer and +// runs the informer to begin processing items. +func (c *AccountCache) Run(stopCh <-chan struct{}) { + c.informer.AddEventHandler(k8scache.ResourceEventHandlerFuncs{ + AddFunc: func(obj interface{}) { + if resourceMatchACKRoleAccountsConfigMap(obj) { + object := obj.(*corev1.ConfigMap).DeepCopy() + c.log.V(1).Info("ack-role-account-map has been created") + c.updateAccountRoleData(object.Data) + c.log.V(1).Info("cached ack-role-account-map data") + } + }, + UpdateFunc: func(orig, desired interface{}) { + if resourceMatchACKRoleAccountsConfigMap(desired) { + object := desired.(*corev1.ConfigMap).DeepCopy() + c.log.V(1).Info("ack-role-account-map has been updated") + //TODO(a-hilaly): compare data checksum before updating the cache + c.updateAccountRoleData(object.Data) + c.log.V(1).Info("cached ack-role-account-map data") + } + }, + DeleteFunc: func(obj interface{}) { + if resourceMatchACKRoleAccountsConfigMap(obj) { + c.log.V(1).Info("ack-role-account-map has been deleted") + newMap := make(map[string]string) + c.updateAccountRoleData(newMap) + c.log.V(1).Info("cleaned up role account map") + } + }, + }) + go c.informer.Run(stopCh) +} + +// GetAccountRoleARN queries the AWS accountID associated Role ARN +// from the cached CARM configmap. This function is thread safe. +func (c *AccountCache) GetAccountRoleARN(accountID string) (string, bool) { + c.RLock() + defer c.RUnlock() + roleARN, ok := c.roleARNs[accountID] + return roleARN, ok +} + +// updateAccountRoleData updates the CARM map. This function is thread safe. +func (c *AccountCache) updateAccountRoleData(data map[string]string) { + c.Lock() + defer c.Unlock() + c.roleARNs = data +} diff --git a/pkg/runtime/cache/account_test.go b/pkg/runtime/cache/account_test.go new file mode 100644 index 0000000000..3a3e783326 --- /dev/null +++ b/pkg/runtime/cache/account_test.go @@ -0,0 +1,143 @@ +// Copyright Amazon.com Inc. or its affiliates. All Rights Reserved. +// +// Licensed under the Apache License, Version 2.0 (the "License"). You may +// not use this file except in compliance with the License. A copy of the +// License is located at +// +// http://aws.amazon.com/apache2.0/ +// +// or in the "license" file accompanying this file. This file is distributed +// on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either +// express or implied. See the License for the specific language governing +// permissions and limitations under the License. + +package cache_test + +import ( + "context" + "testing" + "time" + + "github.com/stretchr/testify/require" + "go.uber.org/zap/zapcore" + corev1 "k8s.io/api/core/v1" + metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" + "k8s.io/apimachinery/pkg/watch" + k8sfake "k8s.io/client-go/kubernetes/fake" + k8stesting "k8s.io/client-go/testing" + ctrlrtzap "sigs.k8s.io/controller-runtime/pkg/log/zap" + + ackrtcache "github.com/aws/aws-controllers-k8s/pkg/runtime/cache" +) + +const ( + testNamespace = "ack-system" + + testAccount1 = "012345678912" + testAccountARN1 = "arn:aws:iam::012345678912:role/S3Access" + testAccount2 = "219876543210" + testAccountARN2 = "arn:aws:iam::012345678912:role/root" +) + +func TestAccountCache(t *testing.T) { + accountsMap1 := map[string]string{ + testAccount1: testAccountARN1, + } + + accountsMap2 := map[string]string{ + testAccount1: testAccountARN1, + testAccount2: testAccountARN2, + } + + // create a fake k8s client and a fake watcher + k8sClient := k8sfake.NewSimpleClientset() + watcher := watch.NewFake() + k8sClient.PrependWatchReactor("configMaps", k8stesting.DefaultWatchReactor(watcher, nil)) + + zapOptions := ctrlrtzap.Options{ + Development: true, + Level: zapcore.InfoLevel, + } + fakeLogger := ctrlrtzap.New(ctrlrtzap.UseFlagOptions(&zapOptions)) + + // initlizing account cache + accountCache := ackrtcache.NewAccountCache(k8sClient, fakeLogger) + stopCh := make(chan struct{}) + accountCache.Run(stopCh) + + // Test create events + k8sClient.CoreV1().ConfigMaps(testNamespace).Create( + context.Background(), + &corev1.ConfigMap{ + ObjectMeta: metav1.ObjectMeta{ + Name: "random-map", + }, + Data: accountsMap1, + }, + metav1.CreateOptions{}, + ) + + time.Sleep(time.Second) + + _, ok := accountCache.GetAccountRoleARN("random-account") + require.False(t, ok) + + k8sClient.CoreV1().ConfigMaps(testNamespace).Create( + context.Background(), + &corev1.ConfigMap{ + ObjectMeta: metav1.ObjectMeta{ + Name: ackrtcache.ACKRoleAccountMap, + Namespace: "ack-system", + }, + Data: accountsMap1, + }, + metav1.CreateOptions{}, + ) + + time.Sleep(time.Second) + + roleARN, ok := accountCache.GetAccountRoleARN(testAccount1) + require.True(t, ok) + require.Equal(t, roleARN, testAccountARN1) + + _, ok = accountCache.GetAccountRoleARN(testAccount2) + require.False(t, ok) + + // Test update events + k8sClient.CoreV1().ConfigMaps("ack-system").Update( + context.Background(), + &corev1.ConfigMap{ + ObjectMeta: metav1.ObjectMeta{ + Name: ackrtcache.ACKRoleAccountMap, + Namespace: "ack-system", + }, + Data: accountsMap2, + }, + metav1.UpdateOptions{}, + ) + + time.Sleep(time.Second) + + roleARN, ok = accountCache.GetAccountRoleARN(testAccount1) + require.True(t, ok) + require.Equal(t, roleARN, testAccountARN1) + + roleARN, ok = accountCache.GetAccountRoleARN(testAccount2) + require.True(t, ok) + require.Equal(t, roleARN, testAccountARN2) + + // Test delete events + k8sClient.CoreV1().ConfigMaps("ack-system").Delete( + context.Background(), + ackrtcache.ACKRoleAccountMap, + metav1.DeleteOptions{}, + ) + + time.Sleep(time.Second) + + _, ok = accountCache.GetAccountRoleARN(testAccount1) + require.False(t, ok) + _, ok = accountCache.GetAccountRoleARN(testAccount2) + require.False(t, ok) + +} diff --git a/pkg/runtime/cache/cache.go b/pkg/runtime/cache/cache.go new file mode 100644 index 0000000000..db88251f28 --- /dev/null +++ b/pkg/runtime/cache/cache.go @@ -0,0 +1,84 @@ +// Copyright Amazon.com Inc. or its affiliates. All Rights Reserved. +// +// Licensed under the Apache License, Version 2.0 (the "License"). You may +// not use this file except in compliance with the License. A copy of the +// License is located at +// +// http://aws.amazon.com/apache2.0/ +// +// or in the "license" file accompanying this file. This file is distributed +// on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either +// express or implied. See the License for the specific language governing +// permissions and limitations under the License. + +package cache + +import ( + "os" + "time" + + "github.com/go-logr/logr" + kubernetes "k8s.io/client-go/kubernetes" +) + +const ( + // defaultNamespace is the default namespace to use if the environment + // variable NAMESPACE is not found. The NAMESPACE variable is injected + // using the kubernetes downward api. + defaultNamespace = "ack-system" + + // informerDefaultResyncPeriod is the period at which ShouldResync + // is considered. + informerResyncPeriod = 0 * time.Second +) + +// currentNamespace is the namespace in which the current service +// controller Pod is running +var currentNamespace string + +func init() { + currentNamespace = os.Getenv("K8S_NAMESPACE") + if currentNamespace == "" { + currentNamespace = defaultNamespace + } +} + +// Caches is used to interact with the different caches +type Caches struct { + // stopCh is a channel use to stop all the + // owned caches + stopCh chan struct{} + + // Accounts cache + Accounts *AccountCache + + // Namespaces cache + Namespaces *NamespaceCache +} + +// New creates a new Caches object from a kubernetes.Interface and +// a logr.Logger +func New(clientset kubernetes.Interface, log logr.Logger) Caches { + return Caches{ + Accounts: NewAccountCache(clientset, log), + Namespaces: NewNamespaceCache(clientset, log), + } +} + +// Run runs all the owned caches +func (c Caches) Run() { + stopCh := make(chan struct{}) + if c.Accounts != nil { + c.Accounts.Run(stopCh) + } + if c.Namespaces != nil { + c.Namespaces.Run(stopCh) + } + c.stopCh = stopCh +} + +// Stop closes the stop channel and cause all the SharedInformers +// by caches to stop running +func (c Caches) Stop() { + close(c.stopCh) +} diff --git a/pkg/runtime/cache/namespace.go b/pkg/runtime/cache/namespace.go new file mode 100644 index 0000000000..ee9bb6c72e --- /dev/null +++ b/pkg/runtime/cache/namespace.go @@ -0,0 +1,172 @@ +// Copyright Amazon.com Inc. or its affiliates. All Rights Reserved. +// +// Licensed under the Apache License, Version 2.0 (the "License"). You may +// not use this file except in compliance with the License. A copy of the +// License is located at +// +// http://aws.amazon.com/apache2.0/ +// +// or in the "license" file accompanying this file. This file is distributed +// on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either +// express or implied. See the License for the specific language governing +// permissions and limitations under the License. + +package cache + +import ( + "sync" + + "github.com/go-logr/logr" + corev1 "k8s.io/api/core/v1" + informersv1 "k8s.io/client-go/informers/core/v1" + kubernetes "k8s.io/client-go/kubernetes" + k8scache "k8s.io/client-go/tools/cache" + + ackv1alpha1 "github.com/aws/aws-controllers-k8s/apis/core/v1alpha1" +) + +// namespaceInfo contains annotations ACK controllers care about +type namespaceInfo struct { + // services.k8s.aws/default-region Annotation + defaultRegion string + // services.k8s.aws/owner-account-id Annotation + ownerAccountID string +} + +// getDefaultRegion returns the default region value +func (n *namespaceInfo) getDefaultRegion() string { + if n == nil { + return "" + } + return n.defaultRegion +} + +// getOwnerAccountID returns the namespace owner Account ID +func (n *namespaceInfo) getOwnerAccountID() string { + if n == nil { + return "" + } + return n.ownerAccountID +} + +// NamespaceCache is reponsible of keeping track of namespaces +// annotations, and caching those related to the ACK controller. +type NamespaceCache struct { + sync.RWMutex + + log logr.Logger + // Namespace informer + informer k8scache.SharedInformer + // namespaceInfos maps namespaces names to their known namespaceInfo + namespaceInfos map[string]*namespaceInfo +} + +// NewNamespaceCache makes a new NamespaceCache from a +// kubernetes.Interface and a logr.Logger +func NewNamespaceCache(clientset kubernetes.Interface, log logr.Logger) *NamespaceCache { + sharedInformer := informersv1.NewNamespaceInformer( + clientset, + informerResyncPeriod, + k8scache.Indexers{}, + ) + return &NamespaceCache{ + informer: sharedInformer, + log: log.WithName("NamespaceCache"), + namespaceInfos: make(map[string]*namespaceInfo), + } +} + +// isIgnoredNamespace returns true if an object is of type corev1.Namespace +// and it metadata name is one of 'ack-system', 'kube-system' or 'kube-public' +func isIgnoredNamespace(raw interface{}) bool { + object, ok := raw.(*corev1.Namespace) + return ok && + (object.ObjectMeta.Name == "ack-system" || + object.ObjectMeta.Name == "kube-system" || + object.ObjectMeta.Name == "kube-public") +} + +// Run adds event handler functions to the SharedInformer and +// runs the informer to begin processing items. +func (c *NamespaceCache) Run(stopCh <-chan struct{}) { + c.informer.AddEventHandler(k8scache.ResourceEventHandlerFuncs{ + AddFunc: func(obj interface{}) { + if !isIgnoredNamespace(obj) { + c.log.V(1).Info("namespace has been created") + c.setNamespaceInfoFromK8sObject(obj.(*corev1.Namespace)) + c.log.V(1).Info("cached namespace ACK related annotations") + } + }, + + UpdateFunc: func(orig, desired interface{}) { + if !isIgnoredNamespace(desired) { + c.log.V(1).Info("namespace has been updated") + c.setNamespaceInfoFromK8sObject(desired.(*corev1.Namespace)) + c.log.V(1).Info("cached namespace ACK related annotations") + } + }, + DeleteFunc: func(obj interface{}) { + if !isIgnoredNamespace(obj) { + c.log.V(1).Info("namespace has been deleted") + c.deleteNamespaceInfo(obj.(*corev1.Namespace).ObjectMeta.Name) + c.log.V(1).Info("cleaned up namespace informations from cache") + } + }, + }) + go c.informer.Run(stopCh) +} + +// GetDefaultRegion returns the default region if it it exists +func (c *NamespaceCache) GetDefaultRegion(namespace string) (string, bool) { + info, ok := c.getNamespaceInfo(namespace) + if ok { + r := info.getDefaultRegion() + return r, r != "" + } + return "", false +} + +// GetOwnerAccountID returns the owner account ID if it exists +func (c *NamespaceCache) GetOwnerAccountID(namespace string) (string, bool) { + info, ok := c.getNamespaceInfo(namespace) + if ok { + a := info.getOwnerAccountID() + return a, a != "" + } + return "", false +} + +// getNamespaceInfo reads a namespace cached annotations and +// return a given namespace default aws region and owner account id. +// This function is thread safe. +func (c *NamespaceCache) getNamespaceInfo(ns string) (*namespaceInfo, bool) { + c.RLock() + defer c.RUnlock() + namespaceInfo, ok := c.namespaceInfos[ns] + return namespaceInfo, ok +} + +// setNamespaceInfoFromK8sObject takes a corev1.Namespace object and sets the +// namespace ACK related annotations in the cache map +func (c *NamespaceCache) setNamespaceInfoFromK8sObject(ns *corev1.Namespace) { + nsa := ns.ObjectMeta.Annotations + nsInfo := &namespaceInfo{} + DefaultRegion, ok := nsa[ackv1alpha1.AnnotationDefaultRegion] + if ok { + nsInfo.defaultRegion = DefaultRegion + } + OwnerAccountID, ok := nsa[ackv1alpha1.AnnotationOwnerAccountID] + if ok { + nsInfo.ownerAccountID = OwnerAccountID + } + c.Lock() + defer c.Unlock() + c.namespaceInfos[ns.ObjectMeta.Name] = nsInfo +} + +// deleteNamespace deletes an entry from cache map +func (c *NamespaceCache) deleteNamespaceInfo(ns string) { + c.Lock() + defer c.Unlock() + delete(c.namespaceInfos, ns) +} diff --git a/pkg/runtime/cache/namespace_test.go b/pkg/runtime/cache/namespace_test.go new file mode 100644 index 0000000000..1ae1cb8c97 --- /dev/null +++ b/pkg/runtime/cache/namespace_test.go @@ -0,0 +1,118 @@ +// Copyright Amazon.com Inc. or its affiliates. All Rights Reserved. +// +// Licensed under the Apache License, Version 2.0 (the "License"). You may +// not use this file except in compliance with the License. A copy of the +// License is located at +// +// http://aws.amazon.com/apache2.0/ +// +// or in the "license" file accompanying this file. This file is distributed +// on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either +// express or implied. See the License for the specific language governing +// permissions and limitations under the License. + +package cache_test + +import ( + "context" + "testing" + "time" + + "github.com/stretchr/testify/require" + "go.uber.org/zap/zapcore" + corev1 "k8s.io/api/core/v1" + metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" + "k8s.io/apimachinery/pkg/watch" + k8sfake "k8s.io/client-go/kubernetes/fake" + k8stesting "k8s.io/client-go/testing" + ctrlrtzap "sigs.k8s.io/controller-runtime/pkg/log/zap" + + ackv1alpha1 "github.com/aws/aws-controllers-k8s/apis/core/v1alpha1" + ackrtcache "github.com/aws/aws-controllers-k8s/pkg/runtime/cache" +) + +const ( + testNamespace1 = "production" +) + +func TestNamespaceCache(t *testing.T) { + // create a fake k8s client and fake watcher + k8sClient := k8sfake.NewSimpleClientset() + watcher := watch.NewFake() + k8sClient.PrependWatchReactor("production", k8stesting.DefaultWatchReactor(watcher, nil)) + + // New logger writing to specific buffer + zapOptions := ctrlrtzap.Options{ + Development: true, + Level: zapcore.InfoLevel, + } + fakeLogger := ctrlrtzap.New(ctrlrtzap.UseFlagOptions(&zapOptions)) + + // initlizing account cache + namespaceCache := ackrtcache.NewNamespaceCache(k8sClient, fakeLogger) + stopCh := make(chan struct{}) + + namespaceCache.Run(stopCh) + + // Test create events + k8sClient.CoreV1().Namespaces().Create( + context.Background(), + &corev1.Namespace{ + ObjectMeta: metav1.ObjectMeta{ + Name: "production", + Annotations: map[string]string{ + ackv1alpha1.AnnotationDefaultRegion: "us-west-2", + ackv1alpha1.AnnotationOwnerAccountID: "012345678912", + }, + }, + }, + metav1.CreateOptions{}, + ) + + time.Sleep(time.Second) + + defaultRegion, ok := namespaceCache.GetDefaultRegion("production") + require.True(t, ok) + require.Equal(t, "us-west-2", defaultRegion) + + ownerAccountID, ok := namespaceCache.GetOwnerAccountID("production") + require.True(t, ok) + require.Equal(t, "012345678912", ownerAccountID) + + // Test update events + k8sClient.CoreV1().Namespaces().Update( + context.Background(), + &corev1.Namespace{ + ObjectMeta: metav1.ObjectMeta{ + Name: "production", + Annotations: map[string]string{ + ackv1alpha1.AnnotationDefaultRegion: "us-est-1", + ackv1alpha1.AnnotationOwnerAccountID: "21987654321", + }, + }, + }, + metav1.UpdateOptions{}, + ) + + time.Sleep(time.Second) + + defaultRegion, ok = namespaceCache.GetDefaultRegion("production") + require.True(t, ok) + require.Equal(t, "us-est-1", defaultRegion) + + ownerAccountID, ok = namespaceCache.GetOwnerAccountID("production") + require.True(t, ok) + require.Equal(t, "21987654321", ownerAccountID) + + // Test delete events + k8sClient.CoreV1().Namespaces().Delete( + context.Background(), + "production", + metav1.DeleteOptions{}, + ) + + time.Sleep(time.Second) + + _, ok = namespaceCache.GetDefaultRegion(testNamespace1) + require.False(t, ok) +} diff --git a/pkg/runtime/reconciler.go b/pkg/runtime/reconciler.go index 8e8241517a..f8918e0abb 100644 --- a/pkg/runtime/reconciler.go +++ b/pkg/runtime/reconciler.go @@ -21,12 +21,14 @@ import ( corev1 "k8s.io/api/core/v1" apierrors "k8s.io/apimachinery/pkg/api/errors" metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" + kubernetes "k8s.io/client-go/kubernetes" ctrlrt "sigs.k8s.io/controller-runtime" "sigs.k8s.io/controller-runtime/pkg/client" ackv1alpha1 "github.com/aws/aws-controllers-k8s/apis/core/v1alpha1" ackerr "github.com/aws/aws-controllers-k8s/pkg/errors" "github.com/aws/aws-controllers-k8s/pkg/requeue" + ackrtcache "github.com/aws/aws-controllers-k8s/pkg/runtime/cache" acktypes "github.com/aws/aws-controllers-k8s/pkg/types" ) @@ -38,11 +40,12 @@ import ( // controller-runtime.Controller objects (each containing a single reconciler // object)s and sharing watch and informer queues across those controllers. type reconciler struct { - kc client.Client - rmf acktypes.AWSResourceManagerFactory - rd acktypes.AWSResourceDescriptor - log logr.Logger - cfg Config + kc client.Client + rmf acktypes.AWSResourceManagerFactory + rd acktypes.AWSResourceDescriptor + log logr.Logger + cfg Config + cache ackrtcache.Caches } // GroupKind returns the string containing the API group and kind reconciled by @@ -60,7 +63,13 @@ func (r *reconciler) BindControllerManager(mgr ctrlrt.Manager) error { if r.rmf == nil { return ackerr.NilResourceManagerFactory } + clusterConfig := mgr.GetConfig() + clientset, err := kubernetes.NewForConfig(clusterConfig) + if err != nil { + return err + } r.kc = mgr.GetClient() + r.cache = ackrtcache.New(clientset, r.log) rd := r.rmf.ResourceDescriptor() return ctrlrt.NewControllerManagedBy( mgr, @@ -335,12 +344,28 @@ func (r *reconciler) getOwnerAccountID( } // getRegion returns the AWS region that the given resource is in or should be -// created in. If the Namespace has a region associated with it, that is used, -// otherwise the region specified in the configuration is used. +// created in. If the CR have a region associated with it, it is used. Otherwise +// we look for the namespace associated region, if that is set we use it. Finally +// if none of these annotations are set we use the use the region specified in the +// configuration is used func (r *reconciler) getRegion( res acktypes.AWSResource, ) ackv1alpha1.AWSRegion { - // TODO(jaypipes): Do the Namespace region lookup... + // look for region in CR metadata annotations + resAnnotations := res.MetaObject().GetAnnotations() + region, ok := resAnnotations[ackv1alpha1.AnnotationRegion] + if ok { + return ackv1alpha1.AWSRegion(region) + } + + // look for default region in namespace metadata annotations + ns := res.MetaObject().GetNamespace() + defaultRegion, ok := r.cache.Namespaces.GetDefaultRegion(ns) + if ok { + return ackv1alpha1.AWSRegion(defaultRegion) + } + + // use controller configuration region return ackv1alpha1.AWSRegion(r.cfg.Region) } diff --git a/pkg/runtime/registry.go b/pkg/runtime/registry.go index b4915d29cf..10dbf5cfd7 100644 --- a/pkg/runtime/registry.go +++ b/pkg/runtime/registry.go @@ -39,7 +39,7 @@ func (r *Registry) GetResourceManagerFactories() []types.AWSResourceManagerFacto return res } -// RegisterManagerFactory registers a resource manager factory with the +// RegisterResourceManagerFactory registers a resource manager factory with the // package's registry func (r *Registry) RegisterResourceManagerFactory(f types.AWSResourceManagerFactory) { r.Lock() diff --git a/pkg/runtime/service_controller_test.go b/pkg/runtime/service_controller_test.go index 26039f1203..24e960307b 100644 --- a/pkg/runtime/service_controller_test.go +++ b/pkg/runtime/service_controller_test.go @@ -18,6 +18,7 @@ import ( "testing" "github.com/stretchr/testify/require" + "go.uber.org/zap/zapcore" "k8s.io/apimachinery/pkg/api/meta" metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" "k8s.io/apimachinery/pkg/runtime" @@ -28,6 +29,7 @@ import ( "sigs.k8s.io/controller-runtime/pkg/cache" "sigs.k8s.io/controller-runtime/pkg/client" "sigs.k8s.io/controller-runtime/pkg/healthz" + ctrlrtzap "sigs.k8s.io/controller-runtime/pkg/log/zap" ctrlmanager "sigs.k8s.io/controller-runtime/pkg/manager" k8sscheme "sigs.k8s.io/controller-runtime/pkg/scheme" "sigs.k8s.io/controller-runtime/pkg/webhook" @@ -66,7 +68,7 @@ func (m *fakeManager) AddMetricsExtraHandler(path string, handler http.Handler) func (m *fakeManager) AddHealthzCheck(name string, check healthz.Checker) error { return nil } func (m *fakeManager) AddReadyzCheck(name string, check healthz.Checker) error { return nil } func (m *fakeManager) Start(<-chan struct{}) error { return nil } -func (m *fakeManager) GetConfig() *rest.Config { return nil } +func (m *fakeManager) GetConfig() *rest.Config { return &rest.Config{} } func (m *fakeManager) GetScheme() *runtime.Scheme { return scheme } func (m *fakeManager) GetClient() client.Client { return nil } func (m *fakeManager) GetFieldIndexer() client.FieldIndexer { return nil } @@ -99,6 +101,12 @@ func TestServiceController(t *testing.T) { sc := ackrt.NewServiceController("bookstore", "bookstore.services.k8s.aws") require.NotNil(sc) + zapOptions := ctrlrtzap.Options{ + Development: true, + Level: zapcore.InfoLevel, + } + fakeLogger := ctrlrtzap.New(ctrlrtzap.UseFlagOptions(&zapOptions)) + sc.WithLogger(fakeLogger) sc.WithResourceManagerFactories(reg.GetResourceManagerFactories()) recons := sc.GetReconcilers() diff --git a/templates/config/controller/deployment.yaml.tpl b/templates/config/controller/deployment.yaml.tpl index f93bb1abfa..76a4c2fb62 100644 --- a/templates/config/controller/deployment.yaml.tpl +++ b/templates/config/controller/deployment.yaml.tpl @@ -41,4 +41,9 @@ spec: requests: cpu: 100m memory: 200Mi + env: + - name: K8S_NAMESPACE + valueFrom: + fieldRef: + fieldPath: metadata.namespace terminationGracePeriodSeconds: 10