Added tag lookup backoff logic when no tags are found.

configbutler/resolvers.py

@@ -2,6 +2,7 @@
 import logging
 import socket
 import boto3
+import time
 
 from string import Template
 from botocore.exceptions import ClientError
@@ -103,6 +104,8 @@ def resolve(self, parts, current_properties):
 
 class AWSTagResolver(BaseResolver):
 
+    RETRY_COUNT = 5
+
     def __init__(self):
         super(AWSTagResolver, self).__init__()
         self.client = None
@@ -122,15 +125,28 @@ def _ec2_client(self):
     def resolve(self, key, current_properties):
 
         if self.tags is None:
-            response = self._ec2_client().describe_tags(
-                Filters=[
-                    {
-                        'Name': 'resource-id',
-                        'Values': [self._metadata().instance_id]
-                    },
-                ]
-            )
-            self.tags = response["Tags"]
+            self.tags = []
+            backoff_time = 1
+            count = 0
+
+            while len(self.tags) == 0 and count < self.RETRY_COUNT:
+                response = self._ec2_client().describe_tags(
+                    Filters=[
+                        {
+                            'Name': 'resource-id',
+                            'Values': [self._metadata().instance_id]
+                        },
+                    ]
+                )
+                self.tags = response["Tags"]
+                if len(self.tags) == 0:
+                    logger.error("No AWS::tag values found, waiting {}sec to retry.".format(backoff_time))
+                    time.sleep(backoff_time)
+                    backoff_time = backoff_time * 2
+                    count += 1
+
+        if len(self.tags) == 0:
+            logger.error("No AWS::tag values found, continuing with no tags.")
 
         return self.lookup_tag(key=self.resolve_embedded(key, current_properties), tags=self.tags)
 
@@ -140,6 +156,7 @@ def lookup_tag(key, tags):
             if tag["Key"] == key:
                 return tag["Value"]
 
+        logger.error("Unable to find AWS::tag named '{}'".format(key))
         return None
 
 

tests/test_aws_tags_resolver.py

@@ -0,0 +1,118 @@
+import unittest
+
+import StringIO
+from ec2_metadata import EC2Metadata
+from mock import call, Mock, MagicMock
+import mock
+import logging
+
+from configbutler.resolvers import AWSTagResolver
+
+
+logger = logging.getLogger("configbutler")
+logging.basicConfig()
+
+
+class TestAWSTagsResolver(unittest.TestCase):
+
+    @mock.patch("configbutler.resolvers.logger")
+    def test_no_tags_retries(self, mock_logger):
+        undertest = AWSTagResolver()
+        undertest.RETRY_COUNT = 2
+        undertest.metadata = mock.create_autospec(EC2Metadata)
+        undertest.metadata.instance_id = "i-12345"
+
+        mock_tags = {
+            "Tags": []
+        }
+
+        undertest.client = Mock()
+        undertest.client.describe_tags = MagicMock(return_value=mock_tags)
+
+        self.assertEqual(None, undertest.resolve("blart", dict))
+
+        self.assertEqual([call(Filters=[{'Values': ['i-12345'], 'Name': 'resource-id'}]),
+                          call(Filters=[{'Values': ['i-12345'], 'Name': 'resource-id'}])],
+                         undertest.client.describe_tags.mock_calls)
+
+        self.assertEqual([call.error('No AWS::tag values found, waiting 1sec to retry.'),
+                          call.error('No AWS::tag values found, waiting 2sec to retry.'),
+                          call.error('No AWS::tag values found, continuing with no tags.'),
+                          call.error("Unable to find AWS::tag named 'blart'")],
+                         mock_logger.mock_calls)
+
+    @mock.patch("configbutler.resolvers.logger")
+    def test_with_second_retries(self, mock_logger):
+
+        no_tags = {
+            "Tags": []
+        }
+
+        some_tags = {
+            "Tags": [{"Key": "blart", "Value": "bling"}]
+        }
+
+        incr_return_values = [no_tags, some_tags]
+
+        def mock_tags_lookup(**args):
+            return incr_return_values.pop(0)
+
+        undertest = AWSTagResolver()
+        undertest.RETRY_COUNT = 2
+        undertest.metadata = mock.create_autospec(EC2Metadata)
+        undertest.metadata.instance_id = "i-12345"
+
+        undertest.client = Mock()
+        undertest.client.describe_tags = mock_tags_lookup
+
+        self.assertEqual("bling", undertest.resolve("blart", dict))
+
+        # Should be no values left in the array
+        self.assertEqual(0, len(incr_return_values))
+
+        self.assertEqual([call.error('No AWS::tag values found, waiting 1sec to retry.')],
+                         mock_logger.mock_calls)
+
+    @mock.patch("configbutler.resolvers.logger")
+    def test_with_missing_tag(self, mock_logger):
+        undertest = AWSTagResolver()
+        undertest.RETRY_COUNT = 2
+        undertest.metadata = mock.create_autospec(EC2Metadata)
+        undertest.metadata.instance_id = "i-12345"
+
+        mock_tags = {
+            "Tags": [{"Key": "aKey", "Value": "aValue"}]
+        }
+
+        undertest.client = Mock()
+        undertest.client.describe_tags = MagicMock(return_value=mock_tags)
+
+        self.assertEqual(None, undertest.resolve("blart", dict))
+
+        self.assertEqual([call(Filters=[{'Values': ['i-12345'], 'Name': 'resource-id'}])],
+                         undertest.client.describe_tags.mock_calls)
+
+        self.assertEqual([call.error("Unable to find AWS::tag named 'blart'")],
+                         mock_logger.mock_calls)
+
+    @mock.patch("configbutler.resolvers.logger")
+    def test_with_actual_tag(self, mock_logger):
+        undertest = AWSTagResolver()
+        undertest.RETRY_COUNT = 2
+        undertest.metadata = mock.create_autospec(EC2Metadata)
+        undertest.metadata.instance_id = "i-12345"
+
+        mock_tags = {
+            "Tags": [{"Key": "blart", "Value": "bling"}]
+        }
+
+        undertest.client = Mock()
+        undertest.client.describe_tags = MagicMock(return_value=mock_tags)
+
+        self.assertEqual("bling", undertest.resolve("blart", dict))
+
+        self.assertEqual([call(Filters=[{'Values': ['i-12345'], 'Name': 'resource-id'}])],
+                         undertest.client.describe_tags.mock_calls)
+
+        self.assertEqual([],
+                         mock_logger.mock_calls)