Merge pull request #2418 from DataDog/413-response

better 413 response handling

checks/check_status.py

@@ -761,7 +761,7 @@ class ForwarderStatus(AgentStatus):
     NAME = 'Forwarder'
 
     def __init__(self, queue_length=0, queue_size=0, flush_count=0, transactions_received=0,
-                 transactions_flushed=0):
+                 transactions_flushed=0, too_big_count=0):
         AgentStatus.__init__(self)
         self.queue_length = queue_length
         self.queue_size = queue_size
@@ -771,6 +771,7 @@ def __init__(self, queue_length=0, queue_size=0, flush_count=0, transactions_rec
         self.proxy_data = get_config(parse_args=False).get('proxy_settings')
         self.hidden_username = None
         self.hidden_password = None
+        self.too_big_count = too_big_count
         if self.proxy_data and self.proxy_data.get('user'):
             username = self.proxy_data.get('user')
             hidden = len(username) / 2 if len(username) <= 7 else len(username) - 4
@@ -784,6 +785,7 @@ def body_lines(self):
             "Flush Count: %s" % self.flush_count,
             "Transactions received: %s" % self.transactions_received,
             "Transactions flushed: %s" % self.transactions_flushed,
+            "Transactions rejected: %s" % self.too_big_count,
             ""
         ]
 
@@ -815,6 +817,7 @@ def to_dict(self):
             'proxy_data': self.proxy_data,
             'hidden_username': self.hidden_username,
             'hidden_password': self.hidden_password,
+            'too_big_count': self.too_big_count,
 
         })
         return status_info

ddagent.py

@@ -266,7 +266,10 @@ def flush(self):
     def on_response(self, response):
         if response.error:
             log.error("Response: %s" % response)
-            self._trManager.tr_error(self)
+            if response.code == 413:
+                self._trManager.tr_error_too_big(self)
+            else:
+                self._trManager.tr_error(self)
         else:
             self._trManager.tr_success(self)
 

transaction.py

@@ -83,6 +83,8 @@ def __init__(self, max_wait_for_replay, max_queue_size, throttling_delay):
         self._transactions_received = 0
         self._transactions_flushed = 0
 
+        self._too_big_count = 0
+
         # Global counter to assign a number to each transaction: we may have an issue
         #  if this overlaps
         self._counter = 0
@@ -173,7 +175,8 @@ def flush(self):
             queue_size=self._total_size,
             flush_count=self._flush_count,
             transactions_received=self._transactions_received,
-            transactions_flushed=self._transactions_flushed).persist()
+            transactions_flushed=self._transactions_flushed,
+            too_big_count=self._too_big_count).persist()
 
     def flush_next(self):
 
@@ -216,6 +219,24 @@ def tr_error(self,tr):
           (tr.get_id(), tr.get_error_count(), plural(tr.get_error_count()),
            tr.get_next_flush()))
 
+    def tr_error_too_big(self,tr):
+        tr.inc_error_count()
+        log.warn("Transaction %d is %sKB, it has been rejected as too large. \
+          It will not be replayed." % (tr.get_id(), tr.get_size() / 1024))
+        self._transactions.remove(tr)
+        self._total_count -= 1
+        self._total_size -= tr.get_size()
+        self._transactions_flushed += 1
+        self.print_queue_stats()
+        self._too_big_count += 1
+        ForwarderStatus(
+            queue_length=self._total_count,
+            queue_size=self._total_size,
+            flush_count=self._flush_count,
+            transactions_received=self._transactions_received,
+            transactions_flushed=self._transactions_flushed,
+            too_big_count=self._too_big_count).persist()
+
     def tr_success(self,tr):
         log.debug("Transaction %d completed" % tr.get_id())
         self._transactions.remove(tr)

win32/status.html

@@ -242,6 +242,9 @@ <h2>Forwarder</h2>
             <dt>Flush count</dt>        <dd>{{ forwarder['flush_count'] }}</dd>
             <dt>Queue size</dt>         <dd>{{ forwarder['queue_size'] }} bytes</dd>
             <dt>Queue length</dt>       <dd>{{ forwarder['queue_length'] }}</dd>
+            <dt>Transactions received</dt> <dd>{{ forwarder['transactions_received'] }}</dd>
+            <dt>Transactions flushed</dt> <dd>{{ forwarder['transactions_flushed'] }}</dd>
+            <dt>Transactions rejected</dt>       <dd>{{ forwarder['too_big_count'] }}</dd>
           </dl>
 
           {% if forwarder['proxy_data'] %}