Merge pull request #145 from yahoo/develop

Python 2 & 3 compatible string and bytes handling
yahoo · Nov 20, 2019 · 889db6c · 889db6c
2 parents 5ab0ec7 + ac0fe38
commit 889db6c
Show file tree

Hide file tree

Showing 4 changed files with 17 additions and 8 deletions.
diff --git a/tests/mock_kafka_consumer.py b/tests/mock_kafka_consumer.py
@@ -118,10 +118,10 @@ def poll(self, timeout_ms):
         return {
             u'400000005d73185508707bfc': [ConsumerRecord(
                 topic=u'panoptes-metrics', partition=49, offset=704152, timestamp=-1, timestamp_type=0,
-                key=u'class:subclass:type', value=json.dumps(value), checksum=-1526904207, serialized_key_size=19,
+                key=b'class:subclass:type', value=json.dumps(value), checksum=-1526904207, serialized_key_size=19,
                 serialized_value_size=1140)],
             u'400000005d731855164bb9bc': [ConsumerRecord(
                 topic=u'panoptes-metrics', partition=49, offset=704152, timestamp=-1, timestamp_type=0,
-                key=u'class:subclass:type::', value=json.dumps(value), checksum=-1526904207, serialized_key_size=19,
+                key=b'class:subclass:type::', value=json.dumps(value), checksum=-1526904207, serialized_key_size=19,
                 serialized_value_size=1140)]
         }
diff --git a/yahoo_panoptes/framework/utilities/consumer.py b/yahoo_panoptes/framework/utilities/consumer.py
@@ -380,7 +380,8 @@ def start_consumer(self):
                     logger.debug(u'Processing consumer record with key: "%s" and value: "%s"' % (
                         consumer_record.key, consumer_record.value))
 
-                    if self.keys and consumer_record.key not in self.keys:
+                    consumer_record_key = consumer_record.key.decode(u'utf-8')
+                    if self.keys and consumer_record_key not in self.keys:
                         logger.debug(
                                 u'Consumer record key "%s" does not match any of the provided keys, skipping' %
                                 consumer_record.key)
@@ -403,7 +404,7 @@ def start_consumer(self):
                             consumer_records_validation_failed += 1
                             continue
                     try:
-                        callback_succeeded = self._callback(consumer_record.key, consumer_record_object)
+                        callback_succeeded = self._callback(consumer_record_key, consumer_record_object)
                         # If the callback fails even for one consumer record, we want to fail (not update the committed)
                         # offset for the entire the batch, so exit
                         if not callback_succeeded:
@@ -498,3 +499,5 @@ def __init__(self, panoptes_context, client_id, group, keys, poll_timeout, callb
                                                         session_timeout=session_timeout,
                                                         max_poll_records=max_poll_records,
                                                         max_partition_fetch_bytes=max_partition_fetch_bytes)
+
+
diff --git a/yahoo_panoptes/framework/utilities/snmp/variable.py b/yahoo_panoptes/framework/utilities/snmp/variable.py
@@ -22,7 +22,10 @@ def __init__(self, queried_oid, oid, index, value, snmp_type):
         self._snmp_type = snmp_type
 
     def __repr__(self):
-        printable_value = strip_non_printable(self._value)
+
+        printable_value = self.value.decode(u'ascii', u'backslashreplace') if hasattr(self.value, u'decode') \
+            else self.value
+
         return (
             u"<{0} value={1} (oid={2}, index={3}, snmp_type={4})>".format(
                     self.__class__.__name__,

diff --git a/yahoo_panoptes/plugins/enrichment/interface/plugin_enrichment_interface.py b/yahoo_panoptes/plugins/enrichment/interface/plugin_enrichment_interface.py
@@ -27,7 +27,9 @@ def _build_enrichments_map(self):
                 self._enrichments_map[varbind.oid + '.' + varbind.index] = varbind.value
 
     def get_interface_name(self, index):
-        return self._enrichments_map.get(ifName + '.' + index, self._MISSING_VALUE_STRING)
+        return self._enrichments_map.get(ifName + '.' + index, self._MISSING_VALUE_STRING)  \
+                   .encode(u'ascii', u'ignore') \
+                   .decode(u'ascii')
 
     @property
     def host(self):
@@ -99,15 +101,16 @@ def get_parent_interface_configured_speed(self, index):
             return self._MISSING_METRIC_VALUE
 
     def get_description(self, index):
-        return self._enrichments_map.get(ifDescr + '.' + index, self._MISSING_VALUE_STRING)
+        return self._enrichments_map.get(ifDescr + '.' + index, self._MISSING_VALUE_STRING) \
+                   .encode(u'ascii', u'ignore').decode(u'ascii')
 
     def get_media_type(self, index):
         type_index = self._enrichments_map.get(ifType + '.' + index)
         return getIfTypeDesc(type_index) if type_index is not None else self._MISSING_VALUE_STRING
 
     def get_alias(self, index):
         alias = self._enrichments_map.get(ifAlias + '.' + index, self._MISSING_VALUE_STRING)
-        return alias if len(alias) > 0 else self._MISSING_VALUE_STRING
+        return alias.encode(u'ascii', u'ignore').decode(u'ascii') if len(alias) > 0 else self._MISSING_VALUE_STRING
 
     def get_if_speed(self, index):
         return int(self._enrichments_map.get(ifSpeed + '.' + index, self._MISSING_METRIC_VALUE))