Merge pull request #120 from yahoo/develop

Add Python 3 support

setup.cfg

@@ -26,13 +26,14 @@ install_requires =
     Cerberus
     configobj
     faulthandler;python_version<"3.3"
+    future
     gevent
     influxdb
     ipaddress
     json-schema-validator
     kazoo
     kafka-python==1.3.5
-    mmh3
+    pymmh3
     pyparsing
     ratelimitingfilter
     redis
@@ -44,6 +45,7 @@ packages = find:
 
 [options.extras_require]
 dev =
+    future
     tox
     nose
     nose-cov

tests/consumers/influxdb/test_influxdb_consumer.py

@@ -38,13 +38,13 @@ def get_list_database(self):
         return [{u'name': u'db1'}, {u'name': u'db2'}, {u'name': u'db3'}]
 
     def ping(self):
-        return '1.6.2'
+        return u'1.6.2'
 
 
 class TestPanoptesInfluxDBConsumer(unittest.TestCase):
     def test_influxdb_consumer_parser_exception(self):
         mock_argument_parser = Mock()
-        attributes = {'parse_known_args.side_effect': Exception}
+        attributes = {u'parse_known_args.side_effect': Exception}
         mock_argument_parser.configure_mock(**attributes)
         with patch('yahoo_panoptes.consumers.influxdb.consumer.argparse.ArgumentParser', mock_argument_parser):
             with self.assertRaises(SystemExit):
@@ -103,25 +103,24 @@ def test_panoptes_influxdb_connection_create_database_exception(self):
     def test_panoptes_influxdb_consumer(self):
         """Test sending metrics through the InfluxDB client"""
 
-        output_data_file = os.path.join(os.path.dirname(os.path.abspath(__file__)), 'output/influx_line00.data')
+        output_data_file = os.path.join(os.path.dirname(os.path.abspath(__file__)), u'output/influx_line00.data')
 
         output_data = open(output_data_file).read()
 
         MockPanoptesConsumer.files = [
-            'consumers/influxdb/input/metrics_group00.json',
+            u'consumers/influxdb/input/metrics_group00.json',
         ]
 
         with requests_mock.Mocker() as m:
             m.register_uri('POST', "http://127.0.0.1:8086/write", status_code=204)
             PanoptesInfluxDBConsumer.factory()
-            self.assertEquals(m.last_request.body, output_data)
+            self.assertEquals(m.last_request.body.decode("utf-8"), output_data)
 
         # Fail on first write to try _send_one_by_one
         with requests_mock.Mocker() as m:
             m.register_uri('POST', "http://127.0.0.1:8086/write", response_list=[
-                {'status_code': 400}, {'status_code': 204}])
+                {u'status_code': 400}, {u'status_code': 204}])
             PanoptesInfluxDBConsumer.factory()
-            self.assertEquals(m.last_request.body, output_data)
+            self.assertEquals(m.last_request.body.decode("utf-8"), output_data)
 
         # TODO: Test sending points in a single batch
-

tests/enrichment/test_enrichment_plugin_scheduler.py

@@ -37,11 +37,11 @@ def setUp(self):
                                                  key_value_store_class_list=[PanoptesTestKeyValueStore],
                                                  create_message_producer=False, async_message_producer=False,
                                                  create_zookeeper_client=True)
-        self._celery_config = PanoptesCeleryConfig(app_name="Enrichment Plugin Test")
+        self._celery_config = PanoptesCeleryConfig(app_name=u"Enrichment Plugin Test")
         self._scheduler = PanoptesPluginScheduler(
             panoptes_context=self._panoptes_context,
-            plugin_type="enrichment",
-            plugin_type_display_name="Enrichment",
+            plugin_type=u"enrichment",
+            plugin_type_display_name=u"Enrichment",
             celery_config=self._celery_config,
             lock_timeout=1,
             plugin_scheduler_task=_callback

tests/helpers.py

@@ -8,6 +8,6 @@
 
 def get_test_conf_file():
     my_dir = os.path.dirname(os.path.realpath(__file__))
-    panoptes_test_conf_file = os.path.join(my_dir, 'config_files/test_panoptes_config.ini')
+    panoptes_test_conf_file = os.path.join(my_dir, u'config_files/test_panoptes_config.ini')
 
     return my_dir, panoptes_test_conf_file

tests/mock_kafka_consumer.py

@@ -2,14 +2,15 @@
 Copyright 2019, Verizon Media
 Licensed under the terms of the Apache 2.0 license. See LICENSE file in project root for terms.
 """
+from builtins import object
 import json
 from kafka.consumer.fetcher import ConsumerRecord
 
 
 class Subscription(object):
 
     def all_consumed_offsets(self):
-        return 'Subscription.all_consumed_offsets'
+        return u'Subscription.all_consumed_offsets'
 
 
 class MockKafkaConsumer(object):
@@ -72,55 +73,55 @@ def committed(self, topic_partition):
     def poll(self, timeout_ms):
 
         value = {
-            "metrics_group_interval": 60,
-            "resource": {
-                "resource_site": "test_site",
-                "resource_id": "test_id",
-                "resource_class": "network",
-                "resource_plugin": "test_plugin",
-                "resource_creation_timestamp": 1567823517.46,
-                "resource_subclass": "test_subclass",
-                "resource_endpoint": "test_endpoint",
-                "resource_metadata": {
-                    "test_metadata_key": "test_metadata_value",
-                    "_resource_ttl": "604800"
+            u"metrics_group_interval": 60,
+            u"resource": {
+                u"resource_site": u"test_site",
+                u"resource_id": u"test_id",
+                u"resource_class": u"network",
+                u"resource_plugin": u"test_plugin",
+                u"resource_creation_timestamp": 1567823517.46,
+                u"resource_subclass": u"test_subclass",
+                u"resource_endpoint": u"test_endpoint",
+                u"resource_metadata": {
+                    u"test_metadata_key": u"test_metadata_value",
+                    u"_resource_ttl": u"604800"
                 },
-                "resource_type": "test_type"
+                u"resource_type": u"test_type"
             },
-            "dimensions": [
+            u"dimensions": [
                 {
-                    "dimension_name": "cpu_name",
-                    "dimension_value": "test_cpu_name_value"
+                    u"dimension_name": u"cpu_name",
+                    u"dimension_value": u"test_cpu_name_value"
                 },
                 {
-                    "dimension_name": "cpu_no",
-                    "dimension_value": "test_cpu_no_value"
+                    u"dimension_name": u"cpu_no",
+                    u"dimension_value": u"test_cpu_no_value"
                 },
                 {
-                    "dimension_name": "cpu_type",
-                    "dimension_value": "test_cpu_type_value"
+                    u"dimension_name": u"cpu_type",
+                    u"dimension_value": u"test_cpu_type_value"
                 }
             ],
-            "metrics_group_type": "cpu",
-            "metrics": [
+            u"metrics_group_type": u"cpu",
+            u"metrics": [
                 {
-                    "metric_creation_timestamp": 1567823946.72,
-                    "metric_type": "gauge",
-                    "metric_name": "cpu_utilization",
-                    "metric_value": 0
+                    u"metric_creation_timestamp": 1567823946.72,
+                    u"metric_type": u"gauge",
+                    u"metric_name": u"cpu_utilization",
+                    u"metric_value": 0
                 }
             ],
-            "metrics_group_creation_timestamp": 1567823946.72,
-            "metrics_group_schema_version": "0.2"
+            u"metrics_group_creation_timestamp": 1567823946.72,
+            u"metrics_group_schema_version": u"0.2"
         }
 
         return {
-            '400000005d73185508707bfc': [ConsumerRecord(
-                topic='panoptes-metrics', partition=49, offset=704152, timestamp=-1, timestamp_type=0,
-                key='class:subclass:type', value=json.dumps(value), checksum=-1526904207, serialized_key_size=19,
+            u'400000005d73185508707bfc': [ConsumerRecord(
+                topic=u'panoptes-metrics', partition=49, offset=704152, timestamp=-1, timestamp_type=0,
+                key=u'class:subclass:type', value=json.dumps(value), checksum=-1526904207, serialized_key_size=19,
                 serialized_value_size=1140)],
-            '400000005d731855164bb9bc': [ConsumerRecord(
-                topic='panoptes-metrics', partition=49, offset=704152, timestamp=-1, timestamp_type=0,
-                key='class:subclass:type::', value=json.dumps(value), checksum=-1526904207, serialized_key_size=19,
+            u'400000005d731855164bb9bc': [ConsumerRecord(
+                topic=u'panoptes-metrics', partition=49, offset=704152, timestamp=-1, timestamp_type=0,
+                key=u'class:subclass:type::', value=json.dumps(value), checksum=-1526904207, serialized_key_size=19,
                 serialized_value_size=1140)]
         }

tests/mock_panoptes_consumer.py

@@ -3,6 +3,7 @@
 Licensed under the terms of the Apache 2.0 license. See LICENSE file in project root for terms.
 """
 
+from builtins import object
 import json
 import os
 import time
@@ -13,7 +14,7 @@
 
 # Mocking this since the underlying functions used in the original are only available on Linux systems
 def mock_get_client_id(prefix):
-    return '_'.join([prefix, 'localhost', '1234'])
+    return u'_'.join([prefix, u'localhost', u'1234'])
 
 
 class MockPanoptesConsumer(object):
@@ -49,12 +50,11 @@ def start_consumer(self):
                 # Skip for resource objects
 
                 try:
-                    if consumer_record_object['metrics_group_creation_timestamp'] < 0:
-                        consumer_record_object['metrics_group_creation_timestamp'] += time.time()
+                    if consumer_record_object[u'metrics_group_creation_timestamp'] < 0:
+                        consumer_record_object[u'metrics_group_creation_timestamp'] += time.time()
                 except:
                     pass
-
-                self._callback('key:foo:foo', consumer_record_object)
+                self._callback(u'key:foo:foo', consumer_record_object)
 
 
 class MockPanoptesResourcesConsumer(MockPanoptesConsumer):

tests/mock_panoptes_lock.py

@@ -2,6 +2,7 @@
 Copyright 2018, Oath Inc.
 Licensed under the terms of the Apache 2.0 license. See LICENSE file in project root for terms.
 """
+from builtins import object
 
 
 class MockPanoptesLock(object):

tests/mock_panoptes_producer.py

@@ -2,13 +2,14 @@
 Copyright 2019, Verizon Media Inc.
 Licensed under the terms of the Apache 2.0 license. See LICENSE file in project root for terms.
 """
+from builtins import object
 
 
 class MockPanoptesMessageProducer(object):
 
     def __init__(self):
         self._kafka_client = dict()
-        self._kafka_client['stopped'] = False
+        self._kafka_client[u'stopped'] = False
         self._kafka_producer = []
 
     def __del__(self):
@@ -27,18 +28,18 @@ def _send_messages(self, topic, partition, messages, key):
     def send_messages(self, topic, key, messages, partitioning_key=None):
 
         self._kafka_producer.append({
-            'topic': topic,
-            'key': key,
-            'message': messages
+            u'topic': topic,
+            u'key': key,
+            u'message': messages
         })
 
     def ensure_topic_exists(self, topic):
         return True
 
     def stop(self):
 
-        if not self._kafka_client['stopped']:
-            self._kafka_client['stopped'] = True
+        if not self._kafka_client[u'stopped']:
+            self._kafka_client[u'stopped'] = True
 
 
 class MockPanoptesKeyedProducer(MockPanoptesMessageProducer):

tests/mock_redis.py

@@ -12,7 +12,7 @@ def __init__(self, bad_connection=False, timeout=False, **kwargs):
         if bad_connection:
             raise ConnectionError
         super(PanoptesMockRedis, self).__init__(**kwargs)
-        self.connection_pool = 'mockredis connection pool'
+        self.connection_pool = u'mockredis connection pool'
         self.timeout = timeout
 
     def get(self, key):

tests/mock_requests_responses.py

@@ -2,18 +2,19 @@
 Copyright 2018, Oath Inc.
 Licensed under the terms of the Apache 2.0 license. See LICENSE file in project root for terms.
 """
+from builtins import object
 
 
 class MockRequestsResponseBadRequest(object):
     status_code = 400
-    content = 'Bad Request'
+    content = u'Bad Request'
 
 
 class MockRequestsResponseServerFailure(object):
     status_code = 500
-    content = 'Internal Server Failure'
+    content = u'Internal Server Failure'
 
 
 class MockRequestsResponseOK(object):
     status_code = 200
-    content = 'OK'
+    content = u'OK'

tests/plugins/enrichment/generic/snmp/cisco/ios/test_plugin_enrichment_cisco_ios_device_metrics.py

@@ -37,14 +37,14 @@ class TestPluginCiscoIOSEnrichment(helpers.SNMPEnrichmentPluginTestFramework, un
     results_data_file = '4948E.results.json'
     resource_backplane = 'backplane'
     resource_model = '4948E'
-    plugin_conf = {'Core': {'name': 'Test Plugin', 'module': 'test_plugin'},
-                   'main': {'execute_frequency': '60', 'enrichment_ttl': '300',
-                            'resource_filter': 'resource_class = "network" AND resource_type = "cisco" AND '
-                                               'resource_metadata.os_name LIKE "CISCO IOS%"',
-                            'polling_frequency': '300'},
-                   'snmp': {'timeout': 5, 'retries': 2},
-                   'enrichment': {'preload': 'self:metrics'},
-                   'x509': {'x509_secured_requests': 0}
+    plugin_conf = {u'Core': {u'name': u'Test Plugin', u'module': u'test_plugin'},
+                   u'main': {u'execute_frequency': 60, u'enrichment_ttl': 300,
+                             u'resource_filter': u'resource_class = "network" AND resource_type = "cisco" AND '
+                                                 u'resource_metadata.os_name LIKE "CISCO IOS%"',
+                             u'polling_frequency': 300},
+                   u'snmp': {u'timeout': 5, u'retries': 2},
+                   u'enrichment': {u'preload': u'self:metrics'},
+                   u'x509': {u'x509_secured_requests': 0}
                    }
     plugin_class = plugin_enrichment_cisco_ios_device_metrics.CiscoIOSPluginEnrichmentMetrics
     use_enrichment = False

tests/plugins/enrichment/generic/snmp/cisco/nxos/test_plugin_enrichment_cisco_nxos_device_metrics.py

@@ -37,15 +37,15 @@ class TestPluginCiscoNXOSEnrichment(helpers.SNMPEnrichmentPluginTestFramework, u
     snmp_community = '3048'
     results_data_file = '3048.results.json'
     resource_backplane = 'backplane'
-    plugin_conf = {'Core': {'name': 'Test Plugin', 'module': 'test_plugin'},
-                   'main': {'execute_frequency': '60', 'enrichment_ttl': '300',
-                            'resource_filter': 'resource_class = "network" AND resource_type = "cisco" AND '
-                                               'resource_metadata.os_name LIKE "CISCO NX-OS%"',
-                            'polling_frequency': '300',
-                            'n3k_models': ["3048T", "3064-X", "3064"]},
-                   'snmp': {'timeout': 5, 'retries': 2},
-                   'enrichment': {'preload': 'self:metrics'},
-                   'x509': {'x509_secured_requests': 0}
+    plugin_conf = {u'Core': {u'name': u'Test Plugin', u'module': u'test_plugin'},
+                   u'main': {u'execute_frequency': 60, u'enrichment_ttl': 300,
+                             u'resource_filter': u'resource_class = "network" AND resource_type = "cisco" AND '
+                                                 u'resource_metadata.os_name LIKE "CISCO NX-OS%"',
+                             u'polling_frequency': 300,
+                             u'n3k_models': [u"3048T", u"3064-X", u"3064"]},
+                   u'snmp': {u'timeout': 5, u'retries': 2},
+                   u'enrichment': {u'preload': u'self:metrics'},
+                   u'x509': {u'x509_secured_requests': 0}
                    }
     plugin_class = plugin_enrichment_cisco_nxos_device_metrics.CiscoNXOSPluginEnrichmentMetrics
     use_enrichment = False

tests/plugins/enrichment/generic/snmp/juniper/data/ex_results.json

@@ -112,7 +112,7 @@
               },
               "cpu_util": {
                 "method": "bulk_walk",
-                "oid": ".1.3.6.1.4.1.2636.3.1.13.1.22"
+                "oid": ".1.3.6.1.4.1.2636.3.1.13.1.21"
               },
               "fans": {
                 "method": "static",

tests/plugins/enrichment/generic/snmp/juniper/data/mx2020_results.json

@@ -388,7 +388,7 @@
               },
               "cpu_util": {
                 "method": "bulk_walk",
-                "oid": ".1.3.6.1.4.1.2636.3.1.13.1.22"
+                "oid": ".1.3.6.1.4.1.2636.3.1.13.1.21"
               },
               "fans": {
                 "method": "static",

tests/plugins/enrichment/generic/snmp/juniper/data/qfx_results.json

@@ -110,7 +110,7 @@
               },
               "cpu_util": {
                 "method": "bulk_walk",
-                "oid": ".1.3.6.1.4.1.2636.3.1.13.1.22"
+                "oid": ".1.3.6.1.4.1.2636.3.1.13.1.21"
               },
               "fans": {
                 "method": "static",

tests/plugins/enrichment/generic/snmp/juniper/data/qfx_simulate_power_timeout_results.json

@@ -89,7 +89,7 @@
               },
               "cpu_util": {
                 "method": "bulk_walk",
-                "oid": ".1.3.6.1.4.1.2636.3.1.13.1.22"
+                "oid": ".1.3.6.1.4.1.2636.3.1.13.1.21"
               },
               "fans": {
                 "method": "static",