From ed3910b96f6e50ff75493699229ed2688ed9c489 Mon Sep 17 00:00:00 2001 From: Daniel Wang Date: Fri, 8 Mar 2019 15:24:41 -0800 Subject: [PATCH] Python 3 migration: BREAKING CHANGE All in on Python 3 for Python 2.7 EOL in 2020 --- .circleci/config.yml | 2 +- configs/config.dev.yaml | 3 +- ops/entrypoint.py | 18 +++--- setup.py | 12 ++-- src/iris/__main__.py | 2 +- src/iris/api.py | 91 ++++++++++++++-------------- src/iris/app_stats.py | 6 +- src/iris/bin/app_stats.py | 4 +- src/iris/bin/owasync.py | 2 +- src/iris/bin/run_server.py | 5 +- src/iris/bin/sender.py | 44 +++++++------- src/iris/bin/sync_targets.py | 6 +- src/iris/cache.py | 1 - src/iris/config.py | 4 +- src/iris/coordinator/kazoo.py | 2 - src/iris/coordinator/noncluster.py | 2 - src/iris/gmail.py | 4 +- src/iris/metrics/__init__.py | 2 +- src/iris/metrics/influx.py | 2 +- src/iris/metrics/prometheus.py | 2 +- src/iris/plugins/core.py | 3 +- src/iris/sender/cache.py | 20 +++--- src/iris/sender/oneclick.py | 7 ++- src/iris/sender/quota.py | 10 +-- src/iris/sender/rpc.py | 4 +- src/iris/sphinx_extension.py | 4 +- src/iris/ui/__init__.py | 4 +- src/iris/ui/auth/ldap.py | 1 - src/iris/utils.py | 12 ++-- src/iris/vendors/__init__.py | 8 +-- src/iris/vendors/iris_twilio.py | 8 +-- src/iris/webhooks/alertmanager.py | 2 - src/iris/webhooks/grafana.py | 2 - test/e2eserver.py | 2 +- test/e2etest.py | 44 +++++++------- test/test_config.py | 2 +- test/test_coordinator.py | 18 +++--- test/test_iris_vendor_hipchat.py | 6 +- test/test_iris_vendor_messagebird.py | 2 +- test/test_iris_vendor_slack.py | 2 +- test/test_iris_vendor_smtp.py | 2 +- test/test_iris_vendor_twilio.py | 6 +- test/test_irisapi.py | 44 +++++++------- test/test_sender.py | 62 +++++++++---------- 44 files changed, 236 insertions(+), 253 deletions(-) diff --git a/.circleci/config.yml b/.circleci/config.yml index 8b4b5814..109e15bf 100644 --- a/.circleci/config.yml +++ b/.circleci/config.yml @@ -4,7 +4,7 @@ jobs: services: - docker docker: - - image: circleci/python:2.7-stretch-browsers + - image: circleci/python:3.7-stretch-browsers - image: mysql/mysql-server:5.7 environment: - MYSQL_ALLOW_EMPTY_PASSWORD=1 diff --git a/configs/config.dev.yaml b/configs/config.dev.yaml index 1949b0b6..feb086d8 100644 --- a/configs/config.dev.yaml +++ b/configs/config.dev.yaml @@ -55,7 +55,8 @@ qr_login_url: Null ## Refer to their class name in this array. ## The webhook will be available at /v0/webhooks/$name and requires being called with ## ?application=xxx&key=xxx -#webhooks: [] +webhooks: + - alertmanager ##################### ### Iris-sender diff --git a/ops/entrypoint.py b/ops/entrypoint.py index 82b983fa..4a8455da 100644 --- a/ops/entrypoint.py +++ b/ops/entrypoint.py @@ -14,7 +14,7 @@ def load_sqldump(config, sqlfile, one_db=True): - print 'Importing %s...' % sqlfile + print('Importing %s...' % sqlfile) with open(sqlfile) as h: cmd = ['/usr/bin/mysql', '-h', config['host'], '-u', config['user'], '-p' + config['password']] @@ -24,17 +24,17 @@ def load_sqldump(config, sqlfile, one_db=True): proc.communicate() if proc.returncode == 0: - print 'DB successfully loaded ' + sqlfile + print('DB successfully loaded ' + sqlfile) return True else: - print ('Ran into problems during DB bootstrap. ' + print(('Ran into problems during DB bootstrap. ' 'IRIS will likely not function correctly. ' - 'mysql exit code: %s for %s') % (proc.returncode, sqlfile) + 'mysql exit code: %s for %s') % (proc.returncode, sqlfile)) return False def wait_for_mysql(config): - print 'Checking MySQL liveness on %s...' % config['host'] + print('Checking MySQL liveness on %s...' % config['host']) db_address = (config['host'], 3306) tries = 0 while True: @@ -45,17 +45,17 @@ def wait_for_mysql(config): break except socket.error: if tries > 20: - print 'Waited too long for DB to come up. Bailing.' + print('Waited too long for DB to come up. Bailing.') sys.exit(1) - print 'DB not up yet. Waiting a few seconds..' + print('DB not up yet. Waiting a few seconds..') time.sleep(2) tries += 1 continue def initialize_mysql_schema(config): - print 'Initializing Iris database' + print('Initializing Iris database') # disable one_db to let schema_0.sql create the database re = load_sqldump(config, os.path.join(dbpath, 'schema_0.sql'), one_db=False) if not re: @@ -71,7 +71,7 @@ def initialize_mysql_schema(config): sys.stderr.write('Failed to load dummy data.') with open(initializedfile, 'w'): - print 'Wrote %s so we don\'t bootstrap db again' % initializedfile + print('Wrote %s so we don\'t bootstrap db again' % initializedfile) def main(): diff --git a/setup.py b/setup.py index 1179f674..bb1f7873 100644 --- a/setup.py +++ b/setup.py @@ -15,20 +15,20 @@ include_package_data=True, install_requires=[ 'streql==3.0.2', - 'dnspython==1.14.0', + 'dnspython==1.16.0', 'phonenumbers==7.4.1', 'twilio==4.5.0', 'google-api-python-client==1.4.2', 'oauth2client==1.4.12', 'slackclient==0.16', - 'PyYAML==3.11', - 'gevent==1.1.2', + 'PyYAML==3.13', + 'gevent==1.4.0', 'falcon==1.1.0', 'falcon-cors==1.1.2', 'ujson==1.35', 'requests==2.20.0', - 'PyMySQL==0.7.2', - 'SQLAlchemy==1.0.11', + 'PyMySQL==0.9.3', + 'SQLAlchemy==1.3.0', 'Jinja2==2.10', 'importlib==1.0.3', 'Markdown==2.4.1', @@ -47,7 +47,7 @@ 'pyqrcode==1.2.1' ], extras_require={ - 'kazoo': ['kazoo==2.3.1'], + 'kazoo': ['kazoo==2.6.1'], # plugin deps 'influxdb': ['influxdb'], 'prometheus': ['prometheus_client'], diff --git a/src/iris/__main__.py b/src/iris/__main__.py index 3c86abff..f5100962 100644 --- a/src/iris/__main__.py +++ b/src/iris/__main__.py @@ -15,5 +15,5 @@ app = API(config) logging.basicConfig() server = config['server'] - print 'LISTENING: %(host)s:%(port)d' % server + print('LISTENING: %(host)s:%(port)d' % server) WSGIServer((server['host'], server['port']), app).serve_forever() diff --git a/src/iris/api.py b/src/iris/api.py index 7cfafb86..a387d405 100644 --- a/src/iris/api.py +++ b/src/iris/api.py @@ -1,8 +1,5 @@ # Copyright (c) LinkedIn Corporation. All rights reserved. Licensed under the BSD-2 Clause license. # See LICENSE in the project root for license information. - -from __future__ import absolute_import - from gevent import spawn, sleep, socket import msgpack @@ -16,7 +13,7 @@ import logging import jinja2 from jinja2.sandbox import SandboxedEnvironment -from urlparse import parse_qs +from urllib.parse import parse_qs import ujson from falcon import (HTTP_200, HTTP_201, HTTP_204, HTTPBadRequest, HTTPNotFound, HTTPUnauthorized, HTTPForbidden, HTTPFound, @@ -662,7 +659,7 @@ def gen_where_filter_clause(connection, filters, filter_types, kwargs): 4. (optional) transform escaped value through filter_escaped_value_transforms[col](value) ''' where = [] - for key, values in kwargs.iteritems(): + for key, values in kwargs.items(): col, _, op = key.partition('__') # Skip columns that don't exist if col not in filters: @@ -670,7 +667,7 @@ def gen_where_filter_clause(connection, filters, filter_types, kwargs): col_type = filter_types.get(col, str) # Format strings because Falcon splits on ',' but not on '%2C' # TODO: Get rid of this by setting request options on Falcon 1.1 - if isinstance(values, basestring): + if isinstance(values, str): values = values.split(',') for val in values: try: @@ -804,7 +801,7 @@ def process_resource(self, req, resp, resource, params): # pragma: no cover qs = req.env['QUERY_STRING'] if qs: path = path + '?' + qs - body = req.context['body'] + body = req.context['body'].decode('utf-8') auth = req.get_header('AUTHORIZATION') if auth and auth.startswith('hmac '): username_header = req.get_header('X-IRIS-USERNAME') @@ -825,9 +822,9 @@ def process_resource(self, req, resp, resource, params): # pragma: no cover text = '%s %s %s %s %s' % (window, method, path, body, username_header) else: text = '%s %s %s %s' % (window, method, path, body) - HMAC = hmac.new(api_key, text, hashlib.sha512) + HMAC = hmac.new(api_key.encode('utf-8'), text.encode('utf-8'), hashlib.sha512) digest = base64.urlsafe_b64encode(HMAC.digest()) - if equals(client_digest, digest): + if equals(client_digest.encode('utf-8'), digest): req.context['app'] = app if username_header: req.context['username'] = username_header @@ -838,9 +835,9 @@ def process_resource(self, req, resp, resource, params): # pragma: no cover text = '%s %s %s %s %s' % (window - 1, method, path, body, username_header) else: text = '%s %s %s %s' % (window - 1, method, path, body) - HMAC = hmac.new(api_key, text, hashlib.sha512) + HMAC = hmac.new(api_key.encode('utf-8'), text.encode('utf-8'), hashlib.sha512) digest = base64.urlsafe_b64encode(HMAC.digest()) - if equals(client_digest, digest): + if equals(client_digest.encode('utf-8'), digest): req.context['app'] = app if username_header: req.context['username'] = username_header @@ -2023,13 +2020,13 @@ def on_post(self, req, resp): # message body, which happens if both body and template are not # specified, or if we don't have email_html if 'template' in message: - if not isinstance(message['template'], basestring): + if not isinstance(message['template'], str): raise HTTPBadRequest('template needs to be a string') elif 'body' in message: - if not isinstance(message['body'], basestring): + if not isinstance(message['body'], str): raise HTTPBadRequest('body needs to be a string') elif 'email_html' in message: - if not isinstance(message['email_html'], basestring): + if not isinstance(message['email_html'], str): raise HTTPBadRequest('email_html needs to be a string') # Handle the edge-case where someone is only specifying email_html # and not the others. Avoid KeyError's later on in sender @@ -2207,8 +2204,8 @@ def on_post(self, req, resp): content = template_params.pop('content') contents = [] template_env = SandboxedEnvironment(autoescape=True) - for _application, modes in content.iteritems(): - for _mode, _content in modes.iteritems(): + for _application, modes in content.items(): + for _mode, _content in modes.items(): _content['mode'] = _mode _content['application'] = _application try: @@ -2292,7 +2289,7 @@ def on_post(self, req, resp, username): # Configure priority -> mode for a single application if app is not None: - for p, m in mode_params.iteritems(): + for p, m in mode_params.items(): if m != 'default': session.execute(insert_target_application_modes_query, {'name': username, 'priority': p, 'mode': m, 'app': app}) @@ -2304,8 +2301,8 @@ def on_post(self, req, resp, username): # Configure priority -> mode for multiple applications in one call (avoid MySQL deadlocks) elif multiple_apps is not None: - for app, app_modes in multiple_apps.iteritems(): - for p, m in app_modes.iteritems(): + for app, app_modes in multiple_apps.items(): + for p, m in app_modes.items(): if m != 'default': session.execute(insert_target_application_modes_query, {'name': username, 'priority': p, 'mode': m, 'app': app}) @@ -2314,7 +2311,7 @@ def on_post(self, req, resp, username): {'name': username, 'priority': p, 'app': app}) # Also configure global defaults in the same call if they're specified - for p in mode_params.viewkeys() & modes.viewkeys(): + for p in mode_params.keys() & modes.keys(): m = mode_params[p] if m != 'default': session.execute(insert_user_modes_query, @@ -2326,7 +2323,7 @@ def on_post(self, req, resp, username): # Configure user's global priority -> mode which covers all # applications that don't have defaults set else: - for p, m in mode_params.iteritems(): + for p, m in mode_params.items(): if m != 'default': session.execute(insert_user_modes_query, {'name': username, 'priority': p, 'mode': m}) @@ -2675,8 +2672,8 @@ def on_put(self, req, resp, app_name): JOIN `priority` on `priority`.`id` = `default_application_mode`.`priority_id` WHERE `default_application_mode`.`application_id` = :application_id''', {'application_id': app['id']})} - kill_priorities = existing_priorities - default_modes.viewkeys() - for priority, mode in default_modes.iteritems(): + kill_priorities = existing_priorities - default_modes.keys() + for priority, mode in default_modes.items(): # If we disabled this mode for this app in the code block # above, avoid the expected integrity error here by bailing # early @@ -2786,7 +2783,7 @@ def on_post(self, req, resp, app_name): except ValueError: raise HTTPBadRequest('Invalid json in post body') - if data.viewkeys() != required_quota_keys: + if data.keys() != required_quota_keys: raise HTTPBadRequest('Missing required keys in post body') try: @@ -3102,7 +3099,7 @@ def on_put(self, req, resp, app_name): {'app_name': app_name, 'email_addresses': email_addresses}) # Configure new/existing ones - for email_address, plan_name in email_to_plans.iteritems(): + for email_address, plan_name in email_to_plans.items(): # If this plan does not have steps that support this app, block this app_template_count = session.execute(''' SELECT EXISTS ( @@ -3254,7 +3251,7 @@ def on_get(self, req, resp, app_name): fields = [f for f in fields if f in plan_columns] if fields else None req.params.pop('fields', None) if not fields: - fields = plan_columns.keys() + fields = list(plan_columns.keys()) connection = db.engine.raw_connection() cursor = connection.cursor(db.dict_cursor) @@ -3605,7 +3602,7 @@ def create_response(self, msg_id, source, content): def create_email_message(self, application, dest, subject, body): if application not in cache.applications: - return False, 'Application "%s" not found in %s.' % (application, cache.applications.keys()) + return False, 'Application "%s" not found in %s.' % (application, list(cache.applications.keys())) app = cache.applications[application] @@ -3671,12 +3668,12 @@ def validate_app(app): with db.guarded_session() as session: is_batch = False is_claim_all = False - if isinstance(msg_id, int) or (isinstance(msg_id, basestring) and msg_id.isdigit()): + if isinstance(msg_id, int) or (isinstance(msg_id, str) and msg_id.isdigit()): # FIXME: return error if message not found for id app = get_app_from_msg_id(session, msg_id) validate_app(app) self.create_response(msg_id, source, content) - elif isinstance(msg_id, basestring) and uuid4hex.match(msg_id): + elif isinstance(msg_id, str) and uuid4hex.match(msg_id): # msg id is not pure digit, might be a batch id sql = 'SELECT message.id FROM message WHERE message.batch=:batch_id' results = session.execute(sql, {'batch_id': msg_id}) @@ -3705,7 +3702,7 @@ def validate_app(app): apps_to_message[msg_app].append(mid) # Case where we want to give back a custom message as there was nothing to claim - elif msg_id is None and isinstance(content, basestring): + elif msg_id is None and isinstance(content, str): session.close() return '', content else: @@ -3717,20 +3714,20 @@ def validate_app(app): try: plugin_output = { app: find_plugin(app).handle_response(mode, msg_ids, source, content, batch=is_batch) - for app, msg_ids in apps_to_message.iteritems() + for app, msg_ids in apps_to_message.items() } except Exception as e: logger.exception( 'Failed to handle %s response for mode %s for apps %s during claim all', - content, mode, apps_to_message.keys()) + content, mode, list(apps_to_message.keys())) raise HTTPBadRequest('Failed to handle response', 'failed to handle response: %s' % str(e)) if len(plugin_output) > 1: return plugin_output, '\n'.join('%s: %s' % (app, output) - for app, output in plugin_output.iteritems()) + for app, output in plugin_output.items()) else: - return plugin_output, '\n'.join(plugin_output.itervalues()) + return plugin_output, '\n'.join(plugin_output.values()) else: try: @@ -3755,7 +3752,7 @@ def on_post(self, req, resp): raise HTTPBadRequest('Missing source', msg) to = email_headers.get('To', []) # 'To' will either be string of single recipient or list of several - if isinstance(to, basestring): + if isinstance(to, str): to = [to] # source is in the format of "First Last ", # but we only want the email part @@ -3824,7 +3821,7 @@ def on_post(self, req, resp): session.close() resp.status = HTTP_204 # Pass the new incident id back through a header so we can test this - resp.set_header('X-IRIS-INCIDENT', incident_id) + resp.set_header('X-IRIS-INCIDENT', str(incident_id)) return session.close() @@ -3845,7 +3842,7 @@ def on_post(self, req, resp): # When processing a claim all scenario, the first item returned by handle_user_response # will be a dict mapping the app to its plugin output. if isinstance(app, dict): - for app_name, app_response in app.iteritems(): + for app_name, app_response in app.items(): app_response = '%s: %s' % (app_name, app_response) success, re = self.create_email_message( app_name, source, 'Re: %s' % subject, app_response) @@ -3892,14 +3889,14 @@ def on_post(self, req, resp): post_dict = parse_qs(req.context['body']) msg_id = req.get_param('message_id', required=True) - if 'Digits' not in post_dict: + if b'Digits' not in post_dict: raise HTTPBadRequest('Digits argument not found') # For phone call callbacks, To argument is the target and From is the # twilio number - if 'To' not in post_dict: + if b'To' not in post_dict: raise HTTPBadRequest('To argument not found') - digits = post_dict['Digits'][0] - source = post_dict['To'][0] + digits = post_dict[b'Digits'][0].decode('utf-8') + source = post_dict[b'To'][0].decode('utf-8') try: _, response = self.handle_user_response('call', msg_id, source, digits) @@ -3914,13 +3911,13 @@ def on_post(self, req, resp): class ResponseTwilioMessages(ResponseMixin): def on_post(self, req, resp): post_dict = parse_qs(req.context['body']) - if 'Body' not in post_dict: + if b'Body' not in post_dict: raise HTTPBadRequest('SMS body not found', 'Missing Body argument in post body') - if 'From' not in post_dict: + if b'From' not in post_dict: raise HTTPBadRequest('From argument not found', 'Missing From in post body') - source = post_dict['From'][0] - body = post_dict['Body'][0] + source = post_dict[b'From'][0].decode('utf-8') + body = post_dict[b'Body'][0].decode('utf-8') try: msg_id, content = utils.parse_response(body.strip(), 'sms', source) except (ValueError, IndexError): @@ -3963,7 +3960,7 @@ class TwilioDeliveryUpdate(object): allow_read_no_auth = False def on_post(self, req, resp): - post_dict = falcon.uri.parse_query_string(req.context['body']) + post_dict = falcon.uri.parse_query_string(req.context['body'].decode('utf-8')) sid = post_dict.get('MessageSid', post_dict.get('CallSid')) status = post_dict.get('MessageStatus', post_dict.get('CallStatus')) @@ -3977,7 +3974,7 @@ def on_post(self, req, resp): cursor = connection.cursor() try: max_retries = 3 - for i in xrange(max_retries): + for i in range(max_retries): try: affected = cursor.execute( '''UPDATE `twilio_delivery_status` diff --git a/src/iris/app_stats.py b/src/iris/app_stats.py index 49fc48e8..eae48c1b 100644 --- a/src/iris/app_stats.py +++ b/src/iris/app_stats.py @@ -53,7 +53,7 @@ def calculate_app_stats(app, connection, cursor, fields_filter=None): query_data = {'application_id': app['id']} stats = {} - fields = queries.viewkeys() + fields = queries.keys() if fields_filter: fields &= set(fields_filter) for key in fields: @@ -98,7 +98,7 @@ def calculate_app_stats(app, connection, cursor, fields_filter=None): logger.info('App Stats (%s) mode status query took %s seconds', app['name'], round(time.time() - start, 2)) for mode, status, count in cursor: - if isinstance(status, basestring) and status.isdigit(): + if isinstance(status, str) and status.isdigit(): status = int(status) mode_status[mode][status] = count @@ -181,7 +181,7 @@ def calculate_global_stats(connection, cursor, fields_filter=None): } stats = {} - fields = queries.viewkeys() + fields = queries.keys() if fields_filter: fields &= set(fields_filter) diff --git a/src/iris/bin/app_stats.py b/src/iris/bin/app_stats.py index e6f0e1c9..031719fc 100644 --- a/src/iris/bin/app_stats.py +++ b/src/iris/bin/app_stats.py @@ -41,7 +41,7 @@ def set_global_stats(stats, connection, cursor): - for stat, val in stats.iteritems(): + for stat, val in stats.items(): if val is not None: cursor.execute('''INSERT INTO `global_stats` (`statistic`, `value`, `timestamp`) VALUES (%s, %s, NOW()) @@ -51,7 +51,7 @@ def set_global_stats(stats, connection, cursor): def set_app_stats(app, stats, connection, cursor): - for stat, val in stats.iteritems(): + for stat, val in stats.items(): if val is not None: cursor.execute('''INSERT INTO `application_stats` (`application_id`, `statistic`, `value`, `timestamp`) VALUES (%s, %s, %s, NOW()) diff --git a/src/iris/bin/owasync.py b/src/iris/bin/owasync.py index 377d468e..0dd3da32 100644 --- a/src/iris/bin/owasync.py +++ b/src/iris/bin/owasync.py @@ -164,7 +164,7 @@ def relay(message, iris_client): # of the created incident; otherwise, the header will not exist or it will be a textual # error message. incident_header = req.headers.get('X-IRIS-INCIDENT') - if isinstance(incident_header, basestring) and incident_header.isdigit(): + if isinstance(incident_header, str) and incident_header.isdigit(): metrics.incr('incident_created_count') else: diff --git a/src/iris/bin/run_server.py b/src/iris/bin/run_server.py index d60a0c4a..a32cb0b2 100755 --- a/src/iris/bin/run_server.py +++ b/src/iris/bin/run_server.py @@ -11,6 +11,7 @@ from gunicorn.six import iteritems import iris import iris.config +import imp class StandaloneApplication(gunicorn.app.base.BaseApplication): @@ -28,8 +29,8 @@ def load_config(self): def load(self): import iris - reload(iris) - reload(iris.config) + imp.reload(iris) + imp.reload(iris.config) config = iris.config.load_config(sys.argv[1]) import iris.api diff --git a/src/iris/bin/sender.py b/src/iris/bin/sender.py index e31933cb..59b791ba 100644 --- a/src/iris/bin/sender.py +++ b/src/iris/bin/sender.py @@ -387,7 +387,7 @@ def deactivate(): max_retries = 3 # this deadlocks sometimes. try until it doesn't. - for i in xrange(1, max_retries + 1): + for i in range(1, max_retries + 1): try: cursor.execute(GET_INACTIVE_IDS_SQL) ids = tuple(r[0] for r in cursor) @@ -499,7 +499,7 @@ def escalate(): else: escalations[n['incident_id']] = (n['plan_id'], n['current_step'] + 1) - for incident_id, (plan_id, step) in escalations.iteritems(): + for incident_id, (plan_id, step) in escalations.items(): plan = cache.plans[plan_id] steps = plan['steps'].get(step, []) if steps: @@ -535,7 +535,7 @@ def aggregate(now): all_actives = {r[0] for r in cursor} cursor.close() connection.close() - for key in queues.keys(): + for key in list(queues.keys()): aggregation_window = cache.plans[key[0]]['aggregation_window'] if now - sent.get(key, 0) >= aggregation_window: aggregated_message_ids = queues[key] @@ -561,7 +561,7 @@ def aggregate(now): logger.info('[-] purged %s from messages %s remaining', active_message_ids, len(messages)) del queues[key] sent[key] = now - inactive_message_ids = messages.viewkeys() - all_actives + inactive_message_ids = messages.keys() - all_actives logger.info('[x] dropped %s inactive messages from claimed incidents, %s remain', len(inactive_message_ids), len(messages)) @@ -650,13 +650,13 @@ def fetch_and_prepare_message(): # does this message trigger aggregation? window = plan_aggregate_windows.setdefault(key, defaultdict(int)) - for bucket in window.keys(): + for bucket in list(window.keys()): if now - bucket > plan['threshold_window']: del window[bucket] window[now] += 1 - if sum(window.itervalues()) > plan['threshold_count']: + if sum(window.values()) > plan['threshold_count']: # too many messages for the aggregation key - enqueue # add message id to aggregation queue @@ -918,7 +918,7 @@ def mark_message_as_sent(message): max_retries = 3 # this deadlocks sometimes. try until it doesn't. - for i in xrange(1, max_retries + 1): + for i in range(1, max_retries + 1): try: cursor.execute(sql, params) connection.commit() @@ -954,7 +954,7 @@ def mark_message_as_sent(message): max_retries = 3 # this deadlocks sometimes. try until it doesn't. - for i in xrange(1, max_retries + 1): + for i in range(1, max_retries + 1): try: cursor.execute(UPDATE_MESSAGE_BODY_SQL, (message['body'], message['subject'], update_ids)) connection.commit() @@ -1258,12 +1258,12 @@ def maintain_workers(config): # Remove all smtp vendor objects so they don't get initialized unnecessarily config['vendors'] = [vendor for vendor in config['vendors'] if vendor['type'] != 'iris_smtp'] - logger.info('Workers per mode: %s', ', '.join('%s: %s' % count for count in workers_per_mode.iteritems())) + logger.info('Workers per mode: %s', ', '.join('%s: %s' % count for count in iter(workers_per_mode.items()))) # Make sure all the counts and distributions for "normal" workers are proper, including email if we're not doing MX record # autoscaling - for mode, worker_count in workers_per_mode.iteritems(): + for mode, worker_count in workers_per_mode.items(): mode_tasks = worker_tasks[mode] if mode_tasks: for task in mode_tasks: @@ -1273,7 +1273,7 @@ def maintain_workers(config): task.update({'greenlet': spawn(worker, per_mode_send_queues[mode], config, kill_set), 'kill_set': kill_set}) metrics.incr('workers_respawn_cnt') else: - for x in xrange(worker_count): + for x in range(worker_count): kill_set = gevent.event.Event() mode_tasks.append({'greenlet': spawn(worker, per_mode_send_queues[mode], config, kill_set), 'kill_set': kill_set}) @@ -1284,7 +1284,7 @@ def maintain_workers(config): tasks_to_kill = [] # Adjust worker count - for mx, correct_worker_count in email_smtp_workers.iteritems(): + for mx, correct_worker_count in email_smtp_workers.items(): mx_workers = autoscale_email_worker_tasks[mx] current_task_count = len(mx_workers) @@ -1305,7 +1305,7 @@ def maintain_workers(config): worker_config = copy.deepcopy(config) worker_config['vendors'] = [email_vendor_config] - for x in xrange(new_task_count): + for x in range(new_task_count): kill_set = gevent.event.Event() mx_workers.append({'greenlet': spawn(worker, email_queue, worker_config, kill_set), 'kill_set': kill_set}) @@ -1313,14 +1313,14 @@ def maintain_workers(config): elif current_task_count > correct_worker_count: kill_task_count = current_task_count - correct_worker_count logger.info('Auto scaling MX record %s DOWN %d tasks', mx, kill_task_count) - for x in xrange(kill_task_count): + for x in range(kill_task_count): try: tasks_to_kill.append(mx_workers.pop()) except IndexError: break # Kill MX records no longer in use - kill_mx = autoscale_email_worker_tasks.viewkeys() - email_smtp_workers.viewkeys() + kill_mx = autoscale_email_worker_tasks.keys() - email_smtp_workers.keys() for mx in kill_mx: workers = autoscale_email_worker_tasks[mx] if workers: @@ -1329,7 +1329,7 @@ def maintain_workers(config): del autoscale_email_worker_tasks[mx] # Make sure all existing workers are alive - for mx, mx_tasks in autoscale_email_worker_tasks.iteritems(): + for mx, mx_tasks in autoscale_email_worker_tasks.items(): email_vendor_config = copy.deepcopy(email_vendor) email_vendor_config['smtp_server'] = mx email_vendor_config.pop('smtp_gateway', None) @@ -1419,21 +1419,21 @@ def sender_shutdown(): # Stop sender RPC server rpc.shutdown() - for tasks in worker_tasks.itervalues(): + for tasks in worker_tasks.values(): for task in tasks: task['kill_set'].set() - for tasks in autoscale_email_worker_tasks.itervalues(): + for tasks in autoscale_email_worker_tasks.values(): for task in tasks: task['kill_set'].set() logger.info('Waiting for sender workers to shut down') - for tasks in worker_tasks.itervalues(): + for tasks in worker_tasks.values(): for task in tasks: task['greenlet'].join() - for tasks in autoscale_email_worker_tasks.itervalues(): + for tasks in autoscale_email_worker_tasks.values(): for task in tasks: task['greenlet'].join() @@ -1478,7 +1478,7 @@ def init_sender(config): process_title = config['sender'].get('process_title') - if process_title and isinstance(process_title, basestring): + if process_title and isinstance(process_title, str): setproctitle.setproctitle(process_title) logger.info('Changing process name to %s', process_title) @@ -1617,7 +1617,7 @@ def main(): metrics.incr('task_failure') send_task = spawn(send) - for mode, send_queue in per_mode_send_queues.iteritems(): + for mode, send_queue in per_mode_send_queues.items(): # Set metric for size of worker queue metrics.set('send_queue_%s_size' % mode, len(send_queue)) diff --git a/src/iris/bin/sync_targets.py b/src/iris/bin/sync_targets.py index 8ef5cd33..f3c51695 100644 --- a/src/iris/bin/sync_targets.py +++ b/src/iris/bin/sync_targets.py @@ -187,13 +187,13 @@ def sync_from_oncall(config, engine, purge_old_users=True): continue contacts[row.mode] = row.destination - iris_usernames = iris_users.viewkeys() + iris_usernames = iris_users.keys() # users from the oncall endpoints and config files metrics.set('users_found', len(oncall_users)) metrics.set('teams_found', len(oncall_team_names)) oncall_users.update(get_predefined_users(config)) - oncall_usernames = oncall_users.viewkeys() + oncall_usernames = oncall_users.keys() # set of users not presently in iris users_to_insert = oncall_usernames - iris_usernames @@ -225,7 +225,7 @@ def sync_from_oncall(config, engine, purge_old_users=True): logger.exception('Failed to add user %s' % username) continue metrics.incr('users_added') - for key, value in oncall_users[username].iteritems(): + for key, value in oncall_users[username].items(): if value and key in modes: logger.info('%s: %s -> %s' % (username, key, value)) engine.execute(target_contact_add_sql, (target_id, modes[key], value, value)) diff --git a/src/iris/cache.py b/src/iris/cache.py index fb85fd07..2f97392c 100644 --- a/src/iris/cache.py +++ b/src/iris/cache.py @@ -1,7 +1,6 @@ # Copyright (c) LinkedIn Corporation. All rights reserved. Licensed under the BSD-2 Clause license. # See LICENSE in the project root for license information. -from __future__ import absolute_import from . import db import logging diff --git a/src/iris/config.py b/src/iris/config.py index f371ef76..c402a7d4 100644 --- a/src/iris/config.py +++ b/src/iris/config.py @@ -15,8 +15,8 @@ def load_config_file(path=None): ''' Get config from path to file, defaulting to cli arg. This can easily be monkey patched. ''' if not path: if len(sys.argv) <= 1: - print 'ERROR: missing config file.' - print 'usage: %s API_CONFIG_FILE' % sys.argv[0] + print('ERROR: missing config file.') + print('usage: %s API_CONFIG_FILE' % sys.argv[0]) sys.exit(1) path = sys.argv[1] diff --git a/src/iris/coordinator/kazoo.py b/src/iris/coordinator/kazoo.py index 39f7c773..c79b05c6 100644 --- a/src/iris/coordinator/kazoo.py +++ b/src/iris/coordinator/kazoo.py @@ -1,8 +1,6 @@ # Copyright (c) LinkedIn Corporation. All rights reserved. Licensed under the BSD-2 Clause license. # See LICENSE in the project root for license information. -from __future__ import absolute_import - from kazoo.client import KazooClient, KazooState from kazoo.handlers.gevent import SequentialGeventHandler from kazoo.recipe.party import Party diff --git a/src/iris/coordinator/noncluster.py b/src/iris/coordinator/noncluster.py index 71e928f6..200e7437 100644 --- a/src/iris/coordinator/noncluster.py +++ b/src/iris/coordinator/noncluster.py @@ -1,8 +1,6 @@ # Copyright (c) LinkedIn Corporation. All rights reserved. Licensed under the BSD-2 Clause license. # See LICENSE in the project root for license information. -from __future__ import absolute_import - from itertools import cycle import logging logger = logging.getLogger(__name__) diff --git a/src/iris/gmail.py b/src/iris/gmail.py index a65983fe..affd5d43 100644 --- a/src/iris/gmail.py +++ b/src/iris/gmail.py @@ -171,9 +171,9 @@ def send_message( """ self.connect() if not body: - kwargs = dict({ + kwargs = dict(list({ 'sender': self.config.get('sub') - }.items() + kwargs.items()) + }.items()) + list(kwargs.items())) body = self.create_message(**kwargs) ret = {} try: diff --git a/src/iris/metrics/__init__.py b/src/iris/metrics/__init__.py index 6d885458..6668593d 100644 --- a/src/iris/metrics/__init__.py +++ b/src/iris/metrics/__init__.py @@ -40,7 +40,7 @@ def add_new_metrics(default_stats): stats_reset.update(default_stats) # avoid clobbering existing metrics if they are already present - for key, default_value in default_stats.iteritems(): + for key, default_value in default_stats.items(): stats.setdefault(key, default_value) diff --git a/src/iris/metrics/influx.py b/src/iris/metrics/influx.py index d62149e8..7f6bb936 100644 --- a/src/iris/metrics/influx.py +++ b/src/iris/metrics/influx.py @@ -33,7 +33,7 @@ def send_metrics(self, metrics): return now = str(datetime.now()) payload = [] - for metric, value in metrics.iteritems(): + for metric, value in metrics.items(): data = { 'measurement': self.appname, 'tags': {}, diff --git a/src/iris/metrics/prometheus.py b/src/iris/metrics/prometheus.py index fff43034..f590f369 100644 --- a/src/iris/metrics/prometheus.py +++ b/src/iris/metrics/prometheus.py @@ -31,7 +31,7 @@ def __init__(self, config, appname): def send_metrics(self, metrics): if not self.enable_metrics: return - for metric, value in metrics.iteritems(): + for metric, value in metrics.items(): if metric not in self.gauges: self.gauges[metric] = Gauge(self.appname + '_' + metric, '') self.gauges[metric].set_to_current_time() diff --git a/src/iris/plugins/core.py b/src/iris/plugins/core.py index fe30d122..88f64dff 100644 --- a/src/iris/plugins/core.py +++ b/src/iris/plugins/core.py @@ -3,7 +3,6 @@ # -*- coding:utf-8 -*- -from __future__ import absolute_import from .. import utils import logging logger = logging.getLogger(__name__) @@ -34,7 +33,7 @@ def __init__(self, config): def get_phone_menu_text(cls): if cls.phone_response_menu: return ' '.join([item['title'] for (_, item) in - cls.phone_response_menu.iteritems()]) + cls.phone_response_menu.items()]) else: return '' diff --git a/src/iris/sender/cache.py b/src/iris/sender/cache.py index f072732a..dc0b6699 100644 --- a/src/iris/sender/cache.py +++ b/src/iris/sender/cache.py @@ -1,8 +1,6 @@ # Copyright (c) LinkedIn Corporation. All rights reserved. Licensed under the BSD-2 Clause license. # See LICENSE in the project root for license information. -from __future__ import absolute_import - from collections import deque import jinja2 from jinja2.sandbox import SandboxedEnvironment @@ -56,7 +54,7 @@ def purge(self): connection = self.engine.raw_connection() cursor = connection.cursor() cursor.execute(self.active, [tuple(self.data)]) - for key in self.data.viewkeys() - {row[0] for row in cursor}: + for key in self.data.keys() - {row[0] for row in cursor}: del self.data[key] cursor.close() connection.close() @@ -169,8 +167,8 @@ def refresh(self): active = {item['id']: item['name'] for item in templates_response} - new_active_ids = active.viewkeys() - old_active_ids = self.active.viewkeys() + new_active_ids = active.keys() + old_active_ids = self.active.keys() old_ids = old_active_ids - new_active_ids new_ids = new_active_ids - old_active_ids @@ -220,7 +218,7 @@ def __getitem__(self, key): if plan['tracking_template']: tracking_template = plan['tracking_template'] if plan['tracking_type'] == 'email': - for application, application_templates in tracking_template.iteritems(): + for application, application_templates in tracking_template.items(): try: tracking_template[application] = { 'email_subject': self.template_env.from_string(application_templates['email_subject']), @@ -233,7 +231,7 @@ def __getitem__(self, key): logger.exception('[-] error parsing Plan template for %s: %s', key, application) continue else: - for application, application_templates in tracking_template.iteritems(): + for application, application_templates in tracking_template.items(): try: tracking_template[application] = { 'body': self.template_env.from_string(application_templates['body']), @@ -259,8 +257,8 @@ def refresh(self): active = {item['id']: item['name'] for item in plans_response} - new_active_ids = active.viewkeys() - old_active_ids = self.active.viewkeys() + new_active_ids = active.keys() + old_active_ids = self.active.keys() old_ids = old_active_ids - new_active_ids new_ids = new_active_ids - old_active_ids @@ -311,8 +309,8 @@ def refresh(self): cursor.close() connection.close() - current = rates.viewkeys() - old = self.rates.viewkeys() + current = rates.keys() + old = self.rates.keys() # purge old rate entries for key in old - current: diff --git a/src/iris/sender/oneclick.py b/src/iris/sender/oneclick.py index 3edd41ef..3d2aabac 100644 --- a/src/iris/sender/oneclick.py +++ b/src/iris/sender/oneclick.py @@ -3,7 +3,7 @@ import hmac import hashlib -import urllib +import urllib.parse import base64 oneclick_email_markup = ''' @@ -27,6 +27,7 @@ def generate_oneclick_url(config, data): keys = ('msg_id', 'email_address', 'cmd') # Order here needs to match order in iris-relay - HMAC = hmac.new(config['gmail_one_click_url_key'], ' '.join(str(data[key]) for key in keys), hashlib.sha512) + HMAC = hmac.new(config['gmail_one_click_url_key'].encode('utf-8'), + (' '.join(str(data[key]) for key in keys).encode('utf-8')), hashlib.sha512) data['token'] = base64.urlsafe_b64encode(HMAC.digest()) - return config['gmail_one_click_url_endpoint'] + '?' + urllib.urlencode(data) + return config['gmail_one_click_url_endpoint'] + '?' + urllib.parse.urlencode(data) diff --git a/src/iris/sender/quota.py b/src/iris/sender/quota.py index cd43c532..4b3c7703 100644 --- a/src/iris/sender/quota.py +++ b/src/iris/sender/quota.py @@ -93,10 +93,10 @@ def refresh(self): new_rates = {} for application, hard_limit, soft_limit, hard_duration, soft_duration, target_name, target_role, plan_name, wait_time in self.get_new_rules(): - new_rates[application] = (hard_limit, soft_limit, hard_duration / 60, soft_duration / 60, wait_time, plan_name, (target_name, target_role)) + new_rates[application] = (hard_limit, soft_limit, hard_duration // 60, soft_duration // 60, wait_time, plan_name, (target_name, target_role)) - old_keys = self.rates.viewkeys() - new_keys = new_rates.viewkeys() + old_keys = self.rates.keys() + new_keys = new_rates.keys() # Remove old application entries for key in old_keys - new_keys: @@ -158,7 +158,7 @@ def allow_send(self, message): hard_usage_pct = 0 if hard_limit > 0: - hard_usage_pct = (hard_quota_usage / hard_limit) * 100 + hard_usage_pct = (hard_quota_usage // hard_limit) * 100 metrics.set('app_%s_quota_hard_usage_pct' % application, hard_usage_pct) if hard_quota_usage > hard_limit: @@ -172,7 +172,7 @@ def allow_send(self, message): soft_usage_pct = 0 if soft_limit > 0: - soft_usage_pct = (soft_quota_usage / soft_limit) * 100 + soft_usage_pct = (soft_quota_usage // soft_limit) * 100 metrics.set('app_%s_quota_soft_usage_pct' % application, soft_usage_pct) if soft_quota_usage > soft_limit: diff --git a/src/iris/sender/rpc.py b/src/iris/sender/rpc.py index 4156c08d..bc1351ec 100644 --- a/src/iris/sender/rpc.py +++ b/src/iris/sender/rpc.py @@ -1,8 +1,6 @@ # Copyright (c) LinkedIn Corporation. All rights reserved. Licensed under the BSD-2 Clause license. # See LICENSE in the project root for license information. -from __future__ import absolute_import - import os from gevent import Timeout, socket from gevent.server import StreamServer @@ -116,7 +114,7 @@ def handle_api_notification_request(socket, address, req): # fill in dummy iris meta data notification['context']['iris'] = {} elif 'email_html' in notification: - if not isinstance(notification['email_html'], basestring): + if not isinstance(notification['email_html'], str): logger.warn('Dropping OOB message with invalid email_html from app %s: %s', notification['application'], notification['email_html']) reject_api_request(socket, address, 'INVALID email_html') diff --git a/src/iris/sphinx_extension.py b/src/iris/sphinx_extension.py index ce7d78b1..b1bdb309 100644 --- a/src/iris/sphinx_extension.py +++ b/src/iris/sphinx_extension.py @@ -1,8 +1,6 @@ #!/usr/bin/env python # -*- coding:utf-8 -*- -from __future__ import print_function - from docutils import nodes from docutils.statemachine import ViewList @@ -49,7 +47,7 @@ def make_rst(self, section_title_set): app = autohttp_import_object(self.arguments[0]) for method, path, handler in get_routes(app): docstring = handler.__doc__ - if not isinstance(docstring, unicode): + if not isinstance(docstring, str): analyzer = ModuleAnalyzer.for_module(handler.__module__) docstring = force_decode(docstring, analyzer.encoding) if not docstring and 'include-empty-docstring' not in self.options: diff --git a/src/iris/ui/__init__.py b/src/iris/ui/__init__.py index 4ea745f3..11b77f44 100644 --- a/src/iris/ui/__init__.py +++ b/src/iris/ui/__init__.py @@ -276,7 +276,7 @@ def on_get(self, req, resp): last_flash=get_flash(req)) def on_post(self, req, resp): - form_body = uri.parse_query_string(req.context['body']) + form_body = uri.parse_query_string(req.context['body'].decode('utf-8')) try: username = form_body['username'] @@ -348,7 +348,7 @@ class JinjaValidate(): frontend_route = True def on_post(self, req, resp): - form_body = uri.parse_query_string(req.context['body']) + form_body = uri.parse_query_string(req.context['body'].decode('utf-8')) try: template_subject = form_body['templateSubject'] diff --git a/src/iris/ui/auth/ldap.py b/src/iris/ui/auth/ldap.py index 40b147a5..1a6d45e9 100644 --- a/src/iris/ui/auth/ldap.py +++ b/src/iris/ui/auth/ldap.py @@ -1,7 +1,6 @@ # Copyright (c) LinkedIn Corporation. All rights reserved. Licensed under the BSD-2 Clause license. # See LICENSE in the project root for license information. -from __future__ import absolute_import import logging import ldap import os diff --git a/src/iris/utils.py b/src/iris/utils.py index 2d6add76..23c5f107 100644 --- a/src/iris/utils.py +++ b/src/iris/utils.py @@ -3,7 +3,7 @@ # -*- coding:utf-8 -*- -from __future__ import absolute_import + from phonenumbers import (format_number as pn_format_number, parse as pn_parse, PhoneNumberFormat) from gevent import sleep @@ -209,7 +209,7 @@ def claim_incident(incident_id, owner): max_retries = 3 - for i in xrange(max_retries): + for i in range(max_retries): cursor = connection.cursor() try: cursor.execute('''UPDATE `incident` @@ -313,14 +313,14 @@ def claim_incidents_from_batch_id(batch_id, owner): def msgpack_unpack_msg_from_socket(socket): - unpacker = msgpack.Unpacker() + unpacker = msgpack.Unpacker(encoding='utf-8') while True: buf = socket.recv(1024) if not buf: break unpacker.feed(buf) try: - item = unpacker.next() + item = next(unpacker) except StopIteration: pass else: @@ -329,8 +329,8 @@ def msgpack_unpack_msg_from_socket(socket): def sanitize_unicode_dict(d): '''Properly decode unicode strings in d to avoid breaking jinja2 renderer''' - for key, value in d.iteritems(): - if isinstance(value, basestring): + for key, value in d.items(): + if isinstance(value, bytes): try: d[key] = value.decode('utf-8') except UnicodeError: diff --git a/src/iris/vendors/__init__.py b/src/iris/vendors/__init__.py index 32e3f274..b5c3f2f0 100644 --- a/src/iris/vendors/__init__.py +++ b/src/iris/vendors/__init__.py @@ -44,16 +44,16 @@ def __init__(self, vendors, application_vendors): for mode in vendor_cls.supports: vendor_instance = vendor_cls(copy.deepcopy(vendor_config)) vendor_instances[mode].append(vendor_instance) - for application_name, application_cls in applications.iteritems(): + for application_name, application_cls in applications.items(): app_vendor_instances[application_name][mode].append(application_cls(vendor_instance)) - for mode, instances in vendor_instances.iteritems(): + for mode, instances in vendor_instances.items(): random.shuffle(instances) self.all_vendor_instances += instances self.vendors_iter[mode] = itertools.cycle(instances) - for application_name, modes in app_vendor_instances.iteritems(): - for mode_name, instances in modes.iteritems(): + for application_name, modes in app_vendor_instances.items(): + for mode_name, instances in modes.items(): random.shuffle(instances) self.app_specific_vendors_iter[application_name][mode_name] = itertools.cycle(instances) diff --git a/src/iris/vendors/iris_twilio.py b/src/iris/vendors/iris_twilio.py index ac257b86..03abb570 100644 --- a/src/iris/vendors/iris_twilio.py +++ b/src/iris/vendors/iris_twilio.py @@ -9,7 +9,7 @@ from iris import db from sqlalchemy.exc import IntegrityError import time -import urllib +import urllib.parse import logging logger = logging.getLogger(__name__) @@ -46,7 +46,7 @@ def generate_message_text(self, message): for key in ('subject', 'body'): value = message.get(key) - if not isinstance(value, basestring): + if not isinstance(value, str): continue value = value.strip() if value: @@ -118,11 +118,11 @@ def send_call(self, message): payload['message_id'] = message_id payload['instruction'] = plugin.get_phone_menu_text() relay_cb_url = '%s%s?%s' % ( - self.config['relay_base_url'], self.gather_endpoint, urllib.urlencode({k: unicode(v).encode('utf-8') for k, v in payload.iteritems()}) + self.config['relay_base_url'], self.gather_endpoint, urllib.parse.urlencode({k: str(v).encode('utf-8') for k, v in payload.items()}) ) else: relay_cb_url = '%s%s?%s' % ( - self.config['relay_base_url'], self.say_endpoint, urllib.urlencode({k: unicode(v).encode('utf-8') for k, v in payload.iteritems()}) + self.config['relay_base_url'], self.say_endpoint, urllib.parse.urlencode({k: str(v).encode('utf-8') for k, v in payload.items()}) ) start = time.time() diff --git a/src/iris/webhooks/alertmanager.py b/src/iris/webhooks/alertmanager.py index d571eca5..9e688a9f 100644 --- a/src/iris/webhooks/alertmanager.py +++ b/src/iris/webhooks/alertmanager.py @@ -1,5 +1,3 @@ -from __future__ import absolute_import - import datetime import logging import ujson diff --git a/src/iris/webhooks/grafana.py b/src/iris/webhooks/grafana.py index 589a6ce5..ef38efff 100644 --- a/src/iris/webhooks/grafana.py +++ b/src/iris/webhooks/grafana.py @@ -1,5 +1,3 @@ -from __future__ import absolute_import - import datetime import logging import ujson diff --git a/test/e2eserver.py b/test/e2eserver.py index c88f15cd..4b40230d 100644 --- a/test/e2eserver.py +++ b/test/e2eserver.py @@ -20,6 +20,6 @@ def clean_exit(a, b): config = load_config(sys.argv[1]) addr = (config['server']['host'], config['server']['port']) - print 'Listening on %s...' % (addr,) + print('Listening on %s...' % (addr,)) application = get_api(config) WSGIServer(addr, application).serve_forever() diff --git a/test/e2etest.py b/test/e2etest.py index 92e68250..faba97ef 100644 --- a/test/e2etest.py +++ b/test/e2etest.py @@ -425,7 +425,7 @@ def test_api_response_batch_phone_call(fake_batch_id, sample_phone, fake_iris_nu 'message_id': fake_batch_id, }, data=data) assert re.status_code == 200 - assert re.content == '{"app_response":"All iris incidents claimed for batch id %s."}' % fake_batch_id + assert re.content.decode('utf-8') == '{"app_response":"All iris incidents claimed for batch id %s."}' % fake_batch_id def test_api_response_sms(fake_message_id, fake_incident_id, sample_phone): @@ -486,7 +486,7 @@ def test_api_response_batch_sms(fake_batch_id, sample_phone): re = requests.post(base_url + 'response/twilio/messages', data=data) assert re.status_code == 200 - assert re.content == '{"app_response":"All iris incidents claimed for batch id %s."}' % fake_batch_id + assert re.content.decode('utf-8') == '{"app_response":"All iris incidents claimed for batch id %s."}' % fake_batch_id data = base_body.copy() data['Body'] = '%s claim arg1 arg2' % '*(fasdf' @@ -835,7 +835,7 @@ def test_api_response_batch_email(fake_batch_id, sample_email): assert re.status_code == 204 data = { - 'body': u'I\u0131d claim', + 'body': 'I\u0131d claim', 'headers': [ {'name': 'From', 'value': sample_email}, {'name': 'Subject', 'value': 'fooject'}, @@ -847,7 +847,7 @@ def test_api_response_batch_email(fake_batch_id, sample_email): def test_plan_routing(): re = requests.get(base_url + 'plans/TESTDOOOOT') - assert re.content == "" + assert re.content == b"" assert re.status_code == 404 @@ -918,8 +918,8 @@ def test_post_plan(sample_user, sample_team, sample_template_name): # Test post to plans endpoint (create plan) re = requests.post(base_url + 'plans', json=data, headers=username_header(sample_user)) assert re.status_code == 201 - plan_id = re.content.strip() - new_data = requests.get(base_url + 'plans/' + str(plan_id)).json() + plan_id = re.content.strip().decode('utf-8') + new_data = requests.get(base_url + 'plans/' + plan_id).json() assert new_data['name'] == data['name'] assert new_data['creator'] == data['creator'] assert new_data['description'] == data['description'] @@ -936,7 +936,7 @@ def test_post_plan(sample_user, sample_team, sample_template_name): # Test post to plan endpoint (mark active/inactive) re = requests.post(base_url + 'plans/' + plan_id, json={'active': 0}) assert re.status_code == 200 - assert re.content == '0' + assert re.content == b'0' # Malformed requests re = requests.post(base_url + 'plans/' + plan_id, json={}) @@ -952,7 +952,7 @@ def test_post_plan(sample_user, sample_team, sample_template_name): re = requests.post(base_url + 'plans/' + plan_id, json={'active': 1}) assert re.status_code == 200 - assert re.content == '1' + assert re.content == b'1' # Test get plan endpoint (plan search) re = requests.get(base_url + 'plans?active=1&name__contains=%s-test-foo' % sample_user) @@ -1064,7 +1064,7 @@ def test_post_dynamic_plan(sample_user, sample_team, sample_template_name): # Test post to plans endpoint (create plan) re = requests.post(base_url + 'plans', json=data, headers=username_header(sample_user)) assert re.status_code == 201 - plan_id = re.content.strip() + plan_id = int(re.content.strip()) new_data = requests.get(base_url + 'plans/' + str(plan_id)).json() assert new_data['name'] == data['name'] assert new_data['creator'] == data['creator'] @@ -1132,7 +1132,7 @@ def test_delete_plan(sample_user, sample_team, sample_template_name, sample_appl # Test creating and deleting by ID re = requests.post(base_url + 'plans', json=data, headers=username_header(sample_user)) assert re.status_code == 201 - plan_id = re.content.strip() + plan_id = int(re.content.strip()) assert plan_id re = requests.get(base_url + 'plans/%s' % plan_id) @@ -1147,7 +1147,7 @@ def test_delete_plan(sample_user, sample_team, sample_template_name, sample_appl # Test creating and deleting by name re = requests.post(base_url + 'plans', json=data, headers=username_header(sample_user)) assert re.status_code == 201 - plan_id = re.content.strip() + plan_id = int(re.content.strip()) assert plan_id re = requests.get(base_url + 'plans/%s' % plan_id) @@ -1165,7 +1165,7 @@ def test_delete_plan(sample_user, sample_team, sample_template_name, sample_appl re = requests.post(base_url + 'plans', json=data_cant_kill, headers=username_header(sample_user)) assert re.status_code == 201 - plan_id = re.content.strip() + plan_id = int(re.content.strip()) assert plan_id assert create_incident_with_message(sample_application_name, data_cant_kill['name'], sample_user, 'email') @@ -1271,7 +1271,7 @@ def test_post_incident(sample_user, sample_team, sample_application_name, sample }, headers={'Authorization': 'hmac %s:abc' % sample_application_name}) incident_id = int(re.content) assert re.status_code == 201 - re = requests.get(base_url + 'incidents/%s' % re.content.strip()) + re = requests.get(base_url + 'incidents/%s' % incident_id) assert re.status_code == 200 # Test claiming incident @@ -1342,7 +1342,7 @@ def test_post_dynamic_incident(sample_user, sample_team, sample_application_name }, headers={'Authorization': 'hmac %s:abc' % sample_application_name}) incident_id = int(re.content) assert re.status_code == 201 - re = requests.get(base_url + 'incidents/%s' % re.content.strip()) + re = requests.get(base_url + 'incidents/%s' % incident_id) assert re.status_code == 200 # Claim @@ -1403,7 +1403,7 @@ def test_post_incident_change_application(sample_user, sample_application_name, }, headers={'Authorization': 'hmac %s:abc' % superuser_application}) incident_id = int(re.content) assert re.status_code == 201 - re = requests.get(base_url + 'incidents/%s' % re.content.strip()) + re = requests.get(base_url + 'incidents/%s' % incident_id) assert re.status_code == 200 assert re.json()['application'] == sample_application_name @@ -1569,7 +1569,7 @@ def test_api_get_nested_context(sample_user, sample_team, sample_template_name, assert re.status_code == 201 iid = re.content.strip() - re = requests.get(base_url + 'incidents/' + iid) + re = requests.get(base_url + 'incidents/' + iid.decode('utf-8')) assert re.status_code == 200 assert re.json()['context']['nodes'] == ctx['nodes'] @@ -1673,7 +1673,7 @@ def test_get_invalid_incident(iris_incidents): re = requests.get(base_url + 'incidents?id=job') assert re.status_code == 400 - assert 'id should be ' in re.content + assert b'id should be ' in re.content def test_post_user_modes(sample_user): @@ -2144,7 +2144,7 @@ def test_get_user(sample_user, sample_email, sample_admin_user): re = requests.get(base_url + 'users/' + sample_user, headers=username_header(sample_user)) assert re.status_code == 200 data = re.json() - assert data.viewkeys() == {'teams', 'modes', 'per_app_modes', 'admin', 'contacts', 'name'} + assert data.keys() == {'teams', 'modes', 'per_app_modes', 'admin', 'contacts', 'name'} assert data['contacts']['email'] == sample_email assert data['name'] == sample_user @@ -2158,7 +2158,7 @@ def test_healthcheck(): f.write('GOOD') re = requests.get(server + 'healthcheck') assert re.status_code == 200 - assert re.content == 'GOOD' + assert re.content == b'GOOD' def test_stats(): @@ -2173,7 +2173,7 @@ def test_stats(): re = requests.get(base_url + 'stats?fields=total_active_users&fields=total_plans') assert re.status_code == 200 - assert re.json().viewkeys() == {'total_active_users', 'total_plans'} + assert re.json().keys() == {'total_active_users', 'total_plans'} re = requests.get(base_url + 'stats?fields=fakefield') assert re.status_code == 200 @@ -2702,7 +2702,7 @@ def test_view_app_key(sample_application_name, sample_admin_user): re = requests.get(base_url + 'applications/%s/key' % sample_application_name, headers=username_header(sample_admin_user)) assert re.status_code == 200 - assert re.json().viewkeys() == {'key'} + assert re.json().keys() == {'key'} def test_change_app_key(sample_application_name, sample_admin_user): @@ -3201,7 +3201,7 @@ def test_comment(sample_user, sample_team, sample_application_name, sample_templ }, headers={'Authorization': 'hmac %s:abc' % sample_application_name}) incident_id = int(re.content) assert re.status_code == 201 - re = requests.get(base_url + 'incidents/%s' % re.content.strip()) + re = requests.get(base_url + 'incidents/%s' % incident_id) assert re.status_code == 200 # Post a few comments for the incident diff --git a/test/test_config.py b/test/test_config.py index 56f0222e..da723821 100644 --- a/test/test_config.py +++ b/test/test_config.py @@ -11,7 +11,7 @@ def test_load_config(mocker): mocker.patch.dict(os.environ, {'IRIS_CFG_DB_USER': 'iris_dev'}) with NamedTemporaryFile() as temp_config: - temp_config.write(''' + temp_config.write(b''' db: conn: kwargs: diff --git a/test/test_coordinator.py b/test/test_coordinator.py index 419d285c..8bbcbc5e 100644 --- a/test/test_coordinator.py +++ b/test/test_coordinator.py @@ -14,7 +14,7 @@ def setup_class(cls): @classmethod def teardown_class(cls): - for instance in cls.instances.itervalues(): + for instance in cls.instances.values(): instance.leave_cluster() def test_failover(self): @@ -43,12 +43,12 @@ def test_failover(self): # Verify it became slave sleep(3) assert self.instances['c1'].slave_count == 1 - assert self.instances['c1'].slaves.next() == (u'testinstance', 1002) + assert next(self.instances['c1'].slaves) == ('testinstance', 1002) # Verify API can see these instances self.instances['api'] = Coordinator(zk_url, None, None, False) - assert self.instances['api'].get_current_master() == (u'testinstance', 1001) - assert (u'testinstance', 1002) in self.instances['api'].get_current_slaves() + assert self.instances['api'].get_current_master() == ('testinstance', 1001) + assert ('testinstance', 1002) in self.instances['api'].get_current_slaves() # Kill off first master and see if slave becomes master with no slaves self.instances['c1'].leave_cluster() @@ -65,7 +65,7 @@ def test_failover(self): # It should show up as a slave to self.instances['c2'] which is now master assert self.instances['c2'].am_i_master() assert self.instances['c2'].slave_count == 1 - assert self.instances['c2'].slaves.next() == (u'testinstance', 1001) + assert next(self.instances['c2'].slaves) == ('testinstance', 1001) def test_non_cluster(): @@ -82,7 +82,7 @@ def test_non_cluster(): assert master_with_slaves.slave_count == 2 slaves = master_with_slaves.slaves - assert slaves.next() == ('testinstance', 1001) - assert slaves.next() == ('testinstance', 1002) - assert slaves.next() == ('testinstance', 1001) - assert slaves.next() == ('testinstance', 1002) + assert next(slaves) == ('testinstance', 1001) + assert next(slaves) == ('testinstance', 1002) + assert next(slaves) == ('testinstance', 1001) + assert next(slaves) == ('testinstance', 1002) diff --git a/test/test_iris_vendor_hipchat.py b/test/test_iris_vendor_hipchat.py index a92e6b8a..1b60bac6 100644 --- a/test/test_iris_vendor_hipchat.py +++ b/test/test_iris_vendor_hipchat.py @@ -13,7 +13,7 @@ def test_message_construction_for_incident(): fake_msg = { 'application': 'grafana', 'incident_id': 123, - 'body': u'test body', + 'body': 'test body', 'message_id': 456, 'destination': '@user1', } @@ -30,7 +30,7 @@ def test_destination_parsing_for_incident(): fake_msg = { 'application': 'grafana', 'incident_id': 123, - 'body': u'test body', + 'body': 'test body', 'message_id': 456, } destination = '1234;testtoken;@user1' @@ -59,7 +59,7 @@ def test_destination_parsing_defaults_for_incident(): fake_msg = { 'application': 'grafana', 'incident_id': 123, - 'body': u'test body', + 'body': 'test body', 'message_id': 456, } destination = 'user_missing_@' diff --git a/test/test_iris_vendor_messagebird.py b/test/test_iris_vendor_messagebird.py index be5efcef..a86400c9 100644 --- a/test/test_iris_vendor_messagebird.py +++ b/test/test_iris_vendor_messagebird.py @@ -11,7 +11,7 @@ def test_message_construction_for_incident(): fake_msg = { 'application': 'grafana', 'incident_id': 123, - 'body': u'test body', + 'body': 'test body', 'message_id': 456, 'destination': '0612342341' } diff --git a/test/test_iris_vendor_slack.py b/test/test_iris_vendor_slack.py index facf39f3..dc7596d3 100644 --- a/test/test_iris_vendor_slack.py +++ b/test/test_iris_vendor_slack.py @@ -17,7 +17,7 @@ def test_atttachments_construction_for_incident(): fake_msg = { 'application': 'grafana', 'incident_id': 123, - 'body': u'test body', + 'body': 'test body', 'message_id': 456, 'destination': 'user1' } diff --git a/test/test_iris_vendor_smtp.py b/test/test_iris_vendor_smtp.py index ff39afb2..0d42461d 100644 --- a/test/test_iris_vendor_smtp.py +++ b/test/test_iris_vendor_smtp.py @@ -57,5 +57,5 @@ def test_smtp_unicode(mocker): smtp_vendor.send_email({ 'destination': 'foo@bar', 'subject': 'hello', - 'body': u'\u201c', + 'body': '\u201c', }) diff --git a/test/test_iris_vendor_twilio.py b/test/test_iris_vendor_twilio.py index d6b1dff2..0b5b6c93 100644 --- a/test/test_iris_vendor_twilio.py +++ b/test/test_iris_vendor_twilio.py @@ -39,7 +39,7 @@ def test_twilio_notification_call_generate(mocker): if_machine='Continue', url=relay_base_url + ( '/api/v0/twilio/calls/say?content=Hello.+World&' - 'source=iris-sender&loop=3'), + 'loop=3&source=iris-sender'), status_callback=relay_base_url + '/api/v0/twilio/status' ) @@ -75,7 +75,7 @@ def test_twilio_incident_call_generate(mocker): if_machine='Continue', url=relay_base_url + ( '/api/v0/twilio/calls/gather?content=Hello.+World&' - 'source=iris-sender&instruction=Press+1+to+pay&' - 'message_id=1&loop=3'), + 'loop=3&source=iris-sender&' + 'message_id=1&instruction=Press+1+to+pay'), status_callback=relay_base_url + '/api/v0/twilio/status' ) diff --git a/test/test_irisapi.py b/test/test_irisapi.py index cd608afc..87097577 100644 --- a/test/test_irisapi.py +++ b/test/test_irisapi.py @@ -31,12 +31,12 @@ def test_parse_response(self): class TestHealthcheck(falcon.testing.TestCase): def test_healthcheck(self): - with patch('__builtin__.open', mock_open(read_data='GOOD')) as m: + with patch('iris.api.open', mock_open(read_data='GOOD')) as m: self.api.add_route('/healthcheck', Healthcheck('healthcheck_path')) result = self.simulate_get(path='/healthcheck') m.assert_called_once_with('healthcheck_path') self.assertEqual(result.status_code, 200) - self.assertEqual(result.content, 'GOOD') + self.assertEqual(result.content, b'GOOD') class TestAuth(falcon.testing.TestCase): @@ -57,32 +57,32 @@ def test_auth(self): window = int(time.time()) // 5 text = '%s %s %s %s' % (window, 'GET', '/foo/bar', '') - HMAC = hmac.new('key', text, hashlib.sha512) - digest = base64.urlsafe_b64encode(HMAC.digest()) + HMAC = hmac.new(b'key', text.encode('utf-8'), hashlib.sha512) + digest = base64.urlsafe_b64encode(HMAC.digest()).decode('utf-8') auth = 'hmac app:%s' % digest result = self.simulate_get(path='/foo/bar', headers={'Authorization': auth}) self.assertEqual(result.status_code, 200) - self.assertEqual(result.content, 'Hello world') + self.assertEqual(result.content, b'Hello world') # Test query string window = int(time.time()) // 5 text = '%s %s %s %s' % (window, 'GET', '/foo/bar?baz=123', '') - HMAC = hmac.new('key', text, hashlib.sha512) - digest = base64.urlsafe_b64encode(HMAC.digest()) + HMAC = hmac.new(b'key', text.encode('utf-8'), hashlib.sha512) + digest = base64.urlsafe_b64encode(HMAC.digest()).decode('utf-8') auth = 'hmac app:%s' % digest result = self.simulate_get(path='/foo/bar', query_string='baz=123', headers={'Authorization': auth}) self.assertEqual(result.status_code, 200) - self.assertEqual(result.content, 'Hello world') + self.assertEqual(result.content, b'Hello world') # Test trailng slash window = int(time.time()) // 5 text = '%s %s %s %s' % (window, 'GET', '/foo/bar/', '') - HMAC = hmac.new('key', text, hashlib.sha512) - digest = base64.urlsafe_b64encode(HMAC.digest()) + HMAC = hmac.new(b'key', text.encode('utf-8'), hashlib.sha512) + digest = base64.urlsafe_b64encode(HMAC.digest()).decode('utf-8') auth = 'hmac app:%s' % digest result = self.simulate_get(path='/foo/bar/', headers={'Authorization': auth}) self.assertEqual(result.status_code, 200) - self.assertEqual(result.content, 'Hello world') + self.assertEqual(result.content, b'Hello world') # Test no auth header result = self.simulate_get(path='/foo/bar') @@ -97,7 +97,7 @@ def test_auth(self): dummy.allow_read_no_auth = True result = self.simulate_get(path='/foo/bar') self.assertEqual(result.status_code, 200) - self.assertEqual(result.content, 'Hello world') + self.assertEqual(result.content, b'Hello world') def test_secondary_auth(self): iris.cache.applications = {'app': {'key': 'asdf', 'secondary_key': 'key'}} @@ -108,32 +108,32 @@ def test_secondary_auth(self): window = int(time.time()) // 5 text = '%s %s %s %s' % (window, 'GET', '/foo/bar', '') - HMAC = hmac.new('key', text, hashlib.sha512) - digest = base64.urlsafe_b64encode(HMAC.digest()) + HMAC = hmac.new(b'key', text.encode('utf-8'), hashlib.sha512) + digest = base64.urlsafe_b64encode(HMAC.digest()).decode('utf-8') auth = 'hmac app:%s' % digest result = self.simulate_get(path='/foo/bar', headers={'Authorization': auth}) self.assertEqual(result.status_code, 200) - self.assertEqual(result.content, 'Hello world') + self.assertEqual(result.content, b'Hello world') # Test query string window = int(time.time()) // 5 text = '%s %s %s %s' % (window, 'GET', '/foo/bar?baz=123', '') - HMAC = hmac.new('key', text, hashlib.sha512) - digest = base64.urlsafe_b64encode(HMAC.digest()) + HMAC = hmac.new(b'key', text.encode('utf-8'), hashlib.sha512) + digest = base64.urlsafe_b64encode(HMAC.digest()).decode('utf-8') auth = 'hmac app:%s' % digest result = self.simulate_get(path='/foo/bar', query_string='baz=123', headers={'Authorization': auth}) self.assertEqual(result.status_code, 200) - self.assertEqual(result.content, 'Hello world') + self.assertEqual(result.content, b'Hello world') # Test trailng slash window = int(time.time()) // 5 text = '%s %s %s %s' % (window, 'GET', '/foo/bar/', '') - HMAC = hmac.new('key', text, hashlib.sha512) - digest = base64.urlsafe_b64encode(HMAC.digest()) + HMAC = hmac.new(b'key', text.encode('utf-8'), hashlib.sha512) + digest = base64.urlsafe_b64encode(HMAC.digest()).decode('utf-8') auth = 'hmac app:%s' % digest result = self.simulate_get(path='/foo/bar/', headers={'Authorization': auth}) self.assertEqual(result.status_code, 200) - self.assertEqual(result.content, 'Hello world') + self.assertEqual(result.content, b'Hello world') # Test no auth header result = self.simulate_get(path='/foo/bar') @@ -148,4 +148,4 @@ def test_secondary_auth(self): dummy.allow_read_no_auth = True result = self.simulate_get(path='/foo/bar') self.assertEqual(result.status_code, 200) - self.assertEqual(result.content, 'Hello world') + self.assertEqual(result.content, b'Hello world') diff --git a/test/test_sender.py b/test/test_sender.py index b71d6177..7cc1ed49 100644 --- a/test/test_sender.py +++ b/test/test_sender.py @@ -74,27 +74,27 @@ def test_configure(mocker): } fake_plan = { - u'name': u'find-test-user', - u'threshold_count': 10, - u'creator': u'test-user', - u'created': 1470444636, - u'aggregation_reset': 300, - u'aggregation_window': 300, - u'threshold_window': 900, - u'tracking_type': None, - u'steps': [ - [{u'repeat': 0, u'target': u'test-user', u'id': 178243, u'priority': u'low', u'step': 1, - u'role': u'user', u'template': u'test-app Default', u'wait': 0}, - {u'repeat': 1, u'target': u'test-user', u'id': 178252, u'priority': u'high', u'step': 1, - u'role': u'user', u'template': u'test-app Default', u'wait': 300}], - [{u'repeat': 3, u'target': u'test-user', u'id': 178261, u'priority': u'urgent', u'step': 2, - u'role': u'user', u'template': u'test-app Default', u'wait': 900}] + 'name': 'find-test-user', + 'threshold_count': 10, + 'creator': 'test-user', + 'created': 1470444636, + 'aggregation_reset': 300, + 'aggregation_window': 300, + 'threshold_window': 900, + 'tracking_type': None, + 'steps': [ + [{'repeat': 0, 'target': 'test-user', 'id': 178243, 'priority': 'low', 'step': 1, + 'role': 'user', 'template': 'test-app Default', 'wait': 0}, + {'repeat': 1, 'target': 'test-user', 'id': 178252, 'priority': 'high', 'step': 1, + 'role': 'user', 'template': 'test-app Default', 'wait': 300}], + [{'repeat': 3, 'target': 'test-user', 'id': 178261, 'priority': 'urgent', 'step': 2, + 'role': 'user', 'template': 'test-app Default', 'wait': 900}] ], - u'tracking_template': None, - u'tracking_key': None, - u'active': 1, - u'id': 19546, - u'description': u"please don't abuse this plan :)" + 'tracking_template': None, + 'tracking_key': None, + 'active': 1, + 'id': 19546, + 'description': "please don't abuse this plan :)" } @@ -330,9 +330,9 @@ def test_generate_slave_message_payload(): } result = generate_msgpack_message_payload(data) assert msgpack.unpackb(result) == { - 'endpoint': 'v0/slave_send', - 'data': { - 'ids': [1, 2, 3, 4] + b'endpoint': b'v0/slave_send', + b'data': { + b'ids': [1, 2, 3, 4] } } @@ -343,8 +343,8 @@ def test_quotas(mocker): from gevent import sleep mocker.patch('iris.sender.quota.ApplicationQuota.get_new_rules', return_value=[( - u'testapp', 5, 2, 120, 120, u'testuser', - u'user', u'iris-plan', 10 + 'testapp', 5, 2, 120, 120, 'testuser', + 'user', 'iris-plan', 10 )]) mocker.patch('iris.sender.quota.ApplicationQuota.notify_incident') mocker.patch('iris.sender.quota.ApplicationQuota.notify_target') @@ -376,7 +376,7 @@ def test_quotas(mocker): assert stats['app_testapp_quota_hard_usage_pct'] == 100 assert stats['app_testapp_quota_soft_usage_pct'] == 200 - for _ in xrange(10): + for _ in range(10): assert quotas.allow_send({'application': 'app_without_quota'}) @@ -511,16 +511,16 @@ def test_handle_api_notification_request_invalid_message(mocker): def test_sanitize_unicode_dict(): - import pytest from jinja2.sandbox import SandboxedEnvironment from iris.utils import sanitize_unicode_dict # Use jinja the same way as in sender env = SandboxedEnvironment(autoescape=False) template = env.from_string('{{var}} {{var2}} {{nested.nest}}') - bad_context = {'var': '\xe2\x80\x99', 'var2': 2, 'nested': {'nest': '\xe2\x80\x99'}} + bad_context = {'var': b'\xe2\x80\x99', 'var2': 2, 'nested': {'nest': b'\xe2\x80\x99'}} - with pytest.raises(UnicodeDecodeError): - template.render(**bad_context) + assert '\\xe2' in template.render(**bad_context) - template.render(**sanitize_unicode_dict(bad_context)) + good_render = template.render(**sanitize_unicode_dict(bad_context)) + assert '\\xe2' not in good_render + assert b'\xe2\x80\x99'.decode('utf-8') in good_render