From e34e033d45cef773064534ce3907fb797d776182 Mon Sep 17 00:00:00 2001 From: Sebastien Billion Date: Wed, 30 Nov 2016 11:29:49 +0100 Subject: [PATCH 1/3] Begin work for python 3.5 compat --- requirements.txt | 1 + test_env.json | 3 ++ test_settings.json | 2 +- tests/tests.py | 71 +++++++++++++++++++++++++++++++-------- tests/tests_middleware.py | 39 ++++++++++++++------- zappa/cli.py | 63 +++++++++++++++++++++------------- zappa/handler.py | 27 ++++++++------- zappa/middleware.py | 19 ++++++++--- zappa/util.py | 7 ++-- zappa/wsgi.py | 10 ++++-- zappa/zappa.py | 22 +++++++----- 11 files changed, 181 insertions(+), 83 deletions(-) create mode 100644 test_env.json diff --git a/requirements.txt b/requirements.txt index 9822fb9e7..3a396250f 100644 --- a/requirements.txt +++ b/requirements.txt @@ -16,3 +16,4 @@ troposphere==1.9.0 Werkzeug==0.11.11 wheel==0.29.0 wsgi-request-logger==0.4.6 +letsencrypt \ No newline at end of file diff --git a/test_env.json b/test_env.json new file mode 100644 index 000000000..04a83e926 --- /dev/null +++ b/test_env.json @@ -0,0 +1,3 @@ +{ + "hello": "world" +} \ No newline at end of file diff --git a/test_settings.json b/test_settings.json index 7561c7935..cea3d8f3b 100644 --- a/test_settings.json +++ b/test_settings.json @@ -55,7 +55,7 @@ }, "extendo2": { "extends": "extendo", - "s3_bucket": "lmbda2", + "s3_bucket": "lmbda2" }, "depricated_remote_env": { "s3_bucket": "lmbda", diff --git a/tests/tests.py b/tests/tests.py index 20f5e3fe5..17a00717d 100644 --- a/tests/tests.py +++ b/tests/tests.py @@ -1,4 +1,8 @@ # -*- coding: utf8 -*- +from __future__ import absolute_import + +import platform + import base64 import collections import json @@ -25,11 +29,15 @@ from zappa.wsgi import create_wsgi_request, common_log from zappa.zappa import Zappa, ASSUME_POLICY, ATTACH_POLICY + + def random_string(length): return ''.join(random.choice(string.printable) for _ in range(length)) class TestZappa(unittest.TestCase): + python_version = '.'.join(platform.python_version_tuple()[:2]) + def setUp(self): self.sleep_patch = mock.patch('time.sleep', return_value=None) # Tests expect us-east-1. @@ -65,7 +73,8 @@ def test_create_lambda_package(self): # mock the pip.get_installed_distributions() to include a package in lambda_packages so that the code # for zipping pre-compiled packages gets called mock_named_tuple = collections.namedtuple('mock_named_tuple', ['project_name']) - mock_return_val = [mock_named_tuple(lambda_packages.keys()[0])] # choose name of 1st package in lambda_packages + mock_return_val = [mock_named_tuple(list(lambda_packages.keys())[0])] + # choose name of 1st package in lambda_packages with mock.patch('pip.get_installed_distributions', return_value=mock_return_val): z = Zappa() path = z.create_lambda_zip(handler_file=os.path.realpath(__file__)) @@ -329,8 +338,17 @@ def test_b64_pattern(self): self.assertRegexpMatches(document, pattern) for bad_code in ['200', '301', '302']: - document = base64.b64encode(head + bad_code + random_string(50)) - self.assertNotRegexpMatches(document, pattern) + try: + document = base64.b64encode(head + bad_code + + random_string(50)) + except TypeError: + document = base64.b64encode(str.encode(head + bad_code + + random_string(50))) + try: + self.assertNotRegexpMatches(document, pattern) + except TypeError: + self.assertNotRegexpMatches(bytes.decode(document), + pattern) def test_200_pattern(self): pattern = Zappa.selection_pattern('200') @@ -581,7 +599,7 @@ def test_handler(self, session): # Annoyingly, this will fail during record, but # the result will actually be okay to use in playback. # See: https://github.com/garnaat/placebo/issues/48 - self.assertEqual(os.environ['hello'], 'world') + self.assertEqual(os.environ.get('hello'), 'world') event = { "body": {}, @@ -896,10 +914,17 @@ def test_cli_init(self): # Via http://stackoverflow.com/questions/2617057/how-to-supply-stdin-files-and-environment-variable-inputs-to-python-unit-tests inputs = ['dev', 'lmbda', 'test_settings', 'y', ''] + def test_for(inputs): input_generator = (i for i in inputs) - with mock.patch('__builtin__.raw_input', lambda prompt: next(input_generator)): - zappa_cli.init() + if int(self.python_version[0]) < 3: + with mock.patch('__builtin__.input', lambda prompt: next(input_generator)): + zappa_cli.init() + else: + with mock.patch('builtins.input', lambda prompt: next( + input_generator)): + zappa_cli.init() + if os.path.isfile('zappa_settings.json'): os.remove('zappa_settings.json') @@ -914,10 +939,17 @@ def test_for(inputs): # Test via handle() input_generator = (i for i in inputs) - with mock.patch('__builtin__.raw_input', lambda prompt: next(input_generator)): - zappa_cli = ZappaCLI() - argv = ['init'] - zappa_cli.handle(argv) + if int(self.python_version[0]) < 3: + with mock.patch('__builtin__.input', lambda prompt: next(input_generator)): + zappa_cli = ZappaCLI() + argv = ['init'] + zappa_cli.handle(argv) + else: + with mock.patch('builtins.input', lambda prompt: next( + input_generator)): + zappa_cli = ZappaCLI() + argv = ['init'] + zappa_cli.handle(argv) if os.path.isfile('zappa_settings.json'): os.remove('zappa_settings.json') @@ -1141,8 +1173,10 @@ def test_dj_wsgi(self): app = get_django_wsgi('dj_test_settings') os.remove('dj_test_settings.py') - os.remove('dj_test_settings.pyc') - + try: + os.remove('dj_test_settings.pyc') + except: + pass ## # Util / Misc ## @@ -1229,7 +1263,11 @@ def test_remote_env_package(self): with zipfile.ZipFile(zappa_cli.zip_path, 'r') as lambda_zip: content = lambda_zip.read('zappa_settings.py') zappa_cli.remove_local_zip() - m = re.search("REMOTE_ENV='(.*)'", content) + try: + m = re.search("REMOTE_ENV='(.*)'", content) + except TypeError: + m = re.search("REMOTE_ENV='(.*)'", bytes.decode(content)) + self.assertEqual(m.group(1), 's3://lmbda-env/dev/env.json') zappa_cli = ZappaCLI() @@ -1240,7 +1278,12 @@ def test_remote_env_package(self): with zipfile.ZipFile(zappa_cli.zip_path, 'r') as lambda_zip: content = lambda_zip.read('zappa_settings.py') zappa_cli.remove_local_zip() - m = re.search("REMOTE_ENV='(.*)'", content) + try: + + m = re.search("REMOTE_ENV='(.*)'", content) + except: + m = re.search("REMOTE_ENV='(.*)'", bytes.decode(content)) + self.assertEqual(m.group(1), 's3://lmbda-env/prod/env.json') diff --git a/tests/tests_middleware.py b/tests/tests_middleware.py index 497ee02de..05603c3a5 100644 --- a/tests/tests_middleware.py +++ b/tests/tests_middleware.py @@ -1,4 +1,6 @@ # -*- coding: utf8 -*- +from __future__ import absolute_import +from __future__ import print_function import unittest import base58 import json @@ -8,6 +10,7 @@ from zappa.wsgi import create_wsgi_request from zappa.middleware import ZappaWSGIMiddleware +import six class TestWSGIMockMiddleWare(unittest.TestCase): @@ -213,9 +216,12 @@ def simple_app(environ, start_response): self.assertEqual(''.join(resp), body) def test_wsgi_middleware_uglystring(self): - ugly_string = unicode("˝ÓÔÒÚÆ☃ЗИЙКЛМФХЦЧШ차를 타고 온 펲시맨(╯°□°）╯︵ ┻━┻)" - "לֹהִים, אֵת הַשָּׁמַיִם, וְאֵת הָt͔̦h̞̲e̢̤ ͍̬̲͖f̴̘͕̣è͖ẹ̥̩l͖͔͚i͓͚̦͠n͖͍̗͓̳̮g͍ ̨ 𝕢𝕦𝕚𝕔𝕜 𝕓𝕣𝕠𝕨", - encoding='utf8') + try: + ugly_string = unicode("˝ÓÔÒÚÆ☃ЗИЙКЛМФХЦЧШ차를 타고 온 펲시맨(╯°□°）╯︵ ┻━┻)" + "לֹהִים, אֵת הַשָּׁמַיִם, וְאֵת הָt͔̦h̞̲e̢̤ ͍̬̲͖f̴̘͕̣è͖ẹ̥̩l͖͔͚i͓͚̦͠n͖͍̗͓̳̮g͍ ̨ 𝕢𝕦𝕚𝕔𝕜 𝕓𝕣𝕠𝕨") + except: + ugly_string = "˝ÓÔÒÚÆ☃ЗИЙКЛМФХЦЧШ차를 타고 온 펲시맨(╯°□°）╯︵ ┻━┻)" + \ + "לֹהִים, אֵת הַשָּׁמַיִם, וְאֵת הָt͔̦h̞̲e̢̤ ͍̬̲͖f̴̘͕̣è͖ẹ̥̩l͖͔͚i͓͚̦͠n͖͍̗͓̳̮g͍ ̨ 𝕢𝕦𝕚𝕔𝕜 𝕓𝕣𝕠𝕨" # Pass some unicode through the middleware body def simple_app(environ, start_response): @@ -230,7 +236,7 @@ def simple_app(environ, start_response): # Call with empty WSGI Environment resp = app(dict(), self._start_response) - print(''.join(resp)) + print((''.join(resp))) # Pass some unicode through the middleware headers def simple_app(environ, start_response): @@ -245,7 +251,7 @@ def simple_app(environ, start_response): # Call with empty WSGI Environment resp = app(dict(), self._start_response) - print(''.join(resp)) + print((''.join(resp))) def test_wsgi_middleware_expiry(self): # Setting the cookies @@ -376,7 +382,7 @@ def test_wsgi_middleware_realcall(self): def set_cookies(environ, start_response): status = '200 OK' - print environ + print(environ) response_headers = [('Set-Cookie', 'foo=123'), ('Set-Cookie', 'bar=456'), ('Set-Cookie', 'baz=789')] @@ -445,7 +451,7 @@ def set_cookies(environ, start_response): def change_cookie(environ, start_response): status = '200 OK' - print 'environ', environ + print('environ', environ) response_headers = [('Set-Cookie', 'foo=new_value')] start_response(status, response_headers) return ['Set cookies!'] @@ -459,10 +465,17 @@ def change_cookie(environ, start_response): self.assertEqual(len(zappa_cookie), 1) zappa_cookie1 = zappa_cookie[0] self.assertTrue(zappa_cookie1.startswith('zappa=')) + zdict = parse_cookie(zappa_cookie1) - print 'zdict', zdict - zdict2 = json.loads(base58.b58decode(zdict['zappa'])) - print 'zdict2', zdict2 + print('zdict', zdict) + try: + zdict2 = json.loads(base58.b58decode(zdict['zappa'])) + + except TypeError: + print(base58.b58decode(zdict['zappa'])) + zdict2 = json.loads(zdict.get('zappa')) + + print('zdict2', zdict2) self.assertEqual(len(zdict2), 3) self.assertEqual(zdict2['foo'], 'new_value') self.assertEqual(zdict2['bar'], '456') @@ -476,7 +489,7 @@ def change_cookie(environ, start_response): def read_cookies(environ, start_response): status = '200 OK' - print 'environ', environ + print('environ', environ) response_headers = [] start_response(status, response_headers) return [environ['HTTP_COOKIE']] @@ -487,14 +500,14 @@ def read_cookies(environ, start_response): trailing_slash=False) response = Response.from_app(app, environ) - print "response", response + print("response", response) # Filter the headers for Set-Cookie header zappa_cookie = [x[1] for x in response.headers if x[0] == 'Set-Cookie'] self.assertEqual(len(zappa_cookie), 1) zappa_cookie1 = zappa_cookie[0] self.assertTrue(zappa_cookie1.startswith('zappa=')) zdict = parse_cookie(zappa_cookie1) - print 'zdict', zdict + print('zdict', zdict) cookies = json.loads(base58.b58decode(zdict['zappa'])) self.assertEqual(cookies['foo'], 'new_value') self.assertEqual(cookies['bar'], '456') diff --git a/zappa/cli.py b/zappa/cli.py index 8dc843901..4d691d7a2 100644 --- a/zappa/cli.py +++ b/zappa/cli.py @@ -10,6 +10,8 @@ from __future__ import unicode_literals from __future__ import division +from __future__ import absolute_import +from __future__ import print_function import argparse import base64 @@ -35,9 +37,11 @@ from click.exceptions import ClickException from dateutil import parser from datetime import datetime,timedelta -from zappa import Zappa, logger, API_GATEWAY_REGIONS -from util import (check_new_version_available, detect_django_settings, - detect_flask_apps, parse_s3_url) + +from zappa.util import detect_django_settings, parse_s3_url, \ + check_new_version_available +from zappa.util import detect_flask_apps +from zappa.zappa import API_GATEWAY_REGIONS, Zappa CUSTOM_SETTINGS = [ @@ -231,13 +235,13 @@ def handle(self, argv=None): environments = [] if all_environments: # All envs! - environments = self.zappa_settings.keys() + environments = list(self.zappa_settings.keys()) else: # Just one env. if len(self.command_env) < 2: # pragma: no cover # If there's only one environment defined in the settings, # use that as the default. - if len(self.zappa_settings.keys()) is 1: - environments.append(self.zappa_settings.keys()[0]) + if len(list(self.zappa_settings.keys())) is 1: + environments.append(list(self.zappa_settings.keys())[0]) else: parser.error("Please supply an environment to interact with.") sys.exit(1) @@ -591,7 +595,7 @@ def undeploy(self, noconfirm=False, remove_logs=False): """ if not noconfirm: # pragma: no cover - confirm = raw_input("Are you sure you want to undeploy? [y/n] ") + confirm = input("Are you sure you want to undeploy? [y/n] ") if confirm != 'y': return @@ -893,9 +897,15 @@ def check_environment(self, environment): """ non_strings = [] - for k,v in environment.iteritems(): - if not isinstance(v, basestring): - non_strings.append(k) + + for k,v in environment.items(): + try: + if not isinstance(v, basestring): + non_strings.append(k) + except NameError: + if not isinstance(v,(str,bytes)): + non_strings.append(k) + if non_strings: raise ValueError("The following environment variables are not strings: {}".format(", ".join(non_strings))) else: @@ -915,8 +925,9 @@ def init(self, settings_file="zappa_settings.json"): raise ClickException("This project is " + click.style("already initialized", fg="red", bold=True) + "!") # Ensure P2 until Lambda supports it. - if sys.version_info >= (3,0): # pragma: no cover - raise ClickException("Zappa curently only works with Python 2, until AWS Lambda adds Python 3 support.") + #if sys.version_info >= (3,0): # pragma: no cover + #raise ClickException("Zappa curently only works with Python 2, + # until AWS Lambda adds Python 3 support.") # Ensure inside virtualenv. if not ( hasattr(sys, 'prefix') or hasattr(sys, 'real_prefix') or hasattr(sys, 'base_prefix') ): # pragma: no cover @@ -943,7 +954,7 @@ def init(self, settings_file="zappa_settings.json"): click.echo("Your Zappa configuration can support multiple production environments, like '" + click.style("dev", bold=True) + "', '" + click.style("staging", bold=True) + "', and '" + click.style("production", bold=True) + "'.") - env = raw_input("What do you want to call this environment (default 'dev'): ") or "dev" + env = input("What do you want to call this environment (default 'dev'): ") or "dev" try: self.check_stage_name(env) break @@ -954,7 +965,7 @@ def init(self, settings_file="zappa_settings.json"): click.echo("\nYour Zappa deployments will need to be uploaded to a " + click.style("private S3 bucket", bold=True) + ".") click.echo("If you don't have a bucket yet, we'll create one for you too.") default_bucket = "zappa-" + ''.join(random.choice(string.ascii_lowercase + string.digits) for _ in range(9)) - bucket = raw_input("What do you want call your bucket? (default '%s'): " % default_bucket) or default_bucket + bucket = input("What do you want call your bucket? (default '%s'): " % default_bucket) or default_bucket # TODO actually create bucket. # Detect Django/Flask @@ -981,10 +992,10 @@ def init(self, settings_file="zappa_settings.json"): while django_settings in [None, '']: if matches: click.echo("We discovered: " + click.style(', '.join('{}'.format(i) for v, i in enumerate(matches)), bold=True)) - django_settings = raw_input("Where are your project's settings? (default '%s'): " % matches[0]) or matches[0] + django_settings = input("Where are your project's settings? (default '%s'): " % matches[0]) or matches[0] else: click.echo("(This will likely be something like 'your_project.settings')") - django_settings = raw_input("Where are your project's settings?: ") + django_settings = input("Where are your project's settings?: ") django_settings = django_settings.replace("'", "") django_settings = django_settings.replace('"', "") else: @@ -998,9 +1009,9 @@ def init(self, settings_file="zappa_settings.json"): while app_function in [None, '']: if matches: click.echo("We discovered: " + click.style(', '.join('{}'.format(i) for v, i in enumerate(matches)), bold=True)) - app_function = raw_input("Where is your app's function? (default '%s'): " % matches[0]) or matches[0] + app_function = input("Where is your app's function? (default '%s'): " % matches[0]) or matches[0] else: - app_function = raw_input("Where is your app's function?: ") + app_function = input("Where is your app's function?: ") app_function = app_function.replace("'", "") app_function = app_function.replace('"', "") @@ -1014,7 +1025,7 @@ def init(self, settings_file="zappa_settings.json"): click.echo("If you are using Zappa for the first time, you probably don't want to do this!") global_deployment = False while True: - global_type = raw_input("Would you like to deploy this application to " + click.style("globally", bold=True) + "? (default 'n') [y/n/(p)rimary]: ") + global_type = input("Would you like to deploy this application to " + click.style("globally", bold=True) + "? (default 'n') [y/n/(p)rimary]: ") if not global_type: break if global_type.lower() in ["y", "yes", "p", "primary"]: @@ -1038,7 +1049,7 @@ def init(self, settings_file="zappa_settings.json"): for each_env in envs: # Honestly, this could be cleaner. - env_name = each_env.keys()[0] + env_name = list(each_env.keys())[0] env_dict = each_env[env_name] env_bucket = bucket @@ -1050,7 +1061,7 @@ def init(self, settings_file="zappa_settings.json"): 's3_bucket': env_bucket, } } - if env_dict.has_key('aws_region'): + if 'aws_region' in env_dict: env_zappa_settings[env_name]['aws_region'] = env_dict.get('aws_region') zappa_settings.update(env_zappa_settings) @@ -1066,7 +1077,7 @@ def init(self, settings_file="zappa_settings.json"): click.echo("\nOkay, here's your " + click.style("zappa_settings.js", bold=True) + ":\n") click.echo(click.style(zappa_settings_json, fg="yellow", bold=False)) - confirm = raw_input("\nDoes this look " + click.style("okay", bold=True, fg="green") + "? (default 'y') [y/n]: ") or 'yes' + confirm = input("\nDoes this look " + click.style("okay", bold=True, fg="green") + "? (default 'y') [y/n]: ") or 'yes' if confirm[0] not in ['y', 'Y', 'yes', 'YES']: click.echo("" + click.style("Sorry", bold=True, fg='red') + " to hear that! Please init again.") return @@ -1140,7 +1151,7 @@ def certify(self, no_cleanup=False): from shutil import copyfile copyfile(account_key_location, '/tmp/account.key') else: - if not cert_location or not cert_key_location or not cert_chain_location: + if not cert_key_location or not cert_chain_location: raise ClickException("Can't certify a domain without " + click.style("certificate, certificate_key and certificate_chain", fg="red", bold=True) + " configured!") @@ -1467,7 +1478,11 @@ def create_package(self): # Lambda requires a specific chmod temp_settings = tempfile.NamedTemporaryFile(delete=False) os.chmod(temp_settings.name, 0o644) - temp_settings.write(settings_s) + try: + temp_settings.write(settings_s) + except TypeError: + temp_settings.write(str.encode(settings_s)) + temp_settings.close() lambda_zip.write(temp_settings.name, 'zappa_settings.py') os.remove(temp_settings.name) diff --git a/zappa/handler.py b/zappa/handler.py index 94a303c7d..8c1967118 100644 --- a/zappa/handler.py +++ b/zappa/handler.py @@ -1,4 +1,5 @@ from __future__ import unicode_literals +from __future__ import absolute_import import datetime import importlib @@ -15,16 +16,12 @@ # This file may be copied into a project's root, # so handle both scenarios. -try: - from zappa.cli import ZappaCLI - from zappa.middleware import ZappaWSGIMiddleware - from zappa.wsgi import create_wsgi_request, common_log - from zappa.util import parse_s3_url -except ImportError as e: # pragma: no cover - from .cli import ZappaCLI - from .middleware import ZappaWSGIMiddleware - from .wsgi import create_wsgi_request, common_log - from .util import parse_s3_url + +from zappa.cli import ZappaCLI +from zappa.middleware import ZappaWSGIMiddleware +from zappa.wsgi import create_wsgi_request, common_log +from zappa.util import parse_s3_url + # Set up logging @@ -74,7 +71,7 @@ class LambdaHandler(object): def __new__(cls, settings_name="zappa_settings", session=None): """Singleton instance to avoid repeat setup""" if LambdaHandler.__instance is None: - LambdaHandler.__instance = object.__new__(cls, settings_name, session) + LambdaHandler.__instance = super(LambdaHandler, cls).__new__(cls) return LambdaHandler.__instance def __init__(self, settings_name="zappa_settings", session=None): @@ -153,6 +150,9 @@ def load_remote_settings(self, remote_bucket, remote_file): try: content = remote_env_object['Body'].read().decode('utf-8') + except AttributeError: + content = remote_env_object['Body'].getvalue() + except Exception as e: # pragma: no cover # catch everything aws might decide to raise print('Exception while reading remote settings file.', e) @@ -160,8 +160,9 @@ def load_remote_settings(self, remote_bucket, remote_file): try: settings_dict = json.loads(content) - except (ValueError, TypeError): # pragma: no cover - print('Failed to parse remote settings!') + except (ValueError, TypeError) as e: # pragma: no cover + print(content) + print('Failed to parse remote settings! : ' + content) return # add each key-value to environment - overwrites existing keys! diff --git a/zappa/middleware.py b/zappa/middleware.py index 6f2d038e7..5b81e7420 100644 --- a/zappa/middleware.py +++ b/zappa/middleware.py @@ -1,3 +1,4 @@ + import base58 import json import time @@ -54,7 +55,7 @@ def __call__(self, environ, start_response): only passing cookies that are still valid to the WSGI app. """ self.start_response = start_response - + print(environ) # Parse cookies from the WSGI environment parsed = parse_cookie(environ) @@ -122,7 +123,10 @@ def encode_response(self, status, headers, exc_info=None): # JSON-ify the cookie and encode it. pack_s = json.dumps(self.request_cookies) - encoded = base58.b58encode(pack_s) + try: + encoded = base58.b58encode(pack_s) + except TypeError: + encoded = base58.b58encode(str.encode(pack_s)) # Set the result as the zappa cookie new_headers.append( @@ -152,9 +156,15 @@ def decode_zappa_cookie(self, encoded_zappa): Eat our Zappa cookie. Save the parsed cookies, as we need to send them back on every update. """ - self.decoded_zappa = base58.b58decode(encoded_zappa) + + try: + self.decoded_zappa = base58.b58decode(encoded_zappa) + except TypeError: + self.decoded_zappa = base58.b58decode(str.encode(encoded_zappa)) + self.request_cookies = json.loads(self.decoded_zappa) + def filter_expired_cookies(self): """ Remove any expired cookies from our internal state. @@ -173,10 +183,9 @@ def iter_cookies_expires(self): Yield name and expires of cookies. """ for name, value in self.request_cookies.items(): - cookie = (name + '=' + value).encode('utf-8') + cookie = (b'%s=%s' % (name ,value)).encode('utf-8') if cookie.count('=') is 1: continue - kvps = cookie.split(';') for kvp in kvps: kvp = kvp.strip() diff --git a/zappa/util.py b/zappa/util.py index e334a2cd8..54da6f917 100644 --- a/zappa/util.py +++ b/zappa/util.py @@ -4,7 +4,10 @@ import requests import shutil import stat -import urlparse +try: + from urllib.parse import urlparse +except ImportError: + from urlparse import urlparse def copytree(src, dst, symlinks=False, ignore=None): @@ -237,7 +240,7 @@ def parse_s3_url(url): bucket = '' path = '' if url: - result = urlparse.urlparse(url) + result = urlparse(url) bucket = result.netloc path = result.path.strip('/') return bucket, path diff --git a/zappa/wsgi.py b/zappa/wsgi.py index 155f83965..1c5c7e26d 100644 --- a/zappa/wsgi.py +++ b/zappa/wsgi.py @@ -1,9 +1,15 @@ +from __future__ import absolute_import import logging import base64 -from urllib import urlencode +try: + from urllib import urlencode + from StringIO import StringIO +except ImportError: + from urllib.parse import urlencode + from io import StringIO + from requestlogger import ApacheFormatter -from StringIO import StringIO from werkzeug import urls diff --git a/zappa/zappa.py b/zappa/zappa.py index 595087331..61611a690 100644 --- a/zappa/zappa.py +++ b/zappa/zappa.py @@ -1,4 +1,6 @@ from __future__ import print_function +from __future__ import absolute_import +import platform import boto3 import botocore @@ -25,7 +27,7 @@ from tqdm import tqdm # Zappa imports -from util import copytree, add_event_source, remove_event_source +from zappa.util import copytree, add_event_source, remove_event_source logging.basicConfig(format='%(levelname)s:%(message)s') logger = logging.getLogger(__name__) @@ -350,7 +352,7 @@ def splitpath(path): parts.append(tail) (path, tail) = os.path.split(path) parts.append(os.path.join(path, tail)) - return map(os.path.normpath, parts)[::-1] + return list(map(os.path.normpath, parts))[::-1] split_venv = splitpath(venv) split_cwd = splitpath(cwd) @@ -374,10 +376,12 @@ def splitpath(path): # Then, do the site-packages.. temp_package_path = os.path.join(tempfile.gettempdir(), str(int(time.time() + 1))) + python_version = '.'.join(platform.python_version_tuple()[:2]) if os.sys.platform == 'win32': site_packages = os.path.join(venv, 'Lib', 'site-packages') else: - site_packages = os.path.join(venv, 'lib', 'python2.7', 'site-packages') + site_packages = os.path.join(venv, 'lib', 'python%s' % python_version, + 'site-packages') if minify: excludes = ZIP_EXCLUDES + exclude copytree(site_packages, temp_package_path, symlinks=False, ignore=shutil.ignore_patterns(*excludes)) @@ -385,7 +389,7 @@ def splitpath(path): copytree(site_packages, temp_package_path, symlinks=False) # We may have 64-bin specific packages too. - site_packages_64 = os.path.join(venv, 'lib64', 'python2.7', 'site-packages') + site_packages_64 = os.path.join(venv, 'lib64', 'python%s' % python_version, 'site-packages') if os.path.exists(site_packages_64): if minify: excludes = ZIP_EXCLUDES + exclude @@ -510,7 +514,7 @@ def get_manylinux_wheel(self, package): for f in data['releases'][version]: if f['filename'].endswith('cp27mu-manylinux1_x86_64.whl'): return f['url'] - except Exception, e: # pragma: no cover + except Exception as e: # pragma: no cover return None return None @@ -1148,7 +1152,7 @@ def update_stack(self, name, working_bucket, wait=False, update_only=False): TemplateURL=url, Tags=tags) print('Waiting for stack {0} to update..'.format(name)) - except botocore.client.ClientError as e: + except botocore.exceptions.ClientError as e: if e.response['Error']['Message'] == 'No updates are to be performed.': wait = False else: @@ -1621,7 +1625,7 @@ def delete_rule(self, rule_name): if error_code == 'AccessDeniedException': raise else: - logger.debug('No target found for this rule: {} {}'.format(rule_name, e.message)) + logger.debug('No target found for this rule: {} {}'.format(rule_name, e)) return if 'Targets' in targets and targets['Targets']: @@ -1696,10 +1700,10 @@ def _clear_policy(self, lambda_name): else: logger.debug('Failed to load Lambda function policy: {}'.format(policy_response)) except ClientError as e: - if e.message.find('ResourceNotFoundException') > -1: + if 'ResourceNotFoundException' in e.response: logger.debug('No policy found, must be first run.') else: - logger.error('Unexpected client error {}'.format(e.message)) + logger.error('Unexpected client error {}'.format(e)) ## # CloudWatch Logging From ce0bee76ba752aaf61a0f8774604d60f2cfdd0cd Mon Sep 17 00:00:00 2001 From: Sebastien Billion Date: Wed, 30 Nov 2016 11:41:36 +0100 Subject: [PATCH 2/3] Remove useless file created before --- test_env.json | 3 --- 1 file changed, 3 deletions(-) delete mode 100644 test_env.json diff --git a/test_env.json b/test_env.json deleted file mode 100644 index 04a83e926..000000000 --- a/test_env.json +++ /dev/null @@ -1,3 +0,0 @@ -{ - "hello": "world" -} \ No newline at end of file From ccf8f66a8cc34d7b9516bb008422238867c2debb Mon Sep 17 00:00:00 2001 From: Sebastien Billion Date: Wed, 30 Nov 2016 13:14:29 +0100 Subject: [PATCH 3/3] Add python 3.5 to travis --- .travis.yml | 1 + tests/tests.py | 7 ++++--- tests/tests_middleware.py | 4 +--- tests/utils.py | 20 ++++++++++++++++---- zappa/handler.py | 1 - zappa/middleware.py | 6 +++--- 6 files changed, 25 insertions(+), 14 deletions(-) diff --git a/.travis.yml b/.travis.yml index 5e92d89fe..a7b3180a1 100644 --- a/.travis.yml +++ b/.travis.yml @@ -1,6 +1,7 @@ language: python python: - "2.7" + - "3.5" # command to install dependencies install: - "pip install setuptools --upgrade; pip install -r test_requirements.txt; pip install -e git+https://github.com/django/django-contrib-comments.git#egg=django-contrib-comments; python setup.py install" diff --git a/tests/tests.py b/tests/tests.py index 550ceb153..867255ad3 100644 --- a/tests/tests.py +++ b/tests/tests.py @@ -6,7 +6,7 @@ import base64 import collections import json -from contextlib import nested + import mock import os @@ -83,8 +83,9 @@ def test_copy_editable_packages(self, mock_remove, mock_find_packages): temp_egg_link = os.path.join(temp_package_dir, 'package-python.egg-link') z = Zappa() - with nested( - patch_open(), mock.patch('glob.glob'), mock.patch('zappa.zappa.copytree') + with ( + patch_open(), mock.patch('glob.glob'), mock.patch( + 'zappa.util.copytree') ) as ((mock_open, mock_file), mock_glob, mock_copytree): # We read in the contents of the egg-link file mock_file.read.return_value = "{}\n.".format(egg_path) diff --git a/tests/tests_middleware.py b/tests/tests_middleware.py index 05603c3a5..2d76faafa 100644 --- a/tests/tests_middleware.py +++ b/tests/tests_middleware.py @@ -470,10 +470,8 @@ def change_cookie(environ, start_response): print('zdict', zdict) try: zdict2 = json.loads(base58.b58decode(zdict['zappa'])) - except TypeError: - print(base58.b58decode(zdict['zappa'])) - zdict2 = json.loads(zdict.get('zappa')) + zdict2 = json.loads(bytes.decode(base58.b58decode(zdict['zappa']))) print('zdict2', zdict2) self.assertEqual(len(zdict2), 3) diff --git a/tests/utils.py b/tests/utils.py index 0404f2593..7af6f52c3 100644 --- a/tests/utils.py +++ b/tests/utils.py @@ -1,8 +1,11 @@ +import platform + import placebo import boto3 import os import functools from contextlib import contextmanager + from mock import patch, MagicMock PLACEBO_DIR = os.path.join(os.path.dirname(__file__), 'placebo') @@ -59,13 +62,22 @@ def patch_open(): yielded. Yields the mock for "open" and "file", respectively.""" mock_open = MagicMock(spec=open) - mock_file = MagicMock(spec=file) - + try: + mock_file = MagicMock(spec=file) + except NameError: + mock_file = MagicMock(spec=open) @contextmanager def stub_open(*args, **kwargs): mock_open(*args, **kwargs) yield mock_file - with patch('__builtin__.open', stub_open): - yield mock_open, mock_file + python_version = '.'.join(platform.python_version_tuple()[:2]) + + if int(python_version[0]) < 3: + + with patch('__builtin__.open', stub_open): + yield mock_open, mock_file + else: + with patch('builtins.open', stub_open): + yield mock_open, mock_file diff --git a/zappa/handler.py b/zappa/handler.py index 8c1967118..d8c23c7ec 100644 --- a/zappa/handler.py +++ b/zappa/handler.py @@ -161,7 +161,6 @@ def load_remote_settings(self, remote_bucket, remote_file): try: settings_dict = json.loads(content) except (ValueError, TypeError) as e: # pragma: no cover - print(content) print('Failed to parse remote settings! : ' + content) return diff --git a/zappa/middleware.py b/zappa/middleware.py index 5b81e7420..858700c98 100644 --- a/zappa/middleware.py +++ b/zappa/middleware.py @@ -55,7 +55,6 @@ def __call__(self, environ, start_response): only passing cookies that are still valid to the WSGI app. """ self.start_response = start_response - print(environ) # Parse cookies from the WSGI environment parsed = parse_cookie(environ) @@ -162,7 +161,7 @@ def decode_zappa_cookie(self, encoded_zappa): except TypeError: self.decoded_zappa = base58.b58decode(str.encode(encoded_zappa)) - self.request_cookies = json.loads(self.decoded_zappa) + self.request_cookies = json.loads(bytes.decode(self.decoded_zappa)) def filter_expired_cookies(self): @@ -183,7 +182,8 @@ def iter_cookies_expires(self): Yield name and expires of cookies. """ for name, value in self.request_cookies.items(): - cookie = (b'%s=%s' % (name ,value)).encode('utf-8') + cookie = '{}={}'.format(name ,value).encode('utf-8') + if cookie.count('=') is 1: continue kvps = cookie.split(';')