From 833aeb7e39dae0383358181a17d8557d6df1723c Mon Sep 17 00:00:00 2001 From: Eric Plaster Date: Tue, 29 Nov 2016 12:31:35 -0600 Subject: [PATCH 1/2] This will find all .egg-link files and add the correct python modules to the zip package. --- zappa/zappa.py | 31 +++++++++++++++++-------------- 1 file changed, 17 insertions(+), 14 deletions(-) diff --git a/zappa/zappa.py b/zappa/zappa.py index 715c442f9..6f11e19bb 100644 --- a/zappa/zappa.py +++ b/zappa/zappa.py @@ -1,5 +1,8 @@ from __future__ import print_function +import glob +from setuptools import find_packages + import boto3 import botocore import json @@ -373,37 +376,37 @@ def splitpath(path): copytree(cwd, temp_project_path, symlinks=False) # Then, do the site-packages.. + egg_links = [] temp_package_path = os.path.join(tempfile.gettempdir(), str(int(time.time() + 1))) if os.sys.platform == 'win32': site_packages = os.path.join(venv, 'Lib', 'site-packages') else: site_packages = os.path.join(venv, 'lib', 'python2.7', 'site-packages') + egg_links.extend(glob.glob(os.path.join(site_packages, '*.egg-link'))) + if minify: excludes = ZIP_EXCLUDES + exclude copytree(site_packages, temp_package_path, symlinks=False, ignore=shutil.ignore_patterns(*excludes)) + else: - copytree(site_packages, temp_package_path, symlinks=False) + copytree(site_packages, temp_package_path, symlinks=False, ignore=shutil.ignore_patterns('*.egg-link')) # We may have 64-bin specific packages too. site_packages_64 = os.path.join(venv, 'lib64', 'python2.7', 'site-packages') if os.path.exists(site_packages_64): + egg_links.extend(glob.glob(os.path.join(site_packages_64, '*.egg-link'))) if minify: excludes = ZIP_EXCLUDES + exclude copytree(site_packages_64, temp_package_path, symlinks=False, ignore=shutil.ignore_patterns(*excludes)) else: - copytree(site_packages_64, temp_package_path, symlinks=False) - - # Then, do the src/.. for things installed with pip install -e git:// etc - if os.sys.platform == 'win32': - src_packages = os.path.join(venv, 'Src') - else: - src_packages = os.path.join(venv, 'src') - if os.path.exists(src_packages): - if minify: - excludes = ZIP_EXCLUDES + exclude - copytree(src_packages, temp_package_path, symlinks=False, ignore=shutil.ignore_patterns(*excludes)) - else: - copytree(src_packages, temp_package_path, symlinks=False) + copytree(site_packages_64, temp_package_path, symlinks=False, ignore=shutil.ignore_patterns('*.egg-link')) + + for egg_link in egg_links: + with open(egg_link) as df: + line = df.readlines()[0] # egg-link files all ways has two lines, the first being the path + pkgs = set([x.split(".")[0] for x in find_packages(line, exclude=['test', 'tests'])]) + for pkg in pkgs: + copytree(pkg, temp_package_path, symlinks=False) copy_tree(temp_package_path, temp_project_path, update=True) From c3a6f0ed8d68c107ff6fc717f4205723cd833a37 Mon Sep 17 00:00:00 2001 From: Eric Plaster Date: Tue, 29 Nov 2016 17:39:23 -0600 Subject: [PATCH 2/2] Added test case for copying editable packages. --- tests/tests.py | 39 ++++++++++++++++++++++++++++++++++++++- tests/utils.py | 20 ++++++++++++++++++++ zappa/zappa.py | 24 ++++++++++++++++-------- 3 files changed, 74 insertions(+), 9 deletions(-) diff --git a/tests/tests.py b/tests/tests.py index 3046d5bf6..f093da5dc 100644 --- a/tests/tests.py +++ b/tests/tests.py @@ -2,6 +2,8 @@ import base64 import collections import json +from contextlib import nested + import mock import os import random @@ -13,7 +15,7 @@ from click.exceptions import ClickException from lambda_packages import lambda_packages -from .utils import placebo_session +from .utils import placebo_session, patch_open from zappa.cli import ZappaCLI, shamelessly_promote from zappa.ext.django_zappa import get_django_wsgi @@ -61,6 +63,41 @@ def test_zappa(self): self.assertTrue(True) Zappa() + @mock.patch('zappa.zappa.find_packages') + @mock.patch('os.remove') + def test_copy_editable_packages(self, mock_remove, mock_find_packages): + temp_package_dir = '/var/folders/rn/9tj3_p0n1ln4q4jn1lgqy4br0000gn/T/1480455339' + egg_links = [ + '/user/test/.virtualenvs/test/lib/python2.7/site-packages/package-python.egg-link' + ] + egg_path = "/some/other/directory/package" + mock_find_packages.return_value = ["package", "package.subpackage", "package.another"] + temp_egg_link = os.path.join(temp_package_dir, 'package-python.egg-link') + + z = Zappa() + with nested( + patch_open(), mock.patch('glob.glob'), mock.patch('zappa.zappa.copytree') + ) as ((mock_open, mock_file), mock_glob, mock_copytree): + # We read in the contents of the egg-link file + mock_file.read.return_value = "{}\n.".format(egg_path) + + # we use glob.glob to get the egg-links in the temp packages directory + mock_glob.return_value = [temp_egg_link] + + z.copy_editable_packages(egg_links, temp_package_dir) + + # make sure we copied the right directories + mock_copytree.assert_called_with( + os.path.join(egg_path, 'package'), + os.path.join(temp_package_dir, 'package'), + symlinks=False + ) + self.assertEqual(mock_copytree.call_count, 1) + + # make sure it removes the egg-link from the temp packages directory + mock_remove.assert_called_with(temp_egg_link) + self.assertEqual(mock_remove.call_count, 1) + def test_create_lambda_package(self): # mock the pip.get_installed_distributions() to include a package in lambda_packages so that the code # for zipping pre-compiled packages gets called diff --git a/tests/utils.py b/tests/utils.py index 7a61e8f76..0404f2593 100644 --- a/tests/utils.py +++ b/tests/utils.py @@ -2,6 +2,8 @@ import boto3 import os import functools +from contextlib import contextmanager +from mock import patch, MagicMock PLACEBO_DIR = os.path.join(os.path.dirname(__file__), 'placebo') @@ -49,3 +51,21 @@ def wrapper(*args, **kwargs): return function(*args, **kwargs) return wrapper + + +@contextmanager +def patch_open(): + """Patch open() to allow mocking both open() itself and the file that is + yielded. + Yields the mock for "open" and "file", respectively.""" + mock_open = MagicMock(spec=open) + mock_file = MagicMock(spec=file) + + @contextmanager + def stub_open(*args, **kwargs): + mock_open(*args, **kwargs) + yield mock_file + + with patch('__builtin__.open', stub_open): + yield mock_open, mock_file + diff --git a/zappa/zappa.py b/zappa/zappa.py index 6f11e19bb..08c3f6675 100644 --- a/zappa/zappa.py +++ b/zappa/zappa.py @@ -304,6 +304,18 @@ def cache_param(self, value): ## # Packaging ## + def copy_editable_packages(self, egg_links, temp_package_path): + for egg_link in egg_links: + with open(egg_link) as df: + egg_path = df.read().decode('utf-8').splitlines()[0].strip() + pkgs = set([x.split(".")[0] for x in find_packages(egg_path, exclude=['test', 'tests'])]) + for pkg in pkgs: + copytree(os.path.join(egg_path, pkg), os.path.join(temp_package_path, pkg), symlinks=False) + + if temp_package_path: + # now remove any egg-links as they will cause issues if they still exist + for link in glob.glob(os.path.join(temp_package_path, "*.egg-link")): + os.remove(link) def create_lambda_zip(self, prefix='lambda_package', handler_file=None, minify=True, exclude=None, use_precompiled_packages=True, include=None, venv=None): @@ -389,7 +401,7 @@ def splitpath(path): copytree(site_packages, temp_package_path, symlinks=False, ignore=shutil.ignore_patterns(*excludes)) else: - copytree(site_packages, temp_package_path, symlinks=False, ignore=shutil.ignore_patterns('*.egg-link')) + copytree(site_packages, temp_package_path, symlinks=False) # We may have 64-bin specific packages too. site_packages_64 = os.path.join(venv, 'lib64', 'python2.7', 'site-packages') @@ -399,14 +411,10 @@ def splitpath(path): excludes = ZIP_EXCLUDES + exclude copytree(site_packages_64, temp_package_path, symlinks=False, ignore=shutil.ignore_patterns(*excludes)) else: - copytree(site_packages_64, temp_package_path, symlinks=False, ignore=shutil.ignore_patterns('*.egg-link')) + copytree(site_packages_64, temp_package_path, symlinks=False) - for egg_link in egg_links: - with open(egg_link) as df: - line = df.readlines()[0] # egg-link files all ways has two lines, the first being the path - pkgs = set([x.split(".")[0] for x in find_packages(line, exclude=['test', 'tests'])]) - for pkg in pkgs: - copytree(pkg, temp_package_path, symlinks=False) + if egg_links: + self.copy_editable_packages(egg_links, temp_package_path) copy_tree(temp_package_path, temp_project_path, update=True)