Merge pull request #497 from xuru/Fix_requirements_editable_packages

This will find all .egg-link files and add the correct python module to the zip package.

tests/tests.py

@@ -2,6 +2,8 @@
 import base64
 import collections
 import json
+from contextlib import nested
+
 import mock
 import os
 import random
@@ -13,7 +15,7 @@
 from click.exceptions import ClickException
 from lambda_packages import lambda_packages
 
-from .utils import placebo_session
+from .utils import placebo_session, patch_open
 
 from zappa.cli import ZappaCLI, shamelessly_promote
 from zappa.ext.django_zappa import get_django_wsgi
@@ -61,6 +63,41 @@ def test_zappa(self):
         self.assertTrue(True)
         Zappa()
 
+    @mock.patch('zappa.zappa.find_packages')
+    @mock.patch('os.remove')
+    def test_copy_editable_packages(self, mock_remove, mock_find_packages):
+        temp_package_dir = '/var/folders/rn/9tj3_p0n1ln4q4jn1lgqy4br0000gn/T/1480455339'
+        egg_links = [
+            '/user/test/.virtualenvs/test/lib/python2.7/site-packages/package-python.egg-link'
+        ]
+        egg_path = "/some/other/directory/package"
+        mock_find_packages.return_value = ["package", "package.subpackage", "package.another"]
+        temp_egg_link = os.path.join(temp_package_dir, 'package-python.egg-link')
+
+        z = Zappa()
+        with nested(
+                patch_open(), mock.patch('glob.glob'), mock.patch('zappa.zappa.copytree')
+        ) as ((mock_open, mock_file), mock_glob, mock_copytree):
+            # We read in the contents of the egg-link file
+            mock_file.read.return_value = "{}\n.".format(egg_path)
+
+            # we use glob.glob to get the egg-links in the temp packages directory
+            mock_glob.return_value = [temp_egg_link]
+
+            z.copy_editable_packages(egg_links, temp_package_dir)
+
+            # make sure we copied the right directories
+            mock_copytree.assert_called_with(
+                os.path.join(egg_path, 'package'),
+                os.path.join(temp_package_dir, 'package'),
+                symlinks=False
+            )
+            self.assertEqual(mock_copytree.call_count, 1)
+
+            # make sure it removes the egg-link from the temp packages directory
+            mock_remove.assert_called_with(temp_egg_link)
+            self.assertEqual(mock_remove.call_count, 1)
+
     def test_create_lambda_package(self):
         # mock the pip.get_installed_distributions() to include a package in lambda_packages so that the code
         # for zipping pre-compiled packages gets called

tests/utils.py

@@ -2,6 +2,8 @@
 import boto3
 import os
 import functools
+from contextlib import contextmanager
+from mock import patch, MagicMock
 
 PLACEBO_DIR = os.path.join(os.path.dirname(__file__), 'placebo')
 
@@ -49,3 +51,21 @@ def wrapper(*args, **kwargs):
         return function(*args, **kwargs)
 
     return wrapper
+
+
+@contextmanager
+def patch_open():
+    """Patch open() to allow mocking both open() itself and the file that is
+    yielded.
+    Yields the mock for "open" and "file", respectively."""
+    mock_open = MagicMock(spec=open)
+    mock_file = MagicMock(spec=file)
+
+    @contextmanager
+    def stub_open(*args, **kwargs):
+        mock_open(*args, **kwargs)
+        yield mock_file
+
+    with patch('__builtin__.open', stub_open):
+        yield mock_open, mock_file
+

zappa/zappa.py

@@ -1,5 +1,8 @@
 from __future__ import print_function
 
+import glob
+from setuptools import find_packages
+
 import boto3
 import botocore
 import json
@@ -301,6 +304,18 @@ def cache_param(self, value):
     ##
     # Packaging
     ##
+    def copy_editable_packages(self, egg_links, temp_package_path):
+        for egg_link in egg_links:
+            with open(egg_link) as df:
+                egg_path = df.read().decode('utf-8').splitlines()[0].strip()
+                pkgs = set([x.split(".")[0] for x in find_packages(egg_path, exclude=['test', 'tests'])])
+                for pkg in pkgs:
+                    copytree(os.path.join(egg_path, pkg), os.path.join(temp_package_path, pkg), symlinks=False)
+
+        if temp_package_path:
+            # now remove any egg-links as they will cause issues if they still exist
+            for link in glob.glob(os.path.join(temp_package_path, "*.egg-link")):
+                os.remove(link)
 
     def create_lambda_zip(self, prefix='lambda_package', handler_file=None,
                           minify=True, exclude=None, use_precompiled_packages=True, include=None, venv=None):
@@ -373,37 +388,33 @@ def splitpath(path):
             copytree(cwd, temp_project_path, symlinks=False)
 
         # Then, do the site-packages..
+        egg_links = []
         temp_package_path = os.path.join(tempfile.gettempdir(), str(int(time.time() + 1)))
         if os.sys.platform == 'win32':
             site_packages = os.path.join(venv, 'Lib', 'site-packages')
         else:
             site_packages = os.path.join(venv, 'lib', 'python2.7', 'site-packages')
+        egg_links.extend(glob.glob(os.path.join(site_packages, '*.egg-link')))
+
         if minify:
             excludes = ZIP_EXCLUDES + exclude
             copytree(site_packages, temp_package_path, symlinks=False, ignore=shutil.ignore_patterns(*excludes))
+
         else:
             copytree(site_packages, temp_package_path, symlinks=False)
 
         # We may have 64-bin specific packages too.
         site_packages_64 = os.path.join(venv, 'lib64', 'python2.7', 'site-packages')
         if os.path.exists(site_packages_64):
+            egg_links.extend(glob.glob(os.path.join(site_packages_64, '*.egg-link')))
             if minify:
                 excludes = ZIP_EXCLUDES + exclude
                 copytree(site_packages_64, temp_package_path, symlinks=False, ignore=shutil.ignore_patterns(*excludes))
             else:
                 copytree(site_packages_64, temp_package_path, symlinks=False)
 
-        # Then, do the src/.. for things installed with pip install -e git:// etc
-        if os.sys.platform == 'win32':
-            src_packages = os.path.join(venv, 'Src')
-        else:
-            src_packages = os.path.join(venv, 'src')
-        if os.path.exists(src_packages):
-            if minify:
-                excludes = ZIP_EXCLUDES + exclude
-                copytree(src_packages, temp_package_path, symlinks=False, ignore=shutil.ignore_patterns(*excludes))
-            else:
-                copytree(src_packages, temp_package_path, symlinks=False)
+        if egg_links:
+            self.copy_editable_packages(egg_links, temp_package_path)
 
         copy_tree(temp_package_path, temp_project_path, update=True)