Merge 618aa8e into 1202044

zappa/zappa.py

@@ -11,6 +11,7 @@
 import requests
 import shutil
 import string
+import subprocess
 import sys
 import tarfile
 import tempfile
@@ -22,6 +23,7 @@
 from lambda_packages import lambda_packages
 from os.path import expanduser
 from tqdm import tqdm
+from platform import machine
 
 logger = logging.getLogger(__name__)
 
@@ -177,6 +179,8 @@
 
 ZIP_EXCLUDES =  ['*.exe', '*.DS_Store', '*.Python', '*.git', '*.zip', '*.tar.gz','*.hg']
 
+STANDARD_CONDA_PACKAGES = ['openssl','pip','python','readline','sqlite','wheel', 'boto3', 'botocore']
+
 ##
 # Classes
 ##
@@ -235,7 +239,7 @@ def __init__(self, boto_session=None, profile_name=None):
     ##
 
     def create_lambda_zip(self, prefix='lambda_package', handler_file=None,
-                          minify=True, exclude=None, use_precompiled_packages=True, include=None, venv=None):
+                          minify=True, exclude=None, use_precompiled_packages=True, include=None, venv=None, exclude_conda_packages=STANDARD_CONDA_PACKAGES):
         """
         Creates a Lambda-ready zip file of the current virtualenvironment and working directory.
 
@@ -245,11 +249,26 @@ def create_lambda_zip(self, prefix='lambda_package', handler_file=None,
         print("Packaging project as zip...")
 
         if not venv:
-            try:
-                venv = os.environ['VIRTUAL_ENV']
-            except KeyError as e: # pragma: no cover
-                print("Zappa requires an active virtual environment.")
+            virtual_env = os.environ.get('VIRTUAL_ENV', None)
+            conda_env = os.environ.get('CONDA_ENV_PATH', None)
+            if virtual_env is None and conda_env is None:
+                print("Zappa requires an active virtual or conda environment.")
+                quit()
+            elif virtual_env is not None and conda_env is not None:
+                print('Ambiguous environment: both virtual environments and conda environments are active.')
                 quit()
+            else:
+                venv = virtual_env if virtual_env is not None else conda_env
+                if conda_env is not None:
+                    if (('linux' not in sys.platform)
+                            or (machine() in ['armv6l', 'armv7l', 'ppc64le'])
+                            or (8 * tuple.__itemsize__ != 64)):
+                        warnings.warn('The local architecture is different than that of AWS lambda (linux-64).\nMake sure the binaries in your conda environment are compatible with AWS lambda')
+                    if sys.version_info[:2] != (2,7):
+                        warnings.warn('The local python version is different than that of AWS lambda (2.7.x).\nMake sure the binaries in your conda environment are compatible with AWS lambda')
+                    if use_precompiled_packages:
+                        warnings.warn('Using conda while use_precompiled_packages is set to true is not recommended: it may lead to overwriting conda packages')
+                USE_CONDA = conda_env is not None
 
         cwd = os.getcwd()
         zip_fname = prefix + '-' + str(int(time.time())) + '.zip'
@@ -279,7 +298,8 @@ def splitpath(path):
             print("Warning! Your project and virtualenv have the same name! You may want to re-create your venv with a new name, or explicitly define a 'project_name', as this may cause errors.")
 
         # First, do the project..
-        temp_project_path = os.path.join(tempfile.gettempdir(), str(int(time.time())))
+        temp_zappa_folder = os.path.join(tempfile.gettempdir(), str(int(time.time())))
+        temp_project_path = os.path.join(temp_zappa_folder, 'project')
 
         if minify:
             excludes = ZIP_EXCLUDES + exclude + [split_venv[-1]]
@@ -289,16 +309,39 @@ def splitpath(path):
 
         # Then, do the site-packages..
         # TODO Windows: %VIRTUAL_ENV%\Lib\site-packages
-        temp_package_path = os.path.join(tempfile.gettempdir(), str(int(time.time() + 1)))
-        site_packages = os.path.join(venv, 'lib', 'python2.7', 'site-packages')
+        temp_package_path = os.path.join(temp_zappa_folder, 'package')
+        if not USE_CONDA:
+            site_packages = os.path.join(venv, 'lib', 'python2.7', 'site-packages')
 
-        if minify:
-            excludes = ZIP_EXCLUDES + exclude
-            shutil.copytree(site_packages, temp_package_path, symlinks=False, ignore=shutil.ignore_patterns(*excludes))
-        else:
-            shutil.copytree(site_packages, temp_package_path, symlinks=False)
+            if minify:
+                excludes = ZIP_EXCLUDES + exclude
+                shutil.copytree(site_packages, temp_package_path, symlinks=False, ignore=shutil.ignore_patterns(*excludes))
+            else:
+                shutil.copytree(site_packages, temp_package_path, symlinks=False)
 
-        copy_tree(temp_package_path, temp_project_path, update=True)
+            copy_tree(temp_package_path, temp_project_path, update=True)
+        else:
+            temp_package_path = os.path.join(temp_zappa_folder,'conda_env')
+            site_packages = os.path.join(temp_package_path, 'lib', 'python2.7', 'site-packages')
+            if minify:
+                excludes = ZIP_EXCLUDES + exclude
+                shutil.copytree(conda_env, temp_package_path, symlinks=True, ignore=shutil.ignore_patterns(*excludes))
+                # Use conda cli to remove standard packages like python, pip, ...
+                subprocess.call(['conda','remove','-p',temp_package_path,'--force','--yes']+exclude_conda_packages)
+            else:
+                shutil.copytree(conda_env, temp_package_path, symlinks=True)
+
+            # Extracts all egg files (e.g. setuptools)
+            egg_files = [f for f in os.listdir(site_packages) if os.path.isfile(os.path.join(site_packages, f)) and f.split('.')[-1]=='egg']
+            for egg_file in egg_files:
+                print('Extracting '+ egg_file)
+                with zipfile.ZipFile(os.path.join(site_packages,egg_file)) as zf:
+                    zf.extractall(os.path.join(site_packages))
+                os.remove(os.path.join(site_packages,egg_file))
+            # Put site-packages at the root of the environment
+            copy_tree(site_packages, temp_package_path, update=True)
+            shutil.rmtree(site_packages)
+            copy_tree(temp_package_path, temp_project_path, update=True)
 
         # Then the pre-compiled packages..
         if use_precompiled_packages:
@@ -355,8 +398,7 @@ def splitpath(path):
         zipf.close()
 
         # Trash the temp directory
-        shutil.rmtree(temp_project_path)
-        shutil.rmtree(temp_package_path)
+        shutil.rmtree(temp_zappa_folder)
 
         # Warn if this is too large for Lambda.
         file_stats = os.stat(zip_path)