Unit & Integration test for python_pip workflow (#18)

Fixes #14

.coveragerc

@@ -1,5 +1,7 @@
 [run]
 branch = True
+omit =
+    aws_lambda_builders/workflows/python_pip/compat.py
 [report]
 exclude_lines =
     pragma: no cover

aws_lambda_builders/__init__.py

@@ -1 +1,4 @@
+"""
+AWS Lambda Builder Library
+"""
 __version__ = '0.0.1'

aws_lambda_builders/workflow.py

@@ -49,6 +49,7 @@ def __new__(mcs, name, bases, class_dict):
         if not isinstance(cls.CAPABILITY, Capability):
             raise ValueError("Workflow '{}' must register valid capabilities".format(cls.NAME))
 
+        LOG.debug("Registering workflow '%s' with capability '%s'", cls.NAME, cls.CAPABILITY)
         DEFAULT_REGISTRY[cls.CAPABILITY] = cls
 
         return cls

aws_lambda_builders/workflows/__init__.py

@@ -0,0 +1,5 @@
+"""
+Officially supported builder workflows
+"""
+
+import aws_lambda_builders.workflows.python_pip

aws_lambda_builders/workflows/python_pip/__init__.py

@@ -0,0 +1,5 @@
+"""
+Builds Python Lambda functions using PIP dependency manager
+"""
+
+from .workflow import PythonPipWorkflow

aws_lambda_builders/workflows/python_pip/actions.py

@@ -1,10 +1,15 @@
-from aws_lambda_builders.actions import BaseAction
-from .packager import PythonPipDependencyBuilder
+"""
+Action to resolve Python dependencies using PIP
+"""
+
+from aws_lambda_builders.actions import BaseAction, Purpose, ActionFailedError
+from .packager import PythonPipDependencyBuilder, PackagerError
 
 
 class PythonPipBuildAction(BaseAction):
 
     NAME = 'PythonPipBuildAction'
+    PURPOSE = Purpose.RESOLVE_DEPENDENCIES
 
     def __init__(self, artifacts_dir, manifest_path, runtime):
         self.artifacts_dir = artifacts_dir
@@ -13,8 +18,11 @@ def __init__(self, artifacts_dir, manifest_path, runtime):
         self.package_builder = PythonPipDependencyBuilder()
 
     def execute(self):
-        self.package_builder.build_dependencies(
-            self.artifacts_dir,
-            self.manifest_path,
-            self.runtime,
-        )
+        try:
+            self.package_builder.build_dependencies(
+                self.artifacts_dir,
+                self.manifest_path,
+                self.runtime,
+            )
+        except PackagerError as ex:
+            raise ActionFailedError(str(ex))

aws_lambda_builders/workflows/python_pip/packager.py

@@ -1,3 +1,7 @@
+"""
+Installs packages using PIP
+"""
+
 import sys
 import re
 import subprocess
@@ -20,24 +24,34 @@
 """
 
 
-class InvalidSourceDistributionNameError(Exception):
+class PackagerError(Exception):
+    pass
+
+
+class InvalidSourceDistributionNameError(PackagerError):
     pass
 
 
-class MissingDependencyError(Exception):
+class RequirementsFileNotFoundError(PackagerError):
+    def __init__(self, requirements_path):
+        super(RequirementsFileNotFoundError, self).__init__(
+            'Requirements file not found: %s' % requirements_path)
+
+
+class MissingDependencyError(PackagerError):
     """Raised when some dependencies could not be packaged for any reason."""
     def __init__(self, missing):
         self.missing = missing
 
 
-class NoSuchPackageError(Exception):
+class NoSuchPackageError(PackagerError):
     """Raised when a package name or version could not be found."""
     def __init__(self, package_name):
         super(NoSuchPackageError, self).__init__(
             'Could not satisfy the requirement: %s' % package_name)
 
 
-class PackageDownloadError(Exception):
+class PackageDownloadError(PackagerError):
     """Generic networking error during a package download."""
     pass
 
@@ -54,10 +68,12 @@ def __init__(self, osutils=None, dependency_builder=None):
         :param dependency_builder: This class will be used to build the
             dependencies of the project.
         """
+        self.osutils = osutils
         if osutils is None:
-            osutils = OSUtils()
+            self.osutils = OSUtils()
+
         if dependency_builder is None:
-            dependency_builder = DependencyBuilder(osutils)
+            dependency_builder = DependencyBuilder(self.osutils)
         self._dependency_builder = dependency_builder
 
     def build_dependencies(self, artifacts_dir_path, requirements_path,
@@ -93,6 +109,10 @@ def build_dependencies(self, artifacts_dir_path, requirements_path,
         # correct version of python. We need to enforce that assumption here
         # by finding/creating a virtualenv of the correct version and when
         # pip is called set the appropriate env vars.
+
+        if not self.osutils.file_exists(requirements_path):
+            raise RequirementsFileNotFoundError(requirements_path)
+
         self._dependency_builder.build_site_packages(
             requirements_path, artifacts_dir_path)
 

aws_lambda_builders/workflows/python_pip/utils.py

@@ -1,3 +1,7 @@
+"""
+Commonly used utilities
+"""
+
 import io
 import os
 import zipfile
@@ -10,26 +14,10 @@
 
 
 class OSUtils(object):
-    ZIP_DEFLATED = zipfile.ZIP_DEFLATED
 
     def environ(self):
         return os.environ
 
-    def open(self, filename, mode):
-        return open(filename, mode)
-
-    def open_zip(self, filename, mode, compression=ZIP_DEFLATED):
-        return zipfile.ZipFile(filename, mode, compression=compression)
-
-    def remove_file(self, filename):
-        """Remove a file, noop if file does not exist."""
-        # Unlike os.remove, if the file does not exist,
-        # then this method does nothing.
-        try:
-            os.remove(filename)
-        except OSError:
-            pass
-
     def file_exists(self, filename):
         return os.path.isfile(filename)
 
@@ -46,14 +34,6 @@ def get_file_contents(self, filename, binary=True, encoding='utf-8'):
         with io.open(filename, mode, encoding=encoding) as f:
             return f.read()
 
-    def set_file_contents(self, filename, contents, binary=True):
-        if binary:
-            mode = 'wb'
-        else:
-            mode = 'w'
-        with open(filename, mode) as f:
-            f.write(contents)
-
     def extract_zipfile(self, zipfile_path, unpack_dir):
         with zipfile.ZipFile(zipfile_path, 'r') as z:
             z.extractall(unpack_dir)
@@ -71,18 +51,9 @@ def get_directory_contents(self, path):
     def makedirs(self, path):
         os.makedirs(path)
 
-    def dirname(self, path):
-        return os.path.dirname(path)
-
-    def abspath(self, path):
-        return os.path.abspath(path)
-
     def joinpath(self, *args):
         return os.path.join(*args)
 
-    def walk(self, path):
-        return os.walk(path)
-
     def copytree(self, source, destination):
         if not os.path.exists(destination):
             self.makedirs(destination)
@@ -98,12 +69,6 @@ def copytree(self, source, destination):
     def rmtree(self, directory):
         shutil.rmtree(directory)
 
-    def copy(self, source, destination):
-        shutil.copy(source, destination)
-
-    def move(self, source, destination):
-        shutil.move(source, destination)
-
     @contextlib.contextmanager
     def tempdir(self):
         tempdir = tempfile.mkdtemp()
@@ -122,19 +87,3 @@ def mtime(self, path):
     @property
     def pipe(self):
         return subprocess.PIPE
-
-
-class UI(object):
-    def __init__(self, out=None, err=None):
-        if out is None:
-            out = sys.stdout
-        if err is None:
-            err = sys.stderr
-        self._out = out
-        self._err = err
-
-    def write(self, msg):
-        self._out.write(msg)
-
-    def error(self, msg):
-        self._err.write(msg)

aws_lambda_builders/workflows/python_pip/workflow.py

@@ -1,11 +1,20 @@
-from aws_lambda_builders.workflow import BaseWorkflow
+"""
+Python PIP Workflow
+"""
+
+from aws_lambda_builders.workflow import BaseWorkflow, Capability
 from aws_lambda_builders.actions import CopySourceAction
 
 from .actions import PythonPipBuildAction
 
 
 class PythonPipWorkflow(BaseWorkflow):
 
+    NAME = "PythonPipWorkflow"
+    CAPABILITY = Capability(language="python",
+                            dependency_manager="pip",
+                            application_framework=None)
+
     def __init__(self,
                  source_dir,
                  artifacts_dir,

tests/integration/workflows/python_pip/test_python_pip.py

@@ -0,0 +1,58 @@
+
+import os
+import shutil
+import tempfile
+from unittest import TestCase
+
+from aws_lambda_builders.builder import LambdaBuilder
+from aws_lambda_builders.exceptions import WorkflowFailedError
+
+
+class TestPythonPipWorkflow(TestCase):
+    """
+    Verifies that `python_pip` workflow works by building a Lambda that requires Numpy
+    """
+
+    TEST_DATA_FOLDER = os.path.join(os.path.dirname(__file__), "testdata")
+
+    def setUp(self):
+        self.source_dir = self.TEST_DATA_FOLDER
+        self.artifacts_dir = tempfile.mkdtemp()
+        self.scratch_dir = tempfile.mkdtemp()
+
+        self.manifest_path_valid = os.path.join(self.TEST_DATA_FOLDER, "requirements-numpy.txt")
+        self.manifest_path_invalid = os.path.join(self.TEST_DATA_FOLDER, "requirements-invalid.txt")
+
+        self.test_data_files = set(os.listdir(self.TEST_DATA_FOLDER))
+
+        self.builder = LambdaBuilder(language="python",
+                                     dependency_manager="pip",
+                                     application_framework=None)
+
+    def tearDown(self):
+        shutil.rmtree(self.artifacts_dir)
+        shutil.rmtree(self.scratch_dir)
+
+    def test_must_build_python_project(self):
+        self.builder.build(self.source_dir, self.artifacts_dir, None, self.manifest_path_valid,
+                           runtime="python2.7")
+
+        expected_files = self.test_data_files.union({"numpy", "numpy-1.15.4.data", "numpy-1.15.4.dist-info"})
+        output_files = set(os.listdir(self.artifacts_dir))
+        self.assertEquals(expected_files, output_files)
+
+    def test_must_fail_to_resolve_dependencies(self):
+
+        with self.assertRaises(WorkflowFailedError) as ctx:
+            self.builder.build(self.source_dir, self.artifacts_dir, None, self.manifest_path_invalid,
+                               runtime="python2.7")
+
+        self.assertIn("Invalid requirement: 'adfasf=1.2.3'", str(ctx.exception))
+
+    def test_must_fail_if_requirements_not_found(self):
+
+        with self.assertRaises(WorkflowFailedError) as ctx:
+            self.builder.build(self.source_dir, self.artifacts_dir, None, os.path.join("non", "existent", "manifest"),
+                               runtime="python2.7")
+
+        self.assertIn("Requirements file not found", str(ctx.exception))

tests/integration/workflows/python_pip/testdata/main.py

@@ -0,0 +1,6 @@
+import numpy
+
+
+def lambda_handler(event, context):
+    # Just return the value of PI with two decimals - 3.14
+    return "{0:.2f}".format(numpy.pi)

tests/integration/workflows/python_pip/testdata/requirements-invalid.txt

@@ -0,0 +1 @@
+adfasf=1.2.3

tests/integration/workflows/python_pip/testdata/requirements-numpy.txt

@@ -0,0 +1 @@
+numpy==1.15.4

tests/unit/workflows/python_pip/test_actions.py

@@ -0,0 +1,31 @@
+
+from unittest import TestCase
+from mock import patch
+
+from aws_lambda_builders.actions import ActionFailedError
+
+from aws_lambda_builders.workflows.python_pip.actions import PythonPipBuildAction
+from aws_lambda_builders.workflows.python_pip.packager import PackagerError
+
+
+class TestPythonPipBuildAction(TestCase):
+
+    @patch("aws_lambda_builders.workflows.python_pip.actions.PythonPipDependencyBuilder")
+    def test_action_must_call_builder(self, PythonPipDependencyBuilderMock):
+        builder_instance = PythonPipDependencyBuilderMock.return_value
+
+        action = PythonPipBuildAction("artifacts", "manifest", "runtime")
+        action.execute()
+
+        builder_instance.build_dependencies.assert_called_with("artifacts", "manifest", "runtime")
+
+    @patch("aws_lambda_builders.workflows.python_pip.actions.PythonPipDependencyBuilder")
+    def test_must_raise_exception_on_failure(self, PythonPipDependencyBuilderMock):
+        builder_instance = PythonPipDependencyBuilderMock.return_value
+        builder_instance.build_dependencies.side_effect = PackagerError()
+
+        action = PythonPipBuildAction("artifacts", "manifest", "runtime")
+
+        with self.assertRaises(ActionFailedError):
+            action.execute()
+

tests/unit/workflows/python_pip/test_packager.py

@@ -88,15 +88,18 @@ def popen(self, *args, **kwargs):
 class TestPythonPipDependencyBuilder(object):
     def test_can_call_dependency_builder(self, osutils):
         mock_dep_builder = mock.Mock(spec=DependencyBuilder)
+        osutils_mock = mock.Mock(spec=osutils)
         builder = PythonPipDependencyBuilder(
-            osutils=osutils,
+            osutils=osutils_mock,
             dependency_builder=mock_dep_builder,
         )
         builder.build_dependencies(
             'artifacts/path/', 'path/to/requirements.txt', 'python3.6'
         )
         mock_dep_builder.build_site_packages.assert_called_once_with(
             'path/to/requirements.txt', 'artifacts/path/')
+        osutils_mock.file_exists.assert_called_once_with(
+            'path/to/requirements.txt')
 
 
 class TestPackage(object):