Merge remote-tracking branch 'origin/master' into feature/django2-on-…

…tests

.travis.yml

@@ -9,7 +9,7 @@ install:
   - "pip install setuptools --upgrade; pip install -r test_requirements.txt; pip install -e git+https://github.com/django/django-contrib-comments.git#egg=django-contrib-comments; python setup.py install" 
 # command to run tests
 env:
-  - TESYCASE=tests/tests_docs.py
+  - TESTCASE=tests/tests_docs.py
   - TESTCASE=tests/test_handler.py
   - TESTCASE=tests/tests_middleware.py
   - TESTCASE=tests/tests_placebo.py

CHANGELOG.md

@@ -1,5 +1,21 @@
 # Zappa Changelog
 
+## 0.47.0
+* Support for SQS events
+* Added test to enforce running of doctoc
+* Add support for running django as a WSGI app (for NewRelic and others)
+* Updates AWS regions for lambda and API Gateway
+* Fix support for gcloud and other packages with slim_handler
+* Add --disable-keep-open to zappa tail
+* Dependency updates
+* Fix pyenv invocation
+* Add custom base_path stripping support
+* Multiple documentation fixes and improvements
+* first iteration of a documented deploy policy
+
+## 0.46.2
+* hotfix for creating virtual environments
+
 ## 0.46.1
 * Hotfix for pipenv support (pip >10.0.1)
 * Adds AWS GovCloud support!

README.md

@@ -59,6 +59,7 @@
     - [IAM Policy](#iam-policy)
     - [API Gateway Lambda Authorizers](#api-gateway-lambda-authorizers)
     - [Cognito User Pool Authorizer](#cognito-user-pool-authorizer)
+    - [API Gateway Resource Policy](#api-gateway-resource-policy)
   - [Setting Environment Variables](#setting-environment-variables)
     - [Local Environment Variables](#local-environment-variables)
     - [Remote AWS Environment Variables](#remote-aws-environment-variables)
@@ -156,7 +157,7 @@ This will automatically detect your application type (Flask/Django - Pyramid use
     // The name of your stage
     "dev": {
         // The name of your S3 bucket
-        "s3_bucket": "lmbda",
+        "s3_bucket": "lambda",
 
         // The modular python path to your WSGI application function.
         // In Flask and Bottle, this is your 'app' object.
@@ -174,7 +175,7 @@ or for Django:
 ```javascript
 {
     "dev": { // The name of your stage
-       "s3_bucket": "lmbda", // The name of your S3 bucket
+       "s3_bucket": "lambda", // The name of your S3 bucket
        "django_settings": "your_project.settings" // The python path to your Django settings.
     }
 }
@@ -460,7 +461,7 @@ However, it's now far easier to use Route 53-based DNS authentication, which wil
 
 ## Executing in Response to AWS Events
 
-Similarly, you can have your functions execute in response to events that happen in the AWS ecosystem, such as S3 uploads, DynamoDB entries, Kinesis streams, and SNS messages.
+Similarly, you can have your functions execute in response to events that happen in the AWS ecosystem, such as S3 uploads, DynamoDB entries, Kinesis streams, SNS messages, and SQS queues.
 
 In your *zappa_settings.json* file, define your [event sources](http://docs.aws.amazon.com/lambda/latest/dg/invoking-lambda-function.html) and the function you wish to execute. For instance, this will execute `your_module.process_upload_function` in response to new objects in your `my-bucket` S3 bucket. Note that `process_upload_function` must accept `event` and `context` parameters.
 
@@ -558,6 +559,21 @@ Optionally you can add [SNS message filters](http://docs.aws.amazon.com/sns/late
        ]
 ```
 
+[SQS](https://docs.aws.amazon.com/lambda/latest/dg/with-sqs.html) is also pulling messages from a stream.  At this time, [only "Standard" queues can trigger lambda events, not "FIFO" queues](https://docs.aws.amazon.com/lambda/latest/dg/with-sqs.html).  Read the AWS Documentation carefully since Lambda calls the SQS DeleteMessage API on your behalf once your function completes successfully.  
+
+```javascript
+       "events": [
+           {
+               "function": "your_module.process_messages",
+               "event_source": {
+                    "arn":  "arn:aws:sqs:us-east-1:12341234:your-queue-name-arn",
+                    "batch_size": 10, // Max: 10. Use 1 to trigger immediate processing
+                    "enabled": true // Default is false
+               }
+           }
+       ]
+```
+
 For configuring Lex Bot's intent triggered events:
 ```javascript
 	"bot_events": [
@@ -805,6 +821,7 @@ to change Zappa's behavior. Use these at your own risk!
         "apigateway_description": "My funky application!", // Define a custom description for the API Gateway console. Default None.
         "assume_policy": "my_assume_policy.json", // optional, IAM assume policy JSON file
         "attach_policy": "my_attach_policy.json", // optional, IAM attach policy JSON file
+        "apigateway_policy": "my_apigateway_policy.json", // optional, API Gateway resource policy JSON file
         "async_source": "sns", // Source of async tasks. Defaults to "lambda"
         "async_resources": true, // Create the SNS topic and DynamoDB table to use. Defaults to true.
         "async_response_table": "your_dynamodb_table_name",  // the DynamoDB table name to use for captured async responses; defaults to None (can't capture)
@@ -906,7 +923,7 @@ to change Zappa's behavior. Use these at your own risk!
             "Key": "Value",  // Example Key and value
             "Key2": "Value2",
             },
-        "timeout_seconds": 30, // Maximum lifespan for the Lambda function (default 30, max 300.)
+        "timeout_seconds": 30, // Maximum lifespan for the Lambda function (default 30, max 900.)
         "touch": true, // GET the production URL upon initial deployment (default True)
         "touch_path": "/", // The endpoint path to GET when checking the initial deployment (default "/")
         "use_precompiled_packages": true, // If possible, use C-extension packages which have been pre-compiled for AWS Lambda. Default true.
@@ -1049,6 +1066,31 @@ You can also use AWS Cognito User Pool Authorizer by adding:
 }
 ```
 
+#### API Gateway Resource Policy
+
+You can also use API Gateway Resource Policies. Example of IP Whitelisting:
+
+```javascript
+{
+    "Version": "2012-10-17",
+    "Statement": [
+        {
+            "Effect": "Allow",
+            "Principal": "*",
+            "Action": "execute-api:Invoke",
+            "Resource": "execute-api:/*",
+            "Condition": {
+                "IpAddress": {
+                    "aws:SourceIp": [
+                        "1.2.3.4/32"
+                    ]
+                }
+            }
+        }
+    ]
+}
+```
+
 ### Setting Environment Variables
 
 #### Local Environment Variables

requirements.txt

@@ -1,5 +1,4 @@
 argcomplete==1.9.3
-base58==1.0.0
 botocore>=1.7.19
 boto3>=1.4.7
 docutils>=0.12
@@ -15,6 +14,8 @@ python-dateutil>=2.6.1, <2.7.0
 python-slugify==1.2.4
 PyYAML==3.13
 requests>=2.10.0
+# requests has issues with urllib3 1.24
+urllib3<=1.23
 six>=1.11.0
 toml>=0.9.4
 tqdm==4.19.1

test_requirements.txt

@@ -7,3 +7,6 @@ mock>=2.0.0
 nose>=1.3.7
 nose-timer==0.6.0
 placebo>=0.8.1
+# requests has issues with urllib3 1.24
+# and if we install 1.24 here python setup.py install keeps both versions
+urllib3<=1.23

test_settings.py

@@ -19,7 +19,8 @@
     'arn:aws:s3:1': 'test_settings.aws_s3_event',
     'arn:aws:sns:1': 'test_settings.aws_sns_event',
     'arn:aws:dynamodb:1': 'test_settings.aws_dynamodb_event',
-    'arn:aws:kinesis:1': 'test_settings.aws_kinesis_event'
+    'arn:aws:kinesis:1': 'test_settings.aws_kinesis_event',
+    'arn:aws:sqs:1': 'test_settings.aws_sqs_event'
 }
 
 ENVIRONMENT_VARIABLES={'testenv': 'envtest'}
@@ -54,9 +55,14 @@ def aws_kinesis_event(event, content):
     return "AWS KINESIS EVENT"
 
 
+def aws_sqs_event(event, content):
+    return "AWS SQS EVENT"
+
+
 def authorizer_event(event, content):
     return "AUTHORIZER_EVENT"
 
 
 def command():
     print("command")
+

tests/tests.py

@@ -1463,6 +1463,7 @@ def test_get_domain_respects_route53_setting(self, client, template):
 
         # And that the route53 path still works
         zappa_core.route53.list_hosted_zones.return_value = {
+            'IsTruncated': False,
             'HostedZones': [
                 {
                     'Id': 'somezone'
@@ -1483,6 +1484,70 @@ def test_get_domain_respects_route53_setting(self, client, template):
         zappa_core.route53.list_resource_record_sets.assert_called_once_with(
             HostedZoneId='somezone')
 
+    @mock.patch('botocore.client')
+    def test_get_all_zones_normal_case(self, client):
+        zappa_core = Zappa(
+            boto_session=mock.Mock(),
+            profile_name="test",
+            aws_region="test",
+            load_credentials=False
+        )
+        zappa_core.route53 = mock.Mock()
+
+        # Check that it handle the normal case
+        zappa_core.route53.list_hosted_zones.return_value = {
+            'IsTruncated': False,
+            'HostedZones': [
+                {
+                    'Id': 'somezone'
+                }
+            ]
+        }
+
+        zones = zappa_core.get_all_zones()
+        zappa_core.route53.list_hosted_zones.assert_called_with(MaxItems='100')
+        self.assertListEqual(zones['HostedZones'], [{'Id': 'somezone'}])
+
+    @mock.patch('botocore.client')
+    def test_get_all_zones_two_pages(self, client):
+        zappa_core = Zappa(
+            boto_session=mock.Mock(),
+            profile_name="test",
+            aws_region="test",
+            load_credentials=False
+        )
+        zappa_core.route53 = mock.Mock()
+
+        # Check that it handle the normal case
+        zappa_core.route53.list_hosted_zones.side_effect = [
+            {
+                'IsTruncated': True,
+                'HostedZones': [
+                    {
+                        'Id': 'zone1'
+                    }
+                ],
+                'NextMarker': "101"
+            },
+            {
+                'IsTruncated': False,
+                'HostedZones': [
+                    {
+                        'Id': 'zone2'
+                    }
+                ]
+            }
+        ]
+
+        zones = zappa_core.get_all_zones()
+        zappa_core.route53.list_hosted_zones.assert_has_calls(
+            [
+                mock.call(MaxItems='100'),
+                mock.call(MaxItems='100', Marker='101'),
+            ]
+        )
+        self.assertListEqual(zones['HostedZones'], [{'Id': 'zone1'}, {'Id': 'zone2'}])
+
     ##
     # Django
     ##

tests/tests_placebo.py

@@ -376,6 +376,29 @@ def test_handler(self, session):
         }
         self.assertEqual("AWS KINESIS EVENT", lh.handler(event, None))
 
+        # Test AWS SQS event
+        event = {
+            u"Records": [
+                {
+                    u"messageId": u"c80e8021-a70a-42c7-a470-796e1186f753",
+                    u"receiptHandle": u"AQEBJQ+/u6NsnT5t8Q/VbVxgdUl4TMKZ5FqhksRdIQvLBhwNvADoBxYSOVeCBXdnS9P+erlTtwEALHsnBXynkfPLH3BOUqmgzP25U8kl8eHzq6RAlzrSOfTO8ox9dcp6GLmW33YjO3zkq5VRYyQlJgLCiAZUpY2D4UQcE5D1Vm8RoKfbE+xtVaOctYeINjaQJ1u3mWx9T7tork3uAlOe1uyFjCWU5aPX/1OHhWCGi2EPPZj6vchNqDOJC/Y2k1gkivqCjz1CZl6FlZ7UVPOx3AMoszPuOYZ+Nuqpx2uCE2MHTtMHD8PVjlsWirt56oUr6JPp9aRGo6bitPIOmi4dX0FmuMKD6u/JnuZCp+AXtJVTmSHS8IXt/twsKU7A+fiMK01NtD5msNgVPoe9JbFtlGwvTQ==",
+                    u"body": u"{\"foo\":\"bar\"}",
+                    u"attributes": {
+                        u"ApproximateReceiveCount": u"3",
+                        u"SentTimestamp": u"1529104986221",
+                        u"SenderId": u"594035263019",
+                        u"ApproximateFirstReceiveTimestamp": u"1529104986230"
+                        },
+                    u"messageAttributes": {},
+                    u"md5OfBody": u"9bb58f26192e4ba00f01e2e7b136bbd8",
+                    u"eventSource": u"aws:sqs",
+                    u"eventSourceARN": u"arn:aws:sqs:1",
+                    u"awsRegion": u"us-east-1"
+                    }
+                ]
+            }
+        self.assertEqual("AWS SQS EVENT", lh.handler(event, None))
+
         # Test Authorizer event
         event = {u'authorizationToken': u'hubtoken1', u'methodArn': u'arn:aws:execute-api:us-west-2:1234:xxxxx/dev/GET/v1/endpoint/param', u'type': u'TOKEN'}
         self.assertEqual("AUTHORIZER_EVENT", lh.handler(event, None))

zappa/__init__.py

@@ -11,4 +11,4 @@
               'Zappa (and AWS Lambda) support the following versions of Python: {}'.format(formatted_supported_versions)
     raise RuntimeError(err_msg)
 
-__version__ = '0.46.2'
+__version__ = '0.47.0'

zappa/cli.py

@@ -52,6 +52,7 @@
 
 
 CUSTOM_SETTINGS = [
+    'apigateway_policy',
     'assume_policy',
     'attach_policy',
     'aws_region',
@@ -661,7 +662,8 @@ def template(self, lambda_arn, role_arn, output=None, json=False):
                                             iam_authorization=self.iam_authorization,
                                             authorizer=self.authorizer,
                                             cors_options=self.cors,
-                                            description=self.apigateway_description
+                                            description=self.apigateway_description,
+                                            policy=self.apigateway_policy
                                         )
 
         if not output:
@@ -1018,6 +1020,9 @@ def update(self, source_zip=None, no_upload=False):
         if endpoint_url and 'https://' not in endpoint_url:
             endpoint_url = 'https://' + endpoint_url
 
+        if self.base_path:
+            endpoint_url += '/' + self.base_path
+
         deployed_string = "Your updated Zappa deployment is " + click.style("live", fg='green', bold=True) + "!"
         if self.use_apigateway:
             deployed_string = deployed_string + ": " + click.style("{}".format(endpoint_url), bold=True)
@@ -1459,8 +1464,11 @@ def tabular_print(title, value):
             # There literally isn't a better way to do this.
             # AWS provides no way to tie a APIGW domain name to its Lambda function.
             domain_url = self.stage_config.get('domain', None)
+            base_path = self.stage_config.get('base_path', None)
             if domain_url:
                 status_dict["Domain URL"] = 'https://' + domain_url
+                if base_path:
+                    status_dict["Domain URL"] += '/' + base_path
             else:
                 status_dict["Domain URL"] = "None Supplied"
 
@@ -2005,6 +2013,7 @@ def load_settings(self, settings_file=None, session=None):
         self.profile_name = self.stage_config.get('profile_name', None)
         self.log_level = self.stage_config.get('log_level', "DEBUG")
         self.domain = self.stage_config.get('domain', None)
+        self.base_path = self.stage_config.get('base_path', None)
         self.timeout_seconds = self.stage_config.get('timeout_seconds', 30)
         dead_letter_arn = self.stage_config.get('dead_letter_arn', '')
         self.dead_letter_config = {'TargetArn': dead_letter_arn} if dead_letter_arn else {}
@@ -2271,6 +2280,11 @@ def create_package(self, output=None):
             else:
                 settings_s = settings_s + "DOMAIN=None\n"
 
+            if self.base_path:
+                settings_s = settings_s + "BASE_PATH='{0!s}'\n".format((self.base_path))
+            else:
+                settings_s = settings_s + "BASE_PATH=None\n"
+
             # Pass through remote config bucket and path
             if self.remote_env:
                 settings_s = settings_s + "REMOTE_ENV='{0!s}'\n".format(