Added Lambda subparser to handle creating archive and upload to S3

Create an archive (zip) and upload to S3 for use in a CFN stack.
Outputs the S3 url for use in your stack.

Makefile

@@ -21,12 +21,12 @@ test:
 	@echo "=== Testing ==="
 	python -m unittest discover -v test "*.py"
 
-deploy_test: build
+deploy_test: lint test build
 	@echo "=== Deploy test.pypi ==="
 	@twine upload dist/* -r testpypi
 	# pip install --user -i https://test.pypi.org/simple/ cfnctl==0.3.3
 
-deploy: build
+deploy: lint test build
 	@echo "=== Deploy pypi ==="
 	@twine upload dist/*
 

README.md

@@ -22,18 +22,56 @@ pip install cfnctl
 # Usage
 
 ```
-usage: cfnctl [-h] [-p AWS_PROFILE] [-r REGION] {deploy} ...
+usage: cfnctl [-h] [-p AWS_PROFILE] [-r REGION] {deploy,lambda} ...
 
 Launch and manage CloudFormation stacks
 
+positional arguments:
+  {deploy,lambda}
+    deploy         creates a changeset and executes to create or update stack
+    lambda         creates an archive and loads it to S3 to create a lambda
+                   from
+
+optional arguments:
+  -h, --help       show this help message and exit
+  -p AWS_PROFILE   AWS Profile
+  -r REGION        Region name
+```
+
+### Deploy
+
+```
+usage: cfnctl deploy [-h] -s STACK_NAME -t TEMPLATE [-b BUCKET] [-nr]
+                     [-p PARAMETERS]
+
+optional arguments:
+  -h, --help     show this help message and exit
+
+required arguments:
+  -s STACK_NAME  Stack name
+  -t TEMPLATE    CFN Template from local file or URL
+
 optional arguments:
-  -h, --help      show this help message and exit
-  -p AWS_PROFILE  AWS Profile
-  -r REGION       Region name
+  -b BUCKET      Bucket to upload template to
+  -nr            Do not rollback
+  -p PARAMETERS  Local parameters JSON file
+```
+
+### Lambda
 
-subcommands:
-  command to run
+Package a folder into a zip archive and upload to S3. Creates the bucket
+if it does not exist. Outputs the S3 url for use in a stack.
+
+```
+usage: cfnctl lambda [-h] -s SOURCE [-o OUTPUT] [-b BUCKET]
 
-  {deploy}
-    deploy        creates a changeset and executes to create or update stack
+optional arguments:
+  -h, --help  show this help message and exit
+
+required arguments:
+  -s SOURCE   Source folder to zip and upload
+
+optional arguments:
+  -o OUTPUT   Destination of the archive file
+  -b BUCKET   Bucket to upload archive to
 ```

cfnctl/cfnctl.py

@@ -35,10 +35,10 @@ def arg_deploy(parser, action):
     '''
     Deploy subcommand and arguments
     '''
-    command = parser.add_subparsers(description='command to run',
-                                    dest='parser_name')
+    # command = parser.add_subparsers(description='command to run',
+    #                                 dest='deploy')
 
-    command_deploy = command.add_parser(
+    command_deploy = parser.add_parser(
         'deploy', help='creates a changeset and executes to create or update stack')
     required_group = command_deploy.add_argument_group('required arguments')
     required_group.add_argument(
@@ -55,14 +55,33 @@ def arg_deploy(parser, action):
     command_deploy.set_defaults(func=action)
     return parser
 
+def arg_lambda(parser, action):
+    '''
+    Lambda subcommand and arguments
+    '''
+    # command = parser.add_subparsers(description='command to run',
+    #                                 dest='lambda')
+
+    command_lambda = parser.add_parser(
+        'lambda', help='creates an archive and loads it to S3 to create a lambda from')
+    required_group = command_lambda.add_argument_group('required arguments')
+    required_group.add_argument(
+        '-s', dest='source', required=True, help='Source folder to zip and upload')
+    optional_group = command_lambda.add_argument_group('optional arguments')
+    optional_group.add_argument(
+        '-o', dest='output', required=False, help='Destination of the archive file')
+    optional_group.add_argument(
+        '-b', dest='bucket', required=False, help='Bucket to upload archive to')
+
+    command_lambda.set_defaults(func=action)
+    return parser
 
 def arg_parser():
     '''
     Create an argparse object with global arguments and return
     '''
     parser = argparse.ArgumentParser(prog='cfnctl',
                                      description='Launch and manage CloudFormation stacks')
-
     parser.add_argument('-p', dest='aws_profile',
                         required=False, help='AWS Profile')
     parser.add_argument('-r', dest='region', required=False, help="Region name")
@@ -79,7 +98,9 @@ def main():
     CFNCTL entrypoint
     '''
     parser = arg_parser()
-    parser = arg_deploy(parser, commands.deploy)
+    subparsers = parser.add_subparsers()
+    arg_deploy(subparsers, commands.deploy)
+    arg_lambda(subparsers, commands.lambda_command)
     args = parser.parse_args()
     args.func(args)
 

cfnctl/commands/__init__.py

@@ -2,3 +2,4 @@
 CFNCTL subcommand logic
 '''
 from cfnctl.commands.deploy import deploy
+from cfnctl.commands.lambda_command import lambda_command

cfnctl/commands/deploy.py

@@ -10,69 +10,11 @@
 import logging
 import json
 import os
-import re
 import sys
 import boto3
 import botocore.exceptions
 from jinja2 import Environment, FileSystemLoader
-
-def _upload_template(simple_storage_service, stack, bucket, template_name):
-    '''Upload the cfn template
-    to S3
-    '''
-    logging.info('Uploading template file')
-    if _is_url(template_name):
-        return None
-    file_path = os.path.abspath(template_name)
-    s3_path = _s3_path(stack, template_name)
-    return simple_storage_service.upload_file(file_path, bucket, s3_path)
-
-
-def _bucket_exists(simple_storage_service, name):
-    '''Check if a bucket exists
-    by name
-
-    return bool
-    '''
-    logging.info('Verifying S3 bucket exists')
-    buckets = simple_storage_service.list_buckets()
-    exists = False
-    for bucket in buckets['Buckets']:
-        if bucket['Name'] == name:
-            exists = True
-            break
-    return exists
-
-
-def _maybe_make_bucket(simple_storage_service, region, account_id):
-    '''Make a bucket to upload
-    the cfn template to if
-    it does not exist
-
-    return string - bucket name
-    '''
-    logging.info('Maybe make S3 bucket')
-    stack_bucket = 'cfnctl-staging-bucket-{}-{}'.format(
-        region, account_id)
-    if _bucket_exists(simple_storage_service, stack_bucket):
-        logging.info('Bucket exists')
-        return stack_bucket
-
-    logging.info('No S3 bucket found, creating %s', stack_bucket)
-    simple_storage_service.create_bucket(
-        Bucket=stack_bucket,
-        CreateBucketConfiguration={
-            'LocationConstraint': region
-        })
-    simple_storage_service.put_bucket_versioning(
-        Bucket=stack_bucket,
-        VersioningConfiguration={
-            'MFADelete': 'Enabled',
-            'Status': 'Enabled'
-        },
-    )
-    return stack_bucket
-
+import cfnctl.lib as lib
 
 def _stack_exists(client, name):
     '''Check if a cfn stack exists
@@ -249,22 +191,6 @@ def _get_parameters(parameter_file):
     )
     return json.loads(rendered)
 
-def _is_url(search):
-    return re.match('https?://', search) is not None
-
-def _s3_path(stack, template):
-    return '{}/{}'.format(stack, os.path.basename(template))
-
-def _get_template_url(bucket, stack, template_name):
-    '''Get the cfn template url
-    with the bucket name
-
-    return string - template URL
-    '''
-    if _is_url(template_name):
-        return template_name
-    return 'https://s3.amazonaws.com/{}/{}'.format(bucket, _s3_path(stack, template_name))
-
 
 def deploy(args):
     '''Deploy a cloudformation stack
@@ -275,12 +201,12 @@ def deploy(args):
     simple_storage_service = boto3.client('s3')
     account_id = boto3.client('sts').get_caller_identity().get('Account')
     region = args.region or boto3.session.Session().region_name
-    bucket = args.bucket or _maybe_make_bucket(simple_storage_service, region, account_id)
-    _upload_template(simple_storage_service, stack, bucket, args.template)
+    bucket = args.bucket or lib.bucket.maybe_make_bucket(simple_storage_service, region, account_id)
+    lib.bucket.upload_file(simple_storage_service, stack, bucket, args.template)
     changeset = _make_change_set(
         client,
         stack,
-        _get_template_url(bucket, stack, args.template),
+        bucket.get_file_url(bucket, stack, args.template),
         _get_parameters(args.parameters)
     )
     ready = _wait_for_changeset(client, changeset, stack)

cfnctl/commands/lambda_command.py

@@ -0,0 +1,57 @@
+'''
+Deploy subcommand logic
+Handles creating an S3 bucket (if required) and uploading
+template to the bucket
+Creates and executes a changeset to either create a new
+stack or update an existing stack
+'''
+import logging
+import os
+import zipfile
+import boto3
+import cfnctl.lib.bucket as bucket
+
+def write_zip(path, ziph):
+    '''
+    Write files to a zip file
+    path {string} absolute path to directory to zip
+    ziph {ZipFile} ZipFile handle
+    '''
+    basedir = os.path.dirname(path)
+    for root, _, files in os.walk(path):
+        for filename in files:
+            abspath = os.path.join(root, filename)
+            ziph.write(abspath, abspath.replace(basedir, ''))
+
+def zip_dir(path, name):
+    '''
+    Zip a directory
+    path {string} absolute path to directory to zip
+    name {string} absolute path to archive directory location/name
+    '''
+    if not name.endswith('.zip'):
+        name = ''.join([name, '.zip'])
+    logging.info('writing contents of %s to archive %s', path, name)
+    zipf = zipfile.ZipFile(name, 'w', zipfile.ZIP_DEFLATED)
+    write_zip(path, zipf)
+    zipf.close()
+
+def lambda_command(args):
+    '''Deploy a lambda function
+    '''
+    logging.info('Calling lambda_command')
+    simple_storage_service = boto3.client('s3')
+    account_id = boto3.client('sts').get_caller_identity().get('Account')
+    region = args.region or boto3.session.Session().region_name
+    bucket_name = args.bucket or bucket.maybe_make_bucket(
+        simple_storage_service,
+        region,
+        account_id
+    )
+    outfile = os.path.abspath(args.output or ''.join([args.source, '.zip']))
+    source = os.path.abspath(args.source)
+    zip_dir(source, outfile)
+    bucket.upload_file(simple_storage_service, 'lambda', bucket_name, outfile)
+    logging.info('Finished uploading archive')
+    file_url = bucket.get_file_url(bucket_name, 'lambda', os.path.basename(outfile))
+    logging.info(file_url)

setup.py

@@ -29,7 +29,7 @@
 def open_file(fname):
     return open(os.path.join(os.path.dirname(__file__), fname))
 
-_version = "0.3.6"
+_version = "0.4.1"
 
 console_scripts = [ 'cfnctl = cfnctl.cfnctl:main',
                    ]
@@ -53,6 +53,7 @@ def open_file(fname):
     include_package_data=True,
     install_requires=[
         'boto3>=1.9.59',
+        'jinja2>=2.10'
     ],
     packages=find_packages(),
     keywords='aws cfn control cfnctl cloudformation stack stackset',

test/commands/deploy.py

@@ -1,60 +1,11 @@
 import os
 import unittest
 import datetime
-import boto3
-import botocore
 from test.mocks.s3 import S3
 from test.mocks.cloudformation import Cloudformation
-from botocore.stub import Stubber
-from cfnctl.commands.deploy import _get_template_url, _upload_template, _wait_for_stack
+from cfnctl.commands.deploy import _wait_for_stack
 
-class TestDeploy(unittest.TestCase):
-
-    def test_template_url(self):
-        local_url = _get_template_url('foo', 'baz', 'bar.template')
-        file_url = _get_template_url('foo', 'baz', 'file:///test/bar.template')
-        http_url = _get_template_url('foo', 'baz', 'https://www.templates.com/bar.template')
-        self.assertEqual(local_url, 'https://s3.amazonaws.com/foo/baz/bar.template')
-        self.assertEqual(file_url, 'https://s3.amazonaws.com/foo/baz/bar.template')
-        self.assertEqual(http_url, 'https://www.templates.com/bar.template')
-
-    def test_upload_template(self):
-        '''upload a template with just a name
-        '''
-        def upload(file_path, bucket, template_name):
-            self.assertEqual(template_name, 'baz/bar.template')
-            self.assertEqual(bucket, 'foo')
-            self.assertEqual(file_path, os.path.abspath('bar.template'))
-            return
-        client = S3()
-        client.mock('upload_file', upload)
-        _upload_template(client, 'baz', 'foo', 'bar.template')
-        self.assertEqual(client.called['upload_file'], 1)
-        self.assertEqual(True, True)
-
-    def test_upload_template_url(self):
-        '''should not attempt to upload a template that is already a url
-        '''
-        def upload(file_path, bucket, template_name):
-            return
-        client = S3()
-        client.mock('upload_file', upload)
-        _upload_template(client, 'baz', 'foo', 'http://templates.com/bar.template')
-        self.assertEqual(client.called['upload_file'], 0)
-
-    def test_upload_template_file_path(self):
-        '''upload a template with just a name
-        '''
-        def upload(file_path, bucket, template_name):
-            self.assertEqual(template_name, 'baz/bar.template')
-            self.assertEqual(bucket, 'foo')
-            self.assertEqual(file_path, os.path.abspath('file:///foo/bar.template'))
-            return
-        client = S3()
-        client.mock('upload_file', upload)
-        _upload_template(client, 'baz', 'foo', 'file:///foo/bar.template')
-        self.assertEqual(client.called['upload_file'], 1)
-        self.assertEqual(True, True)
+class TestCommandDeploy(unittest.TestCase):
 
     def test_wait_for_stack(self):
         def describe_stack_events(StackName, NextToken):