mailer smtp integration, priority headers, cleanup / refactoring, fixed broken logging

.gitignore

@@ -1,5 +1,6 @@
 *.pyc
 *.egg-info/
+*.swp
 deps/
 include/
 lib/

.travis.yml

@@ -11,6 +11,8 @@ install:
   - pip install --upgrade pytest pytest-xdist pytest-cov
   - pip install --upgrade coveralls
   - python setup.py develop
-script: make sphinx && py.test -v -n auto --cov=c7n tests tools/c7n_*
+script:
+  - flake8 tools/c7n_mailer
+  - make sphinx && py.test -v -n auto --cov=c7n tests tools/c7n_*
 after_success:
   coveralls

tools/c7n_mailer/README.md

@@ -51,6 +51,7 @@ policies:
     actions:
       - type: notify
         template: default
+        priority_header: 2
         subject: testing the c7n mailer
         to:
           - you@example.com
@@ -107,6 +108,11 @@ schema](./c7n_mailer/cli.py#L11-L41) to which the file must conform, here is
 | ✅  | `queue_url`          | string           | the queue to listen to for messages |
 | ✅  | `from_address`       | string           | default from address                |
 |           | `contact_tags`       | array of strings | tags that we should look at for address information |
+|           | `smtp_server`        | string           | if this is unset, aws ses is used by default. To configure your lambda role to talk to smtpd in your private vpc, see [here](https://docs.aws.amazon.com/lambda/latest/dg/vpc.html) |
+|           | `smtp_port`          | integer          | smtp port                           |
+|           | `smtp_ssl`           | boolean          | this defaults to True               |
+|           | `smtp_username`      | string           |                                     |
+|           | `smtp_password`      | string           |                                     |
 
 
 #### Standard Lambda Function Config
@@ -156,6 +162,7 @@ policies:
     actions:
       - type: notify
         template: default
+        priority_header: 1
         subject: fix your tags
         to:
           - resource-owner
@@ -176,6 +183,7 @@ are either
   `OwnerContact` tag on the resource that matched the policy, or
 - `event-owner` for push-based/realtime policies that will send to the user
   that was responsible for the underlying event.
+- `priority_header` indicate the importannce of an email with [headers](https://www.chilkatsoft.com/p/p_471.asp). Different emails clients will display stars, exclamation points or flags depending on the value. Should be an integer from 1 to 5.
 
 Both of these special values are best effort, i.e., if no `OwnerContact` tag is
 specified then `resource-owner` email will not be delivered, and in the case of
@@ -191,6 +199,7 @@ For reference purposes, the JSON Schema of the `notify` action:
     "type": {"enum": ["notify"]},
     "to": {"type": "array", "items": {"type": "string"}},
     "subject": {"type": "string"},
+    "priority_header": {"type": "integer"},
     "template": {"type": "string"},
     "transport": {
       "type": "object",

tools/c7n_mailer/c7n_mailer/cli.py

@@ -27,6 +27,11 @@
 
         # Mailer Infrastructure Config
         'cache': {'type': 'string'},
+        'smtp_server': {'type': 'string'},
+        'smtp_port': {'type': 'integer'},
+        'smtp_ssl': {'type': 'boolean'},
+        'smtp_username': {'type': 'string'},
+        'smtp_password': {'type': 'string'},
         'ldap_uri': {'type': 'string'},
         'ldap_bind_dn': {'type': 'string'},
         'ldap_bind_user': {'type': 'string'},
@@ -59,7 +64,8 @@ def main():
     options = parser.parse_args()
 
     import logging
-    logging.basicConfig(level=logging.DEBUG)
+    log_format = '%(asctime)s - %(name)s - %(levelname)s - %(message)s'
+    logging.basicConfig(level=logging.DEBUG, format=log_format)
     logging.getLogger('botocore').setLevel(logging.WARNING)
 
     with open(options.config) as fh:

tools/c7n_mailer/c7n_mailer/deploy.py

@@ -23,12 +23,16 @@
 
 entry_source = """\
 import logging
-logging.root.setLevel(logging.DEBUG)
 
 from c7n_mailer import handle
 
+logger = logging.getLogger('custodian.mailer')
+log_format = '%(asctime)s - %(name)s - %(levelname)s - %(message)s'
+logging.basicConfig(level=logging.DEBUG, format=log_format)
+logging.getLogger('botocore').setLevel(logging.WARNING)
+
 def dispatch(event, context):
-    return handle.run(event, context)
+    return handle.start_c7n_mailer(event, context, logger)
 """
 
 
@@ -70,7 +74,6 @@ def provision(config, session_factory):
                 prefix="")
         ])
 
-
     archive = get_archive(config)
     func = LambdaFunction(func_config, archive)
     manager = LambdaManager(session_factory)

tools/c7n_mailer/c7n_mailer/handle.py

@@ -17,20 +17,13 @@
 
 import base64
 import boto3
-import getpass
 import json
-import logging
 import os
 
+from .sqs_queue_processor import MailerSqsQueueProcessor
 
-logging.root.setLevel(logging.DEBUG)
-logging.getLogger('botocore').setLevel(logging.WARNING)
 
-log = logging.getLogger('custodian.mailer')
-
-
-def bootstrap():
-    log.debug("Initializing")
+def config_setup(session):
     task_dir = os.environ.get('LAMBDA_TASK_ROOT')
     os.environ['PYTHONPATH'] = "%s:%s" % (task_dir, os.environ.get('PYTHONPATH', ''))
     with open(os.path.join(task_dir, 'config.json')) as fh:
@@ -46,24 +39,12 @@ def bootstrap():
         os.environ['https_proxy'] = config['https_proxy']
     return config
 
-session = boto3.Session()
-config = bootstrap()
-
 
-def run(event, context):
+def start_c7n_mailer(event, context, logger):
     try:
-        from markupsafe import Markup
-        from jinja2 import utils
-        from .worker import Worker
-        from .processor import Processor
+        session = boto3.Session()
+        config = config_setup(session)
+        mailer_sqs_queue_processor = MailerSqsQueueProcessor(config, session, logger)
+        mailer_sqs_queue_processor.run()
     except Exception as e:
-        log.exception("import failed %s", e)
-
-    try:
-        log.info("Worker Run")
-        w = Worker(config, context, session)
-        w.run()
-    except:
-        log.exception("Error processing worker \n DebugEnv: %s \n User: %s \n" % (
-            os.environ['PYTHONPATH'],
-            getpass.getuser()))
+        logger.exception("Error starting mailer MailerSqsQueueProcessor(). \n Error: %s \n" % (e))

tools/c7n_mailer/c7n_mailer/record.py

@@ -58,7 +58,7 @@ def resource_format(resource, resource_type):
     elif resource_type == 's3':
         return "%s" % (resource['Name'])
     elif resource_type == 'ebs':
-        return "%s %s %s %s" %(
+        return "%s %s %s %s" % (
             resource['VolumeId'],
             resource['Size'],
             resource['State'],
@@ -138,7 +138,7 @@ def resource_format(resource, resource_type):
             resource['SnapshotId'],
             resource['StartTime'])
     elif resource_type == 'subnet':
-        return "%s %s %s %s %s %s" %(
+        return "%s %s %s %s %s %s" % (
             resource['SubnetId'],
             resource['VpcId'],
             resource['AvailabilityZone'],