Merge 8ffaeaf into 627898c

Makefile

@@ -21,9 +21,11 @@ wheel:
 	pipenv run python setup.py bdist_wheel
 
 container:
-	docker build -t $(ECR_REGISTRY)/carbon:latest \
-		-t $(ECR_REGISTRY)/carbon:`git describe --always` \
-		-t carbon:latest .
+	docker build -t $(ECR_REGISTRY)/carbon-stage:latest \
+		-t $(ECR_REGISTRY)/carbon-stage:`git describe --always` \
+		-t $(ECR_REGISTRY)/carbon-prod:latest \
+		-t $(ECR_REGISTRY)/carbon-prod:`git describe --always` \
+		-t carbon-stage:latest .
 
 dist: deps wheel container ## Build docker image
 	@tput setaf 2
@@ -51,5 +53,7 @@ update: ## Update all python dependencies
 
 publish: ## Push and tag the latest image (use `make dist && make publish`)
 	$$(aws ecr get-login --no-include-email --region us-east-1)
-	docker push $(ECR_REGISTRY)/carbon:latest
-	docker push $(ECR_REGISTRY)/carbon:`git describe --always`
+	docker push $(ECR_REGISTRY)/carbon-stage:latest
+	docker push $(ECR_REGISTRY)/carbon-stage:`git describe --always`
+	docker push $(ECR_REGISTRY)/carbon-prod:latest
+	docker push $(ECR_REGISTRY)/carbon-prod:`git describe --always`

carbon/app.py

@@ -2,12 +2,14 @@
 from __future__ import absolute_import
 from contextlib import contextmanager, closing
 from datetime import datetime
-from functools import partial
+from functools import partial, update_wrapper
 from ftplib import FTP, FTP_TLS
 import os
 import re
 import threading
 
+import boto3
+import click
 from lxml import etree as ET
 from sqlalchemy import func, select
 
@@ -368,3 +370,46 @@ def run(self):
                                 int(self.config['FTP_PORT']),
                                 self.ssl_ctx)
             PipeWriter(out=fp_w).pipe(ftp_rdr).write(feed_type)
+
+
+def sns_log(f):
+    """AWS SNS log decorator for wrapping a click command.
+
+    This can be used as a decorator for a click command. It will wrap
+    execution of the click command in a try/except so that any exception
+    can be logged to the SNS topic before being re-raised.
+    """
+    msg_start = ("[{}] Starting carbon run for the {} feed in the {} "
+                 "environment.")
+    msg_success = ("[{}] Finished carbon run for the {} feed in the {} "
+                   "environment.")
+    msg_fail = ("[{}] The following problem was encountered during the "
+                "carbon run for the {} feed in the {} environment:\n\n"
+                "{}")
+
+    @click.pass_context
+    def wrapped(ctx, *args, **kwargs):
+        sns_id = ctx.params.get('sns_topic')
+        if sns_id:
+            client = boto3.client('sns')
+            stage = ctx.params.get('ftp_path', '').lstrip('/').split('/')[0]
+            feed = ctx.params.get('feed_type', '')
+            client.publish(TopicArn=sns_id, Subject="Carbon run",
+                           Message=msg_start.format(
+                               datetime.utcnow().isoformat(), feed, stage))
+            try:
+                res = ctx.invoke(f, *args, **kwargs)
+            except Exception as e:
+                client.publish(TopicArn=sns_id, Subject="Carbon run",
+                               Message=msg_fail.format(
+                                   datetime.utcnow().isoformat(), feed, stage,
+                                   e))
+                raise
+            else:
+                client.publish(TopicArn=sns_id, Subject="Carbon run",
+                               Message=msg_success.format(
+                                   datetime.utcnow().isoformat(), feed, stage))
+        else:
+            res = ctx.invoke(f, *args, **kwargs)
+        return res
+    return update_wrapper(wrapped, f)

carbon/cli.py

@@ -6,16 +6,15 @@
 import boto3
 import click
 
-from carbon.app import Config, FTPFeeder, Writer
+from carbon.app import Config, FTPFeeder, Writer, sns_log
 from carbon.db import engine
 
 
 @click.command()
 @click.version_option()
 @click.argument('feed_type', type=click.Choice(['people', 'articles']))
 @click.option('--db', envvar='CARBON_DB', help='Database connection string')
-@click.option('-o', '--out', help='Output file', type=click.File('wb'),
-              default='-')
+@click.option('-o', '--out', help='Output file', type=click.File('wb'))
 @click.option('--ftp', is_flag=True, help='Send output to FTP server; do not '
                                           'use this with the -o/--out option')
 @click.option('--ftp-host', envvar='FTP_HOST', help='Hostname of FTP server',
@@ -29,8 +28,13 @@
 @click.option('--secret-id', help='AWS Secrets id containing DB connection '
                                   'string and FTP password. If given, will '
                                   'override other command line options.')
+@click.option('--sns-topic', help='AWS SNS Topic ARN. If given, a message '
+                                  'will be sent when the load begins and '
+                                  'then another message will be sent with '
+                                  'the outcome of the load.')
+@sns_log
 def main(feed_type, db, out, ftp, ftp_host, ftp_port, ftp_user, ftp_pass,
-         ftp_path, secret_id):
+         ftp_path, secret_id, sns_topic):
     """Generate feeds for Symplectic Elements.
 
     Specify which FEED_TYPE should be generated. This should be either
@@ -60,6 +64,8 @@ def main(feed_type, db, out, ftp, ftp_host, ftp_port, ftp_user, ftp_pass,
 
     engine.configure(cfg['CARBON_DB'])
     if ftp:
+        click.echo("Starting carbon run for {}".format(feed_type))
         FTPFeeder({'feed_type': feed_type}, None, cfg).run()
+        click.echo("Finished carbon run for {}".format(feed_type))
     else:
         Writer(out=out).write(feed_type)

tests/test_cli.py

@@ -19,14 +19,14 @@ def runner():
 
 
 def test_people_returns_people(runner, xml_data):
-    res = runner.invoke(main, ['--db', 'sqlite://', 'people'])
+    res = runner.invoke(main, ['--db', 'sqlite://', '-o', '-', 'people'])
     assert res.exit_code == 0
     assert res.stdout_bytes == \
         ET.tostring(xml_data, encoding="UTF-8", xml_declaration=True)
 
 
 def test_articles_returns_articles(runner, articles_data):
-    res = runner.invoke(main, ['--db', 'sqlite://', 'articles'])
+    res = runner.invoke(main, ['--db', 'sqlite://', '-o', '-', 'articles'])
     assert res.exit_code == 0
     assert res.stdout_bytes == \
         ET.tostring(articles_data, encoding='UTF-8', xml_declaration=True)