From f8b87eac00463e780ee2ab9683e17a8fd62b7a28 Mon Sep 17 00:00:00 2001 From: Mike Graves Date: Thu, 2 Aug 2018 15:35:20 -0400 Subject: [PATCH 1/2] Add support for running as AWS Lambda This adds a Lambda handler function. There are a few assumptions being made here. This assumes the Lambda will be triggered by a CloudWatch event and that the event will pass a JSON structure that indicates the type of feed to generate. The function is configured through environment variables. Env vars that contain sensitive information should be added to an AWS Secrets object. I'm making some guesses here as to how the event data and secrets get passed to the Lambda. The documentation suggests it's possible to pass JSON in both cases, but it's not entirely clear. Some adjustment may be required after attempting to run this in AWS. --- .gitignore | 1 + carbon/app.py | 30 ++++++++++++++++++++++++++++++ carbon/cli.py | 12 ++++-------- lambda.py | 16 ++++++++++++++++ 4 files changed, 51 insertions(+), 8 deletions(-) create mode 100644 lambda.py diff --git a/.gitignore b/.gitignore index 71f31f2..7dbd5f6 100644 --- a/.gitignore +++ b/.gitignore @@ -6,3 +6,4 @@ build/ dist/ tests/db/ +.pytest_cache diff --git a/carbon/app.py b/carbon/app.py index 4e60bae..100b28a 100644 --- a/carbon/app.py +++ b/carbon/app.py @@ -4,6 +4,7 @@ from datetime import datetime from functools import partial import ftplib +import os import re import threading @@ -293,3 +294,32 @@ def _add_person(xf, person): add_child(record, 'field', person.get('HR_ORG_LEVEL5_NAME'), name='[Generic05]') xf.write(record) + + +class Config(dict): + @classmethod + def from_env(cls): + cfg = cls() + for var in ['FTP_USER', 'FTP_PASS', 'FTP_PATH', 'FTP_HOST', + 'CARBON_DB',]: + cfg[var] = os.environ.get(var) + return cfg + + +class Lambda: + def __init__(self, event, context, config): + self.event = event + self.context = context + self.config = config + + def run(self): + r, w = os.pipe() + feed_type = self.event['feed_type'] + with open(r, 'rb') as fp_r, open(w, 'wb') as fp_w: + ftp_rdr = FTPReader(fp_r, + self.config['FTP_USER'], + self.config['FTP_PASS'], + self.config['FTP_PATH'], + self.config['FTP_HOST'], + self.config['FTP_PORT']) + PipeWriter(out=fp_w).pipe(ftp_rdr).write(feed_type) diff --git a/carbon/cli.py b/carbon/cli.py index 38e067f..09ea4ba 100644 --- a/carbon/cli.py +++ b/carbon/cli.py @@ -1,11 +1,9 @@ # -*- coding: utf-8 -*- from __future__ import absolute_import -import os - import click -from carbon.app import FTPReader, PipeWriter, Writer +from carbon.app import Config, Lambda, Writer from carbon.db import engine @@ -48,10 +46,8 @@ def main(feed_type, db, out, ftp, ftp_host, ftp_port, ftp_user, ftp_pass, """ engine.configure(db) if ftp: - r, w = os.pipe() - with open(r, 'rb') as fp_r, open(w, 'wb') as fp_w: - ftp_rdr = FTPReader(fp_r, ftp_user, ftp_pass, ftp_path, ftp_host, - ftp_port) - PipeWriter(out=fp_w).pipe(ftp_rdr).write(feed_type) + cfg = Config(FTP_USER=ftp_user, FTP_PASS=ftp_pass, FTP_PATH=ftp_path, + FTP_HOST=ftp_host, FTP_PORT=ftp_port) + Lambda({'feed_type': feed_type}, None, cfg).run() else: Writer(out=out).write(feed_type) diff --git a/lambda.py b/lambda.py new file mode 100644 index 0000000..315dac9 --- /dev/null +++ b/lambda.py @@ -0,0 +1,16 @@ +import json +import os + +import boto3 + +from carbon.app import Config, Lambda +from carbon.db import engine + + +def handler(event, context): + client = boto3.client('secretsmanager') + secret_env = client.get_secret_value(SecretId=os.environ['SECRET_ID']) + cfg = Config.from_env() + cfg.update(json.loads(secret_env)) + engine.configure(cfg['CARBON_DB']) + Lambda(event, context, cfg).run() From 3c2dc2df023e5dd95fed86597b78b2324aa87ee8 Mon Sep 17 00:00:00 2001 From: Mike Graves Date: Thu, 2 Aug 2018 16:17:54 -0400 Subject: [PATCH 2/2] Change the name of the Lambda class Use a name that better reflects what the class does. --- carbon/app.py | 2 +- carbon/cli.py | 4 ++-- lambda.py | 4 ++-- 3 files changed, 5 insertions(+), 5 deletions(-) diff --git a/carbon/app.py b/carbon/app.py index 100b28a..fc1b5f1 100644 --- a/carbon/app.py +++ b/carbon/app.py @@ -306,7 +306,7 @@ def from_env(cls): return cfg -class Lambda: +class FTPFeeder: def __init__(self, event, context, config): self.event = event self.context = context diff --git a/carbon/cli.py b/carbon/cli.py index 09ea4ba..6584dcb 100644 --- a/carbon/cli.py +++ b/carbon/cli.py @@ -3,7 +3,7 @@ import click -from carbon.app import Config, Lambda, Writer +from carbon.app import Config, FTPFeeder, Writer from carbon.db import engine @@ -48,6 +48,6 @@ def main(feed_type, db, out, ftp, ftp_host, ftp_port, ftp_user, ftp_pass, if ftp: cfg = Config(FTP_USER=ftp_user, FTP_PASS=ftp_pass, FTP_PATH=ftp_path, FTP_HOST=ftp_host, FTP_PORT=ftp_port) - Lambda({'feed_type': feed_type}, None, cfg).run() + FTPFeeder({'feed_type': feed_type}, None, cfg).run() else: Writer(out=out).write(feed_type) diff --git a/lambda.py b/lambda.py index 315dac9..93e9f28 100644 --- a/lambda.py +++ b/lambda.py @@ -3,7 +3,7 @@ import boto3 -from carbon.app import Config, Lambda +from carbon.app import Config, FTPFeeder from carbon.db import engine @@ -13,4 +13,4 @@ def handler(event, context): cfg = Config.from_env() cfg.update(json.loads(secret_env)) engine.configure(cfg['CARBON_DB']) - Lambda(event, context, cfg).run() + FTPFeeder(event, context, cfg).run()