Merge pull request #16 from kyleknap/s3-delete-event

Add steps for s3 delete portion

code/media-query/05-s3-delete-event/.chalice/config.json

@@ -0,0 +1,10 @@
+{
+  "version": "2.0",
+  "app_name": "media-query",
+  "stages": {
+    "dev": {
+      "api_gateway_stage": "api",
+      "autogen_policy": false
+    }
+  }
+}

code/media-query/05-s3-delete-event/.chalice/policy-dev.json

@@ -0,0 +1,51 @@
+{
+  "Version": "2012-10-17",
+  "Statement": [
+    {
+      "Action": [
+        "logs:CreateLogGroup",
+        "logs:CreateLogStream",
+        "logs:PutLogEvents"
+      ],
+      "Resource": "arn:aws:logs:*:*:*",
+      "Effect": "Allow"
+    },
+    {
+      "Action": [
+        "dynamodb:PutItem",
+        "dynamodb:DeleteItem",
+        "dynamodb:UpdateItem",
+        "dynamodb:GetItem",
+        "dynamodb:Scan",
+        "dynamodb:Query"
+      ],
+      "Resource": [
+        "arn:aws:dynamodb:*:*:table/media-query-*"
+      ],
+      "Effect": "Allow"
+    },
+    {
+      "Action": [
+        "rekognition:DetectLabels",
+        "rekognition:StartLabelDetection",
+        "rekognition:GetLabelDetection"
+      ],
+      "Resource": "*",
+      "Effect": "Allow"
+    },
+    {
+      "Action": [
+         "s3:GetObject"
+      ],
+      "Resource": "arn:aws:s3:::media-query*",
+      "Effect": "Allow"
+    },
+    {
+      "Action": [
+         "iam:PassRole"
+      ],
+      "Resource": "arn:aws:iam::*:role/media-query-*",
+      "Effect": "Allow"
+    }
+  ]
+}

code/media-query/05-s3-delete-event/app.py

@@ -0,0 +1,48 @@
+import os
+
+import boto3
+from chalice import Chalice
+from chalicelib import db
+from chalicelib import rekognition
+
+app = Chalice(app_name='media-query')
+
+_MEDIA_DB = None
+_REKOGNITION_CLIENT = None
+_SUPPORTED_IMAGE_EXTENSIONS = [
+    '.jpg',
+    '.png',
+]
+
+
+def get_media_db():
+    global _MEDIA_DB
+    if _MEDIA_DB is None:
+        _MEDIA_DB = db.DynamoMediaDB(
+            boto3.resource('dynamodb').Table(
+                os.environ['MEDIA_TABLE_NAME']))
+    return _MEDIA_DB
+
+
+def get_rekognition_client():
+    global _REKOGNITION_CLIENT
+    if _REKOGNITION_CLIENT is None:
+        _REKOGNITION_CLIENT = rekognition.RekognitonClient(
+            boto3.client('rekognition'))
+    return _REKOGNITION_CLIENT
+
+
+@app.on_s3_event(bucket=os.environ['MEDIA_BUCKET_NAME'],
+                 events=['s3:ObjectCreated:*'])
+def handle_object_created(event):
+    if _is_image(event.key):
+        _handle_created_image(bucket=event.bucket, key=event.key)
+
+
+def _is_image(key):
+    return any([key.endswith(ext) for ext in _SUPPORTED_IMAGE_EXTENSIONS])
+
+
+def _handle_created_image(bucket, key):
+    labels = get_rekognition_client().get_image_labels(bucket=bucket, key=key)
+    get_media_db().add_media_file(key, media_type=db.IMAGE_TYPE, labels=labels)

code/media-query/05-s3-delete-event/chalicelib/db.py

@@ -0,0 +1,77 @@
+from functools import reduce
+
+from boto3.dynamodb.conditions import Attr
+
+
+IMAGE_TYPE = 'image'
+VIDEO_TYPE = 'video'
+
+
+class MediaDB(object):
+    def list_media_files(self, label=None):
+        pass
+
+    def add_media_file(self, name, media_type, labels=None):
+        pass
+
+    def get_media_file(self, name):
+        pass
+
+    def delete_media_file(self, name):
+        pass
+
+
+class DynamoMediaDB(MediaDB):
+    def __init__(self, table_resource):
+        self._table = table_resource
+
+    def list_media_files(self, startswith=None, media_type=None, label=None):
+        scan_params = {}
+        filter_expression = None
+        if startswith is not None:
+            filter_expression = self._add_to_filter_expression(
+                filter_expression, Attr('name').begins_with(startswith)
+            )
+        if media_type is not None:
+            filter_expression = self._add_to_filter_expression(
+                filter_expression, Attr('type').eq(media_type)
+            )
+        if label is not None:
+            filter_expression = self._add_to_filter_expression(
+                filter_expression, Attr('labels').contains(label)
+            )
+        if filter_expression:
+            scan_params['FilterExpression'] = filter_expression
+        response = self._table.scan(**scan_params)
+        return response['Items']
+
+    def add_media_file(self, name, media_type, labels=None):
+        if labels is None:
+            labels = []
+        self._table.put_item(
+            Item={
+                'name': name,
+                'type': media_type,
+                'labels': labels,
+            }
+        )
+
+    def get_media_file(self, name):
+        response = self._table.get_item(
+            Key={
+                'name': name,
+            },
+        )
+        return response.get('Item')
+
+    def delete_media_file(self, name):
+        self._table.delete_item(
+            Key={
+                'name': name,
+            }
+        )
+
+    def _add_to_filter_expression(self, expression, condition):
+        if expression is None:
+            return condition
+        return expression & condition

code/media-query/05-s3-delete-event/chalicelib/rekognition.py

@@ -0,0 +1,53 @@
+import uuid
+
+
+class RekognitonClient(object):
+    def __init__(self, boto3_client):
+        self._boto3_client = boto3_client
+
+    def get_image_labels(self, bucket, key):
+        response = self._boto3_client.detect_labels(
+            Image={
+                'S3Object': {
+                    'Bucket': bucket,
+                    'Name': key
+                },
+            },
+            MinConfidence=50.0
+        )
+        return [label['Name'] for label in response['Labels']]
+
+    def start_video_label_job(self, bucket, key, topic_arn, role_arn):
+        response = self._boto3_client.start_label_detection(
+            Video={
+                'S3Object': {
+                    'Bucket': bucket,
+                    'Name': key
+                }
+            },
+            ClientRequestToken=str(uuid.uuid4()),
+            NotificationChannel={
+                'SNSTopicArn': topic_arn,
+                'RoleArn': role_arn
+            },
+            MinConfidence=50.0
+        )
+        return response['JobId']
+
+    def get_video_job_labels(self, job_id):
+        labels = set()
+        client_kwargs = {
+            'JobId': job_id,
+        }
+        response = self._boto3_client.get_label_detection(**client_kwargs)
+        self._collect_video_labels(labels, response)
+        while 'NextToken' in response:
+            client_kwargs['NextToken'] = response['NextToken']
+            response = self._boto3_client.get_label_detection(**client_kwargs)
+            self._collect_video_labels(labels, response)
+        return list(labels)
+
+    def _collect_video_labels(self, labels, response):
+        for label in response['Labels']:
+            label_name = label['Label']['Name']
+            labels.add(label_name)

code/media-query/05-s3-delete-event/recordresources.py

@@ -0,0 +1,41 @@
+import argparse
+import json
+import os
+
+import boto3
+from botocore import xform_name
+
+
+def record_as_env_var(stack_name, stage):
+    cloudformation = boto3.client('cloudformation')
+    response = cloudformation.describe_stacks(
+        StackName=stack_name
+    )
+    outputs = response['Stacks'][0]['Outputs']
+    with open(os.path.join('.chalice', 'config.json')) as f:
+        data = json.load(f)
+        data['stages'].setdefault(stage, {}).setdefault(
+            'environment_variables', {}
+        )
+        for output in outputs:
+            data['stages'][stage]['environment_variables'][
+                _to_env_var_name(output['OutputKey'])] = output['OutputValue']
+    with open(os.path.join('.chalice', 'config.json'), 'w') as f:
+        serialized = json.dumps(data, indent=2, separators=(',', ': '))
+        f.write(serialized + '\n')
+
+
+def _to_env_var_name(name):
+    return xform_name(name).upper()
+
+
+def main():
+    parser = argparse.ArgumentParser()
+    parser.add_argument('-s', '--stage', default='dev')
+    parser.add_argument('--stack-name', required=True)
+    args = parser.parse_args()
+    record_as_env_var(stack_name=args.stack_name, stage=args.stage)
+
+
+if __name__ == '__main__':
+    main()

code/media-query/05-s3-delete-event/requirements.txt

@@ -0,0 +1 @@
+boto3<1.8.0

code/media-query/05-s3-delete-event/resources.json

@@ -0,0 +1,40 @@
+{
+  "Outputs": {
+    "MediaBucketName": {
+      "Value": {
+        "Ref": "MediaBucket"
+      }
+    },
+    "MediaTableName": {
+      "Value": {
+        "Ref": "MediaTable"
+      }
+    }
+  },
+  "Resources": {
+    "MediaBucket": {
+      "Type": "AWS::S3::Bucket"
+    },
+    "MediaTable": {
+      "Properties": {
+        "AttributeDefinitions": [
+          {
+            "AttributeName": "name",
+            "AttributeType": "S"
+          }
+        ],
+        "KeySchema": [
+          {
+            "AttributeName": "name",
+            "KeyType": "HASH"
+          }
+        ],
+        "ProvisionedThroughput": {
+          "ReadCapacityUnits": 5,
+          "WriteCapacityUnits": 5
+        }
+      },
+      "Type": "AWS::DynamoDB::Table"
+    }
+  }
+}

code/media-query/06-web-api/app.py

@@ -0,0 +1,56 @@
+import os
+
+import boto3
+from chalice import Chalice
+from chalicelib import db
+from chalicelib import rekognition
+
+
+app = Chalice(app_name='media-query')
+
+_MEDIA_DB = None
+_REKOGNITION_CLIENT = None
+_SUPPORTED_IMAGE_EXTENSIONS = [
+    '.jpg',
+    '.png',
+]
+
+
+def get_media_db():
+    global _MEDIA_DB
+    if _MEDIA_DB is None:
+        _MEDIA_DB = db.DynamoMediaDB(
+            boto3.resource('dynamodb').Table(
+                os.environ['MEDIA_TABLE_NAME']))
+    return _MEDIA_DB
+
+
+def get_rekognition_client():
+    global _REKOGNITION_CLIENT
+    if _REKOGNITION_CLIENT is None:
+        _REKOGNITION_CLIENT = rekognition.RekognitonClient(
+            boto3.client('rekognition'))
+    return _REKOGNITION_CLIENT
+
+
+@app.on_s3_event(bucket=os.environ['MEDIA_BUCKET_NAME'],
+                 events=['s3:ObjectCreated:*'])
+def handle_object_created(event):
+    if _is_image(event.key):
+        _handle_created_image(bucket=event.bucket, key=event.key)
+
+
+@app.on_s3_event(bucket=os.environ['MEDIA_BUCKET_NAME'],
+                 events=['s3:ObjectRemoved:*'])
+def handle_object_removed(event):
+    if _is_image(event.key):
+        get_media_db().delete_media_file(event.key)
+
+
+def _is_image(key):
+    return any([key.endswith(ext) for ext in _SUPPORTED_IMAGE_EXTENSIONS])
+
+
+def _handle_created_image(bucket, key):
+    labels = get_rekognition_client().get_image_labels(bucket=bucket, key=key)
+    get_media_db().add_media_file(key, media_type=db.IMAGE_TYPE, labels=labels)