-
Notifications
You must be signed in to change notification settings - Fork 42
Commit
This commit does not belong to any branch on this repository, and may belong to a fork outside of the repository.
Merge pull request #18 from kyleknap/videos
Flesh out video process part of tutorial
- Loading branch information
Showing
10 changed files
with
752 additions
and
18 deletions.
There are no files selected for viewing
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
Original file line number | Diff line number | Diff line change |
---|---|---|
@@ -0,0 +1,10 @@ | ||
{ | ||
"version": "2.0", | ||
"app_name": "media-query", | ||
"stages": { | ||
"dev": { | ||
"api_gateway_stage": "api", | ||
"autogen_policy": false | ||
} | ||
} | ||
} |
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
Original file line number | Diff line number | Diff line change |
---|---|---|
@@ -0,0 +1,51 @@ | ||
{ | ||
"Version": "2012-10-17", | ||
"Statement": [ | ||
{ | ||
"Action": [ | ||
"logs:CreateLogGroup", | ||
"logs:CreateLogStream", | ||
"logs:PutLogEvents" | ||
], | ||
"Resource": "arn:aws:logs:*:*:*", | ||
"Effect": "Allow" | ||
}, | ||
{ | ||
"Action": [ | ||
"dynamodb:PutItem", | ||
"dynamodb:DeleteItem", | ||
"dynamodb:UpdateItem", | ||
"dynamodb:GetItem", | ||
"dynamodb:Scan", | ||
"dynamodb:Query" | ||
], | ||
"Resource": [ | ||
"arn:aws:dynamodb:*:*:table/media-query-*" | ||
], | ||
"Effect": "Allow" | ||
}, | ||
{ | ||
"Action": [ | ||
"rekognition:DetectLabels", | ||
"rekognition:StartLabelDetection", | ||
"rekognition:GetLabelDetection" | ||
], | ||
"Resource": "*", | ||
"Effect": "Allow" | ||
}, | ||
{ | ||
"Action": [ | ||
"s3:GetObject" | ||
], | ||
"Resource": "arn:aws:s3:::media-query*", | ||
"Effect": "Allow" | ||
}, | ||
{ | ||
"Action": [ | ||
"iam:PassRole" | ||
], | ||
"Resource": "arn:aws:iam::*:role/media-query-*", | ||
"Effect": "Allow" | ||
} | ||
] | ||
} |
Empty file.
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
Original file line number | Diff line number | Diff line change |
---|---|---|
@@ -0,0 +1,77 @@ | ||
from functools import reduce | ||
|
||
from boto3.dynamodb.conditions import Attr | ||
|
||
|
||
IMAGE_TYPE = 'image' | ||
VIDEO_TYPE = 'video' | ||
|
||
|
||
class MediaDB(object): | ||
def list_media_files(self, label=None): | ||
pass | ||
|
||
def add_media_file(self, name, media_type, labels=None): | ||
pass | ||
|
||
def get_media_file(self, name): | ||
pass | ||
|
||
def delete_media_file(self, name): | ||
pass | ||
|
||
|
||
class DynamoMediaDB(MediaDB): | ||
def __init__(self, table_resource): | ||
self._table = table_resource | ||
|
||
def list_media_files(self, startswith=None, media_type=None, label=None): | ||
scan_params = {} | ||
filter_expression = None | ||
if startswith is not None: | ||
filter_expression = self._add_to_filter_expression( | ||
filter_expression, Attr('name').begins_with(startswith) | ||
) | ||
if media_type is not None: | ||
filter_expression = self._add_to_filter_expression( | ||
filter_expression, Attr('type').eq(media_type) | ||
) | ||
if label is not None: | ||
filter_expression = self._add_to_filter_expression( | ||
filter_expression, Attr('labels').contains(label) | ||
) | ||
if filter_expression: | ||
scan_params['FilterExpression'] = filter_expression | ||
response = self._table.scan(**scan_params) | ||
return response['Items'] | ||
|
||
def add_media_file(self, name, media_type, labels=None): | ||
if labels is None: | ||
labels = [] | ||
self._table.put_item( | ||
Item={ | ||
'name': name, | ||
'type': media_type, | ||
'labels': labels, | ||
} | ||
) | ||
|
||
def get_media_file(self, name): | ||
response = self._table.get_item( | ||
Key={ | ||
'name': name, | ||
}, | ||
) | ||
return response.get('Item') | ||
|
||
def delete_media_file(self, name): | ||
self._table.delete_item( | ||
Key={ | ||
'name': name, | ||
} | ||
) | ||
|
||
def _add_to_filter_expression(self, expression, condition): | ||
if expression is None: | ||
return condition | ||
return expression & condition |
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
Original file line number | Diff line number | Diff line change |
---|---|---|
@@ -0,0 +1,53 @@ | ||
import uuid | ||
|
||
|
||
class RekognitonClient(object): | ||
def __init__(self, boto3_client): | ||
self._boto3_client = boto3_client | ||
|
||
def get_image_labels(self, bucket, key): | ||
response = self._boto3_client.detect_labels( | ||
Image={ | ||
'S3Object': { | ||
'Bucket': bucket, | ||
'Name': key | ||
}, | ||
}, | ||
MinConfidence=50.0 | ||
) | ||
return [label['Name'] for label in response['Labels']] | ||
|
||
def start_video_label_job(self, bucket, key, topic_arn, role_arn): | ||
response = self._boto3_client.start_label_detection( | ||
Video={ | ||
'S3Object': { | ||
'Bucket': bucket, | ||
'Name': key | ||
} | ||
}, | ||
ClientRequestToken=str(uuid.uuid4()), | ||
NotificationChannel={ | ||
'SNSTopicArn': topic_arn, | ||
'RoleArn': role_arn | ||
}, | ||
MinConfidence=50.0 | ||
) | ||
return response['JobId'] | ||
|
||
def get_video_job_labels(self, job_id): | ||
labels = set() | ||
client_kwargs = { | ||
'JobId': job_id, | ||
} | ||
response = self._boto3_client.get_label_detection(**client_kwargs) | ||
self._collect_video_labels(labels, response) | ||
while 'NextToken' in response: | ||
client_kwargs['NextToken'] = response['NextToken'] | ||
response = self._boto3_client.get_label_detection(**client_kwargs) | ||
self._collect_video_labels(labels, response) | ||
return list(labels) | ||
|
||
def _collect_video_labels(self, labels, response): | ||
for label in response['Labels']: | ||
label_name = label['Label']['Name'] | ||
labels.add(label_name) |
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
Original file line number | Diff line number | Diff line change |
---|---|---|
@@ -0,0 +1,41 @@ | ||
import argparse | ||
import json | ||
import os | ||
|
||
import boto3 | ||
from botocore import xform_name | ||
|
||
|
||
def record_as_env_var(stack_name, stage): | ||
cloudformation = boto3.client('cloudformation') | ||
response = cloudformation.describe_stacks( | ||
StackName=stack_name | ||
) | ||
outputs = response['Stacks'][0]['Outputs'] | ||
with open(os.path.join('.chalice', 'config.json')) as f: | ||
data = json.load(f) | ||
data['stages'].setdefault(stage, {}).setdefault( | ||
'environment_variables', {} | ||
) | ||
for output in outputs: | ||
data['stages'][stage]['environment_variables'][ | ||
_to_env_var_name(output['OutputKey'])] = output['OutputValue'] | ||
with open(os.path.join('.chalice', 'config.json'), 'w') as f: | ||
serialized = json.dumps(data, indent=2, separators=(',', ': ')) | ||
f.write(serialized + '\n') | ||
|
||
|
||
def _to_env_var_name(name): | ||
return xform_name(name).upper() | ||
|
||
|
||
def main(): | ||
parser = argparse.ArgumentParser() | ||
parser.add_argument('-s', '--stage', default='dev') | ||
parser.add_argument('--stack-name', required=True) | ||
args = parser.parse_args() | ||
record_as_env_var(stack_name=args.stack_name, stage=args.stage) | ||
|
||
|
||
if __name__ == '__main__': | ||
main() |
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
Original file line number | Diff line number | Diff line change |
---|---|---|
@@ -0,0 +1 @@ | ||
boto3<1.8.0 |
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
Original file line number | Diff line number | Diff line change |
---|---|---|
@@ -0,0 +1,104 @@ | ||
{ | ||
"Outputs": { | ||
"MediaBucketName": { | ||
"Value": { | ||
"Ref": "MediaBucket" | ||
} | ||
}, | ||
"MediaTableName": { | ||
"Value": { | ||
"Ref": "MediaTable" | ||
} | ||
}, | ||
"VideoTopicArn": { | ||
"Value": { | ||
"Ref": "VideoTopic" | ||
} | ||
}, | ||
"VideoTopicName": { | ||
"Value": { | ||
"Fn::GetAtt": [ | ||
"VideoTopic", | ||
"TopicName" | ||
] | ||
} | ||
}, | ||
"VideoRoleArn": { | ||
"Value": { | ||
"Fn::GetAtt": [ | ||
"VideoRole", | ||
"Arn" | ||
] | ||
} | ||
} | ||
}, | ||
"Resources": { | ||
"MediaBucket": { | ||
"Type": "AWS::S3::Bucket" | ||
}, | ||
"MediaTable": { | ||
"Properties": { | ||
"AttributeDefinitions": [ | ||
{ | ||
"AttributeName": "name", | ||
"AttributeType": "S" | ||
} | ||
], | ||
"KeySchema": [ | ||
{ | ||
"AttributeName": "name", | ||
"KeyType": "HASH" | ||
} | ||
], | ||
"ProvisionedThroughput": { | ||
"ReadCapacityUnits": 5, | ||
"WriteCapacityUnits": 5 | ||
} | ||
}, | ||
"Type": "AWS::DynamoDB::Table" | ||
}, | ||
"VideoTopic": { | ||
"Type": "AWS::SNS::Topic" | ||
}, | ||
"VideoRole": { | ||
"Properties": { | ||
"AssumeRolePolicyDocument": { | ||
"Statement": [ | ||
{ | ||
"Effect": "Allow", | ||
"Action": [ | ||
"sts:AssumeRole" | ||
], | ||
"Principal": { | ||
"Service": [ | ||
"rekognition.amazonaws.com" | ||
] | ||
} | ||
} | ||
] | ||
}, | ||
"Policies": [ | ||
{ | ||
"PolicyName": "RekognitionPublish", | ||
"PolicyDocument": { | ||
"Statement": [ | ||
{ | ||
"Effect": "Allow", | ||
"Action": [ | ||
"sns:Publish" | ||
], | ||
"Resource": [ | ||
{ | ||
"Ref": "VideoTopic" | ||
} | ||
] | ||
} | ||
] | ||
} | ||
} | ||
] | ||
}, | ||
"Type": "AWS::IAM::Role" | ||
} | ||
} | ||
} |
Oops, something went wrong.