-
Notifications
You must be signed in to change notification settings - Fork 13
/
serverless.yml
93 lines (84 loc) · 2.38 KB
/
serverless.yml
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
service: serverless-data-pipeline
provider:
name: aws
region: eu-central-1
runtime: python3.7
stage: ${opt:stage}
# add your own lambda_function role ARN here
role: <YOUR_LAMBDA_ROLE_ARN_MUST_COME_HERE>
plugins:
- serverless-s3-remover
custom:
bucket: ${self:service}-${self:provider.stage}
bucket_glue_scripts: ${self:custom.bucket}-glue-scripts
glue_crawler: load-${self:provider.stage}
glue_job: unzip-${self:provider.stage}
glue_job_script_location: s3://${self:custom.bucket_glue_scripts}/unzip.py
# add your own glue role ARN here
glue_role: <YOUR_GLUE_ROLE_ARN_MUST_COME_HERE>
athena_db: ${self:custom.bucket}
remover:
buckets:
- ${self:custom.bucket}
- ${self:custom.bucket_glue_scripts}
functions:
unzip:
handler: lambda_function/unzip.handler
events:
- s3:
bucket: ${self:custom.bucket}
event: s3:ObjectCreated:*
rules:
- prefix: raw/
- suffix: .zip
environment:
DEST_BUCKET: ${self:custom.bucket}
DEST_KEY: unzip/
GLUE_JOB: ${self:custom.glue_job}
extract:
handler: lambda_function/extract.handler
events:
- s3:
bucket: ${self:custom.bucket}
event: s3:ObjectCreated:*
rules:
- prefix: unzip/
- suffix: .eml
environment:
DEST_BUCKET: ${self:custom.bucket}
DEST_KEY: extract/
load:
handler: lambda_function/load.handler
events:
- s3:
bucket: ${self:custom.bucket}
event: s3:ObjectCreated:*
rules:
- prefix: extract/
- suffix: .json
environment:
GLUE_CRAWLER: ${self:custom.glue_crawler}
resources:
Resources:
GlueJob:
Type: AWS::Glue::Job
Properties:
Role: ${self:custom.glue_role}
Command:
Name: pythonshell
ScriptLocation: ${self:custom.glue_job_script_location}
Name: ${self:custom.glue_job}
S3GlueScripts:
Type: AWS::S3::Bucket
Properties:
BucketName: ${self:custom.bucket_glue_scripts}
GlueCrawler:
Type: AWS::Glue::Crawler
Properties:
DatabaseName: ${self:custom.athena_db}
Name: ${self:custom.glue_crawler}
Role: ${self:custom.glue_role}
TablePrefix: messages_
Targets:
S3Targets:
- Path: s3://${self:custom.bucket}/extract/