-
Notifications
You must be signed in to change notification settings - Fork 0
/
template.yaml
223 lines (201 loc) · 6.52 KB
/
template.yaml
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
125
126
127
128
129
130
131
132
133
134
135
136
137
138
139
140
141
142
143
144
145
146
147
148
149
150
151
152
153
154
155
156
157
158
159
160
161
162
163
164
165
166
167
168
169
170
171
172
173
174
175
176
177
178
179
180
181
182
183
184
185
186
187
188
189
190
191
192
193
194
195
196
197
198
199
200
201
202
203
204
205
206
207
208
209
210
211
212
213
214
215
216
217
218
219
220
221
222
223
AWSTemplateFormatVersion: '2010-09-09'
Transform: AWS::Serverless-2016-10-31
Description: >
maintain and report on crowdsourced bathymetry resources
#
# expects the following resources to be defined outside the template
# - IAM role: arn:aws:iam::282856304593:role/csb-pilot-role
# - Athena database/table: dcdb.bathy-orders
# - Athena database/table: dcdb.csb_parquet
# - S3 bucket: aws-athena-query-results-282856304593-us-east-1
# - S3 bucket: order-pickup
# - SNS Topic: arn:aws:sns:us-east-1:282856304593:csb-email
Globals:
Function:
Timeout: 3
MemorySize: 128
Runtime: python3.9
Architectures:
- x86_64
Handler: app.lambda_handler
Environment:
Variables:
LOGLEVEL: WARNING
ORDERS_TABLE: 'bathy-orders'
ATHENA_DATABASE: 'dcdb'
ATHENA_TABLE: 'csb_parquet'
ATHENA_QUERIES_TABLE: !Ref QueryTrackingTable
DELIVERY_BUCKET_NAME: 'order-pickup'
ATHENA_RESULTS_BUCKET_NAME: 'aws-athena-query-results-282856304593-us-east-1'
Resources:
# HelloWorldFunction:
# Type: AWS::Serverless::Function # More info about Function Resource: https://github.com/awslabs/serverless-application-model/blob/master/versions/2016-10-31.md#awsserverlessfunction
# Properties:
# CodeUri: hello_world/
# Handler: app.lambda_handler
# Role: arn:aws:iam::282856304593:role/csb-pilot-role
# Runtime: python3.9
# Architectures:
# - x86_64
# Events:
# TriggerEvent:
# Type: ScheduleV2
# Properties:
# Description: execute state machine to drop and re-create the CSB parquet-format table
# Name: !Sub '${AWS::StackName}-hello-world-trigger'
# RoleArn: arn:aws:iam::282856304593:role/csb-cwevents-role
# ScheduleExpression: rate(2 minutes)
# State: ENABLED
# Input: |-
# {
# "globals": {
# "ATHENA_DATABASE": "dcdb",
# "CSV_TABLE": "csb_h3",
# "PARQUET_TABLE": "csb_parquet",
# "S3_BUCKET": "dcdb-data"
# }
# }
CountByH3ResultsFunction:
Type: AWS::Serverless::Function
Properties:
CodeUri: count_by_h3_results/
Role: arn:aws:iam::282856304593:role/csb-pilot-role
Events:
Trigger:
Type: CloudWatchEvent
Properties:
Pattern:
source:
- aws.athena
detail-type:
- 'Athena Query State Change'
detail:
currentState:
- 'SUCCEEDED'
- 'FAILED'
CountBathyOrdersFunction:
Type: AWS::Serverless::Function
Properties:
CodeUri: count_bathy_orders/
Role: arn:aws:iam::282856304593:role/csb-pilot-role
CountByH3QueryFunction:
Type: AWS::Serverless::Function
Properties:
CodeUri: count_by_h3_query/
Role: arn:aws:iam::282856304593:role/csb-pilot-role
FYCountsInputFunction:
Type: AWS::Serverless::Function
Properties:
CodeUri: fy_counts_input/
Role: arn:aws:iam::282856304593:role/csb-pilot-role
UpdateCsbStatisticsFunction:
Type: AWS::Serverless::Function
Properties:
CodeUri: update_csb_statistics/
Role: arn:aws:iam::282856304593:role/csb-pilot-role
#
# Step Functions
#
CsbReportingStateMachine:
Type: AWS::Serverless::StateMachine
Properties:
DefinitionUri: statemachine/csb_reporting.asl.json
Role: arn:aws:iam::282856304593:role/csb-pilot-role
DefinitionSubstitutions:
UpdateCsbStatisticsFunction: !GetAtt UpdateCsbStatisticsFunction.Arn
FYCountsInputFunction: !GetAtt FYCountsInputFunction.Arn
CountByH3QueryFunction: !GetAtt CountByH3QueryFunction.Arn
CountBathyOrdersFunction: !GetAtt CountBathyOrdersFunction.Arn
# rebuild Parquet table statemachine
CsbRebuildTableStateMachine:
Type: AWS::Serverless::StateMachine
Properties:
DefinitionUri: statemachine/rebuild_csb_table.asl.json
Role: arn:aws:iam::282856304593:role/csb-pilot-role
DefinitionSubstitutions:
EmailSnsTopic: "arn:aws:sns:us-east-1:282856304593:csb-email"
#
# EventBridge Rules
#
RebuildCsbTableRule:
Type: AWS::Events::Rule
Properties:
Description: Rebuild the parquet-format CSB table by executing a state machine to drop and re-create the table
EventBusName: default
Name: !Sub '${AWS::StackName}-rebuild-csb-table-rule'
ScheduleExpression: rate(2 days)
State: ENABLED
Targets:
- Id: CsbRebuildTableStateMachine
Arn: !Ref CsbRebuildTableStateMachine
RoleArn: arn:aws:iam::282856304593:role/csb-cwevents-role
Input: |-
{
"globals": {
"ATHENA_DATABASE": "dcdb",
"CSV_TABLE": "csb_h3",
"PARQUET_TABLE": "csb_parquet",
"S3_BUCKET": "dcdb-data"
}
}
RetryPolicy:
MaximumRetryAttempts: 1
GenerateCsbDataSummaryRule:
Type: AWS::Events::Rule
Properties:
Description: trigger the step function used to generate data for the CSB dashboard
EventBusName: default
Name: !Sub '${AWS::StackName}-csb-data-summary-rule'
ScheduleExpression: rate(2 days)
State: ENABLED
Targets:
- Id: CsbReportingStateMachine
Arn: !Ref CsbReportingStateMachine
RoleArn: arn:aws:iam::282856304593:role/csb-cwevents-role
Input: |-
{
"TableName": "dcdb.csb_parquet"
}
# TODO
# DeadLetterConfig:
# Arn: arn:aws:sqs:us-east-1:282856304593:CsbSummaryResultsQueue
RetryPolicy:
MaximumRetryAttempts: 5
#
# SNS Topics
#
# TODO
#
# SQS Queues
#
# TODO
#
# Tables
#
# associate the Athena QueryExecutionId with the StepFunction TaskToken
QueryTrackingTable:
Type: AWS::DynamoDB::Table
Properties:
TableName: !Sub '${AWS::StackName}-queries'
TimeToLiveSpecification:
AttributeName: ttl,
Enabled: true
Tags:
- Key: 'stack-name'
Value: !Ref AWS::StackName
- Key: project
Value: bathymetry
AttributeDefinitions:
- AttributeName: id
AttributeType: S
- AttributeName: label
AttributeType: S
KeySchema:
- AttributeName: id
KeyType: HASH
- AttributeName: label
KeyType: RANGE
BillingMode: PAY_PER_REQUEST
Outputs:
QueryLookupTableName:
Value: !Ref QueryTrackingTable