-
Notifications
You must be signed in to change notification settings - Fork 97
/
create-solution.sh
executable file
·201 lines (163 loc) · 5.01 KB
/
create-solution.sh
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
125
126
127
128
129
130
131
132
133
134
135
136
137
138
139
140
141
142
143
144
145
146
147
148
149
150
151
152
153
154
155
156
157
158
159
160
161
162
163
164
165
166
167
168
169
170
171
172
173
174
175
176
177
178
179
180
181
182
183
184
185
186
187
188
189
190
191
192
193
194
195
196
197
198
199
200
201
#!/bin/bash
# Strict mode, fail on any error
set -euo pipefail
on_error() {
set +e
echo "There was an error, execution halted" >&2
echo "Error at line $1"
exit 1
}
trap 'on_error $LINENO' ERR
export PREFIX=''
export LOCATION="eastus"
export TESTTYPE="1"
export STEPS="CIDPTMV"
usage() {
echo "Usage: $0 -d <deployment-name> [-s <steps>] [-t <test-type>] [-l <location>]"
echo "-s: specify which steps should be executed. Default=$STEPS"
echo " Possible values:"
echo " C=COMMON"
echo " I=INGESTION"
echo " D=DATABASE"
echo " P=PROCESSING"
echo " T=TEST clients"
echo " M=METRICS reporting"
echo " V=VERIFY deployment"
echo "-t: test 1,5,10 thousands msgs/sec. Default=$TESTTYPE"
echo "-l: where to create the resources. Default=$LOCATION"
exit 1;
}
# Initialize parameters specified from command line
while getopts ":d:s:t:l:" arg; do
case "${arg}" in
d)
PREFIX=${OPTARG}
;;
s)
STEPS=${OPTARG}
;;
t)
TESTTYPE=${OPTARG}
;;
l)
LOCATION=${OPTARG}
;;
esac
done
shift $((OPTIND-1))
if [[ -z "$PREFIX" ]]; then
echo "Enter a name for this deployment."
usage
fi
# 10000 messages/sec
if [ "$TESTTYPE" == "10" ]; then
export EVENTHUB_PARTITIONS=16
export EVENTHUB_CAPACITY=12
export COSMOSDB_RU=100000
export SIMULATOR_INSTANCES=5
export DATABRICKS_NODETYPE=Standard_DS3_v2
export DATABRICKS_WORKERS=16
export DATABRICKS_MAXEVENTSPERTRIGGER=100000
fi
# 5000 messages/sec
if [ "$TESTTYPE" == "5" ]; then
export EVENTHUB_PARTITIONS=8
export EVENTHUB_CAPACITY=6
export COSMOSDB_RU=50000
export SIMULATOR_INSTANCES=3
export DATABRICKS_NODETYPE=Standard_DS3_v2
export DATABRICKS_WORKERS=8
export DATABRICKS_MAXEVENTSPERTRIGGER=50000
fi
# 1000 messages/sec
if [ "$TESTTYPE" == "1" ]; then
export EVENTHUB_PARTITIONS=2
export EVENTHUB_CAPACITY=2
export COSMOSDB_RU=20000
export SIMULATOR_INSTANCES=1
export DATABRICKS_NODETYPE=Standard_DS3_v2
export DATABRICKS_WORKERS=2
export DATABRICKS_MAXEVENTSPERTRIGGER=10000
fi
# last checks and variables setup
if [ -z ${SIMULATOR_INSTANCES+x} ]; then
usage
fi
export RESOURCE_GROUP=$PREFIX
# remove log.txt if exists
rm -f log.txt
echo "Checking pre-requisites..."
source ../assert/has-local-az.sh
source ../assert/has-local-jq.sh
source ../assert/has-local-databrickscli.sh
echo
echo "Streaming at Scale with Azure Databricks and CosmosDB"
echo "====================================================="
echo
echo "Steps to be executed: $STEPS"
echo
echo "Configuration: "
echo ". Resource Group => $RESOURCE_GROUP"
echo ". Region => $LOCATION"
echo ". EventHubs => TU: $EVENTHUB_CAPACITY, Partitions: $EVENTHUB_PARTITIONS"
echo ". Databricks => VM: $DATABRICKS_NODETYPE, Workers: $DATABRICKS_WORKERS, maxEventsPerTrigger: $DATABRICKS_MAXEVENTSPERTRIGGER"
echo ". CosmosDB => RU: $COSMOSDB_RU"
echo ". Simulators => $SIMULATOR_INSTANCES"
echo
echo "Deployment started..."
echo
echo "***** [C] Setting up COMMON resources"
export AZURE_STORAGE_ACCOUNT=$PREFIX"storage"
RUN=`echo $STEPS | grep C -o || true`
if [ ! -z "$RUN" ]; then
source ../components/azure-common/create-resource-group.sh
source ../components/azure-storage/create-storage-account.sh
fi
echo
echo "***** [I] Setting up INGESTION"
export EVENTHUB_NAMESPACE=$PREFIX"eventhubs"
export EVENTHUB_NAME=$PREFIX"in-"$EVENTHUB_PARTITIONS
export EVENTHUB_CG="cosmos"
RUN=`echo $STEPS | grep I -o || true`
if [ ! -z "$RUN" ]; then
source ../components/azure-event-hubs/create-event-hub.sh
fi
echo
echo "***** [D] Setting up DATABASE"
export COSMOSDB_SERVER_NAME=$PREFIX"cosmosdb"
export COSMOSDB_DATABASE_NAME="streaming"
export COSMOSDB_COLLECTION_NAME="rawdata"
RUN=`echo $STEPS | grep D -o || true`
if [ ! -z "$RUN" ]; then
source ../components/azure-cosmosdb/create-cosmosdb.sh
fi
echo
echo "***** [P] Setting up PROCESSING"
export ADB_WORKSPACE=$PREFIX"databricks"
export ADB_TOKEN_KEYVAULT=$PREFIX"kv" #NB AKV names are limited to 24 characters
RUN=`echo $STEPS | grep P -o || true`
if [ ! -z "$RUN" ]; then
source ../components/azure-databricks/create-databricks.sh
source ../streaming/databricks/runners/eventhubs-to-cosmosdb.sh
fi
echo
echo "***** [T] Starting up TEST clients"
RUN=`echo $STEPS | grep T -o || true`
if [ ! -z "$RUN" ]; then
source ../simulator/run-generator-eventhubs.sh
fi
echo
echo "***** [M] Starting METRICS reporting"
RUN=`echo $STEPS | grep M -o || true`
if [ ! -z "$RUN" ]; then
source ../components/azure-event-hubs/report-throughput.sh
fi
echo
echo "***** [V] Starting deployment VERIFICATION"
RUN=`echo $STEPS | grep V -o || true`
if [ ! -z "$RUN" ]; then
source ../components/azure-databricks/create-databricks.sh
source ../streaming/databricks/runners/verify-cosmosdb.sh
fi
echo
echo "***** Done"