-
Notifications
You must be signed in to change notification settings - Fork 3
/
setup.sh
395 lines (302 loc) · 14.7 KB
/
setup.sh
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
125
126
127
128
129
130
131
132
133
134
135
136
137
138
139
140
141
142
143
144
145
146
147
148
149
150
151
152
153
154
155
156
157
158
159
160
161
162
163
164
165
166
167
168
169
170
171
172
173
174
175
176
177
178
179
180
181
182
183
184
185
186
187
188
189
190
191
192
193
194
195
196
197
198
199
200
201
202
203
204
205
206
207
208
209
210
211
212
213
214
215
216
217
218
219
220
221
222
223
224
225
226
227
228
229
230
231
232
233
234
235
236
237
238
239
240
241
242
243
244
245
246
247
248
249
250
251
252
253
254
255
256
257
258
259
260
261
262
263
264
265
266
267
268
269
270
271
272
273
274
275
276
277
278
279
280
281
282
283
284
285
286
287
288
289
290
291
292
293
294
295
296
297
298
299
300
301
302
303
304
305
306
307
308
309
310
311
312
313
314
315
316
317
318
319
320
321
322
323
324
325
326
327
328
329
330
331
332
333
334
335
336
337
338
339
340
341
342
343
344
345
346
347
348
349
350
351
352
353
354
355
356
357
358
359
360
361
362
363
364
365
366
367
368
369
370
371
372
373
374
375
376
377
378
379
380
381
382
383
384
385
386
387
388
389
390
391
392
393
394
395
clear
echo "┏━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━┓"
echo "┃ Starting to create all the demo ┃"
echo "┗━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━┛"
echo " *** Thanks Paul Vidal *** "
echo " *** Thanks Andre *** "
echo " *** Thanks Dan ***"
echo ""
echo "▔▔▔▔▔▔▔▔▔▔▔▔▔▔▔▔▔▔▔▔▔▔▔▔▔▔▔▔▔▔▔▔▔▔▔▔▔▔▔▔▔▔"
echo "⏱ $(date +%H%Mhrs)"
echo "▔▔▔▔▔▔▔▔▔▔▔▔▔▔▔▔▔▔▔▔▔▔▔▔▔▔▔▔▔▔▔▔▔▔▔▔▔▔▔▔▔▔"
echo ""
echo ""
# Run NiFi setupnifi.sh
# TODO
# Build NiFi Atlas Connection
# https://www.datainmotion.dev/2020/02/connecting-apache-nifi-to-apache-atlas.html
# Add NiFi SQL Reporting Tasks for JVM, Provenance, ...
# Add NiFi HTTP Listener for Kafka Alerts
# Build Kafka Alerts / Notifier
# Repos
yum -y install git curl wget
# Andre's script
cd /tmp/resources
./reset-to-lab.sh 12
# Created for edge2ai
cd /opt/demo
git clone https://github.com/tspannhw/meetup-sensors
git clone https://github.com/tspannhw/FlinkSQLDemo
git clone https://github.com/tspannhw/ApacheConAtHome2020
git clone https://github.com/tspannhw/retail-dynamic-shelf-pricing
git clone https://github.com/tspannhw/SmartWeather
git clone https://github.com/tspannhw/ClouderaPublicCloudCDFWorkshop
git clone https://github.com/tspannhw/SmartStocks
chmod -R 777 /opt/demo/SmartStocks
chmod -R 777 /opt/demo/FlinkSQLDemo
chmod -R 777 /opt/demo/meetup-sensors
chmod -R 777 /opt/demo/ApacheConAtHome2020
chmod -R 777 /opt/demo/retail-dynamic-shelf-pricing
# No Kerberos
echo ""
echo ""
echo "▔▔▔▔▔▔▔▔▔▔▔▔▔▔▔▔▔▔▔▔▔▔▔▔▔▔▔▔▔▔▔▔▔▔▔▔▔▔▔▔▔▔"
echo " Build HDFS Directories"
echo ""
echo "▔▔▔▔▔▔▔▔▔▔▔▔▔▔▔▔▔▔▔▔▔▔▔▔▔▔▔▔▔▔▔▔▔▔▔▔▔▔▔▔▔▔"
echo ""
echo ""
HADOOP_USER_NAME=hdfs hdfs dfs -mkdir /user/admin
HADOOP_USER_NAME=hdfs hdfs dfs -mkdir /user/root
HADOOP_USER_NAME=hdfs hdfs dfs -chown root:root /user/root
HADOOP_USER_NAME=hdfs hdfs dfs -chown admin:admin /user/admin
HADOOP_USER_NAME=hdfs hdfs dfs -chmod -R 777 /user
HADOOP_USER_NAME=hdfs hdfs dfs -mkdir /user/kafka
HADOOP_USER_NAME=hdfs hdfs dfs -mkdir /tmp/sensors
HADOOP_USER_NAME=hdfs hdfs dfs -chmod -R 777 /tmp/sensors
HADOOP_USER_NAME=hdfs hdfs dfs -chown kafka:kafka /user/kafka
HADOOP_USER_NAME=hdfs hdfs dfs -chown kafka:kafka /tmp/sensors
HADOOP_USER_NAME=hdfs hdfs dfs -mkdir /tmp/itemprice
HADOOP_USER_NAME=hdfs hdfs dfs -chmod -R 777 /tmp/itemprice
HADOOP_USER_NAME=hdfs hdfs dfs -chown kafka:kafka /tmp/itemprice
HADOOP_USER_NAME=hdfs hdfs dfs -mkdir /tmp/twitter
HADOOP_USER_NAME=hdfs hdfs dfs -chmod -R 777 /tmp/twitter
HADOOP_USER_NAME=hdfs hdfs dfs -chown kafka:kafka /tmp/twitter
HADOOP_USER_NAME=hdfs hdfs dfs -mkdir /tmp/globaltwitter
HADOOP_USER_NAME=hdfs hdfs dfs -chmod -R 777 /tmp/globaltwitter
HADOOP_USER_NAME=hdfs hdfs dfs -chown kafka:kafka /tmp/globaltwitter
HADOOP_USER_NAME=hdfs hdfs dfs -mkdir /tmp/stocks
HADOOP_USER_NAME=hdfs hdfs dfs -chmod -R 777 /tmp/stocks
HADOOP_USER_NAME=hdfs hdfs dfs -chmod -R 777 /warehouse/tablespace/managed/hive
HADOOP_USER_NAME=hdfs hdfs dfs -mkdir -p /iextradingnews
HADOOP_USER_NAME=hdfs hdfs dfs -mkdir -p /iextradingquote
HADOOP_USER_NAME=hdfs hdfs dfs -mkdir -p /iextradingchart
HADOOP_USER_NAME=hdfs hdfs dfs -mkdir -p /stocks
HADOOP_USER_NAME=hdfs hdfs dfs -mkdir -p /cyber
HADOOP_USER_NAME=hdfs hdfs dfs -mkdir /tmp/weatherapp
HADOOP_USER_NAME=hdfs hdfs dfs -mkdir -p /tmp/spl
HADOOP_USER_NAME=hdfs hdfs dfs -chmod -R 777 /tmp/spl
HADOOP_USER_NAME=hdfs hdfs dfs -mkdir -p /user/tspann/prices
HADOOP_USER_NAME=hdfs hdfs dfs -chmod -R 777 /user/tspann/prices
HADOOP_USER_NAME=hdfs hdfs dfs -chmod -R 777 /tmp/weatherapp
HADOOP_USER_NAME=hdfs hdfs dfs -chmod -R 777 /
HADOOP_USER_NAME=hdfs hdfs dfs -chmod -R 777 /warehouse/tablespace/managed/hive/
# Need permissions for hive tables
# Permission denied: user=nifi, access=EXECUTE, inode="/warehouse/tablespace/managed/hive/hv_stocks":hive:hive:drwxrwx---
cd /opt/demo/ApacheConAtHome2020
echo ""
echo ""
echo "▔▔▔▔▔▔▔▔▔▔▔▔▔▔▔▔▔▔▔▔▔▔▔▔▔▔▔▔▔▔▔▔▔▔▔▔▔▔▔▔▔▔"
echo "Build HBase Tables"
echo "▔▔▔▔▔▔▔▔▔▔▔▔▔▔▔▔▔▔▔▔▔▔▔▔▔▔▔▔▔▔▔▔▔▔▔▔▔▔▔▔▔▔"
echo ""
echo ""
hbase shell /opt/demo/ApacheConAtHome2020/hbase/tables.tbl
echo ""
echo ""
echo "▔▔▔▔▔▔▔▔▔▔▔▔▔▔▔▔▔▔▔▔▔▔▔▔▔▔▔▔▔▔▔▔▔▔▔▔▔▔▔▔▔▔"
echo " Building Kudu Impala Tables"
echo ""
echo "▔▔▔▔▔▔▔▔▔▔▔▔▔▔▔▔▔▔▔▔▔▔▔▔▔▔▔▔▔▔▔▔▔▔▔▔▔▔▔▔▔▔"
echo ""
echo ""
# Build Kudu tables
impala-shell -i edge2ai-1.dim.local -d default -f /opt/demo/ApacheConAtHome2020/sql/kudu.sql
echo ""
echo ""
echo "▔▔▔▔▔▔▔▔▔▔▔▔▔▔▔▔▔▔▔▔▔▔▔▔▔▔▔▔▔▔▔▔▔▔▔▔▔▔▔▔▔▔"
echo " Building Apache Hive ORC Tx Tables"
echo ""
echo "▔▔▔▔▔▔▔▔▔▔▔▔▔▔▔▔▔▔▔▔▔▔▔▔▔▔▔▔▔▔▔▔▔▔▔▔▔▔▔▔▔▔"
echo ""
echo ""
# Build Hive Tables
beeline -u jdbc:hive2://edge2ai-1.dim.local:10000/default -f /opt/demo/ApacheConAtHome2020/sql/hive.sql
# Postgresql Tables
# Phoenix Tables
# Druid Tables
# HBase Tables
# MySQL Tables
# External ORC Hive Tables
# Parquet Impala Tables
# CREATE EXTERNAL TABLE ingest_existing_files LIKE PARQUET '/user/etl/destination/datafile1.dat'
# STORED AS PARQUET
# LOCATION '/user/etl/destination';
echo ""
echo ""
echo "▔▔▔▔▔▔▔▔▔▔▔▔▔▔▔▔▔▔▔▔▔▔▔▔▔▔▔▔▔▔▔▔▔▔▔▔▔▔▔▔▔▔"
echo " Building Schemas"
echo ""
echo "▔▔▔▔▔▔▔▔▔▔▔▔▔▔▔▔▔▔▔▔▔▔▔▔▔▔▔▔▔▔▔▔▔▔▔▔▔▔▔▔▔▔"
echo ""
echo ""
#
# Get Schema Registry URL
curl -X GET "http://edge2ai-1.dim.local:8585/api/v1/admin/schemas/registryInfo" -H "accept: application/json"
echo ""
echo ""
# Load Schemas into Schema Registry
# https://registry-project.readthedocs.io/en/latest/schema-registry.html#api-examples
# http://edge2ai-1.dim.local:7788/swagger
for f in /opt/demo/ApacheConAtHome2020/schemas/*.avsc
do
echo "Uploading Schema File $f"
schemaname="`echo $f | awk -F'[/.]' '{print $(NF-1)}'`"
echo "Schema [$schemaname]"
echo ""
curl -X POST "http://edge2ai-1.dim.local:7788/api/v1/schemaregistry/schemas" -H "accept: application/json" -H "Content-Type: application/json" -d "{ \"type\": \"avro\", \"schemaGroup\": \"Kafka\", \"name\": \"$schemaname\", \"description\": \"schemaname\", \"compatibility\": \"BOTH\", \"validationLevel\": \"LATEST\"}"
echo ""
# Upload body
curl -X POST "http://edge2ai-1.dim.local:7788/api/v1/schemaregistry/schemas/$schemaname/versions/upload?branch=MASTER&disableCanonicalCheck=false" -H "accept: application/json" -H "Content-Type: multipart/form-data" -F "file=@/opt/demo/ApacheConAtHome2020/schemas/$schemaname.avsc;type=application/json" -F "description=schemaname"
echo "Added."
echo ""
done
echo ""
echo ""
echo "▔▔▔▔▔▔▔▔▔▔▔▔▔▔▔▔▔▔▔▔▔▔▔▔▔▔▔▔▔▔▔▔▔▔▔▔▔▔▔▔▔▔"
echo "Integrate Flink and Atlas"
echo "▔▔▔▔▔▔▔▔▔▔▔▔▔▔▔▔▔▔▔▔▔▔▔▔▔▔▔▔▔▔▔▔▔▔▔▔▔▔▔▔▔▔"
echo ""
# Atlas Flink Integration
#
#curl -k -u admin:supersecret1 --location --request POST 'http://edge2ai-1.dim.local:31000/api/atlas/v2/types/typedefs' \
#--header 'Content-Type: application/json' \
#--data-raw '{
# Kafka Topics (SMM or NiFi could auto build)
echo ""
echo ""
echo "▔▔▔▔▔▔▔▔▔▔▔▔▔▔▔▔▔▔▔▔▔▔▔▔▔▔▔▔▔▔▔▔▔▔▔▔▔▔▔▔▔▔"
echo "Kafka Topics"
echo "▔▔▔▔▔▔▔▔▔▔▔▔▔▔▔▔▔▔▔▔▔▔▔▔▔▔▔▔▔▔▔▔▔▔▔▔▔▔▔▔▔▔"
echo ""
echo ""
for f in /opt/demo/ApacheConAtHome2020/schemas/*.avsc
do
schemaname="`echo $f | awk -F'[/_.]' '{print $(NF-1)}'`"
echo "Creating Kafka Topic [$schemaname]"
echo ""
/opt/cloudera/parcels/CDH/bin/kafka-topics --create --bootstrap-server edge2ai-1.dim.local:9092 --replication-factor 1 --partitions 1 --topic $schemaname
done
/opt/cloudera/parcels/CDH/bin/kafka-topics --create --bootstrap-server edge2ai-1.dim.local:9092 --replication-factor 1 --partitions 1 --topic global_sensor_events
# Kafka Connect
# https://docs.cloudera.com/runtime/7.2.0/smm-rest-api-reference/index.html#/Kafka_Connect_operations
# Uses SMM REST API
# http://edge2ai-1.dim.local:8585/swagger
echo ""
echo ""
echo "▔▔▔▔▔▔▔▔▔▔▔▔▔▔▔▔▔▔▔▔▔▔▔▔▔▔▔▔▔▔▔▔▔▔▔▔▔▔▔▔▔▔"
echo "Kafka Connect"
echo "▔▔▔▔▔▔▔▔▔▔▔▔▔▔▔▔▔▔▔▔▔▔▔▔▔▔▔▔▔▔▔▔▔▔▔▔▔▔▔▔▔▔"
echo ""
echo ""
curl -X GET "http://edge2ai-1.dim.local:8585/api/v1/admin/metrics/producers?state=all&duration=LAST_THIRTY_DAYS" -H "accept: application/json"
echo ""
echo ""
# Is Kafka Connect Configured
# TODO check = false
curl -X GET "http://edge2ai-1.dim.local:8585/api/v1/admin/kafka-connect/is-configured" -H "accept: application/json"
echo ""
echo ""
# Check to see if HDFS Sink is there
curl -X GET "http://edge2ai-1.dim.local:8585/api/v1/admin/kafka-connect/connector-plugins" -H "accept: application/json"
echo ""
echo ""
# load that file TODO
# itemprice is connector name
for f in /opt/demo/ApacheConAtHome2020/kafkaconnect/*.json
do
schemaname="`echo $f | awk -F'[/_.]' '{print $(NF-1)}'`"
echo "Creating Kafka Connectors for each [$schemaname]"
echo ""
curl -X PUT "http://edge2ai-1.dim.local:8585/api/v1/admin/kafka-connect/connectors/$schemaname" -H "accept: application/json" -H "Content-Type: application/json" -d @/opt/demo/ApacheConAtHome2020/kafkaconnect/$schemaname.json
done
echo ""
echo ""
# Check Service
curl -X GET "http://edge2ai-1.dim.local:8585/api/v1/admin/kafka-connect/connectors" -H "accept: application/json"
echo ""
echo ""
# Check KC Metrics
curl -X GET "http://ec2-54-167-28-79.compute-1.amazonaws.com:8585/api/v1/admin/metrics/connect/workers" -H "accept: application/json"
# Build Alerts
# edge2ai-1.dim.local
echo ""
echo ""
echo "▔▔▔▔▔▔▔▔▔▔▔▔▔▔▔▔▔▔▔▔▔▔▔▔▔▔▔▔▔▔▔▔▔▔▔▔▔▔▔▔▔▔"
echo "Build Kafka Alerts"
echo "▔▔▔▔▔▔▔▔▔▔▔▔▔▔▔▔▔▔▔▔▔▔▔▔▔▔▔▔▔▔▔▔▔▔▔▔▔▔▔▔▔▔"
echo ""
echo ""
curl -X GET "http://edge2ai-1.dim.local:9991/api/v1/admin/alert/notifications?limit=11&startOffset=-1"
# POST to build A Notifier
curl -X POST "http://edge2ai-1.dim.local:9991/api/v1/admin/notifiers" -H "accept: application/json" -H "Content-Type: application/json" -d '{"id":null,"name":"http","description":"http","notifierProviderId":"http","rateLimiterConfig":{"count":1,"duration":"MINUTE"},"config":{"URL":"http://edge2ai-1.dim.local:9999/alerts","ConnectionTimeoutInMilliSecs":30000,"ReadTimeoutInMilliSecs":30000}}'
echo ""
# GET to list that notifier
curl -X GET "http://edge2ai-1.dim.local:9991/api/v1/admin/notifiers"
echo ""
# POST to Add a new alert Policy
for f in /opt/demo/ApacheConAtHome2020/alerts/*.json
do
echo "Load alert $f"
curl -X POST "http://edge2ai-1.dim.local:8585/api/v1/admin/alertPolicy" -H "accept: application/json" -H "Content-Type: application/json" -d @$f
echo ""
done
# GET the list of alert policies created
curl -X GET "http://edge2ai-1.dim.local:9991/api/v1/admin/alertPolicy"
# To Disable all policies, iterate all policies, go by id
# http://edge2ai-1.dim.local:9991/api/v1/admin/alertPolicy/45/disable
# To DELETE all policies, iterate all policies, go by id
# http://edge2ai-1.dim.local:9991/api/v1/admin/alertPolicy/6
echo ""
echo ""
echo "▔▔▔▔▔▔▔▔▔▔▔▔▔▔▔▔▔▔▔▔▔▔▔▔▔▔▔▔▔▔▔▔▔▔▔▔▔▔▔▔▔▔"
echo "Flink SQL Setup"
echo "▔▔▔▔▔▔▔▔▔▔▔▔▔▔▔▔▔▔▔▔▔▔▔▔▔▔▔▔▔▔▔▔▔▔▔▔▔▔▔▔▔▔"
echo ""
echo ""
# Flink SQL Session
flink-yarn-session -tm 2048 -s 2 -d
# Starts Flink SQL
# flink-sql-client embedded -e /opt/demo/ApacheConAtHome2020/conf/sql-env.yaml
echo ""
echo ""
echo "▔▔▔▔▔▔▔▔▔▔▔▔▔▔▔▔▔▔▔▔▔▔▔▔▔▔▔▔▔▔▔▔▔▔▔▔▔▔▔▔▔▔"
echo "Server NiFi Flow Setup"
echo "▔▔▔▔▔▔▔▔▔▔▔▔▔▔▔▔▔▔▔▔▔▔▔▔▔▔▔▔▔▔▔▔▔▔▔▔▔▔▔▔▔▔"
echo ""
echo ""
# Get Toolkit
#wget https://archive.apache.org/dist/nifi/1.11.4/nifi-toolkit-1.11.4-bin.tar.gz
#tar -xvf nifi-toolkit*.tar.gz
#/opt/demo/ApacheConAtHome2020/scripts/nifi-toolkit-1.11.4/bin/cli.sh registry list-buckets -u http://edge2ai-1.dim.local:18080
#registry create-flow -verbose -u http://edge2ai-1.dim.local:18080 -b 250a5ae5-ced8-4f4e-8b3b-01eb9d47a0d9 --flowName iotFlow
#registry import-flow-version -verbose -u http://somesite.compute-1.amazonaws.com:18080 -f a5a4ac59-9aeb-416e-937f-e601ca8beba9 -i flows/iot-1.json
# registry list-flows -u http://ec2-35-171-154-174.compute-1.amazonaws.com:18080 -b 250a5ae5-ced8-4f4e-8b3b-01eb9d47a0d9
# https://www.datainmotion.dev/2020/03/using-nifi-cli-to-restore-nifi-flows.html
# https://www.datainmotion.dev/2019/09/backup-and-restore-nifi-registry.html
# Get NiFi Python API
# https://pypi.org/project/nipyapi/
# already installed as part of My Setup
# pip3 install nipyapi
# python3 loadflow.py
# Restore NiFi Flow
# Add parameter context and parameters
# https://www.datainmotion.dev/2020/09/devops-working-with-parameter-contexts.html
# Final HDFS Check (demo only, use Apache Ranger in SDX for real workloads).
HADOOP_USER_NAME=hdfs hdfs dfs -chmod -R 777 /
# Stop MiNiFi and EFM
/opt/cloudera/cem/minifi-0.6.0.1.2.1.0-24/bin/minifi.sh stop
/opt/cloudera/cem/efm/bin/efm.sh stop
# Run NiFi Parms Load
source /opt/demo/ApacheConAtHome2020/scripts/setupnifi.sh
# Get Postgresql jdbc
cd /opt/demo
wget https://jdbc.postgresql.org/download/postgresql-42.2.18.jar
echo ""
echo ""
echo "▔▔▔▔▔▔▔▔▔▔▔▔▔▔▔▔▔▔▔▔▔▔▔▔▔▔▔▔▔▔▔▔▔▔▔▔▔▔▔▔▔▔"
echo ""
echo "Completed."
echo ""
echo "⏱ $(date +%H%Mhrs)"
echo "▔▔▔▔▔▔▔▔▔▔▔▔▔▔▔▔▔▔▔▔▔▔▔▔▔▔▔▔▔▔▔▔▔▔▔▔▔▔▔▔▔▔"