/
commands.sh
124 lines (97 loc) · 3.57 KB
/
commands.sh
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
#!/bin/bash
########
# JAVA #
########
sudo apt update
sudo apt install openjdk-11-jre-headless -y
#########
# KAFKA #
#########
wget https://downloads.apache.org/kafka/3.5.1/kafka_2.13-3.5.1.tgz
sudo mkdir /usr/local/kafka
sudo chown -R $(whoami) /usr/local/kafka
tar \
-xzf kafka_2.13-3.5.1.tgz \
-C /usr/local/kafka \
--strip-components=1
# set up kraft
export uuid=$(/usr/local/kafka/bin/kafka-storage.sh random-uuid)
/usr/local/kafka/bin/kafka-storage.sh format \
-t $uuid \
-c /usr/local/kafka/config/kraft/server.properties
# start kafka
/usr/local/kafka/bin/kafka-server-start.sh \
-daemon \
/usr/local/kafka/config/kraft/server.properties
###############
# KAFKA TOPIC #
###############
/usr/local/kafka/bin/kafka-topics.sh \
--create \
--topic mytopic \
--bootstrap-server localhost:9092 \
--partitions 10
/usr/local/kafka/bin/kafka-topics.sh \
--create \
--topic deadletter \
--bootstrap-server localhost:9092 \
--partitions 10
#################
# KAFKA CONNECT #
#################
mkdir /usr/local/kafka/plugins /usr/local/kafka/plugins/camel-postgresql-sink-kafka-connector
echo "plugin.path=/usr/local/kafka/plugins" >> /usr/local/kafka/config/connect-distributed.properties
# download connector
wget https://repo.maven.apache.org/maven2/org/apache/camel/kafkaconnector/camel-postgresql-sink-kafka-connector/3.18.2/camel-postgresql-sink-kafka-connector-3.18.2-package.tar.gz
tar \
-xzf camel-postgresql-sink-kafka-connector-3.18.2-package.tar.gz \
-C /usr/local/kafka/plugins/camel-postgresql-sink-kafka-connector \
--strip-components=1
# download postgresql driver
wget https://jdbc.postgresql.org/download/postgresql-42.6.0.jar
mv postgresql-42.6.0.jar /usr/local/kafka/plugins/camel-postgresql-sink-kafka-connector
# start kafka connect
/usr/local/kafka/bin/connect-distributed.sh \
-daemon \
/usr/local/kafka/config/connect-distributed.properties
# loop that waits for kafka connect to come online
# only useful when this file is run as a whole
# disregard when manually executing commands
while true; do
curl -X GET http://localhost:8083 < /dev/null
if [ $? -eq 0 ]; then
break
fi
sleep 15
done
# timescale-sink config
echo '{
"name": "timescale-sink",
"config": {
"connector.class":"org.apache.camel.kafkaconnector.postgresqlsink.CamelPostgresqlsinkSinkConnector",
"errors.tolerance":"all",
"errors.deadletterqueue.topic.name":"deadletter",
"tasks.max":10,
"value.converter":"org.apache.kafka.connect.storage.StringConverter",
"key.converter":"org.apache.kafka.connect.storage.StringConverter",
"topics":"mytopic",
"camel.kamelet.postgresql-sink.databaseName":"tsdb",
"camel.kamelet.postgresql-sink.password":"password",
"camel.kamelet.postgresql-sink.query":"INSERT INTO accounts (name,city) VALUES (:#name,:#city)",
"camel.kamelet.postgresql-sink.serverName":"service_id.project_id.tsdb.cloud.timescale.com",
"camel.kamelet.postgresql-sink.serverPort":"5432",
"camel.kamelet.postgresql-sink.username":"tsdbadmin"
}
}' > timescale-sink.properties
# add timescale-sink connector
cat timescale-sink.properties | curl -X POST -d @- http://localhost:8083/connectors -H "Content-Type: application/json"
# validate if connector shows up
curl -X GET http://localhost:8083/connectors
############
# KAFKACAT #
############
sudo apt install kafkacat -y
#send message on topic
echo '{"name":"Mathis","city":"Salt Lake City"}' | kafkacat -P -b localhost:9092 -t mytopic
echo '{"name":"Oliver","city":"Moab"}' | kafkacat -P -b localhost:9092 -t mytopic
echo '{"name":"Mark","city":"Park City"}' | kafkacat -P -b localhost:9092 -t mytopic