-
Notifications
You must be signed in to change notification settings - Fork 198
/
run.sh
88 lines (58 loc) · 3.59 KB
/
run.sh
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
#!/bin/bash
### SETUP
# Download Confluent Platform 5.1.1 https://www.confluent.io/download/
# Unzip and add confluent-5.1.1/bin to your PATH
# Download and install Docker for Mac / Windows / Linux and do
docker-compose up -d
# Alternatively start postgres manually on your laptop at port 5432 and username/password = postgres/postgres
# Start the Confluent platform using the Confluent CLI
confluent start
# Create all the topics we're going to use for this demo
kafka-topics --create --topic udemy-reviews --partitions 3 --replication-factor 1 --zookeeper localhost:2181
kafka-topics --create --topic udemy-reviews-valid --partitions 3 --replication-factor 1 --zookeeper localhost:2181
kafka-topics --create --topic udemy-reviews-fraud --partitions 3 --replication-factor 1 --zookeeper localhost:2181
kafka-topics --create --topic long-term-stats --partitions 3 --replication-factor 1 --zookeeper localhost:2181
kafka-topics --create --topic recent-stats --partitions 3 --replication-factor 1 --zookeeper localhost:2181
# Build and package the different project components (make sure you have maven installed)
mvn clean package
### PLAYING
## Step 1: Review Producer
# Start an avro consumer on our reviews topic
kafka-avro-console-consumer --topic udemy-reviews --bootstrap-server localhost:9092
# And launch our first producer in another terminal !
export COURSE_ID=1075642 # Kafka for Beginners Course
java -jar udemy-reviews-producer/target/uber-udemy-reviews-producer-1.0-SNAPSHOT.jar
# This pulls overs 1000 reviews with some intentional delay of 50 ms between each send so you can see it stream in your consumer
## Step 2: Kafka Streams - Fraud Detector
# New terminal: Start an avro consumer on our valid reviews topic
kafka-avro-console-consumer --topic udemy-reviews-valid --bootstrap-server localhost:9092
# New terminal: Start an avro consumer on our fraud reviews topic
kafka-avro-console-consumer --topic udemy-reviews-fraud --bootstrap-server localhost:9092
# And launch our fraud Kafka Streams application in another terminal !
java -jar udemy-reviews-fraud/target/uber-udemy-reviews-fraud-1.0-SNAPSHOT.jar
# Keep it running.
# The execution is very quick!
# You'll see that ~50 messages (5%) were conducted to the fraud topic, the rest to the valid topic
## Step 3: Kafka Streams - Reviews Aggregator
# New terminal: Start an avro consumer on our recent stats topic
kafka-avro-console-consumer --topic recent-stats --bootstrap-server localhost:9092
# New terminal: Start an avro consumer on our long term stats topic
kafka-avro-console-consumer --topic long-term-stats --bootstrap-server localhost:9092
# Launch our review aggregator app
java -jar udemy-reviews-aggregator/target/uber-udemy-reviews-aggregator-1.0-SNAPSHOT.jar
# as we can see the recent topic only has the reviews for the past 90 days, while the long term has them all
## Step 4: Kafka Connect
# Load the recent and long term stats into Postgres using Kafka Connect Sink!
confluent load SinkTopics -d kafka-connectors/SinkTopicsInPostgres.properties
## Step 5: Play some more
# Make sure the four components are running (you can shut down the consumers)
# and fire off more producers
export COURSE_ID=1141696 # Kafka Connect Course
java -jar udemy-reviews-producer/target/uber-udemy-reviews-producer-1.0-SNAPSHOT.jar
export COURSE_ID=1141702 # Kafka Setup and Administration Course
java -jar udemy-reviews-producer/target/uber-udemy-reviews-producer-1.0-SNAPSHOT.jar
export COURSE_ID=1294188 # Kafka Streams Course
java -jar udemy-reviews-producer/target/uber-udemy-reviews-producer-1.0-SNAPSHOT.jar
## Step 6: Clean up
docker-compose down
confluent destroy