-
Notifications
You must be signed in to change notification settings - Fork 11
/
AggregateExamples.kt
101 lines (87 loc) · 4.16 KB
/
AggregateExamples.kt
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
package com.perkss.kafka.reactive.examples
import com.perkss.social.media.model.PostCreated
import io.confluent.kafka.streams.serdes.avro.SpecificAvroSerde
import org.apache.kafka.common.serialization.Serdes
import org.apache.kafka.streams.StreamsBuilder
import org.apache.kafka.streams.Topology
import org.apache.kafka.streams.kstream.*
import org.apache.kafka.streams.kstream.Suppressed.BufferConfig.unbounded
import org.slf4j.LoggerFactory
import java.time.Duration
object AggregateExamples {
private val logger = LoggerFactory.getLogger(AggregateExamples::class.java)
fun buildUserSocialMediaPostsTotalCountTopology(
inputTopic: String,
outputTopic: String,
postCreatedSerde: SpecificAvroSerde<PostCreated>
): Topology {
val builder = StreamsBuilder()
// consume the post created
val input = builder.stream(inputTopic, Consumed.with(Serdes.String(), postCreatedSerde))
// Build a table of the counts
val aggregated: KTable<String, Long> = input
.groupBy { _, value -> value.userId } // group by the user who created the post
.count()
// stream the total counts per user keyed by user ID and the count
aggregated.toStream().to(outputTopic, Produced.with(Serdes.String(), Serdes.Long()))
return builder.build()
}
fun buildSocialMediaPostsWindowedTotalCountTopology(
inputTopic: String,
outputTopic: String,
postCreatedSerde: SpecificAvroSerde<PostCreated>
): Topology {
val builder = StreamsBuilder()
// consume the post created
val input = builder.stream(
inputTopic, Consumed.with(
Serdes.String(), postCreatedSerde,
PostCreatedTimestampExtractor, Topology.AutoOffsetReset.EARLIEST
)
)
// Build a table of the counts
val aggregated: KTable<Windowed<String>, Long> = input
.peek { key, postCreated -> logger.info("Key {} and value {}", key, postCreated) }
.groupBy { _, value -> value.userId } // group by the user who created the post
.windowedBy(
TimeWindows.of(Duration.ofSeconds(30))
)
.count()
// stream the total counts per user keyed by user ID and the count
aggregated.toStream { windowedKey, _ -> windowedKey.key() }
.peek { key, value -> logger.info("Sending on Key {} value {}", key, value) }
.to(outputTopic, Produced.with(Serdes.String(), Serdes.Long()))
return builder.build()
}
fun buildSocialMediaPostFinalWindowCountTopology(
inputTopic: String,
outputTopic: String,
postCreatedSerde: SpecificAvroSerde<PostCreated>
): Topology {
val builder = StreamsBuilder()
// consume the post created
val input = builder.stream(
inputTopic, Consumed.with(
Serdes.String(), postCreatedSerde,
PostCreatedTimestampExtractor, Topology.AutoOffsetReset.EARLIEST
)
)
// Build a table of the counts
val aggregated: KTable<Windowed<String>, Long> = input
.peek { key, postCreated -> logger.info("Key {} and value {}", key, postCreated) }
.groupBy { _, value -> value.userId } // group by the user who created the post
.windowedBy(
TimeWindows.of(Duration.ofSeconds(30)).grace(Duration.ZERO)
)
// Note we need to materialize here as serdes changes
.count(Materialized.with(Serdes.String(), Serdes.Long()))
// Suppress the output so only output of a window is emitted when the window closes.
// Provide a name so it is constant in the KStream internally.
.suppress(Suppressed.untilWindowCloses(unbounded()).withName("SocialMediaCountsSuppression"))
// stream the total counts per user keyed by user ID and the count
aggregated.toStream { windowedKey, _ -> windowedKey.key() }
.peek { key, value -> logger.info("Sending on Key {} value {}", key, value) }
.to(outputTopic, Produced.with(Serdes.String(), Serdes.Long()))
return builder.build()
}
}