-
Notifications
You must be signed in to change notification settings - Fork 64
/
consumer.clj
155 lines (140 loc) · 7.87 KB
/
consumer.clj
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
125
126
127
128
129
130
131
132
133
134
135
136
137
138
139
140
141
142
143
144
145
146
147
148
149
150
151
152
153
154
155
(ns ziggurat.messaging.consumer
(:require [ziggurat.mapper :as mpr]
[ziggurat.message-payload :as mp]
[clojure.tools.logging :as log]
[langohr.basic :as lb]
[langohr.channel :as lch]
[langohr.consumers :as lcons]
[ziggurat.kafka-consumer.consumer-handler :as ch]
[schema.core :as s]
[sentry-clj.async :as sentry]
[taoensso.nippy :as nippy]
[ziggurat.config :refer [get-in-config]]
[ziggurat.messaging.connection :refer [connection]]
[ziggurat.sentry :refer [sentry-reporter]]
[ziggurat.messaging.util :refer :all]
[ziggurat.metrics :as metrics]
[ziggurat.util.error :refer [report-error]]))
(defn- convert-to-message-payload
"This function is used for migration from Ziggurat Version 2.x to 3.x. It checks if the message is a message payload or a message(pushed by Ziggurat version < 3.0.0) and converts messages to
message-payload to pass onto the mapper-fn.
If the `:retry-count` key is absent in the `message`, then it puts `0` as the value for `:retry-count` in `MessagePayload`.
It also converts the topic-entity into a keyword while constructing MessagePayload."
[message topic-entity]
(try
(s/validate mp/message-payload-schema message)
(catch Exception e
(log/info "old message format read, converting to message-payload: " message)
(let [retry-count (or (:retry-count message) 0)
message-payload (mp/->MessagePayload (dissoc message :retry-count) (keyword topic-entity))]
(assoc message-payload :retry-count retry-count)))))
(defn convert-and-ack-message
"De-serializes the message payload (`payload`) using `nippy/thaw` and converts it to `MessagePayload`. Acks the message
if `ack?` is true."
[ch {:keys [delivery-tag] :as meta} ^bytes payload ack? topic-entity]
(try
(let [message (nippy/thaw payload)]
(when ack?
(lb/ack ch delivery-tag))
(convert-to-message-payload message topic-entity))
(catch Exception e
(lb/reject ch delivery-tag false)
(report-error e "Error while decoding message")
(metrics/increment-count ["rabbitmq-message" "conversion"] "failure" {:topic_name (name topic-entity)})
nil)))
(defn- ack-message
[ch delivery-tag]
(lb/ack ch delivery-tag))
(defn process-message-from-queue [ch meta payload topic-entity processing-fn]
(let [delivery-tag (:delivery-tag meta)
message-payload (convert-and-ack-message ch meta payload false topic-entity)]
(when message-payload
(log/infof "Processing message [%s] from RabbitMQ " message-payload)
(try
(log/debug "Calling processor-fn with the message-payload - " message-payload " with retry count - " (:retry-count message-payload))
(processing-fn message-payload)
(ack-message ch delivery-tag)
(catch Exception e
(lb/reject ch delivery-tag true)
(report-error e "Error while processing message-payload from RabbitMQ")
(metrics/increment-count ["rabbitmq-message" "process"] "failure" {:topic_name (name topic-entity)}))))))
(defn read-message-from-queue [ch queue-name topic-entity ack?]
(try
(let [[meta payload] (lb/get ch queue-name false)]
(when (some? payload)
(convert-and-ack-message ch meta payload ack? topic-entity)))
(catch Exception e
(report-error e "Error while consuming the dead set message")
(metrics/increment-count ["rabbitmq-message" "consumption"] "failure" {:topic_name (name topic-entity)}))))
(defn- construct-queue-name
([topic-entity]
(construct-queue-name topic-entity nil))
([topic-entity channel]
(if (nil? channel)
(prefixed-queue-name topic-entity (get-in-config [:rabbit-mq :dead-letter :queue-name]))
(prefixed-channel-name topic-entity channel (get-in-config [:rabbit-mq :dead-letter :queue-name])))))
(defn get-dead-set-messages
"This method can be used to read and optionally ack messages in dead-letter queue, based on the value of `ack?`.
For example, this method can be used to delete messages from dead-letter queue if `ack?` is set to true."
([topic-entity count]
(get-dead-set-messages topic-entity nil count))
([topic-entity channel count]
(remove nil?
(with-open [ch (lch/open connection)]
(doall (for [_ (range count)]
(read-message-from-queue ch (construct-queue-name topic-entity channel) topic-entity false)))))))
(defn process-dead-set-messages
"This method reads and processes `count` number of messages from RabbitMQ dead-letter queue for topic `topic-entity` and
channel specified by `channel`. Executes `processing-fn` for every message read from the queue."
([topic-entity count processing-fn]
(process-dead-set-messages topic-entity nil count processing-fn))
([topic-entity channel count processing-fn]
(with-open [ch (lch/open connection)]
(doall (for [_ (range count)]
(let [queue-name (construct-queue-name topic-entity channel)
[meta payload] (lb/get ch queue-name false)]
(when (some? payload)
(process-message-from-queue ch meta payload topic-entity processing-fn))))))))
(defn- message-handler [wrapped-mapper-fn topic-entity]
(fn [ch meta ^bytes payload]
(process-message-from-queue ch meta payload topic-entity wrapped-mapper-fn)))
(defn- start-subscriber* [ch prefetch-count queue-name wrapped-mapper-fn topic-entity]
(lb/qos ch prefetch-count)
(let [consumer-tag (lcons/subscribe ch
queue-name
(message-handler wrapped-mapper-fn topic-entity)
{:handle-shutdown-signal-fn (fn [consumer_tag reason]
(log/infof "channel closed with consumer tag: %s, reason: %s " consumer_tag, reason))
:handle-consume-ok-fn (fn [consumer_tag]
(log/infof "consumer started for %s with consumer tag %s " queue-name consumer_tag))})]))
(defn start-retry-subscriber* [handler-fn topic-entity]
(when (get-in-config [:retry :enabled])
(dotimes [_ (get-in-config [:jobs :instant :worker-count])]
(start-subscriber* (lch/open connection)
(get-in-config [:jobs :instant :prefetch-count])
(prefixed-queue-name topic-entity (get-in-config [:rabbit-mq :instant :queue-name]))
handler-fn
topic-entity))))
(defn start-channels-subscriber [channels topic-entity]
(doseq [channel channels]
(let [channel-key (first channel)
channel-handler-fn (second channel)]
(dotimes [_ (get-in-config [:stream-router topic-entity :channels channel-key :worker-count])]
(start-subscriber* (lch/open connection)
1
(prefixed-channel-name topic-entity channel-key (get-in-config [:rabbit-mq :instant :queue-name]))
(mpr/channel-mapper-func channel-handler-fn channel-key)
topic-entity)))))
(defn start-subscribers
"Starts the subscriber to the instant queue of the rabbitmq"
[stream-routes batch-routes]
(doseq [stream-route stream-routes]
(let [topic-entity (first stream-route)
handler (-> stream-route second :handler-fn)
channels (-> stream-route second (dissoc :handler-fn))]
(start-channels-subscriber channels topic-entity)
(start-retry-subscriber* (mpr/mapper-func handler (keys channels)) topic-entity)))
(doseq [batch-route batch-routes]
(let [topic-entity (first batch-route)
handler (-> batch-route second :handler-fn)]
(start-retry-subscriber* (fn [message] (ch/process handler message)) topic-entity))))