-
Notifications
You must be signed in to change notification settings - Fork 3
/
consumers.clj
93 lines (78 loc) · 3.63 KB
/
consumers.clj
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
(ns piped.consumers
"Code relating to reading SQS messages from channels and processing them."
(:require [clojure.core.async :as async]
[clojure.tools.logging :as log]
[piped.utils :as utils]
[piped.sqs :as sqs]))
(defn- make-consumer [client input-chan ack-chan nack-chan message-fn]
(async/go-loop [msg nil task nil]
(if (and (nil? msg) (nil? task))
(if-some [msg (async/<! input-chan)]
(recur msg (message-fn msg))
:complete)
(case (async/alt!
[task] ([action] action)
[(utils/message->deadline msg)] :extend
:priority true)
nil
(recur nil nil)
:ack
(do (async/>! ack-chan msg) (recur nil nil))
:nack
(do (async/>! nack-chan msg) (recur nil nil))
:extend
(recur
(let [message-id (utils/message->identifier msg)
queue-url (utils/message->queue-url msg)
old-timeout (utils/message->timeout msg)
new-timeout (* 2 old-timeout)
response (async/<! (sqs/change-visibility-one client msg new-timeout))]
(if (utils/anomaly? response)
(log/error "Error extending visibility timeout of inflight message." (pr-str response))
(log/infof "Extended visibility for inflight message %s in queue %s from %d to %d seconds." message-id queue-url old-timeout new-timeout))
(-> msg (utils/with-deadline (* new-timeout 1000)) (utils/with-timeout new-timeout)))
task)))))
(defn spawn-consumer-async
"Spawns a consumer fit for cpu bound or asynchronous tasks. Uses the core.async dispatch thread pool.
:client - an aws-api sqs client instance
:input-chan - a channel of incoming sqs messages
:ack-chan - a channel that accepts messages that should be acked
:nack-chan - a channel that accepts messages that should be nacked
:consumer-fn - a function of a message that either returns a result directly
or may return a core.async channel that emits once (like a
promise chan) when finished processing the message. Must not
block.
"
[client input-chan ack-chan nack-chan consumer-fn]
(make-consumer client input-chan ack-chan nack-chan
(fn [msg]
(async/go
(try
(loop [result (consumer-fn msg)]
(if (utils/channel? result)
(recur (async/<! result))
(if (contains? #{:ack :nack} result) result :ack)))
(catch Exception e
(log/error e "Exception processing sqs message in async consumer.")
:nack))))))
(defn spawn-consumer-blocking
"Spawns a consumer fit for synchronous blocking tasks. Uses a dedicated thread when processing a message.
:client - an aws-api sqs client instance
:input-chan - a channel of incoming sqs messages
:ack-chan - a channel that accepts messages that should be acked
:nack-chan - a channel that accepts messages that should be nacked
:consumer-fn - a function of a message that may perform blocking io with
the message.
"
[client input-chan ack-chan nack-chan consumer-fn]
(make-consumer client input-chan ack-chan nack-chan
(fn [msg]
(async/thread
(try
(loop [result (consumer-fn msg)]
(if (utils/channel? result)
(recur (async/<!! result))
(if (contains? #{:ack :nack} result) result :ack)))
(catch Exception e
(log/error e "Exception processing sqs message in blocking consumer.")
:nack))))))