-
Notifications
You must be signed in to change notification settings - Fork 160
/
consumer.clj
64 lines (52 loc) · 2.6 KB
/
consumer.clj
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
(ns crux.tx.consumer
(:require [clojure.tools.logging :as log]
[crux.db :as db]
[crux.node :as n]
[crux.tx :as tx]
[crux.codec :as c])
(:import java.io.Closeable
java.time.Duration))
(defn- index-tx-log [{:keys [!error], ::n/keys [tx-log indexer document-store]} {::keys [^Duration poll-sleep-duration]}]
(log/info "Started tx-consumer")
(try
(while true
(let [consumed-txs? (with-open [tx-log (db/open-tx-log tx-log (::tx/tx-id (db/latest-completed-tx indexer)))]
(let [tx-log (iterator-seq tx-log)
consumed-txs? (not (empty? tx-log))]
(doseq [{:keys [crux.tx.event/tx-events] :as tx} tx-log
:let [tx (select-keys tx [::tx/tx-time ::tx/tx-id])]]
(db/index-docs indexer (db/fetch-docs document-store
(->> tx-events
(into #{} (comp (mapcat tx/tx-event->doc-hashes)
(map c/new-id))))))
(db/index-tx indexer tx tx-events)
(when (Thread/interrupted)
(throw (InterruptedException.))))
consumed-txs?))]
(when (Thread/interrupted)
(throw (InterruptedException.)))
(when-not consumed-txs?
(Thread/sleep (.toMillis poll-sleep-duration)))))
(catch InterruptedException e)
(catch Exception e
(reset! !error e)
(log/error e "Error consuming transactions")))
(log/info "Shut down tx-consumer"))
(defrecord TxConsumer [^Thread executor-thread, !error]
db/TxConsumer
(consumer-error [_] @!error)
Closeable
(close [_]
(.interrupt executor-thread)
(.join executor-thread)))
(def tx-consumer
{:start-fn (fn [deps args]
(let [!error (atom nil)]
(->TxConsumer (doto (Thread. #(index-tx-log (assoc deps :!error !error) args))
(.setName "crux-tx-consumer")
(.start))
!error)))
:deps [::n/indexer ::n/document-store ::n/tx-log]
:args {::poll-sleep-duration {:default (Duration/ofMillis 100)
:doc "How long to sleep between polling for new transactions"
:crux.config/type :crux.config/duration}}})