|
1 | 1 | (ns mysql-queue.core
|
2 | 2 | "A MySQL-backed durable queue implementation with scheduled jobs support."
|
3 | 3 | (:require [mysql-queue.queries :as queries]
|
4 |
| - [mysql-queue.utils :refer [while-let fn-options with-error-handler profile-block meter ns->ms]] |
| 4 | + [mysql-queue.utils :refer [while-let fn-options with-error-handler profile-block meter ns->ms numeric-stats]] |
5 | 5 | [clojure.string :as string]
|
6 | 6 | [clojure.set :as clj-set]
|
7 | 7 | [clojure.edn :as edn]
|
|
21 | 21 | (stop [worker timeout-secs]))
|
22 | 22 |
|
23 | 23 | (defrecord Worker
|
24 |
| - [db-conn input-chan status sieve status-threads consumer-threads scheduler-thread recovery-thread options] |
| 24 | + [db-conn fn-bindings input-chan status sieve status-threads consumer-threads scheduler-thread recovery-thread options] |
25 | 25 | Stoppable
|
26 | 26 | (stop [this timeout-secs]
|
27 | 27 | (when (:running @status)
|
|
58 | 58 | (clojure.core/name name)
|
59 | 59 | (clojure.core/name status)
|
60 | 60 | (pr-str parameters)
|
61 |
| - attempt]) |
| 61 | + attempt |
| 62 | + (Date.)]) |
62 | 63 | Persistent
|
63 | 64 | (persist [this conn]
|
64 | 65 | (if id
|
|
161 | 162 | (cleanup job db-conn)
|
162 | 163 | (try
|
163 | 164 | (log-fn :info job "Executing job " job)
|
164 |
| - (let [[status params] (-> (meter m :job-fn (job-fn status parameters)) |
| 165 | + (let [[status params] (-> (meter m :user (job-fn status parameters)) |
165 | 166 | job-result-or-nil (or [:done nil]))]
|
166 | 167 | (-> job (beget status params) (persist db-conn)))
|
167 | 168 | (catch Exception e
|
|
183 | 184 | (log-fn :info job "Executing job " job)
|
184 | 185 | (-> job beget (persist db-conn)))))
|
185 | 186 |
|
| 187 | +(defn- bound-job-names |
| 188 | + "Takes a map of job bindings and returns a sequence of names as strings." |
| 189 | + [fn-bindings] |
| 190 | + (map name (keys fn-bindings))) |
| 191 | + |
186 | 192 | (defn- get-scheduled-jobs
|
187 | 193 | "Searches for ready scheduled jobs and attempts to insert root jobs for each of those.
|
188 | 194 | Returns the number of jobs added, or false if channel was closed."
|
189 | 195 | [db-conn n fn-bindings sieve]
|
190 |
| - (->> (queries/select-n-ready-scheduled-jobs db-conn (map name (keys fn-bindings)) sieve n) |
| 196 | + (->> (queries/select-n-ready-scheduled-jobs db-conn (bound-job-names fn-bindings) sieve n) |
191 | 197 | (map #(scheduled-job % fn-bindings))))
|
192 | 198 |
|
193 | 199 | (defn- get-stuck-jobs
|
|
197 | 203 | [db-conn n fn-bindings threshold sieve]
|
198 | 204 | (->> (queries/select-n-stuck-jobs db-conn
|
199 | 205 | (map name ultimate-job-states)
|
200 |
| - (map name (keys fn-bindings)) |
| 206 | + (bound-job-names fn-bindings) |
201 | 207 | sieve
|
202 | 208 | threshold
|
203 | 209 | n)
|
|
246 | 252 | (try
|
247 | 253 | (log-fn :debug job "Consumer received job " job)
|
248 | 254 | (loop [current-job job
|
| 255 | + last-job job |
249 | 256 | metrics {}]
|
250 | 257 | (if current-job
|
251 | 258 | (do
|
252 | 259 | (>! status-chan {:id id :state :running-job :job current-job})
|
253 | 260 | (let [[next-job dmetrics] (<! (thread (execute current-job db-conn log-fn err-fn)))]
|
254 |
| - (recur next-job (merge-with + metrics dmetrics)))) |
| 261 | + (recur next-job current-job (merge-with + metrics dmetrics)))) |
255 | 262 | (do
|
256 |
| - (>! status-chan {:id id :state :finished-job :job current-job :metrics metrics}) |
257 |
| - (log-fn :info current-job "Completed job " job " in " (ns->ms (:full metrics)) "ms") |
| 263 | + (>! status-chan {:id id :state :finished-job :job last-job :metrics metrics}) |
| 264 | + (log-fn :info last-job "Completed job " last-job " in " (ns->ms (:full metrics)) "ms") |
258 | 265 | nil)))
|
259 | 266 | (catch Exception e
|
260 | 267 | (>! status-chan {:id id :state :error :job job})
|
|
350 | 357 | (close! ch)))
|
351 | 358 | (go-loop [occupations {}]
|
352 | 359 | (if-let [v (<! ch)]
|
353 |
| - (let [[v ch] (alts! (map #(vector %1 v) out-chs))] |
| 360 | + (let [[ret ch] (alts! (map #(vector %1 v) out-chs))] |
354 | 361 | (when-let [occupation (occupations ch)]
|
355 | 362 | (swap! sieve disj occupation))
|
356 |
| - (recur (assoc occupations ch v))) |
| 363 | + (when ret |
| 364 | + (recur (assoc occupations ch v)))) |
357 | 365 | (doseq [out-ch out-chs]
|
358 | 366 | (close! out-ch))))
|
359 | 367 | [in-ch out-chs sieve]))
|
|
403 | 411 | (persist db-conn)
|
404 | 412 | :id))
|
405 | 413 |
|
| 414 | +(defn status |
| 415 | + "Returns a map describing the current status of the worker." |
| 416 | + [{:keys [db-conn fn-bindings] |
| 417 | + {recovery-threshold :recovery-threshold-mins} :options |
| 418 | + :as worker}] |
| 419 | + (let [serialize-job (fn [job] |
| 420 | + (and job |
| 421 | + (select-keys job [:id :name :status :metrics :processed-at]))) |
| 422 | + sieve @(:sieve worker) |
| 423 | + {:keys [consumers] :as raw-status} @(:status worker) |
| 424 | + recent-jobs (mapcat :recent-jobs (:consumers raw-status)) |
| 425 | + scheduled-status (queries/select-scheduled-jobs-status db-conn (bound-job-names fn-bindings)) |
| 426 | + in-progress-status (queries/select-jobs-status |
| 427 | + db-conn |
| 428 | + (bound-job-names fn-bindings) |
| 429 | + ultimate-job-states |
| 430 | + recovery-threshold) |
| 431 | + consumers (map #(-> % |
| 432 | + (dissoc :recent-jobs) |
| 433 | + (update :last-job serialize-job)) |
| 434 | + consumers)] |
| 435 | + (-> raw-status |
| 436 | + (assoc :db-queue {:scheduled-jobs scheduled-status :jobs in-progress-status}) |
| 437 | + (assoc :prefetched-jobs (map serialize-job sieve)) |
| 438 | + (assoc :recent-jobs-stats |
| 439 | + {:job-types (frequencies (map :name recent-jobs)) |
| 440 | + :performance {:user (numeric-stats (keep #(get-in % [:metrics :user]) recent-jobs)) |
| 441 | + :full (numeric-stats (keep #(get-in % [:metrics :full]) recent-jobs))}}) |
| 442 | + (assoc :recent-jobs (map serialize-job recent-jobs)) |
| 443 | + (assoc :consumers consumers)))) |
| 444 | + |
406 | 445 | (defn worker
|
407 | 446 | "Creates a new worker. Takes a database connection db-conn,
|
408 | 447 | a map of fn-bindings binding job names to job functions, and a number
|
|
412 | 451 | when the publisher will block. Default 10.
|
413 | 452 | * prefetch - the number of jobs a publisher fetches from the database at once.
|
414 | 453 | Default 10.
|
| 454 | + * num-stats-jobs - the number of jobs to keep in memory for statistical purpose. |
| 455 | + Per consumer thread. Default 50. |
415 | 456 | * num-consumer-threads - the number of concurrent threads that run jobs at the
|
416 | 457 | same time.
|
417 | 458 | * min-scheduler-sleep-interval - the minimum time in seconds the scheduler will sleep
|
|
505 | 546 | log-fn)]
|
506 | 547 | (log-fn :info nil "Starting a new worker...")
|
507 | 548 | (->Worker db-conn
|
| 549 | + fn-bindings |
508 | 550 | in-ch
|
509 | 551 | status
|
510 | 552 | sieve
|
|
0 commit comments