/
plugin.clj
123 lines (110 loc) · 4.27 KB
/
plugin.clj
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
(ns kaocha-retry.plugin
"Plugin to retry tests multiple times, useful for flakey tests."
(:require [clojure.test :as te]
[kaocha.hierarchy :as h]
[kaocha.plugin :refer [defplugin]]
[kaocha.testable :as testable]))
(def default-max-retries 3)
(def default-wait-time 100)
(def current-retries (atom 0))
(defn- with-capture-report [t to-report]
(with-redefs [te/report (fn [& args]
(swap! to-report concat args))]
(try
(t)
[(empty? (filter h/fail-type? @to-report)) nil]
(catch Exception e
[false e]))))
(defn run-with-retry [t max-retries wait-time]
(fn []
(loop [attempts 0]
(reset! current-retries attempts)
(let [to-report (atom [])
[passed? exc] (with-capture-report t to-report)
report #(doseq [tr @to-report]
(te/report tr))]
(if passed?
(do (report) true)
(if (= attempts max-retries)
(do (report)
;; just rethrow the exception
(when (some? exc)
(throw exc)))
(do
(Thread/sleep wait-time)
(recur (inc attempts)))))))))
(defn format-retries [retried max-retries]
(let [[success fails]
(->> retried
(sort-by second)
(split-with #(< (second %) max-retries)))]
(when (seq fails)
(println (format "* Tests failed even after %s retries" max-retries))
(doseq [[t-id] fails]
(println (format "- %s" t-id))))
(when (seq success)
(println "* Tests succeeded after retrying `n` times")
(doseq [[t-id retries] success]
(println (format "- %s: %s" t-id retries))))))
(defn should-retry? [test-plan testable]
(let [test-name (str (:kaocha.var/name testable))
test-regexps (:kaocha-retry.plugin/retrying-tests-regexes test-plan)]
(boolean
(and (::retry? test-plan)
(h/leaf? testable)
(or (empty? test-regexps)
(some some?
(map #(re-find (re-pattern %) test-name)
test-regexps)))))))
(defplugin kaocha-retry.plugin/retry
(cli-options [opts]
(conj opts
[nil "--[no-]retry" "Retry tests"]
[nil "--max-retries NUM" "Number of times to retry the tests"
:parse-fn #(Integer/parseInt %)]
[nil "--retry-interval INTERVAL" "How many milliseconds to wait before retrying"
:parse-fn #(Integer/parseInt %)]))
(config [config]
(let [retry? (get-in config
[:kaocha/cli-options :retry]
(if (contains? config ::retry?)
(::retry? config)
true))
max-retries (get-in config
[:kaocha/cli-options :max-retries]
(or (::max-retries config)
default-max-retries))
retry-interval (get-in config
[:kaocha/cli-options :retry-interval]
(or (::retry-interval config)
default-wait-time))]
(assoc config
::retry? retry?
::max-retries max-retries
::retry-interval retry-interval)))
;; can I get the retry? config from the config to each testable??
(pre-test [testable test-plan]
(reset! current-retries 0)
;; these two are not actually being fetched correctly
(if (should-retry? test-plan testable)
(update testable
:kaocha.testable/wrap
conj
(fn [t]
(run-with-retry t
(::max-retries test-plan)
(::retry-interval test-plan))))
testable))
(post-test [testable test-plan]
(cond-> testable
(pos? @current-retries) (assoc ::retries @current-retries)))
(post-summary [test-result]
(let [retried
(for [t (testable/test-seq test-result)
:let [retries (::retries t)]
:when (and retries
(h/leaf? t)
(pos? retries))]
[(:kaocha.testable/id t) retries])]
(format-retries retried (::max-retries test-result))
test-result)))