/
test.clj
190 lines (159 loc) · 5.6 KB
/
test.clj
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
125
126
127
128
129
130
131
132
133
134
135
136
137
138
139
140
141
142
143
144
145
146
147
148
149
150
151
152
153
154
155
156
157
158
159
160
161
162
163
164
165
166
167
168
169
170
171
172
173
174
175
176
177
178
179
180
181
182
183
184
185
186
187
188
189
190
(ns greenlight.test
"A _test_ is a collection of steps which are run in sequence to exercise a
specific usage scenario."
(:require
[clojure.spec.alpha :as s]
[greenlight.step :as step])
(:import
java.time.Instant
java.time.temporal.ChronoUnit))
;; ## Test Configuration
;; Namespace where the test is defined.
(s/def ::ns symbol?)
;; Source line where the test is defined.
(s/def ::line integer?)
;; Title of the test run.
(s/def ::title string?)
;; Human-friendly description of the scenario the test covers.
(s/def ::description string?)
;; Test execution group tag. Tests within the same group
;; are executed in serial.
(s/def ::group keyword?)
;; Sequence of steps to take for this test.
(s/def ::steps
(s/coll-of ::step/config
:kind vector?
:min-count 1))
;; Initial and final context map for the test.
(s/def ::context map?)
;; The test case map defines metadata about the test and its steps.
(s/def ::case
(s/keys :req [::title
::steps]
:opt [::ns
::line
::description
::context]))
;; Collection of test cases.
(s/def ::suite
(s/coll-of ::case
:kind? vector?
:min-count 1))
(defn- contains-ns?
"Returns true if at least one key is in the namespace provided."
[m ns]
(some #(= ns (namespace (key %))) m))
(defn- attr-map?
"Returns true if the map contains at least one key in the
greenlight.test namespace."
[o]
(and (map? o) (contains-ns? o "greenlight.test")))
(defmacro deftest
"Defines a new integration test. In the first position, the value can
either be an optional docstring or an optional test configuration
map. An integration test is a collection of individual steps or an
arbitrarily nested sequential collection of steps."
[test-sym & body]
(let [docstring (when (string? (first body))
(first body))
body (if (string? (first body))
(rest body)
body)
attr-map (when (attr-map? (first body))
(first body))
steps (if (attr-map? (first body))
(rest body)
body)
base (cond-> {}
docstring (assoc ::description docstring)
attr-map (merge attr-map))]
`(defn ~(vary-meta test-sym assoc ::test true)
[]
(assoc ~base
::title ~(str test-sym)
::ns '~(symbol (str *ns*))
::line ~(:line (meta &form))
::steps (vec (flatten (list ~@steps)))))))
;; ## Test Results
;; Final outcome of the test case.
(s/def ::outcome ::step/outcome)
;; When the test run started.
(s/def ::started-at inst?)
;; When the test run ended.
(s/def ::ended-at inst?)
(defn elapsed
"Calculates the elapsed time a test took. Returns the duration in fractional
seconds, or 0.0 if started-at or ended-at is missing."
[result]
(let [started-at (::started-at result)
ended-at (::ended-at result)]
(if (and started-at ended-at)
(/ (.between ChronoUnit/MILLIS started-at ended-at) 1e3)
0.0)))
;; ## Test Execution
(defn ^:dynamic *report*
"Dynamic reporting function which is called at various points in the test
execution. The event data should be a map containing at least a `:type` key."
[event]
; Default no-op action.
nil)
; TODO: between steps, write out current state to a local file?
(defn- run-steps!
"Executes a sequence of test steps by running them in order until one fails.
Returns a tuple with the enriched vector of steps run and the final context
map."
[system ctx steps]
(loop [history []
ctx ctx
steps steps]
(if-let [step (first steps)]
; Run next step to advance the test.
(let [step (step/initialize step ctx)
_ (*report* {:type :step-start
:step step})
[step' ctx'] (step/advance! system step ctx)
history' (conj history step')]
(*report* {:type :step-end
:step step'})
; Continue while steps pass.
(if (= :pass (::step/outcome step'))
(recur history' ctx' (next steps))
[(vec (concat history' (rest steps))) ctx']))
; No more steps.
[history ctx])))
(defn- run-cleanup!
"Clean up after a test run by cleaning up all the reported resources in
reverse order."
[system history]
(doseq [step (reverse history)]
(when-let [cleanups (seq (::step/cleanup step))]
(doseq [[resource-type parameters] (reverse cleanups)]
(try
(*report* {:type :cleanup-resource
:resource-type resource-type
:parameters parameters})
(step/clean! system resource-type parameters)
(catch Exception ex
(*report* {:type :cleanup-error
:resource-type resource-type
:parameters parameters
:error ex})))))))
(defn run-test!
"Execute a test. Returns the updated test map."
[system test-case]
(*report* {:type :test-start
:test test-case})
(let [started-at (Instant/now)
ctx (::context test-case {})
[history ctx] (run-steps! system ctx (::steps test-case))
_ (run-cleanup! system history)
ended-at (Instant/now)
test-case (assoc test-case
::steps history
::context ctx
::outcome (last (keep ::step/outcome history))
::started-at started-at
::ended-at ended-at)]
(*report* {:type :test-end
:test test-case})
test-case))