Permalink
Browse files

Improve failure reporting with failing result data and location.

This addresses TCHECK-34 ("reporting is uniformative").

This adds a clojure.test/assert-expr which parses the check results
and emits appropriate messages via clojure.test/do-report. For failed
tests, the arguments to clojure-test/assert-check are returned as the
:actual result, with a placeholder {:result true} as the :expected result.

The file and location provided in ClojureScript environments is not
particularly useful as it reports the file and line location in the
relevant JavaScript file rather than of the source ClojureScript file.

Change-Id: I372c307115d0cc1d62d7460eb253031cd269f750
  • Loading branch information...
grzm authored and Gary Fredericks committed Jul 6, 2017
1 parent 05a5360 commit 05ad8701def0e283eacadd50c89aedf18f91115c
@@ -11,6 +11,7 @@
(:require #?(:clj [clojure.test :as ct]
:cljs [cljs.test :as ct :include-macros true])
[clojure.test.check :as tc]
[clojure.test.check.clojure-test.assertions]
[clojure.test.check.impl :refer [get-current-time-millis
exception-like?]]
[clojure.test.check.results :as results]))
@@ -21,7 +22,7 @@
(if (and (not (results/passing? result))
(exception-like? (:clojure.test.check.properties/error result-data)))
(throw (:clojure.test.check.properties/error result-data))
(ct/is result)))
(ct/is (clojure.test.check.clojure-test/check? m))))
(def ^:dynamic *default-test-count* 100)
@@ -0,0 +1,48 @@
(ns clojure.test.check.clojure-test.assertions
#?(:cljs (:require-macros [clojure.test.check.clojure-test.assertions.cljs]))
(:require [clojure.string :as str]
#?(:clj [clojure.test :as t]
:cljs [cljs.test :as t])
[clojure.test.check.results :as results]))
#?(:clj
(defn test-context-stacktrace [st]
(drop-while
#(let [class-name (.getClassName ^StackTraceElement %)]
(or (clojure.string/starts-with? class-name "java.lang")
(clojure.string/starts-with? class-name "clojure.test$")
(clojure.string/starts-with? class-name "clojure.test.check.clojure_test$")
(clojure.string/starts-with? class-name "clojure.test.check.clojure_test.assertions")))
st)))
#?(:clj
(defn file-and-line*
[stacktrace]
(if (seq stacktrace)
(let [^StackTraceElement s (first stacktrace)]
{:file (.getFileName s) :line (.getLineNumber s)})
{:file nil :line nil})))
(defn check-results [{:keys [result] :as m}]
(if (results/passing? result)
(t/do-report
{:type :pass
:message (dissoc m :result)})
(t/do-report
(merge {:type :fail
:expected {:result true}
:actual m}
#?(:clj (file-and-line*
(test-context-stacktrace (.getStackTrace (Thread/currentThread))))
:cljs (t/file-and-line (js/Error.) 4))))))
(defn check?
[_ form]
`(let [m# ~(nth form 1)]
(check-results m#)))
#?(:clj
(defmethod t/assert-expr 'clojure.test.check.clojure-test/check?
[_ form]
(check? _ form)))
@@ -0,0 +1,8 @@
(ns clojure.test.check.clojure-test.assertions.cljs
(:require
[cljs.test]
[clojure.test.check.clojure-test.assertions :as assertions]))
(defmethod cljs.test/assert-expr 'clojure.test.check.clojure-test/check?
[_ msg form]
(assertions/check? msg form))
@@ -8,7 +8,9 @@
; You must not remove this notice, or any other, from this software.
(ns clojure.test.check.clojure-test-test
(:require #?@(:cljs
(:require [clojure.set :as set]
[clojure.string :as str]
#?@(:cljs
[[cljs.test
:as test
:include-macros true
@@ -138,14 +140,34 @@
(defspec this-is-supposed-to-fail 100 vector-elements-are-unique)
(deftest can-report-failures
(let [{:keys [test-out]} (capture-test-var #'this-is-supposed-to-fail)
[result-line expected-line actual-line & more] (->> (str/split-lines test-out)
;; skip any ::shrunk messages
(drop-while #(not (re-find #"^FAIL" %))))]
(is (re-find #"^FAIL in \(this-is-supposed-to-fail\) " result-line))
#?(:clj (is (re-find #"\(clojure_test_test\.cljc:\d+\)$" result-line)))
(is (= expected-line "expected: {:result true}"))
(let [actual (read-string (subs actual-line 10))]
(is (set/subset? #{:result :result-data :seed :failing-size :num-tests :fail :shrunk}
(set (keys actual))))
(is (= false (:result actual))))
(is (nil? more))))
(deftest can-report-shrinking
(binding [ct/*report-shrinking* true]
(testing "don't emit Shrinking messages by default"
(let [{:keys [report-counters test-out]} (capture-test-var #'this-is-supposed-to-fail)]
(is (== 1 (:fail report-counters)))
(is (re-seq #?(:clj (java.util.regex.Pattern/compile
"(?s)Shrinking vector-elements-are-unique starting with parameters \\[\\[.+")
:cljs #"Shrinking vector-elements-are-unique starting with parameters \[\[[\s\S]+")
test-out)))))
(is (not (re-find #"Shrinking" test-out)))))
(testing "bind *report-shrinking* to true to emit Shrinking messages"
(binding [ct/*report-shrinking* true]
(let [{:keys [report-counters test-out]} (capture-test-var #'this-is-supposed-to-fail)]
(is (== 1 (:fail report-counters)))
(is (re-seq #?(:clj (java.util.regex.Pattern/compile
"(?s)Shrinking vector-elements-are-unique starting with parameters \\[\\[.+")
:cljs #"Shrinking vector-elements-are-unique starting with parameters \[\[[\s\S]+")
test-out))))))
(deftest tcheck-118-pass-shrunk-input-on-to-clojure-test
(let [{trial ::ct/trial, shrinking ::ct/shrinking, shrunk ::ct/shrunk}

0 comments on commit 05ad870

Please sign in to comment.