Skip to content

HTTPS clone URL

Subversion checkout URL

You can clone with HTTPS or Subversion.

Download ZIP
Browse files

begin overhaul

  • Loading branch information...
commit 992d3011502d7f6ff199dcfe1acf35bac77d3a11 1 parent 88e40f3
@stuarthalloway stuarthalloway authored
View
24 README.md
@@ -5,7 +5,6 @@ Test data generation and execution harness. Very early days.
This API will change. You have been warned.
-
Releases and Dependency Information
========================================
@@ -28,7 +27,6 @@ Latest stable release: 0.1.4
</dependency>
-
Example Usages
========================================
@@ -40,18 +38,13 @@ and a validator:
[^long a ^long b] ;; input spec
(assert (integer? %))) ;; 0 or more validator forms
-Given a var, namespace, or directory, you can run the tests for it:
-
- (test-vars #'integers-closed-over-addition)
- (test-namespaces 'clojure.test.generative-test)
- (test-dirs \"src/test/clojure\")
-
-Succesful test output includes :iterations, :msec, and the :var for
-each test run:
+To generate test data, see the fns in the generators namespace.
- {:iterations 44645, :msec 1429,
- :var #'clojure.test.generative-test/numbers-closed-over-addition}
+To integrate with clojure.test:
+ ;; somewhere in your test suite
+ (:require '[clojure.test.generative.clojure-test :as clojure-test])
+ (clojure-test/run-generative-tests)
Developer Information
========================================
@@ -69,6 +62,13 @@ Developer Information
Change Log
====================
+* Release 0.1.5 (in development)
+ * Can now run tests under clojure.test
+ * Tests produce data events that can be consumed by arbitrary reporting tools
+ * Example reporting integration with logback.
+ * Added `is` macro with more detailed reporting than `clojure.test/is`
+ * Removed collection based input generators. Input generators must be fns.
+ * Removed duplicate input check. Tests can be called multiple times with same input.
* Release 0.1.4 on 2012.01.03
* Initial version
View
0  script/.gitignore → bin/.gitignore
File renamed without changes
View
7 bin/repl
@@ -0,0 +1,7 @@
+#!/bin/bash
+# Note: First you must run mvn dependency:build-classpath -Dmdep.outputFile=bin/maven-classpath
+CLASSPATH=src/main/clojure:src/test/clojure:src/examples/clojure:`cat bin/maven-classpath`
+
+java -server -Xmx2GB -cp $CLASSPATH clojure.main "$@"
+
+
View
16 pom.xml
@@ -51,20 +51,21 @@
</repositories>
<properties>
- <clojure.version>1.3.0</clojure.version>
+ <clojure.version>1.4.0</clojure.version>
</properties>
<dependencies>
<dependency>
<groupId>org.clojure</groupId>
- <artifactId>clojure</artifactId>
- <version>1.3.0-beta1</version>
- </dependency>
- <dependency>
- <groupId>org.clojure</groupId>
<artifactId>tools.namespace</artifactId>
<version>0.1.1</version>
</dependency>
+ <dependency>
+ <groupId>ch.qos.logback</groupId>
+ <artifactId>logback-classic</artifactId>
+ <version>1.0.6</version>
+ <scope>provided</scope>
+ </dependency>
</dependencies>
<build>
@@ -94,7 +95,8 @@
<artifactId>clojure-maven-plugin</artifactId>
<version>1.3.7</version>
<configuration>
- <testScript>script/test_runner.clj</testScript>
+ <vmargs>-Dclojure.test.generative.runner=clojure.test</vmargs>
+ <!-- <testScript>bin/test_runner.clj</testScript> -->
<copiedNamespaces>
<namespace>!.*</namespace>
</copiedNamespaces>
View
7 script/examples
@@ -1,7 +0,0 @@
-#!/bin/sh
-# Note: First you must run mvn dependency:build-classpath -Dmdep.outputFile=script/maven-classpath
-CLASSPATH=src/main/clojure:src/test/clojure:src/examples/clojure:`cat script/maven-classpath`
-
-java -server -Xmx2G -cp $CLASSPATH clojure.main -i script/examples.clj
-
-
View
14 script/examples.clj
@@ -1,14 +0,0 @@
-(use '[clojure.test.generative])
-(try
- (println "Testing on" *cores* "cores for" *msec* "msec.")
- (let [futures (test-dirs "src/examples/clojure")]
- (doseq [f futures]
- @f))
- (catch Throwable t
- (.printStackTrace t)
- (System/exit -1))
- (finally
- (shutdown-agents)))
-(System/exit 0)
-
-
View
13 script/repl
@@ -1,13 +0,0 @@
-#!/bin/bash
-# Note: First you must run mvn dependency:build-classpath -Dmdep.outputFile=script/maven-classpath
-CLASSPATH=src/main/clojure:src/test/clojure:src/examples/clojure:`cat script/maven-classpath`
-
-USER_SCRIPT=$1
-shift
-if [ "$USER_SCRIPT" != "" ]; then
- USER_SCRIPT="-i $USER_SCRIPT -r "
-fi
-
-java -server -Xmx2GB -cp $CLASSPATH clojure.main $USER_SCRIPT
-
-
View
14 script/test_runner.clj
@@ -1,14 +0,0 @@
-(use '[clojure.test.generative])
-(try
- (println "Testing on" *cores* "cores for" *msec* "msec.")
- (let [futures (test-dirs "src/main/clojure" "src/test/clojure")]
- (doseq [f futures]
- @f))
- (catch Throwable t
- (.printStackTrace t)
- (System/exit -1))
- (finally
- (shutdown-agents)))
-(System/exit 0)
-
-
View
83 src/examples/clojure/clojure/test/array_test.clj
@@ -0,0 +1,83 @@
+; Copyright (c) Rich Hickey, Stuart Halloway, and contributors.
+; All rights reserved.
+; The use and distribution terms for this software are covered by the
+; Eclipse Public License 1.0 (http://opensource.org/licenses/eclipse-1.0.php)
+; which can be found in the file epl-v10.html at the root of this distribution.
+; By using this software in any fashion, you are agreeing to be bound by
+; the terms of this license.
+; You must not remove this notice, or any other, from this software.
+
+(ns clojure.test.array-test
+ (:use clojure.test.generative)
+ (:require [clojure.test.generative.generators :as gen]
+ [clojure.test.generative.clojure-test :as clojure-test]))
+
+(clojure-test/run-generative-tests)
+
+(defn nan-or-=
+ [a b]
+ (or (= a b)
+ (and (Double/isNaN a)
+ (Double/isNaN b))))
+
+(defmacro array-access-specs
+ [pdescs]
+ `(do
+ ~@(map
+ (fn [[prim comp]]
+ `(defspec ~(symbol (str prim "-array-access"))
+ vec
+ [~(with-meta 'arr {:tag (list (symbol (str prim "-array")) prim)})]
+ (assert (= (alength ~'arr) (count ~'%)))
+ (dotimes [~'i (count ~'arr)]
+ (assert (~comp (aget ~'arr ~'i) (get ~'% ~'i))))))
+ pdescs)))
+
+(defmacro copy-array-fns
+ [& types]
+ `(do
+ ~@(map
+ (fn [t]
+ `(defn ~(symbol (str "copy-" t "-array"))
+ [~'a]
+ (let [~'copy ~(list (symbol (str t "-array")) 'a)]
+ (dotimes [~'i (alength ~'a)]
+ (aset ~'copy ~'i ~(with-meta `(aget ~'a ~'i)
+ {:tag (symbol (str t "s"))})))
+ ~'copy)))
+ types)))
+
+(copy-array-fns boolean byte char short int long float double object)
+
+(defmacro aset-specs
+ [pdescs]
+ `(do
+ ~@(map
+ (fn [[prim comp]]
+ `(defspec ~(symbol (str prim "-aset-spec"))
+ copy-object-array
+ [~(with-meta 'arr {:tag (list (symbol (str prim "-array")) prim)})]
+ (assert (= (alength ~'arr) (alength ~'%)))
+ (dotimes [~'i (count ~'arr)]
+ (assert (~comp (aget ~'arr ~'i) (aget ~'% ~'i))))))
+ pdescs)))
+
+(array-access-specs {int =
+ long =
+ float nan-or-=
+ double nan-or-=
+ short =
+ byte =
+ char =
+ boolean =})
+
+(aset-specs {int =
+ long =
+ float nan-or-=
+ double nan-or-=
+ short =
+ byte =
+ char =
+ boolean =})
+
+
View
29 src/examples/clojure/clojure/test/java_test.clj
@@ -0,0 +1,29 @@
+; Copyright (c) Rich Hickey, Stuart Halloway, and contributors.
+; All rights reserved.
+; The use and distribution terms for this software are covered by the
+; Eclipse Public License 1.0 (http://opensource.org/licenses/eclipse-1.0.php)
+; which can be found in the file epl-v10.html at the root of this distribution.
+; By using this software in any fashion, you are agreeing to be bound by
+; the terms of this license.
+; You must not remove this notice, or any other, from this software.
+
+(ns clojure.test.java-test
+ (:use clojure.test.generative))
+
+(set! *warn-on-reflection* true)
+
+(defspec entry-set-keys-always-have-values
+ #(java.util.concurrent.ConcurrentHashMap. ^java.util.Map %)
+ [^{:tag (hash-map #(uniform 0 1e7) long #(uniform 1 20000))} m]
+ (let [ks (keys m)]
+ (dotimes [_ 5]
+ (future
+ (doseq [k (shuffle ks)]
+ (.remove ^java.util.Map % k)
+ (Thread/sleep 1)))))
+ (loop [it (-> ^java.util.Map % (.entrySet) (.iterator))]
+ (Thread/sleep 1)
+ (when (.hasNext it)
+ (let [[k v] (.next it)]
+ (when (nil? v) (throw (RuntimeException. "Nil value for " k)))
+ (recur it)))))
View
13 src/examples/clojure/clojure/test/math_test.clj
@@ -78,9 +78,10 @@
(defspec quotient-and-remainder
(fn [a b] (sort [a b]))
[^{:tag `integer} a ^{:tag `integer} b]
- (let [[a d] %
- q (quot a d)
- r (rem a d)]
- (assert (= a
- (+ (* q d) r)
- (unchecked-add (unchecked-multiply q d) r)))))
+ (when-not (zero? (second %))
+ (let [[a d] %
+ q (quot a d)
+ r (rem a d)]
+ (assert (= a
+ (+ (* q d) r)
+ (unchecked-add (unchecked-multiply q d) r))))))
View
377 src/main/clojure/clojure/test/generative.clj
@@ -7,193 +7,10 @@
; the terms of this license.
; You must not remove this notice, or any other, from this software.
-(ns
- ^{:author "Stuart Halloway"
- :doc "Generative testing.
-
-A defspec consists of a name, a function to be tested, an input spec,
-and a validator:
-
- (defspec integers-closed-over-addition
- (fn [a b] (+' a b)) ;; input fn
- [^long a ^long b] ;; input spec
- (assert (integer? %))) ;; 0 or more validator forms
-
-Given a var, namespace, or directory, you can run the tests for it:
-
- (test-vars #'integers-closed-over-addition)
- (test-namespaces 'clojure.test.generative-test)
- (test-dirs \"src/test/clojure\")
-
-Succesful test output includes :iterations, :msec, and the :var for
-each test run:
-
- {:iterations 44645, :msec 1429,
- :var #'clojure.test.generative-test/numbers-closed-over-addition}
-
-By default, an entire test run lasts for 10 seconds, but you can change
-the test run duration by binding *msec*:
-
- ;; go get a snack...
- (binding [*msec* 1000000]
- (test-dirs \"src/test/clojure\"))
-
-The test run will attempt to use all your cores. You can change this
-by binding *cores* to a different number.
-
- ;; no snack needed, your other N cores still available...
- (binding [*msec* 1000000
- *cores* 1]
- (test-dirs \"src/test/clojure\"))
-
-If you want to see the generated inputs, bind *verbose* to true:
-
- (binding [*msec* 1000
- *verbose* true]
- (test-vars #'integers-closed-over-addition))
-
- {:inputs (2 -7312771435595740594)}
- {:inputs (2 6571825658936471225)}
- {:inputs (-5298357432219421459 1)}
- ;; etc.
-
-If you want to generate test data without running any tests, you can
-call generate-test-data.
-
- (take 5 (generate-test-data '[bool (uniform 0 10)]))
- => ((false 6) (true 2) (false 9) (true 2) (true 7))
-
-Common scalar generators:
-
- bool byte long printable-ascii-char string
- symbol keyword
- scalar (chooses scalar type at random)
-
-You can generate collections with a spec of the form
- (coll-type item-type):
-
- (take 5 (generate-test-data '[(vec bool)]))
- => (([true false true true])
- ([true false true false false false])
- ([true false true true true true false true true])
- ([false true false true false])
- ([true false true false false false false false]))
-
-Common collection generators:
-
- vec set hash-map list
- tuple (fixed size vector)
- collection (chooses collection type at random)
-
-You should assume that the generators use a probability distribution
-that is out of a specific test's control. If you prefer, you can
-choose an explicit distribution. (e.g. uniform or geometric).
-
- (take 5 (generate-test-data '[(geometric 0.2)]))
- => ((1) (9) (11) (17) (3))
-
-Generators that create collections (or strings etc.) take an
-optional sizer parameter. Sizers can be explicit numbers, or fns
-that return sizes per some distribution:
-
- (take 3 (generate-test-data '[(vec bool 3)]))
- => (([false false true])
- ([true false false])
- ([false false false]))
-
-When a test fails, output includes the failed form, the iteration,
-the error message, and the random seed.
-
- {:form (user/all-numbers-are-positive -6169532649852302182),
- :iteration 1,
- :seed 42,
- :error \"Assert failed: (pos? l)\"}
-
-Note the :seed value in the error output above. The *rnd* and *seed*
-values live in the clojure.test.generative.generators namespace, and
-can be used to conrol the randomization used to generate test data.
-This can be useful in generating reproducible inputs.
-
-The test runner tracks old inputs, and tries not to submit the same
-input to the same fn twice in a single test run. If the test data
-generator cannot generate enough unique values to drive the test for
-the expected msec duration, it will stop when it runs out of values,
-and mark the test run as :exhausted.
-
-You can write your own generators, either fns or collections.
-
- (def num-piggies [1 2 3])
- (defn houses [] (gen/rand-nth [\"straw\" \"sticks\" \"brick\"]))
-
-When you name a spec with a precending backtick (`), the test data
-generator will look for a var in the current namespace, instead of
-one of the built-in generators:
-
- (take 5 (generate-test-data [`piggy-ordinals `houses]))
- => ((\"first\" \"brick\")
- (\"third\" \"brick\")
- (\"third\" \"straw\")
- (\"first\" \"straw\")
- (\"third\" \"sticks\"))
-
-"}
- clojure.test.generative
- (:use [clojure.tools.namespace :only (find-namespaces-in-dir)]
- [clojure.pprint :only (pprint)]
- [clojure.walk :only (prewalk)])
- (:require [clojure.test.generative.generators :as gen]))
-
-(def last-report (agent nil))
-
-(def report-fn
- "Reporting function, defaults to prn.
- reset! val to customize reporting."
- (atom prn))
-
-(defn report
- "Report a result. Thread-safe, unlike prn."
- [result]
- (send-off last-report
- (fn [_]
- (try
- (@report-fn result)
- (catch Exception e (.printStackTrace e)))
- result)))
-
-(defn- deep-take
- "Recursively convert any collections in form to (take n)
- of those collections. Used as a signature function in
- mostly-unique."
- [n form]
- (prewalk
- #(if (coll? %) (take n %) %)
- form))
-
-(defn- mostly-unique
- "Create a mostly unique series of items taken from coll (which
- is often a lazy, infinite sequence).
-
- The 'mostly' arises from keeping track of the *signature* of
- past values instead of the past values themselves. If you use
- identity as a signature function, 'mostly' becomes 'exactly',
- at the memory cost of holding the entire sequence of values.
-
- Returned sequence ends on nil or when retry-limit consecutive
- attempts to produce a novel value fail."
- [coll signature retry-limit]
- (let [next-val
- (fn [[_ coll past]]
- (loop [i 0
- [candidate & more] coll]
- (when (and candidate (< i retry-limit))
- (let [sig (signature candidate)]
- (if-not (contains? past sig)
- [candidate more (conj past sig)]
- (recur (inc i) more))))))]
- (->> (take-while identity
- (iterate next-val [nil coll #{}]))
- (drop 1)
- (map first))))
+(ns clojure.test.generative
+ (:require [clojure.walk :as walk]
+ [clojure.test.generative.event :as event]
+ [clojure.test.generative.runner :as runner]))
(defn- fully-qualified
"Qualify a name used in :tag metadata. Unqualified names are
@@ -212,7 +29,7 @@ one of the built-in generators:
(defn- dequote
"Remove the backquotes used to call out user-namespaced forms."
[form]
- (prewalk
+ (walk/prewalk
#(if (and (sequential? %)
(= 2 (count %))
(= 'quote (first %)))
@@ -220,190 +37,44 @@ one of the built-in generators:
%)
form))
-(defn- infinite
- "Make a data generator infinite, by cycling collections and by
- calling fns repeatedly."
- [f]
- (if (coll? f)
- (cycle f)
- (repeatedly f)))
-
(defn- tag->gen
"Convert tag to source code form for a test data generator."
[arg]
- (let [form (prewalk (fn [s] (if (symbol? s) (fully-qualified s) s)) (dequote arg))]
+ (let [form (walk/prewalk (fn [s] (if (symbol? s) (fully-qualified s) s)) (dequote arg))]
(if (seq? form)
(list 'fn '[] form)
form)))
-(defn generate-test-data
- "Generate infinite sequece of test data based on tags."
- [tags]
- (let [gens (map #(eval (tag->gen %)) tags)]
- (apply map vector
- (map
- infinite
- gens))))
-
-(defn test-data-generator
- "Create a test data generator based on tags."
- [tags]
- (let [gens (seq (map #(eval (tag->gen %)) tags))]
- (if (seq gens)
- (fn []
- (apply map vector
- (map
- infinite
- gens)))
- (constantly nil))))
-
-(defn run-test
- "Tests function f with generator gen for up to msec
- milliseconds. Returns a map of :msec and :iterations completed"
- [& {:keys [fname f gen-inputs msec verbose]}]
- (binding [gen/*rnd* (java.util.Random. gen/*seed*)]
- (let [start (System/currentTimeMillis)
- times-up? (if msec
- (fn [] (> (System/currentTimeMillis) (+ msec start)))
- (constantly true))]
- (loop [count 0
- [args & more] (mostly-unique (gen-inputs) (partial deep-take 8) 100)]
- (if (or (and (nil? args) (< 0 count)) (times-up?))
- (merge {:msec (- (System/currentTimeMillis) start)
- :iterations count}
- (if (nil? args) {:exhausted true} {}))
+(defmacro fail
+ [& args]
+ (with-meta `(event/report-context :type ::fail
+ :level :warn
+ ~@args)
+ (meta &form)))
+
+(defmacro is
+ ([v] (with-meta `(is ~v nil) (meta &form)))
+ ([v msg]
+ `(let [~'actual ~v ~'expected '~v]
+ (if ~'actual
(do
- (try
- (when verbose (report {:inputs args}))
- (apply f args)
- (catch Throwable t
- (let [failed-form `(~fname ~@args)
- failure {:form failed-form
- :iteration count
- :seed gen/*seed*
- :error (.getMessage t)
- :exception t}]
- (report failure))
- (throw t)))
- (recur (inc count) more)))))))
-
-(defn spec?
- "Is var a spec"
- [v]
- (boolean (::gen-inputs (meta v))))
-
-(defn- find-vars-in-namespaces
- [& nses]
- (when nses
- (apply require nses)
- (reduce (fn [v ns] (into v (vals (ns-interns ns)))) [] nses)))
-
-(defn- find-vars-in-dirs
- [& dirs]
- (->> (mapcat #(find-namespaces-in-dir (java.io.File. ^String %)) dirs)
- (apply find-vars-in-namespaces)))
-
-(defn- find-tests-in-vars
- [& vars]
- (filter spec? vars))
-
-(def ^:dynamic
- *msec*
- "Desired duration for a test run. Defaults to 10 seconds."
- 10000)
-
-(def ^:dynamic
- *verbose*
- "Set to true to print test inputs as they are used."
- false)
-
-(def ^:dynamic
- *cores*
- "Number of cores to attempt to utilize in a test run. Defaults
- to the number of processors available."
- (.availableProcessors (Runtime/getRuntime)))
-
-(def ^:dynamic
- *seeds*
- "Random seeds for different cores. (If you bind this to fewer seeds
- than *cores*, you will get less utilization and less testing!)"
- (into [] (range 42 (+ 1024 42))))
-
-(defn- var-name
- [^clojure.lang.Var s]
- (resolve (symbol (str (.getName (.ns s)) "/" (.sym s)))))
-
-(def ^:private var-gen
- (comp ::gen-inputs meta))
-
-(defn- run-test-vars
- [vars]
- (let [futures
- (into
- []
- (map
- (fn [seed]
- (let [nvars (count vars)]
- (future
- (binding [gen/*seed* seed]
- (doseq [^clojure.lang.Var v vars]
- (report (merge {:var (var-name v) :seed seed}
- (run-test :fname (var-name v)
- :f @v
- :msec (/ *msec* nvars)
- :verbose *verbose*
- :gen-inputs (var-gen v))))
- :done)))))
- (take *cores* *seeds*)))]
- (future (doseq [f futures] @f) (report :run-complete))
- futures))
-
-(defn test-vars
- "Run tests for all vars. Returns vector of *cores* futures."
- [& vars]
- (run-test-vars (apply find-tests-in-vars vars)))
-
-(defn test-namespaces
- "Run tests for all vars in nses. Returns vector of *cores* futures."
- [& nses]
- (run-test-vars (apply find-tests-in-vars (apply find-vars-in-namespaces nses))))
-
-(defn test-dirs
- "Run tests for all vars in source code files in dirs.
- Returns vector of *cores* futures."
- [& dirs]
- (run-test-vars (apply find-tests-in-vars (apply find-vars-in-dirs dirs))))
-
-(defmacro retest
- "Reload nses, and then test-namespaces. For REPL use."
- [& nses]
- `(do
- (require :reload-all ~(clojure.core/vec nses))
- (test-namespaces ~@nses)))
+ (event/report :level :debug :type ::pass)
+ ~'actual)
+ ~(with-meta
+ `(fail ~@(when msg `[:message ~msg]))
+ (meta &form))))))
(defmacro defspec
- "Define a spec (a fn) with name name. When you run a spec var
- with any of the test- fns, inputs for fn-to-test will be generated
- based on the :tag metadata on args in argspecs.
- After each iteration, the validator-body will execute, with
- bindings to the vars in argspecs, plus a binding of '%' to the
- result of calling fn-to-test.
-
- Multiple arities in argspecs are not supported."
[name fn-to-test args & validator-body]
(when-let [missing-tags (->> (map #(list % (-> % meta :tag)) args)
(filter (fn [[_ tag]] (nil? tag)))
seq)]
(throw (IllegalArgumentException. (str "Missing tags for " (seq (map first missing-tags)) " in " name))))
`(defn ~(with-meta name (assoc (meta name)
- ::gen-inputs (test-data-generator (map #(-> % meta :tag) args))))
+ ::inputs (into [] (map #(-> % meta :tag tag->gen eval) args))))
~(into [] (map (fn [a#] (with-meta a# (dissoc (meta a#) :tag))) args))
(let [~'% (apply ~fn-to-test ~args)]
~@validator-body)))
-
-
-
-
View
78 src/main/clojure/clojure/test/generative/clojure_test.clj
@@ -0,0 +1,78 @@
+; Copyright (c) Rich Hickey, Stuart Halloway, and contributors.
+; All rights reserved.
+; The use and distribution terms for this software are covered by the
+; Eclipse Public License 1.0 (http://opensource.org/licenses/eclipse-1.0.php)
+; which can be found in the file epl-v10.html at the root of this distribution.
+; By using this software in any fashion, you are agreeing to be bound by
+; the terms of this license.
+; You must not remove this notice, or any other, from this software.
+
+(ns clojure.test.generative.clojure-test
+ (:require [clojure.test.generative.event :as event]
+ [clojure.test.generative.config :as config]
+ [clojure.test.generative.runner :as runner]
+ [clojure.tools.namespace :as ns]
+ [clojure.test :as ctest]))
+
+(defmulti event->ctevent
+ "Returns an event for a Clojure test reporter, or nil"
+ :type)
+
+(defmethod event->ctevent :default [_] nil)
+(defmethod event->ctevent :clojure.test.generative/pass [_] {:type :pass})
+(defmethod event->ctevent :clojure.test.generative/iter [e] e)
+(defmethod event->ctevent :clojure.test.generative/fail [e] e)
+(defmethod event->ctevent :clojure.test.generative/error [e] e)
+
+(defmethod ctest/report :clojure.test.generative/iter
+ [e]
+ (when (contains? (:tags e) :begin)
+ (ctest/inc-report-counter :test)))
+
+(defmethod ctest/report :clojure.test.generative/fail
+ [e]
+ (ctest/inc-report-counter :fail)
+ (println "\nFAIL\n")
+ (event/pprint e))
+
+(defmethod ctest/report :clojure.test.generative/error
+ [e]
+ (ctest/inc-report-counter :error)
+ (println "\nERROR\n")
+ (event/pprint e))
+
+(defn handler
+ "Event handler that delegates c.t.g events to clojure.test/report"
+ [e]
+ (event/dot-progress e)
+ (when-let [cte (event->ctevent e)]
+ (clojure.test/report cte)))
+
+(defmacro run-generative-tests
+ "Add this macro somewhere in your clojure.test suite to create
+ a clojure.test entry point that will run your generative tests"
+ []
+ (event/add-handler handler)
+ `(ctest/deftest ~'generative-test-adapter
+ []
+ (runner/run-all)))
+
+(defn -main
+ "Command line entry point, runs all tests in dirs using clojure.test"
+ [& dirs]
+ (if (seq dirs)
+ (let [nses (mapcat #(ns/find-namespaces-in-dir (java.io.File. ^String %)) dirs)]
+ (doseq [ns nses] (require ns))
+ (let [{:keys [threads msec]} (config/config)]
+ (try
+ (println "Testing on" threads "threads for" msec "msec.")
+ (apply ctest/run-tests nses)
+ (catch Throwable t
+ (.printStackTrace t)
+ (System/exit -1))
+ (finally
+ (shutdown-agents)))))
+ (do
+ (println "Specify at least one directory with tests")
+ (System/exit -1))))
+
View
27 src/main/clojure/clojure/test/generative/config.clj
@@ -0,0 +1,27 @@
+; Copyright (c) Rich Hickey, Stuart Halloway, and contributors.
+; All rights reserved.
+; The use and distribution terms for this software are covered by the
+; Eclipse Public License 1.0 (http://opensource.org/licenses/eclipse-1.0.php)
+; which can be found in the file epl-v10.html at the root of this distribution.
+; By using this software in any fashion, you are agreeing to be bound by
+; the terms of this license.
+; You must not remove this notice, or any other, from this software.
+
+(ns clojure.test.generative.config)
+
+(def config-mapping
+ [["clojure.test.generative.threads" [:threads] read-string (.availableProcessors (Runtime/getRuntime))]
+ ["clojure.test.generative.msec" [:msec] read-string 10000]
+ ["clojure.test.generative.runner" [:runner] identity nil]])
+
+(defn config
+ []
+ (reduce
+ (fn [m [prop path coerce default]]
+ (let [val (System/getProperty prop)]
+ (if (seq val)
+ (assoc-in m path (coerce val))
+ (assoc-in m path default))))
+ {}
+ config-mapping))
+
View
146 src/main/clojure/clojure/test/generative/event.clj
@@ -0,0 +1,146 @@
+; Copyright (c) Rich Hickey, Stuart Halloway, and contributors.
+; All rights reserved.
+; The use and distribution terms for this software are covered by the
+; Eclipse Public License 1.0 (http://opensource.org/licenses/eclipse-1.0.php)
+; which can be found in the file epl-v10.html at the root of this distribution.
+; By using this software in any fashion, you are agreeing to be bound by
+; the terms of this license.
+; You must not remove this notice, or any other, from this software.
+
+(ns clojure.test.generative.event
+ (:refer-clojure :exclude [pr-str])
+ (:require [clojure.pprint :as pprint]
+ [clojure.string :as str]))
+
+(set! *warn-on-reflection* true)
+
+(def ^long pid
+ "Process id"
+ (read-string (.getName (java.lang.management.ManagementFactory/getRuntimeMXBean))))
+
+(defn create
+ [& args]
+ (let [t (Thread/currentThread)]
+ (apply assoc
+ {:tstamp (System/currentTimeMillis)
+ :thread (.getId t)
+ :thread-name (.getName t)
+ :pid pid
+ :level :info}
+ args)))
+
+(def ^:private serializer (agent nil))
+
+(defn serialized
+ "Returns a function that calls f for side effects, async,
+ serialized by an agent"
+ [f]
+ (fn [& args]
+ (send-off serializer
+ (fn [_]
+ (apply f args)
+ nil))))
+
+;; TODO set from Java property?
+(def ^:private event-print-length 100)
+(def ^:private event-print-level 10)
+
+(defn pr-str
+ "Print with event print settings"
+ [s]
+ (binding [*print-length* event-print-length
+ *print-level* event-print-level]
+ (clojure.core/pr-str s)))
+
+(defn pprint
+ "Print with event print settings"
+ [s]
+ (binding [*print-length* event-print-length
+ *print-level* event-print-level]
+ (pprint/pprint s)
+ (flush)))
+
+(def ^:private report-fns
+ (atom []))
+
+(defn add-handler
+ "Add a handler. Idempotent"
+ [f]
+ (swap!
+ report-fns
+ (fn [v f]
+ (if (some #{f} v)
+ v
+ (conj v f)))
+ f))
+
+(defn remove-handler
+ "Remove a handler. Idempotent"
+ [f]
+ (swap!
+ report-fns
+ (fn [v f]
+ (into (empty v) (remove #{f} v)))
+ f))
+
+(defmacro with-handler
+ "Run with handler temporarily installed."
+ [handler & body]
+ `(let [h# ~handler]
+ (add-handler h#)
+ (try
+ ~@body
+ (finally
+ (remove-handler h#)))))
+
+(defn load-var-val
+ "Load and return the value of a var"
+ [fqname]
+ (when-let [ns (namespace fqname)]
+ (require (symbol ns)))
+ @(resolve fqname))
+
+(defn load-default-handlers
+ []
+ (reset! report-fns [])
+ (doseq [handler (let [s (System/getProperty "clojure.test.generative.event.handlers")]
+ (when (seq s) (str/split s #",")))]
+ (add-handler (load-var-val (symbol handler)))))
+
+(defn report-fn
+ [event]
+ (doseq [f @report-fns]
+ (f event)))
+
+(defmacro report
+ [& args]
+ `(report-fn (create ~@args)))
+
+(defn local-bindings
+ "Produces a map of the names of local bindings to their values."
+ [env]
+ (let [symbols (map key env)]
+ (zipmap (map (fn [sym] `(quote ~sym)) symbols) symbols)))
+
+(defmacro report-context
+ "Report event with contextual ns, file, line, bindings."
+ [& args]
+ `(report-fn
+ (create :bindings ~(local-bindings &env)
+ :file ~*file*
+ :line ~(:line (meta &form))
+ ~@args)))
+
+(def last-dot (atom 0))
+
+(defn dot-progress
+ "Prints a dot per event, throttled to ten dots/sec."
+ [{:keys [tstamp]}]
+ (when (< 100 (- tstamp @last-dot))
+ (reset! last-dot tstamp)
+ (send-off serializer
+ (fn [_]
+ (print ".")
+ (flush)
+ nil))))
+
View
63 src/main/clojure/clojure/test/generative/logback.clj
@@ -0,0 +1,63 @@
+; Copyright (c) Rich Hickey, Stuart Halloway, and contributors.
+; All rights reserved.
+; The use and distribution terms for this software are covered by the
+; Eclipse Public License 1.0 (http://opensource.org/licenses/eclipse-1.0.php)
+; which can be found in the file epl-v10.html at the root of this distribution.
+; By using this software in any fashion, you are agreeing to be bound by
+; the terms of this license.
+; You must not remove this notice, or any other, from this software.
+
+(ns clojure.test.generative.logback
+ (:require [clojure.test.generative.event :as event])
+ (:import [ch.qos.logback.classic.spi ILoggingEvent]
+ [ch.qos.logback.classic Level]
+ [org.slf4j Logger LoggerFactory]))
+
+(set! *warn-on-reflection* true)
+
+(defn level->logback
+ [level]
+ (case level
+ :info Level/INFO
+ :warn Level/WARN
+ :error Level/ERROR
+ :debug Level/DEBUG))
+
+(defn event->logback
+ [event ^ch.qos.logback.classic.Logger logger]
+ (let [msg (delay (binding [*print-length* 50]
+ (pr-str (dissoc event :level :thread :tstamp :name))))
+ level (level->logback (:level event))]
+ (reify ILoggingEvent
+ (getThreadName [_] (str (:thread-name event)))
+ (getLevel [_] level)
+ (getMessage [_] @msg)
+ (getArgumentArray [_])
+ (getFormattedMessage [_] @msg)
+ (getLoggerName [_] (.getName logger))
+ (getLoggerContextVO [_] (.. logger getLoggerContext getLoggerContextRemoteView))
+ (getThrowableProxy [_])
+ (getCallerData [_])
+ (hasCallerData [_])
+ (getMarker [_])
+ (getMdc [_])
+ (getTimeStamp [_] (:tstamp event))
+ (prepareForDeferredProcessing [_]))))
+
+(defn handler
+ [event]
+ (let [name (str (:name event))
+ logger ^ch.qos.logback.classic.Logger (LoggerFactory/getLogger name)]
+ (.callAppenders logger (event->logback event logger))))
+
+
+(comment
+ (require :reload '[clojure.test.generative.event :as event])
+ (require :reload '[clojure.test.generative.event.logback :as la])
+ (in-ns 'clojure.test.generative.event.logback-adapter)
+ (set! *warn-on-reflection* true)
+ (def l (LoggerFactory/getLogger "stu"))
+ (.info l "hi")
+ (report-to-logback (event/create {:level :warn :message "Drat"}))
+
+ )
View
140 src/main/clojure/clojure/test/generative/runner.clj
@@ -0,0 +1,140 @@
+; Copyright (c) Rich Hickey, Stuart Halloway, and contributors.
+; All rights reserved.
+; The use and distribution terms for this software are covered by the
+; Eclipse Public License 1.0 (http://opensource.org/licenses/eclipse-1.0.php)
+; which can be found in the file epl-v10.html at the root of this distribution.
+; By using this software in any fashion, you are agreeing to be bound by
+; the terms of this license.
+; You must not remove this notice, or any other, from this software.
+
+(ns clojure.test.generative.runner
+ (:require
+ [clojure.tools.namespace :as ns]
+ [clojure.test.generative.config :as config]
+ [clojure.test.generative.event :as event]
+ [clojure.test.generative.generators :as gen]))
+
+(set! *warn-on-reflection* true)
+
+(defprotocol Test
+ (test-name [_])
+ (test-fn [_])
+ (test-input [_]))
+
+(extend-protocol Test
+ clojure.lang.Var
+ (test-name
+ [v]
+ (-> (when-let [ns (.ns v)]
+ (str ns "/" (.sym v))
+ (.sym v))
+ symbol))
+ (test-fn
+ [this]
+ @this)
+ (test-input
+ [v]
+ (map #(%) (:clojure.test.generative/inputs (meta v)))))
+
+;; non-nil binding means running inside the framework
+(def ^:dynamic *failed* nil)
+
+(defn run-iter
+ "Run a single test iteration"
+ [test]
+ (let [name (test-name test)
+ f (test-fn test)
+ input (test-input test)]
+ (event/report :name name :args input :type :clojure.test.generative/iter :tags #{:begin})
+ (try
+ (let [result (apply f input)]
+ (when-not (realized? *failed*)
+ (event/report :name name :return result :type :clojure.test.generative/iter :tags #{:end})))
+ (catch Throwable t
+ (deliver *failed* :error)
+ (event/report :name name :exception t :type :clojure.test.generative/error)))))
+
+(defn run-for
+ "Run f (presumably for side effects) repeatedly on n threads,
+ until msec has passed or somebody signals *failed*"
+ [test nthreads msec]
+ (let [start (System/currentTimeMillis)
+ futs (doall
+ (map
+ #(future
+ (try
+ (binding [gen/*seed* (java.util.Random. (+ % 42))
+ *failed* (promise)]
+ (loop [iter 0]
+ (let [result (run-iter test)
+ now (System/currentTimeMillis)]
+ (if (and (< now (+ start msec))
+ (not (realized? *failed*)))
+ (recur (inc iter))
+ (event/report :msec (- now start)
+ :count iter
+ :type :clojure.test.generative/run-completed)))))
+ (catch Throwable t
+ (event/report :level :error :exception t :type :clojure.test.generative/run-error))))
+ (range nthreads)))]
+ (doseq [f futs] @f)))
+
+(defn run-batch
+ "Run a batch of fs on nthreads each. Try to divide msec
+ among the fs. Args like run-for."
+ [tests nthreads msec]
+ (when (seq tests)
+ (let [msec-per-f (quot msec (count tests))]
+ (doseq [test tests]
+ (run-for test nthreads msec-per-f)))))
+
+(defn failed!
+ "Tell the runner that a test failed"
+ []
+ (when *failed*
+ (deliver *failed* :failed)))
+
+(defn set-seed
+ [n]
+ (event/report :seed n)
+ (set! gen/*rnd* (java.util.Random. n)))
+
+(defn run-suite
+ [fs nthreads msec]
+ (event/report :tags #{:begin} :type :clojure.test.generative/suite :nthreads nthreads :ntests (count fs))
+ (try
+ (run-batch
+ fs
+ nthreads
+ msec)
+ (finally
+ (event/report :tags #{:end} :type :clojure.test.generative/suite :nthreads nthreads :ntests (count fs)))))
+
+(defn gentest?
+ [v]
+ (boolean (:clojure.test.generative/inputs (meta v))))
+
+(defn find-vars-in-namespaces
+ [& nses]
+ (when nses
+ (reduce (fn [v ns] (into v (vals (ns-interns ns)))) [] nses)))
+
+(defn find-vars-in-dirs
+ [& dirs]
+ (let [nses (mapcat #(ns/find-namespaces-in-dir (java.io.File. ^String %)) dirs)]
+ (doseq [ns nses] (require ns))
+ (apply find-vars-in-namespaces nses)))
+
+(defn find-gentests-in-vars
+ [& vars]
+ (filter gentest? vars))
+
+(defn run-all
+ []
+ (let [conf (config/config)
+ tests (->> (apply find-vars-in-namespaces (all-ns))
+ (filter gentest?))]
+ (run-suite tests (:threads conf) (:msec conf))))
+
+
+
View
49 src/test/clojure/clojure/test/generative_test.clj
@@ -8,58 +8,31 @@
; You must not remove this notice, or any other, from this software.
(ns clojure.test.generative-test
- (:use clojure.test.generative)
- (:require [clojure.test.generative.generators :as gen]))
+ (:use clojure.test.generative
+ [clojure.test :exclude [is]])
+ (:require [clojure.test.generative.generators :as gen]
+ [clojure.test.generative.event :as event]
+ [clojure.test.generative.clojure-test :as clojure-test]))
+
+(clojure-test/run-generative-tests)
(defspec test-anything-goes
identity
[^anything s])
-(defn little-number
- []
- (gen/uniform 0 3))
-
-(defspec test-exhausted-generation-2
- (constantly nil)
- [^{:tag `little-number} arg1
- ^{:tag `little-number} arg2])
-
-(def digits [0 1 2 3 4 5 6 7 8 9])
-
-(defspec test-collection-based-generator
- identity
- [^{:tag `digits} d]
- (assert (<= 0 d 9)))
-
-(defspec test-vec
- identity
- [^{:tag (vec long)} _]
- (assert (vector? %)))
-
(defspec test-weighted-generation
identity
[^{:tag (vec #(weighted {boolean 8 long 1}))} _]
(let [[longs bools] (split-with number? %)]
- (when (< 10 (count %))
- (assert (< (count longs) (count bools))))
- (assert (= (+ (count longs) (count bools))
+ (is (= (+ (count longs) (count bools))
(count %)))))
(defspec integers-closed-over-addition
(fn [a b] (+' a b))
[^long a ^long b]
- (integer? %))
+ (is (integer? %)))
+
+
-(defmacro long=
- "Returns true if all forms are equal, or if all forms
- throw an ArithmeticException"
- [& forms]
- `(=
- ~@(map
- (fn [form]
- `(try
- ~form
- (catch ArithmeticException e# :ArithmeticException)))
- forms)))
Please sign in to comment.
Something went wrong with that request. Please try again.