diff --git a/test-grid-tests.lisp b/test-grid-tests.lisp index f78cb18..65edffa 100644 --- a/test-grid-tests.lisp +++ b/test-grid-tests.lisp @@ -68,6 +68,19 @@ :test #'string=) (null (getf status :known-to-fail))))) +(defun test-hu.dwim.stefil-api () + (ql:quickload :hu.dwim.stefil-sample-test-suite) + (let ((status (test-grid-testsuites::run-hu.dwim.stefil-test-suite + (read-from-string "hu.dwim.stefil-sample-test-suite::sample-stefil-suite")))) + (and (alexandria:set-equal (getf status :failed-tests) + '("sample-stefil-suite.one-fail-test" + "sample-stefil-suite.two-fails-test" + "sample-stefil-suite.error-test" + "sample-stefil-suite.all-fails-expected-test" + "sample-stefil-suite.not-all-fails-expected-test") + :test #'string=) + (null (getf status :known-to-fail))))) + (defun test-clunit-api () (ql:quickload :clunit-sample-test-suite) (let ((status (test-grid-testsuites::run-clunit-test-suite (read-from-string "clunit-sample-test-suite::NumberSuite")))) @@ -107,6 +120,7 @@ (test-fiveam-api) (test-eos-api) (test-stefil-api) + (test-hu.dwim.stefil-api) (test-clunit-api) (test-nst-api) (test-aggregated-status))))) diff --git a/test-grid-testsuites.asd b/test-grid-testsuites.asd index ea600fa..6c87a58 100644 --- a/test-grid-testsuites.asd +++ b/test-grid-testsuites.asd @@ -18,6 +18,7 @@ #:fiveam-api #:eos-api #:stefil-api + #:hu.dwim.stefil-api #:clunit-api #:nst-api) :components diff --git a/testsuites/apis-of-test-frameworks/hu.dwim.stefil-api-impl.asd b/testsuites/apis-of-test-frameworks/hu.dwim.stefil-api-impl.asd new file mode 100644 index 0000000..fadd380 --- /dev/null +++ b/testsuites/apis-of-test-frameworks/hu.dwim.stefil-api-impl.asd @@ -0,0 +1,11 @@ +;;; -*- Mode: LISP; Syntax: COMMON-LISP; indent-tabs-mode: nil; coding: utf-8; -*- +;;; +;;; Copyright (C) 2011 Anton Vodonosov (avodonosov@yandex.ru) +;;; +;;; See LICENSE for details. + +(asdf:defsystem #:hu.dwim.stefil-api-impl + :version "0.1.0" + :serial t + :depends-on (#:hu.dwim.stefil-api #:hu.dwim.stefil) + :components ((:file "hu.dwim.stefil-api-impl"))) diff --git a/testsuites/apis-of-test-frameworks/hu.dwim.stefil-api-impl.lisp b/testsuites/apis-of-test-frameworks/hu.dwim.stefil-api-impl.lisp new file mode 100644 index 0000000..2af3f7c --- /dev/null +++ b/testsuites/apis-of-test-frameworks/hu.dwim.stefil-api-impl.lisp @@ -0,0 +1,34 @@ +(defpackage #:hu.dwim.stefil-api-impl + (:use #:cl :hu.dwim.stefil-api)) + +(in-package #:hu.dwim.stefil-api-impl) + +(defun print-details (test-suite-result) + (format t "~&~%-------------------------------------------------------------------------------------------------~%") + (format t "The test result details (printed by cl-test-grid in addition to the standard Stefil output above)~%") + (format t "~%(DESCRIBE RESULT):~%") + (describe test-suite-result) + (format t "~&~%Individual failures:~%") + (map 'nil + (lambda (descr) (format t "~A~%" descr)) + (hu.dwim.stefil::failure-descriptions-of test-suite-result))) + +(defun run-test-suite (test-suite-name) + (let* ((*debug-io* *standard-output*) + (result (hu.dwim.stefil:funcall-test-with-feedback-message test-suite-name))) + (print-details result) + result)) + +(defun test-name (failure-description) + (let ((hierarchical-names (mapcar + #'(lambda (context) + (hu.dwim.stefil::name-of + (hu.dwim.stefil::test-of context))) + (hu.dwim.stefil::test-context-backtrace-of failure-description)))) + (format nil "~(~{~A~^.~}~)" (nreverse hierarchical-names)))) + +(defun failed-tests (test-suite-result) + (remove-duplicates (map 'list + #'test-name + (hu.dwim.stefil::failure-descriptions-of test-suite-result)) + :test #'string=)) \ No newline at end of file diff --git a/testsuites/apis-of-test-frameworks/hu.dwim.stefil-api.asd b/testsuites/apis-of-test-frameworks/hu.dwim.stefil-api.asd new file mode 100644 index 0000000..d86823c --- /dev/null +++ b/testsuites/apis-of-test-frameworks/hu.dwim.stefil-api.asd @@ -0,0 +1,11 @@ +;;; -*- Mode: LISP; Syntax: COMMON-LISP; indent-tabs-mode: nil; coding: utf-8; -*- +;;; +;;; Copyright (C) 2011 Anton Vodonosov (avodonosov@yandex.ru) +;;; +;;; See LICENSE for details. + +(asdf:defsystem #:hu.dwim.stefil-api + :version "0.1.0" + :serial t + :depends-on (#:api-dsl) + :components ((:file "hu.dwim.stefil-api"))) diff --git a/testsuites/apis-of-test-frameworks/hu.dwim.stefil-api.lisp b/testsuites/apis-of-test-frameworks/hu.dwim.stefil-api.lisp new file mode 100644 index 0000000..bdfdeca --- /dev/null +++ b/testsuites/apis-of-test-frameworks/hu.dwim.stefil-api.lisp @@ -0,0 +1,25 @@ +(defpackage #:hu.dwim.stefil-api + (:use :cl #:api-dsl) + (:export #:run-test-suite + #:failed-tests)) + +(in-package #:hu.dwim.stefil-api) + +(proclfun run-test-suite ((test-suite-name symbol)) t + "Runs the Stefil test stuite specified by TEST-SUITE-NAME and returns +an opaque result object. The result object may be passed to the FAILED-TESTS +function to retreive the list of failed tests.") + +(proclfun failed-tests ((test-suite-result t)) list + "List of failed test names. Test names are downcased strings. +The TEST-SUITE-RESULT parameter must be a result of the RUN-TEST-SUITE +function.") + +;; Stefil has a macro with-expected-failures, but we don't know +;; how it is intended to be used - to register "known" failues, or for some +;; other purpose. So far none of the libraries added to cl-test-grid +;; use this feature, therefore we do not distinguish the failure-description +;; having expected-p = t from other failure-descriptions. +;; +;; When we meet the first library using with-expected-failures, we will +;; see how to support it. \ No newline at end of file diff --git a/testsuites/apis-of-test-frameworks/stefil-api-impl.asd b/testsuites/apis-of-test-frameworks/stefil-api-impl.asd index de637d9..df3645f 100644 --- a/testsuites/apis-of-test-frameworks/stefil-api-impl.asd +++ b/testsuites/apis-of-test-frameworks/stefil-api-impl.asd @@ -7,5 +7,5 @@ (asdf:defsystem #:stefil-api-impl :version "0.1.0" :serial t - :depends-on (#:stefil-api #:hu.dwim.stefil) + :depends-on (#:stefil-api #:stefil) :components ((:file "stefil-api-impl"))) diff --git a/testsuites/apis-of-test-frameworks/stefil-api-impl.lisp b/testsuites/apis-of-test-frameworks/stefil-api-impl.lisp index f3ae040..d7078b0 100644 --- a/testsuites/apis-of-test-frameworks/stefil-api-impl.lisp +++ b/testsuites/apis-of-test-frameworks/stefil-api-impl.lisp @@ -11,24 +11,24 @@ (format t "~&~%Individual failures:~%") (map 'nil (lambda (descr) (format t "~A~%" descr)) - (hu.dwim.stefil::failure-descriptions-of test-suite-result))) + (stefil::failure-descriptions-of test-suite-result))) (defun run-test-suite (test-suite-name) (let* ((*debug-io* *standard-output*) - (result (hu.dwim.stefil:funcall-test-with-feedback-message test-suite-name))) + (result (stefil:funcall-test-with-feedback-message test-suite-name))) (print-details result) result)) (defun test-name (failure-description) (let ((hierarchical-names (mapcar #'(lambda (context) - (hu.dwim.stefil::name-of - (hu.dwim.stefil::test-of context))) - (hu.dwim.stefil::test-context-backtrace-of failure-description)))) + (stefil::name-of + (stefil::test-of context))) + (stefil::test-context-backtrace-of failure-description)))) (format nil "~(~{~A~^.~}~)" (nreverse hierarchical-names)))) (defun failed-tests (test-suite-result) (remove-duplicates (map 'list #'test-name - (hu.dwim.stefil::failure-descriptions-of test-suite-result)) + (stefil::failure-descriptions-of test-suite-result)) :test #'string=)) \ No newline at end of file diff --git a/testsuites/sample-test-suites/hu.dwim.stefil-sample-test-suite.asd b/testsuites/sample-test-suites/hu.dwim.stefil-sample-test-suite.asd new file mode 100644 index 0000000..de3063a --- /dev/null +++ b/testsuites/sample-test-suites/hu.dwim.stefil-sample-test-suite.asd @@ -0,0 +1,11 @@ +;;; -*- Mode: LISP; Syntax: COMMON-LISP; indent-tabs-mode: nil; coding: utf-8; -*- +;;; +;;; Copyright (C) 2011 Anton Vodonosov (avodonosov@yandex.ru) +;;; +;;; See LICENSE for details. + +(asdf:defsystem #:hu.dwim.stefil-sample-test-suite + :version "0.1.0" + :serial t + :depends-on (#:hu.dwim.stefil) + :components ((:file "hu.dwim.stefil-sample-test-suite"))) diff --git a/testsuites/sample-test-suites/hu.dwim.stefil-sample-test-suite.lisp b/testsuites/sample-test-suites/hu.dwim.stefil-sample-test-suite.lisp new file mode 100644 index 0000000..2d66bb1 --- /dev/null +++ b/testsuites/sample-test-suites/hu.dwim.stefil-sample-test-suite.lisp @@ -0,0 +1,34 @@ +(defpackage #:hu.dwim.stefil-sample-test-suite + (:use #:cl :hu.dwim.stefil)) + +(in-package #:hu.dwim.stefil-sample-test-suite) + +(defsuite sample-stefil-suite) +(in-suite sample-stefil-suite) + +(deftest ok-test () + (is (= (+ 2 2) 4))) + +(deftest one-fail-test () + (is (= (+ 2 2) 3))) + +(deftest two-fails-test () + (is (= (+ 2 2) 3)) + (is (= (+ 1 1) 2)) + (is (= (+ 3 3) 7))) + +(deftest error-test () + (is (= 7 7)) + (is (= (/ 1 0) 1))) + +(deftest all-fails-expected-test () + (with-expected-failures + (is (= (+ 2 2) 3)) + (is (= (+ 1 1) 2)) + (is (= (+ 3 3) 7)))) + +(deftest not-all-fails-expected-test () + (is (= (+ 2 2) 3)) + (with-expected-failures + (is (= (+ 1 1) 2)) + (is (= (+ 3 3) 7)))) diff --git a/testsuites/sample-test-suites/stefil-sample-test-suite.asd b/testsuites/sample-test-suites/stefil-sample-test-suite.asd index 0206a60..3e13884 100644 --- a/testsuites/sample-test-suites/stefil-sample-test-suite.asd +++ b/testsuites/sample-test-suites/stefil-sample-test-suite.asd @@ -7,5 +7,5 @@ (asdf:defsystem #:stefil-sample-test-suite :version "0.1.0" :serial t - :depends-on (#:hu.dwim.stefil) + :depends-on (#:stefil) :components ((:file "stefil-sample-test-suite"))) diff --git a/testsuites/sample-test-suites/stefil-sample-test-suite.lisp b/testsuites/sample-test-suites/stefil-sample-test-suite.lisp index 26d22c6..09c0a8f 100644 --- a/testsuites/sample-test-suites/stefil-sample-test-suite.lisp +++ b/testsuites/sample-test-suites/stefil-sample-test-suite.lisp @@ -1,5 +1,5 @@ (defpackage #:stefil-sample-test-suite - (:use #:cl :hu.dwim.stefil)) + (:use #:cl :stefil)) (in-package #:stefil-sample-test-suite) diff --git a/testsuites/testsuites.lisp b/testsuites/testsuites.lisp index 1469676..57717b1 100644 --- a/testsuites/testsuites.lisp +++ b/testsuites/testsuites.lisp @@ -64,6 +64,7 @@ just passed to the QUICKLISP:QUICKLOAD." ("fiveam-api" . "fiveam-api-impl") ("eos-api" . "eos-api-impl") ("stefil-api" . "stefil-api-impl") + ("hu.dwim.stefil-api" . "hu.dwim.stefil-api-impl") ("clunit-api" . "clunit-api-impl") ("nst-api" . "nst-api-impl"))) (impl-asdf-system (or (cdr (assoc api known-impls :test #'string=)) @@ -137,6 +138,12 @@ just passed to the QUICKLISP:QUICKLOAD." (list :failed-tests (stefil-api:failed-tests result) :known-to-fail '()))) +(defun run-hu.dwim.stefil-test-suite (test-suite-spec) + (require-impl "hu.dwim.stefil-api") + (let ((result (hu.dwim.stefil-api:run-test-suite test-suite-spec))) + (list :failed-tests (hu.dwim.stefil-api:failed-tests result) + :known-to-fail '()))) + (defun run-clunit-test-suite (test-suite-name) (require-impl "clunit-api") (let ((result (clunit-api:run-test-suite test-suite-name))) @@ -215,10 +222,10 @@ just passed to the QUICKLISP:QUICKLOAD." (defmethod libtest ((library-name (eql :babel))) - ;; The test framework used: stefil. + ;; The test framework used: hu.dwim.stefil. (quicklisp:quickload :babel-tests) - (run-stefil-test-suite (intern (string '#:babel-tests) '#:babel-tests))) + (run-hu.dwim.stefil-test-suite (read-from-string "babel-tests::babel-tests"))) (defmethod libtest ((library-name (eql :trivial-features))) @@ -563,9 +570,9 @@ just passed to the QUICKLISP:QUICKLOAD." (run-fiveam-test-suite :it.bese.arnesi)) (defmethod libtest ((library-name (eql :local-time))) - ;; test framework used: Stefil + ;; test framework used: stefil (quicklisp:quickload :local-time.test) - (run-stefil-test-suite (intern (string '#:test) '#:local-time.test))) + (run-stefil-test-suite (read-from-string "local-time.test::test"))) (defmethod libtest ((library-name (eql :s-xml))) ;; test framework used: cl:assert @@ -801,10 +808,10 @@ just passed to the QUICKLISP:QUICKLOAD." (run-rt-test-suite)) (defmethod libtest ((library-name (eql :hu.dwim.stefil))) - ;; The test framework used: stefil. + ;; The test framework used: hu.dwim.stefil. (ql:quickload :hu.dwim.stefil.test) - (run-stefil-test-suite (read-from-string "hu.dwim.stefil.test::test"))) + (run-hu.dwim.stefil-test-suite (read-from-string "hu.dwim.stefil.test::test"))) (defmethod libtest ((library-name (eql :kmrcl))) @@ -829,14 +836,14 @@ just passed to the QUICKLISP:QUICKLOAD." (run-rt-test-suite)) (defmethod libtest ((library-name (eql :hu.dwim.walker))) - ;; The test framework used: stefil. + ;; The test framework used: hu.dwim.stefil. (ql:quickload :hu.dwim.walker.test) - (run-stefil-test-suite (read-from-string "hu.dwim.walker.test::test"))) + (run-hu.dwim.stefil-test-suite (read-from-string "hu.dwim.walker.test::test"))) (defmethod libtest ((library-name (eql :hu.dwim.defclass-star))) - ;; The test framework used: stefil. + ;; The test framework used: hu.dwim.stefil. (ql:quickload :hu.dwim.defclass-star.test) - (run-stefil-test-suite (read-from-string "hu.dwim.defclass-star.test::test"))) + (run-hu.dwim.stefil-test-suite (read-from-string "hu.dwim.defclass-star.test::test"))) (defmethod libtest ((library-name (eql :bknr-datastore))) ;; test framework used: FiveAM @@ -869,9 +876,9 @@ just passed to the QUICKLISP:QUICKLOAD." (run-fiveam-test-suite :it.bese.yaclml)) (defmethod libtest ((library-name (eql :com.google.base))) - ;; The test framework used: stefil. + ;; The test framework used: hu.dwim.stefil. (ql:quickload :com.google.base-test) - (run-stefil-test-suite (read-from-string "com.google.base-test::test-base"))) + (run-hu.dwim.stefil-test-suite (read-from-string "com.google.base-test::test-base"))) (defmethod libtest ((library-name (eql :external-program))) ;; test framework used: FiveAM