Sperate code to run stefil suites and hu.dwim.stefil. The same code c…

…an't be used for both anymore, as they have become incompatible.

test-grid-tests.lisp

@@ -68,6 +68,19 @@
                                :test #'string=)
          (null (getf status :known-to-fail)))))
 
+(defun test-hu.dwim.stefil-api ()
+  (ql:quickload :hu.dwim.stefil-sample-test-suite)
+  (let ((status (test-grid-testsuites::run-hu.dwim.stefil-test-suite
+                   (read-from-string "hu.dwim.stefil-sample-test-suite::sample-stefil-suite"))))
+    (and (alexandria:set-equal (getf status :failed-tests)
+                               '("sample-stefil-suite.one-fail-test"
+                                 "sample-stefil-suite.two-fails-test"
+                                 "sample-stefil-suite.error-test"
+                                 "sample-stefil-suite.all-fails-expected-test"
+                                 "sample-stefil-suite.not-all-fails-expected-test")
+                               :test #'string=)
+         (null (getf status :known-to-fail)))))
+
 (defun test-clunit-api ()
   (ql:quickload :clunit-sample-test-suite)
   (let ((status (test-grid-testsuites::run-clunit-test-suite (read-from-string "clunit-sample-test-suite::NumberSuite"))))
@@ -107,6 +120,7 @@
                        (test-fiveam-api)
                        (test-eos-api)
                        (test-stefil-api)
+                       (test-hu.dwim.stefil-api)
                        (test-clunit-api)
                        (test-nst-api)
                        (test-aggregated-status)))))

test-grid-testsuites.asd

@@ -18,6 +18,7 @@
                #:fiveam-api
                #:eos-api
                #:stefil-api
+               #:hu.dwim.stefil-api
                #:clunit-api
                #:nst-api)
   :components

testsuites/apis-of-test-frameworks/hu.dwim.stefil-api-impl.asd

@@ -0,0 +1,11 @@
+;;; -*- Mode: LISP; Syntax: COMMON-LISP; indent-tabs-mode: nil; coding: utf-8;  -*-
+;;;
+;;; Copyright (C) 2011 Anton Vodonosov (avodonosov@yandex.ru)
+;;;
+;;; See LICENSE for details.
+
+(asdf:defsystem #:hu.dwim.stefil-api-impl
+  :version "0.1.0"
+  :serial t
+  :depends-on (#:hu.dwim.stefil-api #:hu.dwim.stefil)
+  :components ((:file "hu.dwim.stefil-api-impl")))

testsuites/apis-of-test-frameworks/hu.dwim.stefil-api-impl.lisp

@@ -0,0 +1,34 @@
+(defpackage #:hu.dwim.stefil-api-impl
+  (:use #:cl :hu.dwim.stefil-api))
+
+(in-package #:hu.dwim.stefil-api-impl)
+
+(defun print-details (test-suite-result)
+    (format t "~&~%-------------------------------------------------------------------------------------------------~%")
+    (format t "The test result details (printed by cl-test-grid in addition to the standard Stefil output above)~%")
+    (format t "~%(DESCRIBE RESULT):~%")
+    (describe test-suite-result)
+    (format t "~&~%Individual failures:~%")
+    (map 'nil
+         (lambda (descr) (format t "~A~%" descr))
+         (hu.dwim.stefil::failure-descriptions-of test-suite-result)))
+
+(defun run-test-suite (test-suite-name)
+  (let* ((*debug-io* *standard-output*)
+         (result (hu.dwim.stefil:funcall-test-with-feedback-message test-suite-name)))
+    (print-details result)
+    result))
+
+(defun test-name (failure-description)
+  (let ((hierarchical-names  (mapcar 
+                              #'(lambda (context) 
+                                  (hu.dwim.stefil::name-of
+                                   (hu.dwim.stefil::test-of context)))
+                              (hu.dwim.stefil::test-context-backtrace-of failure-description))))
+    (format nil "~(~{~A~^.~}~)" (nreverse hierarchical-names))))
+
+(defun failed-tests (test-suite-result)
+  (remove-duplicates (map 'list
+                          #'test-name
+                          (hu.dwim.stefil::failure-descriptions-of test-suite-result))
+                     :test #'string=))

testsuites/apis-of-test-frameworks/hu.dwim.stefil-api.asd

@@ -0,0 +1,11 @@
+;;; -*- Mode: LISP; Syntax: COMMON-LISP; indent-tabs-mode: nil; coding: utf-8;  -*-
+;;;
+;;; Copyright (C) 2011 Anton Vodonosov (avodonosov@yandex.ru)
+;;;
+;;; See LICENSE for details.
+
+(asdf:defsystem #:hu.dwim.stefil-api
+  :version "0.1.0"
+  :serial t
+  :depends-on (#:api-dsl)
+  :components ((:file "hu.dwim.stefil-api")))

testsuites/apis-of-test-frameworks/hu.dwim.stefil-api.lisp

@@ -0,0 +1,25 @@
+(defpackage #:hu.dwim.stefil-api
+  (:use :cl #:api-dsl)
+  (:export #:run-test-suite
+           #:failed-tests))
+
+(in-package #:hu.dwim.stefil-api)
+
+(proclfun run-test-suite ((test-suite-name symbol)) t
+  "Runs the Stefil test stuite specified by TEST-SUITE-NAME and returns 
+an opaque result object. The result object may be passed to the FAILED-TESTS 
+function to retreive the list of failed tests.")
+
+(proclfun failed-tests ((test-suite-result t)) list
+  "List of failed test names. Test names are downcased strings.
+The TEST-SUITE-RESULT parameter must be a result of the RUN-TEST-SUITE
+function.")
+
+;; Stefil has a macro with-expected-failures, but we don't know
+;; how it is intended to be used - to register "known" failues, or for some
+;; other purpose. So far none of the libraries added to cl-test-grid
+;; use this feature, therefore we do not distinguish the failure-description
+;; having expected-p = t from other failure-descriptions.
+;; 
+;; When we meet the first library using with-expected-failures, we will
+;; see how to support it.

testsuites/apis-of-test-frameworks/stefil-api-impl.asd

@@ -7,5 +7,5 @@
 (asdf:defsystem #:stefil-api-impl
   :version "0.1.0"
   :serial t
-  :depends-on (#:stefil-api #:hu.dwim.stefil)
+  :depends-on (#:stefil-api #:stefil)
   :components ((:file "stefil-api-impl")))

testsuites/apis-of-test-frameworks/stefil-api-impl.lisp

@@ -11,24 +11,24 @@
     (format t "~&~%Individual failures:~%")
     (map 'nil
          (lambda (descr) (format t "~A~%" descr))
-         (hu.dwim.stefil::failure-descriptions-of test-suite-result)))
+         (stefil::failure-descriptions-of test-suite-result)))
 
 (defun run-test-suite (test-suite-name)
   (let* ((*debug-io* *standard-output*)
-         (result (hu.dwim.stefil:funcall-test-with-feedback-message test-suite-name)))
+         (result (stefil:funcall-test-with-feedback-message test-suite-name)))
     (print-details result)
     result))
 
 (defun test-name (failure-description)
   (let ((hierarchical-names  (mapcar 
                               #'(lambda (context) 
-                                  (hu.dwim.stefil::name-of
-                                   (hu.dwim.stefil::test-of context)))
-                              (hu.dwim.stefil::test-context-backtrace-of failure-description))))
+                                  (stefil::name-of
+                                   (stefil::test-of context)))
+                              (stefil::test-context-backtrace-of failure-description))))
     (format nil "~(~{~A~^.~}~)" (nreverse hierarchical-names))))
 
 (defun failed-tests (test-suite-result)
   (remove-duplicates (map 'list
                           #'test-name
-                          (hu.dwim.stefil::failure-descriptions-of test-suite-result))
+                          (stefil::failure-descriptions-of test-suite-result))
                      :test #'string=))

testsuites/sample-test-suites/hu.dwim.stefil-sample-test-suite.asd

@@ -0,0 +1,11 @@
+;;; -*- Mode: LISP; Syntax: COMMON-LISP; indent-tabs-mode: nil; coding: utf-8;  -*-
+;;;
+;;; Copyright (C) 2011 Anton Vodonosov (avodonosov@yandex.ru)
+;;;
+;;; See LICENSE for details.
+
+(asdf:defsystem #:hu.dwim.stefil-sample-test-suite
+  :version "0.1.0"
+  :serial t
+  :depends-on (#:hu.dwim.stefil)
+  :components ((:file "hu.dwim.stefil-sample-test-suite")))

testsuites/sample-test-suites/hu.dwim.stefil-sample-test-suite.lisp

@@ -0,0 +1,34 @@
+(defpackage #:hu.dwim.stefil-sample-test-suite
+  (:use #:cl :hu.dwim.stefil))
+
+(in-package #:hu.dwim.stefil-sample-test-suite)
+
+(defsuite sample-stefil-suite)
+(in-suite sample-stefil-suite)
+
+(deftest ok-test ()
+  (is (= (+ 2 2) 4)))
+
+(deftest one-fail-test ()
+  (is (= (+ 2 2) 3)))
+
+(deftest two-fails-test ()
+  (is (= (+ 2 2) 3))
+  (is (= (+ 1 1) 2))
+  (is (= (+ 3 3) 7)))
+
+(deftest error-test ()
+  (is (= 7 7))
+  (is (= (/ 1 0) 1)))
+
+(deftest all-fails-expected-test ()
+  (with-expected-failures
+    (is (= (+ 2 2) 3))
+    (is (= (+ 1 1) 2))
+    (is (= (+ 3 3) 7))))
+
+(deftest not-all-fails-expected-test ()
+  (is (= (+ 2 2) 3))
+  (with-expected-failures
+    (is (= (+ 1 1) 2))
+    (is (= (+ 3 3) 7))))

testsuites/sample-test-suites/stefil-sample-test-suite.asd

@@ -7,5 +7,5 @@
 (asdf:defsystem #:stefil-sample-test-suite
   :version "0.1.0"
   :serial t
-  :depends-on (#:hu.dwim.stefil)
+  :depends-on (#:stefil)
   :components ((:file "stefil-sample-test-suite")))

testsuites/sample-test-suites/stefil-sample-test-suite.lisp

@@ -1,5 +1,5 @@
 (defpackage #:stefil-sample-test-suite
-  (:use #:cl :hu.dwim.stefil))
+  (:use #:cl :stefil))
 
 (in-package #:stefil-sample-test-suite)
 

testsuites/testsuites.lisp

@@ -64,6 +64,7 @@ just passed to the QUICKLISP:QUICKLOAD."
                         ("fiveam-api" . "fiveam-api-impl")
                         ("eos-api" . "eos-api-impl")
                         ("stefil-api" . "stefil-api-impl")
+                        ("hu.dwim.stefil-api" . "hu.dwim.stefil-api-impl")
                         ("clunit-api" . "clunit-api-impl")
                         ("nst-api" . "nst-api-impl")))
          (impl-asdf-system (or (cdr (assoc api known-impls :test #'string=))
@@ -137,6 +138,12 @@ just passed to the QUICKLISP:QUICKLOAD."
     (list :failed-tests (stefil-api:failed-tests result)
           :known-to-fail '())))
 
+(defun run-hu.dwim.stefil-test-suite (test-suite-spec)
+  (require-impl "hu.dwim.stefil-api")
+  (let ((result (hu.dwim.stefil-api:run-test-suite test-suite-spec)))
+    (list :failed-tests (hu.dwim.stefil-api:failed-tests result)
+          :known-to-fail '())))
+
 (defun run-clunit-test-suite (test-suite-name)
   (require-impl "clunit-api")
   (let ((result (clunit-api:run-test-suite test-suite-name)))
@@ -215,10 +222,10 @@ just passed to the QUICKLISP:QUICKLOAD."
 
 (defmethod libtest ((library-name (eql :babel)))
 
-  ;; The test framework used: stefil.
+  ;; The test framework used: hu.dwim.stefil.
 
   (quicklisp:quickload :babel-tests)
-  (run-stefil-test-suite (intern (string '#:babel-tests) '#:babel-tests)))
+  (run-hu.dwim.stefil-test-suite (read-from-string "babel-tests::babel-tests")))
 
 (defmethod libtest ((library-name (eql :trivial-features)))
 
@@ -563,9 +570,9 @@ just passed to the QUICKLISP:QUICKLOAD."
   (run-fiveam-test-suite :it.bese.arnesi))
 
 (defmethod libtest ((library-name (eql :local-time)))
-  ;; test framework used: Stefil
+  ;; test framework used: stefil
   (quicklisp:quickload :local-time.test)
-  (run-stefil-test-suite (intern (string '#:test) '#:local-time.test)))
+  (run-stefil-test-suite (read-from-string "local-time.test::test")))
 
 (defmethod libtest ((library-name (eql :s-xml)))
   ;; test framework used: cl:assert
@@ -801,10 +808,10 @@ just passed to the QUICKLISP:QUICKLOAD."
   (run-rt-test-suite))
 
 (defmethod libtest ((library-name (eql :hu.dwim.stefil)))
-  ;; The test framework used: stefil.
+  ;; The test framework used: hu.dwim.stefil.
 
   (ql:quickload :hu.dwim.stefil.test)
-  (run-stefil-test-suite (read-from-string "hu.dwim.stefil.test::test")))
+  (run-hu.dwim.stefil-test-suite (read-from-string "hu.dwim.stefil.test::test")))
 
 (defmethod libtest ((library-name (eql  :kmrcl)))
 
@@ -829,14 +836,14 @@ just passed to the QUICKLISP:QUICKLOAD."
   (run-rt-test-suite))
 
 (defmethod libtest ((library-name (eql :hu.dwim.walker)))
-  ;; The test framework used: stefil.
+  ;; The test framework used: hu.dwim.stefil.
   (ql:quickload :hu.dwim.walker.test)
-  (run-stefil-test-suite (read-from-string "hu.dwim.walker.test::test")))
+  (run-hu.dwim.stefil-test-suite (read-from-string "hu.dwim.walker.test::test")))
 
 (defmethod libtest ((library-name (eql :hu.dwim.defclass-star)))
-  ;; The test framework used: stefil.
+  ;; The test framework used: hu.dwim.stefil.
   (ql:quickload :hu.dwim.defclass-star.test)
-  (run-stefil-test-suite (read-from-string "hu.dwim.defclass-star.test::test")))
+  (run-hu.dwim.stefil-test-suite (read-from-string "hu.dwim.defclass-star.test::test")))
 
 (defmethod libtest ((library-name (eql :bknr-datastore)))
   ;; test framework used: FiveAM
@@ -869,9 +876,9 @@ just passed to the QUICKLISP:QUICKLOAD."
   (run-fiveam-test-suite :it.bese.yaclml))
 
 (defmethod libtest ((library-name (eql :com.google.base)))
-  ;; The test framework used: stefil.
+  ;; The test framework used: hu.dwim.stefil.
   (ql:quickload :com.google.base-test)
-  (run-stefil-test-suite (read-from-string "com.google.base-test::test-base")))
+  (run-hu.dwim.stefil-test-suite (read-from-string "com.google.base-test::test-base")))
 
 (defmethod libtest ((library-name (eql :external-program)))
   ;; test framework used: FiveAM