Permalink
Fetching contributors…
Cannot retrieve contributors at this time
1116 lines (906 sloc) 44.9 KB
;;;; -*- Mode: LISP; Syntax: COMMON-LISP; indent-tabs-mode: nil; coding: utf-8; show-trailing-whitespace: t -*-
;;;; Copyright (C) 2011 Anton Vodonosov (avodonosov@yandex.ru)
;;;; See LICENSE for details.
(defpackage #:test-grid-testsuites
(:nicknames :tg-testsuites :tg-suites)
(:use :cl)
(:export ;; list of the libraries added to test-grid
#:*all-libs*
;; the generic function to be implemented for evey library
#:libtest))
(in-package #:test-grid-testsuites)
(defgeneric libtest (library-name)
(:documentation "Define a method for this function
with LIBRARY-NAME eql-specialized for for every library added
to the test grid.
The method should run test suite and return the resulting
status. Status is one of these values:
:OK - all tests passed,
:FAIL - some test failed,
:NO-RESOURCE - test suite can not be run because some required
resource is absent in the environment. For example, CFFI library
test suite needs a small C library compiled to DLL. User must
do it manually. In case the DLL is absent, the LIBTEST method
for CFFI returns :NO-RESOURCE.
Extended status in the form
(:FAILED-TESTS <list of failed tests> :KNOWN-TO-FAIL <list of known failures>)
The test names in these lists are represented as downcased strings.
If any SERIOUS-CONDITION is signalled, this is considered a failure.
For convenience, T may be returned instead of :OK and NIL instead of :FAIL."))
(defun normalize-status (status)
"Normilzies test resul status - converts T to :OK and NIL to :FAIL."
(case status
((t :ok) :ok)
((nil :fail) :fail)
(otherwise status)))
;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;
;; My Require
;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;
(defun require-impl (api)
"Loads an implementation of the specified API (if
it is not loaded yet).
Some of test-grid components have separate package
and ASDF system for API and separate package+ASDF
system for the implementation. This allows test-grid
to be compilable and (partialuly) opereable even
when some components are broken on particular lisp.
For these known test-grid componetns REQUIRE-IMPL loads
the implementation. Otherwise the API parameter is
just passed to the QUICKLISP:QUICKLOAD."
(setf api (string-downcase api))
(let* ((known-impls '(("rt-api" . "rt-api-impl")
("lift-api" . "lift-api-impl")
("fiveam-api" . "fiveam-api-impl")
("eos-api" . "eos-api-impl")
("stefil-api" . "stefil-api-impl")
("hu.dwim.stefil-api" . "hu.dwim.stefil-api-impl")
("clunit-api" . "clunit-api-impl")
("nst-api" . "nst-api-impl")))
(impl-asdf-system (or (cdr (assoc api known-impls :test #'string=))
api)))
(quicklisp:quickload impl-asdf-system)))
;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;
;; LIBTEST implementations for particular libraries
;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;
(defparameter *all-libs*
'(:alexandria :babel :trivial-features :cffi
:cl-ppcre :usocket :flexi-streams :bordeaux-threads
:cl-base64 :cl-fad :trivial-backtrace :puri
:anaphora :parenscript :trivial-garbage :iterate
:metabang-bind :cl-json :cl-containers :metatilities-base
:cl-cont :moptilities :trivial-timeout :metatilities
:named-readtables :arnesi :local-time :s-xml
:cl-oauth :cl-routes :cl-unicode :fiveam
:trivial-utf-8 :yason :cl-annot :cl-openid
:split-sequence :cl-closure-template :cl-interpol :trivial-shell
:let-plus :data-sift :cl-num-utils :ieee-floats
:cl-project :trivial-http :cl-store :hu.dwim.stefil
:kmrcl :cxml-stp :hu.dwim.walker :hu.dwim.defclass-star
:bknr-datastore :yaclml :com.google.base :external-program
:cl-mustache :trivial-gray-streams :drakma :optima
:cl-6502 :doplus :nst :track-best
:cleric :cl-erlang-term :stmx :cl-epmd
:bencode :jsown :nibbles :cl-custom-hash-table
:hyperluminal-mem :cl-python :cl-slug)
"All the libraries, testsuites of which we know how to run.")
(defun clean-rt (&optional (rt-package :rtest))
(require-impl "rt-api")
(rt-api:clean rt-package))
(defun run-rt-test-suite(&optional (rt-package :rtest))
(require-impl "rt-api")
(let (non-compiled-failures all-failures)
(rt-api:do-tests :compiled-p nil :rt-package rt-package)
(setf non-compiled-failures (rt-api:failed-tests rt-package))
(rt-api:do-tests :compiled-p t :rt-package rt-package)
(setf all-failures (union non-compiled-failures
(rt-api:failed-tests rt-package)))
(list :failed-tests (mapcar #'string-downcase all-failures)
:known-to-fail (mapcar #'string-downcase (rt-api:known-to-fail rt-package)))))
(defun run-lift-test-suite (test-suite-name)
(require-impl "lift-api")
(let ((result (lift-api:run-test-suite test-suite-name)))
(list :failed-tests (lift-api:failed-tests result)
:known-to-fail (lift-api:known-to-fail test-suite-name))))
(defun run-fiveam-test-suite (test-suite-spec)
(require-impl "fiveam-api")
(let ((result (fiveam-api:run-test-suite test-suite-spec)))
(list :failed-tests (fiveam-api:failed-tests result)
:known-to-fail '())))
(defun run-eos-test-suites (&rest test-suite-specs)
(require-impl "eos-api")
(let ((result (apply #'eos-api:run-test-suites test-suite-specs)))
(list :failed-tests (eos-api:failed-tests result)
:known-to-fail '())))
(defun run-stefil-test-suite (test-suite-spec)
(require-impl "stefil-api")
(let ((result (stefil-api:run-test-suite test-suite-spec)))
(list :failed-tests (stefil-api:failed-tests result)
:known-to-fail '())))
(defun run-hu.dwim.stefil-test-suite (test-suite-spec)
(require-impl "hu.dwim.stefil-api")
(let ((result (hu.dwim.stefil-api:run-test-suite test-suite-spec)))
(list :failed-tests (hu.dwim.stefil-api:failed-tests result)
:known-to-fail '())))
(defun run-clunit-test-suite (test-suite-name)
(require-impl "clunit-api")
(let ((result (clunit-api:run-test-suite test-suite-name)))
(list :failed-tests (clunit-api:failed-tests result)
:known-to-fail '())))
(defun run-nst-test-suites (&rest test-suite-names)
(require-impl "nst-api")
(let ((result (apply #'nst-api:run-test-suites test-suite-names)))
(list :failed-tests (nst-api:failed-tests result)
:known-to-fail '())))
(defun running-cl-test-more-suite (project-name runner-function)
;; cl-test-more test suites usually run tests during
;; the load time.
;;
;; cl-test-more produces text output in
;; the TAP (Test Anhything Protocol),
;; and does not produce any other programmaticaly
;; inspectable value.
;;
;; We will intercept the test output, and
;; interpret it according to the TAP
;; format (we limit this by just
;; looking for strings starting with "not ok"
;; in the output).
;; Intercepting cl-test-more TAP output
(quicklisp:quickload :cl-test-more)
(let ((test-output-buf (make-string-output-stream)))
(progv
(list (read-from-string "cl-test-more:*test-result-output*"))
(list (make-broadcast-stream *standard-output* test-output-buf))
(funcall runner-function))
;; Now look for a strting starting from "not ok"
;; in the test output
(with-input-from-string (test-output (get-output-stream-string test-output-buf))
(do ((line (read-line test-output nil) (read-line test-output nil)))
((null line))
(when (test-grid-utils::starts-with line "not ok")
(format t "---------------------------------------------------------------------------~%")
(format t "~A test suite has a test failure; the first TAP output failure string:~%~A"
project-name line)
(return-from running-cl-test-more-suite :fail))))
:ok))
(defun combine-extended-libresult (libresult-a libresult-b)
(list :failed-tests (union (getf libresult-a :failed-tests)
(getf libresult-b :failed-tests)
:test #'string=)
:known-to-fail (union (getf libresult-a :known-to-fail)
(getf libresult-b :known-to-fail)
:test #'string=)))
(assert (let ((combined (combine-extended-libresult '(:failed-tests ("a" "b")
:known-to-fail ("c"))
'(:failed-tests ("a2" "c2")
:known-to-fail ("b2")))))
(and (test-grid-utils:set= '("a" "b" "a2" "c2")
(getf combined :failed-tests)
:test #'string=)
(test-grid-utils:set= '("c" "b2")
(getf combined :known-to-fail)
:test #'string=))))
(defmethod libtest ((library-name (eql :alexandria)))
;; The test framework used: rt.
(clean-rt #+sbcl :sb-rt #-sbcl :rtest)
(asdf:clear-system :alexandria-tests)
(quicklisp:quickload :alexandria-tests)
(run-rt-test-suite #+sbcl :sb-rt #-sbcl :rtest))
(defmethod libtest ((library-name (eql :babel)))
;; The test framework used: hu.dwim.stefil.
(quicklisp:quickload :babel-tests)
(run-hu.dwim.stefil-test-suite (read-from-string "babel-tests::babel-tests")))
(defmethod libtest ((library-name (eql :trivial-features)))
;; The test framework used: rt.
(clean-rt)
(asdf:clear-system :trivial-features-tests)
;; Load cffi-grovel which is used in trivial-features-tests.asd,
;; but not in the asdf:defsystem macro (issue #2).
(quicklisp:quickload :cffi-grovel)
(quicklisp:quickload :trivial-features-tests)
(run-rt-test-suite))
(defmethod libtest ((library-name (eql :cffi)))
;; The test framework used: rt.
(clean-rt)
(asdf:clear-system :cffi-tests)
;; Workaround for the quicklisp bug #55
;; (https://github.com/quicklisp/quicklisp-client/issues/55)
(ql:quickload :cffi)
(let ((cffi-dir (make-pathname :name nil :type nil :defaults (ql-dist:find-asdf-system-file "cffi"))))
(pushnew cffi-dir asdf:*central-registry* :test #'equal))
;; now (ql:quickload "cffi-tests") will work
;; CFFI tests work with a small test C library.
;; The user is expected to compile the library manually.
;; If the library is not available, CFFI tests
;; signal cffi:load-foreign-library-error.
(handler-bind ((error #'(lambda (condition)
;; Check if the error is a
;; cffi:load-foreign-library-error.
;; Take into account that it might
;; be some other error which prevents
;; even CFFI system to load,
;; and therefore CFFI package may be
;; absent. That's why use use ignore-errors
;; when looking for a symbol in the CFFI
;; packge.
(when (eq (type-of condition)
(ignore-errors (find-symbol (symbol-name '#:load-foreign-library-error)
'#:cffi)))
;; todo: add the full path to the 'test' directory,
;; where user can find the scripts to compile
;; the test C library, to the error message.
(format t
"~& An error occurred during (quicklisp:quickload :cffi-tests):~%~A~&This means the small C library used by CFFI tests is not available (probably you haven't compiled it). The compilation script for the library may be found in the 'test' directory in the CFFI distribution.~%"
condition)
(return-from libtest :no-resource))
;; resignal the condition
(error condition))))
(quicklisp:quickload :cffi-tests))
(run-rt-test-suite))
(defmethod libtest ((library-name (eql :cl-ppcre)))
#+abcl
(progn
(format t "~&Due to ABCL bug #188 (http://trac.common-lisp.net/armedbear/ticket/188)~%")
(format t "cl-ppcre tests fail, repeating this error huge number of times, in result~%")
(format t "hanging for a long time and producing a huge log.")
(return-from libtest :fail))
;; The test framework used: custom.
;; Workaround the quicklisp issue #225 -
;; https://github.com/quicklisp/quicklisp-projects/issues/225 -
;; first load cl-ppcre-unicode, because otherwise
;; current quicklisp can not find cl-ppcre-unicode-test
(quicklisp:quickload :cl-ppcre-unicode)
(quicklisp:quickload :cl-ppcre-unicode-test)
;; copy/paste from cl-ppcre-unicode.asd
(funcall (intern (symbol-name :run-all-tests) (find-package :cl-ppcre-test))
:more-tests (intern (symbol-name :unicode-test) (find-package :cl-ppcre-test))))
(defmethod libtest ((library-name (eql :usocket)))
;; The test framework used: rt.
(clean-rt)
(asdf:clear-system :usocket-test)
(quicklisp:quickload :usocket-test)
;; TODO: usocket test suite might need manual configuration,
;; see their README. Distinguish the case
;; when the manual configuration hasn't been
;; performed and return :no-resource status.
;;
;; (setf usocket-test::*common-lisp-net*
;; (or usocket-test::*common-lisp-net*
;; "74.115.254.14"))
(run-rt-test-suite))
(defmethod libtest ((library-name (eql :flexi-streams)))
;; The test framework used: custom.
(quicklisp:quickload :flexi-streams-test)
;; copy/paste from flexi-streams.asd
(funcall (intern (symbol-name :run-all-tests)
(find-package :flexi-streams-test))))
(defmethod libtest ((library-name (eql :bordeaux-threads)))
;; The test framework used: fiveam.
(handler-case (ql:quickload :bordeaux-threads/test)
;; in old versions the systems was named :bordeaux-threads-test
(ql:system-not-found () (ql:quickload :bordeaux-threads-test)))
(run-fiveam-test-suite :bordeaux-threads))
(defmethod libtest ((library-name (eql :cl-base64)))
;; The test framework used: ptester.
;; Load cl-base64 first, because cl-base64-tests
;; is defined in cl-base64.asd which confuses
;; quicklisp (some versions of, see issue #1)
(quicklisp:quickload :cl-base64)
(quicklisp:quickload :cl-base64-tests)
(funcall (intern (symbol-name '#:do-tests)
(find-package '#:cl-base64-tests))))
(defmethod libtest ((library-name (eql :cl-fad)))
;; The test framework used: cl:assert + unit-test.
(quicklisp:quickload :cl-fad)
;; loading the tests code:
(or
;; in the most recent version of cl-fad, the tests have their own ASDF system
(ignore-errors (quicklisp:quickload :cl-fad-test) t)
;; in older versions the tests are just kept in a single file
(let ((test-file (or (probe-file (asdf:system-relative-pathname :cl-fad "fad.test.lisp"))
;; in the oldest version the file was named differently
(probe-file (asdf:system-relative-pathname :cl-fad "test.lisp")))))
(when test-file
(load test-file)))
(error "Can not load cl-fad-test system, and can not find fad.test.lisp nor test.lisp in the cl-fad sources"))
;; running the tests
(let (
;; main cl-fad test suite uses cl:assert.
;; I.e. any test faifures are signaled as errors.
(main-ok-p (handler-case
(progn
(format t "~&~%;;; Running main tests~%~%")
(funcall (read-from-string "cl-fad-test:test"))
t)
(error (e)
(format t "cl-fad test suite failed with error: ~A" e)
nil)))
;; newer version of cl-fad has tests for temporary file functionality,
;; these new tests are implemented using unit-test framework
(temp-file-ok-p (if (find-symbol (string 'temporary-file) 'cl-fad-test)
(progn
(format t "~&~%;;; Running temporary file tests~%~%")
(funcall (read-from-string "unit-test:run-all-tests")
:unit
(read-from-string "cl-fad-test::temporary-file")))
t)))
(and main-ok-p temp-file-ok-p)))
(defmethod libtest ((library-name (eql :trivial-backtrace)))
;; The test framework used: lift.
(quicklisp:quickload :trivial-backtrace-test)
(run-lift-test-suite :trivial-backtrace-test))
(defmethod libtest ((library-name (eql :puri)))
;; The test framework used: ptester.
;; Load puri first, because puri-tests
;; is defined in puri.asd which confuses
;; quicklisp (some versions of, see issue #1)
(quicklisp:quickload :puri)
(quicklisp:quickload :puri-tests)
;; copy/paste from puri.asd
(funcall (intern (symbol-name '#:do-tests)
(find-package :puri-tests))))
(defmethod libtest ((library-name (eql :anaphora)))
;; The test framework used: rt.
(clean-rt)
(asdf:clear-system :anaphora-test)
;; anaphora-test is defined in anaphora.asd,
;; therefore to reload :anaphora-test
;; we need to clean the :anaphora system too
(asdf:clear-system :anaphora)
;; Load anaphora first, because anaphora-tests
;; is defined in anaphora.asd which confuses
;; quicklisp (some versions of, see issue #1)
(quicklisp:quickload :anaphora)
(quicklisp:quickload :anaphora-test)
(run-rt-test-suite))
(defmethod libtest ((library-name (eql :parenscript)))
;; The test framework used: eos (similar to FiveAM).
(quicklisp:quickload :parenscript.test)
;; asdf:test-op is not provided for parenscript,
;; only a separate package ps-test with public
;; function run-tests.
;; The test suites to run are taken from the
;; the function run-tests in the file
;; <parenscript sources>/t/test.lisp.
(run-eos-test-suites (intern (string '#:output-tests) :ps-test)
(intern (string '#:package-system-tests) :ps-test)
(intern (string '#:eval-tests) :ps-test)))
(defmethod libtest ((library-name (eql :trivial-garbage)))
;; The test framework used: rt.
(clean-rt)
(asdf:clear-system :trivial-garbage) ; yes, trivial-garbage but not trivial-garbage-tests,
; because the trivial-garbage-tests system is defined
; in the same trivial-garbage.asd and neither
; asdf nor quicklisp can't find trivial-garbage-tests.
(quicklisp:quickload :trivial-garbage); trivial-garbage but not trivial-garbage-tests,
; for the same reasons as explained above.
(asdf:operate 'asdf:load-op :trivial-garbage-tests)
(run-rt-test-suite))
(defmethod libtest ((library-name (eql :iterate)))
;; The test framework used: rt.
(clean-rt #+sbcl :sb-rt #-sbcl :rtest)
(asdf:clear-system :iterate-tests)
(asdf:clear-system :iterate)
;; Load iterate first, because iterate-tests
;; is defined in iterate.asd which confuses
;; quicklisp (some versions of, see issue #1)
(quicklisp:quickload :iterate)
(quicklisp:quickload :iterate-tests)
(run-rt-test-suite #+sbcl :sb-rt #-sbcl :rtest))
(defmethod libtest ((library-name (eql :metabang-bind)))
;; The test framework used: lift.
;; metabang-bind-test includes binding syntax
;; for regular-expression and corresponding
;; tests; but this functionality is only
;; loaded if cl-ppcre is loaded first.
;; (this conditional loading is achieaved
;; with asdf-system-connections).
(quicklisp:quickload :cl-ppcre)
(quicklisp:quickload :metabang-bind-test)
(run-lift-test-suite :metabang-bind-test))
(defmethod libtest ((library-name (eql :cl-json)))
;; The test framework used: fiveam.
(let ((*trace-output* *standard-output*))
;; Load cl-json first, because cl-json.tests
;; is defined in cl-json.asd which confuses
;; quicklisp (some versions of, see issue #1)
(quicklisp:quickload :cl-json)
(quicklisp:quickload :cl-json.test)
(run-fiveam-test-suite (intern (symbol-name '#:json) :json-test))))
(defmethod libtest ((library-name (eql :cl-containers)))
;; The test framework used: lift.
(quicklisp:quickload :cl-containers-test)
(run-lift-test-suite :cl-containers-test))
(defmethod libtest ((library-name (eql :metatilities-base)))
;; The test framework used: lift.
;; Instead of just doing
;; (quicklisp:quickload :metatilities-base-test)
;; we need to workarount quicklisp issue
;; https://github.com/quicklisp/quicklisp-client/issues/58.
(ql:quickload "metatilities-base")
(let* ((lib-dir (make-pathname :name nil
:type nil
:defaults (ql-dist:find-asdf-system-file "metatilities-base")))
(asdf:*central-registry* (cons lib-dir asdf:*central-registry*)))
(ql:quickload "metatilities-base-test"))
;; now run the tests
(run-lift-test-suite :metatilities-base-test))
(defmethod libtest ((library-name (eql :cl-cont)))
;; The test framework used: rt.
(clean-rt)
(asdf:clear-system :cl-cont-test)
(quicklisp:quickload :cl-cont-test)
(run-rt-test-suite))
(defmethod libtest ((library-name (eql :moptilities)))
;; The test framework used: lift.
(quicklisp:quickload :moptilities-test)
(run-lift-test-suite :moptilities-test))
(defmethod libtest ((library-name (eql :trivial-timeout)))
;; The test framework used: lift.
(quicklisp:quickload :trivial-timeout-test)
(run-lift-test-suite :trivial-timeout-test))
(defmethod libtest ((library-name (eql :metatilities)))
;; The test framework used: lift.
(quicklisp:quickload :metatilities-test)
(run-lift-test-suite :metatilities-test))
(defmethod libtest ((library-name (eql :named-readtables)))
;; test framework used: customized RT
(clean-rt :named-readtables-test)
(asdf:clear-system :named-readtables)
(asdf:clear-system :named-readtables-test)
(quicklisp:quickload :named-readtables)
(handler-case (ql:quickload :named-readtables/test)
;; in old versions the systems was named :named-readtables-test
(ql:system-not-found () (ql:quickload :named-readtables-test)))
(run-rt-test-suite :named-readtables-test))
(defmethod libtest ((library-name (eql :arnesi)))
;; test framework used: FiveAM
(quicklisp:quickload :arnesi)
(quicklisp:quickload :arnesi.test)
(run-fiveam-test-suite :it.bese.arnesi))
(defmethod libtest ((library-name (eql :local-time)))
;; test framework used: stefil
(quicklisp:quickload :local-time.test)
(run-stefil-test-suite (read-from-string "local-time.test::test")))
(defmethod libtest ((library-name (eql :s-xml)))
;; test framework used: cl:assert
(quicklisp:quickload :s-xml)
;; s-xml test suite uses cl:assert, and all
;; the assertsions are top level, i.e. executed
;; immediatelly during the system load
(handler-case
(progn
(asdf:operate 'asdf:load-op :s-xml.test :force t)
:ok)
(error (e)
(format t "s-xml test suite failed with error: ~A" e)
:fail)))
(defun is-windows ()
(member :asdf-windows *features* :test #'eq))
;; See coverage.org for more info why IO lib is not added.
;;
;; (defmethod libtest ((library-name (eql :iolib)))
;; ;; test framework used: FiveAM
;;
;; (cond ((is-windows)
;; (format t "IOLib is not implemented for Windows.~%")
;; :no-resource)
;; (t
;; (quicklisp:quickload :iolib-tests)
;; (run-fiveam-test-suite :iolib))))
(defmethod libtest ((library-name (eql :cl-oauth)))
;; test framework used: FiveAM
(quicklisp:quickload :cl-oauth)
(quicklisp:quickload :cl-oauth.tests)
;; the code is based on the method
;; (defmethod asdf:perform ((o asdf:test-op) (c (eql (asdf:find-system :cl-oauth))))
;; from the cl-oauth sources
(let ((original-request-adapter (symbol-value (read-from-string "cl-oauth:*request-adapter*"))))
(unwind-protect
(progn
(funcall (read-from-string "oauth-test::init-test-request-adapter"))
(run-fiveam-test-suite (read-from-string "oauth-test::oauth")))
(setf (symbol-value (read-from-string "cl-oauth:*request-adapter*"))
original-request-adapter))))
(defmethod libtest ((library-name (eql :cl-routes)))
;; The test framework used: lift.
(quicklisp:quickload :routes)
(quicklisp:quickload :routes-test)
;; good way to refer symbols, thanks cl-routes
(run-lift-test-suite (read-from-string "routes.test::routes-test")))
(defmethod libtest ((library-name (eql :cl-unicode)))
;; The test framework used: custom.
(quicklisp:quickload :cl-unicode)
(quicklisp:quickload :cl-unicode-test)
(funcall (read-from-string "cl-unicode-test:run-all-tests")))
(defmethod libtest ((library-name (eql :fiveam)))
;; test framework used: FiveAM
(quicklisp:quickload :fiveam)
(run-fiveam-test-suite :it.bese.fiveam))
(defmethod libtest ((library-name (eql :trivial-utf-8)))
;; test framework used: cl:assert
(quicklisp:quickload :trivial-utf-8)
;; trivial-utf-8-tests test suite uses cl:assert, and all
;; the assertsions are top level, i.e. executed
;; immediatelly during the system load
(handler-case
(progn
(asdf:operate 'asdf:load-op :trivial-utf-8-tests :force t)
:ok)
(error (e)
(format t "trivial-utf-8 test suite failed with error: ~A" e)
:fail)))
(defmethod libtest ((library-name (eql :yason)))
;; test framework used: unit-test
;; Handle package name conflict between cl-json
;; and yason: the package of cl-json is named :json,
;; the package of yason is named :yason, but has
;; :json as a nickname.
;; Temporary rename the :json package if it's loaded
(when (and (find-package :json)
;; make sure it's found not by a nickname
(string= :json (package-name :json)))
(rename-package :json :json-temp-uinuque-name))
(quicklisp:quickload :yason)
;; it doesn't provide an ASDF system for tests,
;; load the test framework manually, and then
;; the test.lisp file.
(quicklisp:quickload :unit-test)
(let* ((yason-dir (make-pathname :name nil :type nil :defaults (ql-dist:find-asdf-system-file "yason")))
(yason-test-file (merge-pathnames "test.lisp" yason-dir)))
(format t "loading ~A~%" yason-test-file)
(load yason-test-file))
;; remove nicknames from the :yason package
(rename-package :yason :yason nil)
;; rename :json back to it's original name
(when (find-package :json-temp-uinuque-name)
(rename-package :json-temp-uinuque-name :json))
;; now run the tests. It returns a boolean
(funcall (read-from-string "unit-test:run-all-tests") :unit :yason))
(defmethod libtest ((library-name (eql :cl-annot)))
;; test framework used: cl-test-more
(running-cl-test-more-suite "cl-annot"
#'(lambda ()
;; ensure it is reloaded, even if it was already loaded before
(asdf:clear-system :cl-annot)
(asdf:clear-system :cl-annot-test)
(quicklisp:quickload :cl-annot-test))))
(defmethod libtest ((library-name (eql :cl-openid)))
;; test framework used: FiveAM
(ql:quickload :cl-openid)
(ql:quickload :cl-openid.test)
(run-fiveam-test-suite :cl-openid))
(defmethod libtest ((library-name (eql :split-sequence)))
;; test framework used: FiveAM
(ql:quickload :split-sequence)
(ql:quickload :split-sequence-tests)
(run-fiveam-test-suite :split-sequence))
(defmethod libtest ((library-name (eql :cl-closure-template)))
;; The test framework used: lift.
(ql:quickload :closure-template)
(ql:quickload :closure-template-test)
(run-lift-test-suite (read-from-string "closure-template.test::closure-template-test")))
(defmethod libtest ((library-name (eql :cl-interpol)))
#+abcl
(progn
(format t "~&We do not run cl-interpol tests on ABCL, but just report a hardcoded failure,~%")
(format t "because as of ABCL 1.0.1 cl-interpol crashes ABCL by ~%")
(format t "\"java.lang.OutOfMemoryError: Java heap space\" and before that~%")
(format t "produces megabytes of logs containing the error message:~%")
(format t " got an unexpected error: The value #<FLEXI-STREAMS:FLEXI-INPUT-STREAM {56E7A30E}> is not of type STREAM.~%")
(format t "Increasing JVM heap size does not help - it just takes longer and produces more~%")
(format t "logs, crashing finally with OutOfMemory anyway.~%")
(return-from libtest :fail))
;; The test framework used: custom.
(ql:quickload :cl-interpol)
(ql:quickload :cl-interpol-test)
(funcall (read-from-string "cl-interpol-test:run-all-tests")))
;; Decided not to add Lift now.
;; See coverage.org for details.
;;
;; (defmethod libtest ((library-name (eql :lift)))
;; ;; The test framework used: lift.
;; (ql:quickload :lift)
;; (ql:quickload :lift-test)
;; (run-lift-test-suite (read-from-string "lift-test::lift-test")))
(defmethod libtest ((library-name (eql :trivial-shell)))
;; The test framework used: lift.
(cond ((is-windows)
(format t "trivial-shell is not implemented for Windows.~%")
:no-resource)
(t
(ql:quickload :trivial-shell)
(ql:quickload :trivial-shell-test)
(run-lift-test-suite (read-from-string "trivial-shell-test::trivial-shell-test")))))
(defmethod libtest ((library-name (eql :let-plus)))
;; The test framework used: lift.
(ql:quickload :let-plus)
(ql:quickload :let-plus-tests)
(run-lift-test-suite (read-from-string "let-plus-tests::let-plus-tests")))
(defmethod libtest ((library-name (eql :data-sift)))
;; The test framework used: lift.
(ql:quickload :data-sift)
(ql:quickload :data-sift-test)
(run-lift-test-suite (read-from-string "data-sift.test::data-sift-test")))
(defmethod libtest ((library-name (eql :cl-num-utils)))
;; The test framework used: lift.
(ql:quickload :cl-num-utils)
(ql:quickload :cl-num-utils-tests)
(if (member "lift"
(asdf::component-load-dependencies (asdf:find-system :cl-num-utils-tests))
:test #'string-equal)
(run-lift-test-suite (read-from-string "cl-num-utils-tests::cl-num-utils-tests"))
(run-clunit-test-suite (read-from-string "cl-num-utils-tests::tests"))))
(defmethod libtest ((library-name (eql :ieee-floats)))
;; test framework used: FiveAM
(ql:quickload :ieee-floats)
(ql:quickload :ieee-floats-tests)
(run-fiveam-test-suite :ieee-floats))
(defmethod libtest ((library-name (eql :cl-project)))
;; test framework used: cl-test-more
(running-cl-test-more-suite "cl-project"
#'(lambda ()
(ql:quickload :cl-project)
(ql:quickload :cl-project-test))))
(defmethod libtest ((library-name (eql :trivial-http)))
;; The test framework used: lift.
(ql:quickload :trivial-http)
(ql:quickload :trivial-http-test)
(run-lift-test-suite :trivial-http-test))
(defmethod libtest ((library-name (eql :cl-store)))
;; The test framework used: rt.
(clean-rt)
(asdf:clear-system :cl-store)
(asdf:clear-system :cl-store-tests)
(ql:quickload :cl-store)
(ql:quickload :cl-store-tests)
(run-rt-test-suite))
(defmethod libtest ((library-name (eql :hu.dwim.stefil)))
;; The test framework used: hu.dwim.stefil.
(ql:quickload :hu.dwim.stefil.test)
(run-hu.dwim.stefil-test-suite (read-from-string "hu.dwim.stefil.test::test")))
(defmethod libtest ((library-name (eql :kmrcl)))
;; The test framework used: rt.
(clean-rt)
(asdf:clear-system :kmrcl-tests)
(ql:quickload :kmrcl-tests)
(run-rt-test-suite))
(defmethod libtest ((library-name (eql :cxml-stp)))
;; The test framework used: rt.
(clean-rt)
(asdf:clear-system :cxml-stp)
(asdf:clear-system :cxml-stp-test)
(ql:quickload :cxml-stp)
(ql:quickload :cxml-stp-test)
(run-rt-test-suite))
(defmethod libtest ((library-name (eql :hu.dwim.walker)))
;; The test framework used: hu.dwim.stefil.
(ql:quickload :hu.dwim.walker.test)
(run-hu.dwim.stefil-test-suite (read-from-string "hu.dwim.walker.test::test")))
(defmethod libtest ((library-name (eql :hu.dwim.defclass-star)))
;; The test framework used: hu.dwim.stefil.
(ql:quickload :hu.dwim.defclass-star.test)
(run-hu.dwim.stefil-test-suite (read-from-string "hu.dwim.defclass-star.test::test")))
(defmethod libtest ((library-name (eql :bknr-datastore)))
;; test framework used: FiveAM
(ql:quickload :bknr.datastore)
(ql:quickload :bknr.datastore.test)
(run-fiveam-test-suite :bknr.datastore))
(defmethod libtest ((library-name (eql :yaclml)))
;; test framework used: FiveAM
;; Handle package name conflict between s-xml and
;; yaclml. s-xml creates a package named :xml,
;; while yaclml uses :xml as a nickname for it's package.
;; Temporary rename the :xml package if it's loaded
(when (and (find-package :xml)
;; make sure it's found not by a nickname
(string= :xml (package-name :xml)))
(rename-package :xml :xml-temp-uinuque-name))
(ql:quickload :yaclml)
;; remove nicknames from the :yaclml package
(rename-package :it.bese.yaclml.xml :it.bese.yaclml.xml nil)
;; rename :json back to it's original name
(when (find-package :xml-temp-uinuque-name)
(rename-package :xml-temp-uinuque-name :xml))
(ql:quickload :yaclml.test)
(run-fiveam-test-suite :it.bese.yaclml))
(defmethod libtest ((library-name (eql :com.google.base)))
;; The test framework used: hu.dwim.stefil.
(ql:quickload :com.google.base-test)
(run-hu.dwim.stefil-test-suite (read-from-string "com.google.base-test::test-base")))
(defmethod libtest ((library-name (eql :external-program)))
;; test framework used: FiveAM
(when (is-windows)
(format t "The external-program test suite uses unix shell commands, like cd, which, and therefor can not be tested on Windows.")
(return-from libtest :no-resource))
(ql:quickload :external-program)
(ql:quickload :external-program-test)
(run-fiveam-test-suite (read-from-string "external-program-tests::tests")))
;; see coverage.org for details why weblocks is not included into the test grid
;;
;; (defmethod libtest ((library-name (eql :weblocks)))
;; ;; The test framework used: lift.
;;
;; (when (find-package :yason)
;; (warn "Deleting package :yason and ASDF system :yason because weblocks depends on cl-json and cl-json has a package name conflict with yason (cl-json main package name is :json, and yason has nickname :json")
;; (delete-package :yason)
;; (asdf:clear-system :yason))
;;
;; (ql:quickload :weblocks-test)
;; (ql:quickload :weblocks-store-test)
;;
;; (combine-extended-libresult (run-lift-test-suite :weblocks-suite)
;; (run-lift-test-suite :store-suite)))
(defmethod libtest ((library-name (eql :cl-mustache)))
;; test framework used: cl-test-more
(running-cl-test-more-suite "cl-mustache"
#'(lambda ()
(quicklisp:quickload :cl-mustache-test)
(funcall (read-from-string "mustache-test:run")))))
(defmacro fncall (funname &rest args)
`(funcall (read-from-string ,funname) ,@args))
(defmethod libtest ((library-name (eql :trivial-gray-streams)))
;; test framework used: custom
(handler-case (ql:quickload :trivial-gray-streams-test)
(serious-condition (c)
(format t "~A~%" c)
(format t "Can't load trivial-gray-streams-test. Most likely we deal with an old version of trivial-gray-streams where test suite is not implemented yet.~%")
(RETURN-FROM libtest :no-resource)))
(list :failed-tests (fncall "trivial-gray-streams-test:failed-test-names"
(fncall "trivial-gray-streams-test:run-tests"))
:known-to-fail nil))
(defmethod libtest ((library-name (eql :drakma)))
;; The test framework used: fiveam.
(handler-case (ql:quickload :drakma-test)
(serious-condition (c)
(format t "~A~%" c)
(format t "Can't load drakma-test. Most likely we deal with an old version of drakma where test suite is not implemented yet.~%")
(RETURN-FROM libtest :no-resource)))
(run-fiveam-test-suite :drakma))
(defmethod libtest ((library-name (eql :optima)))
;; The test framework used: eos.
(ql:quickload :optima.test)
(run-eos-test-suites (read-from-string "optima.test::optima-test")))
(defmethod libtest ((library-name (eql :cl-6502)))
;; The test framework used: fiveam.
(quicklisp:quickload :cl-6502)
(handler-case (ql:quickload :cl-6502-test)
;; in old versions the system was named cl-6502-tests
(ql:system-not-found ()
(format t "Failed to load cl-6502-test - ql:system-not-found. Probably we deal with an old version where the system was called cl-6502-testS. Trying it...~%")
(ql:quickload :cl-6502-tests)))
(run-fiveam-test-suite (read-from-string "6502-tests::6502-tests")))
(defmethod libtest ((library-name (eql :doplus)))
;; The test framework used: eos (similar to FiveAM).
(quicklisp:quickload :doplus-tests)
(run-eos-test-suites (read-from-string "doplus-tests::doplus-suite")))
(defmethod libtest ((library-name (eql :nst)))
;; The test framework used: nst.
(quicklisp:quickload :nst)
;; Quicklisp (as of 2013-06-15) can not find many ASDF
;; systems located in subdirectories of nst project.
;; Workaround this by putting necesary directories
;; into ASDF:*CENTRAL-REGISTRY*
(let ((asdf:*central-registry* (append (mapcar (lambda (subdir)
(asdf:system-relative-pathname :nst subdir))
'("test/asdf/"
"test/direct/"
"test/lisp/comp-set/"
"test/manual/"
"test/meta/"
"test/"
"test/util/"))
asdf:*central-registry*)))
(quicklisp:quickload :nst-test))
(run-nst-test-suites :nst-test
:nst-test-utils
:nst-simple-tests
:nst-simple-tests-interpreted
:nst-meta
#+not :nst-meta-sources
#+not :nst-meta-sources-1
#+not :nst-methods-meta-sources))
(defmethod libtest ((library-name (eql :track-best)))
;; The test framework used: nst.
(quicklisp:quickload :track-best-tests)
(run-nst-test-suites :track-best-tests))
(defmethod libtest ((library-name (eql :cleric)))
;; The test framework used: fiveam.
;; Note, since quicklisp 2013-12-12 it is enough to just
;;
;; (quicklisp:quickload :cleric-test)
;; (run-fiveam-test-suite (read-from-string "cleric-test:all-tests"))
;;
;; but we keep workarounds for old problems to be able to run
;; tests on older Quicklisp. With time we will remove the workarounds
;; and keep only the simple code above.
;; See issue #21.
(quicklisp:quickload :cleric)
(quicklisp:quickload :cleric-test)
(run-fiveam-test-suite (if (find-symbol (string 'all-tests) 'cleric-test)
(read-from-string "cleric-test:all-tests")
(read-from-string "cleric-test::cleric"))))
(defmethod libtest ((library-name (eql :cl-erlang-term)))
;; The test framework used: fiveam.
;; Note, since quicklisp 2013-12-12 it is enough to just
;;
;; (quicklisp:quickload :erlang-term-test)
;; (run-fiveam-test-suite (read-from-string "erlang-term-test:all-tests"))
;;
;; but we keep workarounds for old problems to be able to run
;; tests on older Quicklisp. With time we will remove the workarounds
;; and keep only the simple code above.
;; See issue #21.
(quicklisp:quickload :erlang-term)
(let ((asdf:*central-registry* (cons (asdf:system-source-directory :erlang-term)
asdf:*central-registry*)))
(quicklisp:quickload :erlang-term-test))
(if (find-symbol (string 'all-tests) 'erlang-term-test)
(run-fiveam-test-suite (read-from-string "erlang-term-test:all-tests"))
(reduce #'combine-extended-libresult
(mapcar #'run-fiveam-test-suite
(mapcar #'read-from-string
'("erlang-term-test::bops"
"erlang-term-test::decode"
"erlang-term-test::encode"
"erlang-term-test::erlang-object"
"erlang-term-test::erlang-translatable"))))))
(defmethod libtest ((library-name (eql :stmx)))
;; The test framework used: fiveam.
(quicklisp:quickload :stmx)
(quicklisp:quickload :stmx.test)
(run-fiveam-test-suite (read-from-string "stmx.test::suite")))
(defmethod libtest ((library-name (eql :cl-epmd)))
;; The test framework used: fiveam.
;; Note, since quicklisp 2013-12-12 it is enough to just
;;
;; (quicklisp:quickload :epmd-test)
;; (run-fiveam-test-suite (read-from-string "epmd-test:all-tests"))
;;
;; but we keep workarounds for old problems to be able to run
;; tests on older Quicklisp. With time we will remove the workarounds
;; and keep only the simple code above.
;; See issue #21.
(quicklisp:quickload :epmd-test)
(if (find-symbol (string 'all-tests) 'epmd-test)
(run-fiveam-test-suite (read-from-string "epmd-test:all-tests"))
(reduce #'combine-extended-libresult
(mapcar #'run-fiveam-test-suite
(mapcar #'read-from-string '("epmd-test::epmd-protocol"
"epmd-test::epmd-client"
"epmd-test::epmd-server"))))))
(defmethod libtest ((library-name (eql :bencode)))
;; test framework used: hu.dwim.stefil.
(quicklisp:quickload :bencode-test)
(run-hu.dwim.stefil-test-suite (read-from-string "bencode-test::test-all")))
(defmethod libtest ((library-name (eql :jsown)))
;; test framework used: fiveam.
(quicklisp:quickload :jsown-tests)
(run-fiveam-test-suite (read-from-string "jsown-tests:test-all")))
(defmethod libtest ((library-name (eql :nibbles)))
;; The test framework used: rt.
(clean-rt)
(asdf:clear-system :nibbles)
(asdf:clear-system :nibbles-tests)
(quicklisp:quickload :nibbles-tests)
(run-rt-test-suite))
(defmethod libtest ((library-name (eql :cl-custom-hash-table)))
;; The test framework used: hu.dwim.stefil.
(ql:quickload :cl-custom-hash-table-test)
(run-hu.dwim.stefil-test-suite (read-from-string "cl-custom-hash-table-test::test-suite")))
(defmethod libtest ((library-name (eql :hyperluminal-mem)))
;; The test framework used: fiveam.
(quicklisp:quickload :hyperluminal-mem)
(quicklisp:quickload :hyperluminal-mem-test)
(run-fiveam-test-suite (read-from-string "hyperluminal-mem-test::suite")))
(defmethod libtest ((library-name (eql :cl-python)))
;; The test framework used:
;; clpython internally uses ptester (or on Allegro - Allegro provided util.test)
;; The function clpython.test::run-tests returns T in case of success and NIL otherwise.
(ql:quickload :clpython) ; without this Quicklisp can't find clpython.test
(ql:quickload :clpython.test)
(fncall "clpython.test::run-tests"))
(defmethod libtest ((library-name (eql :cl-slug)))
;; test framework used: prove
(ql:quickload :cl-slug-test)
(fncall "prove:run" :cl-slug-test))