Permalink
Browse files

Make tests easier to run by using a testing framework (LIFT) and hook…

…ing into #'ASDF:TEST-SYSTEM

Signed-off-by: Pixie <pix@kepibu.org>
  • Loading branch information...
mmontone authored and Pixie committed Feb 10, 2015
1 parent 68d81e1 commit eb55e96790aa0103a053217c686bb2a2bc1f026a
Showing with 224 additions and 163 deletions.
  1. +10 −2 README.org
  2. +20 −0 burgled-batteries-tests.asd
  3. +2 −1 burgled-batteries.asd
  4. +18 −16 t/numeric.lisp
  5. +5 −0 t/packages.lisp
  6. +98 −97 t/refcnts.lisp
  7. +36 −28 t/sanity.lisp
  8. +21 −19 t/sequences.lisp
  9. +14 −0 t/tests.lisp
View
@@ -104,14 +104,22 @@ For example, under the default policy of :DISCARD, you would see something like:
If you'd like access to Python types without a known translation, :BARRIER
or :FINALIZE are highly recommended over :PASS-THROUGH. They do, however, come
with [[http://pinterface.livejournal.com/40934.html][some caveats]] which you should be aware of.
* Testing
#+begin_src
(asdf:test-system "burgled-batteries")
#+end_src
Tests have been run under SBCL, Clozure CL, and CLISP.
* To Do
* Output redirection
* Callbacks
* Whole-module import (into own package?)
* Python object <-> CLOS mappings
* Provide facilities for user code to define own Lisp-Python mappings
* Better tests that can be run all at once
* Test on something other than SBCL
* Better integrate Quickcheck tests, so LIFT knows about quickcheck failures
* Pathname support (requires FILE* support)
* Prior Art / Other Solutions
@@ -0,0 +1,20 @@
;;; burgled-batteries-tests.asdf --- system definition for the burgled-batteries
;;; test suite
(defsystem "burgled-batteries-tests"
:name "burgled-batteries-tests"
:description "burgled-batteries tests"
:serial t
:components
((:module "test"
:pathname "t/"
:components
((:file "packages")
(:file "tests")
(:file "sanity")
(:file "numeric")
(:file "refcnts")
(:file "sequences"))
:serial t))
:depends-on (#:burgled-batteries #:lift #:cl-quickcheck)
:perform (test-op (o c) #+asdf3 (uiop:symbol-call '#:python-cffi.test '#:run-tests)))
View
@@ -43,4 +43,5 @@ in #p\"grovel-include-dir.lisp\".
(:file "ffi-interface")
(:file "ffi-conditions")
(:file "ffi-callbacks")
(:file "api")))
(:file "api"))
:in-order-to ((test-op (test-op #:burgled-batteries-tests))))
View
@@ -1,18 +1,20 @@
(in-package #:python-cffi.test)
(quickcheck
(let ((*size* (1- (expt 2 31))))
(for-all ((v an-integer))
(is= v (python.cffi:int.from-long v))
(is= v (burgled-batteries:run (format nil "~d" v))))
(for-all ((v a-boolean))
(is= v (python.cffi:bool.from-long (if v 1 0)))
(is= v (burgled-batteries:run (if v "True" "False"))))
(for-all ((v an-integer))
(is= v (python.cffi:long.from-long v))
(is= v (burgled-batteries:run (format nil "~d" v)))))
(let ((*size* (/ most-positive-single-float 2)))
(for-all ((v a-real))
(let ((v (coerce v 'double-float)))
(is= v (python.cffi:float.from-double v))
(is= v (burgled-batteries:run (format nil "~f" v)))))))
(addtest (burgled-batteries)
numeric-test
(quickcheck
(let ((*size* (1- (expt 2 31))))
(for-all ((v an-integer))
(is= v (python.cffi:int.from-long v))
(is= v (burgled-batteries:run (format nil "~d" v))))
(for-all ((v a-boolean))
(is= v (python.cffi:bool.from-long (if v 1 0)))
(is= v (burgled-batteries:run (if v "True" "False"))))
(for-all ((v an-integer))
(is= v (python.cffi:long.from-long v))
(is= v (burgled-batteries:run (format nil "~d" v)))))
(let ((*size* (/ most-positive-single-float 2)))
(for-all ((v a-real))
(let ((v (coerce v 'double-float)))
(is= v (python.cffi:float.from-double v))
(is= v (burgled-batteries:run (format nil "~f" v))))))))
View
@@ -0,0 +1,5 @@
(defpackage #:python-cffi.test
(:use #:cl #:cl-quickcheck)
(:import-from #:lift #:deftestsuite #:addtest)
(:shadow #:assert)
(:export #:run-tests))
View
@@ -1,118 +1,119 @@
(in-package #:python-cffi.test)
;; TODO: Use an actual test library.
(python.cffi:with-refcnt-barrier
(loop :for (object code) :in `((,python.cffi:+None+ "None")
(,python.cffi:+True+ "True")
(,python.cffi:+False+ "False")
;; CPython implements small numbers as shared references.
(,(burgled-batteries:run* "1") "1"))
:do (symbol-macrolet ((current-refcnt (python.cffi::%object.refcnt object)))
(let ((orig-refcnt current-refcnt))
(flet ((ensure-unchanged-refcnt (python-code)
(burgled-batteries:run python-code)
(assert (= orig-refcnt current-refcnt) ()
"Reference count for ~S was ~A by ~S"
code
(if (> orig-refcnt current-refcnt) "decreased" "increased")
python-code)
(format t ".")))
(ensure-unchanged-refcnt code)
(ensure-unchanged-refcnt (format nil "[~A, ~A, ~A]" code code code))
(ensure-unchanged-refcnt (format nil "(~A, ~A, ~A)" code code code))
(ensure-unchanged-refcnt (format nil "dict(a=~A, b=~A, c=~A)" code code code)))))))
;; unknown translations
(python.cffi:with-refcnt-barrier
(burgled-batteries:run "import datetime")
(burgled-batteries:run "tmp = datetime.date.today()")
(let* ((ptr (cpython::dict.get-item* burgled-batteries::main-module-dict* "tmp"))
(orig-refcnt (cpython::%object.refcnt ptr)))
(loop :for (object code) :in `((,ptr "tmp"))
(addtest (burgled-batteries)
refcnt-same-after-translation
(python.cffi:with-refcnt-barrier
(loop :for (object code) :in `((,python.cffi:+None+ "None")
(,python.cffi:+True+ "True")
(,python.cffi:+False+ "False")
;; CPython implements small numbers as shared references.
(,(burgled-batteries:run* "1") "1"))
:do (symbol-macrolet ((current-refcnt (python.cffi::%object.refcnt object)))
(let ((orig-refcnt current-refcnt))
(flet ((ensure-unchanged-refcnt (python-code)
(cpython:with-refcnt-barrier
(burgled-batteries:run python-code))
(burgled-batteries:run python-code)
(assert (= orig-refcnt current-refcnt) ()
"Reference count for ~S was ~A by ~S"
code
(if (> orig-refcnt current-refcnt) "decreased" "increased")
python-code)
(format t ".")))
python-code)))
(ensure-unchanged-refcnt code)
(ensure-unchanged-refcnt (format nil "[~A, ~A, ~A]" code code code))
(ensure-unchanged-refcnt (format nil "(~A, ~A, ~A)" code code code))
(ensure-unchanged-refcnt (format nil "dict(a=~A, b=~A, c=~A)" code code code))))))
(assert (= orig-refcnt (cpython::%object.refcnt ptr))
()
"Reference count was changed ~D overall."
(- orig-refcnt (cpython::%object.refcnt ptr)))))
(ensure-unchanged-refcnt (format nil "dict(a=~A, b=~A, c=~A)" code code code))))))))
;; unknown translations
(addtest (burgled-batteries)
refcnt-same-outside-refcnt-barrier
(python.cffi:with-refcnt-barrier
(burgled-batteries:run "import datetime")
(burgled-batteries:run "tmp = datetime.date.today()")
(let* ((ptr (cpython::dict.get-item* burgled-batteries::main-module-dict* "tmp"))
(orig-refcnt (cpython::%object.refcnt ptr)))
(loop :for (object code) :in `((,ptr "tmp"))
:do (symbol-macrolet ((current-refcnt (python.cffi::%object.refcnt object)))
(let ((orig-refcnt current-refcnt))
(flet ((ensure-unchanged-refcnt (python-code)
(cpython:with-refcnt-barrier
(burgled-batteries:run python-code))
(assert (= orig-refcnt current-refcnt) ()
"Reference count for ~S was ~A by ~S"
code
(if (> orig-refcnt current-refcnt) "decreased" "increased")
python-code)))
(ensure-unchanged-refcnt code)
(ensure-unchanged-refcnt (format nil "[~A, ~A, ~A]" code code code))
(ensure-unchanged-refcnt (format nil "(~A, ~A, ~A)" code code code))
(ensure-unchanged-refcnt (format nil "dict(a=~A, b=~A, c=~A)" code code code))))))
(assert (= orig-refcnt (cpython::%object.refcnt ptr))
()
"Reference count was changed ~D overall."
(- orig-refcnt (cpython::%object.refcnt ptr))))))
;; Inspired by (read: almost entirely copied from) #'VOODOO in trivial-garbage's
;; tests.
(defun voodoo (expr)
(funcall (compile nil `(lambda () (eval (read-from-string (format nil "~S" ',expr))))))
(values))
(cpython:with-unknown-translation-policy (:finalize)
(burgled-batteries:import "time")
(burgled-batteries:run "v = time.gmtime()")
(let* ((code "v")
(wrapped (burgled-batteries:run code))
(object (python.cffi::wrapped-value wrapped)))
(unwind-protect
(symbol-macrolet ((current-refcnt (python.cffi::%object.refcnt object)))
(let ((orig-refcnt current-refcnt))
(flet ((ensure-unchanged-refcnt (python-code)
(tg:gc :full t)
(voodoo `(burgled-batteries:run ,python-code))
(voodoo `(tg:gc :full t))
(assert (= orig-refcnt current-refcnt) ()
"Reference count (~A, ~A) for ~S was ~A by ~A from ~S"
orig-refcnt current-refcnt
code
(if (> orig-refcnt current-refcnt) "decreased" "increased")
(abs (- orig-refcnt current-refcnt))
python-code)
(format t ".")))
(ensure-unchanged-refcnt code)
(ensure-unchanged-refcnt (format nil "[~A, ~A, ~A]" code code code))
(ensure-unchanged-refcnt (format nil "(~A, ~A, ~A)" code code code))
(ensure-unchanged-refcnt (format nil "dict(a=~A, b=~A, c=~A)" code code code)))))
(burgled-batteries:run "v = None")
(tg:gc :full t))
;; Mostly this is just here to ensure wrapped doesn't get GCed during the tests
(cpython::%object.refcnt (cpython::wrapped-value wrapped))))
(addtest (burgled-batteries)
refcnt-same-after-finalizer-runs
(cpython:with-unknown-translation-policy (:finalize)
(burgled-batteries:import "time")
(burgled-batteries:run "v = time.gmtime()")
(let* ((code "v")
(wrapped (burgled-batteries:run code))
(object (python.cffi::wrapped-value wrapped)))
(unwind-protect
(symbol-macrolet ((current-refcnt (python.cffi::%object.refcnt object)))
(let ((orig-refcnt current-refcnt))
(flet ((ensure-unchanged-refcnt (python-code)
(tg:gc :full t)
(voodoo `(burgled-batteries:run ,python-code))
(voodoo `(tg:gc :full t))
(assert (= orig-refcnt current-refcnt) ()
"Reference count (~A, ~A) for ~S was ~A by ~A from ~S"
orig-refcnt current-refcnt
code
(if (> orig-refcnt current-refcnt) "decreased" "increased")
(abs (- orig-refcnt current-refcnt))
python-code)))
(ensure-unchanged-refcnt code)
(ensure-unchanged-refcnt (format nil "[~A, ~A, ~A]" code code code))
(ensure-unchanged-refcnt (format nil "(~A, ~A, ~A)" code code code))
(ensure-unchanged-refcnt (format nil "dict(a=~A, b=~A, c=~A)" code code code)))))
(burgled-batteries:run "v = None")
(tg:gc :full t))
;; Mostly this is just here to ensure wrapped doesn't get GCed during the tests
(cpython::%object.refcnt (cpython::wrapped-value wrapped)))))
;; unknown translations with stolen references
(cpython:with-refcnt-barrier
(burgled-batteries:run "import datetime")
(burgled-batteries:run "tmp = datetime.date.today()")
(let* ((ptr (cpython::dict.get-item* burgled-batteries::main-module-dict* "tmp"))
(orig-refcnt (cpython::%object.refcnt ptr)))
(loop :for (object code) :in `((,ptr "tmp"))
:do (symbol-macrolet ((current-refcnt (python.cffi::%object.refcnt object)))
(let* ((orig-refcnt current-refcnt)
(tuple* (burgled-batteries:run* "(1, 2, 3, 4)")))
(flet ((ensure-unchanged-refcnt ()
(assert (= orig-refcnt current-refcnt) ()
"Reference count for ~S was ~A"
code
(if (> orig-refcnt current-refcnt) "decreased" "increased")
)
(format t ".")))
(ensure-unchanged-refcnt)
(cpython:with-unknown-translation-policy (:barrier)
(cpython:tuple.set-item tuple* 0 (burgled-batteries:run "tmp")))
(voodoo
`(cpython:with-unknown-translation-policy (:finalize)
(cpython:tuple.set-item ,tuple* 1 (burgled-batteries:run "tmp"))))
(voodoo `(tg:gc :full t))
(cpython:.dec-ref tuple*)
(ensure-unchanged-refcnt)))))
(assert (= orig-refcnt (cpython::%object.refcnt ptr))
()
"Reference count was changed ~D overall."
(- orig-refcnt (cpython::%object.refcnt ptr)))))
(addtest (burgled-batteries)
refcnt-same-for-stolen-references
(cpython:with-refcnt-barrier
(burgled-batteries:run "import datetime")
(burgled-batteries:run "tmp = datetime.date.today()")
(let* ((ptr (cpython::dict.get-item* burgled-batteries::main-module-dict* "tmp"))
(orig-refcnt (cpython::%object.refcnt ptr)))
(loop :for (object code) :in `((,ptr "tmp"))
:do (symbol-macrolet ((current-refcnt (python.cffi::%object.refcnt object)))
(let* ((orig-refcnt current-refcnt)
(tuple* (burgled-batteries:run* "(1, 2, 3, 4)")))
(flet ((ensure-unchanged-refcnt ()
(assert (= orig-refcnt current-refcnt) ()
"Reference count for ~S was ~A"
code
(if (> orig-refcnt current-refcnt) "decreased" "increased"))))
(ensure-unchanged-refcnt)
(cpython:with-unknown-translation-policy (:barrier)
(cpython:tuple.set-item tuple* 0 (burgled-batteries:run "tmp")))
(voodoo
`(cpython:with-unknown-translation-policy (:finalize)
(cpython:tuple.set-item ,tuple* 1 (burgled-batteries:run "tmp"))))
(voodoo `(tg:gc :full t))
(cpython:.dec-ref tuple*)
(ensure-unchanged-refcnt)))))
(assert (= orig-refcnt (cpython::%object.refcnt ptr))
()
"Reference count was changed ~D overall."
(- orig-refcnt (cpython::%object.refcnt ptr))))))
View
@@ -1,34 +1,42 @@
(defpackage #:python-cffi.test
(:use #:cl #:cl-quickcheck))
(in-package #:python-cffi.test)
(burgled-batteries:startup-python)
(addtest (burgled-batteries)
types-have-refcnts-and-are-types
(let (type-vars)
(do-external-symbols (s '#:cpython type-vars)
(let ((symbol-name (symbol-name s)))
(when (and (string= '#:+ symbol-name :end2 1)
(string= '#:.type+ symbol-name :start2 (- (length symbol-name) 6)))
(push s type-vars))))
(loop :for type-var :in type-vars
:do (assert (plusp (cpython::%object.refcnt (eval type-var)))
()
"Python type ~S does not have a positive reference count."
type-var)
(assert (string= "<type '" (cpython:object.str (eval type-var)) :end2 7)
()
"Python type ~S is not stringified as a type."
type-var))))
(let (type-vars)
(do-external-symbols (s '#:cpython type-vars)
(let ((symbol-name (symbol-name s)))
(when (and (string= '#:+ symbol-name :end2 1)
(string= '#:.type+ symbol-name :start2 (- (length symbol-name) 6)))
(push s type-vars))))
(loop :for type-var :in type-vars
:do (assert (plusp (cpython::%object.refcnt (eval type-var)))
()
"Python type ~S does not have a positive reference count."
type-var)
(assert (string= "<type '" (cpython:object.str (eval type-var)) :end2 7)
()
"Python type ~S is not stringified as a type."
type-var)))
(addtest (burgled-batteries)
apply-min
(let ((nums (alexandria:shuffle (list 1 2 3 4 5 6 7 8 9 10))))
(burgled-batteries::with-cpython-pointer (min-fn (burgled-batteries:run* "min"))
(assert (= (apply #'burgled-batteries:apply min-fn nums)
(apply #'min nums))
()
"Something seems to be wrong with APPLY. Have types been switched again?"))))
(let ((nums (alexandria:shuffle (list 1 2 3 4 5 6 7 8 9 10))))
(burgled-batteries::with-cpython-pointer (min-fn (burgled-batteries:run* "min"))
(assert (= (apply #'burgled-batteries:apply min-fn nums)
(apply #'min nums))
(eval-when (:compile-toplevel :load-toplevel)
(burgled-batteries:startup-python))
(addtest (burgled-batteries)
defpyfun-max
(burgled-batteries:defpyfun ("max" pymax) (&rest args))
(let ((nums (alexandria:shuffle (list 1 2 3 4 5 6 7 8 9 10))))
(assert (= (pymax nums) (apply #'max nums))
()
"Something seems to be wrong with APPLY. Have types been switched again?")))
"Either MAX is broken or DEFPYFUN is having issues.")))
(burgled-batteries:defpyfun ("max" pymax) (&rest args))
(let ((nums (alexandria:shuffle (list 1 2 3 4 5 6 7 8 9 10))))
(assert (= (pymax nums) (apply #'max nums))
()
"Either MAX is broken or DEFPYFUN is having issues."))
(eval-when (:compile-toplevel :load-toplevel)
(burgled-batteries:shutdown-python))
Oops, something went wrong.

0 comments on commit eb55e96

Please sign in to comment.