Documentation

README.md

@@ -237,9 +237,12 @@ Things of note:
   another system and call `TEST-OP` on *it*.  If you're doing the
   latter, you need both definitions.
 
+# Reference
+
+[See here for reference](http://rpav.github.com/CheckL/).
+
 # Etc
 
-I wrote this system literally today and, ironically, haven't tested it
-much.  It's likely to undergo some changes when it hits more
-real-world cases, but the API shouldn't change much.  Simplicity and
-lispy workflow is the goal here.
+I've been using this a bit and have added some conveniences.  There
+could certainly be more.  Suggestions welcome, theoretical or
+otherwise.

checkl-docs.asd

@@ -0,0 +1,13 @@
+(eval-when (:compile-toplevel :load-toplevel :execute)
+  (asdf:load-system :cl-gendoc))
+
+(defsystem :checkl-docs
+  :depends-on (:checkl :cl-gendoc)
+
+  :serial t
+  :pathname "doc"
+
+  :components
+  ((:file "generate")))
+
+(gendoc:define-gendoc-load-op :checkl-docs :checkl.docs 'generate)

checkl.lisp

@@ -18,7 +18,9 @@
                      (result-error-value c)
                      (result-error-last c)))))
 
-(defgeneric result-equalp (o1 o2))
+(defgeneric result-equalp (o1 o2)
+  (:documentation "Generic function to compare test results `O1` to
+`O2`.  Defaults to `EQUALP`."))
 
 (defmethod result-equalp (o1 o2)
   (equalp o1 o2))
@@ -89,6 +91,22 @@
           (setf error-p t))))))
 
 (defmacro check ((&key name (category :default) (output-p nil)) &body body)
+  "=> test-results
+
+Run `BODY`.  Check resulting values against the last run using
+`CHECKL:RESULT-EQUALP`, or store them if this is the first run.
+Sameness of the test is determined by comparing the body with
+`EQUALP`, or by `NAME`.
+
+`NAME` may be specified to name a test.  If the test exists but is
+anonymous (i.e., `NAME` has not been specified), specifying `NAME`
+will name the test and it will no longer be anonymous.
+
+`CATEGORY` may be specified for running groups of tests.
+
+If `OUTPUT-P` is `t`, the results will be printed to
+`*standard-output*` as well as returned.  This may be helpful if the
+results are too long to see in your emacs minibuffer."
   (let ((fun (gensym))
         (result (gensym))
         (namesym (gensym))
@@ -107,6 +125,9 @@
          (values-list result-list)))))
 
 (defun run (&rest names)
+  "=> test-results
+
+Run tests named `NAMES`, collecting their results."
   (let ((lambdas (package-tests-lambdas (current-tests))))
     (loop for name in names
           as fn = (gethash name lambdas)
@@ -115,13 +136,17 @@
           finally (return (values-list results)))))
 
 (defun run-all (&optional (category :default) &rest categories)
+  "=> test-results
+
+Run all tests, optionally specifying categories."
   (push category categories)
   (let ((current-categories (package-tests-categories (current-tests))))
     (loop for cat in categories
           appending (gethash cat current-categories) into names
           finally (return (apply #'run names)))))
 
 (defun checkl-store (&optional filespec)
+  "Store package test results to `FILESPEC`"
   (let ((filespec (or filespec (package-tests-default-checkl-store (current-tests))))
         (results (package-tests-results (current-tests))))
     (unless (> (hash-table-count results) 0)
@@ -132,6 +157,7 @@
     (values)))
 
 (defun checkl-load (&optional filespec)
+  "Load package test results from `FILESPEC`"
   (let* ((tests (current-tests))
          (filespec (or filespec (package-tests-default-checkl-store tests))))
     (with-open-file (stream filespec)
@@ -144,6 +170,8 @@
               collect k)))
 
 (defun clear (&rest names)
+  "Clear the test results from the tests `NAMES`.  For clearing anonymous
+test results, see `CLEAR-ANONYMOUS`."
   (let ((tests (current-tests)))
     (loop for name in names do
       (remhash name (package-tests-results tests))
@@ -153,6 +181,7 @@
               (delete name (gethash c (package-tests-categories tests))))))))
 
 (defun clear-anonymous ()
+  "Clear anonymous test results.  For clearing named tests, see `CLEAR`."
   (let ((tests (current-tests)))
     (loop for name being the hash-keys of (package-tests-results tests) do
       (when (not (symbolp name))
@@ -161,3 +190,15 @@
         (do-categories (c tests)
           (setf (gethash c (package-tests-categories tests))
                 (delete name (gethash c (package-tests-categories tests)))))))))
+
+(defmacro check-output (&body body)
+  "Use this within a `CHECK` block.  Rebind `*standard-output*` and
+`*error-output*` and return a `CHECK`-able result."
+  (let ((so (gensym)) (se (gensym)))
+    `(let* ((,so (make-string-output-stream))
+            (,se (make-string-output-stream))
+            (*standard-output* ,so)
+            (*error-output* ,se))
+       ,@body
+       (list (get-output-stream-string ,so)
+             (get-output-stream-string ,se)))))

doc/.gitignore

@@ -0,0 +1 @@
+*.html

doc/generate.lisp

@@ -0,0 +1,12 @@
+(defpackage :checkl.docs
+  (:use #:cl #:gendoc)
+  (:export #:generate))
+
+(in-package :checkl.docs)
+
+(defun generate ()
+  (gendoc (:output-system :checkl-docs
+           :output-filename "index.html"
+           :css "simple.css")
+    (:mdf "intro.md")
+    (:apiref :checkl)))

doc/intro.md

@@ -0,0 +1,238 @@
+# CheckL
+
+Why write programs in Common Lisp but tests like Java?
+
+My workflow for writing Common Lisp tends to be like this:
+
+* Write a bit of lisp, perhaps a function, class, or structure
+* Write a snippet in a scratch buffer to test
+* Fix if necessary and repeat
+
+Testing is already inherent in this process, all we need is a little
+bit of Common Lisp magic to take advantage of it.  Thus, CheckL:
+
+```
+(defun foo ()
+  (+ 1 1))
+
+(check () (foo)) ;; => 2
+
+(defun foo ()
+  (+ 1 2))
+
+(check () (foo))
+
+    |
+    v
+
+Result 0 has changed: 3
+Previous result: 2
+   [Condition of type CHECKL::RESULT-ERROR]
+
+Restarts:
+ 0: [USE-NEW-VALUE] The new value is correct, use it from now on.
+ 1: [SKIP-TEST] Skip this, leaving the old value, but continue testing
+ 2: [RETRY] Retry SLIME interactive evaluation request.
+ 3: [*ABORT] Return to SLIME's top level.
+ 4: [TERMINATE-THREAD] Terminate this thread (#<THREAD "worker" RUNNING {100586AB13}>)
+```
+
+# Usage
+
+## Tests
+
+Presumably you already write code to test.  Possibly you even write
+something like this, evaluating it and manually checking the result:
+
+```
+(progn
+  (function-1 ...)
+  (function-2 ...))
+```
+
+With CheckL, you don't really have to change much.  Your `PROGN`
+becomes a `CHECK ()` and you run it.  CheckL notifies you if something
+changes!
+
+Results are compared with `CHECKL:RESULT-EQUALP`.  This defaults to
+`CL:EQUALP`.  Defining it for other things may be useful.
+
+For very long values, it may be helpful to print them:
+
+```
+(check (:output-p t) (some-very-long-result)) => ...
+```
+
+If you make changes to the test, it becomes another test:
+
+```
+(defun foo () (+ 1 1))
+(check () (foo))      ;; => 2
+(check () (1- (foo))) ;; => 1
+```
+
+However, if you name it before you change it, it'll always compare
+against the same list:
+
+```
+(check () (foo)) ;; => 2
+
+(defun foo () (+ 1 3))
+
+(check (:name :two) (foo)) ;; => Error!
+```
+
+In this case, the old "anonymous" test (actually identified by the
+body of the test) is now named `:two`.  Making changes to the test
+will alter the same test, now named `:two`, and compare against prior
+results.
+
+Finally, you might want to check more than one thing in a single
+`CHECK`.  You can do this with `VALUES` (or variants):
+
+```
+(defun foo () (+ 1 1)
+(defun bar () (- 1 1))
+
+(check (:name :two)
+  (values (foo) (bar))) ;; => 2, 0
+
+(defun bar () (foo))
+
+(check (:name :two)
+  (values (foo) (bar))) ;; => Error!
+```
+
+Or, if you want to run one or more tests:
+
+```
+(run :two ...)
+```
+
+## Categories
+
+So you've been writing a bunch of little tests and want to run them
+all and see if anything has changed:
+
+```
+(run-all)
+```
+
+Easy!  And you haven't had to specifically declare it so in three
+places.  However maybe you want a bit more structure and split up your
+tests when you run them all.  Thus categories:
+
+```
+(check (:name :foo :category :some-category) ...)
+
+(run-all :some-category ...)
+```
+
+The default category is perhaps unsurprisingly called `:default`.
+That's pretty much all there is to categories.
+
+## Remembering results
+
+Since we're not *manually* defining the result, it would be
+unfortunate if we *happened* to quit our lisp while our code still had
+a bug, and then weren't sure what it was.  Easy enough:
+
+```
+(checkl-store "some-file")
+;;; - later -
+(checkl-load "some-file)
+```
+
+This uses `cl-marshal` and `WRITE` to write values to the file
+(overwriting it entirely).  `*PRINT-READABLY*` is forced to `t`, but
+you can otherwise customize the output as per `WRITE`.
+
+Along with revision control, it should be easy to keep track of test
+results and make modifications.
+
+## Formalizing
+
+This is not meant to be a complete replacement for test suites such as
+FiveAM, but more of a "deformalization".
+
+But once you've worked on a bit of code and have your buffer cluttered
+with `(check (...) ...)` forms, you probably don't want to rewrite
+them all as FiveAM constructs.  It'd be nice if you could just sortof
+integrate them all with minimal effort, like this:
+
+```
+;; 5am doesn't have a find-suite, so you have to do this:
+(defsuite :default)
+
+(check-formal (:name :two) (foo))
+```
+
+Well, assuming you had FiveAM loaded before you loaded CheckL, this is
+exactly what you do.  (I didn't want FiveAM to be a strict
+dependency.)  Now you can do one of these, and they still do what they
+should:
+
+```
+(5am:run! :default) ;; => Pretty dots, one per VALUE
+(run-all :default)  ;; => 2
+```
+
+We've gone from *very* informal testing to having things in FiveAM
+with minimal effort!
+
+Note that you can *still* eval the `CHECK-FORMAL` block in your buffer
+and it behaves just like a `CHECK` block, returning its values and
+signaling a condition if they change.
+
+## ASDF
+
+Wouldn't it be nice if ASDF loaded your saved CheckL values, and let
+you call your newly-created FiveAM tests with minimal effort?  Of
+course!
+
+```
+(cl:eval-when (:load-toplevel :execute)
+  (asdf:load-system :fiveam)
+  (asdf:load-system :checkl))
+
+(defsystem :my-system
+  :description "A brand new system"
+  ...)
+
+(defsystem :my-system-tests
+  :description "A system that tests"
+  
+  :depends-on (:fiveam :checkl)
+  :serial t
+  :pathname "t"
+
+  :components ((:file "package")
+               (checkl:tests "some-test")
+               (checkl:test-values "test-values"
+                                   :package :my-system-tests)))
+
+(checkl:define-test-op :my-system :my-system-test)
+(checkl:define-test-op :my-system-test)
+```
+
+That's all!  No long `PERFORM` definitions.  Just make sure to have
+the `EVAL-WHEN` at the top.  Now you can do this:
+
+```
+(asdf:load-system :my-system)
+(asdf:test-system :my-system) ;; => (5am:run! :default)
+```
+
+Things of note:
+
+* New ASDF component: `CHECK:TESTS` loads a file with `CHECK-FORMAL`
+  tests, but does *not* run them.
+
+* New ASDF component: `CHECK:TEST-VALUES` loads a file with test
+  values.  It also sets the path to be the *default* for test values,
+  so you can simply do `(checkl-store)` or `(checkl-load)`.
+
+* `define-test-op SYSTEM &optional OTHER-SYSTEM`: This sets up the
+  `ASDF:PERFORM` method for system to either run tests, or load
+  another system and call `TEST-OP` on *it*.  If you're doing the
+  latter, you need both definitions.