Merge pull request #2 from scicloj/snapshots

switching to a snapshot-based workflow, rather than an incremental one

CHANGELOG.md

@@ -1,6 +1,9 @@
 # Change Log
 All notable changes to this project will be documented in this file. This change log follows the conventions of [keepachangelog.com](http://keepachangelog.com/).
 
+## [1-alpha3-SNAPSHOT]
+- shifting to shapshot-based workflow (WIP)
+
 ## [1-alpha2]
 - exception handling
 

README.md

@@ -4,6 +4,8 @@ Generating tests from Clojure notes
 
 [![Clojars Project](https://img.shields.io/clojars/v/org.scicloj/note-to-test.svg)](https://clojars.org/org.scicloj/note-to-test)
 
+# TODO rewrite this README for the snapshot-based approach
+
 ## Status
 Initial draft
 
@@ -17,23 +19,13 @@ It can automatically generate tests:
 
 Tests are created by running code examples and remembering their outputs. The person writing the documentation is responsible for checking that these outputs are sensible. The tests are responsible for checking these outputs remain the same on future versions.
 
-Tests are accumulated in standard clojure.test files. Each namespace of code examples has its own generated test file.
-
-Old tests are kept until one removes them manually (or explicitly asks to clean them up). New tests are added if they are based on code examples which do not appear in the test files yet.
-
-### Caveats
-
-This method is meaningful only if all code examples are stateless -- that is, only if their output would be the same wherever they are evaluated throughout the whole namespace evaluation process (assuming that all relevant vars are already defined).
-
-A stateful notebook such as
-```clj
-(def x 1)
-(* x 10)
-(def x (inc x))
-(* x 20)
-```
-will result in wrong tests.
+Tests are written in standard clojure.test files. 
 
+Each namespace of code examples has its own generated test file. 
+* The top-level forms in the test namespace correspond to all runnable top-level forms in the source namespace. That is, all top-level forms except for Rich `(comment ...)` blocks. 
+* The namespace definition form is adapted to the test namespace's needs. 
+* Forms that result in a var (e.g., `def`, `defn`, `defonce`) are kept as-is. 
+* All other forms are turned in to `(deftest ...)` test definitions.
 
 ## Usage
 
@@ -50,14 +42,6 @@ Assume you have a namespace, say `dum.dummy` in the file [notebooks/dum/dummy.cl
 ```
 would generate a test namespace 'dum.dummy-generated-test' in the file [test/dum/dummy_generated_test.clj](test/dum/dummy_generated_test.clj) with clojure.test tests verifying that those code examples actually return the values they had at the time we executed that `gentest!` call.
 
-If that namespace already exists, then we keep the existing tests (verifying old values that have been generated in the past). We avoid adding new tests for the code examples that already appear in existing tests.
-
-```clj
-(note-to-test/gentest! "notebooks/dum/dummy.clj"
-                       {:cleanup-existing-tests? true})
-```
-would first clean the test namespace up, removing all existing tests.
-
 ### Command Line
 
 See [scicloj.note-to-test.v1.main/-main](src/scicloj/note_to_test/v1/main)
@@ -78,7 +62,7 @@ clojure -M:dev:gen --verbose
 
 ### Build.tools
 
-Add `gentests!` to your test function in build.clj
+Add `gentests!` to your test function in `build.clj`.
 
 ### Handling special values
 
@@ -119,6 +103,7 @@ will result in a test like
 You see, the output value is represented as a code snippet that would generate that value. Since `tech.ml.dataset` datasets can be compared using the `=` function, this is a valid test for our code example.
 
 ## Wishlist
+- support running from command line
 - support an alternative plain-data output format to help seeing the output changes explicitly
 - make clear error messages:
   - when outputs change
@@ -130,6 +115,7 @@ You see, the output value is represented as a code snippet that would generate t
   - generate tests from code examples in docstrings
   - explore generate docstrings in a structured way (marking code examples explicitly)
 - support approximate comparisons of values for floating-point computations
+- support automatic regeneration (say, using file watch)
 
 ## License
 

build.clj

@@ -4,7 +4,7 @@
             [org.corfield.build :as bb]))
 
 (def lib 'org.scicloj/note-to-test)
-(def version "1-alpha2")
+(def version "1-alpha3-SNAPSHOT")
 #_ ; alternatively, use MAJOR.MINOR.COMMITS:
 (def version (format "1.0.%s" (b/git-count-revs nil)))
 

deps.edn

@@ -9,6 +9,6 @@
            :test  {:extra-paths ["test" "notebooks"]
                    :extra-deps  {org.clojure/test.check {:mvn/version "1.1.1"}
                                  io.github.cognitect-labs/test-runner
-                                 {:git/tag "v0.5.0" :git/sha "48c3c67"}
+                                 {:git/tag "v0.5.1" :git/sha "dfb30dd"}
                                  scicloj/tablecloth     {:mvn/version "7.000-beta-51"}}}
            :gen   {:main-opts ["-m" "scicloj.note-to-test.v1.main"]}}}

notebooks/dum/dummy.clj

@@ -16,16 +16,18 @@
 
 
 (defn f [x]
-  (+ x 9))
+  (+ x 19))
+
+(f 12)
 
-(f 11)
 
 (-> {:x [1 2 3]}
     tc/dataset
     (tc/map-columns :y [:x] (partial * 10)))
 
+(f 13)
+
+
 (comment
   (note-to-test/gentest! "notebooks/dum/dummy.clj")
-  (note-to-test/gentest! "notebooks/dum/dummy.clj"
-                         {:cleanup-existing-tests? true})
   ,)

src/scicloj/note_to_test/v1/api.clj

@@ -7,26 +7,16 @@
 
 (defn gentest!
   "Generate a clojure.test file for the code examples in the file at `source-path`.
-  Optionsl `options`:
-  - :cleanup-existing-tests? - boolean - default `false` - should we create the tests file from scratch (when `true`), or incrementally (when `false`)?
 
-  Examples:
-  Generate tests for a given file incrementally, handling only new code examples.
+  Example:
   ```clj
   (gentest! \"notebooks/dum/dummy.clj\")
   ```
-  Generate tests from scratch:
-  ```clj
-  (gentest! \"notebooks/dum/dummy.clj\"
-        {:cleanup-existing-tests? true})
-  ```
   "
-  ([source-path]
-   (gentest! source-path {}))
-  ([source-path options]
-   (-> source-path
-       (impl/prepare-context options)
-       impl/write-tests!)))
+  [source-path]
+  (-> source-path
+      impl/prepare-context
+      impl/write-tests!))
 
 (defn gentests!
   "Generate tests for all source files discovered in [dirs]."
@@ -39,7 +29,7 @@
              ^File file (file-seq (io/file dir))
              :when (impl/clojure-source? file)]
        (when verbose (println "Loading file" (str file)))
-       (cond-> (gentest! file options)
+       (cond-> (gentest! file)
                verbose (println))))
    [:success]))
 

src/scicloj/note_to_test/v1/impl.clj

@@ -3,15 +3,12 @@
             [clojure.pprint :as pp]
             [clojure.tools.reader]
             [clojure.tools.reader.reader-types]
-            [clojure.java.io :as io])
+            [clojure.java.io :as io]
+            [clojure.java.shell :as shell])
   (:import (java.io File)))
 
 (set! *warn-on-reflection* true)
 
-#_(defn spy [x tag]
-    (pp/pprint [:tag x])
-    x)
-
 (def *special-value-representations
   (atom {}))
 
@@ -62,36 +59,45 @@
                                       {:source-ns source-ns
                                        :code code
                                        :exception e}))))]
-    (format test-template
-            (str "test-" index)
-            (indent code 4)
-            (-> output
-                represent-value
-                pp/pprint
-                with-out-str
-                (indent 4)))))
+    (if (var? output)
+      ;; if the output is a var,
+      ;; just keep the code (so that we run things in order)
+      code
+      ;; else - actually create a test
+      (format test-template
+              (str "test-" index)
+              (indent code 4)
+              (-> output
+                  represent-value
+                  pp/pprint
+                  with-out-str
+                  (indent 4))))))
 
-(defn ->test-ns-symbol [ns-symbol]
-  (-> ns-symbol
-      name
-      (str "-generated-test")
-      symbol))
 
 
+(defn ->test-ns-symbol [ns-symbol]
+  (format "%s-generated-test"
+          (name ns-symbol)))
+
+(defn ->test-path [test-ns-symbol]
+  (-> test-ns-symbol
+      name
+      (string/replace #"-" "_")
+      (string/replace #"\." "/")
+      (->> (format "test/%s.clj"))))
 
 (defn ->test-ns-requires [ns-symbol ns-requires]
   (-> (concat (list
                :require
-               '[clojure.test :refer [deftest is]]
-               [ns-symbol :refer :all])
+               '[clojure.test :refer [deftest is]])
               ns-requires)
       pp/pprint
       with-out-str))
 
-(defn ->test-ns [ns-symbol ns-requires]
+(defn ->test-ns [test-ns-symbol test-ns-requires]
   (format "(ns %s\n%s)"
-          (->test-ns-symbol ns-symbol)
-          (-> (->test-ns-requires ns-symbol ns-requires)
+          test-ns-symbol
+          (-> test-ns-requires
               (indent 2))))
 
 (defn code->forms [code]
@@ -116,11 +122,6 @@
       (and (list? form)
            (-> form first (= value-or-set-of-values))))))
 
-(defn ns-name->test-path [ns-name]
-  (-> ns-name
-      name
-      (string/replace #"\." "/")
-      (->> (format "test/%s_generated_test.clj"))))
 
 
 (defn test-form->original-form [test-form]
@@ -145,20 +146,20 @@
                 {:test-form test-form
                  :original-form (test-form->original-form
                                  test-form)})))))
-(defn delete-file-when-exists [path]
-  (let [file (io/file path)]
-    (when (.exists file)
-      (io/delete-file file))))
 
-(defn prepare-context [source-path {:keys [cleanup-existing-tests?]}]
+#_(defn git-hash []
+    (-> (shell/sh "git" "rev-parse" "HEAD")
+        :out
+        (string/replace #"\n" "")))
+
+(defn prepare-context [source-path]
   (try
     (load-file (str source-path))
     (catch Exception e
       (throw (ex-info "note-to-test: Exception on lode-file"
                       {:source-path source-path
                        :exception e}))))
-  (let [forms ( read-forms
-               source-path)
+  (let [forms (read-forms source-path)
         ns-form (->> forms
                      (filter (begins-with? 'ns))
                      first)
@@ -167,58 +168,39 @@
                              (filter (begins-with? :require))
                              first
                              rest)
-        test-path (ns-name->test-path
-                   ns-symbol)
-        _ (when cleanup-existing-tests?
-            (delete-file-when-exists test-path))
-        existing-tests (read-tests test-path)
-        known-forms (some->> existing-tests
-                             (map :original-form)
-                             set)
+        test-ns-symbol (->test-ns-symbol ns-symbol)
+        test-ns-requires (->test-ns-requires ns-symbol ns-requires)
+        test-path (->test-path test-ns-symbol)
         codes-for-tests (->> forms
-                             (filter (complement
-                                      (begins-with?
-                                       '#{ns
-                                          def
-                                          defonce
-                                          defn
-                                          comment})))
-                             (filter (-> known-forms
-                                         (or #{})
-                                         complement))
+                             (remove (begins-with? '#{ns comment}))
                              (map (fn [form]
                                     (-> form
                                         meta
                                         :source)))
                              (remove nil?))]
     {:ns-symbol ns-symbol
      :ns-requires ns-requires
+     :test-ns-symbol test-ns-symbol
+     :test-ns-requires test-ns-requires
      :test-path test-path
-     :existing-tests existing-tests
      :codes-for-tests codes-for-tests}))
 
 
 (defn write-tests! [context]
   (let [{:keys [ns-symbol
                 ns-requires
+                test-ns-symbol
+                test-ns-requires
                 test-path
-                codes-for-tests
-                existing-tests]} context]
-    (when-not existing-tests
-      (io/make-parents test-path))
-    (let [n-existing-tests (count existing-tests)]
-      (->> codes-for-tests
-           (map-indexed (fn [i code]
-                          (->test code
-                                  (+ i n-existing-tests)
-                                  (find-ns ns-symbol))))
-           (concat
-            (if existing-tests
-              []
-              [(->test-ns ns-symbol
-                          ns-requires)]))
-           (string/join "\n")
-           (#(spit test-path
-                   %
-                   :append true))))
+                codes-for-tests]} context]
+    (io/make-parents test-path)
+    (->> codes-for-tests
+         (map-indexed (fn [i code]
+                        (->test code
+                                i
+                                (find-ns ns-symbol))))
+         (cons (->test-ns test-ns-symbol
+                          test-ns-requires))
+         (string/join "\n")
+         (spit test-path))
     [:wrote test-path]))

src/scicloj/note_to_test/v1/main.clj

@@ -6,7 +6,6 @@
 (def cli-options
   [["-d" "--dirs" :default ["notebooks"]]
    ["-a" "--accept"]
-   ["-c" "--cleanup-existing-tests?"]
    ["-v" "--verbose"]
    ["-h" "--help"]])