jepsen.core/run! now wraps :generator in a Forgettable reference.

We pass the test map around liberally during setup and execution, which
creates lots of opportunities to retain references. If any component
keeps hold of the test, it would also inadvertently retain the head of
the generator--which is usually an infinite, lazy data structure.

This breaking change immediately wraps the generator in a mutable
Forgettable reference type (new in jepsen.util). It responds to IDeref,
in case you *intend* to manipulate the generator for some reason, but is
explicitly forgotten once the generator interpreter starts--allowing it
to be garbage collected.

The memory effects are staggering. Tests which exhausted a 16 GB heap in
an hour can now run in 256 MB with almost nothing making it into oldgen.

Author: aphyr

jepsen/src/jepsen/core.clj

@@ -20,18 +20,18 @@
             [clojure.datafy :refer [datafy]]
             [dom-top.core :as dt :refer [assert+]]
             [fipp.edn :refer [pprint]]
-            [jepsen.util :as util :refer [with-thread-name
+            [jepsen [checker :as checker]
+                    [client :as client]
+                    [control :as control]
+                    [db :as db]
+                    [generator :as generator]
+                    [nemesis :as nemesis]
+                    [store :as store]
+                    [os :as os]
+                    [util :as util :refer [with-thread-name
                                           fcatch
                                           real-pmap
-                                          relative-time-nanos]]
-            [jepsen.os :as os]
-            [jepsen.db :as db]
-            [jepsen.control :as control]
-            [jepsen.generator :as generator]
-            [jepsen.checker :as checker]
-            [jepsen.client :as client]
-            [jepsen.nemesis :as nemesis]
-            [jepsen.store :as store]
+                                          relative-time-nanos]]]
             [jepsen.store.format :as store.format]
             [jepsen.control.util :as cu]
             [jepsen.generator [interpreter :as gen.interpreter]]
@@ -300,11 +300,14 @@
           (store/stop-logging!))))
 
 (defn prepare-test
-  "Takes a test and ensures it has a :start-time, :concurrency, and :barrier
-  field. This operation always succeeds, and is necessary for accessing a
-  test's store directory, which depends on :start-time. You may call this
-  yourself before calling run!, if you need access to the store directory
-  outside the run! context."
+  "Takes a test and prepares it for running. Ensures it has a :start-time,
+  :concurrency, and :barrier field. Wraps its generator in a forgettable
+  reference, to prevent us from inadvertently retaining the head.
+
+  This operation always succeeds, and is necessary for accessing a test's store
+  directory, which depends on :start-time. You may call this yourself before
+  calling run!, if you need access to the store directory outside the run!
+  context."
   [test]
   (cond-> test
     (not (:start-time test)) (assoc :start-time (util/local-time))
@@ -313,7 +316,8 @@
                                  (let [c (count (:nodes test))]
                                    (if (pos? c)
                                      (CyclicBarrier. (count (:nodes test)))
-                                     ::no-barrier)))))
+                                     ::no-barrier)))
+    true (update :generator util/forgettable)))
 
 (defn run!
   "Runs a test. Tests are maps containing

jepsen/src/jepsen/generator.clj

@@ -601,7 +601,15 @@
                 gen')]
 
           ; This generator is exhausted; move on
-          (recur (next this) test ctx))))))
+          (recur (next this) test ctx)))))
+
+  ; Forgettables are transparently unwrapped when treated as generators.
+  jepsen.util.Forgettable
+  (update [this test ctx event]
+    (update @this test ctx event))
+
+  (op [this test ctx]
+    (op @this test ctx)))
 
 (defmacro extend-protocol-runtime
   "Extends a protocol to a runtime-defined class. Helpful because some Clojure

jepsen/src/jepsen/generator/interpreter.clj

@@ -182,13 +182,14 @@
     true))
 
 (defn run!
-  "Takes a test with a :store :handle open. Opens a writer for the test's
-  history using that handle. Creates an initial context from test and evaluates
-  all ops from (:gen test). Spawns a thread for each worker, and hands those
-  workers operations from gen; each thread applies the operation using (:client
-  test) or (:nemesis test), as appropriate. Invocations and completions are
-  journaled to a history on disk. Returns a new test with no :generator and a
-  completed :history.
+  "Takes a test with a :store :handle open. Causes the test's reference to the
+  :generator to be forgotten, to avoid retaining the head of infinite seqs.
+  Opens a writer for the test's history using that handle. Creates an initial
+  context from test and evaluates all ops from (:gen test). Spawns a thread for
+  each worker, and hands those workers operations from gen; each thread applies
+  the operation using (:client test) or (:nemesis test), as appropriate.
+  Invocations and completions are journaled to a history on disk. Returns a new
+  test with no :generator and a completed :history.
 
   Generators are automatically wrapped in friendly-exception and validate.
   Clients are wrapped in a validator as well.
@@ -209,10 +210,11 @@
                             worker-ids)
           invocations (into {} (map (juxt :id :in) workers))
           gen         (->> (:generator test)
+                           deref
                            gen/friendly-exceptions
                            gen/validate)
-          ; Don't retain the head of the generator! If it's a lazy seq, this
-          ; will consume memory forever.
+          ; Forget generator
+          _           (util/forget! (:generator test))
           test        (dissoc test :generator)]
       (try+
         (loop [ctx            ctx

jepsen/src/jepsen/util.clj

@@ -3,19 +3,21 @@
   (:refer-clojure :exclude [parse-long]) ; Clojure added this in 1.11.1
   (:require [clojure.tools.logging :refer [info]]
             [clojure.core.reducers :as r]
-            [clojure [string :as str]]
-            [clojure.pprint :refer [pprint]]
-            [clojure.walk :as walk]
+            [clojure [string :as str]
+                     [pprint :as pprint :refer [pprint]]
+                     [walk :as walk]]
             [clojure.java [io :as io]
                           [shell :as shell]]
             [clj-time.core :as time]
             [clj-time.local :as time.local]
             [clojure.tools.logging :refer [debug info warn]]
             [dom-top.core :as dt :refer [bounded-future]]
             [fipp [edn :as fipp]
+                  [ednize]
                   [engine :as fipp.engine]]
             [jepsen [history :as h]]
             [jepsen.history.fold :refer [loopf]]
+            [potemkin :refer [definterface+]]
             [slingshot.slingshot :refer [try+ throw+]]
             [tesser.core :as t])
   (:import (java.lang.reflect Method)
@@ -1025,3 +1027,63 @@
 
          true
          nil)))
+
+(definterface+ IForgettable
+  (forget! [this]
+           "Allows this forgettable reference to be reclaimed by the GC at some
+           later time. Future attempts to dereference it may throw. Returns
+           self."))
+
+(deftype Forgettable [^:unsynchronized-mutable x]
+  IForgettable
+  (forget! [this]
+    (set! x ::forgotten)
+    this)
+
+  clojure.lang.IDeref
+  (deref [this]
+    (let [x x]
+      (if (identical? x ::forgotten)
+        (throw+ {:type ::forgotten})
+        x)))
+
+  Object
+  (toString [this]
+    (let [x x]
+      (str "#<Forgettable " (if (identical? x ::forgotten)
+                              "?"
+                              x)
+           ">")))
+
+  (equals [this other]
+    (identical? this other)))
+
+(defn forgettable
+  "Constructs a deref-able reference to x which can be explicitly forgotten.
+  Helpful for controlling access to infinite seqs (e.g. the generator) when you
+  don't have firm control over everyone who might see them."
+  [x]
+  (Forgettable. x))
+
+(defmethod pprint/simple-dispatch jepsen.util.Forgettable
+  [^Forgettable f]
+  (let [prefix (format "#<Forgettable ")]
+    (pprint/pprint-logical-block
+      :prefix prefix :suffix ">"
+      (pprint/pprint-indent :block (-> (count prefix) (- 2) -))
+      (pprint/pprint-newline :linear)
+      (pprint/write-out (try+ @f
+                              (catch [:type ::forgotten] e
+                                "?"))))))
+
+(prefer-method pprint/simple-dispatch
+               jepsen.util.Forgettable clojure.lang.IDeref)
+
+(extend-protocol fipp.ednize/IOverride jepsen.util.Forgettable)
+(extend-protocol fipp.ednize/IEdn jepsen.util.Forgettable
+  (-edn [f]
+    (fipp.ednize/tagged-object f
+                               (try+ @f
+                                     (catch [:type ::forgotten] e
+                                       '?)))))
+

jepsen/test/jepsen/util_test.clj

@@ -1,6 +1,8 @@
 (ns jepsen.util-test
   (:refer-clojure :exclude [parse-long])
-  (:require [clojure.test :refer :all]
+  (:require [clojure [pprint :refer [pprint]]
+                     [test :refer :all]]
+            [fipp.edn :as fipp]
             [jepsen [history :as h]
                     [util :refer :all]]))
 
@@ -174,3 +176,15 @@
     (is (< (* target-mean 0.7)
            mean
            (* target-mean 1.3)))))
+
+(deftest forgettable-test
+  (let [f (forgettable :foo)]
+    (is (= :foo @f))
+    (is (= "#<Forgettable :foo>" (str f)))
+    (is (= "#<Forgettable :foo>\n" (with-out-str (pprint f))))
+    (is (re-find #"^#object\[jepsen.util.Forgettable \"0x\w+\" :foo\]\n$"
+                 (with-out-str (fipp/pprint f))))
+    (forget! f)
+    (is (thrown-with-msg? clojure.lang.ExceptionInfo
+                          #"\{:type :jepsen\.util/forgotten\}"
+                          @f))))