Mongo: first failing tests

jepsen-io · Feb 25, 2015 · b3ba335 · b3ba335
1 parent a5567f1
commit b3ba335
Show file tree

Hide file tree

Showing 5 changed files with 25 additions and 15 deletions.
diff --git a/.gitignore b/.gitignore
@@ -19,8 +19,8 @@ build
 /site
 site/**
 bench/**
-target/**
-checkouts/**
+*/target/**
+*/checkouts/**
 */store/**
 */timeline.html
 */resources/datomic/download-key

diff --git a/jepsen/project.clj b/jepsen/project.clj
@@ -5,7 +5,7 @@
 ;                            :exclusions [org.fressian/fressian]]
                  [org.clojure/tools.logging "0.2.6"]
                  [clj-time "0.6.0"]
-                 [knossos "0.2"]
+                 [knossos "0.2.1-SNAPSHOT"]
                  [clj-ssh "0.5.7"]
                  [hiccup "1.0.5"]
                  [org.clojars.achim/multiset "0.1.0-SNAPSHOT"]

diff --git a/jepsen/target/stale/extract-native.dependencies b/jepsen/target/stale/extract-native.dependencies
diff --git a/mongodb/src/mongodb/core.clj b/mongodb/src/mongodb/core.clj
@@ -141,11 +141,21 @@
   (try
     (admin-command! conn :replSetInitiate config)
     (catch ExceptionInfo e
-      ; Some of the time (but not all the time; why?) Mongo returns this error
-      ; from replsetinitiate, which is, as far as I can tell, not actually an
-      ; error (?)
-      (when-not (= "Received replSetInitiate - should come online shortly."
-                   (get-in (ex-data e) [:result "errmsg"]))
+      (condp re-find (get-in (ex-data e) [:result "errmsg"])
+        ; Some of the time (but not all the time; why?) Mongo returns this error
+        ; from replsetinitiate, which is, as far as I can tell, not actually an
+        ; error (?)
+        #"Received replSetInitiate - should come online shortly."
+        nil
+
+        ; This is a hint we should back off and retry; one of the nodes probably
+        ; isn't fully alive yet.
+        #"need all members up to initiate, not ok"
+        (do (info "not all members alive yet; retrying replica set initiate"
+            (Thread/sleep 1000)
+            (replica-set-initiate! conn config)))
+
+        ; Or by default re-throw
         (throw e)))))
 
 (defn replica-set-master?
@@ -385,7 +395,7 @@
   (DocumentCASClient. "jepsen"
                       "jepsen"
                       (ObjectId.)
-                      WriteConcern/JOURNAL_SAFE
+                      WriteConcern/MAJORITY
                       nil
                       nil))
 
@@ -404,11 +414,10 @@
                       (->> gen/cas
                            (gen/delay 1)
                            (gen/nemesis
-                             (gen/seq (cycle [(gen/sleep 30)
-                                              {:type :info :f :start}
-                                              (gen/sleep 30)
-                                              {:type :info :f :stop}])))
-                           (gen/time-limit 30))
+                             (gen/seq (cycle [(gen/sleep 45)
+                                              {:type :info :f :stop}
+                                              {:type :info :f :start}])))
+                           (gen/time-limit 200))
                       (gen/nemesis
                         (gen/once {:type :info :f :stop}))
                       (gen/clients

diff --git a/mongodb/test/mongodb/core_test.clj b/mongodb/test/mongodb/core_test.clj
@@ -15,4 +15,6 @@
 (deftest document-cas-test
   (let [test (jepsen/run! (m/document-cas-test))]
     (is (:valid? (:results test)))
+    (println "history is")
+    (pprint (:history test))
     (-> test :results :linear report/linearizability)))