Skip to content

Commit

Permalink
Contended MV tests. Associative map checker. Parameterized standard g…
Browse files Browse the repository at this point in the history 
…en run length.
  • Loading branch information
@jkni
jkni committed Jul 30, 2015
1 parent cfa83ee commit b389ddc
Show file tree
Hide file tree
Showing 6 changed files with 152 additions and 90 deletions.
6 changes: 3 additions & 3 deletions cassandra/project.clj
View file
Expand Up @@ -5,14 +5,14 @@
:url "http://www.eclipse.org/legal/epl-v10.html"}
:dependencies [[org.clojure/clojure "1.6.0"]
[org.clojure/java.jmx "0.3.1"]
[jepsen "0.0.4"]
[clojurewerkz/cassaforte "2.1.0-beta1"]
[jkni/jepsen "0.0.5-SNAPSHOT"]
[jkni/cassaforte "trunk-SNAPSHOT"]
[com.codahale.metrics/metrics-core "3.0.2"]]
:test-selectors {:steady :steady
:bootstrap :bootstrap
:map :map
:set :set
:mv-set :mv-set
:mv :mv
:batch-set :batch-set
:lwt :lwt
:decommission :decommission
Expand Down
42 changes: 26 additions & 16 deletions cassandra/src/cassandra/core.clj
View file
Expand Up @@ -248,23 +248,24 @@
(defn std-gen
"Takes a client generator and wraps it in a typical schedule and nemesis
causing failover."
[gen]
(gen/phases
(->> gen
(gen/nemesis
(gen/seq (cycle [(gen/sleep (scaled 15))
{:type :info :f :start}
(gen/sleep (scaled 45))
{:type :info :f :stop}])))
(bootstrap 120)
(gen/conductor :decommissioner
(gen/seq (cycle [(gen/sleep (scaled 100))
{:type :info :f :decommission}])))
(gen/time-limit (scaled 400)))
(recover)
(gen/clients
([gen] (std-gen 400 gen))
([duration gen]
(gen/phases
(->> gen
(gen/time-limit (scaled 40))))))
(gen/nemesis
(gen/seq (cycle [(gen/sleep (scaled 15))
{:type :info :f :start}
(gen/sleep (scaled 45))
{:type :info :f :stop}])))
(bootstrap 120)
(gen/conductor :decommissioner
(gen/seq (cycle [(gen/sleep (scaled 100))
{:type :info :f :decommission}])))
(gen/time-limit (scaled duration)))
(recover)
(gen/clients
(->> gen
(gen/time-limit (scaled 40)))))))

(def add {:type :invoke :f :add :value 1})
(def sub {:type :invoke :f :add :value -1})
Expand All @@ -279,6 +280,15 @@
(map (fn [x] {:type :invoke, :f :add, :value x}))
gen/seq))

(defn assocs
"Generator that emits :assoc operations for sequential integers,
mapping x to (f x)"
[f]
(->> (range)
(map (fn [x] {:type :invoke :f :assoc :value {:k x
:v (f x)}}))
gen/seq))

(defn read-once
"A generator which reads exactly once."
[]
Expand Down
112 changes: 60 additions & 52 deletions cassandra/src/cassandra/mv.clj
View file
Expand Up @@ -30,12 +30,13 @@
[cassandra.conductors :as conductors])
(:import (clojure.lang ExceptionInfo)
(com.datastax.driver.core ConsistencyLevel)
(com.datastax.driver.core.policies FallthroughRetryPolicy)
(com.datastax.driver.core.exceptions UnavailableException
WriteTimeoutException
ReadTimeoutException
NoHostAvailableException)))

(defrecord MVSetClient [conn]
(defrecord MVMapClient [conn]
client/Client
(setup! [_ test node]
(locking setup-lock
Expand All @@ -46,38 +47,38 @@
{:class "SimpleStrategy"
:replication_factor 3}}))
(cql/use-keyspace conn "jepsen_keyspace")
(cql/create-table conn "original"
(cql/create-table conn "map"
(if-not-exists)
(column-definitions {:id :int
:mvid :int
:primary-key [:id]}))
(try (cassandra/execute conn (str "CREATE MATERIALIZED VIEW mv AS SELECT"
" * FROM original WHERE mvid IS NOT NULL"
" AND id IS NOT NULL "
"PRIMARY KEY (mvid, id);"))
(column-definitions {:key :int
:value :int
:primary-key [:key]}))
(try (cassandra/execute conn (str "CREATE MATERIALIZED VIEW mvmap AS SELECT"
" * FROM map WHERE value IS NOT NULL"
" AND key IS NOT NULL "
"PRIMARY KEY (value, key);"))
(catch com.datastax.driver.core.exceptions.AlreadyExistsException e))
(->MVSetClient conn))))
(->MVMapClient conn))))
(invoke! [this test op]
(case (:f op)
:add (try (with-consistency-level ConsistencyLevel/QUORUM
(cql/insert conn
"original"
{:id (:value op)
:mvid (:value op)}))
(assoc op :type :ok)
(catch UnavailableException e
(assoc op :type :fail :value (.getMessage e)))
(catch WriteTimeoutException e
(assoc op :type :info :value :timed-out))
(catch NoHostAvailableException e
(info "All nodes are down - sleeping 2s")
(Thread/sleep 2000)
(assoc op :type :fail :value (.getMessage e))))
:assoc (try (with-retry-policy FallthroughRetryPolicy/INSTANCE
(with-consistency-level ConsistencyLevel/QUORUM
(cql/update conn
"map"
{:value (:v (:value op))}
(where [[= :key (:k (:value op))]]))))
(assoc op :type :ok)
(catch UnavailableException e
(assoc op :type :fail :value (.getMessage e)))
(catch WriteTimeoutException e
(assoc op :type :info :value :timed-out))
(catch NoHostAvailableException e
(info "All nodes are down - sleeping 2s")
(Thread/sleep 2000)
(assoc op :type :fail :value (.getMessage e))))
:read (try (let [value (->> (with-retry-policy aggressive-read
(with-consistency-level ConsistencyLevel/ALL
(cql/select conn "mv")))
(map :mvid)
(into (sorted-set)))]
(cql/select conn "mvmap")))
(#(zipmap (map :key %) (map :value %))))]
(assoc op :type :ok :value value))
(catch UnavailableException e
(info "Not enough replicas - failing")
Expand All @@ -92,99 +93,106 @@
(info "Tearing down client with conn" conn)
(cassandra/disconnect! conn)))

(defn mv-set-client
"A set implemented using MV"
(defn mv-map-client
"A map implemented using MV"
[]
(->MVSetClient nil))
(->MVMapClient nil))

(defn mv-set-test
(defn mv-map-test
[name opts]
(merge (cassandra-test (str "mv set " name)
{:client (mv-set-client)
:model (model/set)
(merge (cassandra-test (str "mv contended map " name)
{:client (mv-map-client)
:generator (gen/phases
(->> (gen/clients (adds))
(->> (gen/clients (assocs identity))
(gen/delay 1)
std-gen)
(std-gen 250))
(gen/conductor :replayer
(gen/once {:type :info :f :replay}))
(read-once)
(->> (gen/clients (assocs -))
(gen/delay 1)
(std-gen 250))
(gen/conductor :replayer
(gen/once {:type :info :f :replay}))
(read-once))
:checker (checker/compose
{:set checker/set})})
{:map checker/associative-map})})
(merge-with merge {:conductors {:replayer (conductors/replayer)}} opts)))


;; Uncontended tests
(def bridge-test
(mv-set-test "bridge"
(mv-map-test "bridge"
{:conductors {:nemesis (nemesis/partitioner (comp nemesis/bridge shuffle))}}))

(def halves-test

(mv-set-test "halves"
(mv-map-test "halves"
{:conductors {:nemesis (nemesis/partition-random-halves)}}))

(def isolate-node-test
(mv-set-test "isolate node"
(mv-map-test "isolate node"
{:conductors {:nemesis (nemesis/partition-random-node)}}))

(def crash-subset-test
(mv-set-test "crash"
(mv-map-test "crash"
{:conductors {:nemesis (crash-nemesis)}}))

(def clock-drift-test
(mv-set-test "clock drift"
(mv-map-test "clock drift"
{:conductors {:nemesis (nemesis/clock-scrambler 10000)}}))

(def bridge-test-bootstrap
(mv-set-test "bridge bootstrap"
(mv-map-test "bridge bootstrap"
{:bootstrap (atom #{:n4 :n5})
:conductors {:nemesis (nemesis/partitioner (comp nemesis/bridge shuffle))
:bootstrapper (conductors/bootstrapper)}}))

(def halves-test-bootstrap
(mv-set-test "halves bootstrap"
(mv-map-test "halves bootstrap"
{:bootstrap (atom #{:n4 :n5})
:conductors {:nemesis (nemesis/partition-random-halves)
:bootstrapper (conductors/bootstrapper)}}))

(def isolate-node-test-bootstrap
(mv-set-test "isolate node bootstrap"
(mv-map-test "isolate node bootstrap"
{:bootstrap (atom #{:n4 :n5})
:conductors {:nemesis (nemesis/partition-random-node)
:bootstrapper (conductors/bootstrapper)}}))

(def crash-subset-test-bootstrap
(mv-set-test "crash bootstrap"
(mv-map-test "crash bootstrap"
{:bootstrap (atom #{:n4 :n5})
:conductors {:nemesis (crash-nemesis)
:bootstrapper (conductors/bootstrapper)}}))

(def clock-drift-test-bootstrap
(mv-set-test "clock drift bootstrap"
(mv-map-test "clock drift bootstrap"
{:bootstrap (atom #{:n4 :n5})
:conductors {:nemesis (nemesis/clock-scrambler 10000)
:bootstrapper (conductors/bootstrapper)}}))

(def bridge-test-decommission
(mv-set-test "bridge decommission"
(mv-map-test "bridge decommission"
{:conductors {:nemesis (nemesis/partitioner (comp nemesis/bridge shuffle))
:decommissioner (conductors/decommissioner)}}))

(def halves-test-decommission
(mv-set-test "halves decommission"
(mv-map-test "halves decommission"
{:conductors {:nemesis (nemesis/partition-random-halves)
:decommissioner (conductors/decommissioner)}}))

(def isolate-node-test-decommission
(mv-set-test "isolate node decommission"
(mv-map-test "isolate node decommission"
{:conductors {:nemesis (nemesis/partition-random-node)
:decommissioner (conductors/decommissioner)}}))

(def crash-subset-test-decommission
(mv-set-test "crash decommission"
(mv-map-test "crash decommission"
{:conductors {:nemesis (crash-nemesis)
:decommissioner (conductors/decommissioner)}}))

(def clock-drift-test-decommission
(mv-set-test "clock drift decommission"
(mv-map-test "clock drift decommission"
{:conductors {:nemesis (nemesis/clock-scrambler 10000)
:decommissioner (conductors/decommissioner)}}))
30 changes: 15 additions & 15 deletions cassandra/test/cassandra/mv_test.clj
View file
Expand Up @@ -7,49 +7,49 @@
[report :as report]]))

;; Steady state cluster tests
(deftest ^:mv-set ^:steady mv-set-bridge
(deftest ^:mv ^:steady mv-bridge
(run-test! bridge-test))

(deftest ^:mv-set ^:steady mv-set-isolate-node
(deftest ^:mv ^:steady mv-isolate-node
(run-test! isolate-node-test))

(deftest ^:mv-set ^:steady mv-set-halves
(deftest ^:mv ^:steady mv-halves
(run-test! halves-test))

(deftest ^:mv-set ^:steady mv-set-crash-subset
(deftest ^:mv ^:steady mv-crash-subset
(run-test! crash-subset-test))

(deftest ^:clock mv-set-clock-drift
(deftest ^:clock mv-clock-drift
(run-test! clock-drift-test))

;; Bootstrapping tests
(deftest ^:mv-set ^:bootstrap mv-set-bridge-bootstrap
(deftest ^:mv ^:bootstrap mv-bridge-bootstrap
(run-test! bridge-test-bootstrap))

(deftest ^:mv-set ^:bootstrap mv-set-isolate-node-bootstrap
(deftest ^:mv ^:bootstrap mv-isolate-node-bootstrap
(run-test! isolate-node-test-bootstrap))

(deftest ^:mv-set ^:bootstrap mv-set-halves-bootstrap
(deftest ^:mv ^:bootstrap mv-halves-bootstrap
(run-test! halves-test-bootstrap))

(deftest ^:mv-set ^:bootstrap mv-set-crash-subset-bootstrap
(deftest ^:mv ^:bootstrap mv-crash-subset-bootstrap
(run-test! crash-subset-test-bootstrap))

(deftest ^:clock mv-set-clock-drift-bootstrap
(deftest ^:clock mv-clock-drift-bootstrap
(run-test! clock-drift-test-bootstrap))

;; Decommission tests
(deftest ^:mv-set ^:decommission mv-set-bridge-decommission
(deftest ^:mv ^:decommission mv-bridge-decommission
(run-test! bridge-test-decommission))

(deftest ^:mv-set ^:decommission mv-set-isolate-node-decommission
(deftest ^:mv ^:decommission mv-isolate-node-decommission
(run-test! isolate-node-test-decommission))

(deftest ^:mv-set ^:decommission mv-set-halves-decommission
(deftest ^:mv ^:decommission mv-halves-decommission
(run-test! halves-test-decommission))

(deftest ^:mv-set ^:decommission mv-set-crash-subset-decommission
(deftest ^:mv ^:decommission mv-crash-subset-decommission
(run-test! crash-subset-test-decommission))

(deftest ^:clock mv-set-clock-drift-decommission
(deftest ^:clock mv-clock-drift-decommission
(run-test! clock-drift-test-decommission))
4 changes: 2 additions & 2 deletions jepsen/project.clj
View file
@@ -1,12 +1,12 @@
(defproject jepsen "0.0.4"
(defproject jkni/jepsen "0.0.5-SNAPSHOT"
:description "Call Me Maybe: Network Partitions in Practice"
:dependencies [[org.clojure/clojure "1.6.0"]
[org.clojure/data.fressian "0.2.0"]
; :exclusions [org.fressian/fressian]]
[org.clojure/tools.logging "0.2.6"]
[clj-time "0.6.0"]
[knossos "0.2.2"]
[clj-ssh "0.5.11"]
[jkni/clj-ssh "0.5.12-SNAPSHOT"]
[gnuplot "0.1.0"]
[hiccup "1.0.5"]
[org.clojars.achim/multiset "0.1.0"]
Expand Down

0 comments on commit b389ddc

Please sign in to comment.