Initial setup for hazelcast-raft testing

Author: mdogan

hazelcast/project.clj

@@ -1,9 +1,10 @@
-(defproject jepsen.hazelcast "0.1.0-SNAPSHOT"
-  :description "Jepsen tests for hazelcast"
+(defproject jepsen.hazelcast-raft "0.1.0-SNAPSHOT"
+  :description "Jepsen tests for hazelcast-raft"
   :url "http://jepsen.io/"
   :license {:name "Eclipse Public License"
             :url "http://www.eclipse.org/legal/epl-v10.html"}
+  :repositories [["jitpack" "https://jitpack.io"]]
   :dependencies [[org.clojure/clojure "1.8.0"]
                  [jepsen "0.1.6"]
-                 [com.hazelcast/hazelcast-client "3.8"]]
+                 [com.github.mdogan.hazelcast/hazelcast-raft-client "raft-v3"]]
   :main jepsen.hazelcast)

hazelcast/server/project.clj

@@ -5,11 +5,12 @@
             :url "http://www.eclipse.org/legal/epl-v10.html"}
   :source-paths ["src"]
   :java-source-paths ["java"]
+  :repositories [["jitpack" "https://jitpack.io"]]
   :dependencies [[org.clojure/clojure "1.8.0"]
                  [org.clojure/tools.cli "0.3.5"]
                  [org.clojure/tools.logging "0.3.1"]
                  [spootnik/unilog "0.7.13"]
-                 [com.hazelcast/hazelcast "3.8.3"]]
+                 [com.github.mdogan.hazelcast/hazelcast-raft "raft-v3"]]
   :profiles {:uberjar {:uberjar-name "hazelcast-server.jar"}}
   :main jepsen.hazelcast-server
   :aot [jepsen.hazelcast-server])

hazelcast/server/src/jepsen/hazelcast_server.clj

@@ -7,14 +7,49 @@
   (:import (com.hazelcast.core Hazelcast)
            (com.hazelcast.config Config
                                  LockConfig
+                                 ServiceConfig
                                  MapConfig
-                                 QuorumConfig)))
+                                 QuorumConfig)
+           (com.hazelcast.raft RaftConfig
+                               RaftMember)))
 
 (def opt-spec
   [["-m" "--members MEMBER-LIST" "Comma-separated list of peers to connect to"
     :parse-fn (fn [m]
                 (str/split m #"\s*,\s*"))]])
 
+(defn prepareRaftServiceConfig
+  "Prepare Hazelcast RaftConfig and ServiceConfig"
+  [members]
+  (let [raftConfig (RaftConfig.)
+        serviceConfig (ServiceConfig.)
+
+        ; add raft members
+        _       (doseq [member members]
+                  (info "Adding " member " to raft group")
+                  (.addMember raftConfig (RaftMember. (str member ":5701") member)))
+
+        _ (.setLeaderElectionTimeoutInMillis raftConfig 1000)
+        _ (.setLeaderHeartbeatPeriodInMillis raftConfig 1000)
+
+        ; prepare service config
+        _ (.setEnabled serviceConfig true)
+        _ (.setName serviceConfig com.hazelcast.raft.impl.service.RaftService/SERVICE_NAME)
+        _ (.setClassName serviceConfig (.getName com.hazelcast.raft.impl.service.RaftService))
+        _ (.setConfigObject serviceConfig raftConfig)
+        ]
+    serviceConfig))
+
+(defn prepareAtomicLongServiceConfig
+  "Prepare Raft AtomicLong service config"
+  []
+  (let [serviceConfig (ServiceConfig.)
+        _ (.setEnabled serviceConfig true)
+        _ (.setName serviceConfig com.hazelcast.raft.service.atomiclong.RaftAtomicLongService/SERVICE_NAME)
+        _ (.setClassName serviceConfig (.getName com.hazelcast.raft.service.atomiclong.RaftAtomicLongService))
+        ]
+    serviceConfig))
+
 (defn -main
   "Go go go"
   [& args]
@@ -23,25 +58,30 @@
                 summary
                 errors]} (cli/parse-opts args opt-spec)
         config  (Config.)
+        members (:members options)
 
         ; Timeouts
-        _ (.setProperty config "hazelcast.client.heartbeat.interval" "1000")
-        _ (.setProperty config "hazelcast.client.heartbeat.timeout" "5000")
-        _ (.setProperty config "hazelcast.client.invocation.timeout.seconds" "5")
+        _ (.setProperty config "hazelcast.client.max.no.heartbeat.seconds" "5")
         _ (.setProperty config "hazelcast.heartbeat.interval.seconds" "1")
-        _ (.setProperty config "hazelcast.master.confirmation.interval.seconds" "1")
         _ (.setProperty config "hazelcast.max.no.heartbeat.seconds" "5")
-        _ (.setProperty config "hazelcast.max.no.master.confirmation.seconds" "10")
         _ (.setProperty config "hazelcast.operation.call.timeout.millis" "5000")
+        _ (.setProperty config "hazelcast.wait.seconds.before.join" "0")
+        _ (.setProperty config "hazelcast.merge.first.run.delay.seconds" "1")
+        _ (.setProperty config "hazelcast.merge.next.run.delay.seconds" "1")
 
         ; Network config
         _       (.. config getNetworkConfig getJoin getMulticastConfig
                     (setEnabled false))
         tcp-ip  (.. config getNetworkConfig getJoin getTcpIpConfig)
-        _       (doseq [member (:members options)]
+        _       (doseq [member members]
                   (.addMember tcp-ip member))
         _       (.setEnabled tcp-ip true)
 
+        ; prepare raft services
+        servicesConfig (.getServicesConfig config)
+        _ (.addServiceConfig servicesConfig (prepareRaftServiceConfig members))
+        _ (.addServiceConfig servicesConfig (prepareAtomicLongServiceConfig))
+
         ; Quorum for split-brain protection
         quorum (doto (QuorumConfig.)
                  (.setName "majority")

hazelcast/src/jepsen/hazelcast.clj

@@ -26,8 +26,7 @@
            (com.hazelcast.client.config ClientConfig)
            (com.hazelcast.client HazelcastClient)
            (com.hazelcast.core HazelcastInstance)
-           (com.hazelcast.core IMap)
-           (com.hazelcast.spi.discovery NodeFilter)))
+           (com.hazelcast.core IMap)))
 
 (def local-server-dir
   "Relative path to local server project directory"
@@ -79,7 +78,6 @@
           "/usr/bin/java"
           :-jar jar
           :--members (->> (:nodes test)
-                          (remove #{node})
                           (map cn/ip)
                           (str/join ",")))))
 
@@ -119,11 +117,6 @@
         _ (.setProperty config "hazelcast.client.heartbeat.interval" "1000")
         _ (.setProperty config "hazelcast.client.heartbeat.timeout" "5000")
         _ (.setProperty config "hazelcast.client.invocation.timeout.seconds" "5")
-        _ (.setProperty config "hazelcast.heartbeat.interval.seconds" "1")
-        _ (.setProperty config "hazelcast.master.confirmation.interval.seconds" "1")
-        _ (.setProperty config "hazelcast.max.no.heartbeat.seconds" "5")
-        _ (.setProperty config "hazelcast.max.no.master.confirmation.seconds" "10")
-        _ (.setProperty config "hazelcast.operation.call.timeout.millis" "5000")
 
         net    (doto (.getNetworkConfig config)
                  ; Don't retry operations when network fails (!?)
@@ -140,14 +133,6 @@
         ; Only talk to our node (the client's smart and will try to talk to
         ; everyone, but we're trying to simulate clients in different network
         ; components here)
-        node-filter (reify NodeFilter
-                      (test [this candidate]
-                        (prn node :testing candidate)
-                        (info node :testing candidate)
-                        true))
-        _      (.. net
-                   getDiscoveryConfig
-                   (setNodeFilter node-filter))
         ; Connect to our node
         _      (.addAddress net (into-array String [node]))]
     (HazelcastClient/newHazelcastClient config)))
@@ -168,6 +153,28 @@
     (teardown! [this test]
       (.terminate conn))))
 
+
+(defn create-raft-atomic-long
+  "Creates a new Raft based AtomicLong"
+  [client name test]
+  (com.hazelcast.raft.impl.client.RaftAtomicLong/create client name (count (:nodes test))))
+
+(defn raft-atomic-long-id-client
+  "Generates unique IDs using a Raft based AtomicLong"
+  [conn atomic-long]
+  (reify client/Client
+    (setup! [_ test node]
+      (let [conn (connect node)]
+        (raft-atomic-long-id-client conn
+                   (create-raft-atomic-long conn "jepsen.atomic-long" test))))
+
+    (invoke! [this test op]
+      (assert (= (:f op) :generate))
+      (assoc op :type :ok, :value (.incrementAndGet atomic-long)))
+
+    (teardown! [this test]
+      (.terminate conn))))
+
 (defn atomic-ref-id-client
   "Generates unique IDs using an AtomicReference"
   [conn atomic-ref]
@@ -394,6 +401,10 @@
                       :generator (->> {:type :invoke, :f :generate}
                                       (gen/stagger 1))
                       :checker  (checker/unique-ids)}
+   :raft-atomic-long-ids  {:client (raft-atomic-long-id-client nil nil)
+                      :generator (->> {:type :invoke, :f :generate}
+                                      (gen/stagger 1))
+                      :checker  (checker/unique-ids)}
    :id-gen-ids       {:client    (id-gen-id-client nil nil)
                       :generator {:type :invoke, :f :generate}
                       :checker   (checker/unique-ids)}})