Merge pull request #59 from patc888/master

Updated the tsunami test

core/README.md

@@ -40,15 +40,15 @@ The test will automatically create the necessary tables if it doesn't find them.
 
 In one console, run the gondola agent command. This process listens for commands to start and stop a gondola instance.
 ```
-> ./bin/gondola-agent.sh
+> ./bin/gondola-agent
 2015-09-27 10:07:58,638 INFO Initialize system, kill all gondola processes
 2015-09-27 10:07:59,973 INFO Listening on port 1200
 ...
 ```
 
 In another console, run the tsunami test:
 ```
-> ./bin/tsunami.sh
+> ./bin/tsunami
 Dropping all tables
 2015-09-27 10:20:00,879 INFO Connecting to gondola agent at localhost:1200
 2015-09-27 10:20:00,881 INFO Connecting to gondola agent at localhost:1200

core/bin/gondola

@@ -15,10 +15,10 @@ if [ -z "$JAVA_OPTS" ]; then
   export xJAVA_OPTS="-ea -XX:+UnlockCommercialFeatures -XX:+FlightRecorder"
 fi
 
-# Error if hostid is not specified
+# Error if host is not specified
 function check_hostid {
   if [ -z "$HOSTID" ]; then
-    echo "-hostid is missing"
+    echo "-host is missing"
     sh $GONDOLA_CMD $PARAMS
     exit 1
   fi
@@ -27,7 +27,7 @@ function check_hostid {
 # Find the hostid. To be used for the pid file.
 while test $# -gt 0; do
   case "$1" in
-    -hostid)
+    -host)
       shift
       if test $# -gt 0; then
         export HOSTID=$1

core/bin/gondola-local-test.sh

@@ -2,9 +2,9 @@
 
 function start  {
   rm -rf /tmp/gondola*
-  env JAVA_OPTS="-agentlib:jdwp=transport=dt_socket,server=y,suspend=n,address=5005 -XX:+UnlockCommercialFeatures -XX:+FlightRecorder" nohup bin/gondola.sh -hostid host1 -clusterid cluster1 -workers  10 -conf conf/gondola-sample.conf &
-  env JAVA_OPTS="-agentlib:jdwp=transport=dt_socket,server=y,suspend=n,address=5006 -XX:+UnlockCommercialFeatures -XX:+FlightRecorder" nohup bin/gondola.sh -hostid host2 -clusterid cluster1 -workers  10 -conf conf/gondola-sample.conf &
-  env JAVA_OPTS="-agentlib:jdwp=transport=dt_socket,server=y,suspend=n,address=5007 -XX:+UnlockCommercialFeatures -XX:+FlightRecorder" nohup bin/gondola.sh -hostid host3 -clusterid cluster1 -workers  10 -conf conf/gondola-sample.conf &
+  env JAVA_OPTS="-agentlib:jdwp=transport=dt_socket,server=y,suspend=n,address=5005 -XX:+UnlockCommercialFeatures -XX:+FlightRecorder" nohup bin/gondola -hostid host1 -clusterid cluster1 -workers  10 -conf conf/gondola-sample.conf &
+  env JAVA_OPTS="-agentlib:jdwp=transport=dt_socket,server=y,suspend=n,address=5006 -XX:+UnlockCommercialFeatures -XX:+FlightRecorder" nohup bin/gondola -hostid host2 -clusterid cluster1 -workers  10 -conf conf/gondola-sample.conf &
+  env JAVA_OPTS="-agentlib:jdwp=transport=dt_socket,server=y,suspend=n,address=5007 -XX:+UnlockCommercialFeatures -XX:+FlightRecorder" nohup bin/gondola -hostid host3 -clusterid cluster1 -workers  10 -conf conf/gondola-sample.conf &
 }
 
 function stop {

core/bin/gondola-remote.sh

@@ -3,9 +3,9 @@ PATH=/usr/bin:/usr/local/bin:/home/y/bin:/home/y/sbin:/home/y/bin64:/home/y/sbin
 
 function start {
   rm -rf /tmp/gondola*
-  env JAVA_OPTS="-agentlib:jdwp=transport=dt_socket,server=y,suspend=n,address=5005" nohup bin/gondola.sh -hostid host1 -clusterid cluster1 -port 1099 -config conf/gondola-remote.conf start 2>&1
-  env JAVA_OPTS="-agentlib:jdwp=transport=dt_socket,server=y,suspend=n,address=5006" nohup bin/gondola.sh -hostid host2 -clusterid cluster1 -port 1100 -config conf/gondola-remote.conf start 2>&1
-  env JAVA_OPTS="-agentlib:jdwp=transport=dt_socket,server=y,suspend=n,address=5007" nohup bin/gondola.sh -hostid host3 -clusterid cluster1 -port 1101 -config conf/gondola-remote.conf start 2>&1
+  env JAVA_OPTS="-agentlib:jdwp=transport=dt_socket,server=y,suspend=n,address=5005" nohup bin/gondola -hostid host1 -clusterid cluster1 -port 1099 -config conf/gondola-remote.conf start 2>&1
+  env JAVA_OPTS="-agentlib:jdwp=transport=dt_socket,server=y,suspend=n,address=5006" nohup bin/gondola -hostid host2 -clusterid cluster1 -port 1100 -config conf/gondola-remote.conf start 2>&1
+  env JAVA_OPTS="-agentlib:jdwp=transport=dt_socket,server=y,suspend=n,address=5007" nohup bin/gondola -hostid host3 -clusterid cluster1 -port 1101 -config conf/gondola-remote.conf start 2>&1
 }
 
 function stop {

core/bin/gondola-test.sh

@@ -2,10 +2,10 @@
 
 function start {
   rm -rf /tmp/gondola*
-  env JAVA_OPTS="-agentlib:jdwp=transport=dt_socket,server=y,suspend=n,address=5005" nohup bin/gondola.sh -hostid host1 -clusterid cluster1 -port 1099 -conf conf/gondola-sample.conf start 2>&1
+  env JAVA_OPTS="-agentlib:jdwp=transport=dt_socket,server=y,suspend=n,address=5005" nohup bin/gondola -hostid host1 -clusterid cluster1 -port 1099 -conf conf/gondola-sample.conf start 2>&1
   sleep 1
-  env JAVA_OPTS="-agentlib:jdwp=transport=dt_socket,server=y,suspend=n,address=5006" nohup bin/gondola.sh -hostid host2 -clusterid cluster1 -port 1100 -conf conf/gondola-sample.conf start 2>&1
-  env JAVA_OPTS="-agentlib:jdwp=transport=dt_socket,server=y,suspend=n,address=5007" nohup bin/gondola.sh -hostid host3 -clusterid cluster1 -port 1101 -conf conf/gondola-sample.conf start 2>&1
+  env JAVA_OPTS="-agentlib:jdwp=transport=dt_socket,server=y,suspend=n,address=5006" nohup bin/gondola -hostid host2 -clusterid cluster1 -port 1100 -conf conf/gondola-sample.conf start 2>&1
+  env JAVA_OPTS="-agentlib:jdwp=transport=dt_socket,server=y,suspend=n,address=5007" nohup bin/gondola -hostid host3 -clusterid cluster1 -port 1101 -conf conf/gondola-sample.conf start 2>&1
   echo "Wait for server up and send force leader signal.."
   sleep 1
   echo "F" | nc localhost 1099

core/conf/gondola-command.conf

@@ -7,29 +7,44 @@
 
 gondola {
   hosts = [
-    { hostId = "H1", hostname = "127.0.0.1", port = "2828" }
-    { hostId = "H2", hostname = "127.0.0.1", port = "2829" }
-    { hostId = "H3", hostname = "127.0.0.1", port = "2830" }
-    { hostId = "H4", hostname = "127.0.0.1", port = "2831" }
-    { hostId = "H5", hostname = "127.0.0.1", port = "2832" }
-    { hostId = "H6", hostname = "127.0.0.1", port = "2833" }
+    { hostId = H1, hostname = "127.0.0.1", port = "2828", storeId = DB1 }
+    { hostId = H2, hostname = "127.0.0.1", port = "2829", storeId = DB2 }
+    { hostId = H3, hostname = "127.0.0.1", port = "2830", storeId = DB3 }
+    { hostId = H4, hostname = "127.0.0.1", port = "2831", storeId = DB1 }
+    { hostId = H5, hostname = "127.0.0.1", port = "2832", storeId = DB2 }
+    { hostId = H6, hostname = "127.0.0.1", port = "2833", storeId = DB3 }
   ]
   shards = [
     { shardId = "S1", hosts =
       [
-        {hostId: H1, memberId: 81}
-        {hostId: H2, memberId: 82}
-        {hostId: H3, memberId: 83}
+        { hostId: H1, memberId: 81 }
+        { hostId: H2, memberId: 82 }
+        { hostId: H3, memberId: 83 }
       ]
     }
     { shardId = "S2", hosts =
       [
-        {hostId: H4, memberId: 84}
-        {hostId: H5, memberId: 85}
-        {hostId: H6, memberId: 86}
+        { hostId: H4, memberId: 84 }
+        { hostId: H5, memberId: 85 }
+        { hostId: H6, memberId: 86 }
       ]
     }
   ]
+  stores = [
+    { storeId = DB1, h2.url = "jdbc:h2:/tmp/gondola-db1" }
+    { storeId = DB2, h2.url = "jdbc:h2:/tmp/gondola-db2" }
+  ]
+  sites = [
+    { siteId = DC1, hosts =
+      [ H1, H4 ]
+    }
+    { siteId = DC2, hosts =
+      [ H2, H5 ]
+    }
+    { siteId = DC3, hosts =
+      [ H3, H6 ]
+    }
+  ]
 }
 
 ##################### raft
@@ -58,7 +73,7 @@ gondola {
   tracing {
     summary_period = 30000
     raft_messages = false
-    raft_heartbeats = true
+    raft_heartbeats = false
     command = false
     storage = false
     network = false
@@ -70,6 +85,14 @@ gondola {
 
 storage {
   impl = storage.h2
+
+  h2 {
+    class = com.yahoo.gondola.impl.H2dbStorage
+    url = "jdbc:h2:/tmp/gondola-db-$hostId"
+    user = sa
+    password = ""
+    port = 9092
+  }
 }
 
 ##################### gondola command

core/conf/gondola-tsunami.conf

@@ -44,7 +44,7 @@ gondola {
   tracing {
     summary_period = 30000
     raft_messages = true
-    raft_heartbeats = true
+    raft_heartbeats = false
     command = true
     storage = true
     network = false

core/pom.xml

@@ -76,7 +76,7 @@
         <dependency>
             <groupId>com.zaxxer</groupId>
             <artifactId>HikariCP</artifactId>
-            <version>2.3.8</version>
+            <version>2.4.2</version>
         </dependency>
         <!-- Dependency used in command line tools -->
         <dependency>

core/src/main/java/com/yahoo/gondola/impl/MySqlStorage.java

@@ -41,9 +41,9 @@ public class MySqlStorage implements Storage {
     public MySqlStorage(Gondola gondola, String hostId) throws GondolaException {
         // Get configs
         maxCommandSize = gondola.getConfig().getInt("raft.command_max_size");
-        String user = gondola.getConfig().get("storage_mysql.user");
-        String password = gondola.getConfig().get("storage_mysql.password");
-        String url = gondola.getConfig().get("storage_mysql.url");
+        String user = gondola.getConfig().get("storage.mysql.user");
+        String password = gondola.getConfig().get("storage.mysql.password");
+        String url = gondola.getConfig().get("storage.mysql.url");
         url = url.replace("$hostId", hostId);
 
         logger.info("Initializing MySql storage. maxCommandSize={} url={}", maxCommandSize, url);

core/src/main/java/com/yahoo/gondola/impl/NastyStorage.java

@@ -27,14 +27,14 @@ public class NastyStorage implements Storage {
     boolean enabled;
 
     public NastyStorage(Gondola gondola, String hostId) throws GondolaException {
-        String storageClassName = gondola.getConfig().get(gondola.getConfig().get("storage_nasty.impl") + ".class");
+        String storageClassName = gondola.getConfig().get(gondola.getConfig().get("storage.nasty.impl") + ".class");
         try {
             storage = (Storage) Class.forName(storageClassName).getConstructor(Gondola.class, String.class)
                     .newInstance(gondola, hostId);
         } catch (Exception e) {
             throw new GondolaException(e);
         }
-        tracing = gondola.getConfig().getBoolean("storage_nasty.tracing");
+        tracing = gondola.getConfig().getBoolean("storage.nasty.tracing");
     }
 
     @Override

core/src/main/resources/default.conf

@@ -57,7 +57,7 @@ gondola {
   # (e.g. connect retry failed), only the message is displayed and the stack trace is suppressed.
   # Also when suppression is enabled, some known and noisy exceptions are only printed once a minute,
   # along with the number of times the message was suppressed within the minute.
-  stack_trace_suppression = false
+  stack_trace_suppression = true
 
   tracing {
     # Gondola prints out details of it's internal queues and pools at the specified period (ms)

core/src/test/java/com/yahoo/gondola/cli/GondolaAgent.java

@@ -41,7 +41,7 @@ public class GondolaAgent {
 
     static Logger logger = LoggerFactory.getLogger(GondolaAgent.class);
 
-    static String configFile;
+    static String configFile = "conf/gondola-command.conf";
     static int port;
     static Config config;
     AtomicInteger jdwpPort = new AtomicInteger(5005);
@@ -73,8 +73,6 @@ public static void main(String[] args) throws ParseException, IOException {
         }
         if (commandLine.hasOption("config")) {
             configFile = commandLine.getOptionValue("config");
-        } else {
-            configFile = "conf/gondola-sample.conf";
         }
 
         config = new Config(new File(configFile));
@@ -124,7 +122,7 @@ public void run() {
                     String[] split = line.split(" ");
                     logger.info("Received command: {}", line);
                     switch (split[0]) {
-                        // create <param1> <param2> ... -> gondola.sh <param1> <param2> ....
+                        // create <param1> <param2> ... -> gondola <param1> <param2> ....
                         case "create":
                             if (split.length >= 2) {
                                 execGondola(split[1], line.substring(line.indexOf(split[1]) + split[1].length()));
@@ -221,7 +219,7 @@ private boolean killGondola(String hostId) {
         }
 
         private void execGondola(String hostId, String args) throws IOException {
-            args = "./bin/gondola.sh " + args;
+            args = "./bin/gondola " + args;
             logger.info("Launching gondola: {}", args);
 
             ProcessBuilder pb = new ProcessBuilder(args.split("\\s"));
@@ -232,7 +230,7 @@ private void execGondola(String hostId, String args) throws IOException {
                                   getJdwpPort(hostId)));
 
             // Redirect output to a log file
-            File outFile = new File(String.format("logs/gondola-%s.log", hostId));
+            File outFile = new File(String.format("logs/tsunami-%s.log", hostId));
             pb.redirectOutput(outFile);
             pb.redirectError(outFile);
 

core/src/test/java/com/yahoo/gondola/tsunami/AgentClient.java

@@ -36,7 +36,7 @@ public AgentClient(String hostId, String hostname, int agentPort, CliClient gond
 
     public void createInstance() {
         try {
-            channel.send(String.format("create %s -hostid %s -clusterid cluster1 -port %d -config %s start",
+            channel.send(String.format("create %s -host %s -shard shard1 -port %d -config %s start",
                                        hostId, hostId, gondolaCc.getPort(), "conf/gondola-tsunami.conf"));
             up = true;
         } catch (Exception e) {