HWKMETRICS-404 Cassandra Driver timeouts during the build

api/metrics-api-jaxrs/src/main/java/org/hawkular/metrics/api/jaxrs/MetricsServiceLifecycle.java

@@ -22,6 +22,7 @@
 import static java.util.concurrent.TimeUnit.SECONDS;
 
 import static org.hawkular.metrics.api.jaxrs.config.ConfigurationKey.CASSANDRA_CQL_PORT;
+import static org.hawkular.metrics.api.jaxrs.config.ConfigurationKey.CASSANDRA_DRIVER_READ_TIMEOUT_MS;
 import static org.hawkular.metrics.api.jaxrs.config.ConfigurationKey.CASSANDRA_KEYSPACE;
 import static org.hawkular.metrics.api.jaxrs.config.ConfigurationKey.CASSANDRA_NODES;
 import static org.hawkular.metrics.api.jaxrs.config.ConfigurationKey.CASSANDRA_RESETDB;
@@ -71,6 +72,7 @@
 import com.datastax.driver.core.JdkSSLOptions;
 import com.datastax.driver.core.SSLOptions;
 import com.datastax.driver.core.Session;
+import com.datastax.driver.core.SocketOptions;
 import com.google.common.base.Throwables;
 import com.google.common.util.concurrent.Futures;
 import com.google.common.util.concurrent.Uninterruptibles;
@@ -122,6 +124,11 @@ public enum State {
     @ConfigurationProperty(CASSANDRA_RESETDB)
     private String resetDb;
 
+    @Inject
+    @Configurable
+    @ConfigurationProperty(CASSANDRA_DRIVER_READ_TIMEOUT_MS)
+    private String driverReadTimeout;
+
     @Inject
     @Configurable
     @ConfigurationProperty(WAIT_FOR_SERVICE)
@@ -295,6 +302,16 @@ private Session createSession() {
             clusterBuilder.withoutJMXReporting();
         }
 
+        int drvReadTimeout;
+        try {
+            drvReadTimeout = Integer.parseInt(driverReadTimeout);
+        } catch (NumberFormatException nfe) {
+            int defaultReadTimeout = Integer.parseInt(CASSANDRA_DRIVER_READ_TIMEOUT_MS.defaultValue());
+            log.warnInvalidDriverReadTimeout(driverReadTimeout, defaultReadTimeout);
+            drvReadTimeout = Integer.parseInt(CASSANDRA_DRIVER_READ_TIMEOUT_MS.defaultValue());
+        }
+        clusterBuilder.withSocketOptions(new SocketOptions().setReadTimeoutMillis(drvReadTimeout));
+
         Cluster cluster = clusterBuilder.build();
         cluster.init();
         Session createdSession = null;

api/metrics-api-jaxrs/src/main/java/org/hawkular/metrics/api/jaxrs/config/ConfigurationKey.java

@@ -34,6 +34,7 @@ public enum ConfigurationKey {
     CASSANDRA_KEYSPACE("cassandra.keyspace", "hawkular_metrics", null, false),
     CASSANDRA_RESETDB("cassandra.resetdb", null, null, true),
     CASSANDRA_USESSL("hawkular-metrics.cassandra-use-ssl", "false", "CASSANDRA_USESSL", false),
+    CASSANDRA_DRIVER_READ_TIMEOUT_MS("cassandra.driver.read-timeout-ms", "12000", "CASSANDRA_DRIVER_READ_TIMEOUT_MS", false),
     WAIT_FOR_SERVICE("hawkular.metrics.waitForService", null, null, true),
     USE_VIRTUAL_CLOCK("hawkular.metrics.use-virtual-clock", "false", "USE_VIRTUAL_CLOCK", false),
     DEFAULT_TTL("hawkular.metrics.default-ttl", "7", "DEFAULT_TTL", false),

api/metrics-api-jaxrs/src/main/java/org/hawkular/metrics/api/jaxrs/log/RestLogger.java

@@ -81,4 +81,8 @@ public interface RestLogger extends BasicLogger {
     @LogMessage(level = WARN)
     @Message(id = 200011, value = "Invalid value [%s] for default TTL. Will use a default of %s days")
     void warnInvalidDefaultTTL(String ttl, String defaultTTL);
+
+    @LogMessage(level = WARN)
+    @Message(id = 200012, value = "Invalid Cassandra Driver read timeout %s, not a number. Will use the default of %d ms.")
+    void warnInvalidDriverReadTimeout(String found, int defaultValue);
 }

core/configuration-service/pom.xml

@@ -82,4 +82,20 @@
       <scope>test</scope>
     </dependency>
   </dependencies>
+
+  <build>
+    <plugins>
+      <plugin>
+        <groupId>org.apache.maven.plugins</groupId>
+        <artifactId>maven-surefire-plugin</artifactId>
+        <configuration>
+          <systemProperties>
+            <nodes>${nodes}</nodes>
+            <cassandra.driver.read-timeout-ms>${cassandra.driver.read-timeout-ms}</cassandra.driver.read-timeout-ms>
+          </systemProperties>
+        </configuration>
+      </plugin>
+    </plugins>
+  </build>
+
 </project>

core/configuration-service/src/test/java/org/hawkular/metrics/sysconfig/ConfigurationServiceTest.java

@@ -35,6 +35,7 @@
 import com.datastax.driver.core.ResultSet;
 import com.datastax.driver.core.Row;
 import com.datastax.driver.core.Session;
+import com.datastax.driver.core.SocketOptions;
 import com.google.common.collect.ImmutableMap;
 
 import rx.Observable;
@@ -51,7 +52,14 @@ public class ConfigurationServiceTest {
 
     @BeforeClass
     public void initClass() {
-        Cluster cluster = new Cluster.Builder().addContactPoint("127.0.0.1").build();
+        String nodeAddresses = System.getProperty("nodes", "127.0.0.1");
+        int cassandraDriverReadTimeout =
+                Integer.parseInt(System.getProperty("cassandra.driver.read-timeout-ms", "12000"));
+
+        Cluster cluster = new Cluster.Builder()
+                .addContactPoints(nodeAddresses.split(","))
+                .withSocketOptions(new SocketOptions().setReadTimeoutMillis(cassandraDriverReadTimeout))
+                .build();
         session = cluster.connect();
 
         SchemaService schemaService = new SchemaService();

core/metrics-core-service/pom.xml

@@ -190,6 +190,7 @@
               <systemPropertyVariables>
                 <keyspace>${test.keyspace}</keyspace>
                 <nodes>${nodes}</nodes>
+                <cassandra.driver.read-timeout-ms>${cassandra.driver.read-timeout-ms}</cassandra.driver.read-timeout-ms>
               </systemPropertyVariables>
             </configuration>
             <executions>

core/metrics-core-service/src/test/java/org/hawkular/metrics/core/service/BaseITest.java

@@ -38,6 +38,7 @@
 import com.datastax.driver.core.PreparedStatement;
 import com.datastax.driver.core.ResultSet;
 import com.datastax.driver.core.Session;
+import com.datastax.driver.core.SocketOptions;
 import com.google.common.collect.ImmutableList;
 import com.google.common.util.concurrent.ListenableFuture;
 import com.google.common.util.concurrent.Uninterruptibles;
@@ -63,9 +64,12 @@ public abstract class BaseITest {
     @BeforeSuite
     public static void initSuite() {
         String nodeAddresses = System.getProperty("nodes", "127.0.0.1");
+        int cassandraDriverReadTimeout =
+                Integer.parseInt(System.getProperty("cassandra.driver.read-timeout-ms", "12000"));
         Cluster cluster = new Cluster.Builder()
                 .addContactPoints(nodeAddresses.split(","))
 //                .withProtocolVersion(ProtocolVersion.V4)
+                .withSocketOptions(new SocketOptions().setReadTimeoutMillis(cassandraDriverReadTimeout))
                 .build();
         session = cluster.connect();
         rxSession = new RxSessionImpl(session);

integration-tests/rest-tests-jaxrs/pom.xml

@@ -176,6 +176,7 @@
                     <javaOpt>-Dhawkular.terminal-event.timeout=${terminal-event.timeout}</javaOpt>
                     <javaOpt>-Dhawkular.metrics.allowed-cors-origins=http://test.hawkular.org,https://secure.hawkular.io</javaOpt>
                     <javaOpt>-Dhawkular.metrics.allowed-cors-access-control-allow-headers=random-header1,random-header2</javaOpt>
+                    <javaOpt>-Dcassandra.driver.read-timeout-ms=${cassandra.driver.read-timeout-ms}</javaOpt>
                     <javaOpt>-Xdebug</javaOpt>
                     <javaOpt>-Xrunjdwp:transport=dt_socket,server=y,suspend=n,address=8787</javaOpt>
                   </javaOpts>

pom.xml

@@ -95,6 +95,7 @@
 
   <properties>
     <test.keyspace>hawkulartest</test.keyspace>
+    <cassandra.driver.read-timeout-ms>12000</cassandra.driver.read-timeout-ms> <!-- used to run the tests -->
     <nodes>127.0.0.1</nodes>
     <!-- Dependencies versions -->
     <version.org.cassalog>0.1.2</version.org.cassalog>