METRON-1004: Travis CI - Job Exceeded Maximum Time Limit

.travis.yml

@@ -17,7 +17,9 @@ before_install:
   - export PATH=$M2_HOME/bin:$PATH
 script:
   - |
-    time mvn -q -T 2C -DskipTests install && time mvn -q -T 2C jacoco:prepare-agent surefire:test@unit-tests && mvn -q jacoco:prepare-agent surefire:test@integration-tests  && time mvn -q jacoco:prepare-agent test --projects metron-interface/metron-config && time build_utils/verify_licenses.sh
+    time mvn -q -T 2C -DskipTests install && time mvn -q -T 2C surefire:test@unit-tests && time mvn -q surefire:test@integration-tests  && time mvn -q test --projects metron-interface/metron-config && time build_utils/verify_licenses.sh
+before_cache:
+  - rm -rf $HOME/.m2/org/apache/metron
 cache:
   directories:
   - $HOME/.m2

metron-analytics/metron-maas-service/src/test/java/org/apache/metron/maas/service/MaasIntegrationTest.java

@@ -33,17 +33,14 @@
 import org.apache.curator.framework.CuratorFramework;
 import org.apache.curator.framework.CuratorFrameworkFactory;
 import org.apache.curator.retry.ExponentialBackoffRetry;
-import org.apache.curator.test.TestingServer;
 import org.apache.hadoop.conf.Configuration;
 import org.apache.hadoop.fs.Path;
 import org.apache.hadoop.net.NetUtils;
-import org.apache.hadoop.util.JarFinder;
 import org.apache.hadoop.util.Shell;
 import org.apache.hadoop.yarn.api.records.ApplicationReport;
 import org.apache.hadoop.yarn.api.records.YarnApplicationState;
 import org.apache.hadoop.yarn.client.api.YarnClient;
 import org.apache.hadoop.yarn.conf.YarnConfiguration;
-import org.apache.hadoop.yarn.server.MiniYARNCluster;
 import org.apache.metron.integration.ComponentRunner;
 import org.apache.metron.integration.components.YarnComponent;
 import org.apache.metron.integration.components.ZKServerComponent;
@@ -57,25 +54,25 @@
 import org.apache.metron.test.utils.UnitTestHelper;
 import org.apache.zookeeper.KeeperException;
 import org.junit.After;
+import org.junit.AfterClass;
 import org.junit.Assert;
-import org.junit.Before;
+import org.junit.BeforeClass;
 import org.junit.Test;
 
 public class MaasIntegrationTest {
   private static final Log LOG =
           LogFactory.getLog(MaasIntegrationTest.class);
-  private CuratorFramework client;
-  private ComponentRunner runner;
-  private YarnComponent yarnComponent;
-  private ZKServerComponent zkServerComponent;
-  @Before
-  public void setup() throws Exception {
+  private static CuratorFramework client;
+  private static ComponentRunner runner;
+  private static YarnComponent yarnComponent;
+  private static ZKServerComponent zkServerComponent;
+
+  @BeforeClass
+  public static void setupBeforeClass() throws Exception {
     UnitTestHelper.setJavaLoggingLevel(Level.SEVERE);
     LOG.info("Starting up YARN cluster");
 
-    Map<String, String> properties = new HashMap<>();
     zkServerComponent = new ZKServerComponent();
-
     yarnComponent = new YarnComponent().withApplicationMasterClass(ApplicationMaster.class).withTestName(MaasIntegrationTest.class.getSimpleName());
 
     runner = new ComponentRunner.Builder()
@@ -92,14 +89,19 @@ public void setup() throws Exception {
     client.start();
   }
 
-  @After
-  public void tearDown(){
+  @AfterClass
+  public static void tearDownAfterClass(){
     if(client != null){
       client.close();
     }
     runner.stop();
   }
 
+  @After
+  public void tearDown() {
+    runner.reset();
+  }
+
   @Test(timeout=900000)
   public void testMaaSWithDomain() throws Exception {
     testDSShell(true);

metron-analytics/metron-profiler/src/test/java/org/apache/metron/profiler/integration/ConfigUploadComponent.java

@@ -20,6 +20,7 @@
 package org.apache.metron.profiler.integration;
 
 import org.apache.curator.framework.CuratorFramework;
+import org.apache.curator.framework.imps.CuratorFrameworkState;
 import org.apache.metron.integration.InMemoryComponent;
 import org.apache.metron.integration.UnableToStartException;
 import org.apache.metron.integration.components.ZKServerComponent;
@@ -56,14 +57,25 @@ public void stop() {
     // nothing to do
   }
 
+  public void update()
+      throws UnableToStartException {
+    try {
+      upload();
+    } catch (Exception e) {
+      throw new UnableToStartException(e.getMessage(), e);
+    }
+  }
+
   /**
    * Uploads configuration to Zookeeper.
    * @throws Exception
    */
   private void upload() throws Exception {
     final String zookeeperUrl = topologyProperties.getProperty(ZKServerComponent.ZOOKEEPER_PROPERTY);
     try(CuratorFramework client = getClient(zookeeperUrl)) {
-      client.start();
+      if(client.getState() != CuratorFrameworkState.STARTED) {
+        client.start();
+      }
       uploadGlobalConfig(client);
       uploadProfilerConfig(client);
     }
@@ -87,7 +99,7 @@ private void uploadProfilerConfig(CuratorFramework client) throws Exception {
    * @param client The zookeeper client.
    */
   private void uploadGlobalConfig(CuratorFramework client) throws Exception {
-    if (globalConfiguration == null) {
+    if (globalConfiguration != null) {
       byte[] globalConfig = readGlobalConfigFromFile(globalConfiguration);
       if (globalConfig.length > 0) {
         writeGlobalConfigToZookeeper(readGlobalConfigFromFile(globalConfiguration), client);

metron-analytics/metron-profiler/src/test/java/org/apache/metron/profiler/integration/ProfilerIntegrationTest.java

@@ -33,6 +33,7 @@
 import org.apache.metron.hbase.TableProvider;
 import org.apache.metron.integration.BaseIntegrationTest;
 import org.apache.metron.integration.ComponentRunner;
+import org.apache.metron.integration.UnableToStartException;
 import org.apache.metron.integration.components.FluxTopologyComponent;
 import org.apache.metron.integration.components.KafkaComponent;
 import org.apache.metron.integration.components.ZKServerComponent;
@@ -41,7 +42,10 @@
 import org.apache.metron.statistics.OnlineStatisticsProvider;
 import org.apache.metron.test.mock.MockHTable;
 import org.junit.After;
+import org.junit.AfterClass;
 import org.junit.Assert;
+import org.junit.Before;
+import org.junit.BeforeClass;
 import org.junit.Test;
 
 import java.io.File;
@@ -76,7 +80,7 @@ public class ProfilerIntegrationTest extends BaseIntegrationTest {
    * }
    */
   @Multiline
-  private String message1;
+  private static String message1;
 
   /**
    * {
@@ -87,7 +91,7 @@ public class ProfilerIntegrationTest extends BaseIntegrationTest {
    * }
    */
   @Multiline
-  private String message2;
+  private static String message2;
 
   /**
    * {
@@ -98,15 +102,16 @@ public class ProfilerIntegrationTest extends BaseIntegrationTest {
    * }
    */
   @Multiline
-  private String message3;
+  private static String message3;
 
-  private ColumnBuilder columnBuilder;
-  private ZKServerComponent zkComponent;
-  private FluxTopologyComponent fluxComponent;
-  private KafkaComponent kafkaComponent;
-  private List<byte[]> input;
-  private ComponentRunner runner;
-  private MockHTable profilerTable;
+  private static ColumnBuilder columnBuilder;
+  private static ZKServerComponent zkComponent;
+  private static FluxTopologyComponent fluxComponent;
+  private static KafkaComponent kafkaComponent;
+  private static ConfigUploadComponent configUploadComponent;
+  private static List<byte[]> input;
+  private static ComponentRunner runner;
+  private static MockHTable profilerTable;
 
   private static final String tableName = "profiler";
   private static final String columnFamily = "P";
@@ -133,7 +138,7 @@ public HTableInterface getTable(Configuration config, String tableName) throws I
   @Test
   public void testExample1() throws Exception {
 
-    setup(TEST_RESOURCES + "/config/zookeeper/readme-example-1");
+    update(TEST_RESOURCES + "/config/zookeeper/readme-example-1");
 
     // start the topology and write test messages to kafka
     fluxComponent.submitTopology();
@@ -158,7 +163,7 @@ public void testExample1() throws Exception {
   @Test
   public void testExample2() throws Exception {
 
-    setup(TEST_RESOURCES + "/config/zookeeper/readme-example-2");
+    update(TEST_RESOURCES + "/config/zookeeper/readme-example-2");
 
     // start the topology and write test messages to kafka
     fluxComponent.submitTopology();
@@ -191,7 +196,7 @@ public void testExample2() throws Exception {
   @Test
   public void testExample3() throws Exception {
 
-    setup(TEST_RESOURCES + "/config/zookeeper/readme-example-3");
+    update(TEST_RESOURCES + "/config/zookeeper/readme-example-3");
 
     // start the topology and write test messages to kafka
     fluxComponent.submitTopology();
@@ -216,7 +221,7 @@ public void testExample3() throws Exception {
   @Test
   public void testExample4() throws Exception {
 
-    setup(TEST_RESOURCES + "/config/zookeeper/readme-example-4");
+    update(TEST_RESOURCES + "/config/zookeeper/readme-example-4");
 
     // start the topology and write test messages to kafka
     fluxComponent.submitTopology();
@@ -239,7 +244,8 @@ public void testExample4() throws Exception {
   @Test
   public void testPercentiles() throws Exception {
 
-    setup(TEST_RESOURCES + "/config/zookeeper/percentiles");
+    update(TEST_RESOURCES + "/config/zookeeper/percentiles");
+
 
     // start the topology and write test messages to kafka
     fluxComponent.submitTopology();
@@ -277,9 +283,15 @@ private <T> List<T> read(List<Put> puts, String family, byte[] qualifier, Class<
     return results;
   }
 
-  public void setup(String pathToConfig) throws Exception {
+  @BeforeClass
+  public static void setupBeforeClass() throws UnableToStartException {
     columnBuilder = new ValueOnlyColumnBuilder(columnFamily);
 
+    List<String> inputNew = Stream.of(message1, message2, message3)
+        .map(m -> Collections.nCopies(5, m))
+        .flatMap(l -> l.stream())
+        .collect(Collectors.toList());
+
     // create input messages for the profiler to consume
     input = Stream.of(message1, message2, message3)
             .map(Bytes::toBytes)
@@ -320,10 +332,8 @@ public void setup(String pathToConfig) throws Exception {
             new KafkaComponent.Topic(outputTopic, 1)));
 
     // upload profiler configuration to zookeeper
-    ConfigUploadComponent configUploadComponent = new ConfigUploadComponent()
-            .withTopologyProperties(topologyProperties)
-            .withGlobalConfiguration(pathToConfig)
-            .withProfilerConfiguration(pathToConfig);
+    configUploadComponent = new ConfigUploadComponent()
+            .withTopologyProperties(topologyProperties);
 
     // load flux definition for the profiler topology
     fluxComponent = new FluxTopologyComponent.Builder()
@@ -345,11 +355,32 @@ public void setup(String pathToConfig) throws Exception {
     runner.start();
   }
 
+  public void update(String path) throws Exception {
+    configUploadComponent.withGlobalConfiguration(path)
+        .withProfilerConfiguration(path);
+    configUploadComponent.update();
+  }
+
+  @AfterClass
+  public static void tearDownAfterClass() throws Exception {
+    MockHTable.Provider.clear();
+    if (runner != null) {
+      runner.stop();
+    }
+  }
+
+  @Before
+  public void setup() {
+    // create the mock table
+    profilerTable = (MockHTable) MockHTable.Provider.addToCache(tableName, columnFamily);
+  }
+
   @After
   public void tearDown() throws Exception {
     MockHTable.Provider.clear();
+    profilerTable.clear();
     if (runner != null) {
-      runner.stop();
+      runner.reset();
     }
   }
 }