Migrate Elasticsearch tests to NoSQLUnit Elasticsearch HTTP adapter

graylog-project-parent/pom.xml

@@ -545,7 +545,7 @@
             <dependency>
                 <groupId>com.github.joschi.nosqlunit</groupId>
                 <artifactId>nosqlunit-elasticsearch-http</artifactId>
-                <version>1.0.0-3+jackson</version>
+                <version>1.0.0-4+jackson</version>
                 <scope>test</scope>
             </dependency>
             <dependency>

graylog2-server/pom.xml

@@ -63,8 +63,8 @@
         <!-- to work around filtering bug, which makes maven.build.timestamp inaccessible -->
         <build.timestamp>${maven.build.timestamp}</build.timestamp>
 
-        <it.es.httpPort>9200</it.es.httpPort>
-        <it.es.transportPort>9300</it.es.transportPort>
+        <it.es.httpPort>19200</it.es.httpPort>
+        <it.es.transportPort>19300</it.es.transportPort>
         <it.es.version>5.5.2</it.es.version>
         <it.es.instanceCount>1</it.es.instanceCount>
         <it.es.clusterName>test</it.es.clusterName>

graylog2-server/src/test/java/org/graylog2/ElasticsearchBase.java

@@ -38,6 +38,9 @@
 import io.searchbox.indices.template.DeleteTemplate;
 import io.searchbox.indices.template.GetTemplate;
 import io.searchbox.indices.template.PutTemplate;
+import org.graylog2.indexer.IndexMapping;
+import org.graylog2.indexer.IndexMapping2;
+import org.graylog2.indexer.IndexMapping5;
 import org.junit.Rule;
 
 import javax.inject.Inject;
@@ -57,6 +60,8 @@
 
 public abstract class ElasticsearchBase {
     private static final String PROPERTIES_RESOURCE_NAME = "elasticsearch.properties";
+    private static final String DEFAULT_PORT = "9200";
+
     public static final Duration ES_TIMEOUT = Duration.seconds(5L);
 
     @Rule
@@ -86,7 +91,7 @@ private Properties loadProperties(String resourceName) {
     }
 
     protected String getHttpPort() {
-        return properties.getProperty("httpPort", "9200");
+        return properties.getProperty("httpPort", DEFAULT_PORT);
     }
 
     protected String getServer() {
@@ -113,6 +118,17 @@ protected void assertSucceeded(JestResult jestResult) {
                 .isTrue();
     }
 
+    protected IndexMapping indexMapping() {
+        switch (elasticsearchVersion.getMajorVersion()) {
+            case 2:
+                return new IndexMapping2();
+            case 5:
+                return new IndexMapping5();
+            default:
+                throw new IllegalStateException("Only Elasticsearch 2.x and 5.x are supported");
+        }
+    }
+
     public void createIndex(String index) throws IOException {
         createIndex(index, 1, 0);
     }

graylog2-server/src/test/java/org/graylog2/indexer/indices/IndicesIT.java

@@ -22,6 +22,7 @@
 import com.fasterxml.jackson.databind.ObjectMapper;
 import com.fasterxml.jackson.databind.node.BooleanNode;
 import com.fasterxml.jackson.databind.node.TextNode;
+import com.github.joschi.nosqlunit.elasticsearch.http.ElasticsearchConfiguration;
 import com.google.common.collect.ImmutableMap;
 import com.google.common.eventbus.EventBus;
 import com.google.common.eventbus.Subscribe;
@@ -42,7 +43,6 @@
 import org.graylog2.indexer.indices.events.IndicesDeletedEvent;
 import org.graylog2.indexer.indices.events.IndicesReopenedEvent;
 import org.graylog2.indexer.messages.Messages;
-import org.graylog2.indexer.nosqlunit.IndexCreatingLoadStrategyFactory;
 import org.graylog2.indexer.retention.strategies.DeletionRetentionStrategy;
 import org.graylog2.indexer.retention.strategies.DeletionRetentionStrategyConfig;
 import org.graylog2.indexer.rotation.strategies.MessageCountRotationStrategy;
@@ -75,7 +75,7 @@ public class IndicesIT extends ElasticsearchBase {
     @Rule
     public final MockitoRule mockitoRule = MockitoJUnit.rule();
 
-    private final IndexSetConfig indexSetConfig = IndexSetConfig.builder()
+    private static final IndexSetConfig indexSetConfig = IndexSetConfig.builder()
             .id("index-set-1")
             .title("Index set 1")
             .description("For testing")
@@ -92,14 +92,19 @@ public class IndicesIT extends ElasticsearchBase {
             .indexOptimizationMaxNumSegments(1)
             .indexOptimizationDisabled(false)
             .build();
-    private final IndexSet indexSet = new TestIndexSet(indexSetConfig);
+    private static final IndexSet indexSet = new TestIndexSet(indexSetConfig);
 
     private EventBus eventBus;
     private Indices indices;
     private IndexMappingFactory indexMappingFactory;
 
-    public IndicesIT() {
-        elasticsearchRule.setLoadStrategyFactory(new IndexCreatingLoadStrategyFactory(indexSet, Collections.singleton(INDEX_NAME)));
+    @Override
+    protected ElasticsearchConfiguration.Builder elasticsearchConfiguration() {
+        final Map<String, Map<String, Object>> messageTemplates = Collections.singletonMap("graylog-test-internal", indexMapping().messageTemplate("*", "standard"));
+        return super.elasticsearchConfiguration()
+                .indexTemplates(messageTemplates)
+                .createIndices(false)
+                .deleteAllIndices(true);
     }
 
     @Before
@@ -108,7 +113,7 @@ public void setUp() throws Exception {
         final Node node = new Node(client());
         indexMappingFactory = new IndexMappingFactory(node);
         indices = new Indices(client(),
-                new ObjectMapper(),
+                new ObjectMapperProvider().get(),
                 indexMappingFactory,
                 new Messages(new MetricRegistry(), client()),
                 mock(NodeId.class),
@@ -117,7 +122,6 @@ public void setUp() throws Exception {
     }
 
     @Test
-    @UsingDataSet(loadStrategy = LoadStrategyEnum.CLEAN_INSERT)
     public void testDelete() throws Exception {
         final String index = createRandomIndex("indices_it_");
         indices.delete(index);
@@ -148,18 +152,19 @@ public void testClose() throws Exception {
     }
 
     @Test
-    @UsingDataSet(loadStrategy = LoadStrategyEnum.CLEAN_INSERT)
     public void testAliasExists() throws Exception {
+        final String index = createRandomIndex("indices_it_");
         final String alias = "graylog_alias_exists";
         assertThat(indices.aliasExists(alias)).isFalse();
 
         try {
-            addAliasMapping(INDEX_NAME, alias);
+            addAliasMapping(index, alias);
 
             assertThat(indices.aliasExists(alias)).isTrue();
             assertThat(indices.exists(alias)).isFalse();
         } finally {
-            removeAliasMapping(INDEX_NAME, alias);
+            removeAliasMapping(index, alias);
+            deleteIndex(index);
         }
     }
 
@@ -185,24 +190,24 @@ public void testIndexIfIndexExists() throws Exception {
     }
 
     @Test
-    @UsingDataSet(loadStrategy = LoadStrategyEnum.CLEAN_INSERT)
     public void testExistsIfIndexDoesNotExist() throws Exception {
         final String indexNotAlias = "graylog_index_does_not_exist";
         assertThat(indices.exists(indexNotAlias)).isFalse();
     }
 
     @Test
-    @UsingDataSet(loadStrategy = LoadStrategyEnum.CLEAN_INSERT)
     public void testAliasTarget() throws Exception {
+        final String index = createRandomIndex("indices_it_");
         final String alias = "graylog_alias_target";
         assertThat(indices.aliasTarget(alias)).isEmpty();
 
         try {
-            addAliasMapping(INDEX_NAME, alias);
+            addAliasMapping(index, alias);
 
-            assertThat(indices.aliasTarget(alias)).contains(INDEX_NAME);
+            assertThat(indices.aliasTarget(alias)).contains(index);
         } finally {
-            removeAliasMapping(INDEX_NAME, alias);
+            removeAliasMapping(index, alias);
+            deleteIndex(index);
         }
     }
 
@@ -311,22 +316,19 @@ public void testCreateOverwritesIndexTemplate() throws Exception {
     }
 
     @Test
-    @UsingDataSet(loadStrategy = LoadStrategyEnum.DELETE_ALL)
-    public void indexCreationDateReturnsIndexCreationDateOfExistingIndexAsDateTime() {
-        final String indexName = "index_creation_date_test";
+    public void indexCreationDateReturnsIndexCreationDateOfExistingIndexAsDateTime() throws IOException {
         final DateTime now = DateTime.now(DateTimeZone.UTC);
+        final String indexName = createRandomIndex("indices_it_");
         try {
-            indices.create(indexName, indexSet);
             indices.indexCreationDate(indexName).ifPresent(
                     indexCreationDate -> org.assertj.jodatime.api.Assertions.assertThat(indexCreationDate).isAfterOrEqualTo(now)
             );
         } finally {
-            indices.delete(indexName);
+            deleteIndex(indexName);
         }
     }
 
     @Test
-    @UsingDataSet(loadStrategy = LoadStrategyEnum.DELETE_ALL)
     public void indexCreationDateReturnsEmptyOptionalForNonExistingIndex() {
         assertThat(indices.indexCreationDate("index_missing")).isEmpty();
     }