METRON-2225 Upgrade to Solr 7.4.0 (merrimanr via nickwallen) closes #…

…1492
apache · Aug 28, 2019 · ad71c04 · ad71c04
1 parent 15e5276
commit ad71c04
Show file tree

Hide file tree

Showing 19 changed files with 165 additions and 224 deletions.
diff --git a/dependencies_with_url.csv b/dependencies_with_url.csv
@@ -136,7 +136,7 @@ org.slf4j:slf4j-simple:jar:1.7.7:compile,MIT,http://www.slf4j.org
 org.slf4j:jcl-over-slf4j:jar:1.7.7:compile,MIT,http://www.slf4j.org
 org.slf4j:jcl-over-slf4j:jar:1.7.16:compile,MIT,http://www.slf4j.org
 org.slf4j:jcl-over-slf4j:jar:1.7.21:compile,MIT,http://www.slf4j.org
-org.slf4j:jcl-over-slf4j:jar:1.7.21:compile,MIT,http://www.slf4j.org
+org.slf4j:jcl-over-slf4j:jar:1.7.24:compile,MIT,http://www.slf4j.org
 org.slf4j:jul-to-slf4j:jar:1.7.16:compile,MIT,http://www.slf4j.org
 org.slf4j:jul-to-slf4j:jar:1.7.21:compile,MIT,http://www.slf4j.org
 org.slf4j:jul-to-slf4j:jar:1.7.25:compile,MIT,http://www.slf4j.org
@@ -328,6 +328,7 @@ org.mortbay.jetty:jsp-2.1:jar:6.1.14:compile,ASLv2,
 org.mortbay.jetty:jsp-api-2.1:jar:6.1.14:compile,ASLv2,
 org.mortbay.jetty:servlet-api-2.5:jar:6.1.14:compile,ASLv2,
 org.noggit:noggit:jar:0.6:compile,Apache License, Version 2.0,http://github.com/yonik/noggit
+org.noggit:noggit:jar:0.8:compile,Apache License, Version 2.0,http://github.com/yonik/noggit
 org.scannotation:scannotation:jar:1.0.3:compile,Apache License V2.0,http://scannotation.sf.net
 org.slf4j:log4j-over-slf4j:jar:1.6.6:compile,Apache Software Licenses,http://www.slf4j.org
 org.springframework.integration:spring-integration-core:jar:3.0.0.RELEASE:compile,The Apache Software License, Version 2.0,http://www.springintegration.org/
@@ -453,6 +454,7 @@ org.springframework.kafka:spring-kafka:jar:1.1.1.RELEASE:compile,ASLv2,https://g
 org.springframework.kafka:spring-kafka:jar:2.0.4.RELEASE:compile,ASLv2,https://github.com/spring-projects/spring-kafka
 ch.hsr:geohash:jar:1.3.0:compile,ASLv2,https://github.com/kungfoo/geohash-java
 org.locationtech.spatial4j:spatial4j:jar:0.6:compile,ASLv2,https://github.com/locationtech/spatial4j
+org.locationtech.spatial4j:spatial4j:jar:0.7:compile,ASLv2,https://github.com/locationtech/spatial4j
 com.github.luben:zstd-jni:jar:1.3.2-2:compile,BSD,https://github.com/luben/zstd-jni
 com.github.spullara.mustache.java:compiler:jar:0.9.3:compile,ASLv2,https://github.com/spullara/mustache.java/blob/master/LICENSE
 io.netty:netty-buffer:jar:4.1.13.Final:compile,ASLv2,http://netty.io/

diff --git a/metron-deployment/ansible/roles/solr/defaults/main.yml b/metron-deployment/ansible/roles/solr/defaults/main.yml
@@ -15,7 +15,7 @@
 #  limitations under the License.
 #
 ---
-solr_version: 6.6.2
+solr_version: 7.4.0
 solr_installation_user: "solr"
 solr_user_home: /var/solr
 solr_home: "{{ solr_user_home }}/solr-{{ solr_version }}"

diff --git a/.../java/org/apache/metron/elasticsearch/integration/ElasticsearchSearchIntegrationTest.java b/.../java/org/apache/metron/elasticsearch/integration/ElasticsearchSearchIntegrationTest.java
@@ -270,6 +270,14 @@ public void throws_exception_on_aggregation_queries_on_non_string_non_numeric_fi
     dao.group(request);
   }
 
+  @Test
+  public void different_type_facet_query() throws Exception {
+    thrown.expect(Exception.class);
+    SearchRequest request = JSONUtils.INSTANCE.load(differentTypeFacetQuery, SearchRequest.class);
+    SearchResponse response = getIndexDao().search(request);
+    Assert.assertEquals(3, response.getTotal());
+  }
+
   @Test
   public void different_type_filter_query() throws Exception {
     SearchRequest request = JSONUtils.INSTANCE.load(differentTypeFilterQuery, SearchRequest.class);

diff --git a/...n-indexing-common/src/test/java/org/apache/metron/indexing/dao/SearchIntegrationTest.java b/...n-indexing-common/src/test/java/org/apache/metron/indexing/dao/SearchIntegrationTest.java
@@ -709,14 +709,6 @@ public void missing_type_facet_query() throws Exception {
     response.getFacetCounts();
   }
 
-  @Test
-  public void different_type_facet_query() throws Exception {
-    thrown.expect(Exception.class);
-    SearchRequest request = JSONUtils.INSTANCE.load(differentTypeFacetQuery, SearchRequest.class);
-    SearchResponse response = getIndexDao().search(request);
-    Assert.assertEquals(3, response.getTotal());
-  }
-
   @Test
   public void exceeding_max_results_throws_exception() throws Exception {
     thrown.expect(InvalidSearchException.class);
@@ -934,8 +926,13 @@ public static void stop() {
 
   @Test
   public abstract void returns_column_data_for_multiple_indices() throws Exception;
+
   @Test
   public abstract void returns_column_metadata_for_specified_indices() throws Exception;
+
+  @Test
+  public abstract void different_type_facet_query() throws Exception;
+
   @Test
   public abstract void different_type_filter_query() throws Exception;
 

diff --git a/metron-platform/metron-solr/metron-solr-common/README.md b/metron-platform/metron-solr/metron-solr-common/README.md
@@ -27,7 +27,7 @@ limitations under the License.
 
 ## Introduction
 
-Metron ships with Solr 6.6.2 support. Solr Cloud can be used as the real-time portion of the datastore resulting from [metron-indexing](../metron-indexing/README.md).
+Metron ships with Solr 7.4.0 support. Solr Cloud can be used as the real-time portion of the datastore resulting from [metron-indexing](../metron-indexing/README.md).
 
 ## Configuration
 
@@ -42,15 +42,15 @@ via the global config.  The following settings are possible as part of the globa
   * _WARNING_: If you set this to `false`, then commits will happen based on the SolrClient's internal mechanism and
     worker failure *may* result data being acknowledged in storm but not written in Solr.
 * `solr.commit.soft`
-  * This is a boolean which defines whether the writer makes a soft commit or a durable commit.  See [here](https://lucene.apache.org/solr/guide/6_6/near-real-time-searching.html#NearRealTimeSearching-AutoCommits)  The default is `false`.
+  * This is a boolean which defines whether the writer makes a soft commit or a durable commit.  See [here](https://lucene.apache.org/solr/guide/7_4/near-real-time-searching.html)  The default is `false`.
   * _WARNING_: If you set this to `true`, then commits will happen based on the SolrClient's internal mechanism and
     worker failure *may* result data being acknowledged in storm but not written in Solr.
 * `solr.commit.waitSearcher`
-  * This is a boolean which defines whether the writer blocks the commit until the data is available to search.  See [here](https://lucene.apache.org/solr/guide/6_6/near-real-time-searching.html#NearRealTimeSearching-AutoCommits)  The default is `true`.
+  * This is a boolean which defines whether the writer blocks the commit until the data is available to search.  See [here](https://lucene.apache.org/solr/guide/7_4/near-real-time-searching.html)  The default is `true`.
   * _WARNING_: If you set this to `false`, then commits will happen based on the SolrClient's internal mechanism and
     worker failure *may* result data being acknowledged in storm but not written in Solr.
 * `solr.commit.waitFlush`
-  * This is a boolean which defines whether the writer blocks the commit until the data is flushed.  See [here](https://lucene.apache.org/solr/guide/6_6/near-real-time-searching.html#NearRealTimeSearching-AutoCommits)  The default is `true`.
+  * This is a boolean which defines whether the writer blocks the commit until the data is flushed.  See [here](https://lucene.apache.org/solr/guide/7_4/near-real-time-searching.html)  The default is `true`.
   * _WARNING_: If you set this to `false`, then commits will happen based on the SolrClient's internal mechanism and
     worker failure *may* result data being acknowledged in storm but not written in Solr.
 * `solr.collection`
@@ -86,15 +86,11 @@ The script performs the following tasks
 * Installs Solr
 * Starts Solr Cloud
 
-_Note: for details on setting up Solr Cloud in production mode, see https://lucene.apache.org/solr/guide/6_6/taking-solr-to-production.html_
+Note: for details on setting up Solr Cloud in production mode, see https://lucene.apache.org/solr/guide/7_4/taking-solr-to-production.html
 
 Navigate to `$METRON_HOME/bin` and spin up Solr Cloud by running `install_solr.sh`.  After running this script, 
 Elasticsearch and Kibana will have been stopped and you should now have an instance of Solr Cloud up and running at http://localhost:8983/solr/#/~cloud.  This manner of starting Solr
-will also spin up an embedded Zookeeper instance at port 9983. More information can be found [here](https://lucene.apache.org/solr/guide/6_6/getting-started-with-solrcloud.html)
-
-Solr can also be installed using [HDP Search 3](https://docs.hortonworks.com/HDPDocuments/HDP2/HDP-2.6.4/bk_solr-search-installation/content/ch_hdp_search_30.html).  HDP Search 3 sets the Zookeeper root to 
-`/solr` so this will need to be added to each url in the comma-separated list in Ambari UI -> Services -> Metron -> Configs -> Index Settings -> Solr Zookeeper Urls.  For example, in full dev
-this would be `node1:2181/solr`.
+will also spin up an embedded Zookeeper instance at port 9983. More information can be found [here](https://lucene.apache.org/solr/guide/7_4/getting-started-with-solrcloud.html)
 
 ## Enabling Solr
 
@@ -122,8 +118,8 @@ Any other collections must be created manually before starting the Indexing comp
 
 As of now, we have mapped out the Schemas in `src/main/config/schema`.
 Ambari will eventually install these, but at the moment it's manual and
-you should refer to the Solr documentation [https://lucene.apache.org/solr/guide/6_6](here) in general
-and [here](https://lucene.apache.org/solr/guide/6_6/documents-fields-and-schema-design.html) if you'd like to know more about schemas in Solr.
+you should refer to the Solr documentation [https://lucene.apache.org/solr/guide/7_4](here) in general
+and [here](https://lucene.apache.org/solr/guide/7_4/documents-fields-and-schema-design.html) if you'd like to know more about schemas in Solr.
 
 In Metron's Solr DAO implementation, document updates involve reading a document, applying the update and replacing the original by reindexing the whole document.  
 Indexing LatLonType and PointType field types stores data in internal fields that should not be returned in search results.  For these fields a dynamic field type matching the suffix needs to be added to store the data points.
@@ -146,7 +142,7 @@ If any copy fields are defined, stored and docValues should be set to false.
 Convenience scripts are provided with Metron to create and delete collections.  Ambari uses these scripts to automatically create collections.  To use them outside of Ambari, a few environment variables must be set first:
 ```
 # Path to the zookeeper node used by Solr
-export ZOOKEEPER=node1:2181/solr
+export ZOOKEEPER=node1:9983
 # Set to true if Kerberos is enabled
 export SECURITY_ENABLED=true 
 ```
@@ -167,4 +163,4 @@ The `create_collection.sh` script depends on schemas installed in `$METRON_HOME/
 * error
 
 Additional schemas should be installed in that location if using the `create_collection.sh` script.  Any collection can be deleted with the `delete_collection.sh` script.
-These scripts use the [Solr Collection API](http://lucene.apache.org/solr/guide/6_6/collections-api.html).
+These scripts use the [Solr Collection API](http://lucene.apache.org/solr/guide/7_4/collections-api.html).
diff --git a/metron-platform/metron-solr/metron-solr-common/pom.xml b/metron-platform/metron-solr/metron-solr-common/pom.xml
@@ -39,6 +39,50 @@
             <artifactId>solr-solrj</artifactId>
             <version>${global_solr_version}</version>
         </dependency>
+        <dependency>
+            <groupId>org.noggit</groupId>
+            <artifactId>noggit</artifactId>
+            <version>0.8</version>
+        </dependency>
+        <dependency>
+            <groupId>org.apache.httpcomponents</groupId>
+            <artifactId>httpclient</artifactId>
+            <version>4.5.3</version>
+        </dependency>
+        <dependency>
+            <groupId>org.apache.httpcomponents</groupId>
+            <artifactId>httpcore</artifactId>
+            <version>4.4.6</version>
+        </dependency>
+        <dependency>
+            <groupId>org.apache.httpcomponents</groupId>
+            <artifactId>httpmime</artifactId>
+            <version>4.5.3</version>
+        </dependency>
+        <dependency>
+            <groupId>org.apache.solr</groupId>
+            <artifactId>solr-test-framework</artifactId>
+            <version>${global_solr_version}</version>
+            <scope>test</scope>
+            <exclusions>
+                <exclusion>
+                    <artifactId>fastutil</artifactId>
+                    <groupId>it.unimi.dsi</groupId>
+                </exclusion>
+                <exclusion>
+                    <groupId>com.fasterxml.jackson.core</groupId>
+                    <artifactId>jackson-core</artifactId>
+                </exclusion>
+                <exclusion>
+                    <groupId>com.fasterxml.jackson.core</groupId>
+                    <artifactId>jackson-databind</artifactId>
+                </exclusion>
+                <exclusion>
+                    <artifactId>caffeine</artifactId>
+                    <groupId>com.github.ben-manes.caffeine</groupId>
+                </exclusion>
+            </exclusions>
+        </dependency>
         <dependency>
             <groupId>org.apache.hbase</groupId>
             <artifactId>hbase-client</artifactId>
@@ -93,51 +137,14 @@
                     <groupId>org.apache.httpcomponents</groupId>
                     <artifactId>httpclient</artifactId>
                 </exclusion>
-            </exclusions>
-        </dependency>
-
-        <!-- Test -->
-
-        <dependency>
-            <groupId>org.apache.solr</groupId>
-            <artifactId>solr-test-framework</artifactId>
-            <version>${global_solr_version}</version>
-            <scope>test</scope>
-            <exclusions>
-                <exclusion>
-                    <artifactId>fastutil</artifactId>
-                    <groupId>it.unimi.dsi</groupId>
-                </exclusion>
-                <exclusion>
-                    <groupId>com.fasterxml.jackson.core</groupId>
-                    <artifactId>jackson-core</artifactId>
-                </exclusion>
                 <exclusion>
-                    <groupId>com.fasterxml.jackson.core</groupId>
-                    <artifactId>jackson-annotations</artifactId>
-                </exclusion>
-                <exclusion>
-                    <groupId>com.fasterxml.jackson.core</groupId>
-                    <artifactId>jackson-databind</artifactId>
-                </exclusion>
-                <exclusion>
-                    <artifactId>caffeine</artifactId>
-                    <groupId>com.github.ben-manes.caffeine</groupId>
+                    <groupId>org.apache.logging.log4j</groupId>
+                    <artifactId>log4j-core</artifactId>
                 </exclusion>
             </exclusions>
         </dependency>
-        <dependency>
-            <groupId>org.apache.logging.log4j</groupId>
-            <artifactId>log4j-api</artifactId>
-            <version>${global_log4j_core_version}</version>
-            <scope>test</scope>
-        </dependency>
-        <dependency>
-            <groupId>org.apache.logging.log4j</groupId>
-            <artifactId>log4j-core</artifactId>
-            <version>${global_log4j_core_version}</version>
-            <scope>test</scope>
-        </dependency>
+
+        <!-- Test -->
         <dependency>
             <groupId>org.apache.kafka</groupId>
             <artifactId>kafka-clients</artifactId>

diff --git a/...form/metron-solr/metron-solr-common/src/main/java/org/apache/metron/solr/dao/SolrDao.java b/...form/metron-solr/metron-solr-common/src/main/java/org/apache/metron/solr/dao/SolrDao.java
@@ -40,7 +40,7 @@
 import org.apache.metron.solr.client.SolrClientFactory;
 import org.apache.solr.client.solrj.SolrClient;
 import org.apache.solr.client.solrj.impl.HttpClientUtil;
-import org.apache.solr.client.solrj.impl.Krb5HttpClientConfigurer;
+import org.apache.solr.client.solrj.impl.Krb5HttpClientBuilder;
 import org.slf4j.Logger;
 import org.slf4j.LoggerFactory;
 
@@ -170,7 +170,8 @@ public Document removeCommentFromAlert(CommentAddRemoveRequest request, Document
   }
 
   void enableKerberos() {
-    HttpClientUtil.addConfigurer(new Krb5HttpClientConfigurer());
+    Krb5HttpClientBuilder krb5HttpClientBuilder = new Krb5HttpClientBuilder();
+    HttpClientUtil.setHttpClientBuilder(krb5HttpClientBuilder.getBuilder());
   }
 
   public SolrSearchDao getSolrSearchDao() {

diff --git a/.../metron/solr/writer/MetronSolrClient.java → ...metron/solr/writer/SolrClientFactory.java b/.../metron/solr/writer/MetronSolrClient.java → ...metron/solr/writer/SolrClientFactory.java
@@ -31,21 +31,33 @@
 import org.slf4j.LoggerFactory;
 
 import java.io.IOException;
-import java.util.List;
-import java.util.Map;
+import java.util.*;
 
-public class MetronSolrClient extends CloudSolrClient {
+public class SolrClientFactory {
 
   private static final Logger LOG = LoggerFactory
-          .getLogger(MetronSolrClient.class);
+          .getLogger(SolrClientFactory.class);
 
 
-  public MetronSolrClient(String zkHost) {
-    super(zkHost);
+  public static CloudSolrClient create(String zkHost) {
+    CloudSolrClient.Builder builder = getBuilder(zkHost);
+    return builder.build();
   }
 
-  public MetronSolrClient(String zkHost, Map<String, Object> solrHttpConfig) {
-    super(zkHost, HttpClientUtil.createClient(toSolrProps(solrHttpConfig)));
+  public static CloudSolrClient create(String zkHost, Map<String, Object> solrHttpConfig) {
+    CloudSolrClient.Builder builder = getBuilder(zkHost);
+    builder.withHttpClient(HttpClientUtil.createClient(toSolrProps(solrHttpConfig)));
+    return builder.build();
+  }
+
+
+  public static CloudSolrClient.Builder getBuilder(String zkHost) {
+    String[] parts = zkHost.split("/");
+    Optional<String> zkChroot = Optional.empty();
+    if (parts.length > 1) {
+      zkChroot = Optional.of("/" + parts[1]);
+    }
+    return new CloudSolrClient.Builder(Arrays.asList(parts[0].split(",")), zkChroot);
   }
 
   public static SolrParams toSolrProps(Map<String, Object> config) {
@@ -73,36 +85,4 @@ else if(v instanceof Iterable) {
     }
     return ret;
   }
-
-  public void createCollection(String name, int numShards, int replicationFactor) throws IOException, SolrServerException {
-    if (!listCollections().contains(name)) {
-      request(getCreateCollectionsRequest(name, numShards, replicationFactor));
-    }
-  }
-
-  public QueryRequest getCreateCollectionsRequest(String name, int numShards, int replicationFactor) {
-    ModifiableSolrParams params = new ModifiableSolrParams();
-    params.set(SolrConstants.REQUEST_ACTION, CollectionParams.CollectionAction.CREATE.name());
-    params.set(SolrConstants.REQUEST_NAME, name);
-    params.set(SolrConstants.REQUEST_NUM_SHARDS, numShards);
-    params.set(SolrConstants.REQUEST_REPLICATION_FACTOR, replicationFactor);
-    params.set(SolrConstants.REQUEST_COLLECTION_CONFIG_NAME, name);
-    QueryRequest request = new QueryRequest(params);
-    request.setPath(SolrConstants.REQUEST_COLLECTIONS_PATH);
-    return request;
-  }
-
-  @SuppressWarnings("unchecked")
-  public List<String> listCollections() throws IOException, SolrServerException {
-    NamedList<Object> response = request(getListCollectionsRequest(), null);
-    return (List<String>) response.get(SolrConstants.RESPONSE_COLLECTIONS);
-  }
-
-  public QueryRequest getListCollectionsRequest() {
-    ModifiableSolrParams params = new ModifiableSolrParams();
-    params.set(SolrConstants.REQUEST_ACTION, CollectionParams.CollectionAction.LIST.name());
-    QueryRequest request = new QueryRequest(params);
-    request.setPath(SolrConstants.REQUEST_COLLECTIONS_PATH);
-    return request;
-  }
 }
diff --git a/...etron-solr/metron-solr-common/src/main/java/org/apache/metron/solr/writer/SolrWriter.java b/...etron-solr/metron-solr-common/src/main/java/org/apache/metron/solr/writer/SolrWriter.java
@@ -46,7 +46,7 @@
 import org.apache.solr.client.solrj.SolrServerException;
 import org.apache.solr.client.solrj.impl.HttpClientUtil;
 import org.apache.solr.client.solrj.impl.HttpSolrClient;
-import org.apache.solr.client.solrj.impl.Krb5HttpClientConfigurer;
+import org.apache.solr.client.solrj.impl.Krb5HttpClientBuilder;
 import org.apache.solr.client.solrj.response.UpdateResponse;
 import org.apache.solr.common.SolrException;
 import org.apache.solr.common.SolrInputDocument;
@@ -134,9 +134,9 @@ public <T> T coerceOrDefaultOrExcept(Map<String, Object> globalConfig, Class<T>
   private String defaultCollection;
   private Map<String, Object> solrHttpConfig;
 
-  private MetronSolrClient solr;
+  private org.apache.solr.client.solrj.impl.CloudSolrClient solr;
 
-  public SolrWriter withMetronSolrClient(MetronSolrClient solr) {
+  public SolrWriter withCloudSolrClient(org.apache.solr.client.solrj.impl.CloudSolrClient solr) {
     this.solr = solr;
     return this;
   }
@@ -163,9 +163,12 @@ public void init(Map stormConf, WriterConfiguration configurations) throws IOExc
     LOG.info("Default Collection: {}", "" + defaultCollection );
     if(solr == null) {
       if (isKerberosEnabled(stormConf)) {
-        HttpClientUtil.addConfigurer(new Krb5HttpClientConfigurer());
+        Krb5HttpClientBuilder krb5HttpClientBuilder = new Krb5HttpClientBuilder();
+        HttpClientUtil.setHttpClientBuilder(krb5HttpClientBuilder.getBuilder());
       }
-      solr = new MetronSolrClient(zookeeperUrl, solrHttpConfig);
+
+
+      solr = SolrClientFactory.create(zookeeperUrl, solrHttpConfig);
     }
     solr.setDefaultCollection(defaultCollection);