From ad71c0468888977f1b3ab1aa58fea38b846a1e37 Mon Sep 17 00:00:00 2001 From: merrimanr Date: Wed, 28 Aug 2019 12:47:40 -0400 Subject: [PATCH] METRON-2225 Upgrade to Solr 7.4.0 (merrimanr via nickwallen) closes apache/metron#1492 --- dependencies_with_url.csv | 4 +- .../ansible/roles/solr/defaults/main.yml | 2 +- .../ElasticsearchSearchIntegrationTest.java | 8 ++ .../indexing/dao/SearchIntegrationTest.java | 13 ++- .../metron-solr/metron-solr-common/README.md | 24 +++-- .../metron-solr/metron-solr-common/pom.xml | 89 ++++++++++--------- .../org/apache/metron/solr/dao/SolrDao.java | 5 +- ...SolrClient.java => SolrClientFactory.java} | 60 +++++-------- .../apache/metron/solr/writer/SolrWriter.java | 13 +-- .../src/main/scripts/create_collection.sh | 2 +- .../src/main/scripts/delete_collection.sh | 3 + .../src/main/scripts/install_solr.sh | 2 + .../SolrMetaAlertIntegrationTest.java | 2 +- .../SolrSearchIntegrationTest.java | 19 ++++ .../integration/components/SolrComponent.java | 27 +++--- .../solr/writer/MetronSolrClientTest.java | 83 ----------------- .../metron/solr/writer/SolrWriterTest.java | 7 +- .../SolrIndexingIntegrationTest.java | 23 +++-- pom.xml | 3 +- 19 files changed, 165 insertions(+), 224 deletions(-) rename metron-platform/metron-solr/metron-solr-common/src/main/java/org/apache/metron/solr/writer/{MetronSolrClient.java => SolrClientFactory.java} (54%) delete mode 100644 metron-platform/metron-solr/metron-solr-common/src/test/java/org/apache/metron/solr/writer/MetronSolrClientTest.java diff --git a/dependencies_with_url.csv b/dependencies_with_url.csv index 9d7b0fa28c..66c4766b82 100644 --- a/dependencies_with_url.csv +++ b/dependencies_with_url.csv @@ -136,7 +136,7 @@ org.slf4j:slf4j-simple:jar:1.7.7:compile,MIT,http://www.slf4j.org org.slf4j:jcl-over-slf4j:jar:1.7.7:compile,MIT,http://www.slf4j.org org.slf4j:jcl-over-slf4j:jar:1.7.16:compile,MIT,http://www.slf4j.org org.slf4j:jcl-over-slf4j:jar:1.7.21:compile,MIT,http://www.slf4j.org -org.slf4j:jcl-over-slf4j:jar:1.7.21:compile,MIT,http://www.slf4j.org +org.slf4j:jcl-over-slf4j:jar:1.7.24:compile,MIT,http://www.slf4j.org org.slf4j:jul-to-slf4j:jar:1.7.16:compile,MIT,http://www.slf4j.org org.slf4j:jul-to-slf4j:jar:1.7.21:compile,MIT,http://www.slf4j.org org.slf4j:jul-to-slf4j:jar:1.7.25:compile,MIT,http://www.slf4j.org @@ -328,6 +328,7 @@ org.mortbay.jetty:jsp-2.1:jar:6.1.14:compile,ASLv2, org.mortbay.jetty:jsp-api-2.1:jar:6.1.14:compile,ASLv2, org.mortbay.jetty:servlet-api-2.5:jar:6.1.14:compile,ASLv2, org.noggit:noggit:jar:0.6:compile,Apache License, Version 2.0,http://github.com/yonik/noggit +org.noggit:noggit:jar:0.8:compile,Apache License, Version 2.0,http://github.com/yonik/noggit org.scannotation:scannotation:jar:1.0.3:compile,Apache License V2.0,http://scannotation.sf.net org.slf4j:log4j-over-slf4j:jar:1.6.6:compile,Apache Software Licenses,http://www.slf4j.org org.springframework.integration:spring-integration-core:jar:3.0.0.RELEASE:compile,The Apache Software License, Version 2.0,http://www.springintegration.org/ @@ -453,6 +454,7 @@ org.springframework.kafka:spring-kafka:jar:1.1.1.RELEASE:compile,ASLv2,https://g org.springframework.kafka:spring-kafka:jar:2.0.4.RELEASE:compile,ASLv2,https://github.com/spring-projects/spring-kafka ch.hsr:geohash:jar:1.3.0:compile,ASLv2,https://github.com/kungfoo/geohash-java org.locationtech.spatial4j:spatial4j:jar:0.6:compile,ASLv2,https://github.com/locationtech/spatial4j +org.locationtech.spatial4j:spatial4j:jar:0.7:compile,ASLv2,https://github.com/locationtech/spatial4j com.github.luben:zstd-jni:jar:1.3.2-2:compile,BSD,https://github.com/luben/zstd-jni com.github.spullara.mustache.java:compiler:jar:0.9.3:compile,ASLv2,https://github.com/spullara/mustache.java/blob/master/LICENSE io.netty:netty-buffer:jar:4.1.13.Final:compile,ASLv2,http://netty.io/ diff --git a/metron-deployment/ansible/roles/solr/defaults/main.yml b/metron-deployment/ansible/roles/solr/defaults/main.yml index 61562f8fbf..ca4e289663 100644 --- a/metron-deployment/ansible/roles/solr/defaults/main.yml +++ b/metron-deployment/ansible/roles/solr/defaults/main.yml @@ -15,7 +15,7 @@ # limitations under the License. # --- -solr_version: 6.6.2 +solr_version: 7.4.0 solr_installation_user: "solr" solr_user_home: /var/solr solr_home: "{{ solr_user_home }}/solr-{{ solr_version }}" diff --git a/metron-platform/metron-elasticsearch/metron-elasticsearch-common/src/test/java/org/apache/metron/elasticsearch/integration/ElasticsearchSearchIntegrationTest.java b/metron-platform/metron-elasticsearch/metron-elasticsearch-common/src/test/java/org/apache/metron/elasticsearch/integration/ElasticsearchSearchIntegrationTest.java index 7da94e970e..26e0529a0f 100644 --- a/metron-platform/metron-elasticsearch/metron-elasticsearch-common/src/test/java/org/apache/metron/elasticsearch/integration/ElasticsearchSearchIntegrationTest.java +++ b/metron-platform/metron-elasticsearch/metron-elasticsearch-common/src/test/java/org/apache/metron/elasticsearch/integration/ElasticsearchSearchIntegrationTest.java @@ -270,6 +270,14 @@ public void throws_exception_on_aggregation_queries_on_non_string_non_numeric_fi dao.group(request); } + @Test + public void different_type_facet_query() throws Exception { + thrown.expect(Exception.class); + SearchRequest request = JSONUtils.INSTANCE.load(differentTypeFacetQuery, SearchRequest.class); + SearchResponse response = getIndexDao().search(request); + Assert.assertEquals(3, response.getTotal()); + } + @Test public void different_type_filter_query() throws Exception { SearchRequest request = JSONUtils.INSTANCE.load(differentTypeFilterQuery, SearchRequest.class); diff --git a/metron-platform/metron-indexing/metron-indexing-common/src/test/java/org/apache/metron/indexing/dao/SearchIntegrationTest.java b/metron-platform/metron-indexing/metron-indexing-common/src/test/java/org/apache/metron/indexing/dao/SearchIntegrationTest.java index cfe5752033..7aa60b714f 100644 --- a/metron-platform/metron-indexing/metron-indexing-common/src/test/java/org/apache/metron/indexing/dao/SearchIntegrationTest.java +++ b/metron-platform/metron-indexing/metron-indexing-common/src/test/java/org/apache/metron/indexing/dao/SearchIntegrationTest.java @@ -709,14 +709,6 @@ public void missing_type_facet_query() throws Exception { response.getFacetCounts(); } - @Test - public void different_type_facet_query() throws Exception { - thrown.expect(Exception.class); - SearchRequest request = JSONUtils.INSTANCE.load(differentTypeFacetQuery, SearchRequest.class); - SearchResponse response = getIndexDao().search(request); - Assert.assertEquals(3, response.getTotal()); - } - @Test public void exceeding_max_results_throws_exception() throws Exception { thrown.expect(InvalidSearchException.class); @@ -934,8 +926,13 @@ public static void stop() { @Test public abstract void returns_column_data_for_multiple_indices() throws Exception; + @Test public abstract void returns_column_metadata_for_specified_indices() throws Exception; + + @Test + public abstract void different_type_facet_query() throws Exception; + @Test public abstract void different_type_filter_query() throws Exception; diff --git a/metron-platform/metron-solr/metron-solr-common/README.md b/metron-platform/metron-solr/metron-solr-common/README.md index 081436a90e..cf37589389 100644 --- a/metron-platform/metron-solr/metron-solr-common/README.md +++ b/metron-platform/metron-solr/metron-solr-common/README.md @@ -27,7 +27,7 @@ limitations under the License. ## Introduction -Metron ships with Solr 6.6.2 support. Solr Cloud can be used as the real-time portion of the datastore resulting from [metron-indexing](../metron-indexing/README.md). +Metron ships with Solr 7.4.0 support. Solr Cloud can be used as the real-time portion of the datastore resulting from [metron-indexing](../metron-indexing/README.md). ## Configuration @@ -42,15 +42,15 @@ via the global config. The following settings are possible as part of the globa * _WARNING_: If you set this to `false`, then commits will happen based on the SolrClient's internal mechanism and worker failure *may* result data being acknowledged in storm but not written in Solr. * `solr.commit.soft` - * This is a boolean which defines whether the writer makes a soft commit or a durable commit. See [here](https://lucene.apache.org/solr/guide/6_6/near-real-time-searching.html#NearRealTimeSearching-AutoCommits) The default is `false`. + * This is a boolean which defines whether the writer makes a soft commit or a durable commit. See [here](https://lucene.apache.org/solr/guide/7_4/near-real-time-searching.html) The default is `false`. * _WARNING_: If you set this to `true`, then commits will happen based on the SolrClient's internal mechanism and worker failure *may* result data being acknowledged in storm but not written in Solr. * `solr.commit.waitSearcher` - * This is a boolean which defines whether the writer blocks the commit until the data is available to search. See [here](https://lucene.apache.org/solr/guide/6_6/near-real-time-searching.html#NearRealTimeSearching-AutoCommits) The default is `true`. + * This is a boolean which defines whether the writer blocks the commit until the data is available to search. See [here](https://lucene.apache.org/solr/guide/7_4/near-real-time-searching.html) The default is `true`. * _WARNING_: If you set this to `false`, then commits will happen based on the SolrClient's internal mechanism and worker failure *may* result data being acknowledged in storm but not written in Solr. * `solr.commit.waitFlush` - * This is a boolean which defines whether the writer blocks the commit until the data is flushed. See [here](https://lucene.apache.org/solr/guide/6_6/near-real-time-searching.html#NearRealTimeSearching-AutoCommits) The default is `true`. + * This is a boolean which defines whether the writer blocks the commit until the data is flushed. See [here](https://lucene.apache.org/solr/guide/7_4/near-real-time-searching.html) The default is `true`. * _WARNING_: If you set this to `false`, then commits will happen based on the SolrClient's internal mechanism and worker failure *may* result data being acknowledged in storm but not written in Solr. * `solr.collection` @@ -86,15 +86,11 @@ The script performs the following tasks * Installs Solr * Starts Solr Cloud -_Note: for details on setting up Solr Cloud in production mode, see https://lucene.apache.org/solr/guide/6_6/taking-solr-to-production.html_ +Note: for details on setting up Solr Cloud in production mode, see https://lucene.apache.org/solr/guide/7_4/taking-solr-to-production.html Navigate to `$METRON_HOME/bin` and spin up Solr Cloud by running `install_solr.sh`. After running this script, Elasticsearch and Kibana will have been stopped and you should now have an instance of Solr Cloud up and running at http://localhost:8983/solr/#/~cloud. This manner of starting Solr -will also spin up an embedded Zookeeper instance at port 9983. More information can be found [here](https://lucene.apache.org/solr/guide/6_6/getting-started-with-solrcloud.html) - -Solr can also be installed using [HDP Search 3](https://docs.hortonworks.com/HDPDocuments/HDP2/HDP-2.6.4/bk_solr-search-installation/content/ch_hdp_search_30.html). HDP Search 3 sets the Zookeeper root to -`/solr` so this will need to be added to each url in the comma-separated list in Ambari UI -> Services -> Metron -> Configs -> Index Settings -> Solr Zookeeper Urls. For example, in full dev -this would be `node1:2181/solr`. +will also spin up an embedded Zookeeper instance at port 9983. More information can be found [here](https://lucene.apache.org/solr/guide/7_4/getting-started-with-solrcloud.html) ## Enabling Solr @@ -122,8 +118,8 @@ Any other collections must be created manually before starting the Indexing comp As of now, we have mapped out the Schemas in `src/main/config/schema`. Ambari will eventually install these, but at the moment it's manual and -you should refer to the Solr documentation [https://lucene.apache.org/solr/guide/6_6](here) in general -and [here](https://lucene.apache.org/solr/guide/6_6/documents-fields-and-schema-design.html) if you'd like to know more about schemas in Solr. +you should refer to the Solr documentation [https://lucene.apache.org/solr/guide/7_4](here) in general +and [here](https://lucene.apache.org/solr/guide/7_4/documents-fields-and-schema-design.html) if you'd like to know more about schemas in Solr. In Metron's Solr DAO implementation, document updates involve reading a document, applying the update and replacing the original by reindexing the whole document. Indexing LatLonType and PointType field types stores data in internal fields that should not be returned in search results. For these fields a dynamic field type matching the suffix needs to be added to store the data points. @@ -146,7 +142,7 @@ If any copy fields are defined, stored and docValues should be set to false. Convenience scripts are provided with Metron to create and delete collections. Ambari uses these scripts to automatically create collections. To use them outside of Ambari, a few environment variables must be set first: ``` # Path to the zookeeper node used by Solr -export ZOOKEEPER=node1:2181/solr +export ZOOKEEPER=node1:9983 # Set to true if Kerberos is enabled export SECURITY_ENABLED=true ``` @@ -167,4 +163,4 @@ The `create_collection.sh` script depends on schemas installed in `$METRON_HOME/ * error Additional schemas should be installed in that location if using the `create_collection.sh` script. Any collection can be deleted with the `delete_collection.sh` script. -These scripts use the [Solr Collection API](http://lucene.apache.org/solr/guide/6_6/collections-api.html). \ No newline at end of file +These scripts use the [Solr Collection API](http://lucene.apache.org/solr/guide/7_4/collections-api.html). \ No newline at end of file diff --git a/metron-platform/metron-solr/metron-solr-common/pom.xml b/metron-platform/metron-solr/metron-solr-common/pom.xml index 03468500fe..fd9ef01f3c 100644 --- a/metron-platform/metron-solr/metron-solr-common/pom.xml +++ b/metron-platform/metron-solr/metron-solr-common/pom.xml @@ -39,6 +39,50 @@ solr-solrj ${global_solr_version} + + org.noggit + noggit + 0.8 + + + org.apache.httpcomponents + httpclient + 4.5.3 + + + org.apache.httpcomponents + httpcore + 4.4.6 + + + org.apache.httpcomponents + httpmime + 4.5.3 + + + org.apache.solr + solr-test-framework + ${global_solr_version} + test + + + fastutil + it.unimi.dsi + + + com.fasterxml.jackson.core + jackson-core + + + com.fasterxml.jackson.core + jackson-databind + + + caffeine + com.github.ben-manes.caffeine + + + org.apache.hbase hbase-client @@ -93,51 +137,14 @@ org.apache.httpcomponents httpclient - - - - - - - org.apache.solr - solr-test-framework - ${global_solr_version} - test - - - fastutil - it.unimi.dsi - - - com.fasterxml.jackson.core - jackson-core - - com.fasterxml.jackson.core - jackson-annotations - - - com.fasterxml.jackson.core - jackson-databind - - - caffeine - com.github.ben-manes.caffeine + org.apache.logging.log4j + log4j-core - - org.apache.logging.log4j - log4j-api - ${global_log4j_core_version} - test - - - org.apache.logging.log4j - log4j-core - ${global_log4j_core_version} - test - + + org.apache.kafka kafka-clients diff --git a/metron-platform/metron-solr/metron-solr-common/src/main/java/org/apache/metron/solr/dao/SolrDao.java b/metron-platform/metron-solr/metron-solr-common/src/main/java/org/apache/metron/solr/dao/SolrDao.java index ef68e8687a..0ef67a0947 100644 --- a/metron-platform/metron-solr/metron-solr-common/src/main/java/org/apache/metron/solr/dao/SolrDao.java +++ b/metron-platform/metron-solr/metron-solr-common/src/main/java/org/apache/metron/solr/dao/SolrDao.java @@ -40,7 +40,7 @@ import org.apache.metron.solr.client.SolrClientFactory; import org.apache.solr.client.solrj.SolrClient; import org.apache.solr.client.solrj.impl.HttpClientUtil; -import org.apache.solr.client.solrj.impl.Krb5HttpClientConfigurer; +import org.apache.solr.client.solrj.impl.Krb5HttpClientBuilder; import org.slf4j.Logger; import org.slf4j.LoggerFactory; @@ -170,7 +170,8 @@ public Document removeCommentFromAlert(CommentAddRemoveRequest request, Document } void enableKerberos() { - HttpClientUtil.addConfigurer(new Krb5HttpClientConfigurer()); + Krb5HttpClientBuilder krb5HttpClientBuilder = new Krb5HttpClientBuilder(); + HttpClientUtil.setHttpClientBuilder(krb5HttpClientBuilder.getBuilder()); } public SolrSearchDao getSolrSearchDao() { diff --git a/metron-platform/metron-solr/metron-solr-common/src/main/java/org/apache/metron/solr/writer/MetronSolrClient.java b/metron-platform/metron-solr/metron-solr-common/src/main/java/org/apache/metron/solr/writer/SolrClientFactory.java similarity index 54% rename from metron-platform/metron-solr/metron-solr-common/src/main/java/org/apache/metron/solr/writer/MetronSolrClient.java rename to metron-platform/metron-solr/metron-solr-common/src/main/java/org/apache/metron/solr/writer/SolrClientFactory.java index 5c27cceb45..a662558e04 100644 --- a/metron-platform/metron-solr/metron-solr-common/src/main/java/org/apache/metron/solr/writer/MetronSolrClient.java +++ b/metron-platform/metron-solr/metron-solr-common/src/main/java/org/apache/metron/solr/writer/SolrClientFactory.java @@ -31,21 +31,33 @@ import org.slf4j.LoggerFactory; import java.io.IOException; -import java.util.List; -import java.util.Map; +import java.util.*; -public class MetronSolrClient extends CloudSolrClient { +public class SolrClientFactory { private static final Logger LOG = LoggerFactory - .getLogger(MetronSolrClient.class); + .getLogger(SolrClientFactory.class); - public MetronSolrClient(String zkHost) { - super(zkHost); + public static CloudSolrClient create(String zkHost) { + CloudSolrClient.Builder builder = getBuilder(zkHost); + return builder.build(); } - public MetronSolrClient(String zkHost, Map solrHttpConfig) { - super(zkHost, HttpClientUtil.createClient(toSolrProps(solrHttpConfig))); + public static CloudSolrClient create(String zkHost, Map solrHttpConfig) { + CloudSolrClient.Builder builder = getBuilder(zkHost); + builder.withHttpClient(HttpClientUtil.createClient(toSolrProps(solrHttpConfig))); + return builder.build(); + } + + + public static CloudSolrClient.Builder getBuilder(String zkHost) { + String[] parts = zkHost.split("/"); + Optional zkChroot = Optional.empty(); + if (parts.length > 1) { + zkChroot = Optional.of("/" + parts[1]); + } + return new CloudSolrClient.Builder(Arrays.asList(parts[0].split(",")), zkChroot); } public static SolrParams toSolrProps(Map config) { @@ -73,36 +85,4 @@ else if(v instanceof Iterable) { } return ret; } - - public void createCollection(String name, int numShards, int replicationFactor) throws IOException, SolrServerException { - if (!listCollections().contains(name)) { - request(getCreateCollectionsRequest(name, numShards, replicationFactor)); - } - } - - public QueryRequest getCreateCollectionsRequest(String name, int numShards, int replicationFactor) { - ModifiableSolrParams params = new ModifiableSolrParams(); - params.set(SolrConstants.REQUEST_ACTION, CollectionParams.CollectionAction.CREATE.name()); - params.set(SolrConstants.REQUEST_NAME, name); - params.set(SolrConstants.REQUEST_NUM_SHARDS, numShards); - params.set(SolrConstants.REQUEST_REPLICATION_FACTOR, replicationFactor); - params.set(SolrConstants.REQUEST_COLLECTION_CONFIG_NAME, name); - QueryRequest request = new QueryRequest(params); - request.setPath(SolrConstants.REQUEST_COLLECTIONS_PATH); - return request; - } - - @SuppressWarnings("unchecked") - public List listCollections() throws IOException, SolrServerException { - NamedList response = request(getListCollectionsRequest(), null); - return (List) response.get(SolrConstants.RESPONSE_COLLECTIONS); - } - - public QueryRequest getListCollectionsRequest() { - ModifiableSolrParams params = new ModifiableSolrParams(); - params.set(SolrConstants.REQUEST_ACTION, CollectionParams.CollectionAction.LIST.name()); - QueryRequest request = new QueryRequest(params); - request.setPath(SolrConstants.REQUEST_COLLECTIONS_PATH); - return request; - } } diff --git a/metron-platform/metron-solr/metron-solr-common/src/main/java/org/apache/metron/solr/writer/SolrWriter.java b/metron-platform/metron-solr/metron-solr-common/src/main/java/org/apache/metron/solr/writer/SolrWriter.java index b23a517ff3..242c44a52f 100644 --- a/metron-platform/metron-solr/metron-solr-common/src/main/java/org/apache/metron/solr/writer/SolrWriter.java +++ b/metron-platform/metron-solr/metron-solr-common/src/main/java/org/apache/metron/solr/writer/SolrWriter.java @@ -46,7 +46,7 @@ import org.apache.solr.client.solrj.SolrServerException; import org.apache.solr.client.solrj.impl.HttpClientUtil; import org.apache.solr.client.solrj.impl.HttpSolrClient; -import org.apache.solr.client.solrj.impl.Krb5HttpClientConfigurer; +import org.apache.solr.client.solrj.impl.Krb5HttpClientBuilder; import org.apache.solr.client.solrj.response.UpdateResponse; import org.apache.solr.common.SolrException; import org.apache.solr.common.SolrInputDocument; @@ -134,9 +134,9 @@ public T coerceOrDefaultOrExcept(Map globalConfig, Class private String defaultCollection; private Map solrHttpConfig; - private MetronSolrClient solr; + private org.apache.solr.client.solrj.impl.CloudSolrClient solr; - public SolrWriter withMetronSolrClient(MetronSolrClient solr) { + public SolrWriter withCloudSolrClient(org.apache.solr.client.solrj.impl.CloudSolrClient solr) { this.solr = solr; return this; } @@ -163,9 +163,12 @@ public void init(Map stormConf, WriterConfiguration configurations) throws IOExc LOG.info("Default Collection: {}", "" + defaultCollection ); if(solr == null) { if (isKerberosEnabled(stormConf)) { - HttpClientUtil.addConfigurer(new Krb5HttpClientConfigurer()); + Krb5HttpClientBuilder krb5HttpClientBuilder = new Krb5HttpClientBuilder(); + HttpClientUtil.setHttpClientBuilder(krb5HttpClientBuilder.getBuilder()); } - solr = new MetronSolrClient(zookeeperUrl, solrHttpConfig); + + + solr = SolrClientFactory.create(zookeeperUrl, solrHttpConfig); } solr.setDefaultCollection(defaultCollection); diff --git a/metron-platform/metron-solr/metron-solr-common/src/main/scripts/create_collection.sh b/metron-platform/metron-solr/metron-solr-common/src/main/scripts/create_collection.sh index 7693646e25..087a80a695 100755 --- a/metron-platform/metron-solr/metron-solr-common/src/main/scripts/create_collection.sh +++ b/metron-platform/metron-solr/metron-solr-common/src/main/scripts/create_collection.sh @@ -33,4 +33,4 @@ SOLR_NODE=`$ZOOKEEPER_HOME/bin/zkCli.sh -server $ZOOKEEPER ls /live_nodes | tail zip -rj - $METRON_HOME/config/schema/$1 | curl -X POST $NEGOTIATE --header "Content-Type:text/xml" --data-binary @- "http://$SOLR_NODE/solr/admin/configs?action=UPLOAD&name=$1" # Create the collection -curl -X GET $NEGOTIATE "http://$SOLR_NODE/solr/admin/collections?action=CREATE&name=$1&numShards=1" +curl -X GET $NEGOTIATE "http://$SOLR_NODE/solr/admin/collections?action=CREATE&name=$1&collection.configName=$1&numShards=1" diff --git a/metron-platform/metron-solr/metron-solr-common/src/main/scripts/delete_collection.sh b/metron-platform/metron-solr/metron-solr-common/src/main/scripts/delete_collection.sh index c8b45e7e1b..3898720929 100755 --- a/metron-platform/metron-solr/metron-solr-common/src/main/scripts/delete_collection.sh +++ b/metron-platform/metron-solr/metron-solr-common/src/main/scripts/delete_collection.sh @@ -31,3 +31,6 @@ SOLR_NODE=`$ZOOKEEPER_HOME/bin/zkCli.sh -server $ZOOKEEPER ls /live_nodes | tail # Delete the collection curl -X GET $NEGOTIATE "http://$SOLR_NODE/solr/admin/collections?action=DELETE&name=$1" + +# Delete the config set +curl -X GET $NEGOTIATE "http://$SOLR_NODE/solr/admin/configs?action=DELETE&name=$1" diff --git a/metron-platform/metron-solr/metron-solr-common/src/main/scripts/install_solr.sh b/metron-platform/metron-solr/metron-solr-common/src/main/scripts/install_solr.sh index da04557d1d..352833f99a 100755 --- a/metron-platform/metron-solr/metron-solr-common/src/main/scripts/install_solr.sh +++ b/metron-platform/metron-solr/metron-solr-common/src/main/scripts/install_solr.sh @@ -30,6 +30,8 @@ service kibana stop service elasticsearch stop +yum install -y wget zip lsof + SOLR_VERSION=${global_solr_version} SOLR_USER=solr SOLR_SERVICE=$SOLR_USER diff --git a/metron-platform/metron-solr/metron-solr-common/src/test/java/org/apache/metron/solr/integration/SolrMetaAlertIntegrationTest.java b/metron-platform/metron-solr/metron-solr-common/src/test/java/org/apache/metron/solr/integration/SolrMetaAlertIntegrationTest.java index 084660f367..5f23f38c7b 100644 --- a/metron-platform/metron-solr/metron-solr-common/src/test/java/org/apache/metron/solr/integration/SolrMetaAlertIntegrationTest.java +++ b/metron-platform/metron-solr/metron-solr-common/src/test/java/org/apache/metron/solr/integration/SolrMetaAlertIntegrationTest.java @@ -389,7 +389,7 @@ protected String getSourceTypeField() { @Override protected void commit() throws IOException { try { - List collections = solr.getSolrClient().listCollections(); + List collections = solr.listCollections(); for (String collection : collections) { solr.getSolrClient().commit(collection); } diff --git a/metron-platform/metron-solr/metron-solr-common/src/test/java/org/apache/metron/solr/integration/SolrSearchIntegrationTest.java b/metron-platform/metron-solr/metron-solr-common/src/test/java/org/apache/metron/solr/integration/SolrSearchIntegrationTest.java index de01df99d2..6c28a79480 100644 --- a/metron-platform/metron-solr/metron-solr-common/src/test/java/org/apache/metron/solr/integration/SolrSearchIntegrationTest.java +++ b/metron-platform/metron-solr/metron-solr-common/src/test/java/org/apache/metron/solr/integration/SolrSearchIntegrationTest.java @@ -225,6 +225,25 @@ public void returns_column_data_for_multiple_indices() throws Exception { Assert.assertEquals(null, fieldTypes.get("fake.field")); } + @Test + public void different_type_facet_query() throws Exception { + SearchRequest request = JSONUtils.INSTANCE.load(differentTypeFacetQuery, SearchRequest.class); + SearchResponse response = getIndexDao().search(request); + Assert.assertEquals(10, response.getTotal()); + Assert.assertTrue(response.getFacetCounts().containsKey("ttl")); + Map facetCounts = response.getFacetCounts().get("ttl"); + Assert.assertEquals(1L, facetCounts.get("1").longValue()); + Assert.assertEquals(1L, facetCounts.get("2").longValue()); + Assert.assertEquals(1L, facetCounts.get("3").longValue()); + Assert.assertEquals(1L, facetCounts.get("4").longValue()); + Assert.assertEquals(1L, facetCounts.get("5").longValue()); + Assert.assertEquals(1L, facetCounts.get("data 1").longValue()); + Assert.assertEquals(1L, facetCounts.get("data 2").longValue()); + Assert.assertEquals(1L, facetCounts.get("data 3").longValue()); + Assert.assertEquals(1L, facetCounts.get("data 4").longValue()); + Assert.assertEquals(1L, facetCounts.get("data 5").longValue()); + } + @Test public void different_type_filter_query() throws Exception { thrown.expect(InvalidSearchException.class); diff --git a/metron-platform/metron-solr/metron-solr-common/src/test/java/org/apache/metron/solr/integration/components/SolrComponent.java b/metron-platform/metron-solr/metron-solr-common/src/test/java/org/apache/metron/solr/integration/components/SolrComponent.java index 4bc9f8a469..c91c43a578 100644 --- a/metron-platform/metron-solr/metron-solr-common/src/test/java/org/apache/metron/solr/integration/components/SolrComponent.java +++ b/metron-platform/metron-solr/metron-solr-common/src/test/java/org/apache/metron/solr/integration/components/SolrComponent.java @@ -21,12 +21,12 @@ import java.util.Collection; import java.util.Map.Entry; import java.util.stream.Collectors; -import org.apache.metron.common.Constants; + import org.apache.metron.indexing.dao.metaalert.MetaAlertConstants; import org.apache.metron.integration.InMemoryComponent; import org.apache.metron.integration.UnableToStartException; import org.apache.metron.solr.dao.SolrUtilities; -import org.apache.metron.solr.writer.MetronSolrClient; +import org.apache.metron.solr.writer.SolrClientFactory; import org.apache.solr.client.solrj.SolrQuery; import org.apache.solr.client.solrj.SolrServerException; import org.apache.solr.client.solrj.embedded.JettyConfig; @@ -105,7 +105,7 @@ public void start() throws UnableToStartException { for(String name: collections.keySet()) { String configPath = collections.get(name); miniSolrCloudCluster.uploadConfigSet(new File(configPath).toPath(), name); - CollectionAdminRequest.createCollection(name, 1, 1).process(miniSolrCloudCluster.getSolrClient()); + CollectionAdminRequest.createCollection(name, name,1, 1).process(miniSolrCloudCluster.getSolrClient()); } if (postStartCallback != null) { postStartCallback.apply(this); @@ -134,8 +134,8 @@ public void reset() { } } - public MetronSolrClient getSolrClient() { - return new MetronSolrClient(getZookeeperUrl()); + public CloudSolrClient getSolrClient() { + return SolrClientFactory.create(getZookeeperUrl()); } public MiniSolrCloudCluster getMiniSolrCloudCluster() { @@ -149,19 +149,16 @@ public String getZookeeperUrl() { public void addCollection(String name, String configPath) throws InterruptedException, IOException, KeeperException, SolrServerException { miniSolrCloudCluster.uploadConfigSet(new File(configPath).toPath(), name); - CollectionAdminRequest.createCollection(name, 1, 1) + CollectionAdminRequest.createCollection(name, name,1, 1) .process(miniSolrCloudCluster.getSolrClient()); } - public boolean hasCollection(String collection) { - MetronSolrClient solr = getSolrClient(); - boolean collectionFound = false; - try { - collectionFound = solr.listCollections().contains(collection); - } catch (Exception e) { - e.printStackTrace(); - } - return collectionFound; + public List listCollections() throws IOException, SolrServerException { + return CollectionAdminRequest.listCollections(miniSolrCloudCluster.getSolrClient()); + } + + public boolean hasCollection(String collection) throws IOException, SolrServerException { + return listCollections().contains(collection); } public List> getAllIndexedDocs(String collection) { diff --git a/metron-platform/metron-solr/metron-solr-common/src/test/java/org/apache/metron/solr/writer/MetronSolrClientTest.java b/metron-platform/metron-solr/metron-solr-common/src/test/java/org/apache/metron/solr/writer/MetronSolrClientTest.java deleted file mode 100644 index 37807e6b94..0000000000 --- a/metron-platform/metron-solr/metron-solr-common/src/test/java/org/apache/metron/solr/writer/MetronSolrClientTest.java +++ /dev/null @@ -1,83 +0,0 @@ -/** - * Licensed to the Apache Software Foundation (ASF) under one - * or more contributor license agreements. See the NOTICE file - * distributed with this work for additional information - * regarding copyright ownership. The ASF licenses this file - * to you under the Apache License, Version 2.0 (the - * "License"); you may not use this file except in compliance - * with the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ -package org.apache.metron.solr.writer; - -import org.apache.metron.solr.writer.MetronSolrClient; -import org.apache.solr.client.solrj.request.QueryRequest; -import org.apache.solr.common.params.CollectionParams; -import org.apache.solr.common.util.NamedList; -import org.hamcrest.Description; -import org.junit.Test; -import org.mockito.ArgumentMatcher; -import org.mockito.Mockito; - -import java.util.ArrayList; - -import static org.mockito.Matchers.argThat; -import static org.mockito.Matchers.isNull; -import static org.mockito.Mockito.times; -import static org.mockito.Mockito.verify; - -public class MetronSolrClientTest { - - static class CollectionRequestMatcher extends ArgumentMatcher { - - private String name; - - public CollectionRequestMatcher(String name) { - this.name = name; - } - - @Override - public boolean matches(Object o) { - QueryRequest queryRequest = (QueryRequest) o; - return name.equals(queryRequest.getParams().get("action")); - } - - @Override - public void describeTo(Description description) { - description.appendText(name); - } - } - - @Test - public void testClient() throws Exception { - - final String collection = "metron"; - String zookeeperUrl = "zookeeperUrl"; - MetronSolrClient metronSolrClient = Mockito.spy(new MetronSolrClient(zookeeperUrl)); - - Mockito.doReturn(new NamedList() {{ - add("collections", new ArrayList() {{ - add(collection); - }}); - }}).when(metronSolrClient).request(argThat(new CollectionRequestMatcher(CollectionParams.CollectionAction.LIST.name())), (String) isNull()); - metronSolrClient.createCollection(collection, 1, 1); - verify(metronSolrClient, times(1)).request(argThat(new CollectionRequestMatcher(CollectionParams.CollectionAction.LIST.name())), (String) isNull()); - verify(metronSolrClient, times(0)).request(argThat(new CollectionRequestMatcher(CollectionParams.CollectionAction.CREATE.name())), (String) isNull()); - - metronSolrClient = Mockito.spy(new MetronSolrClient(zookeeperUrl)); - Mockito.doReturn(new NamedList() {{ - add("collections", new ArrayList()); - }}).when(metronSolrClient).request(argThat(new CollectionRequestMatcher(CollectionParams.CollectionAction.LIST.name())), (String) isNull()); - Mockito.doReturn(new NamedList<>()).when(metronSolrClient).request(argThat(new CollectionRequestMatcher(CollectionParams.CollectionAction.CREATE.name())), (String) isNull()); - metronSolrClient.createCollection(collection, 1, 1); - verify(metronSolrClient, times(1)).request(argThat(new CollectionRequestMatcher(CollectionParams.CollectionAction.LIST.name())), (String) isNull()); - verify(metronSolrClient, times(1)).request(argThat(new CollectionRequestMatcher(CollectionParams.CollectionAction.CREATE.name())), (String) isNull()); - } -} diff --git a/metron-platform/metron-solr/metron-solr-common/src/test/java/org/apache/metron/solr/writer/SolrWriterTest.java b/metron-platform/metron-solr/metron-solr-common/src/test/java/org/apache/metron/solr/writer/SolrWriterTest.java index 515f2f8c75..f6ac329caf 100644 --- a/metron-platform/metron-solr/metron-solr-common/src/test/java/org/apache/metron/solr/writer/SolrWriterTest.java +++ b/metron-platform/metron-solr/metron-solr-common/src/test/java/org/apache/metron/solr/writer/SolrWriterTest.java @@ -34,6 +34,7 @@ import org.apache.metron.common.configuration.writer.IndexingWriterConfiguration; import org.apache.metron.common.writer.BulkMessage; import org.apache.metron.enrichment.integration.utils.SampleUtil; +import org.apache.solr.client.solrj.impl.CloudSolrClient; import org.apache.solr.client.solrj.request.QueryRequest; import org.apache.solr.common.SolrInputDocument; import org.hamcrest.Description; @@ -119,8 +120,8 @@ public void testWriter() throws Exception { messages.add(new BulkMessage<>("message2", message2)); String collection = "metron"; - MetronSolrClient solr = Mockito.mock(MetronSolrClient.class); - SolrWriter writer = new SolrWriter().withMetronSolrClient(solr); + CloudSolrClient solr = Mockito.mock(CloudSolrClient.class); + SolrWriter writer = new SolrWriter().withCloudSolrClient(solr); writer.init(null,new IndexingWriterConfiguration("solr", configurations)); verify(solr, times(1)).setDefaultCollection(collection); @@ -128,7 +129,7 @@ public void testWriter() throws Exception { Map globalConfig = configurations.getGlobalConfig(); globalConfig.put("solr.collection", collection); configurations.updateGlobalConfig(globalConfig); - writer = new SolrWriter().withMetronSolrClient(solr); + writer = new SolrWriter().withCloudSolrClient(solr); writer.init(null, new IndexingWriterConfiguration("solr", configurations)); verify(solr, times(1)).setDefaultCollection(collection); diff --git a/metron-platform/metron-solr/metron-solr-storm/src/test/java/org/apache/metron/indexing/integration/SolrIndexingIntegrationTest.java b/metron-platform/metron-solr/metron-solr-storm/src/test/java/org/apache/metron/indexing/integration/SolrIndexingIntegrationTest.java index 2fb48b7721..420cdb20e3 100644 --- a/metron-platform/metron-solr/metron-solr-storm/src/test/java/org/apache/metron/indexing/integration/SolrIndexingIntegrationTest.java +++ b/metron-platform/metron-solr/metron-solr-storm/src/test/java/org/apache/metron/indexing/integration/SolrIndexingIntegrationTest.java @@ -18,6 +18,8 @@ package org.apache.metron.indexing.integration; import com.google.common.base.Function; + +import java.io.IOException; import java.util.List; import java.util.Map; import java.util.Properties; @@ -39,6 +41,7 @@ import org.apache.metron.integration.components.ZKServerComponent; import org.apache.metron.solr.SolrConstants; import org.apache.metron.solr.integration.components.SolrComponent; +import org.apache.solr.client.solrj.SolrServerException; public class SolrIndexingIntegrationTest extends IndexingIntegrationTest { @@ -85,18 +88,22 @@ public Processor>> getProcessor(final List inpu public ReadinessState process(ComponentRunner runner) { SolrComponent solrComponent = runner.getComponent("search", SolrComponent.class); KafkaComponent kafkaComponent = runner.getComponent("kafka", KafkaComponent.class); - if (solrComponent.hasCollection(collection)) { - docs = solrComponent.getAllIndexedDocs(collection); - if (docs.size() < inputMessages.size() ) { - errors = kafkaComponent.readMessages(ERROR_TOPIC); - if(errors.size() > 0 && errors.size() + docs.size() == inputMessages.size()){ + try { + if (solrComponent.hasCollection(collection)) { + docs = solrComponent.getAllIndexedDocs(collection); + if (docs.size() < inputMessages.size() ) { + errors = kafkaComponent.readMessages(ERROR_TOPIC); + if(errors.size() > 0 && errors.size() + docs.size() == inputMessages.size()){ + return ReadinessState.READY; + } + return ReadinessState.NOT_READY; + } else { return ReadinessState.READY; } - return ReadinessState.NOT_READY; } else { - return ReadinessState.READY; + return ReadinessState.NOT_READY; } - } else { + } catch (IOException | SolrServerException e) { return ReadinessState.NOT_READY; } } diff --git a/pom.xml b/pom.xml index 73782ecd5e..3cbd2bc8c0 100644 --- a/pom.xml +++ b/pom.xml @@ -115,7 +115,6 @@ 2.2.5 3.7 1.8 - 6.6.2 1.10.19 1.7.0 3.2.0 @@ -148,6 +147,7 @@ 1.7.25 1.2.1 0.8.0 + 7.4.0 @@ -161,6 +161,7 @@ ${base_storm_version}.${hdp_version}-${build_number} ${base_kafka_version}.${hdp_version}-${build_number} 0.7.3 + 6.6.2