From f64686bb06cec2e31c9560d7e7e7f050311d62f1 Mon Sep 17 00:00:00 2001 From: Steven Date: Mon, 27 Jun 2016 09:30:52 -0400 Subject: [PATCH 1/2] NUTCH-2267 - Solr and Hadoop JAR mismatch Explicitly pass in an instance of SystemDefaultHttpClient to CloudSolrClient, otherwise SolrJ will use a default implementation of CloseableHttpClient, which is not present in the HttpClient and HttpCore JARs in Hadoop < 2.8 (see https://issues.apache.org/jira/browse/SOLR-7948 and https://issues.apache.org/jira/browse/HADOOP-12767). --- .../src/java/org/apache/nutch/indexwriter/solr/SolrUtils.java | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/src/plugin/indexer-solr/src/java/org/apache/nutch/indexwriter/solr/SolrUtils.java b/src/plugin/indexer-solr/src/java/org/apache/nutch/indexwriter/solr/SolrUtils.java index eec0080621..85a9c4c2ba 100644 --- a/src/plugin/indexer-solr/src/java/org/apache/nutch/indexwriter/solr/SolrUtils.java +++ b/src/plugin/indexer-solr/src/java/org/apache/nutch/indexwriter/solr/SolrUtils.java @@ -22,6 +22,7 @@ import org.slf4j.Logger; import org.slf4j.LoggerFactory; import org.apache.hadoop.mapred.JobConf; +import org.apache.http.impl.client.SystemDefaultHttpClient; import org.apache.solr.client.solrj.SolrClient; import org.apache.solr.client.solrj.impl.HttpSolrClient; import org.apache.solr.client.solrj.impl.CloudSolrClient; @@ -60,7 +61,8 @@ public static ArrayList getSolrClients(JobConf job) throws Malformed } public static CloudSolrClient getCloudSolrClient(String url) throws MalformedURLException { - CloudSolrClient sc = new CloudSolrClient(url.replace('|', ',')); + SystemDefaultHttpClient httpClient = new SystemDefaultHttpClient(); + CloudSolrClient sc = new CloudSolrClient(url.replace('|', ','), httpClient); sc.setParallelUpdates(true); sc.connect(); return sc; From 6c1537b16ba5f3e0fc28fff5a60595e01437900d Mon Sep 17 00:00:00 2001 From: Steven Date: Thu, 30 Jun 2016 07:48:31 -0400 Subject: [PATCH 2/2] Use static HttpClient for all SOLR connections Changed HttpClient to static based on http://hc.apache.org/httpclient-3.x/performance.html and added connection all SolrJ connections. --- .../java/org/apache/nutch/indexwriter/solr/SolrUtils.java | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/src/plugin/indexer-solr/src/java/org/apache/nutch/indexwriter/solr/SolrUtils.java b/src/plugin/indexer-solr/src/java/org/apache/nutch/indexwriter/solr/SolrUtils.java index 85a9c4c2ba..d70bc62af5 100644 --- a/src/plugin/indexer-solr/src/java/org/apache/nutch/indexwriter/solr/SolrUtils.java +++ b/src/plugin/indexer-solr/src/java/org/apache/nutch/indexwriter/solr/SolrUtils.java @@ -32,6 +32,7 @@ public class SolrUtils { public static Logger LOG = LoggerFactory.getLogger(SolrUtils.class); + private static HttpClient HTTP_CLIENT = new SystemDefaultHttpClient(); /** * @@ -52,7 +53,7 @@ public static ArrayList getSolrClients(JobConf job) throws Malformed } } else { for (int i = 0; i < urls.length; i++) { - SolrClient sc = new HttpSolrClient(urls[i]); + SolrClient sc = new HttpSolrClient(urls[i], HTTP_CLIENT); solrClients.add(sc); } } @@ -61,15 +62,14 @@ public static ArrayList getSolrClients(JobConf job) throws Malformed } public static CloudSolrClient getCloudSolrClient(String url) throws MalformedURLException { - SystemDefaultHttpClient httpClient = new SystemDefaultHttpClient(); - CloudSolrClient sc = new CloudSolrClient(url.replace('|', ','), httpClient); + CloudSolrClient sc = new CloudSolrClient(url.replace('|', ','), HTTP_CLIENT); sc.setParallelUpdates(true); sc.connect(); return sc; } public static SolrClient getHttpSolrClient(String url) throws MalformedURLException { - SolrClient sc =new HttpSolrClient(url); + SolrClient sc =new HttpSolrClient(url, HTTP_CLIENT); return sc; }