Skip to content

Commit

Permalink
NIFI-7294 Address deprecation issues in solrj and httpclient
Browse files Browse the repository at this point in the history 
Some calls to deprecated methods in httpclient were resulting in
UnsupportedOperationException. Use the new API calls in both httpclient
and solrj. Add an integration test to include test coverage for
org.apache.nifi.processors.solr.SolrUtils.createClient

This closes apache#4171.
  • Loading branch information
@mkazia @jfrazee
mkazia authored and jfrazee committed Apr 1, 2020
1 parent 1c2ba91 commit b28659d
Show file tree
Hide file tree
Showing 6 changed files with 121 additions and 22 deletions.
39 changes: 26 additions & 13 deletions ...-bundle/nifi-solr-processors/src/main/java/org/apache/nifi/processors/solr/SolrUtils.java
View file
Expand Up @@ -21,8 +21,11 @@
import org.apache.commons.io.IOUtils;
import org.apache.commons.lang3.StringUtils;
import org.apache.http.client.HttpClient;
import org.apache.http.conn.scheme.Scheme;
import org.apache.http.conn.ssl.SSLSocketFactory;
import org.apache.http.config.Registry;
import org.apache.http.config.RegistryBuilder;
import org.apache.http.conn.ssl.SSLConnectionSocketFactory;
import org.apache.http.conn.socket.ConnectionSocketFactory;
import org.apache.http.conn.socket.PlainConnectionSocketFactory;
import org.apache.nifi.components.AllowableValue;
import org.apache.nifi.components.PropertyDescriptor;
import org.apache.nifi.context.PropertyContext;
Expand Down Expand Up @@ -230,7 +233,7 @@ public class SolrUtils {

public static final String REPEATING_PARAM_PATTERN = "[\\w\\.]+\\.\\d+$";

public static SolrClient createSolrClient(final PropertyContext context, final String solrLocation) {
public static synchronized SolrClient createSolrClient(final PropertyContext context, final String solrLocation) {
final Integer socketTimeout = context.getProperty(SOLR_SOCKET_TIMEOUT).asTimePeriod(TimeUnit.MILLISECONDS).intValue();
final Integer connectionTimeout = context.getProperty(SOLR_CONNECTION_TIMEOUT).asTimePeriod(TimeUnit.MILLISECONDS).intValue();
final Integer maxConnections = context.getProperty(SOLR_MAX_CONNECTIONS).asInteger();
Expand All @@ -240,26 +243,36 @@ public static SolrClient createSolrClient(final PropertyContext context, final S
final String kerberosPrincipal = context.getProperty(KERBEROS_PRINCIPAL).evaluateAttributeExpressions().getValue();
final String kerberosPassword = context.getProperty(KERBEROS_PASSWORD).getValue();

final ModifiableSolrParams params = new ModifiableSolrParams();
params.set(HttpClientUtil.PROP_SO_TIMEOUT, socketTimeout);
params.set(HttpClientUtil.PROP_CONNECTION_TIMEOUT, connectionTimeout);
params.set(HttpClientUtil.PROP_MAX_CONNECTIONS, maxConnections);
params.set(HttpClientUtil.PROP_MAX_CONNECTIONS_PER_HOST, maxConnectionsPerHost);
// Reset HttpClientBuilder static values
HttpClientUtil.resetHttpClientBuilder();

// has to happen before the client is created below so that correct configurer would be set if needed
if (kerberosCredentialsService != null || (!StringUtils.isBlank(kerberosPrincipal) && !StringUtils.isBlank(kerberosPassword))) {
HttpClientUtil.setHttpClientBuilder(new KerberosHttpClientBuilder().getHttpClientBuilder(Optional.empty()));
}

final HttpClient httpClient = HttpClientUtil.createClient(params);

if (sslContextService != null) {
final SSLContext sslContext = sslContextService.createSSLContext(SSLContextService.ClientAuth.REQUIRED);
final SSLSocketFactory sslSocketFactory = new SSLSocketFactory(sslContext);
final Scheme httpsScheme = new Scheme("https", 443, sslSocketFactory);
httpClient.getConnectionManager().getSchemeRegistry().register(httpsScheme);
final SSLConnectionSocketFactory sslSocketFactory = new SSLConnectionSocketFactory(sslContext);
HttpClientUtil.setSchemaRegistryProvider(new HttpClientUtil.SchemaRegistryProvider() {
@Override
public Registry<ConnectionSocketFactory> getSchemaRegistry() {
RegistryBuilder<ConnectionSocketFactory> builder = RegistryBuilder.create();
builder.register("http", PlainConnectionSocketFactory.getSocketFactory());
builder.register("https", sslSocketFactory);
return builder.build();
}
});
}

final ModifiableSolrParams params = new ModifiableSolrParams();
params.set(HttpClientUtil.PROP_SO_TIMEOUT, socketTimeout);
params.set(HttpClientUtil.PROP_CONNECTION_TIMEOUT, connectionTimeout);
params.set(HttpClientUtil.PROP_MAX_CONNECTIONS, maxConnections);
params.set(HttpClientUtil.PROP_MAX_CONNECTIONS_PER_HOST, maxConnectionsPerHost);

final HttpClient httpClient = HttpClientUtil.createClient(params);

if (SOLR_TYPE_STANDARD.getValue().equals(context.getProperty(SOLR_TYPE).getValue())) {
return new HttpSolrClient.Builder(solrLocation).withHttpClient(httpClient).build();
} else {
Expand Down
92 changes: 89 additions & 3 deletions ...undle/nifi-solr-processors/src/test/java/org/apache/nifi/processors/solr/QuerySolrIT.java
View file
Expand Up @@ -20,10 +20,13 @@
package org.apache.nifi.processors.solr;

import com.google.gson.stream.JsonReader;
import org.apache.nifi.controller.AbstractControllerService;
import org.apache.nifi.json.JsonRecordSetWriter;
import org.apache.nifi.processor.ProcessContext;
import org.apache.nifi.processor.exception.ProcessException;
import org.apache.nifi.reporting.InitializationException;
import org.apache.nifi.schema.access.SchemaAccessUtils;
import org.apache.nifi.ssl.SSLContextService;
import org.apache.nifi.util.MockFlowFile;
import org.apache.nifi.util.TestRunner;
import org.apache.nifi.util.TestRunners;
Expand All @@ -36,25 +39,29 @@
import org.junit.AfterClass;
import org.junit.BeforeClass;
import org.junit.Test;
import org.mockito.Mockito;
import org.xmlunit.matchers.CompareMatcher;

import javax.net.ssl.SSLContext;
import java.io.ByteArrayInputStream;
import java.io.IOException;
import java.io.InputStreamReader;
import java.nio.file.Files;
import java.nio.file.Path;
import java.nio.file.Paths;
import java.text.SimpleDateFormat;
import java.util.Collections;
import java.util.Date;
import java.util.HashMap;
import java.util.List;
import java.util.Locale;
import java.util.Map;
import java.util.Optional;
import java.util.TimeZone;

import static org.hamcrest.MatcherAssert.assertThat;
import static org.junit.Assert.assertEquals;
import static org.junit.Assert.assertFalse;
import static org.junit.Assert.assertThat;

public class QuerySolrIT {
/*
Expand Down Expand Up @@ -117,7 +124,7 @@ public static CloudSolrClient createSolrClient() {
CloudSolrClient solrClient = null;

try {
solrClient = new CloudSolrClient.Builder().withZkHost(SOLR_LOCATION).build();
solrClient = new CloudSolrClient.Builder(Collections.singletonList(SOLR_LOCATION), Optional.empty()).build();
solrClient.setDefaultCollection(SOLR_COLLECTION);
} catch (Exception e) {
e.printStackTrace();
Expand All @@ -141,7 +148,7 @@ private TestRunner createRunnerWithSolrClient(SolrClient solrClient) {

TestRunner runner = TestRunners.newTestRunner(proc);
runner.setProperty(SolrUtils.SOLR_TYPE, SolrUtils.SOLR_TYPE_CLOUD.getValue());
runner.setProperty(SolrUtils.SOLR_LOCATION, "localhost:2181");
runner.setProperty(SolrUtils.SOLR_LOCATION, SOLR_LOCATION);
runner.setProperty(SolrUtils.COLLECTION, SOLR_COLLECTION);

return runner;
Expand Down Expand Up @@ -627,6 +634,24 @@ public void testRecordResponse() throws IOException, InitializationException {
assertEquals(controlScore, 45);
}

@Test
public void testSslContextService() throws IOException, InitializationException {
final QuerySolr proc = Mockito.mock(QuerySolr.class);
TestRunner runner = TestRunners.newTestRunner(proc);
runner.setProperty(SolrUtils.SOLR_TYPE, SolrUtils.SOLR_TYPE_CLOUD.getValue());
runner.setProperty(SolrUtils.SOLR_LOCATION, SOLR_LOCATION);
runner.setProperty(SolrUtils.COLLECTION, SOLR_COLLECTION);

final SSLContextService sslContextService = new MockSSLContextService();
runner.addControllerService("ssl-context", sslContextService);
runner.enableControllerService(sslContextService);

runner.setProperty(SolrUtils.SSL_CONTEXT_SERVICE, "ssl-context");
proc.onScheduled(runner.getProcessContext());
Mockito.verify(proc, Mockito.times(1)).createSolrClient(Mockito.any(ProcessContext.class), Mockito.eq((String)SOLR_LOCATION));

}

// Override createSolrClient and return the passed in SolrClient
private class TestableProcessor extends QuerySolr {
private SolrClient solrClient;
Expand All @@ -639,4 +664,65 @@ protected SolrClient createSolrClient(ProcessContext context, String solrLocatio
return solrClient;
}
}

/**
* Mock implementation so we don't need to have a real keystore/truststore available for testing.
*/
private class MockSSLContextService extends AbstractControllerService implements SSLContextService {

@Override
public SSLContext createSSLContext(ClientAuth clientAuth) throws ProcessException {
return null;
}

@Override
public String getTrustStoreFile() {
return null;
}

@Override
public String getTrustStoreType() {
return null;
}

@Override
public String getTrustStorePassword() {
return null;
}

@Override
public boolean isTrustStoreConfigured() {
return false;
}

@Override
public String getKeyStoreFile() {
return null;
}

@Override
public String getKeyStoreType() {
return null;
}

@Override
public String getKeyStorePassword() {
return null;
}

@Override
public String getKeyPassword() {
return null;
}

@Override
public boolean isKeyStoreConfigured() {
return false;
}

@Override
public String getSslAlgorithm() {
return null;
}
}
}
2 changes: 1 addition & 1 deletion ...undle/nifi-solr-processors/src/test/java/org/apache/nifi/processors/solr/TestGetSolr.java
View file
Expand Up @@ -48,8 +48,8 @@
import java.util.Locale;
import java.util.TimeZone;

import static org.hamcrest.MatcherAssert.assertThat;
import static org.junit.Assert.assertEquals;
import static org.junit.Assert.assertThat;

public class TestGetSolr {

Expand Down
4 changes: 2 additions & 2 deletions ...lr-processors/src/test/java/org/apache/nifi/processors/solr/TestPutSolrContentStream.java
View file
Expand Up @@ -30,7 +30,7 @@
import org.apache.solr.client.solrj.SolrQuery;
import org.apache.solr.client.solrj.SolrRequest;
import org.apache.solr.client.solrj.SolrServerException;
import org.apache.solr.client.solrj.impl.HttpSolrClient;
import org.apache.solr.client.solrj.impl.BaseHttpSolrClient;
import org.apache.solr.client.solrj.response.QueryResponse;
import org.apache.solr.common.SolrDocument;
import org.apache.solr.common.SolrException;
Expand Down Expand Up @@ -318,7 +318,7 @@ public void testSolrExceptionShouldRouteToFailure() throws IOException, SolrServ

@Test
public void testRemoteSolrExceptionShouldRouteToFailure() throws IOException, SolrServerException {
final Throwable throwable = new HttpSolrClient.RemoteSolrException(
final Throwable throwable = new BaseHttpSolrClient.RemoteSolrException(
"host", 401, "error", new NumberFormatException());
final ExceptionThrowingProcessor proc = new ExceptionThrowingProcessor(throwable);

Expand Down
4 changes: 2 additions & 2 deletions ...nifi-solr-processors/src/test/java/org/apache/nifi/processors/solr/TestPutSolrRecord.java
View file
Expand Up @@ -36,7 +36,7 @@
import org.apache.solr.client.solrj.SolrQuery;
import org.apache.solr.client.solrj.SolrRequest;
import org.apache.solr.client.solrj.SolrServerException;
import org.apache.solr.client.solrj.impl.HttpSolrClient;
import org.apache.solr.client.solrj.impl.BaseHttpSolrClient;
import org.apache.solr.client.solrj.response.QueryResponse;
import org.apache.solr.common.SolrDocument;
import org.apache.solr.common.SolrException;
Expand Down Expand Up @@ -440,7 +440,7 @@ public void testSolrExceptionShouldRouteToFailure() throws IOException, SolrServ

@Test
public void testRemoteSolrExceptionShouldRouteToFailure() throws IOException, SolrServerException, InitializationException {
final Throwable throwable = new HttpSolrClient.RemoteSolrException(
final Throwable throwable = new BaseHttpSolrClient.RemoteSolrException(
"host", 401, "error", new NumberFormatException());
final ExceptionThrowingProcessor proc = new ExceptionThrowingProcessor(throwable);

Expand Down
2 changes: 1 addition & 1 deletion ...dle/nifi-solr-processors/src/test/java/org/apache/nifi/processors/solr/TestQuerySolr.java
View file
Expand Up @@ -48,9 +48,9 @@
import java.util.Map;
import java.util.TimeZone;

import static org.hamcrest.MatcherAssert.assertThat;
import static org.junit.Assert.assertEquals;
import static org.junit.Assert.assertFalse;
import static org.junit.Assert.assertThat;

public class TestQuerySolr {
static final String DEFAULT_SOLR_CORE = "testCollection";
Expand Down

0 comments on commit b28659d

Please sign in to comment.