From cb498e7058bae9df11f3c379118000e1072ae390 Mon Sep 17 00:00:00 2001 From: "Goutam.Mandal" Date: Thu, 13 Apr 2017 14:32:52 +0530 Subject: [PATCH 1/5] Added support to specify connection and socket read timeout. --- .../dataimport/SolrEntityProcessor.java | 30 +++++++++++++++---- 1 file changed, 25 insertions(+), 5 deletions(-) diff --git a/solr/contrib/dataimporthandler/src/java/org/apache/solr/handler/dataimport/SolrEntityProcessor.java b/solr/contrib/dataimporthandler/src/java/org/apache/solr/handler/dataimport/SolrEntityProcessor.java index 6d8726f91dcb..22898d7eb82b 100644 --- a/solr/contrib/dataimporthandler/src/java/org/apache/solr/handler/dataimport/SolrEntityProcessor.java +++ b/solr/contrib/dataimporthandler/src/java/org/apache/solr/handler/dataimport/SolrEntityProcessor.java @@ -63,9 +63,14 @@ public class SolrEntityProcessor extends EntityProcessorBase { public static final String SOLR_SERVER = "url"; public static final String QUERY = "query"; - public static final String TIMEOUT = "timeout"; + public static final String TIMEOUT = "timeout"; // kept for backward compatibility + public static final String CONNECTION_TIMEOUT = "connectionTimeout"; + public static final String READ_TIMEOUT = "readTimeout"; + public static final String QUERY_TIMEOUT = "queryTimeout"; - public static final int TIMEOUT_SECS = 5 * 60; // 5 minutes + public static final int CONNECTION_TIMEOUT_SECS = 10; // 10 seconds + public static final int READ_TIMEOUT_SECS = 30; // 30 seconds + public static final int QUERY_TIMEOUT_SECS = 5 * 60; // 5 minutes public static final int ROWS_DEFAULT = 50; private SolrClient solrClient = null; @@ -74,7 +79,9 @@ public class SolrEntityProcessor extends EntityProcessorBase { private String[] filterQueries; private String[] fields; private String requestHandler;// 'qt' param - private int timeout = TIMEOUT_SECS; + private int connectionTimeout = CONNECTION_TIMEOUT_SECS; + private int readTimeout = READ_TIMEOUT_SECS; + private int queryTimeout = QUERY_TIMEOUT_SECS; @Override public void destroy() { @@ -126,6 +133,19 @@ protected void firstInit(Context context) { .build(); LOG.info("using BinaryResponseParser"); } + /* + * Added connection and read timeout for HTTP connection + */ + String connectionTimeoutAsString = context.getResolvedEntityAttribute(CONNECTION_TIMEOUT); + if (connectionTimeoutAsString != null) { + this.connectionTimeout = Integer.parseInt(connectionTimeoutAsString); + } + String readTimeoutAsString = context.getResolvedEntityAttribute(READ_TIMEOUT); + if (readTimeoutAsString != null) { + this.readTimeout = Integer.parseInt(readTimeoutAsString); + } + ((HttpSolrClient)solrClient).setConnectionTimeout(connectionTimeout * 1000); + ((HttpSolrClient)solrClient).setSoTimeout(readTimeout * 1000); } catch (MalformedURLException e) { throw new DataImportHandlerException(DataImportHandlerException.SEVERE, e); } @@ -274,10 +294,10 @@ protected Iterator> createNextPageIterator(QueryResponse resp protected void passNextPage(SolrQuery solrQuery) { String timeoutAsString = context.getResolvedEntityAttribute(TIMEOUT); if (timeoutAsString != null) { - SolrEntityProcessor.this.timeout = Integer.parseInt(timeoutAsString); + SolrEntityProcessor.this.queryTimeout = Integer.parseInt(timeoutAsString); } - solrQuery.setTimeAllowed(timeout * 1000); + solrQuery.setTimeAllowed(queryTimeout * 1000); solrQuery.setStart(getStart() + getSize()); } From f8b02ca1ddee628f875f5a566e51aa75de8e89bc Mon Sep 17 00:00:00 2001 From: "Goutam.Mandal" Date: Thu, 13 Apr 2017 14:36:14 +0530 Subject: [PATCH 2/5] Revert "Added support to specify connection and socket read timeout." This reverts commit cb498e7058bae9df11f3c379118000e1072ae390. --- .../dataimport/SolrEntityProcessor.java | 30 ++++--------------- 1 file changed, 5 insertions(+), 25 deletions(-) diff --git a/solr/contrib/dataimporthandler/src/java/org/apache/solr/handler/dataimport/SolrEntityProcessor.java b/solr/contrib/dataimporthandler/src/java/org/apache/solr/handler/dataimport/SolrEntityProcessor.java index 22898d7eb82b..6d8726f91dcb 100644 --- a/solr/contrib/dataimporthandler/src/java/org/apache/solr/handler/dataimport/SolrEntityProcessor.java +++ b/solr/contrib/dataimporthandler/src/java/org/apache/solr/handler/dataimport/SolrEntityProcessor.java @@ -63,14 +63,9 @@ public class SolrEntityProcessor extends EntityProcessorBase { public static final String SOLR_SERVER = "url"; public static final String QUERY = "query"; - public static final String TIMEOUT = "timeout"; // kept for backward compatibility - public static final String CONNECTION_TIMEOUT = "connectionTimeout"; - public static final String READ_TIMEOUT = "readTimeout"; - public static final String QUERY_TIMEOUT = "queryTimeout"; + public static final String TIMEOUT = "timeout"; - public static final int CONNECTION_TIMEOUT_SECS = 10; // 10 seconds - public static final int READ_TIMEOUT_SECS = 30; // 30 seconds - public static final int QUERY_TIMEOUT_SECS = 5 * 60; // 5 minutes + public static final int TIMEOUT_SECS = 5 * 60; // 5 minutes public static final int ROWS_DEFAULT = 50; private SolrClient solrClient = null; @@ -79,9 +74,7 @@ public class SolrEntityProcessor extends EntityProcessorBase { private String[] filterQueries; private String[] fields; private String requestHandler;// 'qt' param - private int connectionTimeout = CONNECTION_TIMEOUT_SECS; - private int readTimeout = READ_TIMEOUT_SECS; - private int queryTimeout = QUERY_TIMEOUT_SECS; + private int timeout = TIMEOUT_SECS; @Override public void destroy() { @@ -133,19 +126,6 @@ protected void firstInit(Context context) { .build(); LOG.info("using BinaryResponseParser"); } - /* - * Added connection and read timeout for HTTP connection - */ - String connectionTimeoutAsString = context.getResolvedEntityAttribute(CONNECTION_TIMEOUT); - if (connectionTimeoutAsString != null) { - this.connectionTimeout = Integer.parseInt(connectionTimeoutAsString); - } - String readTimeoutAsString = context.getResolvedEntityAttribute(READ_TIMEOUT); - if (readTimeoutAsString != null) { - this.readTimeout = Integer.parseInt(readTimeoutAsString); - } - ((HttpSolrClient)solrClient).setConnectionTimeout(connectionTimeout * 1000); - ((HttpSolrClient)solrClient).setSoTimeout(readTimeout * 1000); } catch (MalformedURLException e) { throw new DataImportHandlerException(DataImportHandlerException.SEVERE, e); } @@ -294,10 +274,10 @@ protected Iterator> createNextPageIterator(QueryResponse resp protected void passNextPage(SolrQuery solrQuery) { String timeoutAsString = context.getResolvedEntityAttribute(TIMEOUT); if (timeoutAsString != null) { - SolrEntityProcessor.this.queryTimeout = Integer.parseInt(timeoutAsString); + SolrEntityProcessor.this.timeout = Integer.parseInt(timeoutAsString); } - solrQuery.setTimeAllowed(queryTimeout * 1000); + solrQuery.setTimeAllowed(timeout * 1000); solrQuery.setStart(getStart() + getSize()); } From d4bca6ec5a0814080b21d1e3c4b2301938c99176 Mon Sep 17 00:00:00 2001 From: gmandal Date: Thu, 13 Apr 2017 14:44:51 +0530 Subject: [PATCH 3/5] Added support to specify connection and socket read timeout in DataImportHandler for SolrEntityProcessor --- .../dataimport/SolrEntityProcessor.java | 30 +++++++++++++++---- 1 file changed, 25 insertions(+), 5 deletions(-) diff --git a/solr/contrib/dataimporthandler/src/java/org/apache/solr/handler/dataimport/SolrEntityProcessor.java b/solr/contrib/dataimporthandler/src/java/org/apache/solr/handler/dataimport/SolrEntityProcessor.java index 6d8726f91dcb..22898d7eb82b 100644 --- a/solr/contrib/dataimporthandler/src/java/org/apache/solr/handler/dataimport/SolrEntityProcessor.java +++ b/solr/contrib/dataimporthandler/src/java/org/apache/solr/handler/dataimport/SolrEntityProcessor.java @@ -63,9 +63,14 @@ public class SolrEntityProcessor extends EntityProcessorBase { public static final String SOLR_SERVER = "url"; public static final String QUERY = "query"; - public static final String TIMEOUT = "timeout"; + public static final String TIMEOUT = "timeout"; // kept for backward compatibility + public static final String CONNECTION_TIMEOUT = "connectionTimeout"; + public static final String READ_TIMEOUT = "readTimeout"; + public static final String QUERY_TIMEOUT = "queryTimeout"; - public static final int TIMEOUT_SECS = 5 * 60; // 5 minutes + public static final int CONNECTION_TIMEOUT_SECS = 10; // 10 seconds + public static final int READ_TIMEOUT_SECS = 30; // 30 seconds + public static final int QUERY_TIMEOUT_SECS = 5 * 60; // 5 minutes public static final int ROWS_DEFAULT = 50; private SolrClient solrClient = null; @@ -74,7 +79,9 @@ public class SolrEntityProcessor extends EntityProcessorBase { private String[] filterQueries; private String[] fields; private String requestHandler;// 'qt' param - private int timeout = TIMEOUT_SECS; + private int connectionTimeout = CONNECTION_TIMEOUT_SECS; + private int readTimeout = READ_TIMEOUT_SECS; + private int queryTimeout = QUERY_TIMEOUT_SECS; @Override public void destroy() { @@ -126,6 +133,19 @@ protected void firstInit(Context context) { .build(); LOG.info("using BinaryResponseParser"); } + /* + * Added connection and read timeout for HTTP connection + */ + String connectionTimeoutAsString = context.getResolvedEntityAttribute(CONNECTION_TIMEOUT); + if (connectionTimeoutAsString != null) { + this.connectionTimeout = Integer.parseInt(connectionTimeoutAsString); + } + String readTimeoutAsString = context.getResolvedEntityAttribute(READ_TIMEOUT); + if (readTimeoutAsString != null) { + this.readTimeout = Integer.parseInt(readTimeoutAsString); + } + ((HttpSolrClient)solrClient).setConnectionTimeout(connectionTimeout * 1000); + ((HttpSolrClient)solrClient).setSoTimeout(readTimeout * 1000); } catch (MalformedURLException e) { throw new DataImportHandlerException(DataImportHandlerException.SEVERE, e); } @@ -274,10 +294,10 @@ protected Iterator> createNextPageIterator(QueryResponse resp protected void passNextPage(SolrQuery solrQuery) { String timeoutAsString = context.getResolvedEntityAttribute(TIMEOUT); if (timeoutAsString != null) { - SolrEntityProcessor.this.timeout = Integer.parseInt(timeoutAsString); + SolrEntityProcessor.this.queryTimeout = Integer.parseInt(timeoutAsString); } - solrQuery.setTimeAllowed(timeout * 1000); + solrQuery.setTimeAllowed(queryTimeout * 1000); solrQuery.setStart(getStart() + getSize()); } From 0e9a25901b1e598e718ce6e0375db9713b7e4f2c Mon Sep 17 00:00:00 2001 From: gmandal Date: Thu, 13 Apr 2017 14:54:27 +0530 Subject: [PATCH 4/5] SOLR-10487: Added support to specify connection and socket read timeout in DataImportHandler for SolrEntityProcessor --- .../dataimport/SolrEntityProcessor.java | 30 +++++++++++++++---- 1 file changed, 25 insertions(+), 5 deletions(-) diff --git a/solr/contrib/dataimporthandler/src/java/org/apache/solr/handler/dataimport/SolrEntityProcessor.java b/solr/contrib/dataimporthandler/src/java/org/apache/solr/handler/dataimport/SolrEntityProcessor.java index 6d8726f91dcb..22898d7eb82b 100644 --- a/solr/contrib/dataimporthandler/src/java/org/apache/solr/handler/dataimport/SolrEntityProcessor.java +++ b/solr/contrib/dataimporthandler/src/java/org/apache/solr/handler/dataimport/SolrEntityProcessor.java @@ -63,9 +63,14 @@ public class SolrEntityProcessor extends EntityProcessorBase { public static final String SOLR_SERVER = "url"; public static final String QUERY = "query"; - public static final String TIMEOUT = "timeout"; + public static final String TIMEOUT = "timeout"; // kept for backward compatibility + public static final String CONNECTION_TIMEOUT = "connectionTimeout"; + public static final String READ_TIMEOUT = "readTimeout"; + public static final String QUERY_TIMEOUT = "queryTimeout"; - public static final int TIMEOUT_SECS = 5 * 60; // 5 minutes + public static final int CONNECTION_TIMEOUT_SECS = 10; // 10 seconds + public static final int READ_TIMEOUT_SECS = 30; // 30 seconds + public static final int QUERY_TIMEOUT_SECS = 5 * 60; // 5 minutes public static final int ROWS_DEFAULT = 50; private SolrClient solrClient = null; @@ -74,7 +79,9 @@ public class SolrEntityProcessor extends EntityProcessorBase { private String[] filterQueries; private String[] fields; private String requestHandler;// 'qt' param - private int timeout = TIMEOUT_SECS; + private int connectionTimeout = CONNECTION_TIMEOUT_SECS; + private int readTimeout = READ_TIMEOUT_SECS; + private int queryTimeout = QUERY_TIMEOUT_SECS; @Override public void destroy() { @@ -126,6 +133,19 @@ protected void firstInit(Context context) { .build(); LOG.info("using BinaryResponseParser"); } + /* + * Added connection and read timeout for HTTP connection + */ + String connectionTimeoutAsString = context.getResolvedEntityAttribute(CONNECTION_TIMEOUT); + if (connectionTimeoutAsString != null) { + this.connectionTimeout = Integer.parseInt(connectionTimeoutAsString); + } + String readTimeoutAsString = context.getResolvedEntityAttribute(READ_TIMEOUT); + if (readTimeoutAsString != null) { + this.readTimeout = Integer.parseInt(readTimeoutAsString); + } + ((HttpSolrClient)solrClient).setConnectionTimeout(connectionTimeout * 1000); + ((HttpSolrClient)solrClient).setSoTimeout(readTimeout * 1000); } catch (MalformedURLException e) { throw new DataImportHandlerException(DataImportHandlerException.SEVERE, e); } @@ -274,10 +294,10 @@ protected Iterator> createNextPageIterator(QueryResponse resp protected void passNextPage(SolrQuery solrQuery) { String timeoutAsString = context.getResolvedEntityAttribute(TIMEOUT); if (timeoutAsString != null) { - SolrEntityProcessor.this.timeout = Integer.parseInt(timeoutAsString); + SolrEntityProcessor.this.queryTimeout = Integer.parseInt(timeoutAsString); } - solrQuery.setTimeAllowed(timeout * 1000); + solrQuery.setTimeAllowed(queryTimeout * 1000); solrQuery.setStart(getStart() + getSize()); } From debcfd6683f3f3d54cdfb881e4f573ad913cdc05 Mon Sep 17 00:00:00 2001 From: "Goutam.Mandal" Date: Thu, 13 Apr 2017 14:32:52 +0530 Subject: [PATCH 5/5] SOLR-10487: Added support to specify connection and socket read timeout in DataImportHandler for SolrEntityProcessor. --- .../dataimport/SolrEntityProcessor.java | 30 +++++++++++++++---- 1 file changed, 25 insertions(+), 5 deletions(-) diff --git a/solr/contrib/dataimporthandler/src/java/org/apache/solr/handler/dataimport/SolrEntityProcessor.java b/solr/contrib/dataimporthandler/src/java/org/apache/solr/handler/dataimport/SolrEntityProcessor.java index 6d8726f91dcb..22898d7eb82b 100644 --- a/solr/contrib/dataimporthandler/src/java/org/apache/solr/handler/dataimport/SolrEntityProcessor.java +++ b/solr/contrib/dataimporthandler/src/java/org/apache/solr/handler/dataimport/SolrEntityProcessor.java @@ -63,9 +63,14 @@ public class SolrEntityProcessor extends EntityProcessorBase { public static final String SOLR_SERVER = "url"; public static final String QUERY = "query"; - public static final String TIMEOUT = "timeout"; + public static final String TIMEOUT = "timeout"; // kept for backward compatibility + public static final String CONNECTION_TIMEOUT = "connectionTimeout"; + public static final String READ_TIMEOUT = "readTimeout"; + public static final String QUERY_TIMEOUT = "queryTimeout"; - public static final int TIMEOUT_SECS = 5 * 60; // 5 minutes + public static final int CONNECTION_TIMEOUT_SECS = 10; // 10 seconds + public static final int READ_TIMEOUT_SECS = 30; // 30 seconds + public static final int QUERY_TIMEOUT_SECS = 5 * 60; // 5 minutes public static final int ROWS_DEFAULT = 50; private SolrClient solrClient = null; @@ -74,7 +79,9 @@ public class SolrEntityProcessor extends EntityProcessorBase { private String[] filterQueries; private String[] fields; private String requestHandler;// 'qt' param - private int timeout = TIMEOUT_SECS; + private int connectionTimeout = CONNECTION_TIMEOUT_SECS; + private int readTimeout = READ_TIMEOUT_SECS; + private int queryTimeout = QUERY_TIMEOUT_SECS; @Override public void destroy() { @@ -126,6 +133,19 @@ protected void firstInit(Context context) { .build(); LOG.info("using BinaryResponseParser"); } + /* + * Added connection and read timeout for HTTP connection + */ + String connectionTimeoutAsString = context.getResolvedEntityAttribute(CONNECTION_TIMEOUT); + if (connectionTimeoutAsString != null) { + this.connectionTimeout = Integer.parseInt(connectionTimeoutAsString); + } + String readTimeoutAsString = context.getResolvedEntityAttribute(READ_TIMEOUT); + if (readTimeoutAsString != null) { + this.readTimeout = Integer.parseInt(readTimeoutAsString); + } + ((HttpSolrClient)solrClient).setConnectionTimeout(connectionTimeout * 1000); + ((HttpSolrClient)solrClient).setSoTimeout(readTimeout * 1000); } catch (MalformedURLException e) { throw new DataImportHandlerException(DataImportHandlerException.SEVERE, e); } @@ -274,10 +294,10 @@ protected Iterator> createNextPageIterator(QueryResponse resp protected void passNextPage(SolrQuery solrQuery) { String timeoutAsString = context.getResolvedEntityAttribute(TIMEOUT); if (timeoutAsString != null) { - SolrEntityProcessor.this.timeout = Integer.parseInt(timeoutAsString); + SolrEntityProcessor.this.queryTimeout = Integer.parseInt(timeoutAsString); } - solrQuery.setTimeAllowed(timeout * 1000); + solrQuery.setTimeAllowed(queryTimeout * 1000); solrQuery.setStart(getStart() + getSize()); }