Merge pull request #919 from untergeek/fix/remote_version

Remote reindex issues addressed

curator/actions.py

@@ -1011,7 +1011,8 @@ def __init__(self, ilo, request_body, refresh=True,
                     'client_key={7} '
                     'aws_key={8} '
                     'aws_secret_key={9} '
-                    'aws_region={10}'.format(
+                    'aws_region={10}'
+                    'skip_version_test=True'.format(
                         rhost,
                         rhttp_auth,
                         remote_url_prefix,
@@ -1039,6 +1040,7 @@ def __init__(self, ilo, request_body, refresh=True,
                         aws_key=remote_aws_key,
                         aws_secret_key=remote_aws_secret_key,
                         aws_region=remote_aws_region,
+                        skip_version_test=True,
                     )
                 except Exception as e:
                     self.loggit.error(

curator/utils.py

@@ -650,6 +650,8 @@ def get_client(**kwargs):
         not work if `hosts` has more than one value.**  It will raise an
         Exception in that case.
     :type master_only: bool
+    :arg skip_version_test: If `True`, skip the version check as part of the 
+        client connection.
     :rtype: :class:`elasticsearch.Elasticsearch`
     """
     if 'url_prefix' in kwargs:
@@ -667,6 +669,10 @@ def get_client(**kwargs):
     kwargs['hosts'] = '127.0.0.1' if not 'hosts' in kwargs else kwargs['hosts']
     kwargs['master_only'] = False if not 'master_only' in kwargs \
         else kwargs['master_only']
+    if 'skip_version_test' in kwargs:
+        skip_version_test = kwargs.pop('skip_version_test')
+    else:
+        skip_version_test = False
     kwargs['use_ssl'] = False if not 'use_ssl' in kwargs else kwargs['use_ssl']
     kwargs['ssl_no_validate'] = False if not 'ssl_no_validate' in kwargs \
         else kwargs['ssl_no_validate']
@@ -737,8 +743,14 @@ def get_client(**kwargs):
             )
     try:
         client = elasticsearch.Elasticsearch(**kwargs)
-        # Verify the version is acceptable.
-        check_version(client)
+        if skip_version_test:
+            logger.warn(
+                'Skipping Elasticsearch version verification. This is '
+                'acceptable for remote reindex operations.'
+            )
+        else:
+            # Verify the version is acceptable.
+            check_version(client)
         # Verify "master_only" status, if applicable
         check_master(client, master_only=master_only)
         return client

docs/Changelog.rst

@@ -11,7 +11,8 @@ production! There `will` be many more changes before 5.0.0 is released.
 **New Features**
 
   * Reindex is here! The new reindex action has a ton of flexibility. You 
-    can even reindex from remote locations!
+    can even reindex from remote locations, so long as the remote cluster is
+    Elasticsearch 1.4 or newer.
 
   * Added the ``period`` filter (#733). This allows you to select indices 
     or snapshots, based on whether they fit within a period of hours, days, 
@@ -35,7 +36,7 @@ production! There `will` be many more changes before 5.0.0 is released.
 
   * Bumped ``click`` (python module) version dependency to 6.7
   * Bumped ``urllib3`` (python module) version dependency to 1.20
-  * Bumped ``elasticsearch`` (python module) version dependency to 5.2
+  * Bumped ``elasticsearch`` (python module) version dependency to 5.3
   * Refactored a ton of code to be cleaner and hopefully more consistent.
 
 **Bug Fixes**

docs/asciidoc/actions.asciidoc

@@ -568,6 +568,17 @@ Optional settings
 TIP: See an example of this action in an <<actionfile,actionfile>>
     <<ex_reindex,here>>.
 
+=== Compatibility
+
+Generally speaking, the Curator should be able to perform a remote reindex from
+any version of Elasticsearch, 1.4 and newer. Strictly speaking, the Reindex API
+in Elasticsearch _is_ able to reindex from older clusters, but Curator cannot be
+used to facilitate this due to Curator's dependency on changes released in 1.4.
+
+However, there is a https://github.com/elastic/elasticsearch/pull/23805[known bug]
+with Elasticsearch 5.3.0 not being able to reindex from remote clusters older
+than 2.0.  The patch is available in Elasticsearch 5.3.1.  Earlier versions of
+Elasticsearch 5.x do not suffer from this bug.
 
 
 [[replicas]]

test/integration/test_reindex.py

@@ -125,7 +125,8 @@ def test_reindex_from_remote(self):
 
         # Build remote client
         try:
-            rclient = curator.get_client(host=rhost, port=rport)
+            rclient = curator.get_client(
+                host=rhost, port=rport, skip_version_test=True)
         except:
             raise SkipTest(
                 'Unable to connect to host at {0}:{1}'.format(rhost, rport))
@@ -201,7 +202,8 @@ def test_reindex_from_remote_no_indices(self):
 
         # Build remote client
         try:
-            rclient = curator.get_client(host=rhost, port=rport)
+            rclient = curator.get_client(
+                host=rhost, port=rport, skip_version_test=True)
         except:
             raise SkipTest(
                 'Unable to connect to host at {0}:{1}'.format(rhost, rport))