Add retries for failed ES queries.

I hope this will make trees fail less often due to transient network issues. Examples: https://jenkins-dxr.mozilla.org/job/mozilla-aurora/116/console and https://jenkins-dxr.mozilla.org/job/mozilla-beta/92/console.

docs/source/configuration.rst

@@ -67,6 +67,10 @@ containing the config file.
     The number of seconds DXR should wait for elasticsearch responses during
     indexing. Default: 60
 
+``es_indexing_retries``
+    How many other ES nodes to try if a query to one during indexing times out
+    or the connection fails. This is an experimental feature. Default: 0
+
 ``es_refresh_interval``
     The number of seconds between elasticsearch's consolidation passes during
     indexing. Set to -1 to do no refreshes at all, except directly after an

dxr/build.py

@@ -55,7 +55,9 @@ def index_and_deploy_tree(tree, verbose=False):
 
     """
     config = tree.config
-    es = ElasticSearch(config.es_hosts, timeout=config.es_indexing_timeout)
+    es = ElasticSearch(config.es_hosts,
+                       timeout=config.es_indexing_timeout,
+                       max_retries=config.es_indexing_retries)
     index_name = index_tree(tree, es, verbose=verbose)
     if 'index' not in tree.config.skip_stages:
         deploy_tree(tree, es, index_name)

dxr/config.py

@@ -133,8 +133,13 @@ def __init__(self, input, relative_to=None):
                         lambda v: v >= 0,
                         error='"es_indexing_timeout" must be a non-negative '
                               'integer.'),
+                Optional('es_indexing_retries', default=0):
+                    And(Use(int),
+                        lambda v: v >= 0,
+                        error='"es_indexing_retries" must be a non-negative '
+                              'integer.'),
                 Optional('es_refresh_interval', default=60):
-                    Use(int, error='"es_indexing_timeout" must be an integer.')
+                    Use(int, error='"es_refresh_interval" must be an integer.')
             },
             basestring: dict
         })