Merge pull request #932 from dmlond/DDS-811_batch_retries-prod

[Production] DDS-811 batch retries

lib/tasks/elasticsearch.rake

@@ -33,8 +33,27 @@ def index_documents
             data: f.__elasticsearch__.as_indexed_json }
           }
       }
-      Elasticsearch::Model.client.bulk body: current_batch
-      $stderr.print "+" * current_batch.length
+      trys = 0
+      error_ids = []
+      while trys < 5
+        bulk_response = Elasticsearch::Model.client.bulk body: current_batch
+        if bulk_response["errors"]
+          trys += 1
+          error_ids = bulk_response["items"].select {|item|
+            item["index"]["status"] >= 400
+          }.map {|i|
+            i["index"]["_id"]
+          }
+          current_batch = current_batch.select {|b| error_ids.include? b[:index][:_id] }
+        else
+          trys = 5
+        end
+      end
+      unless error_ids.empty?
+        $stderr.puts "page #{page_num} Ids Not Loaded after #{trys} tries:"
+        $stderr.puts error_ids.join(',')
+      end
+      $stderr.print "+" * (current_batch.length - error_ids.length)
     end
   end
 end