bulk_inference: ensure results are returned in correct order

The `ThreadPoolProcessor.map` method does not guarantee that results are
returned in the correct order. This PR sorts the results correctly.

This behavior was evident only on pypy builds, but the API contract
of not guaranteeing in-order results is present on both cPython and
Pypy.

Author: mhaas

sap/aibus/dar/client/inference_client.py

@@ -2,7 +2,7 @@
 Client API for the Inference microservice.
 """
 from concurrent.futures import ThreadPoolExecutor
-from typing import List, Union
+from typing import List, Union, Tuple
 
 from requests import RequestException
 
@@ -154,12 +154,13 @@ def do_bulk_inference(
         :return: the aggregated ObjectPrediction dictionaries
         """
 
-        def predict_call(work_package):
+        def predict_call(work_package: Tuple[int, list]) -> Tuple[int, list]:
+            work_package_index, objects_list = work_package
             try:
                 response = self.create_inference_request(
-                    model_name, work_package, top_n=top_n, retry=retry
+                    model_name, objects_list, top_n=top_n, retry=retry
                 )
-                return response["predictions"]
+                return (work_package_index, response["predictions"])
             except (DARHTTPException, RequestException) as exc:
                 self.log.warning(
                     "Caught %s during bulk inference. "
@@ -174,20 +175,25 @@ def predict_call(work_package):
                         "labels": None,
                         "_sdk_error": "{}: {}".format(exc.__class__.__name__, str(exc)),
                     }
-                    for inference_object in work_package
+                    for inference_object in objects_list
                 ]
-                return prediction_error
+                return (work_package_index, prediction_error)
 
-        results = []
+        # Because Executor.map may return results out of order, we add an index
+        # to each list of objects and later restore the correct order
+        input_data_indexed = enumerate(split_list(objects, LIMIT_OBJECTS_PER_CALL))
 
+        results_buffer = []
         with ThreadPoolExecutor(max_workers=4) as pool:
-            results_iterator = pool.map(
-                predict_call, split_list(objects, LIMIT_OBJECTS_PER_CALL)
-            )
-
+            results_iterator = pool.map(predict_call, input_data_indexed)
             for predictions in results_iterator:
-                results.extend(predictions)
+                results_buffer.append(predictions)
 
+        # sort by index and remove index
+        results_buffer.sort(key=lambda x: x[0])
+        results = []
+        for result in results_buffer:
+            results.extend(result[1])
         return results
 
     def create_inference_request_with_url(