handle multiple api calls together

biothings_explorer/api_call_handler.py

@@ -139,7 +139,7 @@ def call_api(self, uri_value_dict, endpoint_name):
             print('This API call returns no results. The URI given is {}, the endpoint given is {}'.format(uri_value_dict, endpoint_name))
             return
         """
-        return (endpoint_name, rsults)
+        return (endpoint_name, results)
 
     def preprocess_json_doc(self, json_doc, endpoint_name):
         """
@@ -189,11 +189,13 @@ def extract_output(self, json_doc, endpoint_name, output_uri, predicate):
         if output_type == 'Entity':
             jsonld_context = self.registry.endpoint_info[endpoint_name]['jsonld_context']
             outputs = json2nquads(json_doc, jsonld_context, output_uri, predicate)
-            if outputs:
-                return (outputs, self.registry.bioentity_info[output_uri]['preferred_name'])
-            else:
-                print("No output could be found from the given json_doc and output type!")
-                return
+            results = []
+            for _output in outputs:
+                if _output:
+                    results.append(_output, self.registry.bioentity_info[output_uri]['preferred_name'])
+                else:
+                    results.append(None)
+            return results
         # if output_type is object, use OpenAPI specs to extract the output
         else:
             response = self.endpoint_info[endpoint_name]['get']['responses']['200']['x-responseValueType']
@@ -241,17 +243,9 @@ def input2output(self, input_type, input_value, endpoint_name, output_type, pred
         rs = (grequests.get(u, params=v) for (u,v) in api_call_params)
         responses = grequests.map(rs)
             #api_call_response = self.call_api(uri_value, endpoint_name)
-        for api_call_response in responses:
-            if api_call_response.status_code == 200:
-                json_doc = self.preprocess_json_doc(api_call_response.json(), endpoint_name)
-            else:
-                print('This API call returns no response. Endpoint name: {}, Input value type: {}. Input value: {}'
-                      .format(endpoint_name, input_type, input_value))
-            # extract the output based on output_type & predicate
-            output_info = self.extract_output(json_doc, endpoint_name, output_type, predicate=predicate)
-            if output_info:
-                final_results.append({'input': (_input_value, self.registry.bioentity_info[input_type]['preferred_name']), 'output': (output_info)})
-            else:
-                print('The API call returns response. But no appropriate output could be extracted. Output type given is : {}. Predicate given is: {}'
-                      .format(output_type, predicate))
+        valid_responses = [self.preprocess_json_doc(api_call_response.json(), endpoint_name) if api_call_response.status_code == 200 else {} for api_call_response in responses]
+        outputs = self.extract_output(valid_responses, endpoint_name, output_type, predicate=predicate)
+        for i in range(len(outputs)):
+            if outputs[i]:
+                final_results.append({'input': (processed_input[i], self.registry.bioentity_info[input_type]['preferred_name']), 'output': (outputs[i])})
         return final_results

biothings_explorer/jsonld_processor.py

@@ -11,7 +11,7 @@
 t = jsonld.JsonLdProcessor()
 
 
-def jsonld2nquads(jsonld_doc, mode='request'):
+def jsonld2nquads(jsonld_doc, mode='batch'):
     """
     Given a JSON-LD annotated document,
     Fetch it's corresponding NQUADs file from JSON-LD playground
@@ -28,17 +28,25 @@ def jsonld2nquads(jsonld_doc, mode='request'):
         JSON-LD annotated document
     """
     # need to skip html escapes
-    if mode == 'request':
+    if mode != 'batch':
         nquads = requests.post('http://jsonld.biothings.io/?action=nquads', data={'doc': json.dumps(jsonld_doc).replace('>', ">").replace(' ', '')})
         if nquads.status_code != 413:
             # remove the log line from the nquads
             nquads = re.sub('Parsed .*second.\n', '', nquads.json()['output'])
             return t.parse_nquads(nquads)
-    elif mode == 'grequest':
+    elif mode == 'batch':
         responses = []
         for _jsonld_doc in jsonld_doc:
-            responses.append(grequest.post('http://jsonld.biothings.io/?action=nquads', data={'doc': json.dumps(jsonld_doc).replace('>', ">").replace(' ', '')}))
-        return grequests.map(iter(responses))
+            responses.append(grequests.post('http://jsonld.biothings.io/?action=nquads', data={'doc': json.dumps(_jsonld_doc).replace('>', ">").replace(' ', '')}))
+        responses = grequests.map(iter(responses))
+        results = []
+        for _response in responses:
+            if _response.status_code != 413:
+                nquads = re.sub('Parsed .*second.\n', '', _response.json()['output'])
+                results.append(t.parse_nquads(nquads))
+            else:
+                results.append(None)
+        return results
 
 def fetchvalue(nquads, object_uri, predicate=None):
     """
@@ -95,7 +103,7 @@ def find_base(d, relation=defaultdict(set)):
             find_base(v, relation=relation)
     return relation
 
-def json2nquads(json_doc, context_file_path, output, predicate=None):
+def json2nquads(json_doc, context_file_path, output_type, predicate=None):
     """
     Given a JSON document, perform the following actions
     1) Find the json-ld context file based on endpoint_name
@@ -114,11 +122,15 @@ def json2nquads(json_doc, context_file_path, output, predicate=None):
         NQUADS predicate, default is None
     """
     context_file = readFile(context_file_path)
-    json_doc.update(context_file)
-    nquads = jsonld2nquads(json_doc)
-    output = fetchvalue(nquads, output, predicate=predicate)
-    if output:
-        outputs = list(set(output))
-        return outputs
-    else:
-        return None
+    for _json_doc in json_doc:
+        _json_doc.update(context_file)
+    nquads = jsonld2nquads(json_doc, mode='batch')
+    results = []
+    for _nquad in nquads:
+        output = fetchvalue(_nquad, output_type, predicate=predicate)
+        if output:
+            outputs = list(set(output))
+            results.append(outputs)
+        else:
+            results.append(None)
+    return results