AB-394 - Bulk get items with a single database query

db/data.py

@@ -334,6 +334,7 @@ def write_high_level(mbid, ll_id, data, build_sha1):
             for model_name, data in json_high.items():
                 write_high_level_item(connection, model_name, model_version, ll_id, version_id, data)
 
+
 def load_low_level(mbid, offset=0):
     """Load lowlevel data with the given mbid as a dictionary.
     If no offset is given, return the first. If an offset is
@@ -359,6 +360,48 @@ def load_low_level(mbid, offset=0):
         return row[0]
 
 
+def load_many_low_level(recordings):
+    """Uses a single query method to collect low-level JSON data for multiple recordings.
+
+    Args:
+        recordings: A list of tuples (mbid, offset).
+
+    Returns:
+
+        {"mbid1": {"offset1": {lowlevel_data},
+                  "offsetn": {lowlevel_data}}
+         ...
+         ...
+
+        "mbidn": {"offset1": {lowlevel_data}}
+        }
+
+    """
+
+    with db.engine.connect() as connection:
+        query = text("""
+            SELECT gid, submission_offset, data
+              FROM (
+            SELECT id, gid,
+        ROW_NUMBER () 
+              OVER (PARTITION BY gid 
+          ORDER BY submitted DESC) - 1 submission_offset
+              FROM lowlevel
+            )   AS partitions
+         LEFT JOIN lowlevel_json 
+             USING (id)
+             WHERE (gid, submission_offset) IN :recordings
+        """)
+
+        result = connection.execute(query, { 'recordings': tuple(recordings) })
+
+        recordings_info = {}
+        for row in result.fetchall():
+            recordings_info.setdefault(str(row['gid']), {})[str(row['submission_offset'])] = row['data']
+
+        return recordings_info
+
+
 def load_high_level(mbid, offset=0):
     """Load high-level data for a given MBID."""
     with db.engine.connect() as connection:

webserver/views/api/v1/core.py

@@ -183,6 +183,7 @@ def get_data_for_multiple_recordings(collect_data):
 
     recording_details = {}
 
+    # Loading low level data with multiple query method
     for recording_id, offset in recordings:
         try:
             recording_details.setdefault(recording_id, {})[offset] = collect_data(recording_id, offset)
@@ -219,9 +220,22 @@ def get_many_lowlevel():
       Takes the form `mbid[:offset];mbid[:offset]`. Offsets are optional, and should
       be >= 0
 
+    :query query_method: *Optional.* Determines which query method should be used
+
+      `single` will use the single-query method. No parameter or another value will revert
+      to the multiple-query method
+
     :resheader Content-Type: *application/json*
     """
-    recording_details = get_data_for_multiple_recordings(db.data.load_low_level)
+    # Query option allowed for testing purposes:
+    if request.args.get("query_method") == "single":
+        # Use single-query method
+        recordings = check_bad_request_for_multiple_recordings()
+        recording_details = db.data.load_many_low_level(recordings)
+        return jsonify(recording_details)    
+    else:
+        # Use multiple-query method
+        recording_details = get_data_for_multiple_recordings(db.data.load_low_level)
     return recording_details