Speedup for build_models (#173)

Dockerfile.test

@@ -1,7 +1,7 @@
 FROM python:3.8.5-slim-buster
 
 RUN apt-get -y update && apt-get -y upgrade && \
-    apt-get install --no-install-recommends -y wait-for-it vim gcc libpq-dev libc-dev postgresql-client redis-tools
+    apt-get install --no-install-recommends -y wait-for-it vim gcc libpq-dev libc-dev postgresql-client redis-tools curl
 
 WORKDIR /app
 ENV PYTHONPATH /app/

datastore/shared/postgresql_backend/sql_read_database_backend_service.py

@@ -177,21 +177,23 @@ def build_models_from_result(
         self, result, mapped_fields_per_fqid: Dict[str, List[str]]
     ) -> Dict[str, Model]:
         result_map = {}
+        has_empty_entries = self.query_helper.mapped_fields_map_has_empty_entry(
+            mapped_fields_per_fqid
+        )
         for row in result:
             fqid = row["fqid"]
 
-            if self.query_helper.mapped_fields_map_has_empty_entry(
-                mapped_fields_per_fqid
-            ):
-                # at least one collection needs all fields, so we just selected data
-                model = row["data"]
-            else:
-                model = row.copy()
+            if has_empty_entries:
+                # at least one collection needs all fields, so we need to select data
+                row = row["data"]
 
             if fqid in mapped_fields_per_fqid and len(mapped_fields_per_fqid[fqid]) > 0:
-                for key in list(model.keys()):
-                    if key not in mapped_fields_per_fqid[fqid] or model[key] is None:
-                        del model[key]
+                model = {}
+                for field in mapped_fields_per_fqid[fqid]:
+                    if row.get(field) is not None:
+                        model[field] = row[field]
+            else:
+                model = row
             result_map[fqid] = model
 
         return result_map

tests/shared/unit/postgresql_backend/test_sql_read_database_backend_service.py

@@ -186,19 +186,10 @@ def test_fetch_models_mapped_fields(
 def test_build_models_from_result(
     read_database: ReadDatabase, query_helper: SqlQueryHelper
 ):
-    row = MagicMock()
-    row["fqid"] = MagicMock()
-    row.copy = lambda: row
-    row.keys = MagicMock(return_value=[MagicMock()])
-    mfpc = MagicMock()
-    mfpc.__getitem__ = MagicMock(return_value=[MagicMock()])
-    mfpc.__contains__ = MagicMock(return_value=True)
-
-    query_helper.mapped_fields_map_has_empty_entry = MagicMock(return_value=False)
-
+    row = {"fqid": "a/1", "field": "a"}
+    mfpc = {"a/1": ["field"]}
     result = read_database.build_models_from_result([row], mfpc)
-
-    assert result == {row["fqid"]: row}
+    assert result == {row["fqid"]: {"field": "a"}}
 
 
 def test_build_model_ignore_deleted(