Change the prod api queries to return an iterator so that we can filt…

…er them rather than keep the whole array in memory

intrahospital_api/apis/prod_api.py

@@ -348,7 +348,6 @@ def __init__(self):
             )
 
     def execute_hospital_query(self, query, params=None):
-        result = []
         with pytds.connect(
             self.hospital_settings["ip_address"],
             self.hospital_settings["database"],
@@ -364,14 +363,13 @@ def execute_hospital_query(self, query, params=None):
                 while True:
                     latest = cur.fetchmany(1000)
                     if latest:
-                        result.extend(latest)
+                        for row in latest:
+                            logger.debug(row)
+                            yield row
                     else:
                         break
-        logger.debug(result)
-        return result
 
     def execute_trust_query(self, query, params=None):
-        result = []
         with pytds.connect(
             self.trust_settings["ip_address"],
             self.trust_settings["database"],
@@ -387,11 +385,11 @@ def execute_trust_query(self, query, params=None):
                 while True:
                     latest = cur.fetchmany(1000)
                     if latest:
-                        result.extend(latest)
+                        for row in latest:
+                            logger.debug(row)
+                            yield row
                     else:
                         break
-        logger.debug(result)
-        return result
 
     @property
     def pathology_demographics_query(self):
@@ -469,23 +467,23 @@ def raw_data(self, hospital_number, lab_number=None, test_type=None):
         db_date = datetime.date.today() - datetime.timedelta(365)
 
         if lab_number:
-            return self.execute_trust_query(
+            return list(self.execute_trust_query(
                 self.all_data_query_for_lab_number,
                 params=dict(
                     hospital_number=hospital_number,
                     since=db_date,
                     lab_number=lab_number
                 )
-            )
+            ))
         if test_type:
-            return self.execute_trust_query(
+            return list(self.execute_trust_query(
                 self.all_data_query_for_lab_test_type,
                 params=dict(
                     hospital_number=hospital_number,
                     since=db_date,
                     test_type=test_type
                 )
-            )
+            ))
         else:
             return self.execute_trust_query(
                 self.all_data_for_hospital_number_query,
@@ -515,9 +513,16 @@ def data_deltas(self, some_datetime):
         all_rows = self.data_delta_query(some_datetime)
 
         hospital_number_to_rows = defaultdict(list)
+        our_patient_hospital_numbers = set()
 
         for row in all_rows:
-            hospital_number_to_rows[row.get_hospital_number()].append(row)
+            hn = row.get_hospital_number()
+            if hn not in our_patient_hospital_numbers:
+                if Demographics.objects.filter(hospital_number=hn).exists():
+                    our_patient_hospital_numbers.add(hn)
+
+            if hn in our_patient_hospital_numbers:
+                hospital_number_to_rows[row.get_hospital_number()].append(row)
 
         result = []
 

intrahospital_api/test/test_prod_api.py

@@ -438,12 +438,12 @@ def test_execute_hospital_query_with_params(self, pytds):
         api = self.get_api()
         conn = pytds.connect().__enter__()
         cursor = conn.cursor().__enter__()
-        cursor.fetchmany.return_value = ["some_results"]
+        cursor.fetchmany.side_effect = [["some_results"], None]
         result = api.execute_hospital_query(
             "some query", dict(hospital_number="1231222222")
         )
         self.assertEqual(
-            result, ["some_results"]
+            list(result), ["some_results"]
         )
         cursor.execute.assert_called_once_with(
             "some query",
@@ -456,10 +456,10 @@ def test_execute_hospital_query_without_params(self, pytds):
         api = self.get_api()
         conn = pytds.connect().__enter__()
         cursor = conn.cursor().__enter__()
-        cursor.fetchmany.return_value = ["some_results"]
+        cursor.fetchmany.side_effect = [["some_results"], None]
         result = api.execute_hospital_query("some query")
         self.assertEqual(
-            result, ["some_results"]
+            list(result), ["some_results"]
         )
         cursor.execute.assert_called_once_with("some query", None)
         self.assertTrue(cursor.fetchmany.called)