backend: Ingest only first report in a push to avoid timeouts

Author: Bastien Abadie

backend/code_coverage_backend/gcp.py

@@ -78,40 +78,47 @@ def __init__(self, reports_dir=None):
             for report in self.list_reports(repo, nb=1):
                 self.download_report(report)
 
-    def ingest_pushes(self, repository, min_push_id=None, nb_pages=3):
+    def ingest_pushes(self, repository, platform, suite, min_push_id=None, nb_pages=3):
         """
         Ingest HGMO changesets and pushes into our Redis Cache
         The pagination goes from oldest to newest, starting from the optional min_push_id
         """
+        ingested = False
         for push_id, push in hgmo_pushes(repository, min_push_id, nb_pages):
             for changeset in push["changesets"]:
                 # TODO: look all neighboring reports on GCP
                 report = Report(
                     self.reports_dir,
                     repository,
                     changeset,
+                    platform,
+                    suite,
                     push_id=push_id,
                     date=push["date"],
                 )
-                if self.ingest_report(report):
+
+                # Always link changeset to push to find closest available report
+                self.redis.hmset(
+                    KEY_CHANGESET.format(
+                        repository=report.repository, changeset=report.changeset
+                    ),
+                    {"push": report.push_id, "date": report.date},
+                )
+
+                if not ingested and self.ingest_report(report):
                     logger.info(
                         "Found report in that push", push_id=push_id, report=str(report)
                     )
 
+                    # Only ingest first report found in a push in order to stay below 30s response time
+                    ingested = True
+
     def ingest_report(self, report):
         """
         When a report exist for a changeset, download it and update redis data
         """
         assert isinstance(report, Report)
 
-        # Always link changeset to push to find closest available report
-        self.redis.hmset(
-            KEY_CHANGESET.format(
-                repository=report.repository, changeset=report.changeset
-            ),
-            {"push": report.push_id, "date": report.date},
-        )
-
         # Download the report
         if not self.download_report(report):
             logger.info("Report not available", report=str(report))
@@ -228,7 +235,9 @@ def find_closest_report(
             push_id, _ = hgmo_revision_details(repository, changeset)
 
             # Ingest pushes as we clearly don't have it in cache
-            self.ingest_pushes(repository, min_push_id=push_id - 1, nb_pages=1)
+            self.ingest_pushes(
+                repository, platform, suite, min_push_id=push_id - 1, nb_pages=1
+            )
 
         # Load report from that push
         return self.find_report(

backend/tests/test_gcp.py

@@ -122,7 +122,7 @@ def test_ingest_hgmo(mock_cache, mock_hgmo):
     # Ingest last pushes
     assert mock_cache.list_reports("myrepo") == []
     assert len(mock_cache.redis.keys("changeset:myrepo:*")) == 0
-    mock_cache.ingest_pushes("myrepo")
+    mock_cache.ingest_pushes("myrepo", "all", "all")
     assert len(mock_cache.redis.keys("changeset:myrepo:*")) > 0
     assert mock_cache.list_reports("myrepo") == [
         Report(mock_cache.reports_dir, "myrepo", rev, push_id=1, date=995)