Bug 1372581 - Fix exception in error reporting for missing Repository (…

…#2624)

tests/etl/test_resultset_loader.py

@@ -7,7 +7,9 @@
 from treeherder.etl.resultset_loader import (GithubPullRequestTransformer,
                                              GithubPushTransformer,
                                              HgPushTransformer,
+                                             PulseResultsetError,
                                              ResultsetLoader)
+from treeherder.model.models import Push
 
 
 @pytest.fixture
@@ -93,6 +95,12 @@ def test_get_transformer_class(exchange, transformer_class):
     assert rsl.get_transformer_class(exchange) == transformer_class
 
 
+def test_unsupported_exchange():
+    with pytest.raises(PulseResultsetError):
+        rsl = ResultsetLoader()
+        rsl.get_transformer_class("meh")
+
+
 def test_ingest_github_pull_request(test_repository, github_pr, transformed_github_pr,
                                     mock_github_pr_commits):
     xformer = GithubPullRequestTransformer(github_pr)
@@ -114,15 +122,17 @@ def test_ingest_hg_push(test_repository, hg_push, transformed_hg_push,
     assert transformed_hg_push == resultset
 
 
+@pytest.mark.django_db
 def test_ingest_hg_push_bad_repo(hg_push):
-    """Test that a message from an unknown HG repo doesn't throw an error"""
+    """Test graceful handling of an unknown HG repo"""
     hg_push["payload"]["repo_url"] = "https://bad.repo.com"
     ResultsetLoader().process(hg_push, "exchange/hgpushes/v1")
-    # No exception here means it passed.
+    assert Push.objects.count() == 0
 
 
+@pytest.mark.django_db
 def test_ingest_github_push_bad_repo(github_push):
-    """Test that a message from an unknown GH repo doesn't throw an error"""
+    """Test graceful handling of an unknown GH repo"""
     github_push["details"]["event.head.repo.url"] = "https://bad.repo.com"
-    ResultsetLoader().process(hg_push, "exchange/taskcluster-github/v1/push")
-    # No exception here means it passed.
+    ResultsetLoader().process(github_push, "exchange/taskcluster-github/v1/push")
+    assert Push.objects.count() == 0

treeherder/etl/resultset_loader.py

@@ -17,34 +17,32 @@ class ResultsetLoader:
     """Transform and load a list of Resultsets"""
 
     def process(self, message_body, exchange):
+        transformer = self.get_transformer_class(exchange)(message_body)
         try:
-            transformer = self.get_transformer_class(exchange)(message_body)
-            try:
-                repo = Repository.objects.get(url=transformer.repo_url,
-                                              branch=transformer.branch,
-                                              active_status="active")
-
-            except ObjectDoesNotExist:
-                repo_info = message_body.get("details",
-                                             message_body["payload"])
-                newrelic.agent.record_custom_event("skip_unknown_repository",
-                                                   repo_info)
-                logger.warn("Skipping unsupported repo: {} {}".format(
-                    transformer.repo_url,
-                    transformer.branch))
-                return
-
-            transformed_data = transformer.transform(repo.name)
-
-            logger.info("Storing resultset for {} {} {}".format(
-                repo.name,
+            repo = Repository.objects.get(url=transformer.repo_url,
+                                          branch=transformer.branch,
+                                          active_status="active")
+
+        except ObjectDoesNotExist:
+            repo_info = transformer.get_info()
+            repo_info.update({
+                "url": transformer.repo_url,
+                "branch": transformer.branch,
+            })
+            newrelic.agent.record_custom_event("skip_unknown_repository",
+                                               repo_info)
+            logger.warn("Skipping unsupported repo: {} {}".format(
                 transformer.repo_url,
                 transformer.branch))
-            store_result_set_data(repo, [transformed_data])
+            return
 
-        except Exception as ex:
-            newrelic.agent.record_exception(exc=ex)
-            logger.exception("Error transforming resultset", exc_info=ex)
+        transformed_data = transformer.transform(repo.name)
+
+        logger.info("Storing resultset for {} {} {}".format(
+            repo.name,
+            transformer.repo_url,
+            transformer.branch))
+        store_result_set_data(repo, [transformed_data])
 
     def get_transformer_class(self, exchange):
         if "github" in exchange:
@@ -74,6 +72,15 @@ def __init__(self, message_body):
     def get_branch(self):
         return self.message_body["details"]["event.base.repo.branch"]
 
+    def get_info(self):
+        # flatten the data a bit so it will show in new relic as fields
+        info = self.message_body["details"].copy()
+        info.update({
+            "organization": self.message_body["organization"],
+            "repository": self.message_body["repository"]
+        })
+        return info
+
     def fetch_resultset(self, url, repository, sha=None):
         params = {"sha": sha} if sha else {}
         params.update(self.CREDENTIALS)
@@ -223,6 +230,9 @@ def __init__(self, message_body):
         self.repo_url = message_body["payload"]["repo_url"]
         self.branch = None
 
+    def get_info(self):
+        return self.message_body["payload"]
+
     def transform(self, repository):
         logger.info("transforming for {}".format(repository))
         url = self.message_body["payload"]["pushlog_pushes"][0]["push_full_json_url"]