Add a test for retriggers and unclassified failures.

mozilla-services · Aug 1, 2020 · 8be8c29 · 8be8c29
1 parent 02748fe
commit 8be8c29
Show file tree

Hide file tree

Showing 4 changed files with 52 additions and 6 deletions.
diff --git a/tests/functionality.py b/tests/functionality.py
@@ -27,6 +27,7 @@
 
 from tests.mock_commandprovider import TestCommandProvider
 from tests.mock_treeherder_server import MockTreeherderServer
+from tests.treeherder_api import RETRIGGER_RESPONSE
 
 try:
     from localconfig import localconfig
@@ -101,12 +102,13 @@ def DEFAULT_EXPECTED_VALUES(revision):
 
 def COMMAND_MAPPINGS(expected_values):
     return {
-            "./mach vendor": expected_values.library_version_id,
-            "./mach try auto": TRY_OUTPUT(expected_values.try_revision_id),
-            "arc diff --verbatim": ARC_OUTPUT,
-            "echo '{\"constraints\"": CONDUIT_USERNAME_SEARCH_OUTPUT,
-            "echo '{\"transactions\":": CONDUIT_EDIT_OUTPUT
-        }
+        "./mach vendor": expected_values.library_version_id,
+        "./mach try auto": TRY_OUTPUT(expected_values.try_revision_id),
+        "arc diff --verbatim": ARC_OUTPUT,
+        "echo '{\"constraints\"": CONDUIT_USERNAME_SEARCH_OUTPUT,
+        "echo '{\"transactions\":": CONDUIT_EDIT_OUTPUT,
+        "echo -n": RETRIGGER_RESPONSE
+    }
 
 
 class MockedBugzillaProvider(BaseProvider):
@@ -332,6 +334,43 @@ def testExistingJobClassifiedFailure(self):
             u.dbProvider.delete_job(l, expected_values.library_version_id)
 
     # Create -> Jobs are Running -> Awaiting Retriggers -> Unclassified Failure
+    def testExistingJobUnclassifiedFailure(self):
+        # Setup
+        try_revision = "ab2232a04301f1d2dbeea7050488f8ec2dde5451"
+        expected_values = DEFAULT_EXPECTED_VALUES(try_revision)
+        self.configs['Bugzilla']['filed_bug_id'] = expected_values.filed_bug_id
+        self.configs['Command']['test_mappings'] = COMMAND_MAPPINGS(expected_values)
+
+        # Make it
+        library_filter = 'dav1d'
+        u = Updatebot(self.configs, self.providers)
+        _check_jobs = functools.partial(self._check_jobs, u, library_filter, expected_values)
+
+        # Ensure we don't have a dirty database with existing jobs
+        for l in u.dbProvider.get_libraries():
+            j = u.dbProvider.get_job(l, expected_values.library_version_id)
+            self.assertEqual(j, None, "When running testExistingJobUnclassifiedFailure, we found an existing job, indicating the database is dirty and should be cleaned.")
+
+        # Run it
+        u.run(library_filter=library_filter)
+        # Check that we created the job successfully
+        _check_jobs(JOBSTATUS.AWAITING_TRY_RESULTS, JOBOUTCOME.PENDING)
+        # Run it again, this time we'll tell it the jobs are still in process
+        u.run(library_filter=library_filter)
+        # Should still be Awaiting Try Results
+        _check_jobs(JOBSTATUS.AWAITING_TRY_RESULTS, JOBOUTCOME.PENDING)
+        # Run it again, this time we'll tell it a test failed
+        u.run(library_filter=library_filter)
+        # Should be DONE and Failed.
+        _check_jobs(JOBSTATUS.AWAITING_RETRIGGER_RESULTS, JOBOUTCOME.PENDING)
+        # Run it again, this time we'll tell it all the tests failed
+        u.run(library_filter=library_filter)
+        # Should be DONE and Failed.
+        _check_jobs(JOBSTATUS.DONE, JOBOUTCOME.UNCLASSIFIED_FAILURES)
+
+        # Cleanup
+        for l in u.dbProvider.get_libraries():
+            u.dbProvider.delete_job(l, expected_values.library_version_id)
 
 
 if __name__ == '__main__':

diff --git a/tests/mock_treeherder_server.py b/tests/mock_treeherder_server.py
@@ -21,6 +21,7 @@
     'e152bb86666565ee6619c15f60156cd6c79580a9': "{\"results\":[{\"id\":2}]}",
     '55ca6286e3e4f4fba5d0448333fa99fc5a404a73': "{\"results\":[{\"id\":3}]}",
     '56082fc4acfacba40993e47ef8302993c59e264e': "{\"results\":[{\"id\":4}]}",
+    'ab2232a04301f1d2dbeea7050488f8ec2dde5451': "{\"results\":[{\"id\":5}]}",
 }
 
 PUSH_IDS = {
@@ -41,6 +42,10 @@
     # testExistingJobClassifiedFailure
     '4_1_1': "treeherder_api_response_jobs_still_running.txt",
     '4_1_2': "treeherder_api_response_classified_failure.txt",
+    # testExistingJobUnclassifiedFailure
+    '5_1_1': "treeherder_api_response_jobs_still_running.txt",
+    '5_1_2': "treeherder_api_response_jobs_one_unclassified_failure.txt",
+    '5_1_3': "treeherder_api_response_jobs_unclassified_failure_retriggers_done.txt",
 }
 
 seen_counters = {}

diff --git a/tests/treeherder_api_response_jobs_one_unclassified_failure.txt b/tests/treeherder_api_response_jobs_one_unclassified_failure.txt
@@ -0,0 +1 @@
+{"count":23,"next":null,"previous":null,"results":[[1,311165077,"unknown","?","Gecko Decision Task","D","2020-07-27T18:15:27.887809","gecko-decision",757557,"2a1026d5db99ce10f9946aec8ca65bf13956dee7","success","2aa083621bb989d6acf1151667288d5fe9616178","completed",1,"Lani1fJlTNqQ7bITYylzug",0,6,"opt"],[1,311165521,"pedantic checks","pedantic","source-test-mozlint-file-whitespace","file-whitespace","2020-07-27T18:18:29.893295","lint",757557,"2a1026d5db99ce10f9946aec8ca65bf13956dee7","success","8762ce85f312620a908db2d256608d1c8c23fffa","completed",2,"ARQElqlkTCuvTXE5hf5GdQ",0,1,"opt"],[1,311165522,"Python 3 unit tests","py3","source-test-python-taskgraph-tests-py3","tg","2020-07-27T18:19:31.876695","linux1804-64",757557,"2a1026d5db99ce10f9946aec8ca65bf13956dee7","success","3048dfcb8901c1343c7b020ba48911b330ca0c8d","completed",2,"cMkRDhdXSXmPzRbArt6vHA",0,2,"opt"],[1,311165523,"Check on texts","text","source-test-mozlint-codespell","spell","2020-07-27T18:18:07.506696","lint",757557,"2a1026d5db99ce10f9946aec8ca65bf13956dee7","success","6ac7e23dc92982745cef4fe37846a7be8b578fed","completed",2,"b3pHOsmTT9uIluiM-VTxiA",0,1,"opt"],[1,311165524,"Python 2 unit tests","py2","source-test-python-mozbuild-macosx1014-64/opt-py2","mbu","2020-07-27T18:24:33.002518","macosx1014-64",757557,"2a1026d5db99ce10f9946aec8ca65bf13956dee7","success","8e3d8e2e69c8d04ce6ac7270568505846b0ba426","completed",2,"FkJXw4FWQwqTt_TT2YANZg",0,9,"opt"],[1,311165525,"Python 2 unit tests","py2","source-test-python-mach-windows10-64/opt-py2","mach","2020-07-27T18:24:01.410399","windows10-64",757557,"2a1026d5db99ce10f9946aec8ca65bf13956dee7","testfailed","d616552aac93b4267f1b1897bee53ceb61e473a9","completed",2,"KB4P_rJeRR-Dtxa1Mem9-g",0,2,"opt"],[1,311165526,"Python 3 unit tests","py3","source-test-python-mach-macosx1014-64/opt-py3","mach","2020-07-27T18:19:59.107339","macosx1014-64",757557,"2a1026d5db99ce10f9946aec8ca65bf13956dee7","success","bd48f8dd80af9aec00545d8ba0842e325e096dd6","completed",2,"fu3_jSXtSSGxWzBIx4jQtQ",0,4,"opt"],[1,311165527,"Python checks","py","source-test-mozlint-py-black","black","2020-07-27T18:18:26.648819","lint",757557,"2a1026d5db99ce10f9946aec8ca65bf13956dee7","success","17aef7b6f84f6c23ce1205391b95efc6014543bf","completed",1,"SGAFgn-CR5CA5PfmcKI8fg",0,1,"opt"],[1,311165528,"Python 2 unit tests","py2","source-test-python-mach-macosx1014-64/opt-py2","mach","2020-07-27T18:20:14.544684","macosx1014-64",757557,"2a1026d5db99ce10f9946aec8ca65bf13956dee7","success","62f6fe8a09ffa3d50d93b5f79498ca3496d8ce4f","completed",2,"J96VpxkISO-F2_mnqZeGqg",0,5,"opt"],[1,311165529,"Python 2 unit tests","py2","source-test-python-mozbuild-linux1804-64/opt-py2","mbu","2020-07-27T18:21:48.408890","linux1804-64",757557,"2a1026d5db99ce10f9946aec8ca65bf13956dee7","success","13d624f5915bc64326a309ba656c45206775d34e","completed",2,"ZqEL7Wo3SnSrXPRlFhkIow",0,4,"opt"],[1,311165530,"Python 3 unit tests","py3","source-test-python-mozbuild-windows10-64/opt-py3","mbu","2020-07-27T18:30:01.067588","windows10-64",757557,"2a1026d5db99ce10f9946aec8ca65bf13956dee7","success","eb0423302d420e059634a499b1158aedca7efbc9","completed",2,"F8l9UrmyRbCXCPKMYcF3ug",0,8,"opt"],[1,311165531,"Check on texts","text","source-test-mozlint-rejected-words","rejected-words","2020-07-27T18:23:29.211842","lint",757557,"2a1026d5db99ce10f9946aec8ca65bf13956dee7","success","f22644154e5ef26d261fa39ac64de643582d66b5","completed",2,"UBz2OlSMTi-lVtJk-ifltw",0,6,"opt"],[1,311165532,"Python 2 unit tests","py2","source-test-python-mach-linux1804-64/opt-py2","mach","2020-07-27T18:18:26.628501","linux1804-64",757557,"2a1026d5db99ce10f9946aec8ca65bf13956dee7","success","319a91a88d5d8ff67b492e3508ba5c1f43f32569","completed",2,"SFHLn--yTTiK5dVcSu7UKA",0,1,"opt"],[1,311165533,"Python 3 unit tests","py3","source-test-python-mozbuild-macosx1014-64/opt-py3","mbu","2020-07-27T18:24:40.479098","macosx1014-64",757557,"2a1026d5db99ce10f9946aec8ca65bf13956dee7","success","37a120d444f0da9c6f17b86ae47a149267b57763","completed",2,"U6m1glyzRs-ZVFgOmiQnsw",0,9,"opt"],[1,311165534,"Python 3 unit tests","py3","source-test-python-mach-windows10-64/opt-py3","mach","2020-07-27T18:24:04.465018","windows10-64",757557,"2a1026d5db99ce10f9946aec8ca65bf13956dee7","success","a45233c915e6a4238557fdc8b3429b8683dc463d","completed",2,"Cn40cPY1RneMVsKw_SEgkg",0,3,"opt"],[1,311165535,"Python 3 unit tests","py3","source-test-python-mach-linux1804-64/opt-py3","mach","2020-07-27T18:18:44.062917","linux1804-64",757557,"2a1026d5db99ce10f9946aec8ca65bf13956dee7","success","bbe460d9929f6559c83dc1d126ffff926fc2272d","completed",2,"YMeF5lFeSL-HNkuCN_TIqQ",0,1,"opt"],[1,311165536,"Python 2 unit tests","py2","source-test-python-taskgraph-tests-py2","tg","2020-07-27T18:19:13.324477","linux1804-64",757557,"2a1026d5db99ce10f9946aec8ca65bf13956dee7","success","8c69f7ee372f449324141b7eab55d9c1d1895221","completed",2,"AOOrdboKRTWu8w8pCI9kig",0,2,"opt"],[1,311165537,"pedantic checks","pedantic","source-test-mozlint-license","license","2020-07-27T18:18:44.133072","lint",757557,"2a1026d5db99ce10f9946aec8ca65bf13956dee7","success","6b73bd01b6cf9a0b87c15c032e2da81bedddd157","completed",1,"QBoAslx1SnGZB5u_Mjfifg",0,1,"opt"],[1,311165538,"Python 2 unit tests","py2","source-test-python-mozbuild-windows10-64/opt-py2","mbu","2020-07-27T18:29:43.631280","windows10-64",757557,"2a1026d5db99ce10f9946aec8ca65bf13956dee7","success","16b00baca91e1874abf70c2c941215bf8f6ef476","completed",2,"JCwek_FeT0yHXdHdXTkoxg",0,8,"opt"],[1,311165539,"Python checks","py","source-test-mozlint-py-flake8","f8","2020-07-27T18:20:01.027950","lint",757557,"2a1026d5db99ce10f9946aec8ca65bf13956dee7","success","2ce9f9a4e5b3f60910d9f5089038f0b19a361bde","completed",1,"V52oLnrNS2OXkPIPQPAvXA",0,2,"opt"],[1,311165540,"Python 3 unit tests","py3","source-test-python-mozbuild-linux1804-64/opt-py3","mbu","2020-07-27T18:22:32.261043","linux1804-64",757557,"2a1026d5db99ce10f9946aec8ca65bf13956dee7","success","5de80d3e5983f342b57719e7040798f330323abb","completed",2,"BnTuqbKrSV-PPg1P4BXtBQ",0,4,"opt"],[1,311165541,"Python checks","py","source-test-mozlint-py-pylint","pylint","2020-07-27T18:18:00.816054","lint",757557,"2a1026d5db99ce10f9946aec8ca65bf13956dee7","success","d044d20e142c0ef8ff2379111baf89c39a9082e3","completed",1,"ffoF-mGxSMeDxsVA3qUfnA",0,1,"opt"],[1,311165542,"Python checks","py","source-test-mozlint-py-compat","py-compat","2020-07-27T18:18:22.369138","lint",757557,"2a1026d5db99ce10f9946aec8ca65bf13956dee7","success","d01146ab758b5cccc61197552eba610360ff4a4c","completed",1,"OS19rdhWRPqmZDo9LfDvuQ",0,1,"opt"]],"job_property_names":["failure_classification_id","id","job_group_name","job_group_symbol","job_type_name","job_type_symbol","last_modified","platform","push_id","push_revision","result","signature","state","tier","task_id","retry_id","duration","platform_option"]}