Bug 1429030 - automatically suggest Bugzilla product and component when bug filer gets opened

docker/entrypoint_prod.sh

@@ -76,6 +76,9 @@ elif [ "$1" == "run_intermittents_commenter" ]; then
 elif [ "$1" == "update_bugscache" ]; then
     newrelic-admin run-program ./manage.py update_bugscache
 
+elif [ "$1" == "update_files_bugzilla_map" ]; then
+    newrelic-admin run-program ./manage.py update_files_bugzilla_map
+
 elif [ "$1" == "cache_failure_history" ]; then
     newrelic-admin run-program ./manage.py cache_failure_history
 

newrelic.ini

@@ -30,4 +30,4 @@ shutdown_timeout = 15
 # List finite-duration commands here to enable their annotation by the agent.
 # For infinite duration commands (such as `pulse_listener_*`) see:
 # https://docs.newrelic.com/docs/agents/python-agent/supported-features/python-background-tasks#wrapping
-instrumentation.scripts.django_admin = update_changelog check cycle_data load_initial_data migrate update_bugscache run_intermittents_commenter synthesize_backfill_report backfill_text_log_error_jobs
+instrumentation.scripts.django_admin = update_changelog check cycle_data load_initial_data migrate update_bugscache update_files_bugzilla_map run_intermittents_commenter synthesize_backfill_report backfill_text_log_error_jobs

tests/conftest.py

@@ -11,6 +11,7 @@
 import responses
 from _pytest.monkeypatch import MonkeyPatch
 from django.conf import settings
+from django.core.management import call_command
 from rest_framework.test import APIClient
 
 from treeherder.etl.jobs import store_job_data
@@ -64,6 +65,14 @@ def pytest_runtest_setup(item):
     cache.clear()
 
 
+@pytest.fixture
+def setup_repository_data(django_db_setup, django_db_blocker):
+    with django_db_blocker.unblock():
+        call_command('loaddata', join(SAMPLE_DATA_PATH, 'repository_group.json'))
+    with django_db_blocker.unblock():
+        call_command('loaddata', join(SAMPLE_DATA_PATH, 'repository.json'))
+
+
 @pytest.fixture(scope="session", autouse=True)
 def block_unmocked_requests():
     """
@@ -631,6 +640,40 @@ def authorized_sheriff_client(client, test_sheriff):
     return client
 
 
+@pytest.fixture
+def mock_file_bugzilla_map_request(monkeypatch):
+    """
+    Mock fetch_json() used by files_bugzilla_map ETL to return local sample
+    files which map source files to Bugzilla components.
+    """
+    import treeherder.etl.files_bugzilla_map
+
+    def _fetch_data(self, project):
+        url = (
+            'https://firefox-ci-tc.services.mozilla.com/api/index/v1/task/gecko.v2.%s.latest.source.source-bugzilla-info/artifacts/public/components.json'
+            % project
+        )
+        files_bugzilla_data = None
+        file_name = "files_bugzilla_map_%s_%s.json" % (project, self.run_id)
+        exception = None
+        try:
+            tests_folder = os.path.dirname(__file__)
+            data_path = os.path.join(tests_folder, "sample_data", "files_bugzilla_map", file_name)
+            with open(data_path) as f:
+                files_bugzilla_data = json.load(f)
+        except Exception as e:
+            exception = e
+        return {
+            "url": url,
+            "files_bugzilla_data": files_bugzilla_data,
+            "exception": exception,
+        }
+
+    monkeypatch.setattr(
+        treeherder.etl.files_bugzilla_map.FilesBugzillaMapProcess, 'fetch_data', _fetch_data
+    )
+
+
 @pytest.fixture
 def text_log_error_lines(test_job, failure_lines):
     from tests.autoclassify.utils import create_text_log_errors

tests/log_parser/test_tasks.py

@@ -49,6 +49,6 @@ def test_create_error_summary(
 
     # We really need to add some tests that check the values of each entry
     # in bug_suggestions, but for now this is better than nothing.
-    expected_keys = set(["search", "search_terms", "bugs", "line_number"])
+    expected_keys = set(["search", "path_end", "search_terms", "bugs", "line_number"])
     for failure_line in bug_suggestions:
         assert set(failure_line.keys()) == expected_keys

tests/model/test_error_summary.py

@@ -3,7 +3,7 @@
 from treeherder.model.error_summary import (
     get_cleaned_line,
     get_crash_signature,
-    get_error_search_term,
+    get_error_search_term_and_path,
 )
 
 LINE_CLEANING_TEST_CASES = (
@@ -39,31 +39,43 @@ def test_get_cleaned_line(line_raw, exp_line_cleaned):
             '| chrome://mochitests/content/browser/browser/components/loop/test/mochitest/browser_fxa_login.js '
             '| Check settings tab URL - Got http://mochi.test:8888/browser/browser/components/loop/test/mochitest/loop_fxa.sjs'
         ),
-        'browser_fxa_login.js',
+        {
+            'path_end': 'chrome://mochitests/content/browser/browser/components/loop/test/mochitest/browser_fxa_login.js',
+            'search_term': 'browser_fxa_login.js',
+        },
     ),
     (
         (
             'REFTEST TEST-UNEXPECTED-FAIL '
             '| file:///C:/slave/test/build/tests/reftest/tests/layout/reftests/layers/component-alpha-exit-1.html '
             '| image comparison (==), max difference: 255, number of differing pixels: 251'
         ),
-        'component-alpha-exit-1.html',
+        {
+            'path_end': 'file:///C:/slave/test/build/tests/reftest/tests/layout/reftests/layers/component-alpha-exit-1.html',
+            'search_term': 'component-alpha-exit-1.html',
+        },
     ),
     (
         (
             '2423 INFO TEST-UNEXPECTED-FAIL '
             '| /tests/dom/media/tests/mochitest/test_dataChannel_basicAudio.html '
             '| undefined assertion name - Result logged after SimpleTest.finish()'
         ),
-        'test_dataChannel_basicAudio.html',
+        {
+            'path_end': '/tests/dom/media/tests/mochitest/test_dataChannel_basicAudio.html',
+            'search_term': 'test_dataChannel_basicAudio.html',
+        },
     ),
     (
         (
             r"TEST-UNEXPECTED-FAIL "
             r"| mainthreadio "
             r"| File 'c:\users\cltbld~1.t-w' was accessed and we were not expecting it: {'Count': 6, 'Duration': 0.112512, 'RunCount': 6}"
         ),
-        'mainthreadio',
+        {
+            'path_end': 'mainthreadio',
+            'search_term': 'mainthreadio',
+        },
     ),
     (
         (
@@ -72,7 +84,10 @@ def test_get_cleaned_line(line_raw, exp_line_cleaned):
             "http://10.0.2.2:8854/tests/dom/canvas/test/reftest/wrapper.html?green.png "
             "| application crashed [@ jemalloc_crash]"
         ),
-        'webgl-resize-test.html',
+        {
+            'path_end': 'http://10.0.2.2:8854/tests/dom/canvas/test/reftest/webgl-resize-test.html',
+            'search_term': 'webgl-resize-test.html',
+        },
     ),
     (
         (
@@ -81,32 +96,30 @@ def test_get_cleaned_line(line_raw, exp_line_cleaned):
             "http://10.0.2.2:8854/tests/dom/canvas/test/reftest/wrapper.html?green.png "
             "| application crashed [@ jemalloc_crash]"
         ),
-        'webgl-resize-test.html',
+        {
+            'path_end': 'http://10.0.2.2:8854/tests/dom/canvas/test/reftest/webgl-resize-test.html',
+            'search_term': 'webgl-resize-test.html',
+        },
     ),
     (
         (
             "TEST-UNEXPECTED-FAIL "
             "| /tests/dom/events/test/pointerevents/pointerevent_touch-action-table-test_touch-manual.html "
             "| touch-action attribute test on the cell: assert_true: scroll received while shouldn't expected true got false"
         ),
-        'pointerevent_touch-action-table-test_touch-manual.html',
-    ),
-    (
-        (
-            "TEST-UNEXPECTED-FAIL "
-            "| /tests/dom/events/test/pointerevents/pointerevent_touch-action-table-test_touch-manual.html "
-            "| touch-action attribute test on the cell: assert_true: scroll received while shouldn't expected true got false"
-        ),
-        'pointerevent_touch-action-table-test_touch-manual.html',
+        {
+            'path_end': '/tests/dom/events/test/pointerevents/pointerevent_touch-action-table-test_touch-manual.html',
+            'search_term': 'pointerevent_touch-action-table-test_touch-manual.html',
+        },
     ),
 )
 
 
-@pytest.mark.parametrize(("line", "exp_search_term"), PIPE_DELIMITED_LINE_TEST_CASES)
-def test_get_delimited_search_term(line, exp_search_term):
+@pytest.mark.parametrize(("line", "exp_search_info"), PIPE_DELIMITED_LINE_TEST_CASES)
+def test_get_delimited_search_term(line, exp_search_info):
     """Test search term extraction for a pipe delimited error line"""
-    actual_search_term = get_error_search_term(line)
-    assert actual_search_term == exp_search_term
+    actual_search_info = get_error_search_term_and_path(line)
+    assert actual_search_info == exp_search_info
 
 
 LEAK_LINE_TEST_CASES = (
@@ -117,7 +130,10 @@ def test_get_delimited_search_term(line, exp_search_term):
             '(BackstagePass, CallbackObject, DOMEventTargetHelper, '
             'EventListenerManager, EventTokenBucket, ...)'
         ),
-        'BackstagePass, CallbackObject, DOMEventTargetHelper, EventListenerManager, EventTokenBucket, ...',
+        {
+            'path_end': None,
+            'search_term': 'BackstagePass, CallbackObject, DOMEventTargetHelper, EventListenerManager, EventTokenBucket, ...',
+        },
     ),
     (
         (
@@ -126,43 +142,55 @@ def test_get_delimited_search_term(line, exp_search_term):
             '(AsyncLatencyLogger, AsyncTransactionTrackersHolder, AudioOutputObserver, '
             'BufferRecycleBin, CipherSuiteChangeObserver, ...)'
         ),
-        'AsyncLatencyLogger, AsyncTransactionTrackersHolder, AudioOutputObserver, BufferRecycleBin, CipherSui',
+        {
+            'path_end': None,
+            'search_term': 'AsyncLatencyLogger, AsyncTransactionTrackersHolder, AudioOutputObserver, BufferRecycleBin, CipherSui',
+        },
     ),
     (
         (
             'TEST-UNEXPECTED-FAIL '
             '| LeakSanitizer | leak at '
             'MakeUnique, nsThread::nsChainedEventQueue::nsChainedEventQueue, nsThread, nsThreadManager::Init'
         ),
-        'MakeUnique, nsThread::nsChainedEventQueue::nsChainedEventQueue, nsThread, nsThreadManager::Init',
+        {
+            'path_end': None,
+            'search_term': 'MakeUnique, nsThread::nsChainedEventQueue::nsChainedEventQueue, nsThread, nsThreadManager::Init',
+        },
     ),
 )
 
 
-@pytest.mark.parametrize(("line", "exp_search_term"), LEAK_LINE_TEST_CASES)
-def test_get_leak_search_term(line, exp_search_term):
+@pytest.mark.parametrize(("line", "exp_search_info"), LEAK_LINE_TEST_CASES)
+def test_get_leak_search_term(line, exp_search_info):
     """tests the search term extracted from a leak error line is correct"""
-    actual_search_term = get_error_search_term(line)
-    assert actual_search_term == exp_search_term
+    actual_search_info = get_error_search_term_and_path(line)
+    assert actual_search_info == exp_search_info
 
 
 FULL_LINE_FALLBACK_TEST_CASES = (
     (
         'Automation Error: No crash directory (/mnt/sdcard/tests/profile/minidumps/) found on remote device',
-        'Automation Error: No crash directory (/mnt/sdcard/tests/profile/minidumps/) found on remote device',
+        {
+            'path_end': None,
+            'search_term': 'Automation Error: No crash directory (/mnt/sdcard/tests/profile/minidumps/) found on remote device',
+        },
     ),
     (
         'PROCESS-CRASH | Automation Error: Missing end of test marker (process crashed?)',
-        'Automation Error: Missing end of test marker (process crashed?)',
+        {
+            'path_end': None,
+            'search_term': 'Automation Error: Missing end of test marker (process crashed?)',
+        },
     ),
 )
 
 
-@pytest.mark.parametrize(("line", "exp_search_term"), FULL_LINE_FALLBACK_TEST_CASES)
-def test_get_full_line_search_term(line, exp_search_term):
+@pytest.mark.parametrize(("line", "exp_search_info"), FULL_LINE_FALLBACK_TEST_CASES)
+def test_get_full_line_search_term(line, exp_search_info):
     """Test that the full error line is used as a fall-back if no test name found"""
-    actual_search_term = get_error_search_term(line)
-    assert actual_search_term == exp_search_term
+    actual_search_info = get_error_search_term_and_path(line)
+    assert actual_search_info == exp_search_info
 
 
 LONG_LINE_TEST_CASES = (
@@ -177,29 +205,33 @@ def test_get_full_line_search_term(line, exp_search_term):
             '\'b2g-inbound\', \'--download-symbols\', \'ondemand\'], '
             'attempting to kill'
         ),
-        (
-            'command timed out: 2400 seconds without output running '
-            '[\'/tools/buildbot/bin/python\', \'scripts/scrip'
-        ),
+        {
+            'path_end': None,
+            'search_term': 'command timed out: 2400 seconds without output running '
+            '[\'/tools/buildbot/bin/python\', \'scripts/scrip',
+        },
     ),
     (
         (
             'TEST-UNEXPECTED-FAIL '
             '| test_switch_frame.py TestSwitchFrame.test_should_be_able_to_carry_on_working_if_the_frame_is_deleted_from_under_us '
             '| AssertionError: 0 != 1'
         ),
-        'test_switch_frame.py TestSwitchFrame.test_should_be_able_to_carry_on_working_if_the_frame_is_deleted',
+        {
+            'path_end': 'test_switch_frame.py TestSwitchFrame.test_should_be_able_to_carry_on_working_if_the_frame_is_deleted_from_under_us',
+            'search_term': 'test_switch_frame.py TestSwitchFrame.test_should_be_able_to_carry_on_working_if_the_frame_is_deleted',
+        },
     ),
 )
 
 # command timed out: 2400 seconds without output running ['/tools/buildbot/bin/python', 'scripts/scripts/android_emulator_unittest.py', '--cfg', 'android/androidx86.py', '--test-suite', 'robocop-1', '--test-suite', 'robocop-2', '--test-suite', 'robocop-3', '--test-suite', 'xpcshell', '--blob-upload-branch', 'b2g-inbound', '--download-symbols', 'ondemand'], attempting to kill
 
 
-@pytest.mark.parametrize(("line", "exp_search_term"), LONG_LINE_TEST_CASES)
-def test_get_long_search_term(line, exp_search_term):
+@pytest.mark.parametrize(("line", "exp_search_info"), LONG_LINE_TEST_CASES)
+def test_get_long_search_term(line, exp_search_info):
     """tests that long search terms are capped at 100 characters"""
-    actual_search_term = get_error_search_term(line)
-    assert actual_search_term == exp_search_term
+    actual_search_info = get_error_search_term_and_path(line)
+    assert actual_search_info == exp_search_info
 
 
 CRASH_LINE_TEST_CASES = (
@@ -214,31 +246,43 @@ def test_get_long_search_term(line, exp_search_term):
 )
 
 
-@pytest.mark.parametrize(("line", "exp_search_term"), CRASH_LINE_TEST_CASES)
-def test_get_crash_signature(line, exp_search_term):
+@pytest.mark.parametrize(("line", "exp_search_info"), CRASH_LINE_TEST_CASES)
+def test_get_crash_signature(line, exp_search_info):
     """tests the search term extracted from an error line is correct"""
-    actual_search_term = get_crash_signature(line)
-    assert actual_search_term == exp_search_term
+    actual_search_info = get_crash_signature(line)
+    assert actual_search_info == exp_search_info
 
 
 BLACKLIST_TEST_CASES = (
     (
         'TEST-UNEXPECTED-FAIL | remoteautomation.py | application timed out after 330 seconds with no output',
-        'remoteautomation.py | application timed out after 330 seconds with no output',
+        {
+            'path_end': 'remoteautomation.py',
+            'search_term': 'remoteautomation.py | application timed out after 330 seconds with no output',
+        },
+    ),
+    (
+        'Return code: 1',
+        {
+            'path_end': None,
+            'search_term': None,
+        },
     ),
-    ('Return code: 1', None),
     (
         (
             'REFTEST PROCESS-CRASH | file:///home/worker/workspace/build/tests/reftest/tests/layout/reftests/font-inflation/video-1.html '
             '| application crashed [@ mozalloc_abort]'
         ),
-        'video-1.html',
+        {
+            'path_end': 'file:///home/worker/workspace/build/tests/reftest/tests/layout/reftests/font-inflation/video-1.html',
+            'search_term': 'video-1.html',
+        },
     ),
 )
 
 
-@pytest.mark.parametrize(("line", "exp_search_term"), BLACKLIST_TEST_CASES)
-def test_get_blacklisted_search_term(line, exp_search_term):
+@pytest.mark.parametrize(("line", "exp_search_info"), BLACKLIST_TEST_CASES)
+def test_get_blacklisted_search_term(line, exp_search_info):
     """Test search term extraction for lines that contain a blacklisted term"""
-    actual_search_term = get_error_search_term(line)
-    assert actual_search_term == exp_search_term
+    actual_search_info = get_error_search_term_and_path(line)
+    assert actual_search_info == exp_search_info