Upgrade common requirements: Utils

manage.py

@@ -8,26 +8,6 @@
 # https://docs.python.org/3.7/library/warnings.html#default-warning-filters
 warnings.simplefilter('default', DeprecationWarning)
 
-# Suppress noisy warnings from dependencies
-
-# Reported in https://support.newrelic.com/tickets/338064
-warnings.filterwarnings('ignore', category=DeprecationWarning, module='newrelic')
-
-# https://github.com/encode/django-rest-framework/issues/6317
-warnings.filterwarnings('ignore', category=DeprecationWarning, module='markdown.util')
-
-# "Using or importing the ABCs from 'collections' instead of from 'collections.abc' is deprecated, and in 3.8 it will stop working"
-# corsheaders/checks.py -> https://github.com/ottoyiu/django-cors-headers/issues/374
-# jinja2/runtime.py -> https://github.com/pallets/jinja/pull/867
-# orderedmultidict/orderedmultidict.py -> https://github.com/gruns/orderedmultidict/pull/20
-# promise/promise_list.py -> https://github.com/syrusakbary/promise/pull/67
-warnings.filterwarnings(
-    'ignore', category=DeprecationWarning, message=r'Using or importing the ABCs .*'
-)
-
-# "the imp module is deprecated in favour of importlib; see the module's documentation for alternative uses"
-warnings.filterwarnings('ignore', category=DeprecationWarning, module='celery.utils.imports')
-
 if __name__ == "__main__":
     os.environ["DJANGO_SETTINGS_MODULE"] = "treeherder.config.settings"
 

requirements/common.in

@@ -1,47 +1,44 @@
 # Packages that are shared between deployment and dev environments.
 gunicorn==20.1.0
-whitenoise[brotli]==5.2.0  # Used by Whitenoise to provide Brotli-compressed versions of static files.
+whitenoise[brotli]==6.5.0  # Used by Whitenoise to provide Brotli-compressed versions of static files.
 Django==4.1.9
 celery==5.3.1  # celery needed for data ingestion
 cached-property==1.5.2 # needed for kombu with --require-hashes
-simplejson==3.17.2  # import simplejson
-newrelic==5.22.1.152
-certifi==2022.12.7
+simplejson==3.19.1  # import simplejson
+newrelic==8.8.0
+certifi==2023.5.7
 
 mysqlclient==2.1.1   # Required by Django
 
-jsonschema==3.2.0  # import jsonschema
+jsonschema==4.17.3  # import jsonschema
 djangorestframework==3.14.0  # Imported as rest_framework
 django-cors-headers==4.1.0  # Listed as 3rd party app on settings.py
-mozlog==7.1.0
-
-coreapi==2.3.3  # `coreapi` must be installed for schema support.
+mozlog==8.0.0
 
 # Used directly and also by Django's YAML serializer.
-PyYAML==5.4.1  # Imported as yaml
+PyYAML==6.0  # Imported as yaml
 django-environ==0.10.0  # Imported as environ
 
 python-dateutil==2.8.2
 django-filter==23.2  # Listed in DEFAULT_FILTER_BACKENDS on settings.py
 django-redis==5.3.0  # Listed in CACHES on settings.py
 
-taskcluster==42.1.1  # import taskcluster
-python-jose[pycryptodome]==3.2.0  # from jose import jwt
+taskcluster==53.2.1  # import taskcluster
+python-jose[pycryptodome]==3.3.0  # from jose import jwt
 
-furl==2.1.0  # Imported as furl
+furl==2.1.3  # Imported as furl
 first==2.0.2  # Imported as first
-json-e==4.4.1  # import jsone
+json-e==4.5.2  # import jsone
 django-cache-memoize==0.1.10  # Imported as cache_memoize
 
 # Required for Push Health
-mozci==1.12.6
-
-importlib-metadata==3.7.3
+mozci[cache]==2.3.2
 
 # Dockerflow/CloudOps APIs
 dockerflow==2022.8.0
 
 # Measuring noise of perf data
-moz-measure-noise==2.60.0.2
+moz-measure-noise==2.60.1
 
-typed-ast==1.5.2
+# Used in the intermittents commenter
+jinja2==3.1.2

requirements/dev.in

@@ -39,6 +39,3 @@ pip-tools==6.13.0
 
 requests==2.31.0
 urllib3==2.0.3
-
-# TODO: Clean this once other packages are upgraded, see https://github.com/pypa/pip/issues/11975
-setuptools==65.6.3

requirements/dev.txt

@@ -313,15 +313,15 @@ pip-tools==6.13.0 \
     --hash=sha256:50943f151d87e752abddec8158622c34ad7f292e193836e90e30d87da60b19d9 \
     --hash=sha256:61d46bd2eb8016ed4a924e196e6e5b0a268cd3babd79e593048720db23522bb1
     # via -r requirements/dev.in
-platformdirs==3.5.3 \
-    --hash=sha256:0ade98a4895e87dc51d47151f7d2ec290365a585151d97b4d8d6312ed6132fed \
-    --hash=sha256:e48fabd87db8f3a7df7150a4a5ea22c546ee8bc39bc2473244730d4b56d2cc4e
+platformdirs==3.8.0 \
+    --hash=sha256:b0cabcb11063d21a0b261d557acb0a9d2126350e63b70cdf7db6347baea456dc \
+    --hash=sha256:ca9ed98ce73076ba72e092b23d3c93ea6c4e186b3f1c3dad6edd98ff6ffcca2e
     # via
     #   black
     #   virtualenv
-pluggy==1.0.0 \
-    --hash=sha256:4224373bacce55f955a878bf9cfa763c1e360858e330072059e10bad68531159 \
-    --hash=sha256:74134bbf457f031a36d68416e1509f34bd5ccc019f0bcc952c7b909d06b37bd3
+pluggy==1.2.0 \
+    --hash=sha256:c2fd55a7d7a3863cba1a013e4e2414658b1d07b6bc57b3919e0c63c9abb99849 \
+    --hash=sha256:d12f0c4b579b15f5e054301bb226ee85eeeba08ffec228092f8defbaa3a4c4b3
     # via
     #   pypom
     #   pytest
@@ -493,9 +493,9 @@ types-pyyaml==6.0.12.10 \
     --hash=sha256:662fa444963eff9b68120d70cda1af5a5f2aa57900003c2006d7626450eaae5f \
     --hash=sha256:ebab3d0700b946553724ae6ca636ea932c1b0868701d4af121630e78d695fc97
     # via responses
-typing-extensions==4.6.3 \
-    --hash=sha256:88a4153d8505aabbb4e13aacb7c486c2b4a33ca3b3f807914a9b4c844c471c26 \
-    --hash=sha256:d91d5919357fe7f681a9f2b5b4cb2a5f1ef0a1e9f59c4d8ff0d3491e05c0ffd5
+typing-extensions==4.7.0 \
+    --hash=sha256:5d8c9dac95c27d20df12fb1d97b9793ab8b2af8a3a525e68c80e21060c161771 \
+    --hash=sha256:935ccf31549830cda708b42289d44b6f74084d616a00be651601a4f968e77c82
     # via
     #   asgiref
     #   black
@@ -507,9 +507,9 @@ urllib3[socks]==2.0.3 \
     #   requests
     #   responses
     #   selenium
-virtualenv==20.23.0 \
-    --hash=sha256:6abec7670e5802a528357fdc75b26b9f57d5d92f29c5462ba0fbe45feacc685e \
-    --hash=sha256:a85caa554ced0c0afbd0d638e7e2d7b5f92d23478d05d17a76daeac8f279f924
+virtualenv==20.23.1 \
+    --hash=sha256:34da10f14fea9be20e0fd7f04aba9732f84e593dac291b757ce42e3368a39419 \
+    --hash=sha256:8ff19a38c1021c742148edc4f81cb43d7f8c6816d2ede2ab72af5b84c749ade1
     # via pre-commit
 watchdog==3.0.0 \
     --hash=sha256:0e06ab8858a76e1219e68c7573dfeba9dd1c0219476c5a44d5333b01d7e1743a \
@@ -552,9 +552,9 @@ zope-component==6.0 \
     --hash=sha256:96d0a04db39643caf2dfaec152340f3e914df1dc3fa32fbb913782620dc6c3c6 \
     --hash=sha256:9a0a0472ad201b94b4fe6741ce9ac2c30b8bb22c516077bf03692dec4dfb6906
     # via pypom
-zope-event==4.6 \
-    --hash=sha256:73d9e3ef750cca14816a9c322c7250b0d7c9dbc337df5d1b807ff8d3d0b9e97c \
-    --hash=sha256:81d98813046fc86cc4136e3698fee628a3282f9c320db18658c21749235fce80
+zope-event==5.0 \
+    --hash=sha256:2832e95014f4db26c47a13fdaef84cef2f4df37e66b59d8f1f4a8f319a632c26 \
+    --hash=sha256:bac440d8d9891b4068e2b5a2c5e2c9765a9df762944bda6955f96bb9b91e67cd
     # via zope-component
 zope-hookable==5.4 \
     --hash=sha256:0054539ed839751b7f511193912cba393f0b8b5f7dfe9f3601c65b2d3b74e731 \
@@ -634,11 +634,10 @@ pip==23.1.2 \
     --hash=sha256:0e7c86f486935893c708287b30bd050a36ac827ec7fe5e43fe7cb198dd835fba \
     --hash=sha256:3ef6ac33239e4027d9a5598a381b9d30880a1477e50039db2eac6e8a8f6d1b18
     # via pip-tools
-setuptools==65.6.3 \
-    --hash=sha256:57f6f22bde4e042978bcd50176fdb381d7c21a9efa4041202288d3737a0c6a54 \
-    --hash=sha256:a7620757bf984b58deaf32fc8a4577a9bbc0850cf92c20e1ce41c38c19e5fb75
+setuptools==68.0.0 \
+    --hash=sha256:11e52c67415a381d10d6b462ced9cfb97066179f0e871399e006c4ab101fc85f \
+    --hash=sha256:baf1fdb41c6da4cd2eae722e135500da913332ab3f2f5c7d33af9b492acb5235
     # via
-    #   -r requirements/dev.in
     #   nodeenv
     #   pip-tools
     #   zope-component

setup.cfg

@@ -21,10 +21,14 @@ filterwarnings =
     error
     ignore::ImportWarning
     ignore::PendingDeprecationWarning
+    # Gunicorn uses deprecated calls to pkg_resources, this issue is tracked in
+    # https://github.com/benoitc/gunicorn/issues/2840 and will likely be fixed in
+    # version 21.0.0.
+    ignore:.*pkg_resources.*:DeprecationWarning
     # WhiteNoise warns if either `.django-static/` or `.build/` do not exist at startup,
     # however this is expected when running tests since Django collectstatic and yarn build
     # (which create those directories) typically aren't run apart from during deployments.
-    ignore:No directory at.*:UserWarning:whitenoise.base
+    ignore:No directory at.*:UserWarning
 markers =
     slow: mark a test as slow.
 xfail_strict = true

tests/push_health/test_compare.py

@@ -35,26 +35,7 @@ def mock_rev(test_push):
 def mock_json_pushes(test_push):
     responses.add(
         responses.GET,
-        f'https://hg.mozilla.org/{test_push.repository.name}/json-pushes?version=2&startID=536017&endID=536018',
-        json={
-            "pushes": {
-                "536018": {
-                    "changesets": [
-                        "abcdef77949168d16c03a4cba167678b7ab65f76",
-                        "084c7f0fcde34f813013a96423e6bec18837ead4",
-                        "81ab76256e9a2198a2c9f368d260c2f46ef749a0",
-                    ],
-                    "date": 1589318625,
-                    "user": "user@example.org",
-                }
-            },
-        },
-        content_type='application/json',
-        status=200,
-    )
-    responses.add(
-        responses.GET,
-        f'https://hg.mozilla.org/{test_push.repository.name}/json-pushes?version=2&startID=536015&endID=536016',
+        f'https://hg.mozilla.org/{test_push.repository.name}/json-pushes?version=2&full=1&startID=536015&endID=536016',
         json={
             "pushes": {
                 "536016": {

tests/test_middleware.py

@@ -5,7 +5,7 @@
 from treeherder.middleware import CustomWhiteNoise
 
 URLS_IMMUTABLE = [
-    # Assets generated by Neutrino.
+    # Assets generated by Yarn.
     '/assets/2.379789df.css',
     '/assets/dancing_cat.fa5552a5.gif',
     '/assets/fontawesome-webfont.af7ae505.woff2',
@@ -24,7 +24,7 @@
     '/revision.txt',
     '/tree_open.png',
     '/docs/schema.js',
-    # The unhashed Neutrino/webpack output if using `yarn build --mode development`.
+    # The unhashed Yarn/webpack output if using `yarn build --mode development`.
     '/assets/runtime.js',
     '/assets/vendors~index.js',
     # The unhashed Django static asset originals (used in development).

treeherder/config/settings.py

@@ -9,6 +9,7 @@
 from kombu import Exchange, Queue
 
 from treeherder.config.utils import connection_should_use_tls
+from treeherder.middleware import add_headers_function
 
 # TODO: Switch to pathlib once using Python 3.
 SRC_DIR = dirname(dirname(dirname(abspath(__file__))))
@@ -394,21 +395,23 @@
         'rest_framework.renderers.JSONRenderer',
         'rest_framework.renderers.BrowsableAPIRenderer',
     ),
-    'DEFAULT_SCHEMA_CLASS': 'rest_framework.schemas.coreapi.AutoSchema',
+    'DEFAULT_SCHEMA_CLASS': 'rest_framework.schemas.openapi.AutoSchema',
     'DEFAULT_VERSION': '1.0',
     'DEFAULT_VERSIONING_CLASS': 'rest_framework.versioning.AcceptHeaderVersioning',
     'TEST_REQUEST_DEFAULT_FORMAT': 'json',
 }
 
 # Whitenoise
-# http://whitenoise.evans.io/en/stable/django.html#available-settings
+# https://whitenoise.readthedocs.io/en/stable/django.html#available-settings
 # Files in this directory will be served by WhiteNoise at the site root.
 WHITENOISE_ROOT = join(SRC_DIR, ".build")
 # Serve index.html for URLs ending in a trailing slash.
 WHITENOISE_INDEX_FILE = True
 # Only output the hashed filename version of static files and not the originals.
 # Halves the time spent performing Brotli/gzip compression during deploys.
 WHITENOISE_KEEP_ONLY_HASHED_FILES = True
+# Add a `Content-Security-Policy` header to all static file responses.
+WHITENOISE_ADD_HEADERS_FUNCTION = add_headers_function
 
 # Templating
 TEMPLATES = [

treeherder/config/urls.py

@@ -1,6 +1,6 @@
 from django.conf import settings
 from django.urls import include, re_path
-from rest_framework.documentation import include_docs_urls
+from rest_framework.schemas import get_schema_view
 
 from treeherder.webapp.api import urls as api_urls
 from django.views.generic.base import TemplateView
@@ -18,6 +18,12 @@
 
 urlpatterns += [
     re_path(r'^api/', include(api_urls)),
-    re_path(r'^docs/', include_docs_urls(title='REST API Docs')),
+    re_path(r'^openapi/', get_schema_view(title='REST API Docs'), name='openapi-schema'),
+    re_path(
+        r'^docs/',
+        TemplateView.as_view(
+            template_name='redoc.html', extra_context={'schema_url': 'openapi-schema'}
+        ),
+    ),
     re_path(r'', TemplateView.as_view(template_name='index.html')),
 ]

treeherder/etl/classification_loader.py

@@ -106,15 +106,15 @@ def get_push(self, task_route):
             raise
 
         try:
-            newrelic.agent.add_custom_parameter("project", project)
+            newrelic.agent.add_custom_attribute("project", project)
 
             repository = Repository.objects.get(name=project)
         except Repository.DoesNotExist:
             logger.info("Job with unsupported project: %s", project)
             raise
 
         try:
-            newrelic.agent.add_custom_parameter("revision", revision)
+            newrelic.agent.add_custom_attribute("revision", revision)
 
             revision_field = 'revision__startswith' if len(revision) < 40 else 'revision'
             filter_kwargs = {'repository': repository, revision_field: revision}

treeherder/etl/job_loader.py

@@ -61,7 +61,7 @@ def process_job(self, pulse_job, root_url):
         if self._is_valid_job(pulse_job):
             try:
                 project = pulse_job["origin"]["project"]
-                newrelic.agent.add_custom_parameter("project", project)
+                newrelic.agent.add_custom_attribute("project", project)
 
                 repository = Repository.objects.get(name=project)
                 if repository.active_status != 'active':

treeherder/etl/jobs.py

@@ -462,7 +462,7 @@ def store_job_data(repository, originalData):
             # make more fields visible in new relic for the job
             # where we encountered the error
             datum.update(datum.get("job", {}))
-            newrelic.agent.record_exception(params=datum)
+            newrelic.agent.notice_error(attributes=datum)
 
             # skip any jobs that hit errors in these stages.
             continue

treeherder/etl/push_loader.py

@@ -20,15 +20,15 @@ class PushLoader:
     def process(self, message_body, exchange, root_url):
         transformer = self.get_transformer_class(exchange)(message_body)
         try:
-            newrelic.agent.add_custom_parameter("url", transformer.repo_url)
-            newrelic.agent.add_custom_parameter("branch", transformer.branch)
+            newrelic.agent.add_custom_attribute("url", transformer.repo_url)
+            newrelic.agent.add_custom_attribute("branch", transformer.branch)
             repos = Repository.objects
             if transformer.branch:
                 repos = repos.filter(branch__regex="(^|,)%s($|,)" % transformer.branch)
             else:
                 repos = repos.filter(branch=None)
             repo = repos.get(url=transformer.repo_url, active_status="active")
-            newrelic.agent.add_custom_parameter("repository", repo.name)
+            newrelic.agent.add_custom_attribute("repository", repo.name)
         except ObjectDoesNotExist:
             repo_info = transformer.get_info()
             repo_info.update(
@@ -257,7 +257,7 @@ def transform(self, repository):
         return self.fetch_push(url, repository)
 
     def fetch_push(self, url, repository, sha=None):
-        newrelic.agent.add_custom_parameter("sha", sha)
+        newrelic.agent.add_custom_attribute("sha", sha)
 
         logger.debug("fetching for %s %s", repository, url)
         # there will only ever be one, with this url

treeherder/etl/pushlog.py

@@ -126,7 +126,7 @@ def run(self, source_url, repository_name, changeset=None, last_push_id=None):
             try:
                 store_push(repository, self.transform_push(push))
             except Exception:
-                newrelic.agent.record_exception()
+                newrelic.agent.notice_error()
                 errors.append(
                     {
                         "project": repository,

treeherder/etl/tasks/pulse_tasks.py

@@ -23,8 +23,8 @@ def store_pulse_tasks(
     Fetches tasks from Taskcluster
     """
     loop = asyncio.get_event_loop()
-    newrelic.agent.add_custom_parameter("exchange", exchange)
-    newrelic.agent.add_custom_parameter("routing_key", routing_key)
+    newrelic.agent.add_custom_attribute("exchange", exchange)
+    newrelic.agent.add_custom_attribute("routing_key", routing_key)
     # handleMessage expects messages in this format
     runs = loop.run_until_complete(
         handleMessage(
@@ -47,8 +47,8 @@ def store_pulse_pushes(
     """
     Fetches the pushes pending from pulse exchanges and loads them.
     """
-    newrelic.agent.add_custom_parameter("exchange", exchange)
-    newrelic.agent.add_custom_parameter("routing_key", routing_key)
+    newrelic.agent.add_custom_attribute("exchange", exchange)
+    newrelic.agent.add_custom_attribute("routing_key", routing_key)
 
     PushLoader().process(body, exchange, root_url)
 
@@ -63,7 +63,7 @@ def store_pulse_tasks_classification(
     By default, it should listen to the Community cluster as classifications
     are only running there for the moment
     """
-    newrelic.agent.add_custom_parameter("exchange", exchange)
-    newrelic.agent.add_custom_parameter("routing_key", routing_key)
+    newrelic.agent.add_custom_attribute("exchange", exchange)
+    newrelic.agent.add_custom_attribute("routing_key", routing_key)
 
     ClassificationLoader().process(pulse_job, root_url)