perf: move more implementations to use orjson (#70832)

src/sentry/apidocs/build.py

@@ -1,12 +1,12 @@
 from typing import Any
 
-from sentry.utils import json
+import orjson
 
 
 def get_old_json_paths(filename: str) -> Any:
     try:
-        with open(filename) as f:
-            old_raw_paths = json.load(f)["paths"]
+        with open(filename, "rb") as f:
+            old_raw_paths = orjson.loads(f.read())["paths"]
     except OSError:
         raise Exception(
             "Generate old OpenAPI files before running this command. Run `make build-api-docs` directly."
@@ -16,8 +16,8 @@ def get_old_json_paths(filename: str) -> Any:
 
 def get_old_json_components(filename: str) -> Any:
     try:
-        with open(filename) as f:
-            old_raw_components = json.load(f)["components"]
+        with open(filename, "rb") as f:
+            old_raw_components = orjson.loads(f.read())["components"]
     except OSError:
         raise Exception(
             "Generate old OpenAPI files before running this command. Run `make build-api-docs` directly."

src/sentry/conf/locale.py

@@ -1,7 +1,8 @@
 import os
 
+import orjson
+
 import sentry
-from sentry.utils import json
 
 
 # change locale file dir name to locale code
@@ -12,6 +13,6 @@ def dirname_to_local(dir_name):
     return dir_name
 
 
-with open(os.path.join(os.path.dirname(sentry.__file__), "locale", "catalogs.json")) as f:
-    CATALOGS = json.load(f)["supported_locales"]
+with open(os.path.join(os.path.dirname(sentry.__file__), "locale", "catalogs.json"), "rb") as f:
+    CATALOGS = orjson.loads(f.read())["supported_locales"]
     CATALOGS = [dirname_to_local(dirname) for dirname in CATALOGS]

src/sentry/data_export/models.py

@@ -1,5 +1,6 @@
 import logging
 
+import orjson
 from django.conf import settings
 from django.db import models, router, transaction
 from django.urls import reverse
@@ -17,7 +18,6 @@
 )
 from sentry.db.models.fields.hybrid_cloud_foreign_key import HybridCloudForeignKey
 from sentry.services.hybrid_cloud.user.service import user_service
-from sentry.utils import json
 
 from .base import DEFAULT_EXPIRATION, ExportQueryType, ExportStatus
 
@@ -124,7 +124,7 @@ def email_failure(self, message: str) -> None:
             context={
                 "creation": self.format_date(self.date_added),
                 "error_message": message,
-                "payload": json.dumps(self.payload),
+                "payload": orjson.dumps(self.payload).decode(),
             },
             type="organization.export-data",
             template="sentry/emails/data-export-failure.txt",

src/sentry/datascrubbing.py

@@ -13,14 +13,14 @@
     validate_pii_selector,
 )
 
-from sentry.utils import json, metrics
+from sentry.utils import metrics
 from sentry.utils.safe import safe_execute
 
 
 def get_pii_config(project):
     def _decode(value):
         if value:
-            return safe_execute(json.loads, value, _with_transaction=False)
+            return safe_execute(orjson.loads, value, _with_transaction=False)
 
     # Order of merging is important here. We want to apply organization rules
     # before project rules. For example:

src/sentry/hybridcloud/tasks/deliver_webhooks.py

@@ -2,6 +2,7 @@
 import logging
 from concurrent.futures import ThreadPoolExecutor, as_completed
 
+import orjson
 import sentry_sdk
 from django.db.models import Min, Subquery
 from django.utils import timezone
@@ -23,7 +24,7 @@
 from sentry.silo.client import RegionSiloClient, SiloClientError
 from sentry.tasks.base import instrumented_task
 from sentry.types.region import get_region_by_name
-from sentry.utils import json, metrics
+from sentry.utils import metrics
 
 logger = logging.getLogger(__name__)
 
@@ -378,7 +379,7 @@ def perform_request(payload: WebhookPayload) -> None:
             logging_context["request_method"] = payload.request_method
             logging_context["request_path"] = payload.request_path
 
-            headers = json.loads(payload.request_headers)
+            headers = orjson.loads(payload.request_headers)
             response = client.request(
                 method=payload.request_method,
                 path=payload.request_path,

src/sentry/identity/gitlab/provider.py

@@ -1,12 +1,13 @@
 import logging
 
+import orjson
+
 from sentry import http
 from sentry.auth.exceptions import IdentityNotValid
 from sentry.http import safe_urlopen, safe_urlread
 from sentry.identity.oauth2 import OAuth2Provider
 from sentry.services.hybrid_cloud.identity import identity_service
 from sentry.services.hybrid_cloud.identity.model import RpcIdentity
-from sentry.utils import json
 from sentry.utils.http import absolute_uri
 
 logger = logging.getLogger("sentry.integration.gitlab")
@@ -104,7 +105,7 @@ def refresh_identity(self, identity: RpcIdentity, *args, **kwargs):
 
         try:
             body = safe_urlread(req)
-            payload = json.loads(body)
+            payload = orjson.loads(body)
         except Exception as e:
             # JSONDecodeError's will happen when we get a 301
             # from GitLab, and won't have the `code` attribute

src/sentry/identity/google/provider.py

@@ -1,8 +1,9 @@
+import orjson
+
 from sentry import options
 from sentry.auth.exceptions import IdentityNotValid
 from sentry.auth.provider import MigratingIdentityId
 from sentry.identity.oauth2 import OAuth2Provider
-from sentry.utils import json
 from sentry.utils.signing import urlsafe_b64decode
 
 # When no hosted domain is in use for the authenticated user, we default to the
@@ -41,7 +42,7 @@ def build_identity(self, state):
             raise IdentityNotValid("Unable to decode id_token: %s" % exc)
 
         try:
-            user_data = json.loads(payload)
+            user_data = orjson.loads(payload)
         except ValueError as exc:
             raise IdentityNotValid("Unable to decode id_token payload: %s" % exc)
 

src/sentry/identity/oauth2.py

@@ -5,6 +5,7 @@
 from time import time
 from urllib.parse import parse_qsl, urlencode
 
+import orjson
 from django.http import HttpResponse
 from django.utils.decorators import method_decorator
 from django.views.decorators.csrf import csrf_exempt
@@ -14,7 +15,6 @@
 from sentry.http import safe_urlopen, safe_urlread
 from sentry.pipeline import PipelineView
 from sentry.shared_integrations.exceptions import ApiError
-from sentry.utils import json
 from sentry.utils.http import absolute_uri
 
 from .base import Provider
@@ -194,8 +194,8 @@ def refresh_identity(self, identity, *args, **kwargs):
 
         try:
             body = safe_urlread(req)
-            payload = json.loads(body)
-        except Exception:
+            payload = orjson.loads(body)
+        except orjson.JSONDecodeError:
             payload = {}
 
         self.handle_refresh_error(req, payload)
@@ -288,7 +288,7 @@ def exchange_token(self, request: Request, pipeline, code):
             body = safe_urlread(req)
             if req.headers.get("Content-Type", "").startswith("application/x-www-form-urlencoded"):
                 return dict(parse_qsl(body))
-            return json.loads(body)
+            return orjson.loads(body)
         except SSLError:
             logger.info(
                 "identity.oauth2.ssl-error",
@@ -306,7 +306,7 @@ def exchange_token(self, request: Request, pipeline, code):
                 "error": "Could not connect to host or service",
                 "error_description": f"Ensure that {url} is open to connections",
             }
-        except json.JSONDecodeError:
+        except orjson.JSONDecodeError:
             logger.info("identity.oauth2.json-error", extra={"url": self.access_token_url})
             return {
                 "error": "Could not decode a JSON Response",

src/sentry/identity/vsts/provider.py

@@ -1,3 +1,4 @@
+import orjson
 from django.core.exceptions import PermissionDenied
 from rest_framework.request import Request
 
@@ -117,7 +118,6 @@ def exchange_token(self, request: Request, pipeline, code):
         from urllib.parse import parse_qsl
 
         from sentry.http import safe_urlopen, safe_urlread
-        from sentry.utils import json
         from sentry.utils.http import absolute_uri
 
         req = safe_urlopen(
@@ -134,4 +134,4 @@ def exchange_token(self, request: Request, pipeline, code):
         body = safe_urlread(req)
         if req.headers["Content-Type"].startswith("application/x-www-form-urlencoded"):
             return dict(parse_qsl(body))
-        return json.loads(body)
+        return orjson.loads(body)

src/sentry/incidents/action_handlers.py

@@ -5,6 +5,7 @@
 from collections.abc import Sequence
 from urllib.parse import urlencode
 
+import orjson
 from django.conf import settings
 from django.template.defaultfilters import pluralize
 from django.urls import reverse
@@ -25,7 +26,6 @@
 from sentry.snuba.metrics import format_mri_field, is_mri_field
 from sentry.types.actor import Actor, ActorType
 from sentry.types.integrations import ExternalProviders
-from sentry.utils import json
 from sentry.utils.email import MessageBuilder, get_email_addresses
 
 
@@ -196,7 +196,7 @@ def build_message(self, context, status, user_id) -> MessageBuilder:
             html_template="sentry/emails/incidents/trigger.html",
             type=f"incident.alert_rule_{display.lower()}",
             context=context,
-            headers={"X-SMTPAPI": json.dumps({"category": "metric_alert_email"})},
+            headers={"X-SMTPAPI": orjson.dumps({"category": "metric_alert_email"}).decode()},
         )
 
 

src/sentry/loader/browsersdkversion.py

@@ -3,11 +3,11 @@
 import os
 import re
 
+import orjson
 from django.conf import settings
 from packaging.version import Version
 
 import sentry
-from sentry.utils import json
 
 logger = logging.getLogger("sentry")
 
@@ -22,7 +22,7 @@ def load_registry(path):
     fn = os.path.join(LOADER_FOLDER, path + ".json")
     try:
         with open(fn, "rb") as f:
-            return json.load(f)
+            return orjson.loads(f.read())
     except OSError:
         return None
 

src/sentry/mail/notifications.py

@@ -4,6 +4,7 @@
 from collections.abc import Iterable, Mapping, MutableMapping
 from typing import TYPE_CHECKING, Any
 
+import orjson
 import sentry_sdk
 from django.utils.encoding import force_str
 
@@ -16,7 +17,6 @@
 from sentry.notifications.types import UnsubscribeContext
 from sentry.types.actor import Actor
 from sentry.types.integrations import ExternalProviders
-from sentry.utils import json
 from sentry.utils.email import MessageBuilder, group_id_to_email
 from sentry.utils.linksign import generate_signed_unsubscribe_link
 
@@ -27,7 +27,7 @@
 
 
 def get_headers(notification: BaseNotification) -> Mapping[str, Any]:
-    headers = {"X-SMTPAPI": json.dumps({"category": notification.metrics_key})}
+    headers = {"X-SMTPAPI": orjson.dumps({"category": notification.metrics_key}).decode()}
     if isinstance(notification, ProjectNotification) and notification.project.slug:
         headers["X-Sentry-Project"] = notification.project.slug
 

src/sentry/testutils/factories.py

@@ -15,6 +15,7 @@
 from unittest import mock
 from uuid import uuid4
 
+import orjson
 import petname
 from django.conf import settings
 from django.contrib.auth.models import AnonymousUser
@@ -150,7 +151,7 @@
 from sentry.types.integrations import ExternalProviders
 from sentry.types.region import Region, get_local_region, get_region_by_name
 from sentry.types.token import AuthTokenType
-from sentry.utils import json, loremipsum
+from sentry.utils import loremipsum
 from sentry.utils.performance_issues.performance_problem import PerformanceProblem
 from social_auth.models import UserSocialAuth
 
@@ -285,7 +286,7 @@ def make_word(words=None):
 
 def _patch_artifact_manifest(path, org=None, release=None, project=None, extra_files=None):
     with open(path, "rb") as fp:
-        manifest = json.load(fp)
+        manifest = orjson.loads(fp.read())
     if org:
         manifest["org"] = org
     if release:
@@ -294,7 +295,7 @@ def _patch_artifact_manifest(path, org=None, release=None, project=None, extra_f
         manifest["project"] = project
     for path in extra_files or {}:
         manifest["files"][path] = {"url": path}
-    return json.dumps(manifest)
+    return orjson.dumps(manifest).decode()
 
 
 # TODO(dcramer): consider moving to something more scalable like factoryboy