Skip to content

perf: use orjson in billing metrics #70579

New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Merged
anonrig merged 1 commit into from
May 9, 2024
Merged

perf: use orjson in billing metrics #70579

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
1 commit
Select commit
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
6 changes: 2 additions & 4 deletions src/sentry/ingest/billing_metrics_consumer.py
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -3,6 +3,7 @@
from datetime import datetime, timezone
from typing import Any, cast

import orjson
import sentry_sdk
from arroyo.backends.kafka import KafkaPayload
from arroyo.processing.strategies import (
Expand All @@ -27,7 +28,6 @@
from sentry.sentry_metrics.utils import reverse_resolve_tag_value
from sentry.snuba.metrics import parse_mri
from sentry.snuba.metrics.naming_layer.mri import is_custom_metric
from sentry.utils import json
from sentry.utils.outcomes import Outcome, track_outcome

logger = logging.getLogger(__name__)
Expand Down Expand Up @@ -87,9 +87,7 @@ def submit(self, message: Message[KafkaPayload]) -> None:
self.__next_step.submit(message)

def _get_payload(self, message: Message[KafkaPayload]) -> GenericMetric:
payload = json.loads(
message.payload.value.decode("utf-8"), use_rapid_json=True, skip_trace=True
)
payload = orjson.loads(message.payload.value)
return cast(GenericMetric, payload)

def _count_processed_items(self, generic_metric: GenericMetric) -> Mapping[DataCategory, int]:
Expand Down
4 changes: 2 additions & 2 deletions tests/sentry/ingest/billing_metrics_consumer/test_billing_metrics_consumer_kafka.py
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -4,6 +4,7 @@
from typing import cast
from unittest import mock

import orjson
from arroyo.backends.kafka import KafkaPayload
from arroyo.types import BrokerValue, Message, Partition, Topic
from django.core.cache import cache
Expand All @@ -23,7 +24,6 @@
)
from sentry.sentry_metrics.use_case_id_registry import UseCaseID
from sentry.testutils.pytest.fixtures import django_db_all
from sentry.utils import json
from sentry.utils.outcomes import Outcome


Expand Down Expand Up @@ -206,7 +206,7 @@ def test_outcomes_consumed(track_outcome, factories):
def generate_kafka_message(generic_metric: GenericMetric) -> Message[KafkaPayload]:
nonlocal generate_kafka_message_counter

encoded = json.dumps(generic_metric).encode()
encoded = orjson.dumps(generic_metric)
payload = KafkaPayload(key=None, value=encoded, headers=[])
message = Message(
BrokerValue(
Expand Down
4 changes: 2 additions & 2 deletions tests/sentry/ingest/ingest_consumer/test_dlq.py
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -3,6 +3,7 @@
from unittest.mock import Mock

import msgpack
import orjson
import pytest
from arroyo.backends.kafka import KafkaPayload
from arroyo.dlq import InvalidMessage
Expand All @@ -13,7 +14,6 @@
from sentry.ingest.consumer.factory import IngestStrategyFactory
from sentry.ingest.types import ConsumerType
from sentry.testutils.pytest.fixtures import django_db_all
from sentry.utils import json


def make_message(payload: bytes, partition: Partition, offset: int) -> Message:
Expand Down Expand Up @@ -60,7 +60,7 @@ def test_dlq_invalid_messages(factories, topic_name, consumer_type) -> None:
{
"type": "unsupported type",
"project_id": project.id,
"payload": json.dumps(sample_event).encode("utf-8"),
"payload": orjson.dumps(sample_event),
"start_time": int(time.time()),
"event_id": "aaa",
}
Expand Down
4 changes: 2 additions & 2 deletions tests/sentry/ingest/ingest_consumer/test_ingest_consumer_kafka.py
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -5,6 +5,7 @@
import uuid

import msgpack
import orjson
import pytest
from django.conf import settings

Expand All @@ -15,7 +16,6 @@
from sentry.eventstore.processing import event_processing_store
from sentry.testutils.pytest.fixtures import django_db_all
from sentry.testutils.skips import requires_kafka, requires_snuba
from sentry.utils import json
from sentry.utils.batching_kafka_consumer import create_topics
from sentry.utils.kafka_config import get_topic_definition

Expand Down Expand Up @@ -79,7 +79,7 @@ def inner(type, project=default_project):
"start_time": int(time.time()),
"event_id": event_id,
"project_id": int(project_id),
"payload": json.dumps(normalized_event),
"payload": orjson.dumps(normalized_event),
}

val = msgpack.packb(message)
Expand Down
24 changes: 12 additions & 12 deletions tests/sentry/ingest/ingest_consumer/test_ingest_consumer_processing.py
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -8,6 +8,7 @@
from typing import Any
from unittest.mock import Mock

import orjson
import pytest
from arroyo.backends.kafka.consumer import KafkaPayload
from arroyo.backends.local.backend import LocalBroker
Expand All @@ -30,7 +31,6 @@
from sentry.testutils.pytest.fixtures import django_db_all
from sentry.testutils.skips import requires_snuba
from sentry.usage_accountant import accountant
from sentry.utils import json
from sentry.utils.eventuser import EventUser
from sentry.utils.json import loads

Expand Down Expand Up @@ -87,7 +87,7 @@ def test_deduplication_works(default_project, task_runner, preprocess_event):
for _ in range(2):
process_event(
{
"payload": json.dumps(payload),
"payload": orjson.dumps(payload).decode(),
"start_time": start_time,
"event_id": event_id,
"project_id": project_id,
Expand Down Expand Up @@ -137,7 +137,7 @@ def test_transactions_spawn_save_event_transaction(
start_time = time.time() - 3600
process_event(
{
"payload": json.dumps(payload),
"payload": orjson.dumps(payload).decode(),
"start_time": start_time,
"event_id": event_id,
"project_id": project_id,
Expand Down Expand Up @@ -186,7 +186,7 @@ def test_accountant_transaction(default_project):
},
}
payload = get_normalized_event(event, default_project)
serialized = json.dumps(payload)
serialized = orjson.dumps(payload).decode()
process_event(
{
"payload": serialized,
Expand Down Expand Up @@ -239,7 +239,7 @@ def test_feedbacks_spawn_save_event_feedback(
start_time = time.time() - 3600
process_event(
{
"payload": json.dumps(payload),
"payload": orjson.dumps(payload).decode(),
"start_time": start_time,
"event_id": event_id,
"project_id": project_id,
Expand Down Expand Up @@ -291,7 +291,7 @@ def test_with_attachments(default_project, task_runner, missing_chunks, monkeypa
with task_runner():
process_event(
{
"payload": json.dumps(payload),
"payload": orjson.dumps(payload).decode(),
"start_time": start_time,
"event_id": event_id,
"project_id": project_id,
Expand Down Expand Up @@ -361,7 +361,7 @@ def test_deobfuscate_view_hierarchy(default_project, task_runner):

process_attachment_chunk(
{
"payload": json.dumps_htmlsafe(obfuscated_view_hierarchy).encode(),
"payload": orjson.dumps(obfuscated_view_hierarchy),
"event_id": event_id,
"project_id": project_id,
"id": attachment_id,
Expand All @@ -372,7 +372,7 @@ def test_deobfuscate_view_hierarchy(default_project, task_runner):
with task_runner():
process_event(
{
"payload": json.dumps(payload),
"payload": orjson.dumps(payload).decode(),
"start_time": start_time,
"event_id": event_id,
"project_id": project_id,
Expand Down Expand Up @@ -499,14 +499,14 @@ def test_userreport(django_cache, default_project, monkeypatch):
{
"type": "user_report",
"start_time": start_time,
"payload": json.dumps(
"payload": orjson.dumps(
{
"name": "Hans Gans",
"event_id": event_id,
"comments": "hello world",
"email": "markus+dontatme@sentry.io",
}
),
).decode(),
"project_id": default_project.id,
},
project=default_project,
Expand All @@ -530,14 +530,14 @@ def test_userreport_reverse_order(django_cache, default_project, monkeypatch):
{
"type": "user_report",
"start_time": start_time,
"payload": json.dumps(
"payload": orjson.dumps(
{
"name": "Hans Gans",
"event_id": event_id,
"comments": "hello world",
"email": "markus+dontatme@sentry.io",
}
),
).decode(),
"project_id": default_project.id,
},
project=default_project,
Expand Down