diff --git a/pyproject.toml b/pyproject.toml index 126d1253f5610c..98b8774849d3eb 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -50,7 +50,7 @@ dependencies = [ "mmh3>=4.0.0", "msgspec>=0.19.0", "msgpack>=1.1.0", - "objectstore-client>=0.0.5", + "objectstore-client>=0.0.10", "openai>=1.3.5", "orjson>=3.10.10", "packaging>=24.1", @@ -797,7 +797,6 @@ module = [ "tests.sentry.notifications.notifications.organization_request.*", "tests.sentry.notifications.platform.*", "tests.sentry.notifications.utils.*", - "tests.sentry.objectstore.*", "tests.sentry.onboarding_tasks.*", "tests.sentry.organizations.*", "tests.sentry.partnerships.*", diff --git a/src/sentry/attachments/__init__.py b/src/sentry/attachments/__init__.py index 9eabaa957d77ad..a0cb6557d04204 100644 --- a/src/sentry/attachments/__init__.py +++ b/src/sentry/attachments/__init__.py @@ -15,8 +15,7 @@ import sentry_sdk from django.conf import settings -from sentry.objectstore import Client as ObjectstoreClient -from sentry.objectstore import get_attachments_client +from sentry.objectstore import get_attachments_session from sentry.options.rollout import in_random_rollout from sentry.utils.cache import cache_key_for_event from sentry.utils.imports import import_string @@ -81,13 +80,12 @@ def delete_cached_and_ratelimited_attachments( Non-ratelimited attachments which are already stored in `objectstore` will be retained there for long-term storage. """ - client: ObjectstoreClient | None = None for attachment in attachments: # deletes from objectstore if no long-term storage is desired if attachment.rate_limited and attachment.stored_id: - if client is None: - client = get_attachments_client().for_project(project.organization_id, project.id) - client.delete(attachment.stored_id) + get_attachments_session(project.organization_id, project.id).delete( + attachment.stored_id + ) # unconditionally deletes any payloads from the attachment cache attachment.delete() diff --git a/src/sentry/lang/native/symbolicator.py b/src/sentry/lang/native/symbolicator.py index 78be8cc3ccc0be..0567dbd617abed 100644 --- a/src/sentry/lang/native/symbolicator.py +++ b/src/sentry/lang/native/symbolicator.py @@ -26,7 +26,7 @@ from sentry.lang.native.utils import Backoff from sentry.models.project import Project from sentry.net.http import Session -from sentry.objectstore import get_attachments_client +from sentry.objectstore import get_attachments_session from sentry.options.rollout import in_random_rollout from sentry.utils import metrics @@ -180,16 +180,12 @@ def process_minidump( "objectstore.force-stored-symbolication" ) if force_stored_attachment: - client = get_attachments_client().for_project( - self.project.organization_id, self.project.id - ) - minidump.stored_id = client.put(minidump.data) + session = get_attachments_session(self.project.organization_id, self.project.id) + minidump.stored_id = session.put(minidump.data) if minidump.stored_id: - client = get_attachments_client().for_project( - self.project.organization_id, self.project.id - ) - storage_url = client.object_url(minidump.stored_id) + session = get_attachments_session(self.project.organization_id, self.project.id) + storage_url = session.object_url(minidump.stored_id) json: dict[str, Any] = { "platform": platform, "sources": sources, @@ -206,7 +202,7 @@ def process_minidump( return process_response(res) finally: if force_stored_attachment: - client.delete(minidump.stored_id) + session.delete(minidump.stored_id) minidump.stored_id = None data = { @@ -229,16 +225,12 @@ def process_applecrashreport(self, platform: str, report: CachedAttachment): "objectstore.force-stored-symbolication" ) if force_stored_attachment: - client = get_attachments_client().for_project( - self.project.organization_id, self.project.id - ) - report.stored_id = client.put(report.data) + session = get_attachments_session(self.project.organization_id, self.project.id) + report.stored_id = session.put(report.data) if report.stored_id: - client = get_attachments_client().for_project( - self.project.organization_id, self.project.id - ) - storage_url = client.object_url(report.stored_id) + session = get_attachments_session(self.project.organization_id, self.project.id) + storage_url = session.object_url(report.stored_id) json: dict[str, Any] = { "platform": platform, "sources": sources, @@ -254,7 +246,7 @@ def process_applecrashreport(self, platform: str, report: CachedAttachment): return process_response(res) finally: if force_stored_attachment: - client.delete(report.stored_id) + session.delete(report.stored_id) report.stored_id = None data = { diff --git a/src/sentry/models/eventattachment.py b/src/sentry/models/eventattachment.py index 1e5bab96d939a1..aa6807eb503ec4 100644 --- a/src/sentry/models/eventattachment.py +++ b/src/sentry/models/eventattachment.py @@ -18,7 +18,7 @@ from sentry.db.models.fields.bounded import BoundedIntegerField from sentry.db.models.manager.base_query_set import BaseQuerySet from sentry.models.files.utils import get_size_and_checksum, get_storage -from sentry.objectstore import get_attachments_client +from sentry.objectstore import get_attachments_session from sentry.objectstore.metrics import measure_storage_operation from sentry.options.rollout import in_random_rollout from sentry.utils import metrics @@ -130,15 +130,15 @@ def delete(self, *args: Any, **kwargs: Any) -> tuple[int, dict[str, int]]: if blob_path.startswith(V2_PREFIX): try: organization_id = _get_organization(self.project_id) - get_attachments_client().for_project( - organization_id, self.project_id - ).delete(blob_path.removeprefix(V2_PREFIX)) + get_attachments_session(organization_id, self.project_id).delete( + blob_path.removeprefix(V2_PREFIX) + ) except Exception: sentry_sdk.capture_exception() elif self.blob_path.startswith(V2_PREFIX): organization_id = _get_organization(self.project_id) - get_attachments_client().for_project(organization_id, self.project_id).delete( + get_attachments_session(organization_id, self.project_id).delete( self.blob_path.removeprefix(V2_PREFIX) ) @@ -169,9 +169,7 @@ def getfile(self) -> IO[bytes]: elif self.blob_path.startswith(V2_PREFIX): id = self.blob_path.removeprefix(V2_PREFIX) organization_id = _get_organization(self.project_id) - response = ( - get_attachments_client().for_project(organization_id, self.project_id).get(id) - ) + response = get_attachments_session(organization_id, self.project_id).get(id) return response.payload raise NotImplementedError() @@ -203,9 +201,7 @@ def putfile(cls, project_id: int, attachment: CachedAttachment) -> PutfileResult if in_random_rollout("objectstore.double_write.attachments"): try: organization_id = _get_organization(project_id) - get_attachments_client().for_project(organization_id, project_id).put( - data, id=object_key - ) + get_attachments_session(organization_id, project_id).put(data, key=object_key) metrics.incr("storage.attachments.double_write") blob_path += V2_PREFIX except Exception: @@ -220,9 +216,7 @@ def putfile(cls, project_id: int, attachment: CachedAttachment) -> PutfileResult else: organization_id = _get_organization(project_id) - blob_path = V2_PREFIX + get_attachments_client().for_project( - organization_id, project_id - ).put(data) + blob_path = V2_PREFIX + get_attachments_session(organization_id, project_id).put(data) return PutfileResult( content_type=content_type, size=size, sha1=checksum, blob_path=blob_path diff --git a/src/sentry/objectstore/__init__.py b/src/sentry/objectstore/__init__.py index ead6115fcedec2..83144e48fb69c7 100644 --- a/src/sentry/objectstore/__init__.py +++ b/src/sentry/objectstore/__init__.py @@ -1,13 +1,11 @@ from datetime import timedelta -from objectstore_client import Client, ClientBuilder, ClientError, MetricsBackend, TimeToLive +from objectstore_client import Client, MetricsBackend, Session, TimeToLive, Usecase from objectstore_client.metrics import Tags from sentry.utils import metrics as sentry_metrics -__all__ = ["get_attachments_client", "Client", "ClientBuilder", "ClientError"] - -_attachments_client: ClientBuilder | None = None +__all__ = ["get_attachments_session"] class SentryMetricsBackend(MetricsBackend): @@ -35,16 +33,19 @@ def distribution( sentry_metrics.distribution(name, value, tags=tags, unit=unit) -def get_attachments_client() -> ClientBuilder: - global _attachments_client - if not _attachments_client: +_ATTACHMENTS_CLIENT: Client | None = None +_ATTACHMENTS_USECASE = Usecase("attachments", expiration_policy=TimeToLive(timedelta(days=30))) + + +def get_attachments_session(org: int, project: int) -> Session: + global _ATTACHMENTS_CLIENT + if not _ATTACHMENTS_CLIENT: from sentry import options as options_store options = options_store.get("objectstore.config") - _attachments_client = ClientBuilder( + _ATTACHMENTS_CLIENT = Client( options["base_url"], - "attachments", metrics_backend=SentryMetricsBackend(), - default_expiration_policy=TimeToLive(timedelta(days=30)), ) - return _attachments_client + + return _ATTACHMENTS_CLIENT.session(_ATTACHMENTS_USECASE, org=org, project=project) diff --git a/src/sentry/reprocessing2.py b/src/sentry/reprocessing2.py index a6a3b8233ed2a8..bf074bb14c9787 100644 --- a/src/sentry/reprocessing2.py +++ b/src/sentry/reprocessing2.py @@ -97,7 +97,7 @@ from sentry.models.eventattachment import V1_PREFIX, V2_PREFIX, EventAttachment from sentry.models.files.utils import get_storage from sentry.models.project import Project -from sentry.objectstore import get_attachments_client +from sentry.objectstore import get_attachments_session from sentry.options.rollout import in_random_rollout from sentry.services import eventstore from sentry.services.eventstore.models import Event, GroupEvent @@ -412,11 +412,7 @@ def _maybe_copy_attachment_into_cache( else: # otherwise, we store it in objectstore with attachment.getfile() as fp: - stored_id = ( - get_attachments_client() - .for_project(project.organization_id, project.id) - .put(fp) - ) + stored_id = get_attachments_session(project.organization_id, project.id).put(fp) # but we then also make that storage permanent, as otherwise # the codepaths won’t be cleaning up this stored file. # essentially this means we are moving the file from the previous storage diff --git a/tests/sentry/api/endpoints/test_event_attachment_details.py b/tests/sentry/api/endpoints/test_event_attachment_details.py index 8ed840f5abde04..2de27274936ab2 100644 --- a/tests/sentry/api/endpoints/test_event_attachment_details.py +++ b/tests/sentry/api/endpoints/test_event_attachment_details.py @@ -4,7 +4,7 @@ from sentry.attachments.base import CachedAttachment from sentry.models.activity import Activity from sentry.models.eventattachment import V1_PREFIX, V2_PREFIX, EventAttachment -from sentry.objectstore import get_attachments_client +from sentry.objectstore import get_attachments_session from sentry.testutils.cases import APITestCase, PermissionTestCase, TestCase from sentry.testutils.helpers.datetime import before_now from sentry.testutils.helpers.features import with_feature @@ -97,10 +97,8 @@ def test_doublewrite_objectstore(self) -> None: assert attachment.blob_path is not None object_key = attachment.blob_path.removeprefix(V1_PREFIX + V2_PREFIX) # the file should also be available in objectstore - os_response = ( - get_attachments_client() - .for_project(self.organization.id, self.project.id) - .get(object_key) + os_response = get_attachments_session(self.organization.id, self.project.id).get( + object_key ) assert os_response.payload.read() == ATTACHMENT_CONTENT diff --git a/tests/sentry/ingest/ingest_consumer/test_ingest_consumer_processing.py b/tests/sentry/ingest/ingest_consumer/test_ingest_consumer_processing.py index a67cc8fe7a194d..72319419751e1c 100644 --- a/tests/sentry/ingest/ingest_consumer/test_ingest_consumer_processing.py +++ b/tests/sentry/ingest/ingest_consumer/test_ingest_consumer_processing.py @@ -29,7 +29,7 @@ from sentry.models.debugfile import create_files_from_dif_zip from sentry.models.eventattachment import EventAttachment from sentry.models.userreport import UserReport -from sentry.objectstore import get_attachments_client +from sentry.objectstore import get_attachments_session from sentry.services import eventstore from sentry.testutils.factories import get_fixture_path from sentry.testutils.helpers.features import Feature @@ -467,10 +467,8 @@ def test_process_stored_attachment( with open(get_fixture_path("native", "threadnames.dmp"), "rb") as f: attachment_payload = f.read() - stored_id = ( - get_attachments_client() - .for_project(default_project.organization_id, project_id) - .put(attachment_payload) + stored_id = get_attachments_session(default_project.organization_id, project_id).put( + attachment_payload ) with task_runner(): diff --git a/tests/sentry/objectstore/test_objectstore.py b/tests/sentry/objectstore/test_objectstore.py deleted file mode 100644 index 7c7d0881bec385..00000000000000 --- a/tests/sentry/objectstore/test_objectstore.py +++ /dev/null @@ -1,71 +0,0 @@ -import pytest -import zstandard - -from sentry.objectstore import ClientBuilder, ClientError -from sentry.testutils.skips import requires_objectstore - -pytestmark = [requires_objectstore] - - -class Testserver: - url = "http://localhost:8888" - secret = "" - - -def test_object_url() -> None: - server = Testserver() - client = ClientBuilder(server.url, "test").for_project(123, 456) - - assert ( - client.object_url("foo") - == "http://localhost:8888/v1/foo?usecase=test&scope=org.123%2Fproj.456" - ) - - -def test_stores_uncompressed() -> None: - server = Testserver() - client = ClientBuilder(server.url, "test").for_organization(12345) - - body = b"oh hai!" - stored_id = client.put(body, "foo", compression="none") - assert stored_id == "foo" - - result = client.get("foo") - - assert result.metadata.compression is None - assert result.payload.read() == b"oh hai!" - - -def test_uses_zstd_by_default() -> None: - server = Testserver() - client = ClientBuilder(server.url, "test").for_organization(12345) - - body = b"oh hai!" - stored_id = client.put(body, "foo") - assert stored_id == "foo" - - # when the user indicates that it does not want decompression, it gets zstd - result = client.get("foo", decompress=False) - - assert result.metadata.compression == "zstd" - assert zstandard.decompress(result.payload.read(), 1024) == b"oh hai!" - - # otherwise, the client does the decompression - result = client.get("foo") - - assert result.metadata.compression is None - assert result.payload.read() == b"oh hai!" - - -def test_deletes_stored_stuff() -> None: - server = Testserver() - client = ClientBuilder(server.url, "test").for_organization(12345) - - body = b"oh hai!" - stored_id = client.put(body, "foo") - assert stored_id == "foo" - - client.delete("foo") - - with pytest.raises(ClientError): - client.get("foo") diff --git a/uv.lock b/uv.lock index 78a8f34dc16100..3dda25fc0a9113 100644 --- a/uv.lock +++ b/uv.lock @@ -1,5 +1,5 @@ version = 1 -revision = 3 +revision = 2 requires-python = ">=3.13" resolution-markers = [ "sys_platform == 'darwin' or sys_platform == 'linux'", @@ -439,6 +439,14 @@ wheels = [ { url = "https://pypi.devinfra.sentry.io/wheels/filelock-3.13.1-py3-none-any.whl", hash = "sha256:57dbda9b35157b05fb3e58ee91448612eb674172fab98ee235ccb0b5bee19a1c" }, ] +[[package]] +name = "filetype" +version = "1.2.0" +source = { registry = "https://pypi.devinfra.sentry.io/simple" } +wheels = [ + { url = "https://pypi.devinfra.sentry.io/wheels/filetype-1.2.0-py2.py3-none-any.whl", hash = "sha256:7ce71b6880181241cf7ac8697a2f1eb6a8bd9b429f7ad6d27b8db9ba5f1c2d25" }, +] + [[package]] name = "flake8" version = "7.3.0" @@ -1159,15 +1167,16 @@ wheels = [ [[package]] name = "objectstore-client" -version = "0.0.5" +version = "0.0.10" source = { registry = "https://pypi.devinfra.sentry.io/simple" } dependencies = [ + { name = "filetype", marker = "sys_platform == 'darwin' or sys_platform == 'linux'" }, { name = "sentry-sdk", marker = "sys_platform == 'darwin' or sys_platform == 'linux'" }, { name = "urllib3", marker = "sys_platform == 'darwin' or sys_platform == 'linux'" }, { name = "zstandard", marker = "sys_platform == 'darwin' or sys_platform == 'linux'" }, ] wheels = [ - { url = "https://pypi.devinfra.sentry.io/wheels/objectstore_client-0.0.5-py3-none-any.whl", hash = "sha256:80f069d48b325f0420f7bfc3cbddca3fa86012761ff462e0145664e3a206e907" }, + { url = "https://pypi.devinfra.sentry.io/wheels/objectstore_client-0.0.10-py3-none-any.whl", hash = "sha256:464718f1ff3678c522e9bfa26619715801784eabd187e9d7066171f0e617f3ad" }, ] [[package]] @@ -2135,7 +2144,7 @@ requires-dist = [ { name = "mmh3", specifier = ">=4.0.0" }, { name = "msgpack", specifier = ">=1.1.0" }, { name = "msgspec", specifier = ">=0.19.0" }, - { name = "objectstore-client", specifier = ">=0.0.5" }, + { name = "objectstore-client", specifier = ">=0.0.10" }, { name = "openai", specifier = ">=1.3.5" }, { name = "orjson", specifier = ">=3.10.10" }, { name = "packaging", specifier = ">=24.1" },