-
-
Notifications
You must be signed in to change notification settings - Fork 4k
/
sdk.py
492 lines (412 loc) · 20 KB
/
sdk.py
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
125
126
127
128
129
130
131
132
133
134
135
136
137
138
139
140
141
142
143
144
145
146
147
148
149
150
151
152
153
154
155
156
157
158
159
160
161
162
163
164
165
166
167
168
169
170
171
172
173
174
175
176
177
178
179
180
181
182
183
184
185
186
187
188
189
190
191
192
193
194
195
196
197
198
199
200
201
202
203
204
205
206
207
208
209
210
211
212
213
214
215
216
217
218
219
220
221
222
223
224
225
226
227
228
229
230
231
232
233
234
235
236
237
238
239
240
241
242
243
244
245
246
247
248
249
250
251
252
253
254
255
256
257
258
259
260
261
262
263
264
265
266
267
268
269
270
271
272
273
274
275
276
277
278
279
280
281
282
283
284
285
286
287
288
289
290
291
292
293
294
295
296
297
298
299
300
301
302
303
304
305
306
307
308
309
310
311
312
313
314
315
316
317
318
319
320
321
322
323
324
325
326
327
328
329
330
331
332
333
334
335
336
337
338
339
340
341
342
343
344
345
346
347
348
349
350
351
352
353
354
355
356
357
358
359
360
361
362
363
364
365
366
367
368
369
370
371
372
373
374
375
376
377
378
379
380
381
382
383
384
385
386
387
388
389
390
391
392
393
394
395
396
397
398
399
400
401
402
403
404
405
406
407
408
409
410
411
412
413
414
415
416
417
418
419
420
421
422
423
424
425
426
427
428
429
430
431
432
433
434
435
436
437
438
439
440
441
442
443
444
445
446
447
448
449
450
451
452
453
454
455
456
457
458
459
460
461
462
463
464
465
466
467
468
469
470
471
472
473
474
475
476
477
478
479
480
481
482
483
484
485
486
487
488
489
490
491
492
import copy
import inspect
import logging
import random
import sentry_sdk
from django.conf import settings
from django.urls import resolve
# Reexport sentry_sdk just in case we ever have to write another shim like we
# did for raven
from sentry_sdk import capture_exception, capture_message, configure_scope, push_scope # NOQA
from sentry_sdk.client import get_options
from sentry_sdk.transport import make_transport
from sentry_sdk.utils import logger as sdk_logger
from sentry import options
from sentry.utils import metrics
from sentry.utils.db import DjangoAtomicIntegration
from sentry.utils.rust import RustInfoIntegration
logger = logging.getLogger(__name__)
UNSAFE_FILES = (
"sentry/event_manager.py",
"sentry/tasks/process_buffer.py",
"sentry/ingest/ingest_consumer.py",
# This consumer lives outside of sentry but is just as unsafe.
"outcomes_consumer.py",
)
# URLs that should always be sampled
SAMPLED_URL_NAMES = {
# codeowners
"sentry-api-0-project-codeowners": settings.SAMPLED_DEFAULT_RATE,
"sentry-api-0-project-codeowners-details": settings.SAMPLED_DEFAULT_RATE,
# external teams POST, PUT, DELETE
"sentry-api-0-external-team": settings.SAMPLED_DEFAULT_RATE,
"sentry-api-0-external-team-details": settings.SAMPLED_DEFAULT_RATE,
# external users POST, PUT, DELETE
"sentry-api-0-organization-external-user": settings.SAMPLED_DEFAULT_RATE,
"sentry-api-0-organization-external-user-details": settings.SAMPLED_DEFAULT_RATE,
# integration platform
"external-issues": settings.SAMPLED_DEFAULT_RATE,
"sentry-api-0-sentry-app-installation-authorizations": settings.SAMPLED_DEFAULT_RATE,
# integrations
"sentry-extensions-jira-issue-hook": 0.05,
"sentry-extensions-vercel-webhook": settings.SAMPLED_DEFAULT_RATE,
"sentry-extensions-vercel-generic-webhook": settings.SAMPLED_DEFAULT_RATE,
"sentry-extensions-vercel-configure": settings.SAMPLED_DEFAULT_RATE,
"sentry-extensions-vercel-ui-hook": settings.SAMPLED_DEFAULT_RATE,
"sentry-api-0-group-integration-details": settings.SAMPLED_DEFAULT_RATE,
# notification platform
"sentry-api-0-user-notification-settings": settings.SAMPLED_DEFAULT_RATE,
"sentry-api-0-team-notification-settings": settings.SAMPLED_DEFAULT_RATE,
# events
"sentry-api-0-organization-events": 1,
# releases
"sentry-api-0-organization-releases": settings.SAMPLED_DEFAULT_RATE,
"sentry-api-0-organization-release-details": settings.SAMPLED_DEFAULT_RATE,
"sentry-api-0-project-releases": settings.SAMPLED_DEFAULT_RATE,
"sentry-api-0-project-release-details": settings.SAMPLED_DEFAULT_RATE,
# stats
"sentry-api-0-organization-stats": settings.SAMPLED_DEFAULT_RATE,
"sentry-api-0-organization-stats-v2": settings.SAMPLED_DEFAULT_RATE,
"sentry-api-0-project-stats": 0.05, # lower rate because of high TPM
# debug files
"sentry-api-0-assemble-dif-files": 0.1,
# scim
"sentry-api-0-organization-scim-member-index": 0.1,
"sentry-api-0-organization-scim-member-details": 0.1,
"sentry-api-0-organization-scim-team-index": 0.1,
"sentry-api-0-organization-scim-team-details": 0.1,
# members
"sentry-api-0-organization-invite-request-index": settings.SAMPLED_DEFAULT_RATE,
"sentry-api-0-organization-invite-request-detail": settings.SAMPLED_DEFAULT_RATE,
"sentry-api-0-organization-join-request": settings.SAMPLED_DEFAULT_RATE,
# login
"sentry-login": 0.1,
"sentry-auth-organization": 0.2,
"sentry-auth-link-identity": settings.SAMPLED_DEFAULT_RATE,
"sentry-auth-sso": settings.SAMPLED_DEFAULT_RATE,
"sentry-logout": 0.1,
"sentry-register": settings.SAMPLED_DEFAULT_RATE,
"sentry-2fa-dialog": settings.SAMPLED_DEFAULT_RATE,
# reprocessing
"sentry-api-0-issues-reprocessing": settings.SENTRY_REPROCESSING_APM_SAMPLING,
}
if settings.ADDITIONAL_SAMPLED_URLS:
SAMPLED_URL_NAMES.update(settings.ADDITIONAL_SAMPLED_URLS)
# Tasks not included here are not sampled
# If a parent task schedules other tasks you should add it in here or the children
# tasks will not be sampled
SAMPLED_TASKS = {
"sentry.tasks.send_ping": settings.SAMPLED_DEFAULT_RATE,
"sentry.tasks.store.symbolicate_event": settings.SENTRY_SYMBOLICATE_EVENT_APM_SAMPLING,
"sentry.tasks.store.symbolicate_event_from_reprocessing": settings.SENTRY_SYMBOLICATE_EVENT_APM_SAMPLING,
"sentry.tasks.store.process_event": settings.SENTRY_PROCESS_EVENT_APM_SAMPLING,
"sentry.tasks.store.process_event_from_reprocessing": settings.SENTRY_PROCESS_EVENT_APM_SAMPLING,
"sentry.tasks.assemble.assemble_dif": 0.1,
"sentry.tasks.app_store_connect.dsym_download": settings.SENTRY_APPCONNECT_APM_SAMPLING,
"sentry.tasks.app_store_connect.refresh_all_builds": settings.SENTRY_APPCONNECT_APM_SAMPLING,
"sentry.tasks.process_suspect_commits": settings.SENTRY_SUSPECT_COMMITS_APM_SAMPLING,
"sentry.tasks.process_commit_context": settings.SENTRY_SUSPECT_COMMITS_APM_SAMPLING,
"sentry.tasks.post_process.post_process_group": settings.SENTRY_POST_PROCESS_GROUP_APM_SAMPLING,
"sentry.tasks.reprocessing2.handle_remaining_events": settings.SENTRY_REPROCESSING_APM_SAMPLING,
"sentry.tasks.reprocessing2.reprocess_group": settings.SENTRY_REPROCESSING_APM_SAMPLING,
"sentry.tasks.reprocessing2.finish_reprocessing": settings.SENTRY_REPROCESSING_APM_SAMPLING,
"sentry.tasks.relay.build_project_config": settings.SENTRY_RELAY_TASK_APM_SAMPLING,
"sentry.tasks.relay.invalidate_project_config": settings.SENTRY_RELAY_TASK_APM_SAMPLING,
"sentry.tasks.process_buffer.process_incr": 0.01,
"sentry.replays.tasks.delete_recording_segments": settings.SAMPLED_DEFAULT_RATE,
"sentry.tasks.weekly_reports.schedule_organizations": 1.0,
"sentry.tasks.weekly_reports.prepare_organization_report": 0.1,
"sentry.profiles.task.process_profile": 0.01,
"sentry.tasks.derive_code_mappings.process_organizations": settings.SAMPLED_DEFAULT_RATE,
"sentry.tasks.derive_code_mappings.derive_code_mappings": settings.SAMPLED_DEFAULT_RATE,
"sentry.monitors.tasks.check_monitors": 1.0,
"sentry.tasks.auto_enable_codecov": settings.SAMPLED_DEFAULT_RATE,
}
if settings.ADDITIONAL_SAMPLED_TASKS:
SAMPLED_TASKS.update(settings.ADDITIONAL_SAMPLED_TASKS)
UNSAFE_TAG = "_unsafe"
EXPERIMENT_TAG = "_experimental_event"
def is_current_event_safe():
"""
Tests the current stack for unsafe locations that would likely cause
recursion if an attempt to send to Sentry was made.
"""
with configure_scope() as scope:
# Scope was explicitly marked as unsafe
if scope._tags.get(UNSAFE_TAG):
return False
project_id = scope._tags.get("processing_event_for_project")
if project_id and project_id == settings.SENTRY_PROJECT:
return False
for _, filename, _, _, _, _ in inspect.stack():
if filename.endswith(UNSAFE_FILES):
return False
return True
def is_current_event_experimental():
"""
Checks if the event was explicitly marked as experimental.
"""
with configure_scope() as scope:
if scope._tags.get(EXPERIMENT_TAG):
return True
return False
def mark_scope_as_unsafe():
"""
Set the unsafe tag on the SDK scope for outgoing crashes and transactions.
Marking a scope explicitly as unsafe allows the recursion breaker to
decide early, before walking the stack and checking for unsafe files.
"""
with configure_scope() as scope:
scope.set_tag(UNSAFE_TAG, True)
def mark_scope_as_experimental():
"""
Set the experimental tag on the SDK scope for outgoing crashes and transactions.
Marking the scope will cause these crashes and transaction to be sent to a separate experimental dsn.
"""
with configure_scope() as scope:
scope.set_tag(EXPERIMENT_TAG, True)
def set_current_event_project(project_id):
"""
Set the current project on the SDK scope for outgoing crash reports.
This is a dedicated function because it is also important for the recursion
breaker to work. You really should set the project in every task that is
relevant to event processing, or that task may crash ingesting
sentry-internal errors, causing infinite recursion.
"""
with configure_scope() as scope:
scope.set_tag("processing_event_for_project", project_id)
scope.set_tag("project", project_id)
def get_project_key():
from sentry.models.projectkey import ProjectKey
if not settings.SENTRY_PROJECT:
return None
key = None
try:
if settings.SENTRY_PROJECT_KEY is not None:
key = ProjectKey.objects.get(
id=settings.SENTRY_PROJECT_KEY, project=settings.SENTRY_PROJECT
)
else:
key = ProjectKey.get_default(settings.SENTRY_PROJECT)
except Exception as exc:
# if the relation fails to query or is missing completely, lets handle
# it gracefully
sdk_logger.warning(
"internal-error.unable-to-fetch-project",
extra={
"project_id": settings.SENTRY_PROJECT,
"project_key": settings.SENTRY_PROJECT_KEY,
"error_message": str(exc),
},
)
if key is None:
sdk_logger.warning(
"internal-error.no-project-available",
extra={
"project_id": settings.SENTRY_PROJECT,
"project_key": settings.SENTRY_PROJECT_KEY,
},
)
return key
def traces_sampler(sampling_context):
# If there's already a sampling decision, just use that
if sampling_context["parent_sampled"] is not None:
return sampling_context["parent_sampled"]
if "celery_job" in sampling_context:
task_name = sampling_context["celery_job"].get("task")
if task_name in SAMPLED_TASKS:
return SAMPLED_TASKS[task_name]
# Resolve the url, and see if we want to set our own sampling
if "wsgi_environ" in sampling_context:
try:
match = resolve(sampling_context["wsgi_environ"].get("PATH_INFO"))
if match and match.url_name in SAMPLED_URL_NAMES:
return SAMPLED_URL_NAMES[match.url_name]
except Exception:
# On errors or 404, continue to default sampling decision
pass
# Default to the sampling rate in settings
return float(settings.SENTRY_BACKEND_APM_SAMPLING or 0)
def before_send_transaction(event, _):
# Occasionally the span limit is hit and we drop spans from transactions, this helps find transactions where this occurs.
num_of_spans = len(event["spans"])
event["tags"]["spans_over_limit"] = num_of_spans >= 1000
if not event["measurements"]:
event["measurements"] = {}
event["measurements"]["num_of_spans"] = {"value": num_of_spans}
return event
# Patches transport functions to add metrics to improve resolution around events sent to our ingest.
# Leaving this in to keep a permanent measurement of sdk requests vs ingest.
def patch_transport_for_instrumentation(transport, transport_name):
_send_request = transport._send_request
if _send_request:
def patched_send_request(*args, **kwargs):
metrics.incr(f"internal.sent_requests.{transport_name}.events")
return _send_request(*args, **kwargs)
transport._send_request = patched_send_request
return transport
def configure_sdk():
from sentry_sdk.integrations.celery import CeleryIntegration
from sentry_sdk.integrations.django import DjangoIntegration
from sentry_sdk.integrations.logging import LoggingIntegration
from sentry_sdk.integrations.redis import RedisIntegration
from sentry_sdk.integrations.threading import ThreadingIntegration
assert sentry_sdk.Hub.main.client is None
sdk_options = dict(settings.SENTRY_SDK_CONFIG)
relay_dsn = sdk_options.pop("relay_dsn", None)
experimental_dsn = sdk_options.pop("experimental_dsn", None)
internal_project_key = get_project_key()
# Modify SENTRY_SDK_CONFIG in your deployment scripts to specify your desired DSN
upstream_dsn = sdk_options.pop("dsn", None)
sdk_options["traces_sampler"] = traces_sampler
sdk_options["release"] = (
f"backend@{sdk_options['release']}" if "release" in sdk_options else None
)
sdk_options["send_client_reports"] = True
sdk_options["before_send_transaction"] = before_send_transaction
if upstream_dsn:
transport = make_transport(get_options(dsn=upstream_dsn, **sdk_options))
upstream_transport = patch_transport_for_instrumentation(transport, "upstream")
else:
upstream_transport = None
if relay_dsn:
transport = make_transport(get_options(dsn=relay_dsn, **sdk_options))
relay_transport = patch_transport_for_instrumentation(transport, "relay")
elif settings.IS_DEV and not settings.SENTRY_USE_RELAY:
relay_transport = None
elif internal_project_key and internal_project_key.dsn_private:
transport = make_transport(get_options(dsn=internal_project_key.dsn_private, **sdk_options))
relay_transport = patch_transport_for_instrumentation(transport, "relay")
else:
relay_transport = None
if experimental_dsn:
transport = make_transport(get_options(dsn=experimental_dsn, **sdk_options))
experimental_transport = patch_transport_for_instrumentation(transport, "experimental")
else:
experimental_transport = None
if settings.SENTRY_PROFILING_ENABLED:
sdk_options["profiles_sample_rate"] = settings.SENTRY_PROFILES_SAMPLE_RATE
sdk_options["profiler_mode"] = settings.SENTRY_PROFILER_MODE
class MultiplexingTransport(sentry_sdk.transport.Transport):
def capture_envelope(self, envelope):
# Temporarily capture envelope counts to compare to ingested
# transactions.
metrics.incr("internal.captured.events.envelopes")
transaction = envelope.get_transaction_event()
if transaction:
metrics.incr("internal.captured.events.transactions")
# Assume only transactions get sent via envelopes
if options.get("transaction-events.force-disable-internal-project"):
return
self._capture_anything("capture_envelope", envelope)
def capture_event(self, event):
if event.get("type") == "transaction" and options.get(
"transaction-events.force-disable-internal-project"
):
return
self._capture_anything("capture_event", event)
def _capture_anything(self, method_name, *args, **kwargs):
# Experimental events will be sent to the experimental transport.
if experimental_transport:
rate = options.get("store.use-experimental-dsn-sample-rate")
if is_current_event_experimental():
if rate and random.random() < rate:
getattr(experimental_transport, method_name)(*args, **kwargs)
# Experimental events should not be sent to other transports even if they are not sampled.
return
# Upstream should get the event first because it is most isolated from
# the this sentry installation.
if upstream_transport:
metrics.incr("internal.captured.events.upstream")
# TODO(mattrobenolt): Bring this back safely.
# from sentry import options
# install_id = options.get('sentry:install-id')
# if install_id:
# event.setdefault('tags', {})['install-id'] = install_id
getattr(upstream_transport, method_name)(*args, **kwargs)
if relay_transport and options.get("store.use-relay-dsn-sample-rate") == 1:
# If this is a envelope ensure envelope and it's items are distinct references
if method_name == "capture_envelope":
args_list = list(args)
envelope = args_list[0]
relay_envelope = copy.copy(envelope)
relay_envelope.items = envelope.items.copy()
args = [relay_envelope, *args_list[1:]]
if is_current_event_safe():
metrics.incr("internal.captured.events.relay")
getattr(relay_transport, method_name)(*args, **kwargs)
else:
metrics.incr(
"internal.uncaptured.events.relay",
skip_internal=False,
tags={"reason": "unsafe"},
)
sentry_sdk.init(
# set back the upstream_dsn popped above since we need a default dsn on the client
# for dynamic sampling context public_key population
dsn=upstream_dsn,
transport=MultiplexingTransport(),
integrations=[
DjangoAtomicIntegration(),
DjangoIntegration(),
CeleryIntegration(),
LoggingIntegration(event_level=None),
RustInfoIntegration(),
RedisIntegration(),
ThreadingIntegration(propagate_hub=True),
],
**sdk_options,
)
class RavenShim:
"""Wrapper around sentry-sdk in case people are writing their own
integrations that rely on this being here."""
def captureException(self, exc_info=None, **kwargs):
with sentry_sdk.push_scope() as scope:
self._kwargs_into_scope(scope, **kwargs)
return capture_exception(exc_info)
def captureMessage(self, msg, **kwargs):
with sentry_sdk.push_scope() as scope:
self._kwargs_into_scope(scope, **kwargs)
return capture_message(msg)
def tags_context(self, tags):
with sentry_sdk.configure_scope() as scope:
for k, v in tags.items():
scope.set_tag(k, v)
def _kwargs_into_scope(self, scope, extra=None, tags=None, fingerprint=None, request=None):
for key, value in extra.items() if extra else ():
scope.set_extra(key, value)
for key, value in tags.items() if tags else ():
scope.set_tag(key, value)
if fingerprint is not None:
scope.fingerprint = fingerprint
def check_tag(tag_key: str, expected_value: str) -> bool:
"""Detect a tag already set and being different than what we expect.
This function checks if a tag has been already been set and if it differs
from what we want to set it to.
"""
with configure_scope() as scope:
if scope._tags and tag_key in scope._tags and scope._tags[tag_key] != expected_value:
extra = {
f"previous_{tag_key}": scope._tags[tag_key],
f"new_{tag_key}": expected_value,
}
logger.warning(f"Tag already set and different ({tag_key}).", extra=extra)
return True
def bind_organization_context(organization):
# Callable to bind additional context for the Sentry SDK
helper = settings.SENTRY_ORGANIZATION_CONTEXT_HELPER
# XXX(dcramer): this is duplicated in organizationContext.jsx on the frontend
with sentry_sdk.configure_scope() as scope, sentry_sdk.start_span(
op="other", description="bind_organization_context"
):
if check_tag("organization.slug", organization.slug):
# This can be used to find errors that may have been mistagged
scope.set_tag("possible_mistag", True)
scope.set_tag("organization", organization.id)
scope.set_tag("organization.slug", organization.slug)
scope.set_context("organization", {"id": organization.id, "slug": organization.slug})
if helper:
try:
helper(scope=scope, organization=organization)
except Exception:
sdk_logger.exception(
"internal-error.organization-context",
extra={"organization_id": organization.id},
)
def set_measurement(measurement_name, value, unit=None):
try:
transaction = sentry_sdk.Hub.current.scope.transaction
if transaction is not None:
transaction.set_measurement(measurement_name, value, unit)
except Exception:
pass