feat(alert-rule): Instrument analytics for Alert Rule UI Components

src/sentry/analytics/events/alert_created.py

@@ -11,6 +11,8 @@ class AlertCreatedEvent(analytics.Event):
         analytics.Attribute("rule_id"),
         analytics.Attribute("rule_type"),
         analytics.Attribute("is_api_token"),
+        # `alert_rule_ui_component` can be `alert-rule-action`
+        analytics.Attribute("alert_rule_ui_component", required=False),
     )
 
 

src/sentry/analytics/events/alert_rule_ui_component_webhook_sent.py

@@ -0,0 +1,15 @@
+from sentry import analytics
+
+
+class AlertRuleUiComponentWebhookSentEvent(analytics.Event):
+    type = "alert_rule_ui_component_webhook.sent"
+
+    attributes = (
+        # organization_id refers to the organization that installed the sentryapp
+        analytics.Attribute("organization_id"),
+        analytics.Attribute("sentry_app_id"),
+        analytics.Attribute("event"),
+    )
+
+
+analytics.register(AlertRuleUiComponentWebhookSentEvent)

src/sentry/analytics/events/metric_alert_with_ui_component_created.py

@@ -0,0 +1,14 @@
+from sentry import analytics
+
+
+class MetricAlertWithUiComponentCreatedEvent(analytics.Event):
+    type = "metric_alert_with_ui_component.created"
+
+    attributes = (
+        analytics.Attribute("user_id", required=False),
+        analytics.Attribute("alert_rule_id"),
+        analytics.Attribute("organization_id"),
+    )
+
+
+analytics.register(MetricAlertWithUiComponentCreatedEvent)

src/sentry/analytics/events/sentry_app_created.py

@@ -8,6 +8,7 @@ class SentryAppCreatedEvent(analytics.Event):
         analytics.Attribute("user_id"),
         analytics.Attribute("organization_id"),
         analytics.Attribute("sentry_app"),
+        analytics.Attribute("created_alert_rule_ui_component", type=bool, required=False),
     )
 
 

src/sentry/analytics/events/sentry_app_updated.py

@@ -4,7 +4,12 @@
 class SentryAppUpdatedEvent(analytics.Event):
     type = "sentry_app.updated"
 
-    attributes = (analytics.Attribute("user_id"), analytics.Attribute("sentry_app"))
+    attributes = (
+        analytics.Attribute("user_id"),
+        analytics.Attribute("organization_id"),
+        analytics.Attribute("sentry_app"),
+        analytics.Attribute("created_alert_rule_ui_component", type=bool, required=False),
+    )
 
 
 analytics.register(SentryAppUpdatedEvent)

src/sentry/api/endpoints/project_rules.py

@@ -1,4 +1,4 @@
-from typing import Mapping, Sequence
+from typing import Mapping, Optional, Sequence
 
 from rest_framework import serializers, status
 from rest_framework.response import Response
@@ -24,7 +24,8 @@
 
 def trigger_alert_rule_action_creators(
     actions: Sequence[Mapping[str, str]],
-) -> None:
+) -> Optional[str]:
+    created = None
     for action in actions:
         # Only call creator for Sentry Apps with UI Components for alert rules.
         if not action.get("hasSchemaFormConfig"):
@@ -40,6 +41,8 @@ def trigger_alert_rule_action_creators(
             raise serializers.ValidationError(
                 {"sentry_app": f'{install.sentry_app.name}: {result["message"]}'}
             )
+        created = "alert-rule-action"
+    return created
 
 
 class ProjectRulesEndpoint(ProjectEndpoint):
@@ -122,7 +125,9 @@ def post(self, request, project):
                 tasks.find_channel_id_for_rule.apply_async(kwargs=kwargs)
                 return Response(uuid_context, status=202)
 
-            trigger_alert_rule_action_creators(kwargs.get("actions"))
+            created_alert_rule_ui_component = trigger_alert_rule_action_creators(
+                kwargs.get("actions")
+            )
             rule = project_rules.Creator.run(request=request, **kwargs)
             RuleActivity.objects.create(
                 rule=rule, user=request.user, type=RuleActivityType.CREATED.value
@@ -142,6 +147,7 @@ def post(self, request, project):
                 rule_type="issue",
                 sender=self,
                 is_api_token=request.auth is not None,
+                alert_rule_ui_component=created_alert_rule_ui_component,
             )
 
             return Response(serialize(rule, request.user))

src/sentry/incidents/endpoints/organization_alert_rule_details.py

@@ -23,7 +23,7 @@ def get(self, request, organization, alert_rule):
 
     def put(self, request, organization, alert_rule):
         serializer = DrfAlertRuleSerializer(
-            context={"organization": organization, "access": request.access},
+            context={"organization": organization, "access": request.access, "user": request.user},
             instance=alert_rule,
             data=request.data,
         )

src/sentry/incidents/endpoints/organization_alert_rule_index.py

@@ -176,7 +176,8 @@ def post(self, request, organization):
             raise ResourceDoesNotExist
 
         serializer = AlertRuleSerializer(
-            context={"organization": organization, "access": request.access}, data=request.data
+            context={"organization": organization, "access": request.access, "user": request.user},
+            data=request.data,
         )
 
         if serializer.is_valid():

src/sentry/incidents/endpoints/serializers.py

@@ -7,6 +7,7 @@
 from django.utils.encoding import force_text
 from rest_framework import serializers
 
+from sentry import analytics
 from sentry.api.serializers.rest_framework.base import CamelSnakeModelSerializer
 from sentry.api.serializers.rest_framework.environment import EnvironmentField
 from sentry.api.serializers.rest_framework.project import ProjectField
@@ -82,6 +83,7 @@ class AlertRuleTriggerActionSerializer(CamelSnakeModelSerializer):
      - `alert_rule`: The alert_rule related to this action.
      - `organization`: The organization related to this action.
      - `access`: An access object (from `request.access`)
+     - `user`: The user from `request.user`
     """
 
     id = serializers.IntegerField(required=False)
@@ -216,13 +218,21 @@ def validate(self, attrs):
 
     def create(self, validated_data):
         try:
-            return create_alert_rule_trigger_action(
+            action = create_alert_rule_trigger_action(
                 trigger=self.context["trigger"], **validated_data
             )
         except InvalidTriggerActionError as e:
             raise serializers.ValidationError(force_text(e))
         except ApiRateLimitedError as e:
             raise serializers.ValidationError(force_text(e))
+        else:
+            analytics.record(
+                "metric_alert_with_ui_component.created",
+                user_id=getattr(self.context["user"], "id", None),
+                alert_rule_id=getattr(self.context["alert_rule"], "id"),
+                organization_id=getattr(self.context["organization"], "id"),
+            )
+            return action
 
     def update(self, instance, validated_data):
         if "id" in validated_data:
@@ -241,6 +251,7 @@ class AlertRuleTriggerSerializer(CamelSnakeModelSerializer):
      - `alert_rule`: The alert_rule related to this trigger.
      - `organization`: The organization related to this trigger.
      - `access`: An access object (from `request.access`)
+     - `user`: The user from `request.user`
     """
 
     id = serializers.IntegerField(required=False)
@@ -304,6 +315,7 @@ def _handle_actions(self, alert_rule_trigger, actions):
                         "trigger": alert_rule_trigger,
                         "organization": self.context["organization"],
                         "access": self.context["access"],
+                        "user": self.context["user"],
                         "use_async_lookup": self.context.get("use_async_lookup"),
                         "validate_channel_id": self.context.get("validate_channel_id"),
                         "input_channel_id": action_data.pop("input_channel_id", None),
@@ -333,6 +345,7 @@ class AlertRuleSerializer(CamelSnakeModelSerializer):
     Serializer for creating/updating an alert rule. Required context:
      - `organization`: The organization related to this alert rule.
      - `access`: An access object (from `request.access`)
+     - `user`: The user from `request.user`
     """
 
     environment = EnvironmentField(required=False, allow_null=True)
@@ -672,6 +685,7 @@ def _handle_triggers(self, alert_rule, triggers):
                         "alert_rule": alert_rule,
                         "organization": self.context["organization"],
                         "access": self.context["access"],
+                        "user": self.context["user"],
                         "use_async_lookup": self.context.get("use_async_lookup"),
                         "input_channel_id": self.context.get("input_channel_id"),
                         "validate_channel_id": self.context.get("validate_channel_id"),

src/sentry/mediators/sentry_apps/creator.py

@@ -16,8 +16,10 @@
 )
 from sentry.models.sentryapp import default_uuid, generate_slug
 
+from .mixin import SentryAppMixin
 
-class Creator(Mediator):
+
+class Creator(Mediator, SentryAppMixin):
     name = Param((str,))
     author = Param((str,))
     organization = Param("sentry.models.Organization")
@@ -130,4 +132,5 @@ def record_analytics(self):
             user_id=self.user.id,
             organization_id=self.organization.id,
             sentry_app=self.sentry_app.slug,
+            created_alert_rule_ui_component="alert-rule-action" in self.get_schema_types(),
         )

src/sentry/mediators/sentry_apps/mixin.py

@@ -0,0 +1,6 @@
+from typing import Set
+
+
+class SentryAppMixin:
+    def get_schema_types(self) -> Set[str]:
+        return {element["type"] for element in (self.schema or {}).get("elements", [])}

src/sentry/mediators/sentry_apps/updater.py

@@ -1,4 +1,5 @@
 from collections.abc import Iterable
+from typing import Set
 
 from django.db.models import Q
 from django.utils import timezone
@@ -11,8 +12,10 @@
 from sentry.models import ApiToken, SentryAppComponent, SentryAppInstallation, ServiceHook
 from sentry.models.sentryapp import REQUIRED_EVENT_PERMISSIONS
 
+from .mixin import SentryAppMixin
 
-class Updater(Mediator):
+
+class Updater(Mediator, SentryAppMixin):
     sentry_app = Param("sentry.models.SentryApp")
     name = Param((str,), required=False)
     status = Param((str,), required=False)
@@ -143,9 +146,16 @@ def _update_allowed_origins(self):
     @if_param("schema")
     def _update_schema(self):
         self.sentry_app.schema = self.schema
+        self.new_schema_elements = self._get_new_schema_elements()
         self._delete_old_ui_components()
         self._create_ui_components()
 
+    def _get_new_schema_elements(self) -> Set[str]:
+        current = SentryAppComponent.objects.filter(sentry_app=self.sentry_app).values_list(
+            "type", flat=True
+        )
+        return self.get_schema_types() - set(current)
+
     def _delete_old_ui_components(self):
         SentryAppComponent.objects.filter(sentry_app_id=self.sentry_app.id).delete()
 
@@ -157,5 +167,9 @@ def _create_ui_components(self):
 
     def record_analytics(self):
         analytics.record(
-            "sentry_app.updated", user_id=self.user.id, sentry_app=self.sentry_app.slug
+            "sentry_app.updated",
+            user_id=self.user.id,
+            organization_id=self.sentry_app.owner_id,
+            sentry_app=self.sentry_app.slug,
+            created_alert_rule_ui_component="alert-rule-action" in (self.new_schema_elements or {}),
         )

src/sentry/receivers/features.py

@@ -282,7 +282,15 @@ def record_inbound_filter_toggled(project, **kwargs):
 
 @alert_rule_created.connect(weak=False)
 def record_alert_rule_created(
-    user, project, rule, rule_type, is_api_token, referrer=None, session_id=None, **kwargs
+    user,
+    project,
+    rule,
+    rule_type,
+    is_api_token,
+    referrer=None,
+    session_id=None,
+    alert_rule_ui_component=None,
+    **kwargs,
 ):
     if rule_type == "issue" and rule.label == DEFAULT_RULE_LABEL and rule.data == DEFAULT_RULE_DATA:
         return
@@ -308,6 +316,7 @@ def record_alert_rule_created(
         referrer=referrer,
         session_id=session_id,
         is_api_token=is_api_token,
+        alert_rule_ui_component=alert_rule_ui_component,
     )
 
 

src/sentry/rules/actions/notify_event_service.py

@@ -6,6 +6,7 @@
 
 from django import forms
 
+from sentry import analytics
 from sentry.api.serializers import serialize
 from sentry.api.serializers.models.app_platform_event import AppPlatformEvent
 from sentry.api.serializers.models.incident import IncidentSerializer
@@ -71,18 +72,51 @@ def send_incident_alert_notification(action, incident, metric_value=None, method
         )
         return
 
+    app_platform_event = AppPlatformEvent(
+        resource="metric_alert",
+        action=INCIDENT_STATUS[
+            incident_status_info(incident, metric_value, action, method)
+        ].lower(),
+        install=install,
+        data=build_incident_attachment(action, incident, metric_value, method),
+    )
+
+    # Can raise errors if client returns >= 400
     send_and_save_webhook_request(
         sentry_app,
-        AppPlatformEvent(
-            resource="metric_alert",
-            action=INCIDENT_STATUS[
-                incident_status_info(incident, metric_value, action, method)
-            ].lower(),
-            install=install,
-            data=build_incident_attachment(action, incident, metric_value, method),
-        ),
+        app_platform_event,
     )
 
+    # On success, record analytic event for Metric Alert Rule UI Component
+    alert_rule_action_ui_component = find_alert_rule_action_ui_component(app_platform_event)
+
+    if alert_rule_action_ui_component:
+        analytics.record(
+            "alert_rule_ui_component_webhook.sent",
+            organization_id=organization.id,
+            sentry_app_id=sentry_app.id,
+            event=f"{app_platform_event.resource}.{app_platform_event.action}",
+        )
+
+
+def find_alert_rule_action_ui_component(app_platform_event: AppPlatformEvent) -> bool:
+    # Loop through the triggers for the alert rule event. For each trigger, check if an action is an alert rule UI Component
+    triggers = (
+        getattr(app_platform_event, "data", {})
+        .get("metric_alert", {})
+        .get("alert_rule", {})
+        .get("triggers", [])
+    )
+
+    actions = [
+        action
+        for trigger in triggers
+        for action in trigger["actions"]
+        if (action["type"] == "sentry_app" and action["settings"] is not None)
+    ]
+
+    return True if len(actions) else False
+
 
 class NotifyEventServiceForm(forms.Form):
     service = forms.ChoiceField(choices=())