feat: Call webhooks async and add backoff to webhooks (#2932)

Co-authored-by: Kim Gustyr <khvn26@gmail.com>

api/app/settings/common.py

@@ -1073,3 +1073,6 @@
     # The LDAP user username and password used by `sync_ldap_users_and_groups` command
     LDAP_SYNC_USER_USERNAME = env.str("LDAP_SYNC_USER_USERNAME", None)
     LDAP_SYNC_USER_PASSWORD = env.str("LDAP_SYNC_USER_PASSWORD", None)
+
+WEBHOOK_BACKOFF_BASE = env.int("WEBHOOK_BACKOFF_BASE", default=2)
+WEBHOOK_BACKOFF_RETRIES = env.int("WEBHOOK_BACKOFF_RETRIES", default=3)

api/audit/signals.py

@@ -27,7 +27,9 @@ def call_webhooks(sender, instance, **kwargs):
         if instance.project
         else instance.environment.project.organisation
     )
-    call_organisation_webhooks(organisation, data, WebhookEventType.AUDIT_LOG_CREATED)
+    call_organisation_webhooks.delay(
+        args=(organisation.id, data, WebhookEventType.AUDIT_LOG_CREATED.value)
+    )
 
 
 def _get_integration_config(instance, integration_name):

api/edge_api/identities/tasks.py

@@ -74,7 +74,7 @@ def call_environment_webhook_for_feature_state_change(
         else WebhookEventType.FLAG_UPDATED
     )
 
-    call_environment_webhooks(environment, data, event_type=event_type)
+    call_environment_webhooks(environment.id, data, event_type=event_type.value)
 
 
 @register_task_handler(priority=TaskPriority.HIGH)

api/features/tasks.py

@@ -1,5 +1,3 @@
-from threading import Thread
-
 from environments.models import Webhook
 from features.models import FeatureState
 from webhooks.constants import WEBHOOK_DATETIME_FORMAT
@@ -38,19 +36,18 @@ def trigger_feature_state_change_webhooks(
     previous_state = _get_previous_state(history_instance, event_type)
     if previous_state:
         data.update(previous_state=previous_state)
-    Thread(
-        target=call_environment_webhooks,
-        args=(instance.environment, data, event_type),
-    ).start()
 
-    Thread(
-        target=call_organisation_webhooks,
+    call_environment_webhooks.delay(
+        args=(instance.environment.id, data, event_type.value)
+    )
+
+    call_organisation_webhooks.delay(
         args=(
-            instance.environment.project.organisation,
+            instance.environment.project.organisation.id,
             data,
-            event_type,
-        ),
-    ).start()
+            event_type.value,
+        )
+    )
 
 
 def _get_previous_state(

api/task_processor/decorators.py

@@ -59,7 +59,7 @@ def delay(
         delay_until: datetime | None = None,
         # TODO @khvn26 consider typing `args` and `kwargs` with `ParamSpec`
         # (will require a change to the signature)
-        args: tuple[typing.Any] = (),
+        args: tuple[typing.Any, ...] = (),
         kwargs: dict[str, typing.Any] | None = None,
     ) -> Task | None:
         logger.debug("Request to run task '%s' asynchronously.", self.task_identifier)

api/tests/unit/audit/test_unit_audit_models.py

@@ -1,6 +1,8 @@
 from audit.models import AuditLog
 from audit.related_object_type import RelatedObjectType
+from audit.serializers import AuditLogSerializer
 from integrations.datadog.models import DataDogConfiguration
+from webhooks.webhooks import WebhookEventType
 
 
 def test_organisation_webhooks_are_called_when_audit_log_saved(project, mocker):
@@ -13,7 +15,13 @@ def test_organisation_webhooks_are_called_when_audit_log_saved(project, mocker):
     audit_log.save()
 
     # Then
-    mock_call_webhooks.assert_called()
+    mock_call_webhooks.delay.assert_called_once_with(
+        args=(
+            project.organisation.id,
+            AuditLogSerializer(instance=audit_log).data,
+            WebhookEventType.AUDIT_LOG_CREATED.value,
+        )
+    )
 
 
 def test_data_dog_track_event_not_called_on_audit_log_saved_when_not_configured(

api/tests/unit/edge_api/identities/test_unit_edge_api_identities_tasks.py

@@ -58,8 +58,8 @@ def test_call_environment_webhook_for_feature_state_change_with_new_state_only(
     mock_call_environment_webhooks.assert_called_once()
     call_args = mock_call_environment_webhooks.call_args
 
-    assert call_args[0][0] == environment
-    assert call_args[1]["event_type"] == WebhookEventType.FLAG_UPDATED
+    assert call_args[0][0] == environment.id
+    assert call_args[1]["event_type"] == WebhookEventType.FLAG_UPDATED.value
 
     mock_generate_webhook_feature_state_data.assert_called_once_with(
         feature=feature,
@@ -111,8 +111,8 @@ def test_call_environment_webhook_for_feature_state_change_with_previous_state_o
     mock_call_environment_webhooks.assert_called_once()
     call_args = mock_call_environment_webhooks.call_args
 
-    assert call_args[0][0] == environment
-    assert call_args[1]["event_type"] == WebhookEventType.FLAG_DELETED
+    assert call_args[0][0] == environment.id
+    assert call_args[1]["event_type"] == WebhookEventType.FLAG_DELETED.value
 
     mock_generate_webhook_feature_state_data.assert_called_once_with(
         feature=feature,
@@ -182,8 +182,8 @@ def test_call_environment_webhook_for_feature_state_change_with_both_states(
     mock_call_environment_webhooks.assert_called_once()
     call_args = mock_call_environment_webhooks.call_args
 
-    assert call_args[0][0] == environment
-    assert call_args[1]["event_type"] == WebhookEventType.FLAG_UPDATED
+    assert call_args[0][0] == environment.id
+    assert call_args[1]["event_type"] == WebhookEventType.FLAG_UPDATED.value
 
     assert mock_generate_webhook_feature_state_data.call_count == 2
     mock_generate_data_calls = mock_generate_webhook_feature_state_data.call_args_list

api/tests/unit/features/test_unit_features_tasks.py

@@ -1,6 +1,5 @@
-from unittest import mock
-
 import pytest
+from pytest_mock import MockerFixture
 
 from environments.models import Environment
 from features.models import Feature, FeatureState
@@ -11,8 +10,7 @@
 
 
 @pytest.mark.django_db
-@mock.patch("features.tasks.Thread")
-def test_trigger_feature_state_change_webhooks(MockThread):
+def test_trigger_feature_state_change_webhooks(mocker: MockerFixture):
     # Given
     initial_value = "initial"
     new_value = "new"
@@ -30,34 +28,40 @@ def test_trigger_feature_state_change_webhooks(MockThread):
     feature_state.feature_state_value.save()
     feature_state.save()
 
-    MockThread.reset_mock()  # reset mock as it will have been called when setting up the data
+    mock_call_environment_webhooks = mocker.patch(
+        "features.tasks.call_environment_webhooks"
+    )
+    mock_call_organisation_webhooks = mocker.patch(
+        "features.tasks.call_organisation_webhooks"
+    )
 
     # When
     trigger_feature_state_change_webhooks(feature_state)
 
     # Then
-    call_list = MockThread.call_args_list
+    environment_webhook_call_args = (
+        mock_call_environment_webhooks.delay.call_args.kwargs["args"]
+    )
+    organisation_webhook_call_args = (
+        mock_call_organisation_webhooks.delay.call_args.kwargs["args"]
+    )
 
-    environment_webhook_call_args = call_list[0]
-    organisation_webhook_call_args = call_list[1]
+    assert environment_webhook_call_args[0] == environment.id
+    assert organisation_webhook_call_args[0] == organisation.id
 
     # verify that the data for both calls is the same
-    assert (
-        environment_webhook_call_args[1]["args"][1]
-        == organisation_webhook_call_args[1]["args"][1]
-    )
+    assert environment_webhook_call_args[1] == organisation_webhook_call_args[1]
 
-    data = environment_webhook_call_args[1]["args"][1]
-    event_type = environment_webhook_call_args[1]["args"][2]
+    data = environment_webhook_call_args[1]
+    event_type = environment_webhook_call_args[2]
     assert data["new_state"]["feature_state_value"] == new_value
     assert data["previous_state"]["feature_state_value"] == initial_value
-    assert event_type == WebhookEventType.FLAG_UPDATED
+    assert event_type == WebhookEventType.FLAG_UPDATED.value
 
 
 @pytest.mark.django_db
-@mock.patch("features.tasks.Thread")
 def test_trigger_feature_state_change_webhooks_for_deleted_flag(
-    MockThread, organisation, project, environment, feature
+    mocker, organisation, project, environment, feature
 ):
     # Given
     new_value = "new"
@@ -68,23 +72,28 @@ def test_trigger_feature_state_change_webhooks_for_deleted_flag(
     feature_state.feature_state_value.save()
     feature_state.save()
 
-    MockThread.reset_mock()  # reset mock as it will have been called when setting up the data
+    mock_call_environment_webhooks = mocker.patch(
+        "features.tasks.call_environment_webhooks"
+    )
+    mock_call_organisation_webhooks = mocker.patch(
+        "features.tasks.call_organisation_webhooks"
+    )
+
     trigger_feature_state_change_webhooks(feature_state, WebhookEventType.FLAG_DELETED)
 
     # Then
-    call_list = MockThread.call_args_list
-
-    environment_webhook_call_args = call_list[0]
-    organisation_webhook_call_args = call_list[1]
+    environment_webhook_call_args = (
+        mock_call_environment_webhooks.delay.call_args.kwargs["args"]
+    )
+    organisation_webhook_call_args = (
+        mock_call_organisation_webhooks.delay.call_args.kwargs["args"]
+    )
 
     # verify that the data for both calls is the same
-    assert (
-        environment_webhook_call_args[1]["args"][1]
-        == organisation_webhook_call_args[1]["args"][1]
-    )
+    assert environment_webhook_call_args[1] == organisation_webhook_call_args[1]
 
-    data = environment_webhook_call_args[1]["args"][1]
-    event_type = environment_webhook_call_args[1]["args"][2]
+    data = environment_webhook_call_args[1]
+    event_type = environment_webhook_call_args[2]
     assert data["new_state"] is None
     assert data["previous_state"]["feature_state_value"] == new_value
-    assert event_type == WebhookEventType.FLAG_DELETED
+    assert event_type == WebhookEventType.FLAG_DELETED.value