From c65dd018fda9ab305c296dd5a1f6055646bef43f Mon Sep 17 00:00:00 2001 From: Evgeny Seregin Date: Tue, 9 Aug 2022 20:12:54 +0300 Subject: [PATCH 01/16] Initial Huey setup --- sentry_sdk/integrations/huey.py | 5 +++++ tests/integrations/huey/__init__.py | 3 +++ tests/integrations/huey/test_huey.py | 26 ++++++++++++++++++++++++++ tox.ini | 5 +++++ 4 files changed, 39 insertions(+) create mode 100644 sentry_sdk/integrations/huey.py create mode 100644 tests/integrations/huey/__init__.py create mode 100644 tests/integrations/huey/test_huey.py diff --git a/sentry_sdk/integrations/huey.py b/sentry_sdk/integrations/huey.py new file mode 100644 index 0000000000..c7a54e6fbb --- /dev/null +++ b/sentry_sdk/integrations/huey.py @@ -0,0 +1,5 @@ +from sentry_sdk.integrations import Integration + + +class HueyIntegration(Integration): + identifier = "huey" diff --git a/tests/integrations/huey/__init__.py b/tests/integrations/huey/__init__.py new file mode 100644 index 0000000000..448a7eb2f7 --- /dev/null +++ b/tests/integrations/huey/__init__.py @@ -0,0 +1,3 @@ +import pytest + +pytest.importorskip("huey") diff --git a/tests/integrations/huey/test_huey.py b/tests/integrations/huey/test_huey.py new file mode 100644 index 0000000000..ad523a2b4b --- /dev/null +++ b/tests/integrations/huey/test_huey.py @@ -0,0 +1,26 @@ +import pytest + +from sentry_sdk.integrations.huey import HueyIntegration + +from huey.api import RedisExpireHuey + + +@pytest.fixture +def init_huey(sentry_init): + def inner(): + sentry_init( + integrations=[HueyIntegration()], + traces_sample_rate=1.0, + send_default_pii=True, + debug=True, + ) + + return RedisExpireHuey(name="sentry_sdk", url="redis://127.0.0.1:6379") + + return inner + + +@pytest.fixture(autouse=True) +def flush_huey_tasks(init_huey): + huey = init_huey() + huey.flush() diff --git a/tox.ini b/tox.ini index 3eec4a7a11..e853be622a 100644 --- a/tox.ini +++ b/tox.ini @@ -52,6 +52,8 @@ envlist = {pypy,py2.7,py3.5,py3.6,py3.7,py3.8}-celery-{4.3,4.4} {py3.6,py3.7,py3.8,py3.9,py3.10}-celery-5.0 + {py2.7,py3.4,py3.5,py3.6,py3.7,py3.8,py3.9,py3.10}-huey-2 + py3.7-beam-{2.12,2.13,2.32,2.33} # The aws_lambda tests deploy to the real AWS and have their own matrix of Python versions. @@ -180,6 +182,8 @@ deps = py3.5-celery: newrelic<6.0.0 {pypy,py2.7,py3.6,py3.7,py3.8,py3.9,py3.10}-celery: newrelic + huey-2: huey>=2.0 + requests: requests>=2.0 aws_lambda: boto3 @@ -266,6 +270,7 @@ setenv = bottle: TESTPATH=tests/integrations/bottle falcon: TESTPATH=tests/integrations/falcon celery: TESTPATH=tests/integrations/celery + huey: TESTPATH=tests/integrations/huey requests: TESTPATH=tests/integrations/requests aws_lambda: TESTPATH=tests/integrations/aws_lambda gcp: TESTPATH=tests/integrations/gcp From 3a49987f6ede421ab24e8a904d5a200ca99963a5 Mon Sep 17 00:00:00 2001 From: Evgeny Seregin Date: Tue, 9 Aug 2022 20:50:19 +0300 Subject: [PATCH 02/16] Minimal Huey integration --- sentry_sdk/integrations/huey.py | 50 +++++++++++++++++++++++++++- tests/integrations/huey/test_huey.py | 18 +++++++++- 2 files changed, 66 insertions(+), 2 deletions(-) diff --git a/sentry_sdk/integrations/huey.py b/sentry_sdk/integrations/huey.py index c7a54e6fbb..72debf7d33 100644 --- a/sentry_sdk/integrations/huey.py +++ b/sentry_sdk/integrations/huey.py @@ -1,5 +1,53 @@ -from sentry_sdk.integrations import Integration +from datetime import datetime + +from sentry_sdk._types import MYPY +from sentry_sdk import Hub +from sentry_sdk.integrations import DidNotEnable, Integration +from sentry_sdk.tracing import Transaction, TRANSACTION_SOURCE_TASK +from sentry_sdk.utils import capture_internal_exceptions + +if MYPY: + from typing import Any, Optional + +try: + from huey.api import Huey, Task +except ImportError: + raise DidNotEnable("Huey is not installed") class HueyIntegration(Integration): identifier = "huey" + + @staticmethod + def setup_once(): + # type: () -> None + patch_execute() + + +def patch_execute(): + # type: () -> None + old_execute = Huey._execute + + def _sentry_execute(self, task, timestamp=None): + # type: (Huey, Task, Optional[datetime]) -> Any + hub = Hub.current + + if hub.get_integration(HueyIntegration) is None: + return old_execute(self, task, timestamp) + + with hub.push_scope() as scope: + with capture_internal_exceptions(): + scope._name = "huey" + scope.clear_breadcrumbs() + + transaction = Transaction( + name=task.name, + status="ok", + op="huey.task", + source=TRANSACTION_SOURCE_TASK, + ) + + with hub.start_transaction(transaction): + return old_execute(self, task, timestamp) + + Huey._execute = _sentry_execute diff --git a/tests/integrations/huey/test_huey.py b/tests/integrations/huey/test_huey.py index ad523a2b4b..e4fe6006ef 100644 --- a/tests/integrations/huey/test_huey.py +++ b/tests/integrations/huey/test_huey.py @@ -2,7 +2,7 @@ from sentry_sdk.integrations.huey import HueyIntegration -from huey.api import RedisExpireHuey +from huey.api import RedisExpireHuey, Result @pytest.fixture @@ -24,3 +24,19 @@ def inner(): def flush_huey_tasks(init_huey): huey = init_huey() huey.flush() + + +def test_task_result(init_huey): + huey = init_huey() + + @huey.task() + def increase(num): + return num + 1 + + result = increase(3) + + assert isinstance(result, Result) + assert len(huey) == 1 + task = huey.dequeue() + assert huey.execute(task) == 4 + assert result.get() == 4 From 854109d59ef14ce53d3af920b3a05f5e5e82c537 Mon Sep 17 00:00:00 2001 From: Evgeny Seregin Date: Tue, 9 Aug 2022 21:09:33 +0300 Subject: [PATCH 03/16] Added context_processor for tasks --- mypy.ini | 2 ++ sentry_sdk/integrations/huey.py | 24 +++++++++++++++++ tests/integrations/huey/test_huey.py | 40 ++++++++++++++++++++++++++++ 3 files changed, 66 insertions(+) diff --git a/mypy.ini b/mypy.ini index 2a15e45e49..6e8f6b7230 100644 --- a/mypy.ini +++ b/mypy.ini @@ -63,3 +63,5 @@ disallow_untyped_defs = False ignore_missing_imports = True [mypy-flask.signals] ignore_missing_imports = True +[mypy-huey.*] +ignore_missing_imports = True diff --git a/sentry_sdk/integrations/huey.py b/sentry_sdk/integrations/huey.py index 72debf7d33..63c9adc5a4 100644 --- a/sentry_sdk/integrations/huey.py +++ b/sentry_sdk/integrations/huey.py @@ -8,6 +8,7 @@ if MYPY: from typing import Any, Optional + from sentry_sdk._types import EventProcessor, Event, Hint try: from huey.api import Huey, Task @@ -24,6 +25,28 @@ def setup_once(): patch_execute() +def _make_event_processor(task): + # type: (Any) -> EventProcessor + def event_processor(event, hint): + # type: (Event, Hint) -> Optional[Event] + + with capture_internal_exceptions(): + tags = event.setdefault("tags", {}) + tags["huey_task_id"] = task.id + tags["huey_task_retry"] = task.default_retries > task.retries + extra = event.setdefault("extra", {}) + extra["huey-job"] = { + "task": task.name, + "args": task.args, + "kwargs": task.kwargs, + "retry": (task.default_retries or 0) - task.retries, + } + + return event + + return event_processor + + def patch_execute(): # type: () -> None old_execute = Huey._execute @@ -39,6 +62,7 @@ def _sentry_execute(self, task, timestamp=None): with capture_internal_exceptions(): scope._name = "huey" scope.clear_breadcrumbs() + scope.add_event_processor(_make_event_processor(task)) transaction = Transaction( name=task.name, diff --git a/tests/integrations/huey/test_huey.py b/tests/integrations/huey/test_huey.py index e4fe6006ef..6a1805b941 100644 --- a/tests/integrations/huey/test_huey.py +++ b/tests/integrations/huey/test_huey.py @@ -1,4 +1,5 @@ import pytest +from decimal import DivisionByZero from sentry_sdk.integrations.huey import HueyIntegration @@ -26,6 +27,19 @@ def flush_huey_tasks(init_huey): huey.flush() +def execute_huey_task(huey, func, *args, exceptions=None, **kwargs): + result = func(*args, **kwargs) + task = huey.dequeue() + if exceptions is not None: + try: + huey.execute(task) + except exceptions: + pass + else: + huey.execute(task) + return result + + def test_task_result(init_huey): huey = init_huey() @@ -40,3 +54,29 @@ def increase(num): task = huey.dequeue() assert huey.execute(task) == 4 assert result.get() == 4 + + +@pytest.mark.parametrize("task_fails", [True, False], ids=["error", "success"]) +def test_task_transaction(capture_events, init_huey, task_fails): + huey = init_huey() + + @huey.task() + def division(a, b): + return a / b + + events = capture_events() + execute_huey_task( + huey, division, 1, int(not task_fails), exceptions=(DivisionByZero,) + ) + + if task_fails: + error_event = events.pop(0) + assert error_event["exception"]["values"][0]["type"] == "ZeroDivisionError" + + (event,) = events + assert event["type"] == "transaction" + assert event["transaction"] == "division" + assert event["transaction_info"] == {"source": "task"} + + assert "huey_task_id" in event["tags"] + assert "huey_task_retry" in event["tags"] From 9a84b5ea9b78dbed26efdaf359f069b0499e1ab6 Mon Sep 17 00:00:00 2001 From: Evgeny Seregin Date: Tue, 9 Aug 2022 21:25:05 +0300 Subject: [PATCH 04/16] Added huey to extras_require --- setup.py | 1 + 1 file changed, 1 insertion(+) diff --git a/setup.py b/setup.py index 5ed5560b9b..381241ca1c 100644 --- a/setup.py +++ b/setup.py @@ -51,6 +51,7 @@ def get_file_text(file_name): "django": ["django>=1.8"], "sanic": ["sanic>=0.8"], "celery": ["celery>=3"], + "huey": ["huey>=2"], "beam": ["apache-beam>=2.12"], "rq": ["rq>=0.6"], "aiohttp": ["aiohttp>=3.5"], From 81f3827cef23344c56845e4831370c12c8c2188f Mon Sep 17 00:00:00 2001 From: Evgeny Seregin Date: Tue, 9 Aug 2022 21:50:45 +0300 Subject: [PATCH 05/16] Added error capture --- sentry_sdk/integrations/huey.py | 46 ++++++++++++++++++++++++++-- tests/integrations/huey/test_huey.py | 6 ++++ 2 files changed, 50 insertions(+), 2 deletions(-) diff --git a/sentry_sdk/integrations/huey.py b/sentry_sdk/integrations/huey.py index 63c9adc5a4..802ac3f6f6 100644 --- a/sentry_sdk/integrations/huey.py +++ b/sentry_sdk/integrations/huey.py @@ -1,14 +1,19 @@ +import sys from datetime import datetime +from sentry_sdk._compat import reraise from sentry_sdk._types import MYPY from sentry_sdk import Hub from sentry_sdk.integrations import DidNotEnable, Integration from sentry_sdk.tracing import Transaction, TRANSACTION_SOURCE_TASK -from sentry_sdk.utils import capture_internal_exceptions +from sentry_sdk.utils import capture_internal_exceptions, event_from_exception if MYPY: - from typing import Any, Optional + from typing import Any, Callable, Optional, TypeVar from sentry_sdk._types import EventProcessor, Event, Hint + from sentry_sdk.utils import ExcInfo + + F = TypeVar("F", bound=Callable[..., Any]) try: from huey.api import Huey, Task @@ -47,6 +52,39 @@ def event_processor(event, hint): return event_processor +def _capture_exception(exc_info): + # type: (ExcInfo) -> None + hub = Hub.current + + hub.scope.transaction.set_status("internal_error") + event, hint = event_from_exception( + exc_info, + client_options=hub.client.options if hub.client else None, + mechanism={"type": "huey", "handled": False}, + ) + hub.capture_event(event, hint=hint) + + +def _wrap_task_execute(func): + # type: (F) -> F + def _sentry_execute(*args, **kwargs): + # type: (*Any, **Any) -> Any + hub = Hub.current + if hub.get_integration(HueyIntegration) is None: + return func(*args, **kwargs) + + try: + result = func(*args, **kwargs) + except Exception: + exc_info = sys.exc_info() + _capture_exception(exc_info) + reraise(*exc_info) + + return result + + return _sentry_execute # type: ignore + + def patch_execute(): # type: () -> None old_execute = Huey._execute @@ -71,6 +109,10 @@ def _sentry_execute(self, task, timestamp=None): source=TRANSACTION_SOURCE_TASK, ) + if not getattr(task, "_sentry_patched", False): + task.execute = _wrap_task_execute(task.execute) + task._sentry_patched = True + with hub.start_transaction(transaction): return old_execute(self, task, timestamp) diff --git a/tests/integrations/huey/test_huey.py b/tests/integrations/huey/test_huey.py index 6a1805b941..8e90880ba0 100644 --- a/tests/integrations/huey/test_huey.py +++ b/tests/integrations/huey/test_huey.py @@ -72,11 +72,17 @@ def division(a, b): if task_fails: error_event = events.pop(0) assert error_event["exception"]["values"][0]["type"] == "ZeroDivisionError" + assert error_event["exception"]["values"][0]["mechanism"]["type"] == "huey" (event,) = events assert event["type"] == "transaction" assert event["transaction"] == "division" assert event["transaction_info"] == {"source": "task"} + if task_fails: + assert event["contexts"]["trace"]["status"] == "internal_error" + else: + assert event["contexts"]["trace"]["status"] == "ok" + assert "huey_task_id" in event["tags"] assert "huey_task_retry" in event["tags"] From 7ac5d32d54472687524622b874fce7cc37a4f2ff Mon Sep 17 00:00:00 2001 From: Evgeny Seregin Date: Tue, 9 Aug 2022 22:51:34 +0300 Subject: [PATCH 06/16] Added control flow exceptions handling --- sentry_sdk/integrations/huey.py | 8 ++++++++ tests/integrations/huey/test_huey.py | 28 ++++++++++++++++++++++++++++ 2 files changed, 36 insertions(+) diff --git a/sentry_sdk/integrations/huey.py b/sentry_sdk/integrations/huey.py index 802ac3f6f6..693bc3928f 100644 --- a/sentry_sdk/integrations/huey.py +++ b/sentry_sdk/integrations/huey.py @@ -17,10 +17,14 @@ try: from huey.api import Huey, Task + from huey.exceptions import CancelExecution, RetryTask except ImportError: raise DidNotEnable("Huey is not installed") +HUEY_CONTROL_FLOW_EXCEPTIONS = (CancelExecution, RetryTask) + + class HueyIntegration(Integration): identifier = "huey" @@ -56,6 +60,10 @@ def _capture_exception(exc_info): # type: (ExcInfo) -> None hub = Hub.current + if exc_info[0] in HUEY_CONTROL_FLOW_EXCEPTIONS: + hub.scope.transaction.set_status("aborted") + return + hub.scope.transaction.set_status("internal_error") event, hint = event_from_exception( exc_info, diff --git a/tests/integrations/huey/test_huey.py b/tests/integrations/huey/test_huey.py index 8e90880ba0..ee65ab7466 100644 --- a/tests/integrations/huey/test_huey.py +++ b/tests/integrations/huey/test_huey.py @@ -4,6 +4,7 @@ from sentry_sdk.integrations.huey import HueyIntegration from huey.api import RedisExpireHuey, Result +from huey.exceptions import RetryTask @pytest.fixture @@ -86,3 +87,30 @@ def division(a, b): assert "huey_task_id" in event["tags"] assert "huey_task_retry" in event["tags"] + + +def test_task_retry(capture_events, init_huey): + huey = init_huey() + context = {"retry": True} + + @huey.task() + def retry_task(context): + if context["retry"]: + context["retry"] = False + raise RetryTask() + + events = capture_events() + result = execute_huey_task(huey, retry_task, context) + (event,) = events + + assert event["transaction"] == "retry_task" + assert event["tags"]["huey_task_id"] == result.task.id + assert len(huey) == 1 + + task = huey.dequeue() + huey.execute(task) + (event, _) = events + + assert event["transaction"] == "retry_task" + assert event["tags"]["huey_task_id"] == result.task.id + assert len(huey) == 0 From 1ec6ed3844a11dd2403c8898ce8826fab5e3c7e3 Mon Sep 17 00:00:00 2001 From: Evgeny Seregin Date: Wed, 10 Aug 2022 20:05:10 +0300 Subject: [PATCH 07/16] Added tasks enqueue handling --- sentry_sdk/integrations/huey.py | 26 ++++++++++++++++++++++---- tests/integrations/huey/test_huey.py | 23 +++++++++++++++++++++++ 2 files changed, 45 insertions(+), 4 deletions(-) diff --git a/sentry_sdk/integrations/huey.py b/sentry_sdk/integrations/huey.py index 693bc3928f..bee4c9ecb6 100644 --- a/sentry_sdk/integrations/huey.py +++ b/sentry_sdk/integrations/huey.py @@ -9,14 +9,14 @@ from sentry_sdk.utils import capture_internal_exceptions, event_from_exception if MYPY: - from typing import Any, Callable, Optional, TypeVar + from typing import Any, Callable, Optional, Union, TypeVar from sentry_sdk._types import EventProcessor, Event, Hint from sentry_sdk.utils import ExcInfo F = TypeVar("F", bound=Callable[..., Any]) try: - from huey.api import Huey, Task + from huey.api import Huey, Result, ResultGroup, Task from huey.exceptions import CancelExecution, RetryTask except ImportError: raise DidNotEnable("Huey is not installed") @@ -31,9 +31,27 @@ class HueyIntegration(Integration): @staticmethod def setup_once(): # type: () -> None + patch_enqueue() patch_execute() +def patch_enqueue(): + # type: () -> None + old_enqueue = Huey.enqueue + + def _sentry_enqueue(self, task): + # type: (Huey, Task) -> Optional[Union[Result, ResultGroup]] + hub = Hub.current + + if hub.get_integration(HueyIntegration) is None: + return old_enqueue(self, task) + + with hub.start_span(op="huey.enqueue", description=task.name): + return old_enqueue(self, task) + + Huey.enqueue = _sentry_enqueue + + def _make_event_processor(task): # type: (Any) -> EventProcessor def event_processor(event, hint): @@ -117,9 +135,9 @@ def _sentry_execute(self, task, timestamp=None): source=TRANSACTION_SOURCE_TASK, ) - if not getattr(task, "_sentry_patched", False): + if not getattr(task, "_sentry_is_patched", False): task.execute = _wrap_task_execute(task.execute) - task._sentry_patched = True + task._sentry_is_patched = True with hub.start_transaction(transaction): return old_execute(self, task, timestamp) diff --git a/tests/integrations/huey/test_huey.py b/tests/integrations/huey/test_huey.py index ee65ab7466..09a88e62f6 100644 --- a/tests/integrations/huey/test_huey.py +++ b/tests/integrations/huey/test_huey.py @@ -1,6 +1,7 @@ import pytest from decimal import DivisionByZero +from sentry_sdk import start_transaction from sentry_sdk.integrations.huey import HueyIntegration from huey.api import RedisExpireHuey, Result @@ -114,3 +115,25 @@ def retry_task(context): assert event["transaction"] == "retry_task" assert event["tags"]["huey_task_id"] == result.task.id assert len(huey) == 0 + + +def test_huey_enqueue(init_huey, capture_events): + huey = init_huey() + + @huey.task(name="different_task_name") + def dummy_task(): + pass + + events = capture_events() + + with start_transaction() as transaction: + dummy_task() + + (event,) = events + + assert event["contexts"]["trace"]["trace_id"] == transaction.trace_id + assert event["contexts"]["trace"]["span_id"] == transaction.span_id + + assert len(event["spans"]) + assert event["spans"][0]["op"] == "huey.enqueue" + assert event["spans"][0]["description"] == "different_task_name" From 8d37b958533c6fe1f9b9047795d282cc8ee8aeeb Mon Sep 17 00:00:00 2001 From: Evgeny Seregin Date: Thu, 11 Aug 2022 16:11:17 +0300 Subject: [PATCH 08/16] Fixed python 2.7 support --- sentry_sdk/integrations/huey.py | 3 +++ tests/integrations/huey/test_huey.py | 3 ++- 2 files changed, 5 insertions(+), 1 deletion(-) diff --git a/sentry_sdk/integrations/huey.py b/sentry_sdk/integrations/huey.py index bee4c9ecb6..79c69e153c 100644 --- a/sentry_sdk/integrations/huey.py +++ b/sentry_sdk/integrations/huey.py @@ -1,3 +1,5 @@ +from __future__ import absolute_import + import sys from datetime import datetime @@ -10,6 +12,7 @@ if MYPY: from typing import Any, Callable, Optional, Union, TypeVar + from sentry_sdk._types import EventProcessor, Event, Hint from sentry_sdk.utils import ExcInfo diff --git a/tests/integrations/huey/test_huey.py b/tests/integrations/huey/test_huey.py index 09a88e62f6..5a0bf9f31d 100644 --- a/tests/integrations/huey/test_huey.py +++ b/tests/integrations/huey/test_huey.py @@ -29,7 +29,8 @@ def flush_huey_tasks(init_huey): huey.flush() -def execute_huey_task(huey, func, *args, exceptions=None, **kwargs): +def execute_huey_task(huey, func, *args, **kwargs): + exceptions = kwargs.pop("exceptions", None) result = func(*args, **kwargs) task = huey.dequeue() if exceptions is not None: From 1f7697884ab0e0fa988c38b5681514a27b8da22e Mon Sep 17 00:00:00 2001 From: Evgeny Seregin Date: Thu, 11 Aug 2022 18:43:19 +0300 Subject: [PATCH 09/16] Small fix --- tox.ini | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/tox.ini b/tox.ini index e853be622a..97d28f5bcf 100644 --- a/tox.ini +++ b/tox.ini @@ -52,7 +52,7 @@ envlist = {pypy,py2.7,py3.5,py3.6,py3.7,py3.8}-celery-{4.3,4.4} {py3.6,py3.7,py3.8,py3.9,py3.10}-celery-5.0 - {py2.7,py3.4,py3.5,py3.6,py3.7,py3.8,py3.9,py3.10}-huey-2 + {py2.7,py3.5,py3.6,py3.7,py3.8,py3.9,py3.10}-huey-2 py3.7-beam-{2.12,2.13,2.32,2.33} @@ -182,6 +182,7 @@ deps = py3.5-celery: newrelic<6.0.0 {pypy,py2.7,py3.6,py3.7,py3.8,py3.9,py3.10}-celery: newrelic + huey: redis huey-2: huey>=2.0 requests: requests>=2.0 From 56942bcefe10ac3490d9b6625f303ae3cff4e8a5 Mon Sep 17 00:00:00 2001 From: Evgeny Seregin Date: Fri, 11 Nov 2022 23:31:04 +0300 Subject: [PATCH 10/16] Split tox github actions --- .github/workflows/test-integration-huey.yml | 62 +++++++++++++++++++++ 1 file changed, 62 insertions(+) create mode 100644 .github/workflows/test-integration-huey.yml diff --git a/.github/workflows/test-integration-huey.yml b/.github/workflows/test-integration-huey.yml new file mode 100644 index 0000000000..708903a61c --- /dev/null +++ b/.github/workflows/test-integration-huey.yml @@ -0,0 +1,62 @@ +name: Test huey + +on: + push: + branches: + - master + - release/** + + pull_request: + +# Cancel in progress workflows on pull_requests. +# https://docs.github.com/en/actions/using-jobs/using-concurrency#example-using-a-fallback-value +concurrency: + group: ${{ github.workflow }}-${{ github.head_ref || github.run_id }} + cancel-in-progress: true + +permissions: + contents: read + +env: + BUILD_CACHE_KEY: ${{ github.sha }} + CACHED_BUILD_PATHS: | + ${{ github.workspace }}/dist-serverless + +jobs: + test: + name: huey, python ${{ matrix.python-version }}, ${{ matrix.os }} + runs-on: ${{ matrix.os }} + timeout-minutes: 45 + continue-on-error: true + + strategy: + matrix: + python-version: ["2.7","3.5","3.6","3.7","3.8","3.9","3.10"] + os: [ubuntu-latest] + + steps: + - uses: actions/checkout@v3 + - uses: actions/setup-python@v4 + with: + python-version: ${{ matrix.python-version }} + + - name: Setup Test Env + env: + PGHOST: localhost + PGPASSWORD: sentry + run: | + pip install codecov tox + + - name: Test huey + env: + CI_PYTHON_VERSION: ${{ matrix.python-version }} + timeout-minutes: 45 + shell: bash + run: | + set -x # print commands that are executed + coverage erase + + ./scripts/runtox.sh "${{ matrix.python-version }}-huey" --cov=tests --cov=sentry_sdk --cov-report= --cov-branch + coverage combine .coverage* + coverage xml -i + codecov --file coverage.xml From e66cf4de10a051c5675ed286cf91dd40ea0b1850 Mon Sep 17 00:00:00 2001 From: Evgeny Seregin Date: Fri, 11 Nov 2022 23:37:58 +0300 Subject: [PATCH 11/16] Cosmetic improvements --- sentry_sdk/consts.py | 2 ++ sentry_sdk/integrations/huey.py | 7 ++++--- tests/integrations/huey/test_huey.py | 2 +- 3 files changed, 7 insertions(+), 4 deletions(-) diff --git a/sentry_sdk/consts.py b/sentry_sdk/consts.py index c920fc8fa5..ef85574718 100644 --- a/sentry_sdk/consts.py +++ b/sentry_sdk/consts.py @@ -122,7 +122,9 @@ class OP: MIDDLEWARE_STARLETTE_RECEIVE = "middleware.starlette.receive" MIDDLEWARE_STARLETTE_SEND = "middleware.starlette.send" QUEUE_SUBMIT_CELERY = "queue.submit.celery" + QUEUE_SUBMIT_HUEY = "queue.submit.huey" QUEUE_TASK_CELERY = "queue.task.celery" + QUEUE_TASK_HUEY = "queue.task.huey" QUEUE_TASK_RQ = "queue.task.rq" SUBPROCESS = "subprocess" SUBPROCESS_WAIT = "subprocess.wait" diff --git a/sentry_sdk/integrations/huey.py b/sentry_sdk/integrations/huey.py index 79c69e153c..5ea0a12512 100644 --- a/sentry_sdk/integrations/huey.py +++ b/sentry_sdk/integrations/huey.py @@ -6,6 +6,7 @@ from sentry_sdk._compat import reraise from sentry_sdk._types import MYPY from sentry_sdk import Hub +from sentry_sdk.consts import OP from sentry_sdk.integrations import DidNotEnable, Integration from sentry_sdk.tracing import Transaction, TRANSACTION_SOURCE_TASK from sentry_sdk.utils import capture_internal_exceptions, event_from_exception @@ -49,7 +50,7 @@ def _sentry_enqueue(self, task): if hub.get_integration(HueyIntegration) is None: return old_enqueue(self, task) - with hub.start_span(op="huey.enqueue", description=task.name): + with hub.start_span(op=OP.QUEUE_SUBMIT_HUEY, description=task.name): return old_enqueue(self, task) Huey.enqueue = _sentry_enqueue @@ -89,7 +90,7 @@ def _capture_exception(exc_info): event, hint = event_from_exception( exc_info, client_options=hub.client.options if hub.client else None, - mechanism={"type": "huey", "handled": False}, + mechanism={"type": HueyIntegration.identifier, "handled": False}, ) hub.capture_event(event, hint=hint) @@ -134,7 +135,7 @@ def _sentry_execute(self, task, timestamp=None): transaction = Transaction( name=task.name, status="ok", - op="huey.task", + op=OP.QUEUE_TASK_HUEY, source=TRANSACTION_SOURCE_TASK, ) diff --git a/tests/integrations/huey/test_huey.py b/tests/integrations/huey/test_huey.py index 5a0bf9f31d..a12a7acfcb 100644 --- a/tests/integrations/huey/test_huey.py +++ b/tests/integrations/huey/test_huey.py @@ -136,5 +136,5 @@ def dummy_task(): assert event["contexts"]["trace"]["span_id"] == transaction.span_id assert len(event["spans"]) - assert event["spans"][0]["op"] == "huey.enqueue" + assert event["spans"][0]["op"] == "queue.submit.huey" assert event["spans"][0]["description"] == "different_task_name" From 0c0899b4c9735204240e39285a39e0fa066eaee5 Mon Sep 17 00:00:00 2001 From: Evgeny Seregin Date: Wed, 16 Nov 2022 21:36:07 +0300 Subject: [PATCH 12/16] Replacing RedisStorage with MemoryStorage in tests --- tests/integrations/huey/test_huey.py | 4 ++-- tox.ini | 1 - 2 files changed, 2 insertions(+), 3 deletions(-) diff --git a/tests/integrations/huey/test_huey.py b/tests/integrations/huey/test_huey.py index a12a7acfcb..819a4816d7 100644 --- a/tests/integrations/huey/test_huey.py +++ b/tests/integrations/huey/test_huey.py @@ -4,7 +4,7 @@ from sentry_sdk import start_transaction from sentry_sdk.integrations.huey import HueyIntegration -from huey.api import RedisExpireHuey, Result +from huey.api import MemoryHuey, Result from huey.exceptions import RetryTask @@ -18,7 +18,7 @@ def inner(): debug=True, ) - return RedisExpireHuey(name="sentry_sdk", url="redis://127.0.0.1:6379") + return MemoryHuey(name="sentry_sdk") return inner diff --git a/tox.ini b/tox.ini index a0d245d64b..5b381f24de 100644 --- a/tox.ini +++ b/tox.ini @@ -210,7 +210,6 @@ deps = {py3.7}-celery: importlib-metadata<5.0 {py2.7,py3.6,py3.7,py3.8,py3.9,py3.10}-celery: newrelic - huey: redis huey-2: huey>=2.0 requests: requests>=2.0 From 48c2fa8a191feb362c06b109f7eb5528e7caecc9 Mon Sep 17 00:00:00 2001 From: Anton Pirker Date: Wed, 25 Jan 2023 11:15:33 +0100 Subject: [PATCH 13/16] Updated test script --- .github/workflows/test-integration-huey.yml | 29 ++++++++++++++------- 1 file changed, 20 insertions(+), 9 deletions(-) diff --git a/.github/workflows/test-integration-huey.yml b/.github/workflows/test-integration-huey.yml index 708903a61c..4226083299 100644 --- a/.github/workflows/test-integration-huey.yml +++ b/.github/workflows/test-integration-huey.yml @@ -27,12 +27,16 @@ jobs: name: huey, python ${{ matrix.python-version }}, ${{ matrix.os }} runs-on: ${{ matrix.os }} timeout-minutes: 45 - continue-on-error: true strategy: + fail-fast: false matrix: - python-version: ["2.7","3.5","3.6","3.7","3.8","3.9","3.10"] - os: [ubuntu-latest] + python-version: ["2.7","3.5","3.6","3.7","3.8","3.9","3.10","3.11"] + # python3.6 reached EOL and is no longer being supported on + # new versions of hosted runners on Github Actions + # ubuntu-20.04 is the last version that supported python3.6 + # see https://github.com/actions/setup-python/issues/544#issuecomment-1332535877 + os: [ubuntu-20.04] steps: - uses: actions/checkout@v3 @@ -41,15 +45,10 @@ jobs: python-version: ${{ matrix.python-version }} - name: Setup Test Env - env: - PGHOST: localhost - PGPASSWORD: sentry run: | - pip install codecov tox + pip install codecov "tox>=3,<4" - name: Test huey - env: - CI_PYTHON_VERSION: ${{ matrix.python-version }} timeout-minutes: 45 shell: bash run: | @@ -60,3 +59,15 @@ jobs: coverage combine .coverage* coverage xml -i codecov --file coverage.xml + + check_required_tests: + name: All huey tests passed or skipped + needs: test + # Always run this, even if a dependent job failed + if: always() + runs-on: ubuntu-20.04 + steps: + - name: Check for failures + if: contains(needs.test.result, 'failure') + run: | + echo "One of the dependent jobs have failed. You may need to re-run it." && exit 1 From bb9b955ed67032cd355b118ed5c2b2b78035c020 Mon Sep 17 00:00:00 2001 From: Anton Pirker Date: Wed, 25 Jan 2023 11:19:01 +0100 Subject: [PATCH 14/16] Adding missing constants --- sentry_sdk/consts.py | 2 ++ 1 file changed, 2 insertions(+) diff --git a/sentry_sdk/consts.py b/sentry_sdk/consts.py index 1e309837a3..b2d1ae26c7 100644 --- a/sentry_sdk/consts.py +++ b/sentry_sdk/consts.py @@ -72,6 +72,8 @@ class OP: QUEUE_SUBMIT_CELERY = "queue.submit.celery" QUEUE_TASK_CELERY = "queue.task.celery" QUEUE_TASK_RQ = "queue.task.rq" + QUEUE_SUBMIT_HUEY = "queue.submit.huey" + QUEUE_TASK_HUEY = "queue.task.huey" SUBPROCESS = "subprocess" SUBPROCESS_WAIT = "subprocess.wait" SUBPROCESS_COMMUNICATE = "subprocess.communicate" From 94ec102d392a055e43196d41317ada04da7ea590 Mon Sep 17 00:00:00 2001 From: Anton Pirker Date: Wed, 25 Jan 2023 13:11:19 +0100 Subject: [PATCH 15/16] Only send sensitive data when send_default_pii is set to True --- sentry_sdk/integrations/huey.py | 7 ++++--- 1 file changed, 4 insertions(+), 3 deletions(-) diff --git a/sentry_sdk/integrations/huey.py b/sentry_sdk/integrations/huey.py index 5ea0a12512..a4be04fa5f 100644 --- a/sentry_sdk/integrations/huey.py +++ b/sentry_sdk/integrations/huey.py @@ -6,7 +6,8 @@ from sentry_sdk._compat import reraise from sentry_sdk._types import MYPY from sentry_sdk import Hub -from sentry_sdk.consts import OP +from sentry_sdk.consts import OP, SENSITIVE_DATA_SUBSTITUTE +from sentry_sdk.hub import _should_send_default_pii from sentry_sdk.integrations import DidNotEnable, Integration from sentry_sdk.tracing import Transaction, TRANSACTION_SOURCE_TASK from sentry_sdk.utils import capture_internal_exceptions, event_from_exception @@ -68,8 +69,8 @@ def event_processor(event, hint): extra = event.setdefault("extra", {}) extra["huey-job"] = { "task": task.name, - "args": task.args, - "kwargs": task.kwargs, + "args": task.args if _should_send_default_pii() else SENSITIVE_DATA_SUBSTITUTE, + "kwargs": task.kwargs if _should_send_default_pii() else SENSITIVE_DATA_SUBSTITUTE, "retry": (task.default_retries or 0) - task.retries, } From b9e8915cb513e8131cf311acffd130f1e84cf6b1 Mon Sep 17 00:00:00 2001 From: Anton Pirker Date: Wed, 25 Jan 2023 13:15:16 +0100 Subject: [PATCH 16/16] Formatting --- sentry_sdk/integrations/huey.py | 8 ++++++-- 1 file changed, 6 insertions(+), 2 deletions(-) diff --git a/sentry_sdk/integrations/huey.py b/sentry_sdk/integrations/huey.py index a4be04fa5f..8f5f26133c 100644 --- a/sentry_sdk/integrations/huey.py +++ b/sentry_sdk/integrations/huey.py @@ -69,8 +69,12 @@ def event_processor(event, hint): extra = event.setdefault("extra", {}) extra["huey-job"] = { "task": task.name, - "args": task.args if _should_send_default_pii() else SENSITIVE_DATA_SUBSTITUTE, - "kwargs": task.kwargs if _should_send_default_pii() else SENSITIVE_DATA_SUBSTITUTE, + "args": task.args + if _should_send_default_pii() + else SENSITIVE_DATA_SUBSTITUTE, + "kwargs": task.kwargs + if _should_send_default_pii() + else SENSITIVE_DATA_SUBSTITUTE, "retry": (task.default_retries or 0) - task.retries, }