Connection refused on Celery health checks

[lvieirajr]

Getting a connection refused error when trying to send either a ping or a task to Celery via the health check.
But creating a simple view that directly invokes ping on my celery works fine.

Python 3.12.2
django==5.03
django-redis==5.4.0
celery==5.3.6
amqp==5.2.0
kombu==5.3.5
billiard==4.2.0
redis==5.0.3
hiredis==2.3.2

django | 2024-03-14T23:09:43.198232Z [info ] request_started [django_structlog.middlewares.request] ip=192.168.65.1 request=GET /test/ request_id=19693212-3ffe-49c7-a3ad-af8913676e64 user_agent=Mozilla/5.0 (Macintosh; Intel Mac OS X 10_15_7) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/122.0.0.0 Safari/537.36 user_id=None
django | [{'celery@efa73fb42c24': {'ok': 'pong'}}]
django | 2024-03-14T23:09:44.255825Z [info ] request_finished [django_structlog.middlewares.request] code=200 ip=192.168.65.1 request=GET /test/ request_id=19693212-3ffe-49c7-a3ad-af8913676e64 user_id=None

unavailable: Unknown error
django | Traceback (most recent call last):
django | File "/usr/local/lib/python3.12/site-packages/kombu/connection.py", line 472, in _reraise_as_library_errors
django | yield
django | File "/usr/local/lib/python3.12/site-packages/kombu/connection.py", line 459, in _ensure_connection
django | return retry_over_time(
django | ^^^^^^^^^^^^^^^^
django | File "/usr/local/lib/python3.12/site-packages/kombu/utils/functional.py", line 318, in retry_over_time
django | return fun(*args, **kwargs)
django | ^^^^^^^^^^^^^^^^^^^^
django | File "/usr/local/lib/python3.12/site-packages/kombu/connection.py", line 934, in _connection_factory
django | self._connection = self._establish_connection()
django | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^
django | File "/usr/local/lib/python3.12/site-packages/kombu/connection.py", line 860, in _establish_connection
django | conn = self.transport.establish_connection()
django | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^
django | File "/usr/local/lib/python3.12/site-packages/kombu/transport/pyamqp.py", line 203, in establish_connection
django | conn.connect()
django | File "/usr/local/lib/python3.12/site-packages/amqp/connection.py", line 324, in connect
django | self.transport.connect()
django | File "/usr/local/lib/python3.12/site-packages/amqp/transport.py", line 129, in connect
django | self._connect(self.host, self.port, self.connect_timeout)
django | File "/usr/local/lib/python3.12/site-packages/amqp/transport.py", line 184, in _connect
django | self.sock.connect(sa)
django | ConnectionRefusedError: [Errno 111] Connection refused
django |
django | The above exception was the direct cause of the following exception:
django |
django | Traceback (most recent call last):
django | File "/usr/local/lib/python3.12/site-packages/health_check/contrib/celery_ping/backends.py", line 15, in check_status
django | ping_result = app.control.ping(timeout=timeout)
django | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^
django | File "/usr/local/lib/python3.12/site-packages/celery/app/control.py", line 563, in ping
django | return self.broadcast(
django | ^^^^^^^^^^^^^^^
django | File "/usr/local/lib/python3.12/site-packages/celery/app/control.py", line 776, in broadcast
django | return self.mailbox(conn)._broadcast(
django | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^
django | File "/usr/local/lib/python3.12/site-packages/kombu/pidbox.py", line 330, in _broadcast
django | chan = channel or self.connection.default_channel
django | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^
django | File "/usr/local/lib/python3.12/site-packages/kombu/connection.py", line 953, in default_channel
django | self._ensure_connection(**conn_opts)
django | File "/usr/local/lib/python3.12/site-packages/kombu/connection.py", line 458, in _ensure_connection
django | with ctx():
django | File "/usr/local/lib/python3.12/contextlib.py", line 158, in exit
django | self.gen.throw(value)
django | File "/usr/local/lib/python3.12/site-packages/kombu/connection.py", line 476, in _reraise_as_library_errors
django | raise ConnectionError(str(exc)) from exc
django | kombu.exceptions.OperationalError: [Errno 111] Connection refused
django | unavailable: Unknown error
django | Traceback (most recent call last):
django | File "/usr/local/lib/python3.12/site-packages/kombu/utils/functional.py", line 32, in call
django | return self.value
django | ^^^^^^^^^^^^^^
django | AttributeError: 'ChannelPromise' object has no attribute 'value'. Did you mean: 'call'?
django |
django | During handling of the above exception, another exception occurred:
django |
django | Traceback (most recent call last):
django | File "/usr/local/lib/python3.12/site-packages/kombu/connection.py", line 472, in _reraise_as_library_errors
django | yield
django | File "/usr/local/lib/python3.12/site-packages/kombu/connection.py", line 459, in _ensure_connection
django | return retry_over_time(
django | ^^^^^^^^^^^^^^^^
django | File "/usr/local/lib/python3.12/site-packages/kombu/utils/functional.py", line 318, in retry_over_time
django | return fun(*args, **kwargs)
django | ^^^^^^^^^^^^^^^^^^^^
django | File "/usr/local/lib/python3.12/site-packages/kombu/connection.py", line 934, in _connection_factory
django | self._connection = self._establish_connection()
django | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^
django | File "/usr/local/lib/python3.12/site-packages/kombu/connection.py", line 860, in _establish_connection
django | conn = self.transport.establish_connection()
django | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^
django | File "/usr/local/lib/python3.12/site-packages/kombu/transport/pyamqp.py", line 203, in establish_connection
django | conn.connect()
django | File "/usr/local/lib/python3.12/site-packages/amqp/connection.py", line 324, in connect
django | self.transport.connect()
django | File "/usr/local/lib/python3.12/site-packages/amqp/transport.py", line 129, in connect
django | self._connect(self.host, self.port, self.connect_timeout)
django | File "/usr/local/lib/python3.12/site-packages/amqp/transport.py", line 184, in _connect
django | self.sock.connect(sa)
django | ConnectionRefusedError: [Errno 111] Connection refused
django |
django | The above exception was the direct cause of the following exception:
django |
django | Traceback (most recent call last):
django | File "/usr/local/lib/python3.12/site-packages/health_check/contrib/celery/backends.py", line 17, in check_status
django | result = add.apply_async(
django | ^^^^^^^^^^^^^^^^
django | File "/usr/local/lib/python3.12/site-packages/sentry_sdk/integrations/celery.py", line 228, in apply_async
django | return f(*args, **kwargs)
django | ^^^^^^^^^^^^^^^^^^
django | File "/usr/local/lib/python3.12/site-packages/celery/app/task.py", line 594, in apply_async
django | return app.send_task(
django | ^^^^^^^^^^^^^^
django | File "/usr/local/lib/python3.12/site-packages/celery/app/base.py", line 799, in send_task
django | amqp.send_task_message(P, name, message, **options)
django | File "/usr/local/lib/python3.12/site-packages/celery/app/amqp.py", line 518, in send_task_message
django | ret = producer.publish(
django | ^^^^^^^^^^^^^^^^^
django | File "/usr/local/lib/python3.12/site-packages/kombu/messaging.py", line 186, in publish
django | return _publish(
django | ^^^^^^^^^
django | File "/usr/local/lib/python3.12/site-packages/kombu/connection.py", line 556, in _ensured
django | return fun(*args, **kwargs)
django | ^^^^^^^^^^^^^^^^^^^^
django | File "/usr/local/lib/python3.12/site-packages/kombu/messaging.py", line 195, in _publish
django | channel = self.channel
django | ^^^^^^^^^^^^
django | File "/usr/local/lib/python3.12/site-packages/kombu/messaging.py", line 218, in _get_channel
django | channel = self._channel = channel()
django | ^^^^^^^^^
django | File "/usr/local/lib/python3.12/site-packages/kombu/utils/functional.py", line 34, in call
django | value = self.value = self.contract()
django | ^^^^^^^^^^^^^^^^^^^
django | File "/usr/local/lib/python3.12/site-packages/kombu/messaging.py", line 234, in
django | channel = ChannelPromise(lambda: connection.default_channel)
django | ^^^^^^^^^^^^^^^^^^^^^^^^^^
django | File "/usr/local/lib/python3.12/site-packages/kombu/connection.py", line 953, in default_channel
django | self._ensure_connection(**conn_opts)
django | File "/usr/local/lib/python3.12/site-packages/kombu/connection.py", line 458, in _ensure_connection
django | with ctx():
django | File "/usr/local/lib/python3.12/contextlib.py", line 158, in exit
django | self.gen.throw(value)
django | File "/usr/local/lib/python3.12/site-packages/kombu/connection.py", line 476, in _reraise_as_library_errors
django | raise ConnectionError(str(exc)) from exc
django | kombu.exceptions.OperationalError: [Errno 111] Connection refused

[WisdomPill]

to me it looks like a kombu hence celery error, django-redis is not even in the stacktrace... Do you have any reason to think it is django-redis?

[50-Course]

may i suggest @lvieirajr you open a ticket on celery/kombu?

[lvieirajr]

I'm so sorry, I must have clicked the wrong repo.
This Issue was supposed to be opened on the django-health-check repo, not here.