From 6cf77122da845e5b0afa1607348cf06602679329 Mon Sep 17 00:00:00 2001 From: Clemens Stolle Date: Fri, 26 Apr 2024 16:18:29 +0200 Subject: [PATCH 1/6] fix(redis-connection): increase redis retry strategy backoff (#2546) [python] --- python/bullmq/redis_connection.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/python/bullmq/redis_connection.py b/python/bullmq/redis_connection.py index f068c9ab0..2088d884f 100644 --- a/python/bullmq/redis_connection.py +++ b/python/bullmq/redis_connection.py @@ -24,7 +24,7 @@ class RedisConnection: def __init__(self, redisOpts: dict | str | redis.Redis = {}): self.version = None - retry = Retry(ExponentialBackoff(), 3) + retry = Retry(ExponentialBackoff(cap=20, base=1), 20) retry_errors = [BusyLoadingError, ConnectionError, TimeoutError] if isinstance(redisOpts, redis.Redis): From 723457f6eba0cd320bdac32ada807f83bd46e46b Mon Sep 17 00:00:00 2001 From: semantic-release Date: Fri, 26 Apr 2024 14:19:43 +0000 Subject: [PATCH 2/6] 2.7.4 Automatically generated by python-semantic-release --- docs/gitbook/python/changelog.md | 7 +++++++ python/bullmq/__init__.py | 2 +- python/pyproject.toml | 2 +- 3 files changed, 9 insertions(+), 2 deletions(-) diff --git a/docs/gitbook/python/changelog.md b/docs/gitbook/python/changelog.md index a610c5e72..f7a97d556 100644 --- a/docs/gitbook/python/changelog.md +++ b/docs/gitbook/python/changelog.md @@ -2,6 +2,13 @@ +## v2.7.4 (2024-04-26) +### Fix +* **redis-connection:** Increase redis retry strategy backoff (#2546) [python] ([`6cf7712`](https://github.com/taskforcesh/bullmq/commit/6cf77122da845e5b0afa1607348cf06602679329)) + +### Documentation +* **queue:** Minor styling changes for consistency ([#2541](https://github.com/taskforcesh/bullmq/issues/2541)) ([`a7a7f4f`](https://github.com/taskforcesh/bullmq/commit/a7a7f4f34b147490a713d8b4cef7c6941f3f3eed)) + ## v2.7.3 (2024-04-24) ### Fix * **stalled:** Consider ignoreDependencyOnFailure option (python) (#2540) fixes #2531 ([`0140959`](https://github.com/taskforcesh/bullmq/commit/0140959cabd2613794631e41ebe4c2ddee6f91da)) diff --git a/python/bullmq/__init__.py b/python/bullmq/__init__.py index 45dd6db45..04638f5f7 100644 --- a/python/bullmq/__init__.py +++ b/python/bullmq/__init__.py @@ -3,7 +3,7 @@ A background job processor and message queue for Python based on Redis. """ -__version__ = "2.7.3" +__version__ = "2.7.4" __author__ = 'Taskforce.sh Inc.' __credits__ = 'Taskforce.sh Inc.' diff --git a/python/pyproject.toml b/python/pyproject.toml index 42a0ac401..623d73bfd 100644 --- a/python/pyproject.toml +++ b/python/pyproject.toml @@ -4,7 +4,7 @@ build-backend = "setuptools.build_meta" [project] name = "bullmq" -version = "2.7.3" +version = "2.7.4" description='BullMQ for Python' readme="README.md" authors = [ From 9760b85dfbcc9b3c744f616961ef939e8951321d Mon Sep 17 00:00:00 2001 From: Rogger Valverde Date: Fri, 26 Apr 2024 22:11:56 -0600 Subject: [PATCH 3/6] perf(worker): do not call bzpopmin when blockDelay is lower or equal 0 (#2544) ref #2466 --- src/classes/worker.ts | 28 ++++++++++++++++------------ tests/test_worker.ts | 14 ++++---------- 2 files changed, 20 insertions(+), 22 deletions(-) diff --git a/src/classes/worker.ts b/src/classes/worker.ts index 45c572f35..1a23f2dc9 100644 --- a/src/classes/worker.ts +++ b/src/classes/worker.ts @@ -621,20 +621,22 @@ will never work with more accuracy than 1ms. */ if (!this.closing) { let blockTimeout = this.getBlockTimeout(blockUntil); - blockTimeout = this.blockingConnection.capabilities.canDoubleTimeout - ? blockTimeout - : Math.ceil(blockTimeout); + if (blockTimeout > 0) { + blockTimeout = this.blockingConnection.capabilities.canDoubleTimeout + ? blockTimeout + : Math.ceil(blockTimeout); - this.updateDelays(); // reset delays to avoid reusing same values in next iteration - // Markers should only be used for un-blocking, so we will handle them in this - // function only. - const result = await bclient.bzpopmin(this.keys.marker, blockTimeout); + this.updateDelays(); // reset delays to avoid reusing same values in next iteration + // Markers should only be used for un-blocking, so we will handle them in this + // function only. + const result = await bclient.bzpopmin(this.keys.marker, blockTimeout); - if (result) { - const [_key, member, score] = result; + if (result) { + const [_key, member, score] = result; - if (member) { - return parseInt(score); + if (member) { + return parseInt(score); + } } } @@ -658,7 +660,9 @@ will never work with more accuracy than 1ms. */ if (blockUntil) { const blockDelay = blockUntil - Date.now(); // when we reach the time to get new jobs - if (blockDelay < this.minimumBlockTimeout * 1000) { + if (blockDelay <= 0) { + return blockDelay; + } else if (blockDelay < this.minimumBlockTimeout * 1000) { return this.minimumBlockTimeout; } else { // We restrict the maximum block timeout to 10 second to avoid diff --git a/tests/test_worker.ts b/tests/test_worker.ts index 6ebeadde2..4cc39ca46 100644 --- a/tests/test_worker.ts +++ b/tests/test_worker.ts @@ -949,7 +949,7 @@ describe('workers', function () { describe('when blockUntil is greater than 0', () => { describe('when blockUntil is lower than date now value', () => { - it('returns minimumBlockTimeout', async () => { + it('returns blockDelay value lower or equal 0', async () => { const worker = new Worker(queueName, async () => {}, { connection, prefix, @@ -957,15 +957,9 @@ describe('workers', function () { }); await worker.waitUntilReady(); - if (isRedisVersionLowerThan(worker.redisVersion, '7.0.8')) { - expect(worker['getBlockTimeout'](Date.now() - 1)).to.be.equal( - 0.002, - ); - } else { - expect(worker['getBlockTimeout'](Date.now() - 1)).to.be.equal( - 0.001, - ); - } + expect( + worker['getBlockTimeout'](Date.now() - 1), + ).to.be.lessThanOrEqual(0); await worker.close(); }); }); From a8c5831d2b3e86ae9247af7eba70968cbcbfd7a7 Mon Sep 17 00:00:00 2001 From: semantic-release-bot Date: Sat, 27 Apr 2024 04:12:45 +0000 Subject: [PATCH 4/6] chore(release): 5.7.6 [skip ci] ## [5.7.6](https://github.com/taskforcesh/bullmq/compare/v5.7.5...v5.7.6) (2024-04-27) ### Bug Fixes * **redis-connection:** increase redis retry strategy backoff ([#2546](https://github.com/taskforcesh/bullmq/issues/2546)) [python] ([6cf7712](https://github.com/taskforcesh/bullmq/commit/6cf77122da845e5b0afa1607348cf06602679329)) ### Performance Improvements * **worker:** do not call bzpopmin when blockDelay is lower or equal 0 ([#2544](https://github.com/taskforcesh/bullmq/issues/2544)) ref [#2466](https://github.com/taskforcesh/bullmq/issues/2466) ([9760b85](https://github.com/taskforcesh/bullmq/commit/9760b85dfbcc9b3c744f616961ef939e8951321d)) --- docs/gitbook/changelog.md | 12 ++++++++++++ package.json | 2 +- 2 files changed, 13 insertions(+), 1 deletion(-) diff --git a/docs/gitbook/changelog.md b/docs/gitbook/changelog.md index 12a2de055..7214c4d37 100644 --- a/docs/gitbook/changelog.md +++ b/docs/gitbook/changelog.md @@ -1,3 +1,15 @@ +## [5.7.6](https://github.com/taskforcesh/bullmq/compare/v5.7.5...v5.7.6) (2024-04-27) + + +### Bug Fixes + +* **redis-connection:** increase redis retry strategy backoff ([#2546](https://github.com/taskforcesh/bullmq/issues/2546)) [python] ([6cf7712](https://github.com/taskforcesh/bullmq/commit/6cf77122da845e5b0afa1607348cf06602679329)) + + +### Performance Improvements + +* **worker:** do not call bzpopmin when blockDelay is lower or equal 0 ([#2544](https://github.com/taskforcesh/bullmq/issues/2544)) ref [#2466](https://github.com/taskforcesh/bullmq/issues/2466) ([9760b85](https://github.com/taskforcesh/bullmq/commit/9760b85dfbcc9b3c744f616961ef939e8951321d)) + ## [5.7.5](https://github.com/taskforcesh/bullmq/compare/v5.7.4...v5.7.5) (2024-04-24) diff --git a/package.json b/package.json index 3625fa59b..8836f5015 100644 --- a/package.json +++ b/package.json @@ -1,6 +1,6 @@ { "name": "bullmq", - "version": "5.7.5", + "version": "5.7.6", "description": "Queue for messages and jobs based on Redis", "homepage": "https://bullmq.io/", "main": "./dist/cjs/index.js", From d81f210a5f5968fc040e820946fb672deb24bd01 Mon Sep 17 00:00:00 2001 From: Clemens Stolle Date: Sun, 28 Apr 2024 18:58:35 +0200 Subject: [PATCH 5/6] fix(worker): wait for jobs to finalize on close (#2545) [python] --- python/bullmq/worker.py | 7 ++++++- 1 file changed, 6 insertions(+), 1 deletion(-) diff --git a/python/bullmq/worker.py b/python/bullmq/worker.py index be988e95a..803196e5d 100644 --- a/python/bullmq/worker.py +++ b/python/bullmq/worker.py @@ -243,13 +243,18 @@ async def runStalledJobsCheck(self): async def close(self, force: bool = False): """ - Close the worker + Closes the worker and related redis connections. + + This method waits for current jobs to finalize before returning. """ self.closing = True if force: self.forceClosing = True self.cancelProcessing() + if not force and len(self.processing) > 0: + await asyncio.wait(self.processing, return_when=asyncio.ALL_COMPLETED) + await self.blockingRedisConnection.close() await self.redisConnection.close() From 8020f9952a058988d57d1f772f43b2d802e20692 Mon Sep 17 00:00:00 2001 From: semantic-release Date: Sun, 28 Apr 2024 16:59:53 +0000 Subject: [PATCH 6/6] 2.7.5 Automatically generated by python-semantic-release --- docs/gitbook/python/changelog.md | 7 +++++++ python/bullmq/__init__.py | 2 +- python/pyproject.toml | 2 +- 3 files changed, 9 insertions(+), 2 deletions(-) diff --git a/docs/gitbook/python/changelog.md b/docs/gitbook/python/changelog.md index f7a97d556..7e0976607 100644 --- a/docs/gitbook/python/changelog.md +++ b/docs/gitbook/python/changelog.md @@ -2,6 +2,13 @@ +## v2.7.5 (2024-04-28) +### Fix +* **worker:** Wait for jobs to finalize on close (#2545) [python] ([`d81f210`](https://github.com/taskforcesh/bullmq/commit/d81f210a5f5968fc040e820946fb672deb24bd01)) + +### Performance +* **worker:** Do not call bzpopmin when blockDelay is lower or equal 0 (#2544) ref #2466 ([`9760b85`](https://github.com/taskforcesh/bullmq/commit/9760b85dfbcc9b3c744f616961ef939e8951321d)) + ## v2.7.4 (2024-04-26) ### Fix * **redis-connection:** Increase redis retry strategy backoff (#2546) [python] ([`6cf7712`](https://github.com/taskforcesh/bullmq/commit/6cf77122da845e5b0afa1607348cf06602679329)) diff --git a/python/bullmq/__init__.py b/python/bullmq/__init__.py index 04638f5f7..89c346002 100644 --- a/python/bullmq/__init__.py +++ b/python/bullmq/__init__.py @@ -3,7 +3,7 @@ A background job processor and message queue for Python based on Redis. """ -__version__ = "2.7.4" +__version__ = "2.7.5" __author__ = 'Taskforce.sh Inc.' __credits__ = 'Taskforce.sh Inc.' diff --git a/python/pyproject.toml b/python/pyproject.toml index 623d73bfd..f852693a4 100644 --- a/python/pyproject.toml +++ b/python/pyproject.toml @@ -4,7 +4,7 @@ build-backend = "setuptools.build_meta" [project] name = "bullmq" -version = "2.7.4" +version = "2.7.5" description='BullMQ for Python' readme="README.md" authors = [