From 5ebe25ee9e20350fbe55c945de8b8305583739a8 Mon Sep 17 00:00:00 2001 From: mhenrixon Date: Thu, 28 Jul 2022 12:36:33 +0200 Subject: [PATCH] Update changelog --- CHANGELOG.md | 31 +++++++ .../lua/lock_until_expired.lua | 92 +++++++++++++++++++ 2 files changed, 123 insertions(+) create mode 100644 lib/sidekiq_unique_jobs/lua/lock_until_expired.lua diff --git a/CHANGELOG.md b/CHANGELOG.md index c79bbc6d..d8f53514 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -1,5 +1,36 @@ # Changelog +## [Unreleased](https://github.com/mhenrixon/sidekiq-unique-jobs/tree/HEAD) + +[Full Changelog](https://github.com/mhenrixon/sidekiq-unique-jobs/compare/v7.1.26...HEAD) + +**Closed issues:** + +- Memory bloat / dangling keys / reaper not cleaning orphans [\#637](https://github.com/mhenrixon/sidekiq-unique-jobs/issues/637) + +## [v7.1.26](https://github.com/mhenrixon/sidekiq-unique-jobs/tree/v7.1.26) (2022-07-28) + +[Full Changelog](https://github.com/mhenrixon/sidekiq-unique-jobs/compare/v7.1.25...v7.1.26) + +**Implemented enhancements:** + +- Fix\(until\_expired\): Fix test and implementation [\#725](https://github.com/mhenrixon/sidekiq-unique-jobs/pull/725) ([mhenrixon](https://github.com/mhenrixon)) + +**Fixed bugs:** + +- Fix\(until\_and\_while\_executing\): Improve timeouts slightly [\#728](https://github.com/mhenrixon/sidekiq-unique-jobs/pull/728) ([mhenrixon](https://github.com/mhenrixon)) +- Fix\(unlock\): Delete primed keys on last entry [\#726](https://github.com/mhenrixon/sidekiq-unique-jobs/pull/726) ([mhenrixon](https://github.com/mhenrixon)) + +**Merged pull requests:** + +- Ensure batch delete removes expiring locks [\#724](https://github.com/mhenrixon/sidekiq-unique-jobs/pull/724) ([francesmcmullin](https://github.com/francesmcmullin)) +- Chore: Update dependencies [\#722](https://github.com/mhenrixon/sidekiq-unique-jobs/pull/722) ([mhenrixon](https://github.com/mhenrixon)) +- Move until\_expired digests to separate zset [\#721](https://github.com/mhenrixon/sidekiq-unique-jobs/pull/721) ([francesmcmullin](https://github.com/francesmcmullin)) +- Avoid skipping ranges when looping through queues [\#720](https://github.com/mhenrixon/sidekiq-unique-jobs/pull/720) ([francesmcmullin](https://github.com/francesmcmullin)) +- Bump actions/checkout from 2 to 3 [\#718](https://github.com/mhenrixon/sidekiq-unique-jobs/pull/718) ([dependabot[bot]](https://github.com/apps/dependabot)) +- Add Dependabot for GitHub Actions [\#717](https://github.com/mhenrixon/sidekiq-unique-jobs/pull/717) ([petergoldstein](https://github.com/petergoldstein)) +- Fix Sidekiq::Worker.clear\_all override not being applied [\#714](https://github.com/mhenrixon/sidekiq-unique-jobs/pull/714) ([dsander](https://github.com/dsander)) + ## [v7.1.25](https://github.com/mhenrixon/sidekiq-unique-jobs/tree/v7.1.25) (2022-06-13) [Full Changelog](https://github.com/mhenrixon/sidekiq-unique-jobs/compare/v7.1.24...v7.1.25) diff --git a/lib/sidekiq_unique_jobs/lua/lock_until_expired.lua b/lib/sidekiq_unique_jobs/lua/lock_until_expired.lua new file mode 100644 index 00000000..0cbe187c --- /dev/null +++ b/lib/sidekiq_unique_jobs/lua/lock_until_expired.lua @@ -0,0 +1,92 @@ +-------- BEGIN keys --------- +local digest = KEYS[1] +local queued = KEYS[2] +local primed = KEYS[3] +local locked = KEYS[4] +local info = KEYS[5] +local changelog = KEYS[6] +local digests = KEYS[7] +local expiring_digests = KEYS[8] +-------- END keys --------- + + +-------- BEGIN lock arguments --------- +local job_id = ARGV[1] +local pttl = tonumber(ARGV[2]) +local lock_type = ARGV[3] +local limit = tonumber(ARGV[4]) +-------- END lock arguments ----------- + + +-------- BEGIN injected arguments -------- +local current_time = tonumber(ARGV[5]) +local debug_lua = ARGV[6] == "true" +local max_history = tonumber(ARGV[7]) +local script_name = tostring(ARGV[8]) .. ".lua" +local redisversion = ARGV[9] +--------- END injected arguments --------- + + +-------- BEGIN local functions -------- +<%= include_partial "shared/_common.lua" %> +---------- END local functions ---------- + + +--------- BEGIN lock.lua --------- +log_debug("BEGIN lock digest:", digest, "job_id:", job_id) + +if redis.call("HEXISTS", locked, job_id) == 1 then + log_debug(locked, "already locked with job_id:", job_id) + log("Duplicate") + + log_debug("LREM", queued, -1, job_id) + redis.call("LREM", queued, -1, job_id) + + log_debug("LREM", primed, 1, job_id) + redis.call("LREM", primed, 1, job_id) + + return job_id +end + +local locked_count = redis.call("HLEN", locked) +local within_limit = limit > locked_count +local limit_exceeded = not within_limit + +if limit_exceeded then + log_debug("Limit exceeded:", digest, "(", locked_count, "of", limit, ")") + log("Limited") + return nil +end + +log_debug("ZADD", expiring_digests, current_time + pttl, digest) +redis.call("ZADD", expiring_digests, current_time + pttl, digest) + +log_debug("HSET", locked, job_id, current_time) +redis.call("HSET", locked, job_id, current_time) + +log_debug("LREM", queued, -1, job_id) +redis.call("LREM", queued, -1, job_id) + +log_debug("LREM", primed, 1, job_id) +redis.call("LREM", primed, 1, job_id) + +-- The Sidekiq client sets pttl +log_debug("PEXPIRE", digest, pttl) +redis.call("PEXPIRE", digest, pttl) + +log_debug("PEXPIRE", locked, pttl) +redis.call("PEXPIRE", locked, pttl) + +log_debug("PEXPIRE", info, pttl) +redis.call("PEXPIRE", info, pttl) + +log_debug("PEXPIRE", queued, 1000) +redis.call("PEXPIRE", queued, 1000) + +log_debug("PEXPIRE", primed, 1000) +redis.call("PEXPIRE", primed, 1000) + +log("Locked") +log_debug("END lock digest:", digest, "job_id:", job_id) +return job_id +---------- END lock.lua ----------