Skip to content

Commit

Permalink
MDEV-24142 preparation: Add srw_mutex and srw_lock::u_lock()
Browse files Browse the repository at this point in the history 
The PERFORMANCE_SCHEMA insists on distinguishing read-update-write
locks from read-write locks, so we must add
template<bool support_u_lock> in rd_lock() and wr_lock() operations.

rd_lock::read_trylock(): Add template<bool prioritize_updater=false>
which is used by the srw_lock_low::read_lock() loop. As long as
an UPDATE lock has already been granted to some thread, we will grant
subsequent READ lock requests even if a waiting WRITE lock request
exists. This will be necessary to be compatible with existing usage
pattern of InnoDB rw_lock_t where the holder of SX-latch (which we
will rename to UPDATE latch) may acquire an additional S-latch
on the same object. For normal read-write locks without update operations
this should make no difference at all, because the rw_lock::UPDATER
flag would never be set.
  • Loading branch information
@dr-m
dr-m committed Dec 3, 2020
1 parent 3872e58 commit d46b424
Show file tree
Hide file tree
Showing 3 changed files with 220 additions and 20 deletions.
68 changes: 61 additions & 7 deletions storage/innobase/include/rw_lock.h
View file
Original file line number Diff line number Diff line change
Expand Up @@ -20,7 +20,7 @@ this program; if not, write to the Free Software Foundation, Inc.,
#include <atomic>
#include "my_dbug.h"

/** Simple read-write lock based on std::atomic */
/** Simple read-update-write lock based on std::atomic */
class rw_lock
{
/** The lock word */
Expand All @@ -35,6 +35,8 @@ class rw_lock
static constexpr uint32_t WRITER_WAITING= 1U << 30;
/** Flag to indicate that write_lock() or write_lock_wait() is pending */
static constexpr uint32_t WRITER_PENDING= WRITER | WRITER_WAITING;
/** Flag to indicate that an update lock exists */
static constexpr uint32_t UPDATER= 1U << 29;

/** Start waiting for an exclusive lock.
@return current value of the lock word */
Expand All @@ -45,30 +47,68 @@ class rw_lock
@return whether the exclusive lock was acquired */
bool write_lock_wait_try(uint32_t &l)
{
l= WRITER_WAITING;
return lock.compare_exchange_strong(l, WRITER, std::memory_order_acquire,
std::memory_order_relaxed);
}
/** Try to acquire a shared lock.
@tparam prioritize_updater whether to ignore WRITER_WAITING for UPDATER
@param l the value of the lock word
@return whether the lock was acquired */
template<bool prioritize_updater= false>
bool read_trylock(uint32_t &l)
{
l= UNLOCKED;
while (!lock.compare_exchange_strong(l, l + 1, std::memory_order_acquire,
std::memory_order_relaxed))
{
DBUG_ASSERT(!(WRITER & l) || !(~WRITER_PENDING & l));
if (l & WRITER_PENDING)
DBUG_ASSERT((~(WRITER_PENDING | UPDATER) & l) < UPDATER);
if (prioritize_updater
? (WRITER & l) || ((WRITER_WAITING | UPDATER) & l) == WRITER_WAITING
: (WRITER_PENDING & l))
return false;
}
return true;
}
/** Try to acquire an update lock.
@param l the value of the lock word
@return whether the lock was acquired */
bool update_trylock(uint32_t &l)
{
l= UNLOCKED;
while (!lock.compare_exchange_strong(l, l | UPDATER,
std::memory_order_acquire,
std::memory_order_relaxed))
{
DBUG_ASSERT(!(WRITER & l) || !(~WRITER_PENDING & l));
DBUG_ASSERT((~(WRITER_PENDING | UPDATER) & l) < UPDATER);
if ((WRITER_PENDING | UPDATER) & l)
return false;
}
return true;
}
/** Try to upgrade an update lock to an exclusive lock.
@return whether the update lock was upgraded to exclusive */
bool upgrade_trylock()
{
auto l= UPDATER;
while (!lock.compare_exchange_strong(l, l ^ (WRITER | UPDATER),
std::memory_order_acquire,
std::memory_order_relaxed))
{
DBUG_ASSERT(!(~l & (UPDATER - 1)));
DBUG_ASSERT(((WRITER | UPDATER) & l) == UPDATER);
if (~(WRITER_WAITING | UPDATER) & l)
return false;
}
DBUG_ASSERT((l & ~WRITER_WAITING) == UPDATER);
return true;
}
/** Wait for an exclusive lock.
@return whether the exclusive lock was acquired */
bool write_lock_poll()
{
uint32_t l;
auto l= WRITER_WAITING;
if (write_lock_wait_try(l))
return true;
if (!(l & WRITER_WAITING))
Expand All @@ -88,15 +128,26 @@ class rw_lock
bool read_unlock()
{
auto l= lock.fetch_sub(1, std::memory_order_release);
DBUG_ASSERT(~WRITER_PENDING & l); /* at least one read lock */
DBUG_ASSERT(~(WRITER_PENDING | UPDATER) & l); /* at least one read lock */
DBUG_ASSERT(!(l & WRITER)); /* no write lock must have existed */
return (~WRITER_PENDING & l) == 1;
}
/** Release an update lock.
@return whether any writers may have to be woken up */
bool update_unlock()
{
auto l= lock.fetch_and(~UPDATER, std::memory_order_release);
/* the update lock must have existed */
DBUG_ASSERT((l & (WRITER | UPDATER)) == UPDATER);
return !(~(WRITER_PENDING | UPDATER) & l);
}
/** Release an exclusive lock */
void write_unlock()
{
IF_DBUG_ASSERT(auto l=,) lock.fetch_sub(WRITER, std::memory_order_release);
DBUG_ASSERT(l & WRITER); /* the write lock must have existed */
IF_DBUG_ASSERT(auto l=,)
lock.fetch_and(~WRITER, std::memory_order_release);
/* the write lock must have existed */
DBUG_ASSERT((l & (WRITER | UPDATER)) == WRITER);
}
/** Try to acquire a shared lock.
@return whether the lock was acquired */
Expand All @@ -113,6 +164,9 @@ class rw_lock
/** @return whether an exclusive lock is being held by any thread */
bool is_write_locked() const
{ return !!(lock.load(std::memory_order_relaxed) & WRITER); }
/** @return whether an update lock is being held by any thread */
bool is_update_locked() const
{ return !!(lock.load(std::memory_order_relaxed) & UPDATER); }
/** @return whether a shared lock is being held by any thread */
bool is_read_locked() const
{
Expand Down
63 changes: 58 additions & 5 deletions storage/innobase/include/srw_lock.h
View file
Original file line number Diff line number Diff line change
Expand Up @@ -25,6 +25,22 @@ this program; if not, write to the Free Software Foundation, Inc.,
# define SRW_LOCK_DUMMY /* Use dummy implementation for debugging purposes */
#endif

#ifdef SRW_LOCK_DUMMY
/** An exclusive-only variant of srw_lock */
class srw_mutex
{
pthread_mutex_t lock;
public:
void init() { pthread_mutex_init(&lock, nullptr); }
void destroy() { pthread_mutex_destroy(&lock); }
void wr_lock() { pthread_mutex_lock(&lock); }
void wr_unlock() { pthread_mutex_unlock(&lock); }
bool wr_lock_try() { return !pthread_mutex_trylock(&lock); }
};
#else
# define srw_mutex srw_lock_low
#endif

#include "rw_lock.h"

/** Slim reader-writer lock with no recursion */
Expand All @@ -43,8 +59,12 @@ class srw_lock_low final : private rw_lock
/** Wait for a read lock.
@param l lock word from a failed read_trylock() */
void read_lock(uint32_t l);
/** Wait for a write lock after a failed write_trylock() */
void write_lock();
/** Wait for an update lock.
@param l lock word from a failed update_trylock() */
void update_lock(uint32_t l);
/** Wait for a write lock after a failed write_trylock() or upgrade_trylock()
@param holding_u whether we already hold u_lock() */
void write_lock(bool holding_u);
/** Wait for signal
@param l lock word from a failed acquisition */
inline void writer_wait(uint32_t l);
Expand All @@ -65,9 +85,15 @@ class srw_lock_low final : private rw_lock
#endif
bool rd_lock_try() { uint32_t l; return read_trylock(l); }
bool wr_lock_try() { return write_trylock(); }
template<bool support_u_lock= false>
void rd_lock() { uint32_t l; if (!read_trylock(l)) read_lock(l); }
void wr_lock() { if (!write_trylock()) write_lock(); }
void u_lock() { uint32_t l; if (!update_trylock(l)) update_lock(l); }
bool u_lock_try() { uint32_t l; return update_trylock(l); }
void u_wr_upgrade() { if (!upgrade_trylock()) write_lock(true); }
template<bool support_u_lock= false>
void wr_lock() { if (!write_trylock()) write_lock(false); }
void rd_unlock();
void u_unlock();
void wr_unlock();
};

Expand All @@ -87,8 +113,12 @@ class srw_lock
srw_lock_low lock;
PSI_rwlock *pfs_psi;

template<bool support_u_lock>
ATTRIBUTE_NOINLINE void psi_rd_lock(const char *file, unsigned line);
template<bool support_u_lock>
ATTRIBUTE_NOINLINE void psi_wr_lock(const char *file, unsigned line);
ATTRIBUTE_NOINLINE void psi_u_lock(const char *file, unsigned line);
ATTRIBUTE_NOINLINE void psi_u_wr_upgrade(const char *file, unsigned line);
public:
void init(mysql_pfs_key_t key)
{
Expand All @@ -104,10 +134,11 @@ class srw_lock
}
lock.destroy();
}
template<bool support_u_lock= false>
void rd_lock(const char *file, unsigned line)
{
if (psi_likely(pfs_psi != nullptr))
psi_rd_lock(file, line);
psi_rd_lock<support_u_lock>(file, line);
else
lock.rd_lock();
}
Expand All @@ -117,10 +148,24 @@ class srw_lock
PSI_RWLOCK_CALL(unlock_rwlock)(pfs_psi);
lock.rd_unlock();
}
void u_lock(const char *file, unsigned line)
{
if (psi_likely(pfs_psi != nullptr))
psi_u_lock(file, line);
else
lock.u_lock();
}
void u_unlock()
{
if (pfs_psi)
PSI_RWLOCK_CALL(unlock_rwlock)(pfs_psi);
lock.u_unlock();
}
template<bool support_u_lock= false>
void wr_lock(const char *file, unsigned line)
{
if (psi_likely(pfs_psi != nullptr))
psi_wr_lock(file, line);
psi_wr_lock<support_u_lock>(file, line);
else
lock.wr_lock();
}
Expand All @@ -130,7 +175,15 @@ class srw_lock
PSI_RWLOCK_CALL(unlock_rwlock)(pfs_psi);
lock.wr_unlock();
}
void u_wr_upgrade(const char *file, unsigned line)
{
if (psi_likely(pfs_psi != nullptr))
psi_u_wr_upgrade(file, line);
else
lock.u_wr_upgrade();
}
bool rd_lock_try() { return lock.rd_lock_try(); }
bool u_lock_try() { return lock.u_lock_try(); }
bool wr_lock_try() { return lock.wr_lock_try(); }
};
#endif
Loading

0 comments on commit d46b424

Please sign in to comment.