Permalink
Browse files

percore_lazy: fix up implementation

also set some alignment up w/ static variables
  • Loading branch information...
stephentu committed Aug 21, 2013
1 parent 2f4bab9 commit 9934f4b1f7dde00454c2d1c44e3193728d3a8ca2
Showing with 44 additions and 61 deletions.
  1. +19 −26 core.h
  2. +2 −2 rcu.cc
  3. +10 −8 rcu.h
  4. +10 −7 ticker.h
  5. +1 −2 txn_proto2_impl.cc
  6. +2 −16 txn_proto2_impl.h
View
45 core.h
@@ -144,61 +144,54 @@ class percore {
namespace private_ {
template <typename T>
struct buf {
bool init_;
char bytes_[sizeof(T)];
constexpr buf() : init_(false) {}
inline T *
cast()
{
return (T *) &bytes_[0];
}
inline const T *
cast() const
{
return (T *) &bytes_[0];
}
inline T * cast() { return (T *) &bytes_[0]; }
inline const T * cast() const { return (T *) &bytes_[0]; }
};
}
template <typename T>
class percore_lazy : public percore<private_::buf<T>, false> {
class percore_lazy : private percore<private_::buf<T>, false> {
typedef private_::buf<T> buf_t;
public:
percore_lazy(std::function<void(T &)> init = [](T &) {})
: init_(init) {}
percore_lazy()
{
NDB_MEMSET(&flags_[0], 0, sizeof(flags_));
}
template <class... Args>
inline T &
operator[](unsigned i)
get(unsigned i, Args &&... args)
{
buf_t &b = this->elems()[i].elem;
if (unlikely(!b.init_)) {
b.init_ = true;
T *px = new (&b.bytes_[0]) T();
init_(*px);
if (unlikely(!flags_[i])) {
flags_[i] = true;
T *px = new (&b.bytes_[0]) T(std::forward<Args>(args)...);
return *px;
}
return *b.cast();
}
template <class... Args>
inline T &
my()
my(Args &&... args)
{
return (*this)[coreid::core_id()];
return get(coreid::core_id(), std::forward<Args>(args)...);
}
inline T *
view(unsigned i)
{
buf_t &b = this->elems()[i].elem;
return b.init_ ? b.cast() : nullptr;
return flags_[i] ? b.cast() : nullptr;
}
inline const T *
view(unsigned i) const
{
const buf_t &b = this->elems()[i].elem;
return b.init_ ? b.cast() : nullptr;
return flags_[i] ? b.cast() : nullptr;
}
inline const T *
@@ -208,6 +201,6 @@ class percore_lazy : public percore<private_::buf<T>, false> {
}
private:
std::function<void(T &)> init_;
CACHE_PADOUT; // ugh, wasteful
bool flags_[NMAXCORES];
CACHE_PADOUT;
};
View
4 rcu.cc
@@ -239,7 +239,7 @@ rcu::sync::do_cleanup()
evt_avg_rcu_local_delete_queue_len.offer(n);
// try to release memory from allocator slabs back
if (impl_->try_release()) {
if (try_release()) {
#ifdef ENABLE_EVENT_COUNTERS
const uint64_t now = timer::cur_usec();
if (last_release_timestamp_us_ > 0) {
@@ -290,7 +290,7 @@ rcu::fault_region()
}
rcu::rcu()
: syncs_([this](sync &s) { s.impl_ = this; })
: syncs_()
{
// XXX: these should really be instance members of RCU
// we are assuming only one rcu object is ever created
View
18 rcu.h
@@ -57,7 +57,7 @@ class rcu {
// a sync struct
//
// this is also serving as a memory allocator for the time being
struct sync {
class sync {
friend class rcu;
template <bool> friend class scoped_rcu_base;
public:
@@ -80,16 +80,18 @@ class rcu {
// un-released deallocations
public:
sync()
sync(rcu *impl)
: depth_(0)
, last_reaped_epoch_(0)
#ifdef ENABLE_EVENT_COUNTERS
, last_reaped_timestamp_us_(0)
, last_release_timestamp_us_(0)
#endif
, impl_(nullptr)
, impl_(impl)
, pin_cpu_(-1)
{
INVARIANT(((uintptr_t)this % CACHELINE_SIZE) == 0);
queue_.alloc_freelist(NQueueGroups);
scratch_.alloc_freelist(NQueueGroups);
NDB_MEMSET(&arenas_[0], 0, sizeof(arenas_));
@@ -195,7 +197,7 @@ class rcu {
in_rcu_region(uint64_t &rcu_tick) const
{
const sync *s = syncs_.myview();
if (!s)
if (unlikely(!s))
return false;
const bool is_guarded = ticker::s_instance.is_locally_guarded(rcu_tick);
const bool has_depth = s->depth();
@@ -230,21 +232,21 @@ class rcu {
void fault_region();
rcu(); // initer
static rcu s_instance; // system wide instance
static rcu s_instance CACHE_ALIGNED; // system wide instance
static void Test();
private:
rcu(); // private ctor to enforce singleton
static inline uint64_t constexpr
to_rcu_ticks(uint64_t ticks)
{
return ticks / EpochTimeMultiplier;
}
inline sync &mysync() { return syncs_.my(); }
inline sync &mysync() { return syncs_.my(this); }
percore_lazy<sync> syncs_;
};
View
@@ -86,10 +86,6 @@ class ticker {
class guard {
public:
guard()
: impl_(nullptr), core_(0), tick_(0),
depth_(0), start_us_(0) {}
guard(ticker &impl)
: impl_(&impl), core_(coreid::core_id()), start_us_(0)
{
@@ -177,7 +173,7 @@ class ticker {
uint64_t start_us_;
};
static ticker s_instance; // system wide ticker
static ticker s_instance CACHE_ALIGNED; // system wide ticker
private:
@@ -222,18 +218,25 @@ class ticker {
struct tickinfo {
spinlock lock_; // guards current_tick_ and depth_
std::atomic<uint64_t> current_tick_; // last RCU epoch this thread has seen
// (implies completion through current_tick_ - 1)
std::atomic<uint64_t> depth_; // 0 if not in RCU section
std::atomic<uint64_t> start_us_; // 0 if not in RCU section
tickinfo() : current_tick_(1), depth_(0), start_us_(0) {}
tickinfo()
: current_tick_(1), depth_(0), start_us_(0)
{
INVARIANT(((uintptr_t)this % CACHELINE_SIZE) == 0);
}
};
percore<tickinfo> ticks_;
std::atomic<uint64_t> current_tick_; // which tick are we currenlty on?
std::atomic<uint64_t> current_tick_ CACHE_ALIGNED; // which tick are we currenlty on?
std::atomic<uint64_t> last_tick_inclusive_;
// all threads have *completed* ticks <= last_tick_inclusive_
// (< current_tick_)
CACHE_PADOUT;
};
View
@@ -698,8 +698,7 @@ aligned_padded_elem<transaction_proto2_static::hackstruct>
aligned_padded_elem<transaction_proto2_static::flags>
transaction_proto2_static::g_flags;
percore_lazy<transaction_proto2_static::threadctx>
transaction_proto2_static::g_threadctxs(
[](transaction_proto2_static::threadctx &) {});
transaction_proto2_static::g_threadctxs;
event_counter
transaction_proto2_static::g_evt_worker_thread_wait_log_buffer(
"worker_thread_wait_log_buffer");
View
@@ -552,19 +552,6 @@ class transaction_proto2_static {
typedef basic_px_queue<delete_entry, 4096> px_queue;
public:
// stores last commit TID for each core
// XXX: currently has to be public so we can initialize
// g_threadctxs as:
// percore_lazy<transaction_proto2_static::threadctx>
// transaction_proto2_static::g_threadctxs(
// [](transaction_proto2_static::threadctx &) {});
//
// if we don't explicitly provide a no-op init functor, g++-4.7
// will crash with an internal error:
// txn_proto2_impl.cc: In constructor percore_lazy<T>::percore_lazy(std::function<void(T&)>) [with T = transaction_proto2_static::threadctx]
// txn_proto2_impl.cc:642:30: internal compiler error: in tsubst_copy, at cp/pt.c:12141
struct threadctx {
uint64_t last_commit_tid_;
unsigned last_reaped_epoch_;
@@ -581,13 +568,12 @@ class transaction_proto2_static {
, last_reaped_timestamp_us_(0)
#endif
{
INVARIANT(((uintptr_t)this % CACHELINE_SIZE) == 0);
queue_.alloc_freelist(rcu::NQueueGroups);
scratch_.alloc_freelist(rcu::NQueueGroups);
}
};
protected:
static void
clean_up_to_including(threadctx &ctx, uint64_t ro_tick_geq);
@@ -988,7 +974,7 @@ class transaction_proto2 : public transaction<transaction_proto2, Traits>,
gen_commit_tid(const dbtuple_write_info_vec &write_tuples)
{
const size_t my_core_id = this->rcu_guard()->guard()->core();
threadctx &ctx = g_threadctxs[my_core_id];
threadctx &ctx = g_threadctxs.get(my_core_id);
const tid_t l_last_commit_tid = ctx.last_commit_tid_;
INVARIANT(l_last_commit_tid == dbtuple::MIN_TID ||
CoreId(l_last_commit_tid) == my_core_id);

0 comments on commit 9934f4b

Please sign in to comment.