Skip to content

Commit

Permalink
refactored background batch eviction and added DSA support
Browse files Browse the repository at this point in the history
  • Loading branch information
@guptask
guptask committed Oct 4, 2023
1 parent 2041678 commit 4dc3b02
Show file tree
Hide file tree
Showing 16 changed files with 343 additions and 25 deletions.
7 changes: 4 additions & 3 deletions cachelib/allocator/BackgroundMover-inl.h
View file
Original file line number Diff line number Diff line change
Expand Up @@ -21,8 +21,9 @@ template <typename CacheT>
BackgroundMover<CacheT>::BackgroundMover(
Cache& cache,
std::shared_ptr<BackgroundMoverStrategy> strategy,
MoverDir direction)
: cache_(cache), strategy_(strategy), direction_(direction) {
MoverDir direction,
bool dsaEnabled)
: cache_(cache), strategy_(strategy), direction_(direction), dsaEnabled_(dsaEnabled) {
if (direction_ == MoverDir::Evict) {
moverFunc = BackgroundMoverAPIWrapper<CacheT>::traverseAndEvictItems;
} else if (direction_ == MoverDir::Promote) {
Expand Down Expand Up @@ -93,7 +94,7 @@ void BackgroundMover<CacheT>::checkAndRun() {
}

// try moving BATCH items from the class in order to reach free target
auto moved = moverFunc(cache_, tid, pid, cid, batch);
auto moved = moverFunc(cache_, tid, pid, cid, batch, dsaEnabled_);
moves += moved;
moves_per_class_[assignedMemory[i]] += moved;
}
Expand Down
16 changes: 10 additions & 6 deletions cachelib/allocator/BackgroundMover.h
View file
Original file line number Diff line number Diff line change
Expand Up @@ -32,16 +32,18 @@ struct BackgroundMoverAPIWrapper {
unsigned int tid,
unsigned int pid,
unsigned int cid,
size_t batch) {
return cache.traverseAndEvictItems(tid, pid, cid, batch);
size_t batch,
bool dsaEnabled) {
return cache.traverseAndEvictItems(tid, pid, cid, batch, dsaEnabled);
}

static size_t traverseAndPromoteItems(C& cache,
unsigned int tid,
unsigned int pid,
unsigned int cid,
size_t batch) {
return cache.traverseAndPromoteItems(tid, pid, cid, batch);
size_t batch,
bool dsaEnabled) {
return cache.traverseAndPromoteItems(tid, pid, cid, batch, dsaEnabled);
}
};

Expand All @@ -61,7 +63,8 @@ class BackgroundMover : public PeriodicWorker {
// (promoted vs. evicted and how much)
BackgroundMover(Cache& cache,
std::shared_ptr<BackgroundMoverStrategy> strategy,
MoverDir direction_);
MoverDir direction,
bool dsaEnabled = false);

~BackgroundMover() override;

Expand Down Expand Up @@ -101,9 +104,10 @@ class BackgroundMover : public PeriodicWorker {
Cache& cache_;
std::shared_ptr<BackgroundMoverStrategy> strategy_;
MoverDir direction_;
bool dsaEnabled_{false};

std::function<size_t(
Cache&, unsigned int, unsigned int, unsigned int, size_t)>
Cache&, unsigned int, unsigned int, unsigned int, size_t, bool)>
moverFunc;

// implements the actual logic of running the background evictor
Expand Down
2 changes: 2 additions & 0 deletions cachelib/allocator/CMakeLists.txt
View file
Original file line number Diff line number Diff line change
Expand Up @@ -65,6 +65,8 @@ target_link_libraries(cachelib_allocator PUBLIC
cachelib_navy
cachelib_common
cachelib_shm
dml
dl
)

if ((CMAKE_SYSTEM_NAME STREQUAL Linux) AND
Expand Down
206 changes: 200 additions & 6 deletions cachelib/allocator/CacheAllocator-inl.h
View file
Original file line number Diff line number Diff line change
Expand Up @@ -17,6 +17,7 @@
#pragma once

#include <folly/Random.h>
#include <dml/dml.hpp>

namespace facebook {
namespace cachelib {
Expand Down Expand Up @@ -314,7 +315,8 @@ void CacheAllocator<CacheTrait>::initWorkers() {
if (config_.backgroundEvictorEnabled()) {
startNewBackgroundEvictor(config_.backgroundEvictorInterval,
config_.backgroundEvictorStrategy,
config_.backgroundEvictorThreads);
config_.backgroundEvictorThreads,
config_.dsaEnabled);
}

if (config_.backgroundPromoterEnabled()) {
Expand Down Expand Up @@ -1384,6 +1386,178 @@ size_t CacheAllocator<CacheTrait>::wakeUpWaitersLocked(folly::StringPiece key,
return 0;
}

static std::string dsaErrStr(dml::batch_result& result) {
std::string error;
switch (result.status) {
case dml::status_code::false_predicate:
/* Operation completed successfully, but result is unexpected */
error = "false predicate";
break;
case dml::status_code::partial_completion:
/* Operation was partially completed */
error = "partial complte";
break;
case dml::status_code::nullptr_error:
/* One of data pointers is NULL */
error = "nullptr";
break;
case dml::status_code::bad_size:
/* Invalid byte size was specified */
error = "bad size";
break;
case dml::status_code::bad_length:
/* Invalid number of elements was specified */
error = "bad length";
break;
case dml::status_code::inconsistent_size:
/* Input data sizes are different */
error = "inconsistent size";
break;
case dml::status_code::dualcast_bad_padding:
/* Bits 11:0 of the two destination addresses are not the same */
error = "dualcast bad padding";
break;
case dml::status_code::bad_alignment:
/* One of data pointers has invalid alignment */
error = "bad align";
break;
case dml::status_code::buffers_overlapping:
/* Buffers overlap with each other */
error = "buf overlap";
break;
case dml::status_code::delta_bad_size:
/* Invalid delta record size was specified */
error = "delta bad size";
break;
case dml::status_code::delta_delta_empty:
/* Delta record is empty */
error = "delta emptry";
break;
case dml::status_code::batch_overflow:
/* Batch is full */
error = "batch overflow";
break;
case dml::status_code::execution_failed:
/* Unknown execution error */
error = "unknown exec";
break;
case dml::status_code::unsupported_operation:
/* Unknown execution error */
error = "unsupported op";
break;
case dml::status_code::queue_busy:
/* Enqueue failed to one or several queues */
error = "busy";
break;
case dml::status_code::error:
/* Internal library error occurred */
error = "internal";
break;
case dml::status_code::ok:
/* should not be here */
error = "ok";
break;
default:
error = "none of the above - okay!!";
break;
}
return error + " batch copy error";
}

static std::string dsaErrStr(dml::mem_copy_result& result) {
std::string error;
switch (result.status) {
case dml::status_code::false_predicate:
/* Operation completed successfully: but result is unexpected */
error = "false pred";
break;
case dml::status_code::partial_completion:
/* Operation was partially completed */
error = "partial complte";
break;
case dml::status_code::nullptr_error:
/* One of data pointers is NULL */
error = "nullptr";
break;
case dml::status_code::bad_size:
/* Invalid byte size was specified */
error = "bad size";
break;
case dml::status_code::bad_length:
/* Invalid number of elements was specified */
error = "bad len";
break;
case dml::status_code::inconsistent_size:
/* Input data sizes are different */
error = "inconsis size";
break;
case dml::status_code::dualcast_bad_padding:
/* Bits 11:0 of the two destination addresses are not the same */
error = "dualcast bad padding";
break;
case dml::status_code::bad_alignment:
/* One of data pointers has invalid alignment */
error = "bad align";
break;
case dml::status_code::buffers_overlapping:
/* Buffers overlap with each other */
error = "buf overlap";
break;
case dml::status_code::delta_bad_size:
/* Invalid delta record size was specified */
error = "delta bad size";
break;
case dml::status_code::delta_delta_empty:
/* Delta record is empty */
error = "delta emptry";
break;
case dml::status_code::batch_overflow:
/* Batch is full */
error = "batch overflow";
break;
case dml::status_code::execution_failed:
/* Unknown execution error */
error = "unknw exec";
break;
case dml::status_code::unsupported_operation:
/* Unknown execution error */
error = "unsupported op";
break;
case dml::status_code::queue_busy:
/* Enqueue failed to one or several queues */
error = "busy";
break;
case dml::status_code::error:
/* Internal library error occurred */
error = "internal";
break;
case dml::status_code::ok:
/* should not be here */
error = "ok";
break;
default:
error = "none of the above - okay!!";
break;
}
return error + " mem copy error";
}

template <typename CacheTrait>
void CacheAllocator<CacheTrait>::moveRegularItems(std::vector<EvictionData>& evictionData,
std::vector<WriteHandle>& newItemHdls,
bool skipAddInMMContainer,
bool fromBgThread,
bool dsaEnabled,
std::vector<bool>& moved) {
auto i = 0U;
if (dsaEnabled) {
}
for (; i < evictionData.size(); i++) {
moved[i] = moveRegularItem(*evictionData[i].candidate, newItemHdls[i],
skipAddInMMContainer, fromBgThread);
}
}

template <typename CacheTrait>
bool CacheAllocator<CacheTrait>::moveRegularItem(
Item& oldItem, WriteHandle& newItemHdl, bool skipAddInMMContainer, bool fromBgThread) {
Expand Down Expand Up @@ -1752,7 +1926,8 @@ CacheAllocator<CacheTrait>::getNextCandidates(TierId tid,
ClassId cid,
unsigned int batch,
bool markMoving,
bool fromBgThread) {
bool fromBgThread,
bool dsaEnabled) {

std::vector<void*> blankAllocs;
std::vector<Item*> newAllocs;
Expand Down Expand Up @@ -1916,12 +2091,15 @@ CacheAllocator<CacheTrait>::getNextCandidates(TierId tid,
folly::sformat("Was not able to add all new items, failed item {} and handle {}",
newAllocs[added]->toString(),newHandles[added]->toString()));
}

//3. copy item data - don't need to add in mmContainer
std::vector<bool> moved(evictionData.size());
moveRegularItems(evictionData, newHandles, true, true, dsaEnabled, moved);

for (int i = 0; i < evictionData.size(); i++) {
Item *candidate = evictionData[i].candidate;
WriteHandle newHandle = std::move(newHandles[i]);
bool moved = moveRegularItem(*candidate,newHandle, true, true);
if (moved) {
if (moved[i]) {
(*stats_.numWritebacks)[tid][pid][cid].inc();
XDCHECK(candidate->getKey() == newHandle->getKey());
if (markMoving) {
Expand Down Expand Up @@ -3310,6 +3488,8 @@ PoolStats CacheAllocator<CacheTrait>::getPoolStats(PoolId poolId) const {
for (const ClassId cid : classIds) {
uint64_t allocAttempts = 0, evictionAttempts = 0, allocFailures = 0,
fragmentationSize = 0, classHits = 0, chainedItemEvictions = 0,
dsaEvictBatchHwSubmits = 0, dsaEvictBatchSwSubmits = 0,
dsaEvictIndvlHwSubmits = 0, dsaEvictIndvlSwSubmits = 0,
regularItemEvictions = 0, numWritebacks = 0;
MMContainerStat mmContainerStats;
for (TierId tid = 0; tid < getNumTiers(); tid++) {
Expand All @@ -3319,6 +3499,10 @@ PoolStats CacheAllocator<CacheTrait>::getPoolStats(PoolId poolId) const {
fragmentationSize += (*stats_.fragmentationSize)[tid][poolId][cid].get();
classHits += (*stats_.cacheHits)[tid][poolId][cid].get();
chainedItemEvictions += (*stats_.chainedItemEvictions)[tid][poolId][cid].get();
dsaEvictBatchHwSubmits += (*stats_.dsaEvictBatchHwSubmits)[tid][poolId][cid].get();
dsaEvictBatchSwSubmits += (*stats_.dsaEvictBatchSwSubmits)[tid][poolId][cid].get();
dsaEvictIndvlHwSubmits += (*stats_.dsaEvictIndvlHwSubmits)[tid][poolId][cid].get();
dsaEvictIndvlSwSubmits += (*stats_.dsaEvictIndvlSwSubmits)[tid][poolId][cid].get();
regularItemEvictions += (*stats_.regularItemEvictions)[tid][poolId][cid].get();
numWritebacks += (*stats_.numWritebacks)[tid][poolId][cid].get();
mmContainerStats += getMMContainerStat(tid, poolId, cid);
Expand All @@ -3334,6 +3518,10 @@ PoolStats CacheAllocator<CacheTrait>::getPoolStats(PoolId poolId) const {
fragmentationSize,
classHits,
chainedItemEvictions,
dsaEvictBatchHwSubmits,
dsaEvictBatchSwSubmits,
dsaEvictIndvlHwSubmits,
dsaEvictIndvlSwSubmits,
regularItemEvictions,
numWritebacks,
mmContainerStats}});
Expand Down Expand Up @@ -3388,6 +3576,10 @@ PoolStats CacheAllocator<CacheTrait>::getPoolStats(TierId tid, PoolId poolId) co
(*stats_.fragmentationSize)[tid][poolId][cid].get(),
classHits,
(*stats_.chainedItemEvictions)[tid][poolId][cid].get(),
(*stats_.dsaEvictBatchHwSubmits)[tid][poolId][cid].get(),
(*stats_.dsaEvictBatchSwSubmits)[tid][poolId][cid].get(),
(*stats_.dsaEvictIndvlHwSubmits)[tid][poolId][cid].get(),
(*stats_.dsaEvictIndvlSwSubmits)[tid][poolId][cid].get(),
(*stats_.regularItemEvictions)[tid][poolId][cid].get(),
(*stats_.numWritebacks)[tid][poolId][cid].get(),
getMMContainerStat(tid, poolId, cid)}});
Expand Down Expand Up @@ -4702,7 +4894,8 @@ template <typename CacheTrait>
bool CacheAllocator<CacheTrait>::startNewBackgroundEvictor(
std::chrono::milliseconds interval,
std::shared_ptr<BackgroundMoverStrategy> strategy,
size_t threads) {
size_t threads,
bool dsaEnabled) {
XDCHECK(threads > 0);
backgroundEvictor_.resize(threads);
bool result = true;
Expand All @@ -4725,7 +4918,8 @@ bool CacheAllocator<CacheTrait>::startNewBackgroundEvictor(
}

for (size_t i = 0; i < threads; i++) {
auto ret = startNewWorker("BackgroundEvictor" + std::to_string(i), backgroundEvictor_[i], interval, *this, strategy, MoverDir::Evict);
auto ret = startNewWorker("BackgroundEvictor" + std::to_string(i), backgroundEvictor_[i],
interval, *this, strategy, MoverDir::Evict, dsaEnabled);
result = result && ret;
if (result) {
uint32_t tid = i % getNumTiers();
Expand Down
Loading

0 comments on commit 4dc3b02

Please sign in to comment.