Skip to content

Commit

Permalink
Merge pull request #8 from basho/mv-perf-counters
Browse files Browse the repository at this point in the history 
Mv perf counters
  • Loading branch information
matthewvon committed Nov 7, 2012
2 parents 4083764 + cbc3352 commit 3c363e2
Show file tree
Hide file tree
Showing 12 changed files with 561 additions and 200 deletions.
14 changes: 12 additions & 2 deletions Makefile
View file
Original file line number Diff line number Diff line change
Expand Up @@ -52,8 +52,9 @@ TESTS = \
write_batch_test

TOOLS = \
leveldb_repair

leveldb_repair \
perf_dump \
sst_scan

PROGRAMS = db_bench $(TESTS) $(TOOLS)
BENCHMARKS = db_bench_sqlite3 db_bench_tree_db
Expand Down Expand Up @@ -158,9 +159,18 @@ log_test: db/log_test.o $(LIBOBJECTS) $(TESTHARNESS)
table_test: table/table_test.o $(LIBOBJECTS) $(TESTHARNESS)
$(CXX) table/table_test.o $(LIBOBJECTS) $(TESTHARNESS) -o $@ $(LDFLAGS)

perf_dump: tools/perf_dump.o $(LIBOBJECTS)
$(CXX) tools/perf_dump.o $(LIBOBJECTS) -o $@ $(LDFLAGS)

skiplist_test: db/skiplist_test.o $(LIBOBJECTS) $(TESTHARNESS)
$(CXX) db/skiplist_test.o $(LIBOBJECTS) $(TESTHARNESS) -o $@ $(LDFLAGS)

perf_dump: tools/perf_dump.o $(LIBOBJECTS)
$(CXX) tools/perf_dump.o $(LIBOBJECTS) -o $@ $(LDFLAGS)

sst_scan: tools/sst_scan.o $(LIBOBJECTS)
$(CXX) tools/sst_scan.o $(LIBOBJECTS) -o $@ $(LDFLAGS)

leveldb_repair: tools/leveldb_repair.o $(LIBOBJECTS)
$(CXX) tools/leveldb_repair.o $(LIBOBJECTS) -o $@ $(LDFLAGS)

Expand Down
18 changes: 18 additions & 0 deletions db/db_impl.cc
View file
Original file line number Diff line number Diff line change
Expand Up @@ -35,6 +35,7 @@
#include "util/coding.h"
#include "util/logging.h"
#include "util/mutexlock.h"
#include "leveldb/perf_count.h"

namespace leveldb {

Expand Down Expand Up @@ -1301,11 +1302,14 @@ Status DBImpl::Get(const ReadOptions& options,
LookupKey lkey(key, snapshot);
if (mem->Get(lkey, value, &s)) {
// Done
gPerfCounters->Inc(ePerfGetMem);
} else if (imm != NULL && imm->Get(lkey, value, &s)) {
// Done
gPerfCounters->Inc(ePerfGetImm);
} else {
s = current->Get(options, lkey, value, &stats);
have_stat_update = true;
gPerfCounters->Inc(ePerfGetVersion);
}
mutex_.Lock();
}
Expand All @@ -1317,6 +1321,9 @@ Status DBImpl::Get(const ReadOptions& options,
mem->Unref();
if (imm != NULL) imm->Unref();
current->Unref();

gPerfCounters->Inc(ePerfApiGet);

return s;
}

Expand Down Expand Up @@ -1409,6 +1416,8 @@ Status DBImpl::Write(const WriteOptions& options, WriteBatch* my_batch) {
writers_.front()->cv.Signal();
}

gPerfCounters->Inc(ePerfApiWrite);

return status;
}

Expand Down Expand Up @@ -1485,6 +1494,7 @@ Status DBImpl::MakeRoomForWrite(bool force) {
while (true) {
if (!bg_error_.ok()) {
// Yield previous error
gPerfCounters->Inc(ePerfWriteError);
s = bg_error_;
break;
} else if (
Expand All @@ -1499,28 +1509,33 @@ Status DBImpl::MakeRoomForWrite(bool force) {
mutex_.Unlock();
env_->SleepForMicroseconds(1000);
allow_delay = false; // Do not delay a single write more than once
gPerfCounters->Inc(ePerfWriteSleep);
mutex_.Lock();
} else if (!force &&
(mem_->ApproximateMemoryUsage() <= options_.write_buffer_size)) {
// There is room in current memtable
gPerfCounters->Inc(ePerfWriteNoWait);
break;
} else if (imm_ != NULL) {
// We have filled up the current memtable, but the previous
// one is still being compacted, so we wait.
Log(options_.info_log, "waiting 2...\n");
gPerfCounters->Inc(ePerfWriteWaitImm);
MaybeScheduleCompaction();
bg_cv_.Wait();
Log(options_.info_log, "running 2...\n");
} else if (versions_->NumLevelFiles(0) >= config::kL0_StopWritesTrigger) {
// There are too many level-0 files.
Log(options_.info_log, "waiting...\n");
gPerfCounters->Inc(ePerfWriteWaitLevel0);
bg_cv_.Wait();
Log(options_.info_log, "running...\n");
} else {
// Attempt to switch to a new memtable and trigger compaction of old
assert(versions_->PrevLogNumber() == 0);
uint64_t new_log_number = versions_->NewFileNumber();
WritableFile* lfile = NULL;
gPerfCounters->Inc(ePerfWriteNewMem);
s = env_->NewWritableFile(LogFileName(dbname_, new_log_number), &lfile);
if (!s.ok()) {
// Avoid chewing through file number space in a tight loop.
Expand Down Expand Up @@ -1670,6 +1685,9 @@ Status DB::Open(const Options& options, const std::string& dbname,
} else {
delete impl;
}

gPerfCounters->Inc(ePerfApiOpen);

return s;
}

Expand Down
2 changes: 1 addition & 1 deletion db/dbformat.cc
View file
Original file line number Diff line number Diff line change
Expand Up @@ -27,7 +27,7 @@ std::string ParsedInternalKey::DebugString() const {
(unsigned long long) sequence,
int(type));
std::string result = "'";
result += user_key.ToString();
result += HexString(user_key.ToString());
result += buf;
return result;
}
Expand Down
6 changes: 6 additions & 0 deletions db/table_cache.cc
View file
Original file line number Diff line number Diff line change
Expand Up @@ -8,6 +8,7 @@
#include "leveldb/env.h"
#include "leveldb/table.h"
#include "util/coding.h"
#include "leveldb/perf_count.h"

namespace leveldb {

Expand Down Expand Up @@ -70,8 +71,13 @@ Status TableCache::FindTable(uint64_t file_number, uint64_t file_size, int level

// *handle = cache_->Insert(key, tf, file_size, &DeleteEntry);
*handle = cache_->Insert(key, tf, 1, &DeleteEntry);
gPerfCounters->Inc(ePerfTableOpened);
}
}
else
{
gPerfCounters->Inc(ePerfTableCached);
} // else
return s;
}

Expand Down
4 changes: 4 additions & 0 deletions db/version_set.cc
View file
Original file line number Diff line number Diff line change
Expand Up @@ -17,6 +17,7 @@
#include "table/two_level_iterator.h"
#include "util/coding.h"
#include "util/logging.h"
#include "leveldb/perf_count.h"

namespace leveldb {

Expand Down Expand Up @@ -357,6 +358,9 @@ Status Version::Get(const ReadOptions& options,
}
}

if (0!=num_files)
gPerfCounters->Inc(ePerfSearchLevel0 + level);

for (uint32_t i = 0; i < num_files; ++i) {
if (last_file_read != NULL && stats->seek_file == NULL) {
// We have had more than one seek for this read. Charge the 1st file.
Expand Down
183 changes: 95 additions & 88 deletions include/leveldb/perf_count.h
View file
Original file line number Diff line number Diff line change
Expand Up @@ -97,114 +97,121 @@ class SstCounters

extern struct PerformanceCounters * gPerfCounters;

struct PerformanceCounters

enum PerformanceCountersEnum
{
enum
{
eVersion=1, //!< structure versioning
};
//
// array index values/names
// (enum explicitly numbered to allow future edits / moves / inserts)
//
ePerfROFileOpen=0, //!< PosixMmapReadableFile open
ePerfROFileClose=1, //!< closed
ePerfROFileUnmap=2, //!< unmap without close

ePerfRWFileOpen=3, //!< PosixMmapFile open
ePerfRWFileClose=4, //!< closed
ePerfRWFileUnmap=5, //!< unmap without close

ePerfApiOpen=6, //!< Count of DB::Open completions
ePerfApiGet=7, //!< Count of DBImpl::Get completions
ePerfApiWrite=8, //!< Count of DBImpl::Get completions

ePerfWriteSleep=9, //!< DBImpl::MakeRoomForWrite called sleep
ePerfWriteWaitImm=10, //!< DBImpl::MakeRoomForWrite called Wait on Imm compact
ePerfWriteWaitLevel0=11,//!< DBImpl::MakeRoomForWrite called Wait on Level0 compact
ePerfWriteNewMem=12, //!< DBImpl::MakeRoomForWrite created new memory log
ePerfWriteError=13, //!< DBImpl::MakeRoomForWrite saw bg_error_
ePerfWriteNoWait=14, //!< DBImpl::MakeRoomForWrite took no action

ePerfGetMem=15, //!< DBImpl::Get read from memory log
ePerfGetImm=16, //!< DBImpl::Get read from previous memory log
ePerfGetVersion=17, //!< DBImpl::Get read from Version object

// code ASSUMES the levels are in numerical order,
// i.e. based off of ePerfSearchLevel0
ePerfSearchLevel0=18, //!< Version::Get read searched one or more files here
ePerfSearchLevel1=19, //!< Version::Get read searched one or more files here
ePerfSearchLevel2=20, //!< Version::Get read searched one or more files here
ePerfSearchLevel3=21, //!< Version::Get read searched one or more files here
ePerfSearchLevel4=22, //!< Version::Get read searched one or more files here
ePerfSearchLevel5=23, //!< Version::Get read searched one or more files here
ePerfSearchLevel6=24, //!< Version::Get read searched one or more files here

ePerfTableCached=25, //!< TableCache::FindTable found table in cache
ePerfTableOpened=26, //!< TableCache::FindTable had to open table file
ePerfTableGet=27, //!< TableCache::Get used to retrieve a key

ePerfBGCloseUnmap=28, //!< PosixEnv::BGThreaed started Unmap/Close job
ePerfBGCompactImm=29, //!< PosixEnv::BGThreaed started compaction of Imm
ePerfBGNormal=30, //!< PosixEnv::BGThreaed started normal compaction job
ePerfBGCompactLevel0=31,//!< PosixEnv::BGThreaed started compaction of Level0

ePerfBlockFiltered=32, //!< Table::BlockReader search stopped due to filter
ePerfBlockFilterFalse=33,//!< Table::BlockReader gave a false positive for match
ePerfBlockCached=34, //!< Table::BlockReader found block in cache
ePerfBlockRead=35, //!< Table::BlockReader read block from disk
ePerfBlockFilterRead=36,//!< Table::ReadMeta filter loaded from file
ePerfBlockValidGet=37, //!< Table::InternalGet has valid iterator

ePerfDebug0=38, //!< Developer debug counters, moveable
ePerfDebug1=39, //!< Developer debug counters, moveable
ePerfDebug2=40, //!< Developer debug counters, moveable
ePerfDebug3=41, //!< Developer debug counters, moveable
ePerfDebug4=42, //!< Developer debug counters, moveable

ePerfReadBlockError=43, //!< crc or compression error in ReadBlock (format.cc)

// must follow last index name to represent size of array
// (ASSUMES previous enum is highest value)
ePerfCountEnumSize, //!< size of the array described by the enum values

volatile uint32_t m_StructSize; //!< part 1 of object revision identification
volatile uint32_t m_Version; //!< part 2 of object revision identification
ePerfVersion=1, //!< structure versioning
ePerfKey=41207 //!< random number as shared memory identifier
};

volatile uint64_t m_ROFileOpen; //!< PosixMmapReadableFile open
volatile uint64_t m_ROFileClose; //!< closed
volatile uint64_t m_ROFileUnmap; //!< unmap without close
//
// Do NOT use virtual functions. This structure will be aligned at different
// locations in multiple processes. Things can get messy with virtuals.

volatile uint64_t m_RWFileOpen; //!< PosixMmapFile open
volatile uint64_t m_RWFileClose; //!< closed
volatile uint64_t m_RWFileUnmap; //!< unmap without close
struct PerformanceCounters
{
public:
static int m_LastError;

volatile uint64_t m_ApiOpen; //!< Count of DB::Open completions
volatile uint64_t m_ApiGet; //!< Count of DBImpl::Get completions
volatile uint64_t m_ApiWrite; //!< Count of DBImpl::Get completions
protected:
uint32_t m_Version; //!< object revision identification
uint32_t m_CounterSize; //!< number of objects in m_Counter

volatile uint64_t m_WriteSleep; //!< DBImpl::MakeRoomForWrite called sleep
volatile uint64_t m_WriteWaitImm; //!< DBImpl::MakeRoomForWrite called Wait on Imm compact
volatile uint64_t m_WriteWaitLevel0;//!< DBImpl::MakeRoomForWrite called Wait on Level0 compact
volatile uint64_t m_WriteNewMem; //!< DBImpl::MakeRoomForWrite created new memory log
volatile uint64_t m_WriteError; //!< DBImpl::MakeRoomForWrite saw bg_error_
volatile uint64_t m_WriteNoWait; //!< DBImpl::MakeRoomForWrite took no action
volatile uint64_t m_Counter[ePerfCountEnumSize];

volatile uint64_t m_GetMem; //!< DBImpl::Get read from memory log
volatile uint64_t m_GetImm; //!< DBImpl::Get read from previous memory log
volatile uint64_t m_GetVersion; //!< DBImpl::Get read from Version object
static const char * m_PerfCounterNames[];
static int m_PerfSharedId;
static volatile uint64_t m_BogusCounter; //!< for out of range GetPtr calls

volatile uint64_t m_SearchLevel[7]; //!< Version::Get read searched one or more files here
public:
// only called for local object, not for shared memory
PerformanceCounters();

volatile uint64_t m_TableCached; //!< TableCache::FindTable found table in cache
volatile uint64_t m_TableOpened; //!< TableCache::FindTable had to open table file
volatile uint64_t m_TableGet; //!< TableCache::Get used to retrieve a key
//!< does executable's idea of version match shared object?
bool VersionTest()
{return(ePerfCountEnumSize<=m_CounterSize && ePerfVersion==m_Version);};

volatile uint64_t m_BGCloseUnmap; //!< PosixEnv::BGThreaed started Unmap/Close job
volatile uint64_t m_BGCompactImm; //!< PosixEnv::BGThreaed started compaction of Imm or Level0
volatile uint64_t m_BGNormal; //!< PosixEnv::BGThreaed started normal compaction job
static PerformanceCounters * Init(bool IsReadOnly);

volatile uint64_t m_BlockFiltered; //!< Table::BlockReader search stopped due to filter
volatile uint64_t m_BlockFilterFalse;//!< Table::BlockReader gave a false positive for match
volatile uint64_t m_BlockCached; //!< Table::BlockReader found block in cache
volatile uint64_t m_BlockRead; //!< Table::BlockReader read block from disk
volatile uint64_t m_BlockFilterRead;//!< Table::ReadMeta filter loaded from file
volatile uint64_t m_BlockValidGet; //!< Table::InternalGet has valid iterator
uint64_t Inc(unsigned Index);

volatile uint64_t m_Debug[5]; //!< Developer debug counters, moveable
// return value of a counter
uint64_t Value(unsigned Index) const;

//!< does executable's idea of version match shared object?
bool VersionTest()
{return(sizeof(PerformanceCounters)==m_StructSize && eVersion==m_Version);};
volatile const uint64_t * GetPtr(unsigned Index) const;

void Init();
static const char * GetNamePtr(unsigned Index);

void Dump();

#if 0
// there is no CloseSharedMemFile at this time.
// --> really need a Manager object that holds pointer, static struct,
// and fd for process
static bool
OpenSharedMemFile(
const char * FileName)
{
bool good;
int fd;

good=false;
fd = open(FileName, O_CREAT | O_RDWR, 0644);
if (-1!=fd)
{
void * base;
int ret_val;

base=MAP_FAILED;

ret_val=ftruncate(fd, sizeof(PerformanceCounters));
if (-1 != ret_val)
{
base=mmap(NULL, sizeof(PerformanceCounters),
PROT_READ | PROT_WRITE, MAP_SHARED,
fd, 0);
} // if

if (MAP_FAILED != base)
{
PerformanceCounters * perf;

perf=(PerformanceCounters *)base;
if (!perf->VersionTest())
perf->Init();

gPerfCounters=perf;
good=true;
} // if
} // if

return(good);

}; // OpenSharedMemFile
#endif
}; // struct PerformanceCounters

extern PerformanceCounters * gPerfCounters;

} // namespace leveldb

Expand Down
Loading

0 comments on commit 3c363e2

Please sign in to comment.