Skip to content

Commit

Permalink
FT-300 Add the ability to include/exclude certain strategies and clean
Browse files Browse the repository at this point in the history 
up report formatting a bit. Also strengthen malformed trace detection.
  • Loading branch information
@esmet
esmet committed Aug 11, 2014
1 parent f94e36a commit fd25ba7
Showing 1 changed file with 98 additions and 47 deletions.
145 changes: 98 additions & 47 deletions tools/ba_replay.cc
View file
Original file line number Diff line number Diff line change
Expand Up @@ -91,6 +91,7 @@ PATENT RIGHTS GRANT:

#include <db.h>

#include <getopt.h>
#include <math.h>
#include <stdio.h>
#include <string.h>
Expand All @@ -112,8 +113,7 @@ using std::set;
using std::string;
using std::vector;

static bool debug = false;
static bool verbose = false;
static int verbose = false;

static void ba_replay_assert(bool pred, const char *msg, const char *line, int line_num) {
if (!pred) {
Expand All @@ -136,14 +136,14 @@ static int64_t parse_number(char **ptr, int line_num, int base) {

char *new_ptr;
int64_t n = strtoll(line, &new_ptr, base);
ba_replay_assert(n >= 0, "malformed trace", line, line_num);
ba_replay_assert(n >= 0, "malformed trace (bad numeric token)", line, line_num);
ba_replay_assert(new_ptr > *ptr, "malformed trace (missing numeric token)", line, line_num);
*ptr = new_ptr;
return n;
}

static uint64_t parse_uint64(char **ptr, int line_num) {
int64_t n = parse_number(ptr, line_num, 10);
ba_replay_assert(n >= 0, "malformed trace", *ptr, line_num);
// we happen to know that the uint64's we deal with will
// take less than 63 bits (they come from pointers)
return static_cast<uint64_t>(n);
Expand All @@ -156,7 +156,7 @@ static string parse_token(char **ptr, int line_num) {
// parse the first token, which represents the traced function
char token[64];
int r = sscanf(*ptr, "%64s", token);
ba_replay_assert(r == 1, "malformed trace", line, line_num);
ba_replay_assert(r == 1, "malformed trace (missing string token)", line, line_num);
*ptr += strlen(token);
return string(token);
}
Expand All @@ -168,7 +168,7 @@ static block_allocator::blockpair parse_blockpair(char **ptr, int line_num) {
uint64_t offset, size;
int bytes_read;
int r = sscanf(line, "[%" PRIu64 " %" PRIu64 "]%n", &offset, &size, &bytes_read);
ba_replay_assert(r == 2, "malformed trace", line, line_num);
ba_replay_assert(r == 2, "malformed trace (bad offset/size pair)", line, line_num);
*ptr += bytes_read;
return block_allocator::blockpair(offset, size);
}
Expand Down Expand Up @@ -234,28 +234,31 @@ static vector<string> canonicalize_trace_from(FILE *file) {

std::stringstream ss;
if (fn.find("ba_trace_create") != string::npos) {
// either a create or a create_from_blockpairs. either way,
ba_replay_assert(allocator_ids.count(allocator_id) == 0, "corrupted trace: double create", line, line_num);
ba_replay_assert(fn == "ba_trace_create" || fn == "ba_trace_create_from_blockpairs",
"corrupted trace: bad fn", line, line_num);

// we only convert the allocator_id to an allocator_id_seq_num
// in the canonical trace and leave the rest of the line as-is.
ba_replay_assert(allocator_ids.count(allocator_id) == 0, "corrupted trace: double create", line, line_num);
allocator_ids[allocator_id] = allocator_id_seq_num;
ss << fn << ' ' << allocator_id_seq_num << ' ' << trim_whitespace(ptr) << std::endl;
allocator_id_seq_num++;

// For each blockpair created by this traceline, add its offset to the offset seq map
// with asn ASN_NONE so that later canonicalizations of `free' know whether to write
// down the asn or the raw offset.
//
// First, read passed the reserve / alignment values.
(void) parse_uint64(&ptr, line_num);
(void) parse_uint64(&ptr, line_num);
offset_seq_map *map = &offset_to_seq_num_maps[allocator_id];
while (*trim_whitespace(ptr) != '\0') {
const block_allocator::blockpair bp = parse_blockpair(&ptr, line_num);
(*map)[bp.offset] = ASN_NONE;
if (fn == "ba_trace_create_from_blockpairs") {
// For each blockpair created by this traceline, add its offset to the offset seq map
// with asn ASN_NONE so that later canonicalizations of `free' know whether to write
// down the asn or the raw offset.
offset_seq_map *map = &offset_to_seq_num_maps[allocator_id];
while (*trim_whitespace(ptr) != '\0') {
const block_allocator::blockpair bp = parse_blockpair(&ptr, line_num);
(*map)[bp.offset] = ASN_NONE;
}
}
} else if (allocator_ids.count(allocator_id) > 0) {
// this allocator is part of the canonical trace
} else {
ba_replay_assert(allocator_ids.count(allocator_id) > 0, "corrupted trace: unknown allocator", line, line_num);
uint64_t canonical_allocator_id = allocator_ids[allocator_id];

// this is the map that tracks allocations for this allocator
Expand Down Expand Up @@ -296,18 +299,17 @@ static vector<string> canonicalize_trace_from(FILE *file) {

// translate `destroy(ptr_id) to destroy(canonical_id)'
ss << fn << ' ' << canonical_allocator_id << ' ' << std::endl;
} else {
ba_replay_assert(false, "corrupted trace: bad fn", line, line_num);
}
} else {
// traced an alloc/free for an allocator not created as part of this trace, skip
continue;
}
canonicalized_trace.push_back(ss.str());

toku_free(line);
}

if (allocator_ids.size() != 0) {
fprintf(stderr, "warning: leaked allocators. this is ok if the trace is still live");
fprintf(stderr, "warning: leaked allocators. this might be ok if the tracing process is still running");
}

return canonicalized_trace;
Expand Down Expand Up @@ -390,10 +392,6 @@ static void replay_canonicalized_trace(const vector<string> &canonicalized_trace
char *line = toku_strdup(it->c_str());
line = strip_newline(line, nullptr);

if (debug) {
printf("playing canonical trace line #%d: %s", line_num, line);
}

char *ptr = trim_whitespace(line);

// canonical allocator id is in base 10, not 16
Expand Down Expand Up @@ -476,8 +474,7 @@ static void replay_canonicalized_trace(const vector<string> &canonicalized_trace
}
}

// TODO: Put this in the allocation strategy class
static const char *strategy_str(block_allocator::allocation_strategy strategy) {
static const char *strategy_to_cstring(block_allocator::allocation_strategy strategy) {
switch (strategy) {
case block_allocator::allocation_strategy::BA_STRATEGY_FIRST_FIT:
return "first-fit";
Expand All @@ -492,6 +489,23 @@ static const char *strategy_str(block_allocator::allocation_strategy strategy) {
}
}

static block_allocator::allocation_strategy cstring_to_strategy(const char *str) {
if (strcmp(str, "first-fit") == 0) {
return block_allocator::allocation_strategy::BA_STRATEGY_FIRST_FIT;
}
if (strcmp(str, "best-fit") == 0) {
return block_allocator::allocation_strategy::BA_STRATEGY_BEST_FIT;
}
if (strcmp(str, "heat-zone") == 0) {
return block_allocator::allocation_strategy::BA_STRATEGY_HEAT_ZONE;
}
if (strcmp(str, "padded-fit") != 0) {
fprintf(stderr, "bad strategy string: %s\n", str);
abort();
}
return block_allocator::allocation_strategy::BA_STRATEGY_PADDED_FIT;
}

static void print_result_verbose(uint64_t allocator_id,
block_allocator::allocation_strategy strategy,
const struct fragmentation_report &report) {
Expand All @@ -503,7 +517,7 @@ static void print_result_verbose(uint64_t allocator_id,
}

printf(" allocator_id: %20" PRId64 "\n", allocator_id);
printf(" strategy: %20s\n", strategy_str(strategy));
printf(" strategy: %20s\n", strategy_to_cstring(strategy));

for (int i = 0; i < 2; i++) {
const TOKU_DB_FRAGMENTATION_S *r = i == 0 ? &report.beginning : &report.end;
Expand All @@ -528,7 +542,6 @@ static void print_result_verbose(uint64_t allocator_id,

// misc
printf(" largest unused: %20" PRId64 "\n", r->largest_unused_block);
printf("\n");
}
}

Expand All @@ -542,38 +555,76 @@ static void print_result(uint64_t allocator_id,
uint64_t total_end_bytes = end->data_bytes + end->unused_bytes;
if (total_end_bytes + total_beginning_bytes < 32UL * 1024 * 1024) {
if (verbose) {
printf(" ...skipping allocator_id %" PRId64 " (total bytes < 32mb)\n", allocator_id);
printf("\n");
printf(" ...skipping allocator_id %" PRId64 " (total bytes < 32mb)\n", allocator_id);
}
return;
}
printf("\n");
if (verbose) {
print_result_verbose(allocator_id, strategy, report);
} else {
printf(" %-15s: allocator %" PRId64 ", %.3lf used bytes (%.3lf before)\n",
strategy_str(strategy), allocator_id,
strategy_to_cstring(strategy), allocator_id,
static_cast<double>(report.end.data_bytes) / total_end_bytes,
static_cast<double>(report.beginning.data_bytes) / total_beginning_bytes);
}
}

int main(void) {
// Read the raw trace from stdin
vector<string> canonicalized_trace = canonicalize_trace_from(stdin);
static int only_aggregate_reports;

vector<enum block_allocator::allocation_strategy> candidate_strategies;
candidate_strategies.push_back(block_allocator::allocation_strategy::BA_STRATEGY_FIRST_FIT);
candidate_strategies.push_back(block_allocator::allocation_strategy::BA_STRATEGY_BEST_FIT);
candidate_strategies.push_back(block_allocator::allocation_strategy::BA_STRATEGY_PADDED_FIT);
candidate_strategies.push_back(block_allocator::allocation_strategy::BA_STRATEGY_HEAT_ZONE);
static struct option getopt_options[] = {
{ "verbose", no_argument, &verbose, 1 },
{ "only-aggregate-reports", no_argument, &only_aggregate_reports, 1 },
{ "include-strategy", required_argument, nullptr, 'i' },
{ "exclude-strategy", required_argument, nullptr, 'x' },
{ nullptr, 0, nullptr, 0 },
};

printf("\n");
printf("Individual reports, by allocator:\n");
printf("\n");
int main(int argc, char *argv[]) {
int opt;
set<block_allocator::allocation_strategy> candidate_strategies, excluded_strategies;
while ((opt = getopt_long(argc, argv, "", getopt_options, nullptr)) != -1) {
switch (opt) {
case 0:
break;
case 'i':
candidate_strategies.insert(cstring_to_strategy(optarg));
break;
case 'x':
excluded_strategies.insert(cstring_to_strategy(optarg));
break;
case '?':
default:
abort();
};
}
// Default to everything if nothing was explicitly included.
if (candidate_strategies.empty()) {
candidate_strategies.insert(block_allocator::allocation_strategy::BA_STRATEGY_FIRST_FIT);
candidate_strategies.insert(block_allocator::allocation_strategy::BA_STRATEGY_BEST_FIT);
candidate_strategies.insert(block_allocator::allocation_strategy::BA_STRATEGY_PADDED_FIT);
candidate_strategies.insert(block_allocator::allocation_strategy::BA_STRATEGY_HEAT_ZONE);
}
// ..but remove anything that was explicitly excluded
for (set<block_allocator::allocation_strategy>::const_iterator it = excluded_strategies.begin();
it != excluded_strategies.end(); it++) {
candidate_strategies.erase(*it);
}

// Run the real trace
//
// First, read the raw trace from stdin
vector<string> canonicalized_trace = canonicalize_trace_from(stdin);

if (!only_aggregate_reports) {
printf("\n");
printf("Individual reports, by allocator:\n");
}

struct canonical_trace_stats stats;
map<block_allocator::allocation_strategy, struct fragmentation_report> reports_by_strategy;
for (vector<enum block_allocator::allocation_strategy>::const_iterator it = candidate_strategies.begin();
for (set<block_allocator::allocation_strategy>::const_iterator it = candidate_strategies.begin();
it != candidate_strategies.end(); it++) {
const block_allocator::allocation_strategy strategy(*it);

Expand All @@ -592,15 +643,15 @@ int main(void) {
rp != reports.end(); rp++) {
const struct fragmentation_report &report = rp->second;
aggregate_report.merge(report);
print_result(rp->first, strategy, report);
if (!only_aggregate_reports) {
print_result(rp->first, strategy, report);
}
}
reports_by_strategy[strategy] = aggregate_report;
printf("\n");
}

printf("\n");
printf("Aggregate reports, by strategy:\n");
printf("\n");

for (map<block_allocator::allocation_strategy, struct fragmentation_report>::iterator it = reports_by_strategy.begin();
it != reports_by_strategy.end(); it++) {
Expand Down

0 comments on commit fd25ba7

Please sign in to comment.