Skip to content

Commit 4dbb45f

Browse files
esmetesmet
committed
FT-309 Fix a bug in heat zone, pretty-up the replay's output
1 parent 59e976d commit 4dbb45f

File tree

3 files changed

+68
-28
lines changed

3 files changed

+68
-28
lines changed

‎ft/serialize/block_allocator.cc

Lines changed: 4 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -221,6 +221,8 @@ block_allocator::choose_block_to_alloc_after(size_t size, uint64_t heat) {
221221
return block_allocator_strategy::best_fit(_blocks_array, _n_blocks, size, _alignment);
222222
case BA_STRATEGY_HEAT_ZONE:
223223
return block_allocator_strategy::heat_zone(_blocks_array, _n_blocks, size, _alignment, heat);
224+
case BA_STRATEGY_PADDED_FIT:
225+
return block_allocator_strategy::padded_fit(_blocks_array, _n_blocks, size, _alignment);
224226
default:
225227
abort();
226228
}
@@ -260,7 +262,8 @@ void block_allocator::alloc_block(uint64_t size, uint64_t heat, uint64_t *offset
260262
// our allocation strategy chose the space after `bp' to fit the new block
261263
uint64_t answer_offset = align(bp->offset + bp->size, _alignment);
262264
uint64_t blocknum = bp - _blocks_array;
263-
assert(&_blocks_array[blocknum] == bp);
265+
invariant(&_blocks_array[blocknum] == bp);
266+
invariant(blocknum < _n_blocks);
264267
memmove(bp + 2, bp + 1, (_n_blocks - blocknum - 1) * sizeof(*bp));
265268
bp[1].offset = answer_offset;
266269
bp[1].size = size;

‎ft/serialize/block_allocator_strategy.cc

Lines changed: 13 additions & 14 deletions
Original file line numberDiff line numberDiff line change
@@ -115,16 +115,14 @@ _first_fit(struct block_allocator::blockpair *blocks_array,
115115
return nullptr;
116116
}
117117

118-
for (uint64_t n_spaces_to_check = n_blocks - 1,
119-
blocknum = forward ? 0 : n_blocks - 2;
120-
n_spaces_to_check > 0;
121-
n_spaces_to_check--, forward ? blocknum++ : blocknum--) {
122-
invariant(blocknum < n_blocks);
123-
// Consider the space after blocknum
124-
struct block_allocator::blockpair *bp = &blocks_array[blocknum];
118+
struct block_allocator::blockpair *bp = forward ? &blocks_array[0] : &blocks_array[-1];
119+
for (uint64_t n_spaces_to_check = n_blocks - 1; n_spaces_to_check > 0;
120+
n_spaces_to_check--, forward ? bp++ : bp--) {
121+
// Consider the space after bp
125122
uint64_t padded_alignment = max_padding != 0 ? _align(max_padding, alignment) : alignment;
126123
uint64_t possible_offset = _align(bp->offset + bp->size, padded_alignment);
127124
if (possible_offset + size <= bp[1].offset) {
125+
invariant((forward ? bp - blocks_array : blocks_array - bp) < (int64_t) n_blocks);
128126
return bp;
129127
}
130128
}
@@ -178,28 +176,29 @@ block_allocator_strategy::heat_zone(struct block_allocator::blockpair *blocks_ar
178176
uint64_t n_blocks, uint64_t size, uint64_t alignment,
179177
uint64_t heat) {
180178
if (heat > 0) {
179+
struct block_allocator::blockpair *bp, *boundary_bp;
181180
const double hot_zone_threshold = 0.85;
182181

183182
// Hot allocation. Find the beginning of the hot zone.
184-
struct block_allocator::blockpair *bp = &blocks_array[n_blocks - 1];
185-
uint64_t highest_offset = _align(bp->offset + bp->size, alignment);
183+
boundary_bp = &blocks_array[n_blocks - 1];
184+
uint64_t highest_offset = _align(boundary_bp->offset + boundary_bp->size, alignment);
186185
uint64_t hot_zone_offset = static_cast<uint64_t>(hot_zone_threshold * highest_offset);
187186

188-
bp = std::lower_bound(blocks_array, blocks_array + n_blocks, hot_zone_offset);
189-
uint64_t blocks_in_zone = (blocks_array + n_blocks) - bp;
190-
uint64_t blocks_outside_zone = bp - blocks_array;
187+
boundary_bp = std::lower_bound(blocks_array, blocks_array + n_blocks, hot_zone_offset);
188+
uint64_t blocks_in_zone = (blocks_array + n_blocks) - boundary_bp;
189+
uint64_t blocks_outside_zone = boundary_bp - blocks_array;
191190
invariant(blocks_in_zone + blocks_outside_zone == n_blocks);
192191

193192
if (blocks_in_zone > 0) {
194193
// Find the first fit in the hot zone, going forward.
195-
bp = _first_fit(bp, blocks_in_zone, size, alignment, true, 0);
194+
bp = _first_fit(boundary_bp, blocks_in_zone, size, alignment, true, 0);
196195
if (bp != nullptr) {
197196
return bp;
198197
}
199198
}
200199
if (blocks_outside_zone > 0) {
201200
// Find the first fit in the cold zone, going backwards.
202-
bp = _first_fit(bp, blocks_outside_zone, size, alignment, false, 0);
201+
bp = _first_fit(boundary_bp, blocks_outside_zone, size, alignment, false, 0);
203202
if (bp != nullptr) {
204203
return bp;
205204
}

‎tools/ba_replay.cc

Lines changed: 51 additions & 13 deletions
Original file line numberDiff line numberDiff line change
@@ -111,6 +111,8 @@ using std::set;
111111
using std::string;
112112
using std::vector;
113113

114+
static bool verbose = false;
115+
114116
static void ba_replay_assert(bool pred, const char *msg, const char *line, int line_num) {
115117
if (!pred) {
116118
fprintf(stderr, "%s, line (#%d): %s\n", msg, line_num, line);
@@ -259,7 +261,10 @@ static void replay_canonicalized_trace(const vector<string> &canonicalized_trace
259261

260262
char *line = toku_strdup(it->c_str());
261263

262-
printf("playing canonical trace line #%d: %s", line_num, line);
264+
if (verbose) {
265+
printf("playing canonical trace line #%d: %s", line_num, line);
266+
}
267+
263268
char *ptr = tidy_line(line);
264269

265270
// canonical allocator id is in base 10, not 16
@@ -335,30 +340,43 @@ static void print_result(uint64_t allocator_id,
335340
uint64_t total_bytes = report->data_bytes + report->unused_bytes;
336341
uint64_t total_blocks = report->data_blocks + report->unused_blocks;
337342
if (total_bytes < 32UL * 1024 * 1024) {
338-
printf("skipping allocator_id %" PRId64 " (total bytes < 32mb)\n", allocator_id);
343+
printf(" ...skipping allocator_id %" PRId64 " (total bytes < 32mb)\n", allocator_id);
344+
printf("\n");
339345
return;
340346
}
341347

342-
printf("\n");
343-
printf("allocator_id: %20" PRId64 "\n", allocator_id);
344-
printf("strategy: %20s\n", strategy_str(strategy));
348+
printf(" allocator_id: %20" PRId64 "\n", allocator_id);
349+
printf(" strategy: %20s\n", strategy_str(strategy));
345350

346351
// byte statistics
347-
printf("total bytes: %20" PRId64 "\n", total_bytes);
348-
printf("used bytes: %20" PRId64 " (%.3lf)\n", report->data_bytes,
352+
printf(" total bytes: %20" PRId64 "\n", total_bytes);
353+
printf(" used bytes: %20" PRId64 " (%.3lf)\n", report->data_bytes,
349354
static_cast<double>(report->data_bytes) / total_bytes);
350-
printf("unused bytes: %20" PRId64 " (%.3lf)\n", report->unused_bytes,
355+
printf(" unused bytes: %20" PRId64 " (%.3lf)\n", report->unused_bytes,
351356
static_cast<double>(report->unused_bytes) / total_bytes);
352357

353358
// block statistics
354-
printf("total blocks: %20" PRId64 "\n", total_blocks);
355-
printf("used blocks: %20" PRId64 " (%.3lf)\n", report->data_blocks,
359+
printf(" total blocks: %20" PRId64 "\n", total_blocks);
360+
printf(" used blocks: %20" PRId64 " (%.3lf)\n", report->data_blocks,
356361
static_cast<double>(report->data_blocks) / total_blocks);
357-
printf("unused blocks: %20" PRId64 " (%.3lf)\n", report->unused_blocks,
362+
printf(" unused blocks: %20" PRId64 " (%.3lf)\n", report->unused_blocks,
358363
static_cast<double>(report->unused_blocks) / total_blocks);
359364

360365
// misc
361-
printf("largest unused: %20" PRId64 "\n", report->largest_unused_block);
366+
printf(" largest unused: %20" PRId64 "\n", report->largest_unused_block);
367+
printf("\n");
368+
}
369+
370+
static void merge_fragmentation_reports(TOKU_DB_FRAGMENTATION dst,
371+
TOKU_DB_FRAGMENTATION src) {
372+
dst->file_size_bytes += src->file_size_bytes;
373+
dst->data_bytes += src->data_bytes;
374+
dst->data_blocks += src->data_blocks;
375+
dst->checkpoint_bytes_additional += src->checkpoint_bytes_additional;
376+
dst->checkpoint_blocks_additional += src->checkpoint_blocks_additional;
377+
dst->unused_bytes += src->unused_bytes;
378+
dst->unused_blocks += src->unused_blocks;
379+
dst->largest_unused_block += src->largest_unused_block;
362380
}
363381

364382
int main(void) {
@@ -371,6 +389,11 @@ int main(void) {
371389
candidate_strategies.push_back(block_allocator::allocation_strategy::BA_STRATEGY_PADDED_FIT);
372390
candidate_strategies.push_back(block_allocator::allocation_strategy::BA_STRATEGY_HEAT_ZONE);
373391

392+
printf("\n");
393+
printf("Individual reports, by allocator:\n");
394+
printf("\n");
395+
396+
map<block_allocator::allocation_strategy, TOKU_DB_FRAGMENTATION_S> reports_by_strategy;
374397
for (vector<enum block_allocator::allocation_strategy>::const_iterator it = candidate_strategies.begin();
375398
it != candidate_strategies.end(); it++) {
376399
const block_allocator::allocation_strategy strategy(*it);
@@ -381,16 +404,31 @@ int main(void) {
381404
map<uint64_t, block_allocator *> allocator_map;
382405
replay_canonicalized_trace(canonicalized_trace, strategy, &allocator_map);
383406

407+
TOKU_DB_FRAGMENTATION_S aggregate_report;
408+
memset(&aggregate_report, 0, sizeof(aggregate_report));
384409
for (map<uint64_t, block_allocator *>::iterator al = allocator_map.begin();
385410
al != allocator_map.end(); al++) {
386411
block_allocator *ba = al->second;
387412

388413
TOKU_DB_FRAGMENTATION_S report;
414+
memset(&report, 0, sizeof(report));
389415
ba->get_statistics(&report);
390416
ba->destroy();
391417

392-
print_result(al->first, strategy,&report);
418+
merge_fragmentation_reports(&aggregate_report, &report);
419+
print_result(al->first, strategy, &report);
393420
}
421+
reports_by_strategy[strategy] = aggregate_report;
422+
}
423+
424+
printf("\n");
425+
printf("Aggregate reports, by strategy:\n");
426+
printf("\n");
427+
428+
for (map<block_allocator::allocation_strategy, TOKU_DB_FRAGMENTATION_S>::iterator it = reports_by_strategy.begin();
429+
it != reports_by_strategy.end(); it++) {
430+
TOKU_DB_FRAGMENTATION report = &it->second;
431+
print_result(0, it->first, report);
394432
}
395433

396434
return 0;

0 commit comments

Comments
 (0)