Skip to content

Commit

Permalink
WT-6250 Avoid passing reconcile struct to cell layer for stat logging (…
Browse files Browse the repository at this point in the history 
…#5682)

* Don't pass WT_RECONCILE to cell packing layer

* Piggyback stats on exisiting infrastructure

* Add new stats to track disk bytes used for time window
  • Loading branch information
@hbokhari
hbokhari committed May 28, 2020
1 parent dcab3dc commit 7cd05db
Show file tree
Hide file tree
Showing 11 changed files with 267 additions and 210 deletions.
4 changes: 4 additions & 0 deletions dist/stat_data.py
View file
Expand Up @@ -546,6 +546,8 @@ def __init__(self, name, desc, flags=''):
RecStat('rec_time_aggr_oldest_start_ts', 'pages written including an aggregated oldest start timestamp '),
RecStat('rec_time_aggr_oldest_start_txn', 'pages written including an aggregated oldest start transaction ID '),
RecStat('rec_time_aggr_prepared', 'pages written including an aggregated prepare'),
RecStat('rec_time_window_bytes_ts', 'approximate byte size of timestamps in pages written'),
RecStat('rec_time_window_bytes_txn', 'approximate byte size of transaction IDs in pages written'),
RecStat('rec_time_window_durable_start_ts', 'records written including a start durable timestamp'),
RecStat('rec_time_window_durable_stop_ts', 'records written including a stop durable timestamp'),
RecStat('rec_time_window_pages_durable_start_ts', 'pages written including at least one start durable timestamp'),
Expand Down Expand Up @@ -874,6 +876,8 @@ def __init__(self, name, desc, flags=''):
RecStat('rec_time_aggr_oldest_start_ts', 'pages written including an aggregated oldest start timestamp '),
RecStat('rec_time_aggr_oldest_start_txn', 'pages written including an aggregated oldest start transaction ID '),
RecStat('rec_time_aggr_prepared', 'pages written including an aggregated prepare'),
RecStat('rec_time_window_bytes_ts', 'approximate byte size of timestamps in pages written'),
RecStat('rec_time_window_bytes_txn', 'approximate byte size of transaction IDs in pages written'),
RecStat('rec_time_window_durable_start_ts', 'records written including a start durable timestamp'),
RecStat('rec_time_window_durable_stop_ts', 'records written including a stop durable timestamp'),
RecStat('rec_time_window_pages_durable_start_ts', 'pages written including at least one start durable timestamp'),
Expand Down
35 changes: 13 additions & 22 deletions src/include/cell.i
View file
Expand Up @@ -32,8 +32,7 @@ __cell_check_value_validity(WT_SESSION_IMPL *session, WT_TIME_WINDOW *tw, bool e
* Pack the validity window for a value.
*/
static inline void
__cell_pack_value_validity(
WT_SESSION_IMPL *session, WT_RECONCILE *r, uint8_t **pp, WT_TIME_WINDOW *tw)
__cell_pack_value_validity(WT_SESSION_IMPL *session, uint8_t **pp, WT_TIME_WINDOW *tw)
{
uint8_t flags, *flagsp;

Expand Down Expand Up @@ -85,15 +84,8 @@ __cell_pack_value_validity(
LF_SET(WT_CELL_TS_DURABLE_STOP);
}
}
if (LF_ISSET(
WT_CELL_TS_START | WT_CELL_TS_DURABLE_START | WT_CELL_TS_STOP | WT_CELL_TS_DURABLE_STOP))
r->rec_page_cell_with_ts = true;
if (LF_ISSET(WT_CELL_TXN_START | WT_CELL_TXN_STOP))
r->rec_page_cell_with_txn_id = true;
if (tw->prepare) {
if (tw->prepare)
LF_SET(WT_CELL_PREPARE);
r->rec_page_cell_with_prepared_txn = true;
}
*flagsp = flags;
}

Expand Down Expand Up @@ -228,8 +220,8 @@ __wt_cell_pack_addr(WT_SESSION_IMPL *session, WT_CELL *cell, u_int cell_type, ui
* Set a value item's WT_CELL contents.
*/
static inline size_t
__wt_cell_pack_value(WT_SESSION_IMPL *session, WT_RECONCILE *r, WT_CELL *cell, WT_TIME_WINDOW *tw,
uint64_t rle, size_t size)
__wt_cell_pack_value(
WT_SESSION_IMPL *session, WT_CELL *cell, WT_TIME_WINDOW *tw, uint64_t rle, size_t size)
{
uint8_t byte, *p;
bool validity;
Expand All @@ -238,7 +230,7 @@ __wt_cell_pack_value(WT_SESSION_IMPL *session, WT_RECONCILE *r, WT_CELL *cell, W
p = cell->__chunk;
*p = '\0';

__cell_pack_value_validity(session, r, &p, tw);
__cell_pack_value_validity(session, &p, tw);

/*
* Short data cells without a validity window or run-length encoding have 6 bits of data length
Expand Down Expand Up @@ -360,16 +352,16 @@ __wt_cell_pack_value_match(
* Write a copy value cell.
*/
static inline size_t
__wt_cell_pack_copy(WT_SESSION_IMPL *session, WT_RECONCILE *r, WT_CELL *cell, WT_TIME_WINDOW *tw,
uint64_t rle, uint64_t v)
__wt_cell_pack_copy(
WT_SESSION_IMPL *session, WT_CELL *cell, WT_TIME_WINDOW *tw, uint64_t rle, uint64_t v)
{
uint8_t *p;

/* Start building a cell: the descriptor byte starts zero. */
p = cell->__chunk;
*p = '\0';

__cell_pack_value_validity(session, r, &p, tw);
__cell_pack_value_validity(session, &p, tw);

if (rle < 2)
cell->__chunk[0] |= WT_CELL_VALUE_COPY; /* Type */
Expand All @@ -389,8 +381,7 @@ __wt_cell_pack_copy(WT_SESSION_IMPL *session, WT_RECONCILE *r, WT_CELL *cell, WT
* Write a deleted value cell.
*/
static inline size_t
__wt_cell_pack_del(
WT_SESSION_IMPL *session, WT_RECONCILE *r, WT_CELL *cell, WT_TIME_WINDOW *tw, uint64_t rle)
__wt_cell_pack_del(WT_SESSION_IMPL *session, WT_CELL *cell, WT_TIME_WINDOW *tw, uint64_t rle)
{
uint8_t *p;

Expand All @@ -399,7 +390,7 @@ __wt_cell_pack_del(
*p = '\0';

/* FIXME-WT-6124: we should set the time window prepare value. */
__cell_pack_value_validity(session, r, &p, tw);
__cell_pack_value_validity(session, &p, tw);

if (rle < 2)
cell->__chunk[0] |= WT_CELL_DEL; /* Type */
Expand Down Expand Up @@ -485,8 +476,8 @@ __wt_cell_pack_leaf_key(WT_CELL *cell, uint8_t prefix, size_t size)
* Pack an overflow cell.
*/
static inline size_t
__wt_cell_pack_ovfl(WT_SESSION_IMPL *session, WT_RECONCILE *r, WT_CELL *cell, uint8_t type,
WT_TIME_WINDOW *tw, uint64_t rle, size_t size)
__wt_cell_pack_ovfl(WT_SESSION_IMPL *session, WT_CELL *cell, uint8_t type, WT_TIME_WINDOW *tw,
uint64_t rle, size_t size)
{
uint8_t *p;

Expand All @@ -502,7 +493,7 @@ __wt_cell_pack_ovfl(WT_SESSION_IMPL *session, WT_RECONCILE *r, WT_CELL *cell, ui
break;
case WT_CELL_VALUE_OVFL:
case WT_CELL_VALUE_OVFL_RM:
__cell_pack_value_validity(session, r, &p, tw);
__cell_pack_value_validity(session, &p, tw);
break;
}

Expand Down
13 changes: 6 additions & 7 deletions src/include/extern.h
View file
Expand Up @@ -2062,18 +2062,17 @@ static inline int __wt_write(WT_SESSION_IMPL *session, WT_FH *fh, wt_off_t offse
static inline size_t __wt_cell_pack_addr(WT_SESSION_IMPL *session, WT_CELL *cell, u_int cell_type,
uint64_t recno, WT_TIME_AGGREGATE *ta, size_t size)
WT_GCC_FUNC_DECL_ATTRIBUTE((warn_unused_result));
static inline size_t __wt_cell_pack_copy(WT_SESSION_IMPL *session, WT_RECONCILE *r, WT_CELL *cell,
static inline size_t __wt_cell_pack_copy(WT_SESSION_IMPL *session, WT_CELL *cell,
WT_TIME_WINDOW *tw, uint64_t rle, uint64_t v) WT_GCC_FUNC_DECL_ATTRIBUTE((warn_unused_result));
static inline size_t __wt_cell_pack_del(WT_SESSION_IMPL *session, WT_RECONCILE *r, WT_CELL *cell,
WT_TIME_WINDOW *tw, uint64_t rle) WT_GCC_FUNC_DECL_ATTRIBUTE((warn_unused_result));
static inline size_t __wt_cell_pack_del(WT_SESSION_IMPL *session, WT_CELL *cell, WT_TIME_WINDOW *tw,
uint64_t rle) WT_GCC_FUNC_DECL_ATTRIBUTE((warn_unused_result));
static inline size_t __wt_cell_pack_int_key(WT_CELL *cell, size_t size)
WT_GCC_FUNC_DECL_ATTRIBUTE((warn_unused_result));
static inline size_t __wt_cell_pack_leaf_key(WT_CELL *cell, uint8_t prefix, size_t size)
WT_GCC_FUNC_DECL_ATTRIBUTE((warn_unused_result));
static inline size_t __wt_cell_pack_ovfl(WT_SESSION_IMPL *session, WT_RECONCILE *r, WT_CELL *cell,
uint8_t type, WT_TIME_WINDOW *tw, uint64_t rle, size_t size)
WT_GCC_FUNC_DECL_ATTRIBUTE((warn_unused_result));
static inline size_t __wt_cell_pack_value(WT_SESSION_IMPL *session, WT_RECONCILE *r, WT_CELL *cell,
static inline size_t __wt_cell_pack_ovfl(WT_SESSION_IMPL *session, WT_CELL *cell, uint8_t type,
WT_TIME_WINDOW *tw, uint64_t rle, size_t size) WT_GCC_FUNC_DECL_ATTRIBUTE((warn_unused_result));
static inline size_t __wt_cell_pack_value(WT_SESSION_IMPL *session, WT_CELL *cell,
WT_TIME_WINDOW *tw, uint64_t rle, size_t size) WT_GCC_FUNC_DECL_ATTRIBUTE((warn_unused_result));
static inline size_t __wt_cell_total_len(void *unpack_arg)
WT_GCC_FUNC_DECL_ATTRIBUTE((warn_unused_result));
Expand Down
4 changes: 2 additions & 2 deletions src/include/reconcile.h
View file
Expand Up @@ -256,8 +256,8 @@ struct __wt_reconcile {
WT_CURSOR_BTREE update_modify_cbt;

/*
* Variables to track reconciled pages containing cells with time window values and prepared
* transactions.
* Variables to track reconciliation calls for pages containing cells with time window values
* and prepared transactions.
*/
bool rec_page_cell_with_ts;
bool rec_page_cell_with_txn_id;
Expand Down
35 changes: 33 additions & 2 deletions src/include/reconcile.i
View file
Expand Up @@ -85,47 +85,78 @@ __rec_page_time_stats(WT_SESSION_IMPL *session, WT_RECONCILE *r)
{
/* Time window statistics */
if (r->count_durable_start_ts != 0) {
WT_STAT_CONN_INCRV(
session, rec_time_window_bytes_ts, r->count_durable_start_ts * sizeof(wt_timestamp_t));
WT_STAT_DATA_INCRV(
session, rec_time_window_bytes_ts, r->count_durable_start_ts * sizeof(wt_timestamp_t));
WT_STAT_CONN_INCRV(session, rec_time_window_durable_start_ts, r->count_durable_start_ts);
WT_STAT_DATA_INCRV(session, rec_time_window_durable_start_ts, r->count_durable_start_ts);
WT_STAT_CONN_INCR(session, rec_time_window_pages_durable_start_ts);
WT_STAT_DATA_INCR(session, rec_time_window_pages_durable_start_ts);
r->rec_page_cell_with_ts = true;
}
if (r->count_start_ts != 0) {
WT_STAT_CONN_INCRV(
session, rec_time_window_bytes_ts, r->count_start_ts * sizeof(wt_timestamp_t));
WT_STAT_DATA_INCRV(
session, rec_time_window_bytes_ts, r->count_start_ts * sizeof(wt_timestamp_t));
WT_STAT_CONN_INCRV(session, rec_time_window_start_ts, r->count_start_ts);
WT_STAT_DATA_INCRV(session, rec_time_window_start_ts, r->count_start_ts);
WT_STAT_CONN_INCR(session, rec_time_window_pages_start_ts);
WT_STAT_DATA_INCR(session, rec_time_window_pages_start_ts);
r->rec_page_cell_with_ts = true;
}
if (r->count_start_txn != 0) {
WT_STAT_CONN_INCRV(
session, rec_time_window_bytes_txn, r->count_start_txn * sizeof(uint64_t));
WT_STAT_DATA_INCRV(
session, rec_time_window_bytes_txn, r->count_start_txn * sizeof(uint64_t));
WT_STAT_CONN_INCRV(session, rec_time_window_start_txn, r->count_start_txn);
WT_STAT_DATA_INCRV(session, rec_time_window_start_txn, r->count_start_txn);
WT_STAT_CONN_INCR(session, rec_time_window_pages_start_txn);
WT_STAT_DATA_INCR(session, rec_time_window_pages_start_txn);
r->rec_page_cell_with_txn_id = true;
}
if (r->count_durable_stop_ts != 0) {
WT_STAT_CONN_INCRV(
session, rec_time_window_bytes_ts, r->count_durable_stop_ts * sizeof(wt_timestamp_t));
WT_STAT_DATA_INCRV(
session, rec_time_window_bytes_ts, r->count_durable_stop_ts * sizeof(wt_timestamp_t));
WT_STAT_CONN_INCRV(session, rec_time_window_durable_stop_ts, r->count_durable_stop_ts);
WT_STAT_DATA_INCRV(session, rec_time_window_durable_stop_ts, r->count_durable_stop_ts);
WT_STAT_CONN_INCR(session, rec_time_window_pages_durable_stop_ts);
WT_STAT_DATA_INCR(session, rec_time_window_pages_durable_stop_ts);
r->rec_page_cell_with_ts = true;
}
if (r->count_stop_ts != 0) {
WT_STAT_CONN_INCRV(
session, rec_time_window_bytes_ts, r->count_stop_ts * sizeof(wt_timestamp_t));
WT_STAT_DATA_INCRV(
session, rec_time_window_bytes_ts, r->count_stop_ts * sizeof(wt_timestamp_t));
WT_STAT_CONN_INCRV(session, rec_time_window_stop_ts, r->count_stop_ts);
WT_STAT_DATA_INCRV(session, rec_time_window_stop_ts, r->count_stop_ts);
WT_STAT_CONN_INCR(session, rec_time_window_pages_stop_ts);
WT_STAT_DATA_INCR(session, rec_time_window_pages_stop_ts);
r->rec_page_cell_with_ts = true;
}
if (r->count_stop_txn != 0) {
WT_STAT_CONN_INCRV(
session, rec_time_window_bytes_txn, r->count_stop_txn * sizeof(uint64_t));
WT_STAT_DATA_INCRV(
session, rec_time_window_bytes_txn, r->count_stop_txn * sizeof(uint64_t));
WT_STAT_CONN_INCRV(session, rec_time_window_stop_txn, r->count_stop_txn);
WT_STAT_DATA_INCRV(session, rec_time_window_stop_txn, r->count_stop_txn);
WT_STAT_CONN_INCR(session, rec_time_window_pages_stop_txn);
WT_STAT_DATA_INCR(session, rec_time_window_pages_stop_txn);
r->rec_page_cell_with_txn_id = true;
}

if (r->count_prepare != 0) {
WT_STAT_CONN_INCRV(session, rec_time_window_prepared, r->count_prepare);
WT_STAT_DATA_INCRV(session, rec_time_window_prepared, r->count_prepare);
WT_STAT_CONN_INCR(session, rec_time_window_pages_prepared);
WT_STAT_DATA_INCR(session, rec_time_window_pages_prepared);
r->rec_page_cell_with_prepared_txn = true;
}

/* Time aggregate statistics */
Expand Down Expand Up @@ -352,7 +383,7 @@ __wt_rec_cell_build_val(WT_SESSION_IMPL *session, WT_RECONCILE *r, const void *d
}
__rec_cell_tw_stats(r, tw);

val->cell_len = __wt_cell_pack_value(session, r, &val->cell, tw, rle, val->buf.size);
val->cell_len = __wt_cell_pack_value(session, &val->cell, tw, rle, val->buf.size);
val->len = val->cell_len + val->buf.size;

return (0);
Expand Down Expand Up @@ -400,7 +431,7 @@ __wt_rec_dict_replace(
* offset from the beginning of the page.
*/
offset = (uint64_t)WT_PTRDIFF(r->first_free, (uint8_t *)r->cur_ptr->image.mem + dp->offset);
val->len = val->cell_len = __wt_cell_pack_copy(session, r, &val->cell, tw, rle, offset);
val->len = val->cell_len = __wt_cell_pack_copy(session, &val->cell, tw, rle, offset);
val->buf.data = NULL;
val->buf.size = 0;
}
Expand Down
4 changes: 4 additions & 0 deletions src/include/stat.h
View file
Expand Up @@ -615,6 +615,8 @@ struct __wt_connection_stats {
int64_t perf_hist_opwrite_latency_lt1000;
int64_t perf_hist_opwrite_latency_lt10000;
int64_t perf_hist_opwrite_latency_gt10000;
int64_t rec_time_window_bytes_ts;
int64_t rec_time_window_bytes_txn;
int64_t rec_page_delete_fast;
int64_t rec_maximum_seconds;
int64_t rec_pages;
Expand Down Expand Up @@ -887,6 +889,8 @@ struct __wt_dsrc_stats {
int64_t hs_gc_pages_evict;
int64_t hs_gc_pages_removed;
int64_t hs_gc_pages_visited;
int64_t rec_time_window_bytes_ts;
int64_t rec_time_window_bytes_txn;
int64_t rec_dictionary;
int64_t rec_page_delete_fast;
int64_t rec_suffix_compression;
Expand Down

0 comments on commit 7cd05db

Please sign in to comment.