Skip to content

Commit

Permalink
WT-2771 Add a statistic to track per-btree dirty cache usage. (#3207)
Browse files Browse the repository at this point in the history
  • Loading branch information
@michaelcahill @agorrod
michaelcahill authored and agorrod committed Dec 19, 2016
1 parent 84e44d4 commit 9a3d212
Show file tree
Hide file tree
Showing 8 changed files with 105 additions and 68 deletions.
1 change: 1 addition & 0 deletions dist/stat_data.py
View file
Expand Up @@ -477,6 +477,7 @@ def __init__(self, name, desc, flags=''):
##########################################
# Cache and eviction statistics
##########################################
CacheStat('cache_bytes_dirty', 'tracked dirty bytes in the cache', 'no_clear,no_scale,size'),
CacheStat('cache_bytes_inuse', 'bytes currently in the cache', 'no_clear,no_scale,size'),
CacheStat('cache_bytes_read', 'bytes read into cache', 'size'),
CacheStat('cache_bytes_write', 'bytes written from cache', 'size'),
Expand Down
2 changes: 2 additions & 0 deletions src/btree/bt_stat.c
View file
Expand Up @@ -40,6 +40,8 @@ __wt_btree_stat_init(WT_SESSION_IMPL *session, WT_CURSOR_STAT *cst)
WT_STAT_SET(session, stats, btree_maxleafpage, btree->maxleafpage);
WT_STAT_SET(session, stats, btree_maxleafvalue, btree->maxleafvalue);

WT_STAT_SET(session, stats, cache_bytes_dirty,
__wt_btree_dirty_inuse(session));
WT_STAT_SET(session, stats, cache_bytes_inuse,
__wt_btree_bytes_inuse(session));

Expand Down
1 change: 1 addition & 0 deletions src/include/btree.h
View file
Expand Up @@ -131,6 +131,7 @@ struct __wt_btree {
uint64_t write_gen; /* Write generation */

uint64_t bytes_inmem; /* Cache bytes in memory. */
uint64_t bytes_dirty_intl; /* Bytes in dirty internal pages. */
uint64_t bytes_dirty_leaf; /* Bytes in dirty leaf pages. */

WT_REF *evict_ref; /* Eviction thread's location */
Expand Down
44 changes: 34 additions & 10 deletions src/include/btree.i
View file
Expand Up @@ -70,6 +70,23 @@ __wt_btree_bytes_inuse(WT_SESSION_IMPL *session)
return (__wt_cache_bytes_plus_overhead(cache, btree->bytes_inmem));
}

/*
* __wt_btree_dirty_inuse --
* Return the number of dirty bytes in use.
*/
static inline uint64_t
__wt_btree_dirty_inuse(WT_SESSION_IMPL *session)
{
WT_BTREE *btree;
WT_CACHE *cache;

btree = S2BT(session);
cache = S2C(session)->cache;

return (__wt_cache_bytes_plus_overhead(cache,
btree->bytes_dirty_intl + btree->bytes_dirty_leaf));
}

/*
* __wt_btree_dirty_leaf_inuse --
* Return the number of bytes in use by dirty leaf pages.
Expand Down Expand Up @@ -105,11 +122,12 @@ __wt_cache_page_inmem_incr(WT_SESSION_IMPL *session, WT_PAGE *page, size_t size)
(void)__wt_atomic_addsize(&page->memory_footprint, size);
if (__wt_page_is_modified(page)) {
(void)__wt_atomic_addsize(&page->modify->bytes_dirty, size);
if (WT_PAGE_IS_INTERNAL(page))
if (WT_PAGE_IS_INTERNAL(page)) {
(void)__wt_atomic_add64(&btree->bytes_dirty_intl, size);
(void)__wt_atomic_add64(&cache->bytes_dirty_intl, size);
else if (!F_ISSET(btree, WT_BTREE_LSM_PRIMARY)) {
(void)__wt_atomic_add64(&cache->bytes_dirty_leaf, size);
} else if (!F_ISSET(btree, WT_BTREE_LSM_PRIMARY)) {
(void)__wt_atomic_add64(&btree->bytes_dirty_leaf, size);
(void)__wt_atomic_add64(&cache->bytes_dirty_leaf, size);
}
}
/* Track internal size in cache. */
Expand Down Expand Up @@ -238,10 +256,12 @@ __wt_cache_page_byte_dirty_decr(
if (i == 5)
return;

if (WT_PAGE_IS_INTERNAL(page))
if (WT_PAGE_IS_INTERNAL(page)) {
__wt_cache_decr_check_uint64(session, &btree->bytes_dirty_intl,
decr, "WT_BTREE.bytes_dirty_intl");
__wt_cache_decr_check_uint64(session, &cache->bytes_dirty_intl,
decr, "WT_CACHE.bytes_dirty_intl");
else if (!F_ISSET(btree, WT_BTREE_LSM_PRIMARY)) {
} else if (!F_ISSET(btree, WT_BTREE_LSM_PRIMARY)) {
__wt_cache_decr_check_uint64(session, &btree->bytes_dirty_leaf,
decr, "WT_BTREE.bytes_dirty_leaf");
__wt_cache_decr_check_uint64(session, &cache->bytes_dirty_leaf,
Expand Down Expand Up @@ -297,6 +317,7 @@ __wt_cache_dirty_incr(WT_SESSION_IMPL *session, WT_PAGE *page)
*/
size = page->memory_footprint;
if (WT_PAGE_IS_INTERNAL(page)) {
(void)__wt_atomic_add64(&btree->bytes_dirty_intl, size);
(void)__wt_atomic_add64(&cache->bytes_dirty_intl, size);
(void)__wt_atomic_add64(&cache->pages_dirty_intl, 1);
} else {
Expand Down Expand Up @@ -392,17 +413,20 @@ __wt_cache_page_evict(WT_SESSION_IMPL *session, WT_PAGE *page)

/* Update the cache's dirty-byte count. */
if (modify != NULL && modify->bytes_dirty != 0) {
if (WT_PAGE_IS_INTERNAL(page))
if (WT_PAGE_IS_INTERNAL(page)) {
__wt_cache_decr_zero_uint64(session,
&btree->bytes_dirty_intl,
modify->bytes_dirty, "WT_BTREE.bytes_dirty_intl");
__wt_cache_decr_zero_uint64(session,
&cache->bytes_dirty_intl,
modify->bytes_dirty, "WT_CACHE.bytes_dirty_intl");
else if (!F_ISSET(btree, WT_BTREE_LSM_PRIMARY)) {
__wt_cache_decr_zero_uint64(session,
&cache->bytes_dirty_leaf,
modify->bytes_dirty, "WT_CACHE.bytes_dirty_leaf");
} else if (!F_ISSET(btree, WT_BTREE_LSM_PRIMARY)) {
__wt_cache_decr_zero_uint64(session,
&btree->bytes_dirty_leaf,
modify->bytes_dirty, "WT_BTREE.bytes_dirty_leaf");
__wt_cache_decr_zero_uint64(session,
&cache->bytes_dirty_leaf,
modify->bytes_dirty, "WT_CACHE.bytes_dirty_leaf");
}
}

Expand Down
1 change: 1 addition & 0 deletions src/include/stat.h
View file
Expand Up @@ -564,6 +564,7 @@ struct __wt_dsrc_stats {
int64_t cache_pages_requested;
int64_t cache_write;
int64_t cache_write_restore;
int64_t cache_bytes_dirty;
int64_t cache_eviction_clean;
int64_t cache_state_gen_avg_gap;
int64_t cache_state_avg_written_size;
Expand Down
118 changes: 60 additions & 58 deletions src/include/wiredtiger.in
View file
Expand Up @@ -4978,181 +4978,183 @@ extern int wiredtiger_extension_terminate(WT_CONNECTION *connection);
#define WT_STAT_DSRC_CACHE_WRITE 2059
/*! cache: pages written requiring in-memory restoration */
#define WT_STAT_DSRC_CACHE_WRITE_RESTORE 2060
/*! cache: tracked dirty bytes in the cache */
#define WT_STAT_DSRC_CACHE_BYTES_DIRTY 2061
/*! cache: unmodified pages evicted */
#define WT_STAT_DSRC_CACHE_EVICTION_CLEAN 2061
#define WT_STAT_DSRC_CACHE_EVICTION_CLEAN 2062
/*!
* cache_walk: Average difference between current eviction generation
* when the page was last considered, only reported if cache_walk or all
* statistics are enabled
*/
#define WT_STAT_DSRC_CACHE_STATE_GEN_AVG_GAP 2062
#define WT_STAT_DSRC_CACHE_STATE_GEN_AVG_GAP 2063
/*!
* cache_walk: Average on-disk page image size seen, only reported if
* cache_walk or all statistics are enabled
*/
#define WT_STAT_DSRC_CACHE_STATE_AVG_WRITTEN_SIZE 2063
#define WT_STAT_DSRC_CACHE_STATE_AVG_WRITTEN_SIZE 2064
/*!
* cache_walk: Clean pages currently in cache, only reported if
* cache_walk or all statistics are enabled
*/
#define WT_STAT_DSRC_CACHE_STATE_PAGES_CLEAN 2064
#define WT_STAT_DSRC_CACHE_STATE_PAGES_CLEAN 2065
/*!
* cache_walk: Current eviction generation, only reported if cache_walk
* or all statistics are enabled
*/
#define WT_STAT_DSRC_CACHE_STATE_GEN_CURRENT 2065
#define WT_STAT_DSRC_CACHE_STATE_GEN_CURRENT 2066
/*!
* cache_walk: Dirty pages currently in cache, only reported if
* cache_walk or all statistics are enabled
*/
#define WT_STAT_DSRC_CACHE_STATE_PAGES_DIRTY 2066
#define WT_STAT_DSRC_CACHE_STATE_PAGES_DIRTY 2067
/*!
* cache_walk: Entries in the root page, only reported if cache_walk or
* all statistics are enabled
*/
#define WT_STAT_DSRC_CACHE_STATE_ROOT_ENTRIES 2067
#define WT_STAT_DSRC_CACHE_STATE_ROOT_ENTRIES 2068
/*!
* cache_walk: Internal pages currently in cache, only reported if
* cache_walk or all statistics are enabled
*/
#define WT_STAT_DSRC_CACHE_STATE_PAGES_INTERNAL 2068
#define WT_STAT_DSRC_CACHE_STATE_PAGES_INTERNAL 2069
/*!
* cache_walk: Leaf pages currently in cache, only reported if cache_walk
* or all statistics are enabled
*/
#define WT_STAT_DSRC_CACHE_STATE_PAGES_LEAF 2069
#define WT_STAT_DSRC_CACHE_STATE_PAGES_LEAF 2070
/*!
* cache_walk: Maximum difference between current eviction generation
* when the page was last considered, only reported if cache_walk or all
* statistics are enabled
*/
#define WT_STAT_DSRC_CACHE_STATE_GEN_MAX_GAP 2070
#define WT_STAT_DSRC_CACHE_STATE_GEN_MAX_GAP 2071
/*!
* cache_walk: Maximum page size seen, only reported if cache_walk or all
* statistics are enabled
*/
#define WT_STAT_DSRC_CACHE_STATE_MAX_PAGESIZE 2071
#define WT_STAT_DSRC_CACHE_STATE_MAX_PAGESIZE 2072
/*!
* cache_walk: Minimum on-disk page image size seen, only reported if
* cache_walk or all statistics are enabled
*/
#define WT_STAT_DSRC_CACHE_STATE_MIN_WRITTEN_SIZE 2072
#define WT_STAT_DSRC_CACHE_STATE_MIN_WRITTEN_SIZE 2073
/*!
* cache_walk: On-disk page image sizes smaller than a single allocation
* unit, only reported if cache_walk or all statistics are enabled
*/
#define WT_STAT_DSRC_CACHE_STATE_SMALLER_ALLOC_SIZE 2073
#define WT_STAT_DSRC_CACHE_STATE_SMALLER_ALLOC_SIZE 2074
/*!
* cache_walk: Pages created in memory and never written, only reported
* if cache_walk or all statistics are enabled
*/
#define WT_STAT_DSRC_CACHE_STATE_MEMORY 2074
#define WT_STAT_DSRC_CACHE_STATE_MEMORY 2075
/*!
* cache_walk: Pages currently queued for eviction, only reported if
* cache_walk or all statistics are enabled
*/
#define WT_STAT_DSRC_CACHE_STATE_QUEUED 2075
#define WT_STAT_DSRC_CACHE_STATE_QUEUED 2076
/*!
* cache_walk: Pages that could not be queued for eviction, only reported
* if cache_walk or all statistics are enabled
*/
#define WT_STAT_DSRC_CACHE_STATE_NOT_QUEUEABLE 2076
#define WT_STAT_DSRC_CACHE_STATE_NOT_QUEUEABLE 2077
/*!
* cache_walk: Refs skipped during cache traversal, only reported if
* cache_walk or all statistics are enabled
*/
#define WT_STAT_DSRC_CACHE_STATE_REFS_SKIPPED 2077
#define WT_STAT_DSRC_CACHE_STATE_REFS_SKIPPED 2078
/*!
* cache_walk: Size of the root page, only reported if cache_walk or all
* statistics are enabled
*/
#define WT_STAT_DSRC_CACHE_STATE_ROOT_SIZE 2078
#define WT_STAT_DSRC_CACHE_STATE_ROOT_SIZE 2079
/*!
* cache_walk: Total number of pages currently in cache, only reported if
* cache_walk or all statistics are enabled
*/
#define WT_STAT_DSRC_CACHE_STATE_PAGES 2079
#define WT_STAT_DSRC_CACHE_STATE_PAGES 2080
/*! compression: compressed pages read */
#define WT_STAT_DSRC_COMPRESS_READ 2080
#define WT_STAT_DSRC_COMPRESS_READ 2081
/*! compression: compressed pages written */
#define WT_STAT_DSRC_COMPRESS_WRITE 2081
#define WT_STAT_DSRC_COMPRESS_WRITE 2082
/*! compression: page written failed to compress */
#define WT_STAT_DSRC_COMPRESS_WRITE_FAIL 2082
#define WT_STAT_DSRC_COMPRESS_WRITE_FAIL 2083
/*! compression: page written was too small to compress */
#define WT_STAT_DSRC_COMPRESS_WRITE_TOO_SMALL 2083
#define WT_STAT_DSRC_COMPRESS_WRITE_TOO_SMALL 2084
/*! compression: raw compression call failed, additional data available */
#define WT_STAT_DSRC_COMPRESS_RAW_FAIL_TEMPORARY 2084
#define WT_STAT_DSRC_COMPRESS_RAW_FAIL_TEMPORARY 2085
/*! compression: raw compression call failed, no additional data available */
#define WT_STAT_DSRC_COMPRESS_RAW_FAIL 2085
#define WT_STAT_DSRC_COMPRESS_RAW_FAIL 2086
/*! compression: raw compression call succeeded */
#define WT_STAT_DSRC_COMPRESS_RAW_OK 2086
#define WT_STAT_DSRC_COMPRESS_RAW_OK 2087
/*! cursor: bulk-loaded cursor-insert calls */
#define WT_STAT_DSRC_CURSOR_INSERT_BULK 2087
#define WT_STAT_DSRC_CURSOR_INSERT_BULK 2088
/*! cursor: create calls */
#define WT_STAT_DSRC_CURSOR_CREATE 2088
#define WT_STAT_DSRC_CURSOR_CREATE 2089
/*! cursor: cursor-insert key and value bytes inserted */
#define WT_STAT_DSRC_CURSOR_INSERT_BYTES 2089
#define WT_STAT_DSRC_CURSOR_INSERT_BYTES 2090
/*! cursor: cursor-remove key bytes removed */
#define WT_STAT_DSRC_CURSOR_REMOVE_BYTES 2090
#define WT_STAT_DSRC_CURSOR_REMOVE_BYTES 2091
/*! cursor: cursor-update value bytes updated */
#define WT_STAT_DSRC_CURSOR_UPDATE_BYTES 2091
#define WT_STAT_DSRC_CURSOR_UPDATE_BYTES 2092
/*! cursor: insert calls */
#define WT_STAT_DSRC_CURSOR_INSERT 2092
#define WT_STAT_DSRC_CURSOR_INSERT 2093
/*! cursor: next calls */
#define WT_STAT_DSRC_CURSOR_NEXT 2093
#define WT_STAT_DSRC_CURSOR_NEXT 2094
/*! cursor: prev calls */
#define WT_STAT_DSRC_CURSOR_PREV 2094
#define WT_STAT_DSRC_CURSOR_PREV 2095
/*! cursor: remove calls */
#define WT_STAT_DSRC_CURSOR_REMOVE 2095
#define WT_STAT_DSRC_CURSOR_REMOVE 2096
/*! cursor: reset calls */
#define WT_STAT_DSRC_CURSOR_RESET 2096
#define WT_STAT_DSRC_CURSOR_RESET 2097
/*! cursor: restarted searches */
#define WT_STAT_DSRC_CURSOR_RESTART 2097
#define WT_STAT_DSRC_CURSOR_RESTART 2098
/*! cursor: search calls */
#define WT_STAT_DSRC_CURSOR_SEARCH 2098
#define WT_STAT_DSRC_CURSOR_SEARCH 2099
/*! cursor: search near calls */
#define WT_STAT_DSRC_CURSOR_SEARCH_NEAR 2099
#define WT_STAT_DSRC_CURSOR_SEARCH_NEAR 2100
/*! cursor: truncate calls */
#define WT_STAT_DSRC_CURSOR_TRUNCATE 2100
#define WT_STAT_DSRC_CURSOR_TRUNCATE 2101
/*! cursor: update calls */
#define WT_STAT_DSRC_CURSOR_UPDATE 2101
#define WT_STAT_DSRC_CURSOR_UPDATE 2102
/*! reconciliation: dictionary matches */
#define WT_STAT_DSRC_REC_DICTIONARY 2102
#define WT_STAT_DSRC_REC_DICTIONARY 2103
/*! reconciliation: fast-path pages deleted */
#define WT_STAT_DSRC_REC_PAGE_DELETE_FAST 2103
#define WT_STAT_DSRC_REC_PAGE_DELETE_FAST 2104
/*!
* reconciliation: internal page key bytes discarded using suffix
* compression
*/
#define WT_STAT_DSRC_REC_SUFFIX_COMPRESSION 2104
#define WT_STAT_DSRC_REC_SUFFIX_COMPRESSION 2105
/*! reconciliation: internal page multi-block writes */
#define WT_STAT_DSRC_REC_MULTIBLOCK_INTERNAL 2105
#define WT_STAT_DSRC_REC_MULTIBLOCK_INTERNAL 2106
/*! reconciliation: internal-page overflow keys */
#define WT_STAT_DSRC_REC_OVERFLOW_KEY_INTERNAL 2106
#define WT_STAT_DSRC_REC_OVERFLOW_KEY_INTERNAL 2107
/*! reconciliation: leaf page key bytes discarded using prefix compression */
#define WT_STAT_DSRC_REC_PREFIX_COMPRESSION 2107
#define WT_STAT_DSRC_REC_PREFIX_COMPRESSION 2108
/*! reconciliation: leaf page multi-block writes */
#define WT_STAT_DSRC_REC_MULTIBLOCK_LEAF 2108
#define WT_STAT_DSRC_REC_MULTIBLOCK_LEAF 2109
/*! reconciliation: leaf-page overflow keys */
#define WT_STAT_DSRC_REC_OVERFLOW_KEY_LEAF 2109
#define WT_STAT_DSRC_REC_OVERFLOW_KEY_LEAF 2110
/*! reconciliation: maximum blocks required for a page */
#define WT_STAT_DSRC_REC_MULTIBLOCK_MAX 2110
#define WT_STAT_DSRC_REC_MULTIBLOCK_MAX 2111
/*! reconciliation: overflow values written */
#define WT_STAT_DSRC_REC_OVERFLOW_VALUE 2111
#define WT_STAT_DSRC_REC_OVERFLOW_VALUE 2112
/*! reconciliation: page checksum matches */
#define WT_STAT_DSRC_REC_PAGE_MATCH 2112
#define WT_STAT_DSRC_REC_PAGE_MATCH 2113
/*! reconciliation: page reconciliation calls */
#define WT_STAT_DSRC_REC_PAGES 2113
#define WT_STAT_DSRC_REC_PAGES 2114
/*! reconciliation: page reconciliation calls for eviction */
#define WT_STAT_DSRC_REC_PAGES_EVICTION 2114
#define WT_STAT_DSRC_REC_PAGES_EVICTION 2115
/*! reconciliation: pages deleted */
#define WT_STAT_DSRC_REC_PAGE_DELETE 2115
#define WT_STAT_DSRC_REC_PAGE_DELETE 2116
/*! session: object compaction */
#define WT_STAT_DSRC_SESSION_COMPACT 2116
#define WT_STAT_DSRC_SESSION_COMPACT 2117
/*! session: open cursor count */
#define WT_STAT_DSRC_SESSION_CURSOR_OPEN 2117
#define WT_STAT_DSRC_SESSION_CURSOR_OPEN 2118
/*! transaction: update conflicts */
#define WT_STAT_DSRC_TXN_UPDATE_CONFLICT 2118
#define WT_STAT_DSRC_TXN_UPDATE_CONFLICT 2119

/*!
* @}
Expand Down
4 changes: 4 additions & 0 deletions src/support/stat.c
View file
Expand Up @@ -64,6 +64,7 @@ static const char * const __stats_dsrc_desc[] = {
"cache: pages requested from the cache",
"cache: pages written from cache",
"cache: pages written requiring in-memory restoration",
"cache: tracked dirty bytes in the cache",
"cache: unmodified pages evicted",
"cache_walk: Average difference between current eviction generation when the page was last considered",
"cache_walk: Average on-disk page image size seen",
Expand Down Expand Up @@ -225,6 +226,7 @@ __wt_stat_dsrc_clear_single(WT_DSRC_STATS *stats)
stats->cache_pages_requested = 0;
stats->cache_write = 0;
stats->cache_write_restore = 0;
/* not clearing cache_bytes_dirty */
stats->cache_eviction_clean = 0;
/* not clearing cache_state_gen_avg_gap */
/* not clearing cache_state_avg_written_size */
Expand Down Expand Up @@ -372,6 +374,7 @@ __wt_stat_dsrc_aggregate_single(
to->cache_pages_requested += from->cache_pages_requested;
to->cache_write += from->cache_write;
to->cache_write_restore += from->cache_write_restore;
to->cache_bytes_dirty += from->cache_bytes_dirty;
to->cache_eviction_clean += from->cache_eviction_clean;
to->cache_state_gen_avg_gap += from->cache_state_gen_avg_gap;
to->cache_state_avg_written_size +=
Expand Down Expand Up @@ -535,6 +538,7 @@ __wt_stat_dsrc_aggregate(
WT_STAT_READ(from, cache_pages_requested);
to->cache_write += WT_STAT_READ(from, cache_write);
to->cache_write_restore += WT_STAT_READ(from, cache_write_restore);
to->cache_bytes_dirty += WT_STAT_READ(from, cache_bytes_dirty);
to->cache_eviction_clean += WT_STAT_READ(from, cache_eviction_clean);
to->cache_state_gen_avg_gap +=
WT_STAT_READ(from, cache_state_gen_avg_gap);
Expand Down

0 comments on commit 9a3d212

Please sign in to comment.