Import wiredtiger: 943d9cdd2a1cfc4ae357da44ec497b5115ff3139 from bran…

…ch mongodb-4.4

ref: a7af601643..943d9cdd2a
for: 4.4.0-rc8

WT-6168       Ignore history store visibility when resolving prepared updates
WT-6250       Avoid passing reconcile struct to cell layer for stat logging
WT-6302       Move format operations tracing into WiredTiger log files
WT-6303       Fix commit check may fail because of a prepared rollback
WT-6306       Do not force evict HS pages because of deleted items
WT-6307       Increase the page cache size for all the page allocations
WT-6312       Add history store verification
WT-6315       Fix commit check failure because of updates restored from data store
WT-6317       Create variant of compatibility test that runs faster and generates a smaller artifact footprint
WT-6318       Stop assuming exclusive access when writing to history store pages
WT-6320       Allow eviction of pages with older content

src/third_party/wiredtiger/dist/api_data.py

@@ -522,6 +522,14 @@ def __ge__(self, other):
             is not limited to not skewing newest, not favoring leaf pages,
             and modifying the eviction score mechanism.''',
             type='boolean'),
+        Config('log_retention', '0', r'''
+            adjust log archiving to retain at least this number of log files, ignored if set to 0.
+            (Warning: this option can remove log files required for recovery if no checkpoints
+            have yet been done and the number of log files exceeds the configured value. As
+            WiredTiger cannot detect the difference between a system that has not yet checkpointed
+            and one that will never checkpoint, it might discard log files before any checkpoint is
+            done.)''',
+            min='0', max='1024'),
         Config('realloc_exact', 'false', r'''
             if true, reallocation of memory will only provide the exact
             amount requested. This will help with spotting memory allocation

src/third_party/wiredtiger/dist/s_string.ok

@@ -43,7 +43,8 @@ Barack
 BerkeleyDB
 Bitfield
 Bitwise
-BlqRr
+BlqR
+BqRt
 Brueckner
 Bsearch
 Btree
@@ -951,7 +952,9 @@ localkey
 localtime
 localvalue
 logf
+logfile
 logmgr
+logmsg
 lognum
 logop
 logpath
@@ -963,7 +966,6 @@ lookaside
 lookup
 lookups
 lossy
-lqRrt
 lr
 lrt
 lru
@@ -1024,6 +1026,7 @@ multithreaded
 munmap
 mutex
 mutexes
+mx
 mytable
 mytxn
 namespace
@@ -1147,6 +1150,7 @@ pv
 pvA
 pwrite
 py
+qRrT
 qdown
 qqq
 qrrSS

src/third_party/wiredtiger/dist/stat_data.py

@@ -233,7 +233,6 @@ def __init__(self, name, desc, flags=''):
     CacheStat('cache_eviction_fail_active_children_on_an_internal_page', 'pages selected for eviction unable to be evicted because of active children on an internal page'),
     CacheStat('cache_eviction_fail_in_reconciliation', 'pages selected for eviction unable to be evicted because of failure in reconciliation'),
     CacheStat('cache_eviction_fail_parent_has_overflow_items', 'pages selected for eviction unable to be evicted as the parent page has overflow items'),
-    CacheStat('cache_eviction_fail_with_newer_modifications_on_a_clean_page', 'pages selected for eviction unable to be evicted due to newer modifications on a clean page'),
     CacheStat('cache_eviction_force', 'forced eviction - pages selected count'),
     CacheStat('cache_eviction_force_clean', 'forced eviction - pages evicted that were clean count'),
     CacheStat('cache_eviction_force_clean_time', 'forced eviction - pages evicted that were clean time (usecs)'),
@@ -546,6 +545,8 @@ def __init__(self, name, desc, flags=''):
     RecStat('rec_time_aggr_oldest_start_ts', 'pages written including an aggregated oldest start timestamp '),
     RecStat('rec_time_aggr_oldest_start_txn', 'pages written including an aggregated oldest start transaction ID '),
     RecStat('rec_time_aggr_prepared', 'pages written including an aggregated prepare'),
+    RecStat('rec_time_window_bytes_ts', 'approximate byte size of timestamps in pages written'),
+    RecStat('rec_time_window_bytes_txn', 'approximate byte size of transaction IDs in pages written'),
     RecStat('rec_time_window_durable_start_ts', 'records written including a start durable timestamp'),
     RecStat('rec_time_window_durable_stop_ts', 'records written including a stop durable timestamp'),
     RecStat('rec_time_window_pages_durable_start_ts', 'pages written including at least one start durable timestamp'),
@@ -874,6 +875,8 @@ def __init__(self, name, desc, flags=''):
     RecStat('rec_time_aggr_oldest_start_ts', 'pages written including an aggregated oldest start timestamp '),
     RecStat('rec_time_aggr_oldest_start_txn', 'pages written including an aggregated oldest start transaction ID '),
     RecStat('rec_time_aggr_prepared', 'pages written including an aggregated prepare'),
+    RecStat('rec_time_window_bytes_ts', 'approximate byte size of timestamps in pages written'),
+    RecStat('rec_time_window_bytes_txn', 'approximate byte size of transaction IDs in pages written'),
     RecStat('rec_time_window_durable_start_ts', 'records written including a start durable timestamp'),
     RecStat('rec_time_window_durable_stop_ts', 'records written including a stop durable timestamp'),
     RecStat('rec_time_window_pages_durable_start_ts', 'pages written including at least one start durable timestamp'),

src/third_party/wiredtiger/import.data

@@ -2,5 +2,5 @@
     "vendor": "wiredtiger",
     "github": "wiredtiger/wiredtiger.git",
     "branch": "mongodb-4.4",
-    "commit": "a7af6016438884665a89bc7c67f42c7812472848"
+    "commit": "943d9cdd2a1cfc4ae357da44ec497b5115ff3139"
 }

src/third_party/wiredtiger/src/btree/bt_curnext.c

@@ -688,10 +688,15 @@ __wt_btcur_next(WT_CURSOR_BTREE *cbt, bool truncating)
          * and only saw deleted records, try to evict the page when we release it. Otherwise
          * repeatedly deleting from the beginning of a tree can have quadratic performance. Take
          * care not to force eviction of pages that are genuinely empty, in new trees.
+         *
+         * A visible stop timestamp could have been treated as a tombstone and accounted in the
+         * deleted count. Such a page might not have any new updates and be clean, but could benefit
+         * from reconciliation getting rid of the obsolete content. Hence mark the page dirty to
+         * force it through reconciliation.
          */
         if (page != NULL && (cbt->page_deleted_count > WT_BTREE_DELETE_THRESHOLD ||
                               (newpage && cbt->page_deleted_count > 0))) {
-            __wt_page_evict_soon(session, cbt->ref);
+            WT_ERR(__wt_page_dirty_and_evict_soon(session, cbt->ref));
             WT_STAT_CONN_INCR(session, cache_eviction_force_delete);
         }
         cbt->page_deleted_count = 0;

src/third_party/wiredtiger/src/btree/bt_curprev.c

@@ -646,10 +646,15 @@ __wt_btcur_prev(WT_CURSOR_BTREE *cbt, bool truncating)
          * and only saw deleted records, try to evict the page when we release it. Otherwise
          * repeatedly deleting from the beginning of a tree can have quadratic performance. Take
          * care not to force eviction of pages that are genuinely empty, in new trees.
+         *
+         * A visible stop timestamp could have been treated as a tombstone and accounted in the
+         * deleted count. Such a page might not have any new updates and be clean, but could benefit
+         * from reconciliation getting rid of the obsolete content. Hence mark the page dirty to
+         * force it through reconciliation.
          */
         if (page != NULL && (cbt->page_deleted_count > WT_BTREE_DELETE_THRESHOLD ||
                               (newpage && cbt->page_deleted_count > 0))) {
-            __wt_page_evict_soon(session, cbt->ref);
+            WT_ERR(__wt_page_dirty_and_evict_soon(session, cbt->ref));
             WT_STAT_CONN_INCR(session, cache_eviction_force_delete);
         }
         cbt->page_deleted_count = 0;

src/third_party/wiredtiger/src/btree/bt_delete.c

@@ -294,7 +294,7 @@ __wt_delete_page_instantiate(WT_SESSION_IMPL *session, WT_REF *ref)
     WT_ROW *rip;
     WT_TIME_WINDOW tw;
     WT_UPDATE **upd_array, *upd;
-    size_t size;
+    size_t size, total_size;
     uint32_t count, i;
 
     btree = S2BT(session);
@@ -343,7 +343,8 @@ __wt_delete_page_instantiate(WT_SESSION_IMPL *session, WT_REF *ref)
      * because deletes are instantiated after the history store table updates.)
      */
     if (page->entries != 0 && page->modify->mod_row_update == NULL)
-        WT_RET(__wt_calloc_def(session, page->entries, &page->modify->mod_row_update));
+        WT_PAGE_ALLOC_AND_SWAP(
+          session, page, page->modify->mod_row_update, upd_array, page->entries);
 
     /*
      * Allocate the per-reference update array; in the case of instantiating a page deleted in a
@@ -362,15 +363,17 @@ __wt_delete_page_instantiate(WT_SESSION_IMPL *session, WT_REF *ref)
                     ++count;
         }
         WT_RET(__wt_calloc_def(session, count + 1, &page_del->update_list));
+        __wt_cache_page_inmem_incr(session, page, (count + 1) * sizeof(page_del->update_list));
     }
 
     /* Walk the page entries, giving each one a tombstone. */
-    size = 0;
+    size = total_size = 0;
     count = 0;
     upd_array = page->modify->mod_row_update;
     if ((insert = WT_ROW_INSERT_SMALLEST(page)) != NULL)
         WT_SKIP_FOREACH (ins, insert) {
             WT_ERR(__tombstone_update_alloc(session, page_del, &upd, &size));
+            total_size += size;
             upd->next = ins->upd;
             ins->upd = upd;
 
@@ -385,6 +388,7 @@ __wt_delete_page_instantiate(WT_SESSION_IMPL *session, WT_REF *ref)
         __wt_read_row_time_window(session, page, rip, &tw);
         if (!WT_TIME_WINDOW_HAS_STOP(&tw)) {
             WT_ERR(__tombstone_update_alloc(session, page_del, &upd, &size));
+            total_size += size;
             upd->next = upd_array[WT_ROW_SLOT(page, rip)];
             upd_array[WT_ROW_SLOT(page, rip)] = upd;
 
@@ -394,6 +398,7 @@ __wt_delete_page_instantiate(WT_SESSION_IMPL *session, WT_REF *ref)
             if ((insert = WT_ROW_INSERT(page, rip)) != NULL)
                 WT_SKIP_FOREACH (ins, insert) {
                     WT_ERR(__tombstone_update_alloc(session, page_del, &upd, &size));
+                    total_size += size;
                     upd->next = ins->upd;
                     ins->upd = upd;
 
@@ -403,7 +408,7 @@ __wt_delete_page_instantiate(WT_SESSION_IMPL *session, WT_REF *ref)
         }
     }
 
-    __wt_cache_page_inmem_incr(session, page, size);
+    __wt_cache_page_inmem_incr(session, page, total_size);
 
     return (0);
 

src/third_party/wiredtiger/src/btree/bt_page.c

@@ -535,7 +535,7 @@ __inmem_row_leaf(WT_SESSION_IMPL *session, WT_PAGE *page)
     WT_DECL_ITEM(value);
     WT_DECL_RET;
     WT_ROW *rip;
-    WT_UPDATE *tombstone, **upd_array, *upd;
+    WT_UPDATE *tombstone, *upd, **upd_array;
     size_t size, total_size;
     uint32_t i;
     bool instantiate_prepared, prepare;
@@ -599,7 +599,8 @@ __inmem_row_leaf(WT_SESSION_IMPL *session, WT_PAGE *page)
 
         /* Allocate the per-page update array if one doesn't already exist. */
         if (page->entries != 0 && page->modify->mod_row_update == NULL)
-            WT_RET(__wt_calloc_def(session, page->entries, &page->modify->mod_row_update));
+            WT_PAGE_ALLOC_AND_SWAP(
+              session, page, page->modify->mod_row_update, upd_array, page->entries);
 
         /* For each entry in the page */
         size = total_size = 0;
@@ -614,6 +615,7 @@ __inmem_row_leaf(WT_SESSION_IMPL *session, WT_PAGE *page)
                 WT_ERR(__wt_page_cell_data_ref(session, page, &unpack, value));
 
                 WT_ERR(__wt_upd_alloc(session, value, WT_UPDATE_STANDARD, &upd, &size));
+                total_size += size;
                 upd->durable_ts = unpack.tw.durable_start_ts;
                 upd->start_ts = unpack.tw.start_ts;
                 upd->txnid = unpack.tw.start_txn;
@@ -625,22 +627,23 @@ __inmem_row_leaf(WT_SESSION_IMPL *session, WT_PAGE *page)
                  */
                 if (WT_TIME_WINDOW_HAS_STOP(&unpack.tw)) {
                     WT_ERR(__wt_upd_alloc_tombstone(session, &tombstone, &size));
+                    total_size += size;
                     tombstone->durable_ts = WT_TS_NONE;
                     tombstone->start_ts = unpack.tw.stop_ts;
                     tombstone->txnid = unpack.tw.stop_txn;
                     tombstone->prepare_state = WT_PREPARE_INPROGRESS;
-                    F_SET(tombstone, WT_UPDATE_PREPARE_RESTORED_FROM_DISK);
+                    F_SET(tombstone, WT_UPDATE_PREPARE_RESTORED_FROM_DS);
+                    F_SET(upd, WT_UPDATE_RESTORED_FROM_DS);
                     tombstone->next = upd;
                 } else {
                     upd->durable_ts = WT_TS_NONE;
                     upd->prepare_state = WT_PREPARE_INPROGRESS;
-                    F_SET(upd, WT_UPDATE_PREPARE_RESTORED_FROM_DISK);
+                    F_SET(upd, WT_UPDATE_PREPARE_RESTORED_FROM_DS);
                     tombstone = upd;
                 }
 
                 upd_array[WT_ROW_SLOT(page, rip)] = tombstone;
                 tombstone = upd = NULL;
-                total_size += size;
             }
         }