Import wiredtiger: 7db12ec1b5b1843364ae28248b0680b816aab651 from bran…

…ch mongodb-4.2

ref: 45fd19bcb1..7db12ec1b5
for: 4.1.4

WT-3736       Add statistics to measure contention on lookaside cursor
WT-4185       Don't remove all lookaside entries when reading a page
WT-4225       Automate a backup test that simulates volume snapshot via dd
WT-4308       Insert split during sync should not free blocks
WT-4321       Disable the random direct I/O test
WT-4325       Add a WiredTiger-local version of the qsort(3) call

src/third_party/wiredtiger/dist/s_string.ok

@@ -24,6 +24,7 @@ Async
 AsyncOp
 Athanassoulis
 Athlon
+Axxx
 BBBBB
 BBBBBB
 BBBBBBBBBB
@@ -46,6 +47,7 @@ Brueckner
 Bsearch
 Btree
 Buf
+Bxxx
 Bzip
 CAS
 CCCCCCCCCCCCCCCCCC
@@ -500,6 +502,7 @@ bool
 boolean
 br
 breakpoint
+bs
 bswap
 bt
 btcur
@@ -658,6 +661,7 @@ dhandle
 dhandles
 difftime
 dir
+directio
 dirlist
 disjunction
 disjunctions
@@ -814,6 +818,7 @@ idlems
 idx
 ifdef
 ifdef's
+iflag
 iiSii
 iiiS
 iiii
@@ -1044,6 +1049,7 @@ other's
 ovfl
 ownp
 pR
+pS
 packv
 pagesize
 parens
@@ -1151,6 +1157,7 @@ rwlock
 sH
 sHQ
 sT
+sanitizers
 scalability
 sched
 scr

src/third_party/wiredtiger/dist/s_style

@@ -101,6 +101,12 @@ else
 		cat $t
 	}
 
+	if ! expr "$f" : 'src/include/misc.h' > /dev/null &&
+	    grep '[[:space:]]qsort(' $f > $t; then
+		echo "$f: qsort call, use WiredTiger __wt_qsort instead"
+		cat $t
+	fi
+
 	if ! expr "$f" : 'src/.*/os_setvbuf.c' > /dev/null &&
 	    egrep -w 'setvbuf' $f > $t; then
 		echo "$f: setvbuf call, use WiredTiger library replacements"

src/third_party/wiredtiger/dist/stat_data.py

@@ -260,6 +260,8 @@ def __init__(self, name, desc, flags=''):
     CacheStat('cache_hazard_walks', 'hazard pointer check entries walked'),
     CacheStat('cache_inmem_split', 'in-memory page splits'),
     CacheStat('cache_inmem_splittable', 'in-memory page passed criteria to be split'),
+    CacheStat('cache_lookaside_cursor_wait_application', 'cache overflow cursor application thread wait time (usecs)'),
+    CacheStat('cache_lookaside_cursor_wait_internal', 'cache overflow cursor internal thread wait time (usecs)'),
     CacheStat('cache_lookaside_entries', 'cache overflow table entries', 'no_clear,no_scale'),
     CacheStat('cache_lookaside_insert', 'cache overflow table insert calls'),
     CacheStat('cache_lookaside_remove', 'cache overflow table remove calls'),

src/third_party/wiredtiger/import.data

@@ -1,5 +1,5 @@
 {
-    "commit": "45fd19bcb1007fd4e473d77ddd09d3157ff15c7e", 
+    "commit": "7db12ec1b5b1843364ae28248b0680b816aab651", 
     "github": "wiredtiger/wiredtiger.git", 
     "vendor": "wiredtiger", 
     "branch": "mongodb-4.2"

src/third_party/wiredtiger/src/btree/bt_read.c

@@ -541,13 +541,19 @@ __page_read(WT_SESSION_IMPL *session, WT_REF *ref, uint32_t flags)
 	}
 
 	/*
-	 * We no longer need lookaside entries once the page is instantiated.
-	 * There's no reason for the lookaside remove to fail, but ignore it
-	 * if for some reason it fails, we've got a valid page.
+	 * Once the page is instantiated, we no longer need the history in
+	 * lookaside.  We leave the lookaside sweep thread to do most cleanup,
+	 * but it can only remove keys that skew newest (if there are entries
+	 * in the lookaside newer than the page, they need to be read back into
+	 * cache or they will be lost).
+	 *
+	 * There is no reason for the lookaside remove should fail, but ignore
+	 * it if for some reason it fails, we've got a valid page.
 	 *
 	 * Don't free WT_REF.page_las, there may be concurrent readers.
 	 */
-	if (final_state == WT_REF_MEM && ref->page_las != NULL)
+	if (final_state == WT_REF_MEM &&
+	    ref->page_las != NULL && !ref->page_las->skew_newest)
 		WT_IGNORE_RET(__wt_las_remove_block(
 		    session, ref->page_las->las_pageid, false));
 

src/third_party/wiredtiger/src/btree/bt_slvg.c

@@ -259,7 +259,7 @@ __wt_bt_salvage(WT_SESSION_IMPL *session, WT_CKPT *ckptbase, const char *cfg[])
 	 * fixed-length format ranges to overlap during salvage, and I don't
 	 * want to have to retrofit the code later.
 	 */
-	qsort(ss->pages,
+	__wt_qsort(ss->pages,
 	    (size_t)ss->pages_next, sizeof(WT_TRACK *), __slvg_trk_compare_key);
 	if (ss->page_type == WT_PAGE_ROW_LEAF)
 		WT_ERR(__slvg_row_range(session, ss));
@@ -1093,7 +1093,7 @@ __slvg_col_trk_update_start(uint32_t slot, WT_STUFF *ss)
 	}
 	i -= slot;
 	if (i > 1)
-		qsort(ss->pages + slot, (size_t)i,
+		__wt_qsort(ss->pages + slot, (size_t)i,
 		    sizeof(WT_TRACK *), __slvg_trk_compare_key);
 }
 
@@ -1790,7 +1790,7 @@ __slvg_row_trk_update_start(
 	}
 	i -= slot;
 	if (i > 1)
-		qsort(ss->pages + slot, (size_t)i,
+		__wt_qsort(ss->pages + slot, (size_t)i,
 		    sizeof(WT_TRACK *), __slvg_trk_compare_key);
 
 err:	if (page != NULL)
@@ -2160,13 +2160,12 @@ __slvg_ovfl_reconcile(WT_SESSION_IMPL *session, WT_STUFF *ss)
 	 * If an overflow page is referenced more than once, discard leaf pages
 	 * with the lowest LSNs until overflow pages are only referenced once.
 	 *
-	 * This requires sorting the page list by LSN, and the overflow array
-
-	 * by address cookie.
+	 * This requires sorting the page list by LSN, and the overflow array by
+	 * address cookie.
 	 */
-	qsort(ss->pages,
+	__wt_qsort(ss->pages,
 	    (size_t)ss->pages_next, sizeof(WT_TRACK *), __slvg_trk_compare_gen);
-	qsort(ss->ovfl,
+	__wt_qsort(ss->ovfl,
 	    (size_t)ss->ovfl_next, sizeof(WT_TRACK *), __slvg_trk_compare_addr);
 
 	/*

src/third_party/wiredtiger/src/btree/bt_split.c

@@ -625,6 +625,7 @@ static int
 __split_parent(WT_SESSION_IMPL *session, WT_REF *ref, WT_REF **ref_new,
     uint32_t new_entries, size_t parent_incr, bool exclusive, bool discard)
 {
+	WT_BTREE *btree;
 	WT_DECL_ITEM(scr);
 	WT_DECL_RET;
 	WT_IKEY *ikey;
@@ -639,6 +640,7 @@ __split_parent(WT_SESSION_IMPL *session, WT_REF *ref, WT_REF **ref_new,
 	uint32_t hint, i, j;
 	bool empty_parent;
 
+	btree = S2BT(session);
 	parent = ref->home;
 
 	alloc_index = pindex = NULL;
@@ -658,17 +660,23 @@ __split_parent(WT_SESSION_IMPL *session, WT_REF *ref, WT_REF **ref_new,
 	parent_entries = pindex->entries;
 
 	/*
-	 * Remove any refs to deleted pages while we are splitting, we have
-	 * the internal page locked down, and are copying the refs into a new
-	 * array anyway.  Switch them to the special split state, so that any
-	 * reading thread will restart.
+	 * Remove any refs to deleted pages while we are splitting, we have the
+	 * internal page locked down, and are copying the refs into a new array
+	 * anyway.  Switch them to the special split state, so that any reading
+	 * thread will restart.
+	 *
+	 * We can't do this if there is a sync running in the tree in another
+	 * session: removing the refs frees the blocks for the deleted pages,
+	 * which can corrupt the free list calculated by the sync.
 	 */
 	WT_ERR(__wt_scr_alloc(session, 10 * sizeof(uint32_t), &scr));
 	for (deleted_entries = 0, i = 0; i < parent_entries; ++i) {
 		next_ref = pindex->index[i];
 		WT_ASSERT(session, next_ref->state != WT_REF_SPLIT);
 		if ((discard && next_ref == ref) ||
-		    (next_ref->state == WT_REF_DELETED &&
+		    ((!WT_BTREE_SYNCING(btree) ||
+		    WT_SESSION_BTREE_SYNC(session)) &&
+		    next_ref->state == WT_REF_DELETED &&
 		    __wt_delete_page_skip(session, next_ref, true) &&
 		    __wt_atomic_casv32(
 		    &next_ref->state, WT_REF_DELETED, WT_REF_SPLIT))) {

src/third_party/wiredtiger/src/cache/cache_las.c

@@ -310,7 +310,16 @@ __wt_las_cursor(
 			 *
 			 * XXX better as a condition variable.
 			 */
-			__wt_sleep(0, 1000);
+			__wt_sleep(0, WT_THOUSAND);
+			if (F_ISSET(session, WT_SESSION_INTERNAL))
+				WT_STAT_CONN_INCRV(session,
+				    cache_lookaside_cursor_wait_internal,
+				    WT_THOUSAND);
+			else
+				WT_STAT_CONN_INCRV(session,
+				    cache_lookaside_cursor_wait_application,
+				    WT_THOUSAND);
+
 		}
 	}
 

src/third_party/wiredtiger/src/config/config_collapse.c

@@ -399,7 +399,7 @@ __wt_config_merge(WT_SESSION_IMPL *session,
 	 * Sort the array by key and, in the case of identical keys, by
 	 * generation.
 	 */
-	qsort(merge.entries, merge.entries_next,
+	__wt_qsort(merge.entries, merge.entries_next,
 	    sizeof(WT_CONFIG_MERGE_ENTRY), __config_merge_cmp);
 
 	/* Convert the array of entries into a string. */

src/third_party/wiredtiger/src/evict/evict_lru.c

@@ -1257,7 +1257,7 @@ __evict_lru_walk(WT_SESSION_IMPL *session)
 		queue->evict_current = NULL;
 
 	entries = queue->evict_entries;
-	qsort(queue->evict_queue,
+	__wt_qsort(queue->evict_queue,
 	    entries, sizeof(WT_EVICT_ENTRY), __evict_lru_cmp);
 
 	/* Trim empty entries from the end. */

src/third_party/wiredtiger/src/include/misc.h

@@ -186,6 +186,15 @@
 	}								\
 } while (0)
 
+/*
+ * Some C compiler address sanitizers complain if qsort is passed a NULL base
+ * reference, even if there are no elements to compare (note zero elements is
+ * allowed by the IEEE Std 1003.1-2017 standard). Avoid the complaint.
+ */
+#define	__wt_qsort(base, nmemb, size, compar)				\
+	if ((nmemb) != 0)						\
+		qsort(base, nmemb, size, compar)
+
 /*
  * Binary search for an integer key.
  */

src/third_party/wiredtiger/src/include/stat.h

@@ -354,6 +354,8 @@ struct __wt_connection_stats {
 	int64_t cache_bytes_other;
 	int64_t cache_bytes_read;
 	int64_t cache_bytes_write;
+	int64_t cache_lookaside_cursor_wait_application;
+	int64_t cache_lookaside_cursor_wait_internal;
 	int64_t cache_lookaside_score;
 	int64_t cache_lookaside_entries;
 	int64_t cache_lookaside_insert;