Merge branch 'develop' into WT-2056

Conflicts:
	src/cursor/cur_file.c

build_win/filelist.win

@@ -45,6 +45,7 @@ src/btree/col_srch.c
 src/btree/row_key.c
 src/btree/row_modify.c
 src/btree/row_srch.c
+src/cache/cache_las.c
 src/config/config.c
 src/config/config_api.c
 src/config/config_check.c

dist/filelist

@@ -45,6 +45,7 @@ src/btree/col_srch.c
 src/btree/row_key.c
 src/btree/row_modify.c
 src/btree/row_srch.c
+src/cache/cache_las.c
 src/config/config.c
 src/config/config_api.c
 src/config/config_check.c

dist/flags.py

@@ -45,9 +45,10 @@
         'READ_WONT_NEED',
     ],
     'rec_write' : [
+        'EVICT_LOOKASIDE',
         'EVICTING',
-        'SKIP_UPDATE_ERR',
-        'SKIP_UPDATE_RESTORE',
+        'EVICT_UPDATE_RESTORE',
+        'VISIBILITY_ERR',
     ],
     'txn_log_checkpoint' : [
         'TXN_LOG_CKPT_CLEANUP',
@@ -106,16 +107,17 @@
     'session' : [
         'SESSION_CAN_WAIT',
         'SESSION_CLEAR_EVICT_WALK',
+        'SESSION_INTERNAL',
         'SESSION_LOCKED_CHECKPOINT',
         'SESSION_LOCKED_HANDLE_LIST',
         'SESSION_LOCKED_SCHEMA',
         'SESSION_LOCKED_SLOT',
         'SESSION_LOCKED_TABLE',
-        'SESSION_INTERNAL',
         'SESSION_LOGGING_INMEM',
+        'SESSION_LOOKASIDE_CURSOR',
         'SESSION_NO_CACHE',
-        'SESSION_NO_CACHE_CHECK',
         'SESSION_NO_DATA_HANDLES',
+        'SESSION_NO_EVICTION',
         'SESSION_NO_LOGGING',
         'SESSION_NO_SCHEMA_LOCK',
         'SESSION_QUIET_CORRUPT_FILE',

dist/s_string.ok

@@ -157,11 +157,13 @@ KVS
 Kanowski's
 Kounavis
 LANGID
+LAS
 LF
 LLLLLL
 LLLLLLL
 LOGREC
 LOGSCAN
+LOOKASIDE
 LRU
 LRVv
 LSB
@@ -177,6 +179,7 @@ Levyx
 Llqr
 Llqrt
 LockFile
+Lookaside
 Lookup
 MALLOC
 MEM
@@ -211,6 +214,7 @@ NetBSD
 NoAddr
 Noll
 Nul
+OOB
 OPTYPE
 OUTBUFF
 OVFL
@@ -232,6 +236,7 @@ Preload
 Prepend
 Qsort
 RCS
+RECNO
 REF's
 REFs
 RET
@@ -321,6 +326,7 @@ WiredTiger's
 WiredTigerCheckpoint
 WiredTigerException
 WiredTigerInit
+WiredTigerLAS
 WiredTigerLog
 WiredTigerPreplog
 WiredTigerTmplog
@@ -506,6 +512,7 @@ dlh
 dll
 dlopen
 dlsym
+dmalloc
 dmsg
 doxgen
 doxygen
@@ -515,6 +522,7 @@ dsk
 dsrc
 dst
 dstlen
+dstrdup
 dsync
 dumpcmp
 dumpfile
@@ -651,6 +659,7 @@ kvraw
 kvs
 kvsbdb
 lang
+las
 latencies
 lbrace
 lbracket
@@ -678,6 +687,7 @@ logread
 logrec
 logsize
 logtest
+lookaside
 lookup
 lookups
 lossy
@@ -951,6 +961,7 @@ unesc
 unescaped
 uninstantiated
 unistd
+unlinked
 unmap
 unmarshall
 unmarshalled

dist/stat_data.py

@@ -127,9 +127,9 @@ def __init__(self, name, desc, flags=''):
     AsyncStat('async_alloc_race', 'number of allocation state races'),
     AsyncStat('async_alloc_view',
         'number of operation slots viewed for allocation'),
+    AsyncStat('async_cur_queue', 'current work queue length'),
     AsyncStat('async_flush', 'number of flush calls'),
     AsyncStat('async_full', 'number of times operation allocation failed'),
-    AsyncStat('async_cur_queue', 'current work queue length'),
     AsyncStat('async_max_queue',
         'maximum work queue length', 'no_clear,no_scale'),
     AsyncStat('async_nowork', 'number of times worker found no work'),
@@ -156,11 +156,11 @@ def __init__(self, name, desc, flags=''):
     ##########################################
     CacheStat('cache_bytes_dirty',
         'tracked dirty bytes in the cache', 'no_clear,no_scale'),
-    CacheStat('cache_bytes_inuse',
-        'bytes currently in the cache', 'no_clear,no_scale'),
     CacheStat('cache_bytes_internal',
         'tracked bytes belonging to internal pages in the cache',
         'no_clear,no_scale'),
+    CacheStat('cache_bytes_inuse',
+        'bytes currently in the cache', 'no_clear,no_scale'),
     CacheStat('cache_bytes_leaf',
         'tracked bytes belonging to leaf pages in the cache',
         'no_clear,no_scale'),
@@ -172,11 +172,11 @@ def __init__(self, name, desc, flags=''):
     CacheStat('cache_bytes_read', 'bytes read into cache'),
     CacheStat('cache_bytes_write', 'bytes written from cache'),
     CacheStat('cache_eviction_app', 'pages evicted by application threads'),
+    CacheStat('cache_eviction_checkpoint', 'checkpoint blocked page eviction'),
     CacheStat('cache_eviction_clean', 'unmodified pages evicted'),
     CacheStat('cache_eviction_deepen',
         'page split during eviction deepened the tree'),
     CacheStat('cache_eviction_dirty', 'modified pages evicted'),
-    CacheStat('cache_eviction_checkpoint', 'checkpoint blocked page eviction'),
     CacheStat('cache_eviction_fail',
         'pages selected for eviction unable to be evicted'),
     CacheStat('cache_eviction_force',
@@ -204,13 +204,21 @@ def __init__(self, name, desc, flags=''):
     CacheStat('cache_eviction_worker_evicting',
         'eviction worker thread evicting pages'),
     CacheStat('cache_inmem_split', 'in-memory page splits'),
+    CacheStat('cache_lookaside_insert', 'lookaside table insert calls'),
+    CacheStat('cache_lookaside_remove', 'lookaside table remove calls'),
     CacheStat('cache_overhead', 'percentage overhead', 'no_clear,no_scale'),
     CacheStat('cache_pages_dirty',
         'tracked dirty pages in the cache', 'no_clear,no_scale'),
     CacheStat('cache_pages_inuse',
         'pages currently held in the cache', 'no_clear,no_scale'),
     CacheStat('cache_read', 'pages read into cache'),
+    CacheStat('cache_read_lookaside',
+        'pages read into cache requiring lookaside entries'),
     CacheStat('cache_write', 'pages written from cache'),
+    CacheStat('cache_write_lookaside',
+        'page written requiring lookaside records'),
+    CacheStat('cache_write_restore',
+        'pages written requiring in-memory restoration'),
 
     ##########################################
     # Dhandle statistics
@@ -236,8 +244,8 @@ def __init__(self, name, desc, flags=''):
     LogStat('log_compress_len', 'total size of compressed records'),
     LogStat('log_compress_mem', 'total in-memory size of compressed records'),
     LogStat('log_compress_small', 'log records too small to compress'),
-    LogStat('log_compress_writes', 'log records compressed'),
     LogStat('log_compress_write_fails', 'log records not compressed'),
+    LogStat('log_compress_writes', 'log records compressed'),
     LogStat('log_direct_writes', 'log records written directly'),
     LogStat('log_max_filesize', 'maximum log file size', 'no_clear,no_scale'),
     LogStat('log_prealloc_files', 'pre-allocated log files prepared'),
@@ -248,18 +256,17 @@ def __init__(self, name, desc, flags=''):
     LogStat('log_scan_records', 'records processed by log scan'),
     LogStat('log_scan_rereads', 'log scan records requiring two reads'),
     LogStat('log_scans', 'log scan operations'),
-    LogStat('log_sync', 'log sync operations'),
-    LogStat('log_sync_dir', 'log sync_dir operations'),
-    LogStat('log_writes', 'log write operations'),
-    LogStat('log_write_lsn', 'log server thread advances write LSN'),
-
+    LogStat('log_slot_closes', 'consolidated slot closures'),
     LogStat('log_slot_coalesced', 'written slots coalesced'),
     LogStat('log_slot_consolidated', 'logging bytes consolidated'),
-    LogStat('log_slot_closes', 'consolidated slot closures'),
     LogStat('log_slot_joins', 'consolidated slot joins'),
     LogStat('log_slot_races', 'consolidated slot join races'),
     LogStat('log_slot_toobig', 'record size exceeded maximum'),
     LogStat('log_slot_transitions', 'consolidated slot join transitions'),
+    LogStat('log_sync', 'log sync operations'),
+    LogStat('log_sync_dir', 'log sync_dir operations'),
+    LogStat('log_write_lsn', 'log server thread advances write LSN'),
+    LogStat('log_writes', 'log write operations'),
 
     ##########################################
     # Reconciliation statistics
@@ -278,6 +285,8 @@ def __init__(self, name, desc, flags=''):
     TxnStat('txn_checkpoint', 'transaction checkpoints'),
     TxnStat('txn_checkpoint_generation',
         'transaction checkpoint generation', 'no_clear,no_scale'),
+    TxnStat('txn_checkpoint_running',
+        'transaction checkpoint currently running', 'no_clear,no_scale'),
     TxnStat('txn_checkpoint_time_max',
         'transaction checkpoint max time (msecs)', 'no_clear,no_scale'),
     TxnStat('txn_checkpoint_time_min',
@@ -286,17 +295,16 @@ def __init__(self, name, desc, flags=''):
         'transaction checkpoint most recent time (msecs)', 'no_clear,no_scale'),
     TxnStat('txn_checkpoint_time_total',
         'transaction checkpoint total time (msecs)', 'no_clear,no_scale'),
-    TxnStat('txn_checkpoint_running',
-        'transaction checkpoint currently running', 'no_clear,no_scale'),
+    TxnStat('txn_commit', 'transactions committed'),
+    TxnStat('txn_fail_cache',
+        'transaction failures due to cache overflow'),
     TxnStat('txn_pinned_checkpoint_range',
         'transaction range of IDs currently pinned by a checkpoint',
-        'no_clear,no_scale'),
+            'no_clear,no_scale'),
     TxnStat('txn_pinned_range',
         'transaction range of IDs currently pinned', 'no_clear,no_scale'),
-    TxnStat('txn_sync', 'transaction sync calls'),
-    TxnStat('txn_commit', 'transactions committed'),
-    TxnStat('txn_fail_cache', 'transaction failures due to cache overflow'),
     TxnStat('txn_rollback', 'transactions rolled back'),
+    TxnStat('txn_sync', 'transaction sync calls'),
 
     ##########################################
     # LSM statistics
@@ -433,9 +441,9 @@ def __init__(self, name, desc, flags=''):
     ##########################################
     # Block manager statistics
     ##########################################
-    BlockStat('block_alloc', 'blocks allocated'),
     BlockStat('allocation_size',
         'file allocation unit size', 'no_aggregate,no_scale'),
+    BlockStat('block_alloc', 'blocks allocated'),
     BlockStat('block_checkpoint_size', 'checkpoint size', 'no_scale'),
     BlockStat('block_extension', 'allocations requiring file extension'),
     BlockStat('block_free', 'blocks freed'),
@@ -465,17 +473,23 @@ def __init__(self, name, desc, flags=''):
     CacheStat('cache_overflow_value',
         'overflow values cached in memory', 'no_scale'),
     CacheStat('cache_read', 'pages read into cache'),
+    CacheStat('cache_read_lookaside',
+        'pages read into cache requiring lookaside entries'),
     CacheStat('cache_read_overflow', 'overflow pages read into cache'),
     CacheStat('cache_write', 'pages written from cache'),
+    CacheStat('cache_write_lookaside',
+        'page written requiring lookaside records'),
+    CacheStat('cache_write_restore',
+        'pages written requiring in-memory restoration'),
 
     ##########################################
     # Compression statistics
     ##########################################
-    CompressStat('compress_raw_ok', 'raw compression call succeeded'),
     CompressStat('compress_raw_fail',
         'raw compression call failed, no additional data available'),
     CompressStat('compress_raw_fail_temporary',
         'raw compression call failed, additional data available'),
+    CompressStat('compress_raw_ok', 'raw compression call succeeded'),
     CompressStat('compress_read', 'compressed pages read'),
     CompressStat('compress_write', 'compressed pages written'),
     CompressStat('compress_write_fail', 'page written failed to compress'),
@@ -486,21 +500,21 @@ def __init__(self, name, desc, flags=''):
     # Reconciliation statistics
     ##########################################
     RecStat('rec_dictionary', 'dictionary matches'),
+    RecStat('rec_multiblock_internal', 'internal page multi-block writes'),
+    RecStat('rec_multiblock_leaf', 'leaf page multi-block writes'),
+    RecStat('rec_multiblock_max',
+        'maximum blocks required for a page', 'max_aggregate,no_scale'),
     RecStat('rec_overflow_key_internal', 'internal-page overflow keys'),
     RecStat('rec_overflow_key_leaf', 'leaf-page overflow keys'),
     RecStat('rec_overflow_value', 'overflow values written'),
-    RecStat('rec_page_match', 'page checksum matches'),
     RecStat('rec_page_delete', 'pages deleted'),
+    RecStat('rec_page_match', 'page checksum matches'),
     RecStat('rec_pages', 'page reconciliation calls'),
     RecStat('rec_pages_eviction', 'page reconciliation calls for eviction'),
     RecStat('rec_prefix_compression',
         'leaf page key bytes discarded using prefix compression'),
     RecStat('rec_suffix_compression',
         'internal page key bytes discarded using suffix compression'),
-    RecStat('rec_multiblock_internal', 'internal page multi-block writes'),
-    RecStat('rec_multiblock_leaf', 'leaf page multi-block writes'),
-    RecStat('rec_multiblock_max',
-        'maximum blocks required for a page', 'max_aggregate,no_scale'),
 
     ##########################################
     # Transaction statistics

examples/c/ex_log.c

@@ -214,11 +214,13 @@ walk_log(WT_SESSION *session)
 
 		/*
 		 * If the operation is a put, replay it here on the backup
-		 * connection.  Note, we cheat by looking only for fileid 1
-		 * in this example.  The metadata is fileid 0.
+		 * connection.
+		 *
+		 * !!!
+		 * Minor cheat: the metadata is fileid 0, skip its records.
 		 */
-		if (fileid == 1 && rectype == WT_LOGREC_COMMIT &&
-		    optype == WT_LOGOP_ROW_PUT) {
+		if (fileid != 0 &&
+		    rectype == WT_LOGREC_COMMIT && optype == WT_LOGOP_ROW_PUT) {
 			if (!in_txn) {
 				ret = session2->begin_transaction(session2,
 				    NULL);

src/async/async_op.c

@@ -237,7 +237,7 @@ __async_op_init(WT_CONNECTION_IMPL *conn, WT_ASYNC_OP_IMPL *op, uint32_t id)
 	asyncop->c.set_key = __wt_cursor_set_key;
 	asyncop->c.get_value = __wt_cursor_get_value;
 	asyncop->c.set_value = __wt_cursor_set_value;
-	asyncop->c.recno = 0;
+	asyncop->c.recno = WT_RECNO_OOB;
 	memset(asyncop->c.raw_recno_buf, 0, sizeof(asyncop->c.raw_recno_buf));
 	memset(&asyncop->c.key, 0, sizeof(asyncop->c.key));
 	memset(&asyncop->c.value, 0, sizeof(asyncop->c.value));