WT-9368 Extend WT fault injection testing to support LazyFS. (#8797)

* WT-9368 Extend WT fault injection testing to support LazyFS.

* Integrated the LazyFS version of random_abort into cmake. A few small fixes.

* Refactored common LazyFS functionality. Added LazyFS support to two more tests.

* Miscellaneous fixes.

* Fixed a few Windows-related issues.

* Fixed usage().

* Refactored higher-level LazyFS logic into testutil_lazyfs_* functions.

* Fixed a build issue on Windows.

CMakeLists.txt

@@ -52,6 +52,7 @@ include(cmake/configs/base.cmake)
 include(cmake/third_party/aws_sdk.cmake)
 include(cmake/third_party/gcp_sdk.cmake)
 include(cmake/third_party/gperftools.cmake)
+include(cmake/third_party/lazyfs.cmake)
 include(cmake/third_party/lz4.cmake)
 include(cmake/third_party/memkind.cmake)
 include(cmake/third_party/snappy.cmake)

cmake/configs/base.cmake

@@ -281,6 +281,12 @@ config_bool(
     DEFAULT ON
 )
 
+config_bool(
+    ENABLE_LAZYFS
+    "Build LazyFS for testing"
+    DEFAULT OFF
+)
+
 config_bool(
     ENABLE_S3
     "Build the S3 storage extension"

cmake/third_party/lazyfs.cmake

@@ -0,0 +1,25 @@
+include(ExternalProject)
+include(GNUInstallDirs)
+include(${CMAKE_SOURCE_DIR}/cmake/helpers.cmake)
+
+# Skip the LazyFS build step if it is not enabled.
+if(NOT ENABLE_LAZYFS)
+    return()
+endif()
+
+if(TARGET lazyfs)
+    # Avoid redefining the imported library, given this file can be used as an include.
+    return()
+endif()
+
+# Download and install the project into the build directory.
+ExternalProject_Add(lazyfs
+    PREFIX lazyfs
+    GIT_CONFIG advice.detachedHead=false 
+    GIT_REPOSITORY https://github.com/dsrhaslab/lazyfs.git
+    GIT_TAG b0383127
+    CONFIGURE_COMMAND ""
+    BUILD_IN_SOURCE TRUE
+    BUILD_COMMAND cd libs/libpcache && ./build.sh COMMAND cd ../../lazyfs && ./build.sh
+    INSTALL_COMMAND ""
+)

dist/s_string.ok

@@ -37,7 +37,7 @@ BBBBBBBBBB
 BBBBBBBBBBBBB
 BBBBBBBBBBBBBBBB
 BBBBBBBBBBBBBBBBBB
-BCmvxz
+BClmvxz
 BDB
 BDB's
 BDmpvXx
@@ -87,6 +87,8 @@ CURSORs
 CURSTD
 CallsCustDate
 Castagnoli
+CcLlsvz
+Cclmpv
 CentOS
 Checkpointing
 Checksum
@@ -95,7 +97,6 @@ CityHash
 CloseHandle
 CmP
 CmPTh
-Cmvz
 Collet
 Comparator
 Compressibility
@@ -276,6 +277,8 @@ LTE
 LWN
 LZ
 LZO
+LazyFS
+LazyFS's
 LeafGreen
 LevelDB
 Levyx
@@ -351,7 +354,9 @@ Opcode
 Optane
 Outfmt
 PARAM
+PDEATHSIG
 PFX
+PID
 POS
 POSIX
 PPC
@@ -457,6 +462,7 @@ Stoica
 StoreLoad
 StoreStore
 Su
+Subdirectory
 Syscall
 TAILQ
 TCMalloc
@@ -501,6 +507,7 @@ UnlockFile
 Unmap
 UnmapViewOfFile
 Unmarshall
+Unmount
 Unordered
 Uryyb
 VALGRIND
@@ -919,6 +926,7 @@ fgetln
 fgets
 fh
 fhandle
+fifo
 fileX
 filefrag
 filehandle
@@ -963,6 +971,7 @@ fuzzer
 fuzzutil
 fvisibility
 fwrite
+gb
 gc
 gcc
 gdb
@@ -1104,6 +1113,7 @@ kvs
 lang
 las
 latencies
+lazyfs
 lbrace
 lbracket
 ld
@@ -1124,6 +1134,7 @@ libsodium
 libtool
 libwiredtiger
 linkers
+linux
 liveness
 llll
 llu
@@ -1322,6 +1333,7 @@ pclose
 pcpu
 perf
 pfx
+pid
 pluggable
 pmem
 poc
@@ -1353,6 +1365,7 @@ primary's
 printf
 printlog
 priv
+proc
 prog
 progname
 programmatically
@@ -1499,6 +1512,8 @@ strtouq
 struct
 structs
 su
+subdir
+subdirectories
 subdirectory
 subget
 subgetraw
@@ -1507,6 +1522,7 @@ subinit
 sublicense
 subone
 suboptimal
+subprocess
 subsystem's
 subtest
 subtree
@@ -1552,6 +1568,7 @@ tokstart
 toktype
 tolower
 tombstoned
+toml
 toolchain
 toplevel
 totalsec
@@ -1585,6 +1602,7 @@ uS
 ui
 uint
 uintmax
+umount
 unbare
 unbuffered
 uncacheable
@@ -1612,6 +1630,7 @@ unmarshall
 unmarshalled
 unmerged
 unmodify
+unmount
 unordered
 unpackv
 unpadded

test/csuite/CMakeLists.txt

@@ -22,6 +22,15 @@ define_c_test(
     DEPENDS "WT_POSIX"
 )
 
+define_c_test(
+    TARGET test_random_abort_lazyfs
+    SOURCES random_abort/main.c
+    DIR_NAME random_abort
+    EXEC_SCRIPT ${CMAKE_CURRENT_SOURCE_DIR}/random_abort/smoke_lazyfs.sh
+    ARGUMENTS $<TARGET_FILE:test_random_abort_lazyfs>
+    DEPENDS "WT_POSIX" "ENABLE_LAZYFS"
+)
+
 define_c_test(
     TARGET test_random_directio
     SOURCES
@@ -52,6 +61,15 @@ define_c_test(
     DEPENDS "WT_POSIX"
 )
 
+define_c_test(
+    TARGET test_schema_abort_lazyfs
+    SOURCES schema_abort/main.c
+    DIR_NAME schema_abort
+    EXEC_SCRIPT ${CMAKE_CURRENT_SOURCE_DIR}/schema_abort/smoke_lazyfs.sh
+    ARGUMENTS $<TARGET_FILE:test_schema_abort_lazyfs>
+    DEPENDS "WT_POSIX" "ENABLE_LAZYFS"
+)
+
 define_c_test(
     TARGET test_scope
     SOURCES scope/main.c
@@ -91,6 +109,15 @@ define_c_test(
     DEPENDS "WT_POSIX"
 )
 
+define_c_test(
+    TARGET test_timestamp_abort_lazyfs
+    SOURCES timestamp_abort/main.c
+    DIR_NAME timestamp_abort
+    EXEC_SCRIPT ${CMAKE_CURRENT_SOURCE_DIR}/timestamp_abort/smoke_lazyfs.sh
+    ARGUMENTS -b $<TARGET_FILE:test_timestamp_abort_lazyfs>
+    DEPENDS "WT_POSIX" "ENABLE_LAZYFS"
+)
+
 define_c_test(
     TARGET test_truncated_log
     SOURCES truncated_log/main.c

test/csuite/README

@@ -10,3 +10,11 @@ crafted and put into the Evergreen configuration file. We have a utility program
 'test/evergreen/evg_cfg.py' to help with identifying and auto-generating the
 Evergreen configuration for new or missing csuite tests. The program checking
 has been bound into developer workflow through 'dist/s_evergreen'. 
+
+Some csuite tests that check WiredTiger's failure recovery have LazyFS variants,
+which use LazyFS (a FUSE file system) to emulate losing the contents of the buffer
+cache, such as to test the system behavior in face of sudden reboots or power
+failures. To use these tests, you need to have FUSE 3 installed together with the
+development libraries, have 'user_allow_other' enabled in /etc/fuse.conf, and you
+need to also enable LazyFS in cmake via the '-DENABLE_LAZYFS=1' command-line
+option. The tests are not currently available in Evergreen and must be run manually.