WT-6177 direct I/O configurations fail with combined with mmap config…

…urations (#5669)

* Remove trailing newlines from verbose message output.

* Don't allow mmap of any flavor to be configured with direct I/O.

* Don't configure direct I/O with any flavor of mmap.

* config.c:567:9: error: code will never be executed [-Werror,-Wunreachable-code]

* The "mmap" configuration defaults to on, and we have not historically had problems with combining
it with direct I/O. Go back to allowing that combination.

dist/api_data.py

@@ -1001,7 +1001,8 @@ def __ge__(self, other):
         Use memory mapping when accessing files in a read-only mode''',
         type='boolean'),
     Config('mmap_all', 'false', r'''
-        Use memory mapping to read and write all data files''',
+        Use memory mapping to read and write all data files, may not be configured with direct
+        I/O''',
         type='boolean'),
     Config('multiprocess', 'false', r'''
         permit sharing between processes (will automatically start an

src/conn/conn_api.c

@@ -2583,9 +2583,11 @@ wiredtiger_open(const char *home, WT_EVENT_HANDLER *event_handler, const char *c
 
     WT_ERR(__wt_config_gets(session, cfg, "mmap", &cval));
     conn->mmap = cval.val != 0;
-
     WT_ERR(__wt_config_gets(session, cfg, "mmap_all", &cval));
     conn->mmap_all = cval.val != 0;
+    if (conn->direct_io && conn->mmap_all)
+        WT_ERR_MSG(
+          session, EINVAL, "direct I/O configuration is incompatible with mmap_all configuration");
 
     WT_ERR(__wt_config_gets(session, cfg, "operation_timeout_ms", &cval));
     conn->operation_timeout_us = (uint64_t)(cval.val * WT_THOUSAND);

src/include/wiredtiger.in

@@ -2974,8 +2974,8 @@ struct __wt_connection {
  * @config{ ),,}
  * @config{mmap, Use memory mapping when accessing files in a read-only mode., a boolean flag;
  * default \c true.}
- * @config{mmap_all, Use memory mapping to read and write all data files., a boolean flag; default
- * \c false.}
+ * @config{mmap_all, Use memory mapping to read and write all data files\, may not be configured
+ * with direct I/O., a boolean flag; default \c false.}
  * @config{multiprocess, permit sharing between processes (will automatically start an RPC server
  * for primary processes and use RPC for secondary processes). <b>Not yet supported in
  * WiredTiger</b>., a boolean flag; default \c false.}

src/os_posix/os_fs.c

@@ -344,7 +344,7 @@ __posix_file_close(WT_FILE_HANDLE *file_handle, WT_SESSION *wt_session)
     session = (WT_SESSION_IMPL *)wt_session;
     pfh = (WT_FILE_HANDLE_POSIX *)file_handle;
 
-    __wt_verbose(session, WT_VERB_FILEOPS, "%s, file-close: fd=%d\n", file_handle->name, pfh->fd);
+    __wt_verbose(session, WT_VERB_FILEOPS, "%s, file-close: fd=%d", file_handle->name, pfh->fd);
 
     if (pfh->mmap_file_mappable && pfh->mmap_buf != NULL)
         __wt_unmap_file(file_handle, wt_session);
@@ -411,9 +411,7 @@ __posix_file_read(
     session = (WT_SESSION_IMPL *)wt_session;
     pfh = (WT_FILE_HANDLE_POSIX *)file_handle;
 
-    __wt_verbose(session, WT_VERB_READ, "read: %s, fd=%d, offset=%" PRId64
-                                        ","
-                                        "len=%" WT_SIZET_FMT "\n",
+    __wt_verbose(session, WT_VERB_READ, "read: %s, fd=%d, offset=%" PRId64 ", len=%" WT_SIZET_FMT,
       file_handle->name, pfh->fd, offset, len);
 
     /* Assert direct I/O is aligned and a multiple of the alignment. */
@@ -453,9 +451,8 @@ __posix_file_read_mmap(
         return (__posix_file_read(file_handle, wt_session, offset, len, buf));
 
     __wt_verbose(session, WT_VERB_READ,
-      "read-mmap: %s, fd=%d, offset=%" PRId64
-      ","
-      "len=%" WT_SIZET_FMT ", mapped buffer: %p, mapped size = %" PRId64 "\n",
+      "read-mmap: %s, fd=%d, offset=%" PRId64 ", len=%" WT_SIZET_FMT
+      ", mapped buffer: %p, mapped size = %" PRId64,
       file_handle->name, pfh->fd, offset, len, (void *)pfh->mmap_buf, pfh->mmap_size);
 
     /* Indicate that we might be using the mapped area */
@@ -561,10 +558,9 @@ __posix_file_truncate(WT_FILE_HANDLE *file_handle, WT_SESSION *wt_session, wt_of
     session = (WT_SESSION_IMPL *)wt_session;
     pfh = (WT_FILE_HANDLE_POSIX *)file_handle;
 
-    __wt_verbose(session, WT_VERB_FILEOPS, "%s, file-truncate: size=%" PRId64
-                                           ","
-                                           "mapped size=%" PRId64 "\n",
-      file_handle->name, len, pfh->mmap_size);
+    __wt_verbose(session, WT_VERB_FILEOPS,
+      "%s, file-truncate: size=%" PRId64 ", mapped size=%" PRId64, file_handle->name, len,
+      pfh->mmap_size);
 
     remap = (len != pfh->mmap_size);
     if (remap)
@@ -600,9 +596,7 @@ __posix_file_write(
     session = (WT_SESSION_IMPL *)wt_session;
     pfh = (WT_FILE_HANDLE_POSIX *)file_handle;
 
-    __wt_verbose(session, WT_VERB_WRITE, "write: %s, fd=%d, offset=%" PRId64
-                                         ","
-                                         "len=%" WT_SIZET_FMT "\n",
+    __wt_verbose(session, WT_VERB_WRITE, "write: %s, fd=%d, offset=%" PRId64 ", len=%" WT_SIZET_FMT,
       file_handle->name, pfh->fd, offset, len);
 
     /* Assert direct I/O is aligned and a multiple of the alignment. */
@@ -641,9 +635,8 @@ __posix_file_write_mmap(
     pfh = (WT_FILE_HANDLE_POSIX *)file_handle;
 
     __wt_verbose(session, WT_VERB_WRITE,
-      "write-mmap: %s, fd=%d, offset=%" PRId64
-      ","
-      "len=%" WT_SIZET_FMT ", mapped buffer: %p, mapped size = %" PRId64 ".\n",
+      "write-mmap: %s, fd=%d, offset=%" PRId64 ", len=%" WT_SIZET_FMT
+      ", mapped buffer: %p, mapped size = %" PRId64,
       file_handle->name, pfh->fd, offset, len, (void *)pfh->mmap_buf, pfh->mmap_size);
 
     if (!pfh->mmap_file_mappable || pfh->mmap_resizing)
@@ -1024,10 +1017,9 @@ __wt_map_file(WT_FILE_HANDLE *file_handle, WT_SESSION *wt_session)
 
     pfh->mmap_size = file_size;
 
-    __wt_verbose(session, WT_VERB_FILEOPS, "%s: file-mmap: fd=%d, size=%" PRId64
-                                           ", "
-                                           "mapped buffer=%p\n",
-      file_handle->name, pfh->fd, pfh->mmap_size, (void *)pfh->mmap_buf);
+    __wt_verbose(session, WT_VERB_FILEOPS,
+      "%s: file-mmap: fd=%d, size=%" PRId64 ", mapped buffer=%p", file_handle->name, pfh->fd,
+      pfh->mmap_size, (void *)pfh->mmap_buf);
 }
 
 /*
@@ -1063,7 +1055,7 @@ __wt_prepare_remap_resize_file(WT_FILE_HANDLE *file_handle, WT_SESSION *wt_sessi
     if (!pfh->mmap_file_mappable)
         return;
 
-    __wt_verbose(session, WT_VERB_FILEOPS, "%s, prepare-remap-file: buffer=%p\n", file_handle->name,
+    __wt_verbose(session, WT_VERB_FILEOPS, "%s, prepare-remap-file: buffer=%p", file_handle->name,
       (void *)pfh->mmap_buf);
 
 wait:
@@ -1118,7 +1110,7 @@ __wt_remap_resize_file(WT_FILE_HANDLE *file_handle, WT_SESSION *wt_session)
     if (!pfh->mmap_file_mappable)
         return;
 
-    __wt_verbose(session, WT_VERB_FILEOPS, "%s, remap-file: buffer=%p\n", file_handle->name,
+    __wt_verbose(session, WT_VERB_FILEOPS, "%s, remap-file: buffer=%p", file_handle->name,
       (void *)pfh->mmap_buf);
 
     if (pfh->mmap_buf != NULL)
@@ -1145,7 +1137,7 @@ __wt_unmap_file(WT_FILE_HANDLE *file_handle, WT_SESSION *wt_session)
     session = (WT_SESSION_IMPL *)wt_session;
     pfh = (WT_FILE_HANDLE_POSIX *)file_handle;
 
-    __wt_verbose(session, WT_VERB_FILEOPS, "%s, file-unmap: buffer=%p, size=%" PRId64 "\n",
+    __wt_verbose(session, WT_VERB_FILEOPS, "%s, file-unmap: buffer=%p, size=%" PRId64,
       file_handle->name, (void *)pfh->mmap_buf, pfh->mmap_size);
 
     WT_ASSERT(session, pfh->mmap_file_mappable);

test/format/config.c

@@ -552,10 +552,21 @@ config_directio(void)
      */
     if (g.c_backups) {
         if (config_is_perm("backup"))
-            testutil_die(EINVAL, "backup are incompatible with direct I/O");
+            testutil_die(EINVAL, "direct I/O is incompatible with backup configurations");
         config_single("backup=off", false);
     }
 
+    /*
+     * Direct I/O may not work with mmap. Theoretically, Linux ignores direct I/O configurations in
+     * the presence of shared cache configurations (including mmap), but we've seen file corruption
+     * and it doesn't make much sense (the library disallows the combination).
+     */
+    if (g.c_mmap_all != 0) {
+        if (config_is_perm("disk.mmap_all"))
+            testutil_die(EINVAL, "direct I/O is incompatible with mmap_all configurations");
+        config_single("disk.mmap_all=off", false);
+    }
+
     /*
      * Turn off all external programs. Direct I/O is really, really slow on some machines and it can
      * take hours for a job to run. External programs don't have timers running so it looks like
@@ -564,12 +575,12 @@ config_directio(void)
      */
     if (g.c_rebalance) {
         if (config_is_perm("ops.rebalance"))
-            testutil_die(EINVAL, "rebalance is incompatible with direct I/O");
+            testutil_die(EINVAL, "direct I/O is incompatible with rebalance configurations");
         config_single("ops.rebalance=off", false);
     }
     if (g.c_salvage) {
         if (config_is_perm("ops.salvage"))
-            testutil_die(EINVAL, "salvage is incompatible with direct I/O");
+            testutil_die(EINVAL, "direct I/O is incompatible with salvage configurations");
         config_single("ops.salvage=off", false);
     }
 }