Try to use posix_fadvise with CBufferedFile

This primarily affects blocks when bitcoin is launched with -reindex, as
that causes the block files to be loaded as CBufferedFile objects one at
a time as the reindex progresses.

src/streams.h

@@ -8,6 +8,7 @@
 
 #include <support/allocators/zeroafterfree.h>
 #include <serialize.h>
+#include <util.h>
 
 #include <algorithm>
 #include <assert.h>
@@ -593,10 +594,8 @@ class CBufferedFile
 
 public:
     CBufferedFile(FILE *fileIn, uint64_t nBufSize, uint64_t nRewindIn, int nTypeIn, int nVersionIn) :
-        nType(nTypeIn), nVersion(nVersionIn), nSrcPos(0), nReadPos(0), nReadLimit((uint64_t)(-1)), nRewind(nRewindIn), vchBuf(nBufSize, 0)
-    {
-        src = fileIn;
-    }
+        nType(nTypeIn), nVersion(nVersionIn), src(AdviseSequential(fileIn)), nSrcPos(0), nReadPos(0), nReadLimit((uint64_t)(-1)), nRewind(nRewindIn), vchBuf(nBufSize, 0)
+    {}
 
     ~CBufferedFile()
     {
@@ -613,7 +612,7 @@ class CBufferedFile
     void fclose()
     {
         if (src) {
-            ::fclose(src);
+            CloseAndDiscard(src);
             src = nullptr;
         }
     }

src/util.cpp

@@ -18,16 +18,6 @@
 #endif
 
 #ifndef WIN32
-// for posix_fallocate
-#ifdef __linux__
-
-#ifdef _POSIX_C_SOURCE
-#undef _POSIX_C_SOURCE
-#endif
-
-#define _POSIX_C_SOURCE 200112L
-
-#endif // __linux__
 
 #include <algorithm>
 #include <fcntl.h>
@@ -1098,8 +1088,9 @@ void AllocateFileRange(FILE *file, unsigned int offset, unsigned int length) {
         fcntl(fileno(file), F_PREALLOCATE, &fst);
     }
     ftruncate(fileno(file), fst.fst_length);
-#elif defined(__linux__)
-    // Version using posix_fallocate
+#elif _POSIX_C_SOURCE >= 200112L
+    // Use posix_fallocate to advise the kernel how much data we have to write,
+    // if this system supports it.
     off_t nEndPos = (off_t)offset + length;
     posix_fallocate(fileno(file), 0, nEndPos);
 #else
@@ -1119,6 +1110,49 @@ void AllocateFileRange(FILE *file, unsigned int offset, unsigned int length) {
 #endif
 }
 
+FILE* AdviseSequential(FILE *file) {
+#if _POSIX_C_SOURCE >= 200112L
+    // Since this whole thing is advisory anyway, we can ignore any errors
+    // encountered up to and including the posix_fadvise call. However, we must
+    // rewind the file to the appropriate position if we've changed the seek
+    // offset.
+    if (file == nullptr)
+        return nullptr;
+    int fd = fileno(file);
+    if (fd == -1)
+        return file;
+    off_t start = lseek(fd, 0, SEEK_CUR);
+    if (start == -1)
+        return file;
+    off_t end = lseek(fd, 0, SEEK_END);
+    if (end != -1) {
+        posix_fadvise(fd, start, end - start, POSIX_FADV_WILLNEED);
+        posix_fadvise(fd, start, end - start, POSIX_FADV_SEQUENTIAL);
+    }
+    lseek(fd, start, SEEK_SET);
+#endif
+    return file;
+}
+
+int CloseAndDiscard(FILE *file) {
+#if _POSIX_C_SOURCE >= 200112L
+    // Ignore any errors up to and including the posix_fadvise call since it's
+    // advisory.
+    if (file != nullptr) {
+        off_t end;
+        int fd = fileno(file);
+        if (fd == -1)
+            goto close;
+        end = lseek(fd, 0, SEEK_END);
+        if (end == -1)
+            goto close;
+        posix_fadvise(fd, 0, end, POSIX_FADV_DONTNEED);
+    }
+#endif
+ close:
+    return fclose(file);
+}
+
 #ifdef WIN32
 fs::path GetSpecialFolderPath(int nFolder, bool fCreate)
 {

src/util.h

@@ -79,6 +79,15 @@ bool RenameOver(fs::path src, fs::path dest);
 bool LockDirectory(const fs::path& directory, const std::string lockfile_name, bool probe_only=false);
 bool DirIsWritable(const fs::path& directory);
 
+//! Return the original FILE* unchanged. On POSIX systems that support it,
+//! also advise the kernel that the file will be accessed sequentially.
+FILE* AdviseSequential(FILE *file);
+
+//! Close a file and return the result of fclose(). On POSIX systems that
+//! support it, advise the kernel to remove the file contents from the page
+//! cache (which can help on memory-constrained systems).
+int CloseAndDiscard(FILE *file);
+
 /** Release all directory locks. This is used for unit testing only, at runtime
  * the global destructor will take care of the locks.
  */