close unused file-handles to avoid exceeding rlimit

Chia-Network · Dec 4, 2020 · eba3073 · eba3073
1 parent cc88672
commit eba3073
Show file tree

Hide file tree

Showing 2 changed files with 17 additions and 15 deletions.
diff --git a/src/disk.hpp b/src/disk.hpp
@@ -97,22 +97,29 @@ struct FileDisk {
     explicit FileDisk(const fs::path &filename)
     {
         filename_ = filename;
+        Open(false);
+    }
+
+    void Open(bool read_only = true)
+    {
+        // if the file is already open, don't do anything
+        if (f_) return;
 
         // Opens the file for reading and writing
 #ifdef _WIN32
-        f_ = ::_wfopen(filename.c_str(), L"w+b");
+        f_ = ::_wfopen(filename_.c_str(), read_only ? L"r+b" : L"w+b");
 #else
-        f_ = ::fopen(filename.c_str(), "w+b");
+        f_ = ::fopen(filename_.c_str(), read_only ? "r+b" : "w+b");
 #endif
         if (f_ == nullptr) {
             throw InvalidValueException(
-                "Could not open " + filename.string() + ": " + ::strerror(errno));
+                "Could not open " + filename_.string() + ": " + ::strerror(errno));
         }
     }
 
     FileDisk(FileDisk &&fd)
     {
-        filename_ = fd.filename_;
+        filename_ = std::move(fd.filename_);
         f_ = fd.f_;
         fd.f_ = nullptr;
     }
@@ -131,6 +138,7 @@ struct FileDisk {
 
     void Read(uint64_t begin, uint8_t *memcache, uint64_t length)
     {
+        Open();
 #if ENABLE_LOGGING
         disk_log(filename_, op_t::read, begin, length);
 #endif
@@ -160,6 +168,7 @@ struct FileDisk {
 
     void Write(uint64_t begin, const uint8_t *memcache, uint64_t length)
     {
+        Open();
 #if ENABLE_LOGGING
         disk_log(filename_, op_t::write, begin, length);
 #endif
@@ -198,15 +207,6 @@ struct FileDisk {
     {
         Close();
         fs::resize_file(filename_, new_size);
-#ifdef _WIN32
-        f_ = ::_wfopen(filename_.c_str(), L"r+b");
-#else
-        f_ = ::fopen(filename_.c_str(), "r+b");
-#endif
-        if (f_ == nullptr) {
-            throw InvalidValueException(
-                "Could not open " + filename_.string() + ": " + ::strerror(errno));
-        }
     }
 
 private:
@@ -217,7 +217,7 @@ struct FileDisk {
     bool bReading = true;
 
     fs::path filename_;
-    FILE *f_;
+    FILE *f_ = nullptr;
 };
 
 struct BufferedDisk : Disk

diff --git a/src/sort_manager.hpp b/src/sort_manager.hpp
@@ -128,6 +128,8 @@ class SortManager : public Disk {
     {
         for (auto& b : buckets_) {
             b.file.FreeMemory();
+            // the underlying file will be re-opened again on-demand
+            b.underlying_file.Close();
         }
         prev_bucket_buf_.reset();
         memory_start_.reset();
@@ -229,7 +231,7 @@ class SortManager : public Disk {
         BufferedDisk file;
     };
 
-    // Start of the whole memory array. This will be diveded into buckets
+    // The buffer we use to sort buckets in-memory
     std::unique_ptr<uint8_t[]> memory_start_;
     // Size of the whole memory array
     uint64_t memory_size_;