Print extra info in exception message for "no space left on device"

dbms/src/Common/CounterInFile.h

@@ -67,13 +67,13 @@ class CounterInFile
 
         int fd = ::open(path.c_str(), O_RDWR | O_CREAT, 0666);
         if (-1 == fd)
-            DB::throwFromErrno("Cannot open file " + path, DB::ErrorCodes::CANNOT_OPEN_FILE);
+            DB::throwFromErrnoWithPath("Cannot open file " + path, path, DB::ErrorCodes::CANNOT_OPEN_FILE);
 
         try
         {
             int flock_ret = flock(fd, LOCK_EX);
             if (-1 == flock_ret)
-                DB::throwFromErrno("Cannot lock file " + path, DB::ErrorCodes::CANNOT_OPEN_FILE);
+                DB::throwFromErrnoWithPath("Cannot lock file " + path, path, DB::ErrorCodes::CANNOT_OPEN_FILE);
 
             if (!file_doesnt_exists)
             {
@@ -141,7 +141,7 @@ class CounterInFile
 
         int fd = ::open(path.c_str(), O_RDWR | O_CREAT, 0666);
         if (-1 == fd)
-            DB::throwFromErrno("Cannot open file " + path, DB::ErrorCodes::CANNOT_OPEN_FILE);
+            DB::throwFromErrnoWithPath("Cannot open file " + path, path, DB::ErrorCodes::CANNOT_OPEN_FILE);
 
         try
         {

dbms/src/Common/Exception.cpp

@@ -9,6 +9,9 @@
 #include <IO/ReadBufferFromString.h>
 #include <common/demangle.h>
 #include <Common/config_version.h>
+#include <Common/formatReadable.h>
+#include <Storages/MergeTree/DiskSpaceMonitor.h>
+#include <filesystem>
 
 namespace DB
 {
@@ -52,6 +55,11 @@ void throwFromErrno(const std::string & s, int code, int e)
     throw ErrnoException(s + ", " + errnoToString(code, e), code, e);
 }
 
+void throwFromErrnoWithPath(const std::string & s, const std::string & path, int code, int the_errno)
+{
+    throw ErrnoException(s + ", " + errnoToString(code, the_errno), code, the_errno, path);
+}
+
 void tryLogCurrentException(const char * log_name, const std::string & start_of_message)
 {
     tryLogCurrentException(&Logger::get(log_name), start_of_message);
@@ -68,7 +76,52 @@ void tryLogCurrentException(Poco::Logger * logger, const std::string & start_of_
     }
 }
 
-std::string getCurrentExceptionMessage(bool with_stacktrace, bool check_embedded_stacktrace)
+void getNoSpaceLeftInfoMessage(std::filesystem::path path, std::string & msg)
+{
+    path = std::filesystem::absolute(path);
+    /// It's possible to get ENOSPC for non existent file (e.g. if there are no free inodes and creat() fails)
+    /// So try to get info for existent parent directory.
+    while (!std::filesystem::exists(path) && path.has_relative_path())
+        path = path.parent_path();
+
+    auto fs = DiskSpaceMonitor::getStatVFS(path);
+    msg += "\nTotal space: "      + formatReadableSizeWithBinarySuffix(fs.f_blocks * fs.f_bsize)
+         + "\nAvailable space: "  + formatReadableSizeWithBinarySuffix(fs.f_bavail * fs.f_bsize)
+         + "\nTotal inodes: "     + formatReadableQuantity(fs.f_files)
+         + "\nAvailable inodes: " + formatReadableQuantity(fs.f_favail);
+
+    auto mount_point = DiskSpaceMonitor::getMountPoint(path).string();
+    msg += "\nMount point: " + mount_point;
+#if defined(__linux__)
+    msg += "\nFilesystem: " + DiskSpaceMonitor::getFilesystemName(mount_point);
+#endif
+}
+
+std::string getExtraExceptionInfo(const std::exception & e)
+{
+    String msg;
+    try
+    {
+        if (auto file_exception = dynamic_cast<const Poco::FileException *>(&e))
+        {
+            if (file_exception->code() == ENOSPC)
+                getNoSpaceLeftInfoMessage(file_exception->message(), msg);
+        }
+        else if (auto errno_exception = dynamic_cast<const DB::ErrnoException *>(&e))
+        {
+            if (errno_exception->getErrno() == ENOSPC && errno_exception->getPath())
+                getNoSpaceLeftInfoMessage(errno_exception->getPath().value(), msg);
+        }
+    }
+    catch (...)
+    {
+        msg += "\nCannot print extra info: " + getCurrentExceptionMessage(false, false, false);
+    }
+
+    return msg;
+}
+
+std::string getCurrentExceptionMessage(bool with_stacktrace, bool check_embedded_stacktrace /*= false*/, bool with_extra_info /*= true*/)
 {
     std::stringstream stream;
 
@@ -78,15 +131,18 @@ std::string getCurrentExceptionMessage(bool with_stacktrace, bool check_embedded
     }
     catch (const Exception & e)
     {
-        stream << getExceptionMessage(e, with_stacktrace, check_embedded_stacktrace) << " (version " << VERSION_STRING << VERSION_OFFICIAL << ")";
+        stream << getExceptionMessage(e, with_stacktrace, check_embedded_stacktrace)
+               << (with_extra_info ? getExtraExceptionInfo(e) : "")
+               << " (version " << VERSION_STRING << VERSION_OFFICIAL << ")";
     }
     catch (const Poco::Exception & e)
     {
         try
         {
             stream << "Poco::Exception. Code: " << ErrorCodes::POCO_EXCEPTION << ", e.code() = " << e.code()
                 << ", e.displayText() = " << e.displayText()
-                << " (version " << VERSION_STRING << VERSION_OFFICIAL << ")";
+                << (with_extra_info ? getExtraExceptionInfo(e) : "")
+                << " (version " << VERSION_STRING << VERSION_OFFICIAL;
         }
         catch (...) {}
     }
@@ -100,7 +156,9 @@ std::string getCurrentExceptionMessage(bool with_stacktrace, bool check_embedded
             if (status)
                 name += " (demangling status: " + toString(status) + ")";
 
-            stream << "std::exception. Code: " << ErrorCodes::STD_EXCEPTION << ", type: " << name << ", e.what() = " << e.what() << ", version = " << VERSION_STRING << VERSION_OFFICIAL;
+            stream << "std::exception. Code: " << ErrorCodes::STD_EXCEPTION << ", type: " << name << ", e.what() = " << e.what()
+                   << (with_extra_info ? getExtraExceptionInfo(e) : "")
+                   << ", version = " << VERSION_STRING << VERSION_OFFICIAL;
         }
         catch (...) {}
     }

dbms/src/Common/Exception.h

@@ -52,16 +52,18 @@ class Exception : public Poco::Exception
 class ErrnoException : public Exception
 {
 public:
-    ErrnoException(const std::string & msg, int code, int saved_errno_)
-        : Exception(msg, code), saved_errno(saved_errno_) {}
+    ErrnoException(const std::string & msg, int code, int saved_errno_, const std::optional<std::string> & path_ = {})
+        : Exception(msg, code), saved_errno(saved_errno_), path(path_) {}
 
     ErrnoException * clone() const override { return new ErrnoException(*this); }
     void rethrow() const override { throw *this; }
 
     int getErrno() const { return saved_errno; }
+    const std::optional<std::string> getPath() const { return path; }
 
 private:
     int saved_errno;
+    std::optional<std::string> path;
 
     const char * name() const throw() override { return "DB::ErrnoException"; }
     const char * className() const throw() override { return "DB::ErrnoException"; }
@@ -73,6 +75,8 @@ using Exceptions = std::vector<std::exception_ptr>;
 
 std::string errnoToString(int code, int the_errno = errno);
 [[noreturn]] void throwFromErrno(const std::string & s, int code, int the_errno = errno);
+[[noreturn]] void throwFromErrnoWithPath(const std::string & s, const std::string & path, int code,
+                                         int the_errno = errno);
 
 
 /** Try to write an exception to the log (and forget about it).
@@ -87,7 +91,8 @@ void tryLogCurrentException(Poco::Logger * logger, const std::string & start_of_
   * check_embedded_stacktrace - if DB::Exception has embedded stacktrace then
   *  only this stack trace will be printed.
   */
-std::string getCurrentExceptionMessage(bool with_stacktrace, bool check_embedded_stacktrace = false);
+std::string getCurrentExceptionMessage(bool with_stacktrace, bool check_embedded_stacktrace = false,
+                                       bool with_extra_info = true);
 
 /// Returns error code from ErrorCodes
 int getCurrentExceptionCode();

dbms/src/Common/StatusFile.cpp

@@ -51,7 +51,7 @@ StatusFile::StatusFile(const std::string & path_)
     fd = ::open(path.c_str(), O_WRONLY | O_CREAT, 0666);
 
     if (-1 == fd)
-        throwFromErrno("Cannot open file " + path, ErrorCodes::CANNOT_OPEN_FILE);
+        throwFromErrnoWithPath("Cannot open file " + path, path, ErrorCodes::CANNOT_OPEN_FILE);
 
     try
     {
@@ -61,14 +61,14 @@ StatusFile::StatusFile(const std::string & path_)
             if (errno == EWOULDBLOCK)
                 throw Exception("Cannot lock file " + path + ". Another server instance in same directory is already running.", ErrorCodes::CANNOT_OPEN_FILE);
             else
-                throwFromErrno("Cannot lock file " + path, ErrorCodes::CANNOT_OPEN_FILE);
+                throwFromErrnoWithPath("Cannot lock file " + path, path, ErrorCodes::CANNOT_OPEN_FILE);
         }
 
         if (0 != ftruncate(fd, 0))
-            throwFromErrno("Cannot ftruncate " + path, ErrorCodes::CANNOT_TRUNCATE_FILE);
+            throwFromErrnoWithPath("Cannot ftruncate " + path, path, ErrorCodes::CANNOT_TRUNCATE_FILE);
 
         if (0 != lseek(fd, 0, SEEK_SET))
-            throwFromErrno("Cannot lseek " + path, ErrorCodes::CANNOT_SEEK_THROUGH_FILE);
+            throwFromErrnoWithPath("Cannot lseek " + path, path, ErrorCodes::CANNOT_SEEK_THROUGH_FILE);
 
         /// Write information about current server instance to the file.
         {

dbms/src/Common/createHardLink.cpp

@@ -26,16 +26,19 @@ void createHardLink(const String & source_path, const String & destination_path)
             struct stat destination_descr;
 
             if (0 != lstat(source_path.c_str(), &source_descr))
-                throwFromErrno("Cannot stat " + source_path, ErrorCodes::CANNOT_STAT);
+                throwFromErrnoWithPath("Cannot stat " + source_path, source_path, ErrorCodes::CANNOT_STAT);
 
             if (0 != lstat(destination_path.c_str(), &destination_descr))
-                throwFromErrno("Cannot stat " + destination_path, ErrorCodes::CANNOT_STAT);
+                throwFromErrnoWithPath("Cannot stat " + destination_path, destination_path, ErrorCodes::CANNOT_STAT);
 
             if (source_descr.st_ino != destination_descr.st_ino)
-                throwFromErrno("Destination file " + destination_path + " is already exist and have different inode.", ErrorCodes::CANNOT_LINK, link_errno);
+                throwFromErrnoWithPath(
+                        "Destination file " + destination_path + " is already exist and have different inode.",
+                        destination_path, ErrorCodes::CANNOT_LINK, link_errno);
         }
         else
-            throwFromErrno("Cannot link " + source_path + " to " + destination_path, ErrorCodes::CANNOT_LINK);
+            throwFromErrnoWithPath("Cannot link " + source_path + " to " + destination_path, destination_path,
+                                   ErrorCodes::CANNOT_LINK);
     }
 }
 

dbms/src/IO/MMapReadBufferFromFile.cpp

@@ -29,7 +29,8 @@ void MMapReadBufferFromFile::open(const std::string & file_name)
     fd = ::open(file_name.c_str(), O_RDONLY);
 
     if (-1 == fd)
-        throwFromErrno("Cannot open file " + file_name, errno == ENOENT ? ErrorCodes::FILE_DOESNT_EXIST : ErrorCodes::CANNOT_OPEN_FILE);
+        throwFromErrnoWithPath("Cannot open file " + file_name, file_name,
+                               errno == ENOENT ? ErrorCodes::FILE_DOESNT_EXIST : ErrorCodes::CANNOT_OPEN_FILE);
 }
 
 

dbms/src/IO/ReadBufferAIO.cpp

@@ -54,7 +54,7 @@ ReadBufferAIO::ReadBufferAIO(const std::string & filename_, size_t buffer_size_,
     if (fd == -1)
     {
         auto error_code = (errno == ENOENT) ? ErrorCodes::FILE_DOESNT_EXIST : ErrorCodes::CANNOT_OPEN_FILE;
-        throwFromErrno("Cannot open file " + filename, error_code);
+        throwFromErrnoWithPath("Cannot open file " + filename, filename, error_code);
     }
 }
 

dbms/src/IO/ReadBufferFromFile.cpp

@@ -41,12 +41,13 @@ ReadBufferFromFile::ReadBufferFromFile(
     fd = ::open(file_name.c_str(), flags == -1 ? O_RDONLY : flags);
 
     if (-1 == fd)
-        throwFromErrno("Cannot open file " + file_name, errno == ENOENT ? ErrorCodes::FILE_DOESNT_EXIST : ErrorCodes::CANNOT_OPEN_FILE);
+        throwFromErrnoWithPath("Cannot open file " + file_name, file_name,
+                               errno == ENOENT ? ErrorCodes::FILE_DOESNT_EXIST : ErrorCodes::CANNOT_OPEN_FILE);
 #ifdef __APPLE__
     if (o_direct)
     {
         if (fcntl(fd, F_NOCACHE, 1) == -1)
-            throwFromErrno("Cannot set F_NOCACHE on file " + file_name, ErrorCodes::CANNOT_OPEN_FILE);
+            throwFromErrno("Cannot set F_NOCACHE on file " + file_name, file_name, ErrorCodes::CANNOT_OPEN_FILE);
     }
 #endif
 }

dbms/src/IO/ReadBufferFromFileDescriptor.cpp

@@ -61,7 +61,8 @@ bool ReadBufferFromFileDescriptor::nextImpl()
         if (-1 == res && errno != EINTR)
         {
             ProfileEvents::increment(ProfileEvents::ReadBufferFromFileDescriptorReadFailed);
-            throwFromErrno("Cannot read from file " + getFileName(), ErrorCodes::CANNOT_READ_FROM_FILE_DESCRIPTOR);
+            throwFromErrnoWithPath("Cannot read from file " + getFileName(), getFileName(),
+                                   ErrorCodes::CANNOT_READ_FROM_FILE_DESCRIPTOR);
         }
 
         if (res > 0)
@@ -124,7 +125,8 @@ off_t ReadBufferFromFileDescriptor::doSeek(off_t offset, int whence)
         pos = working_buffer.end();
         off_t res = ::lseek(fd, new_pos, SEEK_SET);
         if (-1 == res)
-            throwFromErrno("Cannot seek through file " + getFileName(), ErrorCodes::CANNOT_SEEK_THROUGH_FILE);
+            throwFromErrnoWithPath("Cannot seek through file " + getFileName(), getFileName(),
+                                   ErrorCodes::CANNOT_SEEK_THROUGH_FILE);
         pos_in_file = new_pos;
 
         watch.stop();

dbms/src/IO/WriteBufferAIO.cpp

@@ -62,7 +62,7 @@ WriteBufferAIO::WriteBufferAIO(const std::string & filename_, size_t buffer_size
     if (fd == -1)
     {
         auto error_code = (errno == ENOENT) ? ErrorCodes::FILE_DOESNT_EXIST : ErrorCodes::CANNOT_OPEN_FILE;
-        throwFromErrno("Cannot open file " + filename, error_code);
+        throwFromErrnoWithPath("Cannot open file " + filename, filename, error_code);
     }
 }
 
@@ -96,7 +96,7 @@ void WriteBufferAIO::sync()
     /// Ask OS to flush data to disk.
     int res = ::fsync(fd);
     if (res == -1)
-        throwFromErrno("Cannot fsync " + getFileName(), ErrorCodes::CANNOT_FSYNC);
+        throwFromErrnoWithPath("Cannot fsync " + getFileName(), getFileName(), ErrorCodes::CANNOT_FSYNC);
 }
 
 void WriteBufferAIO::nextImpl()
@@ -173,7 +173,7 @@ void WriteBufferAIO::doTruncate(off_t length)
 
     int res = ::ftruncate(fd, length);
     if (res == -1)
-        throwFromErrno("Cannot truncate file " + filename, ErrorCodes::CANNOT_TRUNCATE_FILE);
+        throwFromErrnoWithPath("Cannot truncate file " + filename, filename, ErrorCodes::CANNOT_TRUNCATE_FILE);
 }
 
 void WriteBufferAIO::flush()
@@ -427,7 +427,7 @@ void WriteBufferAIO::finalize()
         /// Truncate the file to remove unnecessary zeros from it.
         int res = ::ftruncate(fd, max_pos_in_file);
         if (res == -1)
-            throwFromErrno("Cannot truncate file " + filename, ErrorCodes::CANNOT_TRUNCATE_FILE);
+            throwFromErrnoWithPath("Cannot truncate file " + filename, filename, ErrorCodes::CANNOT_TRUNCATE_FILE);
     }
 }
 

dbms/src/IO/WriteBufferFromFile.cpp

@@ -44,13 +44,14 @@ WriteBufferFromFile::WriteBufferFromFile(
     fd = ::open(file_name.c_str(), flags == -1 ? O_WRONLY | O_TRUNC | O_CREAT : flags, mode);
 
     if (-1 == fd)
-        throwFromErrno("Cannot open file " + file_name, errno == ENOENT ? ErrorCodes::FILE_DOESNT_EXIST : ErrorCodes::CANNOT_OPEN_FILE);
+        throwFromErrnoWithPath("Cannot open file " + file_name, file_name,
+                               errno == ENOENT ? ErrorCodes::FILE_DOESNT_EXIST : ErrorCodes::CANNOT_OPEN_FILE);
 
 #ifdef __APPLE__
     if (o_direct)
     {
         if (fcntl(fd, F_NOCACHE, 1) == -1)
-            throwFromErrno("Cannot set F_NOCACHE on file " + file_name, ErrorCodes::CANNOT_OPEN_FILE);
+            throwFromErrno("Cannot set F_NOCACHE on file " + file_name, file_name, ErrorCodes::CANNOT_OPEN_FILE);
     }
 #endif
 }

dbms/src/IO/WriteBufferFromFileDescriptor.cpp

@@ -56,7 +56,8 @@ void WriteBufferFromFileDescriptor::nextImpl()
         if ((-1 == res || 0 == res) && errno != EINTR)
         {
             ProfileEvents::increment(ProfileEvents::WriteBufferFromFileDescriptorWriteFailed);
-            throwFromErrno("Cannot write to file " + getFileName(), ErrorCodes::CANNOT_WRITE_TO_FILE_DESCRIPTOR);
+            throwFromErrnoWithPath("Cannot write to file " + getFileName(), getFileName(),
+                                   ErrorCodes::CANNOT_WRITE_TO_FILE_DESCRIPTOR);
         }
 
         if (res > 0)
@@ -111,15 +112,16 @@ void WriteBufferFromFileDescriptor::sync()
     /// Request OS to sync data with storage medium.
     int res = fsync(fd);
     if (-1 == res)
-        throwFromErrno("Cannot fsync " + getFileName(), ErrorCodes::CANNOT_FSYNC);
+        throwFromErrnoWithPath("Cannot fsync " + getFileName(), getFileName(), ErrorCodes::CANNOT_FSYNC);
 }
 
 
 off_t WriteBufferFromFileDescriptor::doSeek(off_t offset, int whence)
 {
     off_t res = lseek(fd, offset, whence);
     if (-1 == res)
-        throwFromErrno("Cannot seek through file " + getFileName(), ErrorCodes::CANNOT_SEEK_THROUGH_FILE);
+        throwFromErrnoWithPath("Cannot seek through file " + getFileName(), getFileName(),
+                               ErrorCodes::CANNOT_SEEK_THROUGH_FILE);
     return res;
 }
 
@@ -128,7 +130,7 @@ void WriteBufferFromFileDescriptor::doTruncate(off_t length)
 {
     int res = ftruncate(fd, length);
     if (-1 == res)
-        throwFromErrno("Cannot truncate file " + getFileName(), ErrorCodes::CANNOT_TRUNCATE_FILE);
+        throwFromErrnoWithPath("Cannot truncate file " + getFileName(), getFileName(), ErrorCodes::CANNOT_TRUNCATE_FILE);
 }
 
 }

dbms/src/IO/WriteBufferFromTemporaryFile.cpp

@@ -39,7 +39,8 @@ class ReadBufferFromTemporaryWriteBuffer : public ReadBufferFromFile
 
         off_t res = lseek(fd, 0, SEEK_SET);
         if (-1 == res)
-            throwFromErrno("Cannot reread temporary file " + file_name, ErrorCodes::CANNOT_SEEK_THROUGH_FILE);
+            throwFromErrnoWithPath("Cannot reread temporary file " + file_name, file_name,
+                                   ErrorCodes::CANNOT_SEEK_THROUGH_FILE);
 
         return std::make_shared<ReadBufferFromTemporaryWriteBuffer>(fd, file_name, std::move(origin->tmp_file));
     }

dbms/src/Storages/Distributed/DistributedBlockOutputStream.cpp

@@ -565,7 +565,8 @@ void DistributedBlockOutputStream::writeToShard(const Block & block, const std::
         }
 
         if (link(first_file_tmp_path.data(), block_file_path.data()))
-            throwFromErrno("Could not link " + block_file_path + " to " + first_file_tmp_path, ErrorCodes::CANNOT_LINK);
+            throwFromErrnoWithPath("Could not link " + block_file_path + " to " + first_file_tmp_path, block_file_path,
+                                   ErrorCodes::CANNOT_LINK);
     }
 
     /** remove the temporary file, enabling the OS to reclaim inode after all threads