log: do not repeat errors to stderr

If we get an error writing to the log, log it only once to stderr.
This avoids generating, say, 72 GB of ENOSPC errors in
teuthology.log when /var/log fills up.

Fixes: #14616
Signed-off-by: Sage Weil <sage@redhat.com>

src/log/Log.cc

@@ -53,6 +53,7 @@ Log::Log(SubsystemMap *s)
     m_fd(-1),
     m_uid(0),
     m_gid(0),
+    m_fd_last_error(0),
     m_syslog_log(-2), m_syslog_crash(-2),
     m_stderr_log(1), m_stderr_crash(-1),
     m_graylog_log(-3), m_graylog_crash(-3),
@@ -337,9 +338,15 @@ void Log::_flush(EntryQueue *t, EntryQueue *requeue, bool crash)
       if (do_fd) {
         buf[buflen] = '\n';
         int r = safe_write(m_fd, buf, buflen+1);
-        if (r < 0)
-          cerr << "problem writing to " << m_log_file << ": " << cpp_strerror(r)
-	       << std::endl;
+	if (r > 0)
+	  r = 0;
+	if (r != m_fd_last_error) {
+	  if (r < 0)
+	    cerr << "problem writing to " << m_log_file
+		 << ": " << cpp_strerror(r)
+		 << std::endl;
+	  m_fd_last_error = r;
+	}
       }
       if (need_dynamic)
         delete[] buf;

src/log/Log.h

@@ -39,6 +39,8 @@ class Log : private Thread
   uid_t m_uid;
   gid_t m_gid;
 
+  int m_fd_last_error;  ///< last error we say writing to fd (if any)
+
   int m_syslog_log, m_syslog_crash;
   int m_stderr_log, m_stderr_crash;
   int m_graylog_log, m_graylog_crash;