Enable log statement format overriding in Sinks

CHANGELOG.md

@@ -83,6 +83,11 @@
   LOG_INFO(logger, "{} {}", quill::utility::StringRef{sv}, quill::utility::StringRef{"string_literal"});
   ```
 
+- Renamed `write_log_message` to `write_log` in `Sink`. The formatted `log_message` and `process_id` are now also
+  provided. This enhancement supports use cases where the formatted `log_statement` passed to the `Sink` can be ignored
+  and overwritten with a custom format, allowing a single `Logger` to output different formats to various Sinks.
+  ([#476](https://github.com/odygrd/quill/issues/476))
+
 ## v4.4.1
 
 - Fixed multiple definitions of `quill::detail::get_error_message` ([#469](https://github.com/odygrd/quill/issues/469))

examples/CMakeLists.txt

@@ -1,6 +1,7 @@
 add_subdirectory(recommended_usage)
 add_subdirectory(shared_library)
 add_subdirectory(advanced)
+add_subdirectory(single_logger_multiple_sink_formats)
 
 add_executable(quill_example_backend_thread_notify backend_thread_notify.cpp)
 set_common_compile_options(quill_example_backend_thread_notify)

examples/single_logger_multiple_sink_formats/CMakeLists.txt

@@ -0,0 +1,7 @@
+add_executable(quill_single_logger_multiple_sink_formats_1 single_logger_multiple_sink_formats_1.cpp)
+set_common_compile_options(quill_single_logger_multiple_sink_formats_1)
+target_link_libraries(quill_single_logger_multiple_sink_formats_1 quill)
+
+add_executable(quill_single_logger_multiple_sink_formats_2 single_logger_multiple_sink_formats_2.cpp)
+set_common_compile_options(quill_single_logger_multiple_sink_formats_2)
+target_link_libraries(quill_single_logger_multiple_sink_formats_2 quill)

examples/single_logger_multiple_sink_formats/ConsoleSinkWithFormatter.h

@@ -0,0 +1,38 @@
+#pragma once
+
+#include "quill/backend/PatternFormatter.h"
+#include "quill/core/Attributes.h"
+#include "quill/core/Common.h"
+#include "quill/core/Filesystem.h"
+#include "quill/sinks/ConsoleSink.h"
+
+#include <string>
+#include <string_view>
+
+class ConsoleSinkWithFormatter : public quill::ConsoleSink
+{
+public:
+  ConsoleSinkWithFormatter(std::string const& format_pattern, std::string const& time_format,
+                           quill::Timezone timestamp_timezone = quill::Timezone::LocalTime,
+                           bool enable_colours = true, std::string const& stream = "stdout")
+    : quill::ConsoleSink(enable_colours, stream), _formatter(format_pattern, time_format, timestamp_timezone)
+  {
+  }
+
+  void write_log(quill::MacroMetadata const* log_metadata, uint64_t log_timestamp,
+                 std::string_view thread_id, std::string_view thread_name,
+                 std::string const& process_id, std::string_view logger_name, quill::LogLevel log_level,
+                 std::vector<std::pair<std::string, std::string>> const* named_args,
+                 std::string_view log_message, std::string_view) override
+  {
+    std::string_view const formatted_log_statement =
+      _formatter.format(log_timestamp, thread_id, thread_name, process_id, logger_name,
+                        quill::loglevel_to_string(log_level), *log_metadata, named_args, log_message);
+
+    quill::ConsoleSink::write_log(log_metadata, log_timestamp, thread_id, thread_name, process_id,
+                                  logger_name, log_level, named_args, log_message, formatted_log_statement);
+  }
+
+private:
+  quill::PatternFormatter _formatter;
+};

examples/single_logger_multiple_sink_formats/RotatingFileSinkWithFormatter.h

@@ -0,0 +1,40 @@
+#pragma once
+
+#include "quill/backend/PatternFormatter.h"
+#include "quill/core/Attributes.h"
+#include "quill/core/Common.h"
+#include "quill/core/Filesystem.h"
+#include "quill/sinks/RotatingFileSink.h"
+
+#include <string>
+#include <string_view>
+
+class RotatingFileSinkWithFormatter : public quill::RotatingFileSink
+{
+public:
+  RotatingFileSinkWithFormatter(quill::fs::path const& filename, quill::RotatingFileSinkConfig const& config,
+                                std::string const& format_pattern, std::string const& time_format,
+                                quill::Timezone timestamp_timezone = quill::Timezone::LocalTime,
+                                quill::FileEventNotifier file_event_notifier = quill::FileEventNotifier{})
+    : quill::RotatingFileSink(filename, config, file_event_notifier),
+      _formatter(format_pattern, time_format, timestamp_timezone)
+  {
+  }
+
+  void write_log(quill::MacroMetadata const* log_metadata, uint64_t log_timestamp, std::string_view thread_id,
+                                     std::string_view thread_name, std::string const& process_id,
+                                     std::string_view logger_name, quill::LogLevel log_level,
+                                     std::vector<std::pair<std::string, std::string>> const* named_args,
+                                     std::string_view log_message, std::string_view) override
+  {
+    std::string_view const formatted_log_statement =
+      _formatter.format(log_timestamp, thread_id, thread_name, process_id, logger_name,
+                        quill::loglevel_to_string(log_level), *log_metadata, named_args, log_message);
+
+    quill::RotatingFileSink::write_log(log_metadata, log_timestamp, thread_id, thread_name, process_id, logger_name,
+                                       log_level, named_args, log_message, formatted_log_statement);
+  }
+
+private:
+  quill::PatternFormatter _formatter;
+};

examples/single_logger_multiple_sink_formats/single_logger_multiple_sink_formats_1.cpp

@@ -0,0 +1,57 @@
+#include "quill/Backend.h"
+#include "quill/Frontend.h"
+#include "quill/LogMacros.h"
+#include "quill/Logger.h"
+
+#include "RotatingFileSinkWithFormatter.h"
+#include "quill/sinks/ConsoleSink.h"
+
+/**
+ * This example demonstrates how to use a single Logger to output different formats for each Sink.
+ * While the Logger class accepts a single format pattern, you can override this by creating a custom Sink with your own format pattern.
+ */
+int main()
+{
+  // Start the backend thread
+  quill::BackendOptions backend_options;
+  quill::Backend::start(backend_options);
+
+  // Console sink
+  auto console_sink = quill::Frontend::create_or_get_sink<quill::ConsoleSink>("sink_id_1");
+  console_sink->set_log_level_filter(quill::LogLevel::Warning);
+  std::string console_log_pattern =
+    "%(time) [PID %(process_id)] [%(log_level)] [%(logger)] - %(message)";
+  std::string console_time_format = "%Y-%m-%d %H:%M:%S.%Qms";
+
+  // File sink
+  std::string const file_log_pattern = "%(log_level);%(time);%(logger);%(message)";
+  std::string const file_time_format = "%Y%m%dT%H:%M:%S.%Qus";
+
+  auto rotating_file_sink = quill::Frontend::create_or_get_sink<RotatingFileSinkWithFormatter>(
+    "rotating_file.log",
+    []()
+    {
+      // See RotatingFileSinkConfig for more options
+      quill::RotatingFileSinkConfig cfg;
+      cfg.set_open_mode('a');
+      cfg.set_max_backup_files(10);
+      cfg.set_rotation_max_file_size(1024 * 1024);
+      return cfg;
+    }(),
+    file_log_pattern, file_time_format);
+  rotating_file_sink->set_log_level_filter(quill::LogLevel::Info);
+
+  // The Logger is using the console_log_pattern by default
+  // To output our custom format to the file we use our own RotatingFileSinkWithFormatter that is
+  // overwriting the default format
+  quill::Logger* logger = quill::Frontend::create_or_get_logger(
+    "root", {std::move(console_sink), std::move(rotating_file_sink)}, console_log_pattern, console_time_format);
+
+  logger->set_log_level(quill::LogLevel::Debug);
+
+  LOG_INFO(logger, "This is a log info example {}", sizeof(std::string));
+  LOG_WARNING(logger, "This is a log warning example {}", sizeof(std::string));
+  LOG_ERROR(logger, "This is a log error example {}", sizeof(std::string));
+  LOG_CRITICAL(logger, "This is a log critical example {}", sizeof(std::string));
+  return 0;
+}

examples/single_logger_multiple_sink_formats/single_logger_multiple_sink_formats_2.cpp

@@ -0,0 +1,57 @@
+#include "quill/Backend.h"
+#include "quill/Frontend.h"
+#include "quill/LogMacros.h"
+#include "quill/Logger.h"
+
+#include "ConsoleSinkWithFormatter.h"
+#include "quill/sinks/RotatingFileSink.h"
+
+/**
+ * This example demonstrates how to use a single Logger to output different formats for each Sink.
+ * While the Logger class accepts a single format pattern, you can override this by creating a custom Sink with your own format pattern.
+ */
+int main()
+{
+  // Start the backend thread
+  quill::BackendOptions backend_options;
+  quill::Backend::start(backend_options);
+
+  // Console sink
+  std::string console_log_pattern =
+    "%(time) [PID %(process_id)] [%(log_level)] [%(logger)] - %(message)";
+  std::string console_time_format = "%Y-%m-%d %H:%M:%S.%Qms";
+  auto console_sink = quill::Frontend::create_or_get_sink<ConsoleSinkWithFormatter>(
+    "sink_id_1", console_log_pattern, console_time_format);
+  console_sink->set_log_level_filter(quill::LogLevel::Warning);
+
+  // File sink
+  std::string const file_log_pattern = "%(log_level);%(time);%(logger);%(message)";
+  std::string const file_time_format = "%Y%m%dT%H:%M:%S.%Qus";
+
+  auto rotating_file_sink = quill::Frontend::create_or_get_sink<quill::RotatingFileSink>(
+    "rotating_file.log",
+    []()
+    {
+      // See RotatingFileSinkConfig for more options
+      quill::RotatingFileSinkConfig cfg;
+      cfg.set_open_mode('a');
+      cfg.set_max_backup_files(10);
+      cfg.set_rotation_max_file_size(1024 * 1024);
+      return cfg;
+    }());
+  rotating_file_sink->set_log_level_filter(quill::LogLevel::Info);
+
+  // The Logger is using the file_log_pattern by default
+  // To output our custom format to the file we use our own ConsoleSinkWithFormatter that is
+  // overwriting the default format
+  quill::Logger* logger = quill::Frontend::create_or_get_logger(
+    "root", {std::move(console_sink), std::move(rotating_file_sink)}, file_log_pattern, file_time_format);
+
+  logger->set_log_level(quill::LogLevel::Debug);
+
+  LOG_INFO(logger, "This is a log info example {}", sizeof(std::string));
+  LOG_WARNING(logger, "This is a log warning example {}", sizeof(std::string));
+  LOG_ERROR(logger, "This is a log error example {}", sizeof(std::string));
+  LOG_CRITICAL(logger, "This is a log critical example {}", sizeof(std::string));
+  return 0;
+}

examples/user_defined_sink.cpp

@@ -21,15 +21,15 @@ class UserSink final : public quill::Sink
   UserSink() = default;
 
   /***/
-  void write_log_message(quill::MacroMetadata const* log_metadata, uint64_t log_timestamp,
-                         std::string_view thread_id, std::string_view thread_name,
-                         std::string_view logger_name, quill::LogLevel log_level,
-                         std::vector<std::pair<std::string, std::string>> const* named_args,
-                         std::string_view log_message) override
+  void write_log(quill::MacroMetadata const* log_metadata, uint64_t log_timestamp,
+                 std::string_view thread_id, std::string_view thread_name,
+                 std::string const& process_id, std::string_view logger_name, quill::LogLevel log_level,
+                 std::vector<std::pair<std::string, std::string>> const* named_args,
+                 std::string_view log_message, std::string_view log_statement) override
   {
     // Called by the logger backend worker thread for each LOG_* macro
     // last character is '\n' and we exclude it using size() - 1
-    _messages.push_back(std::string{log_message.data(), log_message.size() - 1});
+    _messages.push_back(std::string{log_statement.data(), log_statement.size() - 1});
   }
 
   /***/

quill/include/quill/backend/BackendWorker.h

@@ -789,24 +789,24 @@ class BackendWorker
    */
   QUILL_ATTRIBUTE_HOT void _write_transit_event_to_sinks(TransitEvent const& transit_event) const
   {
-    std::string_view const formatted_log_message = transit_event.logger_base->pattern_formatter->format(
+    std::string_view const log_message =
+      std::string_view{transit_event.formatted_msg.data(), transit_event.formatted_msg.size()};
+
+    std::string_view const log_statement = transit_event.logger_base->pattern_formatter->format(
       transit_event.timestamp, transit_event.thread_id, transit_event.thread_name, _process_id,
       transit_event.logger_base->logger_name, loglevel_to_string(transit_event.log_level()),
-      *transit_event.macro_metadata, transit_event.named_args.get(),
-      std::string_view{transit_event.formatted_msg.data(), transit_event.formatted_msg.size()});
+      *transit_event.macro_metadata, transit_event.named_args.get(), log_message);
 
     for (auto& sink : transit_event.logger_base->sinks)
     {
       // If all filters are okay we write this message to the file
       if (sink->apply_all_filters(transit_event.macro_metadata, transit_event.timestamp,
                                   transit_event.thread_id, transit_event.thread_name,
-                                  transit_event.logger_base->logger_name, transit_event.log_level(),
-                                  formatted_log_message))
+                                  transit_event.logger_base->logger_name, transit_event.log_level(), log_statement))
       {
-        sink->write_log_message(transit_event.macro_metadata, transit_event.timestamp,
-                                transit_event.thread_id, transit_event.thread_name,
-                                transit_event.logger_base->logger_name, transit_event.log_level(),
-                                transit_event.named_args.get(), formatted_log_message);
+        sink->write_log(transit_event.macro_metadata, transit_event.timestamp, transit_event.thread_id,
+                        transit_event.thread_name, _process_id, transit_event.logger_base->logger_name,
+                        transit_event.log_level(), transit_event.named_args.get(), log_message, log_statement);
       }
     }
   }
@@ -1196,8 +1196,7 @@ class BackendWorker
 
     // Format all values to a single string
     std::string formatted_values_str;
-    fmtquill::vformat_to(
-      std::back_inserter(formatted_values_str), format_string,
+    fmtquill::vformat_to(std::back_inserter(formatted_values_str), format_string,
                          fmtquill::basic_format_args<fmtquill::format_context>{
                            format_args_store.get_types(), format_args_store.data()});
 

quill/include/quill/sinks/ConsoleSink.h

@@ -342,16 +342,17 @@ class ConsoleSink : public StreamSink
    * @param log_timestamp log timestamp
    * @param thread_id thread id
    * @param thread_name thread name
+   * @param process_id Process Id
    * @param logger_name logger name
    * @param log_level log level
    * @param named_args vector of key-value pairs of named args
    * @param log_message log message
    */
-  QUILL_ATTRIBUTE_HOT void write_log_message(MacroMetadata const* log_metadata, uint64_t log_timestamp,
-                                             std::string_view thread_id, std::string_view thread_name,
-                                             std::string_view logger_name, LogLevel log_level,
-                                             std::vector<std::pair<std::string, std::string>> const* named_args,
-                                             std::string_view log_message) override
+  QUILL_ATTRIBUTE_HOT void write_log(MacroMetadata const* log_metadata, uint64_t log_timestamp,
+                                     std::string_view thread_id, std::string_view thread_name,
+                                     std::string const& process_id, std::string_view logger_name, LogLevel log_level,
+                                     std::vector<std::pair<std::string, std::string>> const* named_args,
+                                     std::string_view log_message, std::string_view log_statement) override
   {
 #if defined(_WIN32)
     if (_console_colours.using_colours())
@@ -365,7 +366,7 @@ class ConsoleSink : public StreamSink
 
       // Write to console
       bool const write_to_console = WriteConsoleA(
-        out_handle, log_message.data(), static_cast<DWORD>(log_message.size()), nullptr, nullptr);
+        out_handle, log_statement.data(), static_cast<DWORD>(log_statement.size()), nullptr, nullptr);
 
       if (QUILL_UNLIKELY(!write_to_console))
       {
@@ -386,8 +387,8 @@ class ConsoleSink : public StreamSink
     else
     {
       // Write record to file
-      StreamSink::write_log_message(log_metadata, log_timestamp, thread_id, thread_name,
-                                    logger_name, log_level, named_args, log_message);
+      StreamSink::write_log(log_metadata, log_timestamp, thread_id, thread_name, process_id,
+                            logger_name, log_level, named_args, log_message, log_statement);
     }
 #else
     if (_console_colours.can_use_colours())
@@ -398,8 +399,8 @@ class ConsoleSink : public StreamSink
     }
 
     // Write record to file
-    StreamSink::write_log_message(log_metadata, log_timestamp, thread_id, thread_name, logger_name,
-                                  log_level, named_args, log_message);
+    StreamSink::write_log(log_metadata, log_timestamp, thread_id, thread_name, process_id,
+                          logger_name, log_level, named_args, log_message, log_statement);
 
     if (_console_colours.can_use_colours())
     {

quill/include/quill/sinks/JsonConsoleSink.h

@@ -34,15 +34,16 @@ class JsonConsoleSink : public StreamSink
    * @param log_timestamp Timestamp of the log event.
    * @param thread_id ID of the thread.
    * @param thread_name Name of the thread.
+   * @param process_id Process Id
    * @param logger_name Name of the logger.
    * @param log_level Log level of the message.
    * @param named_args Vector of key-value pairs of named args
    */
-  QUILL_ATTRIBUTE_HOT void write_log_message(MacroMetadata const* log_metadata, uint64_t log_timestamp,
-                                             std::string_view thread_id, std::string_view thread_name,
-                                             std::string_view logger_name, LogLevel log_level,
-                                             std::vector<std::pair<std::string, std::string>> const* named_args,
-                                             std::string_view) override
+  QUILL_ATTRIBUTE_HOT void write_log(MacroMetadata const* log_metadata, uint64_t log_timestamp,
+                                     std::string_view thread_id, std::string_view thread_name,
+                                     std::string const& process_id, std::string_view logger_name, LogLevel log_level,
+                                     std::vector<std::pair<std::string, std::string>> const* named_args,
+                                     std::string_view, std::string_view) override
   {
     _json_message.clear();
 
@@ -65,9 +66,9 @@ class JsonConsoleSink : public StreamSink
 
     _json_message.append("}\n");
 
-    StreamSink::write_log_message(log_metadata, log_timestamp, thread_id, thread_name, logger_name,
-                                  log_level, named_args,
-                                  std::string_view{_json_message.data(), _json_message.size()});
+    StreamSink::write_log(log_metadata, log_timestamp, thread_id, thread_name, process_id,
+                          logger_name, log_level, named_args, std::string_view{},
+                          std::string_view{_json_message.data(), _json_message.size()});
   }
 
 private: