Merge branch 'main' into release/10.1.x

CHANGELOG.md

@@ -1,5 +1,24 @@
 # ChangeLog
 
+## [10.1.0-rc3] - 2020-12-02
+### Changed
+- Log: enhance FileSink synchronisation
+
+### Fixed
+- WolfLauncher: fix log file format for a configurated DFIR-Orc
+- Log: add support for SPDLOG_LEVEL env variable
+- Values incorrectly displayed as addresses instead of readable strings
+- Outline: missing 'qfe', 'network', 'environment' sections
+- GetSectors: failure when no disk was specified
+- Column filter processing inserting some empty lines (',,,,,,,,,,,,,,,')
+- GetThis: fix compression level
+
+### Added
+- WolfLauncher: print a 'Ended' line in the journal with statistics
+
+### Removed
+- Dead code: ByteStreamSink
+
 ## [10.1.0-rc2] - 2020-11-20
 ### Changed
 - Column 'ParentName': remove trailing '\'

src/OrcCommand/CMakeLists.txt

@@ -235,6 +235,8 @@ set(SRC_WOLFLAUNCHER
     "WolfLauncher_Run.cpp"
     "WolfTask.cpp"
     "WolfTask.h"
+    "Output/Text/Fmt/WolfPriority.h"
+    "Output/Text/Print/Recipient.h"
     "WolfLauncherSqlSchema.xml"
 )
 

src/OrcCommand/FastFind_Run.cpp

@@ -463,7 +463,7 @@ HRESULT Main::Run()
     if (config.outObject.Type != OutputSpec::Kind::None)
         pObjectTableOutput = TableOutput::GetWriter(config.outObject);
 
-    if (config.outStructured.Type & OutputSpec::Kind::StructuredFile)
+    if (HasFlag(config.outStructured.Type, OutputSpec::Kind::StructuredFile))
     {
         pStructuredOutput = StructuredOutputWriter::GetWriter(config.outStructured, nullptr);
     }

src/OrcCommand/GetSamples_Run.cpp

@@ -473,7 +473,7 @@ HRESULT Main::Run()
         Log::Info(L"Verifying code signatures... Done");
     }
 
-    if (config.sampleinfoOutput.Type & OutputSpec::Kind::TableFile)
+    if (HasFlag(config.sampleinfoOutput.Type, OutputSpec::Kind::TableFile))
     {
         hr = WriteSampleInformation(results);
         if (FAILED(hr))
@@ -483,7 +483,7 @@ HRESULT Main::Run()
         }
     }
 
-    if (config.timelineOutput.Type & OutputSpec::Kind::TableFile)
+    if (HasFlag(config.timelineOutput.Type, OutputSpec::Kind::TableFile))
     {
         // Collect time line related information
         const TaskTracker::TimeLine& timeline = tk.GetTimeLine();

src/OrcCommand/GetSectors_Run.cpp

@@ -117,7 +117,7 @@ std::wstring Main::getBootDiskName()
         return {};
     }
 
-    Guard::ScopeGuard([&hVolume]() {
+    Guard::ScopeGuard sg([&hVolume]() {
         if (hVolume != INVALID_HANDLE_VALUE)
         {
             CloseHandle(hVolume);

src/OrcCommand/Mothership.h

@@ -120,6 +120,8 @@ class ORCUTILS_API Main : public UtilitiesMain
     HRESULT DownloadLocalCopy();
 
 public:
+    void Configure(int argc, const wchar_t* argv[]) override;
+
     static LPCWSTR ToolName() { return L"Mothership"; }
     static LPCWSTR ToolDescription() { return nullptr; }
 

src/OrcCommand/Mothership_Run.cpp

@@ -84,6 +84,11 @@ class MotherShipTerminate : public TerminationHandler
     };
 };
 
+void Main::Configure(int argc, const wchar_t* argv[])
+{
+    UtilitiesMain::Configure(argc, argv);
+}
+
 HRESULT Main::ChangeTemporaryEnvironment()
 {
     HRESULT hr = E_FAIL;

src/OrcCommand/NTFSInfo_Output.cpp

@@ -20,6 +20,7 @@
 #include "Output/Text/Print/Intentions.h"
 #include "Output/Text/Print/Filter.h"
 #include "Output/Text/Print/LocationSet.h"
+#include "Output/Text/Print/Location.h"
 
 #include "Usage.h"
 

src/OrcCommand/Output/Text/Fmt/WolfPriority.h

@@ -0,0 +1,44 @@
+//
+// SPDX-License-Identifier: LGPL-2.1-or-later
+//
+// Copyright © 2020 ANSSI. All Rights Reserved.
+//
+// Author(s): fabienfl (ANSSI)
+//
+
+#pragma once
+
+#include "WolfLauncher.h"
+
+#include <system_error>
+
+#include "Output/Text/Format.h"
+#include "Utils/Iconv.h"
+
+template<>
+struct fmt::formatter<Orc::Command::Wolf::Main::WolfPriority> : public fmt::formatter<std::string_view>
+{
+    template <typename FormatContext>
+    auto format(const Orc::Command::Wolf::Main::WolfPriority& priority, FormatContext& ctx) -> decltype(ctx.out())
+    {
+        std::error_code ec;
+        const auto utf8 = Orc::Utf16ToUtf8(Orc::Command::Wolf::WolfLauncher::ToString(priority), ec);
+        if (ec)
+        {
+            return formatter<std::string_view>::format(Orc::kFailedConversion, ctx);
+        }
+
+        return formatter<std::string_view>::format(utf8, ctx);
+    }
+};
+
+template<>
+struct fmt::formatter<Orc::Command::Wolf::Main::WolfPriority, wchar_t>
+    : public fmt::formatter<std::wstring_view, wchar_t>
+{
+    template <typename FormatContext>
+    auto format(Orc::Command::Wolf::Main::WolfPriority priority, FormatContext& ctx) -> decltype(ctx.out())
+    {
+        return formatter<std::wstring_view, wchar_t>::format(Orc::Command::Wolf::Main::ToString(priority), ctx);
+    }
+};

src/OrcCommand/Output/Text/Print/Recipient.h

@@ -0,0 +1,40 @@
+//
+// SPDX-License-Identifier: LGPL-2.1-or-later
+//
+// Copyright © 2020 ANSSI. All Rights Reserved.
+//
+// Author(s): fabienfl (ANSSI)
+//
+
+#pragma once
+
+#include "Output/Text/Print.h"
+
+#include <boost/algorithm/string/join.hpp>
+
+#include "WolfExecution.h"
+
+namespace Orc {
+namespace Text {
+
+template <>
+struct Printer<Orc::Command::Wolf::WolfExecution::Recipient>
+{
+    template <typename T>
+    static void Output(Orc::Text::Tree<T>& node, const Orc::Command::Wolf::WolfExecution::Recipient& recipient)
+    {
+        std::vector<std::wstring> ArchiveSpec;
+        auto archiveSpecs = boost::join(recipient.ArchiveSpec, L",");
+        if (archiveSpecs.empty())
+        {
+            Print(node, recipient.Name);
+        }
+        else
+        {
+            Print(node, fmt::format(L"{} [{}]", recipient.Name, archiveSpecs));
+        }
+    }
+};
+
+}  // namespace Text
+}  // namespace Orc

src/OrcCommand/RegInfo_Config.cpp

@@ -241,12 +241,12 @@ HRESULT Main::CheckConfiguration()
 {
     config.m_HiveQuery.m_HivesLocation.Consolidate(false, FSVBR::FSType::NTFS);
 
-    if (config.Output.Type == OutputSpec::Kind::None)
+    if (HasFlag(config.Output.Type, OutputSpec::Kind::None))
     {
         Log::Error("No valid output specified (only directory or csv|tsv are allowed");
         return E_INVALIDARG;
     }
-    if (config.Output.Type & OutputSpec::Kind::Archive)
+    if (HasFlag(config.Output.Type, OutputSpec::Kind::Archive))
     {
         Log::Error("Archive output is not supported (only directory or csv|tsv are allowed");
         return E_INVALIDARG;

src/OrcCommand/RegInfo_Run.cpp

@@ -467,7 +467,7 @@ HRESULT Main::Run()
 
         std::shared_ptr<TableOutput::IWriter> pRegInfoWriter;
 
-        if (config.Output.Type & OutputSpec::Kind::TableFile)
+        if (HasFlag(config.Output.Type, OutputSpec::Kind::TableFile))
         {
             pRegInfoWriter = GetRegInfoWriter(config.Output);
             if (nullptr == pRegInfoWriter)
@@ -482,7 +482,7 @@ HRESULT Main::Run()
         {
             auto node = root.AddNode("Parsing hive '{}'", hive.FileName);
 
-            if (config.Output.Type & OutputSpec::Kind::Directory)
+            if (HasFlag(config.Output.Type, OutputSpec::Kind::Directory))
             {
                 std::wstring fileName(hive.FileName);
                 std::replace(fileName.begin(), fileName.end(), L'\\', L'_');

src/OrcCommand/Usage.h

@@ -255,7 +255,7 @@ auto PrintColumnSelectionParameter(
             const ColumnNameDef* pCurCol = pColumnNames;
             while (pCurCol->dwIntention != Intentions::FILEINFO_NONE)
             {
-                if (pCurAlias->dwIntention & pCurCol->dwIntention)
+                if (HasFlag(pCurAlias->dwIntention, pCurCol->dwIntention))
                 {
                     PrintValue(aliasNode, pCurCol->szColumnName, pCurCol->szColumnDesc);
                     dwNumCol++;

src/OrcCommand/UtilitiesLogger.cpp

@@ -11,51 +11,85 @@
 
 #include <optional>
 
+#include <spdlog/cfg/env.h>
+
+#ifdef ORC_BUILD_BOOST_STACKTRACE
+#    include <boost/stacktrace.hpp>
+#endif
+
 #include "UtilitiesLogger.h"
 #include "ParameterCheck.h"
 #include "Utils/Result.h"
 
+using namespace Orc::Command;
+using namespace Orc::Log;
 using namespace Orc;
 
 namespace {
 
-std::shared_ptr<Logger::ConsoleSink> CreateConsoleSink()
+std::unique_ptr<SpdlogLogger> CreateSpdlogLogger(const std::string& name)
+{
+    auto logger = std::make_unique<SpdlogLogger>(name);
+
+    // This is error handler will help to fix log formatting error
+    logger->SetErrorHandler([](const std::string& msg) {
+        std::cerr << msg << std::endl;
+
+#ifdef ORC_BUILD_BOOST_STACKTRACE
+        std::cerr << "Stack trace:" << std::endl;
+        std::cerr << boost::stacktrace::stacktrace();
+#endif
+    });
+
+    // Default upstream log level filter (sinks will not received filtered logs)
+    logger->SetLevel(spdlog::level::debug);
+
+    return logger;
+}
+
+std::shared_ptr<Command::UtilitiesLogger::ConsoleSink> CreateConsoleSink()
 {
-    auto console = std::make_shared<Logger::ConsoleSink>();
-    console->set_level(spdlog::level::critical);
-    return console;
+    auto sink = std::make_shared<Command::UtilitiesLogger::ConsoleSink>();
+    sink->SetLevel(spdlog::level::critical);
+    return sink;
 }
 
-std::shared_ptr<Logger::FileSink> CreateFileSink()
+std::shared_ptr<Command::UtilitiesLogger::FileSink> CreateFileSink()
 {
-    auto file = std::make_shared<Logger::FileSink>();
+    auto sink = std::make_shared<Command::UtilitiesLogger::FileSink>();
 
     // Allow all logs to be print, they will be filtered by the upstream level set by spdlog::set_level
-    file->set_level(spdlog::level::trace);
-    return file;
+    sink->SetLevel(spdlog::level::trace);
+    return sink;
 }
 
-std::pair<std::shared_ptr<spdlog::logger>, std::shared_ptr<spdlog::logger>>
-CreateFacilities(std::shared_ptr<Logger::ConsoleSink> consoleSink, std::shared_ptr<Logger::FileSink> fileSink)
+std::pair<SpdlogLogger::Ptr, SpdlogLogger::Ptr> CreateFacilities(SpdlogSink::Ptr consoleSink, SpdlogSink::Ptr fileSink)
 {
     std::vector<std::shared_ptr<spdlog::logger>> loggers;
 
-    auto defaultLogger = std::make_shared<spdlog::logger>("default", spdlog::sinks_init_list {consoleSink, fileSink});
+    auto defaultLogger = ::CreateSpdlogLogger("default");
+    defaultLogger->Add(consoleSink);
+    defaultLogger->Add(fileSink);
+    defaultLogger->EnableBacktrace(512);
+    defaultLogger->SetFormatter(
+        std::make_unique<spdlog::pattern_formatter>(Log::kDefaultLogPattern, spdlog::pattern_time_type::utc));
 
-    auto fileLogger = std::make_shared<spdlog::logger>("file", fileSink);
-    fileSink->set_level(spdlog::level::trace);  // delegate filtering to the sink
+    auto fileLogger = ::CreateSpdlogLogger("file");
+    fileLogger->Add(fileSink);
+    fileLogger->SetFormatter(
+        std::make_unique<spdlog::pattern_formatter>(Log::kDefaultLogPattern, spdlog::pattern_time_type::utc));
 
-    return {defaultLogger, fileLogger};
+    return {std::move(defaultLogger), std::move(fileLogger)};
 }
 
 }  // namespace
 
 Orc::Command::UtilitiesLogger::UtilitiesLogger()
 {
-    m_fileSink = CreateFileSink();
-    m_consoleSink = CreateConsoleSink();
+    m_fileSink = ::CreateFileSink();
+    m_consoleSink = ::CreateConsoleSink();
 
-    auto [defaultLogger, fileLogger] = CreateFacilities(m_consoleSink, m_fileSink);
+    auto [defaultLogger, fileLogger] = ::CreateFacilities(m_consoleSink, m_fileSink);
 
     auto loggers = {
         std::make_pair(Logger::Facility::kDefault, defaultLogger),
@@ -152,18 +186,22 @@ void Orc::Command::UtilitiesLogger::Configure(int argc, const wchar_t* argv[]) c
         }
     }
 
+    // Load log levels from environment variable (ex: "SPDLOG_LEVEL=info,mylogger=trace")
+    spdlog::cfg::load_env_levels();
+
     if (verbose)
     {
         if (!level)
         {
             level = spdlog::level::debug;
         }
 
-        m_consoleSink->set_level(*level);
+        m_consoleSink->SetLevel(*level);
     }
 
     if (level)
     {
-        spdlog::set_level(*level);
+        m_logger->Get(Logger::Facility::kDefault)->SetLevel(*level);
+        m_logger->Get(Logger::Facility::kLogFile)->SetLevel(*level);
     }
 }