Merge branch 'jean/structured' into fabienfl/spdlog

CMakeLists.txt

@@ -11,6 +11,8 @@ option(ORC_BUILD_ORC        "Build Orc binary" ON)
 option(ORC_BUILD_PARQUET    "Build Parquet module" OFF)
 option(ORC_BUILD_SSDEEP     "Build with ssdeep support" OFF)
 option(ORC_BUILD_JSON       "Build with JSON StructuredOutput enabled" ON)
+option(ORC_BUILD_BACKTRACE  "Build with stack backtrace enabled" ON)
+
 option(ORC_DOWNLOADS_ONLY   "Do not build ORC but only download vcpkg third parties" OFF)
 option(ORC_DISABLE_PRECOMPILED_HEADERS "Disable precompiled headers" OFF)
 
@@ -79,7 +81,6 @@ if(ORC_VCPKG_ROOT)
         boost-multi-index
         boost-outcome
         boost-scope-exit
-        boost-stacktrace
         fmt
         spdlog
         tlsh
@@ -92,6 +93,11 @@ if(ORC_VCPKG_ROOT)
         cli11
     )
 
+    if(ORC_BUILD_BACKTRACE)
+        add_compile_definitions(ORC_BUILD_BACKTRACE)
+        list(APPEND _PACKAGES boost-stacktrace)
+    endif()
+
     if(ORC_BUILD_APACHE_ORC)
         list(APPEND _PACKAGES orc)
     endif()
@@ -101,7 +107,7 @@ if(ORC_VCPKG_ROOT)
     endif()
 
     if(ORC_BUILD_SSDEEP)
-        add_definitions(-DORC_BUILD_SSDEEP)
+        add_compile_definitions(ORC_BUILD_SSDEEP)
         list(APPEND _PACKAGES ssdeep)
     endif()
 

cmake/Orc.cmake

@@ -17,8 +17,6 @@ add_compile_definitions(
     _UNICODE
     NOMINMAX
     BOOST_NO_SWPRINTF
-    _SILENCE_ALL_CXX17_DEPRECATION_WARNINGS
-    _SILENCE_CXX17_CODECVT_HEADER_DEPRECATION_WARNING
 )
 
 if(${TARGET_ARCH} STREQUAL "x64")
@@ -32,7 +30,6 @@ string(REPLACE "/EHsc" "" CMAKE_CXX_FLAGS "${CMAKE_CXX_FLAGS}")
 
 # TODO: Qspectre disable option is not supported until cmake 3.15.2
 add_compile_options(
-    /wd4995
     /EHa      # Enable C++ exception with SEH (required by Robustness.cpp: _set_se_translator)
   # /Gy       # Enable function level linking
   # /JMC      # Debug only Just My Code

src/OrcCommand/ToolEmbed_Config.cpp

@@ -199,7 +199,7 @@ HRESULT Main::GetConfigurationFromConfig(const ConfigItem& configitem)
 
                 if (SUCCEEDED(local_hr = GetNameValuePairFromConfigItem(item, spec)))
                 {
-                    if (spec.Type != EmbeddedResource::EmbedSpec::Void)
+                    if (spec.Type != EmbeddedResource::EmbedSpec::EmbedType::Void)
                         config.ToEmbed.push_back(std::move(spec));
                 }
                 else
@@ -222,7 +222,7 @@ HRESULT Main::GetConfigurationFromConfig(const ConfigItem& configitem)
 
                 if (SUCCEEDED(local_hr = GetAddFileFromConfigItem(item, spec)))
                 {
-                    if (spec.Type != EmbeddedResource::EmbedSpec::Void)
+                    if (spec.Type != EmbeddedResource::EmbedSpec::EmbedType::Void)
                         config.ToEmbed.push_back(std::move(spec));
                 }
                 else
@@ -245,7 +245,7 @@ HRESULT Main::GetConfigurationFromConfig(const ConfigItem& configitem)
 
                 if (SUCCEEDED(local_hr = GetAddArchiveFromConfigItem(item, spec)))
                 {
-                    if (spec.Type != EmbeddedResource::EmbedSpec::Void)
+                    if (spec.Type != EmbeddedResource::EmbedSpec::EmbedType::Void)
                         config.ToEmbed.push_back(std::move(spec));
                 }
                 else

src/OrcCommand/ToolEmbed_Output.cpp

@@ -76,8 +76,8 @@ void Main::PrintParameters()
     std::vector<std::reference_wrapper<const EmbeddedResource::EmbedSpec>> toRemove;
     for (const auto& item : config.ToEmbed)
     {
-        if (item.Type == EmbeddedResource::EmbedSpec::ValuesDeletion
-            || item.Type == EmbeddedResource::EmbedSpec::BinaryDeletion)
+        if (item.Type == EmbeddedResource::EmbedSpec::EmbedType::ValuesDeletion
+            || item.Type == EmbeddedResource::EmbedSpec::EmbedType::BinaryDeletion)
         {
             toRemove.push_back({item});
             continue;

src/OrcCommand/ToolEmbed_Run.cpp

@@ -50,19 +50,19 @@ HRESULT Main::WriteEmbedConfig(
     {
         switch (item.Type)
         {
-            case EmbeddedResource::EmbedSpec::File:
+            case EmbeddedResource::EmbedSpec::EmbedType::File:
                 writer->BeginElement(L"file");
                 writer->WriteNamed(L"name", item.Name.c_str());
                 writer->WriteNamed(L"path", item.Value.c_str());
                 writer->EndElement(L"file");
                 break;
-            case EmbeddedResource::EmbedSpec::NameValuePair:
+            case EmbeddedResource::EmbedSpec::EmbedType::NameValuePair:
                 writer->BeginElement(L"pair");
                 writer->WriteNamed(L"name", item.Name.c_str());
                 writer->WriteNamed(L"value", item.Value.c_str());
                 writer->EndElement(L"pair");
                 break;
-            case EmbeddedResource::EmbedSpec::Archive:
+            case EmbeddedResource::EmbedSpec::EmbedType::Archive:
                 writer->BeginElement(L"archive");
                 writer->WriteNamed(L"name", item.Name.c_str());
                 writer->WriteNamed(L"format", item.ArchiveFormat.c_str());

src/OrcCommand/UtilitiesMain.h

@@ -17,7 +17,9 @@
 #include <boost/algorithm/string/join.hpp>
 #include <boost/algorithm/string/split.hpp>
 #include <boost/algorithm/string/classification.hpp>
-#include <boost/stacktrace.hpp>
+#ifdef ORC_BUILD_BACKTRACE
+#    include <boost/stacktrace.hpp>
+#endif
 #include <concrt.h>
 
 #include "ConfigFile.h"
@@ -542,11 +544,37 @@ class ORCLIB_API UtilitiesMain
     // Option handling
     //
     bool OutputOption(LPCWSTR szArg, LPCWSTR szOption, OutputSpec::Kind supportedTypes, OutputSpec& anOutput);
+
+    template <typename OptionType>
+    bool
+    OutputOption(LPCWSTR szArg, LPCWSTR szOption, OutputSpec::Kind supportedTypes, std::optional<OptionType>& parameter)
+    {
+        OptionType result;
+        if (OutputOption(szArg, szOption, supportedTypes, result))
+        {
+            parameter.emplace(std::move(result));
+            return true;
+        }
+        return false;
+    }
+
     bool OutputOption(LPCWSTR szArg, LPCWSTR szOption, OutputSpec& anOutput)
     {
         return OutputOption(szArg, szOption, anOutput.supportedTypes, anOutput);
     };
 
+    template <typename OptionType>
+    bool OutputOption(LPCWSTR szArg, LPCWSTR szOption, std::optional<OptionType> parameter)
+    {
+        OptionType result;
+        if (OutputOption(szArg, szOption, result))
+        {
+            parameter.emplace(std::move(result));
+            return true;
+        }
+        return false;
+    }
+
     bool OutputFileOption(LPCWSTR szArg, LPCWSTR szOption, std::wstring& strOutputFile);
     template <typename OptionType>
     bool OutputFileOption(LPCWSTR szArg, LPCWSTR szOption, std::optional<OptionType> parameter)
@@ -852,6 +880,7 @@ class ORCLIB_API UtilitiesMain
                 return hr;
             }
         }
+#ifdef ORC_BUILD_BACKTRACE
         catch (std::exception& e)
         {
             std::cerr << "std::exception during execution" << std::endl;
@@ -866,7 +895,20 @@ class ORCLIB_API UtilitiesMain
             boost::stacktrace::stacktrace();
             return E_ABORT;
         }
-
+#else
+        catch (std::exception& e)
+        {
+            std::cerr << "std::exception during execution" << std::endl;
+            std::cerr << "Caught " << e.what() << std::endl;
+            std::cerr << "Type " << typeid(e).name() << std::endl;
+            return E_ABORT;
+        }
+        catch (...)
+        {
+            std::cerr << "Exception during during execution" << std::endl;
+            return E_ABORT;
+        }
+#endif
         GetSystemTime(&Cmd.theFinishTime.value);
         Cmd.theFinishTickCount = GetTickCount();
         Cmd.PrintFooter();

src/OrcLib/ArchiveExtract.cpp

@@ -139,11 +139,16 @@ STDMETHODIMP ArchiveExtract::Extract(
         [pwzExtractRootDir, szSDDL, this](Archive::ArchiveItem& item) -> std::shared_ptr<ByteStream> {
         HRESULT hr = E_FAIL;
 
+        std::wstring fileName;
+        std::replace_copy(
+            begin(item.NameInArchive), end(item.NameInArchive), std::back_insert_iterator(fileName), L'\\', L'_');
+        std::replace(begin(fileName), end(fileName), L'/', L'_');
+
         HANDLE hFile = INVALID_HANDLE_VALUE;
         if (FAILED(
                 hr = UtilGetUniquePath(
                     pwzExtractRootDir,
-                    item.NameInArchive.c_str(),
+                    fileName.c_str(),
                     item.Path,
                     hFile,
                     FILE_ATTRIBUTE_NORMAL | FILE_ATTRIBUTE_TEMPORARY | FILE_ATTRIBUTE_NOT_CONTENT_INDEXED,

src/OrcLib/Buffer.h

@@ -74,13 +74,6 @@ class Buffer
             if (!m_Ptr)
                 throw Orc::Exception(Severity::Fatal, E_OUTOFMEMORY);
         }
-        HeapStore(_T* Ptr, ULONG Elts)
-            : m_Ptr(Ptr)
-            , m_EltsAlloc(Elts)
-        {
-            if (!m_Ptr)
-                throw Orc::Exception(Severity::Fatal, E_POINTER);
-        };
         HeapStore(HeapStore&& other) noexcept
         {
             std::swap(m_Ptr, other.m_Ptr);
@@ -321,7 +314,7 @@ class Buffer
     }
     explicit Buffer(const Buffer& other) { assign(other); }
 
-    Buffer(_In_reads_(Elts) _T* Ptr, _In_ ULONG Elts, _In_ bool Owned, _In_ ULONG Used) { set(Ptr, Elts, Owned, Used); }
+    Buffer(_In_reads_(Elts) const _T* Ptr, _In_ ULONG Elts, _In_ ULONG Used) { set(Ptr, Elts, Used); }
 
     Buffer(const std::initializer_list<_T>& list)
     {
@@ -339,16 +332,20 @@ class Buffer
     bool full() const { return capacity() == size(); }
 
     bool empty() const { return std::holds_alternative<EmptyStore>(m_store); }
-    void set(_In_reads_(Elts) _T* Ptr, _In_ ULONG Elts, _In_ ULONG Used)
+    void set(_In_reads_(Elts) const _T* Ptr, _In_ ULONG Elts, _In_ ULONG Used)
     {
         if (Elts == 0)
             m_store = EmptyStore();
         else if (Elts <= _DeclElts)
         {
-            m_store = InnerStore(Ptr, Ptr + Elts);
+            m_store = InnerStore(Ptr, Ptr + Used);
         }
         else
-            m_store = HeapStore(Ptr, Elts);
+        {
+            HeapStore heap(Elts);
+            heap.assign(Ptr, Used);
+            m_store = std::move(heap);
+        }
 
         if (Used <= Elts)
             m_EltsUsed = Used;
@@ -416,6 +413,46 @@ class Buffer
         view_of(Other.m_Ptr, Other.m_Elts, InUse);
     }
 
+    void append(const _T& Src)
+    {
+        auto capacity_ = capacity();
+        auto size_ = size();
+        auto new_size = size_ + 1;
+
+        if (new_size > capacity_)
+        {
+            auto new_capacity = std::max(new_size, capacity_ + capacity_ / 2);
+            if (new_capacity < _DeclElts)
+                new_capacity = _DeclElts;
+            reserve(new_capacity);
+            capacity_ = new_capacity;
+        }
+
+        if (is_view() && (capacity_ < new_size))  // current store is a view, and is too small to contains the new
+                                                  // elts -> we need to "mutate" to a owning one.
+        {
+            if (new_size <= _DeclElts)
+            {
+                auto new_store = InnerStore();
+
+                std::visit([&new_store](const auto& arg) { new_store.assign(arg.get(), arg.capacity()); }, m_store);
+                m_store = std::move(new_store);
+                capacity_ = _DeclElts;
+            }
+            else
+            {
+                auto new_capacity = std::max(new_size, capacity_ + capacity_ / 2);
+                auto new_store = HeapStore(new_capacity);
+                std::visit([&new_store](const auto& arg) { new_store.assign(arg.get(), arg.capacity()); }, m_store);
+                m_store = std::move(new_store);
+                capacity_ = new_capacity;
+            }
+        }
+
+        std::copy(&Src, &Src + 1, stdext::checked_array_iterator(get() + size_, capacity_ - size_));
+        m_EltsUsed = new_size;
+    }
+
     void append(_In_reads_(Elts) const _T* Src, _In_ ULONG Elts = 1)
     {
         if (!Src)

src/OrcLib/ByteStreamVisitor.h

@@ -30,21 +30,21 @@ class TeeStream;
 class ByteStreamVisitor
 {
 public:
-    virtual void Visit(ByteStream& element) {}
-    virtual void Visit(FileStream& element) {}
-    virtual void Visit(TemporaryStream& element) {}
-    virtual void Visit(MemoryStream& element) {}
-    virtual void Visit(CryptoHashStream& element) {}
-    virtual void Visit(FuzzyHashStream& element) {}
-    virtual void Visit(NTFSStream& element) {}
-    virtual void Visit(UncompressNTFSStream& element) {}
-    virtual void Visit(ResourceStream& element) {}
-    virtual void Visit(PipeStream& element) {}
-    virtual void Visit(OnDiskChunkStream& element) {}
-    virtual void Visit(EncodeMessageStream& element) {}
-    virtual void Visit(JournalingStream& element) {}
-    virtual void Visit(AccumulatingStream& element) {}
-    virtual void Visit(TeeStream& element) {}
+    virtual void Visit(ByteStream&) {}
+    virtual void Visit(FileStream&) {}
+    virtual void Visit(TemporaryStream&) {}
+    virtual void Visit(MemoryStream&) {}
+    virtual void Visit(CryptoHashStream&) {}
+    virtual void Visit(FuzzyHashStream&) {}
+    virtual void Visit(NTFSStream&) {}
+    virtual void Visit(UncompressNTFSStream&) {}
+    virtual void Visit(ResourceStream&) {}
+    virtual void Visit(PipeStream&) {}
+    virtual void Visit(OnDiskChunkStream&) {}
+    virtual void Visit(EncodeMessageStream&) {}
+    virtual void Visit(JournalingStream&) {}
+    virtual void Visit(AccumulatingStream&) {}
+    virtual void Visit(TeeStream&) {}
 };
 
 }  // namespace Orc

src/OrcLib/CMakeLists.txt

@@ -565,8 +565,14 @@ set(SRC_INOUT_TEXT_FMT
     "Output/Text/Fmt/Partition.h"
     "Output/Text/Fmt/PartitionFlags.h"
     "Output/Text/Fmt/PartitionType.h"
+    "Output/Text/Fmt/path.h"
     "Output/Text/Fmt/SYSTEMTIME.h"
     "Output/Text/Fmt/TimeUtc.h"
+)
+
+source_group(In&Out\\Output\\Text\\Fmt FILES ${SRC_INOUT_TEXT_FMT})
+
+set(SRC_INOUT_TEXT_FMT_FWD
     "Output/Text/Fmt/Fwd/ByteQuantity.h"
     "Output/Text/Fmt/Fwd/CryptoHashStreamAlgorithm.h"
     "Output/Text/Fmt/Fwd/error_code.h"
@@ -584,7 +590,8 @@ set(SRC_INOUT_TEXT_FMT
     "Output/Text/Fmt/Fwd/SYSTEMTIME.h"
     "Output/Text/Fmt/Fwd/TimeUtc.h")
 
-source_group(In&Out\\Output\\Text\\Fmt FILES ${SRC_INOUT_TEXT_FMT})
+source_group(In&Out\\Output\\Text\\Fmt\\Fwd FILES ${SRC_INOUT_TEXT_FMT_FWD})
+
 
 set(SRC_INOUT_TEXT_PRINT
     "Output/Text/Print.h"
@@ -832,6 +839,7 @@ add_library(OrcLib
         ${SRC_INOUT_CONSOLE}
         ${SRC_INOUT_TEXT}
         ${SRC_INOUT_TEXT_FMT}
+        ${SRC_INOUT_TEXT_FMT_FWD}
         ${SRC_INOUT_TEXT_PRINT}
         ${SRC_INOUT_UPLOAD}
         ${SRC_LOG}

src/OrcLib/DbgHelpLibrary.cpp

@@ -17,7 +17,7 @@ using namespace Orc;
 using namespace std::string_literals;
 
 DbgHelpLibrary::DbgHelpLibrary()
-    : ExtensionLibrary(L"DbgHelp.dll"s, L"DBGHELP_X86DLL"s, L"DBGHELP_X64DLL"s, L""s)
+    : ExtensionLibrary(L"DbgHelp.dll"s, L"DBGHELP_X86DLL"s, L"DBGHELP_X64DLL"s)
 {
     m_strDesiredName = L"DbgHelp.dll"s;
 }

src/OrcLib/DiskExtent.cpp

@@ -229,7 +229,7 @@ CDiskExtent CDiskExtent::ReOpen(DWORD dwDesiredAccess, DWORD dwShareMode, DWORD
     ext.m_PhysicalSectorSize = m_PhysicalSectorSize;
     ext.m_Start = m_Start;
 
-    const auto k32 = ExtensionLibrary::GetLibrary<Kernel32Extension>(true);
+    const auto k32 = ExtensionLibrary::GetLibrary<Kernel32Extension>();
 
     if (k32 != nullptr)
     {