[Feature] Serialization & Deserialization of market data with protobuf

.github/workflows/macos.yml

@@ -22,7 +22,10 @@ jobs:
           xcode-version: latest-stable
 
       - name: Install dependencies
-        run: brew install ninja
+        run: |
+          brew update && brew upgrade && brew upgrade --cask && brew cleanup
+          brew install ninja protobuf
+          echo "CC=$(echo ${{matrix.compiler}} | sed -e 's/^g++/gcc/' | sed 's/+//g')" >> $GITHUB_ENV
 
       - name: Configure
         run: cmake -DCCT_ENABLE_ASAN=OFF -S . -B build -GNinja

.github/workflows/ubuntu-clang-tidy.yml

@@ -37,6 +37,7 @@ jobs:
           clang-tidy --dump-config
           cmake -S . -B build -DCCT_ENABLE_CLANG_TIDY=ON -DCCT_ENABLE_ASAN=OFF -GNinja
         env:
+          CC: clang-${{matrix.clang-version}}
           CXX: clang++-${{matrix.clang-version}}
           CMAKE_BUILD_TYPE: ${{matrix.buildmode}}
 

.github/workflows/ubuntu-monitoring.yml → .github/workflows/ubuntu-special.yml

@@ -1,4 +1,4 @@
-name: Monitoring
+name: Special
 
 on:
   push:
@@ -7,14 +7,15 @@ on:
   pull_request:
 
 jobs:
-  ubuntu-monitoring-build:
-    name: Build on Ubuntu with monitoring support
+  ubuntu-special-build:
+    name: Build on Ubuntu with monitoring / protobuf support
     runs-on: ubuntu-latest
     strategy:
       matrix:
         compiler: [g++-11]
         buildmode: [Debug]
-        build-prometheus-from-source: [0, 1]
+        build-special-from-source: [0, 1]
+        prometheus-options: ["-DBUILD_SHARED_LIBS=ON -DENABLE_PULL=OFF -DENABLE_PUSH=ON -DENABLE_COMPRESSION=OFF -DENABLE_TESTING=OFF"]
 
     steps:
       - name: Checkout repository code
@@ -24,6 +25,7 @@ jobs:
         run: |
           sudo apt update
           sudo apt install cmake libssl-dev git libcurl4-openssl-dev ninja-build -y --no-install-recommends
+          echo "CC=$(echo ${{matrix.compiler}} | sed -e 's/^g++/gcc/' | sed 's/+//g')" >> $GITHUB_ENV
 
       - name: Install prometheus-cpp
         run: |
@@ -39,10 +41,10 @@ jobs:
         env:
           CXX: ${{matrix.compiler}}
           CMAKE_BUILD_TYPE: ${{matrix.buildmode}}
-        if: matrix.build-prometheus-from-source == 0
+        if: matrix.build-special-from-source == 0
 
       - name: Configure CMake
-        run: cmake -S . -B build -DCCT_BUILD_PROMETHEUS_FROM_SRC=${{matrix.build-prometheus-from-source}} -DCCT_ENABLE_ASAN=OFF -GNinja
+        run: cmake -S . -B build -DCCT_BUILD_PROMETHEUS_FROM_SRC=${{matrix.build-special-from-source}} -DCCT_ENABLE_PROTO=${{matrix.build-special-from-source}} -DCCT_ENABLE_ASAN=OFF -GNinja
         env:
           CXX: ${{matrix.compiler}}
           CMAKE_BUILD_TYPE: ${{matrix.buildmode}}
@@ -56,4 +58,4 @@ jobs:
 
       - name: Sanity check main executable
         working-directory: ${{github.workspace}}/build
-        run: ./coincenter --help
+        run: ./coincenter --help

.github/workflows/ubuntu.yml

@@ -23,22 +23,23 @@ jobs:
         run: |
           sudo apt update
           sudo apt install cmake libssl-dev libcurl4-openssl-dev ninja-build -y --no-install-recommends
+          echo "CC=$(echo ${{matrix.compiler}} | sed -e 's/^g++/gcc/' | sed 's/+//g')" >> $GITHUB_ENV
 
       - name: Install gcc
         run: |
           sudo apt install ${{matrix.compiler}} -y --no-install-recommends
 
           # Temporary workaround for libasan bug stated here: https://github.com/google/sanitizers/issues/1716
           sudo sysctl vm.mmap_rnd_bits=28
-        if: startsWith(matrix.compiler, 'g++')
+        if: startsWith(matrix.compiler, 'g')
 
       - name: Install clang
         run: |
           CLANG_VERSION=$(echo ${{matrix.compiler}} | cut -d- -f2)
           wget https://apt.llvm.org/llvm.sh
           chmod +x llvm.sh
           sudo ./llvm.sh ${CLANG_VERSION}
-        if: startsWith(matrix.compiler, 'clang++')
+        if: startsWith(matrix.compiler, 'clang')
 
       - name: Configure CMake
         run: cmake -S . -B build -GNinja
@@ -55,4 +56,4 @@ jobs:
 
       - name: Sanity check main executable
         working-directory: ${{github.workspace}}/build
-        run: ./coincenter --help
+        run: ./coincenter --help

.github/workflows/windows.yml

@@ -33,4 +33,4 @@ jobs:
 
       - name: Sanity check main executable
         working-directory: ${{github.workspace}}/build/${{matrix.buildmode}}
-        run: .\coincenter.exe --help
+        run: .\coincenter.exe --help

.gitignore

@@ -64,6 +64,7 @@ data/cache
 data/log
 data/secret
 !data/secret/secret_test.json
+data/serialized
 data/static/exchangeconfig.json
 data/static/generalconfig.json
 monitoring/data/grafana/*

CMakeLists.txt

@@ -38,6 +38,7 @@ option(CCT_ENABLE_TESTS "Build the unit tests" ${MAIN_PROJECT})
 option(CCT_BUILD_EXEC "Build an executable instead of a static library" ${MAIN_PROJECT})
 option(CCT_ENABLE_ASAN "Compile with AddressSanitizer" ${CCT_ASAN_BUILD})
 option(CCT_ENABLE_CLANG_TIDY "Compile with clang-tidy checks" OFF)
+option(CCT_ENABLE_PROTO "Compile with protobuf support (to export data to the outside world)" ON)
 option(CCT_BUILD_PROMETHEUS_FROM_SRC "Fetch and build from prometheus-cpp sources" OFF)
 
 set(CCT_DATA_DIR "${CMAKE_CURRENT_SOURCE_DIR}/data" CACHE PATH "Needed data directory for coincenter. Can also be overridden at runtime with this environment variable")
@@ -162,14 +163,68 @@ set(JWT_BUILD_EXAMPLES OFF CACHE BOOL "" FORCE)
 
 list(APPEND fetchContentPackagesToMakeAvailable jwt-cpp) 
 
+# protobuf - serialization / deserialization library
+set(PROTOBUF_FETCHED_CONTENT OFF)
+if(CCT_ENABLE_PROTO)
+  find_package(Protobuf CONFIG)
+  if(Protobuf_FOUND)
+    message(STATUS "Linking with protobuf ${protobuf_VERSION}")
+  else()
+    # Check here for a new version: https://protobuf.dev/support/version-support/#cpp
+    if (NOT PROTOBUF_VERSION)
+      set(PROTOBUF_VERSION v5.26.1)
+    endif()
+
+    message(STATUS "Configuring protobuf ${PROTOBUF_VERSION} from sources")
+
+    # Using git here to simplify cmake code instead of archive download as 
+    # it depends on Abseil behind the scene whose code is not included in the release archives.
+    FetchContent_Declare(
+      protobuf
+      GIT_REPOSITORY https://github.com/protocolbuffers/protobuf.git
+      GIT_TAG ${PROTOBUF_VERSION}
+      GIT_SHALLOW true
+    )
+
+    set(protobuf_BUILD_TESTS OFF CACHE BOOL "" FORCE)
+    set(protobuf_BUILD_SHARED_LIBS OFF CACHE BOOL "" FORCE)
+    set(protobuf_BUILD_LIBUPB OFF CACHE BOOL "" FORCE)
+
+    # Abseil options
+    set(ABSL_CXX_STANDARD ${CMAKE_CXX_STANDARD} CACHE STRING "" FORCE)
+    set(ABSL_PROPAGATE_CXX_STD ON CACHE BOOL "" FORCE)
+    set(ABSL_ENABLE_INSTALL OFF CACHE BOOL "" FORCE)
+    set(ABSL_BUILD_TESTING OFF CACHE BOOL "" FORCE)
+
+    if(CCACHE_PROGRAM)
+      set(protobuf_ALLOW_CCACHE ON CACHE BOOL "" FORCE)
+    endif()
+
+    set(protobuf_INSTALL OFF CACHE BOOL "" FORCE)
+    set(protobuf_BUILD_LIBUPB OFF CACHE BOOL "" FORCE)
+    set(protobuf_WITH_ZLIB ON CACHE BOOL "" FORCE)
+    SET(protobuf_MSVC_STATIC_RUNTIME OFF CACHE INTERNAL "")
+
+
+    list(APPEND fetchContentPackagesToMakeAvailable protobuf)
+
+    set(PROTOBUF_FETCHED_CONTENT ON)
+  endif()
+endif()
+
 # Make fetch content available
 if(fetchContentPackagesToMakeAvailable)
   message(STATUS "Configuring packages ${fetchContentPackagesToMakeAvailable}")
+
   FetchContent_MakeAvailable("${fetchContentPackagesToMakeAvailable}")
+
+  if(PROTOBUF_FETCHED_CONTENT)
+    include(${protobuf_SOURCE_DIR}/cmake/protobuf-generate.cmake)
+  endif()
 endif()
 
 # Unit Tests
-include(cmake/AddUnitTest.cmake)
+include(${CMAKE_CURRENT_LIST_DIR}/cmake/AddUnitTest.cmake)
 
 # Compiler warnings (only in Debug mode)
 if(CMAKE_BUILD_TYPE STREQUAL "Debug")
@@ -215,13 +270,20 @@ if(CCT_ENABLE_PROMETHEUS)
   add_compile_definitions(CCT_ENABLE_PROMETHEUS)
 endif()
 
+if(CCT_ENABLE_PROTO)
+  add_compile_definitions(CCT_ENABLE_PROTO)
+  add_compile_definitions("CCT_PROTOBUF_VERSION=\"${PROTOBUF_VERSION}\"")
+endif()
+
 # Link to sub folders CMakeLists.txt, from the lowest level to the highest level for documentation
 # (beware of cyclic dependencies)
 add_subdirectory(src/tech)
 add_subdirectory(src/monitoring)
 add_subdirectory(src/http-request)
 add_subdirectory(src/objects)
+add_subdirectory(src/serialization)
 add_subdirectory(src/api-objects)
+add_subdirectory(src/trading)
 add_subdirectory(src/api)
 add_subdirectory(src/engine)
 add_subdirectory(src/main)